mortar 0.9.1 → 0.9.2

data/lib/mortar/command/local.rb

@@ -66,12 +66,48 @@ class Mortar::Command::Local < Mortar::Command::Base
       error("No such directory #{project_root}")
     end
     Dir.chdir(project_root)
-
     script = validate_script!(script_name)
     ctrl = Mortar::Local::Controller.new
     ctrl.run(script, pig_parameters)
   end
 
+  # local:characterize 
+  #
+  # Characterize will inspect your input data, inferring a schema and 
+  #    generating keys, if needed. It will output CSV containing various
+  #    statistics about your data (most common values, percent null, etc.)
+  # 
+  # -f, --param-file PARAMFILE # Load pig parameter values from a file
+  #
+  # Load some data and emit statistics.
+  # PARAMFILE:
+  #   LOADER=<full class path of loader function>
+  #   INPUT_SRC=<Location of the input data>
+  #   OUTPUT_PATH=<Relative path from project root for output>
+  #   INFER_TYPES=<when true, recursively infers types for input data>
+  #
+  # Example paramfile:
+  #   LOADER=org.apache.pig.piggybank.storage.JsonLoader()
+  #   INPUT_SRC=s3n://twitter-gardenhose-mortar/example
+  #   OUTPUT_PATH=twitter_char
+  #   INFER_TYPES=true
+  #
+  def characterize
+    validate_arguments!
+
+    #cd into the project root
+    project_root = options[:project_root] ||= Dir.getwd
+    unless File.directory?(project_root)
+      error("No such directory #{project_root}")
+    end
+    Dir.chdir(project_root)
+
+    controlscript_name = "controlscripts/lib/characterize_control.py"
+    script = validate_script!(controlscript_name)
+    ctrl = Mortar::Local::Controller.new
+    ctrl.run(script, pig_parameters)
+  end
+
   # local:illustrate PIGSCRIPT [ALIAS]
   #
   # Locally illustrate the effects and output of a pigscript.

data/lib/mortar/command/plugins.rb

@@ -46,7 +46,7 @@ module Mortar::Command
           plugin.install
           Mortar::Plugin.load_plugin(plugin.name)
         rescue StandardError => e
-          error e
+          error e.message
         end
       end
     end
@@ -68,7 +68,7 @@ module Mortar::Command
         begin
           plugin.uninstall
         rescue Mortar::Plugin::ErrorPluginNotFound => e
-          error e
+          error e.message
         end
       end
     end
@@ -101,7 +101,7 @@ module Mortar::Command
             status "skipped symlink"
           rescue StandardError => e
             status "error"
-            display e
+            display e.message
           end
         end
       end

data/lib/mortar/command/projects.rb

@@ -47,19 +47,31 @@ class Mortar::Command::Projects < Mortar::Command::Base
       error("Usage: mortar projects:delete PROJECTNAME\nMust specify PROJECTNAME.")
     end
     validate_arguments!
-    
-    # delete embedded project mirror if one exists
-    mirror_dir = "#{git.mortar_mirrors_dir()}/#{name}"
-    if File.directory? mirror_dir
-      FileUtils.rm_r mirror_dir
+    projects = api.get_projects().body['projects']
+    project_id = nil
+    if projects.any?
+      projects.each do |project|
+        if project['name'] == name
+          project_id = project['project_id']
+        end
+      end
     end
 
-    # delete Mortar remote
-    project_id = nil
-    action("Sending request to delete project: #{name}") do
-      api.delete_project(name).body["project_id"]
+    if project_id.nil?
+      display "\nNo project with name: #{name}"
+    else
+      # delete embedded project mirror if one exists
+      mirror_dir = "#{git.mortar_mirrors_dir()}/#{name}"
+      if File.directory? mirror_dir
+        FileUtils.rm_r mirror_dir
+      end
+
+      # delete Mortar remote
+      action("Sending request to delete project: #{name}") do
+        api.delete_project(project_id).body['project_id']
+      end
+      display "\nYour project has been deleted."
     end
-    display "\nYour project has been deleted."
     
   end
   

data/lib/mortar/generators/project_generator.rb

@@ -38,6 +38,7 @@ module Mortar
           
           inside "pigscripts" do
             generate_file "pigscript.pig", "#{project_name}.pig"
+            copy_file "characterize.pig", "characterize.pig"
           end
 
           mkdir "controlscripts" 
@@ -46,6 +47,7 @@ module Mortar
             mkdir "lib"
             inside "lib" do
               copy_file "__init__.py", "__init__.py"
+              copy_file "characterize_control.py", "characterize_control.py"
             end
           end
           
@@ -53,6 +55,7 @@ module Mortar
           
           inside "macros" do
             copy_file "gitkeep", ".gitkeep"
+            copy_file "characterize_macro.pig", "characterize_macro.pig"
           end
 
           mkdir "fixtures"
@@ -72,6 +75,7 @@ module Mortar
             mkdir "jython"
             inside "jython" do
               copy_file "gitkeep", ".gitkeep"
+              copy_file "top_5_tuple.py", "top_5_tuple.py"
             end
 
             mkdir "java"
@@ -142,4 +146,4 @@ module Mortar
         end
     end
   end
-end
+end

data/lib/mortar/plugin.rb

@@ -1,4 +1,7 @@
 # based on the Rails Plugin
+
+require "stringio"
+
 module Mortar
   class Plugin
     include Mortar::Helpers
@@ -41,7 +44,40 @@ module Mortar
     end
 
     def self.install_bundle
-      system("bundle install --standalone --without development >> ./../plugin_install.log")
+      # TODO: Deal with the bundler as a runtime dependency issue
+      # before moving these require statements to the top.
+      begin
+        require 'bundler/cli'
+        require 'bundler/friendly_errors'
+      rescue LoadError => e 
+        raise <<-ERROR
+Unable to install this plugin. Make sure you have bundler installed:
+
+$ gem install bundler
+
+ERROR
+      end
+
+      out = StringIO.new
+      $stdout = out
+      begin
+        bundle_def = Bundler.definition({})
+        Bundler.ui = Bundler::UI::Shell.new({})
+        Bundler.ui.level = "silent"
+        Bundler.settings[:path] = "bundle"
+        Bundler::Installer.install(Bundler.root, bundle_def, {
+          :standalone => [],
+        })
+        result = true
+      rescue StandardError => e
+        out.write e.message
+        result = false
+      end
+      open("#{Plugin.directory}/plugin_install.log", 'a') do |f|
+        f.puts out.string
+      end
+      $stdout = STDOUT
+      return result
     end
 
     def self.load!
@@ -111,8 +147,7 @@ ERROR
         Mortar::Plugin.without_bundler_env do
           ENV["BUNDLE_GEMFILE"] = File.expand_path("Gemfile", path)
           if File.exists? ENV["BUNDLE_GEMFILE"]
-            Mortar::Plugin.install_bundle 
-            unless $?.success?
+            unless Mortar::Plugin.install_bundle 
               FileUtils.rm_rf path
               raise Mortar::Plugin::ErrorInstallingDependencies, <<-ERROR
 Unable to install dependencies for #{name}.

data/lib/mortar/templates/project/controlscripts/lib/characterize_control.py

@@ -0,0 +1,23 @@
+from org.apache.pig.scripting import Pig
+import os
+
+if __name__ == "__main__":
+    params       = Pig.getParameters()
+    loader       = params["LOADER"]
+    input_source = params["INPUT_SRC"]
+    output_path  = params["OUTPUT_PATH"]
+    infer_types  = params["INFER_TYPES"]
+
+    Pig.compileFromFile("../pigscripts/characterize.pig").bind({
+        "LOADER"      : loader,
+        "INPUT_SRC"   : input_source,
+        "OUTPUT_PATH" : output_path,
+        "INFER_TYPES" : infer_types
+    }).runSingle()
+
+    for root, _, files in os.walk("../%s" % output_path):
+        for f in files:
+            if f[0] != '.':
+                fullpath = os.path.join(root, f)
+                copypath = os.path.join(root, f + '.csv')
+                os.system ("cp %s %s" % (fullpath, copypath))

data/lib/mortar/templates/project/macros/characterize_macro.pig

@@ -0,0 +1,129 @@
+/*
+    Copyright 2013 Mortar Data Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+/*
+ * A pig macro to characterize some data.
+ * Take an alias to some pre-loaded data and return a relation
+ * containing statistics about that data in a bag.
+ * Each field is listed up to five times (with their five most common
+ * example values) as a tuple of the following form:
+ *     (
+ *      Field Name (embedded fields have their parent's field name prepended), 
+ *      Number of distinct values associated with the field, 
+ *      Total number of values,
+ *      Deciles,
+ *      Number of null appearences,
+ *      Percentage null,
+ *      Minimum value,
+ *      Maximum value,
+ *      Range of values (Max - Min),
+ *      Type,
+ *      Values (tuple of top 5 values, bags/chararrays/tuples are converted to length)
+ *      Value Counts (tuple of corresponding number of value occurrences)
+ *      Original Values (tuple of corresponding original values)
+ *     )
+ *
+ * data: {anything}
+ * inferTypes: when true, characterize will infer numeric types for values
+ *   without an explicit schema (e.g. values in a complex map object)
+ *
+ * Example:
+ * data = LOAD '/path/to/data' USING JsonLoader();
+ * characterized = Characterize(data, 'true')
+ */
+
+REGISTER 's3://mhc-software-mirror/datafu/datafu-0.0.10.jar';
+REGISTER 's3://mhc-software-mirror/bacon-bits/udfs/java/bacon-bits-0.1.0.jar';
+REGISTER '../udfs/jython/top_5_tuple.py' USING jython AS top5;
+
+DEFINE Deciles                  datafu.pig.stats.StreamingQuantile('11');
+
+DEFINE Characterize(data, inferTypes)
+RETURNS out {
+
+DEFINE Characterize__ExtractFields              com.mortardata.pig.ExtractFields('$inferTypes');
+
+raw_fields =  FOREACH $data
+              GENERATE FLATTEN(Characterize__ExtractFields(*)) as 
+              (keyname:chararray, type:chararray, val:double, orig:chararray);
+
+--Group the rows by field name and find the number of unique values for each field in the collection
+key_groups = GROUP raw_fields BY (keyname);
+unique_vals = FOREACH key_groups {
+    v = raw_fields.val;
+    null_fields = filter raw_fields by val is null;
+    unique_v = distinct v;
+    GENERATE flatten(group)  as keyname:chararray,
+             COUNT(unique_v) as num_distinct_vals_count:long, COUNT(v) as num_vals:long, COUNT(null_fields) as num_null:long,
+             (double) COUNT(null_fields) / (double) COUNT(raw_fields) as percentage_null:double, 
+             MIN(unique_v.val) as min_val:double, MAX(unique_v.val) as max_val:double;
+}
+
+-- calculate decile tuples for each key (filtering out null values to make datafu happy)
+no_nulls = filter raw_fields by val is not null;
+no_null_groups = GROUP no_nulls BY keyname;
+key_deciles = FOREACH no_null_groups {
+  GENERATE flatten(group) as keyname:chararray, Deciles(no_nulls.val) as deciles:tuple();
+}      
+
+-- Find the number of times each value occurs for each field
+key_val_groups = GROUP raw_fields BY (keyname, type, val, orig);
+key_val_groups_with_counts =  FOREACH key_val_groups
+                             GENERATE flatten(group),
+                                      COUNT($1) as val_count:long;
+
+-- Find the top 5 most common values for each field
+key_vals = GROUP key_val_groups_with_counts BY (keyname);
+top_5_vals = FOREACH key_vals {
+    ordered_vals = ORDER key_val_groups_with_counts BY val_count DESC;
+    limited_vals = LIMIT ordered_vals 5;
+    GENERATE flatten(limited_vals);
+}
+
+cogroup_result = COGROUP unique_vals BY keyname, 
+                         top_5_vals BY keyname;
+
+flat_vals = FOREACH cogroup_result {
+     top_vals = top5.key_bag_to_tuple(top_5_vals);
+     GENERATE flatten(unique_vals) as 
+              (keyname:chararray, num_distinct_vals_count:long, num_vals:long, 
+               num_null:long, percent_null:double, min_val:double, max_val:double), top_vals.vals as vals,
+               top_vals.type as type:chararray, top_vals.orig as orig:tuple(), top_vals.val_counts as val_counts:tuple();
+}
+
+join_result = JOIN flat_vals BY keyname,
+            key_deciles  BY keyname;
+
+-- Clean up columns (remove duplicate keyname field)
+result =  FOREACH join_result
+         GENERATE flat_vals::keyname as Key,
+                  num_distinct_vals_count as NDistinct,
+                  num_vals as NVals,
+                  deciles as Deciles,        
+                  num_null as NNull,
+                  percent_null as PctNull,
+                  min_val as Min,
+                  max_val as Max,
+                  max_val - min_val as Range,
+                  type as Type,
+                  vals as Vals,
+                  val_counts as ValCounts,
+                  orig as OrigVals;
+
+-- -- Sort by field name and number of values
+$out = ORDER result BY Key;
+
+};

data/lib/mortar/templates/project/pigscripts/characterize.pig

@@ -0,0 +1,6 @@
+IMPORT '../macros/characterize_macro.pig';
+data = LOAD '$INPUT_SRC'
+       USING $LOADER;
+characterize_data = Characterize(data, $INFER_TYPES);
+STORE characterize_data INTO '../$OUTPUT_PATH'
+       USING org.apache.pig.piggybank.storage.CSVExcelStorage(',', 'YES_MULTILINE', 'UNIX', 'WRITE_OUTPUT_HEADER');

data/lib/mortar/templates/project/udfs/jython/top_5_tuple.py

@@ -0,0 +1,27 @@
+@outputSchema("five:tuple(vals:tuple(v1:double, v2:double, v3:double, v4:double, v5:double), type:chararray, orig:tuple(v1:chararray, v2:chararray, v3:chararray, v4:chararray, v5:chararray), val_counts:tuple(v1:long, v2:long, v3:long, v4:long, v5:long))")
+def key_bag_to_tuple(input_bag):
+    output = []
+    #sort by val_count
+    input_bag = sorted(input_bag, key=lambda tup: tup[4], reverse=True)
+    vals = [None, None, None, None, None]
+    val_counts = [None, None, None, None, None]
+    orig_vals = [None, None, None, None, None]
+    for idx, t in enumerate(input_bag):
+        vals[idx] = t[2]
+        orig_vals[idx] = t[3]
+        val_counts[idx] = t[4]
+    val_tup = tuple(vals)
+    val_count_tup = tuple(val_counts)
+    orig_val_tup = tuple(orig_vals)
+    output.append(val_tup)
+    if all(i[1] == "NULL" for i in input_bag):
+        output.append("NULL")
+    else:
+        for t in input_bag:
+            if t[1] != "NULL":
+                output.append(t[1])
+                break
+    output.append(orig_val_tup)
+    output.append(val_count_tup)
+    return tuple(output)
+    

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.9.1"
+  VERSION = "0.9.2"
 end

data/spec/mortar/command/projects_spec.rb

@@ -36,9 +36,11 @@ module Mortar::Command
     
     project1 = {'name' => "Project1",
                 'status' => Mortar::API::Projects::STATUS_ACTIVE,
+                'project_id' => 'abcd1234',
                 'git_url' => "git@github.com:mortarcode-dev/Project1"}
     project2 = {'name' => "Project2",
                 'status' => Mortar::API::Projects::STATUS_ACTIVE,
+                'project_id' => 'defg5678',
                 'git_url' => "git@github.com:mortarcode-dev/Project2"}
         
     context("index") do
@@ -75,12 +77,13 @@ STDERR
       end
       
       it "deletes project" do
-        project_name = "COMMANDO"
+        project_id = "abcd1234"
 
-        mock(Mortar::Auth.api).delete_project(project_name).returns(Excon::Response.new(:body => {}, :status => 200 )) 
-        stderr, stdout = execute("projects:delete COMMANDO")
+        mock(Mortar::Auth.api).get_projects().returns(Excon::Response.new(:body => {"projects" => [project1, project2]}))
+        mock(Mortar::Auth.api).delete_project(project_id).returns(Excon::Response.new(:body => {}, :status => 200 ))
+        stderr, stdout = execute("projects:delete Project1")
         stdout.should == <<-STDOUT
-Sending request to delete project: COMMANDO... done
+Sending request to delete project: Project1... done
 
 Your project has been deleted.
 STDOUT
@@ -127,11 +130,14 @@ STDOUT
 \e[1;32m      create\e[0m  .gitignore
 \e[1;32m      create\e[0m  pigscripts
 \e[1;32m      create\e[0m  pigscripts/some_new_project.pig
+\e[1;32m      create\e[0m  pigscripts/characterize.pig
 \e[1;32m      create\e[0m  controlscripts
 \e[1;32m      create\e[0m  controlscripts/lib
 \e[1;32m      create\e[0m  controlscripts/lib/__init__.py
+\e[1;32m      create\e[0m  controlscripts/lib/characterize_control.py
 \e[1;32m      create\e[0m  macros
 \e[1;32m      create\e[0m  macros/.gitkeep
+\e[1;32m      create\e[0m  macros/characterize_macro.pig
 \e[1;32m      create\e[0m  fixtures
 \e[1;32m      create\e[0m  fixtures/.gitkeep
 \e[1;32m      create\e[0m  udfs
@@ -139,6 +145,7 @@ STDOUT
 \e[1;32m      create\e[0m  udfs/python/some_new_project.py
 \e[1;32m      create\e[0m  udfs/jython
 \e[1;32m      create\e[0m  udfs/jython/.gitkeep
+\e[1;32m      create\e[0m  udfs/jython/top_5_tuple.py
 \e[1;32m      create\e[0m  udfs/java
 \e[1;32m      create\e[0m  udfs/java/.gitkeep
 \e[1;32m      create\e[0m  vendor
@@ -426,4 +433,4 @@ STDERR
       
     end
   end
-end
+end

data/spec/mortar/plugin_spec.rb

@@ -100,15 +100,30 @@ module Mortar
 
       describe "installing plugins with dependencies" do
         it "should install plugin dependencies" do
+          ## Setup Fake Gem
+          mortar_fake_gem_folder = create_fake_gem("/tmp")
           plugin_folder = "/tmp/mortar_plugin"
           FileUtils.mkdir_p(plugin_folder)
-          File.open(plugin_folder + '/Gemfile', 'w') { |f| f.write "# dummy content" }
+          File.open(plugin_folder + '/Gemfile', 'w') { |f|
+            f.write "gem 'mortar_fake_gem', :path => '#{mortar_fake_gem_folder}'" 
+          }
+          File.open(plugin_folder + '/init.rb', 'w') { |f|
+            f.write <<-EOS
+require File.join(File.dirname(__FILE__), "bundle/bundler/setup")
+require "mortar_fake_gem"
+
+PluginTest = MortarFakeGem::WhoIs.awesome?
+EOS
+          }
           `cd #{plugin_folder} && git init && echo 'test' > README && git add . && git commit -m 'my plugin'`
           Plugin.new(plugin_folder).install
           File.directory?("#{@sandbox}/mortar_plugin").should be_true
           File.directory?("#{@sandbox}/mortar_plugin/bundle").should be_true
           File.exist?("#{@sandbox}/mortar_plugin/Gemfile").should be_true
           File.read("#{@sandbox}/mortar_plugin/README").should == "test\n"
+
+          Plugin.load!
+          PluginTest.should be_true
         end
 
         it "should fail to install plugin with bad dependencies" do

data/spec/spec_helper.rb

@@ -249,6 +249,44 @@ def git_create_conflict(git, project)
   filename
 end
 
+def create_fake_gem(path)
+  mortar_fake_gem = <<-EOS
+Gem::Specification.new do |s|
+  s.name = "mortar_fake_gem"
+  s.version = "0.0.1"
+  s.summary = "mortar_fake_gem is the best"
+  s.files = [
+    "lib/mortar_fake_gem.rb"
+  ]
+  s.require_paths = ["lib"]
+end
+EOS
+  
+  mortar_fake_gem_rb = <<-EOS
+module MortarFakeGem
+  class WhoIs
+    def self.awesome?
+      return true
+    end
+  end
+end
+EOS
+
+  files = { 
+    "mortar_fake_gem.gemspec" => mortar_fake_gem,
+    "lib/mortar_fake_gem.rb" => mortar_fake_gem_rb,
+  }
+
+  FileUtils.mkdir_p(File.join(path, "mortar_fake_gem/lib"))
+  files.each { |file_name, contents|
+    File.open(File.join(path, "/mortar_fake_gem/#{file_name}"), 'w') do |f|
+      f.puts contents
+    end
+  }
+
+  return File.join(path, "mortar_fake_gem")
+
+end
 
 def git_add_file(git, project)
   # add a new file

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 57
+  hash: 63
   prerelease: 
   segments: 
   - 0
   - 9
-  - 1
-  version: 0.9.1
+  - 2
+  version: 0.9.2
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-07-08 00:00:00 Z
+date: 2013-07-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mortar-api-ruby
@@ -210,12 +210,16 @@ files:
 - lib/mortar/templates/pigscript/python_udf.py
 - lib/mortar/templates/project/README.md
 - lib/mortar/templates/project/controlscripts/lib/__init__.py
+- lib/mortar/templates/project/controlscripts/lib/characterize_control.py
 - lib/mortar/templates/project/fixtures/gitkeep
 - lib/mortar/templates/project/gitignore
+- lib/mortar/templates/project/macros/characterize_macro.pig
 - lib/mortar/templates/project/macros/gitkeep
+- lib/mortar/templates/project/pigscripts/characterize.pig
 - lib/mortar/templates/project/pigscripts/pigscript.pig
 - lib/mortar/templates/project/udfs/java/gitkeep
 - lib/mortar/templates/project/udfs/jython/gitkeep
+- lib/mortar/templates/project/udfs/jython/top_5_tuple.py
 - lib/mortar/templates/project/udfs/python/python_udf.py
 - lib/mortar/templates/project/vendor/controlscripts/lib/__init__.py
 - lib/mortar/templates/project/vendor/macros/gitkeep
@@ -289,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Client library and CLI to interact with the Mortar service.