mortar 0.6.2 → 0.7.0

data/lib/mortar/local/python.rb

@@ -0,0 +1,184 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/local/installutil"
+
+class Mortar::Local::Python
+  include Mortar::Local::InstallUtil
+
+  PYTHON_DEFAULT_TGZ_URL = "https://s3.amazonaws.com/mortar-public-artifacts/mortar-python-osx.tgz"
+
+  # Path to the python binary that should be used
+  # for running UDFs
+  @command = nil
+
+
+  # Execute either an installation of python or an inspection
+  # of the local system to see if a usable python is available
+  def check_or_install
+    if osx?
+      # We currently only install python for osx
+      install_python_osx
+    else
+      # Otherwise we check that the system supplied python will be sufficient
+      check_system_python
+    end
+  end
+
+  # Performs an installation of python specific to this project, this
+  # install includes pip and virtualenv
+  def install_python_osx
+    @command = "#{local_install_directory}/python/bin/python"
+    if should_do_python_install?
+      FileUtils.mkdir_p(local_install_directory)
+      action "Installing python" do
+        download_file(python_archive_url, local_install_directory)
+        extract_tgz(local_install_directory + "/" + python_archive_file, local_install_directory)
+
+        # This has been seening coming out of the tgz w/o +x so we do
+        # here to be sure it has the necessary permissions
+        FileUtils.chmod(0755, @command)
+        File.delete(local_install_directory + "/" + python_archive_file)
+        note_install("python")
+      end
+    end
+    true
+  end
+
+  # Determines if a python install needs to occur, true if no
+  # python install present or a newer version is available
+  def should_do_python_install?
+    return (osx? and (not (File.exists?(python_directory))))
+  end
+
+
+  # Checks if there is a usable versionpython already installed
+  def check_system_python
+    py_cmd = path_to_local_python()
+    if not py_cmd
+      false
+    else
+      @command = py_cmd
+      true
+    end
+  end
+
+  # Checks if the specified python command has
+  # virtualenv installed
+  def check_virtualenv_installed(python)
+    `#{python} -m virtualenv --help`
+    if (0 != $?.to_i)
+      false
+    else
+      true
+    end
+  end
+
+  def path_to_local_python
+    # Check several python commands in decending level of desirability
+    [ "python#{desired_python_minor_version}", "python" ].each{ |cmd|
+      path_to_python = `which #{cmd}`.to_s.strip
+      if path_to_python != ''
+        # todo: check for a minimum version (in the case of 'python')
+        if check_virtualenv_installed(path_to_python)
+          return path_to_python
+        end
+      end
+    }
+    return nil
+  end
+
+
+  def desired_python_minor_version
+    return "2.7"
+  end
+
+  def pip_requirements_path
+    return ENV.fetch('PIP_REQ_FILE', File.join(Dir.getwd, "udfs", "python", "requirements.txt"))
+  end
+
+  def has_python_requirements
+    return File.exists?(pip_requirements_path)
+  end
+
+  def python_env_dir
+    return "#{local_install_directory}/pythonenv"
+  end
+
+  def python_directory
+    return "#{local_install_directory}/python"
+  end
+
+  def python_archive_url
+    return ENV.fetch('PYTHON_DISTRO_URL', PYTHON_DEFAULT_TGZ_URL)
+  end
+
+  def python_archive_file
+    File.basename(python_archive_url)
+  end
+
+  # Creates a virtualenv in a well known location and installs any packages
+  # necessary for the users python udf
+  def setup_project_python_environment
+    `#{@command} -m virtualenv #{python_env_dir}`
+    if 0 != $?.to_i
+      return false
+    end
+    if should_do_requirements_install
+      action "Installing python UDF dependencies" do
+        pip_output = `. #{python_env_dir}/bin/activate &&
+          #{python_env_dir}/bin/pip install --requirement #{pip_requirements_path}`
+          if 0 != $?.to_i
+            File.open(pip_error_log_path, 'w') { |f|
+              f.write(pip_output)
+            }
+            return false
+          end
+        note_install("pythonenv")
+      end
+    end
+    return true
+  end
+
+  def pip_error_log_path
+    return ENV.fetch('PIP_ERROR_LOG', "dependency_install.log")
+  end
+
+  # Whether or not we need to do a `pip install -r requirements.txt` because
+  # we've never done one before or the dependencies have changed
+  def should_do_requirements_install
+    if has_python_requirements
+      if not install_date('pythonenv')
+        # We've never done an install from requirements.txt before
+        return true
+      else
+        return (requirements_edit_date > install_date('pythonenv'))
+      end
+    else
+      return false
+    end
+  end
+
+  # Date of last change to the requirements file
+  def requirements_edit_date
+    if has_python_requirements
+      return File.mtime(pip_requirements_path).to_i
+    else
+      return nil
+    end
+  end
+
+end

data/lib/mortar/project.rb

@@ -57,6 +57,19 @@ module Mortar
           ".pig")
         @pigscripts
       end
+
+      def controlscripts_path
+        File.join(@root_path, "controlscripts")
+      end
+
+      def controlscripts
+        @controlscripts ||= ControlScripts.new(
+          controlscripts_path,
+          "controlscripts",
+          ".py",
+          :optional => true)
+        @controlscripts
+      end
       
       def tmp_path
         path = File.join(@root_path, "tmp")
@@ -75,10 +88,11 @@ module Mortar
       
       include Enumerable
       
-      def initialize(path, name, filename_extension)
+      def initialize(path, name, filename_extension, optional=false)
         @path = path
         @name = name
         @filename_extension = filename_extension
+        @optional = optional
         @elements = elements
       end
       
@@ -107,14 +121,15 @@ module Mortar
       end
 
       def elements
-        unless File.directory? @path
-          raise ProjectError, "Unable to find #{@name} directory in project"
+        if File.directory? @path
+          # get {script_name => full_path}
+          file_paths = Dir[File.join(@path, "**", "*#{@filename_extension}")]
+          file_paths_hsh = file_paths.collect{|element_path| [element_name(element_path), element(element_name(element_path), element_path)]}.flatten
+          return Hash[*file_paths_hsh]
+        else
+          raise ProjectError, "Unable to find #{@name} directory in project" if not @optional
         end
-
-        # get {script_name => full_path}
-        file_paths = Dir[File.join(@path, "**", "*#{@filename_extension}")]
-        file_paths_hsh = file_paths.collect{|element_path| [element_name(element_path), element(element_name(element_path), element_path)]}.flatten
-        Hash[*file_paths_hsh]
+        return Hash[]
       end
       
       def element(path)
@@ -124,7 +139,13 @@ module Mortar
     
     class PigScripts < ProjectEntity
       def element(name, path)
-        Script.new(name, path)
+        PigScript.new(name, path)
+      end
+    end
+
+    class ControlScripts < ProjectEntity
+      def element(name, path)
+        ControlScript.new(name, path)
       end
     end
 
@@ -156,5 +177,10 @@ module Mortar
       end
     end
     
+    class ControlScript < Script
+    end
+    class PigScript < Script
+    end
+    
   end
 end

data/lib/mortar/templates/project/gitignore

@@ -2,4 +2,6 @@ Gemfile.lock
 *.pyc
 *.class
 *.log
-tmp
+tmp
+log
+.mortar-local

data/lib/mortar/templates/report/illustrate-report.html

@@ -0,0 +1,96 @@
+
+<html>
+  <head>
+    <title>Illustrate Report</title>
+    <link rel="stylesheet" href="resources/css/illustrate-output.css" type="text/css" />
+  </head>
+  
+  <body>
+
+    <% @tables.each_with_index do |table, table_index| %>
+
+      <h3 class="illustrate_alias">
+        <% if (table["op"] == "LOStore") %>
+          Store : 
+        <% end %>
+        <%= table["alias"] %>
+        <% if (table["notices"])   %>
+        &nbsp;
+        <% end %>
+    </h3>
+
+    <table class="table table-bordered illustrate_table expandable">
+      <tbody>
+        <tr>
+          <% table["fields"].each do |field| %>
+            <th><%= field %></th>
+          <% end %>
+        </tr>
+
+        <% table["data"].each_with_index do |row, row_index| %>
+          <tr>
+            <% row.each_with_index do |value, item_index| %>
+                <% if (value) %>
+                    <td>
+                      <div class="mortar-table-expandable-cell-wrapper">
+                        <div class="mortar-table-expandable-cell-container">
+                          <div class="mortar-table-expandable-cell-preview">
+                            <%# <%- $.mortar_data.truncate.truncate_center(value, 30) %>
+                            <%= value[0..30] %>
+                          </div>
+                          <div class="mortar-table-expandable-cell-content" hidden="true">
+                            <!-- TODO: get clipboard working -->
+                            <!-- <div data-clipboard-target="copy-<%- table_index %>-<%- row_index %>-<%- item_index %>" class="clipboard" data->Copy to Clipboard</div>  -->
+                            <div id="copy-<%- table_index %>-<%- row_index %>-<%- item_index %>" class="illustrate_content_wrapper"><%= value %></div>
+                          </div>
+                        </div>
+                      </div>
+                    </td>
+                <% else  %>
+                    <td><div>&nbsp;</div></td>
+                <% end %>
+            <% end %>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+
+    <% if (table["notices"])  %>
+        <% table["notices"].each do |notice| %>
+            <div class="alert">
+                <a class="close" data-dismiss="alert">&times;</a>
+                * <%= notice %>
+            </div>
+        <% end %>
+    <% end %>
+
+  <% end %>
+
+  <% if (@udf_output and (not @udf_output.empty?))   %>
+    <h3 class="illustrate_alias">UDF Output: </h3>
+    </br>
+    <pre id="illustrate_udf_output">
+      <%= @udf_output %>
+    </pre>
+  <% end %>
+
+  <script src="resources/js/jquery-1.7.1.min.js"></script>
+  <script src="resources/js/jquery.transit.js"></script>
+  <script src="resources/js/jquery.stylestack.js"></script>
+  <script src="resources/js/zero_clipboard.js"></script>
+  <script src="resources/js/mortar-table.js"></script>
+  <script>
+    $('.illustrate_table td').mortarTableExpandableCell();
+    // TODO: get this working
+    /*
+    $('.illustrate_table td .clipboard').each(function() {
+       $(this).data('clippy',  new ZeroClipboard( $(this), {
+            moviePath: "resources/flash/zeroclipboard.swf"
+        })); 
+    });
+    */
+  </script>
+
+  </body>
+
+</html>

data/lib/mortar/templates/script/runpig.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+export PIG_HOME=<%= @pig_home %>
+export PIG_CLASSPATH=<%= @pig_classpath %>
+export CLASSPATH=<%= @classpath %>
+export PIG_MAIN_CLASS=com.mortardata.hawk.HawkMain
+export PIG_OPTS="<% @pig_opts.each do |k,v| %>-D<%= k %>=<%= v %> <% end %>"
+
+# UDF paths are relative to this direectory
+cd <%= @project_home %>/pigscripts
+
+# Setup python environment
+source <%= @local_install_dir %>/pythonenv/bin/activate
+
+# Run Pig
+<%= @local_install_dir %>/pig/bin/pig -exectype local \
+    -log4jconf <%= @local_install_dir %>/pig/conf/log4j-cli-local-dev.properties \
+    -propertyFile <%= @local_install_dir %>/pig/conf/pig-hawk-global.properties \
+    -propertyFile <%= @local_install_dir %>/pig/conf/pig-cli-local-dev.properties \
+    -param_file <%= @pig_params_file %> \
+    <%= @pig_sub_command %>

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.6.2"
+  VERSION = "0.7.0"
 end

data/spec/mortar/auth_spec.rb

@@ -152,5 +152,13 @@ module Mortar
 
       lambda { @cli.check }.should raise_error(Mortar::CLI::Errors::InvalidGithubUsername)
     end
+
+    it "encodes the user email as s3 safe" do
+      user_email = "myemail+dontspam@somedomain.com"
+      stub(@cli).user.returns(user_email)
+      @cli.user().should == user_email
+      @cli.user_s3_safe.should == 'myemail-dontspam-somedomain-com'
+    end
+
   end
 end

data/spec/mortar/command/describe_spec.rb

@@ -57,8 +57,21 @@ STDERR
         with_git_initialized_project do |p|
           stderr, stdout = execute("describe does_not_exist my_alias", p, @git)
           stderr.should == <<-STDERR
- !    Unable to find pigscript does_not_exist
+ !    Unable to find a pigscript or controlscript for does_not_exist
+ !    
  !    No pigscripts found
+ !    
+ !    No controlscripts found
+ STDERR
+        end
+      end
+      
+      it "errors when requested with controlscript" do
+        with_git_initialized_project do |p|
+          write_file(File.join(p.controlscripts_path, "my_script.py"))
+          stderr, stdout = execute("describe my_script my_alias", p, @git)
+          stderr.should == <<-STDERR
+ !    Currently Mortar does not support describing control scripts
  STDERR
         end
       end

data/spec/mortar/command/illustrate_spec.rb

@@ -57,8 +57,21 @@ STDERR
         with_git_initialized_project do |p|
           stderr, stdout = execute("illustrate does_not_exist my_alias", p, @git)
           stderr.should == <<-STDERR
- !    Unable to find pigscript does_not_exist
+ !    Unable to find a pigscript or controlscript for does_not_exist
+ !    
  !    No pigscripts found
+ !    
+ !    No controlscripts found
+ STDERR
+        end
+      end
+      
+      it "errors when requested with controlscript" do
+        with_git_initialized_project do |p|
+          write_file(File.join(p.controlscripts_path, "my_script.py"))
+          stderr, stdout = execute("illustrate my_script my_alias", p, @git)
+          stderr.should == <<-STDERR
+ !    Currently Mortar does not support illustrating control scripts
  STDERR
         end
       end