mortar 0.9.3 → 0.9.4

data/lib/mortar/auth.rb

@@ -67,14 +67,6 @@ class Mortar::Auth
         end
       end
     end
-    
-    def default_host
-      "mortardata.com"
-    end
-    
-    def host
-      ENV['MORTAR_HOST'] || default_host
-    end
   
     def reauthorize
       @credentials = ask_for_and_save_credentials

data/lib/mortar/command/local.rb

@@ -16,6 +16,7 @@
 
 require "mortar/local/controller"
 require "mortar/command/base"
+require "mortar/generators/characterize_generator"
 
 # run select pig commands on your local machine
 #
@@ -71,7 +72,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl.run(script, pig_parameters)
   end
 
-  # local:characterize 
+  # local:characterize -f PARAMFILE
   #
   # Characterize will inspect your input data, inferring a schema and 
   #    generating keys, if needed. It will output CSV containing various
@@ -80,7 +81,7 @@ class Mortar::Command::Local < Mortar::Command::Base
   # -f, --param-file PARAMFILE # Load pig parameter values from a file
   #
   # Load some data and emit statistics.
-  # PARAMFILE:
+  # PARAMFILE (Required):
   #   LOADER=<full class path of loader function>
   #   INPUT_SRC=<Location of the input data>
   #   OUTPUT_PATH=<Relative path from project root for output>
@@ -95,17 +96,28 @@ class Mortar::Command::Local < Mortar::Command::Base
   def characterize
     validate_arguments!
 
+    unless options[:param_file]
+      error("Usage: mortar local:characterize -f PARAMFILE.\nMust specify parameter file. For detailed help run:\n\n   mortar local:characterize -h")
+    end
+
     #cd into the project root
     project_root = options[:project_root] ||= Dir.getwd
     unless File.directory?(project_root)
       error("No such directory #{project_root}")
     end
+
     Dir.chdir(project_root)
 
+    gen = Mortar::Generators::CharacterizeGenerator.new
+    gen.generate_characterize
+
     controlscript_name = "controlscripts/lib/characterize_control.py"
+    gen = Mortar::Generators::CharacterizeGenerator.new
+    gen.generate_characterize
     script = validate_script!(controlscript_name)
     ctrl = Mortar::Local::Controller.new
     ctrl.run(script, pig_parameters)
+    gen.cleanup_characterize(project_root)
   end
 
   # local:illustrate PIGSCRIPT [ALIAS]

data/lib/mortar/generators/characterize_generator.rb

@@ -0,0 +1,105 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "fileutils"
+require "mortar/generators/generator_base"
+module Mortar
+  module Generators
+    class CharacterizeGenerator < Base
+
+      def generate_characterize
+        begin
+          inside "pigscripts" do
+            copy_file_char "characterize.pig", "characterize.pig"
+          end
+
+          inside "controlscripts" do
+            inside "lib" do
+              copy_file_char "characterize_control.py", "characterize_control.py"
+            end
+          end
+
+          inside "macros" do
+            copy_file_char "characterize_macro.pig", "characterize_macro.pig"
+          end
+
+          inside "udfs" do
+            inside "jython" do
+              copy_file_char "top_5_tuple.py", "top_5_tuple.py"
+            end
+          end
+
+        # how best to handle exceptions, here?
+        rescue => e 
+          display("\nCharacterize script generation failed.\n\n")
+          raise e
+        end
+      end
+
+      def cleanup_characterize(project_root)
+        @src_path = project_root
+        begin
+          inside "pigscripts" do
+            remove_file "characterize.pig"
+          end
+
+          inside "controlscripts" do
+            inside "lib" do
+              remove_file "characterize_control.py"
+            end
+          end
+
+          inside "macros" do
+            remove_file "characterize_macro.pig"
+          end
+
+          inside "udfs" do
+            inside "jython" do
+              remove_file "top_5_tuple.py"
+            end
+          end
+
+        # how best to handle exceptions, here?
+        rescue => e 
+          display("\nCharacterize script cleanup failed.\n\n")
+          raise e
+        end
+      end
+
+      def remove_file(target_file, options={ :recursive => false })
+        target_path = File.join(@src_path, @rel_path, target_file)
+        msg = File.join(@rel_path, target_file)[1..-1]
+
+        if File.exists?(target_path)
+          FileUtils.rm(target_path)
+        end
+      end
+          
+      def copy_file_char(src_file, dest_file, options={ :recursive => false })
+        src_path = File.join(@src_path, @rel_path, src_file)
+        dest_path = File.join(@dest_path, @rel_path, dest_file)
+        msg = File.join(@rel_path, dest_file)[1..-1]
+
+        unless File.exists?(dest_path) and 
+            FileUtils.compare_file(src_path, dest_path)
+          FileUtils.mkdir_p(File.dirname(dest_path)) if options[:recursive]
+          FileUtils.cp(src_path, dest_path)
+        end
+      end
+
+    end
+  end
+end

data/lib/mortar/generators/generator_base.rb

@@ -104,4 +104,4 @@ module Mortar
 
     end
   end
-end
+end

data/lib/mortar/generators/project_generator.rb

@@ -38,7 +38,6 @@ module Mortar
           
           inside "pigscripts" do
             generate_file "pigscript.pig", "#{project_name}.pig"
-            copy_file "characterize.pig", "characterize.pig"
           end
 
           mkdir "controlscripts" 
@@ -47,7 +46,6 @@ module Mortar
             mkdir "lib"
             inside "lib" do
               copy_file "__init__.py", "__init__.py"
-              copy_file "characterize_control.py", "characterize_control.py"
             end
           end
           
@@ -55,7 +53,6 @@ module Mortar
           
           inside "macros" do
             copy_file "gitkeep", ".gitkeep"
-            copy_file "characterize_macro.pig", "characterize_macro.pig"
           end
 
           mkdir "fixtures"
@@ -75,7 +72,6 @@ module Mortar
             mkdir "jython"
             inside "jython" do
               copy_file "gitkeep", ".gitkeep"
-              copy_file "top_5_tuple.py", "top_5_tuple.py"
             end
 
             mkdir "java"

data/lib/mortar/helpers.rb

@@ -38,6 +38,18 @@ module Mortar
       RUBY_PLATFORM =~ /-darwin\d/
     end
 
+    def default_host
+      "mortardata.com"
+    end
+    
+    def host
+      ENV['MORTAR_HOST'] || default_host
+    end
+
+    def test_name
+      ENV['MORTAR_TEST_NAME']
+    end
+
     def write_to_file(str_data, path, mkdir_p=true)
       if mkdir_p
         FileUtils.mkdir_p File.dirname(path)
@@ -528,5 +540,6 @@ module Mortar
       create_display_method("exists", "1;34")
       create_display_method("identical", "1;34")
       create_display_method("conflict", "1;31")
+      create_display_method("remove", "1;35")
   end
 end

data/lib/mortar/local/controller.rb

@@ -33,9 +33,13 @@ Linux systems please consult the documentation on your relevant package manager.
 EOF
 
   NO_PYTHON_ERROR_MESSAGE = <<EOF
-A suitable python installation with virtualenv could not be located.  Please ensure
-you have python 2.6+ installed on your local system.  If you need to obtain a copy
-of virtualenv it can be located here:
+A suitable python installation could not be located.  Please ensure you have python 2.6+
+installed on your local system.
+EOF
+
+  NO_VIRTENV_ERROR_MESSAGE = <<EOF
+A suitable Python installation was found, but it is required that virtualenv be installed
+as well.  You can install it with pip, or download it directly from:
 https://pypi.python.org/pypi/virtualenv
 EOF
 
@@ -86,6 +90,10 @@ EOF
       error(NO_PYTHON_ERROR_MESSAGE)
     end
 
+    unless py.check_virtualenv
+      error(NO_VIRTENV_ERROR_MESSAGE)
+    end
+
     unless py.setup_project_python_environment
       msg = "\nUnable to setup a python environment with your dependencies, "
       msg += "see #{py.pip_error_log_path} for more details"

data/lib/mortar/local/installutil.rb

@@ -107,13 +107,73 @@ module Mortar
         end
       end
 
-      # Downloads the file at a specified url into the supplied director
-      def download_file(url, dest_dir)
-        dest_file_path = dest_dir + "/" + File.basename(url)
+      # Downloads the file at a specified url into the supplied directory
+      def download_file(url, dest_file_path)
+        response = get_resource(url)
+        
         File.open(dest_file_path, "wb") do |dest_file|
-          contents = Excon.get(url).body
-          dest_file.write(contents)
+          dest_file.write(response.body)
         end
+      
+      end
+
+      # Perform a get request to a url and follow redirects if necessary.
+      def get_resource(url)
+        make_call(url, 'get')
+      end
+
+      # Perform a head request to a url and follow redirects if necessary.
+      def head_resource(url)
+        make_call(url, 'head')
+      end
+
+      # Make a request to a mortar resource url.  Check response for a 
+      # redirect and if necessary call the new url.  Excon doesn't currently 
+      # support automatically following redirects.  Adds parameter that
+      # checks an environment variable to identify the test making this call
+      # (if being run by a test).
+      def make_call(url, call_func, redirect_times=0, errors=0)
+        if redirect_times >= 5
+          raise RuntimeError, "Too many redirects.  Last url: #{url}"
+        end
+
+        if errors >= 5
+          raise RuntimeError, "Server Error at #{url}"
+        end
+
+        
+        query = {}
+        if test_name
+          query[:test_name] = test_name
+        end
+
+        headers = {'User-Agent' => Mortar::USER_AGENT}
+        if call_func == 'head'
+          response = Excon.head( url, 
+                                :headers => headers,
+                                :query => query
+                               )
+        elsif call_func == 'get'
+          response = Excon.get( url, 
+                                :headers => headers,
+                                :query => query
+                              )
+        else
+          raise RuntimeError, "Unknown call type: #{call_func}"
+        end
+        case response.status
+        when 300..303 then 
+          make_call(response.headers['Location'], call_func, redirect_times+1, errors)
+        when 500..599 then
+          sleep(make_call_sleep_seconds)
+          make_call(url, call_func, redirect_times, errors+1)
+        else
+          response
+        end
+      end
+
+      def make_call_sleep_seconds
+        2
       end
 
       def osx?
@@ -126,7 +186,7 @@ module Mortar
       end
 
       def url_date(url)
-        result = Excon.head(url)
+        result = head_resource(url)
         http_date_to_epoch(result.get_header('Last-Modified'))
       end
 

data/lib/mortar/local/jython.rb

@@ -21,7 +21,7 @@ class Mortar::Local::Jython
 
   JYTHON_VERSION = '2.5.2'
   JYTHON_JAR_NAME = 'jython_installer-' + JYTHON_VERSION + '.jar'
-  JYTHON_JAR_DIR = "http://s3.amazonaws.com/hawk-dev-software-mirror/jython/jython-2.5.2/"
+  JYTHON_JAR_DEFAULT_URL_PATH = "resource/jython"
 
   def install_or_update
     if should_install
@@ -40,24 +40,30 @@ class Mortar::Local::Jython
   end
 
   def install
-    unless File.exists?(local_install_directory + '/' + JYTHON_JAR_NAME)
-        download_file(JYTHON_JAR_DIR + JYTHON_JAR_NAME, local_install_directory)
+    jython_file = File.join(local_install_directory, JYTHON_JAR_NAME)
+    unless File.exists?(jython_file)
+        download_file(jython_jar_url, jython_file)
     end
 
     `$JAVA_HOME/bin/java -jar #{local_install_directory + '/' + JYTHON_JAR_NAME} -s -d #{jython_directory}`
     FileUtils.mkdir_p jython_cache_directory
     FileUtils.chmod_R 0777, jython_cache_directory
 
-    FileUtils.rm(local_install_directory + '/' + JYTHON_JAR_NAME)
+    FileUtils.rm(jython_file)
     note_install('jython')
   end
 
   def should_update
-    return is_newer_version('jython', JYTHON_JAR_DIR + JYTHON_JAR_NAME)
+    return is_newer_version('jython', jython_jar_url)
   end
 
   def update
     FileUtils.rm_r(jython_directory)
     install
   end
+
+  def jython_jar_url
+    default_url = host + "/" + JYTHON_JAR_DEFAULT_URL_PATH
+    ENV.fetch('JYTHON_JAR_URL', default_url)
+  end
 end

data/lib/mortar/local/pig.rb

@@ -23,7 +23,8 @@ class Mortar::Local::Pig
   include Mortar::Local::InstallUtil
 
   PIG_LOG_FORMAT = "humanreadable"
-  PIG_TAR_DEFAULT_URL = "https://s3.amazonaws.com/mhc-software-mirror/cli/pig.tgz"
+  PIG_TGZ_NAME = "pig.tgz"
+  PIG_TGZ_DEFAULT_URL_PATH = "resource/pig"
 
   # Tempfile objects have a hook to delete the file when the object is
   # destroyed by the garbage collector.  In practice this means that a
@@ -77,11 +78,8 @@ class Mortar::Local::Pig
   end
 
   def pig_archive_url
-    ENV.fetch('PIG_DISTRO_URL', PIG_TAR_DEFAULT_URL)
-  end
-
-  def pig_archive_file
-    File.basename(pig_archive_url)
+    default_url = host + "/" + PIG_TGZ_DEFAULT_URL_PATH
+    ENV.fetch('PIG_DISTRO_URL', default_url)
   end
 
   # Determines if a pig install needs to occur, true if no pig install present
@@ -96,7 +94,6 @@ class Mortar::Local::Pig
   end
 
   def install_or_update()
-    call_install = false
     if should_do_pig_install?
       action "Installing pig to #{local_install_directory_name}" do
         install()
@@ -111,8 +108,8 @@ class Mortar::Local::Pig
   # Installs pig for this project if it is not already present
   def install
     FileUtils.mkdir_p(local_install_directory)
-    download_file(pig_archive_url, local_install_directory)
-    local_tgz = File.join(local_install_directory, pig_archive_file)
+    local_tgz = File.join(local_install_directory, PIG_TGZ_NAME)
+    download_file(pig_archive_url, local_tgz)
     extract_tgz(local_tgz, local_install_directory)
 
     # This has been seening coming out of the tgz w/o +x so we do

data/lib/mortar/local/python.rb

@@ -19,13 +19,16 @@ require "mortar/local/installutil"
 class Mortar::Local::Python
   include Mortar::Local::InstallUtil
 
-  PYTHON_DEFAULT_TGZ_URL = "https://s3.amazonaws.com/mhc-software-mirror/cli/mortar-python-osx.tgz"
+  PYTHON_OSX_TGZ_NAME = "mortar-python-osx.tgz"
+  PYTHON_OSX_TGZ_DEFAULT_URL_PATH = "resource/python_osx"
 
   # Path to the python binary that should be used
   # for running UDFs
   @command = nil
 
 
+  @candidate_pythons = nil
+
   # Execute either an installation of python or an inspection
   # of the local system to see if a usable python is available
   def check_or_install
@@ -38,6 +41,17 @@ class Mortar::Local::Python
     end
   end
 
+  def check_virtualenv
+    # Assumes you've already called check_or_install(), in which case
+    # we can skip osx as its installation includeds virtualenv
+    if osx?
+      return true
+    else
+      return check_pythons_for_virtenv
+    end
+
+  end
+
   def should_do_update?
     return is_newer_version('python', python_archive_url)
   end
@@ -60,13 +74,14 @@ class Mortar::Local::Python
 
   def install_osx
     FileUtils.mkdir_p(local_install_directory)
-    download_file(python_archive_url, local_install_directory)
-    extract_tgz(local_install_directory + "/" + python_archive_file, local_install_directory)
+    python_tgz_path = File.join(local_install_directory, PYTHON_OSX_TGZ_NAME)
+    download_file(python_archive_url, python_tgz_path)
+    extract_tgz(python_tgz_path, local_install_directory)
 
     # This has been seening coming out of the tgz w/o +x so we do
     # here to be sure it has the necessary permissions
     FileUtils.chmod(0755, @command)
-    File.delete(local_install_directory + "/" + python_archive_file)
+    File.delete(python_tgz_path)
     note_install("python")
   end
 
@@ -76,21 +91,31 @@ class Mortar::Local::Python
     return (osx? and (not (File.exists?(python_directory))))
   end
 
+  def candidates
+    @candidate_pythons.dup
+  end
 
   # Checks if there is a usable versionpython already installed
   def check_system_python
-    py_cmd = path_to_local_python()
-    if not py_cmd
-      false
-    else
-      @command = py_cmd
-      true
-    end
+    @candidate_pythons = lookup_local_pythons
+    return 0 != @candidate_pythons.length
+  end
+
+  # Inspects the list of found python installations and
+  # checks if they have virtualenv installed. The first
+  # one found will be used.
+  def check_pythons_for_virtenv
+    @candidate_pythons.each{ |py|
+      if has_virtualenv_installed(py)
+        @command = py
+        true
+      end
+    }
   end
 
   # Checks if the specified python command has
   # virtualenv installed
-  def check_virtualenv_installed(python)
+  def has_virtualenv_installed(python)
     `#{python} -m virtualenv --help`
     if (0 != $?.to_i)
       false
@@ -99,18 +124,16 @@ class Mortar::Local::Python
     end
   end
 
-  def path_to_local_python
+  def lookup_local_pythons
     # Check several python commands in decending level of desirability
+    found_bins = []
     [ "python#{desired_python_minor_version}", "python" ].each{ |cmd|
       path_to_python = `which #{cmd}`.to_s.strip
       if path_to_python != ''
-        # todo: check for a minimum version (in the case of 'python')
-        if check_virtualenv_installed(path_to_python)
-          return path_to_python
-        end
+        found_bins << path_to_python
       end
     }
-    return nil
+    return found_bins
   end
 
 
@@ -135,11 +158,8 @@ class Mortar::Local::Python
   end
 
   def python_archive_url
-    return ENV.fetch('PYTHON_DISTRO_URL', PYTHON_DEFAULT_TGZ_URL)
-  end
-
-  def python_archive_file
-    File.basename(python_archive_url)
+    default_url = host + "/" + PYTHON_OSX_TGZ_DEFAULT_URL_PATH
+    return ENV.fetch('PYTHON_DISTRO_URL', default_url)
   end
 
   # Creates a virtualenv in a well known location and installs any packages