mortar 0.13.5 → 0.14.0

data/lib/mortar/command/local.rb

@@ -197,4 +197,18 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl.validate(script, pig_version, pig_parameters)
   end
 
+
+  # local:repl
+  #
+  # Start a local Pig REPL session
+  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
+  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  # -g, --pigversion PIG_VERSION  # Set pig version.  Options are <PIG_VERSION_OPTIONS>.
+  #
+  def repl
+    validate_arguments!
+    ctrl = Mortar::Local::Controller.new
+    ctrl.repl(pig_version, pig_parameters)
+  end
+
 end

data/lib/mortar/command/version.rb

@@ -17,13 +17,16 @@
 # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
 #
 
+
 require "mortar/command/base"
 require "mortar/version"
+require "mortar/helpers"
+require "open3"
 
 # display version
 #
 class Mortar::Command::Version < Mortar::Command::Base
-
+  include Mortar::Helpers
   # version
   #
   # show mortar client version
@@ -34,4 +37,36 @@ class Mortar::Command::Version < Mortar::Command::Base
     display(Mortar::USER_AGENT)
   end
 
+  # upgrade [OPTIONAL_VERSION_NUMBER]
+  # version:upgrade [OPTIONAL_VERSION_NUMBER]
+  #
+  # Upgrade the Mortar development framework
+  #
+  # -v, --version   VERSION_NUMBER    # specify which version to upgrade to
+  def upgrade
+    validate_arguments!
+    if running_on_a_mac? 
+      if installed_with_omnibus?
+        version_number = ''
+        if options[:version] 
+          version_number = " -v " + options[:version]
+        end
+        shell_url = ENV.fetch("MORTAR_INSTALL", "http://install.mortardata.com")
+        dir = "/opt/mortar/installer"
+        begin
+          cmd = "sudo curl -sS -L -o #{dir}/install.sh #{shell_url} && sudo bash #{dir}/install.sh#{version_number}"
+          Kernel.system cmd
+        ensure
+          FileUtils.remove_entry_secure dir #removes temporaryily created directory
+        end
+      else
+        error("mortar upgrade is only for installations not conducted with ruby gem.  Please upgrade by running 'gem install mortar'.")
+      end
+    else
+      error("mortar upgrade is currently only supported for OSX.")
+    end
+  end
+
+  alias_command "upgrade", "version:upgrade"
+
 end

data/lib/mortar/helpers.rb

@@ -49,6 +49,10 @@ module Mortar
     def test_name
       ENV['MORTAR_TEST_NAME']
     end
+    
+    def installed_with_omnibus?
+      File.exists?('/opt/mortar/version-manifest.txt')    
+    end
 
     def write_to_file(str_data, path, mkdir_p=true)
       if mkdir_p

data/lib/mortar/local/controller.rb

@@ -194,4 +194,10 @@ EOF
     pig.validate_script(pig_script, pig_version, pig_parameters)
   end
 
+  def repl(pig_version, pig_parameters)
+    install_and_configure(pig_version)
+    pig = Mortar::Local::Pig.new()
+    pig.launch_repl(pig_version, pig_parameters)
+  end
+
 end

data/lib/mortar/local/installutil.rb

@@ -66,6 +66,10 @@ module Mortar
         File.expand_path("../../templates/project/gitignore", __FILE__)
       end
 
+      def ensure_mortar_local_directory(relative_dir)
+        FileUtils.mkdir_p(File.join(local_install_directory, relative_dir))
+      end
+
       # Drops a marker file for an installed package, used
       # to help determine if updates should be performed
       def note_install(subdirectory)

data/lib/mortar/local/pig.rb

@@ -180,6 +180,10 @@ class Mortar::Local::Pig
     run_pig_command(" -check #{pig_script.path}", pig_version, pig_parameters)
   end
 
+  def launch_repl(pig_version, pig_parameters)
+    run_pig_command(" ", pig_version, pig_parameters)
+  end
+
 
   # run the pig script with user supplied pig parameters
   def run_script(pig_script, pig_version, pig_parameters)

data/lib/mortar/local/python.rb

@@ -22,6 +22,8 @@ class Mortar::Local::Python
   PYTHON_OSX_TGZ_NAME = "mortar-python-osx.tgz"
   PYTHON_OSX_TGZ_DEFAULT_URL_PATH = "resource/python_osx"
 
+  MORTAR_PYTHON_PACKAGES = ["luigi"]
+
   # Path to the python binary that should be used
   # for running UDFs
   @command = nil
@@ -164,11 +166,18 @@ class Mortar::Local::Python
     return ENV.fetch('PYTHON_DISTRO_URL', default_url)
   end
 
+  def has_valid_virtualenv?
+    `#{@command} -m virtualenv #{python_env_dir}`
+    if 0 != $?.to_i
+      return false
+    end
+    return true
+  end
+
   # Creates a virtualenv in a well known location and installs any packages
   # necessary for the users python udf
   def setup_project_python_environment
-    `#{@command} -m virtualenv #{python_env_dir}`
-    if 0 != $?.to_i
+    if not has_valid_virtualenv?
       return false
     end
     if should_do_requirements_install
@@ -184,6 +193,11 @@ class Mortar::Local::Python
         note_install("pythonenv")
       end
     end
+    if should_install_python_dependencies?
+      unless install_python_dependencies()
+        return false
+      end
+    end
     return true
   end
 
@@ -215,4 +229,62 @@ class Mortar::Local::Python
     end
   end
 
+  def mortar_package_url(package)
+    return "http://s3.amazonaws.com/mortar-pypi/#{package}/#{package}.tar.gz";
+  end
+
+  def update_mortar_package?(package)
+    return is_newer_version(mortar_package_dir(package), mortar_package_url(package))
+  end
+
+  def mortar_packages_dir
+    return "pythonenv/mortar-packages"
+  end
+
+  def mortar_package_dir(package)
+    package_dir = "#{mortar_packages_dir}/#{package}"
+  end
+
+  def should_install_python_dependencies?
+    MORTAR_PYTHON_PACKAGES.each{ |package|
+      if update_mortar_package? package
+        return true
+      end
+    }
+    return false
+  end
+
+  def install_python_dependencies
+    action "Installing python dependencies to #{local_install_directory_name}" do
+      ensure_mortar_local_directory mortar_packages_dir
+      MORTAR_PYTHON_PACKAGES.each{ |package_name|
+        unless install_mortar_python_package(package_name)
+          return false
+        end
+      }
+    end
+    return true
+  end
+
+  def pip_install package_url
+    pip_output = `. #{python_env_dir}/bin/activate &&
+          #{python_env_dir}/bin/pip install  #{package_url} --use-mirrors`
+    if 0 != $?.to_i
+      File.open(pip_error_log_path, 'w') { |f|
+        f.write(pip_output)
+      }
+      return false
+    else
+      return true
+    end
+  end
+
+  def install_mortar_python_package(package_name)
+    unless pip_install mortar_package_url(package_name)
+        return false
+    end
+    ensure_mortar_local_directory mortar_package_dir(package_name)
+    note_install mortar_package_dir(package_name)
+  end
+
 end

data/lib/mortar/templates/script/runpig.sh

@@ -9,7 +9,9 @@ export PIG_MAIN_CLASS=com.mortardata.hawk.HawkMain
 export PIG_OPTS="<% @pig_opts.each do |k,v| %>-D<%= k %>=<%= v %> <% end %>"
 
 # UDF paths are relative to this direectory
-cd <%= @project_home %>/pigscripts
+if [ -d "<%= @project_home %>/pigscripts" ]; then
+    cd <%= @project_home %>/pigscripts
+fi
 
 # Setup python environment
 source <%= @local_install_dir %>/pythonenv/bin/activate

data/lib/mortar/updater.rb

@@ -45,7 +45,11 @@ module Mortar
       newest_version = self.get_newest_version
 
       if compare_versions(newest_version, local_version) > 0
-        Mortar::Helpers.warning("There is a new version of the Mortar development framework available.  Please run 'gem install mortar' to install the latest version.\n\n")
+        if installed_with_omnibus? and running_on_a_mac? 
+          Mortar::Helpers.warning("There is a new version of the Mortar development framework available.  Please run 'mortar upgrade' to install the latest version.\n\n")
+        else
+          Mortar::Helpers.warning("There is a new version of the Mortar development framework available.  Please run 'gem install mortar' to install the latest version.\n\n")
+        end
       end
     end
   end

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.13.5"
+  VERSION = "0.14.0"
 end

data/spec/mortar/command/version_spec.rb

@@ -0,0 +1,114 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require "spec_helper"
+require "mortar/command/version"
+
+module Mortar::Command
+  describe Version do
+
+    before(:each) do
+      stub_core
+      
+    end
+    
+    base_url  = "http://install.mortardata.com"
+    base_version = "1.0"
+    tmp_dir_dumm = "/opt/mortar/installer"
+    curl_command = "sudo curl -sS -L -o #{tmp_dir_dumm}/install.sh #{base_url} && sudo bash #{tmp_dir_dumm}/install.sh"
+
+    context("version in prod") do
+      mortar_install_env = ENV['MORTAR_INSTALL']
+      before(:each) do
+        ENV['MORTAR_INSTALL'] = nil  
+      end
+
+      after(:all) do
+        ENV['MORTAR_INSTALL'] = mortar_install_env
+      end
+      it "makes a curl request to download default version" do
+        mock(FileUtils).remove_entry_secure(tmp_dir_dumm)
+        mock(Kernel).system (curl_command)
+        any_instance_of(Mortar::Command::Version) do |base|
+          mock(base).installed_with_omnibus? {true}
+          mock(base).running_on_a_mac? {true} 
+          execute("version:upgrade");
+        end
+      end
+      
+      it "makes curl request for different versions when requested" do
+        mortar_version = "1.0"
+        curl_command_with_version = curl_command +  " -v " + mortar_version
+        mock(FileUtils).remove_entry_secure(tmp_dir_dumm)
+        mock(FileUtils).remove_entry_secure(tmp_dir_dumm)
+        mock(Kernel).system( curl_command_with_version)
+        mock(Kernel).system( curl_command_with_version)
+        any_instance_of(Mortar::Command::Version) do |base|
+          mock(base).installed_with_omnibus? {true}
+          mock(base).installed_with_omnibus? {true}
+          mock(base).running_on_a_mac? {true} 
+          mock(base).running_on_a_mac? {true} 
+          execute( "upgrade -v #{mortar_version}");
+          execute( "version:upgrade --version #{mortar_version}");
+        end
+      end
+      
+    end
+
+    context("version dev") do
+      dev_url = "dev_url.com"
+      dev_curl =  "sudo curl -sS -L -o #{tmp_dir_dumm}/install.sh #{dev_url} && sudo bash #{tmp_dir_dumm}/install.sh" 
+      before(:each) do
+        ENV['MORTAR_INSTALL'] = dev_url
+      end
+
+      it "makes a curl request to download default version on dev" do
+        mock(Kernel).system(dev_curl)
+        
+        mock(FileUtils).remove_entry_secure(tmp_dir_dumm)
+        any_instance_of(Mortar::Command::Version) do |base|
+          mock(base).installed_with_omnibus? {true}
+          mock(base).running_on_a_mac? {true} 
+          execute("upgrade");
+        end
+      end
+    end
+
+    context("version not Mac OSX") do
+      it "throws error when not on mac" do
+        any_instance_of(Mortar::Command::Version) do |base|
+          mock(base).running_on_a_mac? {false} 
+          stderr, stdout = execute("version:upgrade");
+          stderr.should == <<-STDERR
+ !    mortar upgrade is currently only supported for OSX.
+STDERR
+        end
+      end
+
+      it "throws an error when running on a mac but installed with ruby gem" do
+        any_instance_of(Mortar::Command::Version) do |base|
+          mock(base).running_on_a_mac? {true}
+          mock(base).installed_with_omnibus? {false}
+          stderr, stdout = execute("version:upgrade");
+          stderr.should == <<-STDERR
+ !    mortar upgrade is only for installations not conducted with ruby gem.  Please upgrade by running 'gem install mortar'.
+STDERR
+        end
+      end
+    end
+  end
+    
+end
+

data/spec/mortar/local/python_spec.rb

@@ -64,5 +64,69 @@ module Mortar::Local
 
     end
 
+    context "mortar python dependencies" do
+      it "no install if not necessary" do
+        python = Mortar::Local::Python.new
+        stub(python).has_valid_virtualenv?.returns(true)
+        stub(python).should_do_requirements_install.returns(false)
+        mock(python).should_install_python_dependencies?.returns(false)
+        stub(python).install_python_dependencies { raise "Shouldn't be called" }
+        python.setup_project_python_environment
+      end
+      it "install if necessary" do
+        python = Mortar::Local::Python.new
+        stub(python).has_valid_virtualenv?.returns(true)
+        stub(python).should_do_requirements_install.returns(false)
+        mock(python).should_install_python_dependencies?.returns(true)
+        mock(python).install_python_dependencies.returns(true)
+        python.setup_project_python_environment
+      end
+
+      it "doesn't detect needing any update if all packages up to date" do
+        python = Mortar::Local::Python.new
+        stub(python).update_mortar_package?.returns(false)
+        expect(python.should_install_python_dependencies?).to equal(false)
+      end
+
+      it "detects necessary install if any packages need any update" do
+        python = Mortar::Local::Python.new
+        stub(python).update_mortar_package?.returns(true)
+        expect(python.should_install_python_dependencies?).to equal(true)
+      end
+
+      it "creates directory where install timestamps are stored" do
+        python = Mortar::Local::Python.new
+        stub(python).install_mortar_python_package.returns(true)
+        FakeFS do
+          FileUtils.mkdir_p "#{python.local_install_directory}/pythonenv"
+          capture_stdout do
+            python.install_python_dependencies
+          end
+          dir_to_be_created = "#{python.local_install_directory}/#{python.mortar_packages_dir}"
+          expect(File.exists?(dir_to_be_created)).to be_true
+        end
+      end
+
+      it "stops if installation failed" do
+        python = Mortar::Local::Python.new
+        stub(python).pip_install.returns(false)
+        stub(python).note_install { raise "Shouldn't be called" }
+        python.install_mortar_python_package "pip"
+      end
+
+      it "creates directory to store package install date" do
+        python = Mortar::Local::Python.new
+        stub(python).pip_install.returns(true)
+        mock(python).note_install(python.mortar_package_dir("pip"))
+        FakeFS do
+          FileUtils.mkdir_p "#{python.local_install_directory}/#{python.mortar_packages_dir}"
+          python.install_mortar_python_package "pip"
+          dir_to_be_created = "#{python.local_install_directory}/#{python.mortar_packages_dir}/pip"
+          expect(File.exists?(dir_to_be_created)).to be_true
+        end
+      end
+
+    end
+
   end
 end

data/spec/mortar/updater_spec.rb

@@ -87,9 +87,17 @@ module Mortar
           Mortar::VERSION = "1.0.0"
         end
         capture_stdout do
+          stub(Updater).installed_with_omnibus?{false}
+          stub(Updater).running_on_a_mac?{false}
           Updater.update_check
         end.should == "WARNING: There is a new version of the Mortar development framework available.  Please run 'gem install mortar' to install the latest version.\n\n"
+        capture_stdout do
+          mock(Updater).installed_with_omnibus?{true}
+          mock(Updater).running_on_a_mac?{true}
+          Updater.update_check
+        end.should == "WARNING: There is a new version of the Mortar development framework available.  Please run 'mortar upgrade' to install the latest version.\n\n"
+
       end
     end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.14.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-04 00:00:00.000000000 Z
+date: 2014-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -302,6 +302,7 @@ files:
 - spec/mortar/command/pigscripts_spec.rb
 - spec/mortar/command/projects_spec.rb
 - spec/mortar/command/validate_spec.rb
+- spec/mortar/command/version_spec.rb
 - spec/mortar/command_spec.rb
 - spec/mortar/git_spec.rb
 - spec/mortar/helpers_spec.rb
@@ -337,7 +338,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Client library and CLI to interact with the Mortar service.