mortar 0.15.20 → 0.15.21

data/lib/mortar/command/base.rb

@@ -461,19 +461,25 @@ protected
   end
 
   def load_defaults(section_name)
-    if File.exists?('.mortar-defaults')
-      default_options = ParseConfig.new('.mortar-defaults')
-      if default_options.groups.include?(section_name)
-        default_options[section_name].each do |k, v|
-          unless @original_options.include? k.to_sym
-            if v == 'true'
-              v = true
-            elsif v == 'false'
-              v = false
-            end
-                
-            @options[k.to_sym] = v
+    if File.exists?('project.properties')
+      load_defaults_from_file('project.properties', section_name)
+    elsif File.exists?('.mortar-defaults')
+      load_defaults_from_file('.mortar-defaults', section_name)
+    end
+  end
+
+  def load_defaults_from_file(file_name, section_name)
+    default_options = ParseConfig.new(file_name)
+    if default_options.groups.include?(section_name)
+      default_options[section_name].each do |k, v|
+        unless @original_options.include? k.to_sym
+          if v == 'true'
+            v = true
+          elsif v == 'false'
+            v = false
           end
+                
+          @options[k.to_sym] = v
         end
       end
     end

data/lib/mortar/command/local.rb

@@ -249,7 +249,14 @@ class Mortar::Command::Local < Mortar::Command::Base
       error("No such directory #{project_root}")
     end
     Dir.chdir(project_root)
+
     script = validate_luigiscript!(script_name)
+    
+    #Set git ref as environment variable for mortar-luigi to use when
+    #running a MortarTask
+    git_ref = sync_code_with_cloud()
+    ENV['MORTAR_LUIGI_GIT_REF'] = git_ref
+
     ctrl = Mortar::Local::Controller.new
     luigi_params = pig_parameters.sort_by { |p| p['name'] }
     luigi_params = luigi_params.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten

data/lib/mortar/command/projects.rb

@@ -37,6 +37,13 @@ class Mortar::Command::Projects < Mortar::Command::Base
       display("You have no projects.")
     end
   end
+
+  # projects:list
+  #
+  # Display the available set of Mortar projects.
+  def list
+    index
+  end
   
   # projects:delete PROJECTNAME
   #

data/lib/mortar/generators/project_generator.rb

@@ -32,8 +32,11 @@ module Mortar
           @dest_path = File.join(@dest_path, project_name)
           
           copy_file "README.md", "README.md"
+          copy_file "project.properties", "project.properties"
+          copy_file "project.manifest", "project.manifest"
+          copy_file "requirements.txt", "requirements.txt"
           copy_file "gitignore", ".gitignore"
-          
+
           mkdir "pigscripts"
           
           inside "pigscripts" do
@@ -65,9 +68,20 @@ module Mortar
             end
           end
 
+          mkdir "luigiscripts"
+
+          inside "luigiscripts" do
+            copy_file "README", "README"
+          end
+
           mkdir "lib"
           inside "lib" do
-            copy_file "gitkeep", ".gitkeep"
+            copy_file "README", "README"
+          end
+
+          mkdir "params"
+          inside "params" do
+            copy_file "README", "README"
           end
 
         rescue => e 

data/lib/mortar/git.rb

@@ -102,7 +102,8 @@ module Mortar
         # Copy code into a temp directory so we don't confuse editors while snapshotting
         curdir = Dir.pwd
         tmpdir = Dir.mktmpdir
-        FileUtils.cp_r(pathlist, tmpdir)
+
+        copy_files_to_dir(pathlist, tmpdir)        
         Dir.chdir(tmpdir)
 
         if block
@@ -114,6 +115,23 @@ module Mortar
         end
       end
 
+      def copy_files_to_dir(pathlist, dest_dir)
+        #Used to copy the pathlist from the manifest to a separate directory
+        #before syncing.
+        pathlist.each do |path|
+          dir, file = File.split(path)
+
+          #For non-root files/directories we need to create the parent
+          #directories before copying.
+          unless dir == "."
+            FileUtils.mkdir_p(File.join(dest_dir, dir))
+          end
+
+          FileUtils.cp_r(path, File.join(dest_dir, dir))
+        end
+
+      end
+
       #
       # Only snapshot filesystem paths that are in a whitelist
       #
@@ -121,37 +139,56 @@ module Mortar
       def mortar_manifest_pathlist(include_dot_git = true)
         ensure_valid_mortar_project_manifest()
 
-        manifest_pathlist = File.read(".mortar-project-manifest").split("\n")
+        manifest_pathlist = File.read(project_manifest_name).split("\n")
         if include_dot_git
           manifest_pathlist << ".git"
         end
 
+        #Strip out comments and empty lines
+        manifest_pathlist = manifest_pathlist.select do |path|
+          s_path = path.strip
+          !s_path.start_with?("#") && !s_path.empty?
+        end
+
         manifest_pathlist.each do |path|
           unless File.exists? path
-            Helpers.error(".mortar-project-manifest includes file/dir \"#{path}\" that is not in the mortar project directory.")
+            Helpers.error("#{project_manifest_name} includes file/dir \"#{path}\" that is not in the mortar project directory.")
           end
         end
         
         manifest_pathlist
       end
 
+      def add_newline_to_file(path)
+        File.open(path, "r+") do |manifest|
+          contents = manifest.read()
+          manifest.seek(0, IO::SEEK_END)
+
+          # `contents` in ruby 1.8.7 is array with entries of the
+          # type Fixnum which isn't semantically comparable with
+          # the \n char, but the ascii code 10 is
+          unless (contents[-1] == "\n" or contents[-1] == 10)
+            manifest.puts "" # ensure file ends with a newline
+          end
+        end
+      end
+
+      def project_manifest_name()
+        if File.exists? "project.manifest"
+          "project.manifest"
+        elsif File.exists? ".mortar-project-manifest"
+          ".mortar-project-manifest"
+        else
+          "project.manifest"
+        end
+      end
+
       #
       # Create a snapshot whitelist file if it doesn't already exist
       #
       def ensure_valid_mortar_project_manifest()
-        if File.exists? ".mortar-project-manifest"
-          File.open(".mortar-project-manifest", "r+") do |manifest|
-            contents = manifest.read()
-            manifest.seek(0, IO::SEEK_END)
-
-            # `contents` in ruby 1.8.7 is array with entries of the
-            # type Fixnum which isn't semantically comparable with
-            # the \n char, but the ascii code 10 is
-            unless (contents[-1] == "\n" or contents[-1] == 10)
-              manifest.puts "" # ensure file ends with a newline
-            end
-
-          end
+        if File.exists? project_manifest_name
+          add_newline_to_file(project_manifest_name)
         else
           create_mortar_project_manifest('.')
         end
@@ -161,10 +198,11 @@ module Mortar
       # Create a project manifest file
       #
       def create_mortar_project_manifest(path)
-        File.open("#{path}/.mortar-project-manifest", 'w') do |manifest|
-          manifest.puts "pigscripts"
+        File.open("#{path}/#{project_manifest_name}", 'w') do |manifest|
           manifest.puts "macros"
+          manifest.puts "pigscripts"
           manifest.puts "udfs"
+
           if File.directory? "#{path}/lib"
             manifest.puts "lib"
           end
@@ -240,7 +278,6 @@ module Mortar
         # the project is not a git repo, so we manage a mirror directory that is a git repo
         # branch is which branch to sync to. this will be master if the cloud repo
         # is being initialized, or a branch based on the user's name in any other circumstance
-
         project_dir = project.root_path
         mirror_dir = "#{mortar_mirrors_dir}/#{project.name}"
 
@@ -260,6 +297,7 @@ module Mortar
           end
 
           # clone mortar-code repo
+          ensure_valid_mortar_project_manifest()
           remote_path = File.open(".mortar-project-remote").read.strip
           clone(remote_path, mirror_dir)
 
@@ -322,7 +360,7 @@ module Mortar
         # mortar_manifest_pathlist(false) means don't copy .git
         FileUtils.rm_rf(Dir.glob("#{mirror_dir}/*"))
         Dir.chdir(project_dir)
-        FileUtils.cp_r(mortar_manifest_pathlist(false), mirror_dir)
+        copy_files_to_dir(mortar_manifest_pathlist(false), mirror_dir)
 
         # update remote branch
         Dir.chdir(mirror_dir)

data/lib/mortar/local/controller.rb

@@ -151,9 +151,32 @@ EOF
       sqoop.install_or_update()
     end
 
+    write_local_readme
+
     ensure_local_install_dirs_in_gitignore
   end
 
+  def write_local_readme()
+    readme_path = File.join(local_install_directory, "README")
+    unless File.exists? readme_path
+      file = File.new(readme_path, "w")
+      file.write(<<-README
+This directory is used by Mortar to install all of the necessary dependencies for
+running mortar local commands.  You should not modify these files/directories as
+they may be removed or updated at any time.
+
+For additional Java dependencies you should place your jars in the root lib folder
+of your project.  These jars will be automatically registered and 
+available for use in your Pig scripts and UDFs.
+
+You can specify additional Python dependencies in the requirements.txt file in 
+the root of your project.
+README
+)
+      file.close
+    end
+  end
+
   def ensure_local_install_dirs_in_gitignore()
     if File.exists? local_project_gitignore
       File.open(local_project_gitignore, 'r+') do |gitignore|

data/lib/mortar/local/pig.rb

@@ -410,6 +410,14 @@ class Mortar::Local::Pig
       params['MORTAR_EMAIL_S3_ESCAPED'] = Mortar::Auth.user_s3_safe(true)
     end
 
+    if ENV['MORTAR_PROJECT_ROOT']
+      params['MORTAR_PROJECT_ROOT'] = ENV['MORTAR_PROJECT_ROOT']
+    else
+      params['MORTAR_PROJECT_ROOT'] = project_root
+      ENV['MORTAR_PROJECT_ROOT'] = params['MORTAR_PROJECT_ROOT']
+    end
+
+
     # Coerce into the same format as pig parameters that were
     # passed in via the command line or a parameter file
     param_list = []

data/lib/mortar/templates/project/lib/README

@@ -0,0 +1 @@
+To make it easy to bundle dependencies, you can include any needed jars in the lib folder of your project. Any jars in this folder will be automatically registered and available for use in your Pig scripts and UDFs.

data/lib/mortar/templates/project/luigiscripts/README

@@ -0,0 +1 @@
+The luigiscripts directory is where your luigi scripts should be stored.  For more information about using Luigi with Mortar see: http://help.mortardata.com/technologies/luigi.

data/lib/mortar/templates/project/params/README

@@ -0,0 +1 @@
+The params directory is a place to store various param files for your project. For more information about how you can use parameters with Mortar see http://help.mortardata.com/technologies/mortar/local_development.

data/lib/mortar/templates/project/project.manifest

@@ -0,0 +1,12 @@
+# This file lists which files and directories the Mortar Development Framework
+# should sync with the cloud. By default, it includes only the standard Mortar
+# directories.  If you have extra resources that your code depends on, you can
+# add them to this file.
+#
+# See http://help.mortardata.com/technologies/mortar/how_mortar_projects_work
+# for more information.
+
+lib
+macros
+pigscripts
+udfs

data/lib/mortar/templates/project/project.properties

@@ -0,0 +1,11 @@
+# This file allows you to set project level properties.
+
+
+# The defaults section can be used to set the default value used for any
+# Mortar parameter.  For more details see:
+#
+#   http://help.mortardata.com/data_apps/build_your_own/running_a_mortar_project#toc_7SettingProjectDefaults
+
+[DEFAULTS]
+# Default new projects to using Pig 0.12
+pigversion=0.12

data/lib/mortar/templates/project/requirements.txt

@@ -0,0 +1,17 @@
+# Mortar comes with the Python standard libraries, NumPy, SciPy, and NLTK 
+# pre-installed. You can install additional Python packages by listing them out
+# here in pip requirements (https://pip.pypa.io/en/latest/reference/pip_install.html#requirements-file-format) format.
+#
+# Examples:
+#
+# 	Version 2.1 of python-dateutil
+#		python-dateutil==2.1
+#
+# 	Version 2.1 or higher of python-dateutil
+#		python-dateutil>=2.1
+#
+# 	Latest version of python-dateutil
+#		python-dateutil
+#
+# 	Your private library stored in S3
+#		s3://my-bucket/my-custom-lib.tar.gz

data/lib/mortar/templates/script/runpython.sh

@@ -11,4 +11,4 @@ export LUIGI_CONFIG_PATH=`pwd`/`dirname <%= @python_script %>`/client.cfg
 <%= @local_install_dir %>/pythonenv/bin/python \
     <%= @python_arugments %> \
     <%= @python_script %> \
-    <%= @script_arguments %>
+    <%= @script_arguments %>

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.20"
+  VERSION = "0.15.21"
 end

data/spec/mortar/command/base_spec.rb

@@ -213,7 +213,7 @@ no_browser=true
         end
       end
 
-      it "obeys proper overrides" do
+      it "obeys proper overrides - deprecated file" do
         stub_core
         git = Mortar::Git::Git.new
 
@@ -241,6 +241,31 @@ polling_interval=10
         end
       end
 
+      it "obeys proper overrides" do
+        stub_core
+        git = Mortar::Git::Git.new
+
+        with_git_initialized_project do |p|
+          text = """
+[DEFAULTS]
+clustersize=5
+no_browser=true
+pigversion=0.12
+"""
+          write_file(File.join(p.root_path, "project.properties"), text)
+
+          describe_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          describe_url = "https://api.mortardata.com/describe/#{describe_id}"
+
+          mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters=>[]) {Excon::Response.new(:body => {"describe_id" => describe_id})}
+          mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_SUCCESS, "status_description" => "Success", "web_result_url" => describe_url})).ordered
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+
+          stderr, stdout, d = execute_and_return_command("describe pigscripts/my_script.pig my_alias --polling_interval 0.05", p, git)
+          d.options.should == {:polling_interval => "0.05", :no_browser => true, :clustersize => "5", :pigversion => "0.12"}
+        end
+      end
+
     end
 
   end

data/spec/mortar/command/local_spec.rb

@@ -210,6 +210,7 @@ PARAMS
           mock(j).install_or_update.returns(true)
         end
         any_instance_of(Mortar::Local::Controller) do |j|
+          mock(j).write_local_readme
           mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
         end
         stderr, stdout = execute("local:configure")
@@ -301,8 +302,13 @@ STDERR
           any_instance_of(Mortar::Local::Controller) do |u|
             mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig09),'luigi')
           end
+          any_instance_of(Mortar::Command::Local) do |u|
+            mock(u).sync_code_with_cloud().returns("some-git-ref")
+          end
+          ENV['MORTAR_LUIGI_GIT_REF'].should be_nil
           stderr, stdout = execute("local:luigi some_luigi_script -p myoption=2 -p myotheroption=3", p)
           stderr.should == ""
+          ENV['MORTAR_LUIGI_GIT_REF'].should == 'some-git-ref'
         end
       end
 

data/spec/mortar/command/projects_spec.rb

@@ -115,11 +115,17 @@ STDOUT
         File.exists?("macros").should be_true
         File.exists?("pigscripts").should be_true
         File.exists?("udfs").should be_true
+        File.exists?("project.properties").should be_true
+        File.exists?("project.manifest").should be_true
+        File.exists?("requirements.txt").should be_true
         File.exists?("README.md").should be_true
         File.exists?("Gemfile").should be_false
         File.exists?("macros/.gitkeep").should be_true
         File.exists?("pigscripts/some_new_project.pig").should be_true
         File.exists?("udfs/python/some_new_project.py").should be_true
+        File.exists?("luigiscripts/README").should be_true
+        File.exists?("lib/README").should be_true
+        File.exists?("params/README").should be_true
 
         File.read("pigscripts/some_new_project.pig").each_line { |line| line.match(/<%.*%>/).should be_nil }
 
@@ -127,6 +133,9 @@ STDOUT
 Sending request to register project: some_new_project... done
 \e[1;32m      create\e[0m  
 \e[1;32m      create\e[0m  README.md
+\e[1;32m      create\e[0m  project.properties
+\e[1;32m      create\e[0m  project.manifest
+\e[1;32m      create\e[0m  requirements.txt
 \e[1;32m      create\e[0m  .gitignore
 \e[1;32m      create\e[0m  pigscripts
 \e[1;32m      create\e[0m  pigscripts/some_new_project.pig
@@ -139,8 +148,12 @@ Sending request to register project: some_new_project... done
 \e[1;32m      create\e[0m  udfs/jython/.gitkeep
 \e[1;32m      create\e[0m  udfs/java
 \e[1;32m      create\e[0m  udfs/java/.gitkeep
+\e[1;32m      create\e[0m  luigiscripts
+\e[1;32m      create\e[0m  luigiscripts/README
 \e[1;32m      create\e[0m  lib
-\e[1;32m      create\e[0m  lib/.gitkeep
+\e[1;32m      create\e[0m  lib/README
+\e[1;32m      create\e[0m  params
+\e[1;32m      create\e[0m  params/README
 \n\r\e[0KStatus: ACTIVE  \n\nYour project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n
 NOTE: You'll need to change to the new directory to use your project:
     cd some_new_project

data/spec/mortar/git_spec.rb

@@ -454,6 +454,46 @@ STASH
       end
     end 
 
+    context "safe_copy for_snapshot" do
+      it "safe copies properly" do 
+        tmp_dir_src = Dir.mktmpdir
+        Dir.chdir(tmp_dir_src)
+
+        root_file = File.join("root_file")
+        root_dir_file = File.join("root_dir", "root_dir_file")
+        non_root_dir_file = File.join("dir", "non_root_dir", "non_root_dir_file")
+        dir_file = File.join("dir", "dir_file")
+
+        ex_root_file = File.join("excluded_root_file")
+        ex_root_dir_file = File.join("excluded_dir", "excluded_dir_file")
+        ex_dir_file = File.join("dir", "excluded_dir_file")
+        ex_non_root_dir_file = File.join("dir", "excluded_non_root_dir", "excluded_non_root_dir_file")
+
+        write_file(root_file)
+        write_file(root_dir_file)
+        write_file(non_root_dir_file)
+        write_file(dir_file)
+        write_file(ex_root_file)
+        write_file(ex_root_dir_file)
+        write_file(ex_dir_file)
+        write_file(ex_non_root_dir_file)
+
+        pathlist = ["root_file", "root_dir", "dir/non_root_dir", "dir/dir_file"]
+        new_tmp_dir = @git.safe_copy(pathlist)
+
+        Dir.chdir(new_tmp_dir)
+        File.exists?(root_file).should be_true
+        File.exists?(root_dir_file).should be_true
+        File.exists?(non_root_dir_file).should be_true
+        File.exists?(dir_file).should be_true
+
+        File.exists?(ex_root_file).should be_false
+        File.exists?(ex_root_dir_file).should be_false
+        File.exists?(ex_dir_file).should be_false
+        File.exists?(ex_non_root_dir_file).should be_false
+      end
+    end
+
       
 =begin
     #TODO: Fix this.

data/spec/mortar/local/controller_spec.rb

@@ -147,6 +147,7 @@ module Mortar::Local
           mock(j).install_or_update
         end
 
+        mock(ctrl).write_local_readme
         mock(ctrl).ensure_local_install_dirs_in_gitignore
         ctrl.install_and_configure
       end
@@ -173,6 +174,7 @@ module Mortar::Local
           mock(s).install_or_update
         end
 
+        mock(ctrl).write_local_readme
         mock(ctrl).ensure_local_install_dirs_in_gitignore
         ctrl.install_and_configure(Mortar::PigVersion::Pig012.new, command, true)
       end

data/spec/spec_helper.rb

@@ -229,7 +229,7 @@ def with_git_initialized_project_with_remote_prefix(remote_prefix, name="myproje
     git.create_mortar_project_manifest(project.root_path)
 
     remote = "mortar"
-    `git add .mortar-project-manifest`
+    `git add project.manifest`
     `git commit -a -m "First commit"`
     `git remote add #{remote} git@github.com:mortarcode-dev/#{remote_prefix}#{project.name}.git`
     project.remote = remote
@@ -366,6 +366,7 @@ def create_and_validate_git_snapshot(git)
   FileUtils.remove_entry_secure(snapshot_dir)
 end
 
+
 require "mortar/helpers"
 module Mortar::Helpers
   @home_directory = Dir.mktmpdir

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.15.20
+  version: 0.15.21
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-24 00:00:00.000000000 Z
+date: 2014-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -303,9 +303,14 @@ files:
 - lib/mortar/templates/project/README.md
 - lib/mortar/templates/project/controlscripts/lib/__init__.py
 - lib/mortar/templates/project/gitignore
-- lib/mortar/templates/project/lib/gitkeep
+- lib/mortar/templates/project/lib/README
+- lib/mortar/templates/project/luigiscripts/README
 - lib/mortar/templates/project/macros/gitkeep
+- lib/mortar/templates/project/params/README
 - lib/mortar/templates/project/pigscripts/pigscript.pig
+- lib/mortar/templates/project/project.manifest
+- lib/mortar/templates/project/project.properties
+- lib/mortar/templates/project/requirements.txt
 - lib/mortar/templates/project/udfs/java/gitkeep
 - lib/mortar/templates/project/udfs/jython/gitkeep
 - lib/mortar/templates/project/udfs/python/python_udf.py