mortar 0.7.5 → 0.7.6

data/lib/mortar/git.rb

@@ -87,7 +87,7 @@ module Mortar
           raise GitError, "No commits found in repository.  You must do an initial commit to initialize the repository."
         end
 
-        safe_copy do
+        safe_copy(mortar_snapshot_pathlist) do
           did_stash_changes = stash_working_dir("Stash for push to master")
           git('push mortar master')
         end
@@ -95,14 +95,14 @@ module Mortar
       end
 
       #
-      # Create a safe copy of the git directory
+      # Create a safe temporary directory with a given list of filesystem paths (files or dirs) copied into it
       #
 
-      def safe_copy(&block)
+      def safe_copy(pathlist, &block)
         # Copy code into a temp directory so we don't confuse editors while snapshotting
         curdir = Dir.pwd
         tmpdir = Dir.mktmpdir
-        FileUtils.cp_r(Dir.glob('*', File::FNM_DOTMATCH) - ['.', '..'], tmpdir)
+        FileUtils.cp_r(pathlist, tmpdir)
         Dir.chdir(tmpdir)
 
         if block
@@ -113,51 +113,77 @@ module Mortar
           return tmpdir
         end
       end
+
+      #
+      # Only snapshot filesystem paths that are in a whitelist
+      #
+
+      def mortar_snapshot_pathlist()
+        ensure_mortar_project_manifest_exists()
+
+        snapshot_pathlist = File.read('.mortar-project-manifest').split("\n")
+        snapshot_pathlist << ".git"
+
+        snapshot_pathlist.each do |path|
+          unless File.exists? path
+            Helpers.error(".mortar-project-manifest includes file/dir \"#{path}\" that is not in the mortar project directory.")
+          end
+        end
+        
+        snapshot_pathlist
+      end
+
+      #
+      # Create a snapshot whitelist file if it doesn't already exist
+      #
+      def ensure_mortar_project_manifest_exists()
+        unless File.exists? '.mortar-project-manifest'
+          create_mortar_project_manifest('.')
+        end
+      end
+
+      #
+      # Create a project manifest file
+      #
+      def create_mortar_project_manifest(path)
+        File.open("#{path}/.mortar-project-manifest", 'w') do |manifest|
+          if File.directory? "#{path}/controlscripts"
+            manifest.puts "controlscripts"
+          end
+          if File.directory? "#{path}/fixtures"
+            manifest.puts "fixtures"
+          end
+          manifest.puts "pigscripts"
+          manifest.puts "macros"
+          manifest.puts "udfs"
+        end
+      end
     
       #    
       # snapshot
       #
 
       def create_snapshot_branch
-
         # TODO: handle Ctrl-C in the middle
-        # TODO: can we do the equivalent of stash without changing the working directory
         unless has_commits?
           raise GitError, "No commits found in repository.  You must do an initial commit to initialize the repository."
         end
 
         # Copy code into a temp directory so we don't confuse editors while snapshotting
         curdir = Dir.pwd
-        tmpdir = safe_copy
+        tmpdir = safe_copy(mortar_snapshot_pathlist)
       
         starting_branch = current_branch
         snapshot_branch = "mortar-snapshot-#{Mortar::UUID.create_random.to_s}"
-        did_stash_changes = stash_working_dir(snapshot_branch)
-        begin
-          # checkout a new branch
-          git("checkout -b #{snapshot_branch}")
-        
-          if did_stash_changes
-            # apply the topmost stash that we just created
-            git("stash apply stash@{0}")
-          end
-        
-          add_untracked_files()
 
-          # commit the changes if there are any
-          if ! is_clean_working_directory?
-            git("commit -a -m \"mortar development snapshot commit\"")
-          end
-        
-        ensure
-        
-          # return to the starting branch
-          git("checkout #{starting_branch}")
+        # checkout a new branch
+        git("checkout -b #{snapshot_branch}")
+      
+        add_untracked_files()
 
-          # rebuild the original state of the working set
-          if did_stash_changes
-            git("stash pop stash@{0}")
-          end
+        # commit the changes if there are any
+        if ! is_clean_working_directory?
+          git("commit -a -m \"mortar development snapshot commit\"")
         end
       
         Dir.chdir(curdir)
@@ -176,11 +202,15 @@ module Mortar
 
         git_ref = Helpers.action("Sending code snapshot to Mortar") do
           # push the code
-          push(project.remote, snapshot_branch)
+          begin
+            push(project.remote, snapshot_branch)
+          rescue
+            retry if retry_snapshot_push?
+            Helpers.error("Could not connect to github remote. Tried #{@snapshot_push_attempts.to_s} times.")
+          end
 
-          # grab the commit hash and clean out the branch from the local branches
+          # grab the commit hash
           ref = git_ref(snapshot_branch)
-          branch_delete(snapshot_branch)
           ref
         end
 
@@ -189,6 +219,19 @@ module Mortar
         return git_ref
       end
 
+      def retry_snapshot_push?
+        @last_snapshot_retry_sleep_time ||= 0
+        @snapshot_retry_sleep_time ||= 1
+
+        sleep(@snapshot_retry_sleep_time)
+        @last_snapshot_retry_sleep_time, @snapshot_retry_sleep_time = 
+          @snapshot_retry_sleep_time, @last_snapshot_retry_sleep_time + @snapshot_retry_sleep_time
+
+        @snapshot_push_attempts ||= 0
+        @snapshot_push_attempts += 1
+        @snapshot_push_attempts < 10
+      end 
+
       #    
       # add
       #    

data/lib/mortar/local/controller.rb

@@ -92,16 +92,30 @@ EOF
     jy = Mortar::Local::Jython.new()
     jy.install_or_update()
 
-    ensure_local_install_dir_in_gitignore
+    ensure_local_install_dirs_in_gitignore
   end
 
-  def ensure_local_install_dir_in_gitignore()
+  def ensure_local_install_dirs_in_gitignore()
     if File.exists? local_project_gitignore
-      open(local_project_gitignore, 'r+') do |gitignore|
-        unless gitignore.read().include? local_install_directory_name
-          gitignore.seek(0, IO::SEEK_END)
+      File.open(local_project_gitignore, 'r+') do |gitignore|
+        contents = gitignore.read()
+        gitignore.seek(0, IO::SEEK_END)
+
+        unless contents[-1] == "\n"
+          gitignore.puts "" # write a newline
+        end
+
+        unless contents.include? local_install_directory_name
           gitignore.puts local_install_directory_name
         end
+
+        unless contents.include? "logs"
+          gitignore.puts "logs"
+        end
+
+        unless contents.include? "illustrate-output"
+          gitignore.puts "illustrate-output"
+        end
       end
     end
   end

data/lib/mortar/local/installutil.rb

@@ -42,8 +42,12 @@ module Mortar
         File.join(project_root, local_install_directory_name)
       end
 
-      def local_pig_logfile
-        project_root + "/local-pig.log"
+      def local_log_dir
+        project_root + "/logs"
+      end
+
+      def local_udf_log_dir
+        local_log_dir + "/udf"
       end
 
       def local_project_gitignore

data/lib/mortar/local/pig.rb

@@ -235,7 +235,7 @@ class Mortar::Local::Pig
   # get it to do something interesting, such as '-f some-file.pig'
   def run_pig_command(cmd, parameters = nil, jython_output = true)
     unset_hadoop_env_vars
-    delete_local_log_file
+    reset_local_logs
     # Generate the script for running the command, then
     # write it to a temp script which will be exectued
     script_text = script_for_command(cmd, parameters)
@@ -255,10 +255,12 @@ class Mortar::Local::Pig
     ENV['HADOOP_CONF_DIR'] = ''
   end
 
-  def delete_local_log_file
-    if File.exists? local_pig_logfile
-      FileUtils.rm local_pig_logfile
+  def reset_local_logs
+    if Dir.exists? local_log_dir
+      FileUtils.rm_rf local_log_dir
     end
+    Dir.mkdir local_log_dir
+    Dir.mkdir local_udf_log_dir
   end
 
   # Generates a bash script which sets up the necessary environment and
@@ -298,7 +300,8 @@ class Mortar::Local::Pig
     opts['fs.s3.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
     opts['fs.s3.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
     opts['pig.events.logformat'] = PIG_LOG_FORMAT
-    opts['pig.logfile'] = local_pig_logfile
+    opts['pig.logfile'] = local_log_dir + "/local-pig.log"
+    opts['pig.udf.scripting.log.dir'] = local_udf_log_dir
     opts['python.verbose'] = 'error'
     opts['jython.output'] = true
     opts['python.home'] = jython_directory

data/lib/mortar/templates/project/gitignore

@@ -3,5 +3,6 @@ Gemfile.lock
 *.class
 *.log
 tmp
-log
 .mortar-local
+logs
+illustrate-output

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.7.5"
+  VERSION = "0.7.6"
 end

data/spec/mortar/command/local_spec.rb

@@ -145,7 +145,7 @@ STDERR
           mock(j).install_or_update.returns(true)
         end
         any_instance_of(Mortar::Local::Controller) do |j|
-          mock(j).ensure_local_install_dir_in_gitignore.returns(true)
+          mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
         end
         stderr, stdout = execute("local:configure")
         stderr.should == ""

data/spec/mortar/git_spec.rb

@@ -16,8 +16,10 @@
 
 require "spec_helper"
 require "mortar/git"
+require "mortar/helpers"
 
 module Mortar
+
   describe Git do
     
     before do
@@ -280,6 +282,24 @@ STASH
         end
       end
       
+      it "retries pushing the snapshot branch if there is a socket error" do
+        with_git_initialized_project do |p|
+          # RR seems to only count a method as being called if it completes
+          # So we expect "never", even though it's actually tried N times (tested below)
+          mock(@git).push.never { raise Exception.new }
+          mock(@git).sleep.times(10).with_any_args
+
+          original_stdin, original_stderr, original_stdout = $stdin, $stderr, $stdout
+          $stdin, $stderr, $stdout = StringIO.new, StringIO.new, StringIO.new
+
+          begin
+            @git.create_and_push_snapshot_branch(p)
+          rescue SystemExit
+          ensure
+            $stdin, $stderr, $stdout = original_stdin, original_stderr, original_stdout
+          end
+        end
+      end
     end
     
 =begin

data/spec/spec_helper.rb

@@ -30,6 +30,7 @@ Excon.defaults[:mock] = true
 require "mortar-api-ruby"
 
 require "mortar/cli"
+require "mortar/git"
 require "rspec"
 require "rr"
 require "fakefs/safe"
@@ -163,6 +164,9 @@ def with_blank_project(&block)
   FileUtils.mkdir_p(File.join(project_path, "controlscripts"))
   FileUtils.mkdir_p(File.join(project_path, "pigscripts"))
   FileUtils.mkdir_p(File.join(project_path, "macros"))
+  FileUtils.mkdir_p(File.join(project_path, "udfs"))
+  FileUtils.mkdir_p(File.join(project_path, "udfs/python"))
+  FileUtils.mkdir_p(File.join(project_path, "udfs/jython"))
 
   Dir.chdir(project_path)
   
@@ -188,9 +192,11 @@ end
 def with_git_initialized_project(&block)
   # wrap block in a proc that does a commit
   commit_proc = Proc.new do |project|
-    write_file(File.join(project.root_path, "README.txt"), "Some README text")
+    git = Mortar::Git::Git.new
+    git.create_mortar_project_manifest(project.root_path)
+
     remote = "mortar"
-    `git add README.txt`
+    `git add .mortar-project-manifest`
     `git commit -a -m "First commit"`
     `git remote add #{remote} git@github.com:mortarcode-dev/4dbbd83cae8d5bf8a4000000_#{project.name}.git`
     project.remote = remote
@@ -269,10 +275,15 @@ def create_and_validate_git_snapshot(git)
   # ensure the snapshot branch exists
   git.git("branch").include?(snapshot_branch).should be_true
 
+  snapshotted_paths = Dir.glob("**/*")
+  snapshotted_paths.should include("controlscripts")
+  snapshotted_paths.should include("pigscripts")
+  snapshotted_paths.should include("macros")
+  snapshotted_paths.should include("udfs/python")
+  snapshotted_paths.should include("udfs/jython")
+
   Dir.chdir(curdir)
   FileUtils.remove_entry_secure(snapshot_dir)
-
-  git.branches.should == initial_git_branches
 end
 
 require "mortar/helpers"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 15
   prerelease: 
   segments: 
   - 0
   - 7
-  - 5
-  version: 0.7.5
+  - 6
+  version: 0.7.6
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-04-18 00:00:00 Z
+date: 2013-04-23 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mortar-api-ruby