git-fastclone 1.0.16 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cfb5c1a85a5af61f83695373c9e087a382c7dbf0
-  data.tar.gz: b07120a322ee4ca8ac13d0cd98c1251967a70744
+  metadata.gz: 6a48b08fe2a5e4f22ce995ab3c7801c93fb0562f
+  data.tar.gz: 43cc697ec456e6d699415fee25977f824677aff6
 SHA512:
-  metadata.gz: 1b5cfe0a83e6232ae2475d284cfca371d840a0a950bcb4de30bca64f2d94f2a55362d242ac6ec8698c5b6d84304c08d1f1e58d990b892cd930faffabff4c89a5
-  data.tar.gz: d5f62ae9d1e60ede4d3e51c991593d4b6874433f8f3c4c55abf17387d1ebdeff19fb9171449e134ba23cef5b5e664a6376fb3f54c5c590d31d1300b6b658c817
+  metadata.gz: 6aea5e53d2b7c774fe5c03c6cfebff3eb95747281f18e52daa81685e4a298919d85bc288f39421d9079e1bf17351636edbb64b3fd41c7577b9ea958aca9b90e3
+  data.tar.gz: 465b9bd78a32f22994c584391a08d20277ff968913337e08ffdd6073f0ca9267eac0338f418aa39a45bbc39c0c32b7bfa4255ff3567b1064cccff879d7aa00d2

data/lib/git-fastclone.rb

@@ -16,6 +16,7 @@ require 'optparse'
 require 'fileutils'
 require 'logger'
 require 'cocaine'
+require 'timeout'
 
 # Contains helper module UrlHelper and execution class GitFastClone::Runner
 module GitFastClone
@@ -32,7 +33,7 @@ module GitFastClone
     module_function :parse_update_info
 
     def reference_repo_name(url)
-      "#{url.gsub(%r{^.*://}, '').gsub(/^[^@]*@/, '').tr('/', '-').tr(':', '-')}"
+      url.gsub(%r{^.*://}, '').gsub(/^[^@]*@/, '').tr('/', '-').tr(':', '-').to_s
     end
     module_function :reference_repo_name
 
@@ -49,6 +50,12 @@ module GitFastClone
       "#{reference_repo_dir(url, reference_dir, using_local_repo)}:submodules.txt"
     end
     module_function :reference_repo_submodule_file
+
+    def reference_repo_lock_file(url, reference_dir, using_local_repo)
+      lock_file_name = "#{reference_repo_dir(url, reference_dir, using_local_repo)}:lock"
+      File.open(lock_file_name, File::RDWR | File::CREAT, 0644)
+    end
+    module_function :reference_repo_lock_file
   end
 
   # Spawns one thread per submodule, and updates them in parallel. They will be
@@ -60,12 +67,13 @@ module GitFastClone
 
     include GitFastClone::UrlHelper
 
-    DEFAULT_REFERENCE_REPO_DIR = '/var/tmp/git-fastclone/reference'
+    DEFAULT_REFERENCE_REPO_DIR = '/var/tmp/git-fastclone/reference'.freeze
 
-    DEFAULT_GIT_ALLOW_PROTOCOL = 'file:git:http:https:ssh'
+    DEFAULT_GIT_ALLOW_PROTOCOL = 'file:git:http:https:ssh'.freeze
 
-    attr_accessor :reference_dir, :prefetch_submodules, :reference_mutex, :reference_updated,
-                  :options, :logger, :abs_clone_path, :using_local_repo, :verbose, :color
+    attr_accessor :reference_dir, :prefetch_submodules, :reference_updated, :reference_mutex,
+                  :options, :logger, :abs_clone_path, :using_local_repo, :verbose, :color,
+                  :flock_timeout_secs
 
     def initialize
       # Prefetch reference repos for submodules we've seen before
@@ -93,6 +101,8 @@ module GitFastClone
       self.verbose = false
 
       self.color = false
+
+      self.flock_timeout_secs = 0
     end
 
     def run
@@ -112,7 +122,7 @@ module GitFastClone
       clone(url, options[:branch], path)
     end
 
-    def parse_inputs
+    def parse_options
       usage = 'Usage: git fastclone [options] <git-url> [path]'
 
       # One option --branch=<branch>  We're not as brittle as clone. That branch
@@ -137,7 +147,16 @@ module GitFastClone
         opts.on('-c', '--color', 'Display colored output') do
           self.color = true
         end
+
+        opts.on('--lock-timeout N', 'Timeout in seconds to acquire a lock on any reference repo.
+                Default is 0 which waits indefinitely.') do |timeout_secs|
+          self.flock_timeout_secs = timeout_secs
+        end
       end.parse!
+    end
+
+    def parse_inputs
+      parse_options
 
       unless ARGV[0]
         STDERR.puts usage
@@ -172,13 +191,15 @@ module GitFastClone
 
       with_git_mirror(url) do |mirror|
         Cocaine::CommandLine.new('git clone', '--quiet --reference :mirror :url :path')
-          .run(mirror: "#{mirror}", url: "#{url}", path: "#{File.join(abs_clone_path, src_dir)}")
+          .run(mirror: mirror.to_s,
+               url: url.to_s,
+               path: File.join(abs_clone_path, src_dir).to_s)
       end
 
       # Only checkout if we're changing branches to a non-default branch
       if rev
         Dir.chdir(File.join(abs_clone_path, src_dir)) do
-          Cocaine::CommandLine.new('git checkout', '--quiet :rev').run(rev: "#{rev}")
+          Cocaine::CommandLine.new('git checkout', '--quiet :rev').run(rev: rev.to_s)
         end
       end
 
@@ -218,14 +239,34 @@ module GitFastClone
       threads << Thread.new do
         with_git_mirror(submodule_url) do |mirror|
           Cocaine::CommandLine.new('cd', ':dir; git submodule update --quiet --reference :mirror :path')
-            .run(dir: "#{File.join(abs_clone_path, pwd)}", mirror: "#{mirror}", path: "#{submodule_path}")
+            .run(dir: File.join(abs_clone_path, pwd).to_s,
+                 mirror: mirror.to_s,
+                 path: submodule_path.to_s)
         end
 
         update_submodules(File.join(pwd, submodule_path), submodule_url)
       end
     end
 
-    def with_reference_repo_lock(url)
+    def with_reference_repo_lock(url, &block)
+      # Sane POSIX implementations remove exclusive flocks when a process is terminated or killed
+      # We block here indefinitely. Waiting for other git-fastclone processes to release the lock.
+      # With the default timeout of 0 we will wait forever, this can be overridden on the command line.
+      lockfile = reference_repo_lock_file(url, reference_dir, using_local_repo)
+      Timeout.timeout(flock_timeout_secs) { lockfile.flock(File::LOCK_EX) }
+      with_reference_repo_thread_lock(url, &block)
+    ensure
+      # Not strictly necessary to do this unlock as an ensure. If ever exception is caught outside this
+      # primitive, ensure protection may come in handy.
+      lockfile.flock(File::LOCK_UN)
+      lockfile.close
+    end
+
+    def with_reference_repo_thread_lock(url)
+      # We also need thread level locking because pre-fetch means multiple threads can
+      # attempt to update the same repository from a single git-fastclone process
+      # file locks in posix are tracked per process, not per userland thread.
+      # This gives us the equivalent of pthread_mutex around these accesses.
       reference_mutex[reference_repo_name(url)].synchronize do
         yield
       end
@@ -275,7 +316,7 @@ module GitFastClone
     def store_updated_repo(url, mirror, repo_name, fail_hard)
       unless Dir.exist?(mirror)
         Cocaine::CommandLine.new('git clone', '--mirror :url :mirror')
-          .run(url: "#{url}", mirror: "#{mirror}")
+          .run(url: url.to_s, mirror: mirror.to_s)
       end
 
       Cocaine::CommandLine.new('cd', ':path; git remote update --prune').run(path: mirror)

data/lib/git-fastclone/version.rb

@@ -1,3 +1,4 @@
+# Version string for git-fastclone
 module GitFastCloneVersion
-  VERSION = '1.0.16'
+  VERSION = '1.1.0'.freeze
 end

data/spec/git_fastclone_runner_spec.rb

@@ -22,6 +22,14 @@ describe GitFastClone::Runner do
   let(:test_reference_repo_dir) { '/var/tmp/git-fastclone/reference/test_reference_dir' }
   let(:placeholder_arg) { 'PH' }
 
+  let(:lockfile) do
+    lockfile = double
+    expect(lockfile).to receive(:flock).with(File::LOCK_EX).once
+    expect(lockfile).to receive(:flock).with(File::LOCK_UN).once
+    expect(lockfile).to receive(:close).once
+    lockfile
+  end
+
   # Modified ARGV, watch out
   ARGV = ['ssh://git@git.com/git-fastclone.git', 'test_reference_dir']
 
@@ -42,7 +50,7 @@ describe GitFastClone::Runner do
   end
 
   describe '.run' do
-    let(:options) { {:branch => placeholder_arg} }
+    let(:options) { { branch: placeholder_arg } }
 
     it 'should run with the correct args' do
       allow(subject).to receive(:parse_inputs) { [placeholder_arg, placeholder_arg, options] }
@@ -58,7 +66,7 @@ describe GitFastClone::Runner do
       subject.options = {}
       allow(FileUtils).to receive(:mkdir_p) {}
 
-      expect(subject.parse_inputs).to eq([test_url_valid, test_reference_dir, {:branch=>nil}])
+      expect(subject.parse_inputs).to eq([test_url_valid, test_reference_dir, { branch: nil }])
     end
   end
 
@@ -104,6 +112,7 @@ describe GitFastClone::Runner do
     it 'should acquire a lock' do
       allow(Mutex).to receive(:synchronize)
       expect(Mutex).to respond_to(:synchronize)
+      expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
 
       subject.with_reference_repo_lock(test_url_valid) do
         yielded << test_url_valid
@@ -111,6 +120,17 @@ describe GitFastClone::Runner do
 
       expect(yielded).to eq([test_url_valid])
     end
+    it 'should un-flock on thrown exception' do
+      allow(Mutex).to receive(:synchronize)
+      expect(Mutex).to respond_to(:synchronize)
+      expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
+
+      expect do
+        subject.with_reference_repo_lock(test_url_valid) do
+          fail placeholder_arg
+        end
+      end.to raise_error(placeholder_arg)
+    end
   end
 
   describe '.update_submodule_reference' do
@@ -130,6 +150,7 @@ describe GitFastClone::Runner do
         allow(subject).to receive(:reference_repo_name) {}
         allow(subject).to receive(:reference_repo_submodule_file) {}
         expect(File).to receive(:open)
+        expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
 
         subject.update_submodule_reference(placeholder_arg, [placeholder_arg, placeholder_arg])
       end
@@ -141,6 +162,7 @@ describe GitFastClone::Runner do
       it 'should grab the children immediately and then store' do
         expect(subject).to receive(:prefetch).once
         expect(subject).to receive(:store_updated_repo).once
+        expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
 
         allow(File).to receive(:exist?) { true }
         subject.prefetch_submodules = true
@@ -153,6 +175,7 @@ describe GitFastClone::Runner do
       it 'should store the updated repo' do
         expect(subject).not_to receive(:prefetch)
         expect(subject).to receive(:store_updated_repo).once
+        expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
 
         allow(File).to receive(:exist?) { true }
         subject.prefetch_submodules = false
@@ -180,6 +203,7 @@ describe GitFastClone::Runner do
       it 'should store' do
         placeholder_hash[placeholder_arg] = false
         expect(subject).to receive(:store_updated_repo)
+        expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
 
         allow(subject).to receive(:reference_repo_name) { placeholder_arg }
         subject.reference_updated = placeholder_hash
@@ -244,6 +268,7 @@ describe GitFastClone::Runner do
     it 'should yield properly' do
       allow(subject).to receive(:update_reference_repo) {}
       expect(subject).to receive(:reference_repo_dir)
+      expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
 
       subject.with_git_mirror(test_url_valid) do
         yielded << test_url_valid

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fastclone
 version: !ruby/object:Gem::Version
-  version: 1.0.16
+  version: 1.1.0
 platform: ruby
 authors:
 - Michael Tauraso
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-28 00:00:00.000000000 Z
+date: 2016-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cocaine
@@ -78,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: git-clone --recursive on steroids!