git-fastclone 1.1.2 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ebc92dd7541a4acc3c73355da48b66541373b29a
-  data.tar.gz: 99c1ebf5cd8df06cfd5d487b04f764301b75ee5e
+SHA256:
+  metadata.gz: 163da1441ec081e545dba9b3fa54832c20307e7936f38cefb4df3e9ad9515887
+  data.tar.gz: 348f6449e1e7da1d8a66bfedf820832572852e03729788a37418dfef3df86848
 SHA512:
-  metadata.gz: 66e543070bdd86fff06f6a18f3e1d55c94319a1bf0dfe73481c36942b56294532fdde406f2bca65c1b9c78a9ee9992ad27e2efcedc5b79be652b78bcc2c94207
-  data.tar.gz: 662e46428e556027005781eef6896ee6c1525726257aabfcd6d3661df20f30df378bfd61ceb37a65a6c68fc4ea4874e371ad066d797655250ed34d2e45bb1fe8
+  metadata.gz: cd539f1544cbaa0d4d02b8657733b71b33683a994202e5e3f66e3a4ccd07a3ab34655c2d3f5764da1cca880722708688aba29333618a39c10c4873f3c67a9dcb
+  data.tar.gz: 5348e5a19d5d7bf863673952e1da137b1d65deac68ba8b76c6da4d190be53a244e776b208eecc248e2b018f88575e457137ad78670ea67cdfd5e97d65b35f3ab

data/README.md

@@ -75,7 +75,7 @@ Before accepting any pull requests, we need you to sign an [Individual Contribut
 
 Acknowledgements
 ----------------
-[thoughtbot/cocaine][3] - jyurek and collaborators
+[thoughtbot/terrapin][3] - jyurek and collaborators
 
 [robolson][4]
 
@@ -104,7 +104,7 @@ License
 
 [1]: https://corner.squareup.com/2015/11/fastclone.html
 [2]: https://docs.google.com/a/squareup.com/forms/d/13WR8m5uZ2nAkJH41k7GdVBXAAbzDk00vxtEYjd6Imzg/viewform?formkey=dDViT2xzUHAwRkI3X3k5Z0lQM091OGc6MQ&ndplr=1
-[3]: https://github.com/thoughtbot/cocaine
+[3]: https://github.com/thoughtbot/terrapin
 [4]: https://github.com/robolson
 [5]: https://github.com/ianchesal
 [6]: https://github.com/mtauraso

data/Rakefile

@@ -1,11 +1,16 @@
-# encoding: utf-8
+# frozen_string_literal: true
 
 require 'bundler/setup'
 
-task default: %w(spec rubocop)
+task default: %w[spec]
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new
 
-require 'rubocop/rake_task'
-RuboCop::RakeTask.new
+begin
+  require 'rubocop/rake_task'
+  RuboCop::RakeTask.new
+  task default: :rubocop
+rescue LoadError => e
+  raise unless e.path == 'rubocop/rake_task'
+end

data/bin/git-fastclone

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 # Copyright 2015 Square Inc.
 

data/lib/git-fastclone.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Copyright 2015 Square Inc.
 
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +17,7 @@
 require 'optparse'
 require 'fileutils'
 require 'logger'
-require 'cocaine'
+require 'terrapin'
 require 'timeout'
 
 # Contains helper module UrlHelper and execution class GitFastClone::Runner
@@ -53,7 +55,7 @@ module GitFastClone
 
     def reference_repo_lock_file(url, reference_dir, using_local_repo)
       lock_file_name = "#{reference_repo_dir(url, reference_dir, using_local_repo)}:lock"
-      File.open(lock_file_name, File::RDWR | File::CREAT, 0644)
+      File.open(lock_file_name, File::RDWR | File::CREAT, 0o644)
     end
     module_function :reference_repo_lock_file
   end
@@ -117,7 +119,7 @@ module GitFastClone
       end
 
       puts "Cloning #{path_from_git_url(url)} to #{File.join(abs_clone_path, path)}"
-      Cocaine::CommandLine.environment['GIT_ALLOW_PROTOCOL'] =
+      Terrapin::CommandLine.environment['GIT_ALLOW_PROTOCOL'] =
         ENV['GIT_ALLOW_PROTOCOL'] || DEFAULT_GIT_ALLOW_PROTOCOL
       clone(url, options[:branch], path)
     end
@@ -139,7 +141,7 @@ module GitFastClone
           logger.formatter = proc do |_severity, _datetime, _progname, msg|
             "#{msg}\n"
           end
-          Cocaine::CommandLine.logger = logger
+          Terrapin::CommandLine.logger = logger
         end
 
         opts.on('-c', '--color', 'Display colored output') do
@@ -172,8 +174,9 @@ module GitFastClone
 
       if Dir.exist?(path)
         msg = "Clone destination #{File.join(abs_clone_path, path)} already exists!"
-        fail msg.red if color
-        fail msg
+        raise msg.red if color
+
+        raise msg
       end
 
       self.reference_dir = ENV['REFERENCE_REPO_DIR'] || DEFAULT_REFERENCE_REPO_DIR
@@ -188,16 +191,16 @@ module GitFastClone
       initial_time = Time.now
 
       with_git_mirror(url) do |mirror|
-        Cocaine::CommandLine.new('git clone', '--quiet --reference :mirror :url :path')
-          .run(mirror: mirror.to_s,
-               url: url.to_s,
-               path: File.join(abs_clone_path, src_dir).to_s)
+        Terrapin::CommandLine.new('git clone', '--quiet --reference :mirror :url :path')
+                             .run(mirror: mirror.to_s,
+                                  url: url.to_s,
+                                  path: File.join(abs_clone_path, src_dir).to_s)
       end
 
       # Only checkout if we're changing branches to a non-default branch
       if rev
         Dir.chdir(File.join(abs_clone_path, src_dir)) do
-          Cocaine::CommandLine.new('git checkout', '--quiet :rev').run(rev: rev.to_s)
+          Terrapin::CommandLine.new('git checkout', '--quiet :rev').run(rev: rev.to_s)
         end
       end
 
@@ -221,8 +224,8 @@ module GitFastClone
       threads = []
       submodule_url_list = []
 
-      Cocaine::CommandLine.new('cd', ':path; git submodule init 2>&1')
-        .run(path: File.join(abs_clone_path, pwd)).split("\n").each do |line|
+      Terrapin::CommandLine.new('cd', ':path; git submodule init 2>&1')
+                           .run(path: File.join(abs_clone_path, pwd)).split("\n").each do |line|
         submodule_path, submodule_url = parse_update_info(line)
         submodule_url_list << submodule_url
 
@@ -236,10 +239,10 @@ module GitFastClone
     def thread_update_submodule(submodule_url, submodule_path, threads, pwd)
       threads << Thread.new do
         with_git_mirror(submodule_url) do |mirror|
-          Cocaine::CommandLine.new('cd', ':dir; git submodule update --quiet --reference :mirror :path')
-            .run(dir: File.join(abs_clone_path, pwd).to_s,
-                 mirror: mirror.to_s,
-                 path: submodule_path.to_s)
+          Terrapin::CommandLine.new('cd', ':dir; git submodule update --quiet --reference :mirror :path')
+                               .run(dir: File.join(abs_clone_path, pwd).to_s,
+                                    mirror: mirror.to_s,
+                                    path: submodule_path.to_s)
         end
 
         update_submodules(File.join(pwd, submodule_path), submodule_url)
@@ -313,15 +316,14 @@ module GitFastClone
     # that this repo has been updated on this run of fastclone
     def store_updated_repo(url, mirror, repo_name, fail_hard)
       unless Dir.exist?(mirror)
-        Cocaine::CommandLine.new('git clone', '--mirror :url :mirror')
-          .run(url: url.to_s, mirror: mirror.to_s)
+        Terrapin::CommandLine.new('git clone', '--mirror :url :mirror')
+                             .run(url: url.to_s, mirror: mirror.to_s)
       end
 
-      Cocaine::CommandLine.new('cd', ':path; git remote update --prune').run(path: mirror)
+      Terrapin::CommandLine.new('cd', ':path; git remote update --prune').run(path: mirror)
 
       reference_updated[repo_name] = true
-
-    rescue Cocaine::ExitStatusError => e
+    rescue Terrapin::ExitStatusError => e
       # To avoid corruption of the cache, if we failed to update or check out we remove
       # the cache directory entirely. This may cause the current clone to fail, but if the
       # underlying error from git is transient it will not affect future clones.
@@ -344,7 +346,33 @@ module GitFastClone
       # ended up doing the update is done with its housekeeping.
       # This makes sure we have control and unlock when the block returns:
       with_reference_repo_lock(url) do
-        yield reference_repo_dir(url, reference_dir, using_local_repo)
+        dir = reference_repo_dir(url, reference_dir, using_local_repo)
+        retries_left = 1
+
+        begin
+          yield dir
+        rescue Terrapin::ExitStatusError => e
+          error_strings = [
+            'fatal: missing blob object',
+            'fatal: remote did not send all necessary objects',
+            /fatal: packed object [a-z0-9]+ \(stored in .*?\) is corrupt/,
+            /fatal: pack has \d+ unresolved deltas/,
+            'error: unable to read sha1 file of ',
+            'fatal: did not receive expected object'
+          ]
+          if e.to_s =~ /^STDERR:\n.+^#{Regexp.union(error_strings)}/m
+            # To avoid corruption of the cache, if we failed to update or check out we remove
+            # the cache directory entirely. This may cause the current clone to fail, but if the
+            # underlying error from git is transient it will not affect future clones.
+            FileUtils.remove_entry_secure(dir, force: true)
+            if retries_left > 0
+              retries_left -= 1
+              retry
+            end
+          end
+
+          raise
+        end
       end
     end
 

data/lib/git-fastclone/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 # Version string for git-fastclone
 module GitFastCloneVersion
-  VERSION = '1.1.2'.freeze
+  VERSION = '1.2.5'.freeze
 end

data/spec/git_fastclone_runner_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Copyright 2015 Square Inc.
 
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,8 +32,9 @@ describe GitFastClone::Runner do
     lockfile
   end
 
-  # Modified ARGV, watch out
-  ARGV = ['ssh://git@git.com/git-fastclone.git', 'test_reference_dir']
+  before do
+    stub_const('ARGV', ['ssh://git@git.com/git-fastclone.git', 'test_reference_dir'])
+  end
 
   let(:yielded) { [] }
 
@@ -72,14 +75,14 @@ describe GitFastClone::Runner do
 
   describe '.clone' do
     it 'should clone correctly' do
-      cocaine_commandline_double = double('new_cocaine_commandline')
+      terrapin_commandline_double = double('new_terrapin_commandline')
       allow(subject).to receive(:with_git_mirror) {}
-      allow(cocaine_commandline_double).to receive(:run) {}
-      allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+      allow(terrapin_commandline_double).to receive(:run) {}
+      allow(Terrapin::CommandLine).to receive(:new) { terrapin_commandline_double }
 
       expect(Time).to receive(:now).twice { 0 }
-      expect(Cocaine::CommandLine).to receive(:new)
-      expect(cocaine_commandline_double).to receive(:run)
+      expect(Terrapin::CommandLine).to receive(:new)
+      expect(terrapin_commandline_double).to receive(:run)
 
       subject.clone(placeholder_arg, placeholder_arg, '.')
     end
@@ -104,7 +107,7 @@ describe GitFastClone::Runner do
   describe '.thread_update_submodule' do
     it 'should update correctly' do
       pending('need to figure out how to test this')
-      fail
+      raise
     end
   end
 
@@ -127,7 +130,7 @@ describe GitFastClone::Runner do
 
       expect do
         subject.with_reference_repo_lock(test_url_valid) do
-          fail placeholder_arg
+          raise placeholder_arg
         end
       end.to raise_error(placeholder_arg)
     end
@@ -184,7 +187,7 @@ describe GitFastClone::Runner do
       end
     end
 
-    let(:placeholder_hash) { Hash.new }
+    let(:placeholder_hash) { {} }
 
     context 'when already have a hash' do
       it 'should not store' do
@@ -218,7 +221,7 @@ describe GitFastClone::Runner do
     it 'should go through the submodule file properly' do
       expect(Thread).to receive(:new).exactly(3).times
 
-      allow(File).to receive(:readlines) { %w(1 2 3) }
+      allow(File).to receive(:readlines) { %w[1 2 3] }
       subject.prefetch_submodules = true
       subject.prefetch(placeholder_arg)
     end
@@ -226,22 +229,22 @@ describe GitFastClone::Runner do
 
   describe '.store_updated_repo' do
     context 'when fail_hard is true' do
-      it 'should raise a Cocaine error' do
-        cocaine_commandline_double = double('new_cocaine_commandline')
-        allow(cocaine_commandline_double).to receive(:run) { fail Cocaine::ExitStatusError }
-        allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+      it 'should raise a Terrapin error' do
+        terrapin_commandline_double = double('new_terrapin_commandline')
+        allow(terrapin_commandline_double).to receive(:run) { raise Terrapin::ExitStatusError }
+        allow(Terrapin::CommandLine).to receive(:new) { terrapin_commandline_double }
         expect(FileUtils).to receive(:remove_entry_secure).with(placeholder_arg, force: true)
         expect do
           subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, true)
-        end.to raise_error(Cocaine::ExitStatusError)
+        end.to raise_error(Terrapin::ExitStatusError)
       end
     end
 
     context 'when fail_hard is false' do
-      it 'should not raise a cocaine error' do
-        cocaine_commandline_double = double('new_cocaine_commandline')
-        allow(cocaine_commandline_double).to receive(:run) { fail Cocaine::ExitStatusError }
-        allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+      it 'should not raise a terrapin error' do
+        terrapin_commandline_double = double('new_terrapin_commandline')
+        allow(terrapin_commandline_double).to receive(:run) { raise Terrapin::ExitStatusError }
+        allow(Terrapin::CommandLine).to receive(:new) { terrapin_commandline_double }
         expect(FileUtils).to receive(:remove_entry_secure).with(placeholder_arg, force: true)
 
         expect do
@@ -250,12 +253,12 @@ describe GitFastClone::Runner do
       end
     end
 
-    let(:placeholder_hash) { Hash.new }
+    let(:placeholder_hash) { {} }
 
     it 'should correctly update the hash' do
-      cocaine_commandline_double = double('new_cocaine_commandline')
-      allow(cocaine_commandline_double).to receive(:run) {}
-      allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+      terrapin_commandline_double = double('new_terrapin_commandline')
+      allow(terrapin_commandline_double).to receive(:run) {}
+      allow(Terrapin::CommandLine).to receive(:new) { terrapin_commandline_double }
       allow(Dir).to receive(:chdir) {}
 
       subject.reference_updated = placeholder_hash
@@ -276,5 +279,142 @@ describe GitFastClone::Runner do
 
       expect(yielded).to eq([test_url_valid])
     end
+
+    it 'should retry when the cache looks corrupted' do
+      allow(subject).to receive(:update_reference_repo) {}
+      expect(subject).to receive(:reference_repo_dir)
+      expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
+
+      responses = [
+        lambda { |_url|
+          raise Terrapin::ExitStatusError, <<-ERROR.gsub(/^ {12}/, '')
+            STDOUT:
+
+            STDERR:
+
+            fatal: bad object ee35b1e14e7c3a53dcc14d82606e5b872f6a05a7
+            fatal: remote did not send all necessary objects
+          ERROR
+        },
+        ->(url) { url }
+      ]
+      subject.with_git_mirror(test_url_valid) do
+        yielded << responses.shift.call(test_url_valid)
+      end
+
+      expect(responses).to be_empty
+      expect(yielded).to eq([test_url_valid])
+    end
+
+    it 'should retry when the clone succeeds but checkout fails' do
+      allow(subject).to receive(:update_reference_repo) {}
+      expect(subject).to receive(:reference_repo_dir)
+      expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
+
+      responses = [
+        lambda { |_url|
+          raise Terrapin::ExitStatusError, <<-ERROR.gsub(/^ {12}/, '')
+            STDOUT:
+
+            STDERR:
+
+            fatal: packed object 7c4d79704f8adf701f38a7bfb3e33ec5342542f1 (stored in /private/var/tmp/git-fastclone/reference/some-repo.git/objects/pack/pack-d37d7ed3e88d6e5f0ac141a7b0a2b32baf6e21a0.pack) is corrupt
+            warning: Clone succeeded, but checkout failed.
+            You can inspect what was checked out with 'git status' and retry with 'git restore --source=HEAD :/'
+          ERROR
+        },
+        ->(url) { url }
+      ]
+      subject.with_git_mirror(test_url_valid) do
+        yielded << responses.shift.call(test_url_valid)
+      end
+
+      expect(responses).to be_empty
+      expect(yielded).to eq([test_url_valid])
+    end
+
+    it 'should retry when deltas are missing' do
+      allow(subject).to receive(:update_reference_repo) {}
+      expect(subject).to receive(:reference_repo_dir)
+      expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
+
+      responses = [
+        lambda { |_url|
+          raise Terrapin::ExitStatusError, <<-ERROR.gsub(/^ {12}/, '')
+            STDOUT:
+
+            STDERR:
+
+            error: Could not read f7fad86d06fee0678f9af7203b6031feabb40c3e
+            fatal: pack has 138063 unresolved deltas
+            fatal: index-pack failed
+          ERROR
+        },
+        ->(url) { url }
+      ]
+      subject.with_git_mirror(test_url_valid) do
+        yielded << responses.shift.call(test_url_valid)
+      end
+
+      expect(responses).to be_empty
+      expect(yielded).to eq([test_url_valid])
+    end
+  end
+
+  it 'should retry when the cache errors with unable to read sha1 file' do
+    allow(subject).to receive(:update_reference_repo) {}
+    expect(subject).to receive(:reference_repo_dir)
+    expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
+
+    responses = [
+      lambda { |_url|
+        raise Terrapin::ExitStatusError, <<-ERROR.gsub(/^ {12}/, '')
+            STDOUT:
+
+            STDERR:
+
+            error: unable to read sha1 file of sqiosbuild/lib/action/action.rb (6113b739af82d8b07731de8a58d6e233301f80ab)
+            fatal: unable to checkout working tree
+            warning: Clone succeeded, but checkout failed.
+            You can inspect what was checked out with 'git status'
+            and retry with 'git restore --source=HEAD :/'
+        ERROR
+      },
+      ->(url) { url }
+    ]
+    subject.with_git_mirror(test_url_valid) do
+      yielded << responses.shift.call(test_url_valid)
+    end
+
+    expect(responses).to be_empty
+    expect(yielded).to eq([test_url_valid])
+  end
+
+  it 'should retry when the cache errors with did not receive expected object' do
+    allow(subject).to receive(:update_reference_repo) {}
+    expect(subject).to receive(:reference_repo_dir)
+    expect(subject).to receive(:reference_repo_lock_file).and_return(lockfile)
+
+    responses = [
+      lambda { |_url|
+        raise Terrapin::ExitStatusError, <<-ERROR.gsub(/^ {12}/, '')
+            STDOUT:
+
+            STDERR:
+
+            error: Could not read 6682dfe81f66656436e60883dd795e7ec6735153
+            error: Could not read 0cd3703c23fa44c0043d97fbc26356a23939f31b
+            fatal: did not receive expected object 3c64c9dd49c79bd09aa13d4b05ac18263ca29ccd
+            fatal: index-pack failed
+        ERROR
+      },
+      ->(url) { url }
+    ]
+    subject.with_git_mirror(test_url_valid) do
+      yielded << responses.shift.call(test_url_valid)
+    end
+
+    expect(responses).to be_empty
+    expect(yielded).to eq([test_url_valid])
   end
 end

data/spec/git_fastclone_url_helper_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Copyright 2015 Square Inc.
 
 # Licensed under the Apache License, Version 2.0 (the "License");

data/spec/spec_helper.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 require 'rspec/core'
 require 'rspec/mocks'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fastclone
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.5
 platform: ruby
 authors:
 - Michael Tauraso
@@ -9,10 +9,10 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-02 00:00:00.000000000 Z
+date: 2020-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: cocaine
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -26,21 +26,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: terrapin
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: A git command that uses reference repositories and threading to quickly
-  and recursively clone repositories with many nested submodules
+        version: 0.6.0
+description: A git command that uses reference repositories and threading to quicklyand
+  recursively clone repositories with many nested submodules
 email:
 - mtauraso@squareup.com
 - jchang@squareup.com
@@ -78,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: git-clone --recursive on steroids!