git-fastclone 1.3.2 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/lib/git-fastclone/version.rb +1 -1
- data/lib/git-fastclone.rb +50 -45
- data/lib/runner_execution.rb +221 -0
- data/spec/git_fastclone_runner_spec.rb +92 -83
- metadata +3 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed5f3f84cbab65351479f22b659b912521699f542c898010527f25b7c3786c84
|
4
|
+
data.tar.gz: 5f02716656bf0962d9a808ae368078f0e151e6bdfd3abba5b25503504f64194b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a9219567c52c31d5219027adb6227c88970dd839fa8dd2f50258fc205adc3e4f9372d44d6f734242849865d7885f6b8f5a3fadfef05f77c88e24d8d85016a849
|
7
|
+
data.tar.gz: c136b5240dfc87480457c924772944282f518f8c77d49473f41724449ae0566d8cc5476e6f698aa86de2b18f1f77a8995d0f5f3e8778546630e390d6ed2d2530
|
data/README.md
CHANGED
@@ -74,6 +74,8 @@ keep the code as readable as possible.
|
|
74
74
|
Before accepting any pull requests, we need you to sign an [Individual Contributor Agreement][2]
|
75
75
|
(Google form).
|
76
76
|
|
77
|
+
Once landed, please reach out to any owner listed in https://rubygems.org/gems/git-fastclone and ask them to help publish the new version.
|
78
|
+
|
77
79
|
|
78
80
|
Acknowledgements
|
79
81
|
----------------
|
data/lib/git-fastclone.rb
CHANGED
@@ -16,9 +16,8 @@
|
|
16
16
|
|
17
17
|
require 'optparse'
|
18
18
|
require 'fileutils'
|
19
|
-
require 'logger'
|
20
|
-
require 'terrapin'
|
21
19
|
require 'timeout'
|
20
|
+
require_relative 'runner_execution'
|
22
21
|
|
23
22
|
# Contains helper module UrlHelper and execution class GitFastClone::Runner
|
24
23
|
module GitFastClone
|
@@ -68,13 +67,14 @@ module GitFastClone
|
|
68
67
|
require 'colorize'
|
69
68
|
|
70
69
|
include GitFastClone::UrlHelper
|
70
|
+
include RunnerExecution
|
71
71
|
|
72
72
|
DEFAULT_REFERENCE_REPO_DIR = '/var/tmp/git-fastclone/reference'
|
73
73
|
|
74
74
|
DEFAULT_GIT_ALLOW_PROTOCOL = 'file:git:http:https:ssh'
|
75
75
|
|
76
76
|
attr_accessor :reference_dir, :prefetch_submodules, :reference_updated, :reference_mutex,
|
77
|
-
:options, :
|
77
|
+
:options, :abs_clone_path, :using_local_repo, :verbose, :color,
|
78
78
|
:flock_timeout_secs
|
79
79
|
|
80
80
|
def initialize
|
@@ -94,8 +94,6 @@ module GitFastClone
|
|
94
94
|
|
95
95
|
self.options = {}
|
96
96
|
|
97
|
-
self.logger = nil # Only set in verbose mode
|
98
|
-
|
99
97
|
self.abs_clone_path = Dir.pwd
|
100
98
|
|
101
99
|
self.using_local_repo = false
|
@@ -119,8 +117,7 @@ module GitFastClone
|
|
119
117
|
end
|
120
118
|
|
121
119
|
puts "Cloning #{path_from_git_url(url)} to #{File.join(abs_clone_path, path)}"
|
122
|
-
|
123
|
-
ENV['GIT_ALLOW_PROTOCOL'] || DEFAULT_GIT_ALLOW_PROTOCOL
|
120
|
+
ENV['GIT_ALLOW_PROTOCOL'] ||= DEFAULT_GIT_ALLOW_PROTOCOL
|
124
121
|
clone(url, options[:branch], path, options[:config])
|
125
122
|
end
|
126
123
|
|
@@ -137,11 +134,6 @@ module GitFastClone
|
|
137
134
|
|
138
135
|
opts.on('-v', '--verbose', 'Verbose mode') do
|
139
136
|
self.verbose = true
|
140
|
-
self.logger = Logger.new($stdout)
|
141
|
-
logger.formatter = proc do |_severity, _datetime, _progname, msg|
|
142
|
-
"#{msg}\n"
|
143
|
-
end
|
144
|
-
Terrapin::CommandLine.logger = logger
|
145
137
|
end
|
146
138
|
|
147
139
|
opts.on('-c', '--color', 'Display colored output') do
|
@@ -154,7 +146,7 @@ module GitFastClone
|
|
154
146
|
|
155
147
|
opts.on('--lock-timeout N', 'Timeout in seconds to acquire a lock on any reference repo.
|
156
148
|
Default is 0 which waits indefinitely.') do |timeout_secs|
|
157
|
-
self.flock_timeout_secs = timeout_secs
|
149
|
+
self.flock_timeout_secs = timeout_secs.to_i
|
158
150
|
end
|
159
151
|
end.parse!
|
160
152
|
end
|
@@ -217,19 +209,16 @@ module GitFastClone
|
|
217
209
|
with_git_mirror(url) do |mirror, attempt_number|
|
218
210
|
clear_clone_dest_if_needed(attempt_number, clone_dest)
|
219
211
|
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
url: url.to_s,
|
225
|
-
path: clone_dest,
|
226
|
-
config: config.to_s)
|
212
|
+
clone_commands = ['git', 'clone', verbose ? '--verbose' : '--quiet']
|
213
|
+
clone_commands << '--reference' << mirror.to_s << url.to_s << clone_dest
|
214
|
+
clone_commands << '--config' << config.to_s unless config.nil?
|
215
|
+
fail_pipe_on_error(clone_commands, quiet: !verbose)
|
227
216
|
end
|
228
217
|
|
229
218
|
# Only checkout if we're changing branches to a non-default branch
|
230
219
|
if rev
|
231
220
|
Dir.chdir(File.join(abs_clone_path, src_dir)) do
|
232
|
-
|
221
|
+
fail_pipe_on_error(['git', 'checkout', '--quiet', rev.to_s], quiet: !verbose)
|
233
222
|
end
|
234
223
|
end
|
235
224
|
|
@@ -252,9 +241,12 @@ module GitFastClone
|
|
252
241
|
|
253
242
|
threads = []
|
254
243
|
submodule_url_list = []
|
244
|
+
output = ''
|
245
|
+
Dir.chdir(File.join(abs_clone_path, pwd).to_s) do
|
246
|
+
output = fail_on_error('git', 'submodule', 'init', quiet: !verbose)
|
247
|
+
end
|
255
248
|
|
256
|
-
|
257
|
-
.run(path: File.join(abs_clone_path, pwd)).split("\n").each do |line|
|
249
|
+
output.split("\n").each do |line|
|
258
250
|
submodule_path, submodule_url = parse_update_info(line)
|
259
251
|
submodule_url_list << submodule_url
|
260
252
|
|
@@ -268,10 +260,12 @@ module GitFastClone
|
|
268
260
|
def thread_update_submodule(submodule_url, submodule_path, threads, pwd)
|
269
261
|
threads << Thread.new do
|
270
262
|
with_git_mirror(submodule_url) do |mirror, _|
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
263
|
+
Dir.chdir(File.join(abs_clone_path, pwd).to_s) do
|
264
|
+
fail_pipe_on_error(
|
265
|
+
['git', 'submodule', verbose ? nil : '--quiet', 'update', '--reference', mirror.to_s,
|
266
|
+
submodule_path.to_s].compact, quiet: !verbose
|
267
|
+
)
|
268
|
+
end
|
275
269
|
end
|
276
270
|
|
277
271
|
update_submodules(File.join(pwd, submodule_path), submodule_url)
|
@@ -343,43 +337,54 @@ module GitFastClone
|
|
343
337
|
# that this repo has been updated on this run of fastclone
|
344
338
|
def store_updated_repo(url, mirror, repo_name, fail_hard)
|
345
339
|
unless Dir.exist?(mirror)
|
346
|
-
|
347
|
-
|
340
|
+
fail_pipe_on_error(
|
341
|
+
['git', 'clone', verbose ? '--verbose' : '--quiet', '--mirror', url.to_s,
|
342
|
+
mirror.to_s], quiet: !verbose
|
343
|
+
)
|
348
344
|
end
|
349
345
|
|
350
|
-
|
351
|
-
|
346
|
+
Dir.chdir(mirror) do
|
347
|
+
cmd = ['git', 'remote', verbose ? '--verbose' : nil, 'update', '--prune'].compact
|
348
|
+
if verbose
|
349
|
+
fail_pipe_on_error(cmd, quiet: !verbose)
|
350
|
+
else
|
351
|
+
# Because above operation might spit out a lot to stderr, we use this to swallow them
|
352
|
+
# and only display if the operation return non 0 exit code
|
353
|
+
fail_on_error(*cmd, quiet: !verbose)
|
354
|
+
end
|
355
|
+
end
|
352
356
|
reference_updated[repo_name] = true
|
353
|
-
rescue
|
357
|
+
rescue RunnerExecutionRuntimeError => e
|
354
358
|
# To avoid corruption of the cache, if we failed to update or check out we remove
|
355
359
|
# the cache directory entirely. This may cause the current clone to fail, but if the
|
356
360
|
# underlying error from git is transient it will not affect future clones.
|
357
|
-
|
361
|
+
clear_cache(mirror, url)
|
358
362
|
raise e if fail_hard
|
359
363
|
end
|
360
364
|
|
361
365
|
def retriable_error?(error)
|
362
366
|
error_strings = [
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
367
|
+
/^fatal: missing blob object/,
|
368
|
+
/^fatal: remote did not send all necessary objects/,
|
369
|
+
/^fatal: packed object [a-z0-9]+ \(stored in .*?\) is corrupt/,
|
370
|
+
/^fatal: pack has \d+ unresolved delta/,
|
371
|
+
/^error: unable to read sha1 file of /,
|
372
|
+
/^fatal: did not receive expected object/,
|
369
373
|
/^fatal: unable to read tree [a-z0-9]+\n^warning: Clone succeeded, but checkout failed/
|
370
374
|
]
|
371
|
-
error.to_s =~
|
375
|
+
error.to_s =~ /.*#{Regexp.union(error_strings)}/m
|
372
376
|
end
|
373
377
|
|
374
378
|
def print_formatted_error(error)
|
375
379
|
indented_error = error.to_s.split("\n").map { |s| "> #{s}\n" }.join
|
376
|
-
puts "Encountered a retriable error:\n#{indented_error}\n
|
380
|
+
puts "[INFO] Encountered a retriable error:\n#{indented_error}\n"
|
377
381
|
end
|
378
382
|
|
379
383
|
# To avoid corruption of the cache, if we failed to update or check out we remove
|
380
384
|
# the cache directory entirely. This may cause the current clone to fail, but if the
|
381
385
|
# underlying error from git is transient it will not affect future clones.
|
382
386
|
def clear_cache(dir, url)
|
387
|
+
puts "[WARN] Removing the fastclone cache at #{dir}"
|
383
388
|
FileUtils.remove_entry_secure(dir, force: true)
|
384
389
|
reference_updated.delete(reference_repo_name(url))
|
385
390
|
end
|
@@ -405,9 +410,9 @@ module GitFastClone
|
|
405
410
|
with_reference_repo_lock(url) do
|
406
411
|
yield dir, attempt_number
|
407
412
|
end
|
408
|
-
rescue
|
409
|
-
if retriable_error?(e)
|
410
|
-
print_formatted_error(e)
|
413
|
+
rescue RunnerExecutionRuntimeError => e
|
414
|
+
if retriable_error?(e.output)
|
415
|
+
print_formatted_error(e.output)
|
411
416
|
clear_cache(dir, url)
|
412
417
|
|
413
418
|
if attempt_number < retries_allowed
|
@@ -416,7 +421,7 @@ module GitFastClone
|
|
416
421
|
end
|
417
422
|
end
|
418
423
|
|
419
|
-
raise
|
424
|
+
raise e
|
420
425
|
end
|
421
426
|
|
422
427
|
def usage
|
@@ -0,0 +1,221 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# rubocop:disable all
|
3
|
+
|
4
|
+
require 'open3'
|
5
|
+
require 'logger'
|
6
|
+
|
7
|
+
# Execution primitives that force explicit error handling and never call the shell.
|
8
|
+
# Cargo-culted from internal BuildExecution code on top of public version: https://github.com/square/build_execution
|
9
|
+
module RunnerExecution
|
10
|
+
class RunnerExecutionRuntimeError < RuntimeError
|
11
|
+
attr_reader :status, :exitstatus, :command, :output
|
12
|
+
|
13
|
+
def initialize(status, command, output = nil)
|
14
|
+
@status = status
|
15
|
+
@exitstatus = status.exitstatus
|
16
|
+
@command = command
|
17
|
+
@output = output
|
18
|
+
|
19
|
+
super "#{status.inspect}\n#{command.inspect}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Wrapper around open3.pipeline_r which fails on error.
|
24
|
+
# and stops users from invoking the shell by accident.
|
25
|
+
def fail_pipe_on_error(*cmd_list, quiet: false, **opts)
|
26
|
+
print_command('Running Pipeline:', cmd_list) unless quiet
|
27
|
+
|
28
|
+
env = opts.delete(:env) { {} }
|
29
|
+
raise ArgumentError, "The :env option must be a hash, not #{env.inspect}" unless env.is_a?(Hash)
|
30
|
+
|
31
|
+
cmd_list.map! { |cmd| shell_safe(cmd).unshift(env) }
|
32
|
+
|
33
|
+
output, *status_list = Open3.pipeline_r(*cmd_list, opts) do |out, wait_threads|
|
34
|
+
out_reader = Thread.new do
|
35
|
+
if quiet
|
36
|
+
output = out.read
|
37
|
+
else
|
38
|
+
# Output from pipeline should go to stdout and also get returned for
|
39
|
+
# processing if necessary.
|
40
|
+
output = tee(out, STDOUT)
|
41
|
+
end
|
42
|
+
out.close
|
43
|
+
output
|
44
|
+
end
|
45
|
+
[out_reader.value] + wait_threads.map(&:value)
|
46
|
+
end
|
47
|
+
exit_on_status(output, cmd_list, status_list, quiet: quiet)
|
48
|
+
end
|
49
|
+
module_function :fail_pipe_on_error
|
50
|
+
|
51
|
+
# Runs a command that fails on error.
|
52
|
+
# Uses popen2e wrapper. Handles bad statuses with potential for retries.
|
53
|
+
def fail_on_error(*cmd, stdin_data: nil, binmode: false, quiet: false, **opts)
|
54
|
+
print_command('Running Shell Safe Command:', [cmd]) unless quiet
|
55
|
+
shell_safe_cmd = shell_safe(cmd)
|
56
|
+
retry_times = opts[:retry] || 0
|
57
|
+
opts.delete(:retry)
|
58
|
+
|
59
|
+
while retry_times >= 0
|
60
|
+
output, status = popen2e_wrapper(*shell_safe_cmd, stdin_data: stdin_data, binmode: binmode,
|
61
|
+
quiet: quiet, **opts)
|
62
|
+
|
63
|
+
break unless status.exitstatus != 0
|
64
|
+
|
65
|
+
logger.debug("Command failed with exit status #{status.exitstatus}, retrying #{retry_times} more time(s).") if retry_times > 0
|
66
|
+
retry_times -= 1
|
67
|
+
end
|
68
|
+
|
69
|
+
# Get out with the status, good or bad.
|
70
|
+
exit_on_status(output, [shell_safe_cmd], [status], quiet: quiet)
|
71
|
+
end
|
72
|
+
module_function :fail_on_error
|
73
|
+
|
74
|
+
# Wrapper around open3.popen2e
|
75
|
+
#
|
76
|
+
# We emulate open3.capture2e with the following changes in behavior:
|
77
|
+
# 1) The command is printed to stdout before execution.
|
78
|
+
# 2) Attempts to use the shell implicitly are blocked.
|
79
|
+
# 3) Nonzero return codes result in the process exiting.
|
80
|
+
# 4) Combined stdout/stderr goes to callers stdout
|
81
|
+
# (continuously streamed) and is returned as a string
|
82
|
+
#
|
83
|
+
# If you're looking for more process/stream control read the spawn
|
84
|
+
# documentation, and pass options directly here
|
85
|
+
def popen2e_wrapper(*shell_safe_cmd, stdin_data: nil, binmode: false,
|
86
|
+
quiet: false, **opts)
|
87
|
+
|
88
|
+
env = opts.delete(:env) { {} }
|
89
|
+
raise ArgumentError, "The :env option must be a hash, not #{env.inspect}" if !env.is_a?(Hash)
|
90
|
+
|
91
|
+
# Most of this is copied from Open3.capture2e in ruby/lib/open3.rb
|
92
|
+
_output, _status = Open3.popen2e(env, *shell_safe_cmd, opts) do |i, oe, t|
|
93
|
+
if binmode
|
94
|
+
i.binmode
|
95
|
+
oe.binmode
|
96
|
+
end
|
97
|
+
|
98
|
+
outerr_reader = Thread.new do
|
99
|
+
if quiet
|
100
|
+
oe.read
|
101
|
+
else
|
102
|
+
# Instead of oe.read, we redirect. Output from command goes to stdout
|
103
|
+
# and also is returned for processing if necessary.
|
104
|
+
tee(oe, STDOUT)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
if stdin_data
|
109
|
+
begin
|
110
|
+
i.write stdin_data
|
111
|
+
rescue Errno::EPIPE
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
i.close
|
116
|
+
[outerr_reader.value, t.value]
|
117
|
+
end
|
118
|
+
end
|
119
|
+
module_function :popen2e_wrapper
|
120
|
+
|
121
|
+
# Look at a cmd list intended for spawn.
|
122
|
+
# determine if spawn will call the shell implicitly, fail in that case.
|
123
|
+
def shell_safe(cmd)
|
124
|
+
# Take the first string and change it to a list of [executable,argv0]
|
125
|
+
# This syntax for calling popen2e (and eventually spawn) avoids
|
126
|
+
# the shell in all cases
|
127
|
+
shell_safe_cmd = Array.new(cmd)
|
128
|
+
if shell_safe_cmd[0].class == String
|
129
|
+
shell_safe_cmd[0] = [shell_safe_cmd[0], shell_safe_cmd[0]]
|
130
|
+
end
|
131
|
+
shell_safe_cmd
|
132
|
+
end
|
133
|
+
module_function :shell_safe
|
134
|
+
|
135
|
+
def debug_print_cmd_list(cmd_list)
|
136
|
+
# Take a list of command argument lists like you'd sent to open3.pipeline or
|
137
|
+
# fail_on_error_pipe and print out a string that would do the same thing when
|
138
|
+
# entered at the shell.
|
139
|
+
#
|
140
|
+
# This is a converter from our internal representation of commands to a subset
|
141
|
+
# of bash that can be executed directly.
|
142
|
+
#
|
143
|
+
# Note this has problems if you specify env or opts
|
144
|
+
# TODO: make this remove those command parts
|
145
|
+
"\"" +
|
146
|
+
cmd_list.map do |cmd|
|
147
|
+
cmd.map do |arg|
|
148
|
+
arg.gsub("\"", "\\\"") # Escape all double quotes in command arguments
|
149
|
+
end.join("\" \"") # Fully quote all command parts, beginning and end.
|
150
|
+
end.join("\" | \"") + "\"" # Pipe commands to one another.
|
151
|
+
end
|
152
|
+
module_function :debug_print_cmd_list
|
153
|
+
|
154
|
+
# Prints a formatted string with command
|
155
|
+
def print_command(message, cmd)
|
156
|
+
logger.debug("#{message} #{debug_print_cmd_list(cmd)}\n")
|
157
|
+
end
|
158
|
+
module_function :print_command
|
159
|
+
|
160
|
+
# Takes in an input stream and an output stream
|
161
|
+
# Redirects data from one to the other until the input stream closes.
|
162
|
+
# Returns all data that passed through on return.
|
163
|
+
def tee(in_stream, out_stream)
|
164
|
+
alldata = ''
|
165
|
+
loop do
|
166
|
+
begin
|
167
|
+
data = in_stream.read_nonblock(4096)
|
168
|
+
alldata += data
|
169
|
+
out_stream.write(data)
|
170
|
+
out_stream.flush
|
171
|
+
rescue IO::WaitReadable
|
172
|
+
IO.select([in_stream])
|
173
|
+
retry
|
174
|
+
rescue IOError
|
175
|
+
break
|
176
|
+
end
|
177
|
+
end
|
178
|
+
alldata
|
179
|
+
end
|
180
|
+
module_function :tee
|
181
|
+
|
182
|
+
# If any of the statuses are bad, exits with the
|
183
|
+
# return code of the first one.
|
184
|
+
#
|
185
|
+
# Otherwise returns first argument (output)
|
186
|
+
def exit_on_status(output, cmd_list, status_list, quiet: false)
|
187
|
+
status_list.each_index do |index|
|
188
|
+
status = status_list[index]
|
189
|
+
cmd = cmd_list[index]
|
190
|
+
check_status(cmd, status, output: output, quiet: quiet)
|
191
|
+
end
|
192
|
+
|
193
|
+
output
|
194
|
+
end
|
195
|
+
module_function :exit_on_status
|
196
|
+
|
197
|
+
def check_status(cmd, status, output: nil, quiet: false)
|
198
|
+
return if status.exited? && status.exitstatus == 0
|
199
|
+
|
200
|
+
# If we exited nonzero or abnormally, print debugging info and explode.
|
201
|
+
if status.exited?
|
202
|
+
logger.debug("Process Exited normally. Exit status:#{status.exitstatus}") unless quiet
|
203
|
+
else
|
204
|
+
# This should only get executed if we're stopped or signaled
|
205
|
+
logger.debug("Process exited abnormally:\nProcessStatus: #{status.inspect}\n" \
|
206
|
+
"Raw POSIX Status: #{status.to_i}\n") unless quiet
|
207
|
+
end
|
208
|
+
|
209
|
+
raise RunnerExecutionRuntimeError.new(status, cmd, output)
|
210
|
+
end
|
211
|
+
module_function :check_status
|
212
|
+
|
213
|
+
DEFAULT_LOGGER = Logger.new(STDOUT)
|
214
|
+
private_constant :DEFAULT_LOGGER
|
215
|
+
|
216
|
+
def logger
|
217
|
+
DEFAULT_LOGGER
|
218
|
+
end
|
219
|
+
module_function :logger
|
220
|
+
end
|
221
|
+
# rubocop:enable all
|
@@ -36,6 +36,7 @@ describe GitFastClone::Runner do
|
|
36
36
|
|
37
37
|
before do
|
38
38
|
stub_const('ARGV', ['ssh://git@git.com/git-fastclone.git', 'test_reference_dir'])
|
39
|
+
allow($stdout).to receive(:puts)
|
39
40
|
end
|
40
41
|
|
41
42
|
let(:yielded) { [] }
|
@@ -50,7 +51,6 @@ describe GitFastClone::Runner do
|
|
50
51
|
expect(subject.reference_mutex).to eq({})
|
51
52
|
expect(subject.reference_updated).to eq({})
|
52
53
|
expect(subject.options).to eq({})
|
53
|
-
expect(subject.logger).to eq(nil)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
@@ -87,10 +87,9 @@ describe GitFastClone::Runner do
|
|
87
87
|
end
|
88
88
|
|
89
89
|
describe '.clone' do
|
90
|
-
let(:
|
90
|
+
let(:runner_execution_double) { double('runner_execution') }
|
91
91
|
before(:each) do
|
92
|
-
allow(
|
93
|
-
expect(Time).to receive(:now).twice { 0 }
|
92
|
+
allow(runner_execution_double).to receive(:fail_pipe_on_error) {}
|
94
93
|
allow(Dir).to receive(:pwd) { '/pwd' }
|
95
94
|
allow(Dir).to receive(:chdir).and_yield
|
96
95
|
allow(subject).to receive(:with_git_mirror).and_yield('/cache', 0)
|
@@ -98,36 +97,37 @@ describe GitFastClone::Runner do
|
|
98
97
|
end
|
99
98
|
|
100
99
|
it 'should clone correctly' do
|
101
|
-
expect(
|
102
|
-
'git
|
103
|
-
|
104
|
-
) {
|
105
|
-
expect(
|
106
|
-
'git
|
107
|
-
|
108
|
-
) {
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
expect(
|
100
|
+
expect(subject).to receive(:fail_pipe_on_error).with(
|
101
|
+
['git', 'checkout', '--quiet', 'PH'],
|
102
|
+
{ quiet: true }
|
103
|
+
) { runner_execution_double }
|
104
|
+
expect(subject).to receive(:fail_pipe_on_error).with(
|
105
|
+
['git', 'clone', '--quiet', '--reference', '/cache', 'PH', '/pwd/.'],
|
106
|
+
{ quiet: true }
|
107
|
+
) { runner_execution_double }
|
108
|
+
|
109
|
+
subject.clone(placeholder_arg, placeholder_arg, '.', nil)
|
110
|
+
end
|
111
|
+
|
112
|
+
it 'should clone correctly with verbose mode on' do
|
113
|
+
subject.verbose = true
|
114
|
+
expect(subject).to receive(:fail_pipe_on_error).with(
|
115
|
+
['git', 'checkout', '--quiet', 'PH'],
|
116
|
+
{ quiet: false }
|
117
|
+
) { runner_execution_double }
|
118
|
+
expect(subject).to receive(:fail_pipe_on_error).with(
|
119
|
+
['git', 'clone', '--verbose', '--reference', '/cache', 'PH', '/pwd/.'],
|
120
|
+
{ quiet: false }
|
121
|
+
) { runner_execution_double }
|
116
122
|
|
117
123
|
subject.clone(placeholder_arg, placeholder_arg, '.', nil)
|
118
124
|
end
|
119
125
|
|
120
126
|
it 'should clone correctly with custom configs' do
|
121
|
-
expect(
|
122
|
-
'git clone',
|
123
|
-
|
124
|
-
) {
|
125
|
-
expect(terrapin_commandline_double).to receive(:run).with(
|
126
|
-
mirror: '/cache',
|
127
|
-
url: placeholder_arg,
|
128
|
-
path: '/pwd/.',
|
129
|
-
config: 'config'
|
130
|
-
)
|
127
|
+
expect(subject).to receive(:fail_pipe_on_error).with(
|
128
|
+
['git', 'clone', '--quiet', '--reference', '/cache', 'PH', '/pwd/.', '--config', 'config'],
|
129
|
+
{ quiet: true }
|
130
|
+
) { runner_execution_double }
|
131
131
|
|
132
132
|
subject.clone(placeholder_arg, nil, '.', 'config')
|
133
133
|
end
|
@@ -294,36 +294,35 @@ describe GitFastClone::Runner do
|
|
294
294
|
|
295
295
|
describe '.store_updated_repo' do
|
296
296
|
context 'when fail_hard is true' do
|
297
|
-
it 'should raise a
|
298
|
-
|
299
|
-
allow(
|
300
|
-
|
297
|
+
it 'should raise a Runtime error and clear cache' do
|
298
|
+
status = double('status')
|
299
|
+
allow(status).to receive(:exitstatus).and_return(1)
|
300
|
+
ex = RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd')
|
301
|
+
allow(subject).to receive(:fail_pipe_on_error) { raise ex }
|
301
302
|
expect(FileUtils).to receive(:remove_entry_secure).with(placeholder_arg, force: true)
|
302
303
|
expect do
|
303
304
|
subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, true)
|
304
|
-
end.to raise_error(
|
305
|
+
end.to raise_error(ex)
|
305
306
|
end
|
306
307
|
end
|
307
308
|
|
308
309
|
context 'when fail_hard is false' do
|
309
|
-
it 'should not raise a
|
310
|
-
|
311
|
-
allow(
|
312
|
-
|
310
|
+
it 'should not raise a Runtime error but clear cache' do
|
311
|
+
status = double('status')
|
312
|
+
allow(status).to receive(:exitstatus).and_return(1)
|
313
|
+
ex = RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd')
|
314
|
+
allow(subject).to receive(:fail_pipe_on_error) { raise ex }
|
313
315
|
expect(FileUtils).to receive(:remove_entry_secure).with(placeholder_arg, force: true)
|
314
|
-
|
315
316
|
expect do
|
316
317
|
subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, false)
|
317
|
-
end.
|
318
|
+
end.to_not raise_error
|
318
319
|
end
|
319
320
|
end
|
320
321
|
|
321
322
|
let(:placeholder_hash) { {} }
|
322
323
|
|
323
324
|
it 'should correctly update the hash' do
|
324
|
-
|
325
|
-
allow(terrapin_commandline_double).to receive(:run) {}
|
326
|
-
allow(Terrapin::CommandLine).to receive(:new) { terrapin_commandline_double }
|
325
|
+
allow(subject).to receive(:fail_pipe_on_error)
|
327
326
|
allow(Dir).to receive(:chdir) {}
|
328
327
|
|
329
328
|
subject.reference_updated = placeholder_hash
|
@@ -335,10 +334,6 @@ describe GitFastClone::Runner do
|
|
335
334
|
describe '.with_git_mirror' do
|
336
335
|
def retriable_error
|
337
336
|
%(
|
338
|
-
STDOUT:
|
339
|
-
|
340
|
-
STDERR:
|
341
|
-
|
342
337
|
fatal: bad object ee35b1e14e7c3a53dcc14d82606e5b872f6a05a7
|
343
338
|
fatal: remote did not send all necessary objects
|
344
339
|
).strip.split("\n").map(&:strip).join("\n")
|
@@ -351,7 +346,11 @@ describe GitFastClone::Runner do
|
|
351
346
|
->(url) { url }
|
352
347
|
else
|
353
348
|
# Simulate failed error response
|
354
|
-
|
349
|
+
lambda { |_url|
|
350
|
+
status = double('status')
|
351
|
+
allow(status).to receive(:exitstatus).and_return(1)
|
352
|
+
raise RunnerExecution::RunnerExecutionRuntimeError.new(status, 'cmd', response)
|
353
|
+
}
|
355
354
|
end
|
356
355
|
end
|
357
356
|
|
@@ -366,19 +365,22 @@ describe GitFastClone::Runner do
|
|
366
365
|
end
|
367
366
|
|
368
367
|
let(:expected_commands) { [] }
|
369
|
-
let(:expected_commands_args) { [] }
|
370
368
|
|
371
369
|
before(:each) do
|
372
|
-
|
373
|
-
|
370
|
+
allow(subject).to receive(:fail_pipe_on_error) { |*params|
|
371
|
+
command = params[0]
|
374
372
|
expect(expected_commands.length).to be > 0
|
375
373
|
expected_command = expected_commands.shift
|
376
|
-
expected_args = expected_commands_args.shift
|
377
374
|
expect(command).to eq(expected_command)
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
375
|
+
}
|
376
|
+
allow(subject).to receive(:fail_on_error) { |*params|
|
377
|
+
# last one is an argument `quiet:`
|
378
|
+
command = params.first(params.size - 1)
|
379
|
+
expect(expected_commands.length).to be > 0
|
380
|
+
expected_command = expected_commands.shift
|
381
|
+
expect(command).to eq(expected_command)
|
382
|
+
}
|
383
|
+
allow(Dir).to receive(:chdir).and_yield
|
382
384
|
|
383
385
|
allow(subject).to receive(:print_formatted_error) {}
|
384
386
|
allow(subject).to receive(:reference_repo_dir).and_return(test_reference_repo_dir)
|
@@ -389,39 +391,46 @@ describe GitFastClone::Runner do
|
|
389
391
|
expect(expected_commands).to be_empty
|
390
392
|
end
|
391
393
|
|
392
|
-
def clone_cmds
|
394
|
+
def clone_cmds(verbose: false)
|
393
395
|
[
|
394
|
-
['git clone', '--
|
395
|
-
|
396
|
-
|
397
|
-
end
|
398
|
-
|
399
|
-
def clone_args
|
400
|
-
[
|
401
|
-
{
|
402
|
-
mirror: test_reference_repo_dir,
|
403
|
-
url: test_url_valid
|
404
|
-
},
|
405
|
-
{
|
406
|
-
path: test_reference_repo_dir
|
407
|
-
}
|
396
|
+
['git', 'clone', verbose ? '--verbose' : '--quiet', '--mirror', test_url_valid,
|
397
|
+
test_reference_repo_dir],
|
398
|
+
['git', 'remote', verbose ? '--verbose' : nil, 'update', '--prune'].compact
|
408
399
|
]
|
409
400
|
end
|
410
401
|
|
411
402
|
context 'expecting 1 clone attempt' do
|
412
|
-
|
413
|
-
|
403
|
+
context 'with verbose mode on' do
|
404
|
+
before { subject.verbose = true }
|
405
|
+
let(:expected_commands) { clone_cmds(verbose: true) }
|
406
|
+
|
407
|
+
it 'should succeed with a successful clone' do
|
408
|
+
expect(subject).not_to receive(:clear_cache)
|
409
|
+
try_with_git_mirror([true], [[test_reference_repo_dir, 0]])
|
410
|
+
end
|
414
411
|
|
415
|
-
|
416
|
-
|
417
|
-
|
412
|
+
it 'should fail after a non-retryable clone error' do
|
413
|
+
expect(subject).not_to receive(:clear_cache)
|
414
|
+
expect do
|
415
|
+
try_with_git_mirror(['Some unexpected error message'], [])
|
416
|
+
end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError)
|
417
|
+
end
|
418
418
|
end
|
419
419
|
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
420
|
+
context 'with verbose mode off' do
|
421
|
+
let(:expected_commands) { clone_cmds }
|
422
|
+
|
423
|
+
it 'should succeed with a successful clone' do
|
424
|
+
expect(subject).not_to receive(:clear_cache)
|
425
|
+
try_with_git_mirror([true], [[test_reference_repo_dir, 0]])
|
426
|
+
end
|
427
|
+
|
428
|
+
it 'should fail after a non-retryable clone error' do
|
429
|
+
expect(subject).not_to receive(:clear_cache)
|
430
|
+
expect do
|
431
|
+
try_with_git_mirror(['Some unexpected error message'], [])
|
432
|
+
end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError)
|
433
|
+
end
|
425
434
|
end
|
426
435
|
end
|
427
436
|
|
@@ -438,14 +447,14 @@ describe GitFastClone::Runner do
|
|
438
447
|
expect(subject).to receive(:clear_cache).twice.and_call_original
|
439
448
|
expect do
|
440
449
|
try_with_git_mirror([retriable_error, retriable_error], [])
|
441
|
-
end.to raise_error(
|
450
|
+
end.to raise_error(RunnerExecution::RunnerExecutionRuntimeError)
|
442
451
|
end
|
443
452
|
end
|
444
453
|
end
|
445
454
|
|
446
455
|
describe '.retriable_error?' do
|
447
456
|
def format_error(error)
|
448
|
-
error_wrapper =
|
457
|
+
error_wrapper = error.to_s
|
449
458
|
error_wrapper.strip.lines.map(&:strip).join("\n")
|
450
459
|
end
|
451
460
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: git-fastclone
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Tauraso
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2023-
|
12
|
+
date: 2023-03-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: colorize
|
@@ -25,20 +25,6 @@ dependencies:
|
|
25
25
|
- - ">="
|
26
26
|
- !ruby/object:Gem::Version
|
27
27
|
version: '0'
|
28
|
-
- !ruby/object:Gem::Dependency
|
29
|
-
name: terrapin
|
30
|
-
requirement: !ruby/object:Gem::Requirement
|
31
|
-
requirements:
|
32
|
-
- - "~>"
|
33
|
-
- !ruby/object:Gem::Version
|
34
|
-
version: 0.6.0
|
35
|
-
type: :runtime
|
36
|
-
prerelease: false
|
37
|
-
version_requirements: !ruby/object:Gem::Requirement
|
38
|
-
requirements:
|
39
|
-
- - "~>"
|
40
|
-
- !ruby/object:Gem::Version
|
41
|
-
version: 0.6.0
|
42
28
|
description: A git command that uses reference repositories and threading to quicklyand
|
43
29
|
recursively clone repositories with many nested submodules
|
44
30
|
email:
|
@@ -55,6 +41,7 @@ files:
|
|
55
41
|
- bin/git-fastclone
|
56
42
|
- lib/git-fastclone.rb
|
57
43
|
- lib/git-fastclone/version.rb
|
44
|
+
- lib/runner_execution.rb
|
58
45
|
- spec/git_fastclone_runner_spec.rb
|
59
46
|
- spec/git_fastclone_url_helper_spec.rb
|
60
47
|
- spec/spec_helper.rb
|