right_scraper 1.0.16 → 1.0.17
Sign up to get free protection for your applications and to get access to all the features.
@@ -117,12 +117,11 @@ module RightScale
|
|
117
117
|
# === Parameters
|
118
118
|
# res(RightScale::WatchResult):: Watcher status to be analyzed
|
119
119
|
# msg_title(String):: Error message title in case of failure
|
120
|
-
# update(TrueClass|FalseClass):: Whether the process was launch to incrementally update the repo
|
121
120
|
# ok_codes:: Successful process return codes, only 0 by default
|
122
121
|
#
|
123
122
|
# === Return
|
124
123
|
# true:: Always return true
|
125
|
-
def handle_watcher_result(res, msg_title,
|
124
|
+
def handle_watcher_result(res, msg_title, ok_codes=[0])
|
126
125
|
if res.status == :timeout
|
127
126
|
@errors << "#{msg_title} is taking more time than #{@watcher.max_seconds / 60} minutes, aborting..."
|
128
127
|
FileUtils.rm_rf(@current_repo_dir)
|
@@ -130,7 +129,7 @@ module RightScale
|
|
130
129
|
@errors << "#{msg_title} is taking more space than #{@watcher.max_bytes / 1048576} MB, aborting..."
|
131
130
|
FileUtils.rm_rf(@current_repo_dir)
|
132
131
|
elsif !ok_codes.include?(res.exit_code)
|
133
|
-
if
|
132
|
+
if @incremental
|
134
133
|
@callback.call("#{msg_title} failed: #{res.output}, reverting to non incremental update", is_step=false) if @callback
|
135
134
|
FileUtils.rm_rf(@current_repo_dir)
|
136
135
|
@incremental = false
|
@@ -37,7 +37,7 @@ module RightScale
|
|
37
37
|
cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output \"#{@current_repo_dir}/#{filename}\" '#{@repo.url}' 2>&1"
|
38
38
|
FileUtils.mkdir_p(@current_repo_dir)
|
39
39
|
res = @watcher.launch_and_watch(cmd, @current_repo_dir)
|
40
|
-
handle_watcher_result(res, 'Download'
|
40
|
+
handle_watcher_result(res, 'Download')
|
41
41
|
if succeeded?
|
42
42
|
unzip_opt = case @repo.url[/\.(.*)$/]
|
43
43
|
when 'bzip', 'bzip2', 'bz2' then 'j'
|
@@ -47,54 +47,52 @@ module RightScale
|
|
47
47
|
msg = @incremental ? 'Pulling ' : 'Cloning '
|
48
48
|
msg += "git repository '#{@repo.display_name}'"
|
49
49
|
@callback.call(msg, is_step=true) if @callback
|
50
|
-
ssh_cmd = ssh_command
|
50
|
+
@ssh_cmd = ssh_command
|
51
51
|
is_tag = is_branch = on_branch = nil
|
52
52
|
has_tag = !@repo.tag.nil? && !@repo.tag.empty?
|
53
53
|
|
54
54
|
if @incremental
|
55
55
|
Dir.chdir(@current_repo_dir) do
|
56
|
-
|
56
|
+
git_fetch(:depth => 1)
|
57
57
|
if succeeded? && @incremental && has_tag
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
58
|
+
analysis = analyze_repo_tag
|
59
|
+
if succeeded?
|
60
|
+
is_tag = analysis[:tag]
|
61
|
+
is_branch = analysis[:branch]
|
62
|
+
on_branch = analysis[:on_branch]
|
63
|
+
checkout = is_tag && !is_branch
|
64
|
+
if is_tag && is_branch
|
65
|
+
@errors << 'Repository tag ambiguous: could be git tag or git branch'
|
66
|
+
elsif !is_tag && !is_branch
|
67
|
+
current_sha = `git rev-parse HEAD`.chomp
|
68
|
+
if current_sha == @repo.tag
|
69
|
+
@callback.call("Nothing to update: already using #{@repo.tag}", is_step=false) if @callback
|
70
|
+
return true
|
71
|
+
else
|
72
|
+
# Probably a SHA, retrieve all commits
|
73
|
+
git_fetch(:depth => 2**31 - 1)
|
74
|
+
checkout = true
|
75
|
+
end
|
71
76
|
end
|
72
77
|
end
|
73
78
|
if succeeded?
|
74
79
|
if checkout || is_branch && !on_branch
|
75
|
-
|
76
|
-
action = 'checkout'
|
80
|
+
git_checkout(@repo.tag)
|
77
81
|
else # Pull latest commits on same branch
|
78
|
-
|
79
|
-
action = 'pull'
|
80
|
-
end
|
81
|
-
if $? != 0
|
82
|
-
@callback.call("Failed to #{action} #{@repo.tag}: #{res}, falling back to cloning", is_step=false) if @callback
|
83
|
-
FileUtils.rm_rf(@current_repo_dir)
|
84
|
-
@incremental = false
|
82
|
+
git_fetch(:merge => true, :remote_tag => @repo.tag)
|
85
83
|
end
|
86
84
|
end
|
87
85
|
end
|
88
86
|
end
|
89
87
|
end
|
90
88
|
if !@incremental && succeeded?
|
91
|
-
git_cmd = "#{ssh_cmd} git clone --quiet --depth 1 \"#{@repo.url}\" \"#{@current_repo_dir}\" 2>&1"
|
89
|
+
git_cmd = "#{@ssh_cmd} git clone --quiet --depth 1 \"#{@repo.url}\" \"#{@current_repo_dir}\" 2>&1"
|
92
90
|
res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)
|
93
|
-
handle_watcher_result(res, 'git clone'
|
91
|
+
handle_watcher_result(res, 'git clone')
|
94
92
|
if has_tag && succeeded?
|
95
93
|
Dir.chdir(@current_repo_dir) do
|
96
94
|
if is_tag.nil?
|
97
|
-
analysis =
|
95
|
+
analysis = analyze_repo_tag
|
98
96
|
is_tag = analysis[:tag]
|
99
97
|
is_branch = analysis[:branch]
|
100
98
|
on_branch = analysis[:on_branch]
|
@@ -108,12 +106,10 @@ module RightScale
|
|
108
106
|
@errors << output if $? != 0
|
109
107
|
end
|
110
108
|
elsif !is_tag # Not a branch nor a tag, SHA ref? fetch everything so we have all SHAs
|
111
|
-
|
112
|
-
@errors << output if $? != 0
|
109
|
+
git_fetch(:depth => 2**31 -1)
|
113
110
|
end
|
114
111
|
if succeeded? && !on_branch
|
115
|
-
|
116
|
-
@errors << output if $? != 0
|
112
|
+
git_checkout(@repo.tag)
|
117
113
|
end
|
118
114
|
end
|
119
115
|
end
|
@@ -202,31 +198,62 @@ module RightScale
|
|
202
198
|
return ''
|
203
199
|
end
|
204
200
|
|
205
|
-
#
|
206
|
-
#
|
207
|
-
#
|
201
|
+
# Fetch remote commits using given depth
|
202
|
+
# Check size of repo and time it takes to retrieve commits
|
203
|
+
# Update errors collection upon failure (check for succeeded? after call)
|
204
|
+
# Note: Assume that current working directory is a git directory
|
205
|
+
#
|
206
|
+
# === Parameters
|
207
|
+
# opts[:depth(Integer):: Git fetch depth argument, full fetch if not set
|
208
|
+
# opts[:merge]:: Do a pull if set
|
209
|
+
# opts[:remote_tag]:: Remote ref to use, use default if not specified
|
210
|
+
#
|
211
|
+
# === Return
|
212
|
+
# true:: Always return true
|
213
|
+
def git_fetch(opts={})
|
214
|
+
depth = opts[:depth] || 2**31 - 1 # Specify max to override depth of already cloned repo
|
215
|
+
remote = opts[:remote_tag] || 'master'
|
216
|
+
action = (opts[:merge] ? 'pull' : 'fetch')
|
217
|
+
git_cmd = "#{@ssh_cmd} git #{action} --tags --depth #{depth} origin #{remote} 2>&1"
|
218
|
+
res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)
|
219
|
+
handle_watcher_result(res, "git #{action}", ok_codes=[0, 1]) # git fetch returns 1 when there is nothing to fetch
|
220
|
+
end
|
221
|
+
|
222
|
+
# Does a git checkout to given tag
|
208
223
|
# Update errors collection upon failure (check for succeeded? after call)
|
209
224
|
# Note: Assume that current working directory is a git directory
|
210
225
|
#
|
211
226
|
# === Parameters
|
212
|
-
#
|
213
|
-
#
|
227
|
+
# tag(String):: Tag to checkout
|
228
|
+
#
|
229
|
+
# === Return
|
230
|
+
# output(String):: Output of git command
|
231
|
+
def git_checkout(tag)
|
232
|
+
output = `git checkout #{tag} 2>&1`
|
233
|
+
@errors << output if $? != 0
|
234
|
+
output
|
235
|
+
end
|
236
|
+
|
237
|
+
# Analyze repository tag to detect whether it's a branch, a tag or neither (i.e. SHA ref)
|
238
|
+
# Also detech wether the branch is already checked out
|
239
|
+
# Update errors collection upon failure (check for succeeded? after call)
|
240
|
+
# Note: Assume that current working directory is a git directory
|
214
241
|
#
|
215
242
|
# === Return
|
216
243
|
# res(Hash)::
|
217
|
-
# - res[:tag]::
|
218
|
-
# - res[:branch]
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
is_tag = is_branch = on_branch = false
|
224
|
-
if succeeded? && (!update || @incremental)
|
244
|
+
# - res[:tag]:: true if git repo has a tag with a name corresponding to the repository tag
|
245
|
+
# - res[:branch]:: true if git repo has a branch with a name corresponding to the repository tag
|
246
|
+
# - res [:on_branch]:: true if branch is already checked out
|
247
|
+
def analyze_repo_tag
|
248
|
+
is_tag = is_branch = on_branch = nil
|
249
|
+
begin
|
225
250
|
is_tag = `git tag`.split("\n").include?(@repo.tag)
|
226
251
|
is_branch = `git branch -r`.split("\n").map { |t| t.strip }.include?("origin/#{@repo.tag}")
|
227
252
|
on_branch = is_branch && !!`git branch`.split("\n").include?("* #{@repo.tag}")
|
253
|
+
rescue Exception => e
|
254
|
+
@errors << "Analysis of repository tag failed with: #{e.message}"
|
228
255
|
end
|
229
|
-
{ :tag => is_tag, :branch => is_branch, :on_branch => on_branch }
|
256
|
+
res = { :tag => is_tag, :branch => is_branch, :on_branch => on_branch }
|
230
257
|
end
|
231
258
|
|
232
259
|
end
|
@@ -71,7 +71,7 @@ module RightScale
|
|
71
71
|
if cookbooks_path.empty?
|
72
72
|
Dir.chdir(@current_repo_dir) do
|
73
73
|
res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
|
74
|
-
handle_watcher_result(res, 'SVN update'
|
74
|
+
handle_watcher_result(res, 'SVN update')
|
75
75
|
end
|
76
76
|
else
|
77
77
|
cookbooks_path.each do |path|
|
@@ -79,7 +79,7 @@ module RightScale
|
|
79
79
|
full_path = File.join(@current_repo_dir, path)
|
80
80
|
Dir.chdir(full_path) do
|
81
81
|
res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
|
82
|
-
handle_watcher_result(res, 'SVN update'
|
82
|
+
handle_watcher_result(res, 'SVN update')
|
83
83
|
end
|
84
84
|
end
|
85
85
|
end
|
@@ -87,12 +87,12 @@ module RightScale
|
|
87
87
|
if !@incremental && succeeded?
|
88
88
|
if cookbooks_path.empty?
|
89
89
|
res = @watcher.launch_and_watch(svn_checkout_cmd, @current_repo_dir)
|
90
|
-
handle_watcher_result(res, 'SVN checkout'
|
90
|
+
handle_watcher_result(res, 'SVN checkout')
|
91
91
|
else
|
92
92
|
cookbooks_path.each do |path|
|
93
93
|
break unless succeeded?
|
94
94
|
res = @watcher.launch_and_watch(svn_checkout_cmd(path), @current_repo_dir)
|
95
|
-
handle_watcher_result(res, 'SVN checkout'
|
95
|
+
handle_watcher_result(res, 'SVN checkout')
|
96
96
|
end
|
97
97
|
end
|
98
98
|
end
|
data/right_scraper.gemspec
CHANGED
@@ -23,7 +23,7 @@ require 'rubygems'
|
|
23
23
|
|
24
24
|
spec = Gem::Specification.new do |spec|
|
25
25
|
spec.name = 'right_scraper'
|
26
|
-
spec.version = '1.0.
|
26
|
+
spec.version = '1.0.17'
|
27
27
|
spec.authors = ['Raphael Simon']
|
28
28
|
spec.email = 'raphael@rightscale.com'
|
29
29
|
spec.homepage = 'https://github.com/rightscale/right_scraper'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: right_scraper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Raphael Simon
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-
|
12
|
+
date: 2010-05-11 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|