puppet 2.7.21 → 2.7.22
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of puppet might be problematic. Click here for more details.
- data/CHANGELOG +14 -0
- data/Gemfile.lock +2 -2
- data/ext/build_defaults.yaml +2 -3
- data/ext/debian/control +1 -1
- data/ext/packaging/README.md +496 -8
- data/ext/packaging/spec/tasks/00_utils_spec.rb +7 -7
- data/ext/packaging/spec/tasks/build_object_spec.rb +3 -0
- data/ext/packaging/tasks/00_utils.rake +2 -2
- data/ext/packaging/tasks/10_setupvars.rake +8 -1
- data/ext/packaging/tasks/build.rake +2 -0
- data/ext/packaging/tasks/deb_repos.rake +48 -15
- data/ext/packaging/tasks/jenkins.rake +30 -2
- data/ext/packaging/tasks/mock.rake +3 -2
- data/ext/packaging/tasks/pe_remote.rake +1 -1
- data/ext/packaging/tasks/pe_ship.rake +4 -5
- data/ext/packaging/tasks/pe_sign.rake +8 -0
- data/ext/packaging/tasks/pe_sles.rake +8 -7
- data/ext/packaging/tasks/pre_tasks.rake +0 -0
- data/ext/packaging/tasks/retrieve.rake +11 -1
- data/ext/packaging/tasks/rpm_repos.rake +71 -49
- data/ext/packaging/tasks/ship.rake +14 -2
- data/ext/packaging/tasks/sign.rake +9 -3
- data/ext/packaging/tasks/tar.rake +5 -0
- data/ext/packaging/tasks/vendor_gems.rake +110 -0
- data/install.rb +1 -1
- data/lib/puppet.rb +11 -0
- data/lib/puppet/indirector/report/processor.rb +1 -1
- data/lib/puppet/indirector/report/rest.rb +7 -0
- data/lib/puppet/indirector/resource/rest.rb +9 -0
- data/lib/puppet/indirector/rest.rb +81 -47
- data/lib/puppet/indirector/run/rest.rb +6 -0
- data/lib/puppet/network/formats.rb +20 -10
- data/lib/puppet/network/http/handler.rb +1 -1
- data/lib/puppet/node.rb +25 -0
- data/lib/puppet/node/facts.rb +23 -4
- data/lib/puppet/resource.rb +2 -4
- data/lib/puppet/resource/status.rb +28 -0
- data/lib/puppet/run.rb +24 -2
- data/lib/puppet/status.rb +6 -2
- data/lib/puppet/transaction/event.rb +19 -0
- data/lib/puppet/transaction/report.rb +39 -0
- data/lib/puppet/util/log.rb +19 -0
- data/lib/puppet/util/metric.rb +6 -0
- data/lib/puppet/util/monkey_patches.rb +0 -16
- data/lib/puppet/vendor.rb +55 -0
- data/lib/puppet/vendor/load_safe_yaml.rb +1 -0
- data/lib/puppet/vendor/require_vendored.rb +4 -0
- data/lib/puppet/vendor/safe_yaml/CHANGES.md +104 -0
- data/lib/puppet/vendor/safe_yaml/Gemfile +11 -0
- data/lib/puppet/vendor/safe_yaml/LICENSE.txt +22 -0
- data/lib/puppet/vendor/safe_yaml/README.md +179 -0
- data/lib/puppet/vendor/safe_yaml/Rakefile +6 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml.rb +253 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/deep.rb +34 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/parse/date.rb +27 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/parse/hexadecimal.rb +12 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/parse/sexagesimal.rb +26 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/psych_handler.rb +92 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/psych_resolver.rb +52 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/resolver.rb +94 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/safe_to_ruby_visitor.rb +17 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/syck_hack.rb +36 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/syck_node_monkeypatch.rb +43 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/syck_resolver.rb +38 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform.rb +41 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_boolean.rb +21 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_date.rb +11 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_float.rb +33 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_integer.rb +25 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_nil.rb +18 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_symbol.rb +13 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/transformation_map.rb +47 -0
- data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/version.rb +3 -0
- data/lib/puppet/vendor/safe_yaml/run_specs_all_ruby_versions.sh +21 -0
- data/lib/puppet/vendor/safe_yaml/safe_yaml.gemspec +18 -0
- data/lib/puppet/vendor/safe_yaml/spec/exploit.1.9.2.yaml +2 -0
- data/lib/puppet/vendor/safe_yaml/spec/exploit.1.9.3.yaml +2 -0
- data/lib/puppet/vendor/safe_yaml/spec/psych_resolver_spec.rb +10 -0
- data/lib/puppet/vendor/safe_yaml/spec/resolver_specs.rb +250 -0
- data/lib/puppet/vendor/safe_yaml/spec/safe_yaml_spec.rb +702 -0
- data/lib/puppet/vendor/safe_yaml/spec/spec_helper.rb +18 -0
- data/lib/puppet/vendor/safe_yaml/spec/support/exploitable_back_door.rb +29 -0
- data/lib/puppet/vendor/safe_yaml/spec/syck_resolver_spec.rb +10 -0
- data/lib/puppet/vendor/safe_yaml/spec/transform/base64_spec.rb +11 -0
- data/lib/puppet/vendor/safe_yaml/spec/transform/to_date_spec.rb +34 -0
- data/lib/puppet/vendor/safe_yaml/spec/transform/to_float_spec.rb +42 -0
- data/lib/puppet/vendor/safe_yaml/spec/transform/to_integer_spec.rb +59 -0
- data/lib/puppet/vendor/safe_yaml/spec/transform/to_symbol_spec.rb +49 -0
- data/lib/puppet/version.rb +1 -1
- data/spec/lib/puppet_spec/matchers.rb +8 -0
- data/spec/unit/file_serving/metadata_spec.rb +20 -28
- data/spec/unit/indirector/report/rest_spec.rb +41 -0
- data/spec/unit/indirector/rest_spec.rb +314 -339
- data/spec/unit/network/formats_spec.rb +36 -27
- data/spec/unit/network/http/handler_spec.rb +3 -12
- data/spec/unit/node_spec.rb +81 -0
- data/spec/unit/resource_spec.rb +5 -35
- data/spec/unit/run_spec.rb +22 -8
- data/spec/unit/status_spec.rb +6 -0
- data/test/network/handler/report.rb +0 -36
- metadata +148 -102
@@ -15,14 +15,14 @@ describe "00_utils" do
|
|
15
15
|
:get_rpmrelease => '1',
|
16
16
|
:is_rc? => false,
|
17
17
|
},
|
18
|
-
'0.7.
|
19
|
-
:git_describe_version => %w{0.7.
|
20
|
-
:get_dash_version => '0.7.
|
21
|
-
:get_ips_version => '0.7.
|
22
|
-
:get_dot_version => '0.7.
|
23
|
-
:get_debversion => '0.7.0-0.
|
18
|
+
'0.7.0rc10' => {
|
19
|
+
:git_describe_version => %w{0.7.0rc10},
|
20
|
+
:get_dash_version => '0.7.0rc10',
|
21
|
+
:get_ips_version => '0.7.0rc10,3.14159-0',
|
22
|
+
:get_dot_version => '0.7.0rc10',
|
23
|
+
:get_debversion => '0.7.0-0.1rc10puppetlabs1',
|
24
24
|
:get_rpmversion => '0.7.0',
|
25
|
-
:get_rpmrelease => '0.
|
25
|
+
:get_rpmrelease => '0.1rc10',
|
26
26
|
:is_rc? => true,
|
27
27
|
},
|
28
28
|
'0.7.0-rc1' => {
|
@@ -2,6 +2,7 @@
|
|
2
2
|
require 'spec_helper'
|
3
3
|
load_task '00_utils.rake'
|
4
4
|
load_task 'build.rake'
|
5
|
+
require 'yaml'
|
5
6
|
|
6
7
|
describe Build::BuildInstance do
|
7
8
|
Build_Params = [:apt_host,
|
@@ -71,6 +72,7 @@ describe Build::BuildInstance do
|
|
71
72
|
:pe_name,
|
72
73
|
:pe_version,
|
73
74
|
:pg_major_version,
|
75
|
+
:pre_tar_task,
|
74
76
|
:privatekey_pem,
|
75
77
|
:random_mockroot,
|
76
78
|
:rc_mocks,
|
@@ -86,6 +88,7 @@ describe Build::BuildInstance do
|
|
86
88
|
:sles_arch_repos,
|
87
89
|
:summary,
|
88
90
|
:tar_excludes,
|
91
|
+
:tar_host,
|
89
92
|
:tarball_path,
|
90
93
|
:team,
|
91
94
|
:version,
|
@@ -92,7 +92,7 @@ def git_sha_or_tag
|
|
92
92
|
end
|
93
93
|
|
94
94
|
def get_temp
|
95
|
-
temp = `mktemp -d -t
|
95
|
+
temp = `mktemp -d -t pkgXXXXXX`.strip
|
96
96
|
end
|
97
97
|
|
98
98
|
def remote_ssh_cmd target, command
|
@@ -202,7 +202,7 @@ def get_base_pkg_version
|
|
202
202
|
dash = get_dash_version
|
203
203
|
if dash.include?("rc")
|
204
204
|
# Grab the rc number
|
205
|
-
rc_num = dash.match(/rc(\d)
|
205
|
+
rc_num = dash.match(/rc(\d+)/)[1]
|
206
206
|
ver = dash.sub(/-?rc[0-9]+/, "-0.#{@build.release}rc#{rc_num}").gsub(/(rc[0-9]+)-(\d+)?-?/, '\1.\2')
|
207
207
|
else
|
208
208
|
ver = dash.gsub('-','.') + "-#{@build.release}"
|
@@ -53,6 +53,7 @@ else
|
|
53
53
|
@build.yum_repo_path = ENV['YUM_REPO'] if ENV['YUM_REPO']
|
54
54
|
@build.apt_host = ENV['APT_HOST'] if ENV['APT_HOST']
|
55
55
|
@build.apt_repo_path = ENV['APT_REPO'] if ENV['APT_REPO']
|
56
|
+
@build.pe_version = ENV['PE_VER'] if ENV['PE_VER']
|
56
57
|
end
|
57
58
|
|
58
59
|
##
|
@@ -77,8 +78,14 @@ end
|
|
77
78
|
# For backwards compatibilty, we set build:@name to build:@project. @name was
|
78
79
|
# renamed to @project in an effort to align the variable names with what has
|
79
80
|
# been supported for parameter names in the params files.
|
80
|
-
#
|
81
81
|
@build.name = @build.project
|
82
|
+
# We also set @tar_host to @yum_host if @tar_host is not set. This is in
|
83
|
+
# another effort to fix dumb mistakes. Early on, we just assumed tarballs would
|
84
|
+
# go to @yum_host (why? probably just laziness) but this is not ideal and does
|
85
|
+
# not make any sense when looking at the code. Now there's a @tar_host
|
86
|
+
# variable, but for backwards compatibility, we'll default back to @yum_host if
|
87
|
+
# @tar_host isn't set.
|
88
|
+
@build.tar_host ||= @build.yum_host
|
82
89
|
|
83
90
|
if @build.debug
|
84
91
|
@build.print_params
|
@@ -74,6 +74,7 @@ module Build
|
|
74
74
|
:pe_name,
|
75
75
|
:pe_version,
|
76
76
|
:pg_major_version,
|
77
|
+
:pre_tar_task,
|
77
78
|
:privatekey_pem,
|
78
79
|
:random_mockroot,
|
79
80
|
:rc_mocks,
|
@@ -89,6 +90,7 @@ module Build
|
|
89
90
|
:sles_arch_repos,
|
90
91
|
:summary,
|
91
92
|
:tar_excludes,
|
93
|
+
:tar_host,
|
92
94
|
:tarball_path,
|
93
95
|
:task,
|
94
96
|
:team,
|
@@ -12,15 +12,16 @@ namespace :pl do
|
|
12
12
|
namespace :jenkins do
|
13
13
|
desc "Create apt repositories of build DEB packages for this SHA on the distributions erver"
|
14
14
|
task :deb_repos => "pl:fetch" do
|
15
|
+
prefix = @build.build_pe ? "pe/" : ""
|
15
16
|
|
16
17
|
# First, we test that artifacts exist and set up the repos directory
|
17
18
|
artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
|
18
19
|
|
19
20
|
cmd = 'echo " Checking for deb build artifacts. Will exit if not found.." ; '
|
20
|
-
cmd << "[ -d #{artifact_directory}/artifacts
|
21
|
+
cmd << "[ -d #{artifact_directory}/artifacts/#{prefix}deb ] || exit 0 ; "
|
21
22
|
# Descend into the deb directory and obtain the list of distributions
|
22
23
|
# we'll be building repos for
|
23
|
-
cmd << "pushd #{artifact_directory}/artifacts
|
24
|
+
cmd << "pushd #{artifact_directory}/artifacts/#{prefix}deb && dists=$(ls) && popd; "
|
24
25
|
# We do one more check here to make sure we actually have distributions
|
25
26
|
# to build for. If deb is empty we want to just exit.
|
26
27
|
#
|
@@ -48,7 +49,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
|
|
48
49
|
# testing only, we'll just add the debs and ignore source files for now.
|
49
50
|
#
|
50
51
|
cmd << "reprepro=$(which reprepro) ; "
|
51
|
-
cmd << "$reprepro includedeb $dist
|
52
|
+
cmd << "$reprepro includedeb $dist ../../#{prefix}deb/$dist/*.deb ; popd ; done ; "
|
52
53
|
cmd << "popd ; popd "
|
53
54
|
|
54
55
|
remote_ssh_cmd(@build.distribution_server, cmd)
|
@@ -58,7 +59,10 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
|
|
58
59
|
|
59
60
|
# Now that we've created our package repositories, we can generate repo
|
60
61
|
# configurations for use with downstream jobs, acceptance clients, etc.
|
61
|
-
Rake::Task["pl:jenkins:
|
62
|
+
Rake::Task["pl:jenkins:generate_deb_repo_configs"].execute
|
63
|
+
|
64
|
+
# Now that we've created the repo configs, we can ship them
|
65
|
+
Rake::Task["pl:jenkins:ship_repo_configs"].execute
|
62
66
|
end
|
63
67
|
|
64
68
|
# Generate apt configuration files that point to the repositories created
|
@@ -69,20 +73,29 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
|
|
69
73
|
# enable clients to install these packages.
|
70
74
|
#
|
71
75
|
desc "Create apt repository configs for package repos for this sha/tag on the distribution server"
|
72
|
-
task :
|
76
|
+
task :generate_deb_repo_configs => "pl:fetch" do
|
73
77
|
|
74
|
-
# This is the standard path to all build
|
75
|
-
# server for this commit
|
78
|
+
# This is the standard path to all debian build artifact repositories on
|
79
|
+
# the distribution server for this commit
|
76
80
|
#
|
77
|
-
|
81
|
+
base_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/apt/"
|
78
82
|
|
79
|
-
# We obtain
|
83
|
+
# We use wget to obtain a directory listing of what are presumably our deb repos
|
80
84
|
#
|
81
|
-
|
82
|
-
if
|
83
|
-
|
85
|
+
repo_urls = []
|
86
|
+
if wget = find_tool("wget")
|
87
|
+
# First test if the directory even exists
|
88
|
+
#
|
89
|
+
wget_results = %x{#{wget} --spider -r -l 1 --no-parent #{base_url} 2>&1}
|
90
|
+
if $?.success?
|
91
|
+
# We want to exclude index and robots files and only include the http: prefixed elements
|
92
|
+
repo_urls = wget_results.split.uniq.reject{|x| x=~ /\?|index|robots/}.select{|x| x =~ /http:/}.map{|x| x.chomp('/')}
|
93
|
+
else
|
94
|
+
puts "No debian repos available for #{@build.project} at #{@build.ref}."
|
95
|
+
exit 0
|
96
|
+
end
|
84
97
|
else
|
85
|
-
warn "
|
98
|
+
warn "Could not find `wget` tool. This is needed for composing the debian repo configurations. Install `wget` and try again."
|
86
99
|
exit 0
|
87
100
|
end
|
88
101
|
|
@@ -91,13 +104,33 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
|
|
91
104
|
# file for every distribution.
|
92
105
|
#
|
93
106
|
mkdir_p File.join("pkg", "repo_configs", "deb")
|
94
|
-
|
107
|
+
repo_urls.each do |url|
|
108
|
+
# We want to skip the base_url, which wget returns as one of the results
|
109
|
+
next if "#{url}/" == base_url
|
110
|
+
dist = url.split('/').last
|
95
111
|
repoconfig = ["# Packages for #{@build.project} built from ref #{@build.ref}",
|
96
|
-
"deb
|
112
|
+
"deb #{url} #{dist} main"]
|
97
113
|
config = File.join("pkg", "repo_configs", "deb", "pl-#{@build.project}-#{@build.ref}-#{dist}.list")
|
98
114
|
File.open(config, 'w') { |f| f.puts repoconfig }
|
99
115
|
end
|
100
116
|
puts "Wrote apt repo configs for #{@build.project} at #{@build.ref} to pkg/repo_configs/deb."
|
101
117
|
end
|
118
|
+
|
119
|
+
desc "Retrieve debian apt repository configs for this sha"
|
120
|
+
task :deb_repo_configs => "pl:fetch" do
|
121
|
+
if wget = find_tool("wget")
|
122
|
+
mkdir_p "pkg/repo_configs"
|
123
|
+
config_url = "#{@build.builds_server}/#{@build.project}/#{@build.ref}/repo_configs/deb/"
|
124
|
+
begin
|
125
|
+
sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{config_url}"
|
126
|
+
rescue
|
127
|
+
warn "Couldn't retrieve deb apt repo configs. See preceding http response for more info."
|
128
|
+
exit 1
|
129
|
+
end
|
130
|
+
else
|
131
|
+
warn "Could not find `wget` tool! wget is required to download the repository configs."
|
132
|
+
exit 1
|
133
|
+
end
|
134
|
+
end
|
102
135
|
end
|
103
136
|
end
|
@@ -178,6 +178,10 @@ namespace :pl do
|
|
178
178
|
else
|
179
179
|
warn "An error occurred submitting the job to jenkins. Take a look at the preceding http response for more info."
|
180
180
|
end
|
181
|
+
|
182
|
+
# Clean up after ourselves
|
183
|
+
rm bundle
|
184
|
+
rm properties
|
181
185
|
end
|
182
186
|
end
|
183
187
|
end
|
@@ -261,10 +265,34 @@ if @build.build_pe
|
|
261
265
|
end
|
262
266
|
end
|
263
267
|
|
268
|
+
# While pl:remote:deb_all does all cows in serially, with jenkins we
|
269
|
+
# parallelize them. This breaks the cows up and posts a build for all of
|
270
|
+
# them. We have to sleep 5 because jenkins drops the builds when we're
|
271
|
+
# DOSing it with our packaging.
|
272
|
+
desc "Queue pe:deb_all on jenkins builder"
|
273
|
+
task :deb_all => "pl:fetch" do
|
274
|
+
check_var("PE_VER", @build.pe_version)
|
275
|
+
@build.cows.split(' ').each do |cow|
|
276
|
+
@build.default_cow = cow
|
277
|
+
invoke_task("pl:jenkins:post_build", "pe:local_deb")
|
278
|
+
sleep 5
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
# This does the mocks in parallel
|
283
|
+
desc "Queue pe:mock-all on jenkins builder"
|
284
|
+
task :mock_all => "pl:fetch" do
|
285
|
+
@build.final_mocks.split(' ').each do |mock|
|
286
|
+
@build.default_mock = mock
|
287
|
+
invoke_task("pl:jenkins:post_build", "pe:local_mock")
|
288
|
+
sleep 5
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
264
292
|
desc "Queue builds of all PE packages for this project in Jenkins"
|
265
293
|
task :uber_build => "pl:fetch" do
|
266
294
|
check_var("PE_VER", @build.pe_version)
|
267
|
-
["deb_all", "mock_all", "sles"].each do |task|
|
295
|
+
["tar", "deb_all", "mock_all", "sles"].each do |task|
|
268
296
|
invoke_task("pe:jenkins:#{task}")
|
269
297
|
sleep 5
|
270
298
|
end
|
@@ -273,7 +301,7 @@ if @build.build_pe
|
|
273
301
|
desc "Retrieve PE packages built by jenkins, sign, and ship all!"
|
274
302
|
task :uber_ship => "pl:fetch" do
|
275
303
|
check_var("PE_VER", @build.pe_version)
|
276
|
-
["pl:jenkins:retrieve", "pe:ship_rpms", "pe:ship_debs"].each do |task|
|
304
|
+
["pl:jenkins:retrieve", "pl:jenkins:sign_all", "pe:ship_rpms", "pe:ship_debs"].each do |task|
|
277
305
|
Rake::Task[task].invoke
|
278
306
|
end
|
279
307
|
Rake::Task["pl:jenkins:ship"].invoke("shipped")
|
@@ -19,6 +19,7 @@
|
|
19
19
|
# pkg/el-5-i386/*.rpm
|
20
20
|
|
21
21
|
def mock(mock_config, srpm)
|
22
|
+
configdir = nil
|
22
23
|
unless mock = find_tool('mock')
|
23
24
|
warn "mock is required for building rpms with mock. Please install mock and try again."
|
24
25
|
exit 1
|
@@ -39,7 +40,7 @@ def mock(mock_config, srpm)
|
|
39
40
|
end
|
40
41
|
sh "#{mock} -r #{mock_config} #{srpm}"
|
41
42
|
# Clean up the configdir
|
42
|
-
rm_r configdir
|
43
|
+
rm_r configdir unless configdir.nil?
|
43
44
|
|
44
45
|
basedir
|
45
46
|
end
|
@@ -124,7 +125,7 @@ def build_rpm_with_mock(mocks, is_rc)
|
|
124
125
|
cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-srpms")
|
125
126
|
when /i.?86/
|
126
127
|
cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
|
127
|
-
when /
|
128
|
+
when /x86_64/
|
128
129
|
cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-x86_64")
|
129
130
|
when /noarch/
|
130
131
|
cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
|
@@ -44,7 +44,7 @@ if @build.build_pe
|
|
44
44
|
|
45
45
|
desc "Execute remote debian, el, and sles builds, sign, and ship pkgs"
|
46
46
|
task :all => ['clean', 'pl:fetch'] do
|
47
|
-
['pe:
|
47
|
+
['pe:deb_all', 'pe:mock_all', 'pe:sles', 'pe:ship_rpms', 'pe:ship_debs'].each do |task|
|
48
48
|
Rake::Task[task].execute
|
49
49
|
end
|
50
50
|
end
|
@@ -16,13 +16,12 @@ if @build.build_pe
|
|
16
16
|
|
17
17
|
desc "Ship PE debs to #{@build.apt_host}"
|
18
18
|
task :ship_debs => "pl:fetch" do
|
19
|
-
|
20
|
-
|
21
|
-
STDERR.puts "The 'pkg/pe/deb/#{dist}' directory has no packages. Did you run rake pe:deb?"
|
19
|
+
if empty_dir?("pkg/pe/deb")
|
20
|
+
STDERR.puts "The 'pkg/pe/deb' directory has no packages!"
|
22
21
|
exit 1
|
23
22
|
else
|
24
|
-
target_path = ENV['APT_REPO'] ? ENV['APT_REPO'] : "#{@build.apt_repo_path}/#{@build.pe_version}/repos/incoming/
|
25
|
-
rsync_to("pkg/pe/deb
|
23
|
+
target_path = ENV['APT_REPO'] ? ENV['APT_REPO'] : "#{@build.apt_repo_path}/#{@build.pe_version}/repos/incoming/disparate/"
|
24
|
+
rsync_to("pkg/pe/deb/", @build.apt_host, target_path)
|
26
25
|
if @build.team == 'release'
|
27
26
|
Rake::Task["pe:remote:freight"].invoke
|
28
27
|
end
|
@@ -24,5 +24,13 @@ if @build.build_pe
|
|
24
24
|
sign_el5(old_rpms) unless old_rpms.empty?
|
25
25
|
sign_modern(modern_rpms) unless modern_rpms.empty?
|
26
26
|
end
|
27
|
+
# This is essentially a duplicate of the logic in pl:sign_deb_changes, but
|
28
|
+
# since the plan is eventually to rip out the PE work, it'll be easier if
|
29
|
+
# this is a separate task we can pull out later.
|
30
|
+
desc "Sign all debian changes files staged in pkg/pe"
|
31
|
+
task :sign_deb_changes do
|
32
|
+
load_keychain if has_tool('keychain')
|
33
|
+
sign_deb_changes("pkg/pe/deb/*/*.changes") unless Dir["pkg/pe/deb/*/*.changes"].empty?
|
34
|
+
end
|
27
35
|
end
|
28
36
|
end
|
@@ -21,6 +21,7 @@ if @build.build_pe
|
|
21
21
|
namespace :pe do
|
22
22
|
# Temporary task to pull down pe dependencies until this is NFS-mounted
|
23
23
|
task :retrieve_sles_deps => 'pl:fetch' do
|
24
|
+
rm_rf FileList["#{ENV['HOME']}/package_repos/*"]
|
24
25
|
rsync_from("#{@build.sles_repo_path}/#{@build.pe_version}/repos/sles-*", @build.sles_repo_host, "#{ENV['HOME']}/package_repos/")
|
25
26
|
FileList["#{ENV['HOME']}/package_repos/*"].each do |f|
|
26
27
|
update_rpm_repo(f) if File.directory?(f)
|
@@ -40,15 +41,15 @@ if @build.build_pe
|
|
40
41
|
check_tool('build')
|
41
42
|
check_tool('linux32')
|
42
43
|
check_tool('linux64')
|
43
|
-
build_root = get_temp
|
44
|
-
work_dir = prep_sles_dir
|
45
|
-
build_source_dir = "#{work_dir}/SOURCES"
|
46
|
-
build_spec_dir = "#{work_dir}/SPECS"
|
47
|
-
build_spec = "#{build_spec_dir}/#{@build.project}.spec"
|
48
44
|
build_dest_dir = "usr/src/packages"
|
49
45
|
noarch = FALSE
|
50
46
|
built_arch = ''
|
51
47
|
@build.sles_arch_repos.each do |arch, deps_repo|
|
48
|
+
build_root = get_temp
|
49
|
+
work_dir = prep_sles_dir
|
50
|
+
build_source_dir = "#{work_dir}/SOURCES"
|
51
|
+
build_spec_dir = "#{work_dir}/SPECS"
|
52
|
+
build_spec = "#{build_spec_dir}/#{@build.project}.spec"
|
52
53
|
if noarch == FALSE
|
53
54
|
bench = Benchmark.realtime do
|
54
55
|
linux_cmd = arch == 'i586' ? 'linux32' : 'linux64'
|
@@ -70,8 +71,6 @@ if @build.build_pe
|
|
70
71
|
cp(rpms, "pkg/pe/rpm/sles-11-#{arch}")
|
71
72
|
cp(srpms, "pkg/pe/rpm/sles-11-srpms")
|
72
73
|
noarch = rpms.exclude(/noarch/).empty?
|
73
|
-
rm_rf build_root
|
74
|
-
rm_rf work_dir
|
75
74
|
end
|
76
75
|
# See 30_metrics.rake to see what this is doing
|
77
76
|
add_metrics({ :dist => 'sles', :bench => bench }) if @build.benchmark
|
@@ -82,6 +81,8 @@ if @build.build_pe
|
|
82
81
|
cp(FileList["pkg/pe/rpm/sles-11-#{built_arch}/*"], "pkg/pe/rpm/sles-11-#{other_arch}")
|
83
82
|
end
|
84
83
|
end
|
84
|
+
rm_rf build_root
|
85
|
+
rm_rf work_dir
|
85
86
|
end
|
86
87
|
post_metrics if @build.benchmark
|
87
88
|
cd 'pkg/pe/rpm' do
|
File without changes
|
@@ -16,7 +16,17 @@ namespace :pl do
|
|
16
16
|
target = args.target || "artifacts"
|
17
17
|
invoke_task("pl:fetch")
|
18
18
|
mkdir_p 'pkg'
|
19
|
-
|
19
|
+
package_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/#{target}"
|
20
|
+
if wget=find_tool("wget")
|
21
|
+
sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg --reject 'index*' #{package_url}/"
|
22
|
+
else
|
23
|
+
warn "Could not find `wget` tool. Falling back to rsyncing from #{@build.distribution_server}"
|
24
|
+
begin
|
25
|
+
rsync_from("#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}/#{target}/", @build.distribution_server, "pkg/")
|
26
|
+
rescue
|
27
|
+
puts "Couldn't download packages from distribution server. Try installing wget!"
|
28
|
+
end
|
29
|
+
end
|
20
30
|
puts "Packages staged in pkg"
|
21
31
|
end
|
22
32
|
end
|
@@ -46,7 +46,10 @@ namespace :pl do
|
|
46
46
|
|
47
47
|
# Now that we've created our repositories, we can create the configs for
|
48
48
|
# them
|
49
|
-
Rake::Task["pl:jenkins:
|
49
|
+
Rake::Task["pl:jenkins:generate_rpm_repo_configs"].execute
|
50
|
+
|
51
|
+
# And once they're created, we can ship them
|
52
|
+
Rake::Task["pl:jenkins:ship_repo_configs"].execute
|
50
53
|
end
|
51
54
|
|
52
55
|
# Generate yum configuration files that point to the repositories created
|
@@ -57,70 +60,89 @@ namespace :pl do
|
|
57
60
|
# clients to install these packages.
|
58
61
|
#
|
59
62
|
desc "Create yum repository configs for package repos for this sha/tag on the distribution server"
|
60
|
-
task :
|
63
|
+
task :generate_rpm_repo_configs => "pl:fetch" do
|
64
|
+
|
65
|
+
# We have a hard requirement on wget because of all the download magicks
|
66
|
+
# we have to do
|
67
|
+
#
|
68
|
+
unless wget = find_tool("wget")
|
69
|
+
warn "Could not find `wget` tool. This is needed for composing the yum repo configurations. Install `wget` and try again."
|
70
|
+
exit 0
|
71
|
+
end
|
61
72
|
|
62
73
|
# This is the standard path to all build artifacts on the distribution
|
63
74
|
# server for this commit
|
64
75
|
#
|
65
|
-
|
76
|
+
base_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/"
|
77
|
+
|
66
78
|
# First check if the artifacts directory exists
|
67
79
|
#
|
68
|
-
cmd = "[ -d #{artifact_directory} ] || exit 0 ; "
|
69
|
-
# Descend into the artifacts directory and test if we have any repos
|
70
|
-
#
|
71
|
-
cmd << "pushd #{artifact_directory} ; "
|
72
|
-
cmd << 'echo "Checking if rpm repos exists, will exit if not.." ; '
|
73
|
-
cmd << '[ -n "$(find repos -name "*.rpm")" ] || exit 0 ; '
|
74
|
-
cmd << "pushd repos ; "
|
75
80
|
|
76
|
-
|
81
|
+
# We have to do two checks here - first that there are directories with
|
82
|
+
# repodata folders in them, and second that those same directories also
|
83
|
+
# contain rpms
|
84
|
+
#
|
85
|
+
repo_urls = %x{#{wget} --spider -r -l 5 --no-parent #{base_url} 2>&1}.split.uniq.reject{ |x| x =~ /\?|index/ }.select{|x| x =~ /http:.*repodata\/$/}
|
77
86
|
|
78
|
-
|
87
|
+
# RPMs will always exist at the same directory level as the repodata
|
88
|
+
# folder, which means if we go up a level we should find rpms
|
89
|
+
#
|
90
|
+
yum_repos = []
|
91
|
+
repo_urls.map{|x| x.chomp('repodata/')}.each do |url|
|
92
|
+
unless %x{#{wget} --spider -r -l 1 --no-parent #{url} 2>&1}.split.uniq.reject{ |x| x =~ /\?|index/ }.select{|x| x =~ /http:.*\.rpm$/}.empty?
|
93
|
+
yum_repos << url
|
94
|
+
end
|
95
|
+
end
|
79
96
|
|
80
|
-
|
81
|
-
|
82
|
-
%x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/rpm_configs'}
|
83
|
-
unless $?.success?
|
84
|
-
warn "No repos were found to generate configs from. Exiting.."
|
97
|
+
if yum_repos.empty?
|
98
|
+
puts "No rpm repos were found to generate configs from. Exiting.."
|
85
99
|
exit 0
|
86
100
|
end
|
87
|
-
mkdir_p "pkg"
|
88
|
-
rsync_from("#{artifact_directory}/repos/rpm_configs", @build.distribution_server, "pkg")
|
89
101
|
|
90
|
-
|
91
|
-
remote_ssh_cmd(@build.distribution_server, "rm #{artifact_directory}/repos/rpm_configs")
|
102
|
+
mkdir_p File.join("pkg","repo_configs","rpm")
|
92
103
|
|
93
|
-
|
94
|
-
|
104
|
+
# Parse the rpm configs file to generate repository configs. Each line in
|
105
|
+
# the rpm_configs file corresponds with a repo directory on the
|
106
|
+
# distribution server.
|
107
|
+
#
|
108
|
+
yum_repos.each do |url|
|
109
|
+
# We ship a base 'srpm' that gets turned into a repo, but we want to
|
110
|
+
# ignore this one because its an extra
|
111
|
+
next if url == "#{base_url}srpm/"
|
112
|
+
elements = url.split('/')
|
113
|
+
|
114
|
+
dist,version,subdir,arch = url.split('/')[-4..-1]
|
95
115
|
|
96
|
-
#
|
97
|
-
# the rpm_configs file corresponds with a repo directory on the
|
98
|
-
# distribution server.
|
116
|
+
# Create an array of lines that will become our yum config
|
99
117
|
#
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
118
|
+
config = ["[pl-#{@build.project}-#{@build.ref}]"]
|
119
|
+
config << ["name=PL Repo for #{@build.project} at commit #{@build.ref}"]
|
120
|
+
config << ["baseurl=#{url}"]
|
121
|
+
config << ["enabled=1"]
|
122
|
+
config << ["gpgcheck=0"]
|
123
|
+
|
124
|
+
# Write the new config to a file under our repo configs dir
|
125
|
+
#
|
126
|
+
config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{@build.project}-#{@build.ref}-#{dist}-#{version}-#{arch}-#{subdir}.repo")
|
127
|
+
File.open(config_file, 'w') { |f| f.puts config }
|
128
|
+
end
|
129
|
+
puts "Wrote yum configuration files for #{@build.project} at #{@build.ref} to pkg/repo_configs/rpm"
|
130
|
+
end
|
131
|
+
|
132
|
+
desc "Retrieve rpm yum repository configs from distribution server"
|
133
|
+
task :rpm_repo_configs => "pl:fetch" do
|
134
|
+
if wget = find_tool("wget")
|
135
|
+
mkdir_p "pkg/repo_configs"
|
136
|
+
config_url = "#{@build.builds_server}/#{@build.project}/#{@build.ref}/repo_configs/rpm/"
|
137
|
+
begin
|
138
|
+
sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{config_url}"
|
139
|
+
rescue
|
140
|
+
warn "Couldn't retrieve rpm yum repo configs. See preceding http response for more info."
|
141
|
+
exit 1
|
121
142
|
end
|
122
|
-
|
123
|
-
|
143
|
+
else
|
144
|
+
warn "Could not find `wget` tool! wget is required to download the repository configs."
|
145
|
+
exit 1
|
124
146
|
end
|
125
147
|
end
|
126
148
|
end
|