puppet 2.7.21 → 2.7.22

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of puppet might be problematic. Click here for more details.

Files changed (101) hide show
  1. data/CHANGELOG +14 -0
  2. data/Gemfile.lock +2 -2
  3. data/ext/build_defaults.yaml +2 -3
  4. data/ext/debian/control +1 -1
  5. data/ext/packaging/README.md +496 -8
  6. data/ext/packaging/spec/tasks/00_utils_spec.rb +7 -7
  7. data/ext/packaging/spec/tasks/build_object_spec.rb +3 -0
  8. data/ext/packaging/tasks/00_utils.rake +2 -2
  9. data/ext/packaging/tasks/10_setupvars.rake +8 -1
  10. data/ext/packaging/tasks/build.rake +2 -0
  11. data/ext/packaging/tasks/deb_repos.rake +48 -15
  12. data/ext/packaging/tasks/jenkins.rake +30 -2
  13. data/ext/packaging/tasks/mock.rake +3 -2
  14. data/ext/packaging/tasks/pe_remote.rake +1 -1
  15. data/ext/packaging/tasks/pe_ship.rake +4 -5
  16. data/ext/packaging/tasks/pe_sign.rake +8 -0
  17. data/ext/packaging/tasks/pe_sles.rake +8 -7
  18. data/ext/packaging/tasks/pre_tasks.rake +0 -0
  19. data/ext/packaging/tasks/retrieve.rake +11 -1
  20. data/ext/packaging/tasks/rpm_repos.rake +71 -49
  21. data/ext/packaging/tasks/ship.rake +14 -2
  22. data/ext/packaging/tasks/sign.rake +9 -3
  23. data/ext/packaging/tasks/tar.rake +5 -0
  24. data/ext/packaging/tasks/vendor_gems.rake +110 -0
  25. data/install.rb +1 -1
  26. data/lib/puppet.rb +11 -0
  27. data/lib/puppet/indirector/report/processor.rb +1 -1
  28. data/lib/puppet/indirector/report/rest.rb +7 -0
  29. data/lib/puppet/indirector/resource/rest.rb +9 -0
  30. data/lib/puppet/indirector/rest.rb +81 -47
  31. data/lib/puppet/indirector/run/rest.rb +6 -0
  32. data/lib/puppet/network/formats.rb +20 -10
  33. data/lib/puppet/network/http/handler.rb +1 -1
  34. data/lib/puppet/node.rb +25 -0
  35. data/lib/puppet/node/facts.rb +23 -4
  36. data/lib/puppet/resource.rb +2 -4
  37. data/lib/puppet/resource/status.rb +28 -0
  38. data/lib/puppet/run.rb +24 -2
  39. data/lib/puppet/status.rb +6 -2
  40. data/lib/puppet/transaction/event.rb +19 -0
  41. data/lib/puppet/transaction/report.rb +39 -0
  42. data/lib/puppet/util/log.rb +19 -0
  43. data/lib/puppet/util/metric.rb +6 -0
  44. data/lib/puppet/util/monkey_patches.rb +0 -16
  45. data/lib/puppet/vendor.rb +55 -0
  46. data/lib/puppet/vendor/load_safe_yaml.rb +1 -0
  47. data/lib/puppet/vendor/require_vendored.rb +4 -0
  48. data/lib/puppet/vendor/safe_yaml/CHANGES.md +104 -0
  49. data/lib/puppet/vendor/safe_yaml/Gemfile +11 -0
  50. data/lib/puppet/vendor/safe_yaml/LICENSE.txt +22 -0
  51. data/lib/puppet/vendor/safe_yaml/README.md +179 -0
  52. data/lib/puppet/vendor/safe_yaml/Rakefile +6 -0
  53. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml.rb +253 -0
  54. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/deep.rb +34 -0
  55. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/parse/date.rb +27 -0
  56. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/parse/hexadecimal.rb +12 -0
  57. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/parse/sexagesimal.rb +26 -0
  58. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/psych_handler.rb +92 -0
  59. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/psych_resolver.rb +52 -0
  60. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/resolver.rb +94 -0
  61. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/safe_to_ruby_visitor.rb +17 -0
  62. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/syck_hack.rb +36 -0
  63. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/syck_node_monkeypatch.rb +43 -0
  64. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/syck_resolver.rb +38 -0
  65. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform.rb +41 -0
  66. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_boolean.rb +21 -0
  67. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_date.rb +11 -0
  68. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_float.rb +33 -0
  69. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_integer.rb +25 -0
  70. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_nil.rb +18 -0
  71. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/to_symbol.rb +13 -0
  72. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/transform/transformation_map.rb +47 -0
  73. data/lib/puppet/vendor/safe_yaml/lib/safe_yaml/version.rb +3 -0
  74. data/lib/puppet/vendor/safe_yaml/run_specs_all_ruby_versions.sh +21 -0
  75. data/lib/puppet/vendor/safe_yaml/safe_yaml.gemspec +18 -0
  76. data/lib/puppet/vendor/safe_yaml/spec/exploit.1.9.2.yaml +2 -0
  77. data/lib/puppet/vendor/safe_yaml/spec/exploit.1.9.3.yaml +2 -0
  78. data/lib/puppet/vendor/safe_yaml/spec/psych_resolver_spec.rb +10 -0
  79. data/lib/puppet/vendor/safe_yaml/spec/resolver_specs.rb +250 -0
  80. data/lib/puppet/vendor/safe_yaml/spec/safe_yaml_spec.rb +702 -0
  81. data/lib/puppet/vendor/safe_yaml/spec/spec_helper.rb +18 -0
  82. data/lib/puppet/vendor/safe_yaml/spec/support/exploitable_back_door.rb +29 -0
  83. data/lib/puppet/vendor/safe_yaml/spec/syck_resolver_spec.rb +10 -0
  84. data/lib/puppet/vendor/safe_yaml/spec/transform/base64_spec.rb +11 -0
  85. data/lib/puppet/vendor/safe_yaml/spec/transform/to_date_spec.rb +34 -0
  86. data/lib/puppet/vendor/safe_yaml/spec/transform/to_float_spec.rb +42 -0
  87. data/lib/puppet/vendor/safe_yaml/spec/transform/to_integer_spec.rb +59 -0
  88. data/lib/puppet/vendor/safe_yaml/spec/transform/to_symbol_spec.rb +49 -0
  89. data/lib/puppet/version.rb +1 -1
  90. data/spec/lib/puppet_spec/matchers.rb +8 -0
  91. data/spec/unit/file_serving/metadata_spec.rb +20 -28
  92. data/spec/unit/indirector/report/rest_spec.rb +41 -0
  93. data/spec/unit/indirector/rest_spec.rb +314 -339
  94. data/spec/unit/network/formats_spec.rb +36 -27
  95. data/spec/unit/network/http/handler_spec.rb +3 -12
  96. data/spec/unit/node_spec.rb +81 -0
  97. data/spec/unit/resource_spec.rb +5 -35
  98. data/spec/unit/run_spec.rb +22 -8
  99. data/spec/unit/status_spec.rb +6 -0
  100. data/test/network/handler/report.rb +0 -36
  101. metadata +148 -102
@@ -15,14 +15,14 @@ describe "00_utils" do
15
15
  :get_rpmrelease => '1',
16
16
  :is_rc? => false,
17
17
  },
18
- '0.7.0rc1' => {
19
- :git_describe_version => %w{0.7.0rc1},
20
- :get_dash_version => '0.7.0rc1',
21
- :get_ips_version => '0.7.0rc1,3.14159-0',
22
- :get_dot_version => '0.7.0rc1',
23
- :get_debversion => '0.7.0-0.1rc1puppetlabs1',
18
+ '0.7.0rc10' => {
19
+ :git_describe_version => %w{0.7.0rc10},
20
+ :get_dash_version => '0.7.0rc10',
21
+ :get_ips_version => '0.7.0rc10,3.14159-0',
22
+ :get_dot_version => '0.7.0rc10',
23
+ :get_debversion => '0.7.0-0.1rc10puppetlabs1',
24
24
  :get_rpmversion => '0.7.0',
25
- :get_rpmrelease => '0.1rc1',
25
+ :get_rpmrelease => '0.1rc10',
26
26
  :is_rc? => true,
27
27
  },
28
28
  '0.7.0-rc1' => {
@@ -2,6 +2,7 @@
2
2
  require 'spec_helper'
3
3
  load_task '00_utils.rake'
4
4
  load_task 'build.rake'
5
+ require 'yaml'
5
6
 
6
7
  describe Build::BuildInstance do
7
8
  Build_Params = [:apt_host,
@@ -71,6 +72,7 @@ describe Build::BuildInstance do
71
72
  :pe_name,
72
73
  :pe_version,
73
74
  :pg_major_version,
75
+ :pre_tar_task,
74
76
  :privatekey_pem,
75
77
  :random_mockroot,
76
78
  :rc_mocks,
@@ -86,6 +88,7 @@ describe Build::BuildInstance do
86
88
  :sles_arch_repos,
87
89
  :summary,
88
90
  :tar_excludes,
91
+ :tar_host,
89
92
  :tarball_path,
90
93
  :team,
91
94
  :version,
@@ -92,7 +92,7 @@ def git_sha_or_tag
92
92
  end
93
93
 
94
94
  def get_temp
95
- temp = `mktemp -d -t tmpXXXXXX`.strip
95
+ temp = `mktemp -d -t pkgXXXXXX`.strip
96
96
  end
97
97
 
98
98
  def remote_ssh_cmd target, command
@@ -202,7 +202,7 @@ def get_base_pkg_version
202
202
  dash = get_dash_version
203
203
  if dash.include?("rc")
204
204
  # Grab the rc number
205
- rc_num = dash.match(/rc(\d)+/)[1]
205
+ rc_num = dash.match(/rc(\d+)/)[1]
206
206
  ver = dash.sub(/-?rc[0-9]+/, "-0.#{@build.release}rc#{rc_num}").gsub(/(rc[0-9]+)-(\d+)?-?/, '\1.\2')
207
207
  else
208
208
  ver = dash.gsub('-','.') + "-#{@build.release}"
@@ -53,6 +53,7 @@ else
53
53
  @build.yum_repo_path = ENV['YUM_REPO'] if ENV['YUM_REPO']
54
54
  @build.apt_host = ENV['APT_HOST'] if ENV['APT_HOST']
55
55
  @build.apt_repo_path = ENV['APT_REPO'] if ENV['APT_REPO']
56
+ @build.pe_version = ENV['PE_VER'] if ENV['PE_VER']
56
57
  end
57
58
 
58
59
  ##
@@ -77,8 +78,14 @@ end
77
78
  # For backwards compatibilty, we set build:@name to build:@project. @name was
78
79
  # renamed to @project in an effort to align the variable names with what has
79
80
  # been supported for parameter names in the params files.
80
- #
81
81
  @build.name = @build.project
82
+ # We also set @tar_host to @yum_host if @tar_host is not set. This is in
83
+ # another effort to fix dumb mistakes. Early on, we just assumed tarballs would
84
+ # go to @yum_host (why? probably just laziness) but this is not ideal and does
85
+ # not make any sense when looking at the code. Now there's a @tar_host
86
+ # variable, but for backwards compatibility, we'll default back to @yum_host if
87
+ # @tar_host isn't set.
88
+ @build.tar_host ||= @build.yum_host
82
89
 
83
90
  if @build.debug
84
91
  @build.print_params
@@ -74,6 +74,7 @@ module Build
74
74
  :pe_name,
75
75
  :pe_version,
76
76
  :pg_major_version,
77
+ :pre_tar_task,
77
78
  :privatekey_pem,
78
79
  :random_mockroot,
79
80
  :rc_mocks,
@@ -89,6 +90,7 @@ module Build
89
90
  :sles_arch_repos,
90
91
  :summary,
91
92
  :tar_excludes,
93
+ :tar_host,
92
94
  :tarball_path,
93
95
  :task,
94
96
  :team,
@@ -12,15 +12,16 @@ namespace :pl do
12
12
  namespace :jenkins do
13
13
  desc "Create apt repositories of build DEB packages for this SHA on the distributions erver"
14
14
  task :deb_repos => "pl:fetch" do
15
+ prefix = @build.build_pe ? "pe/" : ""
15
16
 
16
17
  # First, we test that artifacts exist and set up the repos directory
17
18
  artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
18
19
 
19
20
  cmd = 'echo " Checking for deb build artifacts. Will exit if not found.." ; '
20
- cmd << "[ -d #{artifact_directory}/artifacts/deb ] || exit 0 ; "
21
+ cmd << "[ -d #{artifact_directory}/artifacts/#{prefix}deb ] || exit 0 ; "
21
22
  # Descend into the deb directory and obtain the list of distributions
22
23
  # we'll be building repos for
23
- cmd << "pushd #{artifact_directory}/artifacts/deb && dists=$(ls) && popd; "
24
+ cmd << "pushd #{artifact_directory}/artifacts/#{prefix}deb && dists=$(ls) && popd; "
24
25
  # We do one more check here to make sure we actually have distributions
25
26
  # to build for. If deb is empty we want to just exit.
26
27
  #
@@ -48,7 +49,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
48
49
  # testing only, we'll just add the debs and ignore source files for now.
49
50
  #
50
51
  cmd << "reprepro=$(which reprepro) ; "
51
- cmd << "$reprepro includedeb $dist ../../deb/$dist/*.deb ; popd ; done ; "
52
+ cmd << "$reprepro includedeb $dist ../../#{prefix}deb/$dist/*.deb ; popd ; done ; "
52
53
  cmd << "popd ; popd "
53
54
 
54
55
  remote_ssh_cmd(@build.distribution_server, cmd)
@@ -58,7 +59,10 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
58
59
 
59
60
  # Now that we've created our package repositories, we can generate repo
60
61
  # configurations for use with downstream jobs, acceptance clients, etc.
61
- Rake::Task["pl:jenkins:deb_repo_configs"].execute
62
+ Rake::Task["pl:jenkins:generate_deb_repo_configs"].execute
63
+
64
+ # Now that we've created the repo configs, we can ship them
65
+ Rake::Task["pl:jenkins:ship_repo_configs"].execute
62
66
  end
63
67
 
64
68
  # Generate apt configuration files that point to the repositories created
@@ -69,20 +73,29 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
69
73
  # enable clients to install these packages.
70
74
  #
71
75
  desc "Create apt repository configs for package repos for this sha/tag on the distribution server"
72
- task :deb_repo_configs => "pl:fetch" do
76
+ task :generate_deb_repo_configs => "pl:fetch" do
73
77
 
74
- # This is the standard path to all build artifacts on the distribution
75
- # server for this commit
78
+ # This is the standard path to all debian build artifact repositories on
79
+ # the distribution server for this commit
76
80
  #
77
- artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
81
+ base_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/apt/"
78
82
 
79
- # We obtain the list of distributions in the debian repository with some hackery.
83
+ # We use wget to obtain a directory listing of what are presumably our deb repos
80
84
  #
81
- %x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/apt'}
82
- if $?.success?
83
- dists = %x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/apt'}.split
85
+ repo_urls = []
86
+ if wget = find_tool("wget")
87
+ # First test if the directory even exists
88
+ #
89
+ wget_results = %x{#{wget} --spider -r -l 1 --no-parent #{base_url} 2>&1}
90
+ if $?.success?
91
+ # We want to exclude index and robots files and only include the http: prefixed elements
92
+ repo_urls = wget_results.split.uniq.reject{|x| x=~ /\?|index|robots/}.select{|x| x =~ /http:/}.map{|x| x.chomp('/')}
93
+ else
94
+ puts "No debian repos available for #{@build.project} at #{@build.ref}."
95
+ exit 0
96
+ end
84
97
  else
85
- warn "No repos were found to generate configs from. Exiting.."
98
+ warn "Could not find `wget` tool. This is needed for composing the debian repo configurations. Install `wget` and try again."
86
99
  exit 0
87
100
  end
88
101
 
@@ -91,13 +104,33 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; '
91
104
  # file for every distribution.
92
105
  #
93
106
  mkdir_p File.join("pkg", "repo_configs", "deb")
94
- dists.each do |dist|
107
+ repo_urls.each do |url|
108
+ # We want to skip the base_url, which wget returns as one of the results
109
+ next if "#{url}/" == base_url
110
+ dist = url.split('/').last
95
111
  repoconfig = ["# Packages for #{@build.project} built from ref #{@build.ref}",
96
- "deb http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/apt/#{dist} #{dist} main"]
112
+ "deb #{url} #{dist} main"]
97
113
  config = File.join("pkg", "repo_configs", "deb", "pl-#{@build.project}-#{@build.ref}-#{dist}.list")
98
114
  File.open(config, 'w') { |f| f.puts repoconfig }
99
115
  end
100
116
  puts "Wrote apt repo configs for #{@build.project} at #{@build.ref} to pkg/repo_configs/deb."
101
117
  end
118
+
119
+ desc "Retrieve debian apt repository configs for this sha"
120
+ task :deb_repo_configs => "pl:fetch" do
121
+ if wget = find_tool("wget")
122
+ mkdir_p "pkg/repo_configs"
123
+ config_url = "#{@build.builds_server}/#{@build.project}/#{@build.ref}/repo_configs/deb/"
124
+ begin
125
+ sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{config_url}"
126
+ rescue
127
+ warn "Couldn't retrieve deb apt repo configs. See preceding http response for more info."
128
+ exit 1
129
+ end
130
+ else
131
+ warn "Could not find `wget` tool! wget is required to download the repository configs."
132
+ exit 1
133
+ end
134
+ end
102
135
  end
103
136
  end
@@ -178,6 +178,10 @@ namespace :pl do
178
178
  else
179
179
  warn "An error occurred submitting the job to jenkins. Take a look at the preceding http response for more info."
180
180
  end
181
+
182
+ # Clean up after ourselves
183
+ rm bundle
184
+ rm properties
181
185
  end
182
186
  end
183
187
  end
@@ -261,10 +265,34 @@ if @build.build_pe
261
265
  end
262
266
  end
263
267
 
268
+ # While pl:remote:deb_all does all cows in serially, with jenkins we
269
+ # parallelize them. This breaks the cows up and posts a build for all of
270
+ # them. We have to sleep 5 because jenkins drops the builds when we're
271
+ # DOSing it with our packaging.
272
+ desc "Queue pe:deb_all on jenkins builder"
273
+ task :deb_all => "pl:fetch" do
274
+ check_var("PE_VER", @build.pe_version)
275
+ @build.cows.split(' ').each do |cow|
276
+ @build.default_cow = cow
277
+ invoke_task("pl:jenkins:post_build", "pe:local_deb")
278
+ sleep 5
279
+ end
280
+ end
281
+
282
+ # This does the mocks in parallel
283
+ desc "Queue pe:mock-all on jenkins builder"
284
+ task :mock_all => "pl:fetch" do
285
+ @build.final_mocks.split(' ').each do |mock|
286
+ @build.default_mock = mock
287
+ invoke_task("pl:jenkins:post_build", "pe:local_mock")
288
+ sleep 5
289
+ end
290
+ end
291
+
264
292
  desc "Queue builds of all PE packages for this project in Jenkins"
265
293
  task :uber_build => "pl:fetch" do
266
294
  check_var("PE_VER", @build.pe_version)
267
- ["deb_all", "mock_all", "sles"].each do |task|
295
+ ["tar", "deb_all", "mock_all", "sles"].each do |task|
268
296
  invoke_task("pe:jenkins:#{task}")
269
297
  sleep 5
270
298
  end
@@ -273,7 +301,7 @@ if @build.build_pe
273
301
  desc "Retrieve PE packages built by jenkins, sign, and ship all!"
274
302
  task :uber_ship => "pl:fetch" do
275
303
  check_var("PE_VER", @build.pe_version)
276
- ["pl:jenkins:retrieve", "pe:ship_rpms", "pe:ship_debs"].each do |task|
304
+ ["pl:jenkins:retrieve", "pl:jenkins:sign_all", "pe:ship_rpms", "pe:ship_debs"].each do |task|
277
305
  Rake::Task[task].invoke
278
306
  end
279
307
  Rake::Task["pl:jenkins:ship"].invoke("shipped")
@@ -19,6 +19,7 @@
19
19
  # pkg/el-5-i386/*.rpm
20
20
 
21
21
  def mock(mock_config, srpm)
22
+ configdir = nil
22
23
  unless mock = find_tool('mock')
23
24
  warn "mock is required for building rpms with mock. Please install mock and try again."
24
25
  exit 1
@@ -39,7 +40,7 @@ def mock(mock_config, srpm)
39
40
  end
40
41
  sh "#{mock} -r #{mock_config} #{srpm}"
41
42
  # Clean up the configdir
42
- rm_r configdir
43
+ rm_r configdir unless configdir.nil?
43
44
 
44
45
  basedir
45
46
  end
@@ -124,7 +125,7 @@ def build_rpm_with_mock(mocks, is_rc)
124
125
  cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-srpms")
125
126
  when /i.?86/
126
127
  cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
127
- when /x66_64/
128
+ when /x86_64/
128
129
  cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-x86_64")
129
130
  when /noarch/
130
131
  cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
@@ -44,7 +44,7 @@ if @build.build_pe
44
44
 
45
45
  desc "Execute remote debian, el, and sles builds, sign, and ship pkgs"
46
46
  task :all => ['clean', 'pl:fetch'] do
47
- ['pe:deb', 'pe:mock_all', 'pe:sles', 'pe:ship_rpms', 'pe:ship_debs'].each do |task|
47
+ ['pe:deb_all', 'pe:mock_all', 'pe:sles', 'pe:ship_rpms', 'pe:ship_debs'].each do |task|
48
48
  Rake::Task[task].execute
49
49
  end
50
50
  end
@@ -16,13 +16,12 @@ if @build.build_pe
16
16
 
17
17
  desc "Ship PE debs to #{@build.apt_host}"
18
18
  task :ship_debs => "pl:fetch" do
19
- dist = @build.default_cow.split('-')[1]
20
- if empty_dir?("pkg/pe/deb/#{dist}")
21
- STDERR.puts "The 'pkg/pe/deb/#{dist}' directory has no packages. Did you run rake pe:deb?"
19
+ if empty_dir?("pkg/pe/deb")
20
+ STDERR.puts "The 'pkg/pe/deb' directory has no packages!"
22
21
  exit 1
23
22
  else
24
- target_path = ENV['APT_REPO'] ? ENV['APT_REPO'] : "#{@build.apt_repo_path}/#{@build.pe_version}/repos/incoming/unified/"
25
- rsync_to("pkg/pe/deb/#{dist}/", @build.apt_host, target_path)
23
+ target_path = ENV['APT_REPO'] ? ENV['APT_REPO'] : "#{@build.apt_repo_path}/#{@build.pe_version}/repos/incoming/disparate/"
24
+ rsync_to("pkg/pe/deb/", @build.apt_host, target_path)
26
25
  if @build.team == 'release'
27
26
  Rake::Task["pe:remote:freight"].invoke
28
27
  end
@@ -24,5 +24,13 @@ if @build.build_pe
24
24
  sign_el5(old_rpms) unless old_rpms.empty?
25
25
  sign_modern(modern_rpms) unless modern_rpms.empty?
26
26
  end
27
+ # This is essentially a duplicate of the logic in pl:sign_deb_changes, but
28
+ # since the plan is eventually to rip out the PE work, it'll be easier if
29
+ # this is a separate task we can pull out later.
30
+ desc "Sign all debian changes files staged in pkg/pe"
31
+ task :sign_deb_changes do
32
+ load_keychain if has_tool('keychain')
33
+ sign_deb_changes("pkg/pe/deb/*/*.changes") unless Dir["pkg/pe/deb/*/*.changes"].empty?
34
+ end
27
35
  end
28
36
  end
@@ -21,6 +21,7 @@ if @build.build_pe
21
21
  namespace :pe do
22
22
  # Temporary task to pull down pe dependencies until this is NFS-mounted
23
23
  task :retrieve_sles_deps => 'pl:fetch' do
24
+ rm_rf FileList["#{ENV['HOME']}/package_repos/*"]
24
25
  rsync_from("#{@build.sles_repo_path}/#{@build.pe_version}/repos/sles-*", @build.sles_repo_host, "#{ENV['HOME']}/package_repos/")
25
26
  FileList["#{ENV['HOME']}/package_repos/*"].each do |f|
26
27
  update_rpm_repo(f) if File.directory?(f)
@@ -40,15 +41,15 @@ if @build.build_pe
40
41
  check_tool('build')
41
42
  check_tool('linux32')
42
43
  check_tool('linux64')
43
- build_root = get_temp
44
- work_dir = prep_sles_dir
45
- build_source_dir = "#{work_dir}/SOURCES"
46
- build_spec_dir = "#{work_dir}/SPECS"
47
- build_spec = "#{build_spec_dir}/#{@build.project}.spec"
48
44
  build_dest_dir = "usr/src/packages"
49
45
  noarch = FALSE
50
46
  built_arch = ''
51
47
  @build.sles_arch_repos.each do |arch, deps_repo|
48
+ build_root = get_temp
49
+ work_dir = prep_sles_dir
50
+ build_source_dir = "#{work_dir}/SOURCES"
51
+ build_spec_dir = "#{work_dir}/SPECS"
52
+ build_spec = "#{build_spec_dir}/#{@build.project}.spec"
52
53
  if noarch == FALSE
53
54
  bench = Benchmark.realtime do
54
55
  linux_cmd = arch == 'i586' ? 'linux32' : 'linux64'
@@ -70,8 +71,6 @@ if @build.build_pe
70
71
  cp(rpms, "pkg/pe/rpm/sles-11-#{arch}")
71
72
  cp(srpms, "pkg/pe/rpm/sles-11-srpms")
72
73
  noarch = rpms.exclude(/noarch/).empty?
73
- rm_rf build_root
74
- rm_rf work_dir
75
74
  end
76
75
  # See 30_metrics.rake to see what this is doing
77
76
  add_metrics({ :dist => 'sles', :bench => bench }) if @build.benchmark
@@ -82,6 +81,8 @@ if @build.build_pe
82
81
  cp(FileList["pkg/pe/rpm/sles-11-#{built_arch}/*"], "pkg/pe/rpm/sles-11-#{other_arch}")
83
82
  end
84
83
  end
84
+ rm_rf build_root
85
+ rm_rf work_dir
85
86
  end
86
87
  post_metrics if @build.benchmark
87
88
  cd 'pkg/pe/rpm' do
File without changes
@@ -16,7 +16,17 @@ namespace :pl do
16
16
  target = args.target || "artifacts"
17
17
  invoke_task("pl:fetch")
18
18
  mkdir_p 'pkg'
19
- rsync_from("#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}/#{target}/", @build.distribution_server, "pkg/")
19
+ package_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/#{target}"
20
+ if wget=find_tool("wget")
21
+ sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg --reject 'index*' #{package_url}/"
22
+ else
23
+ warn "Could not find `wget` tool. Falling back to rsyncing from #{@build.distribution_server}"
24
+ begin
25
+ rsync_from("#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}/#{target}/", @build.distribution_server, "pkg/")
26
+ rescue
27
+ puts "Couldn't download packages from distribution server. Try installing wget!"
28
+ end
29
+ end
20
30
  puts "Packages staged in pkg"
21
31
  end
22
32
  end
@@ -46,7 +46,10 @@ namespace :pl do
46
46
 
47
47
  # Now that we've created our repositories, we can create the configs for
48
48
  # them
49
- Rake::Task["pl:jenkins:rpm_repo_configs"].invoke
49
+ Rake::Task["pl:jenkins:generate_rpm_repo_configs"].execute
50
+
51
+ # And once they're created, we can ship them
52
+ Rake::Task["pl:jenkins:ship_repo_configs"].execute
50
53
  end
51
54
 
52
55
  # Generate yum configuration files that point to the repositories created
@@ -57,70 +60,89 @@ namespace :pl do
57
60
  # clients to install these packages.
58
61
  #
59
62
  desc "Create yum repository configs for package repos for this sha/tag on the distribution server"
60
- task :rpm_repo_configs => "pl:fetch" do
63
+ task :generate_rpm_repo_configs => "pl:fetch" do
64
+
65
+ # We have a hard requirement on wget because of all the download magicks
66
+ # we have to do
67
+ #
68
+ unless wget = find_tool("wget")
69
+ warn "Could not find `wget` tool. This is needed for composing the yum repo configurations. Install `wget` and try again."
70
+ exit 0
71
+ end
61
72
 
62
73
  # This is the standard path to all build artifacts on the distribution
63
74
  # server for this commit
64
75
  #
65
- artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
76
+ base_url = "http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/"
77
+
66
78
  # First check if the artifacts directory exists
67
79
  #
68
- cmd = "[ -d #{artifact_directory} ] || exit 0 ; "
69
- # Descend into the artifacts directory and test if we have any repos
70
- #
71
- cmd << "pushd #{artifact_directory} ; "
72
- cmd << 'echo "Checking if rpm repos exists, will exit if not.." ; '
73
- cmd << '[ -n "$(find repos -name "*.rpm")" ] || exit 0 ; '
74
- cmd << "pushd repos ; "
75
80
 
76
- cmd << 'for repo in $(find -name "repodata") ; do dirname $repo >> rpm_configs ; done'
81
+ # We have to do two checks here - first that there are directories with
82
+ # repodata folders in them, and second that those same directories also
83
+ # contain rpms
84
+ #
85
+ repo_urls = %x{#{wget} --spider -r -l 5 --no-parent #{base_url} 2>&1}.split.uniq.reject{ |x| x =~ /\?|index/ }.select{|x| x =~ /http:.*repodata\/$/}
77
86
 
78
- remote_ssh_cmd(@build.distribution_server, cmd)
87
+ # RPMs will always exist at the same directory level as the repodata
88
+ # folder, which means if we go up a level we should find rpms
89
+ #
90
+ yum_repos = []
91
+ repo_urls.map{|x| x.chomp('repodata/')}.each do |url|
92
+ unless %x{#{wget} --spider -r -l 1 --no-parent #{url} 2>&1}.split.uniq.reject{ |x| x =~ /\?|index/ }.select{|x| x =~ /http:.*\.rpm$/}.empty?
93
+ yum_repos << url
94
+ end
95
+ end
79
96
 
80
- # There's a chance there were simply no rpms to make repos for. If so, we
81
- # don't want to proceed.
82
- %x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/rpm_configs'}
83
- unless $?.success?
84
- warn "No repos were found to generate configs from. Exiting.."
97
+ if yum_repos.empty?
98
+ puts "No rpm repos were found to generate configs from. Exiting.."
85
99
  exit 0
86
100
  end
87
- mkdir_p "pkg"
88
- rsync_from("#{artifact_directory}/repos/rpm_configs", @build.distribution_server, "pkg")
89
101
 
90
- # Clean up the remote configs file
91
- remote_ssh_cmd(@build.distribution_server, "rm #{artifact_directory}/repos/rpm_configs")
102
+ mkdir_p File.join("pkg","repo_configs","rpm")
92
103
 
93
- if File.exist?(File.join("pkg", "rpm_configs"))
94
- mkdir_p File.join("pkg","repo_configs","rpm")
104
+ # Parse the rpm configs file to generate repository configs. Each line in
105
+ # the rpm_configs file corresponds with a repo directory on the
106
+ # distribution server.
107
+ #
108
+ yum_repos.each do |url|
109
+ # We ship a base 'srpm' that gets turned into a repo, but we want to
110
+ # ignore this one because its an extra
111
+ next if url == "#{base_url}srpm/"
112
+ elements = url.split('/')
113
+
114
+ dist,version,subdir,arch = url.split('/')[-4..-1]
95
115
 
96
- # Parse the rpm configs file to generate repository configs. Each line in
97
- # the rpm_configs file corresponds with a repo directory on the
98
- # distribution server.
116
+ # Create an array of lines that will become our yum config
99
117
  #
100
- lines = IO.readlines(File.join("pkg","rpm_configs")).map{ |l| l.chomp }.uniq
101
- lines.each do |repo|
102
- dist,version,subdir,arch = repo.split('/')[1..4]
103
-
104
- # Skip any paths that don't have everything we're looking for, e.g.
105
- # the top-level srpms directory that contains the original srpm from
106
- # packaging
107
- next if dist.nil? or version.nil? or subdir.nil? or arch.nil?
108
-
109
- # Create an array of lines that will become our yum config
110
- #
111
- config = ["[pl-#{@build.project}-#{@build.ref}]"]
112
- config << ["name=PL Repo for #{@build.project} at commit #{@build.ref}"]
113
- config << ["baseurl=http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/#{dist}/#{version}/#{subdir}/#{arch}"]
114
- config << ["enabled=1"]
115
- config << ["gpgcheck=0"]
116
-
117
- # Write the new config to a file under our repo configs dir
118
- #
119
- config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{@build.project}-#{@build.ref}-#{dist}-#{version}-#{arch}-#{subdir}.repo")
120
- File.open(config_file, 'w') { |f| f.puts config }
118
+ config = ["[pl-#{@build.project}-#{@build.ref}]"]
119
+ config << ["name=PL Repo for #{@build.project} at commit #{@build.ref}"]
120
+ config << ["baseurl=#{url}"]
121
+ config << ["enabled=1"]
122
+ config << ["gpgcheck=0"]
123
+
124
+ # Write the new config to a file under our repo configs dir
125
+ #
126
+ config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{@build.project}-#{@build.ref}-#{dist}-#{version}-#{arch}-#{subdir}.repo")
127
+ File.open(config_file, 'w') { |f| f.puts config }
128
+ end
129
+ puts "Wrote yum configuration files for #{@build.project} at #{@build.ref} to pkg/repo_configs/rpm"
130
+ end
131
+
132
+ desc "Retrieve rpm yum repository configs from distribution server"
133
+ task :rpm_repo_configs => "pl:fetch" do
134
+ if wget = find_tool("wget")
135
+ mkdir_p "pkg/repo_configs"
136
+ config_url = "#{@build.builds_server}/#{@build.project}/#{@build.ref}/repo_configs/rpm/"
137
+ begin
138
+ sh "#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{config_url}"
139
+ rescue
140
+ warn "Couldn't retrieve rpm yum repo configs. See preceding http response for more info."
141
+ exit 1
121
142
  end
122
- rm File.join("pkg","rpm_configs")
123
- puts "Wrote yum configuration files for #{@build.project} at #{@build.ref} to pkg/repo_configs/rpm"
143
+ else
144
+ warn "Could not find `wget` tool! wget is required to download the repository configs."
145
+ exit 1
124
146
  end
125
147
  end
126
148
  end