packaging 0.101.0 → 0.105.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4a7e26777ec04e6bbb1824c27ec325110f49d6b52f8ca0dd84e933d3937f8fc3
4
- data.tar.gz: 4b1932eb5b21ae8c1c3bb5ea6f21b5fb57479b3fe3e46273e8d075ba4beb86eb
3
+ metadata.gz: 296bfdcecf936f80297894792d9b9a1bb3a83a422a61e4976aca3916dd6c9a2b
4
+ data.tar.gz: 0c76c4482d82367cef9d988fc1e31080f718322e4eab1fe9a5176f4544c8a3ce
5
5
  SHA512:
6
- metadata.gz: e7aca6b96f6e8886d0cfd8b5f381ec1e0bb92337dae30ed30601391feb11d141df89262c19996e9e01751434f318f6668570151421f3eadfd860f6db17a75632
7
- data.tar.gz: b74047c635a19fbb0ecb308d0b45297d01c81437f77fccc3205c6434934f8e7378372f218d96aa90325496496a5b19f83aa5589be0082ce1a90d96e090e417dc
6
+ metadata.gz: 071534bcce41eb8b11517d21592399a9f94c45b20a70259aa5a13c4bf3ec5bfd6cab5e517698e6f8884736a932a62d0212d945cb97ad549bdf8c80242f594f8e
7
+ data.tar.gz: 532cca35cd6eeea218c8a6cdcdf00f0802f0c0ae33fc009f6f42bd7c39056f45b147abaf971879e2f5532541e1d08c0c6267425d32aa69dad9c11f036697c183
@@ -80,7 +80,7 @@ module Pkg
80
80
 
81
81
  dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
82
82
  cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};"\
83
- "find ./artifacts/ -mindepth 2 -type f; fi"
83
+ "find ./artifacts -mindepth 2 -type f; fi"
84
84
  artifacts, _ = Pkg::Util::Net.remote_execute(
85
85
  self.builds_server,
86
86
  cmd,
@@ -95,6 +95,7 @@ module Pkg
95
95
  # the correct place. For 5.x and 6.x release streams the f prefix
96
96
  # has been removed and so tag will equal original_tag
97
97
  original_tag = Pkg::Paths.tag_from_artifact_path(artifact)
98
+ fail "Error: unrecognized artifact \"#{artifact}\"" if original_tag.nil?
98
99
 
99
100
  # Remove the f-prefix from the fedora platform tag keys so that
100
101
  # beaker can rely on consistent keys once we rip out the f for good
@@ -203,15 +204,18 @@ module Pkg
203
204
  # string. Accept an argument for the write target file. If not specified,
204
205
  # the name of the params file is the current git commit sha or tag.
205
206
  #
206
- def config_to_yaml(target = nil)
207
- file = "#{self.ref}.yaml"
208
- target = target.nil? ? File.join(Pkg::Util::File.mktemp, "#{self.ref}.yaml") : File.join(target, file)
209
- Pkg::Util::File.file_writable?(File.dirname(target), :required => true)
210
- File.open(target, 'w') do |f|
207
+ def config_to_yaml(destination_directory = nil)
208
+ destination_directory = Pkg::Util::File.mktemp if destination_directory.nil?
209
+ config_yaml_file_name = "#{self.ref}.yaml"
210
+
211
+ config_yaml_path = File.join(destination_directory, config_yaml_file_name)
212
+
213
+ Pkg::Util::File.file_writable?(File.dirname(config_yaml_path), :required => true)
214
+ File.open(config_yaml_path, 'w') do |f|
211
215
  f.puts self.config_to_hash.to_yaml
212
216
  end
213
- puts target
214
- target
217
+ puts config_yaml_path
218
+ return config_yaml_path
215
219
  end
216
220
 
217
221
  ##
@@ -22,14 +22,6 @@ module Pkg
22
22
  },
23
23
 
24
24
  'debian' => {
25
- '8' => {
26
- codename: 'jessie',
27
- architectures: ['amd64', 'i386', 'powerpc'],
28
- source_architecture: 'source',
29
- package_format: 'deb',
30
- source_package_formats: DEBIAN_SOURCE_FORMATS,
31
- repo: true,
32
- },
33
25
  '9' => {
34
26
  codename: 'stretch',
35
27
  architectures: ['amd64', 'i386'],
@@ -57,14 +49,6 @@ module Pkg
57
49
  },
58
50
 
59
51
  'el' => {
60
- '5' => {
61
- architectures: ['x86_64', 'i386'],
62
- source_architecture: 'SRPMS',
63
- package_format: 'rpm',
64
- source_package_formats: ['src.rpm'],
65
- signature_format: 'v3',
66
- repo: true,
67
- },
68
52
  '6' => {
69
53
  architectures: ['x86_64', 'i386'],
70
54
  source_architecture: 'SRPMS',
@@ -88,26 +72,18 @@ module Pkg
88
72
  source_package_formats: ['src.rpm'],
89
73
  signature_format: 'v4',
90
74
  repo: true,
91
- }
92
- },
93
-
94
- 'fedora' => {
95
- '30' => {
96
- architectures: ['x86_64'],
97
- source_architecture: 'SRPMS',
98
- package_format: 'rpm',
99
- source_package_formats: ['src.rpm'],
100
- signature_format: 'v4',
101
- repo: true,
102
75
  },
103
- '31' => {
76
+ '9' => {
104
77
  architectures: ['x86_64'],
105
78
  source_architecture: 'SRPMS',
106
79
  package_format: 'rpm',
107
80
  source_package_formats: ['src.rpm'],
108
81
  signature_format: 'v4',
109
82
  repo: true,
110
- },
83
+ }
84
+ },
85
+
86
+ 'fedora' => {
111
87
  '32' => {
112
88
  architectures: ['x86_64'],
113
89
  source_architecture: 'SRPMS',
@@ -127,16 +103,6 @@ module Pkg
127
103
  },
128
104
 
129
105
  'osx' => {
130
- '10.13' => {
131
- architectures: ['x86_64'],
132
- package_format: 'dmg',
133
- repo: false,
134
- },
135
- '10.14' => {
136
- architectures: ['x86_64'],
137
- package_format: 'dmg',
138
- repo: false,
139
- },
140
106
  '10.15' => {
141
107
  architectures: ['x86_64'],
142
108
  package_format: 'dmg',
@@ -233,14 +199,6 @@ module Pkg
233
199
  source_package_formats: DEBIAN_SOURCE_FORMATS,
234
200
  repo: true,
235
201
  },
236
- '18.10' => {
237
- codename: 'cosmic',
238
- architectures: ['amd64', 'ppc64el'],
239
- source_architecture: 'source',
240
- package_format: 'deb',
241
- source_package_formats: DEBIAN_SOURCE_FORMATS,
242
- repo: true,
243
- },
244
202
  '20.04' => {
245
203
  codename: 'focal',
246
204
  architectures: ['amd64', 'aarch64'],
@@ -36,7 +36,8 @@ module Pkg::Repo
36
36
  Dir.chdir(File.join('pkg', local_target)) do
37
37
  puts "Info: Archiving #{repo_location} as #{archive_name}"
38
38
  target_tarball = File.join('repos', "#{archive_name}.tar.gz")
39
- tar_command = "#{tar} --owner=0 --group=0 --create --gzip --file #{target_tarball} #{repo_location}"
39
+ tar_command = %W[#{tar} --owner=0 --group=0 --create --gzip
40
+ --file #{target_tarball} #{repo_location}].join(' ')
40
41
  stdout, _, _ = Pkg::Util::Execution.capture3(tar_command)
41
42
  return stdout
42
43
  end
@@ -62,12 +63,12 @@ module Pkg::Repo
62
63
  next
63
64
  end
64
65
 
65
- tar_action = "--create"
66
- if File.exist?(all_repos_tarball_name)
67
- tar_action = "--update"
68
- end
66
+ tar_action = '--create'
67
+ tar_action = '--update' if File.exist?(all_repos_tarball_name)
68
+
69
+ tar_command = %W[#{tar} --owner=0 --group=0 #{tar_action}
70
+ --file #{all_repos_tarball_name} #{repo_tarball_path}].join(' ')
69
71
 
70
- tar_command = "#{tar} --owner=0 --group=0 #{tar_action} --file #{all_repos_tarball_name} #{repo_tarball_path}"
71
72
  stdout, _, _ = Pkg::Util::Execution.capture3(tar_command)
72
73
  puts stdout
73
74
  end
@@ -117,7 +118,8 @@ module Pkg::Repo
117
118
  )
118
119
  return stdout.split
119
120
  rescue => e
120
- fail "Error: Could not retrieve directories that contain #{pkg_ext} packages in #{Pkg::Config.distribution_server}:#{artifact_directory}"
121
+ fail "Error: Could not retrieve directories that contain #{pkg_ext} " \
122
+ "packages in #{Pkg::Config.distribution_server}:#{artifact_directory}: #{e}"
121
123
  end
122
124
 
123
125
  def populate_repo_directory(artifact_parent_directory)
@@ -126,7 +128,8 @@ module Pkg::Repo
126
128
  cmd << 'rsync --archive --verbose --one-file-system --ignore-existing artifacts/ repos/ '
127
129
  Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, cmd)
128
130
  rescue => e
129
- fail "Error: Could not populate repos directory in #{Pkg::Config.distribution_server}:#{artifact_parent_directory}"
131
+ fail "Error: Could not populate repos directory in " \
132
+ "#{Pkg::Config.distribution_server}:#{artifact_parent_directory}: #{e}"
130
133
  end
131
134
 
132
135
  def argument_required?(argument_name, repo_command)
@@ -134,12 +137,12 @@ module Pkg::Repo
134
137
  end
135
138
 
136
139
  def update_repo(remote_host, command, options = {})
137
- fail_message = "Error: Missing required argument '%s', update your build_defaults?"
140
+ fail_message = "Error: Missing required argument '%s', perhaps update build_defaults?"
138
141
  [:repo_name, :repo_path, :repo_host, :repo_url].each do |option|
139
142
  fail fail_message % option.to_s if argument_required?(option.to_s, command) && !options[option]
140
143
  end
141
144
 
142
- whitelist = {
145
+ repo_configuration = {
143
146
  __REPO_NAME__: options[:repo_name],
144
147
  __REPO_PATH__: options[:repo_path],
145
148
  __REPO_HOST__: options[:repo_host],
@@ -149,7 +152,7 @@ module Pkg::Repo
149
152
  }
150
153
  Pkg::Util::Net.remote_execute(
151
154
  remote_host,
152
- Pkg::Util::Misc.search_and_replace(command, whitelist))
155
+ Pkg::Util::Misc.search_and_replace(command, repo_configuration))
153
156
  end
154
157
  end
155
158
  end
@@ -0,0 +1,8 @@
1
+ # Utility methods for handling Apt staging server.
2
+
3
+ module Pkg::Util::AptStagingServer
4
+ def self.send_packages(pkg_directory, apt_component = 'stable')
5
+ %x(apt-stage-artifacts --component=#{apt_component} #{pkg_directory})
6
+ fail 'APT artifact staging failed.' unless $CHILD_STATUS.success?
7
+ end
8
+ end
@@ -0,0 +1,17 @@
1
+ # Utility methods for handling miscellaneous build metadata
2
+
3
+ require 'fileutils'
4
+
5
+ module Pkg::Util::BuildMetadata
6
+ class << self
7
+ def add_misc_json_files(target_directory)
8
+ misc_json_files = Dir.glob('ext/build_metadata*.json')
9
+ misc_json_files.each do |source_file|
10
+ target_file = File.join(
11
+ target_directory, "#{Pkg::Config.ref}.#{File.basename(source_file)}"
12
+ )
13
+ FileUtils.cp(source_file, target_file)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,42 @@
1
+ # Utility methods for the older distribution server
2
+
3
+ require 'fileutils'
4
+
5
+ module Pkg::Util::DistributionServer
6
+ class << self
7
+ def send_packages(local_source_directory, remote_target_directory)
8
+ Pkg::Util::Execution.retry_on_fail(times: 3) do
9
+ Pkg::Util::Net.remote_execute(
10
+ Pkg::Config.distribution_server,
11
+ "mkdir --mode=775 --parents #{remote_target_directory}"
12
+ )
13
+ Pkg::Util::Net.rsync_to(
14
+ "#{local_source_directory}/",
15
+ Pkg::Config.distribution_server, "#{remote_target_directory}/",
16
+ extra_flags: ['--ignore-existing', '--exclude repo_configs']
17
+ )
18
+ end
19
+
20
+ # In order to get a snapshot of what this build looked like at the time
21
+ # of shipping, we also generate and ship the params file
22
+ #
23
+ Pkg::Config.config_to_yaml(local_source_directory)
24
+ Pkg::Util::Execution.retry_on_fail(times: 3) do
25
+ Pkg::Util::Net.rsync_to(
26
+ "#{local_source_directory}/#{Pkg::Config.ref}.yaml",
27
+ Pkg::Config.distribution_server, "#{remote_target_directory}/",
28
+ extra_flags: ["--exclude repo_configs"]
29
+ )
30
+ end
31
+
32
+ # If we just shipped a tagged version, we want to make it immutable
33
+ files = Dir.glob("#{local_source_directory}/**/*")
34
+ .select { |f| File.file?(f) and !f.include? "#{Pkg::Config.ref}.yaml" }
35
+ .map { |f| "#{remote_target_directory}/#{f.sub(/^#{local_source_directory}\//, '')}" }
36
+
37
+ Pkg::Util::Net.remote_set_ownership(Pkg::Config.distribution_server, 'root', 'release', files)
38
+ Pkg::Util::Net.remote_set_permissions(Pkg::Config.distribution_server, '0664', files)
39
+ Pkg::Util::Net.remote_set_immutable(Pkg::Config.distribution_server, files)
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,26 @@
1
+ # Utility methods for handling ezbake
2
+
3
+ require 'fileutils'
4
+
5
+ module Pkg::Util::EZbake
6
+ class << self
7
+ def add_manifest(target_directory)
8
+ ezbake_manifest = File.join('ext', 'ezbake.manifest')
9
+ ezbake_yaml = File.join('ext', 'ezbake.manifest.yaml')
10
+
11
+ if File.exist?(ezbake_manifest)
12
+ FileUtils.cp(
13
+ ezbake_manifest,
14
+ File.join(target_directory, "#{Pkg::Config.ref}.ezbake.manifest")
15
+ )
16
+ end
17
+
18
+ if File.exists?(ezbake_yaml)
19
+ FileUtils.cp(
20
+ ezbake_yaml,
21
+ File.join(target_directory, "#{Pkg::Config.ref}.ezbake.manifest.yaml")
22
+ )
23
+ end
24
+ end
25
+ end
26
+ end
@@ -120,6 +120,85 @@ module Pkg::Util::File
120
120
  end
121
121
  Pkg::Util::Version.versionbump(workdir) if Pkg::Config.update_version_file
122
122
  end
123
+
124
+ # The fetch method pulls down two files from the build-data repo that contain additional
125
+ # data specific to Puppet Labs release infrastructure intended to augment/override any
126
+ # defaults specified in the source project repo, e.g. in ext/build_defaults.yaml
127
+ #
128
+ # It uses curl to download the files, and places them in a temporary
129
+ # directory, e.g. /tmp/somedirectory/{project,team}/Pkg::Config.builder_data_file
130
+ #
131
+ # Retrieve build-data configurations to override/extend local build_defaults
132
+ def fetch
133
+ # Each team has a build-defaults file that specifies local infrastructure targets
134
+ # for things like builders, target locations for build artifacts, etc Since much
135
+ # of these don't change, one file can be maintained for the team. Each project
136
+ # also has a data file for information specific to it. If the project builds
137
+ # both PE and not PE, it has two files, one for PE, and the other for FOSS
138
+ #
139
+ data_repo = Pkg::Config.build_data_repo
140
+
141
+ if Pkg::Config.dev_build
142
+ puts "NOTICE: This is a dev build!"
143
+ project_data_branch = "#{Pkg::Config.project}-dev"
144
+ else
145
+ project_data_branch = Pkg::Config.project
146
+ end
147
+ team_data_branch = Pkg::Config.team
148
+
149
+ if Pkg::Config.build_pe
150
+ project_data_branch = 'pe-' + project_data_branch unless project_data_branch =~ /^pe-/
151
+ team_data_branch = 'pe-' + team_data_branch unless team_data_branch =~ /^pe-/
152
+ end
153
+
154
+ # Remove .packaging directory from old-style extras loading
155
+ FileUtils.rm_rf("#{ENV['HOME']}/.packaging") if File.directory?("#{ENV['HOME']}/.packaging")
156
+
157
+ # Touch the .packaging file which is allows packaging to present remote tasks
158
+ FileUtils.touch("#{ENV['HOME']}/.packaging")
159
+
160
+ begin
161
+ build_data_directory = Pkg::Util::File.mktemp
162
+ %x(git clone #{data_repo} #{build_data_directory})
163
+ unless $?.success?
164
+ fail 'Error: could not fetch the build-data repo. Maybe you do not have the correct permissions?'
165
+ end
166
+
167
+ Dir.chdir(build_data_directory) do
168
+ [team_data_branch, project_data_branch].each do |branch|
169
+ %x(git checkout #{branch})
170
+ unless $?.success?
171
+ warn "Warning: no build_defaults found in branch '#{branch}' of '#{data_repo}'. Skipping."
172
+ next
173
+ end
174
+ load_extras(build_data_directory)
175
+ end
176
+ end
177
+ ensure
178
+ FileUtils.rm_rf(build_data_directory)
179
+ end
180
+
181
+ Pkg::Config.perform_validations
182
+ end
183
+
184
+ # The load_extras method is intended to load variables
185
+ # from the extra yaml file downloaded by the pl:fetch task.
186
+ # The goal is to be able to augment/override settings in the
187
+ # source project's build_data.yaml and project_data.yaml with
188
+ # Puppet Labs-specific data, rather than having to clutter the
189
+ # generic tasks with data not generally useful outside the
190
+ # PL Release team
191
+ def load_extras(temp_directory)
192
+ unless ENV['PARAMS_FILE'] && ENV['PARAMS_FILE'] != ''
193
+ temp_directory = temp_directory
194
+ raise "load_extras requires a directory containing extras data" if temp_directory.nil?
195
+ Pkg::Config.config_from_yaml("#{temp_directory}/#{Pkg::Config.builder_data_file}")
196
+
197
+ # Environment variables take precedence over those loaded from configs,
198
+ # so we make sure that any we clobbered are reset.
199
+ Pkg::Config.load_envvars
200
+ end
201
+ end
123
202
  end
124
203
  end
125
204
 
@@ -159,7 +159,7 @@ module Pkg::Util::Net
159
159
  raise(ArgumentError, "Cannot sync path '#{origin}' because both origin_host and target_host are nil. Perhaps you need to set TEAM=release ?") unless
160
160
  options[:origin_host] || options[:target_host]
161
161
 
162
- cmd = %W(
162
+ cmd = %W[
163
163
  #{options[:bin]}
164
164
  --recursive
165
165
  --hard-links
@@ -169,7 +169,7 @@ module Pkg::Util::Net
169
169
  --no-perms
170
170
  --no-owner
171
171
  --no-group
172
- ) + [*options[:extra_flags]]
172
+ ] + [*options[:extra_flags]]
173
173
 
174
174
  cmd << '--dry-run' if options[:dryrun]
175
175
  cmd << Pkg::Util.pseudo_uri(path: origin, host: options[:origin_host])
@@ -392,9 +392,8 @@ DOC
392
392
  end
393
393
 
394
394
  def remote_bundle_install_command
395
- export_packaging_location = ''
396
395
  export_packaging_location = "export PACKAGING_LOCATION='#{ENV['PACKAGING_LOCATION']}';" if ENV['PACKAGING_LOCATION'] && !ENV['PACKAGING_LOCATION'].empty?
397
- command = "source /usr/local/rvm/scripts/rvm; rvm use ruby-2.5.1; #{export_packaging_location} bundle install --path .bundle/gems ;"
396
+ "source /usr/local/rvm/scripts/rvm; rvm use ruby-2.5.1; #{export_packaging_location} bundle install --path .bundle/gems ;"
398
397
  end
399
398
 
400
399
  # Given a BuildInstance object and a host, send its params to the host. Return
@@ -0,0 +1,18 @@
1
+ # Module for signing all packages to places
2
+
3
+ module Pkg::Util::Repo
4
+ class << self
5
+
6
+ # Create yum repositories of built RPM packages for this SHA on the distribution server
7
+ def rpm_repos
8
+ Pkg::Util::File.fetch
9
+ Pkg::Rpm::Repo.create_remote_repos
10
+ end
11
+
12
+ # Create apt repositories of build DEB packages for this SHA on the distributions server
13
+ def deb_repos
14
+ Pkg::Util::File.fetch
15
+ Pkg::Deb::Repo.create_repos
16
+ end
17
+ end
18
+ end