apt_stage_artifacts 0.4.0 → 0.5.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 24c10c67846c819b0174ac0b94f2a4b7836e73589969335c1fb4b6a2779e59bc
4
- data.tar.gz: 47be9199cdabcc1bb1f242f6cd4e1aa8bea968d3ed83bc215e17c67ebe8d3f2c
3
+ metadata.gz: 93eab470ef8684eb79b47dafcc7f08b345d232542d96666e853ea8ac7e553d3e
4
+ data.tar.gz: bfee3c314fdd1eecd75253d7ad3c71e7debb8d4c396a43a379541e44c3347b5b
5
5
  SHA512:
6
- metadata.gz: 421d5a2a39379edddffa17bba6925c328d8f2a51d3af858743b771729b337ed74a9d4ce8c48a51798a6371f1b477fb16ab38833966339fe2ac6978650e0a8d4a
7
- data.tar.gz: 585ec2d56f0959a19cb83fad46d5934265c1eaa1dce02b89155ada6a6d507dd3ab064e8a7447cffc5be3ea5384b4e9354331fdab1faf146a28a31e85c6fedb69
6
+ metadata.gz: b9ce4b50d62011ae52277ca427de4c6ecbbba6116687fe240097d70327c99b3b0b870c2deb63fea2a3630719f2d89d0f499f69416b7e81c79ac65d02d60e54c0
7
+ data.tar.gz: d71e6d727e0cc5a8ffa56eb7360174eed2de96af56e1877a9235a1e4f02ca3da7c2ec8d57b51f83ee1e197fabafd77fc101295fbb169b5086b187619bc79aa51
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
14
14
 
15
15
  spec.metadata['homepage_uri'] = spec.homepage
16
16
  spec.metadata['source_code_uri'] = spec.homepage
17
- spec.metadata['changelog_uri'] = File.join(spec.homepage, 'CHANGELOG.md')
17
+ spec.metadata['changelog_uri'] = File.join(spec.homepage, 'blob', 'main', 'CHANGELOG.md')
18
18
 
19
19
  # Specify which files should be added to the gem when it is released.
20
20
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -1,4 +1,5 @@
1
1
  require 'version'
2
+ require 'mixins/constants'
2
3
  require 'mixins/logging'
3
4
 
4
5
  require 'English'
@@ -33,11 +34,6 @@ require 'tmpdir'
33
34
  class AptAddToFreightLibrary
34
35
  include Logging
35
36
 
36
- FREIGHT_COMMAND = '/usr/bin/freight'
37
- VALID_PUPPET_VERSIONS = %w[6 7 8]
38
- VALID_REPO_TYPES = %w[archive nightly stable]
39
- VALID_CODENAMES = %w[bionic jessie stretch buster bullseye bookwork focal xenial]
40
-
41
37
  DOCUMENTATION = <<~DOCOPT
42
38
  Adds debian .deb files to a freight library.
43
39
 
@@ -47,9 +43,9 @@ class AptAddToFreightLibrary
47
43
  apt-add-to-freight-library --version
48
44
 
49
45
  Options:
50
- -p --puppet-version=VERSION Puppet version (Valid: #{VALID_PUPPET_VERSIONS.join(', ')})
51
- -r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{VALID_REPO_TYPES.join(', ')})
52
- -c --codename=DEBIAN_CODENAME Debian codename to deliver to (Valid: #{VALID_CODENAMES.join(', ')})
46
+ -p --puppet-version=VERSION Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
47
+ -r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
48
+ -c --codename=DEBIAN_CODENAME Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
53
49
 
54
50
  -v --verbose Be verbose
55
51
  -h --help Show this help
@@ -65,7 +61,7 @@ class AptAddToFreightLibrary
65
61
  @script_name = File.basename($PROGRAM_NAME)
66
62
  @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
67
63
  if @user_options['--version'] # rubocop:disable Style/GuardClause
68
- puts @version_string
64
+ warn @version_string
69
65
  exit 0
70
66
  end
71
67
  end
@@ -73,7 +69,7 @@ class AptAddToFreightLibrary
73
69
  def parse_options(argv = nil)
74
70
  Docopt.docopt(DOCUMENTATION, { argv: argv })
75
71
  rescue Docopt::Exit => e
76
- puts e.message
72
+ warn e.message
77
73
  exit 1
78
74
  end
79
75
 
@@ -81,15 +77,22 @@ class AptAddToFreightLibrary
81
77
  logger.info @version_string
82
78
  @puppet_version = @user_options['--puppet-version']
83
79
  fatal "Invalid Puppet version '#{@puppet_version}'" unless
84
- VALID_PUPPET_VERSIONS.include? @puppet_version
80
+ AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
85
81
 
86
82
  @repo_type = @user_options['--repo-type']
87
- fatal "Invalid repo type '#{@repo_type}'" unless VALID_REPO_TYPES.include? @repo_type
83
+ unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
84
+ fatal "Invalid repo type '#{@repo_type}'"
85
+ end
88
86
 
89
87
  @codename = @user_options['--codename']
90
- fatal "Invalid codename '#{@codename}'" unless VALID_CODENAMES.include? @codename
88
+ unless AptStageArtifacts::VALID_CODENAMES.include? @codename
89
+ fatal "Invalid codename '#{@codename}'"
90
+ end
91
91
 
92
- freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
92
+ freight_config_file = File.join(
93
+ AptStageArtifacts::FREIGHT_CONFIG_DIRECTORY,
94
+ "puppet_#{@puppet_version}_#{@repo_type}.conf"
95
+ )
93
96
  fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
94
97
  File.readable?(freight_config_file)
95
98
 
@@ -109,7 +112,7 @@ class AptAddToFreightLibrary
109
112
 
110
113
  freight_add_command = %W[
111
114
  sudo --user=jenkins --set-home
112
- #{FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
115
+ #{AptStageArtifacts::FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
113
116
  #{deb_artifact_path} "apt/#{@codename}"
114
117
  ].join(' ')
115
118
 
@@ -1,11 +1,12 @@
1
1
  require 'version'
2
2
  require 'mixins/logging'
3
+ require 'mixins/constants'
3
4
 
4
5
  require 'English'
5
6
  require 'docopt'
6
7
  require 'fileutils'
8
+ require 'json'
7
9
  require 'open-uri'
8
- require 'tempfile'
9
10
  require 'yaml'
10
11
 
11
12
  class AptStageArtifacts
@@ -15,35 +16,34 @@ class AptStageArtifacts
15
16
  Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
16
17
 
17
18
  Usage:
18
- apt-stage-artifacts
19
- apt-stage-artifacts <top_directory>
19
+ apt-stage-artifacts [--version]
20
+ apt-stage-artifacts [--version] <top_directory>
20
21
  DOCOPT
21
22
 
22
23
  def initialize(argv)
23
24
  @script_name = File.basename($PROGRAM_NAME)
24
25
  @log_level = Logger::INFO
25
26
  @user_options = parse_options(argv)
26
- @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
27
+ @version_string = "Version #{AptStageArtifacts::VERSION}"
27
28
 
28
29
  @artifacts_tarball_name = 'artifacts.tgz'
29
30
  @local_tarball_path = File.join(Dir.pwd, @artifacts_tarball_name)
30
31
 
31
32
  @builder_data_yaml = 'https://raw.githubusercontent.com/puppetlabs/build-data/release/builder_data.yaml'
32
33
 
33
- @manifest_file_name = 'apt_stage.manifest'
34
34
  @remote_staging_command = 'apt-stage-from-tarball'
35
35
  end
36
36
 
37
37
  def parse_options(argv = nil)
38
38
  Docopt.docopt(DOCUMENTATION, { argv: argv })
39
39
  rescue Docopt::Exit => e
40
- puts e.message
40
+ warn e.message
41
41
  exit 1
42
42
  end
43
43
 
44
44
  def run
45
45
  logger.info @version_string
46
- parse_repo_type
46
+ exit 0 if @user_options['--version']
47
47
  load_builder_data_yaml
48
48
  collect_artifacts
49
49
  create_remote_staging_directory
@@ -66,6 +66,9 @@ class AptStageArtifacts
66
66
  def collect_artifacts
67
67
  if !@user_options['<top_directory>'].to_s.empty?
68
68
  @top_directory = @user_options['<top_directory>']
69
+ if File.directory?(File.join(@top_directory, 'deb'))
70
+ @top_directory = File.join(@top_directory, 'deb')
71
+ end
69
72
  elsif File.directory?('./output/deb')
70
73
  @top_directory = './output/deb'
71
74
  elsif File.directory?('./pkg/deb')
@@ -77,7 +80,7 @@ class AptStageArtifacts
77
80
 
78
81
  logger.info "Searching for .deb packages in #{@top_directory}"
79
82
  Dir.chdir(@top_directory) do
80
- @debian_artifacts = Dir['*/*.deb']
83
+ @debian_artifacts = Dir['**/*.deb']
81
84
  end
82
85
  return unless @debian_artifacts.empty?
83
86
 
@@ -109,36 +112,63 @@ class AptStageArtifacts
109
112
  logger.info "Artifacts will be staged on '#{@staging_server}'."
110
113
  end
111
114
 
112
- # A manifest is a staging to-do list. For each '.deb' file create a colon-delimited line
115
+ # A manifest is a staging to-do list represented in JSON as an array of hashes
113
116
  # containing:
114
- # <.deb filename>:<puppet-version>:<repo-type>:<debian-codename>
117
+ # deb_file_path:
118
+ # codename:
119
+ # puppet_version:
120
+ # repo_type:
115
121
  def generate_manifest
116
- manifest_temporary_file = Tempfile.new('apt_stage_artifacts')
122
+ manifest = []
117
123
 
118
124
  @debian_artifacts.each do |deb_file_path|
119
- # Ug. We only know the codename because it happens to be the first element
120
- # of the file path.
121
- debian_codename = deb_file_path.split('/').first
122
- manifest_temporary_file.printf(
123
- "%s:%s:%s:%s\n",
124
- deb_file_path, @puppet_version, @repo_type, debian_codename
125
- )
125
+ artifact_components = deb_file_path.split('/')
126
+
127
+ repo_name = artifact_components[-2]
128
+ case repo_name
129
+ when /^puppet(\d+)-nightly$/
130
+ puppet_version = Regexp.last_match(1)
131
+ repo_type = 'nightly'
132
+ when /^puppet(\d+)-archive/
133
+ puppet_version = Regexp.last_match(1)
134
+ repo_type = 'archive'
135
+ when /^puppet(\d+)+$/
136
+ puppet_version = Regexp.last_match(1)
137
+ repo_type = 'stable'
138
+ else
139
+ fatal("Unknown repo name \"#{repo_name}\" in #{deb_file_path}")
140
+ end
141
+
142
+ codename = deb_file_path.split('/')[-3]
143
+ unless AptStageArtifacts::VALID_CODENAMES.include?(codename)
144
+ fatal("#{deb_file_path} contains an unknown codename: #{codename}")
145
+ end
146
+
147
+ manifest << {
148
+ deb_file_path: deb_file_path,
149
+ puppet_version: puppet_version,
150
+ codename: codename,
151
+ repo_type: repo_type
152
+ }
126
153
  end
127
154
 
128
- manifest_temporary_file.close
129
- manifest_path = File.join(@top_directory, @manifest_file_name)
130
- FileUtils.mv(manifest_temporary_file.path, manifest_path)
155
+ manifest_path = File.join(@top_directory, AptStageArtifacts::MANIFEST_JSON_FILENAME)
156
+ File.open(manifest_path, 'w') do |f|
157
+ f.write(manifest.to_json)
158
+ end
131
159
  logger.info("Staging manifest written to '#{manifest_path}'")
132
160
  end
133
161
 
134
162
  # Create a tarball that contains the staging manifest and the associated deb files.
135
163
  def create_tarball
164
+ manifest_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
165
+
136
166
  tarball_create_command = %W[
137
167
  tar --create --gzip --file #{@local_tarball_path} --directory=#{@top_directory}
138
168
  ].join(' ')
139
- %x(#{tarball_create_command} #{@manifest_file_name} #{@debian_artifacts.join(' ')})
169
+ %x(#{tarball_create_command} #{manifest_file_name} #{@debian_artifacts.join(' ')})
140
170
  unless $CHILD_STATUS.success?
141
- fatal "#{tarball_create_command} #{@manifest_file_name} #{@debian_artifacts.join(' ')} failed."
171
+ fatal "#{tarball_create_command} #{manifest_file_name} #{@debian_artifacts.join(' ')} failed."
142
172
  end
143
173
 
144
174
  logger.info("'#{@local_tarball_path}' created.")
@@ -148,27 +178,6 @@ class AptStageArtifacts
148
178
  File.delete @local_tarball_path
149
179
  end
150
180
 
151
- # The target apt repository is described in the REPO_NAME environment variable.
152
- # A better solution is called for, perhaps as part of vanagon config or as a commandline
153
- # parameter.
154
- def parse_repo_type
155
- case ENV['REPO_NAME']
156
- when /\A\z/, nil
157
- fatal "The environment variable 'REPO_NAME', containing the target apt repo, is unset.\n" \
158
- "It should be set to something like 'puppet7' or 'puppet7-nightly'."
159
- when /\Apuppet(\d+)\z/
160
- @puppet_version = Regexp.last_match(1)
161
- @repo_type = 'stable'
162
- when /\Apuppet(\d+)-nightly\z/
163
- @puppet_version = Regexp.last_match(1)
164
- @repo_type = 'nightly'
165
- else
166
- fatal "The environment variable 'REPO_NAME' is set to '#{ENV['REPO_NAME']}'\n" \
167
- "This is not a recognized setting.\n" \
168
- "It should be set to something like 'puppet7' or 'puppet7-nightly'."
169
- end
170
- end
171
-
172
181
  # Send the tarball over to the staging server for further processing with freight.
173
182
  def ship_tarball_to_staging_server
174
183
  logger.info "Sending '#{@local_tarball_path}' to '#{@staging_server}:#{@remote_tarball_path}'"
@@ -1,7 +1,9 @@
1
1
  require 'version'
2
+ require 'mixins/constants'
2
3
  require 'mixins/logging'
3
4
 
4
5
  require 'English'
6
+ require 'json'
5
7
  require 'mkmf'
6
8
 
7
9
  class AptStageFromTarball
@@ -9,12 +11,12 @@ class AptStageFromTarball
9
11
 
10
12
  def initialize(tarball_path = nil)
11
13
  @tarball_path = tarball_path
12
- @manifest_file_name = 'apt_stage.manifest'
14
+ @manifest_json_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
13
15
  @log_level = Logger::INFO
14
16
  @repo_types_updated = {}
15
17
 
16
18
  @script_name = File.basename($PROGRAM_NAME)
17
- @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
19
+ @version_string = "Version #{AptStageArtifacts::VERSION}"
18
20
 
19
21
  # External commands
20
22
  @apt_add_to_freight_library_command = 'apt-add-to-freight-library'
@@ -57,7 +59,7 @@ class AptStageFromTarball
57
59
  fatal "#{tarball_extract_command} failed."
58
60
  end
59
61
 
60
- @manifest_path = File.join(@tarball_directory, @manifest_file_name)
62
+ @manifest_path = File.join(@tarball_directory, @manifest_json_file_name)
61
63
 
62
64
  fatal "Staging manifest file '#{@manifest_path}' is missing or unreadable" unless
63
65
  File.readable?(@manifest_path)
@@ -69,16 +71,15 @@ class AptStageFromTarball
69
71
  # library.
70
72
  def add_artifacts_to_freight_library
71
73
  Dir.chdir(@tarball_directory) do
72
- File.readlines(@manifest_path).each do |line|
73
- line.chomp!
74
- debian_filename, puppet_version, repo_type, codename = line.split(':')
74
+ manifest_file = File.read(@manifest_path)
75
+ JSON.parse(manifest_file, symbolize_names: true).each do |artifact|
75
76
  add_to_freight_library_command = %W[
76
77
  #{@apt_add_to_freight_library_command}
77
78
  --verbose
78
- --puppet-version=#{puppet_version}
79
- --repo-type=#{repo_type}
80
- --codename=#{codename}
81
- #{File.join(Dir.pwd, debian_filename)}
79
+ --puppet-version=#{artifact[:puppet_version]}
80
+ --repo-type=#{artifact[:repo_type]}
81
+ --codename=#{artifact[:codename]}
82
+ #{File.join(Dir.pwd, artifact[:deb_file_path])}
82
83
  ].join(' ')
83
84
 
84
85
  %x(#{add_to_freight_library_command})
@@ -86,6 +87,8 @@ class AptStageFromTarball
86
87
 
87
88
  # Keep a hash, key by puppet_version, of repo_types so that we can
88
89
  # efficiently build out the freight caches later.
90
+ puppet_version = artifact[:puppet_version]
91
+ repo_type = artifact[:repo_type]
89
92
  @repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
90
93
  @repo_types_updated[puppet_version] |= [repo_type]
91
94
  end
@@ -1,4 +1,5 @@
1
1
  require 'version'
2
+ require 'mixins/constants'
2
3
  require 'mixins/logging'
3
4
 
4
5
  require 'English'
@@ -26,10 +27,6 @@ require 'yaml'
26
27
  class AptUpdateFreightCache
27
28
  include Logging
28
29
 
29
- FREIGHT_COMMAND = '/usr/bin/freight'
30
- VALID_PUPPET_VERSIONS = %w[6 7 8]
31
- VALID_REPO_TYPES = %w[archive nightly stable]
32
-
33
30
  DOCUMENTATION = <<~DOCOPT
34
31
  Generates a freight cache from a freight library.
35
32
 
@@ -39,8 +36,8 @@ class AptUpdateFreightCache
39
36
  apt-update-freight-cache --version
40
37
 
41
38
  Options:
42
- -p --puppet-version=VERSION Puppet version (Valid: #{VALID_PUPPET_VERSIONS.join(', ')})
43
- -r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{VALID_REPO_TYPES.join(', ')})
39
+ -p --puppet-version=VERSION Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
40
+ -r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
44
41
 
45
42
  -v --verbose Be verbose
46
43
  -h --help Show this help
@@ -56,7 +53,7 @@ class AptUpdateFreightCache
56
53
  @script_name = File.basename($PROGRAM_NAME)
57
54
  @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
58
55
  if @user_options['--version'] # rubocop:disable Style/GuardClause
59
- puts @version_string
56
+ warn @version_string
60
57
  exit 0
61
58
  end
62
59
  end
@@ -73,10 +70,12 @@ class AptUpdateFreightCache
73
70
  load_builder_data_yaml
74
71
  @puppet_version = @user_options['--puppet-version']
75
72
  fatal "Invalid Puppet version '#{@puppet_version}'" unless
76
- VALID_PUPPET_VERSIONS.include? @puppet_version
73
+ AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
77
74
 
78
75
  @repo_type = @user_options['--repo-type']
79
- fatal "Invalid repo type '#{@repo_type}'" unless VALID_REPO_TYPES.include? @repo_type
76
+ unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
77
+ fatal "Invalid repo type '#{@repo_type}'"
78
+ end
80
79
 
81
80
  freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
82
81
  fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
@@ -0,0 +1,22 @@
1
+ ##
2
+ ## Some constants that are used throughout the code
3
+ ##
4
+
5
+ class AptStageArtifacts
6
+ # Details about freight itself
7
+ FREIGHT_COMMAND = '/usr/bin/freight'
8
+ FREIGHT_CONFIG_DIRECTORY = '/etc/freight.conf.d'
9
+
10
+ # Name of the manifest.json file
11
+ MANIFEST_JSON_FILENAME = 'apt_stage_manifest.json'
12
+
13
+ # Debian/Ubuntu codenames that we care about.
14
+ # Some codenames are skipped because we don't/haven't shipped for them.
15
+ DEBIAN_CODENAMES = %w[jessie stretch buster bullseye bookworm trixie]
16
+ UBUNTU_CODENAMES = %w[xenial bionic focal groovy]
17
+ VALID_CODENAMES = DEBIAN_CODENAMES | UBUNTU_CODENAMES
18
+
19
+ # Validation constraints
20
+ VALID_PUPPET_VERSIONS = %w[6 7 8]
21
+ VALID_REPO_TYPES = %w[archive nightly stable]
22
+ end
data/lib/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  class AptStageArtifacts
2
- VERSION = '0.4.0'
2
+ VERSION = '0.5.4'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apt_stage_artifacts
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Puppet Release Engineering
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-04-28 00:00:00.000000000 Z
11
+ date: 2021-06-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pry
@@ -154,6 +154,7 @@ files:
154
154
  - lib/apt_stage_artifacts.rb
155
155
  - lib/apt_stage_from_tarball.rb
156
156
  - lib/apt_update_freight_cache.rb
157
+ - lib/mixins/constants.rb
157
158
  - lib/mixins/logging.rb
158
159
  - lib/version.rb
159
160
  homepage: https://github.com/puppetlabs/apt_stage_artifacts
@@ -162,7 +163,7 @@ metadata:
162
163
  allowed_push_host: https://rubygems.org
163
164
  homepage_uri: https://github.com/puppetlabs/apt_stage_artifacts
164
165
  source_code_uri: https://github.com/puppetlabs/apt_stage_artifacts
165
- changelog_uri: https://github.com/puppetlabs/apt_stage_artifacts/CHANGELOG.md
166
+ changelog_uri: https://github.com/puppetlabs/apt_stage_artifacts/blob/main/CHANGELOG.md
166
167
  post_install_message:
167
168
  rdoc_options: []
168
169
  require_paths: