apt_stage_artifacts 0.6.0 → 0.8.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8e43d6a863a30baa1254fc13176928c876311967637ee2c94aaa525db1d59434
4
- data.tar.gz: 04c20925c5ffceac9f3861d0e5783cc3ffec3b30acc992bb7e8384acbcf6fcfa
3
+ metadata.gz: ee106eeb8e60ac6a9d942ded6cd6fed67bc799ec2c77f4740d59a6c4c0667b27
4
+ data.tar.gz: 1961201b900674777a81a9ec22959873cd5782bc5141b647b89fb893efe20356
5
5
  SHA512:
6
- metadata.gz: f9eab7e0f70f96fa3f960a33c3013f7957e0eb863a7cf381b975e7d1de484ea157ad71635e792dbab50b449c7590bfa996e36959fd98d831f0ebb20795f773e8
7
- data.tar.gz: 71c0f29097b376bb4d7e169e48b97d8e2377cc3a7c0267e7dbd21cba360a78e91b42e1e5aa6626c0b5818ca1b44faf17e1db2b2b4c7d62dd83cc6f1fd58912af
6
+ metadata.gz: 988779891f6d6729d1fb70beacdd721b46ffc08a0d0cb5694f5461f3c987b839f87dcdda8fa557050232d85ce539e501759da7440638a2ffc59eac1dc10e35a0
7
+ data.tar.gz: 04e70a00aaef203277698f49e521ba0daf83c83d7b2f1752a39727e793ab53b1c6c06d25a95f1c4c07ed5c1d620bc599052a32fe311b27cc6ca45d057a6e810e
@@ -10,8 +10,9 @@ require 'tmpdir'
10
10
  ## This is a wrapper for https://github.com/freight-team/freight, which creates APT repos
11
11
  ## from deb files.
12
12
  ##
13
- ## In our case, we maintain a set of debian repos, based on puppet version and a repo-type
14
- ## 'nightly', 'archive' and 'stable'.
13
+ ## In our case, we maintain a set of debian repos, based on puppet version and a APT component
14
+ ## 'nightly' or 'stable' (we've also used 'archive' in the past; there's some remnants of it
15
+ ## left).
15
16
  ##
16
17
  ## freight provides signing services, so this script needs to exist on a machine where GPG
17
18
  ## signing has been set up.
@@ -19,14 +20,14 @@ require 'tmpdir'
19
20
  ## Typical usage would be for a remote process to call this script with a list of .deb
20
21
  ## URLS:
21
22
  ##
22
- ## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=stretch
23
+ ## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=stretch
23
24
  ## https://some-server.net/somepackage-a_0.19.2-1stretch_amd64.deb
24
25
  ## https://some-server.net/somepackage-b_0.19.2-1stretch_amd64.deb
25
26
  ##
26
- ## Alternately one coule scp a bunch of .deb files to the machine
27
+ ## Alternately one could scp a bunch of .deb files to the machine
27
28
  ## this is installed on then call this script with all the deb files as arguments. For example:
28
29
  ##
29
- ## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=buster
30
+ ## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=buster
30
31
  ## /tmp/xxtmpdir/somepackage-a_0.0.0-1buster_amd64.deb
31
32
  ## /tmp/xxtmpdir/somepackage-b_1.2.3-5buster_amd64.deb
32
33
  ## (etc.)
@@ -38,14 +39,14 @@ class AptAddToFreightLibrary
38
39
  Adds debian .deb files to a freight library.
39
40
 
40
41
  Usage:
41
- apt-add-to-freight-library [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE --codename=DEBIAN_CODENAME <deb_file> ...
42
+ apt-add-to-freight-library [--verbose] --puppet-version=VERSION --component=COMPONENT --codename=DEBIAN_CODENAME <deb_file> ...
42
43
  apt-add-to-freight-library --help
43
44
  apt-add-to-freight-library --version
44
45
 
45
46
  Options:
47
+ -c --component=COMPONENT APT component (Valid: #{AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')})
48
+ -n --codename=DEBIAN_CODENAME Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
46
49
  -p --puppet-version=VERSION Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
47
- -r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
48
- -c --codename=DEBIAN_CODENAME Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
49
50
 
50
51
  -v --verbose Be verbose
51
52
  -h --help Show this help
@@ -79,9 +80,9 @@ class AptAddToFreightLibrary
79
80
  fatal "Invalid Puppet version '#{@puppet_version}'" unless
80
81
  AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
81
82
 
82
- @repo_type = @user_options['--repo-type']
83
- unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
84
- fatal "Invalid repo type '#{@repo_type}'"
83
+ @apt_component = @user_options['--component']
84
+ unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
85
+ fatal "Invalid APT component '#{@apt_component}'"
85
86
  end
86
87
 
87
88
  @codename = @user_options['--codename']
@@ -91,7 +92,7 @@ class AptAddToFreightLibrary
91
92
 
92
93
  freight_config_file = File.join(
93
94
  AptStageArtifacts::FREIGHT_CONFIG_DIRECTORY,
94
- "puppet_#{@puppet_version}_#{@repo_type}.conf"
95
+ "puppet#{@puppet_version}.conf"
95
96
  )
96
97
  fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
97
98
  File.readable?(freight_config_file)
@@ -101,11 +102,14 @@ class AptAddToFreightLibrary
101
102
 
102
103
  Dir.mktmpdir do |temporary_directory|
103
104
  deb_artifacts.each do |artifact_path|
104
- logger.info "Adding #{artifact_path} to puppet_#{@puppet_version}_#{@repo_type}"
105
+ logger.info "Adding #{artifact_path} to puppet#{@puppet_version}/#{@apt_component}"
105
106
  deb_artifact_path = artifact_path
106
107
  if deb_artifact_path.start_with?('https://', 'http://')
107
108
  download_data = URI.parse(deb_artifact_path).open
108
- download_path = File.join(temporary_directory, download_data.base_uri.to_s.split('/').last)
109
+ download_path = File.join(
110
+ temporary_directory,
111
+ download_data.base_uri.to_s.split('/').last
112
+ )
109
113
  IO.copy_stream(download_data, download_path)
110
114
  deb_artifact_path = download_path
111
115
  end
@@ -113,7 +117,7 @@ class AptAddToFreightLibrary
113
117
  freight_add_command = %W[
114
118
  sudo --user=jenkins --set-home
115
119
  #{AptStageArtifacts::FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
116
- #{deb_artifact_path} "apt/#{@codename}"
120
+ #{deb_artifact_path} "apt/#{@codename}/#{@apt_component}"
117
121
  ].join(' ')
118
122
 
119
123
  %x(#{freight_add_command})
@@ -16,7 +16,7 @@ class AptStageArtifacts
16
16
  Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
17
17
 
18
18
  Usage:
19
- apt-stage-artifacts [--repo-type=<repo-type>] [<top_directory>]
19
+ apt-stage-artifacts [--component=<apt_component>] [<top_directory>]
20
20
  apt-stage-artifacts --version
21
21
  apt-stage-artifacts --help
22
22
 
@@ -26,7 +26,7 @@ class AptStageArtifacts
26
26
  @script_name = File.basename($PROGRAM_NAME)
27
27
  @log_level = Logger::INFO
28
28
  @user_options = parse_options(argv)
29
- @repo_type = @user_options['--repo-type'] || 'stable'
29
+ @apt_component = @user_options['--component'] || 'stable'
30
30
  @version_string = "Version #{AptStageArtifacts::VERSION}"
31
31
 
32
32
  @artifacts_tarball_name = 'artifacts.tgz'
@@ -47,11 +47,12 @@ class AptStageArtifacts
47
47
  def run
48
48
  logger.info @version_string
49
49
  exit 0 if @user_options['--version']
50
- unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
51
- fatal "Unknown repo_type: \"#{@repo_type}\". Valid repo types are: " +
52
- AptStageArtifacts::VALID_REPO_TYPES.join(', ')
50
+ unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
51
+ fatal "Unknown APT component: \"#{@apt_component}\". Valid APT components are: " +
52
+ AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')
53
53
  end
54
54
  load_builder_data_yaml
55
+ logger.info "Artifacts will be shipped to '#{@staging_server}'."
55
56
  collect_artifacts
56
57
  create_remote_staging_directory
57
58
  generate_manifest
@@ -116,7 +117,6 @@ class AptStageArtifacts
116
117
  yaml_content = URI.parse(@builder_data_yaml).open(&:read)
117
118
  yaml_data = YAML.safe_load(yaml_content)
118
119
  @staging_server = yaml_data['staging_server']
119
- logger.info "Artifacts will be staged on '#{@staging_server}'."
120
120
  end
121
121
 
122
122
  # A manifest is a staging to-do list represented in JSON as an array of hashes
@@ -125,14 +125,14 @@ class AptStageArtifacts
125
125
  # codename: debian codename for this artifact, needed by freight
126
126
  # puppet_version: part of the destination apt repo. We maintain apt repos by puppet major
127
127
  # version
128
- # repo_type: either 'stable' or 'nightly' typically. 'archive' is supported but unused.
128
+ # apt_component: either 'stable' or 'nightly'. 'archive' is supported but unused.
129
129
  def generate_manifest
130
130
  manifest = []
131
131
 
132
132
  @debian_artifacts.each do |deb_file_path|
133
- artifact_components = deb_file_path.split('/')
133
+ artifact_items = deb_file_path.split('/')
134
134
 
135
- repo_name = artifact_components[-2]
135
+ repo_name = artifact_items[-2]
136
136
  case repo_name
137
137
  when /^puppet(\d+)+$/
138
138
  puppet_version = Regexp.last_match(1)
@@ -149,7 +149,7 @@ class AptStageArtifacts
149
149
  deb_file_path: deb_file_path,
150
150
  puppet_version: puppet_version,
151
151
  codename: codename,
152
- repo_type: @repo_type
152
+ apt_component: @apt_component
153
153
  }
154
154
  end
155
155
 
@@ -13,11 +13,13 @@ class AptStageFromTarball
13
13
  @tarball_path = tarball_path
14
14
  @manifest_json_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
15
15
  @log_level = Logger::INFO
16
- @repo_types_updated = {}
17
16
 
18
17
  @script_name = File.basename($PROGRAM_NAME)
19
18
  @version_string = "Version #{AptStageArtifacts::VERSION}"
20
19
 
20
+ # Keep a list of puppet versions that will require a freight-cache update
21
+ @cache_update_versions = []
22
+
21
23
  # External commands
22
24
  @apt_add_to_freight_library_command = 'apt-add-to-freight-library'
23
25
  @apt_update_freight_cache_command = 'apt-update-freight-cache'
@@ -77,7 +79,7 @@ class AptStageFromTarball
77
79
  #{@apt_add_to_freight_library_command}
78
80
  --verbose
79
81
  --puppet-version=#{artifact[:puppet_version]}
80
- --repo-type=#{artifact[:repo_type]}
82
+ --component=#{artifact[:apt_component]}
81
83
  --codename=#{artifact[:codename]}
82
84
  #{File.join(Dir.pwd, artifact[:deb_file_path])}
83
85
  ].join(' ')
@@ -85,29 +87,25 @@ class AptStageFromTarball
85
87
  %x(#{add_to_freight_library_command})
86
88
  fatal "#{add_to_freight_library_command} failed." unless $CHILD_STATUS.success?
87
89
 
88
- # Keep a hash, key by puppet_version, of repo_types so that we can
90
+ # Keep a hash, key by puppet_version, of apt_components so that we can
89
91
  # efficiently build out the freight caches later.
90
- puppet_version = artifact[:puppet_version]
91
- repo_type = artifact[:repo_type]
92
- @repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
93
- @repo_types_updated[puppet_version] |= [repo_type]
92
+ @cache_update_versions << artifact[:puppet_version]
94
93
  end
95
94
  end
95
+ @cache_update_versions = @cache_update_versions.uniq
96
96
  end
97
97
 
98
+ # Executes the 'freight-cache' command through a local wrapper to create an APT repo
98
99
  def update_freight_cache
99
- @repo_types_updated.each do |puppet_version, repo_types|
100
- repo_types.each do |repo_type|
101
- update_freight_cache_command = %W[
102
- #{@apt_update_freight_cache_command}
103
- --verbose
104
- --puppet-version=#{puppet_version}
105
- --repo-type=#{repo_type}
100
+ @cache_update_versions.each do |puppet_version|
101
+ update_freight_cache_command = %W[
102
+ #{@apt_update_freight_cache_command}
103
+ --verbose
104
+ --puppet-version=#{puppet_version}
106
105
  ].join(' ')
107
106
 
108
- %x(#{update_freight_cache_command})
109
- fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
110
- end
107
+ %x(#{update_freight_cache_command})
108
+ fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
111
109
  end
112
110
  end
113
111
  end
@@ -20,9 +20,8 @@ require 'yaml'
20
20
  ## freight provides signing services, so this script needs to exist on a machine where GPG
21
21
  ## signing has been set up.
22
22
  ##
23
- ## Example usages:
24
- ## apt-update-freight-cache --puppet-version=7 --repo-type=stable
25
- ## apt-update-freight-cache --puppet-version=7 --repo-type=nightly
23
+ ## Example usage:
24
+ ## apt-update-freight-cache --puppet-version=7
26
25
 
27
26
  class AptUpdateFreightCache
28
27
  include Logging
@@ -31,13 +30,12 @@ class AptUpdateFreightCache
31
30
  Generates a freight cache from a freight library.
32
31
 
33
32
  Usage:
34
- apt-update-freight-cache [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE
33
+ apt-update-freight-cache [--verbose] --puppet-version=VERSION
35
34
  apt-update-freight-cache --help
36
35
  apt-update-freight-cache --version
37
36
 
38
37
  Options:
39
38
  -p --puppet-version=VERSION Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
40
- -r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
41
39
 
42
40
  -v --verbose Be verbose
43
41
  -h --help Show this help
@@ -69,15 +67,11 @@ class AptUpdateFreightCache
69
67
  logger.info @version_string
70
68
  load_builder_data_yaml
71
69
  @puppet_version = @user_options['--puppet-version']
72
- fatal "Invalid Puppet version '#{@puppet_version}'" unless
73
- AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
74
-
75
- @repo_type = @user_options['--repo-type']
76
- unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
77
- fatal "Invalid repo type '#{@repo_type}'"
70
+ unless AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
71
+ fatal "Invalid Puppet version '#{@puppet_version}'"
78
72
  end
79
73
 
80
- freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
74
+ freight_config_file = "/etc/freight.conf.d/puppet#{@puppet_version}.conf"
81
75
  fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
82
76
  File.readable?(freight_config_file)
83
77
 
@@ -17,6 +17,13 @@ class AptStageArtifacts
17
17
  VALID_CODENAMES = DEBIAN_CODENAMES | UBUNTU_CODENAMES
18
18
 
19
19
  # Validation constraints
20
- VALID_PUPPET_VERSIONS = %w[6 7 8]
21
- VALID_REPO_TYPES = %w[archive nightly stable]
20
+ VALID_PUPPET_VERSIONS = %w[6 7 8 9]
21
+
22
+ # From freight-add:
23
+ # freight-add registers package with one or more manager/distro[/component] pairs (or triples).
24
+ # component is optional and for apt defaults to 'main'.
25
+
26
+ # These are valid APT components. We use 'stable' instead of 'main' established by convention
27
+ # from docker dists (see https://download.docker.com/linux/ubuntu/dists/groovy)
28
+ VALID_APT_COMPONENTS = %w[archive nightly stable]
22
29
  end
data/lib/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  class AptStageArtifacts
2
- VERSION = '0.6.0'
2
+ VERSION = '0.8.1'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apt_stage_artifacts
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.8.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Puppet Release Engineering
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-06-08 00:00:00.000000000 Z
11
+ date: 2021-09-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pry
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
179
179
  - !ruby/object:Gem::Version
180
180
  version: '0'
181
181
  requirements: []
182
- rubygems_version: 3.1.4
182
+ rubygems_version: 3.1.6
183
183
  signing_key:
184
184
  specification_version: 4
185
185
  summary: Stages .deb artifacts to a remote freight repository