apt_stage_artifacts 0.5.4 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93eab470ef8684eb79b47dafcc7f08b345d232542d96666e853ea8ac7e553d3e
-  data.tar.gz: bfee3c314fdd1eecd75253d7ad3c71e7debb8d4c396a43a379541e44c3347b5b
+  metadata.gz: b9075c27ac41c8d3e0863b08e398bfa37aaa0be45db5d3829ee520a7bdd61b71
+  data.tar.gz: 68800922f370cae55767a88ab3db242ff1fdb16c11dd78f70431bd9fb7f9387f
 SHA512:
-  metadata.gz: b9ce4b50d62011ae52277ca427de4c6ecbbba6116687fe240097d70327c99b3b0b870c2deb63fea2a3630719f2d89d0f499f69416b7e81c79ac65d02d60e54c0
-  data.tar.gz: d71e6d727e0cc5a8ffa56eb7360174eed2de96af56e1877a9235a1e4f02ca3da7c2ec8d57b51f83ee1e197fabafd77fc101295fbb169b5086b187619bc79aa51
+  metadata.gz: 257be870ddd630e3c9120698581f23796c0e3ffb98051c85808ebc249fe45a4ca6a4a487aa3cfd90c3d5de168a17f2fdf9b80c306a48b767f29f8f893465de05
+  data.tar.gz: 1620ec2cb8e999c3d5444b9043c71187f564d527f7110a78c03f28f827abf03132ac48d92677d1220f118568a570df995793acd9c6ea72b2e24f6580bb0b8bed

data/lib/apt_add_to_freight_library.rb

@@ -10,8 +10,9 @@ require 'tmpdir'
 ## This is a wrapper for https://github.com/freight-team/freight, which creates APT repos
 ## from deb files.
 ##
-## In our case, we maintain a set of debian repos, based on puppet version and a repo-type
-## 'nightly', 'archive' and 'stable'.
+## In our case, we maintain a set of debian repos, based on puppet version and a APT component
+## 'nightly' or 'stable' (we've also used 'archive' in the past; there's some remnants of it
+## left).
 ##
 ## freight provides signing services, so this script needs to exist on a machine where GPG
 ## signing has been set up.
@@ -19,14 +20,14 @@ require 'tmpdir'
 ## Typical usage would be for a remote process to call this script with a list of .deb
 ## URLS:
 ##
-## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=stretch
+## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=stretch
 ##   https://some-server.net/somepackage-a_0.19.2-1stretch_amd64.deb
 ##   https://some-server.net/somepackage-b_0.19.2-1stretch_amd64.deb
 ##
-## Alternately one coule scp a bunch of .deb files to the machine
+## Alternately one could scp a bunch of .deb files to the machine
 ## this is installed on then call this script with all the deb files as arguments. For example:
 ##
-## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=buster
+## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=buster
 ##   /tmp/xxtmpdir/somepackage-a_0.0.0-1buster_amd64.deb
 ##   /tmp/xxtmpdir/somepackage-b_1.2.3-5buster_amd64.deb
 ##     (etc.)
@@ -38,14 +39,14 @@ class AptAddToFreightLibrary
     Adds debian .deb files to a freight library.
 
     Usage:
-        apt-add-to-freight-library [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE --codename=DEBIAN_CODENAME <deb_file> ...
+        apt-add-to-freight-library [--verbose] --puppet-version=VERSION --component=COMPONENT --codename=DEBIAN_CODENAME <deb_file> ...
         apt-add-to-freight-library --help
         apt-add-to-freight-library --version
 
     Options:
+    -c --component=COMPONENT       APT component (Valid: #{AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')})
+    -n --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
     -p --puppet-version=VERSION    Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
-    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
-    -c --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
 
     -v --verbose  Be verbose
     -h --help     Show this help
@@ -79,9 +80,9 @@ class AptAddToFreightLibrary
     fatal "Invalid Puppet version '#{@puppet_version}'" unless
       AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
 
-    @repo_type = @user_options['--repo-type']
-    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
-      fatal "Invalid repo type '#{@repo_type}'"
+    @apt_component = @user_options['--component']
+    unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
+      fatal "Invalid APT component '#{@apt_component}'"
     end
 
     @codename = @user_options['--codename']
@@ -91,7 +92,7 @@ class AptAddToFreightLibrary
 
     freight_config_file = File.join(
       AptStageArtifacts::FREIGHT_CONFIG_DIRECTORY,
-      "puppet_#{@puppet_version}_#{@repo_type}.conf"
+      "puppet_#{@puppet_version}.conf"
     )
     fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
       File.readable?(freight_config_file)
@@ -101,11 +102,14 @@ class AptAddToFreightLibrary
 
     Dir.mktmpdir do |temporary_directory|
       deb_artifacts.each do |artifact_path|
-        logger.info "Adding #{artifact_path} to puppet_#{@puppet_version}_#{@repo_type}"
+        logger.info "Adding #{artifact_path} to puppet_#{@puppet_version}/#{@apt_component}"
         deb_artifact_path = artifact_path
         if deb_artifact_path.start_with?('https://', 'http://')
           download_data = URI.parse(deb_artifact_path).open
-          download_path = File.join(temporary_directory, download_data.base_uri.to_s.split('/').last)
+          download_path = File.join(
+            temporary_directory,
+            download_data.base_uri.to_s.split('/').last
+          )
           IO.copy_stream(download_data, download_path)
           deb_artifact_path = download_path
         end
@@ -113,7 +117,7 @@ class AptAddToFreightLibrary
         freight_add_command = %W[
          sudo --user=jenkins --set-home
          #{AptStageArtifacts::FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
-         #{deb_artifact_path} "apt/#{@codename}"
+         #{deb_artifact_path} "apt/#{@codename}/#{@apt_component}"
         ].join(' ')
 
         %x(#{freight_add_command})

data/lib/apt_stage_artifacts.rb

@@ -16,14 +16,17 @@ class AptStageArtifacts
     Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
 
     Usage:
-        apt-stage-artifacts [--version]
-        apt-stage-artifacts [--version] <top_directory>
+        apt-stage-artifacts [--component=<apt_component>] [<top_directory>]
+        apt-stage-artifacts --version
+        apt-stage-artifacts --help
+
   DOCOPT
 
   def initialize(argv)
     @script_name = File.basename($PROGRAM_NAME)
     @log_level = Logger::INFO
     @user_options = parse_options(argv)
+    @apt_component = @user_options['--component'] || 'stable'
     @version_string = "Version #{AptStageArtifacts::VERSION}"
 
     @artifacts_tarball_name = 'artifacts.tgz'
@@ -44,7 +47,12 @@ class AptStageArtifacts
   def run
     logger.info @version_string
     exit 0 if @user_options['--version']
+    unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
+      fatal "Unknown APT component: \"#{@apt_component}\". Valid APT components are: " +
+            AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')
+    end
     load_builder_data_yaml
+    logger.info "Artifacts will be shipped to '#{@staging_server}'."
     collect_artifacts
     create_remote_staging_directory
     generate_manifest
@@ -109,32 +117,25 @@ class AptStageArtifacts
     yaml_content = URI.parse(@builder_data_yaml).open(&:read)
     yaml_data = YAML.safe_load(yaml_content)
     @staging_server = yaml_data['staging_server']
-    logger.info "Artifacts will be staged on '#{@staging_server}'."
   end
 
   # A manifest is a staging to-do list represented in JSON as an array of hashes
   # containing:
-  #   deb_file_path:
-  #   codename:
-  #   puppet_version:
-  #   repo_type:
+  #   deb_file_path: relative path to the .deb file
+  #   codename: debian codename for this artifact, needed by freight
+  #   puppet_version: part of the destination apt repo. We maintain apt repos by puppet major
+  #     version
+  #   apt_component: either 'stable' or 'nightly'. 'archive' is supported but unused.
   def generate_manifest
     manifest = []
 
     @debian_artifacts.each do |deb_file_path|
-      artifact_components = deb_file_path.split('/')
+      artifact_items = deb_file_path.split('/')
 
-      repo_name = artifact_components[-2]
+      repo_name = artifact_items[-2]
       case repo_name
-      when /^puppet(\d+)-nightly$/
-        puppet_version = Regexp.last_match(1)
-        repo_type = 'nightly'
-      when /^puppet(\d+)-archive/
-        puppet_version = Regexp.last_match(1)
-        repo_type = 'archive'
       when /^puppet(\d+)+$/
         puppet_version = Regexp.last_match(1)
-        repo_type = 'stable'
       else
         fatal("Unknown repo name \"#{repo_name}\" in #{deb_file_path}")
       end
@@ -148,7 +149,7 @@ class AptStageArtifacts
         deb_file_path: deb_file_path,
         puppet_version: puppet_version,
         codename: codename,
-        repo_type: repo_type
+        apt_component: @apt_component
       }
     end
 

data/lib/apt_stage_from_tarball.rb

@@ -13,11 +13,13 @@ class AptStageFromTarball
     @tarball_path = tarball_path
     @manifest_json_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
     @log_level = Logger::INFO
-    @repo_types_updated = {}
 
     @script_name = File.basename($PROGRAM_NAME)
     @version_string = "Version #{AptStageArtifacts::VERSION}"
 
+    # Keep a list of puppet versions that will require a freight-cache update
+    @cache_update_versions = []
+
     # External commands
     @apt_add_to_freight_library_command = 'apt-add-to-freight-library'
     @apt_update_freight_cache_command = 'apt-update-freight-cache'
@@ -77,7 +79,7 @@ class AptStageFromTarball
            #{@apt_add_to_freight_library_command}
            --verbose
            --puppet-version=#{artifact[:puppet_version]}
-           --repo-type=#{artifact[:repo_type]}
+           --component=#{artifact[:apt_component]}
            --codename=#{artifact[:codename]}
            #{File.join(Dir.pwd, artifact[:deb_file_path])}
         ].join(' ')
@@ -85,29 +87,25 @@ class AptStageFromTarball
         %x(#{add_to_freight_library_command})
         fatal "#{add_to_freight_library_command} failed." unless $CHILD_STATUS.success?
 
-        # Keep a hash, key by puppet_version, of repo_types so that we can
+        # Keep a hash, key by puppet_version, of apt_components so that we can
         # efficiently build out the freight caches later.
-        puppet_version = artifact[:puppet_version]
-        repo_type = artifact[:repo_type]
-        @repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
-        @repo_types_updated[puppet_version] |= [repo_type]
+        @cache_update_versions << artifact[:puppet_version]
       end
     end
+    @cache_update_versions = @cache_update_versions.uniq
   end
 
+  # Executes the 'freight-cache' command through a local wrapper to create an APT repo
   def update_freight_cache
-    @repo_types_updated.each do |puppet_version, repo_types|
-      repo_types.each do |repo_type|
-        update_freight_cache_command = %W[
-          #{@apt_update_freight_cache_command}
-          --verbose
-          --puppet-version=#{puppet_version}
-          --repo-type=#{repo_type}
+    @cache_update_versions.each do |puppet_version|
+      update_freight_cache_command = %W[
+        #{@apt_update_freight_cache_command}
+        --verbose
+        --puppet-version=#{puppet_version}
         ].join(' ')
 
-        %x(#{update_freight_cache_command})
-        fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
-      end
+      %x(#{update_freight_cache_command})
+      fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
     end
   end
 end

data/lib/apt_update_freight_cache.rb

@@ -20,9 +20,8 @@ require 'yaml'
 ## freight provides signing services, so this script needs to exist on a machine where GPG
 ## signing has been set up.
 ##
-## Example usages:
-## apt-update-freight-cache --puppet-version=7 --repo-type=stable
-## apt-update-freight-cache --puppet-version=7 --repo-type=nightly
+## Example usage:
+##   apt-update-freight-cache --puppet-version=7
 
 class AptUpdateFreightCache
   include Logging
@@ -31,13 +30,12 @@ class AptUpdateFreightCache
     Generates a freight cache from a freight library.
 
     Usage:
-        apt-update-freight-cache [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE
+        apt-update-freight-cache [--verbose] --puppet-version=VERSION
         apt-update-freight-cache --help
         apt-update-freight-cache --version
 
     Options:
     -p --puppet-version=VERSION    Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
-    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
 
     -v --verbose  Be verbose
     -h --help     Show this help
@@ -69,15 +67,11 @@ class AptUpdateFreightCache
     logger.info @version_string
     load_builder_data_yaml
     @puppet_version = @user_options['--puppet-version']
-    fatal "Invalid Puppet version '#{@puppet_version}'" unless
-      AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
-
-    @repo_type = @user_options['--repo-type']
-    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
-      fatal "Invalid repo type '#{@repo_type}'"
+    unless AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
+      fatal "Invalid Puppet version '#{@puppet_version}'"
     end
 
-    freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
+    freight_config_file = "/etc/freight.conf.d/puppet#{@puppet_version}.conf"
     fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
       File.readable?(freight_config_file)
 

data/lib/mixins/constants.rb

@@ -17,6 +17,13 @@ class AptStageArtifacts
   VALID_CODENAMES = DEBIAN_CODENAMES | UBUNTU_CODENAMES
 
   # Validation constraints
-  VALID_PUPPET_VERSIONS = %w[6 7 8]
-  VALID_REPO_TYPES = %w[archive nightly stable]
+  VALID_PUPPET_VERSIONS = %w[6 7 8 9]
+
+  # From freight-add:
+  # freight-add registers package with one or more manager/distro[/component] pairs (or triples).
+  # component is optional and for apt defaults to 'main'.
+
+  # These are valid APT components. We use 'stable' instead of 'main' established by convention
+  # from docker dists (see https://download.docker.com/linux/ubuntu/dists/groovy)
+  VALID_APT_COMPONENTS = %w[archive nightly stable]
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 class AptStageArtifacts
-  VERSION = '0.5.4'
+  VERSION = '0.8.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apt_stage_artifacts
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.8.0
 platform: ruby
 authors:
 - Puppet Release Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-06-07 00:00:00.000000000 Z
+date: 2021-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Stages .deb artifacts to a remote freight repository