apt_stage_artifacts 0.6.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2362e49d0e03e37288319844478f235c20cbeae7904b41f558fea40b827448f
-  data.tar.gz: 832385c791029b856dabcffc12e25e80548a4ce619bab804969a3a252b0a1fac
+  metadata.gz: e22e83a8811e9598f4dc9fd27b639c743451fdc06e0502092daceb5cea04a9f8
+  data.tar.gz: 9d19f393ac7f403a8dbdd7e56e6888e6392d556a8da28c9b5190eefc0c0a6098
 SHA512:
-  metadata.gz: e204fe791bd0d14a9d1d98f03e17fdcea4c8a13a4bc4a541b747ca110f3ea562b02614896d3691dff29b43ef8897539b0b2cf0359a4911265c3645214ba1846c
-  data.tar.gz: c57204613136f24ec94c03ce0cc7675347b79c4535f8fa2877b5de651cea3e705a68fd418dd88e6d75c6e97adad2c950d7a4c8d5539083ce0d7d30844ac16643
+  metadata.gz: ae8a11f11b2a4698b684c169d74943770945b9e1f04489c4eaf8a1e3bf28cdfb64d5fdd3629c5febb6eba5f84e4bc5a8e9dd9e3ad1cbab6f659b62eeeabf5a8b
+  data.tar.gz: dcc70849ba88e635c3fbf8d3022f74d0e136fab374f0f06d166208601479858b834765b6833265e7583638f6c0ae0245e8061a96d02ef48a76104896115df1c7

data/lib/apt_add_to_freight_library.rb

@@ -10,8 +10,9 @@ require 'tmpdir'
 ## This is a wrapper for https://github.com/freight-team/freight, which creates APT repos
 ## from deb files.
 ##
-## In our case, we maintain a set of debian repos, based on puppet version and a repo-type
-## 'nightly', 'archive' and 'stable'.
+## In our case, we maintain a set of debian repos, based on puppet version and a APT component
+## 'nightly' or 'stable' (we've also used 'archive' in the past; there's some remnants of it
+## left).
 ##
 ## freight provides signing services, so this script needs to exist on a machine where GPG
 ## signing has been set up.
@@ -19,14 +20,14 @@ require 'tmpdir'
 ## Typical usage would be for a remote process to call this script with a list of .deb
 ## URLS:
 ##
-## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=stretch
+## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=stretch
 ##   https://some-server.net/somepackage-a_0.19.2-1stretch_amd64.deb
 ##   https://some-server.net/somepackage-b_0.19.2-1stretch_amd64.deb
 ##
-## Alternately one coule scp a bunch of .deb files to the machine
+## Alternately one could scp a bunch of .deb files to the machine
 ## this is installed on then call this script with all the deb files as arguments. For example:
 ##
-## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=buster
+## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=buster
 ##   /tmp/xxtmpdir/somepackage-a_0.0.0-1buster_amd64.deb
 ##   /tmp/xxtmpdir/somepackage-b_1.2.3-5buster_amd64.deb
 ##     (etc.)
@@ -38,14 +39,14 @@ class AptAddToFreightLibrary
     Adds debian .deb files to a freight library.
 
     Usage:
-        apt-add-to-freight-library [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE --codename=DEBIAN_CODENAME <deb_file> ...
+        apt-add-to-freight-library [--verbose] --puppet-version=VERSION --component=COMPONENT --codename=DEBIAN_CODENAME <deb_file> ...
         apt-add-to-freight-library --help
         apt-add-to-freight-library --version
 
     Options:
+    -c --component=COMPONENT       APT component (Valid: #{AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')})
+    -n --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
     -p --puppet-version=VERSION    Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
-    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
-    -c --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
 
     -v --verbose  Be verbose
     -h --help     Show this help
@@ -79,9 +80,9 @@ class AptAddToFreightLibrary
     fatal "Invalid Puppet version '#{@puppet_version}'" unless
       AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
 
-    @repo_type = @user_options['--repo-type']
-    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
-      fatal "Invalid repo type '#{@repo_type}'"
+    @apt_component = @user_options['--component']
+    unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
+      fatal "Invalid APT component '#{@apt_component}'"
     end
 
     @codename = @user_options['--codename']
@@ -91,7 +92,7 @@ class AptAddToFreightLibrary
 
     freight_config_file = File.join(
       AptStageArtifacts::FREIGHT_CONFIG_DIRECTORY,
-      "puppet_#{@puppet_version}_#{@repo_type}.conf"
+      "puppet#{@puppet_version}.conf"
     )
     fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
       File.readable?(freight_config_file)
@@ -101,11 +102,14 @@ class AptAddToFreightLibrary
 
     Dir.mktmpdir do |temporary_directory|
       deb_artifacts.each do |artifact_path|
-        logger.info "Adding #{artifact_path} to puppet_#{@puppet_version}_#{@repo_type}"
+        logger.info "Adding #{artifact_path} to puppet#{@puppet_version}/#{@apt_component}"
         deb_artifact_path = artifact_path
         if deb_artifact_path.start_with?('https://', 'http://')
           download_data = URI.parse(deb_artifact_path).open
-          download_path = File.join(temporary_directory, download_data.base_uri.to_s.split('/').last)
+          download_path = File.join(
+            temporary_directory,
+            download_data.base_uri.to_s.split('/').last
+          )
           IO.copy_stream(download_data, download_path)
           deb_artifact_path = download_path
         end
@@ -113,7 +117,7 @@ class AptAddToFreightLibrary
         freight_add_command = %W[
          sudo --user=jenkins --set-home
          #{AptStageArtifacts::FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
-         #{deb_artifact_path} "apt/#{@codename}"
+         #{deb_artifact_path} "apt/#{@codename}/#{@apt_component}"
         ].join(' ')
 
         %x(#{freight_add_command})

data/lib/apt_stage_artifacts.rb

@@ -16,7 +16,7 @@ class AptStageArtifacts
     Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
 
     Usage:
-        apt-stage-artifacts [--repo-type=<repo-type>] [<top_directory>]
+        apt-stage-artifacts [--component=<apt_component>] [<top_directory>]
         apt-stage-artifacts --version
         apt-stage-artifacts --help
 
@@ -26,7 +26,7 @@ class AptStageArtifacts
     @script_name = File.basename($PROGRAM_NAME)
     @log_level = Logger::INFO
     @user_options = parse_options(argv)
-    @repo_type = @user_options['--repo-type'] || 'stable'
+    @apt_component = @user_options['--component'] || 'stable'
     @version_string = "Version #{AptStageArtifacts::VERSION}"
 
     @artifacts_tarball_name = 'artifacts.tgz'
@@ -47,12 +47,12 @@ class AptStageArtifacts
   def run
     logger.info @version_string
     exit 0 if @user_options['--version']
-    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
-      fatal "Unknown repo_type: \"#{@repo_type}\". Valid repo types are: " +
-            AptStageArtifacts::VALID_REPO_TYPES.join(', ')
+    unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
+      fatal "Unknown APT component: \"#{@apt_component}\". Valid APT components are: " +
+            AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')
     end
     load_builder_data_yaml
-    logger.info "Artifacts will be shipped to the '#{@repo_type}' repo on '#{@staging_server}'."
+    logger.info "Artifacts will be shipped to '#{@staging_server}'."
     collect_artifacts
     create_remote_staging_directory
     generate_manifest
@@ -86,13 +86,15 @@ class AptStageArtifacts
       exit 1
     end
 
-    logger.info "Searching for .deb packages in #{@top_directory}"
+    logger.info "Searching for puppetN/*.deb packages in #{@top_directory}"
     Dir.chdir(@top_directory) do
-      @debian_artifacts = Dir['**/*.deb']
+      @debian_artifacts = Dir['*/puppet*/*.deb']
     end
     return unless @debian_artifacts.empty?
 
-    logger.error 'No .deb files found. Nothing to do.'
+    logger.warn "This project does not contain artifacts suitable for shipping to the " \
+                "APT server. Expected repo layout is 'pkg/<debian codename>/puppet<N>/*.deb'. " \
+                "Skipping."
     exit 0
   end
 
@@ -125,19 +127,19 @@ class AptStageArtifacts
   #   codename: debian codename for this artifact, needed by freight
   #   puppet_version: part of the destination apt repo. We maintain apt repos by puppet major
   #     version
-  #   repo_type: either 'stable' or 'nightly' typically. 'archive' is supported but unused.
+  #   apt_component: either 'stable' or 'nightly'. 'archive' is supported but unused.
   def generate_manifest
     manifest = []
 
     @debian_artifacts.each do |deb_file_path|
-      artifact_components = deb_file_path.split('/')
+      artifact_items = deb_file_path.split('/')
 
-      repo_name = artifact_components[-2]
+      repo_name = artifact_items[-2]
       case repo_name
       when /^puppet(\d+)+$/
         puppet_version = Regexp.last_match(1)
       else
-        fatal("Unknown repo name \"#{repo_name}\" in #{deb_file_path}")
+        fatal "Puppet version not found in \"#{deb_file_path}\"."
       end
 
       codename = deb_file_path.split('/')[-3]
@@ -149,10 +151,14 @@ class AptStageArtifacts
         deb_file_path: deb_file_path,
         puppet_version: puppet_version,
         codename: codename,
-        repo_type: @repo_type
+        apt_component: @apt_component
       }
     end
 
+    if manifest.empty?
+      fatal 'No manifest was generated from the artifact list. This is unexpected.'
+    end
+
     manifest_path = File.join(@top_directory, AptStageArtifacts::MANIFEST_JSON_FILENAME)
     File.open(manifest_path, 'w') do |f|
       f.write(manifest.to_json)
@@ -190,7 +196,8 @@ class AptStageArtifacts
 
   # Invoke the remote staging of the sent artifacts
   def invoke_remote_staging
-    %x(ssh #{@staging_server} #{@remote_staging_command} #{@remote_tarball_path})
+    # Do this in a login shell so we get rvm PATH goodness
+    %x(ssh #{@staging_server} '/bin/bash -l -c "#{@remote_staging_command} #{@remote_tarball_path}"')
     return if $CHILD_STATUS.success?
 
     fatal "'ssh #{@staging_server} #{@remote_staging_command} #{@remote_tarball_path}' failed."

data/lib/apt_stage_from_tarball.rb

@@ -13,11 +13,13 @@ class AptStageFromTarball
     @tarball_path = tarball_path
     @manifest_json_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
     @log_level = Logger::INFO
-    @repo_types_updated = {}
 
     @script_name = File.basename($PROGRAM_NAME)
     @version_string = "Version #{AptStageArtifacts::VERSION}"
 
+    # Keep a list of puppet versions that will require a freight-cache update
+    @cache_update_versions = []
+
     # External commands
     @apt_add_to_freight_library_command = 'apt-add-to-freight-library'
     @apt_update_freight_cache_command = 'apt-update-freight-cache'
@@ -77,7 +79,7 @@ class AptStageFromTarball
            #{@apt_add_to_freight_library_command}
            --verbose
            --puppet-version=#{artifact[:puppet_version]}
-           --repo-type=#{artifact[:repo_type]}
+           --component=#{artifact[:apt_component]}
            --codename=#{artifact[:codename]}
            #{File.join(Dir.pwd, artifact[:deb_file_path])}
         ].join(' ')
@@ -85,29 +87,25 @@ class AptStageFromTarball
         %x(#{add_to_freight_library_command})
         fatal "#{add_to_freight_library_command} failed." unless $CHILD_STATUS.success?
 
-        # Keep a hash, key by puppet_version, of repo_types so that we can
+        # Keep a hash, key by puppet_version, of apt_components so that we can
         # efficiently build out the freight caches later.
-        puppet_version = artifact[:puppet_version]
-        repo_type = artifact[:repo_type]
-        @repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
-        @repo_types_updated[puppet_version] |= [repo_type]
+        @cache_update_versions << artifact[:puppet_version]
       end
     end
+    @cache_update_versions = @cache_update_versions.uniq
   end
 
+  # Executes the 'freight-cache' command through a local wrapper to create an APT repo
   def update_freight_cache
-    @repo_types_updated.each do |puppet_version, repo_types|
-      repo_types.each do |repo_type|
-        update_freight_cache_command = %W[
-          #{@apt_update_freight_cache_command}
-          --verbose
-          --puppet-version=#{puppet_version}
-          --repo-type=#{repo_type}
+    @cache_update_versions.each do |puppet_version|
+      update_freight_cache_command = %W[
+        #{@apt_update_freight_cache_command}
+        --verbose
+        --puppet-version=#{puppet_version}
         ].join(' ')
 
-        %x(#{update_freight_cache_command})
-        fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
-      end
+      %x(#{update_freight_cache_command})
+      fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
     end
   end
 end

data/lib/apt_update_freight_cache.rb

@@ -20,9 +20,8 @@ require 'yaml'
 ## freight provides signing services, so this script needs to exist on a machine where GPG
 ## signing has been set up.
 ##
-## Example usages:
-## apt-update-freight-cache --puppet-version=7 --repo-type=stable
-## apt-update-freight-cache --puppet-version=7 --repo-type=nightly
+## Example usage:
+##   apt-update-freight-cache --puppet-version=7
 
 class AptUpdateFreightCache
   include Logging
@@ -31,13 +30,12 @@ class AptUpdateFreightCache
     Generates a freight cache from a freight library.
 
     Usage:
-        apt-update-freight-cache [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE
+        apt-update-freight-cache [--verbose] --puppet-version=VERSION
         apt-update-freight-cache --help
         apt-update-freight-cache --version
 
     Options:
     -p --puppet-version=VERSION    Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
-    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
 
     -v --verbose  Be verbose
     -h --help     Show this help
@@ -69,15 +67,11 @@ class AptUpdateFreightCache
     logger.info @version_string
     load_builder_data_yaml
     @puppet_version = @user_options['--puppet-version']
-    fatal "Invalid Puppet version '#{@puppet_version}'" unless
-      AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
-
-    @repo_type = @user_options['--repo-type']
-    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
-      fatal "Invalid repo type '#{@repo_type}'"
+    unless AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
+      fatal "Invalid Puppet version '#{@puppet_version}'"
     end
 
-    freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
+    freight_config_file = "/etc/freight.conf.d/puppet#{@puppet_version}.conf"
     fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
       File.readable?(freight_config_file)
 

data/lib/mixins/constants.rb

@@ -17,6 +17,13 @@ class AptStageArtifacts
   VALID_CODENAMES = DEBIAN_CODENAMES | UBUNTU_CODENAMES
 
   # Validation constraints
-  VALID_PUPPET_VERSIONS = %w[6 7 8]
-  VALID_REPO_TYPES = %w[archive nightly stable]
+  VALID_PUPPET_VERSIONS = %w[6 7 8 9]
+
+  # From freight-add:
+  # freight-add registers package with one or more manager/distro[/component] pairs (or triples).
+  # component is optional and for apt defaults to 'stable'.
+
+  # These are valid APT components. We use 'stable' instead of 'main' established by convention
+  # from docker dists (see https://download.docker.com/linux/ubuntu/dists/groovy)
+  VALID_APT_COMPONENTS = %w[archive nightly stable]
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 class AptStageArtifacts
-  VERSION = '0.6.1'
+  VERSION = '0.9.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apt_stage_artifacts
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Puppet Release Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-06-08 00:00:00.000000000 Z
+date: 2021-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.28
 signing_key: 
 specification_version: 4
 summary: Stages .deb artifacts to a remote freight repository