apt_stage_artifacts 0.4.0 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24c10c67846c819b0174ac0b94f2a4b7836e73589969335c1fb4b6a2779e59bc
-  data.tar.gz: 47be9199cdabcc1bb1f242f6cd4e1aa8bea968d3ed83bc215e17c67ebe8d3f2c
+  metadata.gz: 93eab470ef8684eb79b47dafcc7f08b345d232542d96666e853ea8ac7e553d3e
+  data.tar.gz: bfee3c314fdd1eecd75253d7ad3c71e7debb8d4c396a43a379541e44c3347b5b
 SHA512:
-  metadata.gz: 421d5a2a39379edddffa17bba6925c328d8f2a51d3af858743b771729b337ed74a9d4ce8c48a51798a6371f1b477fb16ab38833966339fe2ac6978650e0a8d4a
-  data.tar.gz: 585ec2d56f0959a19cb83fad46d5934265c1eaa1dce02b89155ada6a6d507dd3ab064e8a7447cffc5be3ea5384b4e9354331fdab1faf146a28a31e85c6fedb69
+  metadata.gz: b9ce4b50d62011ae52277ca427de4c6ecbbba6116687fe240097d70327c99b3b0b870c2deb63fea2a3630719f2d89d0f499f69416b7e81c79ac65d02d60e54c0
+  data.tar.gz: d71e6d727e0cc5a8ffa56eb7360174eed2de96af56e1877a9235a1e4f02ca3da7c2ec8d57b51f83ee1e197fabafd77fc101295fbb169b5086b187619bc79aa51

data/apt_stage_artifacts.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = File.join(spec.homepage, 'CHANGELOG.md')
+  spec.metadata['changelog_uri'] = File.join(spec.homepage, 'blob', 'main', 'CHANGELOG.md')
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

data/lib/apt_add_to_freight_library.rb

@@ -1,4 +1,5 @@
 require 'version'
+require 'mixins/constants'
 require 'mixins/logging'
 
 require 'English'
@@ -33,11 +34,6 @@ require 'tmpdir'
 class AptAddToFreightLibrary
   include Logging
 
-  FREIGHT_COMMAND = '/usr/bin/freight'
-  VALID_PUPPET_VERSIONS = %w[6 7 8]
-  VALID_REPO_TYPES = %w[archive nightly stable]
-  VALID_CODENAMES = %w[bionic jessie stretch buster bullseye bookwork focal xenial]
-
   DOCUMENTATION = <<~DOCOPT
     Adds debian .deb files to a freight library.
 
@@ -47,9 +43,9 @@ class AptAddToFreightLibrary
         apt-add-to-freight-library --version
 
     Options:
-    -p --puppet-version=VERSION    Puppet version (Valid: #{VALID_PUPPET_VERSIONS.join(', ')})
-    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{VALID_REPO_TYPES.join(', ')})
-    -c --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{VALID_CODENAMES.join(', ')})
+    -p --puppet-version=VERSION    Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
+    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
+    -c --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
 
     -v --verbose  Be verbose
     -h --help     Show this help
@@ -65,7 +61,7 @@ class AptAddToFreightLibrary
     @script_name = File.basename($PROGRAM_NAME)
     @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
     if @user_options['--version'] # rubocop:disable Style/GuardClause
-      puts @version_string
+      warn @version_string
       exit 0
     end
   end
@@ -73,7 +69,7 @@ class AptAddToFreightLibrary
   def parse_options(argv = nil)
     Docopt.docopt(DOCUMENTATION, { argv: argv })
   rescue Docopt::Exit => e
-    puts e.message
+    warn e.message
     exit 1
   end
 
@@ -81,15 +77,22 @@ class AptAddToFreightLibrary
     logger.info @version_string
     @puppet_version = @user_options['--puppet-version']
     fatal "Invalid Puppet version '#{@puppet_version}'" unless
-      VALID_PUPPET_VERSIONS.include? @puppet_version
+      AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
 
     @repo_type = @user_options['--repo-type']
-    fatal "Invalid repo type '#{@repo_type}'" unless VALID_REPO_TYPES.include? @repo_type
+    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
+      fatal "Invalid repo type '#{@repo_type}'"
+    end
 
     @codename = @user_options['--codename']
-    fatal "Invalid codename '#{@codename}'" unless VALID_CODENAMES.include? @codename
+    unless AptStageArtifacts::VALID_CODENAMES.include? @codename
+      fatal "Invalid codename '#{@codename}'"
+    end
 
-    freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
+    freight_config_file = File.join(
+      AptStageArtifacts::FREIGHT_CONFIG_DIRECTORY,
+      "puppet_#{@puppet_version}_#{@repo_type}.conf"
+    )
     fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
       File.readable?(freight_config_file)
 
@@ -109,7 +112,7 @@ class AptAddToFreightLibrary
 
         freight_add_command = %W[
          sudo --user=jenkins --set-home
-         #{FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
+         #{AptStageArtifacts::FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
          #{deb_artifact_path} "apt/#{@codename}"
         ].join(' ')
 

data/lib/apt_stage_artifacts.rb

@@ -1,11 +1,12 @@
 require 'version'
 require 'mixins/logging'
+require 'mixins/constants'
 
 require 'English'
 require 'docopt'
 require 'fileutils'
+require 'json'
 require 'open-uri'
-require 'tempfile'
 require 'yaml'
 
 class AptStageArtifacts
@@ -15,35 +16,34 @@ class AptStageArtifacts
     Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
 
     Usage:
-        apt-stage-artifacts
-        apt-stage-artifacts <top_directory>
+        apt-stage-artifacts [--version]
+        apt-stage-artifacts [--version] <top_directory>
   DOCOPT
 
   def initialize(argv)
     @script_name = File.basename($PROGRAM_NAME)
     @log_level = Logger::INFO
     @user_options = parse_options(argv)
-    @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
+    @version_string = "Version #{AptStageArtifacts::VERSION}"
 
     @artifacts_tarball_name = 'artifacts.tgz'
     @local_tarball_path = File.join(Dir.pwd, @artifacts_tarball_name)
 
     @builder_data_yaml = 'https://raw.githubusercontent.com/puppetlabs/build-data/release/builder_data.yaml'
 
-    @manifest_file_name = 'apt_stage.manifest'
     @remote_staging_command = 'apt-stage-from-tarball'
   end
 
   def parse_options(argv = nil)
     Docopt.docopt(DOCUMENTATION, { argv: argv })
   rescue Docopt::Exit => e
-    puts e.message
+    warn e.message
     exit 1
   end
 
   def run
     logger.info @version_string
-    parse_repo_type
+    exit 0 if @user_options['--version']
     load_builder_data_yaml
     collect_artifacts
     create_remote_staging_directory
@@ -66,6 +66,9 @@ class AptStageArtifacts
   def collect_artifacts
     if !@user_options['<top_directory>'].to_s.empty?
       @top_directory = @user_options['<top_directory>']
+      if File.directory?(File.join(@top_directory, 'deb'))
+        @top_directory = File.join(@top_directory, 'deb')
+      end
     elsif File.directory?('./output/deb')
       @top_directory = './output/deb'
     elsif File.directory?('./pkg/deb')
@@ -77,7 +80,7 @@ class AptStageArtifacts
 
     logger.info "Searching for .deb packages in #{@top_directory}"
     Dir.chdir(@top_directory) do
-      @debian_artifacts = Dir['*/*.deb']
+      @debian_artifacts = Dir['**/*.deb']
     end
     return unless @debian_artifacts.empty?
 
@@ -109,36 +112,63 @@ class AptStageArtifacts
     logger.info "Artifacts will be staged on '#{@staging_server}'."
   end
 
-  # A manifest is a staging to-do list. For each '.deb' file create a colon-delimited line
+  # A manifest is a staging to-do list represented in JSON as an array of hashes
   # containing:
-  # <.deb filename>:<puppet-version>:<repo-type>:<debian-codename>
+  #   deb_file_path:
+  #   codename:
+  #   puppet_version:
+  #   repo_type:
   def generate_manifest
-    manifest_temporary_file = Tempfile.new('apt_stage_artifacts')
+    manifest = []
 
     @debian_artifacts.each do |deb_file_path|
-      # Ug. We only know the codename because it happens to be the first element
-      # of the file path.
-      debian_codename = deb_file_path.split('/').first
-      manifest_temporary_file.printf(
-        "%s:%s:%s:%s\n",
-        deb_file_path, @puppet_version, @repo_type, debian_codename
-      )
+      artifact_components = deb_file_path.split('/')
+
+      repo_name = artifact_components[-2]
+      case repo_name
+      when /^puppet(\d+)-nightly$/
+        puppet_version = Regexp.last_match(1)
+        repo_type = 'nightly'
+      when /^puppet(\d+)-archive/
+        puppet_version = Regexp.last_match(1)
+        repo_type = 'archive'
+      when /^puppet(\d+)+$/
+        puppet_version = Regexp.last_match(1)
+        repo_type = 'stable'
+      else
+        fatal("Unknown repo name \"#{repo_name}\" in #{deb_file_path}")
+      end
+
+      codename = deb_file_path.split('/')[-3]
+      unless AptStageArtifacts::VALID_CODENAMES.include?(codename)
+        fatal("#{deb_file_path} contains an unknown codename: #{codename}")
+      end
+
+      manifest << {
+        deb_file_path: deb_file_path,
+        puppet_version: puppet_version,
+        codename: codename,
+        repo_type: repo_type
+      }
     end
 
-    manifest_temporary_file.close
-    manifest_path = File.join(@top_directory, @manifest_file_name)
-    FileUtils.mv(manifest_temporary_file.path, manifest_path)
+    manifest_path = File.join(@top_directory, AptStageArtifacts::MANIFEST_JSON_FILENAME)
+    File.open(manifest_path, 'w') do |f|
+      f.write(manifest.to_json)
+    end
     logger.info("Staging manifest written to '#{manifest_path}'")
   end
 
   # Create a tarball that contains the staging manifest and the associated deb files.
   def create_tarball
+    manifest_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
+
     tarball_create_command = %W[
       tar --create --gzip --file #{@local_tarball_path} --directory=#{@top_directory}
     ].join(' ')
-    %x(#{tarball_create_command} #{@manifest_file_name} #{@debian_artifacts.join(' ')})
+    %x(#{tarball_create_command} #{manifest_file_name} #{@debian_artifacts.join(' ')})
     unless $CHILD_STATUS.success?
-      fatal "#{tarball_create_command} #{@manifest_file_name} #{@debian_artifacts.join(' ')} failed."
+      fatal "#{tarball_create_command} #{manifest_file_name} #{@debian_artifacts.join(' ')} failed."
     end
 
     logger.info("'#{@local_tarball_path}' created.")
@@ -148,27 +178,6 @@ class AptStageArtifacts
     File.delete @local_tarball_path
   end
 
-  # The target apt repository is described in the REPO_NAME environment variable.
-  # A better solution is called for, perhaps as part of vanagon config or as a commandline
-  # parameter.
-  def parse_repo_type
-    case ENV['REPO_NAME']
-    when /\A\z/, nil
-      fatal "The environment variable 'REPO_NAME', containing the target apt repo, is unset.\n" \
-            "It should be set to something like 'puppet7' or 'puppet7-nightly'."
-    when /\Apuppet(\d+)\z/
-      @puppet_version = Regexp.last_match(1)
-      @repo_type = 'stable'
-    when /\Apuppet(\d+)-nightly\z/
-      @puppet_version = Regexp.last_match(1)
-      @repo_type = 'nightly'
-    else
-      fatal "The environment variable 'REPO_NAME' is set to '#{ENV['REPO_NAME']}'\n" \
-            "This is not a recognized setting.\n" \
-            "It should be set to something like 'puppet7' or 'puppet7-nightly'."
-    end
-  end
-
   # Send the tarball over to the staging server for further processing with freight.
   def ship_tarball_to_staging_server
     logger.info "Sending '#{@local_tarball_path}' to '#{@staging_server}:#{@remote_tarball_path}'"

data/lib/apt_stage_from_tarball.rb

@@ -1,7 +1,9 @@
 require 'version'
+require 'mixins/constants'
 require 'mixins/logging'
 
 require 'English'
+require 'json'
 require 'mkmf'
 
 class AptStageFromTarball
@@ -9,12 +11,12 @@ class AptStageFromTarball
 
   def initialize(tarball_path = nil)
     @tarball_path = tarball_path
-    @manifest_file_name = 'apt_stage.manifest'
+    @manifest_json_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
     @log_level = Logger::INFO
     @repo_types_updated = {}
 
     @script_name = File.basename($PROGRAM_NAME)
-    @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
+    @version_string = "Version #{AptStageArtifacts::VERSION}"
 
     # External commands
     @apt_add_to_freight_library_command = 'apt-add-to-freight-library'
@@ -57,7 +59,7 @@ class AptStageFromTarball
       fatal "#{tarball_extract_command} failed."
     end
 
-    @manifest_path = File.join(@tarball_directory, @manifest_file_name)
+    @manifest_path = File.join(@tarball_directory, @manifest_json_file_name)
 
     fatal "Staging manifest file '#{@manifest_path}' is missing or unreadable" unless
       File.readable?(@manifest_path)
@@ -69,16 +71,15 @@ class AptStageFromTarball
   # library.
   def add_artifacts_to_freight_library
     Dir.chdir(@tarball_directory) do
-      File.readlines(@manifest_path).each do |line|
-        line.chomp!
-        debian_filename, puppet_version, repo_type, codename = line.split(':')
+      manifest_file = File.read(@manifest_path)
+      JSON.parse(manifest_file, symbolize_names: true).each do |artifact|
         add_to_freight_library_command = %W[
            #{@apt_add_to_freight_library_command}
            --verbose
-           --puppet-version=#{puppet_version}
-           --repo-type=#{repo_type}
-           --codename=#{codename}
-           #{File.join(Dir.pwd, debian_filename)}
+           --puppet-version=#{artifact[:puppet_version]}
+           --repo-type=#{artifact[:repo_type]}
+           --codename=#{artifact[:codename]}
+           #{File.join(Dir.pwd, artifact[:deb_file_path])}
         ].join(' ')
 
         %x(#{add_to_freight_library_command})
@@ -86,6 +87,8 @@ class AptStageFromTarball
 
         # Keep a hash, key by puppet_version, of repo_types so that we can
         # efficiently build out the freight caches later.
+        puppet_version = artifact[:puppet_version]
+        repo_type = artifact[:repo_type]
         @repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
         @repo_types_updated[puppet_version] |= [repo_type]
       end

data/lib/apt_update_freight_cache.rb

@@ -1,4 +1,5 @@
 require 'version'
+require 'mixins/constants'
 require 'mixins/logging'
 
 require 'English'
@@ -26,10 +27,6 @@ require 'yaml'
 class AptUpdateFreightCache
   include Logging
 
-  FREIGHT_COMMAND = '/usr/bin/freight'
-  VALID_PUPPET_VERSIONS = %w[6 7 8]
-  VALID_REPO_TYPES = %w[archive nightly stable]
-
   DOCUMENTATION = <<~DOCOPT
     Generates a freight cache from a freight library.
 
@@ -39,8 +36,8 @@ class AptUpdateFreightCache
         apt-update-freight-cache --version
 
     Options:
-    -p --puppet-version=VERSION    Puppet version (Valid: #{VALID_PUPPET_VERSIONS.join(', ')})
-    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{VALID_REPO_TYPES.join(', ')})
+    -p --puppet-version=VERSION    Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
+    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
 
     -v --verbose  Be verbose
     -h --help     Show this help
@@ -56,7 +53,7 @@ class AptUpdateFreightCache
     @script_name = File.basename($PROGRAM_NAME)
     @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
     if @user_options['--version'] # rubocop:disable Style/GuardClause
-      puts @version_string
+      warn @version_string
       exit 0
     end
   end
@@ -73,10 +70,12 @@ class AptUpdateFreightCache
     load_builder_data_yaml
     @puppet_version = @user_options['--puppet-version']
     fatal "Invalid Puppet version '#{@puppet_version}'" unless
-      VALID_PUPPET_VERSIONS.include? @puppet_version
+      AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
 
     @repo_type = @user_options['--repo-type']
-    fatal "Invalid repo type '#{@repo_type}'" unless VALID_REPO_TYPES.include? @repo_type
+    unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
+      fatal "Invalid repo type '#{@repo_type}'"
+    end
 
     freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
     fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless

data/lib/mixins/constants.rb

@@ -0,0 +1,22 @@
+##
+## Some constants that are used throughout the code
+##
+
+class AptStageArtifacts
+  # Details about freight itself
+  FREIGHT_COMMAND = '/usr/bin/freight'
+  FREIGHT_CONFIG_DIRECTORY = '/etc/freight.conf.d'
+
+  # Name of the manifest.json file
+  MANIFEST_JSON_FILENAME = 'apt_stage_manifest.json'
+
+  # Debian/Ubuntu codenames that we care about.
+  # Some codenames are skipped because we don't/haven't shipped for them.
+  DEBIAN_CODENAMES = %w[jessie stretch buster bullseye bookworm trixie]
+  UBUNTU_CODENAMES = %w[xenial bionic focal groovy]
+  VALID_CODENAMES = DEBIAN_CODENAMES | UBUNTU_CODENAMES
+
+  # Validation constraints
+  VALID_PUPPET_VERSIONS = %w[6 7 8]
+  VALID_REPO_TYPES = %w[archive nightly stable]
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 class AptStageArtifacts
-  VERSION = '0.4.0'
+  VERSION = '0.5.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apt_stage_artifacts
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.4
 platform: ruby
 authors:
 - Puppet Release Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-28 00:00:00.000000000 Z
+date: 2021-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -154,6 +154,7 @@ files:
 - lib/apt_stage_artifacts.rb
 - lib/apt_stage_from_tarball.rb
 - lib/apt_update_freight_cache.rb
+- lib/mixins/constants.rb
 - lib/mixins/logging.rb
 - lib/version.rb
 homepage: https://github.com/puppetlabs/apt_stage_artifacts
@@ -162,7 +163,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/puppetlabs/apt_stage_artifacts
   source_code_uri: https://github.com/puppetlabs/apt_stage_artifacts
-  changelog_uri: https://github.com/puppetlabs/apt_stage_artifacts/CHANGELOG.md
+  changelog_uri: https://github.com/puppetlabs/apt_stage_artifacts/blob/main/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths: