apt_stage_artifacts 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 24c10c67846c819b0174ac0b94f2a4b7836e73589969335c1fb4b6a2779e59bc
+  data.tar.gz: 47be9199cdabcc1bb1f242f6cd4e1aa8bea968d3ed83bc215e17c67ebe8d3f2c
+SHA512:
+  metadata.gz: 421d5a2a39379edddffa17bba6925c328d8f2a51d3af858743b771729b337ed74a9d4ce8c48a51798a6371f1b477fb16ab38833966339fe2ac6978650e0a8d4a
+  data.tar.gz: 585ec2d56f0959a19cb83fad46d5934265c1eaa1dce02b89155ada6a6d507dd3ab064e8a7447cffc5be3ea5384b4e9354331fdab1faf146a28a31e85c6fedb69

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,152 @@
+---
+
+require: rubocop-rake
+
+AllCops:
+  Exclude:
+  - Rakefile
+  - Gemfile
+  - bin/*
+  - spec/**/*.rb
+  - templates/*
+
+Gemspec/DateAssignment:
+  Enabled: true
+
+Layout/FirstArrayElementIndentation:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  Enabled: false
+
+Layout/SpaceBeforeBrackets:
+  Enabled: true
+
+Lint/AmbiguousAssignment:
+  Enabled: true
+
+Lint/DeprecatedConstants:
+  Enabled: true
+
+Lint/DuplicateBranch:
+  Enabled: true
+
+Lint/DuplicateRegexpCharacterClassElement:
+  Enabled: true
+
+Lint/EmptyBlock:
+  Enabled: true
+
+Lint/EmptyClass:
+  Enabled: true
+
+Lint/LambdaWithoutLiteralBlock:
+  Enabled: true
+
+Lint/NoReturnInBeginEndBlocks:
+  Enabled: true
+
+Lint/NumberedParameterAssignment:
+  Enabled: true
+
+Lint/OrAssignmentToConstant:
+  Enabled: true
+
+Lint/RedundantDirGlobSort:
+  Enabled: true
+
+Lint/SymbolConversion:
+  Enabled: true
+
+Lint/ToEnumArguments:
+  Enabled: true
+
+Lint/TripleQuotes:
+  Enabled: true
+
+Lint/UnexpectedBlockArity:
+  Enabled: true
+
+Lint/UnmodifiedReduceAccumulator:
+  Enabled: true
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Style/ArgumentsForwarding:
+  Enabled: true
+
+Style/CollectionCompact:
+  Enabled: true
+
+Style/DocumentDynamicEvalDefinition:
+  Enabled: true
+
+Style/Documentation:
+  Enabled: false
+
+Style/EachWithObject:
+  Enabled: false
+
+Style/EndlessMethod:
+  Enabled: true
+
+Style/FormatString:
+  Enabled: false
+
+Style/FormatStringToken:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/HashConversion:
+  Enabled: true
+
+Style/HashExcept:
+  Enabled: true
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/IfWithBooleanLiteralBranches:
+  Enabled: true
+
+Style/MutableConstant:
+  Enabled: false
+
+Style/NegatedIfElseCondition:
+  Enabled: true
+
+Style/NilLambda:
+  Enabled: true
+
+Style/RedundantArgument:
+  Enabled: true
+
+Style/RedundantReturn:
+  Enabled: false
+
+Style/StringChars:
+  Enabled: true
+
+Style/SwapValues:
+  Enabled: true
+
+Style/CommandLiteral:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.4.0] - 2021-04-28
+### Added
+- Initial release
+

data/CODEOWNERS

@@ -0,0 +1,3 @@
+# This repo is owned by Puppet Release Engineering
+
+* @puppetlabs/release-engineering

data/DEVELOPMENT.md

@@ -0,0 +1,18 @@
+# Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Local Build
+
+To create a local build of the gem: `bundle exec rake build`
+
+## Local Install
+
+To install this gem onto your local machine: `bundle exec rake install`.
+
+## New Release
+
+To release a new version:
+
+- update the version number in `lib/version.rb`
+- run `bundle exec rake release` This will create a git tag for the version, push git commits and tags, and push the `.gem` file to [Puppet Artifactory Rubygems Repo ](https://artifactory.delivery.puppetlabs.net/artifactory/webapp/#/artifacts/browse/tree/General/rubygems).

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://artifactory.delivery.puppetlabs.net/artifactory/api/gems/rubygems'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2021 Puppet, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,25 @@
+# apt_stage_artifacts
+
+apt_stage_artifacts is a Ruby gem for Puppet Release Engineering to deliver locally built .deb packages to our freight-based apt repository staging.
+
+It only handles internal staging and not the final push to outside customers.
+
+The gem provides command-line programs for doing the staging.
+
+## Usage
+
+### apt-stage-artifacts
+
+The main user script, `apt-stage-artifacts` expects the current directory to be a vanagon-style build where debian `.deb` files have been built into `output/deb`.
+
+After consulting the [build-data](https://github.com/puppetlabs/build-data/blob/release/builder_data.yaml) repo for the name of the staging machine, it packs up the debian artifacts into a tarball and copies it there via scp.
+
+It then invokes the remaining scripts on the staging machine to do the rest of the staging:
+
+### apt-stage-from-tarball
+
+`apt-stage-from-tarball` runs in the staging server. It unpacks the tarball created by `apt-stage-artifacts` and calls `apt-add-to-freight-library` on each of them. This adds each `.deb` file to the correct freight library (puppet6, puppet7, puppet7-nightly, etc.)
+
+Once all .deb files have been added, `apt-stage-from-tarball` calls `apt-update-freight-cache`, which instructs freight to generate the actual APT repo (which freight calls a "cache"), including GPG signing of the resulting cache.
+
+When completed, the resulting "caches" are signed, functional APT repos which can sync'd to the outside world.

data/Rakefile

@@ -0,0 +1,32 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'yard'
+
+RSpec::Core::RakeTask.new(:spec)
+YARD::Rake::YardocTask.new
+
+task :build => :yard
+task :default => :spec
+
+desc 'Run rubocop'
+task :rubocop do
+  sh 'rubocop || true'
+end
+
+task :clean do
+  sh 'rm -f pkg/*.gem'
+end
+
+task :devtest => [:clean, :build, :install] do
+  test_host = ENV['APT_STAGE_TEST_HOST'].to_s
+  if test_host.empty?
+    puts 'Error: the "APT_STAGE_TEST_HOST" environment variable is unset.'
+    puts 'Set it to the hostname where the test freight environment exists.'
+    exit 1
+  end
+
+  sh "ssh #{test_host} rm -f ~/apt_stage_artifacts\*.gem"
+  sh "scp pkg/apt_stage_artifacts*.gem #{test_host}:~"
+  sh "ssh #{test_host} gem uninstall apt_stage_artifacts"
+  sh "ssh #{test_host} gem install apt_stage_artifacts\*.gem"
+end

data/apt_stage_artifacts.gemspec

@@ -0,0 +1,37 @@
+require_relative 'lib/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'apt_stage_artifacts'
+  spec.version       = AptStageArtifacts::VERSION
+  spec.authors       = ['Puppet Release Engineering']
+  spec.email         = ['release@puppet.com']
+
+  spec.summary       = 'Stages .deb artifacts to a remote freight repository'
+  spec.homepage      = 'https://github.com/puppetlabs/apt_stage_artifacts'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
+
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = File.join(spec.homepage, 'CHANGELOG.md')
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'pry', '~> 0.14'
+  spec.add_development_dependency 'pry-byebug', '~> 3.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop-rake'
+  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'yard', '~> 0.9'
+
+  spec.add_runtime_dependency 'docopt'
+end

data/bin/console

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'apt_add_to_freight_library'
+require 'apt_stage_artifacts'
+require 'apt_stage_from_tarball'
+require 'apt_update_freight_cache'
+
+require 'irb'
+
+puts %{Mininal startup options. Choose one:
+  irb> AptAddToFreightLibrary.new(Array.new)
+  irb> AptStageFromTarball.new(Array.new)
+  irb> AptUpdateFreightCache.new(Array.new)
+  irb> AptStageArtifacts.new(Array.new)
+}
+
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/apt-add-to-freight-library

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'apt_add_to_freight_library'
+
+AptAddToFreightLibrary.new(ARGV).run

data/exe/apt-stage-artifacts

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'apt_stage_artifacts'
+
+AptStageArtifacts.new(ARGV).run

data/exe/apt-stage-from-tarball

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'apt_stage_from_tarball'
+
+AptStageFromTarball.new(ARGV.first).run

data/exe/apt-update-freight-cache

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'apt_update_freight_cache'
+
+AptUpdateFreightCache.new(ARGV).run

data/lib/apt_add_to_freight_library.rb

@@ -0,0 +1,121 @@
+require 'version'
+require 'mixins/logging'
+
+require 'English'
+require 'docopt'
+require 'open-uri'
+require 'tmpdir'
+
+## This is a wrapper for https://github.com/freight-team/freight, which creates APT repos
+## from deb files.
+##
+## In our case, we maintain a set of debian repos, based on puppet version and a repo-type
+## 'nightly', 'archive' and 'stable'.
+##
+## freight provides signing services, so this script needs to exist on a machine where GPG
+## signing has been set up.
+##
+## Typical usage would be for a remote process to call this script with a list of .deb
+## URLS:
+##
+## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=stretch
+##   https://some-server.net/somepackage-a_0.19.2-1stretch_amd64.deb
+##   https://some-server.net/somepackage-b_0.19.2-1stretch_amd64.deb
+##
+## Alternately one coule scp a bunch of .deb files to the machine
+## this is installed on then call this script with all the deb files as arguments. For example:
+##
+## apt-add-to-freight-library --puppet-version=7 --repo-type=nightly --codename=buster
+##   /tmp/xxtmpdir/somepackage-a_0.0.0-1buster_amd64.deb
+##   /tmp/xxtmpdir/somepackage-b_1.2.3-5buster_amd64.deb
+##     (etc.)
+
+class AptAddToFreightLibrary
+  include Logging
+
+  FREIGHT_COMMAND = '/usr/bin/freight'
+  VALID_PUPPET_VERSIONS = %w[6 7 8]
+  VALID_REPO_TYPES = %w[archive nightly stable]
+  VALID_CODENAMES = %w[bionic jessie stretch buster bullseye bookwork focal xenial]
+
+  DOCUMENTATION = <<~DOCOPT
+    Adds debian .deb files to a freight library.
+
+    Usage:
+        apt-add-to-freight-library [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE --codename=DEBIAN_CODENAME <deb_file> ...
+        apt-add-to-freight-library --help
+        apt-add-to-freight-library --version
+
+    Options:
+    -p --puppet-version=VERSION    Puppet version (Valid: #{VALID_PUPPET_VERSIONS.join(', ')})
+    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{VALID_REPO_TYPES.join(', ')})
+    -c --codename=DEBIAN_CODENAME  Debian codename to deliver to (Valid: #{VALID_CODENAMES.join(', ')})
+
+    -v --verbose  Be verbose
+    -h --help     Show this help
+    --version     Show version
+  DOCOPT
+
+  def initialize(argv)
+    @log_level = Logger::INFO
+    @user_options = parse_options(argv)
+    @verbose = ''
+    @verbose = '--verbose' if @user_options['--verbose']
+
+    @script_name = File.basename($PROGRAM_NAME)
+    @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
+    if @user_options['--version'] # rubocop:disable Style/GuardClause
+      puts @version_string
+      exit 0
+    end
+  end
+
+  def parse_options(argv = nil)
+    Docopt.docopt(DOCUMENTATION, { argv: argv })
+  rescue Docopt::Exit => e
+    puts e.message
+    exit 1
+  end
+
+  def run
+    logger.info @version_string
+    @puppet_version = @user_options['--puppet-version']
+    fatal "Invalid Puppet version '#{@puppet_version}'" unless
+      VALID_PUPPET_VERSIONS.include? @puppet_version
+
+    @repo_type = @user_options['--repo-type']
+    fatal "Invalid repo type '#{@repo_type}'" unless VALID_REPO_TYPES.include? @repo_type
+
+    @codename = @user_options['--codename']
+    fatal "Invalid codename '#{@codename}'" unless VALID_CODENAMES.include? @codename
+
+    freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
+    fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
+      File.readable?(freight_config_file)
+
+    logger.info "Using '#{freight_config_file}'"
+    deb_artifacts = @user_options['<deb_file>']
+
+    Dir.mktmpdir do |temporary_directory|
+      deb_artifacts.each do |artifact_path|
+        logger.info "Adding #{artifact_path} to puppet_#{@puppet_version}_#{@repo_type}"
+        deb_artifact_path = artifact_path
+        if deb_artifact_path.start_with?('https://', 'http://')
+          download_data = URI.parse(deb_artifact_path).open
+          download_path = File.join(temporary_directory, download_data.base_uri.to_s.split('/').last)
+          IO.copy_stream(download_data, download_path)
+          deb_artifact_path = download_path
+        end
+
+        freight_add_command = %W[
+         sudo --user=jenkins --set-home
+         #{FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
+         #{deb_artifact_path} "apt/#{@codename}"
+        ].join(' ')
+
+        %x(#{freight_add_command})
+        fatal "#{freight_add_command} failed." unless $CHILD_STATUS.success?
+      end
+    end
+  end
+end

data/lib/apt_stage_artifacts.rb

@@ -0,0 +1,188 @@
+require 'version'
+require 'mixins/logging'
+
+require 'English'
+require 'docopt'
+require 'fileutils'
+require 'open-uri'
+require 'tempfile'
+require 'yaml'
+
+class AptStageArtifacts
+  include Logging
+
+  DOCUMENTATION = <<~DOCOPT
+    Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
+
+    Usage:
+        apt-stage-artifacts
+        apt-stage-artifacts <top_directory>
+  DOCOPT
+
+  def initialize(argv)
+    @script_name = File.basename($PROGRAM_NAME)
+    @log_level = Logger::INFO
+    @user_options = parse_options(argv)
+    @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
+
+    @artifacts_tarball_name = 'artifacts.tgz'
+    @local_tarball_path = File.join(Dir.pwd, @artifacts_tarball_name)
+
+    @builder_data_yaml = 'https://raw.githubusercontent.com/puppetlabs/build-data/release/builder_data.yaml'
+
+    @manifest_file_name = 'apt_stage.manifest'
+    @remote_staging_command = 'apt-stage-from-tarball'
+  end
+
+  def parse_options(argv = nil)
+    Docopt.docopt(DOCUMENTATION, { argv: argv })
+  rescue Docopt::Exit => e
+    puts e.message
+    exit 1
+  end
+
+  def run
+    logger.info @version_string
+    parse_repo_type
+    load_builder_data_yaml
+    collect_artifacts
+    create_remote_staging_directory
+    generate_manifest
+    create_tarball
+    ship_tarball_to_staging_server
+    invoke_remote_staging
+    cleanup_tarball
+    logger.info 'Deb artifact staging successful.'
+  end
+
+  private
+
+  # Locate all the local debian artifacts to be added to the freight repo.
+  # For historical reasons, there is an implied format of:
+  #  <top_directory>/<debian codename>/<artifact>.deb
+  #
+  # We rely on <debian codename> being in the first position below <top_directory>
+  # in order to properly classify the artifacts. This is definitely brittle.
+  def collect_artifacts
+    if !@user_options['<top_directory>'].to_s.empty?
+      @top_directory = @user_options['<top_directory>']
+    elsif File.directory?('./output/deb')
+      @top_directory = './output/deb'
+    elsif File.directory?('./pkg/deb')
+      @top_directory = './pkg/deb'
+    else
+      logger.error 'Cannot find .deb artifacts. Neither pkg/deb nor output/deb exist.'
+      exit 1
+    end
+
+    logger.info "Searching for .deb packages in #{@top_directory}"
+    Dir.chdir(@top_directory) do
+      @debian_artifacts = Dir['*/*.deb']
+    end
+    return unless @debian_artifacts.empty?
+
+    logger.error 'No .deb files found. Nothing to do.'
+    exit 0
+  end
+
+  # Create a working directory on the staging server that can then be used to deliver
+  # to freight repos.
+  def create_remote_staging_directory
+    @remote_staging_directory = %x(ssh #{@staging_server} /bin/mktemp --directory).chomp
+    fatal "/bin/mktemp --directory on #{@staging_server} failed." unless $CHILD_STATUS.success?
+
+    # We need to give jenkins read access to the remote_staging_directory
+    %x(ssh #{@staging_server} /bin/chmod 755 #{@remote_staging_directory})
+    unless $CHILD_STATUS.success?
+      fatal "/bin/chmod 755 #{@remote_staging_directory} on #{@staging_server} failed."
+    end
+
+    @remote_tarball_path = File.join(@remote_staging_directory, @artifacts_tarball_name)
+  end
+
+  # Load details from the builder_data.yaml on github
+  # Keep only the bare necessities. Right now, that's just the staging server name.
+  def load_builder_data_yaml
+    yaml_content = URI.parse(@builder_data_yaml).open(&:read)
+    yaml_data = YAML.safe_load(yaml_content)
+    @staging_server = yaml_data['staging_server']
+    logger.info "Artifacts will be staged on '#{@staging_server}'."
+  end
+
+  # A manifest is a staging to-do list. For each '.deb' file create a colon-delimited line
+  # containing:
+  # <.deb filename>:<puppet-version>:<repo-type>:<debian-codename>
+  def generate_manifest
+    manifest_temporary_file = Tempfile.new('apt_stage_artifacts')
+
+    @debian_artifacts.each do |deb_file_path|
+      # Ug. We only know the codename because it happens to be the first element
+      # of the file path.
+      debian_codename = deb_file_path.split('/').first
+      manifest_temporary_file.printf(
+        "%s:%s:%s:%s\n",
+        deb_file_path, @puppet_version, @repo_type, debian_codename
+      )
+    end
+
+    manifest_temporary_file.close
+    manifest_path = File.join(@top_directory, @manifest_file_name)
+    FileUtils.mv(manifest_temporary_file.path, manifest_path)
+    logger.info("Staging manifest written to '#{manifest_path}'")
+  end
+
+  # Create a tarball that contains the staging manifest and the associated deb files.
+  def create_tarball
+    tarball_create_command = %W[
+      tar --create --gzip --file #{@local_tarball_path} --directory=#{@top_directory}
+    ].join(' ')
+    %x(#{tarball_create_command} #{@manifest_file_name} #{@debian_artifacts.join(' ')})
+    unless $CHILD_STATUS.success?
+      fatal "#{tarball_create_command} #{@manifest_file_name} #{@debian_artifacts.join(' ')} failed."
+    end
+
+    logger.info("'#{@local_tarball_path}' created.")
+  end
+
+  def cleanup_tarball
+    File.delete @local_tarball_path
+  end
+
+  # The target apt repository is described in the REPO_NAME environment variable.
+  # A better solution is called for, perhaps as part of vanagon config or as a commandline
+  # parameter.
+  def parse_repo_type
+    case ENV['REPO_NAME']
+    when /\A\z/, nil
+      fatal "The environment variable 'REPO_NAME', containing the target apt repo, is unset.\n" \
+            "It should be set to something like 'puppet7' or 'puppet7-nightly'."
+    when /\Apuppet(\d+)\z/
+      @puppet_version = Regexp.last_match(1)
+      @repo_type = 'stable'
+    when /\Apuppet(\d+)-nightly\z/
+      @puppet_version = Regexp.last_match(1)
+      @repo_type = 'nightly'
+    else
+      fatal "The environment variable 'REPO_NAME' is set to '#{ENV['REPO_NAME']}'\n" \
+            "This is not a recognized setting.\n" \
+            "It should be set to something like 'puppet7' or 'puppet7-nightly'."
+    end
+  end
+
+  # Send the tarball over to the staging server for further processing with freight.
+  def ship_tarball_to_staging_server
+    logger.info "Sending '#{@local_tarball_path}' to '#{@staging_server}:#{@remote_tarball_path}'"
+    %x(scp #{@local_tarball_path} #{@staging_server}:#{@remote_tarball_path})
+    return if $CHILD_STATUS.success?
+
+    fatal "'scp #{@local_tarball_path} #{@staging_server}:#{@remote_tarball_path}' failed."
+  end
+
+  # Invoke the remote staging of the sent artifacts
+  def invoke_remote_staging
+    %x(ssh #{@staging_server} #{@remote_staging_command} #{@remote_tarball_path})
+    return if $CHILD_STATUS.success?
+
+    fatal "'ssh #{@staging_server} #{@remote_staging_command} #{@remote_tarball_path}' failed."
+  end
+end

data/lib/apt_stage_from_tarball.rb

@@ -0,0 +1,110 @@
+require 'version'
+require 'mixins/logging'
+
+require 'English'
+require 'mkmf'
+
+class AptStageFromTarball
+  include Logging
+
+  def initialize(tarball_path = nil)
+    @tarball_path = tarball_path
+    @manifest_file_name = 'apt_stage.manifest'
+    @log_level = Logger::INFO
+    @repo_types_updated = {}
+
+    @script_name = File.basename($PROGRAM_NAME)
+    @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
+
+    # External commands
+    @apt_add_to_freight_library_command = 'apt-add-to-freight-library'
+    @apt_update_freight_cache_command = 'apt-update-freight-cache'
+  end
+
+  def run
+    logger.info @version_string
+    check_for_external_commands
+    extract_tarball
+    add_artifacts_to_freight_library
+    update_freight_cache
+    # cleanup_tarball_directory
+    logger.info 'Freight artifact staging successful.'
+  end
+
+  private
+
+  def check_for_external_commands
+    %W[#{@apt_add_to_freight_library_command}
+       #{@apt_update_freight_cache_command}].each do |external_command|
+      fatal "'#{external_command}' is missing or not executable" unless
+        find_executable external_command
+    end
+  end
+
+  def extract_tarball
+    fatal "'#{@tarball_path}' does not exist." unless File.exist?(@tarball_path)
+    fatal "'#{@tarball_path}' is not a regular file." unless File.file?(@tarball_path)
+    fatal "'#{@tarball_path}' is not readable." unless File.readable?(@tarball_path)
+
+    @tarball_directory = File.dirname(@tarball_path)
+
+    tarball_extract_command = %W[
+      tar --directory #{@tarball_directory} --extract --gzip --file #{@tarball_path}
+    ].join(' ')
+    %x(#{tarball_extract_command})
+
+    unless $CHILD_STATUS.success?
+      fatal "#{tarball_extract_command} failed."
+    end
+
+    @manifest_path = File.join(@tarball_directory, @manifest_file_name)
+
+    fatal "Staging manifest file '#{@manifest_path}' is missing or unreadable" unless
+      File.readable?(@manifest_path)
+
+    logger.info("'#{@tarball_path}' extracted.")
+  end
+
+  # Read through the staging manifest, adding each line into the correct apt freight
+  # library.
+  def add_artifacts_to_freight_library
+    Dir.chdir(@tarball_directory) do
+      File.readlines(@manifest_path).each do |line|
+        line.chomp!
+        debian_filename, puppet_version, repo_type, codename = line.split(':')
+        add_to_freight_library_command = %W[
+           #{@apt_add_to_freight_library_command}
+           --verbose
+           --puppet-version=#{puppet_version}
+           --repo-type=#{repo_type}
+           --codename=#{codename}
+           #{File.join(Dir.pwd, debian_filename)}
+        ].join(' ')
+
+        %x(#{add_to_freight_library_command})
+        fatal "#{add_to_freight_library_command} failed." unless $CHILD_STATUS.success?
+
+        # Keep a hash, key by puppet_version, of repo_types so that we can
+        # efficiently build out the freight caches later.
+        @repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
+        @repo_types_updated[puppet_version] |= [repo_type]
+      end
+    end
+  end
+
+  def update_freight_cache
+    @repo_types_updated.each do |puppet_version, repo_types|
+      repo_types.each do |repo_type|
+        update_freight_cache_command = %W[
+          #{@apt_update_freight_cache_command}
+          --verbose
+          --puppet-version=#{puppet_version}
+          --repo-type=#{repo_type}
+        ].join(' ')
+
+        %x(#{update_freight_cache_command})
+        fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
+      end
+    end
+  end
+end

data/lib/apt_update_freight_cache.rb

@@ -0,0 +1,102 @@
+require 'version'
+require 'mixins/logging'
+
+require 'English'
+require 'docopt'
+require 'open-uri'
+require 'tempfile'
+require 'yaml'
+
+##
+## This is a wrapper for https://github.com/freight-team/freight, which creates APT repos
+## from deb files. This triggers the 'freight cache' command which creates a proper
+## APT repo (freight calls it a 'cache'), including signing, from a tree of debian
+## files (freight calls it a 'library').
+##
+## In our case, we maintain a set of debian repos, based on puppet version and a repo-type
+## 'nightly', 'archive' and 'stable'.
+##
+## freight provides signing services, so this script needs to exist on a machine where GPG
+## signing has been set up.
+##
+## Example usages:
+## apt-update-freight-cache --puppet-version=7 --repo-type=stable
+## apt-update-freight-cache --puppet-version=7 --repo-type=nightly
+
+class AptUpdateFreightCache
+  include Logging
+
+  FREIGHT_COMMAND = '/usr/bin/freight'
+  VALID_PUPPET_VERSIONS = %w[6 7 8]
+  VALID_REPO_TYPES = %w[archive nightly stable]
+
+  DOCUMENTATION = <<~DOCOPT
+    Generates a freight cache from a freight library.
+
+    Usage:
+        apt-update-freight-cache [--verbose] --puppet-version=VERSION --repo-type=REPO_TYPE
+        apt-update-freight-cache --help
+        apt-update-freight-cache --version
+
+    Options:
+    -p --puppet-version=VERSION    Puppet version (Valid: #{VALID_PUPPET_VERSIONS.join(', ')})
+    -r --repo-type=REPO_TYPE       Puppet debian repo type (Valid: #{VALID_REPO_TYPES.join(', ')})
+
+    -v --verbose  Be verbose
+    -h --help     Show this help
+    --version     Show version
+  DOCOPT
+
+  def initialize(argv)
+    @log_level = Logger::INFO
+    @user_options = parse_options(argv)
+    @verbose = ''
+    @verbose = '--verbose' if @user_options['--verbose']
+    @builder_data_yaml = 'https://raw.githubusercontent.com/puppetlabs/build-data/release/builder_data.yaml'
+    @script_name = File.basename($PROGRAM_NAME)
+    @version_string = "#{@script_name} version #{AptStageArtifacts::VERSION}"
+    if @user_options['--version'] # rubocop:disable Style/GuardClause
+      puts @version_string
+      exit 0
+    end
+  end
+
+  def parse_options(argv = nil)
+    Docopt.docopt(DOCUMENTATION, { argv: argv })
+  rescue Docopt::Exit => e
+    puts e.message
+    exit 1
+  end
+
+  def run
+    logger.info @version_string
+    load_builder_data_yaml
+    @puppet_version = @user_options['--puppet-version']
+    fatal "Invalid Puppet version '#{@puppet_version}'" unless
+      VALID_PUPPET_VERSIONS.include? @puppet_version
+
+    @repo_type = @user_options['--repo-type']
+    fatal "Invalid repo type '#{@repo_type}'" unless VALID_REPO_TYPES.include? @repo_type
+
+    freight_config_file = "/etc/freight.conf.d/puppet_#{@puppet_version}_#{@repo_type}.conf"
+    fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
+      File.readable?(freight_config_file)
+
+    logger.info "Using '#{freight_config_file}'"
+    freight_cache_command = %W[
+      sudo --user=jenkins --set-home
+      /usr/local/bin/freight-cache-wrapper #{@verbose}
+      --conf=#{freight_config_file} --gpg-key=#{@gpg_key}
+    ].join(' ')
+    %x(#{freight_cache_command})
+    fatal "#{tarball_extract_command} failed." unless $CHILD_STATUS.success?
+  end
+
+  # Load details from the builder_data.yaml on github
+  # Keep only the bare necessities. Right now, that's just the GPG key
+  def load_builder_data_yaml
+    yaml_content = URI.parse(@builder_data_yaml).open(&:read)
+    yaml_data = YAML.safe_load(yaml_content)
+    @gpg_key = yaml_data['gpg_key']
+  end
+end

data/lib/mixins/logging.rb

@@ -0,0 +1,29 @@
+# Handles all logging output for Tefoji
+
+module Logging
+  require 'logger'
+
+  # A cheap hack that will instantiate a logger if necessary before sending a log
+  # message. This is the normal interface to logging from outside.
+  def logger
+    Logging.logger(@log_level)
+  end
+
+  # Another cheap hack to wrap 'logger' above. Called as 'fatal' rather than 'logger.fatal'
+  # directly, does the exit for us so we can be lazy.
+  def fatal(message)
+    logger.fatal(message)
+    exit 1
+  end
+
+  # Do the actual logging by wrapping Logger gem.
+  def self.logger(log_level)
+    return @logger unless @logger.nil?
+
+    @logger = Logger.new($stderr, progname: File.basename($PROGRAM_NAME), level: log_level)
+    @logger.formatter = proc do |severity, _, script_name, message|
+      "#{script_name}: #{severity} #{message}\n"
+    end
+    return @logger
+  end
+end

data/lib/version.rb

@@ -0,0 +1,3 @@
+class AptStageArtifacts
+  VERSION = '0.4.0'
+end

metadata

@@ -0,0 +1,185 @@
+--- !ruby/object:Gem::Specification
+name: apt_stage_artifacts
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Puppet Release Engineering
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-04-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: docopt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- release@puppet.com
+executables:
+- apt-add-to-freight-library
+- apt-stage-artifacts
+- apt-stage-from-tarball
+- apt-update-freight-cache
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODEOWNERS
+- DEVELOPMENT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- apt_stage_artifacts.gemspec
+- bin/console
+- bin/setup
+- exe/apt-add-to-freight-library
+- exe/apt-stage-artifacts
+- exe/apt-stage-from-tarball
+- exe/apt-update-freight-cache
+- lib/apt_add_to_freight_library.rb
+- lib/apt_stage_artifacts.rb
+- lib/apt_stage_from_tarball.rb
+- lib/apt_update_freight_cache.rb
+- lib/mixins/logging.rb
+- lib/version.rb
+homepage: https://github.com/puppetlabs/apt_stage_artifacts
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/puppetlabs/apt_stage_artifacts
+  source_code_uri: https://github.com/puppetlabs/apt_stage_artifacts
+  changelog_uri: https://github.com/puppetlabs/apt_stage_artifacts/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Stages .deb artifacts to a remote freight repository
+test_files: []