ood_packaging 0.0.1.r2.0

data/lib/ood_packaging/build_box.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'ood_packaging/utils'
+require 'rake'
+require 'rake/file_utils'
+
+# Build the packaging build boxes
+class OodPackaging::BuildBox
+  include OodPackaging::Utils
+  include FileUtils
+
+  BASE_IMAGES = {
+    'el7'          => 'centos:7',
+    'el8'          => 'rockylinux/rockylinux:8',
+    'ubuntu-20.04' => 'ubuntu:20.04'
+  }.freeze
+
+  CODENAMES = {
+    'ubuntu-20.04' => 'focal'
+  }.freeze
+
+  def initialize(config = {})
+    @config = config
+    raise ArgumentError, 'Must provide dist' if dist.nil?
+
+    # rubocop:disable Style/GuardClause
+    unless valid_dist?(dist)
+      raise ArgumentError, "Invalid dist selected: #{dist}. Valid choices are #{valid_dists.join(' ')}"
+    end
+    # rubocop:enable Style/GuardClause
+  end
+
+  def dist
+    @dist ||= ENV['OOD_PACKAGING_DIST'] || @config[:dist]
+  end
+
+  def rpm?
+    dist.start_with?('el')
+  end
+
+  def deb?
+    !rpm?
+  end
+
+  def dnf?
+    return false unless rpm?
+    return false if dist == 'el7'
+
+    true
+  end
+
+  def scl?
+    return true if dist == 'el7'
+
+    false
+  end
+
+  def package_manager
+    return 'apt' if deb?
+    return 'dnf' if dnf?
+
+    'yum'
+  end
+
+  def valid_dist?(value)
+    BASE_IMAGES.key?(value)
+  end
+
+  def valid_dists
+    BASE_IMAGES.keys
+  end
+
+  def base_image
+    @base_image ||= BASE_IMAGES[dist]
+  end
+
+  def codename
+    @codename ||= CODENAMES[dist]
+  end
+
+  def build_dir
+    File.join(File.dirname(__FILE__), 'build_box/docker-image')
+  end
+
+  def image_registry
+    @config[:builx_box_registry] || ENV['OOD_PACKAGING_BUILD_BOX_REGISTRY'] || nil
+  end
+
+  def image_org
+    @config[:build_box_org] || ENV['OOD_PACKAGING_BUILX_BOX_ORG'] || 'ohiosupercomputer'
+  end
+
+  def image_name
+    @config[:builx_box_name] || ENV['OOD_PACKAGING_BUILD_BOX_NAME'] || 'ood-buildbox'
+  end
+
+  def image_version
+    (ENV['OOD_PACKAGING_BUILD_BOX_VERSION'] || OodPackaging::VERSION).gsub(/^v/, '')
+  end
+
+  def image_tag
+    [image_registry, image_org, "#{image_name}-#{dist}:#{image_version}"].compact.join('/')
+  end
+
+  def build_gem
+    gem = File.join(proj_root, 'pkg', "ood_packaging-#{OodPackaging::VERSION}.gem")
+    Rake::Task['build'].invoke
+    sh "rm -f #{build_dir}/*.gem"
+    sh "cp #{gem} #{build_dir}/"
+  end
+
+  def dockerfile
+    template_file('build_box/docker-image/Dockerfile.erb')
+  end
+
+  def scripts
+    template_file('build_box/docker-image/inituidgid.sh.erb')
+    template_file('build_box/docker-image/install.sh.erb')
+  end
+
+  def build!
+    scripts
+    cmd = [container_runtime, 'build']
+    cmd.concat ['--tag', image_tag]
+    cmd.concat [ENV['OOD_PACKAGING_BUILD_BOX_ARGS']] if ENV['OOD_PACKAGING_BUILD_BOX_ARGS']
+    cmd.concat ['-f', dockerfile]
+    cmd.concat [build_dir]
+    build_gem
+    sh cmd.join(' ')
+  end
+
+  def push!
+    sh [container_runtime, 'push', image_tag].join(' ')
+  end
+
+  def pull!
+    sh [container_runtime, 'pull', image_tag].join(' ')
+  end
+
+  def save!(path)
+    sh [container_runtime, 'save', image_tag, '| gzip >', path].join(' ')
+  end
+end

data/lib/ood_packaging/options.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Handle options for OodPackaging
+module OodPackaging
+  OPTIONS = [:package, :version, :dist, :work_dir, :clean_work_dir, :output_dir, :clean_output_dir, :tar,
+             :tar_only, :skip_download, :gpg_sign, :gpg_name, :gpg_pubkey, :gpg_private_key, :gpg_passphrase,
+             :debug, :attach].freeze
+end

data/lib/ood_packaging/package.rb

@@ -0,0 +1,326 @@
+# frozen_string_literal: true
+
+require 'ood_packaging/build_box'
+require 'ood_packaging/utils'
+require 'ood_packaging/string_ext'
+require 'English'
+require 'pathname'
+require 'ostruct'
+require 'securerandom'
+require 'rake'
+require 'rake/file_utils'
+
+# The interface to build packages using containers
+class OodPackaging::Package
+  include OodPackaging::Utils
+  include FileUtils
+
+  attr_accessor :build_box
+
+  def initialize(config = {})
+    @config = config
+    @config[:dist] = 'el8' if tar_only?
+    @build_box = OodPackaging::BuildBox.new(config)
+    raise ArgumentError, 'Package is required' if package.nil?
+    raise ArgumentError, 'Version is required' if version.nil?
+    raise ArgumentError, "Package #{package} is not a directory" unless Dir.exist?(package)
+    raise ArgumentError, "Package #{package} is not an absolute path" unless (Pathname.new package).absolute?
+  end
+
+  def container_name
+    @container_name ||= SecureRandom.uuid
+  end
+
+  def debug?
+    return true if ENV['OOD_PACKAGING_DEBUG'] == 'true'
+
+    @config[:debug].nil? ? false : @config[:debug]
+  end
+
+  def cmd_suffix
+    return '' if debug?
+
+    ' 2>/dev/null 1>/dev/null'
+  end
+
+  def attach?
+    return true if ENV['OOD_PACKAGING_ATTACH'] == 'true'
+
+    @config[:attach].nil? ? false : @config[:attach]
+  end
+
+  def work_dir
+    @work_dir ||= File.expand_path(@config[:work_dir])
+  end
+
+  def clean_work_dir
+    return false if ENV['OOD_PACKAGING_CLEAN_WORK_DIR'] == 'false'
+
+    @config[:clean_work_dir].nil? ? true : @config[:clean_work_dir]
+  end
+
+  def output_dir
+    @output_dir ||= File.expand_path(@config[:output_dir])
+  end
+
+  def clean_output_dir
+    return false if ENV['OOD_PACKAGING_CLEAN_OUTPUT_DIR'] == 'false'
+
+    @config[:clean_output_dir].nil? ? true : @config[:clean_output_dir]
+  end
+
+  def package
+    @config[:package]
+  end
+
+  def version
+    @config[:version]
+  end
+
+  def tar_version
+    version.gsub(/^v/, '')
+  end
+
+  def tar?
+    @config[:tar].nil? ? build_box.deb? : @config[:tar]
+  end
+
+  def tar_only?
+    @config[:tar_only].nil? ? false : @config[:tar_only]
+  end
+
+  def package_name
+    name = File.basename(package)
+    case name
+    when /deb|rpm/
+      name = if File.basename(File.dirname(package)) == 'packages'
+               File.basename(File.dirname(File.dirname(package)))
+             else
+               File.basename(File.dirname(package))
+             end
+    when 'packaging'
+      name = File.basename(File.dirname(package))
+    end
+    name
+  end
+
+  def gpg_files
+    [
+      OpenStruct.new(private_key: @config[:gpg_private_key], passphrase: @config[:gpg_passphrase]),
+      OpenStruct.new(private_key: ENV['OOD_PACKAGING_GPG_PRIVATE_KEY'],
+                     passphrase:  ENV['OOD_PACKAGING_GPG_PASSPHRASE'])
+    ].each do |gpg|
+      next if gpg.private_key.nil? || gpg.passphrase.nil?
+      return gpg if File.exist?(gpg.private_key) && File.exist?(gpg.passphrase)
+    end
+    nil
+  end
+
+  def gpg_sign
+    return false if ENV['OOD_PACKAGING_GPG_SIGN'].to_s == 'false'
+    return false if @config[:gpg_sign] == false
+
+    !gpg_files.nil?
+  end
+
+  def default_gpg_name
+    'OnDemand Release Signing Key'
+  end
+
+  def gpg_name
+    @config[:gpg_name].nil? ? default_gpg_name : @config[:gpg_name]
+  end
+
+  def container_init
+    '/sbin/init'
+  end
+
+  def tar_path
+    if build_box.deb?
+      path = File.join(package, 'deb')
+      return path if Dir.exist?(path)
+    end
+
+    package
+  end
+
+  def tar_name
+    version = if build_box.rpm?
+                rpm_version
+              else
+                deb_version
+              end
+    "#{package_name}-#{version}"
+  end
+
+  def rpm_tar_dest_dir
+    [
+      File.join(package, 'rpm'),
+      File.join(package, 'packaging/rpm'),
+      File.join(package, 'packaging')
+    ].each do |dir|
+      return dir if Dir.exist?(dir)
+    end
+    File.join(package, 'packaging/rpm')
+  end
+
+  def rpm_version
+    tar_version
+  end
+
+  def deb_tar_dest_dir
+    dir = File.join(package, 'deb')
+    return File.join(dir, 'build') if Dir.exist?(dir)
+
+    File.join(package, 'build')
+  end
+
+  def deb_version
+    tar_version.gsub('-', '.')
+  end
+
+  def exec_launchers
+    [
+      File.join(ctr_scripts_dir, 'inituidgid.sh'),
+      File.join(ctr_scripts_dir, 'setuser.rb'),
+      ctr_user
+    ]
+  end
+
+  def exec_rake
+    cmd = []
+    cmd.concat exec_launchers if docker_runtime?
+    cmd.concat ['scl', 'enable', scl_ruby, '--'] if podman_runtime? && build_box.scl?
+    cmd.concat [File.join(ctr_scripts_dir, 'rake')]
+    cmd.concat ['-q'] unless debug?
+    cmd.concat ['-f', File.join(ctr_scripts_dir, 'Rakefile'), 'ood_packaging:package:build']
+    cmd
+  end
+
+  def exec_attach
+    cmd = []
+    cmd.concat exec_launchers if docker_runtime?
+    cmd.concat ['/bin/bash']
+    cmd
+  end
+
+  def container_mounts
+    args = []
+    args.concat ['-v', "#{package}:/package:ro"]
+    args.concat ['-v', "#{@config[:gpg_pubkey]}:/gpg.pub:ro"] if @config[:gpg_pubkey]
+    args.concat ['-v', "#{work_dir}:/work"]
+    args.concat ['-v', "#{output_dir}:/output"]
+    if gpg_sign
+      args.concat ['-v', "#{gpg_files.private_key}:#{gpg_private_key}:ro"]
+      args.concat ['-v', "#{gpg_files.passphrase}:#{gpg_passphrase}:ro"]
+    end
+    args
+  end
+
+  def clean!
+    sh "rm -rf #{work_dir}", verbose: debug? if clean_work_dir
+    sh "rm -rf #{output_dir}", verbose: debug? if clean_output_dir
+  end
+
+  def bootstrap!
+    sh "mkdir -p #{work_dir}", verbose: debug?
+    sh "mkdir -p #{output_dir}", verbose: debug?
+  end
+
+  def tar!
+    cmd = ['git', 'ls-files', '.', '|', tar, '-c']
+    if build_box.rpm?
+      dir = rpm_tar_dest_dir
+    else
+      dir = deb_tar_dest_dir.tap { |p| sh "mkdir -p #{p}" }
+      cmd.concat ["--transform 'flags=r;s,packaging/deb,debian,'"]
+    end
+    tar_file = "#{dir}/#{tar_name}.tar.gz"
+    cmd.concat ["--transform 's,^,#{tar_name}/,'"]
+    cmd.concat ['-T', '-', '|', "gzip > #{tar_file}"]
+
+    sh "rm #{tar_file}" if File.exist?(tar_file)
+    puts "Create tar archive #{tar_file}".blue
+    Dir.chdir(tar_path) do
+      sh cmd.join(' '), verbose: debug?
+    end
+  end
+
+  def run!
+    if tar_only?
+      tar!
+      return
+    end
+    clean!
+    bootstrap!
+    tar! if tar?
+    container_start!
+    container_exec!(exec_rake)
+    puts "Build SUCCESS: package=#{package} dist=#{build_box.dist}".green
+  rescue RuntimeError
+    puts "Build FAILED package=#{package} dist=#{build_box.dist}".red
+    raise
+  ensure
+    container_exec!(exec_attach, ['-i', '-t']) if attach?
+    container_kill! if container_running? && !attach?
+  end
+
+  def container_running?
+    cmd = "#{container_runtime} inspect #{container_name} 2>/dev/null 1>/dev/null"
+    puts cmd if debug?
+    `#{cmd}`
+    $CHILD_STATUS.success?
+  end
+
+  def container_start!
+    cmd = [container_runtime, 'run', '--detach', '--rm']
+    cmd.concat ['--name', container_name]
+    cmd.concat rt_specific_flags
+    cmd.concat container_mounts
+    cmd.concat [build_box.image_tag]
+    cmd.concat [container_init]
+    cmd.concat ['1>/dev/null'] unless debug?
+    puts "Starting container #{container_name} using image #{build_box.image_tag}".blue
+    sh cmd.join(' '), verbose: debug?
+  end
+
+  def container_exec!(exec_cmd, extra_args = [])
+    cmd = [container_runtime, 'exec']
+    cmd.concat extra_args
+    container_env.each_pair do |k, v|
+      cmd.concat ['-e', "'#{k}=#{v}'"] unless v.nil?
+    end
+    cmd.concat [container_name]
+    cmd.concat exec_cmd
+    puts "Build STARTED: package=#{package} dist=#{build_box.dist} exec=#{exec_cmd[-1]}".blue
+    sh cmd.join(' '), verbose: debug?
+    true
+  rescue RuntimeError
+    container_kill! if container_running? && !attach?
+    raise
+  end
+
+  def container_kill!
+    puts "Killing container #{container_name}".blue
+    cmd = [container_runtime, 'kill', container_name]
+    cmd.concat [cmd_suffix] unless debug?
+    sh cmd.join(' '), verbose: debug?
+  end
+
+  def container_env
+    env = {
+      'DIST'          => build_box.dist,
+      'PACKAGE'       => package_name,
+      'VERSION'       => version,
+      'TAR_NAME'      => "#{tar_name}.tar.gz",
+      'GPG_SIGN'      => gpg_sign,
+      'GPG_NAME'      => gpg_name,
+      'SKIP_DOWNLOAD' => @config[:skip_download],
+      'OOD_UID'       => Process.uid,
+      'OOD_GID'       => Process.gid,
+      'DEBUG'         => debug?
+    }
+    env['GPG_PUBKEY'] = '/gpg.pub' if @config[:gpg_pubkey]
+    env
+  end
+end

data/lib/ood_packaging/rake_task.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'ood_packaging'
+require 'rake'
+require 'rake/tasklib'
+
+# Define OodPackaging rake task library
+class OodPackaging::RakeTask < ::Rake::TaskLib
+  include ::Rake::DSL if defined?(::Rake::DSL)
+  include OodPackaging::Utils
+
+  OodPackaging::OPTIONS.each do |o|
+    attr_accessor o.to_sym
+  end
+
+  def initialize(*args, &task_block)
+    super()
+    @name = args.shift || :package
+
+    define(args, &task_block)
+  end
+
+  def define(args, &task_block)
+    desc 'Build package' unless ::Rake.application.last_description
+
+    task @name, *args do |_, task_args|
+      task_block&.call(*[self, task_args].slice(0, task_block.arity))
+      config = {}
+      OodPackaging::OPTIONS.each do |o|
+        v = instance_variable_get("@#{o}")
+        config[o.to_sym] = v unless v.nil?
+      end
+      OodPackaging::Package.new(config).run!
+    end
+  end
+
+  def name
+    @name.to_s
+  end
+end

data/lib/ood_packaging/string_ext.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# https://stackoverflow.com/a/11482430
+class String
+  def colorize(code)
+    "\e[#{code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def blue
+    colorize(34)
+  end
+end

data/lib/ood_packaging/tasks.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'ood_packaging'
+require 'ood_packaging/rake_task'
+require 'ood_packaging/utils'
+
+namespace :ood_packaging do
+  include OodPackaging::Utils
+
+  desc 'Set version'
+  task :version, [:version] do |_task, args|
+    version_file = File.join(proj_root, 'lib/ood_packaging/version.rb')
+    version = args[:version].gsub(/^v/, '')
+    sh "#{sed} -i -r \"s|  VERSION =.*|  VERSION = '#{version}'|g\" #{version_file}"
+    sh 'bundle install'
+  end
+
+  namespace :buildbox do
+    desc 'Build buildbox image'
+    task :build, [:dist] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.build!
+    end
+
+    desc 'Push buildbox image'
+    task :push, [:dist] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.push!
+    end
+
+    desc 'Pull buildbox image'
+    task :pull, [:dist] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.pull!
+    end
+
+    desc 'Save buildbox image'
+    task :save, [:dist, :path] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.save!(args[:path])
+    end
+  end
+
+  namespace :package do
+    desc 'Build a package (INSIDE CONTAINER ONLY)'
+    task :build do
+      OodPackaging::Build.new.run!
+    end
+
+    OodPackaging::RakeTask.new(:internal, [:package, :dist]) do |t, args|
+      name = args[:package].split(':').last
+      t.package = File.join(proj_root, 'packages', name)
+      dist = args[:dist] || ENV['OOD_PACKAGING_DIST']
+      t.dist = dist
+      t.version = OodPackaging.package_version(name, dist)
+      t.work_dir = File.join(proj_root, 'tmp/work')
+      t.output_dir = File.join(proj_root, 'tmp/output')
+    end
+
+    desc 'Package ondemand-release'
+    task :'ondemand-release', [:dist] do |t, args|
+      ENV['OOD_PACKAGING_GPG_SIGN'] = 'false'
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand-release-latest'
+    task :'ondemand-release-latest', [:dist] do |t, args|
+      ENV['OOD_PACKAGING_GPG_SIGN'] = 'false'
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand-release'
+    task :'ondemand-runtime', [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package passenger'
+    task :passenger, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package cjose'
+    task :cjose, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package mod_auth_openidc'
+    task :mod_auth_openidc, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package sqlite'
+    task :sqlite, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand_exporter'
+    task :ondemand_exporter, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand-compute'
+    task :'ondemand-compute', [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package python-websockify'
+    task :'python-websockify', [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package turbovnc'
+    task :turbovnc, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+  end
+end