ood_packaging 0.0.1.rc.3

data/lib/ood_packaging/build_box.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'ood_packaging/utils'
+require 'rake'
+require 'rake/file_utils'
+
+# Build the packaging build boxes
+class OodPackaging::BuildBox
+  include OodPackaging::Utils
+  include FileUtils
+
+  BASE_IMAGES = {
+    'el7'          => 'centos:7',
+    'el8'          => 'rockylinux/rockylinux:8',
+    'ubuntu-20.04' => 'ubuntu:20.04'
+  }.freeze
+
+  CODENAMES = {
+    'ubuntu-20.04' => 'focal'
+  }.freeze
+
+  def initialize(config = {})
+    @config = config
+    raise ArgumentError, 'Must provide dist' if @config[:dist].nil?
+
+    # rubocop:disable Style/GuardClause
+    unless valid_dist?(@config[:dist])
+      raise ArgumentError, "Invalid dist selected: #{@config[:dist]}. Valid choices are #{valid_dists.join(' ')}"
+    end
+    # rubocop:enable Style/GuardClause
+  end
+
+  def dist
+    @dist ||= @config[:dist]
+  end
+
+  def rpm?
+    dist.start_with?('el')
+  end
+
+  def deb?
+    !rpm?
+  end
+
+  def dnf?
+    return false unless rpm?
+    return false if dist == 'el7'
+
+    true
+  end
+
+  def scl?
+    return true if dist == 'el7'
+
+    false
+  end
+
+  def package_manager
+    return 'apt' if deb?
+    return 'dnf' if dnf?
+
+    'yum'
+  end
+
+  def valid_dist?(value)
+    BASE_IMAGES.key?(value)
+  end
+
+  def valid_dists
+    BASE_IMAGES.keys
+  end
+
+  def base_image
+    @base_image ||= BASE_IMAGES[dist]
+  end
+
+  def codename
+    @codename ||= CODENAMES[dist]
+  end
+
+  def build_dir
+    File.join(File.dirname(__FILE__), 'build_box/docker-image')
+  end
+
+  def image_registry
+    @config[:builx_box_registry] || ENV['OOD_PACKAGING_BUILD_BOX_REGISTRY'] || nil
+  end
+
+  def image_org
+    @config[:build_box_org] || ENV['OOD_PACKAGING_BUILX_BOX_ORG'] || 'ohiosupercomputer'
+  end
+
+  def image_name
+    @config[:builx_box_name] || ENV['OOD_PACKAGING_BUILD_BOX_NAME'] || 'ood-buildbox'
+  end
+
+  def image_version
+    ENV['OOD_PACKAGING_BUILD_BOX_VERSION'] || OodPackaging::VERSION
+  end
+
+  def image_tag
+    [image_registry, image_org, "#{image_name}-#{dist}:#{image_version}"].compact.join('/')
+  end
+
+  def build_gem
+    gem = File.join(proj_root, 'pkg', "ood_packaging-#{OodPackaging::VERSION}.gem")
+    Rake::Task['build'].invoke
+    sh "rm -f #{build_dir}/*.gem"
+    sh "cp #{gem} #{build_dir}/"
+  end
+
+  def dockerfile
+    template_file('build_box/docker-image/Dockerfile.erb')
+  end
+
+  def scripts
+    template_file('build_box/docker-image/inituidgid.sh.erb')
+    template_file('build_box/docker-image/install.sh.erb')
+  end
+
+  def build!
+    scripts
+    cmd = [container_runtime, 'build']
+    cmd.concat ['--tag', image_tag]
+    cmd.concat [ENV['OOD_PACKAGING_BUILD_BOX_ARGS']] if ENV['OOD_PACKAGING_BUILD_BOX_ARGS']
+    cmd.concat ['-f', dockerfile]
+    cmd.concat [build_dir]
+    build_gem
+    sh cmd.join(' ')
+  end
+
+  def push!
+    sh [container_runtime, 'push', image_tag].join(' ')
+  end
+
+  def save!(path)
+    sh [container_runtime, 'save', image_tag, '| gzip >', path].join(' ')
+  end
+end

data/lib/ood_packaging/options.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Handle options for OodPackaging
+module OodPackaging
+  OPTIONS = [:package, :version, :dist, :work_dir, :clean_work_dir, :output_dir, :clean_output_dir, :tar,
+             :tar_only, :skip_download, :gpg_sign, :gpg_name, :gpg_pubkey, :gpg_private_key, :gpg_passphrase,
+             :debug, :attach].freeze
+end

data/lib/ood_packaging/package.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+require 'ood_packaging/build_box'
+require 'ood_packaging/utils'
+require 'ood_packaging/string_ext'
+require 'English'
+require 'pathname'
+require 'ostruct'
+require 'securerandom'
+require 'rake'
+require 'rake/file_utils'
+
+# The interface to build packages using containers
+class OodPackaging::Package
+  include OodPackaging::Utils
+  include FileUtils
+
+  attr_accessor :build_box
+
+  def initialize(config = {})
+    @config = config
+    @config[:dist] = 'el8' if tar_only?
+    @build_box = OodPackaging::BuildBox.new(config)
+    @clean_work_dir = config[:clean_work_dir].nil? ? true : config[:clean_work_dir]
+    @clean_output_dir = config[:clean_output_dir].nil? ? true : config[:clean_output_dir]
+    raise ArgumentError, 'Package is required' if package.nil?
+    raise ArgumentError, 'Version is required' if version.nil?
+    raise ArgumentError, "Package #{package} is not a directory" unless Dir.exist?(package)
+    raise ArgumentError, "Package #{package} is not an absolute path" unless (Pathname.new package).absolute?
+  end
+
+  def container_name
+    @container_name ||= SecureRandom.uuid
+  end
+
+  def debug
+    return true if ENV['OOD_PACKAGING_DEBUG'] == 'true'
+
+    @config[:debug].nil? ? false : @config[:debug]
+  end
+
+  def attach?
+    return true if ENV['OOD_PACKAGING_ATTACH'] == 'true'
+
+    @config[:attach].nil? ? false : @config[:attach]
+  end
+
+  def work_dir
+    @work_dir ||= File.expand_path(@config[:work_dir])
+  end
+
+  def output_dir
+    @output_dir ||= File.expand_path(@config[:output_dir])
+  end
+
+  def package
+    @config[:package]
+  end
+
+  def version
+    @config[:version]
+  end
+
+  def tar_version
+    version.gsub(/^v/, '')
+  end
+
+  def tar?
+    @config[:tar].nil? ? build_box.deb? : @config[:tar]
+  end
+
+  def tar_only?
+    @config[:tar_only].nil? ? false : @config[:tar_only]
+  end
+
+  def package_name
+    name = File.basename(package)
+    if name =~ /deb|rpm/
+      name = if File.basename(File.dirname(package)) == 'packages'
+               File.basename(File.dirname(File.dirname(package)))
+             else
+               File.basename(File.dirname(package))
+             end
+    end
+    name
+  end
+
+  def gpg_files
+    [
+      OpenStruct.new(private_key: File.join(proj_root, 'ondemand.sec'), passphrase: File.join(proj_root, '.gpgpass')),
+      OpenStruct.new(private_key: File.join(package, 'ondemand.sec'), passphrase: File.join(package, '.gpgpass')),
+      OpenStruct.new(private_key: @config[:gpg_private_key], passphrase: @config[:gpg_passphrase]),
+      OpenStruct.new(private_key: ENV['OOD_PACKAGING_GPG_PRIVATE_KEY'],
+                     passphrase:  ENV['OOD_PACKAGING_GPG_PASSPHRASE'])
+    ].each do |gpg|
+      next if gpg.private_key.nil? || gpg.passphrase.nil?
+      return gpg if File.exist?(gpg.private_key) && File.exist?(gpg.passphrase)
+    end
+    nil
+  end
+
+  def gpg_sign
+    return false if ENV['OOD_PACKAGING_GPG_SIGN'].to_s == 'false'
+    return false if @config[:gpg_sign] == false
+
+    !gpg_files.nil?
+  end
+
+  def gpg_name
+    @config[:gpg_name].nil? ? 'OnDemand Release Signing Key' : @config[:gpg_name]
+  end
+
+  def container_init
+    '/sbin/init'
+  end
+
+  def tar_path
+    if build_box.deb?
+      path = File.join(package, 'deb')
+      return path if Dir.exist?(path)
+    end
+
+    package
+  end
+
+  def rpm_tar_dest_dir
+    dir = File.join(package, 'rpm')
+    return dir if Dir.exist?(dir)
+
+    File.join(package, 'packaging/rpm')
+  end
+
+  def rpm_version
+    tar_version
+  end
+
+  def deb_tar_dest_dir
+    dir = File.join(package, 'deb')
+    return File.join(dir, 'build') if Dir.exist?(dir)
+
+    File.join(package, 'build')
+  end
+
+  def deb_version
+    tar_version.gsub('-', '.')
+  end
+
+  def exec_launchers
+    [
+      File.join(ctr_scripts_dir, 'inituidgid.sh'),
+      File.join(ctr_scripts_dir, 'setuser.rb'),
+      ctr_user
+    ]
+  end
+
+  def exec_rake
+    cmd = []
+    cmd.concat exec_launchers if docker_runtime?
+    cmd.concat ['scl', 'enable', scl_ruby, '--'] if podman_runtime? && build_box.scl?
+    cmd.concat ['rake']
+    cmd.concat ['-q'] unless debug
+    cmd.concat ['-f', File.join(ctr_scripts_dir, 'Rakefile'), 'ood_packaging:package:build']
+    cmd
+  end
+
+  def exec_attach
+    cmd = []
+    cmd.concat exec_launchers if docker_runtime?
+    cmd.concat ['/bin/bash']
+    cmd
+  end
+
+  def container_mounts
+    args = []
+    args.concat ['-v', "#{package}:/package:ro"]
+    args.concat ['-v', "#{@config[:gpg_pubkey]}:/gpg.pub:ro"] if @config[:gpg_pubkey]
+    args.concat ['-v', "#{work_dir}:/work"]
+    args.concat ['-v', "#{output_dir}:/output"]
+    if gpg_sign
+      args.concat ['-v', "#{gpg_files.private_key}:#{gpg_private_key}:ro"]
+      args.concat ['-v', "#{gpg_files.passphrase}:#{gpg_passphrase}:ro"]
+    end
+    args
+  end
+
+  def clean!
+    sh "rm -rf #{work_dir}", verbose: debug if @clean_work_dir
+    sh "rm -rf #{output_dir}", verbose: debug if @clean_output_dir
+  end
+
+  def bootstrap!
+    sh "mkdir -p #{work_dir}", verbose: debug
+    sh "mkdir -p #{output_dir}", verbose: debug
+  end
+
+  def tar!
+    cmd = ['git', 'ls-files', '.', '|', tar, '-c']
+    if build_box.rpm?
+      dir = rpm_tar_dest_dir
+      version = rpm_version
+    else
+      dir = deb_tar_dest_dir.tap { |p| sh "mkdir -p #{p}" }
+      version = deb_version
+      cmd.concat ["--transform 'flags=r;s,packaging/deb,debian,'"]
+    end
+    tar_file = "#{dir}/#{package_name}-#{version}.tar.gz"
+    cmd.concat ["--transform 's,^,#{package_name}-#{version}/,'"]
+    cmd.concat ['-T', '-', '|', "gzip > #{tar_file}"]
+
+    sh "rm #{tar_file}" if File.exist?(tar_file)
+    puts "Create tar archive #{tar_file}".blue
+    Dir.chdir(tar_path) do
+      sh cmd.join(' '), verbose: debug
+    end
+  end
+
+  def run!
+    if tar_only?
+      tar!
+      return
+    end
+    clean!
+    bootstrap!
+    tar! if tar?
+    container_start!
+    container_exec!(exec_rake)
+  rescue RuntimeError
+    # ret = 1
+    puts "Build FAILED package=#{package} dist=#{build_box.dist}".red
+    raise
+  else
+    puts "Build SUCCESS: package=#{package} dist=#{build_box.dist}".green
+  ensure
+    container_exec!(exec_attach, ['-i', '-t']) if attach?
+    container_kill! if container_running?
+  end
+
+  def container_running?
+    `#{container_runtime} inspect #{container_name} 2>/dev/null 1>/dev/null`
+    $CHILD_STATUS.success?
+  end
+
+  def container_start!
+    cmd = [container_runtime, 'run', '--detach', '--rm']
+    cmd.concat ['--name', container_name]
+    cmd.concat rt_specific_flags
+    cmd.concat container_mounts
+    cmd.concat [build_box.image_tag]
+    cmd.concat [container_init]
+    cmd.concat ['1>/dev/null'] unless debug
+    puts "Starting container #{container_name} using image #{build_box.image_tag}".blue
+    sh cmd.join(' '), verbose: debug
+  end
+
+  def container_exec!(exec_cmd, extra_args = [])
+    cmd = [container_runtime, 'exec']
+    cmd.concat extra_args
+    container_env.each_pair do |k, v|
+      cmd.concat ['-e', "'#{k}=#{v}'"] unless v.nil?
+    end
+    cmd.concat [container_name]
+    cmd.concat exec_cmd
+    puts "Build STARTED: package=#{package} dist=#{build_box.dist} exec=#{exec_cmd[-1]}".blue
+    sh cmd.join(' '), verbose: debug
+  end
+
+  def container_kill!
+    puts "Killing container #{container_name}".blue
+    cmd = [container_runtime, 'kill', container_name]
+    cmd.concat ['1>/dev/null', '2>/dev/null'] unless debug
+    sh cmd.join(' '), verbose: debug
+  end
+
+  def container_env
+    env = {
+      'DIST'          => build_box.dist,
+      'PACKAGE'       => package_name,
+      'GPG_SIGN'      => gpg_sign,
+      'GPG_NAME'      => gpg_name,
+      'SKIP_DOWNLOAD' => @config[:skip_download],
+      'OOD_UID'       => Process.uid,
+      'OOD_GID'       => Process.gid,
+      'DEBUG'         => debug
+    }
+    env['VERSION'] = rpm_version if build_box.rpm?
+    env['VERSION'] = deb_version if build_box.deb?
+    env['GPG_PUBKEY'] = '/gpg.pub' if @config[:gpg_pubkey]
+    env
+  end
+end

data/lib/ood_packaging/rake_task.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'ood_packaging/options'
+require 'ood_packaging/package'
+require 'ood_packaging/utils'
+require 'rake'
+require 'rake/tasklib'
+
+# Define OodPackaging rake task library
+class OodPackaging::RakeTask < ::Rake::TaskLib
+  include ::Rake::DSL if defined?(::Rake::DSL)
+  include OodPackaging::Utils
+
+  OodPackaging::OPTIONS.each do |o|
+    attr_accessor o.to_sym
+  end
+
+  def initialize(*args, &task_block)
+    super()
+    @name = args.shift || :package
+
+    define(args, &task_block)
+  end
+
+  def define(args, &task_block)
+    desc 'Build package' unless ::Rake.application.last_description
+
+    task @name, *args do |_, task_args|
+      task_block&.call(*[self, task_args].slice(0, task_block.arity))
+      config = {}
+      OodPackaging::OPTIONS.each do |o|
+        v = instance_variable_get("@#{o}")
+        config[o.to_sym] = v unless v.nil?
+      end
+      OodPackaging::Package.new(config).run!
+    end
+  end
+
+  def name
+    @name.to_s
+  end
+end

data/lib/ood_packaging/string_ext.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# https://stackoverflow.com/a/11482430
+class String
+  def colorize(code)
+    "\e[#{code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def blue
+    colorize(34)
+  end
+end

data/lib/ood_packaging/tasks.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'ood_packaging'
+require 'ood_packaging/rake_task'
+require 'ood_packaging/utils'
+
+namespace :ood_packaging do
+  include OodPackaging::Utils
+
+  namespace :buildbox do
+    desc 'Build buildbox image'
+    task :build, [:dist] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.build!
+    end
+
+    desc 'Push buildbox image'
+    task :push, [:dist] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.push!
+    end
+
+    desc 'Save buildbox image'
+    task :save, [:dist, :path] do |_task, args|
+      @build_box = OodPackaging::BuildBox.new(args)
+      @build_box.save!(args[:path])
+    end
+  end
+
+  namespace :package do
+    desc 'Build a package (INSIDE CONTAINER ONLY)'
+    task :build do
+      OodPackaging::Build.new.run!
+    end
+
+    OodPackaging::RakeTask.new(:internal, [:package, :dist]) do |t, args|
+      name = args[:package].split(':').last
+      t.package = File.join(proj_root, 'packages', name)
+      t.dist = args[:dist]
+      t.version = OodPackaging.package_version(name, args[:dist])
+      t.work_dir = File.join(proj_root, 'tmp/work')
+      t.output_dir = File.join(proj_root, 'tmp/output')
+    end
+
+    desc 'Package ondemand-release'
+    task :'ondemand-release', [:dist] do |t, args|
+      ENV['OOD_PACKAGING_GPG_SIGN'] = 'false'
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand-release-latest'
+    task :'ondemand-release-latest', [:dist] do |t, args|
+      ENV['OOD_PACKAGING_GPG_SIGN'] = 'false'
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand-release'
+    task :'ondemand-runtime', [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package passenger'
+    task :passenger, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package cjose'
+    task :cjose, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package mod_auth_openidc'
+    task :mod_auth_openidc, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package sqlite'
+    task :sqlite, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand_exporter'
+    task :ondemand_exporter, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package ondemand-compute'
+    task :'ondemand-compute', [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package python-websockify'
+    task :'python-websockify', [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+
+    desc 'Package turbovnc'
+    task :turbovnc, [:dist] do |t, args|
+      Rake::Task['ood_packaging:package:internal'].invoke(t.name, args[:dist])
+    end
+  end
+end

data/lib/ood_packaging/utils.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'erb'
+
+# Shared utility functions
+module OodPackaging::Utils
+  def proj_root
+    File.expand_path(File.join(File.dirname(__FILE__), '../..'))
+  end
+
+  def tar
+    @tar ||= begin
+      `which gtar 1>/dev/null 2>&1`
+      $CHILD_STATUS.success? ? 'gtar' : 'tar'
+    end
+  end
+
+  def podman_runtime?
+    @podman_runtime ||= ENV['CONTAINER_RT'] == 'podman' || ENV['container'] == 'podman'
+  end
+
+  def docker_runtime?
+    !podman_runtime?
+  end
+
+  def container_runtime
+    podman_runtime? ? 'podman' : 'docker'
+  end
+
+  def rt_specific_flags
+    if podman_runtime?
+      # SELinux doesn't like it if you're mounting from $HOME
+      [
+        '--security-opt', 'label=disable'
+      ]
+    else
+      ['--privileged', '--tty']
+    end
+  end
+
+  def template_file(filename)
+    cwd = File.expand_path(__dir__).to_s
+    src = File.join(cwd, filename)
+    dest = File.join(cwd, filename.gsub('.erb', ''))
+    content = ERB.new(File.read(src), trim_mode: '-').result(binding)
+    File.open(dest, 'w') { |f| f.write(content) }
+    dest
+  end
+
+  def ondemand_repo_version
+    '2.1'
+  end
+
+  def ruby_version
+    '2.7'
+  end
+
+  def scl_ruby
+    "rh-ruby#{ruby_version.tr('.', '')}"
+  end
+
+  def nodejs_version
+    '14'
+  end
+
+  def ctr_scripts_dir
+    '/ondemand-packaging'
+  end
+
+  def gpg_private_key
+    File.join(ctr_scripts_dir, 'ondemand.sec')
+  end
+
+  def gpg_passphrase
+    File.join(ctr_scripts_dir, '.gpgpass')
+  end
+
+  def ctr_user
+    'ood'
+  end
+
+  def ctr_home
+    File.join('/home', ctr_user)
+  end
+
+  def ctr_rpmmacros
+    File.join(ctr_home, '.rpmmacros')
+  end
+
+  def ctr_gpg_dir
+    File.join(ctr_home, '.gnupg')
+  end
+end