litbuild 1.0.1

data/lib/litbuild/bash_script_visitor.rb

@@ -0,0 +1,547 @@
+# frozen_string_literal: true
+
+require 'litbuild/service_dir'
+require 'litbuild/source_code_manager'
+require 'litbuild/visitor'
+
+module Litbuild
+  ##
+  # This class writes bash scripts to directories, rooted at the
+  # SCRIPT_DIR directory. It adds a sequential number prefix on each
+  # script written to a directory, and ensures that no duplicate scripts
+  # are written: if a script for a specific blueprint (or blueprint
+  # phase) has already been written to any directory, BashScriptVisitor
+  # will ignore subsequent requests to write that blueprint (phase).
+  class BashScriptVisitor < Visitor
+    INSTALL_GID = 9999
+
+    attr_reader :blueprint_dir
+
+    def initialize(parameters:)
+      @parameters = parameters
+      super(directory: @parameters['SCRIPT_DIR'])
+      @written = Hash.new { |hash, key| hash[key] = [] }
+      @all_written_targets = []
+      @all_commands = []
+      @blueprint_dir = quote(File.expand_path('.'))
+      @scm = SourceCodeManager.new(@parameters['TARFILE_DIR'],
+                                   @parameters['PATCH_DIR'])
+    end
+
+    def visit_commands(commands:)
+      write(blueprint: commands, location: cwd) do |script|
+        phase = commands.active_phase
+        restart_file = if phase
+                         "#{commands.name}::#{phase.tr(' ', '_')}"
+                       else
+                         commands.name
+                       end
+        render_restart_header(script, restart_file)
+        log = commands.logfile(commands.name, phase)
+        cmds = commands['commands'] || []
+        files = handle_file_directives(commands)
+        render_servicedirs(script: script,
+                           dirs: commands['servicedir'],
+                           pipelines: commands['service-pipeline'])
+        cmds = [files, cmds].flatten
+        cmds.each do |command|
+          render_command(script, command, log)
+        end
+        render_cfgrepo_trailer(script, commands, log)
+        render_restart_trailer(script, restart_file)
+      end
+    end
+
+    def visit_package(package:)
+      write(blueprint: package, location: cwd) do |script|
+        if (File.stat('/etc').gid == INSTALL_GID) &&
+           Process.uid.zero? &&
+           (ENV['LITBUILD_PKGUSR'] != 'false')
+          pkgusr = { 'name' => [package.name],
+                     'description' => package['full-name'] }
+          package.directives['package-user'] ||= [pkgusr]
+        end
+        if package.directives.include?('package-user')
+          render_package_user(package, script)
+        else
+          render_standard_package(package, script)
+        end
+      end
+    end
+
+    def visit_section(section:)
+      section_dir = File.join(cwd, section.name)
+      write(blueprint: section, location: cwd) do |script|
+        render_restart_header(script, section.name)
+        script.puts("cd $(dirname $0)/#{section.name}")
+        skip_line(script)
+        write_components(location: section_dir, script: script)
+        render_restart_trailer(script, section.name)
+      end
+    end
+
+    def write_sudoers
+      script_dir = @parameters['SCRIPT_DIR']
+      sudoers = sudoers_entries.sort
+      return if sudoers.empty?
+
+      full_path = File.join(script_dir, 'lb-sudoers')
+      File.open(full_path, 'w') do |f|
+        sudoers.each do |s|
+          f.puts(s)
+        end
+      end
+    end
+
+    def write_toplevel_script(target:)
+      script_dir = @parameters['SCRIPT_DIR']
+      return if @written[script_dir].empty?
+
+      script_name = File.join(script_dir, "#{target}.sh")
+      File.open(script_name, 'w') do |f|
+        f.puts("#!#{find_bash}/bash")
+        skip_line(f)
+        f.puts("trap 'echo UTTER FAILURE on line $LINENO' ERR")
+        f.puts('set -e -v')
+        skip_line(f)
+        write_components(location: script_dir, script: f)
+        f.puts('set +v')
+        f.puts('echo TOTAL SUCCESS')
+      end
+      FileUtils.chmod('ugo+x', script_name)
+    end
+
+    private
+
+    def write(blueprint:, location:, &block)
+      return if @all_written_targets.include?(blueprint.target_name)
+
+      FileUtils.mkdir_p(location)
+      script_name = format('%<count>02d-%<script>s',
+                           count: @written[location].size,
+                           script: blueprint.file_name + '.sh')
+      script_path = File.join(location, script_name)
+      File.open(script_path, 'w') do |f|
+        f.write(to_bash_script(blueprint, block))
+      end
+      FileUtils.chmod('ugo+x', script_path)
+      envcmds = environment_commands(blueprint).reject { |c| c =~ /^mkdir/ }
+      unless envcmds.empty?
+        envscript = File.join(location, "#{blueprint.file_name}_env.sh")
+        File.open(envscript, 'w') do |f|
+          envcmds.each { |cmd| f.puts(cmd) }
+        end
+        FileUtils.chmod('ugo+x', envscript)
+      end
+      @written[location] << script_name
+      @all_written_targets << blueprint.target_name
+    end
+
+    def to_bash_script(blueprint, block)
+      if blueprint.phases? && !blueprint.active_phase
+        raise(ParameterMissing,
+              "Phase must be set to render script for #{blueprint.name}")
+      end
+
+      script = StringIO.new
+      script.puts("#!#{find_bash}/bash")
+      skip_line(script)
+      script.puts("export LB_BLUEPRINT_DIR=#{blueprint_dir}")
+      script.puts("trap 'echo #{blueprint.failure_line} on line $LINENO' ERR")
+      script.puts('set -e -v')
+      skip_line(script)
+      env_cmds = environment_commands(blueprint)
+      unless env_cmds.empty?
+        env_cmds.each do |cmd|
+          script.puts(cmd)
+        end
+        skip_line(script)
+      end
+      block.call(script)
+      skip_line(script)
+      script.puts('set +v')
+      script.puts("echo #{blueprint.success_line}")
+      script.string
+    end
+
+    def write_components(location:, script:)
+      @written[location].map do |target|
+        script.puts("echo \"At $(date): Beginning #{target}:\"")
+        script.puts("./#{target}")
+      end
+    end
+
+    def find_bash
+      ENV['PATH'].split(':').detect { |pe| File.exist?(File.join(pe, 'bash')) }
+    end
+
+    def handle_file_directives(blueprint)
+      blueprint.files.keys.sort.map do |name|
+        accum = StringIO.new
+        accum.puts("cat > #{name} <<'LBEOF'")
+        accum.puts(blueprint.files[name].string)
+        accum.puts('LBEOF')
+        accum.string
+      end
+    end
+
+    def sudoers_entries
+      uid = `id -u`.strip
+      return [] if uid == '0'
+
+      raw_sudo_cmds = @all_commands.select do |c|
+        c =~ /sudo / && c.lines.size < 2
+      end.uniq
+      sudo_cmds = raw_sudo_cmds.map do |c|
+        command = c.sub(/.*sudo /, '')
+        unless command.start_with?('/')
+          base_command = command.sub(/ .*$/, '')
+          raise(RelativeSudo,
+                "Need absolute path for \"#{base_command}\" in \"#{c}\"")
+        end
+        sudoed_cmd = c.sub(/^.*sudo (.*)$/, '\\1')
+        sudoed_cmd = sudoed_cmd.sub(/;.*$/, '') if sudoed_cmd.match?(/;/)
+        sudoed_cmd.gsub(/([,:=\\])/, '\\\\\1')
+      end
+      username = `id -un`.strip
+      sudo_cmds.map do |c|
+        "#{username} ALL = NOPASSWD: #{c}"
+      end
+    end
+
+    def render_standard_package(package, script)
+      script.puts("export LB_SOURCE_DIR=#{quote(source_dir(package))}")
+      phase = package.active_phase
+      restart_file = if phase
+                       "#{package.name}::#{phase.tr(' ', '_')}"
+                     else
+                       package.name
+                     end
+      render_restart_header(script, restart_file, package.version)
+      render_prepare_source(package, script)
+      build_location = dir_for_build(package)
+
+      if package.build_dir
+        render_command(script, "mkdir -p #{build_location}", '/dev/null')
+        skip_line(script)
+      end
+
+      Package::BUILD_STAGES.each do |stage|
+        log = package.logfile(stage, phase)
+        render_in_dir(script, build_location) do
+          package.build_commands(stage).each do |command|
+            render_command(script, command, log)
+          end
+        end
+      end
+      render_restart_trailer(script, restart_file, package.version)
+    end
+
+    def dir_for_build(package)
+      if (dir = package.build_dir)
+        File.expand_path(File.join(source_dir(package), dir))
+      else
+        source_dir(package)
+      end
+    end
+
+    def render_prepare_source(package, script)
+      log = package.logfile('prepare')
+      render_command(script, "if [ ! -d #{source_dir(package)} ]; then", log)
+      render_command(script, "mkdir -p #{work_site}", log)
+      render_in_dir(script, work_site) do
+        render_command(script, @scm.untar_command_for(package), log)
+      end
+      render_in_tree_sources(package, script, log)
+      render_patch_commands(package, script, log)
+      render_command(script, 'fi', log)
+    end
+
+    def work_site
+      @parameters['WORK_SITE']
+    end
+
+    def source_dir(package)
+      File.join(work_site, package.name_and_version)
+    end
+
+    def render_in_tree_sources(package, script, log)
+      intree_commands = @scm.intree_untar_commands_for(package)
+      return if intree_commands.empty?
+
+      render_in_dir(script, source_dir(package)) do
+        intree_commands.each do |cmd|
+          render_command(script, cmd, log)
+        end
+      end
+    end
+
+    def render_patch_commands(package, script, log)
+      patch_commands = @scm.patch_commands_for(package)
+      return if patch_commands.empty?
+
+      render_in_dir(script, source_dir(package)) do
+        patch_commands.each do |command|
+          render_command(script, command, log)
+        end
+      end
+    end
+
+    def render_package_user(package, script)
+      pkgusr = package.pkgusr_name
+      log = package.logfile('pkgusr')
+      pkgusr_dir = "~#{pkgusr}"
+      package.directives['configuration-files'] ||= []
+      package.directives['configuration-files'] << "~#{pkgusr}/options"
+      restart_file = ".#{package.active_phase || 'default'}"
+      render_restart_header(script, restart_file, package.version, pkgusr_dir)
+      pkgusr_srcdir = File.join(pkgusr_dir, package.name_and_version)
+      render_add_package_user(package, script, log)
+      @scm.copy_source_files_commands(package).each do |cp_command|
+        render_command(script, cp_command, log)
+      end
+      script.puts("export LB_SOURCE_DIR=#{quote(pkgusr_srcdir)}")
+      skip_line(script)
+      render_command(script, generate_options_file(package, pkgusr_srcdir), log)
+      skip_line(script)
+      pre_build(package, script, log)
+      render_command(script,
+                     "su -l -c /usr/libexec/pkgusr/build #{pkgusr}",
+                     log)
+      post_build(package, script, log)
+      render_restart_trailer(script, restart_file, package.version, pkgusr_dir)
+    end
+
+    def pre_build(package, script, log)
+      render_servicedirs(script: script,
+                         dirs: package['servicedir'],
+                         pipelines: package['service-pipeline'])
+      return unless (bbar = package['before-build-as-root'])
+
+      bbar.each { |cmd| render_command(script, cmd, log) }
+    end
+
+    def post_build(package, script, log)
+      if (aiar = package['after-install-as-root'])
+        aiar.each { |cmd| render_command(script, cmd, log) }
+      end
+      render_cfgrepo_trailer(script, package, log)
+      render_command(script, 'set_install_dirs', log)
+      render_command(script, 'ldconfig', log)
+    end
+
+    def render_add_package_user(package, script, log)
+      pkgusr = package.value('package-user')
+      desc = pkgusr['description'].first || package.value('full_name')
+      render_command(script,
+                     "add_package_user '#{desc}' #{package.pkgusr_name}",
+                     log)
+    end
+
+    def generate_options_file(package, srcdir)
+      options = StringIO.new
+      options.puts("cat > ~#{package.pkgusr_name}/options <<'LBEOF'")
+      options.puts("export version=#{package.version}")
+      options.puts("export LB_SOURCE_DIR=#{quote(srcdir)}")
+      environment_commands(package).each { |cmd| options.puts(cmd) }
+      package.build_dir && options.puts("export build_dir=#{package.build_dir}")
+      render_intree_commands(package, options)
+      Package::BUILD_STAGES.each do |stage|
+        options.puts("function #{stage}_commands()\n{")
+        cmds = package.build_commands(stage)
+        if cmds.empty?
+          options.puts(':')
+        else
+          cmds.each { |cmd| options.puts(cmd) }
+        end
+        options.puts('}')
+      end
+      render_patches(package, options)
+      options.puts('LBEOF')
+      options.string
+    end
+
+    def render_intree_commands(package, options)
+      return if package.in_tree.empty?
+
+      options.puts('declare -A in_tree')
+      package.in_tree.sort.each do |basename, version, path|
+        options.puts("in_tree[#{basename}-#{version}]=#{path}")
+      end
+    end
+
+    def render_patches(package, options)
+      return if package.patch_files.empty?
+
+      options.puts('declare -a patches')
+      package.patch_files.each_with_index do |name, idx|
+        options.puts("patches[#{idx}]=#{name}")
+      end
+    end
+
+    def render_restart_header(script,
+                              filename,
+                              content = 'COMPLETE',
+                              restart_dir = '"$LITBUILDDBDIR"')
+      path = "#{restart_dir}/#{filename}"
+      script.puts("if [ -d #{restart_dir} -a -w #{restart_dir} ]")
+      script.puts('then')
+      script.puts("if [ -f #{path} ]")
+      script.puts('then')
+      script.puts("grep -q '^#{content}$' #{path} && exit 0")
+      script.puts('fi')
+      script.puts('fi')
+      skip_line(script)
+    end
+
+    def render_restart_trailer(script,
+                               filename,
+                               content = 'COMPLETE',
+                               restart_dir = '"$LITBUILDDBDIR"')
+      path = "#{restart_dir}/#{filename}"
+      skip_line(script)
+      script.puts("if [ -d #{restart_dir} -a -w #{restart_dir} ]")
+      script.puts('then')
+      script.puts("echo \"#{content}\" > #{path}")
+      script.puts('fi')
+    end
+
+    def render_command(script, command, log)
+      @all_commands << command
+      if command.match?(/>/)
+        # redirecting output of command, can't put stdout in log.
+        script.puts(command)
+      else
+        script.puts("#{command} >> #{log} 2>&1")
+      end
+    end
+
+    def render_in_dir(script, dir)
+      script.puts("pushd #{dir}")
+      yield
+      script.puts('popd')
+    end
+
+    def skip_line(script)
+      script.puts
+    end
+
+    def environment_commands(blueprint)
+      commands = []
+      if blueprint['environment']
+        env_directives = blueprint['environment']
+        merged_and_flattened = {}
+        env_directives.each do |env|
+          env.each { |key, val| merged_and_flattened[key] = val.first }
+        end
+        merged_and_flattened.each do |k, v|
+          commands << (v.empty? ? "unset #{k}" : "export #{k}=#{quote(v)}")
+        end
+        if merged_and_flattened.key?('LITBUILDDBDIR') &&
+           !merged_and_flattened['LITBUILDDBDIR'].empty?
+          commands << "mkdir -p #{merged_and_flattened['LITBUILDDBDIR']}"
+        end
+      end
+      commands
+    end
+
+    def render_servicedirs(script:, dirs:, pipelines:)
+      return unless dirs || pipelines
+
+      script.puts('pushd /etc/s6-rc/source')
+      skip_line(script)
+      pipelines&.each { |pipeline| render_service_pipeline(script, pipeline) }
+      dirs&.each { |sdir| render_service_dir(script, sdir) }
+      script.puts('popd')
+      skip_line(script)
+    end
+
+    def render_service_pipeline(script, spipe)
+      pname = spipe['name'].first
+      spipe['bundle']&.each do |bundle|
+        script.puts("grep -q '^#{pname}$' #{bundle}/contents || " \
+                    "echo #{pname} >> #{bundle}/contents")
+      end
+      sdirs = spipe['servicedirs']
+      sdirs.each_with_index do |sdir, i|
+        render_service_dir(script, sdir)
+        if i == sdirs.size - 1
+          script.puts("echo #{pname} > #{sdir['name'].first}/pipeline-name")
+        end
+        if i < sdirs.size - 1
+          next_svc = sdirs[i + 1]['name'].first
+          script.puts("echo #{next_svc} > #{sdir['name'].first}/producer-for")
+        end
+        unless i.zero?
+          prev_svc = sdirs[i - 1]['name'].first
+          script.puts("echo #{prev_svc} > #{sdir['name'].first}/consumer-for")
+        end
+        skip_line(script)
+      end
+    end
+
+    def render_service_dir(script, sdir)
+      sd = ServiceDir.new(sdir)
+      if sd.bundle
+        script.puts("grep -q '^#{sd.name}$' #{sd.bundle}/contents || " \
+                    "echo #{sd.name} >> #{sd.bundle}/contents")
+      end
+      script.puts("mkdir -p #{sd.name}")
+      sd.oneline_files.keys.sort.each do |fn|
+        script.puts("echo #{sd.oneline_files[fn]} > #{sd.name}/#{fn}")
+      end
+      multiline = sd.multiline_files
+      deps = sd.dependencies
+      multiline['dependencies'] = deps.join("\n") unless deps.empty?
+      multiline.keys.sort.each do |filename|
+        # I always terminate here documents in litbuild-generated
+        # scripts with "LBEOF", partly because I'm thinking "Litbuild
+        # End-of-File" but also partly because it makes me think of
+        # Shia LaBoeuf, and then I remember the Rob Cantor song of
+        # that name and giggle.
+        script.puts("cat > #{sd.name}/#{filename} <<'LBEOF'")
+        script.puts(multiline[filename])
+        script.puts('LBEOF')
+      end
+      env = sd.env
+      unless env.empty?
+        script.puts("mkdir -p #{sd.name}/env")
+        env.keys.sort.each do |envvar|
+          script.puts("echo '#{env[envvar]}' > #{sd.name}/env/#{envvar}")
+        end
+      end
+      skip_line(script)
+    end
+
+    def render_cfgrepo_trailer(script, blueprint, log)
+      cfgs = blueprint['configuration-files']
+      return unless cfgs
+
+      render_command(script, "cfggit add #{cfgs.join(' ')}", log)
+      render_command(script, 'cfggit stageall', log)
+      bp = "#{blueprint.class.name.split('::').last} #{blueprint.name}"
+      cmd = "cfggit as-default -m 'Configuration files for #{bp}'"
+      render_command(script, cmd, log)
+    end
+
+    # quote a string suitably for a bash script
+    def quote(value)
+      # if value contains embedded backslash and newline characters, get
+      # rid of those first.
+      oneline = value.gsub(/\\\n  /m, '')
+
+      # now, if the value contains ' -- backslash-quote everything (incl spaces)
+      # if the value contains " -- single-quote the whole thing
+      # if the value contains other punctuation -- double-quote the whole thing
+      if oneline.match?(/'/)
+        oneline.gsub(/([\\ "'`$])/, '\\\\\1')
+      elsif oneline.match?(/"/)
+        "'#{oneline}'"
+      elsif oneline.match?(/[\\ `]/)
+        "\"#{oneline}\""
+      else
+        oneline
+      end
+    end
+  end
+end