litbuild 1.0.1

data/lib/litbuild/blueprint.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: false
+
+require 'stringio'
+require 'litbuild/blueprint_parser'
+require 'litbuild/errors'
+require 'litbuild/logfile_namer'
+require 'litbuild/visitor'
+
+module Litbuild
+  class Blueprint
+    class <<self
+      # This should simply be the name of the directory where blueprints
+      # of each specific type are expected to be found by Driver.
+      def self.directory_name
+        raise(SubclassResponsibility)
+      end
+    end
+
+    attr_reader :active_phase
+    attr_reader :base_grafs
+    attr_writer :logfile_namer
+
+    def self.descendants
+      ObjectSpace.each_object(Class).select { |c| c < self }
+    end
+
+    # WARNING: in most cases, a freshly-created blueprint cannot be used
+    # until it has also been _prepared_. (See below.)
+    def initialize(text:)
+      parser = BlueprintParser.new(text)
+      @base_directives = parser.base_directives
+      @phase_directives = parser.phase_directives
+      @base_grafs = parser.base_grafs
+      @phase_grafs = parser.phase_grafs
+      @active_phase = @base_directives['default-phase']&.first
+    end
+
+    # This prepares a blueprint for use. This is separate from
+    # initialize because parameters are not available until after all
+    # blueprints have been parsed, and the logfile namer is not
+    # available until after the LOGFILE_DIR parameter can be resolved.
+    #
+    # Parameters is a set of configuration parameters
+    # Logfile_namer is used to generate log file names
+    # Library is the BlueprintLibrary, used (for example) to find
+    # dependencies.
+    def prepare(parameters:, logfile_namer:, library:)
+      @logfile_namer = logfile_namer
+      @bp_library = library
+      [@base_directives,
+       @phase_directives,
+       @base_grafs,
+       @phase_grafs].each { |component| resolve_all(parameters, component) }
+    end
+
+    # Dependencies need to be handled before their dependants; that's
+    # handled here. Subclasses should run `super` before doing anything
+    # else with the Visitor, or call the send_to_dependencies method
+    # directly, so that this happens properly!
+    def accept(visitor:)
+      send_to_dependencies(visitor: visitor)
+    end
+
+    # Return a copy of this blueprint that is initialized to operate on
+    # the specified phase.
+    def for_phase(new_active_phase)
+      return self unless phases?
+
+      phased_blueprint = clone
+      phased_blueprint.phase = new_active_phase if new_active_phase
+      phased_blueprint
+    end
+
+    def phases
+      @phase_directives.keys
+    end
+
+    def phases?
+      !@phase_directives.empty?
+    end
+
+    def phase_grafs
+      @phase_grafs[active_phase]
+    end
+
+    def directives
+      if active_phase
+        @phase_directives[active_phase]
+      else
+        @base_directives
+      end
+    end
+
+    def [](directive)
+      directives[directive]
+    end
+
+    # This is just like the [] operator method except that it always
+    # returns only the first value for a directive -- which is helpful
+    # for all the directives that are only ever supposed to have a
+    # single value, like `name` or `full-name`.
+    def value(directive)
+      self[directive]&.first
+    end
+
+    def name
+      value('name')
+    end
+
+    def name_with_phase
+      if active_phase
+        "#{name}::#{active_phase}"
+      else
+        name
+      end
+    end
+
+    def full_name
+      value('full-name')
+    end
+
+    def file_name
+      if active_phase
+        "#{name}-#{active_phase}"
+      else
+        name
+      end.tr(' ', '-')
+    end
+
+    def header_text
+      value('full-name')
+    end
+
+    def header_text_with_phase
+      "#{header_text} (#{active_phase} phase)"
+    end
+
+    def logfile(stage_name, phase_name = nil)
+      phase = phase_name&.tr(' ', '_')
+      @logfile_namer.path_for(name, phase, stage_name)
+    end
+
+    def parameter_defaults
+      values = {}
+      all_directive_sets = [@base_directives, @phase_directives.values].flatten
+      all_directive_sets.each do |dirset|
+        next unless dirset.key?('parameter')
+
+        dirset['parameter'].each do |a_param|
+          val = a_param['default'].first
+          values[a_param['name'].first] = if val == '(empty)'
+                                            ''
+                                          else
+                                            val
+                                          end
+        end
+      end
+      values
+    end
+
+    def target_name
+      if active_phase
+        "#{name}::#{active_phase}"
+      else
+        name
+      end
+    end
+
+    def success_line
+      'SUCCESS'
+    end
+
+    def failure_line
+      'FAILURE'
+    end
+
+    # If a dependency is declared both without a phase (typically in the
+    # base directives for the blueprint) and with a phase (typically in
+    # a phase of that blueprint), throw away the phaseless declaration
+    # to avoid including two versions of the dependency.
+    def deduped_dependency_names
+      dep_names = self['depends-on'].clone || []
+      deps_with_phase = dep_names.select { |dep| dep.match?(/::/) }
+      deps_with_phase.each do |dep|
+        dep_without_phase = dep.split('::')[0]
+        dep_names.delete(dep_without_phase)
+      end
+      dep_names
+    end
+
+    protected
+
+    def send_to_dependencies(visitor:)
+      @bp_library.dependencies_for(self).each do |bp|
+        bp.accept(visitor: visitor)
+      end
+    end
+
+    def phase=(phase)
+      bptype = self.class.name.split('::').last
+      msg = "#{bptype} #{name} does not have a phase '#{phase}'"
+      raise(InvalidParameter, msg) unless phase.nil? || phases.include?(phase)
+
+      @active_phase = phase
+    end
+
+    private
+
+    def resolve_all(parameters, something)
+      case something
+      when Hash
+        something.values.each { |element| resolve_all(parameters, element) }
+      when Array
+        something.each { |element| resolve_all(parameters, element) }
+      when String
+        resolve(parameters: parameters, a_string: something)
+      end
+    end
+
+    def resolve(parameters:, a_string:)
+      params = a_string.scan(/PARAM\[([A-Z_]+)\]/).flatten.sort.uniq
+      params.each do |p|
+        unless parameters[p]
+          raise(ParameterMissing, "Parameter #{p} is not defined")
+        end
+
+        a_string.gsub!(/PARAM\[#{p}\]/, parameters[p])
+      end
+      a_string
+    end
+  end
+end

data/lib/litbuild/blueprint_library.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'litbuild/commands.rb'
+require 'litbuild/narrative.rb'
+require 'litbuild/package.rb'
+require 'litbuild/section.rb'
+
+module Litbuild
+  # BlueprintLibrary initializes and sets configuration parameters for
+  # blueprints, and provides a common access point for blueprints and
+  # parameters. It always loads blueprints from the *current working
+  # directory* and uses config parameters from *the environment*.
+  class BlueprintLibrary
+    REQUIRED_PARAMS = %w[DOCUMENT_DIR LOGFILE_DIR PATCH_DIR
+                         SCRIPT_DIR TARFILE_DIR WORK_SITE].freeze
+
+    attr_reader :blueprints, :parameters
+
+    def initialize(logfile_namer_class:)
+      @blueprints = {}
+      Blueprint.descendants.each do |blueprint_type|
+        load_blueprints_of_type(blueprint_type)
+      end
+      @parameters = resolve_parameter_values
+      log_namer = logfile_namer_class.new(@parameters['LOGFILE_DIR'])
+      @blueprints.each_value do |bp|
+        bp.prepare(parameters: @parameters,
+                   logfile_namer: log_namer,
+                   library: self)
+      end
+    end
+
+    def blueprint_for(target:)
+      name, phase = split_name_and_phase(target: target)
+      bp = blueprints[name]
+      unless bp
+        raise(UnknownBlueprint, "Blueprint #{name} not found in library")
+      end
+
+      bp.for_phase(phase)
+    end
+
+    # Convert the dependency declarations found in a `depends-on`
+    # directive to actual blueprints. See the `Dependencies` section of
+    # doc/blueprints.txt for details on how this works.
+    def dependencies_for(blueprint)
+      dep_names = blueprint.deduped_dependency_names
+      dep_names.map do |dep|
+        if dep.match?(/::/) # Explicit phase specified
+          blueprint_for(target: dep)
+        else
+          bp = blueprint_for(target: dep)
+          if bp.phases&.include?(blueprint.active_phase)
+            bp.for_phase(blueprint.active_phase)
+          else
+            bp
+          end
+        end
+      end
+    end
+
+    private
+
+    # split a target name, like linux::headers, into a blueprint name
+    # (linux) and a phase name (headers).
+    def split_name_and_phase(target:)
+      if target.match?(/::/)
+        target.strip.match(/(.*)::(.*)/)[1..2]
+      else
+        [target.strip, nil]
+      end
+    end
+
+    # These methods are used during initialization, be cautious when
+    # modifying them.
+
+    def load_blueprints_of_type(blueprint_type)
+      # Sometimes, starting in ruby 2.3, there is an extra singleton
+      # descendant of Blueprint, or something. (This happens about a
+      # thrid of the time, so I really mean "soemtime"s.) IDK what's
+      # going on and I don't feel like fussing with it; this kludge
+      # works around the issue, whatever it is.
+      return unless blueprint_type.name
+
+      Dir.glob("./#{blueprint_type.directory_name}/*.txt").each do |a_file|
+        bp_text = File.read(a_file)
+        begin
+          # All blueprints have a name and a full-name. If a blueprint
+          # has no `name:` or `full-name:` directive, they will be
+          # provided at load time.
+          %w[name full-name].each do |directive|
+            unless bp_text.match?(/^#{directive}:/)
+              bp_text = "#{directive}: #{File.basename(a_file, '.txt')}\n\n" +
+                        bp_text
+            end
+          end
+          bp = blueprint_type.new(text: bp_text)
+        rescue ArgumentError => e
+          raise(Litbuild::ParseError, "Could not parse #{a_file}: #{e.message}")
+        end
+        if @blueprints[bp.name]
+          raise(DuplicateBlueprint,
+                "Duplicate blueprint #{bp.name} found in #{a_file}")
+        else
+          @blueprints[bp.name] = bp
+        end
+      end
+    end
+
+    def resolve_parameter_values
+      values = {}
+      REQUIRED_PARAMS.each { |key| values[key] = 'UNSET' }
+      @blueprints.each_value do |bp|
+        values.merge!(bp.parameter_defaults)
+      end
+      ENV.each do |k, v|
+        values[k] = v if values.key?(k)
+      end
+      @parameters = values
+    end
+  end
+end

data/lib/litbuild/blueprint_parser.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: false
+
+require 'litbuild/errors'
+require 'json'
+
+module Litbuild
+  # This is a kludgy hand-built parser. Blueprint structure is not (at
+  # least, at this point) complicated enough that I can see any point in
+  # defining a grammar and using a parser generator. The structure of
+  # blueprint directives is described informally in
+  # `doc/blueprints.txt` (and blueprint-type-specific files under `doc`
+  # as well).
+  #
+  # tl;dr: each paragraph can be either directives or narrative.
+  # Narrative is AsciiDoc and does not get parsed or transformed.
+  # Directives are basically a simplified version of YAML.
+  class BlueprintParser
+    # _directives_ are for use in scripts. _grafs_ are for use in
+    # documents.
+    attr_reader :base_directives, :phase_directives, :base_grafs, :phase_grafs
+
+    def initialize(file_text)
+      @text = file_text
+      @phase_directives = {}
+      @phase_grafs = {}
+      phase_chunks = @text.split(/^phase: ([a-z -_]+)$/)
+
+      # The first part of the blueprint, before any phase directive,
+      # becomes the base directives and base narrative for the
+      # blueprint. If there are no phase directives, this is the entire
+      # blueprint, obvs.
+      @base_grafs = []
+      base = parse_phase(phase_chunks.shift, {}, @base_grafs)
+      base['full-name'] ||= base['name']
+      @base_directives = base
+
+      # The rest of the blueprint, if any, consists of directives and
+      # narrative for specific phases. The directives for each phase
+      # include all the base directives, as well as those specific to
+      # the phase.
+      until phase_chunks.empty?
+        phase_name = phase_chunks.shift
+        phase_contents = phase_chunks.shift
+        grafs = []
+        @phase_directives[phase_name] = parse_phase(phase_contents, base, grafs)
+        @phase_grafs[phase_name] = grafs
+      end
+
+      # Any directives at the beginning of a blueprint are actually a
+      # file header that should not be rendered as part of the narrative
+      # for it.
+      @base_grafs.shift while @base_grafs[0].is_a?(Hash)
+    rescue StandardError => e
+      msg = "Cannot parse blueprint starting: #{@text.lines[0..3].join}"
+      raise(Litbuild::ParseError, "#{msg} -- #{e}")
+    end
+
+    private
+
+    # Parse all directive paragraphs found in blueprint_text. Also add
+    # all parsed directive paragraphs, and all narrative paragraphs, to
+    # a collecting parameter.
+    def parse_phase(blueprint_text, start_with_directives, graf_collector)
+      directives = JSON.parse(JSON.generate(start_with_directives))
+      paragraphs = blueprint_text.split(/\n\n+/m)
+      paragraphs.each do |paragraph|
+        if directives?(paragraph)
+          parsed = parse_paragraph(paragraph)
+          add_directives(directives, parsed)
+          graf_collector << parsed
+        else
+          graf_collector << paragraph
+        end
+      end
+      handle_servicedirs(directives)
+      directives
+    end
+
+    # All s6-rc service directories should be tracked in the
+    # configuration file repository, so add them to
+    # `configuration-files`.
+    def handle_servicedirs(directives)
+      cfgs = directives['configuration-files'] || []
+      if (spipes = directives['service-pipeline'])
+        spipes.each do |a_pipe|
+          a_pipe['servicedirs'].each do |a_svc|
+            cfgs << "/etc/s6-rc/source/#{a_svc['name'].first}"
+          end
+        end
+      end
+      if (sdirs = directives['servicedir'])
+        sdirs.each do |a_svc|
+          cfgs << "/etc/s6-rc/source/#{a_svc['name'].first}"
+        end
+      end
+      directives['configuration-files'] = cfgs unless cfgs.empty?
+    end
+
+    def directives?(paragraph)
+      paragraph.split(' ').first =~ /^[a-z-]+:/
+    end
+
+    def add_directives(directives, parsed)
+      directives.merge!(parsed) do |_key, old_val, new_val|
+        [old_val, new_val].flatten
+      end
+    end
+
+    def parse_paragraph(paragraph)
+      lines_to_process = paragraph.lines.map(&:rstrip)
+      parse_lines(lines_to_process)
+    end
+
+    def parse_lines(lines_to_process)
+      graf_directives = Hash.new { |h, k| h[k] = [] }
+      until lines_to_process.empty?
+        directive_line = lines_to_process.shift
+        md = /^([A-Za-z0-9_-]+): *(.*)/.match(directive_line)
+        unless md
+          raise(Litbuild::ParseError,
+                "Expected '#{directive_line}' to be a directive")
+        end
+        directive_name = md[1]
+        firstline_value = md[2]
+        value_lines = related_lines(directive_line, lines_to_process)
+        value = parse_directive_value(firstline_value, value_lines)
+        if value == "''"
+          # two literal apostrophes is a special case, we treat it as an
+          # empty string. Probably ought to document that.
+          graf_directives[directive_name] << ''
+        else
+          graf_directives[directive_name] << value
+        end
+        graf_directives[directive_name].flatten!
+      end
+      graf_directives
+    end
+
+    # Regardless of what kind of directive structure we're dealing with,
+    # all of the lines that are indented more than the initial directive
+    # line are part of that directive.  This method finds all those
+    # related lines and strips away the common initial indentation.
+    def related_lines(directive_line, lines_to_process)
+      directive_indent = indent_for(directive_line)
+      related = []
+      while !lines_to_process.empty? &&
+            (indent_for(lines_to_process[0]) > directive_indent)
+        related << lines_to_process.shift
+      end
+      common_indent = related.map { |l| indent_for(l) }.min
+      related.map { |l| l.slice(common_indent..-1) }
+    end
+
+    # What kind of directive are we dealing with? Could be a simple
+    # value (with zero or more continuation lines), or a multiline
+    # value, or an array, or a subdirective block.
+    def parse_directive_value(firstline_value, other_lines)
+      if firstline_value == '|'
+        multiline_value(other_lines)
+      elsif other_lines.empty?
+        firstline_value
+      elsif other_lines[0].match?(/^ *- /)
+        array_value(other_lines)
+      elsif other_lines[0].match?(/^[A-Za-z_-]+: /)
+        subdirective_value(other_lines)
+      else
+        folded_continuation_lines(firstline_value, other_lines)
+      end
+    end
+
+    # any time a directive line is indented more than the previous line,
+    # without being a part of an array or sub-directive or multi-line
+    # directive, it's a continuation of the previous line.
+    def folded_continuation_lines(first_line, related)
+      stripped = related.map(&:strip)
+      stripped.unshift(first_line)
+      stripped.join(" \\\n  ")
+    end
+
+    def multiline_value(lines)
+      lines.join("\n") + "\n"
+    end
+
+    def array_value(lines)
+      value = []
+      until lines.empty?
+        array_member = lines.shift
+        firstline_value = /^- *(.*)$/.match(array_member)[1]
+        related = related_lines(array_member, lines)
+        value << parse_directive_value(firstline_value, related)
+      end
+      value
+    rescue StandardError
+      raise(Litbuild::ParseError, "Problem parsing: #{lines}")
+    end
+
+    # It turns out that sub-directives are the easiest thing in the
+    # world to parse, since we can just take the value and parse it as a
+    # new directive block.
+    def subdirective_value(lines)
+      parse_lines(lines)
+    end
+
+    # Utility method to find the amount of blank space at the beginning
+    # of a line.
+    def indent_for(line)
+      /^([[:blank:]]*).*/.match(line)[1].size
+    end
+  end
+end

data/lib/litbuild/commands.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'litbuild/blueprint'
+
+module Litbuild
+  class Commands < Blueprint
+    def self.directory_name
+      'commands'
+    end
+
+    def accept(visitor:)
+      super
+      visitor.visit_commands(commands: self)
+    end
+
+    def files
+      return @files if @files
+
+      @files = {}
+      (directives['file'] || []).each do |a_file|
+        add_file_content(a_file)
+      end
+      @files
+    end
+
+    protected
+
+    def add_file_content(directive)
+      unless directive['name'] && directive['content']
+        raise(InvalidDirective, 'file directive missing name or content')
+      end
+
+      content = @files[directive['name'].first] ||= StringIO.new
+      content.puts(directive['content'].first)
+    end
+  end
+end

data/lib/litbuild/driver.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'litbuild/blueprint_library'
+require 'litbuild/ascii_doc_visitor'
+require 'litbuild/bash_script_visitor'
+require 'litbuild/source_files_visitor'
+require 'litbuild/url_visitor'
+
+module Litbuild
+  # This is what the command-line program `lb` uses to do all the work,
+  # aside from parsing the command line. It initializes a blueprint
+  # library and provides methods for exercising all the functionality of
+  # litbuild: writing bash scripts, writing documents, querying package
+  # details, etc.
+  #
+  # All the actual work is done by Visitors that are handed to the
+  # top-level blueprint target (typically specified on the command line).
+  class Driver
+    REQUIRED_PARAMS = %w[DOCUMENT_DIR LOGFILE_DIR PATCH_DIR
+                         SCRIPT_DIR TARFILE_DIR WORK_SITE].freeze
+
+    def initialize(logfile_namer_class: Litbuild::LogfileNamer)
+      @bplib = BlueprintLibrary.new(logfile_namer_class: logfile_namer_class)
+    end
+
+    def library
+      @bplib
+    end
+
+    def params
+      @bplib.parameters
+    end
+
+    def download_urls_for(target:)
+      uv = UrlVisitor.new
+      dispatch(visitor: uv, target: target)
+      uv.urls
+    end
+
+    def source_files_for(target:)
+      sfv = SourceFilesVisitor.new
+      dispatch(visitor: sfv, target: target)
+      sfv.files
+    end
+
+    def write_scripts_for(target:)
+      bsv = BashScriptVisitor.new(parameters: params)
+      dispatch(visitor: bsv, target: target)
+      bsv.write_sudoers
+      bsv.write_toplevel_script(target: target)
+    end
+
+    def write_document_for(target:)
+      adv = AsciiDocVisitor.new(parameters: params)
+      bp = library.blueprint_for(target: target)
+      bp.accept(visitor: adv)
+      adv.write_toplevel_doc(blueprint: bp)
+    end
+
+    private
+
+    def dispatch(visitor:, target:)
+      library.blueprint_for(target: target).accept(visitor: visitor)
+    end
+  end
+end

data/lib/litbuild/errors.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Litbuild
+  # Superclass for all Litbuild errors
+  class Error < RuntimeError; end
+
+  # One or more required configuration parameters has not been provided
+  class ParameterMissing < Error; end
+
+  # A child class of Blueprint is expected to have defined a method, but
+  # has not done so.
+  class SubclassResponsibility < Error; end
+
+  # A blueprint was asked to build a phase that has not been defined.
+  class InvalidParameter < Error; end
+
+  # A blueprint directive is missing the expected structure
+  class InvalidDirective < Error; end
+
+  # The source tarfile and/or patches for a package were not found in
+  # the expected location.
+  class MissingSource < Error; end
+
+  # One of the blueprints referenced in the build set is not defined.
+  class UnknownBlueprint < Error; end
+
+  # A blueprint directive block could not be parsed
+  class ParseError < Error; end
+
+  # More than one blueprint exists with some name.
+  class DuplicateBlueprint < Error; end
+
+  # A command executed through sudo does not have an absolute path.
+  class RelativeSudo < Error; end
+
+  # A blueprints directive specifies a blueprint that has already been
+  # included elsewhere.
+  class UnrenderedComponent < Error; end
+end