qrpm 0.2.0 → 0.3.0

data/lib/qrpm/compiler.rb

@@ -0,0 +1,238 @@
+require 'open3'
+
+module Qrpm
+  # The main result data are #defs and #deps. #keys and #values are the parsed
+  # result of the keys and values of #deps and is used to interpolate strings
+  # when values of dependent variables are known
+  #
+  # #defs is also partitioned into QRPM variables and directories
+  class Compiler
+    # Root node
+    attr_reader :ast # TODO: Rename AST
+
+    # Variable definitions. Map from path to Node
+    attr_reader :defs
+
+    # Map from path to list of variables it depends on. Paths with no
+    # dependencies have an empty list as value
+    attr_reader :deps
+
+    # Dictionary. The dictionary object are compiled into the AST before it is
+    # evaluated so the dictionary elements can be refered to with the usual
+    # $var notation. #dict is automatically augmented with the default system
+    # directory definitions unless :system_dirs is false
+    attr_reader :dict
+
+    # Defaults. Map from key to source expression
+    attr_reader :defaults
+
+    # If :srcdir is true, a default $srcdir variable is prefixed to all local
+    # paths. This is the default. +srcdir:false+ is only used when testing
+    def initialize(dict, system_dirs: true, defaults: true, srcdir: true)
+      constrain dict, Hash
+      @ast = nil
+      @defs = {}
+      @deps = {}
+      @dict = system_dirs ? INSTALL_DIRS.merge(dict) : dict
+      @use_system_dirs = system_dirs
+      @use_defaults = defaults
+      @use_srcdir = srcdir
+    end
+
+    # Parse the YAML source to an AST of Node objects and assign it to #ast.
+    # Returns the AST
+    def parse(yaml)
+      # Root node
+      @ast = RootNode.new
+
+      # Compile standard directories. Also enter them into @defs so that #analyze
+      # can access them by name before @defs is built by #collect_variables
+      STANDARD_DIRS.each { |name| @defs[name] = StandardDirNode.new(@ast, name) } if @use_system_dirs
+
+      # Parse yaml node
+      yaml.each { |key, value|
+        builtin_array = FIELDS[key]&.include?(ArrayNode) || false
+        case [key.to_s, value, builtin_array]
+          in [/[\/\$]/, _, _]; parse_directory_node(@ast, key, value)
+          in [/^#{PATH_RE}$/, String, true]; parse_node(@ast, key, [value])
+          in [/^#{PATH_RE}$/, Array, false]; parse_directory_node(@ast, key, value)
+          in [/^#{PATH_RE}$/, _, _]; parse_node(@ast, key, value)
+        else
+          error "Illegal key: #{key.inspect}"
+        end
+      }
+
+      # Compile and add dictionary to the AST. This allows command line
+      # assignments to override spec file values
+      dict.each { |k,v| [k, ValueNode.new(ast, k.to_s, Fragment::Fragment.parse(v))] }
+
+      # Add defaults
+      DEFAULTS.each { |k,v|
+        next if k == "srcdir" # Special handling of $srcdir below
+        parse_node(@ast, k, v) if !@ast.key?(k)
+      } if @use_defaults
+
+      # Only add a default $srcdir node when :srcdir is true
+      parse_node(@ast, "srcdir", DEFAULTS["srcdir"]) if @use_srcdir && !@ast.key?("srcdir")
+
+      @ast
+    end
+
+    # Analyze and decorate the AST tree. Returns the AST
+    #
+    # The various +check_*+ arguments are only used while testing to allow
+    # illegal but short expressions by suppressing specified checks in
+    # #analyze. The default is to apply all checks
+    def analyze(
+        check_undefined: true, 
+        check_mandatory: true, 
+        check_field_types: true, 
+        check_directory_types: true)
+
+      # Set package/system directory of standard directories depending on the
+      # number of files in the directory
+      dirs = @ast.values.select { |n| n.is_a? DirectoryNode }
+      freq = dirs.map { |d| (d.key_variables & STANDARD_DIRS) * dirs.size }.flatten.tally
+      freq.each { |name, count|
+        node = @defs[name]
+        if count == 1
+          node.setsys
+        else
+          node.setpck
+        end
+      }
+
+      # Collect definitions and dependencies
+      ast.values.each { |node| collect_variables(node) }
+
+      # Detect undefined variables and references to hashes or arrays
+      if check_undefined
+        deps.each { |path, path_deps|
+          path_deps.each { |dep|
+            if !defs.key?(dep)
+              error "Undefined variable '#{dep}' in definition of '#{path}'"
+            elsif !defs[dep].is_a?(ValueNode)
+              error "Can't reference non-variable '#{dep}' in definition of '#{path}'"
+            end
+          }
+        }
+      end
+
+      # Check for mandatory variables
+      if check_mandatory
+        missing = MANDATORY_FIELDS.select { |f| @defs[f].to_s.empty? }
+        missing.empty? or error "Missing mandatory fields '#{missing.join("', '")}'"
+      end
+
+      # Check types of built-in variables
+      if check_field_types
+        FIELDS.each { |f,ts|
+          ast.key?(f) or next
+          ts.any? { |t| ast[f].class <= t } or error "Illegal type of field '#{f}'"
+        }
+      end
+
+      @ast
+    end
+
+    # Compile YAML into a Qrpm object
+    def compile(yaml)
+      parse(yaml)
+      analyze
+      Qrpm.new(defs, deps)
+    end
+
+    def dump
+      puts "Defs"
+      indent { defs.each { |k,v| puts "#{k}: #{v.signature}" if v.interpolated? }}
+      puts "Deps"
+      indent { deps.each { |k,v| puts "#{k}: #{v.join(", ")}" if !v.empty? } }
+    end
+
+  private
+    # Shorthand
+    def error(msg) 
+      raise CompileError, msg, caller
+    end
+    
+    def parse_file_node(parent, hash)
+      hash = { "file" => hash } if hash.is_a?(String)
+
+      # Check for unknown keys
+      unknown_keys = hash.keys - FILE_KEYS
+      unknown_keys.empty? or 
+          error "Illegal file attribute(s): #{unknown_keys.join(", ")}"
+
+      # Check that exactly one of "file", "symlink", or "reflink" is defined
+      (hash.keys & %w(file symlink reflink)).size == 1 or 
+          error "Exactly one of 'file', 'symlink', or 'reflink' should be defined"
+
+      # Check that perm is not used together with symlink or reflink
+      (hash.keys & %w(symlink reflink)).empty? || !hash.key?("perm") or
+          error "Can't use 'perm' together with 'symlink' or 'reflink'"
+
+      # Normalize perm (YAML parses a literal 0644 as the integer 420!)
+      hash["perm"] = sprintf "%04o", hash["perm"] if hash["perm"].is_a?(Integer)
+
+      # Update file with srcdir
+      hash["file"] &&= "$srcdir/#{hash["file"]}" if @use_srcdir
+
+      # Create file node and add members
+      FileNode.make(parent, hash)
+    end
+
+    def parse_directory_node(parent, key, value)
+      constrain parent, RootNode
+      constrain key, String, Symbol
+      constrain value, String, Array, Hash
+      dir = DirectoryNode.new(parent, Fragment::Fragment.parse(key.to_s))
+      case value
+        when String, Hash; parse_file_node(dir, value)
+        when Array; value.each { |elem| parse_file_node(dir, elem) }
+      end
+      dir
+    end
+
+    def parse_node(parent, key, value)
+      constrain parent, HashNode, ArrayNode
+      constrain key, String, Symbol
+      constrain value, String, Integer, Float, Hash, Array, nil
+      key = key.to_s
+      case value
+        when String, Integer, Float
+          ValueNode.new(parent, key, Fragment::Fragment.parse(value))
+        when Hash
+          node = HashNode.new(parent, key)
+          value.each { |key, value| parse_node(node, key, value) }
+          node
+        when Array
+          node = ArrayNode.new(parent, key)
+          value.each.with_index { |value, idx| parse_node(node, idx.to_s, value) }
+          node
+        when nil
+          ValueNode.new(parent, key, nil)
+      end
+    end
+
+    # Collect variables from keys and values and add them to @defs and @deps
+    #
+    # Values and array have dependencies on their own while HashNode does not
+    # and just forwards to its members
+    def collect_variables(node)
+      case node
+        when StandardDirNode
+          # (standard dirs are already added to @defs)
+          @deps[node.path] = node.variables
+        when ValueNode
+          @defs[node.path] = node
+          @deps[node.path] = node.variables
+        when ArrayNode
+          @defs[node.path] = node
+          @deps[node.path] = node.traverse.map(&:variables).flatten.uniq
+        when HashNode
+          node.values.map { |n| collect_variables(n) }.flatten
+      end
+    end
+  end
+end
+

data/lib/qrpm/fragment.rb

@@ -0,0 +1,177 @@
+
+# TODO: Create (and use) a Fragment.parse method
+
+module Qrpm
+  module Fragment
+    # A part of a key or value in the QRPM configuration file
+    class Fragment
+      # Source string of fragment. The top-level expression fragment has
+      # the whole string as its source
+      attr_reader :source
+
+      # List of embedded fragments
+      attr_reader :fragments
+
+      def initialize(source, fragments = [])
+        @source = source 
+        @fragments = Array(fragments)
+      end
+
+      # Return true if this is a NilFragment. False except for NilFragment objects
+      def is_nil?() false end
+
+      # List of variables in the fragment (incl. child fragments)
+      def variables() fragments.map(&:variables).flatten.uniq end
+
+      # Interpolates fragment using dict and returns the result as a String
+      def interpolate(dict) source end
+
+      # Emit source
+      def to_s = source
+
+      # String representation of fragment. Used for debug and test
+      def signature()
+        r = "#{self.class.to_s.sub(/^.*::/, "")}(#{identifier})"
+        r += "{" + fragments.map { |f| f.signature }.join(", ") + "}" if !fragments.empty?
+        r
+      end
+
+      # Parse a JSON value into an Expression source
+      def Fragment.parse(value)
+        constrain value, String, Integer, Float, true, false, nil
+        node = value.nil? ? NilFragment.new : parse_string(value.to_s)
+        Expression.new(value.to_s, node)
+      end
+
+    protected
+      # Used by #signature to display an identifier for a node
+      def identifier() source.inspect.gsub('"', "'") end
+
+
+    private
+      # Parse string and return an array of Fragment sources. The string is
+      # scanned for $NAME, ${NAME}, $(COMMAND), and ${{NAME}} inside $(COMMAND)
+      # interpolations
+      #
+      # The string is parsed into Fragments to be able to interpolate it
+      # without re-parsing
+      #
+      # Variable names needs to be '{}'-quoted if they're followed by a letter
+      # or digit but not '.'. This makes it possible to refer to nested
+      # variables without quotes. Eg '/home/$pck.home/dir' will be parsed as
+      # '/home/${pck.home}/dir'
+      def Fragment.parse_string(string)
+        res = []
+        string.scan(/(.*?)(\\*)(\$#{PATH_RE}|\$\{#{PATH_RE}\}|\$\(.+\)|$)/)[0..-2].each { 
+            |prefix, backslashes, expr|
+          expr.delete_suffix(".")
+          text = prefix + '\\' * (backslashes.size / 2)
+          if expr != ""
+            if backslashes.size % 2 == 0
+              if expr[1] == "(" # $()
+                var = CommandFragment.new(expr)
+                var.fragments.concat parse_shell_string(var.command)
+              else # $NAME or ${NAME}
+                var = VariableFragment.new(expr)
+              end
+            else
+              text += expr
+            end
+          end
+          res << TextFragment.new(text) if text != ""
+          res << var if var
+        }
+        res
+      end
+
+      # Parse shell command string and return an array of Node sources
+      def Fragment.parse_shell_string(string)
+        res = []
+        string.scan(/(.*?)(\\*)(\$\{\{#{PATH_RE}\}\}|$)/).each { |prefix, backslashes, expr|
+          text = prefix + backslashes
+          if expr != ""
+            if backslashes.size % 2 == 0
+              var = CommandVariableFragment.new(expr)
+            else
+              text += expr
+            end
+          end
+          res << TextFragment.new(text) if text != ""
+          res << var if var
+        }
+        res
+      end
+    end
+
+    # Text without variables
+    class TextFragment < Fragment
+    end
+
+    # Nil value
+    class NilFragment < Fragment
+      def is_nil?() true end
+      def initialize() super(nil) end
+    end
+
+    # $NAME or ${NAME}
+    class VariableFragment < Fragment
+      # Name of the variable excluding '$' and '${}'
+      attr_reader :name
+
+      def initialize(source)
+        super
+        @name = source.sub(/^\W+(.*?)\W*$/, '\1')
+      end
+
+      def variables() [name] end
+      def interpolate(dict) dict[name] end
+
+    protected
+      def identifier() name end
+    end
+
+    # ${{NAME}} in $(COMMAND) interpolations
+    class CommandVariableFragment < VariableFragment
+      def interpolate(dict) dict[name] end # FIXME: Proper shell escape
+    end
+
+    # Common code for CommandFragment and Expression
+    class FragmentContainer < Fragment
+      def interpolate(dict) fragments.map { |f| f.interpolate(dict) }.flatten.join end
+    end
+
+    # $(COMMAND)
+    class CommandFragment < FragmentContainer
+      # Command line excl. '$()'
+      attr_reader :command
+
+      def initialize(string)
+        super(string)
+        @command = string[2..-2]
+      end
+
+      def interpolate(dict)
+        cmd = "set -eo pipefail; #{super}"
+        stdout, stderr, status = Open3.capture3(cmd)
+        status == 0 or raise Error.new "Failed expanding '$(#{cmd})'\n#{stderr}"
+        stdout.chomp
+      end
+    end
+
+    # A key or value as a list of Fragments
+    class Expression < FragmentContainer
+      def initialize(...)
+        super
+        @seen = {}
+      end
+
+      def interpolate(...)
+        r = super
+        !@seen.key?(r.inspect) or raise CompileError.new "Duplicate interpolation: #{r.inspect}"
+        @seen[r.inspect] = true
+        r
+      end
+    end
+  end
+end
+

data/lib/qrpm/lexer.rb

@@ -0,0 +1,41 @@
+
+module Qrpm
+  class Lexer
+    attr_reader :reldirs
+    attr_reader :absdirs
+
+    def initialize(reldirs, absdirs)
+      @reldirs = reldirs.map { |p| File.expand_path(p) }
+      @absdirs = absdirs.map { |p| File.expand_path(p) }
+    end
+
+    def self.load_yaml(file) # Is actually a kind of lexer
+      text = IO.read(file).sub(/^__END__.*/m, "")
+      YAML.load(text)
+    end
+
+    def lex(file)
+      yaml = {}
+      source = IO.read(file).sub(/^__END__.*/m, "")
+      YAML.load(source).each { |k,v|
+        if k == "include"
+          includes = v.is_a?(String) ? [v] : v
+          includes.each { |f| Lexer.load_yaml(search f).each { |k,v| yaml[k] = v } }
+        else
+          yaml[k] = v
+        end
+      }
+      yaml
+    end
+
+    def search(file)
+      case file
+        when /^\.\.\/(.*)/; reldirs.map { |d| "#{d}/../#$1" }.find { |f| File.exist? f }
+        when /^\.\/(.*)/; reldirs.map { |d| "#{d}/#$1" }.find { |f| File.exist? f }
+        when /^\//; File.exist?(file) && file
+      else
+        absdirs.map { |d| "#{d}/#{file}" }.find { |f| File.exist? f }
+      end or raise Error, "Can't find #{file}"
+    end
+  end
+end