birman 0.2.2 → 0.3.0

data/lib/birman/compiler/literals.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Birman
+  class Compiler
+    # Values written into the generated parser as Ruby literals: the
+    # constants copied actions read, and .means values.
+    # [module.Compiler.Literals]
+    # purpose = "Which values can be written as literals, and how"
+    module Literals
+      module_function
+
+      def literal?(value)
+        case value
+        when String, Symbol, Integer, true, false, nil then true
+        when Float then value.finite?
+        when Array then value.all? { literal?(_1) }
+        when Hash then value.all? { |k, v| literal?(k) && literal?(v) }
+        else false
+        end
+      end
+
+      # A frozen Array or Hash stays frozen.
+      def source(value)
+        frozen = value.frozen? && (value.is_a?(Array) || value.is_a?(Hash))
+        frozen ? "#{value.inspect}.freeze" : value.inspect
+      end
+    end
+  end
+end

data/lib/birman/compiler/nodes.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Birman
+  class Compiler
+    # The generated counterpart of each node(...) class a grammar builds:
+    # a plain class under BirmanRT::Node (so a rule whose value is one of
+    # several node classes keeps a single type), with readers, a span,
+    # == ignoring the span, and Birman::Node's s-expression inspect. A
+    # node's own initialize can give fields defaults (see Initializer),
+    # which are written where each node is built.
+    # [class.Compiler.Nodes]
+    # purpose = "Collects the grammar's node classes and writes their generated versions"
+    class Nodes
+      def initialize
+        @classes = {}
+        @defaults = {}
+      end
+
+      # The generated class's name for node, registering it.
+      def name_for(node, rule)
+        unless node.is_a?(Class) && node.include?(Birman::Node)
+          raise Error, "rule :#{rule} builds #{node.inspect}, which isn't a node(...) class"
+        end
+
+        name = node.name&.split("::")&.last or raise Error, "rule :#{rule} builds an anonymous node class"
+        check(node, name, rule)
+        other = @classes[name]
+        raise Error, "#{other.name} and #{node.name} would both be generated as #{name}" if other && other != node
+
+        @classes[name] = node
+        name
+      end
+
+      def sources = @classes.map { |name, node| source(name, node.members) }
+
+      def names = @classes.keys
+
+      # The source of the default for each field node's initialize gives
+      # one, by field.
+      def defaults(node) = @defaults.fetch(node, {})
+
+      private
+
+      def check(node, name, rule)
+        if RESERVED_CONSTANTS.include?(name)
+          raise Error, "rule :#{rule} builds #{node.name}, but #{name} is a name the generated parser needs"
+        end
+
+        own = node.instance_methods(false) - node.members + node.private_instance_methods(false)
+        return if own.empty?
+        return @defaults[node] ||= Initializer.new(node, rule).defaults if own == [:initialize]
+
+        raise Error, "rule :#{rule} builds #{node.name}, whose methods (#{own.join(", ")}) can't be compiled yet"
+      end
+
+      def source(name, fields)
+        <<~RUBY
+          class #{name} < BirmanRT::Node
+            #{"attr_reader #{fields.map(&:inspect).join(", ")}" unless fields.empty?}
+
+            def initialize(#{[*fields, :span].join(", ")})
+          #{[*fields, :span].map { "    @#{_1} = #{_1}" }.join("\n")}
+            end
+
+            def ==(other)
+              #{["other.is_a?(#{name})", *fields.map { "@#{_1} == other.#{_1}" }].join(" && ")}
+            end
+
+            def inspect
+              "(#{[name, *fields.map { "\#{@#{_1}.inspect}" }].join(" ")})"
+            end
+
+            def dump
+              "(#{["#{name}[\#{@span.byte_from},\#{@span.byte_to}]", *fields.map { "\#{BirmanRT.dump(@#{_1})}" }].join(" ")})"
+            end
+          end
+        RUBY
+      end
+    end
+
+    # A node's own initialize, read from its source with Prism. One that
+    # takes exactly its fields as keywords, some with literal defaults,
+    # and only calls super, as in
+    #
+    #   node(:name, :args) { def initialize(name:, args: []) = super }
+    #
+    # gives the defaults' sources: they're written where each node is
+    # built, so a [] is a new Array every time, as in Ruby. Anything
+    # more is refused.
+    class Initializer
+      def initialize(node, rule)
+        @node = node
+        @rule = rule
+      end
+
+      def defaults
+        found = find
+        check_parameters(found.parameters)
+        refuse("does more than call super, so it can't be compiled") unless bare_super?(found.body)
+
+        (found.parameters&.keywords || []).each_with_object({}) do |keyword, defaults|
+          next unless keyword.is_a?(Prism::OptionalKeywordParameterNode)
+
+          default = keyword.value
+          unless literal?(default)
+            refuse("gives #{keyword.name} a default (#{default.slice}) that isn't a literal " \
+                   "(nil, true, false, numbers, strings, symbols, and arrays and hashes of them)")
+          end
+          defaults[keyword.name] = default.slice
+        end
+      end
+
+      private
+
+      def find
+        file, line = @node.instance_method(:initialize).source_location
+        found = (MethodFinder.new(:initialize, line).find(MethodFinder.parse(file)) if file && File.exist?(file))
+        found&.first or refuse("isn't written with def, so it can't be compiled")
+      end
+
+      def check_parameters(parameters)
+        keywords = parameters&.keywords || []
+        return if others(parameters).empty? && keywords.map(&:name).sort == @node.members.sort
+
+        refuse("must take exactly its fields (#{@node.members.join(", ")}) as keywords, and nothing else, " \
+               "to be compiled")
+      end
+
+      # Every parameter but the keywords.
+      def others(parameters)
+        return [] unless parameters
+
+        [*parameters.requireds, *parameters.optionals, parameters.rest, *parameters.posts,
+         parameters.keyword_rest, parameters.block].compact
+      end
+
+      def bare_super?(body)
+        body.is_a?(Prism::StatementsNode) && body.body.size == 1 &&
+          body.body.first.is_a?(Prism::ForwardingSuperNode) && body.body.first.block.nil?
+      end
+
+      def literal?(value)
+        case value
+        when Prism::NilNode, Prism::TrueNode, Prism::FalseNode, Prism::IntegerNode, Prism::FloatNode,
+             Prism::SymbolNode
+          true
+        when Prism::StringNode then !value.opening&.start_with?("<<") # a heredoc's slice is only its opening
+        when Prism::ArrayNode then value.elements.all? { literal?(_1) }
+        when Prism::HashNode
+          value.elements.all? { _1.is_a?(Prism::AssocNode) && literal?(_1.key) && literal?(_1.value) }
+        else false
+        end
+      end
+
+      def refuse(problem) = raise(Error, "rule :#{@rule} builds #{@node.name}, whose initialize #{problem}")
+    end
+  end
+end

data/lib/birman/compiler/regexps.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Birman
+  class Compiler
+    # Regexps for StringScanner#skip at the scan position. The interpreter
+    # matches with fixed_anchor: true, so \A, ^, lookbehind, and a \b or
+    # \B before anything is matched see the input before the position;
+    # the generated parser's scanner starts its subject at the position,
+    # where they would silently mean something else. Those are refused.
+    # Everything else gets a leading \A: in Spinel an unanchored miss
+    # searches the rest of the input.
+    # [module.Compiler.Regexps]
+    # purpose = "Checking and anchoring the grammar's regexps"
+    module Regexps
+      module_function
+
+      # The regexp literal to emit for re.
+      def anchored(re)
+        extended = re.options.anybits?(Regexp::EXTENDED)
+        problem = Check.new(re.source, extended).problem
+        raise Error, "#{re.inspect} can't be compiled: it uses #{problem}" if problem
+
+        # With /x, a trailing comment would swallow the closing paren.
+        Regexp.new("\\A(?:#{re.source}#{"\n" if extended})", re.options).inspect
+      end
+
+      # Walks a regexp's source, tracking whether something is certain to
+      # have been matched (consumed) at each point. A \b or \B is allowed
+      # only where it is. Conservative: what it can't follow, it refuses.
+      class Check
+        # One open group: whether input was consumed when it opened, in
+        # every alternative finished so far, and in the current one; and,
+        # for spotting (?>x{m,n}), how many atoms it holds and the wide
+        # interval among them.
+        Frame = Struct.new(:start, :all, :now, :lookaround, :atomic, :atoms, :wide, :alternated)
+
+        # A {m,n} that lets more than WIDE lengths be tried.
+        Wide = Struct.new(:text, :safe)
+        WIDE = 16
+
+        SPINEL_BACKTRACKING = "%s with a lookaround, atomic group, backreference or lazy interval: Spinel's " \
+                              "regexp engine tries every length it allows (exponential; it raises past about 18). " \
+                              "Make the repeat atomic on its own, (?>x%s), if nothing after it needs a shorter " \
+                              "match, or use + with a separate length check"
+
+        OPTIONAL = /[?*]|\{,|\{0+[,}]/
+        QUANTIFIER = /(?:[?*+]|\{\d*,?\d*\})[?+]?/
+
+        def initialize(source, extended)
+          @scanner = StringScanner.new(source)
+          @extended = extended
+          @frames = [Frame.new(false, true, false, false, false, 0)]
+          @backtracks = false
+          @wide = []
+        end
+
+        def problem
+          until @scanner.eos?
+            problem = step
+            return problem if problem
+          end
+          spinel_problem
+        end
+
+        private
+
+        def frame = @frames.last
+
+        # Spinel runs a regexp on its backtracking engine if it has any of
+        # these, and there a wide {m,n} goes exponential when every length
+        # fails. Only a group that is (?>x{m,n}) and nothing else is safe.
+        def spinel_problem
+          wide = @wide.find { !_1.safe }
+          format(SPINEL_BACKTRACKING, wide.text, wide.text) if @backtracks && wide
+        end
+
+        def step # rubocop:disable Metrics
+          s = @scanner
+          if s.skip(/\\A/) then "\\A"
+          elsif s.skip(/\\[bB]/) then frame.now ? nil : "\\b or \\B before anything is matched"
+          elsif s.skip(/\\[zZG]/) then nil
+          elsif s.skip(/\\(?:\d|k<\w+>|k'\w+')/) then backreference
+          elsif s.skip(/\\[pPu]\{[^}]*\}|\\./m) then atom
+          elsif s.skip(/\(\?<[=!]/) then "lookbehind"
+          elsif s.skip(/\(\?#[^)]*\)/) then nil
+          elsif s.skip(/\(\?[=!]/) then open_group(lookaround: true)
+          elsif s.skip(/\(\?>/) then open_group(lookaround: false, atomic: true)
+          elsif s.skip(/\(\?[imx]*(?:-[imx]*)?\)/) then inline_options(s.matched)
+          elsif s.skip(/\((?:\?(?:[imx]*(?:-[imx]*)?:|>|<\w+>|'\w+'))?/) then open_group(lookaround: false)
+          elsif s.skip(/\|/) then alternative
+          elsif s.skip(/\)/) then close
+          elsif s.skip(/\^/) then "^"
+          elsif s.skip(/\[/) then character_class
+          elsif @extended && s.skip(/\s+|#[^\n]*/) then nil
+          else
+            s.getch
+            atom
+          end
+        end
+
+        # Something that matches at least one character, unless a
+        # quantifier after it makes it optional.
+        def atom = quantified(true)
+
+        def quantified(consumes)
+          optional = @scanner.match?(OPTIONAL)
+          quantifier(@scanner.scan(QUANTIFIER))
+          frame.now ||= consumes && !optional
+          nil
+        end
+
+        def backreference
+          @backtracks = true
+          quantified(false)
+        end
+
+        # Counts the atom just read, and notes its quantifier if it's a
+        # wide or lazy interval.
+        def quantifier(text)
+          frame.atoms += 1
+          frame.wide = nil
+          bounds = text&.match(/\A\{(\d*),(\d+)\}(\?)?/) or return
+          @backtracks ||= !bounds[3].nil?
+          return unless bounds[2].to_i - bounds[1].to_i > WIDE
+
+          @wide << (frame.wide = Wide.new(text, false))
+        end
+
+        def open_group(lookaround:, atomic: false)
+          @backtracks ||= lookaround || atomic
+          @frames << Frame.new(frame.now, true, frame.now, lookaround, atomic, 0)
+          nil
+        end
+
+        def alternative
+          frame.all &&= frame.now
+          frame.now = frame.start
+          frame.alternated = true
+          nil
+        end
+
+        def close
+          return "an unmatched )" if @frames.size == 1
+
+          group = @frames.pop
+          group.wide.safe = true if atomic_interval?(group)
+          consumed = !group.lookaround && group.all && group.now
+          optional = @scanner.match?(OPTIONAL)
+          quantifier(@scanner.scan(QUANTIFIER))
+          frame.now ||= consumed && !optional
+          nil
+        end
+
+        # (?>x{m,n}) and nothing else in the group.
+        def atomic_interval?(group)
+          group.atomic && group.wide && group.atoms == 1 && !group.alternated
+        end
+
+        # (?x) turns on extended mode for the rest (conservatively, even
+        # past the end of its group).
+        def inline_options(text)
+          @extended ||= text[/\A\(\?([imx]*)/, 1].include?("x")
+          nil
+        end
+
+        # A character class (nested ones included), after its "[".
+        def character_class
+          depth = 1
+          @scanner.skip(/\^?\]?/)
+          while depth.positive? && !@scanner.eos?
+            if @scanner.skip(/\\./m) then next
+            elsif @scanner.skip(/\[/) then depth += 1
+            elsif @scanner.skip(/\]/) then depth -= 1
+            else @scanner.getch
+            end
+          end
+          atom
+        end
+      end
+    end
+  end
+end