birman 0.2.1 → 0.3.0

data/lib/birman/compiler/types.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Birman
+  class Compiler
+    # A coarse static type for each matcher's value, so every value
+    # instance variable starts as a literal of the type it will hold
+    # (Spinel boxes a variable whose writes disagree). :poly means it
+    # varies, or isn't known; nil, nodes and :poly all start as nil.
+    # [class.Compiler.Types]
+    # purpose = "Infers the type a matcher's value has, for initializers"
+    class Types
+      INITIAL = { str: '""', int: "0", float: "0.0", sym: ':""', array: "[]", hash: "{}" }.freeze
+
+      SENDS = { to_i: :int, to_f: :float, to_sym: :sym, to_s: :str, join: :str, to_h: :hash,
+                upcase: :str, downcase: :str, strip: :str, to_a: :array }.freeze
+
+      def initialize(grammar)
+        @grammar = grammar
+      end
+
+      # The initializer literal for a value of matcher's type.
+      def initial(matcher) = INITIAL.fetch(of(matcher), "nil")
+
+      def of(matcher) # rubocop:disable Metrics
+        case matcher
+        when Str, Pat then :str
+        when Lit then of(@grammar.literal_matcher(matcher.text))
+        when Seq then seq(matcher)
+        when Pick then of(matcher.parts[matcher.index])
+        when SepBy, Rep then :array
+        when Captures then :hash
+        when Choice then matcher.alts.map { of(_1) }.reduce { |a, b| join(a, b) }
+        when Not, And then :nil
+        when Both then of(matcher.right)
+        when Const then constant(matcher.constant)
+        when Send then SENDS.fetch(matcher.method_name, :poly)
+        when To then :node
+        when Infix then infix(of(matcher.operand))
+        when Ref then rules.fetch(matcher.name, :poly)
+        when Label, Check, Quiet, Named, Lexeme then of(matcher.inner)
+        else :poly
+        end
+      end
+
+      # A String, or nil where it's optional: str.maybe.
+      def str_or_nil?(matcher, seen = {})
+        case matcher
+        when Maybe then of(matcher.inner) == :str
+        when Label, Check, Quiet, Named, Lexeme then str_or_nil?(matcher.inner, seen)
+        when Ref
+          !seen.key?(matcher.name) && (seen[matcher.name] = true) &&
+            str_or_nil?(@grammar.matcher_for(matcher.name), seen)
+        else false
+        end
+      end
+
+      private
+
+      def rules = @rules ||= solve
+
+      def seq(matcher)
+        return of(matcher.parts[matcher.only_at]) if matcher.only_at
+
+        matcher.parts.any?(&:capture_key) ? :hash : :array
+      end
+
+      def constant(value)
+        { String => :str, Integer => :int, Float => :float, Symbol => :sym, NilClass => :nil }
+          .fetch(value.class, :poly)
+      end
+
+      # Every rule's type at once, so none depends on which was asked about
+      # first: starting from nothing known (nil) for each, a rule's type is
+      # widened by what its body gives until nothing changes. Rules refer
+      # to each other in cycles (expr -> atom -> expr); one that still
+      # gives nothing (it only recurses) is :poly.
+      def solve
+        names = rule_names
+        @rules = names.to_h { [_1, nil] }
+        changed = true
+        while changed
+          changed = false
+          names.each do |name|
+            type = join(@rules[name], of(@grammar.matcher_for(name)))
+            changed ||= type != @rules[name]
+            @rules[name] = type
+          end
+        end
+        @rules.transform_values { _1 || :poly }
+      end
+
+      # The rules reachable from the root and the ignore rule.
+      def rule_names
+        names = []
+        queue = [@grammar.class.root, @grammar.class.ignore].compact
+        while (name = queue.shift)
+          next if names.include?(name)
+
+          names << name
+          queue.concat(refs(@grammar.matcher_for(name)))
+        end
+        names
+      end
+
+      # Rule names a matcher refers to (custom matchers, which the emitter
+      # refuses, refer to none).
+      def refs(value)
+        case value
+        when Ref then [value.name]
+        when Lit then refs(@grammar.literal_matcher(value.text))
+        when Data then value.to_h.values.flat_map { refs(_1) }
+        when Array then value.flat_map { refs(_1) }
+        else []
+        end
+      end
+
+      # The type of a value that is either: nil is nothing known yet.
+      def join(one, other)
+        return other if one.nil?
+        return one if other.nil? || one == other
+
+        :poly
+      end
+
+      def infix(operand) = operand.nil? || operand == :node ? operand : :poly
+    end
+  end
+end

data/lib/birman/compiler.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+# [file]
+# purpose = "Turns a grammar into a standalone Ruby parser (Grammar.to_ruby)"
+# responsibility = "Walking the rules' matchers and assembling the generated source"
+# dependencies = ["Birman::Compiler::Emitter", "Birman::Compiler::Actions", "compiler/runtime.rb"]
+# pattern = "Code generation from the immutable matcher tree"
+module Birman
+  # Generates Ruby source for a parser that gives exactly the results
+  # Grammar.parse gives, values, spans, errors and all, without Birman.
+  # The output is written in the Ruby subset Spinel compiles to a native
+  # binary: the runtime (compiler/runtime.rb), a class per node, and a
+  # Parser with one method per matcher.
+  #
+  #   source = Calc::Grammar.to_ruby(namespace: "CalcParser")
+  #   CalcParser::Parser.new(input).parse   # => Success or Failure
+  #
+  # Every matcher method takes a byte position and returns where its
+  # match ended, or FAIL; its value, if anything reads it, goes in an
+  # instance variable. Grammars using .map with a block, or a custom
+  # matcher class, can't be compiled: Error names the rule.
+  # [class.Compiler]
+  # purpose = "Generates a standalone parser from a grammar class"
+  # collaborators = ["Emitter (matcher templates)", "Actions (copies .action methods)", "Nodes"]
+  class Compiler
+    # Raised for a grammar the compiler can't translate faithfully.
+    class Error < StandardError; end
+
+    RUNTIME = File.join(__dir__, "compiler", "runtime.rb")
+
+    # Constants the generated code and its runtime refer to. A node class
+    # or copied constant with one of these names would sit in the same
+    # lexical scope and shadow it, so it's refused.
+    RESERVED_CONSTANTS = %w[
+      BirmanRT Parser Node Span Success Failure Committed TooDeep FAIL IN_PROGRESS ABSENT NONE TEN19 TWO64
+      CHUNK_LIMIT FLOAT_CHUNK LOOKAHEAD_DEPTH
+      StringScanner Hash Array String Integer Float Symbol Regexp Kernel Encoding
+      StandardError ArgumentError RangeError TypeError FloatDomainError SystemStackError
+    ].freeze
+
+    # Spinel allows 64 begin blocks active at once and exits the process
+    # past that. Each lookahead (absent, present, &) holds one while its
+    # inner matcher runs, so nesting deeper than this is a Failure ("input
+    # is nested too deeply") instead. A binary built with a bigger
+    # -DSP_EXC_STACK_MAX can take a bigger depth.
+    LOOKAHEAD_DEPTH = 56
+
+    def initialize(grammar_class, namespace:, lookahead_depth: LOOKAHEAD_DEPTH)
+      unless namespace.match?(/\A[A-Z]\w*(::[A-Z]\w*)*\z/)
+        raise ArgumentError, "namespace must be a constant path, like \"CalcParser\""
+      end
+      unless lookahead_depth.is_a?(Integer) && lookahead_depth.positive?
+        raise ArgumentError, "lookahead_depth must be a positive Integer"
+      end
+
+      @lookahead_depth = lookahead_depth
+
+      @grammar_class = grammar_class
+      @grammar = grammar_class.new
+      @namespace = namespace
+    end
+
+    # [method."Compiler#to_ruby"]
+    # returns = "String: Ruby source defining <namespace>::Parser, its node classes and BirmanRT"
+    # raises = [
+    #   "ArgumentError for left-recursive cycles with no rule in common, or a cycle that can't grow, as Grammar#parse",
+    #   "Compiler::Error for what can't be compiled (named by rule)"
+    # ]
+    def to_ruby
+      @grammar.left_recursion
+      emitter = emit
+      check_namespace(emitter.node_names)
+      body = [*emitter.node_classes, parser_class(emitter)].join("\n")
+      wrap_in_namespace("#{runtime}\n#{body}")
+    end
+
+    private
+
+    # Values nobody reads (spacing, keywords) are never kept. Which those
+    # are is found by compiling: start with only the root's value, and
+    # compile again with every rule found to be read, until none is new.
+    def emit
+      valued = [@grammar_class.root]
+      loop do
+        emitter = Emitter.new(@grammar, valued)
+        emitter.compile
+        return emitter if (emitter.read_rules - valued).empty?
+
+        valued |= emitter.read_rules
+      end
+    end
+
+    # Generating into a module that already holds other classes of the
+    # same names (the grammar's own nodes, say) would reopen them with a
+    # different superclass.
+    def check_namespace(node_names)
+      return unless Object.const_defined?(@namespace)
+
+      existing = Object.const_get(@namespace)
+      node_names.each do |name|
+        next unless existing.is_a?(Module) && existing.const_defined?(name, false)
+
+        found = existing.const_get(name, false)
+        next if found.is_a?(Class) && found.superclass.name == "#{@namespace}::BirmanRT::Node"
+
+        raise Error, "#{@namespace} already defines #{name}; generate the parser into another namespace"
+      end
+    end
+
+    def header
+      <<~RUBY
+        # Generated by Birman #{VERSION} from #{@grammar_class.name || "an anonymous grammar"}
+        # (Grammar.to_ruby). Regenerate it rather than editing it.
+        # Parse with #{@namespace}::Parser.new(input).parse, which returns
+        # a Success (value) or a Failure (pos, expected, message).
+        require "strscan"
+      RUBY
+    end
+
+    # The runtime file, after its require.
+    def runtime = File.read(RUNTIME).split(/^require "strscan"\n\n/, 2).last
+
+    def parser_class(emitter)
+      <<~RUBY
+        class Parser < BirmanRT::Parser
+          include BirmanRT
+
+          # How deeply lookaheads may nest (Grammar.to_ruby's lookahead_depth).
+          LOOKAHEAD_DEPTH = #{@lookahead_depth}
+
+        #{indent(emitter.parser_body, 1)}
+        end
+      RUBY
+    end
+
+    def wrap_in_namespace(body)
+      modules = @namespace.split("::")
+      opening = modules.each_with_index.map { |name, depth| indent("module #{name}", depth) }
+      closing = modules.each_index.reverse_each.map { |depth| indent("end", depth) }
+      [header, *opening, indent(body.chomp, modules.size), *closing].join("\n") << "\n"
+    end
+
+    def indent(text, depth)
+      pad = "  " * depth
+      text.lines.map { |line| line.strip.empty? ? "\n" : "#{pad}#{line}" }.join.chomp
+    end
+  end
+end
+
+require_relative "compiler/literals"
+require_relative "compiler/regexps"
+require_relative "compiler/types"
+require_relative "compiler/actions"
+require_relative "compiler/nodes"
+require_relative "compiler/emitter"

data/lib/birman/context.rb

@@ -8,23 +8,39 @@ module Birman
   # non-ASCII string by character offset costs time proportional to
   # the offset, which made parsing quadratic. Failure and Span convert
   # back to characters for anything public.
-  class Context
+  class Context # rubocop:disable Metrics/ClassLength
     # Marks a memo entry whose rule is still being matched.
     IN_PROGRESS = Object.new.freeze
 
+    # The expectations of a frame that hasn't failed anywhere yet. Shared
+    # and frozen; fail_at replaces it rather than appending.
+    NOTHING_EXPECTED = [].freeze
+
     # position: where the most recently entered rule started. When the
     # input nests too deeply for Ruby's stack, that's where it happened.
-    attr_reader :input, :grammar, :memo, :furthest, :expected, :position
+    # ignore_ref: a Ref to the grammar's ignore rule, or nil. Lexemes
+    # read it here, so the grammar running them decides what they skip.
+    attr_reader :input, :grammar, :ignore_ref, :furthest, :expected, :position, :isolated_furthest, :isolated_expected
 
-    def initialize(input, grammar)
+    def initialize(input, grammar, ignore_rule)
       @input = input
       @grammar = grammar
+      @ignore_ref = ignore_rule && Ref.new(ignore_rule) # built once per parse, not per token
       @memo = {}
+      @growing = grammar.left_recursion.leaders.then { _1 unless _1.empty? } # nil: nothing to look up
+      @seeds = {}
+      # { leader of other rules => { pos it grows at => this round's memo
+      # for the other rules } }
+      @rounds = grammar.left_recursion.leaders.values.tally.filter_map { |leader, n| [leader, {}] if n > 1 }.to_h
       @furthest = 0
-      @expected = []
+      @expected = NOTHING_EXPECTED
       @token_ends = {}
     end
 
+    # Where the parse begins: past whatever the ignore rule (if any)
+    # matches at the start of the input.
+    def start = @ignore_ref&.match(self, 0)&.pos || 0
+
     # A lexeme ended at token_end, then the ignore rule consumed spacing
     # (whitespace, comments) up to pos. Spans use this to stop at the
     # token instead of including what was ignored after it.
@@ -46,25 +62,40 @@ module Birman
         @furthest = pos
         @expected = [what]
       elsif pos == @furthest
-        @expected << what
+        @expected = @expected.frozen? ? [what] : @expected << what
       end
       nil
     end
 
-    def merge(pos, expected) = expected.each { fail_at(pos, _1) }
+    # Record a finished frame's expectations (a memo hit, a label, a
+    # check) at pos. Ones already recorded there are skipped: a rule
+    # reached from several alternatives at one position is merged in
+    # once per alternative, and copying its whole list each time grew
+    # it exponentially. Failure#expected is uniq'd anyway, so this
+    # changes no error, only the work behind it. fail_at itself doesn't
+    # check: it runs far more often, and repeats there grow only
+    # linearly.
+    def merge(pos, expected)
+      expected.each { fail_at(pos, _1) unless pos == @furthest && @expected.include?(_1) }
+    end
 
     # Run the block with fresh failure tracking starting at pos, then
-    # restore the outer state. Returns [result, furthest, expected] so
-    # the caller decides what to keep: Ref memoizes it, Label rewrites
-    # it, Not throws it away.
+    # restore the outer state, and return the block's result. What the
+    # block recorded is left in isolated_furthest and isolated_expected
+    # for the caller to keep or drop: Ref memoizes it, Label rewrites
+    # it, Not throws it away. No arrays: this runs for every rule call.
     def isolate(pos)
-      outer = [@furthest, @expected]
+      outer_furthest = @furthest
+      outer_expected = @expected
       @furthest = pos
-      @expected = []
+      @expected = NOTHING_EXPECTED
       result = yield
-      [result, @furthest, @expected]
+      @isolated_furthest = @furthest
+      @isolated_expected = @expected
+      result
     ensure
-      @furthest, @expected = outer
+      @furthest = outer_furthest
+      @expected = outer_expected
     end
 
     def commit_failure! = raise(Committed.new(@furthest, @expected.dup))
@@ -74,9 +105,12 @@ module Birman
     # moves past, so they're dropped. If it fails, that's recorded at
     # pos, as what it expected there or as the lookahead itself.
     def peek?(matcher, pos)
-      result, furthest, expected = isolate(pos) { lookahead { matcher.match(self, pos) } }
+      result = isolate(pos) { lookahead { matcher.match(self, pos) } }
       return true if result
 
+      furthest = isolated_furthest
+      expected = isolated_expected
+
       if furthest == pos && expected.any?
         merge(pos, expected)
       else
@@ -86,38 +120,126 @@ module Birman
     end
 
     # Rule name's memo entry at pos, computed by the block the first
-    # time. Meeting the rule again at the same position while it is
-    # still being matched means it reaches itself without consuming
-    # input: left recursion, which would never terminate. That's a bug
-    # in the grammar, not the input, so it raises.
-    def memoize(name, pos, &)
-      key = [name, pos]
-      entry = @memo[key]
+    # time. The block returns [result, furthest, expected].
+    #
+    # A cycle of rules that call each other before consuming input (left
+    # recursion) is grown from a seed at its leader (see
+    # LeftRecursion#leaders and grow). While the leader grows at pos, the
+    # cycle's other rules are memoized there in a table of the round's
+    # own, not the memo: what they match there depends on the seed, which
+    # changes every round. (A match of theirs already in the memo there is
+    # final: it was made after the leader finished, or without it.)
+    # Every way back to a rule of the
+    # cycle at pos passes through the leader's seed, so meeting any rule
+    # again at the same position while it is still being matched means
+    # it reaches itself without consuming input in a way the grammar's
+    # ASTs don't show (through a custom matcher): that would never
+    # terminate, and it's a bug in the grammar, not the input, so it
+    # raises.
+    def memoize(name, pos, &) # rubocop:disable Metrics
+      table = @memo[name] ||= {}
+      entry = table[pos]
       return entry if entry && !entry.equal?(IN_PROGRESS)
-      raise ArgumentError, "left recursion in rule :#{name}: it can reach itself without consuming input" if entry
+
+      leader = @growing&.[](name) # nil without left recursion: no further lookups
+      round = @rounds[leader]&.[](pos) if leader
+      return in_round(round, name, pos, &) if round && !leader.equal?(name)
+      return seed(name, pos) if entry
 
       @position = pos
-      compute(key, &)
+      return grow(name, table, pos, &) if leader.equal?(name)
+
+      compute(table, pos, &)
     end
 
     # Run a lookahead's inner match. A lookahead only asks whether
     # something would match here, so a commit failure inside it is an
-    # ordinary failure (nil) rather than the end of the parse.
+    # ordinary failure (nil) rather than the end of the parse. What it
+    # expected is kept, as generated parsers keep it: the frames it
+    # unwound through restored what they started with, so without this a
+    # failure where the lookahead began would only be named, not said.
     def lookahead
       yield
-    rescue Committed
+    rescue Committed => e
+      @furthest = e.pos
+      @expected = e.expected
       nil
     end
 
     private
 
-    def compute(key)
-      @memo[key] = IN_PROGRESS
-      @memo[key] = yield
+    # The memo is a Hash per rule, keyed by position, so no key arrays.
+    # (A round's table is keyed by rule instead.)
+    def compute(table, key)
+      table[key] = IN_PROGRESS
+      table[key] = yield
     ensure
       # A commit failure caught by a lookahead unwinds through here; the
       # rule may be tried again at this position afterwards.
-      @memo.delete(key) if @memo[key].equal?(IN_PROGRESS)
+      table.delete(key) if table[key].equal?(IN_PROGRESS)
+    end
+
+    # A rule of a cycle whose leader grows at pos, memoized in the round's
+    # table (keyed by rule): once a round, however many alternatives call
+    # it, or a cycle of n rules with k alternatives each would take
+    # k**(n - 1) matches a round. Back at itself without the leader's seed
+    # between, it raises as memoize does.
+    def in_round(round, name, pos, &)
+      entry = round[name]
+      return entry if entry && !entry.equal?(IN_PROGRESS)
+      return seed(name, pos) if entry
+
+      @position = pos
+      compute(round, name, &)
+    end
+
+    # Seed-growing (Warth et al. 2008), at a cycle's leader only, as pegen
+    # does. The rule is matched with its call to itself here failing,
+    # then again with that call returning the last match (the seed), for
+    # as long as the match gets further. It's a loop, so a longer chain
+    # uses no more stack. The memo entry stays IN_PROGRESS until the end;
+    # every way back to it passes through its seed. A leader of other
+    # rules starts each round with an empty table for them at pos.
+    #
+    # Each attempt's diagnostics are kept, not just the best one's: the
+    # last attempt, which got no further, is usually the one that knows
+    # what was missing ("1 +" expected a number after the "+").
+    def grow(name, table, pos) # rubocop:disable Metrics
+      seeds = @seeds[name] ||= {}
+      seeds[pos] = best = [nil, pos, NOTHING_EXPECTED]
+      table[pos] = IN_PROGRESS
+      rounds = @rounds[name]
+      diagnostics = [pos, NOTHING_EXPECTED]
+      # Not loop: its block adds frames to every level of nesting.
+      while true # rubocop:disable Style/InfiniteLoop
+        rounds[pos] = {} if rounds
+        result, furthest, expected = yield
+        diagnostics = further(diagnostics, furthest, expected)
+        break unless result && result.pos > (best[0]&.pos || -1)
+
+        seeds[pos] = best = [result, pos, NOTHING_EXPECTED]
+      end
+      table[pos] = [best[0], *diagnostics]
+    ensure
+      seeds&.delete(pos) # nil if the stack ran out before it was set
+      rounds&.delete(pos)
+      table.delete(pos) if table[pos].equal?(IN_PROGRESS)
+    end
+
+    # Two frames' diagnostics as one, as fail_at would: the further
+    # position wins, and at the same one the expectations are merged.
+    def further(diagnostics, furthest, expected)
+      return [furthest, expected] if furthest > diagnostics[0]
+      return diagnostics if furthest < diagnostics[0] || expected.empty?
+
+      [furthest, diagnostics[1] | expected]
+    end
+
+    # The rule's match so far at pos, while it grows there. Its
+    # diagnostics are grow's to keep, so it carries none.
+    def seed(name, pos)
+      @seeds[name]&.[](pos) || raise(ArgumentError,
+                                     "left recursion in rule :#{name}: it can reach itself without consuming input")
     end
   end
 end

data/lib/birman/error_formatter.rb

@@ -1,11 +1,17 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Default wording for parse errors"
+# pattern = "Strategy: anything with #call(failure) can replace it"
 module Birman
   # Turns a Failure into readable text. Anything that responds to
   # #call(failure) works in its place, for custom wording or i18n:
   #
   #   failure.message(MyFormatter.new)
   class ErrorFormatter
+    # [method.call]
+    # params = ["failure: Birman::Failure"]
+    # returns = "String: \"line L, column C: expected ..., found ...\" plus the source line and a caret"
     def call(failure)
       source = source_line(failure)
       <<~MSG.chomp