birman 0.2.1 → 0.3.0

data/lib/birman/combinators.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "The combinators available inside every rule body"
+# responsibility = "Building matchers: raw strings and patterns, literals, sequences, choices, lists, wrappers"
+# dependencies = ["Birman::Matcher", "matchers.rb (Seq, Choice, Pick, SepBy, Infix, Lexeme)"]
+# pattern = "Parser combinators (PEG)"
 module Birman
   # Marker for seq: everything after it is committed (see Matcher#>).
   COMMIT = Object.new.tap { |o| def o.inspect = "Birman::COMMIT" }.freeze
@@ -15,36 +20,78 @@ module Birman
   # a rule body, after >> or |) is a literal: it skips trailing spacing
   # when the grammar declares `ignore`, and word-like ones are keywords.
   # Use str and match for exact, raw matching.
+  # [module.Combinators]
+  # purpose = "Matcher-building methods mixed into every grammar (via Birman::Grammar)"
+  # input = "Strings (literals), Regexps (raw patterns) and matchers"
+  # output = "Immutable matcher objects; nothing is matched until Grammar#parse"
+  # notes = "Method names here are reserved: a rule can't share one"
   module Combinators
     # --- Raw matchers ---
 
-    def str(s)    = Str.new(s)
+    # [method.str]
+    # params = ["s: String"]
+    # returns = "Matcher that matches exactly s; never skips spacing, never a keyword"
+    # value = "s"
+    def str(s) = Str.new(s)
+
+    # [method.match]
+    # params = ["re: Regexp (anchored at the current position; \\A, ^ and lookbehind see the whole input)"]
+    # returns = "Matcher for the regexp; raw, nothing skipped"
+    # value = "the matched text"
     def match(re) = Pat.new(re)
-    def any       = Pat.new(/./m)
-    def eof       = any.absent.label("end of input")
-    def never     = Pat.new(/(?!)/)
+
+    # [method.any]
+    # returns = "Matcher for any one character, newlines included"
+    def any = Pat.new(/./m)
+
+    # [method.eof]
+    # returns = "Matcher that succeeds only at the end of the input, consuming nothing"
+    def eof = any.absent.label("end of input")
+
+    # [method.never]
+    # returns = "Matcher that always fails (a placeholder for an empty choice)"
+    def never = Pat.new(/(?!)/)
 
     # A whole word: "if" matches in "if x" but not in "iffy".
-    def keyword(word) = (str(word) >> match(/\w/).absent).map(&:first).label(word.inspect)
+    # [method.keyword]
+    # params = ["word: String"]
+    # returns = "Matcher for word when not followed by a word character"
+    # value = "word"
+    def keyword(word) = Pick.new([str(word), match(/\w/).absent], 0).label(word.inspect)
 
     # --- Literals and the readable forms ---
 
     # Anything usable as a matcher: strings become literals, regexps
     # raw patterns, and matchers pass through unchanged.
+    # [method.lit]
+    # params = ["x: String | Regexp | Matcher"]
+    # returns = "A matcher: String -> literal, Regexp -> raw pattern, matcher unchanged"
+    # raises = ["ArgumentError (anything else, or only(...) outside seq)"]
     def lit(x) = Birman.coerce(x)
 
     # The matcher a literal stands for: "let" is a keyword, "=" an exact
-    # string, either way followed by the grammar's ignore rule. Built
-    # once per grammar instance.
+    # string, either way followed by the ignore rule of the grammar
+    # running it. Built once per grammar instance.
+    # [method.literal_matcher]
+    # params = ["text: String"]
+    # returns = "The matcher a literal stands for: keyword or exact string, then the ignore rule"
+    # side_effects = ["Caches the matcher on the grammar instance"]
+    # notes = "Called by Lit at match time; rarely needed directly"
     def literal_matcher(text)
       (@literal_matchers ||= {})[text] ||= begin
         base = text.match?(/\A\w+\z/) ? keyword(text) : str(text)
-        Lexeme.new(base, ignore_rule, text)
+        Lexeme.new(base, text)
       end
     end
 
     # Sequence. Put `commit` where the construct becomes unambiguous:
     #   seq("let", commit, name.as(:name), "=", expr.as(:value))
+    # [method.seq]
+    # params = ["*items: String | Regexp | Matcher | commit | only(...)"]
+    # returns = "Matcher for the items in order"
+    # value = "Array of item values; a Hash of the .as-named parts if any; the only(...) part's value if used"
+    # raises = ["ArgumentError (commit first; more than one only; only mixed with .as names)",
+    #           "ArgumentError (a name used twice; a nested seq's names the value would drop)"]
     def seq(*items)
       commit_at = items.index(COMMIT)
       raise ArgumentError, "commit can't come first in a seq: nothing has matched yet" if commit_at&.zero?
@@ -54,38 +101,76 @@ module Birman
       Seq.new(parts.map { _1.is_a?(Only) ? _1.matcher : lit(_1) }, commit_at, only_at)
     end
 
+    # [method.commit]
+    # returns = "Marker for seq: if anything after it fails, the whole parse fails there"
+    # notes = "Lookaheads contain a commit failure; rep and maybe don't"
     def commit = COMMIT
 
     # Inside seq, makes this part the seq's whole value; the rest still
     # has to match but is dropped:
     #   seq("[", commit, only(sep_by(value, ",")), "]")   # => [...]
+    # [method.only]
+    # params = ["x: String | Regexp | Matcher"]
+    # returns = "Marker for seq making x's value the seq's whole value"
+    # notes = "x's .as name, if any, passes to the enclosing seq"
     def only(x) = Only.new(lit(x))
 
     # Ordered choice: the first alternative that matches wins.
+    # [method.either]
+    # params = ["*alts: String | Regexp | Matcher"]
+    # returns = "Ordered choice: the first alternative that matches wins"
+    # value = "The winning alternative's value (a name all the alternatives share passes outward)"
     def either(*alts) = alts.size == 1 ? lit(alts.first) : Choice.new(alts.map { lit(_1) })
 
     # One of several literals. Given a hash, each literal produces its
     # value: one_of("true" => true, "null" => nil). They're tried in
     # order, so put longer ones first ("<=" before "<").
+    # [method.one_of]
+    # params = ["*choices: String literals", "or one Hash of literal => value"]
+    # returns = "Ordered choice of the literals"
+    # value = "The matched literal, or its Hash value"
     def one_of(*choices)
       return either(*choices) unless choices in [Hash => table]
 
-      either(*table.map { |text, value| lit(text).map { value } })
+      either(*table.map { |text, value| lit(text).means(value) })
     end
 
     # Left-associative binary operators, folded into nodes:
     #   infix(product, "+", "-", node: BinOp)   # BinOp = node(:op, :left, :right)
+    # [method.infix]
+    # params = ["operand: Matcher", "*ops: String literals", "node: node class with op, left, right fields"]
+    # returns = "Matcher for operand (op operand)*, folded left-associatively"
+    # value = "operand's value, or nested node instances with spans"
+    # raises = ["ArgumentError at parse time if node lacks op/left/right"]
     def infix(operand, *ops, node:) = Infix.new(Birman.coerce(operand), either(*ops).label("operator"), node)
 
     # One or more items separated by sep; value is an array of items.
-    def sep_by1(item, sep) = SepBy.new(lit(item), lit(sep))
-
-    # Zero or more items separated by sep.
-    def sep_by(item, sep) = sep_by1(item, sep).maybe.map { _1 || [] }
+    # With trailing: true, one separator may follow the last item:
+    #   seq("[", only(sep_by(value, ",", trailing: true)), "]")   # [1, 2,]
+    # [method.sep_by1]
+    # params = ["item: String | Regexp | Matcher", "sep: String | Regexp | Matcher", "trailing: Boolean = false"]
+    # returns = "Matcher for one or more items separated by sep (and, if trailing, maybe one more sep)"
+    # value = "Array of item values (item's .as name passes outward)"
+    def sep_by1(item, sep, trailing: false) = SepBy.new(lit(item), lit(sep), 1, trailing)
+
+    # Zero or more items separated by sep. A trailing separator needs an
+    # item before it: "," alone isn't an empty list.
+    # [method.sep_by]
+    # params = ["item: String | Regexp | Matcher", "sep: String | Regexp | Matcher", "trailing: Boolean = false"]
+    # returns = "Matcher for zero or more items separated by sep (and, if trailing, maybe one more sep)"
+    # value = "Array of item values, [] when there are none"
+    def sep_by(item, sep, trailing: false) = SepBy.new(lit(item), lit(sep), 0, trailing)
 
     # inner wrapped in open/close; value is inner's.
+    # [method.between]
+    # params = ["open: String | Matcher", "close: String | Matcher", "inner: Matcher"]
+    # returns = "Matcher for open inner close"
+    # value = "inner's value (inner's .as name passes outward)"
     def between(open, close, inner) = Pick.new([lit(open), Birman.coerce(inner), lit(close)], 1)
 
+    # [method.parens]
+    # params = ["inner: Matcher"]
+    # returns = "between(\"(\", \")\", inner)"
     def parens(inner) = between("(", ")", inner)
 
     private
@@ -99,4 +184,11 @@ module Birman
 
     def ignore_rule = self.class.respond_to?(:ignore) ? self.class.ignore : nil
   end
+
+  # Birman.seq, Birman.str and the rest, for building matchers outside a
+  # grammar (see Birman.parse). literal_matcher stays private: it's what
+  # a grammar runs for a literal, and Birman isn't one.
+  extend Combinators
+
+  singleton_class.send(:private, :literal_matcher)
 end

data/lib/birman/compiler/actions.rb

@@ -0,0 +1,309 @@
+# frozen_string_literal: true
+
+module Birman
+  class Compiler
+    # Copies the grammar methods that .action(:name) and .check(:name) call into the
+    # generated Parser, read from their source with Prism, together with
+    # the grammar methods they call and the constants they read (as
+    # literals, looked up lexically from where the method was written).
+    #
+    # Copied as written, so Integer(), String#to_i, #hex and #oct, which
+    # in a Spinel binary raise past 63 bits (or give 2**63 - 1, when the
+    # receiver's type isn't known), are refused (see Overflows):
+    # large_integer (Grammar's, and the generated parser's) is the way to
+    # read a number.
+    # [class.Compiler.Actions]
+    # purpose = "Copies .action and .check methods, their helpers and constants, into the generated parser"
+    # dependencies = ["prism (bundled from Ruby 3.3; a gem on 3.2), loaded on first use"]
+    class Actions
+      # Methods of the generated Parser and its runtime base class that a
+      # copied method must not replace.
+      RESERVED = /\A(?:rule_|ref_|lit_|init_|skip_|str_)/
+      RUNTIME_METHODS = %i[
+        initialize input parse reset skip_ignored fail_at merge commit_failure ignored_after token_end span_end
+        span failure failure_because invalid_input label_end check_end peek_end bytes_at? abandon left_recursion
+        large_integer
+      ].freeze
+
+      # Methods the generated parser's runtime provides as Birman::Grammar
+      # does, which copied methods may call and actions may name.
+      PROVIDED = %i[large_integer].freeze
+
+      # The constant paths copied methods may read: Float::INFINITY and
+      # Float::NAN, which have no literal.
+      FLOAT_CONSTANTS = %i[INFINITY NAN].freeze
+
+      def initialize(grammar)
+        @grammar = grammar
+        @methods = {}
+        @constants = CopiedConstants.new
+        @trees = {}
+      end
+
+      # Copies grammar method name (and what it needs), once.
+      # form: "action" or "check", for the error if there's no such method.
+      def copy(name, rule, form = "action")
+        return if @methods.key?(name) || provided?(name)
+
+        @methods[name] = nil # reserved first: methods can call each other
+        @methods[name] = method_source(find(name, rule, form), rule)
+      end
+
+      def any? = !@methods.empty?
+
+      def constants = @constants.sources
+
+      def constant_names = @constants.names
+
+      def methods = @methods.values
+
+      def names = @methods.keys
+
+      private
+
+      def find(name, rule, form)
+        unless @grammar.respond_to?(name, true)
+          raise Error, "rule :#{rule} uses .#{form}(:#{name}), but the grammar has no method #{name}"
+        end
+        if RESERVED.match?(name) || RUNTIME_METHODS.include?(name)
+          raise Error, "the grammar method #{name} would clash with the generated parser's own #{name}"
+        end
+
+        method = @grammar.method(name)
+        file, line = method.source_location
+        raise Error, "rule :#{rule}: can't find the source of #{name}" unless file && File.exist?(file)
+
+        MethodFinder.new(name, line).find(tree(file)) ||
+          raise(Error, "rule :#{rule}: #{name} isn't written with def, so it can't be copied")
+      end
+
+      def tree(file) = @trees[file] ||= MethodFinder.parse(file)
+
+      def method_source(found, rule)
+        node, scopes = found
+        walk(node) { visit(_1, scopes, rule) }
+        MethodFinder.source(node)
+      end
+
+      def visit(node, scopes, rule)
+        case node
+        when Prism::ConstantReadNode then @constants.read(node.name, scopes, rule)
+        when Prism::ConstantPathNode then constant_path(node, rule)
+        when Prism::SuperNode, Prism::ForwardingSuperNode
+          raise Error, "rule :#{rule}: copied methods can't call super"
+        when Prism::BlockArgumentNode then Overflows.check_block(node, rule)
+        when Prism::CallNode then call(node, rule)
+        end
+      end
+
+      # Float::INFINITY and Float::NAN, which the generated parser reads
+      # as written (its Float is checked like any core class's).
+      def constant_path(node, rule)
+        float = node.parent.is_a?(Prism::ConstantReadNode) && node.parent.name == :Float
+        return if float && FLOAT_CONSTANTS.include?(node.name)
+
+        raise Error, "rule :#{rule}: copied methods can't read #{node.slice}; use a constant of the grammar"
+      end
+
+      def call(node, rule)
+        if Overflows::DISPATCH.include?(node.name) then dispatch(node, rule)
+        elsif self_call?(node) then own_call(node.name, rule)
+        else Overflows.check_call(node, rule)
+        end
+      end
+
+      def self_call?(node) = node.receiver.nil? || node.receiver.is_a?(Prism::SelfNode)
+
+      # A call on self (written or not): a grammar method is copied too;
+      # Birman's own API (rules, keywords...) isn't there to call, and
+      # Kernel's Integer overflows.
+      def own_call(name, rule)
+        return if provided?(name)
+        raise Error, "rule :#{rule}: copied methods can't call the grammar's #{name}" if birman_api?(name)
+        return copy(name, rule) if grammar_method?(name)
+
+        Overflows.check(name, rule)
+      end
+
+      # send(:name), method(:name)...: checked as a call of name would be.
+      def dispatch(node, rule)
+        name = Overflows.dispatched(node, rule)
+        self_call?(node) ? own_call(name, rule) : Overflows.check(name, rule)
+      end
+
+      # Birman::Grammar's own, not one the grammar defines over it.
+      def provided?(name)
+        PROVIDED.include?(name) && @grammar.method(name).owner == Birman::Grammar
+      end
+
+      # A method of the grammar's, or of a rule module's (Rules::Strings'
+      # decode_escape): not Ruby's, and not part of Birman's grammar API.
+      def grammar_method?(name)
+        return false unless @grammar.respond_to?(name, true)
+
+        ![Kernel, Object, BasicObject].include?(@grammar.method(name).owner)
+      end
+
+      # Rules (which return matchers) and the methods Birman::Grammar and
+      # Combinators give every grammar.
+      def birman_api?(name)
+        return false unless @grammar.respond_to?(name, true)
+        return true if @grammar.respond_to?(:"__rule_#{name}", true)
+
+        owner = @grammar.method(name).owner
+        [Birman::Grammar, Birman::Combinators].include?(owner)
+      end
+
+      def walk(node, &)
+        yield node
+        node.compact_child_nodes.each { walk(_1, &) }
+      end
+    end
+
+    # Methods that raise past 63 bits in a Spinel binary (or, on a receiver
+    # Spinel can't type, silently give 2**63 - 1), refused in copied
+    # methods, by call or by name (map(&:to_i), send(:hex), method(:Integer)).
+    # Whether a receiver is a String can't be told, so it's taken to be one;
+    # only to_i on an Integer literal is let through. A net for the common
+    # forms, not a proof: Float(t).round overflows too, maybe silently.
+    module Overflows
+      INTEGER = "Integer()/to_i overflow past 63 bits in a Spinel binary; use large_integer(...)"
+      MESSAGES = {
+        Integer: INTEGER, to_i: INTEGER,
+        hex: "hex overflows past 63 bits in a Spinel binary; use large_integer(text, 16)",
+        oct: "oct overflows past 63 bits in a Spinel binary; use large_integer(text, 8)"
+      }.freeze
+
+      def self.check(name, rule)
+        message = MESSAGES[name]
+        raise Error, "rule :#{rule}: #{message}" if message
+      end
+
+      def self.check_call(node, rule)
+        check(node.name, rule) unless node.name == :to_i && node.receiver.is_a?(Prism::IntegerNode)
+      end
+
+      # map(&:to_i) calls to_i on each element.
+      def self.check_block(node, rule)
+        check(node.expression.unescaped.to_sym, rule) if node.expression.is_a?(Prism::SymbolNode)
+      end
+
+      # Methods that call a method named by their first argument.
+      DISPATCH = %i[send public_send __send__ method public_method].freeze
+
+      # The method a DISPATCH call names, which must be written as a
+      # symbol to be checked.
+      def self.dispatched(node, rule)
+        target = node.arguments&.arguments&.first
+        return target.unescaped.to_sym if target.is_a?(Prism::SymbolNode)
+
+        raise Error, "rule :#{rule}: copied methods can't #{node.name} a computed name (#{node.slice}); " \
+                     "call the method by name"
+      end
+    end
+
+    # The constants copied methods read, each written as a literal. Looked
+    # up lexically from where the method was written; core classes are
+    # left as they are, and anything else is refused.
+    class CopiedConstants
+      def initialize
+        @constants = {}
+      end
+
+      def sources = @constants.map { |name, (_owner, source)| "#{name} = #{source}" }
+
+      def names = @constants.keys.map(&:to_s)
+
+      # Classes Ruby (or the runtime) provides, which copied code may name.
+      CORE_CLASSES = %i[
+        Integer Float String Symbol Array Hash Regexp Comparable Math Kernel Encoding NilClass
+        StandardError ArgumentError KeyError RangeError TypeError FloatDomainError
+      ].freeze
+
+      def read(name, scopes, rule)
+        owner = scopes.find { _1.const_defined?(name) } or
+          raise Error, "rule :#{rule}: copied methods read #{name}, which can't be found"
+        value = owner.const_get(name)
+        return core_class(name, value, rule) if value.is_a?(Module)
+
+        check_constant(name, owner, value, rule)
+        @constants[name] = [owner, Literals.source(value)]
+      end
+
+      def core_class(name, value, rule)
+        return if CORE_CLASSES.include?(name) && Object.const_get(name).equal?(value)
+
+        raise Error, "rule :#{rule}: copied methods use #{value.inspect}, which the generated parser won't have"
+      end
+
+      def check_constant(name, owner, value, rule)
+        if RESERVED_CONSTANTS.include?(name.to_s)
+          raise Error, "rule :#{rule}: copied methods read #{owner}::#{name}, a name the generated parser needs"
+        end
+        unless Literals.literal?(value)
+          raise Error, "rule :#{rule}: #{name} must be a literal (strings, numbers, symbols, arrays, hashes)"
+        end
+
+        seen, = @constants[name]
+        return if seen.nil? || seen == owner
+
+        raise Error, "copied methods read both #{seen}::#{name} and #{owner}::#{name}; rename one"
+      end
+    end
+
+    # The def of name at line, with the modules it was written in
+    # (innermost first, then Object), for looking up its constants.
+    class MethodFinder
+      def initialize(name, line)
+        @name = name
+        @line = line
+      end
+
+      # file's syntax tree, read with Prism.
+      def self.parse(file)
+        load_prism
+        Prism.parse_file(file).value
+      end
+
+      def self.load_prism
+        require "prism"
+      rescue LoadError
+        raise Error, "copying .action and .check methods and node initializers needs the prism gem " \
+                     "(bundled from Ruby 3.3): add it to your Gemfile"
+      end
+
+      # The def's source, its lines indented as if written at the left
+      # margin.
+      def self.source(node)
+        "#{reindent(node.slice, node.location.start_column)}\n"
+      end
+
+      # The method's later lines are indented for where it was written.
+      def self.reindent(source, column)
+        first, *rest = source.lines
+        [first, *rest.map { _1.sub(/\A {0,#{column}}/, "") }].join
+      end
+
+      def find(node, nesting = [])
+        case node
+        when Prism::DefNode
+          return [node, scopes(nesting)] if node.name == @name && node.location.start_line == @line
+        when Prism::ClassNode, Prism::ModuleNode
+          nesting = [*nesting, node.constant_path.slice]
+        end
+        node.compact_child_nodes.each do |child|
+          found = find(child, nesting)
+          return found if found
+        end
+        nil
+      end
+
+      private
+
+      def scopes(nesting)
+        paths = nesting.each_index.map { nesting[0.._1].join("::") }.reverse
+        [*paths.filter_map { Object.const_get(_1) if Object.const_defined?(_1) }, Object]
+      end
+    end
+  end
+end