birman 0.1.0

data/lib/birman/grammar.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Birman
+  # Include this to make a class a grammar. The class body also gets
+  # `node` (see Nodes), so AST nodes can live beside their rules.
+  #
+  # Including it in a module makes a module of shared rules: whatever
+  # includes that module (a class, or another such module) becomes a
+  # grammar too.
+  module Grammar
+    include Combinators
+
+    def self.included(base)
+      base.extend(RuleSet)
+      base.extend(Nodes)
+      base.extend(base.is_a?(Class) ? ClassMethods : Shared)
+    end
+
+    # Extended onto a module that includes Grammar.
+    module Shared
+      def included(base)
+        super
+        base.include(Grammar)
+      end
+    end
+
+    # Settings are inherited by subclasses. Passing nil clears an
+    # inherited one: `ignore nil`, `keywords nil`.
+    module ClassMethods
+      UNSET = Object.new.freeze
+
+      # The rule a parse starts from.
+      def root(name = UNSET) = setting(:root, name)
+
+      # A rule the parser matches and discards after every literal and
+      # lexeme, and at the start of input. Usually :spacing (from
+      # Birman::Rules::Whitespace or Comments).
+      def ignore(name = UNSET) = setting(:ignore, name)
+
+      # Reserved words, which Birman::Rules::Identifiers won't accept.
+      def keywords(*words)
+        return setting(:keywords) || [] if words.empty?
+
+        setting(:keywords, words == [nil] ? [] : words.map(&:to_s))
+      end
+
+      # These share one instance, so each rule's matcher is built once
+      # rather than on every parse. Defining a rule or changing a
+      # setting starts a fresh one.
+      def parse(input)  = instance.parse(input)
+      def parse!(input, **) = instance.parse!(input, **)
+      def to_ast        = instance.to_ast
+      def to_ebnf       = EBNF.call(to_ast)
+
+      private
+
+      def instance = @instance ||= new
+
+      def setting(key, value = UNSET)
+        ivar = :"@#{key}"
+        unless value.equal?(UNSET)
+          @instance = nil
+          return instance_variable_set(ivar, value)
+        end
+        return instance_variable_get(ivar) if instance_variable_defined?(ivar)
+
+        superclass.public_send(key) if superclass.respond_to?(key)
+      end
+    end
+
+    def keywords = self.class.keywords
+
+    # Returns Success(value) or Failure(...). Never raises on bad input.
+    def parse(input)
+      ctx = Context.new(input, self)
+      start = (ignore_rule && Ref.new(ignore_rule).match(ctx, 0)&.pos) || 0
+      result = Ref.new(root_rule).match(ctx, start)
+      return Success.new(result.value) if result && result.pos == input.bytesize
+
+      ctx.fail_at(result.pos, "end of input") if result
+      build_failure(input, ctx.furthest, ctx.expected)
+    rescue Committed => e
+      build_failure(input, e.pos, e.expected, committed: true)
+    rescue SystemStackError
+      build_failure(input, ctx.position, [], reason: "input is nested too deeply")
+    end
+
+    # Returns the value or raises ParseError, worded by formatter.
+    def parse!(input, formatter: ErrorFormatter.new) = parse(input).value!(formatter)
+
+    def matcher_for(name)
+      (@matchers ||= {})[name] ||= begin
+        body = :"__rule_#{name}"
+        raise ArgumentError, "undefined rule :#{name}" unless respond_to?(body, true)
+
+        Birman.coerce(send(body))
+      end
+    end
+
+    # { rule_name => ast } for every rule reachable from the root (and
+    # the ignore rule), in first-use order.
+    def to_ast
+      rules = {}
+      queue = [root_rule, ignore_rule].compact
+      while (name = queue.shift)
+        next if rules.key?(name)
+
+        rules[name] = matcher_for(name).to_ast
+        queue.concat(AST.refs(rules[name]))
+      end
+      rules
+    end
+
+    private
+
+    def build_failure(input, byte_pos, expected, committed: false, reason: nil)
+      Failure.new(input:, pos: Birman.char_offset(input, byte_pos), expected: expected.uniq, committed:, reason:)
+    end
+
+    def root_rule = self.class.root || raise(ArgumentError, "#{self.class} has no root rule")
+  end
+end

data/lib/birman/matchers.rb

@@ -0,0 +1,396 @@
+# frozen_string_literal: true
+
+module Birman
+  def self.coerce(x)
+    case x
+    when Matcher then x
+    when String  then Str.new(x)
+    when Regexp  then Pat.new(x)
+    else raise ArgumentError, "can't make a matcher from #{x.inspect}"
+    end
+  end
+
+  # Combinator operators, mixed into every matcher type.
+  #
+  # Ruby's operator precedence shapes the notation, tightest first:
+  #
+  #   >>   sequence
+  #   &    conjunction
+  #   |    ordered choice
+  #   >    commit
+  #
+  # So `a > b >> c` means `a > (b >> c)`, and a commit inside a choice
+  # needs parentheses: `(keyword("if") > body) | other`.
+  module Matcher
+    def >>(other) = Seq.new([self, Birman.coerce(other)])
+    def |(other)  = Choice.new([self, Birman.coerce(other)])
+
+    # Conjunction: self must match here (without consuming input),
+    # then other is matched from the same spot. Value is other's.
+    def &(other) = Both.new(self, Birman.coerce(other))
+
+    # Commit: once self matches, everything after it must match too, or
+    # parsing stops right here with an error instead of backtracking.
+    # This is dry-logic's implication operator, applied to parsing.
+    def >(other)
+      left  = uncommitted_parts(self)
+      right = uncommitted_parts(Birman.coerce(other))
+      Seq.new([*left, *right], left.size)
+    end
+
+    def rep(min = 0, max = nil) = Rep.new(self, min, max)
+    def maybe       = rep(0, 1).map(&:first)
+    def absent      = Not.new(self)
+    def present     = And.new(self)
+    def map(&fn)    = Action.new(self, fn)
+    def label(name) = Label.new(self, name)
+
+    # Never mention this matcher in error messages (for whitespace,
+    # comments, and other things nobody wants to be told to type).
+    def quiet = Quiet.new(self)
+
+    # Name this part of a sequence. A sequence with any named parts
+    # produces a hash of just those parts instead of an array.
+    def as(key) = Named.new(self, key)
+
+    # Build a node from the value: a hash of named parts becomes
+    # keyword arguments; anything else fills the node's first field.
+    # Nodes made with Birman.node get a source span.
+    def to(node) = To.new(self, node)
+
+    # The name given by #as, seen through wrappers like labels.
+    def capture_key = nil
+
+    private
+
+    def uncommitted_parts(m) = m.is_a?(Seq) && m.commit_at.nil? ? m.parts : [m]
+  end
+
+  # Literal string.
+  Str = Data.define(:str) do
+    include Matcher
+
+    def match(ctx, pos)
+      if ctx.input.byteslice(pos, str.bytesize) == str
+        Match.new(pos + str.bytesize, str)
+      else
+        ctx.fail_at(pos, str.inspect)
+      end
+    end
+
+    def to_ast = [:str, str]
+  end
+
+  # Regex, anchored at the current position with \G.
+  Pat = Data.define(:re, :source) do
+    include Matcher
+
+    def initialize(re:, source: re.inspect) = super
+
+    def match(ctx, pos)
+      scanner = ctx.scanner
+      scanner.pos = pos
+      len = scanner.match?(re) or return ctx.fail_at(pos, source)
+      Match.new(pos + len, scanner.matched)
+    end
+
+    def to_ast = [:pat, source]
+  end
+
+  # Sequence: parts match in order. The value is an array, or a hash
+  # of the named parts if any are named (see Matcher#as). Parts at
+  # index `commit_at` and beyond are committed: if one fails, parsing stops.
+  Seq = Data.define(:parts, :commit_at) do
+    include Matcher
+
+    # Arrays are copied and frozen so a matcher is immutable all the way down.
+    def initialize(parts:, commit_at: nil) = super(parts: parts.dup.freeze, commit_at:)
+
+    def >>(other) = with(parts: [*parts, Birman.coerce(other)])
+
+    def match(ctx, pos)
+      values = parts.each_with_index.map do |part, i|
+        r = part.match(ctx, pos)
+        unless r
+          fail_committed!(ctx, part, pos) if commit_at && i >= commit_at
+          return nil
+        end
+        pos = r.pos
+        r.value
+      end
+      Match.new(pos, shape(values))
+    end
+
+    def to_ast = [:seq, commit_at, parts.map(&:to_ast)]
+
+    private
+
+    # If the failed part recorded nothing at or past pos (it's quiet,
+    # or a negative lookahead), describe the part itself so the error
+    # still says what was expected.
+    def fail_committed!(ctx, part, pos)
+      ctx.fail_at(pos, EBNF.expr(part.to_ast, EBNF::CHOICE)) if ctx.furthest < pos || ctx.expected.empty?
+      ctx.commit_failure!
+    end
+
+    def shape(values)
+      keys = parts.map(&:capture_key)
+      return values if keys.none?
+
+      keys.zip(values).select(&:first).to_h
+    end
+  end
+
+  # Parts in order, keeping only the value (and the name, see
+  # Matcher#as) of the part at index. Unlike Seq it never flattens
+  # or reshapes, so the kept part comes through as it was.
+  Pick = Data.define(:parts, :index) do
+    include Matcher
+
+    def initialize(parts:, index:) = super(parts: parts.dup.freeze, index:)
+
+    def match(ctx, pos)
+      value = nil
+      parts.each_with_index do |part, i|
+        r = part.match(ctx, pos) or return nil
+        value = r.value if i == index
+        pos = r.pos
+      end
+      Match.new(pos, value)
+    end
+
+    def to_ast = [:seq, nil, parts.map(&:to_ast)]
+    def capture_key = parts[index].capture_key
+  end
+
+  # One or more items separated by sep. The value is an array of the
+  # items, and the name (see Matcher#as) is the item's.
+  SepBy = Data.define(:item, :sep) do
+    include Matcher
+
+    def match(ctx, pos)
+      r = item.match(ctx, pos) or return nil
+      values = [r.value]
+      pos = r.pos
+      loop do
+        s = sep.match(ctx, pos) or break
+        r = item.match(ctx, s.pos) or break
+        break if r.pos == pos # an empty separator and item would loop forever
+
+        values << r.value
+        pos = r.pos
+      end
+      Match.new(pos, values)
+    end
+
+    def to_ast = [:seq, nil, [item.to_ast, [:rep, 0, nil, [:seq, nil, [sep.to_ast, item.to_ast]]]]]
+    def capture_key = item.capture_key
+  end
+
+  # Ordered choice: the first alternative that matches wins.
+  Choice = Data.define(:alts) do
+    include Matcher
+
+    def initialize(alts:) = super(alts: alts.dup.freeze)
+
+    def |(other) = with(alts: [*alts, Birman.coerce(other)])
+
+    def match(ctx, pos)
+      alts.each do |alt|
+        r = alt.match(ctx, pos) and return r
+      end
+      nil
+    end
+
+    def to_ast = [:choice, alts.map(&:to_ast)]
+  end
+
+  # Repetition with min/max; value is an array.
+  Rep = Data.define(:inner, :min, :max) do
+    include Matcher
+
+    def match(ctx, pos)
+      values = []
+      while max.nil? || values.size < max
+        r = inner.match(ctx, pos) or break
+        break if r.pos == pos # an empty match would loop forever
+
+        values << r.value
+        pos = r.pos
+      end
+      values.size >= min ? Match.new(pos, values) : nil
+    end
+
+    def to_ast = [:rep, min, max, inner.to_ast]
+    def capture_key = inner.capture_key
+  end
+
+  # Negative lookahead. The inner attempt's failures are discarded,
+  # since failing is exactly what we hoped for.
+  Not = Data.define(:inner) do
+    include Matcher
+
+    def match(ctx, pos)
+      r, = ctx.isolate(pos) { ctx.lookahead { inner.match(ctx, pos) } }
+      r ? nil : Match.new(pos, nil)
+    end
+
+    def to_ast = [:not, inner.to_ast]
+  end
+
+  # Positive lookahead: succeeds without consuming if inner matches.
+  And = Data.define(:inner) do
+    include Matcher
+
+    def match(ctx, pos) = ctx.peek?(inner, pos) ? Match.new(pos, nil) : nil
+
+    def to_ast = [:and, inner.to_ast]
+  end
+
+  # Conjunction (see Matcher#&).
+  Both = Data.define(:left, :right) do
+    include Matcher
+
+    def match(ctx, pos) = ctx.peek?(left, pos) ? right.match(ctx, pos) : nil
+
+    def to_ast = [:both, left.to_ast, right.to_ast]
+  end
+
+  # Semantic action: transform the value of a successful match.
+  Action = Data.define(:inner, :fn) do
+    include Matcher
+
+    def match(ctx, pos)
+      r = inner.match(ctx, pos) or return nil
+      r.with(value: fn.call(r.value))
+    end
+
+    def to_ast = [:action, inner.to_ast]
+    def capture_key = inner.capture_key
+  end
+
+  # Names a matcher for error messages. If inner fails without getting
+  # past its starting position, the error says "expected <name>"
+  # instead of listing every low-level alternative. If it got further,
+  # the deeper (more specific) error is kept.
+  Label = Data.define(:inner, :name) do
+    include Matcher
+
+    def match(ctx, pos)
+      result, furthest, expected = ctx.isolate(pos) { inner.match(ctx, pos) }
+      if result.nil? && furthest == pos
+        ctx.fail_at(pos, name)
+      else
+        ctx.merge(furthest, expected)
+      end
+      result
+    end
+
+    def to_ast = [:label, name, inner.to_ast]
+    def capture_key = inner.capture_key
+  end
+
+  # See Matcher#as. Passes the value through; the enclosing Seq reads
+  # the key.
+  Named = Data.define(:inner, :key) do
+    include Matcher
+
+    def match(ctx, pos) = inner.match(ctx, pos)
+    def to_ast = inner.to_ast
+    def capture_key = key
+  end
+
+  # See Matcher#to.
+  To = Data.define(:inner, :node) do
+    include Matcher
+
+    def match(ctx, pos)
+      r = inner.match(ctx, pos) or return nil
+      fields = r.value.is_a?(Hash) ? r.value : { node.members.first => r.value }
+      r.with(value: Birman.build(node, fields, ctx, pos, r.pos))
+    end
+
+    def to_ast = [:action, inner.to_ast]
+    def capture_key = inner.capture_key
+  end
+
+  # A token: inner, then the grammar's ignore rule (usually spacing),
+  # keeping inner's value. With no ignore rule it's just inner. `source`
+  # is how a literal shows up in EBNF ("let" rather than its internals).
+  Lexeme = Data.define(:inner, :ignore, :source) do
+    include Matcher
+
+    def initialize(inner:, ignore:, source: nil) = super
+
+    def match(ctx, pos)
+      r = inner.match(ctx, pos) or return nil
+      return r unless ignore
+
+      s = Ref.new(ignore).match(ctx, r.pos) or return r
+      ctx.ignored_after(r.pos, s.pos)
+      Match.new(s.pos, r.value)
+    end
+
+    def to_ast = source ? [:str, source] : inner.to_ast
+    def capture_key = inner.capture_key
+  end
+
+  # Left-associative binary operators: operand (op operand)*, folded
+  # into nodes built as node.new(op:, left:, right:), so the node needs
+  # exactly those fields, in any order.
+  # Each node's span covers its whole subexpression.
+  Infix = Data.define(:operand, :op, :node) do
+    include Matcher
+
+    def match(ctx, pos)
+      start = pos
+      r = operand.match(ctx, pos) or return nil
+      left = r.value
+      pos = r.pos
+
+      loop do
+        o = op.match(ctx, pos) or break
+        right = operand.match(ctx, o.pos) or break
+        fields = { op: o.value, left:, right: right.value }
+        left = Birman.build(node, fields, ctx, start, right.pos)
+        pos = right.pos
+      end
+
+      Match.new(pos, left)
+    end
+
+    def to_ast
+      [:seq, nil, [operand.to_ast, [:rep, 0, nil, [:seq, nil, [op.to_ast, operand.to_ast]]]]]
+    end
+  end
+
+  # See Matcher#quiet.
+  Quiet = Data.define(:inner) do
+    include Matcher
+
+    def match(ctx, pos)
+      result, = ctx.isolate(pos) { inner.match(ctx, pos) }
+      result
+    end
+
+    def to_ast = [:quiet, inner.to_ast]
+    def capture_key = inner.capture_key
+  end
+
+  # A reference to a named rule, resolved through the grammar carried
+  # in the context. Each (rule, position) is parsed once and memoized
+  # along with its diagnostics: packrat parsing.
+  Ref = Data.define(:name) do
+    include Matcher
+
+    def match(ctx, pos)
+      result, furthest, expected = ctx.memoize(name, pos) do
+        ctx.isolate(pos) { ctx.grammar.matcher_for(name).match(ctx, pos) }
+      end
+      ctx.merge(furthest, expected)
+      result
+    end
+
+    def to_ast = [:ref, name]
+  end
+end

data/lib/birman/nodes.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Birman
+  # Where a node came from in the source. Stored as byte offsets (see
+  # Context); from, to, line and column count characters.
+  Span = Data.define(:input, :byte_from, :byte_to) do
+    def from   = Birman.char_offset(input, byte_from)
+    def to     = Birman.char_offset(input, byte_to)
+    def text   = input.byteslice(byte_from...byte_to)
+    def line   = before.count("\n") + 1
+    def column = before.length - (before.rindex("\n") || -1)
+    def to_s   = "line #{line}, column #{column}"
+    def inspect = "#<Birman::Span #{line}:#{column} #{text.inspect}>"
+
+    private
+
+    def before = input.byteslice(0, byte_from)
+  end
+
+  # Mixed into every class made with Nodes#node. Adds an optional
+  # source span that the parser fills in. The span is deliberately
+  # not a Data member, so equality and pattern matching ignore it:
+  # `in Num(value)` still works, and two nodes with the same contents
+  # are equal wherever they came from.
+  module Node
+    attr_reader :span
+
+    # node_class.new(**fields) with a span. The span is set before
+    # initialize runs rather than passed to it, so a node can define
+    # its own initialize (for field defaults, say) without knowing
+    # about spans.
+    def self.new_with_span(node_class, span, fields)
+      node = node_class.allocate
+      node.instance_variable_set(:@span, span)
+      node.__send__(:initialize, **fields)
+      node
+    end
+
+    # span: is still accepted here, for building nodes by hand.
+    def initialize(span: nil, **fields)
+      @span = span if span
+      super(**fields)
+    end
+
+    def with(**changes) = Node.new_with_span(self.class, span, to_h.merge(changes))
+
+    # S-expression style: (BinOp "+" (Num 1) (Num 2))
+    def inspect
+      name = self.class.name&.split("::")&.last || "node"
+      "#{["(#{name}", *to_h.values.map(&:inspect)].join(" ")})"
+    end
+
+    def pretty_print(q) = q.text(inspect)
+  end
+
+  # Extend a module with this to define AST nodes:
+  #
+  #   module MyLang
+  #     extend Birman::Nodes
+  #     Num = node(:value)
+  #   end
+  #
+  # Also available as Birman.node.
+  module Nodes
+    def node(*members, &body)
+      raise ArgumentError, "a node can't have a field named span: every node already has one" if members.include?(:span)
+
+      Data.define(*members) do
+        include Node
+
+        class_eval(&body) if body
+      end
+    end
+  end
+
+  extend Nodes
+
+  # Build a node (or any Data class) from a hash of fields, attaching a
+  # span when the class supports one. The span ends where the last
+  # token did, leaving out spacing the ignore rule consumed after it;
+  # any whitespace consumed some other way (Rules' token helper, say)
+  # is trimmed too.
+  def self.build(node, fields, ctx, from, to)
+    return node.new(**fields) unless node.include?(Node)
+
+    to = ctx.token_end(to)
+    span = Span.new(ctx.input, from, from + ctx.input.byteslice(from, to - from).rstrip.bytesize)
+    Node.new_with_span(node, span, fields)
+  end
+end

data/lib/birman/result.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Birman
+  # Internal: one matcher succeeded, ending at `pos` with `value`.
+  Match = Data.define(:pos, :value)
+
+  # Returned by Grammar#parse when the whole input parsed.
+  Success = Data.define(:value) do
+    def success? = true
+    def failure? = false
+    def value!(_formatter = nil) = value
+  end
+
+  # Returned by Grammar#parse when it didn't. Pure data: rendering it
+  # as text is the ErrorFormatter's job.
+  #
+  # committed: true when a match after a commit (>) failed, meaning the
+  # parser knew exactly which construct it was in and stopped there.
+  #
+  # reason: set when the failure isn't about what was expected, such as
+  # input nested too deeply for Ruby's stack.
+  Failure = Data.define(:input, :pos, :expected, :committed, :reason) do
+    def initialize(input:, pos:, expected:, committed:, reason: nil) = super
+
+    def success? = false
+    def failure? = true
+    def value!(formatter = ErrorFormatter.new) = raise ParseError.new(self, formatter)
+
+    def line   = input[0, pos].count("\n") + 1
+    def column = pos - (input[0, pos].rindex("\n") || -1)
+    def found  = input[pos]&.inspect || "end of input"
+
+    def message(formatter = ErrorFormatter.new) = formatter.call(self)
+
+    # The input can be any size, so only its start is shown.
+    def inspect
+      shown = input.size > 40 ? "#{input[0, 40]}..." : input
+      fields = ["pos=#{pos}", "expected=#{expected.inspect}", "committed=#{committed}"]
+      fields << "reason=#{reason.inspect}" if reason
+      "#<data Birman::Failure #{fields.join(" ")} input=#{shown.inspect}>"
+    end
+  end
+
+  # Raised by parse! (and Failure#value!) with the formatted message.
+  class ParseError < StandardError
+    attr_reader :failure
+
+    def initialize(failure, formatter = ErrorFormatter.new)
+      @failure = failure
+      super(formatter.call(failure))
+    end
+  end
+
+  # Internal: raised when a match after a commit fails. Carries the diagnostics from
+  # the moment of failure. Grammar#parse turns it into a Failure.
+  class Committed < StandardError
+    attr_reader :pos, :expected
+
+    def initialize(pos, expected)
+      @pos = pos
+      @expected = expected
+      super("committed failure at #{pos}")
+    end
+  end
+end

data/lib/birman/rule_set.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Birman
+  # The `rule` and `lexeme` macros. Extend any module or class with it.
+  #
+  # Each rule becomes two ordinary methods: `name` returns a Ref, and a
+  # private `__rule_name` builds the matcher. Because rules are just
+  # methods, they travel through `include` and can be overridden like
+  # any other Ruby method.
+  module RuleSet
+    def rule(name, &)
+      check_rule_name!(name)
+      define_method(:"__rule_#{name}", &)
+      private :"__rule_#{name}"
+      define_method(name) { Ref.new(name) }
+      @instance = nil # a grammar class's cached instance (see Grammar::ClassMethods)
+    end
+
+    # A rule for a token (a number, a string, an identifier): it skips
+    # trailing spacing when the grammar declares `ignore`.
+    def lexeme(name, &body)
+      rule(name) { Lexeme.new(Birman.coerce(instance_exec(&body)), ignore_rule) }
+    end
+
+    private
+
+    # A rule named after a method the grammar relies on (send, class,
+    # match, token...) would silently replace it. Redefining a rule is
+    # fine, and so are private Kernel names like print or select.
+    def check_rule_name!(name)
+      return if private_method_defined?(:"__rule_#{name}")
+
+      owner = [Object, Combinators, Grammar].find { _1.public_method_defined?(name) } ||
+              [Combinators, Grammar].find { _1.private_method_defined?(name) } ||
+              (instance_method(name).owner if method_defined?(name))
+      raise ArgumentError, "rule :#{name} would replace #{owner}##{name}; give the rule another name" if owner
+    end
+  end
+end