birman 0.2.1 → 0.3.0

data/lib/birman/rule_set.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "The rule and lexeme macros"
+# responsibility = "Turning rule blocks into methods, and guarding rule names"
+# pattern = "Class-level DSL (extend)"
 module Birman
   # The `rule` and `lexeme` macros. Extend any module or class with it.
   #
@@ -7,7 +11,18 @@ module Birman
   # private `__rule_name` builds the matcher. Because rules are just
   # methods, they travel through `include` and can be overridden like
   # any other Ruby method.
+  # [module.RuleSet]
+  # purpose = "Defines rules; extend a module with it to write a reusable rule module"
+  # collaborators = ["Grammar (extends its classes with this)", "Ref"]
   module RuleSet
+    # [method.rule]
+    # params = ["name: Symbol", "block: returns the rule's matcher (String, Regexp or Matcher)"]
+    # returns = "nil"
+    # side_effects = [
+    #   "Defines name (returns a Ref) and a private __rule_name",
+    #   "Resets a grammar class's cached instance"
+    # ]
+    # raises = ["ArgumentError if name would replace a method the grammar relies on (send, match, token...)"]
     def rule(name, &)
       check_rule_name!(name)
       define_method(:"__rule_#{name}", &)
@@ -18,22 +33,39 @@ module Birman
 
     # A rule for a token (a number, a string, an identifier): it skips
     # trailing spacing when the grammar declares `ignore`.
+    # [method.lexeme]
+    # params = ["name: Symbol", "body: block returning the token's matcher"]
+    # returns = "nil"
+    # side_effects = ["Defines a rule that skips trailing spacing via the grammar's ignore rule"]
     def lexeme(name, &body)
-      rule(name) { Lexeme.new(Birman.coerce(instance_exec(&body)), ignore_rule) }
+      rule(name) { Lexeme.new(Birman.coerce(instance_exec(&body))) }
     end
 
     private
 
     # A rule named after a method the grammar relies on (send, class,
-    # match, token...) would silently replace it. Redefining a rule is
-    # fine, and so are private Kernel names like print or select.
+    # match, token, or a private helper in any of Birman's modules) would
+    # silently replace it. Redefining a rule is fine, and so are private
+    # Kernel names like print or select, and the grammar's own private
+    # helpers: those are the grammar's to replace.
     def check_rule_name!(name)
       return if private_method_defined?(:"__rule_#{name}")
 
-      owner = [Object, Combinators, Grammar].find { _1.public_method_defined?(name) } ||
-              [Combinators, Grammar].find { _1.private_method_defined?(name) } ||
+      birman = [Object, Combinators, Grammar] | ancestors.select { _1.name&.start_with?("Birman::") }
+      owner = birman.find { _1.public_method_defined?(name) } ||
+              (birman - [Object]).find { _1.private_method_defined?(name, false) } ||
               (instance_method(name).owner if method_defined?(name))
-      raise ArgumentError, "rule :#{name} would replace #{owner}##{name}; give the rule another name" if owner
+      return unless owner
+
+      raise ArgumentError, "rule :#{name} would replace #{owner}##{name}, #{why_taken(owner, birman)}, " \
+                           "since rules are methods too; give the rule another name"
+    end
+
+    def why_taken(owner, birman)
+      if owner == Object then "a method every Ruby object has"
+      elsif birman.include?(owner) then "which Birman relies on"
+      else "a method of this grammar"
+      end
     end
   end
 end

data/lib/birman/rules.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Ready-made rule modules"
+# responsibility = "Whitespace, comments, numbers, strings and identifiers"
+# usage = "include the modules a grammar needs; later includes override earlier ones"
 module Birman
   # A standard library of rule modules. Include the ones you need;
   # later includes override earlier ones (Comments replaces
@@ -10,6 +14,9 @@ module Birman
   module Rules
     # Whitespace as the `spacing` rule, plus a `token` helper for grammars
     # that don't declare `ignore`.
+    # [module.Rules.Whitespace]
+    # rules = ["spacing: zero or more whitespace characters (quiet)"]
+    # helpers = ["token(m): m then spacing, for grammars without ignore"]
     module Whitespace
       extend RuleSet
 
@@ -22,6 +29,9 @@ module Birman
 
     # Whitespace including comments. Works on its own or after
     # Whitespace. Override comment_pattern for other comment syntaxes.
+    # [module.Rules.Comments]
+    # rules = ["spacing: whitespace and comments (quiet)", "comment: one comment"]
+    # helpers = ["comment_pattern: override for other syntaxes (default /#[^\\n]*/)", "token(m)"]
     module Comments
       extend RuleSet
 
@@ -33,35 +43,47 @@ module Birman
     end
 
     # Integers and floats, as Ruby numbers.
+    # [module.Rules.Numbers]
+    # rules = ["integer: Integer", "float: Float", "number: float or integer"]
     module Numbers
       extend RuleSet
 
-      lexeme(:integer) { match(/\d+/).map(&:to_i).label("integer") }
-      lexeme(:float)   { match(/\d+\.\d+/).map(&:to_f).label("float") }
+      lexeme(:integer) { match(/\d+/).map(:to_i).label("integer") }
+      lexeme(:float)   { match(/\d+\.\d+/).map(:to_f).label("float") }
       lexeme(:number)  { (float | integer).label("number") }
     end
 
     # Double-quoted strings with backslash escapes. The opening quote
     # commits, so an unterminated string is reported where it breaks.
+    # [module.Rules.Strings]
+    # rules = ["string: double-quoted, escapes \\n \\t \\r \\0, value is the String"]
+    # helpers = ["decode_escape(text): the character a backslash escape stands for"]
+    # notes = "The opening quote commits, so an unterminated string is reported where it breaks"
     module Strings
       extend RuleSet
 
       ESCAPES = { "n" => "\n", "t" => "\t", "r" => "\r", "0" => "\0" }.freeze
 
       lexeme(:string) do
-        (str('"') > (string_part.rep >> str('"').label("closing quote")))
-          .map { |(_, parts, _)| parts.join }
+        seq(str('"'), commit, only(string_part.rep), str('"').label("closing quote"))
+          .map(:join)
           .label("string")
       end
 
       rule(:string_part) do
-        (match(/\\./).map { ESCAPES.fetch(_1[1], _1[1]) } |
+        (match(/\\./).action(:decode_escape) |
           match(/[^"\\\n]+/)).label("string character")
       end
+
+      private
+
+      def decode_escape(text) = ESCAPES.fetch(text[1], text[1])
     end
 
     # Identifiers that can't be reserved words. Declare those in the
     # grammar with `keywords "if", "then", ...`.
+    # [module.Rules.Identifiers]
+    # rules = ["identifier: [A-Za-z_]\\w*, not one of the grammar's keywords"]
     module Identifiers
       extend RuleSet
 

data/lib/birman/standalone.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# [file]
+# purpose = "Running a matcher without writing a grammar class"
+# responsibility = "Birman.parse and Birman.parse!, via a cached anonymous grammar per matcher"
+# dependencies = ["Birman::Grammar", "Birman::AST"]
+module Birman
+  # Parses input with matcher as the whole grammar: Success or Failure,
+  # exactly as Grammar.parse, and bad input never raises.
+  #
+  #   Birman.parse(Birman.seq("let", Birman.match(/\w+/)), "let x", ignore: /\s*/)
+  #   # => #<data Birman::Success value=["let", "x"]>
+  #
+  # ignore is a matcher (a String, Regexp or Matcher), not a rule name:
+  # there is no grammar here for a name to refer to. Literals skip it
+  # after themselves, and it's skipped at the start of input.
+  # [method."Birman.parse"]
+  # params = ["matcher: String | Regexp | Matcher", "input: String", "ignore: String | Regexp | Matcher | nil"]
+  # returns = "Birman::Success(value) when all of input parses, else Birman::Failure"
+  # raises = ["ArgumentError if matcher or ignore refers to a rule, or an .action/.check method Birman lacks"]
+  def self.parse(matcher, input, ignore: nil) = Standalone.grammar(matcher, ignore).parse(input)
+
+  # [method."Birman.parse!"]
+  # params = ["matcher, input, ignore: as Birman.parse", "formatter: #call(failure) = ErrorFormatter.new"]
+  # returns = "The parsed value"
+  # raises = ["Birman::ParseError", "ArgumentError, as Birman.parse"]
+  def self.parse!(matcher, input, ignore: nil, **) = Standalone.grammar(matcher, ignore).parse!(input, **)
+
+  # The anonymous grammar classes behind Birman.parse: the matcher is
+  # the root rule and ignore the ignore rule. Matchers are immutable
+  # values, so equal ones share a class, and with it the per-grammar
+  # caches (literal matchers, rule matchers). The cache is bounded, so
+  # matchers built afresh for each parse (with a new .map block, say)
+  # don't pile up.
+  # [module.Standalone]
+  # purpose = "Builds and caches the anonymous grammar Birman.parse runs"
+  module Standalone
+    ROOT = :__standalone_root
+    IGNORE = :__standalone_ignore
+    LIMIT = 256
+
+    @grammars = {}
+    @lock = Mutex.new
+
+    # [method."Standalone.grammar"]
+    # params = ["matcher: String | Regexp | Matcher", "ignore: String | Regexp | Matcher | nil"]
+    # returns = "A grammar class whose root rule is matcher (to_ebnf, to_ruby and parse work on it)"
+    # raises = ["ArgumentError if matcher or ignore refers to a rule, or an .action/.check method Birman lacks"]
+    def self.grammar(matcher, ignore)
+      key = [matcher, ignore].freeze
+      @lock.synchronize do
+        @grammars[key] ||= begin
+          @grammars.shift if @grammars.size >= LIMIT
+          build(matcher, ignore)
+        end
+      end
+    end
+
+    def self.build(matcher, ignore)
+      grammar = Class.new do
+        include Grammar
+
+        root ROOT
+        rule(ROOT) { matcher }
+        next unless ignore
+
+        rule(IGNORE) { ignore }
+        self.ignore IGNORE
+      end
+      check!(grammar, [matcher, ignore].compact.map { Birman.coerce(_1) })
+      grammar
+    end
+
+    # A rule reference, or an action or check calling a method only a
+    # grammar of your own would have, raises only when the parse reaches
+    # it (and the method as a NoMethodError on an anonymous class), so a
+    # mistake could hide in an untried alternative. Checked up front.
+    def self.check!(grammar, matchers)
+      matchers.each { each_matcher(_1) { |m| check_matcher!(grammar, m) } }
+    end
+
+    def self.check_matcher!(grammar, matcher)
+      case matcher
+      when Ref
+        raise ArgumentError, "undefined rule #{matcher.name.inspect}: " \
+                             "a matcher run by Birman.parse can't refer to rules; define them in a grammar class"
+      when Action, Check
+        name = matcher.method_name
+        return if grammar.method_defined?(name) || grammar.private_method_defined?(name)
+
+        raise ArgumentError, "undefined method #{name.inspect} for .#{matcher.is_a?(Action) ? "action" : "check"}: " \
+                             "a matcher run by Birman.parse can only use Birman's own methods there; " \
+                             "define yours in a grammar class"
+      end
+    end
+
+    # Every matcher inside root, root included, once each.
+    def self.each_matcher(root)
+      queue = [root]
+      seen = {}.compare_by_identity
+      while (m = queue.shift)
+        next if seen[m]
+
+        seen[m] = true
+        yield m
+        queue.concat(m.to_h.values.flat_map { Array(_1) }.grep(Matcher)) if m.is_a?(Data)
+      end
+    end
+    private_class_method :build, :check!, :check_matcher!, :each_matcher
+  end
+end

data/lib/birman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Birman
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/birman.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Entry point: require \"birman\" loads the whole library"
+# dependencies = ["strscan (standard library)"]
+# requires_ruby = ">= 3.2"
+
 # A small PEG parser library for Ruby 3.2+.
 #
 #   Greeting = Birman.node(:name)
@@ -22,9 +27,31 @@ module Birman
   # The character offset of a byte offset into input. Free for ASCII
   # input (Ruby caches ascii_only?), linear otherwise; used once per
   # parse for a Failure, and only on demand for a Span.
+  # [method."Birman.char_offset"]
+  # params = ["input: String", "byte_pos: Integer"]
+  # returns = "Integer: the character offset of byte_pos"
   def self.char_offset(input, byte_pos)
     input.ascii_only? ? byte_pos : input.byteslice(0, byte_pos).length
   end
+
+  # The byte offset of the first byte in input that doesn't start a
+  # valid character, or nil if input is valid. Grammar#parse fails
+  # there before a regexp can raise on it. Ruby splits invalid bytes
+  # into characters of their own, so everything before it is valid,
+  # and a Failure's line and column count as usual.
+  # [method."Birman.invalid_byte"]
+  # params = ["input: String"]
+  # returns = "Integer | nil: the byte offset of the first invalid byte"
+  def self.invalid_byte(input)
+    return if input.valid_encoding?
+
+    pos = 0
+    input.each_char do |char|
+      return pos unless char.valid_encoding?
+
+      pos += char.bytesize
+    end
+  end
 end
 
 require "strscan"
@@ -37,6 +64,9 @@ require_relative "birman/matchers"
 require_relative "birman/rule_set"
 require_relative "birman/combinators"
 require_relative "birman/grammar"
+require_relative "birman/standalone"
 require_relative "birman/error_formatter"
 require_relative "birman/ast"
+require_relative "birman/left_recursion"
 require_relative "birman/rules"
+require_relative "birman/compiler"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: birman
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Tad Thorley
@@ -23,14 +23,24 @@ files:
 - lib/birman.rb
 - lib/birman/ast.rb
 - lib/birman/combinators.rb
+- lib/birman/compiler.rb
+- lib/birman/compiler/actions.rb
+- lib/birman/compiler/emitter.rb
+- lib/birman/compiler/literals.rb
+- lib/birman/compiler/nodes.rb
+- lib/birman/compiler/regexps.rb
+- lib/birman/compiler/runtime.rb
+- lib/birman/compiler/types.rb
 - lib/birman/context.rb
 - lib/birman/error_formatter.rb
 - lib/birman/grammar.rb
+- lib/birman/left_recursion.rb
 - lib/birman/matchers.rb
 - lib/birman/nodes.rb
 - lib/birman/result.rb
 - lib/birman/rule_set.rb
 - lib/birman/rules.rb
+- lib/birman/standalone.rb
 - lib/birman/version.rb
 homepage: https://github.com/phaedryx/birman
 licenses: