birman 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e56dfecb27c1584f8aefdb873503084e140abf83d53f2b85ed453ad1e25dbbe
-  data.tar.gz: 35b4ee8725298c3809fbde953f51fca4c550ac207a68a2eade43ad7745e02b15
+  metadata.gz: 9b052c872052f6d727f345e06626e7a11f080717c04474bb6bfdcd3a2ad9464a
+  data.tar.gz: 0daa868d84a98399a7ed835301ae320962d9867b949aa83e1dc830e1322fab7c
 SHA512:
-  metadata.gz: abf8ca7031fe77792fb4a3e0a2c8480358320b2edfeebe0e8da7de0814cf71cbccc5229cf8fb710ec1ddebd49208a9b6f7cf8d9a3b6a066830b57b7271e90827
-  data.tar.gz: bdf4b37b3adcfc42ce7f5eb1bee447c83bbe5cc25ab4795a6478991775f9d913554426abf5c6b1392314f072fced9e7f53b8615ffb4fe5eeb37d0483764ece40
+  metadata.gz: b3c9f4bf0d79656d051c51fd105634272d61ba0d05cf9d5e770fcd6326463011dcfdd94f3c158be2729729dacfb281d8e4fc0b16446aa3e2999780549604a926
+  data.tar.gz: 3f62acbd983729b04bb3d632a6b7cb37925f0d3c0d4f24ca116f7c389576e4829ff88bd53cd4a562fe23de26c6af386eeb2df018203c2bb767e81b9eeb21162b

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.2.2] - 2026-10-17
+
+- **Faster:** about 1.75x on the JSON benchmark (159 KB: 0.37s to 0.21s),
+  from allocating 60% fewer objects per parse: no temporary arrays in
+  failure tracking, a per-rule memo table instead of `[rule, pos]` keys,
+  a plain `Match` class instead of `Data`, and literals checked in place
+  instead of sliced out. No API changes.
+- API documentation as TOML comment blocks in `lib/`, and a Limits
+  section in the README.
+
 ## [0.2.1] - 2026-10-17
 
 - **Fixed:** `rep` and `maybe` over something that can match empty.

data/README.md

@@ -20,7 +20,7 @@ bundle add birman
 or in your Gemfile:
 
 ```ruby
-gem "birman", "~> 0.2.1"
+gem "birman", "~> 0.2.2"
 ```
 
 ```ruby
@@ -165,8 +165,8 @@ has one.
   them; pass `nil` to clear one (`ignore nil`, `keywords nil`)
 - `Grammar.parse` returns `Success`/`Failure`; `parse!` raises `ParseError`
   (`parse!(input, formatter: f)` words it your way).
-  Input nested too deeply for Ruby's stack (several hundred levels)
-  is a `Failure` too, with `reason` set.
+  Input nested too deeply for Ruby's stack is a `Failure` too, with
+  `reason` set (see Limits).
 - `Failure#message(formatter)` takes anything with `#call(failure)`
 - `Grammar.to_ast` and `Grammar.to_ebnf` for introspection
 - Rules are methods, so they compose and override through `include`.
@@ -178,6 +178,29 @@ has one.
   `class`, `match`, `token`...). Defining one raises `ArgumentError`
   instead of silently breaking the grammar. Redefining a rule is fine.
 
+## Limits
+
+- **Deep nesting.** Each level of nesting in the input uses Ruby
+  stack. With default settings a simple nested grammar handles about
+  750 levels on the main thread, but only about 250 on other threads
+  (Puma, Sidekiq, `Thread.new`). Past that, `parse` returns a `Failure`
+  whose `reason` is "input is nested too deeply". To allow more, raise
+  Ruby's thread stack sizes when starting the process:
+
+  ```sh
+  RUBY_THREAD_VM_STACK_SIZE=16777216 RUBY_THREAD_MACHINE_STACK_SIZE=67108864 ruby app.rb
+  ```
+
+  That took the same grammar to about 12,000 levels on a thread. The
+  main thread gets less (about 2,000), because its machine stack comes
+  from the operating system (`ulimit -s`).
+- **Changing rules after parsing.** `Grammar.parse` reuses one
+  instance per class, so each rule is built once. Defining a rule or
+  changing a setting on a class resets that class's instance, but a
+  subclass that has already parsed keeps its own: it won't see a rule
+  redefined later on its parent. Define rules before parsing, or call
+  `Subclass.new.parse(input)` to get a fresh instance.
+
 ## Not yet
 
 - Left recursion. A rule that can reach itself without consuming input

data/lib/birman/combinators.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "The combinators available inside every rule body"
+# responsibility = "Building matchers: raw strings and patterns, literals, sequences, choices, lists, wrappers"
+# dependencies = ["Birman::Matcher", "matchers.rb (Seq, Choice, Pick, SepBy, Infix, Lexeme)"]
+# pattern = "Parser combinators (PEG)"
 module Birman
   # Marker for seq: everything after it is committed (see Matcher#>).
   COMMIT = Object.new.tap { |o| def o.inspect = "Birman::COMMIT" }.freeze
@@ -15,27 +20,63 @@ module Birman
   # a rule body, after >> or |) is a literal: it skips trailing spacing
   # when the grammar declares `ignore`, and word-like ones are keywords.
   # Use str and match for exact, raw matching.
+  # [module.Combinators]
+  # purpose = "Matcher-building methods mixed into every grammar (via Birman::Grammar)"
+  # input = "Strings (literals), Regexps (raw patterns) and matchers"
+  # output = "Immutable matcher objects; nothing is matched until Grammar#parse"
+  # notes = "Method names here are reserved: a rule can't share one"
   module Combinators
     # --- Raw matchers ---
 
-    def str(s)    = Str.new(s)
+    # [method.str]
+    # params = ["s: String"]
+    # returns = "Matcher that matches exactly s; never skips spacing, never a keyword"
+    # value = "s"
+    def str(s) = Str.new(s)
+
+    # [method.match]
+    # params = ["re: Regexp (anchored at the current position; \\A, ^ and lookbehind see the whole input)"]
+    # returns = "Matcher for the regexp; raw, nothing skipped"
+    # value = "the matched text"
     def match(re) = Pat.new(re)
-    def any       = Pat.new(/./m)
-    def eof       = any.absent.label("end of input")
-    def never     = Pat.new(/(?!)/)
+
+    # [method.any]
+    # returns = "Matcher for any one character, newlines included"
+    def any = Pat.new(/./m)
+
+    # [method.eof]
+    # returns = "Matcher that succeeds only at the end of the input, consuming nothing"
+    def eof = any.absent.label("end of input")
+
+    # [method.never]
+    # returns = "Matcher that always fails (a placeholder for an empty choice)"
+    def never = Pat.new(/(?!)/)
 
     # A whole word: "if" matches in "if x" but not in "iffy".
+    # [method.keyword]
+    # params = ["word: String"]
+    # returns = "Matcher for word when not followed by a word character"
+    # value = "word"
     def keyword(word) = (str(word) >> match(/\w/).absent).map(&:first).label(word.inspect)
 
     # --- Literals and the readable forms ---
 
     # Anything usable as a matcher: strings become literals, regexps
     # raw patterns, and matchers pass through unchanged.
+    # [method.lit]
+    # params = ["x: String | Regexp | Matcher"]
+    # returns = "A matcher: String -> literal, Regexp -> raw pattern, matcher unchanged"
+    # raises = ["ArgumentError (anything else, or only(...) outside seq)"]
     def lit(x) = Birman.coerce(x)
 
     # The matcher a literal stands for: "let" is a keyword, "=" an exact
     # string, either way followed by the grammar's ignore rule. Built
     # once per grammar instance.
+    # [method.literal_matcher]
+    # params = ["text: String"]
+    # returns = "The matcher a literal stands for: keyword or exact string, then the ignore rule"
+    # side_effects = ["Caches the matcher on the grammar instance"]
+    # notes = "Called by Lit at match time; rarely needed directly"
     def literal_matcher(text)
       (@literal_matchers ||= {})[text] ||= begin
         base = text.match?(/\A\w+\z/) ? keyword(text) : str(text)
@@ -45,6 +86,11 @@ module Birman
 
     # Sequence. Put `commit` where the construct becomes unambiguous:
     #   seq("let", commit, name.as(:name), "=", expr.as(:value))
+    # [method.seq]
+    # params = ["*items: String | Regexp | Matcher | commit | only(...)"]
+    # returns = "Matcher for the items in order"
+    # value = "Array of item values; a Hash of the .as-named parts if any; the only(...) part's value if used"
+    # raises = ["ArgumentError (commit first; more than one only; only mixed with .as names)"]
     def seq(*items)
       commit_at = items.index(COMMIT)
       raise ArgumentError, "commit can't come first in a seq: nothing has matched yet" if commit_at&.zero?
@@ -54,19 +100,34 @@ module Birman
       Seq.new(parts.map { _1.is_a?(Only) ? _1.matcher : lit(_1) }, commit_at, only_at)
     end
 
+    # [method.commit]
+    # returns = "Marker for seq: if anything after it fails, the whole parse fails there"
+    # notes = "Lookaheads contain a commit failure; rep and maybe don't"
     def commit = COMMIT
 
     # Inside seq, makes this part the seq's whole value; the rest still
     # has to match but is dropped:
     #   seq("[", commit, only(sep_by(value, ",")), "]")   # => [...]
+    # [method.only]
+    # params = ["x: String | Regexp | Matcher"]
+    # returns = "Marker for seq making x's value the seq's whole value"
+    # notes = "x's .as name, if any, passes to the enclosing seq"
     def only(x) = Only.new(lit(x))
 
     # Ordered choice: the first alternative that matches wins.
+    # [method.either]
+    # params = ["*alts: String | Regexp | Matcher"]
+    # returns = "Ordered choice: the first alternative that matches wins"
+    # value = "The winning alternative's value"
     def either(*alts) = alts.size == 1 ? lit(alts.first) : Choice.new(alts.map { lit(_1) })
 
     # One of several literals. Given a hash, each literal produces its
     # value: one_of("true" => true, "null" => nil). They're tried in
     # order, so put longer ones first ("<=" before "<").
+    # [method.one_of]
+    # params = ["*choices: String literals", "or one Hash of literal => value"]
+    # returns = "Ordered choice of the literals"
+    # value = "The matched literal, or its Hash value"
     def one_of(*choices)
       return either(*choices) unless choices in [Hash => table]
 
@@ -75,17 +136,37 @@ module Birman
 
     # Left-associative binary operators, folded into nodes:
     #   infix(product, "+", "-", node: BinOp)   # BinOp = node(:op, :left, :right)
+    # [method.infix]
+    # params = ["operand: Matcher", "*ops: String literals", "node: node class with op, left, right fields"]
+    # returns = "Matcher for operand (op operand)*, folded left-associatively"
+    # value = "operand's value, or nested node instances with spans"
+    # raises = ["ArgumentError at parse time if node lacks op/left/right"]
     def infix(operand, *ops, node:) = Infix.new(Birman.coerce(operand), either(*ops).label("operator"), node)
 
     # One or more items separated by sep; value is an array of items.
+    # [method.sep_by1]
+    # params = ["item: String | Regexp | Matcher", "sep: String | Regexp | Matcher"]
+    # returns = "Matcher for one or more items separated by sep"
+    # value = "Array of item values (item's .as name passes outward)"
     def sep_by1(item, sep) = SepBy.new(lit(item), lit(sep))
 
     # Zero or more items separated by sep.
+    # [method.sep_by]
+    # params = ["item: String | Regexp | Matcher", "sep: String | Regexp | Matcher"]
+    # returns = "Matcher for zero or more items separated by sep"
+    # value = "Array of item values, [] when there are none"
     def sep_by(item, sep) = sep_by1(item, sep).maybe.map { _1 || [] }
 
     # inner wrapped in open/close; value is inner's.
+    # [method.between]
+    # params = ["open: String | Matcher", "close: String | Matcher", "inner: Matcher"]
+    # returns = "Matcher for open inner close"
+    # value = "inner's value (inner's .as name passes outward)"
     def between(open, close, inner) = Pick.new([lit(open), Birman.coerce(inner), lit(close)], 1)
 
+    # [method.parens]
+    # params = ["inner: Matcher"]
+    # returns = "between(\"(\", \")\", inner)"
     def parens(inner) = between("(", ")", inner)
 
     private

data/lib/birman/context.rb

@@ -12,16 +12,20 @@ module Birman
     # Marks a memo entry whose rule is still being matched.
     IN_PROGRESS = Object.new.freeze
 
+    # The expectations of a frame that hasn't failed anywhere yet. Shared
+    # and frozen; fail_at replaces it rather than appending.
+    NOTHING_EXPECTED = [].freeze
+
     # position: where the most recently entered rule started. When the
     # input nests too deeply for Ruby's stack, that's where it happened.
-    attr_reader :input, :grammar, :memo, :furthest, :expected, :position
+    attr_reader :input, :grammar, :furthest, :expected, :position, :isolated_furthest, :isolated_expected
 
     def initialize(input, grammar)
       @input = input
       @grammar = grammar
       @memo = {}
       @furthest = 0
-      @expected = []
+      @expected = NOTHING_EXPECTED
       @token_ends = {}
     end
 
@@ -46,7 +50,7 @@ module Birman
         @furthest = pos
         @expected = [what]
       elsif pos == @furthest
-        @expected << what
+        @expected = @expected.frozen? ? [what] : @expected << what
       end
       nil
     end
@@ -54,17 +58,22 @@ module Birman
     def merge(pos, expected) = expected.each { fail_at(pos, _1) }
 
     # Run the block with fresh failure tracking starting at pos, then
-    # restore the outer state. Returns [result, furthest, expected] so
-    # the caller decides what to keep: Ref memoizes it, Label rewrites
-    # it, Not throws it away.
+    # restore the outer state, and return the block's result. What the
+    # block recorded is left in isolated_furthest and isolated_expected
+    # for the caller to keep or drop: Ref memoizes it, Label rewrites
+    # it, Not throws it away. No arrays: this runs for every rule call.
     def isolate(pos)
-      outer = [@furthest, @expected]
+      outer_furthest = @furthest
+      outer_expected = @expected
       @furthest = pos
-      @expected = []
+      @expected = NOTHING_EXPECTED
       result = yield
-      [result, @furthest, @expected]
+      @isolated_furthest = @furthest
+      @isolated_expected = @expected
+      result
     ensure
-      @furthest, @expected = outer
+      @furthest = outer_furthest
+      @expected = outer_expected
     end
 
     def commit_failure! = raise(Committed.new(@furthest, @expected.dup))
@@ -74,9 +83,12 @@ module Birman
     # moves past, so they're dropped. If it fails, that's recorded at
     # pos, as what it expected there or as the lookahead itself.
     def peek?(matcher, pos)
-      result, furthest, expected = isolate(pos) { lookahead { matcher.match(self, pos) } }
+      result = isolate(pos) { lookahead { matcher.match(self, pos) } }
       return true if result
 
+      furthest = isolated_furthest
+      expected = isolated_expected
+
       if furthest == pos && expected.any?
         merge(pos, expected)
       else
@@ -91,13 +103,13 @@ module Birman
     # input: left recursion, which would never terminate. That's a bug
     # in the grammar, not the input, so it raises.
     def memoize(name, pos, &)
-      key = [name, pos]
-      entry = @memo[key]
+      table = @memo[name] ||= {}
+      entry = table[pos]
       return entry if entry && !entry.equal?(IN_PROGRESS)
       raise ArgumentError, "left recursion in rule :#{name}: it can reach itself without consuming input" if entry
 
       @position = pos
-      compute(key, &)
+      compute(table, pos, &)
     end
 
     # Run a lookahead's inner match. A lookahead only asks whether
@@ -111,13 +123,14 @@ module Birman
 
     private
 
-    def compute(key)
-      @memo[key] = IN_PROGRESS
-      @memo[key] = yield
+    # The memo is a Hash per rule, keyed by position, so no key arrays.
+    def compute(table, pos)
+      table[pos] = IN_PROGRESS
+      table[pos] = yield
     ensure
       # A commit failure caught by a lookahead unwinds through here; the
       # rule may be tried again at this position afterwards.
-      @memo.delete(key) if @memo[key].equal?(IN_PROGRESS)
+      table.delete(pos) if table[pos].equal?(IN_PROGRESS)
     end
   end
 end

data/lib/birman/error_formatter.rb

@@ -1,11 +1,17 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Default wording for parse errors"
+# pattern = "Strategy: anything with #call(failure) can replace it"
 module Birman
   # Turns a Failure into readable text. Anything that responds to
   # #call(failure) works in its place, for custom wording or i18n:
   #
   #   failure.message(MyFormatter.new)
   class ErrorFormatter
+    # [method.call]
+    # params = ["failure: Birman::Failure"]
+    # returns = "String: \"line L, column C: expected ..., found ...\" plus the source line and a caret"
     def call(failure)
       source = source_line(failure)
       <<~MSG.chomp

data/lib/birman/grammar.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Makes a class (or a module of shared rules) a grammar"
+# responsibility = "Settings, parse entry points, rule lookup and introspection"
+# dependencies = ["Birman::Combinators", "Birman::RuleSet", "Birman::Nodes", "Birman::Context"]
+# pattern = "Mixin with class-level DSL"
 module Birman
   # Include this to make a class a grammar. The class body also gets
   # `node` (see Nodes), so AST nodes can live beside their rules.
@@ -7,6 +12,10 @@ module Birman
   # Including it in a module makes a module of shared rules: whatever
   # includes that module (a class, or another such module) becomes a
   # grammar too.
+  # [module.Grammar]
+  # purpose = "include this in a class to make it a grammar"
+  # provides = ["rule / lexeme (RuleSet)", "node (Nodes)", "combinators", "settings and parse"]
+  # collaborators = ["Context", "Success", "Failure", "ErrorFormatter"]
   module Grammar
     include Combinators
 
@@ -26,18 +35,30 @@ module Birman
 
     # Settings are inherited by subclasses. Passing nil clears an
     # inherited one: `ignore nil`, `keywords nil`.
+    # [module.Grammar.ClassMethods]
+    # purpose = "Settings and parse entry points on a grammar class"
+    # notes = "Settings are inherited by subclasses; pass nil to clear one"
     module ClassMethods
       UNSET = Object.new.freeze
 
       # The rule a parse starts from.
+      # [method.root]
+      # params = ["name: Symbol | nil (omit to read)"]
+      # returns = "The start rule's name"
       def root(name = UNSET) = setting(:root, name)
 
       # A rule the parser matches and discards after every literal and
       # lexeme, and at the start of input. Usually :spacing (from
       # Birman::Rules::Whitespace or Comments).
+      # [method.ignore]
+      # params = ["name: Symbol | nil (omit to read)"]
+      # returns = "The ignore rule's name (matched and discarded after literals and lexemes)"
       def ignore(name = UNSET) = setting(:ignore, name)
 
       # Reserved words, which Birman::Rules::Identifiers won't accept.
+      # [method.keywords]
+      # params = ["*words: String (none to read; nil to clear)"]
+      # returns = "Array of reserved words (Rules::Identifiers rejects them as names)"
       def keywords(*words)
         return setting(:keywords) || [] if words.empty?
 
@@ -47,10 +68,25 @@ module Birman
       # These share one instance, so each rule's matcher is built once
       # rather than on every parse. Defining a rule or changing a
       # setting starts a fresh one.
-      def parse(input)  = instance.parse(input)
+      # [method."Grammar.parse"]
+      # params = ["input: String"]
+      # returns = "Success(value) or Failure; see Grammar#parse"
+      # notes = "Uses one cached instance; defining a rule or changing a setting resets it"
+      def parse(input) = instance.parse(input)
+
+      # [method."Grammar.parse!"]
+      # params = ["input: String", "formatter: #call(failure) = ErrorFormatter.new"]
+      # returns = "The parsed value"
+      # raises = ["Birman::ParseError"]
       def parse!(input, **) = instance.parse!(input, **)
-      def to_ast        = instance.to_ast
-      def to_ebnf       = EBNF.call(to_ast)
+
+      # [method."Grammar.to_ast"]
+      # returns = "Hash of rule name => AST, for every rule reachable from root and ignore"
+      def to_ast = instance.to_ast
+
+      # [method."Grammar.to_ebnf"]
+      # returns = "String: the grammar in PEG-flavored EBNF"
+      def to_ebnf = EBNF.call(to_ast)
 
       private
 
@@ -71,6 +107,11 @@ module Birman
     def keywords = self.class.keywords
 
     # Returns Success(value) or Failure(...). Never raises on bad input.
+    # [method.parse]
+    # params = ["input: String"]
+    # returns = "Birman::Success(value) when all of input parses, else Birman::Failure"
+    # raises = ["ArgumentError for grammar mistakes (no root, undefined rule, left recursion)"]
+    # notes = "Bad input never raises: too-deep nesting is a Failure with reason set"
     def parse(input)
       ctx = Context.new(input, self)
       start = (ignore_rule && Ref.new(ignore_rule).match(ctx, 0)&.pos) || 0
@@ -86,6 +127,10 @@ module Birman
     end
 
     # Returns the value or raises ParseError, worded by formatter.
+    # [method."parse!"]
+    # params = ["input: String", "formatter: #call(failure) = ErrorFormatter.new"]
+    # returns = "The parsed value"
+    # raises = ["Birman::ParseError (message from formatter)"]
     def parse!(input, formatter: ErrorFormatter.new) = parse(input).value!(formatter)
 
     def matcher_for(name)

data/lib/birman/matchers.rb

@@ -1,6 +1,16 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "The matcher types and the methods every matcher responds to"
+# responsibility = "Matching at a byte position against a Context; building ASTs for EBNF"
+# dependencies = ["Birman::Context", "StringScanner (via Context#scanner)"]
+# pattern = "Immutable Data objects composed into a tree; packrat memoization at rule refs"
+# public_api = "Birman::Matcher (operators and modifiers below); the Data classes are internal"
 module Birman
+  # [method."Birman.coerce"]
+  # params = ["x: String | Regexp | Matcher"]
+  # returns = "Matcher: String -> literal (Lit), Regexp -> raw pattern (Pat), matcher unchanged"
+  # raises = ["ArgumentError (only(...) outside seq, or anything else)"]
   def self.coerce(x)
     case x
     when Matcher then x
@@ -22,41 +32,91 @@ module Birman
   #
   # So `a > b >> c` means `a > (b >> c)`, and a commit inside a choice
   # needs parentheses: `(keyword("if") > body) | other`.
+  # [module.Matcher]
+  # purpose = "Operators and modifiers shared by every matcher"
+  # notes = "Arguments go through Birman.coerce, so bare strings are literals"
   module Matcher
+    # [method.">>"]
+    # params = ["other: String | Regexp | Matcher"]
+    # returns = "Sequence of self then other (flattens into an existing sequence)"
     def >>(other) = Seq.new([self, Birman.coerce(other)])
-    def |(other)  = Choice.new([self, Birman.coerce(other)])
+
+    # [method."|"]
+    # params = ["other: String | Regexp | Matcher"]
+    # returns = "Ordered choice: self, else other"
+    def |(other) = Choice.new([self, Birman.coerce(other)])
 
     # Conjunction: self must match here (without consuming input),
     # then other is matched from the same spot. Value is other's.
+    # [method."&"]
+    # params = ["other: String | Regexp | Matcher"]
+    # returns = "self as a guard (lookahead), then other from the same position"
+    # value = "other's value"
     def &(other) = Both.new(self, Birman.coerce(other))
 
     # Commit: once self matches, everything after it must match too, or
     # parsing stops right here with an error instead of backtracking.
     # This is dry-logic's implication operator, applied to parsing.
+    # [method.">"]
+    # params = ["other: String | Regexp | Matcher"]
+    # returns = "Sequence of self then other, committed after self (like seq(self, commit, other))"
     def >(other)
       left  = uncommitted_parts(self)
       right = uncommitted_parts(Birman.coerce(other))
       Seq.new([*left, *right], left.size)
     end
 
+    # [method.rep]
+    # params = ["min: Integer = 0", "max: Integer | nil = nil (unbounded)"]
+    # returns = "Matcher for self repeated min..max times"
+    # value = "Array of values; an empty first match counts once, then ends the loop"
     def rep(min = 0, max = nil) = Rep.new(self, min, max)
-    def maybe       = rep(0, 1).map(&:first)
-    def absent      = Not.new(self)
-    def present     = And.new(self)
-    def map(&fn)    = Action.new(self, fn)
+
+    # [method.maybe]
+    # returns = "Optional self"
+    # value = "self's value, or nil when absent"
+    def maybe = rep(0, 1).map(&:first)
+
+    # [method.absent]
+    # returns = "Negative lookahead: succeeds, consuming nothing, when self doesn't match here"
+    # value = "nil"
+    def absent = Not.new(self)
+
+    # [method.present]
+    # returns = "Positive lookahead: succeeds, consuming nothing, when self matches here"
+    # value = "nil"
+    def present = And.new(self)
+
+    # [method.map]
+    # params = ["fn: block taking self's value"]
+    # returns = "Matcher whose value is fn's result (self's .as name is kept)"
+    def map(&fn) = Action.new(self, fn)
+
+    # [method.label]
+    # params = ["name: String"]
+    # returns = "self, reported as \"expected <name>\" when it fails without consuming input"
     def label(name) = Label.new(self, name)
 
     # Never mention this matcher in error messages (for whitespace,
     # comments, and other things nobody wants to be told to type).
+    # [method.quiet]
+    # returns = "self, never mentioned in error messages"
     def quiet = Quiet.new(self)
 
     # Name this part of a sequence. A sequence with any named parts
     # produces a hash of just those parts instead of an array.
+    # [method.as]
+    # params = ["key: Symbol"]
+    # returns = "self, named: the enclosing seq's value becomes a Hash of named parts"
     def as(key) = Named.new(self, key)
 
     # Build a node from the value: a hash of named parts becomes
     # keyword arguments; anything else fills the node's first field.
     # Nodes made with Birman.node get a source span.
+    # [method.to]
+    # params = ["node: node class (or any Data class)"]
+    # returns = "Matcher whose value is node built from self's value"
+    # value = "node.new(**hash), or node.new(first_field => value); with a span for node(...) classes"
     def to(node) = To.new(self, node)
 
     # The name given by #as, seen through wrappers like labels.
@@ -81,12 +141,16 @@ module Birman
   Str = Data.define(:str) do
     include Matcher
 
+    # The error text is worked out once, not on every failed attempt.
+    def initialize(str:)
+      @expected = str.inspect.freeze
+      super
+    end
+
     def match(ctx, pos)
-      if ctx.input.byteslice(pos, str.bytesize) == str
-        Match.new(pos + str.bytesize, str)
-      else
-        ctx.fail_at(pos, str.inspect)
-      end
+      scanner = ctx.scanner
+      scanner.pos = pos
+      scanner.match?(str) ? Match.new(pos + str.bytesize, str) : ctx.fail_at(pos, @expected)
     end
 
     def to_ast = [:str, str]
@@ -126,14 +190,11 @@ module Birman
     def >>(other) = with(parts: [*parts, Birman.coerce(other)])
 
     def match(ctx, pos)
-      values = parts.each_with_index.map do |part, i|
-        r = part.match(ctx, pos)
-        unless r
-          fail_committed!(ctx, part, pos) if commit_at && i >= commit_at
-          return nil
-        end
+      values = Array.new(parts.size)
+      parts.each_with_index do |part, i|
+        r = part.match(ctx, pos) or return failed(ctx, part, pos, i)
+        values[i] = r.value
         pos = r.pos
-        r.value
       end
       Match.new(pos, shape(values))
     end
@@ -145,6 +206,11 @@ module Birman
 
     private
 
+    def failed(ctx, part, pos, index)
+      fail_committed!(ctx, part, pos) if commit_at && index >= commit_at
+      nil
+    end
+
     # If the failed part recorded nothing at or past pos (it's quiet,
     # or a negative lookahead), describe the part itself so the error
     # still says what was expected.
@@ -259,7 +325,7 @@ module Birman
     include Matcher
 
     def match(ctx, pos)
-      r, = ctx.isolate(pos) { ctx.lookahead { inner.match(ctx, pos) } }
+      r = ctx.isolate(pos) { ctx.lookahead { inner.match(ctx, pos) } }
       r ? nil : Match.new(pos, nil)
     end
 
@@ -290,7 +356,7 @@ module Birman
 
     def match(ctx, pos)
       r = inner.match(ctx, pos) or return nil
-      r.with(value: fn.call(r.value))
+      Match.new(r.pos, fn.call(r.value))
     end
 
     def to_ast = [:action, inner.to_ast]
@@ -306,7 +372,9 @@ module Birman
     include Matcher
 
     def match(ctx, pos)
-      result, furthest, expected = ctx.isolate(pos) { inner.match(ctx, pos) }
+      result = ctx.isolate(pos) { inner.match(ctx, pos) }
+      furthest = ctx.isolated_furthest
+      expected = ctx.isolated_expected
       if furthest == pos && (result.nil? || (result.pos == pos && expected.any?))
         ctx.fail_at(pos, name)
       else
@@ -336,7 +404,7 @@ module Birman
     def match(ctx, pos)
       r = inner.match(ctx, pos) or return nil
       fields = r.value.is_a?(Hash) ? r.value : { node.members.first => r.value }
-      r.with(value: Birman.build(node, fields, ctx, pos, r.pos))
+      Match.new(r.pos, Birman.build(node, fields, ctx, pos, r.pos))
     end
 
     def to_ast = [:action, inner.to_ast]
@@ -349,13 +417,16 @@ module Birman
   Lexeme = Data.define(:inner, :ignore, :source) do
     include Matcher
 
-    def initialize(inner:, ignore:, source: nil) = super
+    def initialize(inner:, ignore:, source: nil)
+      @ignore_ref = ignore && Ref.new(ignore) # built once, not per token
+      super
+    end
 
     def match(ctx, pos)
       r = inner.match(ctx, pos) or return nil
       return r unless ignore
 
-      s = Ref.new(ignore).match(ctx, r.pos) or return r
+      s = @ignore_ref.match(ctx, r.pos) or return r
       ctx.ignored_after(r.pos, s.pos)
       Match.new(s.pos, r.value)
     end
@@ -398,8 +469,7 @@ module Birman
     include Matcher
 
     def match(ctx, pos)
-      result, = ctx.isolate(pos) { inner.match(ctx, pos) }
-      result
+      ctx.isolate(pos) { inner.match(ctx, pos) }
     end
 
     def to_ast = [:quiet, inner.to_ast]
@@ -414,7 +484,8 @@ module Birman
 
     def match(ctx, pos)
       result, furthest, expected = ctx.memoize(name, pos) do
-        ctx.isolate(pos) { ctx.grammar.matcher_for(name).match(ctx, pos) }
+        r = ctx.isolate(pos) { ctx.grammar.matcher_for(name).match(ctx, pos) }
+        [r, ctx.isolated_furthest, ctx.isolated_expected]
       end
       ctx.merge(furthest, expected)
       result

data/lib/birman/nodes.rb

@@ -1,8 +1,16 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "AST nodes with source spans"
+# responsibility = "node(...) classes, Span, and building nodes during a parse"
+# pattern = "Data classes plus a mixin"
 module Birman
   # Where a node came from in the source. Stored as byte offsets (see
   # Context); from, to, line and column count characters.
+  # [class.Span]
+  # purpose = "Where a node came from in the input"
+  # fields = ["input: String", "byte_from, byte_to: Integer (byte offsets)"]
+  # methods = ["from, to: character offsets", "text", "line", "column", "to_s => \"line L, column C\""]
   Span = Data.define(:input, :byte_from, :byte_to) do
     def from   = Birman.char_offset(input, byte_from)
     def to     = Birman.char_offset(input, byte_to)
@@ -22,6 +30,9 @@ module Birman
   # not a Data member, so equality and pattern matching ignore it:
   # `in Num(value)` still works, and two nodes with the same contents
   # are equal wherever they came from.
+  # [module.Node]
+  # purpose = "Mixed into every node(...) class"
+  # adds = ["span (ignored by == and pattern matching)", "with(**changes) keeping the span", "s-expression inspect"]
   module Node
     attr_reader :span
 
@@ -61,7 +72,13 @@ module Birman
   #   end
   #
   # Also available as Birman.node.
+  # [module.Nodes]
+  # purpose = "Provides node(...); grammars get it automatically, other modules can extend it"
   module Nodes
+    # [method.node]
+    # params = ["*members: Symbol (field names; not :span)", "body: optional block of methods (may define initialize)"]
+    # returns = "A Data class including Birman::Node"
+    # raises = ["ArgumentError if a field is named span"]
     def node(*members, &body)
       raise ArgumentError, "a node can't have a field named span: every node already has one" if members.include?(:span)
 

data/lib/birman/result.rb

@@ -1,10 +1,27 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "What a parse returns"
+# responsibility = "Success and Failure values, and the ParseError that parse! raises"
+# pattern = "Result objects (Data), pattern-matchable"
 module Birman
-  # Internal: one matcher succeeded, ending at `pos` with `value`.
-  Match = Data.define(:pos, :value)
+  # Internal: one matcher succeeded, ending at `pos` with `value`. A
+  # plain class rather than Data: one is made for every successful
+  # match, and Data.new allocates a keyword Hash each time.
+  class Match
+    attr_reader :pos, :value
+
+    def initialize(pos, value)
+      @pos = pos
+      @value = value
+    end
+  end
 
   # Returned by Grammar#parse when the whole input parsed.
+  # [class.Success]
+  # purpose = "The whole input parsed"
+  # fields = ["value: whatever the root rule produced"]
+  # usage = "case grammar.parse(input) in Birman::Success(value) then ..."
   Success = Data.define(:value) do
     def success? = true
     def failure? = false
@@ -19,6 +36,16 @@ module Birman
   #
   # reason: set when the failure isn't about what was expected, such as
   # input nested too deeply for Ruby's stack.
+  # [class.Failure]
+  # purpose = "The input didn't parse"
+  # fields = [
+  # "input: String",
+  # "pos: Integer (character offset of the furthest failure)",
+  # "expected: Array of String (what would have matched there)",
+  # "committed: Boolean (failed after a commit)",
+  # "reason: String | nil (set when it isn't about expectations, e.g. nesting too deep)"
+  # ]
+  # methods = ["line", "column", "found", "message(formatter)", "value!(formatter) -> raises ParseError"]
   Failure = Data.define(:input, :pos, :expected, :committed, :reason) do
     def initialize(input:, pos:, expected:, committed:, reason: nil) = super
 
@@ -42,6 +69,9 @@ module Birman
   end
 
   # Raised by parse! (and Failure#value!) with the formatted message.
+  # [class.ParseError]
+  # purpose = "Raised by parse! and Failure#value!"
+  # fields = ["failure: Birman::Failure", "message: formatted by the formatter given"]
   class ParseError < StandardError
     attr_reader :failure
 

data/lib/birman/rule_set.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "The rule and lexeme macros"
+# responsibility = "Turning rule blocks into methods, and guarding rule names"
+# pattern = "Class-level DSL (extend)"
 module Birman
   # The `rule` and `lexeme` macros. Extend any module or class with it.
   #
@@ -7,7 +11,18 @@ module Birman
   # private `__rule_name` builds the matcher. Because rules are just
   # methods, they travel through `include` and can be overridden like
   # any other Ruby method.
+  # [module.RuleSet]
+  # purpose = "Defines rules; extend a module with it to write a reusable rule module"
+  # collaborators = ["Grammar (extends its classes with this)", "Ref"]
   module RuleSet
+    # [method.rule]
+    # params = ["name: Symbol", "block: returns the rule's matcher (String, Regexp or Matcher)"]
+    # returns = "nil"
+    # side_effects = [
+    #   "Defines name (returns a Ref) and a private __rule_name",
+    #   "Resets a grammar class's cached instance"
+    # ]
+    # raises = ["ArgumentError if name would replace a method the grammar relies on (send, match, token...)"]
     def rule(name, &)
       check_rule_name!(name)
       define_method(:"__rule_#{name}", &)
@@ -18,6 +33,10 @@ module Birman
 
     # A rule for a token (a number, a string, an identifier): it skips
     # trailing spacing when the grammar declares `ignore`.
+    # [method.lexeme]
+    # params = ["name: Symbol", "body: block returning the token's matcher"]
+    # returns = "nil"
+    # side_effects = ["Defines a rule that skips trailing spacing via the grammar's ignore rule"]
     def lexeme(name, &body)
       rule(name) { Lexeme.new(Birman.coerce(instance_exec(&body)), ignore_rule) }
     end

data/lib/birman/rules.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Ready-made rule modules"
+# responsibility = "Whitespace, comments, numbers, strings and identifiers"
+# usage = "include the modules a grammar needs; later includes override earlier ones"
 module Birman
   # A standard library of rule modules. Include the ones you need;
   # later includes override earlier ones (Comments replaces
@@ -10,6 +14,9 @@ module Birman
   module Rules
     # Whitespace as the `spacing` rule, plus a `token` helper for grammars
     # that don't declare `ignore`.
+    # [module.Rules.Whitespace]
+    # rules = ["spacing: zero or more whitespace characters (quiet)"]
+    # helpers = ["token(m): m then spacing, for grammars without ignore"]
     module Whitespace
       extend RuleSet
 
@@ -22,6 +29,9 @@ module Birman
 
     # Whitespace including comments. Works on its own or after
     # Whitespace. Override comment_pattern for other comment syntaxes.
+    # [module.Rules.Comments]
+    # rules = ["spacing: whitespace and comments (quiet)", "comment: one comment"]
+    # helpers = ["comment_pattern: override for other syntaxes (default /#[^\\n]*/)", "token(m)"]
     module Comments
       extend RuleSet
 
@@ -33,6 +43,8 @@ module Birman
     end
 
     # Integers and floats, as Ruby numbers.
+    # [module.Rules.Numbers]
+    # rules = ["integer: Integer", "float: Float", "number: float or integer"]
     module Numbers
       extend RuleSet
 
@@ -43,6 +55,9 @@ module Birman
 
     # Double-quoted strings with backslash escapes. The opening quote
     # commits, so an unterminated string is reported where it breaks.
+    # [module.Rules.Strings]
+    # rules = ["string: double-quoted, escapes \\n \\t \\r \\0, value is the String"]
+    # notes = "The opening quote commits, so an unterminated string is reported where it breaks"
     module Strings
       extend RuleSet
 
@@ -62,6 +77,8 @@ module Birman
 
     # Identifiers that can't be reserved words. Declare those in the
     # grammar with `keywords "if", "then", ...`.
+    # [module.Rules.Identifiers]
+    # rules = ["identifier: [A-Za-z_]\\w*, not one of the grammar's keywords"]
     module Identifiers
       extend RuleSet
 

data/lib/birman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Birman
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/birman.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# [file]
+# purpose = "Entry point: require \"birman\" loads the whole library"
+# dependencies = ["strscan (standard library)"]
+# requires_ruby = ">= 3.2"
+
 # A small PEG parser library for Ruby 3.2+.
 #
 #   Greeting = Birman.node(:name)
@@ -22,6 +27,9 @@ module Birman
   # The character offset of a byte offset into input. Free for ASCII
   # input (Ruby caches ascii_only?), linear otherwise; used once per
   # parse for a Failure, and only on demand for a Span.
+  # [method."Birman.char_offset"]
+  # params = ["input: String", "byte_pos: Integer"]
+  # returns = "Integer: the character offset of byte_pos"
   def self.char_offset(input, byte_pos)
     input.ascii_only? ? byte_pos : input.byteslice(0, byte_pos).length
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: birman
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Tad Thorley