birman 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +204 -0
- data/README.md +381 -14
- data/lib/birman/ast.rb +10 -1
- data/lib/birman/combinators.rb +25 -14
- data/lib/birman/compiler/actions.rb +309 -0
- data/lib/birman/compiler/emitter.rb +940 -0
- data/lib/birman/compiler/literals.rb +29 -0
- data/lib/birman/compiler/nodes.rb +158 -0
- data/lib/birman/compiler/regexps.rb +182 -0
- data/lib/birman/compiler/runtime.rb +758 -0
- data/lib/birman/compiler/types.rb +128 -0
- data/lib/birman/compiler.rb +155 -0
- data/lib/birman/context.rb +124 -15
- data/lib/birman/grammar.rb +93 -16
- data/lib/birman/left_recursion.rb +409 -0
- data/lib/birman/matchers.rb +224 -38
- data/lib/birman/result.rb +10 -1
- data/lib/birman/rule_set.rb +19 -6
- data/lib/birman/rules.rb +10 -5
- data/lib/birman/standalone.rb +111 -0
- data/lib/birman/version.rb +1 -1
- data/lib/birman.rb +22 -0
- metadata +11 -1
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Birman
|
|
4
|
+
class Compiler
|
|
5
|
+
# A coarse static type for each matcher's value, so every value
|
|
6
|
+
# instance variable starts as a literal of the type it will hold
|
|
7
|
+
# (Spinel boxes a variable whose writes disagree). :poly means it
|
|
8
|
+
# varies, or isn't known; nil, nodes and :poly all start as nil.
|
|
9
|
+
# [class.Compiler.Types]
|
|
10
|
+
# purpose = "Infers the type a matcher's value has, for initializers"
|
|
11
|
+
class Types
|
|
12
|
+
INITIAL = { str: '""', int: "0", float: "0.0", sym: ':""', array: "[]", hash: "{}" }.freeze
|
|
13
|
+
|
|
14
|
+
SENDS = { to_i: :int, to_f: :float, to_sym: :sym, to_s: :str, join: :str, to_h: :hash,
|
|
15
|
+
upcase: :str, downcase: :str, strip: :str, to_a: :array }.freeze
|
|
16
|
+
|
|
17
|
+
def initialize(grammar)
|
|
18
|
+
@grammar = grammar
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The initializer literal for a value of matcher's type.
|
|
22
|
+
def initial(matcher) = INITIAL.fetch(of(matcher), "nil")
|
|
23
|
+
|
|
24
|
+
def of(matcher) # rubocop:disable Metrics
|
|
25
|
+
case matcher
|
|
26
|
+
when Str, Pat then :str
|
|
27
|
+
when Lit then of(@grammar.literal_matcher(matcher.text))
|
|
28
|
+
when Seq then seq(matcher)
|
|
29
|
+
when Pick then of(matcher.parts[matcher.index])
|
|
30
|
+
when SepBy, Rep then :array
|
|
31
|
+
when Captures then :hash
|
|
32
|
+
when Choice then matcher.alts.map { of(_1) }.reduce { |a, b| join(a, b) }
|
|
33
|
+
when Not, And then :nil
|
|
34
|
+
when Both then of(matcher.right)
|
|
35
|
+
when Const then constant(matcher.constant)
|
|
36
|
+
when Send then SENDS.fetch(matcher.method_name, :poly)
|
|
37
|
+
when To then :node
|
|
38
|
+
when Infix then infix(of(matcher.operand))
|
|
39
|
+
when Ref then rules.fetch(matcher.name, :poly)
|
|
40
|
+
when Label, Check, Quiet, Named, Lexeme then of(matcher.inner)
|
|
41
|
+
else :poly
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# A String, or nil where it's optional: str.maybe.
|
|
46
|
+
def str_or_nil?(matcher, seen = {})
|
|
47
|
+
case matcher
|
|
48
|
+
when Maybe then of(matcher.inner) == :str
|
|
49
|
+
when Label, Check, Quiet, Named, Lexeme then str_or_nil?(matcher.inner, seen)
|
|
50
|
+
when Ref
|
|
51
|
+
!seen.key?(matcher.name) && (seen[matcher.name] = true) &&
|
|
52
|
+
str_or_nil?(@grammar.matcher_for(matcher.name), seen)
|
|
53
|
+
else false
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def rules = @rules ||= solve
|
|
60
|
+
|
|
61
|
+
def seq(matcher)
|
|
62
|
+
return of(matcher.parts[matcher.only_at]) if matcher.only_at
|
|
63
|
+
|
|
64
|
+
matcher.parts.any?(&:capture_key) ? :hash : :array
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def constant(value)
|
|
68
|
+
{ String => :str, Integer => :int, Float => :float, Symbol => :sym, NilClass => :nil }
|
|
69
|
+
.fetch(value.class, :poly)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Every rule's type at once, so none depends on which was asked about
|
|
73
|
+
# first: starting from nothing known (nil) for each, a rule's type is
|
|
74
|
+
# widened by what its body gives until nothing changes. Rules refer
|
|
75
|
+
# to each other in cycles (expr -> atom -> expr); one that still
|
|
76
|
+
# gives nothing (it only recurses) is :poly.
|
|
77
|
+
def solve
|
|
78
|
+
names = rule_names
|
|
79
|
+
@rules = names.to_h { [_1, nil] }
|
|
80
|
+
changed = true
|
|
81
|
+
while changed
|
|
82
|
+
changed = false
|
|
83
|
+
names.each do |name|
|
|
84
|
+
type = join(@rules[name], of(@grammar.matcher_for(name)))
|
|
85
|
+
changed ||= type != @rules[name]
|
|
86
|
+
@rules[name] = type
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
@rules.transform_values { _1 || :poly }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# The rules reachable from the root and the ignore rule.
|
|
93
|
+
def rule_names
|
|
94
|
+
names = []
|
|
95
|
+
queue = [@grammar.class.root, @grammar.class.ignore].compact
|
|
96
|
+
while (name = queue.shift)
|
|
97
|
+
next if names.include?(name)
|
|
98
|
+
|
|
99
|
+
names << name
|
|
100
|
+
queue.concat(refs(@grammar.matcher_for(name)))
|
|
101
|
+
end
|
|
102
|
+
names
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Rule names a matcher refers to (custom matchers, which the emitter
|
|
106
|
+
# refuses, refer to none).
|
|
107
|
+
def refs(value)
|
|
108
|
+
case value
|
|
109
|
+
when Ref then [value.name]
|
|
110
|
+
when Lit then refs(@grammar.literal_matcher(value.text))
|
|
111
|
+
when Data then value.to_h.values.flat_map { refs(_1) }
|
|
112
|
+
when Array then value.flat_map { refs(_1) }
|
|
113
|
+
else []
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# The type of a value that is either: nil is nothing known yet.
|
|
118
|
+
def join(one, other)
|
|
119
|
+
return other if one.nil?
|
|
120
|
+
return one if other.nil? || one == other
|
|
121
|
+
|
|
122
|
+
:poly
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def infix(operand) = operand.nil? || operand == :node ? operand : :poly
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# [file]
|
|
4
|
+
# purpose = "Turns a grammar into a standalone Ruby parser (Grammar.to_ruby)"
|
|
5
|
+
# responsibility = "Walking the rules' matchers and assembling the generated source"
|
|
6
|
+
# dependencies = ["Birman::Compiler::Emitter", "Birman::Compiler::Actions", "compiler/runtime.rb"]
|
|
7
|
+
# pattern = "Code generation from the immutable matcher tree"
|
|
8
|
+
module Birman
|
|
9
|
+
# Generates Ruby source for a parser that gives exactly the results
|
|
10
|
+
# Grammar.parse gives, values, spans, errors and all, without Birman.
|
|
11
|
+
# The output is written in the Ruby subset Spinel compiles to a native
|
|
12
|
+
# binary: the runtime (compiler/runtime.rb), a class per node, and a
|
|
13
|
+
# Parser with one method per matcher.
|
|
14
|
+
#
|
|
15
|
+
# source = Calc::Grammar.to_ruby(namespace: "CalcParser")
|
|
16
|
+
# CalcParser::Parser.new(input).parse # => Success or Failure
|
|
17
|
+
#
|
|
18
|
+
# Every matcher method takes a byte position and returns where its
|
|
19
|
+
# match ended, or FAIL; its value, if anything reads it, goes in an
|
|
20
|
+
# instance variable. Grammars using .map with a block, or a custom
|
|
21
|
+
# matcher class, can't be compiled: Error names the rule.
|
|
22
|
+
# [class.Compiler]
|
|
23
|
+
# purpose = "Generates a standalone parser from a grammar class"
|
|
24
|
+
# collaborators = ["Emitter (matcher templates)", "Actions (copies .action methods)", "Nodes"]
|
|
25
|
+
class Compiler
|
|
26
|
+
# Raised for a grammar the compiler can't translate faithfully.
|
|
27
|
+
class Error < StandardError; end
|
|
28
|
+
|
|
29
|
+
RUNTIME = File.join(__dir__, "compiler", "runtime.rb")
|
|
30
|
+
|
|
31
|
+
# Constants the generated code and its runtime refer to. A node class
|
|
32
|
+
# or copied constant with one of these names would sit in the same
|
|
33
|
+
# lexical scope and shadow it, so it's refused.
|
|
34
|
+
RESERVED_CONSTANTS = %w[
|
|
35
|
+
BirmanRT Parser Node Span Success Failure Committed TooDeep FAIL IN_PROGRESS ABSENT NONE TEN19 TWO64
|
|
36
|
+
CHUNK_LIMIT FLOAT_CHUNK LOOKAHEAD_DEPTH
|
|
37
|
+
StringScanner Hash Array String Integer Float Symbol Regexp Kernel Encoding
|
|
38
|
+
StandardError ArgumentError RangeError TypeError FloatDomainError SystemStackError
|
|
39
|
+
].freeze
|
|
40
|
+
|
|
41
|
+
# Spinel allows 64 begin blocks active at once and exits the process
|
|
42
|
+
# past that. Each lookahead (absent, present, &) holds one while its
|
|
43
|
+
# inner matcher runs, so nesting deeper than this is a Failure ("input
|
|
44
|
+
# is nested too deeply") instead. A binary built with a bigger
|
|
45
|
+
# -DSP_EXC_STACK_MAX can take a bigger depth.
|
|
46
|
+
LOOKAHEAD_DEPTH = 56
|
|
47
|
+
|
|
48
|
+
def initialize(grammar_class, namespace:, lookahead_depth: LOOKAHEAD_DEPTH)
|
|
49
|
+
unless namespace.match?(/\A[A-Z]\w*(::[A-Z]\w*)*\z/)
|
|
50
|
+
raise ArgumentError, "namespace must be a constant path, like \"CalcParser\""
|
|
51
|
+
end
|
|
52
|
+
unless lookahead_depth.is_a?(Integer) && lookahead_depth.positive?
|
|
53
|
+
raise ArgumentError, "lookahead_depth must be a positive Integer"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
@lookahead_depth = lookahead_depth
|
|
57
|
+
|
|
58
|
+
@grammar_class = grammar_class
|
|
59
|
+
@grammar = grammar_class.new
|
|
60
|
+
@namespace = namespace
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# [method."Compiler#to_ruby"]
|
|
64
|
+
# returns = "String: Ruby source defining <namespace>::Parser, its node classes and BirmanRT"
|
|
65
|
+
# raises = [
|
|
66
|
+
# "ArgumentError for left-recursive cycles with no rule in common, or a cycle that can't grow, as Grammar#parse",
|
|
67
|
+
# "Compiler::Error for what can't be compiled (named by rule)"
|
|
68
|
+
# ]
|
|
69
|
+
def to_ruby
|
|
70
|
+
@grammar.left_recursion
|
|
71
|
+
emitter = emit
|
|
72
|
+
check_namespace(emitter.node_names)
|
|
73
|
+
body = [*emitter.node_classes, parser_class(emitter)].join("\n")
|
|
74
|
+
wrap_in_namespace("#{runtime}\n#{body}")
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
# Values nobody reads (spacing, keywords) are never kept. Which those
|
|
80
|
+
# are is found by compiling: start with only the root's value, and
|
|
81
|
+
# compile again with every rule found to be read, until none is new.
|
|
82
|
+
def emit
|
|
83
|
+
valued = [@grammar_class.root]
|
|
84
|
+
loop do
|
|
85
|
+
emitter = Emitter.new(@grammar, valued)
|
|
86
|
+
emitter.compile
|
|
87
|
+
return emitter if (emitter.read_rules - valued).empty?
|
|
88
|
+
|
|
89
|
+
valued |= emitter.read_rules
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Generating into a module that already holds other classes of the
|
|
94
|
+
# same names (the grammar's own nodes, say) would reopen them with a
|
|
95
|
+
# different superclass.
|
|
96
|
+
def check_namespace(node_names)
|
|
97
|
+
return unless Object.const_defined?(@namespace)
|
|
98
|
+
|
|
99
|
+
existing = Object.const_get(@namespace)
|
|
100
|
+
node_names.each do |name|
|
|
101
|
+
next unless existing.is_a?(Module) && existing.const_defined?(name, false)
|
|
102
|
+
|
|
103
|
+
found = existing.const_get(name, false)
|
|
104
|
+
next if found.is_a?(Class) && found.superclass.name == "#{@namespace}::BirmanRT::Node"
|
|
105
|
+
|
|
106
|
+
raise Error, "#{@namespace} already defines #{name}; generate the parser into another namespace"
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def header
|
|
111
|
+
<<~RUBY
|
|
112
|
+
# Generated by Birman #{VERSION} from #{@grammar_class.name || "an anonymous grammar"}
|
|
113
|
+
# (Grammar.to_ruby). Regenerate it rather than editing it.
|
|
114
|
+
# Parse with #{@namespace}::Parser.new(input).parse, which returns
|
|
115
|
+
# a Success (value) or a Failure (pos, expected, message).
|
|
116
|
+
require "strscan"
|
|
117
|
+
RUBY
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# The runtime file, after its require.
|
|
121
|
+
def runtime = File.read(RUNTIME).split(/^require "strscan"\n\n/, 2).last
|
|
122
|
+
|
|
123
|
+
def parser_class(emitter)
|
|
124
|
+
<<~RUBY
|
|
125
|
+
class Parser < BirmanRT::Parser
|
|
126
|
+
include BirmanRT
|
|
127
|
+
|
|
128
|
+
# How deeply lookaheads may nest (Grammar.to_ruby's lookahead_depth).
|
|
129
|
+
LOOKAHEAD_DEPTH = #{@lookahead_depth}
|
|
130
|
+
|
|
131
|
+
#{indent(emitter.parser_body, 1)}
|
|
132
|
+
end
|
|
133
|
+
RUBY
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def wrap_in_namespace(body)
|
|
137
|
+
modules = @namespace.split("::")
|
|
138
|
+
opening = modules.each_with_index.map { |name, depth| indent("module #{name}", depth) }
|
|
139
|
+
closing = modules.each_index.reverse_each.map { |depth| indent("end", depth) }
|
|
140
|
+
[header, *opening, indent(body.chomp, modules.size), *closing].join("\n") << "\n"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def indent(text, depth)
|
|
144
|
+
pad = " " * depth
|
|
145
|
+
text.lines.map { |line| line.strip.empty? ? "\n" : "#{pad}#{line}" }.join.chomp
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
require_relative "compiler/literals"
|
|
151
|
+
require_relative "compiler/regexps"
|
|
152
|
+
require_relative "compiler/types"
|
|
153
|
+
require_relative "compiler/actions"
|
|
154
|
+
require_relative "compiler/nodes"
|
|
155
|
+
require_relative "compiler/emitter"
|
data/lib/birman/context.rb
CHANGED
|
@@ -8,7 +8,7 @@ module Birman
|
|
|
8
8
|
# non-ASCII string by character offset costs time proportional to
|
|
9
9
|
# the offset, which made parsing quadratic. Failure and Span convert
|
|
10
10
|
# back to characters for anything public.
|
|
11
|
-
class Context
|
|
11
|
+
class Context # rubocop:disable Metrics/ClassLength
|
|
12
12
|
# Marks a memo entry whose rule is still being matched.
|
|
13
13
|
IN_PROGRESS = Object.new.freeze
|
|
14
14
|
|
|
@@ -18,17 +18,29 @@ module Birman
|
|
|
18
18
|
|
|
19
19
|
# position: where the most recently entered rule started. When the
|
|
20
20
|
# input nests too deeply for Ruby's stack, that's where it happened.
|
|
21
|
-
|
|
21
|
+
# ignore_ref: a Ref to the grammar's ignore rule, or nil. Lexemes
|
|
22
|
+
# read it here, so the grammar running them decides what they skip.
|
|
23
|
+
attr_reader :input, :grammar, :ignore_ref, :furthest, :expected, :position, :isolated_furthest, :isolated_expected
|
|
22
24
|
|
|
23
|
-
def initialize(input, grammar)
|
|
25
|
+
def initialize(input, grammar, ignore_rule)
|
|
24
26
|
@input = input
|
|
25
27
|
@grammar = grammar
|
|
28
|
+
@ignore_ref = ignore_rule && Ref.new(ignore_rule) # built once per parse, not per token
|
|
26
29
|
@memo = {}
|
|
30
|
+
@growing = grammar.left_recursion.leaders.then { _1 unless _1.empty? } # nil: nothing to look up
|
|
31
|
+
@seeds = {}
|
|
32
|
+
# { leader of other rules => { pos it grows at => this round's memo
|
|
33
|
+
# for the other rules } }
|
|
34
|
+
@rounds = grammar.left_recursion.leaders.values.tally.filter_map { |leader, n| [leader, {}] if n > 1 }.to_h
|
|
27
35
|
@furthest = 0
|
|
28
36
|
@expected = NOTHING_EXPECTED
|
|
29
37
|
@token_ends = {}
|
|
30
38
|
end
|
|
31
39
|
|
|
40
|
+
# Where the parse begins: past whatever the ignore rule (if any)
|
|
41
|
+
# matches at the start of the input.
|
|
42
|
+
def start = @ignore_ref&.match(self, 0)&.pos || 0
|
|
43
|
+
|
|
32
44
|
# A lexeme ended at token_end, then the ignore rule consumed spacing
|
|
33
45
|
# (whitespace, comments) up to pos. Spans use this to stop at the
|
|
34
46
|
# token instead of including what was ignored after it.
|
|
@@ -55,7 +67,17 @@ module Birman
|
|
|
55
67
|
nil
|
|
56
68
|
end
|
|
57
69
|
|
|
58
|
-
|
|
70
|
+
# Record a finished frame's expectations (a memo hit, a label, a
|
|
71
|
+
# check) at pos. Ones already recorded there are skipped: a rule
|
|
72
|
+
# reached from several alternatives at one position is merged in
|
|
73
|
+
# once per alternative, and copying its whole list each time grew
|
|
74
|
+
# it exponentially. Failure#expected is uniq'd anyway, so this
|
|
75
|
+
# changes no error, only the work behind it. fail_at itself doesn't
|
|
76
|
+
# check: it runs far more often, and repeats there grow only
|
|
77
|
+
# linearly.
|
|
78
|
+
def merge(pos, expected)
|
|
79
|
+
expected.each { fail_at(pos, _1) unless pos == @furthest && @expected.include?(_1) }
|
|
80
|
+
end
|
|
59
81
|
|
|
60
82
|
# Run the block with fresh failure tracking starting at pos, then
|
|
61
83
|
# restore the outer state, and return the block's result. What the
|
|
@@ -98,39 +120,126 @@ module Birman
|
|
|
98
120
|
end
|
|
99
121
|
|
|
100
122
|
# Rule name's memo entry at pos, computed by the block the first
|
|
101
|
-
# time.
|
|
102
|
-
#
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
|
|
123
|
+
# time. The block returns [result, furthest, expected].
|
|
124
|
+
#
|
|
125
|
+
# A cycle of rules that call each other before consuming input (left
|
|
126
|
+
# recursion) is grown from a seed at its leader (see
|
|
127
|
+
# LeftRecursion#leaders and grow). While the leader grows at pos, the
|
|
128
|
+
# cycle's other rules are memoized there in a table of the round's
|
|
129
|
+
# own, not the memo: what they match there depends on the seed, which
|
|
130
|
+
# changes every round. (A match of theirs already in the memo there is
|
|
131
|
+
# final: it was made after the leader finished, or without it.)
|
|
132
|
+
# Every way back to a rule of the
|
|
133
|
+
# cycle at pos passes through the leader's seed, so meeting any rule
|
|
134
|
+
# again at the same position while it is still being matched means
|
|
135
|
+
# it reaches itself without consuming input in a way the grammar's
|
|
136
|
+
# ASTs don't show (through a custom matcher): that would never
|
|
137
|
+
# terminate, and it's a bug in the grammar, not the input, so it
|
|
138
|
+
# raises.
|
|
139
|
+
def memoize(name, pos, &) # rubocop:disable Metrics
|
|
106
140
|
table = @memo[name] ||= {}
|
|
107
141
|
entry = table[pos]
|
|
108
142
|
return entry if entry && !entry.equal?(IN_PROGRESS)
|
|
109
|
-
|
|
143
|
+
|
|
144
|
+
leader = @growing&.[](name) # nil without left recursion: no further lookups
|
|
145
|
+
round = @rounds[leader]&.[](pos) if leader
|
|
146
|
+
return in_round(round, name, pos, &) if round && !leader.equal?(name)
|
|
147
|
+
return seed(name, pos) if entry
|
|
110
148
|
|
|
111
149
|
@position = pos
|
|
150
|
+
return grow(name, table, pos, &) if leader.equal?(name)
|
|
151
|
+
|
|
112
152
|
compute(table, pos, &)
|
|
113
153
|
end
|
|
114
154
|
|
|
115
155
|
# Run a lookahead's inner match. A lookahead only asks whether
|
|
116
156
|
# something would match here, so a commit failure inside it is an
|
|
117
|
-
# ordinary failure (nil) rather than the end of the parse.
|
|
157
|
+
# ordinary failure (nil) rather than the end of the parse. What it
|
|
158
|
+
# expected is kept, as generated parsers keep it: the frames it
|
|
159
|
+
# unwound through restored what they started with, so without this a
|
|
160
|
+
# failure where the lookahead began would only be named, not said.
|
|
118
161
|
def lookahead
|
|
119
162
|
yield
|
|
120
|
-
rescue Committed
|
|
163
|
+
rescue Committed => e
|
|
164
|
+
@furthest = e.pos
|
|
165
|
+
@expected = e.expected
|
|
121
166
|
nil
|
|
122
167
|
end
|
|
123
168
|
|
|
124
169
|
private
|
|
125
170
|
|
|
126
171
|
# The memo is a Hash per rule, keyed by position, so no key arrays.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
table[
|
|
172
|
+
# (A round's table is keyed by rule instead.)
|
|
173
|
+
def compute(table, key)
|
|
174
|
+
table[key] = IN_PROGRESS
|
|
175
|
+
table[key] = yield
|
|
130
176
|
ensure
|
|
131
177
|
# A commit failure caught by a lookahead unwinds through here; the
|
|
132
178
|
# rule may be tried again at this position afterwards.
|
|
179
|
+
table.delete(key) if table[key].equal?(IN_PROGRESS)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# A rule of a cycle whose leader grows at pos, memoized in the round's
|
|
183
|
+
# table (keyed by rule): once a round, however many alternatives call
|
|
184
|
+
# it, or a cycle of n rules with k alternatives each would take
|
|
185
|
+
# k**(n - 1) matches a round. Back at itself without the leader's seed
|
|
186
|
+
# between, it raises as memoize does.
|
|
187
|
+
def in_round(round, name, pos, &)
|
|
188
|
+
entry = round[name]
|
|
189
|
+
return entry if entry && !entry.equal?(IN_PROGRESS)
|
|
190
|
+
return seed(name, pos) if entry
|
|
191
|
+
|
|
192
|
+
@position = pos
|
|
193
|
+
compute(round, name, &)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Seed-growing (Warth et al. 2008), at a cycle's leader only, as pegen
|
|
197
|
+
# does. The rule is matched with its call to itself here failing,
|
|
198
|
+
# then again with that call returning the last match (the seed), for
|
|
199
|
+
# as long as the match gets further. It's a loop, so a longer chain
|
|
200
|
+
# uses no more stack. The memo entry stays IN_PROGRESS until the end;
|
|
201
|
+
# every way back to it passes through its seed. A leader of other
|
|
202
|
+
# rules starts each round with an empty table for them at pos.
|
|
203
|
+
#
|
|
204
|
+
# Each attempt's diagnostics are kept, not just the best one's: the
|
|
205
|
+
# last attempt, which got no further, is usually the one that knows
|
|
206
|
+
# what was missing ("1 +" expected a number after the "+").
|
|
207
|
+
def grow(name, table, pos) # rubocop:disable Metrics
|
|
208
|
+
seeds = @seeds[name] ||= {}
|
|
209
|
+
seeds[pos] = best = [nil, pos, NOTHING_EXPECTED]
|
|
210
|
+
table[pos] = IN_PROGRESS
|
|
211
|
+
rounds = @rounds[name]
|
|
212
|
+
diagnostics = [pos, NOTHING_EXPECTED]
|
|
213
|
+
# Not loop: its block adds frames to every level of nesting.
|
|
214
|
+
while true # rubocop:disable Style/InfiniteLoop
|
|
215
|
+
rounds[pos] = {} if rounds
|
|
216
|
+
result, furthest, expected = yield
|
|
217
|
+
diagnostics = further(diagnostics, furthest, expected)
|
|
218
|
+
break unless result && result.pos > (best[0]&.pos || -1)
|
|
219
|
+
|
|
220
|
+
seeds[pos] = best = [result, pos, NOTHING_EXPECTED]
|
|
221
|
+
end
|
|
222
|
+
table[pos] = [best[0], *diagnostics]
|
|
223
|
+
ensure
|
|
224
|
+
seeds&.delete(pos) # nil if the stack ran out before it was set
|
|
225
|
+
rounds&.delete(pos)
|
|
133
226
|
table.delete(pos) if table[pos].equal?(IN_PROGRESS)
|
|
134
227
|
end
|
|
228
|
+
|
|
229
|
+
# Two frames' diagnostics as one, as fail_at would: the further
|
|
230
|
+
# position wins, and at the same one the expectations are merged.
|
|
231
|
+
def further(diagnostics, furthest, expected)
|
|
232
|
+
return [furthest, expected] if furthest > diagnostics[0]
|
|
233
|
+
return diagnostics if furthest < diagnostics[0] || expected.empty?
|
|
234
|
+
|
|
235
|
+
[furthest, diagnostics[1] | expected]
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# The rule's match so far at pos, while it grows there. Its
|
|
239
|
+
# diagnostics are grow's to keep, so it carries none.
|
|
240
|
+
def seed(name, pos)
|
|
241
|
+
@seeds[name]&.[](pos) || raise(ArgumentError,
|
|
242
|
+
"left recursion in rule :#{name}: it can reach itself without consuming input")
|
|
243
|
+
end
|
|
135
244
|
end
|
|
136
245
|
end
|
data/lib/birman/grammar.rb
CHANGED
|
@@ -88,6 +88,21 @@ module Birman
|
|
|
88
88
|
# returns = "String: the grammar in PEG-flavored EBNF"
|
|
89
89
|
def to_ebnf = EBNF.call(to_ast)
|
|
90
90
|
|
|
91
|
+
# A standalone parser for this grammar, as Ruby source: the same
|
|
92
|
+
# results as parse, from code Spinel can compile to a native binary.
|
|
93
|
+
# [method."Grammar.to_ruby"]
|
|
94
|
+
# params = [
|
|
95
|
+
# "namespace: String, the module the parser is defined in (\"CalcParser\")",
|
|
96
|
+
# "lookahead_depth: Integer = 56, how deeply lookaheads may nest (see Compiler)"
|
|
97
|
+
# ]
|
|
98
|
+
# returns = "String: Ruby source defining <namespace>::Parser (new(input).parse => Success or Failure)"
|
|
99
|
+
# raises = [
|
|
100
|
+
# "ArgumentError for left-recursive cycles with no rule in common, or a cycle that can't grow, as parse",
|
|
101
|
+
# "Birman::Compiler::Error for what can't be compiled (.map blocks, custom matchers...)",
|
|
102
|
+
# "Birman::Compiler::Error for actions calling Integer() or to_i: use large_integer"
|
|
103
|
+
# ]
|
|
104
|
+
def to_ruby(namespace:, **) = Compiler.new(self, namespace:, **).to_ruby
|
|
105
|
+
|
|
91
106
|
private
|
|
92
107
|
|
|
93
108
|
def instance = @instance ||= new
|
|
@@ -110,20 +125,21 @@ module Birman
|
|
|
110
125
|
# [method.parse]
|
|
111
126
|
# params = ["input: String"]
|
|
112
127
|
# returns = "Birman::Success(value) when all of input parses, else Birman::Failure"
|
|
113
|
-
# raises = [
|
|
114
|
-
#
|
|
128
|
+
# raises = [
|
|
129
|
+
# "ArgumentError on the first parse, whatever the input, for no root rule",
|
|
130
|
+
# "ArgumentError on the first parse for left-recursive cycles with no rule in common",
|
|
131
|
+
# "ArgumentError on the first parse for a left-recursive cycle with no base case, or a base case first",
|
|
132
|
+
# "ArgumentError on the first parse for a reachable rule whose matcher can't be built (unnamed captures...)",
|
|
133
|
+
# "ArgumentError for an undefined rule, or left recursion through a custom matcher, when the parse reaches it"
|
|
134
|
+
# ]
|
|
135
|
+
# notes = "Bad input never raises: invalid UTF-8 and too-deep nesting are Failures with reason set"
|
|
115
136
|
def parse(input)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
137
|
+
left_recursion
|
|
138
|
+
if (bad = Birman.invalid_byte(input))
|
|
139
|
+
return build_failure(input, bad, [], reason: "invalid #{input.encoding}")
|
|
140
|
+
end
|
|
120
141
|
|
|
121
|
-
|
|
122
|
-
build_failure(input, ctx.furthest, ctx.expected)
|
|
123
|
-
rescue Committed => e
|
|
124
|
-
build_failure(input, e.pos, e.expected, committed: true)
|
|
125
|
-
rescue SystemStackError
|
|
126
|
-
build_failure(input, ctx.position, [], reason: "input is nested too deeply")
|
|
142
|
+
__match_all(input)
|
|
127
143
|
end
|
|
128
144
|
|
|
129
145
|
# Returns the value or raises ParseError, worded by formatter.
|
|
@@ -144,19 +160,80 @@ module Birman
|
|
|
144
160
|
|
|
145
161
|
# { rule_name => ast } for every rule reachable from the root (and
|
|
146
162
|
# the ignore rule), in first-use order.
|
|
147
|
-
def to_ast
|
|
163
|
+
def to_ast = __reachable_asts { matcher_for(_1).to_ast }
|
|
164
|
+
|
|
165
|
+
# The grammar's left-call analysis (see LeftRecursion), built on the
|
|
166
|
+
# first parse or to_ruby. It raises then, and every time after, if
|
|
167
|
+
# the grammar has left-recursive cycles with no rule in common, or a
|
|
168
|
+
# cycle that can't grow (see LeftRecursion#check!); later parses only
|
|
169
|
+
# read the ivar. Context reads its leaders, to grow them.
|
|
170
|
+
def left_recursion = @left_recursion ||= LeftRecursion.new(__analyzable_asts, ignore: ignore_rule).check!
|
|
171
|
+
|
|
172
|
+
private
|
|
173
|
+
|
|
174
|
+
def __match_all(input)
|
|
175
|
+
ctx = Context.new(input, self, ignore_rule)
|
|
176
|
+
result = Ref.new(root_rule).match(ctx, ctx.start)
|
|
177
|
+
return Success.new(result.value) if result && result.pos == input.bytesize
|
|
178
|
+
|
|
179
|
+
ctx.fail_at(result.pos, "end of input") if result
|
|
180
|
+
build_failure(input, ctx.furthest, ctx.expected)
|
|
181
|
+
rescue Committed => e
|
|
182
|
+
build_failure(input, e.pos, e.expected, committed: true)
|
|
183
|
+
rescue SystemStackError
|
|
184
|
+
build_failure(input, ctx.position, [], reason: "input is nested too deeply")
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# As to_ast, leaving out what it can't describe: undefined rules
|
|
188
|
+
# (the parse reports them if it gets there) and rules using a
|
|
189
|
+
# matcher with no to_ast. The analysis takes those to consume input
|
|
190
|
+
# and to call nothing, so a cycle through one is left to
|
|
191
|
+
# Context#memoize, and rules only it refers to aren't analyzed.
|
|
192
|
+
def __analyzable_asts
|
|
193
|
+
__reachable_asts do |name|
|
|
194
|
+
next unless respond_to?(:"__rule_#{name}", true)
|
|
195
|
+
|
|
196
|
+
matcher_for(name).to_ast
|
|
197
|
+
rescue NoMethodError => e
|
|
198
|
+
raise unless e.name == :to_ast && e.receiver.is_a?(Matcher)
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# { rule_name => yield(rule_name) } for every rule reachable from the
|
|
203
|
+
# root and the ignore rule, in first-use order, skipping nil results.
|
|
204
|
+
def __reachable_asts
|
|
148
205
|
rules = {}
|
|
149
206
|
queue = [root_rule, ignore_rule].compact
|
|
150
207
|
while (name = queue.shift)
|
|
151
208
|
next if rules.key?(name)
|
|
152
209
|
|
|
153
|
-
rules[name] =
|
|
210
|
+
rules[name] = yield(name)
|
|
154
211
|
queue.concat(AST.refs(rules[name]))
|
|
155
212
|
end
|
|
156
|
-
rules
|
|
213
|
+
rules.compact
|
|
157
214
|
end
|
|
158
215
|
|
|
159
|
-
|
|
216
|
+
# Kernel#Integer(value, base), for actions: use it instead of Integer
|
|
217
|
+
# or String#to_i in a grammar you'll generate a parser from (see
|
|
218
|
+
# to_ruby). Past 63 bits, a Spinel binary's Integer and to_i raise or
|
|
219
|
+
# give 2**63 - 1; the generated parser's large_integer gives Bignums,
|
|
220
|
+
# as Ruby's does. Bases: Integers, or Floats truncated (Rational and
|
|
221
|
+
# NaN bases differ from Kernel#Integer in a generated parser). A base
|
|
222
|
+
# that isn't a number raises TypeError, as in Ruby 3.3+; Ruby 3.2's
|
|
223
|
+
# Integer ignores it.
|
|
224
|
+
# [method.large_integer]
|
|
225
|
+
# params = ["value: String or Numeric", "base: Integer = 0 (from the prefix, as Integer's)"]
|
|
226
|
+
# notes = "In a Spinel binary too, Floats past 63 bits convert exactly; negative bases raise there"
|
|
227
|
+
# returns = "Integer(value, base)"
|
|
228
|
+
# raises = ["ArgumentError or TypeError, as Kernel#Integer in Ruby 3.3+"]
|
|
229
|
+
def large_integer(value, base = 0)
|
|
230
|
+
raise TypeError, "no implicit conversion from nil to integer" if base.nil?
|
|
231
|
+
unless base.respond_to?(:to_int)
|
|
232
|
+
raise TypeError, "no implicit conversion of #{[true, false].include?(base) ? base : base.class} into Integer"
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
Integer(value, base)
|
|
236
|
+
end
|
|
160
237
|
|
|
161
238
|
def build_failure(input, byte_pos, expected, committed: false, reason: nil)
|
|
162
239
|
Failure.new(input:, pos: Birman.char_offset(input, byte_pos), expected: expected.uniq, committed:, reason:)
|