ibex 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +43 -0
- data/CHANGELOG.md +30 -0
- data/LICENSE.txt +21 -0
- data/README.md +194 -0
- data/Rakefile +25 -0
- data/Steepfile +10 -0
- data/docs/architecture.md +99 -0
- data/docs/compat-notes.md +37 -0
- data/docs/grammar-reference.md +127 -0
- data/docs/lexer-coverage.md +14 -0
- data/docs/phase10-extensions.md +27 -0
- data/docs/racc-migration.md +47 -0
- data/exe/ibex +6 -0
- data/gemfiles/Gemfile +7 -0
- data/gemfiles/Gemfile.lock +98 -0
- data/lib/ibex/analysis/sets.rb +194 -0
- data/lib/ibex/analysis.rb +8 -0
- data/lib/ibex/cli/counterexample_options.rb +38 -0
- data/lib/ibex/cli/outputs.rb +88 -0
- data/lib/ibex/cli.rb +333 -0
- data/lib/ibex/codegen/dot.rb +52 -0
- data/lib/ibex/codegen/html.rb +110 -0
- data/lib/ibex/codegen/rbs.rb +54 -0
- data/lib/ibex/codegen/report.rb +149 -0
- data/lib/ibex/codegen/ruby.rb +235 -0
- data/lib/ibex/codegen/symbol_labels.rb +18 -0
- data/lib/ibex/error.rb +7 -0
- data/lib/ibex/frontend/action_scanner.rb +266 -0
- data/lib/ibex/frontend/ast.rb +146 -0
- data/lib/ibex/frontend/bootstrap_parser.rb +163 -0
- data/lib/ibex/frontend/dsl.rb +196 -0
- data/lib/ibex/frontend/generated_parser.rb +434 -0
- data/lib/ibex/frontend/generated_parser_base.rb +273 -0
- data/lib/ibex/frontend/grammar.y +156 -0
- data/lib/ibex/frontend/lexer.rb +165 -0
- data/lib/ibex/frontend/parser/declarations.rb +163 -0
- data/lib/ibex/frontend/parser/rules.rb +154 -0
- data/lib/ibex/frontend/parser.rb +23 -0
- data/lib/ibex/frontend/regenerator.rb +41 -0
- data/lib/ibex/frontend/source_cursor.rb +94 -0
- data/lib/ibex/frontend/token_adapter/declaration_state.rb +248 -0
- data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +37 -0
- data/lib/ibex/frontend/token_adapter/rule_state.rb +162 -0
- data/lib/ibex/frontend/token_adapter.rb +108 -0
- data/lib/ibex/frontend.rb +28 -0
- data/lib/ibex/ir/automaton_ir.rb +109 -0
- data/lib/ibex/ir/grammar_ir.rb +192 -0
- data/lib/ibex/ir/serialize.rb +154 -0
- data/lib/ibex/ir.rb +53 -0
- data/lib/ibex/lalr/builder.rb +270 -0
- data/lib/ibex/lalr/conflict.rb +120 -0
- data/lib/ibex/lalr/conflict_search.rb +332 -0
- data/lib/ibex/lalr/conflict_search_limits.rb +39 -0
- data/lib/ibex/lalr/counterexample.rb +138 -0
- data/lib/ibex/lalr/default_reductions.rb +67 -0
- data/lib/ibex/lalr.rb +27 -0
- data/lib/ibex/normalize/declarations.rb +60 -0
- data/lib/ibex/normalize/diagnostics.rb +99 -0
- data/lib/ibex/normalize/expander.rb +218 -0
- data/lib/ibex/normalize/expression.rb +54 -0
- data/lib/ibex/normalize.rb +166 -0
- data/lib/ibex/runtime/parser.rb +360 -0
- data/lib/ibex/runtime.rb +8 -0
- data/lib/ibex/tables.rb +125 -0
- data/lib/ibex/version.rb +6 -0
- data/lib/ibex.rb +23 -0
- data/sig/ibex/analysis/sets.rbs +72 -0
- data/sig/ibex/analysis.rbs +6 -0
- data/sig/ibex/cli/counterexample_options.rbs +16 -0
- data/sig/ibex/cli/outputs.rbs +31 -0
- data/sig/ibex/cli.rbs +108 -0
- data/sig/ibex/codegen/dot.rbs +19 -0
- data/sig/ibex/codegen/html.rbs +35 -0
- data/sig/ibex/codegen/rbs.rbs +31 -0
- data/sig/ibex/codegen/report.rbs +39 -0
- data/sig/ibex/codegen/ruby.rbs +92 -0
- data/sig/ibex/codegen/symbol_labels.rbs +11 -0
- data/sig/ibex/error.rbs +7 -0
- data/sig/ibex/frontend/action_scanner.rbs +74 -0
- data/sig/ibex/frontend/ast.rbs +223 -0
- data/sig/ibex/frontend/bootstrap_parser.rbs +80 -0
- data/sig/ibex/frontend/dsl.rbs +124 -0
- data/sig/ibex/frontend/generated_parser.rbs +163 -0
- data/sig/ibex/frontend/generated_parser_base.rbs +114 -0
- data/sig/ibex/frontend/lexer.rbs +61 -0
- data/sig/ibex/frontend/parser/declarations.rbs +56 -0
- data/sig/ibex/frontend/parser/rules.rbs +48 -0
- data/sig/ibex/frontend/parser.rbs +16 -0
- data/sig/ibex/frontend/regenerator.rbs +16 -0
- data/sig/ibex/frontend/source_cursor.rbs +73 -0
- data/sig/ibex/frontend/token_adapter/declaration_state.rbs +97 -0
- data/sig/ibex/frontend/token_adapter/delimiter_tracker.rbs +25 -0
- data/sig/ibex/frontend/token_adapter/rule_state.rbs +77 -0
- data/sig/ibex/frontend/token_adapter.rbs +67 -0
- data/sig/ibex/frontend.rbs +15 -0
- data/sig/ibex/ir/automaton_ir.rbs +77 -0
- data/sig/ibex/ir/grammar_ir.rbs +143 -0
- data/sig/ibex/ir/serialize.rbs +50 -0
- data/sig/ibex/ir.rbs +43 -0
- data/sig/ibex/lalr/builder.rbs +85 -0
- data/sig/ibex/lalr/conflict.rbs +47 -0
- data/sig/ibex/lalr/conflict_search.rbs +131 -0
- data/sig/ibex/lalr/conflict_search_limits.rbs +29 -0
- data/sig/ibex/lalr/counterexample.rbs +53 -0
- data/sig/ibex/lalr/default_reductions.rbs +20 -0
- data/sig/ibex/lalr.rbs +23 -0
- data/sig/ibex/normalize/declarations.rbs +20 -0
- data/sig/ibex/normalize/diagnostics.rbs +32 -0
- data/sig/ibex/normalize/expander.rbs +66 -0
- data/sig/ibex/normalize/expression.rbs +21 -0
- data/sig/ibex/normalize.rbs +96 -0
- data/sig/ibex/runtime/parser.rbs +167 -0
- data/sig/ibex/runtime.rbs +6 -0
- data/sig/ibex/tables.rbs +50 -0
- data/sig/ibex/version.rbs +6 -0
- data/sig/ibex.rbs +6 -0
- metadata +161 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ibex
|
|
4
|
+
module IR
|
|
5
|
+
SCHEMA_VERSION = 1
|
|
6
|
+
|
|
7
|
+
# @rbs (untyped value) -> untyped
|
|
8
|
+
def deep_freeze(value)
|
|
9
|
+
case value
|
|
10
|
+
when Array then value.each { |item| deep_freeze(item) }
|
|
11
|
+
when Hash
|
|
12
|
+
value.each do |key, item|
|
|
13
|
+
deep_freeze(key)
|
|
14
|
+
deep_freeze(item)
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
value.freeze
|
|
18
|
+
end
|
|
19
|
+
module_function :deep_freeze
|
|
20
|
+
|
|
21
|
+
# An interned terminal or nonterminal.
|
|
22
|
+
class GrammarSymbol
|
|
23
|
+
attr_reader :id #: Integer
|
|
24
|
+
attr_reader :name #: String
|
|
25
|
+
attr_reader :kind #: Symbol
|
|
26
|
+
attr_reader :reserved #: bool
|
|
27
|
+
attr_reader :precedence #: precedence?
|
|
28
|
+
attr_reader :location #: location?
|
|
29
|
+
|
|
30
|
+
# @rbs (id: Integer, name: String, kind: Symbol, ?reserved: bool, ?precedence: precedence?,
|
|
31
|
+
# ?location: location?) -> void
|
|
32
|
+
def initialize(id:, name:, kind:, reserved: false, precedence: nil, location: nil)
|
|
33
|
+
@id = id
|
|
34
|
+
@name = name.freeze
|
|
35
|
+
@kind = kind.to_sym
|
|
36
|
+
@reserved = reserved
|
|
37
|
+
@precedence = IR.deep_freeze(precedence)
|
|
38
|
+
@location = IR.deep_freeze(location)
|
|
39
|
+
freeze
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @rbs () -> bool
|
|
43
|
+
def terminal? = @kind == :terminal
|
|
44
|
+
# @rbs () -> bool
|
|
45
|
+
def nonterminal? = @kind == :nonterminal
|
|
46
|
+
|
|
47
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
48
|
+
def to_h
|
|
49
|
+
{ id: @id, name: @name, kind: @kind, reserved: @reserved, prec: @precedence, loc: @location }
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Opaque Ruby semantic action metadata.
|
|
54
|
+
class Action
|
|
55
|
+
attr_reader :code #: String
|
|
56
|
+
attr_reader :location #: location
|
|
57
|
+
attr_reader :named_refs #: Array[named_ref]
|
|
58
|
+
attr_reader :context_length #: Integer
|
|
59
|
+
|
|
60
|
+
# @rbs (code: String, location: location, ?named_refs: Array[named_ref], ?context_length: Integer) -> void
|
|
61
|
+
def initialize(code:, location:, named_refs: [], context_length: 0)
|
|
62
|
+
@code = code.freeze
|
|
63
|
+
@location = IR.deep_freeze(location)
|
|
64
|
+
@named_refs = IR.deep_freeze(named_refs)
|
|
65
|
+
@context_length = context_length
|
|
66
|
+
freeze
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
70
|
+
def to_h
|
|
71
|
+
{ code: @code, loc: @location, named_refs: @named_refs, context_length: @context_length }
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# A normalized BNF production using symbol ids.
|
|
76
|
+
class Production
|
|
77
|
+
attr_reader :id #: Integer
|
|
78
|
+
attr_reader :lhs #: Integer
|
|
79
|
+
attr_reader :rhs #: Array[Integer]
|
|
80
|
+
attr_reader :action #: Action?
|
|
81
|
+
attr_reader :precedence_override #: Integer?
|
|
82
|
+
attr_reader :origin #: Hash[Symbol, untyped]
|
|
83
|
+
|
|
84
|
+
# @rbs (id: Integer, lhs: Integer, rhs: Array[Integer], action: Action?, precedence_override: Integer?,
|
|
85
|
+
# origin: Hash[Symbol, untyped]) -> void
|
|
86
|
+
def initialize(id:, lhs:, rhs:, action:, precedence_override:, origin:)
|
|
87
|
+
@id = id
|
|
88
|
+
@lhs = lhs
|
|
89
|
+
@rhs = rhs.freeze
|
|
90
|
+
@action = action
|
|
91
|
+
@precedence_override = precedence_override
|
|
92
|
+
@origin = IR.deep_freeze(origin)
|
|
93
|
+
freeze
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
97
|
+
def to_h
|
|
98
|
+
{ id: @id, lhs: @lhs, rhs: @rhs, action: @action&.to_h, prec_override: @precedence_override,
|
|
99
|
+
origin: @origin }
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# One opaque user-code block and the location of its first code line.
|
|
104
|
+
class UserCodeChunk
|
|
105
|
+
attr_reader :code #: String
|
|
106
|
+
attr_reader :location #: location
|
|
107
|
+
|
|
108
|
+
# @rbs (code: String, location: location) -> void
|
|
109
|
+
def initialize(code:, location:)
|
|
110
|
+
@code = code.freeze
|
|
111
|
+
@location = IR.deep_freeze(location)
|
|
112
|
+
freeze
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
116
|
+
def to_h
|
|
117
|
+
{ code: @code, loc: @location }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Immutable normalized grammar exchanged between pipeline stages.
|
|
122
|
+
class Grammar
|
|
123
|
+
attr_reader :class_name #: String
|
|
124
|
+
attr_reader :superclass #: String?
|
|
125
|
+
attr_reader :start #: String
|
|
126
|
+
attr_reader :expect #: Integer
|
|
127
|
+
attr_reader :options #: grammar_options
|
|
128
|
+
attr_reader :symbols #: Array[GrammarSymbol]
|
|
129
|
+
attr_reader :productions #: Array[Production]
|
|
130
|
+
attr_reader :user_code #: Hash[String, String]
|
|
131
|
+
attr_reader :user_code_chunks #: user_code_chunks
|
|
132
|
+
attr_reader :conversions #: Hash[String, String]
|
|
133
|
+
attr_reader :warnings #: Array[grammar_warning]
|
|
134
|
+
attr_reader :schema_version #: Integer
|
|
135
|
+
|
|
136
|
+
# @rbs (class_name: String, superclass: String?, start: String, expect: Integer, options: grammar_options,
|
|
137
|
+
# symbols: Array[GrammarSymbol], productions: Array[Production], user_code: Hash[String, String],
|
|
138
|
+
# conversions: Hash[String, String], warnings: Array[grammar_warning], ?user_code_chunks: user_code_chunks?,
|
|
139
|
+
# ?schema_version: Integer) -> void
|
|
140
|
+
def initialize(class_name:, superclass:, start:, expect:, options:, symbols:, productions:, user_code:,
|
|
141
|
+
conversions:, warnings:, user_code_chunks: nil, schema_version: SCHEMA_VERSION)
|
|
142
|
+
@class_name = class_name.freeze
|
|
143
|
+
@superclass = superclass&.freeze
|
|
144
|
+
@start = start.freeze
|
|
145
|
+
@expect = expect
|
|
146
|
+
@options = IR.deep_freeze(options)
|
|
147
|
+
@symbols = symbols.freeze
|
|
148
|
+
@productions = productions.freeze
|
|
149
|
+
@user_code = IR.deep_freeze(user_code)
|
|
150
|
+
@user_code_chunks = IR.deep_freeze(user_code_chunks || {})
|
|
151
|
+
validate_user_code_chunks
|
|
152
|
+
@conversions = IR.deep_freeze(conversions)
|
|
153
|
+
@warnings = IR.deep_freeze(warnings)
|
|
154
|
+
@schema_version = schema_version
|
|
155
|
+
@symbols_by_name = @symbols.to_h { |symbol| [symbol.name, symbol] }.freeze
|
|
156
|
+
@symbols_by_id = @symbols.to_h { |symbol| [symbol.id, symbol] }.freeze
|
|
157
|
+
freeze
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# @rbs (String name) -> GrammarSymbol?
|
|
161
|
+
def symbol(name) = @symbols_by_name[name]
|
|
162
|
+
# @rbs (Integer? id) -> GrammarSymbol?
|
|
163
|
+
def symbol_by_id(id) = @symbols_by_id[id]
|
|
164
|
+
# @rbs () -> Array[GrammarSymbol]
|
|
165
|
+
def terminals = @symbols.select(&:terminal?)
|
|
166
|
+
# @rbs () -> Array[GrammarSymbol]
|
|
167
|
+
def nonterminals = @symbols.select(&:nonterminal?)
|
|
168
|
+
|
|
169
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
170
|
+
def to_h
|
|
171
|
+
value = { ibex_ir: "grammar", schema_version: @schema_version, class_name: @class_name, superclass: @superclass,
|
|
172
|
+
start: @start, expect: @expect, options: @options, symbols: @symbols.map(&:to_h),
|
|
173
|
+
productions: @productions.map(&:to_h), user_code: @user_code, conversions: @conversions,
|
|
174
|
+
warnings: @warnings } #: Hash[Symbol, untyped]
|
|
175
|
+
value[:user_code_chunks] = @user_code_chunks.transform_values { |chunks| chunks.map(&:to_h) } \
|
|
176
|
+
unless @user_code_chunks.empty?
|
|
177
|
+
value
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
private
|
|
181
|
+
|
|
182
|
+
# @rbs () -> void
|
|
183
|
+
def validate_user_code_chunks
|
|
184
|
+
@user_code_chunks.each do |name, chunks|
|
|
185
|
+
next if chunks.map(&:code).join == @user_code.fetch(name, "")
|
|
186
|
+
|
|
187
|
+
raise Ibex::Error, "(ir):1:1: user-code chunks do not match the concatenated #{name} section"
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Ibex
|
|
6
|
+
module IR
|
|
7
|
+
# Stable JSON serialization for versioned pipeline IR.
|
|
8
|
+
module Serialize
|
|
9
|
+
# @rbs!
|
|
10
|
+
# private def validate_version: (untyped data) -> untyped
|
|
11
|
+
# private def self.validate_version: (untyped data) -> untyped
|
|
12
|
+
# private def load_grammar: (untyped data) -> untyped
|
|
13
|
+
# private def self.load_grammar: (untyped data) -> untyped
|
|
14
|
+
# private def load_automaton: (untyped data) -> untyped
|
|
15
|
+
# private def self.load_automaton: (untyped data) -> untyped
|
|
16
|
+
# private def load_state: (untyped state, untyped grammar) -> untyped
|
|
17
|
+
# private def self.load_state: (untyped state, untyped grammar) -> untyped
|
|
18
|
+
# private def symbol_keyed: (untyped values, untyped grammar, ?actions: untyped) -> untyped
|
|
19
|
+
# private def self.symbol_keyed: (untyped values, untyped grammar, ?actions: untyped) -> untyped
|
|
20
|
+
# private def normalize_action: (untyped value) -> untyped
|
|
21
|
+
# private def self.normalize_action: (untyped value) -> untyped
|
|
22
|
+
# private def load_production: (untyped production) -> untyped
|
|
23
|
+
# private def self.load_production: (untyped production) -> untyped
|
|
24
|
+
# private def load_user_code_chunks: (untyped chunks) -> untyped
|
|
25
|
+
# private def self.load_user_code_chunks: (untyped chunks) -> untyped
|
|
26
|
+
# private def symbolize: (untyped value) -> untyped
|
|
27
|
+
# private def self.symbolize: (untyped value) -> untyped
|
|
28
|
+
|
|
29
|
+
# @rbs (Grammar | Automaton value) -> String
|
|
30
|
+
def dump(value)
|
|
31
|
+
"#{JSON.pretty_generate(value.to_h)}\n"
|
|
32
|
+
end
|
|
33
|
+
module_function :dump
|
|
34
|
+
|
|
35
|
+
# @rbs (String source) -> (Grammar | Automaton)
|
|
36
|
+
def load(source)
|
|
37
|
+
data = JSON.parse(source)
|
|
38
|
+
type = data.fetch("ibex_ir") { raise Ibex::Error, "(ir):1:1: missing ibex_ir discriminator" }
|
|
39
|
+
validate_version(data)
|
|
40
|
+
return load_grammar(data) if type == "grammar"
|
|
41
|
+
return load_automaton(data) if type == "automaton"
|
|
42
|
+
|
|
43
|
+
raise Ibex::Error, "(ir):1:1: unsupported IR type #{type.inspect}"
|
|
44
|
+
rescue JSON::ParserError => e
|
|
45
|
+
raise Ibex::Error, "(ir):1:1: invalid JSON: #{e.message}"
|
|
46
|
+
end
|
|
47
|
+
module_function :load
|
|
48
|
+
|
|
49
|
+
# @rbs skip
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
# @rbs skip
|
|
53
|
+
def validate_version(data)
|
|
54
|
+
version = data["schema_version"]
|
|
55
|
+
return if version == SCHEMA_VERSION
|
|
56
|
+
|
|
57
|
+
raise Ibex::Error, "(ir):1:1: unsupported schema_version #{version.inspect}; expected #{SCHEMA_VERSION}"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# @rbs skip
|
|
61
|
+
def load_grammar(data)
|
|
62
|
+
empty_chunks = {} #: Hash[String, untyped]
|
|
63
|
+
symbols = data.fetch("symbols").map do |symbol|
|
|
64
|
+
GrammarSymbol.new(id: symbol.fetch("id"), name: symbol.fetch("name"), kind: symbol.fetch("kind"),
|
|
65
|
+
reserved: symbol.fetch("reserved"), precedence: symbolize(symbol["prec"]),
|
|
66
|
+
location: symbolize(symbol["loc"]))
|
|
67
|
+
end
|
|
68
|
+
productions = data.fetch("productions").map { |production| load_production(production) }
|
|
69
|
+
Grammar.new(class_name: data.fetch("class_name"), superclass: data["superclass"], start: data.fetch("start"),
|
|
70
|
+
expect: data.fetch("expect"), options: symbolize(data.fetch("options")), symbols: symbols,
|
|
71
|
+
productions: productions, user_code: data.fetch("user_code"),
|
|
72
|
+
conversions: data.fetch("conversions"), warnings: symbolize(data.fetch("warnings")),
|
|
73
|
+
user_code_chunks: load_user_code_chunks(data.fetch("user_code_chunks", empty_chunks)))
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# @rbs skip
|
|
77
|
+
def load_automaton(data)
|
|
78
|
+
grammar = load_grammar(data.fetch("grammar"))
|
|
79
|
+
states = data.fetch("states").map { |state| load_state(state, grammar) }
|
|
80
|
+
Automaton.new(grammar: grammar, states: states, conflict_summary: symbolize(data.fetch("conflict_summary")),
|
|
81
|
+
algorithm: data.fetch("algorithm"), grammar_digest: data.fetch("grammar_digest"))
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @rbs skip
|
|
85
|
+
def load_state(state, grammar)
|
|
86
|
+
items = state.fetch("items").map do |item|
|
|
87
|
+
lookaheads = item.fetch("lookaheads").map { |name| grammar.symbol(name).id }
|
|
88
|
+
AutomatonItem.new(production: item.fetch("production"), dot: item.fetch("dot"), lookaheads: lookaheads)
|
|
89
|
+
end
|
|
90
|
+
AutomatonState.new(id: state.fetch("id"), items: items,
|
|
91
|
+
transitions: symbol_keyed(state.fetch("transitions"), grammar),
|
|
92
|
+
actions: symbol_keyed(state.fetch("actions"), grammar, actions: true),
|
|
93
|
+
gotos: symbol_keyed(state.fetch("gotos"), grammar),
|
|
94
|
+
default_action: normalize_action(state["default_action"]),
|
|
95
|
+
conflicts: symbolize(state.fetch("conflicts")))
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @rbs skip
|
|
99
|
+
def symbol_keyed(values, grammar, actions: false)
|
|
100
|
+
values.to_h do |name, value|
|
|
101
|
+
[grammar.symbol(name).id, actions ? normalize_action(value) : value]
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# @rbs skip
|
|
106
|
+
def normalize_action(value)
|
|
107
|
+
return nil unless value
|
|
108
|
+
|
|
109
|
+
action = symbolize(value)
|
|
110
|
+
action[:type] = action[:type].to_sym
|
|
111
|
+
action
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# @rbs skip
|
|
115
|
+
def load_production(production)
|
|
116
|
+
action_data = production["action"]
|
|
117
|
+
action = if action_data
|
|
118
|
+
Action.new(code: action_data.fetch("code"), location: symbolize(action_data["loc"]),
|
|
119
|
+
named_refs: symbolize(action_data.fetch("named_refs")),
|
|
120
|
+
context_length: action_data.fetch("context_length"))
|
|
121
|
+
end
|
|
122
|
+
Production.new(id: production.fetch("id"), lhs: production.fetch("lhs"), rhs: production.fetch("rhs"),
|
|
123
|
+
action: action, precedence_override: production["prec_override"],
|
|
124
|
+
origin: symbolize(production.fetch("origin")))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# @rbs skip
|
|
128
|
+
def load_user_code_chunks(chunks)
|
|
129
|
+
chunks.to_h do |name, values|
|
|
130
|
+
loaded = values.map do |value|
|
|
131
|
+
UserCodeChunk.new(code: value.fetch("code"), location: symbolize(value.fetch("loc")))
|
|
132
|
+
end
|
|
133
|
+
[name, loaded]
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @rbs skip
|
|
138
|
+
def symbolize(value)
|
|
139
|
+
case value
|
|
140
|
+
when Array then value.map { |item| symbolize(item) }
|
|
141
|
+
when Hash then value.to_h { |key, item| [key.to_sym, symbolize(item)] }
|
|
142
|
+
else value
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
module_function :validate_version, :load_grammar, :load_automaton, :load_state, :symbol_keyed,
|
|
146
|
+
:normalize_action, :load_production, :load_user_code_chunks, :symbolize
|
|
147
|
+
|
|
148
|
+
class << self
|
|
149
|
+
private :validate_version, :load_grammar, :load_automaton, :load_state, :symbol_keyed,
|
|
150
|
+
:normalize_action, :load_production, :load_user_code_chunks, :symbolize
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
data/lib/ibex/ir.rb
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "ir/grammar_ir"
|
|
4
|
+
require_relative "ir/automaton_ir"
|
|
5
|
+
require_relative "ir/serialize"
|
|
6
|
+
|
|
7
|
+
module Ibex
|
|
8
|
+
module IR
|
|
9
|
+
# Shared static shapes used across analysis, automaton construction, and code generation.
|
|
10
|
+
# @rbs!
|
|
11
|
+
# type location = { file: String, line: Integer, column: Integer }
|
|
12
|
+
# type precedence = { associativity: Symbol, level: Integer }
|
|
13
|
+
# type named_ref = { name: String, index: Integer }
|
|
14
|
+
# type user_code_chunks = Hash[String, Array[UserCodeChunk]]
|
|
15
|
+
# type grammar_options = { result_var: bool, omit_action_call: bool }
|
|
16
|
+
# type grammar_warning = {
|
|
17
|
+
# type: Symbol,
|
|
18
|
+
# ?symbol: String,
|
|
19
|
+
# ?production: Integer,
|
|
20
|
+
# ?original: Integer,
|
|
21
|
+
# loc: location?
|
|
22
|
+
# }
|
|
23
|
+
# type shift_action = { type: :shift, state: Integer }
|
|
24
|
+
# type reduce_action = { type: :reduce, production: Integer }
|
|
25
|
+
# type accept_action = { type: :accept }
|
|
26
|
+
# type error_action = { type: :error }
|
|
27
|
+
# type parser_action = shift_action | reduce_action | accept_action | error_action
|
|
28
|
+
# type runtime_action = [:shift, Integer] | [:reduce, Integer] | [:accept] | [:error]
|
|
29
|
+
# type conflict_resolution = { by: Symbol, chose: Symbol | Integer, ?associativity: Symbol }
|
|
30
|
+
# type shift_reduce_conflict =
|
|
31
|
+
# { type: :shift_reduce, symbol: String, shift_to: Integer, reduce: Integer, resolution: conflict_resolution }
|
|
32
|
+
# type reduce_reduce_conflict =
|
|
33
|
+
# { type: :reduce_reduce, symbol: String, reductions: Array[Integer], resolution: conflict_resolution }
|
|
34
|
+
# type conflict = shift_reduce_conflict | reduce_reduce_conflict
|
|
35
|
+
# type conflict_summary = {
|
|
36
|
+
# sr: Integer,
|
|
37
|
+
# resolved_sr: Integer,
|
|
38
|
+
# rr: Integer,
|
|
39
|
+
# expected_sr: Integer,
|
|
40
|
+
# expectation_met: bool
|
|
41
|
+
# }
|
|
42
|
+
# type interpretation = Hash[Symbol, untyped]
|
|
43
|
+
# type counterexample = {
|
|
44
|
+
# state: Integer,
|
|
45
|
+
# type: Symbol,
|
|
46
|
+
# symbol_path: Array[String],
|
|
47
|
+
# sentence: Array[String],
|
|
48
|
+
# lookahead_index: Integer,
|
|
49
|
+
# unifying: bool,
|
|
50
|
+
# interpretations: Array[interpretation]
|
|
51
|
+
# }
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "set"
|
|
4
|
+
|
|
5
|
+
module Ibex
|
|
6
|
+
module LALR
|
|
7
|
+
# Builds canonical LR(1) states and merges states with equal LR(0) cores.
|
|
8
|
+
class Builder
|
|
9
|
+
AUGMENTED_PRODUCTION = -1 #: Integer
|
|
10
|
+
|
|
11
|
+
ALGORITHMS = %i[slr lalr lr1].freeze #: Array[Symbol]
|
|
12
|
+
|
|
13
|
+
# @rbs @grammar: IR::Grammar
|
|
14
|
+
# @rbs @algorithm: Symbol
|
|
15
|
+
# @rbs @sets: Analysis::Sets
|
|
16
|
+
# @rbs @productions_by_lhs: Hash[Integer, Array[IR::Production]]
|
|
17
|
+
# @rbs @resolver: ConflictResolver
|
|
18
|
+
|
|
19
|
+
# @rbs (IR::Grammar grammar, ?algorithm: Symbol | String) -> void
|
|
20
|
+
def initialize(grammar, algorithm: :lalr)
|
|
21
|
+
unless ALGORITHMS.include?(algorithm.to_sym)
|
|
22
|
+
raise ArgumentError, "unknown parser algorithm #{algorithm.inspect}"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
@grammar = grammar
|
|
26
|
+
@algorithm = algorithm.to_sym
|
|
27
|
+
@sets = Analysis::Sets.new(grammar)
|
|
28
|
+
@productions_by_lhs = grammar.productions.group_by(&:lhs)
|
|
29
|
+
@resolver = ConflictResolver.new(grammar)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @rbs () -> IR::Automaton
|
|
33
|
+
def build
|
|
34
|
+
canonical_states, canonical_transitions = canonical_collection
|
|
35
|
+
merged_items, merged_transitions = automaton_items(canonical_states, canonical_transitions)
|
|
36
|
+
states = build_states(merged_items, merged_transitions)
|
|
37
|
+
states = DefaultReductions.apply(states, terminal_ids: @grammar.terminals.map(&:id))
|
|
38
|
+
conflicts = states.flat_map(&:conflicts)
|
|
39
|
+
shift_reduce = conflicts.select { |item| item[:type] == :shift_reduce }
|
|
40
|
+
counted_shift_reduce = shift_reduce.count { |item| item.dig(:resolution, :by) == :default_shift }
|
|
41
|
+
summary = { sr: counted_shift_reduce,
|
|
42
|
+
resolved_sr: shift_reduce.length - counted_shift_reduce,
|
|
43
|
+
rr: conflicts.count { |item| item[:type] == :reduce_reduce },
|
|
44
|
+
expected_sr: @grammar.expect,
|
|
45
|
+
expectation_met: counted_shift_reduce == @grammar.expect } #: IR::conflict_summary
|
|
46
|
+
IR::Automaton.new(grammar: @grammar, states: states, conflict_summary: summary,
|
|
47
|
+
algorithm: @algorithm == :lalr ? "lalr1" : @algorithm.to_s)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
# @rbs () -> [Array[item_set], transitions]
|
|
53
|
+
def canonical_collection
|
|
54
|
+
seed = Set[[AUGMENTED_PRODUCTION, 0, 0]] #: item_set
|
|
55
|
+
states = [closure(seed)]
|
|
56
|
+
transitions = [] #: transitions
|
|
57
|
+
indexes = { item_key(states.first) => 0 }
|
|
58
|
+
cursor = 0
|
|
59
|
+
while cursor < states.length
|
|
60
|
+
transitions[cursor] = {}
|
|
61
|
+
next_symbols(states[cursor]).each do |symbol_id|
|
|
62
|
+
target = go_to(states[cursor], symbol_id)
|
|
63
|
+
key = item_key(target)
|
|
64
|
+
target_id = indexes[key] ||= begin
|
|
65
|
+
states << target
|
|
66
|
+
states.length - 1
|
|
67
|
+
end
|
|
68
|
+
transitions[cursor][symbol_id] = target_id
|
|
69
|
+
end
|
|
70
|
+
cursor += 1
|
|
71
|
+
end
|
|
72
|
+
[states, transitions]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# @rbs (item_set seed) -> item_set
|
|
76
|
+
def closure(seed)
|
|
77
|
+
items = seed.dup
|
|
78
|
+
queue = seed.to_a
|
|
79
|
+
until queue.empty?
|
|
80
|
+
production_id, dot, lookahead = queue.shift
|
|
81
|
+
rhs = rhs_for(production_id)
|
|
82
|
+
grammar_symbol = @grammar.symbol_by_id(rhs[dot])
|
|
83
|
+
next unless grammar_symbol&.nonterminal?
|
|
84
|
+
|
|
85
|
+
lookaheads = suffix_lookaheads(rhs.drop(dot + 1), lookahead)
|
|
86
|
+
@productions_by_lhs.fetch(grammar_symbol.id, Array.new(0)).each do |production|
|
|
87
|
+
lookaheads.each do |token_id|
|
|
88
|
+
item = [production.id, 0, token_id] #: lr_item
|
|
89
|
+
enqueue_item(items, queue, item)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
items
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @rbs (Array[Integer] suffix, Integer inherited) -> Array[Integer]
|
|
97
|
+
def suffix_lookaheads(suffix, inherited)
|
|
98
|
+
bits = @sets.first_of_sequence(suffix)
|
|
99
|
+
bits |= (1 << inherited) if @sets.sequence_nullable?(suffix)
|
|
100
|
+
@grammar.terminals.filter_map { |terminal| terminal.id if bits.anybits?(1 << terminal.id) }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# @rbs (item_set items, Array[lr_item] queue, lr_item item) -> void
|
|
104
|
+
def enqueue_item(items, queue, item)
|
|
105
|
+
return if items.include?(item)
|
|
106
|
+
|
|
107
|
+
items << item
|
|
108
|
+
queue << item
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# @rbs (item_set items) -> Array[Integer]
|
|
112
|
+
def next_symbols(items)
|
|
113
|
+
items.filter_map { |production_id, dot, _lookahead| rhs_for(production_id)[dot] }.uniq.sort
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# @rbs (item_set items, Integer symbol_id) -> item_set
|
|
117
|
+
def go_to(items, symbol_id)
|
|
118
|
+
moved = items.filter_map do |production_id, dot, lookahead|
|
|
119
|
+
next unless rhs_for(production_id)[dot] == symbol_id
|
|
120
|
+
|
|
121
|
+
[production_id, dot + 1, lookahead] #: lr_item
|
|
122
|
+
end
|
|
123
|
+
closure(Set.new(moved))
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# @rbs (Array[item_set] states, transitions transitions) -> [Array[packed_items], transitions]
|
|
127
|
+
def merge_lalr(states, transitions)
|
|
128
|
+
groups = {} #: Hash[Array[item_core], Integer]
|
|
129
|
+
state_groups = states.map do |items|
|
|
130
|
+
core = core_key(items)
|
|
131
|
+
groups[core] ||= groups.length
|
|
132
|
+
end
|
|
133
|
+
merged = Array.new(groups.length) do
|
|
134
|
+
Hash.new { |hash, key| hash[key] = Set.new } #: packed_items
|
|
135
|
+
end
|
|
136
|
+
states.each_with_index do |items, state_id|
|
|
137
|
+
items.each { |production, dot, lookahead| merged[state_groups[state_id]][[production, dot]] << lookahead }
|
|
138
|
+
end
|
|
139
|
+
merged_transitions = Array.new(groups.length) do
|
|
140
|
+
{} #: Hash[Integer, Integer]
|
|
141
|
+
end
|
|
142
|
+
transitions.each_with_index do |edges, state_id|
|
|
143
|
+
edges.each { |symbol, target| merged_transitions[state_groups[state_id]][symbol] = state_groups[target] }
|
|
144
|
+
end
|
|
145
|
+
[merged, merged_transitions]
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# @rbs (Array[item_set] canonical_states, transitions canonical_transitions) -> [Array[packed_items], transitions]
|
|
149
|
+
def automaton_items(canonical_states, canonical_transitions)
|
|
150
|
+
return [pack_canonical_items(canonical_states), canonical_transitions] if @algorithm == :lr1
|
|
151
|
+
|
|
152
|
+
items, transitions = merge_lalr(canonical_states, canonical_transitions)
|
|
153
|
+
apply_slr_lookaheads(items) if @algorithm == :slr
|
|
154
|
+
[items, transitions]
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# @rbs (Array[item_set] states) -> Array[packed_items]
|
|
158
|
+
def pack_canonical_items(states)
|
|
159
|
+
states.map do |items|
|
|
160
|
+
packed = Hash.new { |hash, key| hash[key] = Set.new } #: packed_items
|
|
161
|
+
items.each { |production, dot, lookahead| packed[[production, dot]] << lookahead }
|
|
162
|
+
packed
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# @rbs (Array[packed_items] states) -> void
|
|
167
|
+
def apply_slr_lookaheads(states)
|
|
168
|
+
states.each do |items|
|
|
169
|
+
items.each do |(production_id, dot), lookaheads|
|
|
170
|
+
next unless dot == rhs_for(production_id).length
|
|
171
|
+
|
|
172
|
+
lookaheads.replace(slr_lookaheads(production_id))
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# @rbs (Integer production_id) -> Array[Integer]
|
|
178
|
+
def slr_lookaheads(production_id)
|
|
179
|
+
return [0] if production_id == AUGMENTED_PRODUCTION
|
|
180
|
+
|
|
181
|
+
lhs = @grammar.productions.fetch(production_id).lhs
|
|
182
|
+
bits = @sets.follow_bits.fetch(lhs)
|
|
183
|
+
@grammar.terminals.filter_map { |terminal| terminal.id if bits.anybits?(1 << terminal.id) }
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# @rbs (Array[packed_items] merged_items, transitions transitions) -> Array[IR::AutomatonState]
|
|
187
|
+
def build_states(merged_items, transitions)
|
|
188
|
+
merged_items.each_with_index.map do |item_map, state_id|
|
|
189
|
+
items = item_map.sort.map do |(production, dot), lookaheads|
|
|
190
|
+
IR::AutomatonItem.new(production: production, dot: dot, lookaheads: lookaheads.to_a)
|
|
191
|
+
end
|
|
192
|
+
build_state(state_id, items, transitions[state_id])
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# @rbs (Integer state_id, Array[IR::AutomatonItem] items, Hash[Integer, Integer] transitions) -> IR::AutomatonState
|
|
197
|
+
def build_state(state_id, items, transitions)
|
|
198
|
+
candidates = Hash.new { |hash, key| hash[key] = Array.new(0) } #: Hash[Integer, Array[IR::parser_action]]
|
|
199
|
+
gotos = {} #: Hash[Integer, Integer]
|
|
200
|
+
transitions.each do |symbol_id, target|
|
|
201
|
+
grammar_symbol = @grammar.symbol_by_id(symbol_id)
|
|
202
|
+
raise Ibex::Error, "missing grammar symbol id #{symbol_id}" unless grammar_symbol
|
|
203
|
+
|
|
204
|
+
if grammar_symbol.terminal?
|
|
205
|
+
candidates[symbol_id] << { type: :shift, state: target }
|
|
206
|
+
else
|
|
207
|
+
gotos[symbol_id] = target
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
add_completed_actions(items, candidates)
|
|
211
|
+
actions, conflicts = resolve_actions(candidates)
|
|
212
|
+
IR::AutomatonState.new(id: state_id, items: items, transitions: transitions, actions: actions,
|
|
213
|
+
gotos: gotos, conflicts: conflicts)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# @rbs (Array[IR::AutomatonItem] items, Hash[Integer, Array[IR::parser_action]] candidates) -> void
|
|
217
|
+
def add_completed_actions(items, candidates)
|
|
218
|
+
items.each do |item|
|
|
219
|
+
next unless item.dot == rhs_for(item.production).length
|
|
220
|
+
|
|
221
|
+
item.lookaheads.each do |lookahead|
|
|
222
|
+
action = if item.production == AUGMENTED_PRODUCTION
|
|
223
|
+
{ type: :accept } #: IR::accept_action
|
|
224
|
+
else
|
|
225
|
+
{ type: :reduce, production: item.production } #: IR::reduce_action
|
|
226
|
+
end
|
|
227
|
+
candidates[lookahead] << action
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# @rbs (Hash[Integer, Array[IR::parser_action]] candidates) ->
|
|
233
|
+
# [Hash[Integer, IR::parser_action], Array[IR::conflict]]
|
|
234
|
+
def resolve_actions(candidates)
|
|
235
|
+
actions = {} #: Hash[Integer, IR::parser_action]
|
|
236
|
+
conflicts = [] #: Array[IR::conflict]
|
|
237
|
+
candidates.keys.sort.each do |token_id|
|
|
238
|
+
action, found = @resolver.resolve(token_id, candidates[token_id])
|
|
239
|
+
raise Ibex::Error, "empty parser action candidates" unless action
|
|
240
|
+
|
|
241
|
+
actions[token_id] = action
|
|
242
|
+
conflicts.concat(found)
|
|
243
|
+
end
|
|
244
|
+
[actions, conflicts]
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# @rbs (Integer production_id) -> Array[Integer]
|
|
248
|
+
def rhs_for(production_id)
|
|
249
|
+
if production_id == AUGMENTED_PRODUCTION
|
|
250
|
+
start = @grammar.symbol(@grammar.start) || raise(Ibex::Error, "missing start symbol")
|
|
251
|
+
return [start.id]
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
@grammar.productions.fetch(production_id).rhs
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# @rbs (item_set items) -> Array[item_core]
|
|
258
|
+
def core_key(items)
|
|
259
|
+
items.map do |production, dot, _lookahead|
|
|
260
|
+
[production, dot] #: item_core
|
|
261
|
+
end.uniq.sort
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# @rbs (item_set items) -> Array[lr_item]
|
|
265
|
+
def item_key(items)
|
|
266
|
+
items.to_a.sort
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|