ibex 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. checksums.yaml +7 -0
  2. data/.rubocop.yml +43 -0
  3. data/CHANGELOG.md +30 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +194 -0
  6. data/Rakefile +25 -0
  7. data/Steepfile +10 -0
  8. data/docs/architecture.md +99 -0
  9. data/docs/compat-notes.md +37 -0
  10. data/docs/grammar-reference.md +127 -0
  11. data/docs/lexer-coverage.md +14 -0
  12. data/docs/phase10-extensions.md +27 -0
  13. data/docs/racc-migration.md +47 -0
  14. data/exe/ibex +6 -0
  15. data/gemfiles/Gemfile +7 -0
  16. data/gemfiles/Gemfile.lock +98 -0
  17. data/lib/ibex/analysis/sets.rb +194 -0
  18. data/lib/ibex/analysis.rb +8 -0
  19. data/lib/ibex/cli/counterexample_options.rb +38 -0
  20. data/lib/ibex/cli/outputs.rb +88 -0
  21. data/lib/ibex/cli.rb +333 -0
  22. data/lib/ibex/codegen/dot.rb +52 -0
  23. data/lib/ibex/codegen/html.rb +110 -0
  24. data/lib/ibex/codegen/rbs.rb +54 -0
  25. data/lib/ibex/codegen/report.rb +149 -0
  26. data/lib/ibex/codegen/ruby.rb +235 -0
  27. data/lib/ibex/codegen/symbol_labels.rb +18 -0
  28. data/lib/ibex/error.rb +7 -0
  29. data/lib/ibex/frontend/action_scanner.rb +266 -0
  30. data/lib/ibex/frontend/ast.rb +146 -0
  31. data/lib/ibex/frontend/bootstrap_parser.rb +163 -0
  32. data/lib/ibex/frontend/dsl.rb +196 -0
  33. data/lib/ibex/frontend/generated_parser.rb +434 -0
  34. data/lib/ibex/frontend/generated_parser_base.rb +273 -0
  35. data/lib/ibex/frontend/grammar.y +156 -0
  36. data/lib/ibex/frontend/lexer.rb +165 -0
  37. data/lib/ibex/frontend/parser/declarations.rb +163 -0
  38. data/lib/ibex/frontend/parser/rules.rb +154 -0
  39. data/lib/ibex/frontend/parser.rb +23 -0
  40. data/lib/ibex/frontend/regenerator.rb +41 -0
  41. data/lib/ibex/frontend/source_cursor.rb +94 -0
  42. data/lib/ibex/frontend/token_adapter/declaration_state.rb +248 -0
  43. data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +37 -0
  44. data/lib/ibex/frontend/token_adapter/rule_state.rb +162 -0
  45. data/lib/ibex/frontend/token_adapter.rb +108 -0
  46. data/lib/ibex/frontend.rb +28 -0
  47. data/lib/ibex/ir/automaton_ir.rb +109 -0
  48. data/lib/ibex/ir/grammar_ir.rb +192 -0
  49. data/lib/ibex/ir/serialize.rb +154 -0
  50. data/lib/ibex/ir.rb +53 -0
  51. data/lib/ibex/lalr/builder.rb +270 -0
  52. data/lib/ibex/lalr/conflict.rb +120 -0
  53. data/lib/ibex/lalr/conflict_search.rb +332 -0
  54. data/lib/ibex/lalr/conflict_search_limits.rb +39 -0
  55. data/lib/ibex/lalr/counterexample.rb +138 -0
  56. data/lib/ibex/lalr/default_reductions.rb +67 -0
  57. data/lib/ibex/lalr.rb +27 -0
  58. data/lib/ibex/normalize/declarations.rb +60 -0
  59. data/lib/ibex/normalize/diagnostics.rb +99 -0
  60. data/lib/ibex/normalize/expander.rb +218 -0
  61. data/lib/ibex/normalize/expression.rb +54 -0
  62. data/lib/ibex/normalize.rb +166 -0
  63. data/lib/ibex/runtime/parser.rb +360 -0
  64. data/lib/ibex/runtime.rb +8 -0
  65. data/lib/ibex/tables.rb +125 -0
  66. data/lib/ibex/version.rb +6 -0
  67. data/lib/ibex.rb +23 -0
  68. data/sig/ibex/analysis/sets.rbs +72 -0
  69. data/sig/ibex/analysis.rbs +6 -0
  70. data/sig/ibex/cli/counterexample_options.rbs +16 -0
  71. data/sig/ibex/cli/outputs.rbs +31 -0
  72. data/sig/ibex/cli.rbs +108 -0
  73. data/sig/ibex/codegen/dot.rbs +19 -0
  74. data/sig/ibex/codegen/html.rbs +35 -0
  75. data/sig/ibex/codegen/rbs.rbs +31 -0
  76. data/sig/ibex/codegen/report.rbs +39 -0
  77. data/sig/ibex/codegen/ruby.rbs +92 -0
  78. data/sig/ibex/codegen/symbol_labels.rbs +11 -0
  79. data/sig/ibex/error.rbs +7 -0
  80. data/sig/ibex/frontend/action_scanner.rbs +74 -0
  81. data/sig/ibex/frontend/ast.rbs +223 -0
  82. data/sig/ibex/frontend/bootstrap_parser.rbs +80 -0
  83. data/sig/ibex/frontend/dsl.rbs +124 -0
  84. data/sig/ibex/frontend/generated_parser.rbs +163 -0
  85. data/sig/ibex/frontend/generated_parser_base.rbs +114 -0
  86. data/sig/ibex/frontend/lexer.rbs +61 -0
  87. data/sig/ibex/frontend/parser/declarations.rbs +56 -0
  88. data/sig/ibex/frontend/parser/rules.rbs +48 -0
  89. data/sig/ibex/frontend/parser.rbs +16 -0
  90. data/sig/ibex/frontend/regenerator.rbs +16 -0
  91. data/sig/ibex/frontend/source_cursor.rbs +73 -0
  92. data/sig/ibex/frontend/token_adapter/declaration_state.rbs +97 -0
  93. data/sig/ibex/frontend/token_adapter/delimiter_tracker.rbs +25 -0
  94. data/sig/ibex/frontend/token_adapter/rule_state.rbs +77 -0
  95. data/sig/ibex/frontend/token_adapter.rbs +67 -0
  96. data/sig/ibex/frontend.rbs +15 -0
  97. data/sig/ibex/ir/automaton_ir.rbs +77 -0
  98. data/sig/ibex/ir/grammar_ir.rbs +143 -0
  99. data/sig/ibex/ir/serialize.rbs +50 -0
  100. data/sig/ibex/ir.rbs +43 -0
  101. data/sig/ibex/lalr/builder.rbs +85 -0
  102. data/sig/ibex/lalr/conflict.rbs +47 -0
  103. data/sig/ibex/lalr/conflict_search.rbs +131 -0
  104. data/sig/ibex/lalr/conflict_search_limits.rbs +29 -0
  105. data/sig/ibex/lalr/counterexample.rbs +53 -0
  106. data/sig/ibex/lalr/default_reductions.rbs +20 -0
  107. data/sig/ibex/lalr.rbs +23 -0
  108. data/sig/ibex/normalize/declarations.rbs +20 -0
  109. data/sig/ibex/normalize/diagnostics.rbs +32 -0
  110. data/sig/ibex/normalize/expander.rbs +66 -0
  111. data/sig/ibex/normalize/expression.rbs +21 -0
  112. data/sig/ibex/normalize.rbs +96 -0
  113. data/sig/ibex/runtime/parser.rbs +167 -0
  114. data/sig/ibex/runtime.rbs +6 -0
  115. data/sig/ibex/tables.rbs +50 -0
  116. data/sig/ibex/version.rbs +6 -0
  117. data/sig/ibex.rbs +6 -0
  118. metadata +161 -0
@@ -0,0 +1,192 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ibex
4
+ module IR
5
+ SCHEMA_VERSION = 1
6
+
7
+ # @rbs (untyped value) -> untyped
8
+ def deep_freeze(value)
9
+ case value
10
+ when Array then value.each { |item| deep_freeze(item) }
11
+ when Hash
12
+ value.each do |key, item|
13
+ deep_freeze(key)
14
+ deep_freeze(item)
15
+ end
16
+ end
17
+ value.freeze
18
+ end
19
+ module_function :deep_freeze
20
+
21
+ # An interned terminal or nonterminal.
22
+ class GrammarSymbol
23
+ attr_reader :id #: Integer
24
+ attr_reader :name #: String
25
+ attr_reader :kind #: Symbol
26
+ attr_reader :reserved #: bool
27
+ attr_reader :precedence #: precedence?
28
+ attr_reader :location #: location?
29
+
30
+ # @rbs (id: Integer, name: String, kind: Symbol, ?reserved: bool, ?precedence: precedence?,
31
+ # ?location: location?) -> void
32
+ def initialize(id:, name:, kind:, reserved: false, precedence: nil, location: nil)
33
+ @id = id
34
+ @name = name.freeze
35
+ @kind = kind.to_sym
36
+ @reserved = reserved
37
+ @precedence = IR.deep_freeze(precedence)
38
+ @location = IR.deep_freeze(location)
39
+ freeze
40
+ end
41
+
42
+ # @rbs () -> bool
43
+ def terminal? = @kind == :terminal
44
+ # @rbs () -> bool
45
+ def nonterminal? = @kind == :nonterminal
46
+
47
+ # @rbs () -> Hash[Symbol, untyped]
48
+ def to_h
49
+ { id: @id, name: @name, kind: @kind, reserved: @reserved, prec: @precedence, loc: @location }
50
+ end
51
+ end
52
+
53
+ # Opaque Ruby semantic action metadata.
54
+ class Action
55
+ attr_reader :code #: String
56
+ attr_reader :location #: location
57
+ attr_reader :named_refs #: Array[named_ref]
58
+ attr_reader :context_length #: Integer
59
+
60
+ # @rbs (code: String, location: location, ?named_refs: Array[named_ref], ?context_length: Integer) -> void
61
+ def initialize(code:, location:, named_refs: [], context_length: 0)
62
+ @code = code.freeze
63
+ @location = IR.deep_freeze(location)
64
+ @named_refs = IR.deep_freeze(named_refs)
65
+ @context_length = context_length
66
+ freeze
67
+ end
68
+
69
+ # @rbs () -> Hash[Symbol, untyped]
70
+ def to_h
71
+ { code: @code, loc: @location, named_refs: @named_refs, context_length: @context_length }
72
+ end
73
+ end
74
+
75
+ # A normalized BNF production using symbol ids.
76
+ class Production
77
+ attr_reader :id #: Integer
78
+ attr_reader :lhs #: Integer
79
+ attr_reader :rhs #: Array[Integer]
80
+ attr_reader :action #: Action?
81
+ attr_reader :precedence_override #: Integer?
82
+ attr_reader :origin #: Hash[Symbol, untyped]
83
+
84
+ # @rbs (id: Integer, lhs: Integer, rhs: Array[Integer], action: Action?, precedence_override: Integer?,
85
+ # origin: Hash[Symbol, untyped]) -> void
86
+ def initialize(id:, lhs:, rhs:, action:, precedence_override:, origin:)
87
+ @id = id
88
+ @lhs = lhs
89
+ @rhs = rhs.freeze
90
+ @action = action
91
+ @precedence_override = precedence_override
92
+ @origin = IR.deep_freeze(origin)
93
+ freeze
94
+ end
95
+
96
+ # @rbs () -> Hash[Symbol, untyped]
97
+ def to_h
98
+ { id: @id, lhs: @lhs, rhs: @rhs, action: @action&.to_h, prec_override: @precedence_override,
99
+ origin: @origin }
100
+ end
101
+ end
102
+
103
+ # One opaque user-code block and the location of its first code line.
104
+ class UserCodeChunk
105
+ attr_reader :code #: String
106
+ attr_reader :location #: location
107
+
108
+ # @rbs (code: String, location: location) -> void
109
+ def initialize(code:, location:)
110
+ @code = code.freeze
111
+ @location = IR.deep_freeze(location)
112
+ freeze
113
+ end
114
+
115
+ # @rbs () -> Hash[Symbol, untyped]
116
+ def to_h
117
+ { code: @code, loc: @location }
118
+ end
119
+ end
120
+
121
+ # Immutable normalized grammar exchanged between pipeline stages.
122
+ class Grammar
123
+ attr_reader :class_name #: String
124
+ attr_reader :superclass #: String?
125
+ attr_reader :start #: String
126
+ attr_reader :expect #: Integer
127
+ attr_reader :options #: grammar_options
128
+ attr_reader :symbols #: Array[GrammarSymbol]
129
+ attr_reader :productions #: Array[Production]
130
+ attr_reader :user_code #: Hash[String, String]
131
+ attr_reader :user_code_chunks #: user_code_chunks
132
+ attr_reader :conversions #: Hash[String, String]
133
+ attr_reader :warnings #: Array[grammar_warning]
134
+ attr_reader :schema_version #: Integer
135
+
136
+ # @rbs (class_name: String, superclass: String?, start: String, expect: Integer, options: grammar_options,
137
+ # symbols: Array[GrammarSymbol], productions: Array[Production], user_code: Hash[String, String],
138
+ # conversions: Hash[String, String], warnings: Array[grammar_warning], ?user_code_chunks: user_code_chunks?,
139
+ # ?schema_version: Integer) -> void
140
+ def initialize(class_name:, superclass:, start:, expect:, options:, symbols:, productions:, user_code:,
141
+ conversions:, warnings:, user_code_chunks: nil, schema_version: SCHEMA_VERSION)
142
+ @class_name = class_name.freeze
143
+ @superclass = superclass&.freeze
144
+ @start = start.freeze
145
+ @expect = expect
146
+ @options = IR.deep_freeze(options)
147
+ @symbols = symbols.freeze
148
+ @productions = productions.freeze
149
+ @user_code = IR.deep_freeze(user_code)
150
+ @user_code_chunks = IR.deep_freeze(user_code_chunks || {})
151
+ validate_user_code_chunks
152
+ @conversions = IR.deep_freeze(conversions)
153
+ @warnings = IR.deep_freeze(warnings)
154
+ @schema_version = schema_version
155
+ @symbols_by_name = @symbols.to_h { |symbol| [symbol.name, symbol] }.freeze
156
+ @symbols_by_id = @symbols.to_h { |symbol| [symbol.id, symbol] }.freeze
157
+ freeze
158
+ end
159
+
160
+ # @rbs (String name) -> GrammarSymbol?
161
+ def symbol(name) = @symbols_by_name[name]
162
+ # @rbs (Integer? id) -> GrammarSymbol?
163
+ def symbol_by_id(id) = @symbols_by_id[id]
164
+ # @rbs () -> Array[GrammarSymbol]
165
+ def terminals = @symbols.select(&:terminal?)
166
+ # @rbs () -> Array[GrammarSymbol]
167
+ def nonterminals = @symbols.select(&:nonterminal?)
168
+
169
+ # @rbs () -> Hash[Symbol, untyped]
170
+ def to_h
171
+ value = { ibex_ir: "grammar", schema_version: @schema_version, class_name: @class_name, superclass: @superclass,
172
+ start: @start, expect: @expect, options: @options, symbols: @symbols.map(&:to_h),
173
+ productions: @productions.map(&:to_h), user_code: @user_code, conversions: @conversions,
174
+ warnings: @warnings } #: Hash[Symbol, untyped]
175
+ value[:user_code_chunks] = @user_code_chunks.transform_values { |chunks| chunks.map(&:to_h) } \
176
+ unless @user_code_chunks.empty?
177
+ value
178
+ end
179
+
180
+ private
181
+
182
+ # @rbs () -> void
183
+ def validate_user_code_chunks
184
+ @user_code_chunks.each do |name, chunks|
185
+ next if chunks.map(&:code).join == @user_code.fetch(name, "")
186
+
187
+ raise Ibex::Error, "(ir):1:1: user-code chunks do not match the concatenated #{name} section"
188
+ end
189
+ end
190
+ end
191
+ end
192
+ end
@@ -0,0 +1,154 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Ibex
6
+ module IR
7
+ # Stable JSON serialization for versioned pipeline IR.
8
+ module Serialize
9
+ # @rbs!
10
+ # private def validate_version: (untyped data) -> untyped
11
+ # private def self.validate_version: (untyped data) -> untyped
12
+ # private def load_grammar: (untyped data) -> untyped
13
+ # private def self.load_grammar: (untyped data) -> untyped
14
+ # private def load_automaton: (untyped data) -> untyped
15
+ # private def self.load_automaton: (untyped data) -> untyped
16
+ # private def load_state: (untyped state, untyped grammar) -> untyped
17
+ # private def self.load_state: (untyped state, untyped grammar) -> untyped
18
+ # private def symbol_keyed: (untyped values, untyped grammar, ?actions: untyped) -> untyped
19
+ # private def self.symbol_keyed: (untyped values, untyped grammar, ?actions: untyped) -> untyped
20
+ # private def normalize_action: (untyped value) -> untyped
21
+ # private def self.normalize_action: (untyped value) -> untyped
22
+ # private def load_production: (untyped production) -> untyped
23
+ # private def self.load_production: (untyped production) -> untyped
24
+ # private def load_user_code_chunks: (untyped chunks) -> untyped
25
+ # private def self.load_user_code_chunks: (untyped chunks) -> untyped
26
+ # private def symbolize: (untyped value) -> untyped
27
+ # private def self.symbolize: (untyped value) -> untyped
28
+
29
+ # @rbs (Grammar | Automaton value) -> String
30
+ def dump(value)
31
+ "#{JSON.pretty_generate(value.to_h)}\n"
32
+ end
33
+ module_function :dump
34
+
35
+ # @rbs (String source) -> (Grammar | Automaton)
36
+ def load(source)
37
+ data = JSON.parse(source)
38
+ type = data.fetch("ibex_ir") { raise Ibex::Error, "(ir):1:1: missing ibex_ir discriminator" }
39
+ validate_version(data)
40
+ return load_grammar(data) if type == "grammar"
41
+ return load_automaton(data) if type == "automaton"
42
+
43
+ raise Ibex::Error, "(ir):1:1: unsupported IR type #{type.inspect}"
44
+ rescue JSON::ParserError => e
45
+ raise Ibex::Error, "(ir):1:1: invalid JSON: #{e.message}"
46
+ end
47
+ module_function :load
48
+
49
+ # @rbs skip
50
+ private
51
+
52
+ # @rbs skip
53
+ def validate_version(data)
54
+ version = data["schema_version"]
55
+ return if version == SCHEMA_VERSION
56
+
57
+ raise Ibex::Error, "(ir):1:1: unsupported schema_version #{version.inspect}; expected #{SCHEMA_VERSION}"
58
+ end
59
+
60
+ # @rbs skip
61
+ def load_grammar(data)
62
+ empty_chunks = {} #: Hash[String, untyped]
63
+ symbols = data.fetch("symbols").map do |symbol|
64
+ GrammarSymbol.new(id: symbol.fetch("id"), name: symbol.fetch("name"), kind: symbol.fetch("kind"),
65
+ reserved: symbol.fetch("reserved"), precedence: symbolize(symbol["prec"]),
66
+ location: symbolize(symbol["loc"]))
67
+ end
68
+ productions = data.fetch("productions").map { |production| load_production(production) }
69
+ Grammar.new(class_name: data.fetch("class_name"), superclass: data["superclass"], start: data.fetch("start"),
70
+ expect: data.fetch("expect"), options: symbolize(data.fetch("options")), symbols: symbols,
71
+ productions: productions, user_code: data.fetch("user_code"),
72
+ conversions: data.fetch("conversions"), warnings: symbolize(data.fetch("warnings")),
73
+ user_code_chunks: load_user_code_chunks(data.fetch("user_code_chunks", empty_chunks)))
74
+ end
75
+
76
+ # @rbs skip
77
+ def load_automaton(data)
78
+ grammar = load_grammar(data.fetch("grammar"))
79
+ states = data.fetch("states").map { |state| load_state(state, grammar) }
80
+ Automaton.new(grammar: grammar, states: states, conflict_summary: symbolize(data.fetch("conflict_summary")),
81
+ algorithm: data.fetch("algorithm"), grammar_digest: data.fetch("grammar_digest"))
82
+ end
83
+
84
+ # @rbs skip
85
+ def load_state(state, grammar)
86
+ items = state.fetch("items").map do |item|
87
+ lookaheads = item.fetch("lookaheads").map { |name| grammar.symbol(name).id }
88
+ AutomatonItem.new(production: item.fetch("production"), dot: item.fetch("dot"), lookaheads: lookaheads)
89
+ end
90
+ AutomatonState.new(id: state.fetch("id"), items: items,
91
+ transitions: symbol_keyed(state.fetch("transitions"), grammar),
92
+ actions: symbol_keyed(state.fetch("actions"), grammar, actions: true),
93
+ gotos: symbol_keyed(state.fetch("gotos"), grammar),
94
+ default_action: normalize_action(state["default_action"]),
95
+ conflicts: symbolize(state.fetch("conflicts")))
96
+ end
97
+
98
+ # @rbs skip
99
+ def symbol_keyed(values, grammar, actions: false)
100
+ values.to_h do |name, value|
101
+ [grammar.symbol(name).id, actions ? normalize_action(value) : value]
102
+ end
103
+ end
104
+
105
+ # @rbs skip
106
+ def normalize_action(value)
107
+ return nil unless value
108
+
109
+ action = symbolize(value)
110
+ action[:type] = action[:type].to_sym
111
+ action
112
+ end
113
+
114
+ # @rbs skip
115
+ def load_production(production)
116
+ action_data = production["action"]
117
+ action = if action_data
118
+ Action.new(code: action_data.fetch("code"), location: symbolize(action_data["loc"]),
119
+ named_refs: symbolize(action_data.fetch("named_refs")),
120
+ context_length: action_data.fetch("context_length"))
121
+ end
122
+ Production.new(id: production.fetch("id"), lhs: production.fetch("lhs"), rhs: production.fetch("rhs"),
123
+ action: action, precedence_override: production["prec_override"],
124
+ origin: symbolize(production.fetch("origin")))
125
+ end
126
+
127
+ # @rbs skip
128
+ def load_user_code_chunks(chunks)
129
+ chunks.to_h do |name, values|
130
+ loaded = values.map do |value|
131
+ UserCodeChunk.new(code: value.fetch("code"), location: symbolize(value.fetch("loc")))
132
+ end
133
+ [name, loaded]
134
+ end
135
+ end
136
+
137
+ # @rbs skip
138
+ def symbolize(value)
139
+ case value
140
+ when Array then value.map { |item| symbolize(item) }
141
+ when Hash then value.to_h { |key, item| [key.to_sym, symbolize(item)] }
142
+ else value
143
+ end
144
+ end
145
+ module_function :validate_version, :load_grammar, :load_automaton, :load_state, :symbol_keyed,
146
+ :normalize_action, :load_production, :load_user_code_chunks, :symbolize
147
+
148
+ class << self
149
+ private :validate_version, :load_grammar, :load_automaton, :load_state, :symbol_keyed,
150
+ :normalize_action, :load_production, :load_user_code_chunks, :symbolize
151
+ end
152
+ end
153
+ end
154
+ end
data/lib/ibex/ir.rb ADDED
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "ir/grammar_ir"
4
+ require_relative "ir/automaton_ir"
5
+ require_relative "ir/serialize"
6
+
7
+ module Ibex
8
+ module IR
9
+ # Shared static shapes used across analysis, automaton construction, and code generation.
10
+ # @rbs!
11
+ # type location = { file: String, line: Integer, column: Integer }
12
+ # type precedence = { associativity: Symbol, level: Integer }
13
+ # type named_ref = { name: String, index: Integer }
14
+ # type user_code_chunks = Hash[String, Array[UserCodeChunk]]
15
+ # type grammar_options = { result_var: bool, omit_action_call: bool }
16
+ # type grammar_warning = {
17
+ # type: Symbol,
18
+ # ?symbol: String,
19
+ # ?production: Integer,
20
+ # ?original: Integer,
21
+ # loc: location?
22
+ # }
23
+ # type shift_action = { type: :shift, state: Integer }
24
+ # type reduce_action = { type: :reduce, production: Integer }
25
+ # type accept_action = { type: :accept }
26
+ # type error_action = { type: :error }
27
+ # type parser_action = shift_action | reduce_action | accept_action | error_action
28
+ # type runtime_action = [:shift, Integer] | [:reduce, Integer] | [:accept] | [:error]
29
+ # type conflict_resolution = { by: Symbol, chose: Symbol | Integer, ?associativity: Symbol }
30
+ # type shift_reduce_conflict =
31
+ # { type: :shift_reduce, symbol: String, shift_to: Integer, reduce: Integer, resolution: conflict_resolution }
32
+ # type reduce_reduce_conflict =
33
+ # { type: :reduce_reduce, symbol: String, reductions: Array[Integer], resolution: conflict_resolution }
34
+ # type conflict = shift_reduce_conflict | reduce_reduce_conflict
35
+ # type conflict_summary = {
36
+ # sr: Integer,
37
+ # resolved_sr: Integer,
38
+ # rr: Integer,
39
+ # expected_sr: Integer,
40
+ # expectation_met: bool
41
+ # }
42
+ # type interpretation = Hash[Symbol, untyped]
43
+ # type counterexample = {
44
+ # state: Integer,
45
+ # type: Symbol,
46
+ # symbol_path: Array[String],
47
+ # sentence: Array[String],
48
+ # lookahead_index: Integer,
49
+ # unifying: bool,
50
+ # interpretations: Array[interpretation]
51
+ # }
52
+ end
53
+ end
@@ -0,0 +1,270 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+
5
+ module Ibex
6
+ module LALR
7
+ # Builds canonical LR(1) states and merges states with equal LR(0) cores.
8
+ class Builder
9
+ AUGMENTED_PRODUCTION = -1 #: Integer
10
+
11
+ ALGORITHMS = %i[slr lalr lr1].freeze #: Array[Symbol]
12
+
13
+ # @rbs @grammar: IR::Grammar
14
+ # @rbs @algorithm: Symbol
15
+ # @rbs @sets: Analysis::Sets
16
+ # @rbs @productions_by_lhs: Hash[Integer, Array[IR::Production]]
17
+ # @rbs @resolver: ConflictResolver
18
+
19
+ # @rbs (IR::Grammar grammar, ?algorithm: Symbol | String) -> void
20
+ def initialize(grammar, algorithm: :lalr)
21
+ unless ALGORITHMS.include?(algorithm.to_sym)
22
+ raise ArgumentError, "unknown parser algorithm #{algorithm.inspect}"
23
+ end
24
+
25
+ @grammar = grammar
26
+ @algorithm = algorithm.to_sym
27
+ @sets = Analysis::Sets.new(grammar)
28
+ @productions_by_lhs = grammar.productions.group_by(&:lhs)
29
+ @resolver = ConflictResolver.new(grammar)
30
+ end
31
+
32
+ # @rbs () -> IR::Automaton
33
+ def build
34
+ canonical_states, canonical_transitions = canonical_collection
35
+ merged_items, merged_transitions = automaton_items(canonical_states, canonical_transitions)
36
+ states = build_states(merged_items, merged_transitions)
37
+ states = DefaultReductions.apply(states, terminal_ids: @grammar.terminals.map(&:id))
38
+ conflicts = states.flat_map(&:conflicts)
39
+ shift_reduce = conflicts.select { |item| item[:type] == :shift_reduce }
40
+ counted_shift_reduce = shift_reduce.count { |item| item.dig(:resolution, :by) == :default_shift }
41
+ summary = { sr: counted_shift_reduce,
42
+ resolved_sr: shift_reduce.length - counted_shift_reduce,
43
+ rr: conflicts.count { |item| item[:type] == :reduce_reduce },
44
+ expected_sr: @grammar.expect,
45
+ expectation_met: counted_shift_reduce == @grammar.expect } #: IR::conflict_summary
46
+ IR::Automaton.new(grammar: @grammar, states: states, conflict_summary: summary,
47
+ algorithm: @algorithm == :lalr ? "lalr1" : @algorithm.to_s)
48
+ end
49
+
50
+ private
51
+
52
+ # @rbs () -> [Array[item_set], transitions]
53
+ def canonical_collection
54
+ seed = Set[[AUGMENTED_PRODUCTION, 0, 0]] #: item_set
55
+ states = [closure(seed)]
56
+ transitions = [] #: transitions
57
+ indexes = { item_key(states.first) => 0 }
58
+ cursor = 0
59
+ while cursor < states.length
60
+ transitions[cursor] = {}
61
+ next_symbols(states[cursor]).each do |symbol_id|
62
+ target = go_to(states[cursor], symbol_id)
63
+ key = item_key(target)
64
+ target_id = indexes[key] ||= begin
65
+ states << target
66
+ states.length - 1
67
+ end
68
+ transitions[cursor][symbol_id] = target_id
69
+ end
70
+ cursor += 1
71
+ end
72
+ [states, transitions]
73
+ end
74
+
75
+ # @rbs (item_set seed) -> item_set
76
+ def closure(seed)
77
+ items = seed.dup
78
+ queue = seed.to_a
79
+ until queue.empty?
80
+ production_id, dot, lookahead = queue.shift
81
+ rhs = rhs_for(production_id)
82
+ grammar_symbol = @grammar.symbol_by_id(rhs[dot])
83
+ next unless grammar_symbol&.nonterminal?
84
+
85
+ lookaheads = suffix_lookaheads(rhs.drop(dot + 1), lookahead)
86
+ @productions_by_lhs.fetch(grammar_symbol.id, Array.new(0)).each do |production|
87
+ lookaheads.each do |token_id|
88
+ item = [production.id, 0, token_id] #: lr_item
89
+ enqueue_item(items, queue, item)
90
+ end
91
+ end
92
+ end
93
+ items
94
+ end
95
+
96
+ # @rbs (Array[Integer] suffix, Integer inherited) -> Array[Integer]
97
+ def suffix_lookaheads(suffix, inherited)
98
+ bits = @sets.first_of_sequence(suffix)
99
+ bits |= (1 << inherited) if @sets.sequence_nullable?(suffix)
100
+ @grammar.terminals.filter_map { |terminal| terminal.id if bits.anybits?(1 << terminal.id) }
101
+ end
102
+
103
+ # @rbs (item_set items, Array[lr_item] queue, lr_item item) -> void
104
+ def enqueue_item(items, queue, item)
105
+ return if items.include?(item)
106
+
107
+ items << item
108
+ queue << item
109
+ end
110
+
111
+ # @rbs (item_set items) -> Array[Integer]
112
+ def next_symbols(items)
113
+ items.filter_map { |production_id, dot, _lookahead| rhs_for(production_id)[dot] }.uniq.sort
114
+ end
115
+
116
+ # @rbs (item_set items, Integer symbol_id) -> item_set
117
+ def go_to(items, symbol_id)
118
+ moved = items.filter_map do |production_id, dot, lookahead|
119
+ next unless rhs_for(production_id)[dot] == symbol_id
120
+
121
+ [production_id, dot + 1, lookahead] #: lr_item
122
+ end
123
+ closure(Set.new(moved))
124
+ end
125
+
126
+ # @rbs (Array[item_set] states, transitions transitions) -> [Array[packed_items], transitions]
127
+ def merge_lalr(states, transitions)
128
+ groups = {} #: Hash[Array[item_core], Integer]
129
+ state_groups = states.map do |items|
130
+ core = core_key(items)
131
+ groups[core] ||= groups.length
132
+ end
133
+ merged = Array.new(groups.length) do
134
+ Hash.new { |hash, key| hash[key] = Set.new } #: packed_items
135
+ end
136
+ states.each_with_index do |items, state_id|
137
+ items.each { |production, dot, lookahead| merged[state_groups[state_id]][[production, dot]] << lookahead }
138
+ end
139
+ merged_transitions = Array.new(groups.length) do
140
+ {} #: Hash[Integer, Integer]
141
+ end
142
+ transitions.each_with_index do |edges, state_id|
143
+ edges.each { |symbol, target| merged_transitions[state_groups[state_id]][symbol] = state_groups[target] }
144
+ end
145
+ [merged, merged_transitions]
146
+ end
147
+
148
+ # @rbs (Array[item_set] canonical_states, transitions canonical_transitions) -> [Array[packed_items], transitions]
149
+ def automaton_items(canonical_states, canonical_transitions)
150
+ return [pack_canonical_items(canonical_states), canonical_transitions] if @algorithm == :lr1
151
+
152
+ items, transitions = merge_lalr(canonical_states, canonical_transitions)
153
+ apply_slr_lookaheads(items) if @algorithm == :slr
154
+ [items, transitions]
155
+ end
156
+
157
+ # @rbs (Array[item_set] states) -> Array[packed_items]
158
+ def pack_canonical_items(states)
159
+ states.map do |items|
160
+ packed = Hash.new { |hash, key| hash[key] = Set.new } #: packed_items
161
+ items.each { |production, dot, lookahead| packed[[production, dot]] << lookahead }
162
+ packed
163
+ end
164
+ end
165
+
166
+ # @rbs (Array[packed_items] states) -> void
167
+ def apply_slr_lookaheads(states)
168
+ states.each do |items|
169
+ items.each do |(production_id, dot), lookaheads|
170
+ next unless dot == rhs_for(production_id).length
171
+
172
+ lookaheads.replace(slr_lookaheads(production_id))
173
+ end
174
+ end
175
+ end
176
+
177
+ # @rbs (Integer production_id) -> Array[Integer]
178
+ def slr_lookaheads(production_id)
179
+ return [0] if production_id == AUGMENTED_PRODUCTION
180
+
181
+ lhs = @grammar.productions.fetch(production_id).lhs
182
+ bits = @sets.follow_bits.fetch(lhs)
183
+ @grammar.terminals.filter_map { |terminal| terminal.id if bits.anybits?(1 << terminal.id) }
184
+ end
185
+
186
+ # @rbs (Array[packed_items] merged_items, transitions transitions) -> Array[IR::AutomatonState]
187
+ def build_states(merged_items, transitions)
188
+ merged_items.each_with_index.map do |item_map, state_id|
189
+ items = item_map.sort.map do |(production, dot), lookaheads|
190
+ IR::AutomatonItem.new(production: production, dot: dot, lookaheads: lookaheads.to_a)
191
+ end
192
+ build_state(state_id, items, transitions[state_id])
193
+ end
194
+ end
195
+
196
+ # @rbs (Integer state_id, Array[IR::AutomatonItem] items, Hash[Integer, Integer] transitions) -> IR::AutomatonState
197
+ def build_state(state_id, items, transitions)
198
+ candidates = Hash.new { |hash, key| hash[key] = Array.new(0) } #: Hash[Integer, Array[IR::parser_action]]
199
+ gotos = {} #: Hash[Integer, Integer]
200
+ transitions.each do |symbol_id, target|
201
+ grammar_symbol = @grammar.symbol_by_id(symbol_id)
202
+ raise Ibex::Error, "missing grammar symbol id #{symbol_id}" unless grammar_symbol
203
+
204
+ if grammar_symbol.terminal?
205
+ candidates[symbol_id] << { type: :shift, state: target }
206
+ else
207
+ gotos[symbol_id] = target
208
+ end
209
+ end
210
+ add_completed_actions(items, candidates)
211
+ actions, conflicts = resolve_actions(candidates)
212
+ IR::AutomatonState.new(id: state_id, items: items, transitions: transitions, actions: actions,
213
+ gotos: gotos, conflicts: conflicts)
214
+ end
215
+
216
+ # @rbs (Array[IR::AutomatonItem] items, Hash[Integer, Array[IR::parser_action]] candidates) -> void
217
+ def add_completed_actions(items, candidates)
218
+ items.each do |item|
219
+ next unless item.dot == rhs_for(item.production).length
220
+
221
+ item.lookaheads.each do |lookahead|
222
+ action = if item.production == AUGMENTED_PRODUCTION
223
+ { type: :accept } #: IR::accept_action
224
+ else
225
+ { type: :reduce, production: item.production } #: IR::reduce_action
226
+ end
227
+ candidates[lookahead] << action
228
+ end
229
+ end
230
+ end
231
+
232
+ # @rbs (Hash[Integer, Array[IR::parser_action]] candidates) ->
233
+ # [Hash[Integer, IR::parser_action], Array[IR::conflict]]
234
+ def resolve_actions(candidates)
235
+ actions = {} #: Hash[Integer, IR::parser_action]
236
+ conflicts = [] #: Array[IR::conflict]
237
+ candidates.keys.sort.each do |token_id|
238
+ action, found = @resolver.resolve(token_id, candidates[token_id])
239
+ raise Ibex::Error, "empty parser action candidates" unless action
240
+
241
+ actions[token_id] = action
242
+ conflicts.concat(found)
243
+ end
244
+ [actions, conflicts]
245
+ end
246
+
247
+ # @rbs (Integer production_id) -> Array[Integer]
248
+ def rhs_for(production_id)
249
+ if production_id == AUGMENTED_PRODUCTION
250
+ start = @grammar.symbol(@grammar.start) || raise(Ibex::Error, "missing start symbol")
251
+ return [start.id]
252
+ end
253
+
254
+ @grammar.productions.fetch(production_id).rhs
255
+ end
256
+
257
+ # @rbs (item_set items) -> Array[item_core]
258
+ def core_key(items)
259
+ items.map do |production, dot, _lookahead|
260
+ [production, dot] #: item_core
261
+ end.uniq.sort
262
+ end
263
+
264
+ # @rbs (item_set items) -> Array[lr_item]
265
+ def item_key(items)
266
+ items.to_a.sort
267
+ end
268
+ end
269
+ end
270
+ end