ibex 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +43 -0
- data/CHANGELOG.md +30 -0
- data/LICENSE.txt +21 -0
- data/README.md +194 -0
- data/Rakefile +25 -0
- data/Steepfile +10 -0
- data/docs/architecture.md +99 -0
- data/docs/compat-notes.md +37 -0
- data/docs/grammar-reference.md +127 -0
- data/docs/lexer-coverage.md +14 -0
- data/docs/phase10-extensions.md +27 -0
- data/docs/racc-migration.md +47 -0
- data/exe/ibex +6 -0
- data/gemfiles/Gemfile +7 -0
- data/gemfiles/Gemfile.lock +98 -0
- data/lib/ibex/analysis/sets.rb +194 -0
- data/lib/ibex/analysis.rb +8 -0
- data/lib/ibex/cli/counterexample_options.rb +38 -0
- data/lib/ibex/cli/outputs.rb +88 -0
- data/lib/ibex/cli.rb +333 -0
- data/lib/ibex/codegen/dot.rb +52 -0
- data/lib/ibex/codegen/html.rb +110 -0
- data/lib/ibex/codegen/rbs.rb +54 -0
- data/lib/ibex/codegen/report.rb +149 -0
- data/lib/ibex/codegen/ruby.rb +235 -0
- data/lib/ibex/codegen/symbol_labels.rb +18 -0
- data/lib/ibex/error.rb +7 -0
- data/lib/ibex/frontend/action_scanner.rb +266 -0
- data/lib/ibex/frontend/ast.rb +146 -0
- data/lib/ibex/frontend/bootstrap_parser.rb +163 -0
- data/lib/ibex/frontend/dsl.rb +196 -0
- data/lib/ibex/frontend/generated_parser.rb +434 -0
- data/lib/ibex/frontend/generated_parser_base.rb +273 -0
- data/lib/ibex/frontend/grammar.y +156 -0
- data/lib/ibex/frontend/lexer.rb +165 -0
- data/lib/ibex/frontend/parser/declarations.rb +163 -0
- data/lib/ibex/frontend/parser/rules.rb +154 -0
- data/lib/ibex/frontend/parser.rb +23 -0
- data/lib/ibex/frontend/regenerator.rb +41 -0
- data/lib/ibex/frontend/source_cursor.rb +94 -0
- data/lib/ibex/frontend/token_adapter/declaration_state.rb +248 -0
- data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +37 -0
- data/lib/ibex/frontend/token_adapter/rule_state.rb +162 -0
- data/lib/ibex/frontend/token_adapter.rb +108 -0
- data/lib/ibex/frontend.rb +28 -0
- data/lib/ibex/ir/automaton_ir.rb +109 -0
- data/lib/ibex/ir/grammar_ir.rb +192 -0
- data/lib/ibex/ir/serialize.rb +154 -0
- data/lib/ibex/ir.rb +53 -0
- data/lib/ibex/lalr/builder.rb +270 -0
- data/lib/ibex/lalr/conflict.rb +120 -0
- data/lib/ibex/lalr/conflict_search.rb +332 -0
- data/lib/ibex/lalr/conflict_search_limits.rb +39 -0
- data/lib/ibex/lalr/counterexample.rb +138 -0
- data/lib/ibex/lalr/default_reductions.rb +67 -0
- data/lib/ibex/lalr.rb +27 -0
- data/lib/ibex/normalize/declarations.rb +60 -0
- data/lib/ibex/normalize/diagnostics.rb +99 -0
- data/lib/ibex/normalize/expander.rb +218 -0
- data/lib/ibex/normalize/expression.rb +54 -0
- data/lib/ibex/normalize.rb +166 -0
- data/lib/ibex/runtime/parser.rb +360 -0
- data/lib/ibex/runtime.rb +8 -0
- data/lib/ibex/tables.rb +125 -0
- data/lib/ibex/version.rb +6 -0
- data/lib/ibex.rb +23 -0
- data/sig/ibex/analysis/sets.rbs +72 -0
- data/sig/ibex/analysis.rbs +6 -0
- data/sig/ibex/cli/counterexample_options.rbs +16 -0
- data/sig/ibex/cli/outputs.rbs +31 -0
- data/sig/ibex/cli.rbs +108 -0
- data/sig/ibex/codegen/dot.rbs +19 -0
- data/sig/ibex/codegen/html.rbs +35 -0
- data/sig/ibex/codegen/rbs.rbs +31 -0
- data/sig/ibex/codegen/report.rbs +39 -0
- data/sig/ibex/codegen/ruby.rbs +92 -0
- data/sig/ibex/codegen/symbol_labels.rbs +11 -0
- data/sig/ibex/error.rbs +7 -0
- data/sig/ibex/frontend/action_scanner.rbs +74 -0
- data/sig/ibex/frontend/ast.rbs +223 -0
- data/sig/ibex/frontend/bootstrap_parser.rbs +80 -0
- data/sig/ibex/frontend/dsl.rbs +124 -0
- data/sig/ibex/frontend/generated_parser.rbs +163 -0
- data/sig/ibex/frontend/generated_parser_base.rbs +114 -0
- data/sig/ibex/frontend/lexer.rbs +61 -0
- data/sig/ibex/frontend/parser/declarations.rbs +56 -0
- data/sig/ibex/frontend/parser/rules.rbs +48 -0
- data/sig/ibex/frontend/parser.rbs +16 -0
- data/sig/ibex/frontend/regenerator.rbs +16 -0
- data/sig/ibex/frontend/source_cursor.rbs +73 -0
- data/sig/ibex/frontend/token_adapter/declaration_state.rbs +97 -0
- data/sig/ibex/frontend/token_adapter/delimiter_tracker.rbs +25 -0
- data/sig/ibex/frontend/token_adapter/rule_state.rbs +77 -0
- data/sig/ibex/frontend/token_adapter.rbs +67 -0
- data/sig/ibex/frontend.rbs +15 -0
- data/sig/ibex/ir/automaton_ir.rbs +77 -0
- data/sig/ibex/ir/grammar_ir.rbs +143 -0
- data/sig/ibex/ir/serialize.rbs +50 -0
- data/sig/ibex/ir.rbs +43 -0
- data/sig/ibex/lalr/builder.rbs +85 -0
- data/sig/ibex/lalr/conflict.rbs +47 -0
- data/sig/ibex/lalr/conflict_search.rbs +131 -0
- data/sig/ibex/lalr/conflict_search_limits.rbs +29 -0
- data/sig/ibex/lalr/counterexample.rbs +53 -0
- data/sig/ibex/lalr/default_reductions.rbs +20 -0
- data/sig/ibex/lalr.rbs +23 -0
- data/sig/ibex/normalize/declarations.rbs +20 -0
- data/sig/ibex/normalize/diagnostics.rbs +32 -0
- data/sig/ibex/normalize/expander.rbs +66 -0
- data/sig/ibex/normalize/expression.rbs +21 -0
- data/sig/ibex/normalize.rbs +96 -0
- data/sig/ibex/runtime/parser.rbs +167 -0
- data/sig/ibex/runtime.rbs +6 -0
- data/sig/ibex/tables.rbs +50 -0
- data/sig/ibex/version.rbs +6 -0
- data/sig/ibex.rbs +6 -0
- metadata +161 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ibex
|
|
4
|
+
module LALR
|
|
5
|
+
# Applies yacc-compatible precedence and ordering rules to action candidates.
|
|
6
|
+
class ConflictResolver
|
|
7
|
+
# @rbs (IR::Grammar grammar) -> void
|
|
8
|
+
def initialize(grammar)
|
|
9
|
+
@grammar = grammar
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# @rbs (Integer token_id, Array[IR::parser_action] candidates) -> [IR::parser_action?, Array[IR::conflict]]
|
|
13
|
+
def resolve(token_id, candidates)
|
|
14
|
+
actions = candidates.uniq
|
|
15
|
+
return [actions.first, Array.new(0)] if actions.length <= 1
|
|
16
|
+
|
|
17
|
+
accept = actions.find { |action| action[:type] == :accept }
|
|
18
|
+
return [accept, Array.new(0)] if accept
|
|
19
|
+
|
|
20
|
+
shift = actions.find { |action| action[:type] == :shift } #: IR::shift_action?
|
|
21
|
+
reductions = reduction_actions(actions)
|
|
22
|
+
chosen_reduce, conflicts = resolve_reductions(token_id, reductions)
|
|
23
|
+
all_conflicts = widen_conflicts(conflicts)
|
|
24
|
+
return [chosen_reduce, all_conflicts] unless shift
|
|
25
|
+
return [shift, all_conflicts] unless chosen_reduce
|
|
26
|
+
|
|
27
|
+
chosen, conflict = resolve_shift_reduce(token_id, shift, chosen_reduce)
|
|
28
|
+
[chosen, all_conflicts << conflict]
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# @rbs (Array[IR::parser_action] actions) -> Array[IR::reduce_action]
|
|
34
|
+
def reduction_actions(actions)
|
|
35
|
+
reductions = [] #: Array[IR::reduce_action]
|
|
36
|
+
actions.each do |action|
|
|
37
|
+
next unless action[:type] == :reduce
|
|
38
|
+
|
|
39
|
+
reduction = action #: IR::reduce_action
|
|
40
|
+
reductions << reduction
|
|
41
|
+
end
|
|
42
|
+
reductions.sort_by { |action| action[:production] }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @rbs (Array[IR::reduce_reduce_conflict] conflicts) -> Array[IR::conflict]
|
|
46
|
+
def widen_conflicts(conflicts)
|
|
47
|
+
conflicts.map(&:itself) #: Array[IR::conflict]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# @rbs (Integer token_id, Array[IR::reduce_action] reductions) ->
|
|
51
|
+
# [IR::reduce_action?, Array[IR::reduce_reduce_conflict]]
|
|
52
|
+
def resolve_reductions(token_id, reductions)
|
|
53
|
+
return [nil, Array.new(0)] if reductions.empty?
|
|
54
|
+
return [reductions.first, Array.new(0)] if reductions.length == 1
|
|
55
|
+
|
|
56
|
+
chosen = reductions.first
|
|
57
|
+
conflict = { type: :reduce_reduce, symbol: token_name(token_id),
|
|
58
|
+
reductions: reductions.map { |action| action[:production] },
|
|
59
|
+
resolution: { by: :definition_order, chose: chosen[:production] } } #: IR::reduce_reduce_conflict
|
|
60
|
+
[chosen, [conflict]]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @rbs (Integer token_id, IR::shift_action shift, IR::reduce_action reduction) ->
|
|
64
|
+
# [IR::parser_action, IR::shift_reduce_conflict]
|
|
65
|
+
def resolve_shift_reduce(token_id, shift, reduction)
|
|
66
|
+
token_precedence = required_symbol_by_id(token_id).precedence
|
|
67
|
+
production_precedence = precedence_for_production(reduction[:production])
|
|
68
|
+
chosen, resolution = precedence_choice(shift, reduction, token_precedence, production_precedence)
|
|
69
|
+
conflict = { type: :shift_reduce, symbol: token_name(token_id), shift_to: shift[:state],
|
|
70
|
+
reduce: reduction[:production], resolution: resolution } #: IR::shift_reduce_conflict
|
|
71
|
+
[chosen, conflict]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# @rbs (IR::shift_action shift, IR::reduce_action reduction, IR::precedence? token_precedence,
|
|
75
|
+
# IR::precedence? production_precedence) -> [IR::parser_action, IR::conflict_resolution]
|
|
76
|
+
def precedence_choice(shift, reduction, token_precedence, production_precedence)
|
|
77
|
+
return [shift, { by: :default_shift, chose: :shift }] unless token_precedence && production_precedence
|
|
78
|
+
|
|
79
|
+
comparison = token_precedence[:level] <=> production_precedence[:level]
|
|
80
|
+
return [shift, { by: :precedence, chose: :shift }] if comparison.positive?
|
|
81
|
+
return [reduction, { by: :precedence, chose: :reduce }] if comparison.negative?
|
|
82
|
+
|
|
83
|
+
associativity_choice(shift, reduction, token_precedence[:associativity])
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# @rbs (IR::shift_action shift, IR::reduce_action reduction, Symbol associativity) ->
|
|
87
|
+
# [IR::parser_action, IR::conflict_resolution]
|
|
88
|
+
def associativity_choice(shift, reduction, associativity)
|
|
89
|
+
case associativity.to_sym
|
|
90
|
+
when :left then [reduction, { by: :associativity, associativity: :left, chose: :reduce }]
|
|
91
|
+
when :right then [shift, { by: :associativity, associativity: :right, chose: :shift }]
|
|
92
|
+
when :nonassoc then [{ type: :error }, { by: :associativity, associativity: :nonassoc, chose: :error }]
|
|
93
|
+
else raise Ibex::Error, "unknown associativity #{associativity.inspect}"
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @rbs (Integer production_id) -> IR::precedence?
|
|
98
|
+
def precedence_for_production(production_id)
|
|
99
|
+
production = @grammar.productions.fetch(production_id)
|
|
100
|
+
return required_symbol_by_id(production.precedence_override).precedence if production.precedence_override
|
|
101
|
+
|
|
102
|
+
production.rhs.reverse_each do |symbol_id|
|
|
103
|
+
grammar_symbol = required_symbol_by_id(symbol_id)
|
|
104
|
+
return grammar_symbol.precedence if grammar_symbol.terminal? && grammar_symbol.precedence
|
|
105
|
+
end
|
|
106
|
+
nil
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# @rbs (Integer token_id) -> String
|
|
110
|
+
def token_name(token_id)
|
|
111
|
+
required_symbol_by_id(token_id).name
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# @rbs (Integer id) -> IR::GrammarSymbol
|
|
115
|
+
def required_symbol_by_id(id)
|
|
116
|
+
@grammar.symbol_by_id(id) || raise(Ibex::Error, "missing grammar symbol id #{id}")
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ibex
|
|
4
|
+
module LALR
|
|
5
|
+
# Searches the parser state space for one input accepted through both sides of a conflict.
|
|
6
|
+
# rubocop:disable Metrics/ClassLength -- inline type contracts make the focused search implementation longer.
|
|
7
|
+
class ConflictSearch
|
|
8
|
+
include ConflictSearchLimits
|
|
9
|
+
|
|
10
|
+
DEFAULT_MAX_TOKENS = ConflictSearchLimits::DEFAULT_MAX_TOKENS #: Integer
|
|
11
|
+
DEFAULT_MAX_CONFIGURATIONS = ConflictSearchLimits::DEFAULT_MAX_CONFIGURATIONS #: Integer
|
|
12
|
+
|
|
13
|
+
Configuration = Struct.new(
|
|
14
|
+
:states, #: Array[Integer]
|
|
15
|
+
:nodes, #: Array[derivation_node]
|
|
16
|
+
keyword_init: true
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# @rbs @automaton: IR::Automaton
|
|
20
|
+
# @rbs @grammar: IR::Grammar
|
|
21
|
+
# @rbs @state: IR::AutomatonState
|
|
22
|
+
# @rbs @conflict: IR::conflict
|
|
23
|
+
# @rbs @lookahead: Integer?
|
|
24
|
+
# @rbs @max_tokens: Integer
|
|
25
|
+
# @rbs @max_configurations: Integer
|
|
26
|
+
# @rbs @explored: Integer
|
|
27
|
+
# @rbs @input_tokens: Array[Integer]
|
|
28
|
+
|
|
29
|
+
# @rbs (IR::Automaton automaton, IR::AutomatonState state, IR::conflict conflict,
|
|
30
|
+
# ?max_tokens: Integer, ?max_configurations: Integer) -> void
|
|
31
|
+
def initialize(automaton, state, conflict, max_tokens: DEFAULT_MAX_TOKENS,
|
|
32
|
+
max_configurations: DEFAULT_MAX_CONFIGURATIONS)
|
|
33
|
+
ConflictSearchLimits.validate!(max_tokens: max_tokens, max_configurations: max_configurations)
|
|
34
|
+
@automaton = automaton
|
|
35
|
+
@grammar = automaton.grammar
|
|
36
|
+
@state = state
|
|
37
|
+
@conflict = conflict
|
|
38
|
+
@lookahead = @grammar.symbol(conflict[:symbol])&.id
|
|
39
|
+
@max_tokens = max_tokens
|
|
40
|
+
@max_configurations = max_configurations
|
|
41
|
+
@explored = 0
|
|
42
|
+
@input_tokens = @grammar.terminals.reject(&:reserved).sort_by(&:name).map(&:id)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @rbs () -> search_result?
|
|
46
|
+
def call
|
|
47
|
+
return unless @lookahead
|
|
48
|
+
|
|
49
|
+
queue = [[configuration([0], Array.new(0)), Array.new(0)]] #: Array[[Configuration, Array[Integer]]]
|
|
50
|
+
visited = { [0] => true }
|
|
51
|
+
until queue.empty? || exhausted?
|
|
52
|
+
current, prefix = queue.shift
|
|
53
|
+
conflict_configurations(current).each do |candidate|
|
|
54
|
+
result = unify_from(candidate, prefix)
|
|
55
|
+
return result if result
|
|
56
|
+
end
|
|
57
|
+
enqueue_prefixes(queue, visited, current, prefix)
|
|
58
|
+
end
|
|
59
|
+
nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
# @rbs (Configuration current) -> Array[Configuration]
|
|
65
|
+
def conflict_configurations(current)
|
|
66
|
+
advance(current, required_lookahead, stop_at: @state.id, branch_conflicts: true)
|
|
67
|
+
.filter_map { |status, candidate| candidate if status == :conflict }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# @rbs (Array[[Configuration, Array[Integer]]] queue, Hash[Array[Integer], bool] visited,
|
|
71
|
+
# Configuration current, Array[Integer] prefix) -> void
|
|
72
|
+
def enqueue_prefixes(queue, visited, current, prefix)
|
|
73
|
+
return unless room_for_token?(prefix, [])
|
|
74
|
+
|
|
75
|
+
@input_tokens.each do |token_id|
|
|
76
|
+
advance(current, token_id, branch_conflicts: true).each do |status, candidate|
|
|
77
|
+
next unless status == :shifted
|
|
78
|
+
next if visited[candidate.states]
|
|
79
|
+
|
|
80
|
+
visited[candidate.states] = true
|
|
81
|
+
queue << [candidate, prefix + [token_id]]
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# @rbs (Configuration current, Array[Integer] prefix) -> search_result?
|
|
87
|
+
def unify_from(current, prefix)
|
|
88
|
+
return unless within_token_budget?(prefix, [])
|
|
89
|
+
|
|
90
|
+
conflict_actions.combination(2) do |left_action, right_action|
|
|
91
|
+
next unless left_action && right_action
|
|
92
|
+
|
|
93
|
+
left_results = advance(current, required_lookahead, forced_action: left_action, branch_conflicts: true)
|
|
94
|
+
right_results = advance(current, required_lookahead, forced_action: right_action, branch_conflicts: true)
|
|
95
|
+
left_results.product(right_results).each do |left, right|
|
|
96
|
+
result = search_common_suffix(left, right, prefix, left_action, right_action)
|
|
97
|
+
return result if result
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
nil
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# @rbs (search_entry left, search_entry right, Array[Integer] prefix, IR::parser_action left_action,
|
|
104
|
+
# IR::parser_action right_action) -> search_result?
|
|
105
|
+
def search_common_suffix(left, right, prefix, left_action, right_action)
|
|
106
|
+
return accepted_result(prefix, [], left, right, left_action, right_action) if accepted_pair?(left, right)
|
|
107
|
+
return unless shifted_pair?(left, right)
|
|
108
|
+
|
|
109
|
+
queue = [[left.last, right.last, Array.new(0)]] #: Array[[Configuration, Configuration, Array[Integer]]]
|
|
110
|
+
visited = { pair_key(left.last, right.last) => true }
|
|
111
|
+
until queue.empty? || exhausted?
|
|
112
|
+
left_config, right_config, suffix = queue.shift
|
|
113
|
+
accepted = accept_with_eof(left_config, right_config)
|
|
114
|
+
return accepted_result(prefix, suffix, accepted[0], accepted[1], left_action, right_action) if accepted
|
|
115
|
+
next unless room_for_token?(prefix, suffix)
|
|
116
|
+
|
|
117
|
+
enqueue_suffixes(queue, visited, left_config, right_config, suffix)
|
|
118
|
+
end
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# @rbs (Configuration left_config, Configuration right_config) -> [search_entry, search_entry]?
|
|
123
|
+
def accept_with_eof(left_config, right_config)
|
|
124
|
+
left = advance(left_config, 0, branch_conflicts: true).select { |entry| entry.first == :accepted }
|
|
125
|
+
right = advance(right_config, 0, branch_conflicts: true).select { |entry| entry.first == :accepted }
|
|
126
|
+
left.product(right).first
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# @rbs (Array[[Configuration, Configuration, Array[Integer]]] queue,
|
|
130
|
+
# Hash[[Array[Integer], Array[Integer]], bool] visited, Configuration left_config,
|
|
131
|
+
# Configuration right_config, Array[Integer] suffix) -> void
|
|
132
|
+
def enqueue_suffixes(queue, visited, left_config, right_config, suffix)
|
|
133
|
+
@input_tokens.each do |token_id|
|
|
134
|
+
left = shifted_results(left_config, token_id)
|
|
135
|
+
right = shifted_results(right_config, token_id)
|
|
136
|
+
left.product(right).each do |left_candidate, right_candidate|
|
|
137
|
+
key = pair_key(left_candidate, right_candidate)
|
|
138
|
+
next if visited[key]
|
|
139
|
+
|
|
140
|
+
visited[key] = true
|
|
141
|
+
queue << [left_candidate, right_candidate, suffix + [token_id]]
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# @rbs (Configuration current, Integer token_id) -> Array[Configuration]
|
|
147
|
+
def shifted_results(current, token_id)
|
|
148
|
+
advance(current, token_id, branch_conflicts: true)
|
|
149
|
+
.filter_map { |status, candidate| candidate if status == :shifted }
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# @rbs (Array[Integer] prefix, Array[Integer] suffix, search_entry left, search_entry right,
|
|
153
|
+
# IR::parser_action left_action, IR::parser_action right_action) -> search_result?
|
|
154
|
+
def accepted_result(prefix, suffix, left, right, left_action, right_action)
|
|
155
|
+
return unless within_token_budget?(prefix, suffix)
|
|
156
|
+
|
|
157
|
+
sentence = prefix.dup
|
|
158
|
+
lookahead = required_lookahead
|
|
159
|
+
sentence << lookahead unless lookahead.zero?
|
|
160
|
+
sentence.concat(suffix)
|
|
161
|
+
{ sentence_ids: sentence, lookahead_index: prefix.length,
|
|
162
|
+
interpretations: [interpretation(left_action, left.last), interpretation(right_action, right.last)] }
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# @rbs (IR::parser_action action, Configuration current) -> IR::interpretation
|
|
166
|
+
def interpretation(action, current)
|
|
167
|
+
details = { kind: action[:type], tree: current.nodes.last } #: IR::interpretation
|
|
168
|
+
if action[:type] == :shift
|
|
169
|
+
shift_action = action #: IR::shift_action
|
|
170
|
+
details[:state] = shift_action[:state]
|
|
171
|
+
elsif action[:type] == :reduce
|
|
172
|
+
reduce_action = action #: IR::reduce_action
|
|
173
|
+
details[:production] = reduce_action[:production]
|
|
174
|
+
end
|
|
175
|
+
details
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# @rbs (search_entry left, search_entry right) -> bool
|
|
179
|
+
def accepted_pair?(left, right)
|
|
180
|
+
left.first == :accepted && right.first == :accepted
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# @rbs (search_entry left, search_entry right) -> bool
|
|
184
|
+
def shifted_pair?(left, right)
|
|
185
|
+
left.first == :shifted && right.first == :shifted
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# @rbs () -> Array[IR::parser_action]
|
|
189
|
+
def conflict_actions
|
|
190
|
+
case @conflict[:type]
|
|
191
|
+
when :shift_reduce
|
|
192
|
+
conflict = @conflict #: IR::shift_reduce_conflict
|
|
193
|
+
shift = { type: :shift, state: conflict[:shift_to] } #: IR::shift_action
|
|
194
|
+
reduce = { type: :reduce, production: conflict[:reduce] } #: IR::reduce_action
|
|
195
|
+
[shift, reduce]
|
|
196
|
+
when :reduce_reduce
|
|
197
|
+
conflict = @conflict #: IR::reduce_reduce_conflict
|
|
198
|
+
conflict[:reductions].map do |production|
|
|
199
|
+
{ type: :reduce, production: production } #: IR::reduce_action
|
|
200
|
+
end
|
|
201
|
+
else
|
|
202
|
+
[]
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# @rbs (Configuration initial, Integer token_id, ?forced_action: IR::parser_action?, ?stop_at: Integer?,
|
|
207
|
+
# ?branch_conflicts: bool) -> Array[search_entry]
|
|
208
|
+
def advance(initial, token_id, forced_action: nil, stop_at: nil, branch_conflicts: false)
|
|
209
|
+
queue = [[initial, forced_action]] #: Array[[Configuration, IR::parser_action?]]
|
|
210
|
+
visited = {} #: Hash[[Array[Integer], bool], bool]
|
|
211
|
+
results = [] #: Array[search_entry]
|
|
212
|
+
until queue.empty? || exhausted?
|
|
213
|
+
current, forced = queue.shift
|
|
214
|
+
if stop_at == current.states.last
|
|
215
|
+
results << [:conflict, current]
|
|
216
|
+
next
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
key = [current.states, !forced.nil?] #: [Array[Integer], bool]
|
|
220
|
+
next if visited[key]
|
|
221
|
+
|
|
222
|
+
visited[key] = true
|
|
223
|
+
count_configuration
|
|
224
|
+
actions_for(current.states.last, token_id, forced, branch_conflicts).each do |action|
|
|
225
|
+
apply_action(results, queue, current, action, token_id)
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
results
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# @rbs (Integer state_id, Integer token_id, IR::parser_action? forced, bool branch_conflicts) ->
|
|
232
|
+
# Array[IR::parser_action]
|
|
233
|
+
def actions_for(state_id, token_id, forced, branch_conflicts)
|
|
234
|
+
return [forced] if forced
|
|
235
|
+
|
|
236
|
+
state = @automaton.states.fetch(state_id)
|
|
237
|
+
actions = [state.actions[token_id] || state.default_action].compact
|
|
238
|
+
return actions unless branch_conflicts
|
|
239
|
+
|
|
240
|
+
token_name = symbol_name(token_id)
|
|
241
|
+
state.conflicts.each do |conflict|
|
|
242
|
+
next unless conflict[:symbol] == token_name
|
|
243
|
+
|
|
244
|
+
actions.concat(actions_from(conflict))
|
|
245
|
+
end
|
|
246
|
+
actions.uniq
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# @rbs (IR::conflict conflict) -> Array[IR::parser_action]
|
|
250
|
+
def actions_from(conflict)
|
|
251
|
+
case conflict[:type]
|
|
252
|
+
when :shift_reduce
|
|
253
|
+
shift_reduce = conflict #: IR::shift_reduce_conflict
|
|
254
|
+
shift = { type: :shift, state: shift_reduce[:shift_to] } #: IR::shift_action
|
|
255
|
+
reduce = { type: :reduce, production: shift_reduce[:reduce] } #: IR::reduce_action
|
|
256
|
+
[shift, reduce]
|
|
257
|
+
when :reduce_reduce
|
|
258
|
+
reduce_reduce = conflict #: IR::reduce_reduce_conflict
|
|
259
|
+
reduce_reduce[:reductions].map do |production|
|
|
260
|
+
{ type: :reduce, production: production } #: IR::reduce_action
|
|
261
|
+
end
|
|
262
|
+
else
|
|
263
|
+
[]
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
# @rbs (Array[search_entry] results, Array[[Configuration, IR::parser_action?]] queue,
|
|
268
|
+
# Configuration current, IR::parser_action action, Integer token_id) -> void
|
|
269
|
+
def apply_action(results, queue, current, action, token_id)
|
|
270
|
+
case action[:type]
|
|
271
|
+
when :shift
|
|
272
|
+
shift_action = action #: IR::shift_action
|
|
273
|
+
results << [:shifted, shift(current, shift_action[:state], token_id)]
|
|
274
|
+
when :reduce
|
|
275
|
+
reduce_action = action #: IR::reduce_action
|
|
276
|
+
reduced = reduce(current, reduce_action[:production])
|
|
277
|
+
queue << [reduced, nil] if reduced
|
|
278
|
+
when :accept
|
|
279
|
+
results << [:accepted, current]
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# @rbs (Configuration current, Integer state_id, Integer token_id) -> Configuration
|
|
284
|
+
def shift(current, state_id, token_id)
|
|
285
|
+
symbol = symbol_name(token_id)
|
|
286
|
+
configuration(current.states + [state_id], current.nodes + [{ symbol: symbol, token: symbol }])
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# @rbs (Configuration current, Integer production_id) -> Configuration?
|
|
290
|
+
def reduce(current, production_id)
|
|
291
|
+
production = @grammar.productions.fetch(production_id)
|
|
292
|
+
length = production.rhs.length
|
|
293
|
+
return if length >= current.states.length
|
|
294
|
+
|
|
295
|
+
states = current.states.take(current.states.length - length)
|
|
296
|
+
children = current.nodes.last(length)
|
|
297
|
+
nodes = current.nodes.take(current.nodes.length - length)
|
|
298
|
+
target = @automaton.states.fetch(states.last).gotos[production.lhs]
|
|
299
|
+
return unless target
|
|
300
|
+
|
|
301
|
+
symbol = symbol_name(production.lhs)
|
|
302
|
+
configuration(states + [target], nodes + [{ symbol: symbol, production: production_id, children: children }])
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# @rbs (Array[Integer] states, Array[derivation_node] nodes) -> Configuration
|
|
306
|
+
def configuration(states, nodes) = Configuration.new(states: states.freeze, nodes: nodes.freeze)
|
|
307
|
+
|
|
308
|
+
# @rbs () -> Integer
|
|
309
|
+
def required_lookahead
|
|
310
|
+
@lookahead || raise(Ibex::Error, "missing conflict lookahead")
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# @rbs (Integer id) -> String
|
|
314
|
+
def symbol_name(id)
|
|
315
|
+
symbol = @grammar.symbol_by_id(id) || raise(Ibex::Error, "missing grammar symbol id #{id}")
|
|
316
|
+
symbol.name
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# @rbs (Configuration left, Configuration right) -> [Array[Integer], Array[Integer]]
|
|
320
|
+
def pair_key(left, right)
|
|
321
|
+
[left.states, right.states]
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# @rbs () -> Integer
|
|
325
|
+
def count_configuration = (@explored += 1)
|
|
326
|
+
|
|
327
|
+
# @rbs () -> bool
|
|
328
|
+
def exhausted? = @explored >= @max_configurations
|
|
329
|
+
end
|
|
330
|
+
# rubocop:enable Metrics/ClassLength
|
|
331
|
+
end
|
|
332
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ibex
|
|
4
|
+
module LALR
|
|
5
|
+
# Validates search limits and measures sentences against the token budget.
|
|
6
|
+
module ConflictSearchLimits
|
|
7
|
+
DEFAULT_MAX_TOKENS = 32 #: Integer
|
|
8
|
+
DEFAULT_MAX_CONFIGURATIONS = 50_000 #: Integer
|
|
9
|
+
|
|
10
|
+
# @rbs (max_tokens: Integer, max_configurations: Integer) -> void
|
|
11
|
+
def self.validate!(max_tokens:, max_configurations:)
|
|
12
|
+
validate_limit!(:max_tokens, max_tokens)
|
|
13
|
+
validate_limit!(:max_configurations, max_configurations)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# @rbs (Symbol name, Integer value) -> Integer
|
|
17
|
+
def self.validate_limit!(name, value)
|
|
18
|
+
return value if value.is_a?(Integer) && value.positive?
|
|
19
|
+
|
|
20
|
+
raise ArgumentError, "#{name} must be a positive Integer"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
# @rbs (Array[Integer] prefix, Array[Integer] suffix) -> bool
|
|
26
|
+
def within_token_budget?(prefix, suffix)
|
|
27
|
+
prefix.length + suffix.length + conflict_lookahead_length <= @max_tokens
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @rbs (Array[Integer] prefix, Array[Integer] suffix) -> bool
|
|
31
|
+
def room_for_token?(prefix, suffix)
|
|
32
|
+
prefix.length + suffix.length + conflict_lookahead_length < @max_tokens
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @rbs () -> Integer
|
|
36
|
+
def conflict_lookahead_length = @lookahead.zero? ? 0 : 1
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ibex
|
|
4
|
+
module LALR
|
|
5
|
+
# Produces shortest-path conflict witnesses using Automaton IR only.
|
|
6
|
+
class Counterexample
|
|
7
|
+
DEFAULT_MAX_TOKENS = ConflictSearch::DEFAULT_MAX_TOKENS #: Integer
|
|
8
|
+
DEFAULT_MAX_CONFIGURATIONS = ConflictSearch::DEFAULT_MAX_CONFIGURATIONS #: Integer
|
|
9
|
+
|
|
10
|
+
# @rbs (IR::Automaton automaton, ?max_tokens: Integer, ?max_configurations: Integer) -> void
|
|
11
|
+
def initialize(automaton, max_tokens: DEFAULT_MAX_TOKENS, max_configurations: DEFAULT_MAX_CONFIGURATIONS)
|
|
12
|
+
ConflictSearchLimits.validate!(max_tokens: max_tokens, max_configurations: max_configurations)
|
|
13
|
+
@automaton = automaton
|
|
14
|
+
@grammar = automaton.grammar
|
|
15
|
+
@max_tokens = max_tokens
|
|
16
|
+
@max_configurations = max_configurations
|
|
17
|
+
@shortest_yields = compute_shortest_yields
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# @rbs () -> Array[IR::counterexample]
|
|
21
|
+
def all
|
|
22
|
+
@automaton.states.flat_map do |state|
|
|
23
|
+
state.conflicts.map { |conflict| build_example(state, conflict) }
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# @rbs (IR::AutomatonState state, IR::conflict conflict) -> IR::counterexample
|
|
30
|
+
def build_example(state, conflict)
|
|
31
|
+
unifying = ConflictSearch.new(
|
|
32
|
+
@automaton, state, conflict, max_tokens: @max_tokens, max_configurations: @max_configurations
|
|
33
|
+
).call
|
|
34
|
+
return unifying_example(state, conflict, unifying) if unifying
|
|
35
|
+
|
|
36
|
+
reachability_example(state, conflict)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @rbs (IR::AutomatonState state, IR::conflict conflict, Hash[Symbol, untyped] result) -> IR::counterexample
|
|
40
|
+
def unifying_example(state, conflict, result)
|
|
41
|
+
{ state: state.id, type: conflict[:type], symbol_path: Array.new(0),
|
|
42
|
+
sentence: names(result[:sentence_ids]), lookahead_index: result[:lookahead_index], unifying: true,
|
|
43
|
+
interpretations: result[:interpretations] }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @rbs (IR::AutomatonState state, IR::conflict conflict) -> IR::counterexample
|
|
47
|
+
def reachability_example(state, conflict)
|
|
48
|
+
path = shortest_state_path(state.id)
|
|
49
|
+
lookahead = @grammar.symbol(conflict[:symbol])
|
|
50
|
+
terminal_ids = path.flat_map { |symbol_id| @shortest_yields[symbol_id] || [] }
|
|
51
|
+
terminal_ids << lookahead.id if lookahead
|
|
52
|
+
{ state: state.id, type: conflict[:type], symbol_path: names(path), sentence: names(terminal_ids),
|
|
53
|
+
lookahead_index: terminal_ids.length - 1, unifying: false, interpretations: interpretations(conflict) }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @rbs (Integer target) -> Array[Integer]
|
|
57
|
+
def shortest_state_path(target)
|
|
58
|
+
queue = [[0, Array.new(0)]] #: Array[[Integer, Array[Integer]]]
|
|
59
|
+
visited = { 0 => true }
|
|
60
|
+
until queue.empty?
|
|
61
|
+
state_id, path = queue.shift
|
|
62
|
+
return path if state_id == target
|
|
63
|
+
|
|
64
|
+
@automaton.states.fetch(state_id).transitions.sort.each do |symbol_id, next_state|
|
|
65
|
+
next if visited[next_state]
|
|
66
|
+
|
|
67
|
+
visited[next_state] = true
|
|
68
|
+
queue << [next_state, path + [symbol_id]]
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
[]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# @rbs () -> Hash[Integer, Array[Integer]]
|
|
75
|
+
def compute_shortest_yields
|
|
76
|
+
yields = @grammar.terminals.to_h { |terminal| [terminal.id, [terminal.id]] }
|
|
77
|
+
loop do
|
|
78
|
+
changed = false
|
|
79
|
+
@grammar.productions.each do |production|
|
|
80
|
+
candidate = production.rhs.flat_map { |id| yields[id] || [] }
|
|
81
|
+
next unless production.rhs.all? { |id| yields.key?(id) }
|
|
82
|
+
next unless better_yield?(candidate, yields[production.lhs])
|
|
83
|
+
|
|
84
|
+
yields[production.lhs] = candidate
|
|
85
|
+
changed = true
|
|
86
|
+
end
|
|
87
|
+
return yields unless changed
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @rbs (Array[Integer] candidate, Array[Integer]? current) -> bool
|
|
92
|
+
def better_yield?(candidate, current)
|
|
93
|
+
return true unless current
|
|
94
|
+
return candidate.length < current.length if candidate.length != current.length
|
|
95
|
+
|
|
96
|
+
comparison = names(candidate) <=> names(current)
|
|
97
|
+
comparison ? comparison.negative? : false
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# @rbs (IR::conflict conflict) -> Array[IR::interpretation]
|
|
101
|
+
def interpretations(conflict)
|
|
102
|
+
case conflict[:type]
|
|
103
|
+
when :shift_reduce
|
|
104
|
+
shift_reduce = conflict #: IR::shift_reduce_conflict
|
|
105
|
+
[shift_interpretation(shift_reduce), reduce_interpretation(shift_reduce[:reduce])]
|
|
106
|
+
when :reduce_reduce
|
|
107
|
+
reduce_reduce = conflict #: IR::reduce_reduce_conflict
|
|
108
|
+
reduce_reduce[:reductions].map { |production_id| reduce_interpretation(production_id) }
|
|
109
|
+
else []
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# @rbs (IR::shift_reduce_conflict conflict) -> IR::interpretation
|
|
114
|
+
def shift_interpretation(conflict)
|
|
115
|
+
{ kind: :shift, state: conflict[:shift_to], tree: { token: conflict[:symbol] } }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# @rbs (Integer production_id) -> IR::interpretation
|
|
119
|
+
def reduce_interpretation(production_id)
|
|
120
|
+
production = @grammar.productions.fetch(production_id)
|
|
121
|
+
lhs = symbol_name(production.lhs)
|
|
122
|
+
rhs = names(production.rhs)
|
|
123
|
+
{ kind: :reduce, production: production_id, tree: { symbol: lhs, children: rhs } }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# @rbs (Array[Integer] symbol_ids) -> Array[String]
|
|
127
|
+
def names(symbol_ids)
|
|
128
|
+
symbol_ids.map { |id| symbol_name(id) }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# @rbs (Integer id) -> String
|
|
132
|
+
def symbol_name(id)
|
|
133
|
+
symbol = @grammar.symbol_by_id(id) || raise(Ibex::Error, "missing grammar symbol id #{id}")
|
|
134
|
+
symbol.name
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|