ibex-runtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +337 -0
- data/lib/ibex/runtime/ast_data.rb +23 -0
- data/lib/ibex/runtime/cst/annotation.rb +16 -0
- data/lib/ibex/runtime/cst/cursor.rb +68 -0
- data/lib/ibex/runtime/cst/diff.rb +80 -0
- data/lib/ibex/runtime/cst/editing.rb +56 -0
- data/lib/ibex/runtime/cst/editor.rb +94 -0
- data/lib/ibex/runtime/cst/green/builder.rb +211 -0
- data/lib/ibex/runtime/cst/green/cache.rb +150 -0
- data/lib/ibex/runtime/cst/green/node.rb +78 -0
- data/lib/ibex/runtime/cst/green/token.rb +91 -0
- data/lib/ibex/runtime/cst/green/trivia.rb +36 -0
- data/lib/ibex/runtime/cst/incremental/blender.rb +233 -0
- data/lib/ibex/runtime/cst/incremental/lexed_syntax.rb +28 -0
- data/lib/ibex/runtime/cst/incremental/parse_memo.rb +95 -0
- data/lib/ibex/runtime/cst/incremental/relexer.rb +134 -0
- data/lib/ibex/runtime/cst/incremental/session.rb +229 -0
- data/lib/ibex/runtime/cst/incremental/token_memo.rb +66 -0
- data/lib/ibex/runtime/cst/kind.rb +133 -0
- data/lib/ibex/runtime/cst/parse_result.rb +41 -0
- data/lib/ibex/runtime/cst/rewriter.rb +75 -0
- data/lib/ibex/runtime/cst/serialize.rb +127 -0
- data/lib/ibex/runtime/cst/serialized_tree.rb +44 -0
- data/lib/ibex/runtime/cst/source_text.rb +112 -0
- data/lib/ibex/runtime/cst/syntax_node.rb +432 -0
- data/lib/ibex/runtime/cst/syntax_token.rb +160 -0
- data/lib/ibex/runtime/cst/text_edit.rb +77 -0
- data/lib/ibex/runtime/cst/typed_node.rb +87 -0
- data/lib/ibex/runtime/cst/validator.rb +428 -0
- data/lib/ibex/runtime/cst.rb +38 -0
- data/lib/ibex/runtime/event.rb +44 -0
- data/lib/ibex/runtime/event_jsonl_tracer.rb +37 -0
- data/lib/ibex/runtime/event_sanitizer.rb +273 -0
- data/lib/ibex/runtime/generated_lexer.rb +514 -0
- data/lib/ibex/runtime/jsonl_tracer.rb +75 -0
- data/lib/ibex/runtime/lexer_input.rb +107 -0
- data/lib/ibex/runtime/location_span.rb +114 -0
- data/lib/ibex/runtime/observation.rb +82 -0
- data/lib/ibex/runtime/parser.rb +3027 -0
- data/lib/ibex/runtime/parser_sync_recovery.rb +108 -0
- data/lib/ibex/runtime/repair.rb +126 -0
- data/lib/ibex/runtime/repair_priority_queue.rb +60 -0
- data/lib/ibex/runtime/repair_search.rb +318 -0
- data/lib/ibex/runtime/resource_limits.rb +64 -0
- data/lib/ibex/runtime/table_format.rb +11 -0
- data/lib/ibex/runtime/version.rb +7 -0
- data/lib/ibex/runtime.rb +27 -0
- data/lib/ibex/tables/compact.rb +186 -0
- data/lib/ibex/tables/compact_actions.rb +135 -0
- data/lib/ibex/tables/compact_productions.rb +167 -0
- data/sig/ibex/runtime/ast_data.rbs +12 -0
- data/sig/ibex/runtime/cst/annotation.rbs +13 -0
- data/sig/ibex/runtime/cst/cursor.rbs +34 -0
- data/sig/ibex/runtime/cst/diff.rbs +29 -0
- data/sig/ibex/runtime/cst/editing.rbs +23 -0
- data/sig/ibex/runtime/cst/editor.rbs +43 -0
- data/sig/ibex/runtime/cst/green/builder.rbs +84 -0
- data/sig/ibex/runtime/cst/green/cache.rbs +60 -0
- data/sig/ibex/runtime/cst/green/node.rbs +49 -0
- data/sig/ibex/runtime/cst/green/token.rbs +59 -0
- data/sig/ibex/runtime/cst/green/trivia.rbs +29 -0
- data/sig/ibex/runtime/cst/incremental/blender.rbs +114 -0
- data/sig/ibex/runtime/cst/incremental/lexed_syntax.rbs +20 -0
- data/sig/ibex/runtime/cst/incremental/parse_memo.rbs +59 -0
- data/sig/ibex/runtime/cst/incremental/relexer.rbs +54 -0
- data/sig/ibex/runtime/cst/incremental/session.rbs +89 -0
- data/sig/ibex/runtime/cst/incremental/token_memo.rbs +31 -0
- data/sig/ibex/runtime/cst/kind.rbs +93 -0
- data/sig/ibex/runtime/cst/parse_result.rbs +35 -0
- data/sig/ibex/runtime/cst/rewriter.rbs +31 -0
- data/sig/ibex/runtime/cst/serialize.rbs +39 -0
- data/sig/ibex/runtime/cst/serialized_tree.rbs +34 -0
- data/sig/ibex/runtime/cst/source_text.rbs +51 -0
- data/sig/ibex/runtime/cst/syntax_node.rbs +195 -0
- data/sig/ibex/runtime/cst/syntax_token.rbs +112 -0
- data/sig/ibex/runtime/cst/text_edit.rbs +33 -0
- data/sig/ibex/runtime/cst/typed_node.rbs +50 -0
- data/sig/ibex/runtime/cst/validator.rbs +108 -0
- data/sig/ibex/runtime/cst.rbs +9 -0
- data/sig/ibex/runtime/event.rbs +26 -0
- data/sig/ibex/runtime/event_jsonl_tracer.rbs +25 -0
- data/sig/ibex/runtime/event_sanitizer.rbs +92 -0
- data/sig/ibex/runtime/generated_lexer.rbs +169 -0
- data/sig/ibex/runtime/jsonl_tracer.rbs +42 -0
- data/sig/ibex/runtime/lexer_input.rbs +44 -0
- data/sig/ibex/runtime/location_span.rbs +64 -0
- data/sig/ibex/runtime/observation.rbs +35 -0
- data/sig/ibex/runtime/parser.rbs +926 -0
- data/sig/ibex/runtime/parser_sync_recovery.rbs +60 -0
- data/sig/ibex/runtime/repair.rbs +92 -0
- data/sig/ibex/runtime/repair_priority_queue.rbs +27 -0
- data/sig/ibex/runtime/repair_search.rbs +114 -0
- data/sig/ibex/runtime/resource_limits.rbs +39 -0
- data/sig/ibex/runtime/table_format.rbs +11 -0
- data/sig/ibex/runtime/version.rbs +7 -0
- data/sig/ibex/runtime.rbs +6 -0
- data/sig/ibex/tables/compact.rbs +70 -0
- data/sig/ibex/tables/compact_actions.rbs +57 -0
- data/sig/ibex/tables/compact_productions.rbs +68 -0
- metadata +144 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ibex
|
|
4
|
+
module Runtime
|
|
5
|
+
# Panic-mode synchronization used only when yacc's explicit `error` token
|
|
6
|
+
# cannot be shifted. Included by Parser to share its table and event paths.
|
|
7
|
+
module ParserSyncRecovery
|
|
8
|
+
ERROR_ACTION = [:error].freeze #: [:error]
|
|
9
|
+
|
|
10
|
+
private_constant :ERROR_ACTION
|
|
11
|
+
|
|
12
|
+
# @rbs!
|
|
13
|
+
# private def parser_tables: () -> Hash[Symbol, untyped]
|
|
14
|
+
# private def table_lookup: (untyped, Integer, Integer) -> untyped
|
|
15
|
+
# private def default_action: (Integer) -> untyped
|
|
16
|
+
# private def token_to_str: (untyped) -> String
|
|
17
|
+
# private def trace: (String) -> void
|
|
18
|
+
# private def continue_recovery: () -> untyped
|
|
19
|
+
# private def reject_recovery_eof: () -> [:done, nil]
|
|
20
|
+
# private def finish_recovery: (untyped, untyped, untyped, untyped, Integer, Array[untyped],
|
|
21
|
+
# Hash[String, untyped]?, String, Array[Proc]?) -> [:continue]
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
# @rbs () -> bool
|
|
26
|
+
def sync_recovery_configured?
|
|
27
|
+
tokens = parser_tables[:recovery_sync_tokens]
|
|
28
|
+
tokens.is_a?(Array) && !tokens.empty?
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @rbs () -> bool
|
|
32
|
+
def sync_recovery_active?
|
|
33
|
+
!@sync_recovery_context.nil?
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @rbs (Hash[Symbol, untyped] context, Hash[String, untyped]? token_data,
|
|
37
|
+
# Array[Proc]? observers) -> untyped
|
|
38
|
+
def begin_sync_recovery(context, token_data, observers)
|
|
39
|
+
@sync_recovery_context = context
|
|
40
|
+
@sync_recovery_token_data = token_data
|
|
41
|
+
@sync_recovery_observers = observers
|
|
42
|
+
continue_sync_recovery
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @rbs () -> untyped
|
|
46
|
+
def continue_sync_recovery
|
|
47
|
+
return reject_sync_recovery_eof if @lookahead == Parser::EOF_TOKEN
|
|
48
|
+
return finish_sync_recovery if sync_token?(@lookahead) && synchronize_for_current_token
|
|
49
|
+
|
|
50
|
+
continue_recovery
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @rbs (untyped token_id) -> bool
|
|
54
|
+
def sync_token?(token_id)
|
|
55
|
+
tokens = parser_tables[:recovery_sync_tokens]
|
|
56
|
+
tokens.is_a?(Array) && tokens.include?(token_id)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @rbs () -> bool
|
|
60
|
+
def synchronize_for_current_token
|
|
61
|
+
loop do
|
|
62
|
+
action = sync_action(@state_stack.last, @lookahead)
|
|
63
|
+
return true unless action.nil? || action.first == :error
|
|
64
|
+
return false if @state_stack.length == 1
|
|
65
|
+
|
|
66
|
+
trace("recover: pop state #{@state_stack.last} for sync token") if @yydebug
|
|
67
|
+
@state_stack.pop
|
|
68
|
+
@value_stack.pop
|
|
69
|
+
@location_stack&.pop
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @rbs (Integer state, Integer token_id) -> untyped
|
|
74
|
+
def sync_action(state, token_id)
|
|
75
|
+
return ERROR_ACTION unless parser_tables.fetch(:token_names).key?(token_id)
|
|
76
|
+
|
|
77
|
+
table_lookup(parser_tables.fetch(:actions), state, token_id) || default_action(state) || ERROR_ACTION
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# @rbs () -> [:continue]
|
|
81
|
+
def finish_sync_recovery
|
|
82
|
+
context = @sync_recovery_context || raise(ParseError, "(recovery):1:1: missing synchronization context")
|
|
83
|
+
token_data = @sync_recovery_token_data
|
|
84
|
+
observers = @sync_recovery_observers
|
|
85
|
+
clear_sync_recovery
|
|
86
|
+
@recovery_shifts = Parser::RECOVERY_SHIFTS
|
|
87
|
+
trace("recover: synchronized before #{token_to_str(@lookahead)} in state #{@state_stack.last}") if @yydebug
|
|
88
|
+
finish_recovery(
|
|
89
|
+
context[:token_id], context[:token_display], context[:value], context[:location], context[:state],
|
|
90
|
+
context.fetch(:value_stack), token_data, context.fetch(:reason), observers
|
|
91
|
+
)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @rbs () -> [:done, nil]
|
|
95
|
+
def reject_sync_recovery_eof
|
|
96
|
+
clear_sync_recovery
|
|
97
|
+
reject_recovery_eof
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# @rbs () -> void
|
|
101
|
+
def clear_sync_recovery
|
|
102
|
+
@sync_recovery_context = nil
|
|
103
|
+
@sync_recovery_token_data = nil
|
|
104
|
+
@sync_recovery_observers = nil
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module Ibex
|
|
5
|
+
module Runtime
|
|
6
|
+
# Explicit bounded cost and search policy for automatic token repair.
|
|
7
|
+
class RepairPolicy
|
|
8
|
+
attr_reader :insert_cost #: Integer
|
|
9
|
+
attr_reader :delete_cost #: Integer
|
|
10
|
+
attr_reader :replace_cost #: Integer
|
|
11
|
+
attr_reader :max_cost #: Integer
|
|
12
|
+
attr_reader :max_configurations #: Integer
|
|
13
|
+
attr_reader :max_lookahead #: Integer
|
|
14
|
+
attr_reader :success_shifts #: Integer
|
|
15
|
+
attr_reader :max_stack #: Integer
|
|
16
|
+
|
|
17
|
+
# @rbs (?insert_cost: Integer, ?delete_cost: Integer, ?replace_cost: Integer, ?max_cost: Integer,
|
|
18
|
+
# ?max_configurations: Integer, ?max_lookahead: Integer, ?success_shifts: Integer,
|
|
19
|
+
# ?max_stack: Integer) -> void
|
|
20
|
+
def initialize(insert_cost: 1, delete_cost: 1, replace_cost: 2, max_cost: 3, max_configurations: 5_000,
|
|
21
|
+
max_lookahead: 8, success_shifts: 3, max_stack: 256)
|
|
22
|
+
values = {
|
|
23
|
+
insert_cost: insert_cost,
|
|
24
|
+
delete_cost: delete_cost,
|
|
25
|
+
replace_cost: replace_cost,
|
|
26
|
+
max_cost: max_cost,
|
|
27
|
+
max_configurations: max_configurations,
|
|
28
|
+
max_lookahead: max_lookahead,
|
|
29
|
+
success_shifts: success_shifts,
|
|
30
|
+
max_stack: max_stack
|
|
31
|
+
}
|
|
32
|
+
invalid = values.find { |_name, value| !value.is_a?(Integer) || !value.positive? }
|
|
33
|
+
raise ArgumentError, "#{invalid.first} must be a positive integer" if invalid
|
|
34
|
+
raise ArgumentError, "success_shifts must not exceed max_lookahead" if success_shifts > max_lookahead
|
|
35
|
+
|
|
36
|
+
values.each { |name, value| instance_variable_set(:"@#{name}", value) }
|
|
37
|
+
freeze
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# One immutable insertion, deletion, or replacement.
|
|
42
|
+
class RepairEdit
|
|
43
|
+
KINDS = %i[insert delete replace].freeze #: Array[Symbol]
|
|
44
|
+
|
|
45
|
+
attr_reader :kind #: Symbol
|
|
46
|
+
attr_reader :position #: Integer
|
|
47
|
+
attr_reader :token_id #: Integer
|
|
48
|
+
attr_reader :token_name #: String
|
|
49
|
+
attr_reader :cost #: Integer
|
|
50
|
+
|
|
51
|
+
# @rbs (kind: Symbol, position: Integer, token_id: Integer, token_name: String, cost: Integer) -> void
|
|
52
|
+
def initialize(kind:, position:, token_id:, token_name:, cost:)
|
|
53
|
+
raise ArgumentError, "unknown repair edit #{kind.inspect}" unless KINDS.include?(kind)
|
|
54
|
+
raise ArgumentError, "repair edit position must be nonnegative" if position.negative?
|
|
55
|
+
raise ArgumentError, "repair edit cost must be positive" unless cost.positive?
|
|
56
|
+
|
|
57
|
+
@kind = kind
|
|
58
|
+
@position = position
|
|
59
|
+
@token_id = token_id
|
|
60
|
+
@token_name = token_name.dup.freeze
|
|
61
|
+
@cost = cost
|
|
62
|
+
freeze
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
66
|
+
def to_h
|
|
67
|
+
{ kind: @kind, position: @position, token_id: @token_id, token_name: @token_name, cost: @cost }.freeze
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Deterministically selected repair.
|
|
72
|
+
class RepairPlan
|
|
73
|
+
attr_reader :edits #: Array[RepairEdit]
|
|
74
|
+
attr_reader :cost #: Integer
|
|
75
|
+
attr_reader :configurations #: Integer
|
|
76
|
+
|
|
77
|
+
# @rbs (edits: Array[RepairEdit], configurations: Integer) -> void
|
|
78
|
+
def initialize(edits:, configurations:)
|
|
79
|
+
raise ArgumentError, "repair plan must contain an edit" if edits.empty?
|
|
80
|
+
|
|
81
|
+
@edits = edits.dup.freeze
|
|
82
|
+
@cost = @edits.sum(&:cost)
|
|
83
|
+
@configurations = configurations
|
|
84
|
+
freeze
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
88
|
+
def to_h
|
|
89
|
+
{ cost: @cost, configurations: @configurations, edits: @edits.map(&:to_h).freeze }.freeze
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Internal normalized input record retained while repair looks ahead.
|
|
94
|
+
class RepairInput
|
|
95
|
+
attr_reader :token_id #: Integer
|
|
96
|
+
attr_reader :token_name #: String
|
|
97
|
+
attr_reader :value #: untyped
|
|
98
|
+
attr_reader :location #: untyped
|
|
99
|
+
|
|
100
|
+
# @rbs (token_id: Integer, token_name: String, value: untyped, location: untyped) -> void
|
|
101
|
+
def initialize(token_id:, token_name:, value:, location:)
|
|
102
|
+
@token_id = token_id
|
|
103
|
+
@token_name = token_name.dup.freeze
|
|
104
|
+
@value = value
|
|
105
|
+
@location = location
|
|
106
|
+
freeze
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# @rbs () -> bool
|
|
110
|
+
def eof? = @token_id.zero?
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Internal result of simulating reductions followed by one table action.
|
|
114
|
+
class RepairAdvance
|
|
115
|
+
attr_reader :status #: :shift | :accept
|
|
116
|
+
attr_reader :stack #: Array[Integer]
|
|
117
|
+
|
|
118
|
+
# @rbs (status: :shift | :accept, stack: Array[Integer]) -> void
|
|
119
|
+
def initialize(status:, stack:)
|
|
120
|
+
@status = status
|
|
121
|
+
@stack = stack
|
|
122
|
+
freeze
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module Ibex
|
|
5
|
+
module Runtime
|
|
6
|
+
# Minimal binary heap ordered by an immutable Array priority.
|
|
7
|
+
class RepairPriorityQueue
|
|
8
|
+
# @rbs @entries: Array[[Array[untyped], untyped]]
|
|
9
|
+
|
|
10
|
+
# @rbs () -> void
|
|
11
|
+
def initialize
|
|
12
|
+
@entries = []
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @rbs () -> bool
|
|
16
|
+
def empty? = @entries.empty?
|
|
17
|
+
|
|
18
|
+
# @rbs (Array[untyped] priority, untyped value) -> void
|
|
19
|
+
def push(priority, value)
|
|
20
|
+
entry = [priority, value] #: [Array[untyped], untyped]
|
|
21
|
+
@entries << entry
|
|
22
|
+
index = @entries.length - 1
|
|
23
|
+
while index.positive?
|
|
24
|
+
parent = (index - 1) / 2
|
|
25
|
+
break if compare(@entries[parent], entry) <= 0
|
|
26
|
+
|
|
27
|
+
@entries[index] = @entries[parent]
|
|
28
|
+
index = parent
|
|
29
|
+
end
|
|
30
|
+
@entries[index] = entry
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @rbs () -> [Array[untyped], untyped]?
|
|
34
|
+
def pop
|
|
35
|
+
first = @entries.first
|
|
36
|
+
tail = @entries.pop
|
|
37
|
+
return first if @entries.empty? || !first || !tail
|
|
38
|
+
|
|
39
|
+
index = 0
|
|
40
|
+
while (child = (index * 2) + 1) < @entries.length
|
|
41
|
+
right = child + 1
|
|
42
|
+
child = right if right < @entries.length && compare(@entries[right], @entries[child]).negative?
|
|
43
|
+
break if compare(tail, @entries[child]) <= 0
|
|
44
|
+
|
|
45
|
+
@entries[index] = @entries[child]
|
|
46
|
+
index = child
|
|
47
|
+
end
|
|
48
|
+
@entries[index] = tail
|
|
49
|
+
first
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
# @rbs ([Array[untyped], untyped] left, [Array[untyped], untyped] right) -> Integer
|
|
55
|
+
def compare(left, right)
|
|
56
|
+
(left.fetch(0) <=> right.fetch(0)) || 0
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require_relative "repair_priority_queue" unless defined?(Ibex::Runtime::RepairPriorityQueue)
|
|
5
|
+
|
|
6
|
+
module Ibex
|
|
7
|
+
module Runtime
|
|
8
|
+
# Bounded Dijkstra search over LR state stacks. Semantic actions never run.
|
|
9
|
+
# rubocop:disable Metrics/ClassLength -- queue policy and LR transitions form one bounded search invariant.
|
|
10
|
+
class RepairSearch
|
|
11
|
+
NEED_INPUT = Object.new.freeze #: Object
|
|
12
|
+
LIMIT = Object.new.freeze #: Object
|
|
13
|
+
Configuration = Struct.new(:stack, :input_index, :shifts, :cost, :edits, :goal, keyword_init: true)
|
|
14
|
+
|
|
15
|
+
# @rbs @tables: Hash[Symbol, untyped]
|
|
16
|
+
# @rbs @policy: RepairPolicy
|
|
17
|
+
# @rbs @tokens: Array[RepairInput]
|
|
18
|
+
# @rbs @complete: bool
|
|
19
|
+
# @rbs @configurations: Integer
|
|
20
|
+
# @rbs @heap: RepairPriorityQueue
|
|
21
|
+
# @rbs @best: Hash[Array[untyped], Array[untyped]]
|
|
22
|
+
# @rbs @candidate_ids: Array[Integer]
|
|
23
|
+
|
|
24
|
+
# @rbs (Hash[Symbol, untyped] tables, RepairPolicy policy, Array[RepairInput] tokens,
|
|
25
|
+
# complete: bool) -> void
|
|
26
|
+
def initialize(tables, policy, tokens, complete:)
|
|
27
|
+
@tables = tables
|
|
28
|
+
@policy = policy
|
|
29
|
+
@tokens = tokens
|
|
30
|
+
@complete = complete
|
|
31
|
+
@configurations = 0
|
|
32
|
+
@heap = RepairPriorityQueue.new
|
|
33
|
+
@best = {}
|
|
34
|
+
reserved = [Parser::EOF_TOKEN, Parser::ERROR_TOKEN]
|
|
35
|
+
@candidate_ids = tables.fetch(:token_names).keys.reject { |id| reserved.include?(id) }.sort.freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @rbs (Array[Integer] state_stack) -> (RepairPlan | Object | nil)
|
|
39
|
+
def search(state_stack)
|
|
40
|
+
empty_edits = [] #: Array[RepairEdit]
|
|
41
|
+
push(
|
|
42
|
+
Configuration.new(
|
|
43
|
+
stack: state_stack.dup.freeze,
|
|
44
|
+
input_index: 0,
|
|
45
|
+
shifts: 0,
|
|
46
|
+
cost: 0,
|
|
47
|
+
edits: empty_edits.freeze,
|
|
48
|
+
goal: false
|
|
49
|
+
)
|
|
50
|
+
)
|
|
51
|
+
needs_input = false
|
|
52
|
+
until @heap.empty?
|
|
53
|
+
configuration = pop
|
|
54
|
+
return nil if configuration.equal?(LIMIT)
|
|
55
|
+
next unless configuration.is_a?(Configuration)
|
|
56
|
+
|
|
57
|
+
result = visit(configuration)
|
|
58
|
+
return nil if result.equal?(LIMIT)
|
|
59
|
+
return result if result.is_a?(RepairPlan)
|
|
60
|
+
|
|
61
|
+
needs_input = true if result.equal?(NEED_INPUT)
|
|
62
|
+
end
|
|
63
|
+
needs_input ? NEED_INPUT : nil
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
# @rbs (Configuration configuration) -> (RepairPlan | Object | nil)
|
|
69
|
+
def visit(configuration)
|
|
70
|
+
@configurations += 1
|
|
71
|
+
return LIMIT if @configurations > @policy.max_configurations
|
|
72
|
+
return plan(configuration.edits) if configuration.goal
|
|
73
|
+
if configuration.input_index >= @tokens.length
|
|
74
|
+
return @complete ? nil : NEED_INPUT
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
expand(configuration)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# @rbs (Configuration configuration) -> (RepairPlan | Object | nil)
|
|
81
|
+
def expand(configuration)
|
|
82
|
+
current = @tokens.fetch(configuration.input_index)
|
|
83
|
+
unchanged = advance(configuration.stack, current.token_id)
|
|
84
|
+
return LIMIT if unchanged.equal?(LIMIT)
|
|
85
|
+
|
|
86
|
+
if unchanged.is_a?(RepairAdvance)
|
|
87
|
+
if unchanged.status == :accept && !configuration.edits.empty?
|
|
88
|
+
push(configuration_with(configuration, stack: unchanged.stack, goal: true))
|
|
89
|
+
return
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
if unchanged.status == :shift
|
|
93
|
+
shifts = configuration.shifts + 1
|
|
94
|
+
if shifts >= @policy.success_shifts && !configuration.edits.empty?
|
|
95
|
+
push(configuration_with(configuration, stack: unchanged.stack, shifts: shifts, goal: true))
|
|
96
|
+
return
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
push(
|
|
100
|
+
configuration_with(
|
|
101
|
+
configuration, stack: unchanged.stack, input_index: configuration.input_index + 1, shifts: shifts
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
return unless configuration.cost < @policy.max_cost
|
|
107
|
+
|
|
108
|
+
expand_edits(configuration, current)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# @rbs (Configuration configuration, RepairInput current) -> (RepairPlan | Object | nil)
|
|
112
|
+
def expand_edits(configuration, current)
|
|
113
|
+
push_delete(configuration, current) unless current.eof?
|
|
114
|
+
inserted = expand_candidates(configuration, :insert)
|
|
115
|
+
return inserted if inserted
|
|
116
|
+
|
|
117
|
+
unless current.eof?
|
|
118
|
+
replaced = expand_candidates(configuration, :replace)
|
|
119
|
+
return replaced if replaced
|
|
120
|
+
end
|
|
121
|
+
nil
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# @rbs (Configuration configuration, :insert | :replace kind)
|
|
125
|
+
# -> (RepairPlan | Object | nil)
|
|
126
|
+
def expand_candidates(configuration, kind)
|
|
127
|
+
cost = kind == :insert ? @policy.insert_cost : @policy.replace_cost
|
|
128
|
+
return if configuration.cost + cost > @policy.max_cost
|
|
129
|
+
|
|
130
|
+
@candidate_ids.each do |token_id|
|
|
131
|
+
result = expand_candidate(configuration, kind, token_id, cost)
|
|
132
|
+
return LIMIT if result.equal?(LIMIT)
|
|
133
|
+
end
|
|
134
|
+
nil
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @rbs (Configuration configuration, :insert | :replace kind, Integer token_id, Integer cost) -> Object?
|
|
138
|
+
def expand_candidate(configuration, kind, token_id, cost)
|
|
139
|
+
advanced = advance(configuration.stack, token_id)
|
|
140
|
+
return LIMIT if advanced.equal?(LIMIT)
|
|
141
|
+
return unless advanced
|
|
142
|
+
|
|
143
|
+
edits = candidate_edits(configuration, kind, token_id, cost)
|
|
144
|
+
return unless advanced.is_a?(RepairAdvance)
|
|
145
|
+
|
|
146
|
+
shifts = kind == :replace ? configuration.shifts + 1 : configuration.shifts
|
|
147
|
+
input_index = kind == :replace ? configuration.input_index + 1 : configuration.input_index
|
|
148
|
+
push(
|
|
149
|
+
configuration_with(
|
|
150
|
+
configuration,
|
|
151
|
+
stack: advanced.stack,
|
|
152
|
+
input_index: input_index,
|
|
153
|
+
shifts: shifts,
|
|
154
|
+
cost: configuration.cost + cost,
|
|
155
|
+
edits: edits,
|
|
156
|
+
goal: advanced.status == :accept || shifts >= @policy.success_shifts
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
nil
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# @rbs (Configuration configuration, Symbol kind, Integer token_id, Integer cost) -> Array[RepairEdit]
|
|
163
|
+
def candidate_edits(configuration, kind, token_id, cost)
|
|
164
|
+
edit = RepairEdit.new(
|
|
165
|
+
kind: kind,
|
|
166
|
+
position: configuration.input_index,
|
|
167
|
+
token_id: token_id,
|
|
168
|
+
token_name: token_name(token_id),
|
|
169
|
+
cost: cost
|
|
170
|
+
)
|
|
171
|
+
(configuration.edits + [edit]).freeze
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# @rbs (Configuration configuration, RepairInput current) -> void
|
|
175
|
+
def push_delete(configuration, current)
|
|
176
|
+
cost = configuration.cost + @policy.delete_cost
|
|
177
|
+
return if cost > @policy.max_cost
|
|
178
|
+
|
|
179
|
+
edit = RepairEdit.new(
|
|
180
|
+
kind: :delete,
|
|
181
|
+
position: configuration.input_index,
|
|
182
|
+
token_id: current.token_id,
|
|
183
|
+
token_name: current.token_name,
|
|
184
|
+
cost: @policy.delete_cost
|
|
185
|
+
)
|
|
186
|
+
push(
|
|
187
|
+
configuration_with(
|
|
188
|
+
configuration, input_index: configuration.input_index + 1,
|
|
189
|
+
cost: cost,
|
|
190
|
+
edits: (configuration.edits + [edit]).freeze
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# @rbs (Array[Integer] stack, Integer token_id) -> (RepairAdvance | Object | nil)
|
|
196
|
+
def advance(stack, token_id)
|
|
197
|
+
states = stack.dup
|
|
198
|
+
seen = {} #: Hash[Array[Integer], bool]
|
|
199
|
+
loop do
|
|
200
|
+
return nil if seen[states]
|
|
201
|
+
|
|
202
|
+
seen[states.dup.freeze] = true
|
|
203
|
+
@configurations += 1
|
|
204
|
+
return LIMIT if @configurations > @policy.max_configurations
|
|
205
|
+
|
|
206
|
+
action = action_for(states.last, token_id)
|
|
207
|
+
case action.first
|
|
208
|
+
when :shift
|
|
209
|
+
states << action.fetch(1)
|
|
210
|
+
return nil if states.length > @policy.max_stack
|
|
211
|
+
|
|
212
|
+
return RepairAdvance.new(status: :shift, stack: states.freeze)
|
|
213
|
+
when :reduce then return nil unless apply_reduction(states, action.fetch(1))
|
|
214
|
+
when :accept then return RepairAdvance.new(status: :accept, stack: states.freeze)
|
|
215
|
+
else return nil
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# @rbs (Array[Integer] states, Integer production_id) -> Array[Integer]?
|
|
221
|
+
def apply_reduction(states, production_id)
|
|
222
|
+
production = @tables.fetch(:productions).fetch(production_id)
|
|
223
|
+
length = production.fetch(:length)
|
|
224
|
+
return if length >= states.length
|
|
225
|
+
|
|
226
|
+
states.pop(length)
|
|
227
|
+
goto = table_lookup(@tables.fetch(:gotos), states.last, production.fetch(:lhs))
|
|
228
|
+
return unless goto
|
|
229
|
+
|
|
230
|
+
states << goto
|
|
231
|
+
states if states.length <= @policy.max_stack
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# @rbs (Integer state, Integer token_id) -> untyped
|
|
235
|
+
def action_for(state, token_id)
|
|
236
|
+
explicit = table_lookup(@tables.fetch(:actions), state, token_id)
|
|
237
|
+
return explicit if explicit
|
|
238
|
+
|
|
239
|
+
@tables.fetch(:default_actions, Parser::EMPTY_ROW)[state] || [:error]
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# @rbs (untyped table, Integer row, Integer column) -> untyped
|
|
243
|
+
def table_lookup(table, row, column)
|
|
244
|
+
return table.lookup(row, column) if table.respond_to?(:lookup)
|
|
245
|
+
|
|
246
|
+
table.fetch(row, Parser::EMPTY_ROW)[column]
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# @rbs (Configuration source, ?stack: Array[Integer], ?input_index: Integer, ?shifts: Integer,
|
|
250
|
+
# ?cost: Integer, ?edits: Array[RepairEdit], ?goal: bool) -> Configuration
|
|
251
|
+
def configuration_with(source, stack: source.stack, input_index: source.input_index, shifts: source.shifts,
|
|
252
|
+
cost: source.cost, edits: source.edits, goal: source.goal)
|
|
253
|
+
Configuration.new(
|
|
254
|
+
stack: stack, input_index: input_index, shifts: shifts, cost: cost, edits: edits, goal: goal
|
|
255
|
+
)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# @rbs (Array[RepairEdit] edits) -> RepairPlan
|
|
259
|
+
def plan(edits)
|
|
260
|
+
RepairPlan.new(edits: edits, configurations: @configurations)
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# @rbs (Integer token_id) -> String
|
|
264
|
+
def token_name(token_id)
|
|
265
|
+
@tables.fetch(:token_names).fetch(token_id, token_id.to_s)
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# @rbs (Configuration configuration) -> void
|
|
269
|
+
def push(configuration)
|
|
270
|
+
priority = priority_for(configuration)
|
|
271
|
+
key = [configuration.stack, configuration.input_index, configuration.shifts, configuration.goal]
|
|
272
|
+
previous = @best[key]
|
|
273
|
+
comparison = previous <=> priority if previous
|
|
274
|
+
return if comparison && comparison <= 0
|
|
275
|
+
|
|
276
|
+
@best[key] = priority
|
|
277
|
+
@heap.push(priority, configuration)
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# @rbs () -> (Configuration | Object)
|
|
281
|
+
def pop
|
|
282
|
+
loop do
|
|
283
|
+
entry = @heap.pop
|
|
284
|
+
return LIMIT unless entry
|
|
285
|
+
|
|
286
|
+
priority, configuration = entry
|
|
287
|
+
return LIMIT unless configuration.is_a?(Configuration)
|
|
288
|
+
|
|
289
|
+
key = [configuration.stack, configuration.input_index, configuration.shifts, configuration.goal]
|
|
290
|
+
return configuration if @best[key] == priority
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
# @rbs (Configuration configuration) -> Array[untyped]
|
|
295
|
+
def priority_for(configuration)
|
|
296
|
+
edit_key = configuration.edits.map do |edit|
|
|
297
|
+
rank = { delete: 0, insert: 1, replace: 2 }.fetch(edit.kind)
|
|
298
|
+
[edit.position, rank, edit.token_id]
|
|
299
|
+
end
|
|
300
|
+
goal_rank = configuration.goal ? 0 : 1
|
|
301
|
+
risk = configuration.edits.sum { |edit| semantic_value_risk(edit) }
|
|
302
|
+
[
|
|
303
|
+
configuration.cost, risk, edit_key, goal_rank, -configuration.shifts,
|
|
304
|
+
configuration.input_index, configuration.stack
|
|
305
|
+
]
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Prefer punctuation edits when equal-cost repairs are otherwise
|
|
309
|
+
# ambiguous; inventing or discarding a word-like token is more likely to
|
|
310
|
+
# fabricate or lose an application semantic value.
|
|
311
|
+
# @rbs (RepairEdit edit) -> Integer
|
|
312
|
+
def semantic_value_risk(edit)
|
|
313
|
+
edit.token_name.delete_prefix(":").match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/) ? 1 : 0
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
# rubocop:enable Metrics/ClassLength
|
|
317
|
+
end
|
|
318
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module Ibex
|
|
5
|
+
module Runtime
|
|
6
|
+
# Immutable per-parser-session resource budgets.
|
|
7
|
+
class ResourceLimits
|
|
8
|
+
DEFAULT_MAX_STACK_DEPTH = 10_000 #: Integer
|
|
9
|
+
DEFAULT_MAX_RECOVERY_ATTEMPTS = 100 #: Integer
|
|
10
|
+
DEFAULT_MAX_INCREMENTAL_DECOMPOSED_NODES = 100_000 #: Integer
|
|
11
|
+
DEFAULT_MAX_SESSION_MEMO_BYTES = 64 * 1024 * 1024 #: Integer
|
|
12
|
+
|
|
13
|
+
attr_reader :max_stack_depth #: Integer
|
|
14
|
+
attr_reader :max_recovery_attempts #: Integer
|
|
15
|
+
attr_reader :max_incremental_decomposed_nodes #: Integer
|
|
16
|
+
attr_reader :max_session_memo_bytes #: Integer
|
|
17
|
+
|
|
18
|
+
# @rbs (?max_stack_depth: Integer, ?max_recovery_attempts: Integer,
|
|
19
|
+
# ?max_incremental_decomposed_nodes: Integer, ?max_session_memo_bytes: Integer) -> void
|
|
20
|
+
def initialize(
|
|
21
|
+
max_stack_depth: DEFAULT_MAX_STACK_DEPTH,
|
|
22
|
+
max_recovery_attempts: DEFAULT_MAX_RECOVERY_ATTEMPTS,
|
|
23
|
+
max_incremental_decomposed_nodes: DEFAULT_MAX_INCREMENTAL_DECOMPOSED_NODES,
|
|
24
|
+
max_session_memo_bytes: DEFAULT_MAX_SESSION_MEMO_BYTES
|
|
25
|
+
)
|
|
26
|
+
validate_positive_integer(max_stack_depth, :max_stack_depth)
|
|
27
|
+
validate_nonnegative_integer(max_recovery_attempts, :max_recovery_attempts)
|
|
28
|
+
validate_nonnegative_integer(max_incremental_decomposed_nodes, :max_incremental_decomposed_nodes)
|
|
29
|
+
validate_positive_integer(max_session_memo_bytes, :max_session_memo_bytes)
|
|
30
|
+
@max_stack_depth = max_stack_depth
|
|
31
|
+
@max_recovery_attempts = max_recovery_attempts
|
|
32
|
+
@max_incremental_decomposed_nodes = max_incremental_decomposed_nodes
|
|
33
|
+
@max_session_memo_bytes = max_session_memo_bytes
|
|
34
|
+
freeze
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @rbs () -> Hash[Symbol, Integer]
|
|
38
|
+
def to_h
|
|
39
|
+
{
|
|
40
|
+
max_stack_depth: @max_stack_depth,
|
|
41
|
+
max_recovery_attempts: @max_recovery_attempts,
|
|
42
|
+
max_incremental_decomposed_nodes: @max_incremental_decomposed_nodes,
|
|
43
|
+
max_session_memo_bytes: @max_session_memo_bytes
|
|
44
|
+
}.freeze
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
# @rbs (Integer value, Symbol name) -> void
|
|
50
|
+
def validate_positive_integer(value, name)
|
|
51
|
+
return if value.is_a?(Integer) && value.positive?
|
|
52
|
+
|
|
53
|
+
raise ArgumentError, "#{name} must be a positive Integer"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @rbs (Integer value, Symbol name) -> void
|
|
57
|
+
def validate_nonnegative_integer(value, name)
|
|
58
|
+
return if value.is_a?(Integer) && value >= 0
|
|
59
|
+
|
|
60
|
+
raise ArgumentError, "#{name} must be a nonnegative Integer"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module Ibex
|
|
5
|
+
module Runtime
|
|
6
|
+
# Current parser-table shape emitted by the generator.
|
|
7
|
+
PARSER_TABLE_FORMAT_VERSION = 6 #: Integer
|
|
8
|
+
# Parser-table shapes this runtime can execute.
|
|
9
|
+
SUPPORTED_PARSER_TABLE_FORMAT_VERSIONS = [1, 2, 3, 4, 5, PARSER_TABLE_FORMAT_VERSION].freeze #: Array[Integer]
|
|
10
|
+
end
|
|
11
|
+
end
|