ibex-runtime 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +100 -272
- data/lib/ibex/runtime/cst/green/node.rb +1 -1
- data/lib/ibex/runtime/cst/green/token.rb +1 -1
- data/lib/ibex/runtime/cst/green/trivia.rb +1 -1
- data/lib/ibex/runtime/cst/incremental/blender.rb +1 -1
- data/lib/ibex/runtime/cst/incremental/lexed_syntax.rb +2 -2
- data/lib/ibex/runtime/cst/incremental/parse_memo.rb +2 -2
- data/lib/ibex/runtime/cst/incremental/relexer.rb +3 -0
- data/lib/ibex/runtime/cst/incremental/session.rb +26 -2
- data/lib/ibex/runtime/cst/parse_result.rb +7 -7
- data/lib/ibex/runtime/cst/serialize.rb +6 -3
- data/lib/ibex/runtime/cst/syntax_node.rb +4 -4
- data/lib/ibex/runtime/cst/syntax_token.rb +5 -5
- data/lib/ibex/runtime/cst/typed_node.rb +1 -1
- data/lib/ibex/runtime/cst/validator.rb +40 -30
- data/lib/ibex/runtime/embedded_source.rb +98 -0
- data/lib/ibex/runtime/event.rb +7 -4
- data/lib/ibex/runtime/event_jsonl_tracer.rb +1 -1
- data/lib/ibex/runtime/event_sanitizer.rb +32 -26
- data/lib/ibex/runtime/generated_lexer.rb +23 -16
- data/lib/ibex/runtime/lexer_input.rb +1 -1
- data/lib/ibex/runtime/location_span.rb +15 -15
- data/lib/ibex/runtime/observation.rb +1 -1
- data/lib/ibex/runtime/parser.rb +309 -191
- data/lib/ibex/runtime/parser_sync_recovery.rb +19 -15
- data/lib/ibex/runtime/repair.rb +36 -5
- data/lib/ibex/runtime/repair_priority_queue.rb +10 -6
- data/lib/ibex/runtime/repair_search.rb +92 -29
- data/lib/ibex/runtime/syntax_repair.rb +490 -0
- data/lib/ibex/runtime/syntax_session.rb +416 -0
- data/lib/ibex/runtime/table_format.rb +1 -1
- data/lib/ibex/runtime/version.rb +1 -1
- data/lib/ibex/runtime.rb +2 -1
- data/lib/ibex/tables/compact.rb +16 -12
- data/lib/ibex/tables/compact_actions.rb +45 -33
- data/lib/ibex/tables/compact_productions.rb +18 -15
- data/sig/ibex/runtime/cst/green/node.rbs +2 -2
- data/sig/ibex/runtime/cst/green/token.rbs +2 -2
- data/sig/ibex/runtime/cst/green/trivia.rbs +2 -2
- data/sig/ibex/runtime/cst/incremental/blender.rbs +2 -2
- data/sig/ibex/runtime/cst/incremental/lexed_syntax.rbs +3 -3
- data/sig/ibex/runtime/cst/incremental/parse_memo.rbs +4 -4
- data/sig/ibex/runtime/cst/incremental/session.rbs +8 -2
- data/sig/ibex/runtime/cst/parse_result.rbs +9 -9
- data/sig/ibex/runtime/cst/serialize.rbs +8 -6
- data/sig/ibex/runtime/cst/syntax_node.rbs +6 -6
- data/sig/ibex/runtime/cst/syntax_token.rbs +10 -10
- data/sig/ibex/runtime/cst/typed_node.rbs +2 -2
- data/sig/ibex/runtime/cst/validator.rbs +46 -42
- data/sig/ibex/runtime/embedded_source.rbs +21 -0
- data/sig/ibex/runtime/event.rbs +7 -5
- data/sig/ibex/runtime/event_jsonl_tracer.rbs +1 -1
- data/sig/ibex/runtime/event_sanitizer.rbs +50 -44
- data/sig/ibex/runtime/generated_lexer.rbs +31 -24
- data/sig/ibex/runtime/lexer_input.rbs +2 -2
- data/sig/ibex/runtime/location_span.rbs +28 -28
- data/sig/ibex/runtime/observation.rbs +2 -2
- data/sig/ibex/runtime/parser.rbs +266 -252
- data/sig/ibex/runtime/parser_sync_recovery.rbs +15 -15
- data/sig/ibex/runtime/repair.rbs +28 -8
- data/sig/ibex/runtime/repair_priority_queue.rbs +9 -7
- data/sig/ibex/runtime/repair_search.rbs +38 -19
- data/sig/ibex/runtime/syntax_repair.rbs +177 -0
- data/sig/ibex/runtime/syntax_session.rbs +186 -0
- data/sig/ibex/tables/compact.rbs +12 -12
- data/sig/ibex/tables/compact_actions.rbs +10 -17
- data/sig/ibex/tables/compact_productions.rbs +19 -19
- metadata +12 -6
- data/lib/ibex/runtime/jsonl_tracer.rb +0 -75
- data/sig/ibex/runtime/jsonl_tracer.rbs +0 -42
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require_relative "event_sanitizer" unless defined?(Ibex::Runtime::EventSanitizer)
|
|
5
|
+
|
|
6
|
+
module Ibex
|
|
7
|
+
module Runtime
|
|
8
|
+
# Syntax-session additions to the frozen Parser signature surface.
|
|
9
|
+
# @rbs!
|
|
10
|
+
# class Parser
|
|
11
|
+
# def self.syntax_execution_profile: () -> Symbol
|
|
12
|
+
#
|
|
13
|
+
# def self.syntax_session: (String | CST::SourceText source, ?execution_profile: Symbol?,
|
|
14
|
+
# ?resource_limits: ResourceLimits?, ?limits: SyntaxSessionLimits?,
|
|
15
|
+
# ?cancellation: CancellationToken?, ?blender: bool) -> SyntaxSession
|
|
16
|
+
# end
|
|
17
|
+
|
|
18
|
+
# Raised when a caller has not acknowledged the generated artifact's
|
|
19
|
+
# execution profile.
|
|
20
|
+
class SyntaxSessionTrustError < ArgumentError; end
|
|
21
|
+
|
|
22
|
+
# Raised cooperatively at syntax-session operation boundaries and parser
|
|
23
|
+
# observation checkpoints after cancellation is requested.
|
|
24
|
+
class SyntaxSessionCancelled < StandardError; end
|
|
25
|
+
|
|
26
|
+
# Raised when a syntax-service-specific resource bound is exceeded.
|
|
27
|
+
class SyntaxSessionResourceLimitError < StandardError
|
|
28
|
+
attr_reader :resource #: Symbol
|
|
29
|
+
attr_reader :limit #: Integer
|
|
30
|
+
attr_reader :observed #: Integer
|
|
31
|
+
|
|
32
|
+
# @rbs (resource: Symbol, limit: Integer, observed: Integer) -> void
|
|
33
|
+
def initialize(resource:, limit:, observed:)
|
|
34
|
+
@resource = resource
|
|
35
|
+
@limit = limit
|
|
36
|
+
@observed = observed
|
|
37
|
+
super("syntax session resource limit exceeded: #{resource} is #{observed}, configured maximum is #{limit}")
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Thread-safe cooperative cancellation state for one or more operations.
|
|
42
|
+
class CancellationToken
|
|
43
|
+
# @rbs () -> void
|
|
44
|
+
def initialize
|
|
45
|
+
@mutex = Mutex.new #: Mutex
|
|
46
|
+
@cancelled = false #: bool
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @rbs () -> true
|
|
50
|
+
# rubocop:disable Naming/PredicateMethod -- command mutation convention.
|
|
51
|
+
def cancel!
|
|
52
|
+
@mutex.synchronize { @cancelled = true }
|
|
53
|
+
true
|
|
54
|
+
end
|
|
55
|
+
# rubocop:enable Naming/PredicateMethod
|
|
56
|
+
|
|
57
|
+
# @rbs () -> bool
|
|
58
|
+
def cancelled?
|
|
59
|
+
@mutex.synchronize { @cancelled }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Immutable syntax-service bounds layered over parser ResourceLimits.
|
|
64
|
+
class SyntaxSessionLimits
|
|
65
|
+
DEFAULT_MAX_SOURCE_BYTES = 16 * 1024 * 1024 #: Integer
|
|
66
|
+
DEFAULT_MAX_EDITS_PER_OPERATION = 1_024 #: Integer
|
|
67
|
+
DEFAULT_MAX_INSERTED_BYTES = 4 * 1024 * 1024 #: Integer
|
|
68
|
+
|
|
69
|
+
attr_reader :max_source_bytes #: Integer
|
|
70
|
+
attr_reader :max_edits_per_operation #: Integer
|
|
71
|
+
attr_reader :max_inserted_bytes #: Integer
|
|
72
|
+
|
|
73
|
+
# @rbs (?max_source_bytes: Integer, ?max_edits_per_operation: Integer,
|
|
74
|
+
# ?max_inserted_bytes: Integer) -> void
|
|
75
|
+
def initialize(
|
|
76
|
+
max_source_bytes: DEFAULT_MAX_SOURCE_BYTES,
|
|
77
|
+
max_edits_per_operation: DEFAULT_MAX_EDITS_PER_OPERATION,
|
|
78
|
+
max_inserted_bytes: DEFAULT_MAX_INSERTED_BYTES
|
|
79
|
+
)
|
|
80
|
+
validate_positive_integer(max_source_bytes, :max_source_bytes)
|
|
81
|
+
validate_positive_integer(max_edits_per_operation, :max_edits_per_operation)
|
|
82
|
+
validate_nonnegative_integer(max_inserted_bytes, :max_inserted_bytes)
|
|
83
|
+
@max_source_bytes = max_source_bytes
|
|
84
|
+
@max_edits_per_operation = max_edits_per_operation
|
|
85
|
+
@max_inserted_bytes = max_inserted_bytes
|
|
86
|
+
freeze
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private
|
|
90
|
+
|
|
91
|
+
# @rbs (Object? value, Symbol name) -> void
|
|
92
|
+
def validate_positive_integer(value, name)
|
|
93
|
+
return if value.is_a?(Integer) && value.positive?
|
|
94
|
+
|
|
95
|
+
raise ArgumentError, "#{name} must be a positive Integer"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @rbs (Object? value, Symbol name) -> void
|
|
99
|
+
def validate_nonnegative_integer(value, name)
|
|
100
|
+
return if value.is_a?(Integer) && value >= 0
|
|
101
|
+
|
|
102
|
+
raise ArgumentError, "#{name} must be a nonnegative Integer"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Immutable evidence from one completed syntax-session operation.
|
|
107
|
+
class SyntaxSessionMetrics
|
|
108
|
+
attr_reader :reused_ratio #: Float
|
|
109
|
+
attr_reader :reused_tokens #: Integer
|
|
110
|
+
attr_reader :token_count #: Integer
|
|
111
|
+
attr_reader :fallback_reasons #: Array[Symbol]
|
|
112
|
+
|
|
113
|
+
# @rbs (reused_ratio: Float, reused_tokens: Integer, token_count: Integer,
|
|
114
|
+
# fallback_reasons: Array[Symbol]) -> void
|
|
115
|
+
def initialize(reused_ratio:, reused_tokens:, token_count:, fallback_reasons:)
|
|
116
|
+
unless reused_ratio.is_a?(Float) && reused_ratio.finite? && reused_ratio.between?(0.0, 1.0)
|
|
117
|
+
raise ArgumentError, "reused_ratio must be a finite Float in 0.0..1.0"
|
|
118
|
+
end
|
|
119
|
+
unless reused_tokens.is_a?(Integer) && reused_tokens >= 0
|
|
120
|
+
raise ArgumentError, "reused_tokens must be a nonnegative Integer"
|
|
121
|
+
end
|
|
122
|
+
unless token_count.is_a?(Integer) && token_count >= 0
|
|
123
|
+
raise ArgumentError, "token_count must be a nonnegative Integer"
|
|
124
|
+
end
|
|
125
|
+
unless fallback_reasons.is_a?(Array) && fallback_reasons.all?(Symbol)
|
|
126
|
+
raise ArgumentError, "fallback_reasons must contain only Symbols"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
@reused_ratio = reused_ratio
|
|
130
|
+
@reused_tokens = reused_tokens
|
|
131
|
+
@token_count = token_count
|
|
132
|
+
@fallback_reasons = fallback_reasons.dup.freeze
|
|
133
|
+
freeze
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# @rbs () -> bool
|
|
137
|
+
def fallback?
|
|
138
|
+
!@fallback_reasons.empty?
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Immutable, data-only copy of one parser diagnostic. Application-owned
|
|
143
|
+
# values and locations are sanitized rather than retained by identity.
|
|
144
|
+
class SyntaxSessionDiagnostic
|
|
145
|
+
attr_reader :kind #: Symbol
|
|
146
|
+
attr_reader :data #: Hash[String, EventSanitizer::json_value]
|
|
147
|
+
|
|
148
|
+
# @rbs (kind: Symbol, data: Hash[Object?, Object?]) -> void
|
|
149
|
+
def initialize(kind:, data:)
|
|
150
|
+
raise ArgumentError, "diagnostic kind must be a Symbol" unless kind.is_a?(Symbol)
|
|
151
|
+
raise ArgumentError, "diagnostic data must be a Hash" unless data.is_a?(Hash)
|
|
152
|
+
|
|
153
|
+
@kind = kind
|
|
154
|
+
@data = EventSanitizer.data(data)
|
|
155
|
+
freeze
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# @rbs () -> Hash[String, EventSanitizer::json_value]
|
|
159
|
+
def to_h
|
|
160
|
+
@data
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Immutable public snapshot of one completed syntax-session operation.
|
|
165
|
+
class SyntaxSessionResult
|
|
166
|
+
attr_reader :revision #: Integer
|
|
167
|
+
attr_reader :syntax_root #: CST::SyntaxNode
|
|
168
|
+
attr_reader :diagnostics #: Array[SyntaxSessionDiagnostic]
|
|
169
|
+
attr_reader :expected_tokens #: Array[String]
|
|
170
|
+
attr_reader :metrics #: SyntaxSessionMetrics
|
|
171
|
+
attr_reader :status #: Symbol
|
|
172
|
+
|
|
173
|
+
# @rbs (revision: Integer, syntax_root: CST::SyntaxNode, diagnostics: Array[SyntaxSessionDiagnostic],
|
|
174
|
+
# expected_tokens: Array[String], metrics: SyntaxSessionMetrics, status: Symbol) -> void
|
|
175
|
+
def initialize(revision:, syntax_root:, diagnostics:, expected_tokens:, metrics:, status:)
|
|
176
|
+
raise ArgumentError, "revision must be a nonnegative Integer" unless revision.is_a?(Integer) && revision >= 0
|
|
177
|
+
unless diagnostics.is_a?(Array) && diagnostics.all?(SyntaxSessionDiagnostic)
|
|
178
|
+
raise ArgumentError, "diagnostics must contain only SyntaxSessionDiagnostic values"
|
|
179
|
+
end
|
|
180
|
+
unless expected_tokens.is_a?(Array) && expected_tokens.all?(String)
|
|
181
|
+
raise ArgumentError, "expected_tokens must contain only Strings"
|
|
182
|
+
end
|
|
183
|
+
unless metrics.is_a?(SyntaxSessionMetrics)
|
|
184
|
+
raise ArgumentError, "metrics must be an Ibex::Runtime::SyntaxSessionMetrics"
|
|
185
|
+
end
|
|
186
|
+
raise ArgumentError, "status must be :ok or :syntax_error" unless %i[ok syntax_error].include?(status)
|
|
187
|
+
|
|
188
|
+
@revision = revision
|
|
189
|
+
@syntax_root = syntax_root
|
|
190
|
+
@diagnostics = diagnostics.dup.freeze
|
|
191
|
+
@expected_tokens = expected_tokens.map { |token| token.dup.freeze }.freeze
|
|
192
|
+
@metrics = metrics
|
|
193
|
+
@status = status
|
|
194
|
+
freeze
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# @rbs () -> bool
|
|
198
|
+
def success?
|
|
199
|
+
@status == :ok
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# Generated-language syntax boundary backed by the existing Red/Green CST
|
|
204
|
+
# incremental engine. It does not execute parser production actions.
|
|
205
|
+
class SyntaxSession
|
|
206
|
+
TRUSTED_PROFILE = :trusted_application_code #: Symbol
|
|
207
|
+
|
|
208
|
+
attr_reader :execution_profile #: Symbol
|
|
209
|
+
attr_reader :limits #: SyntaxSessionLimits
|
|
210
|
+
|
|
211
|
+
# @rbs (Class parser_class, String | CST::SourceText source,
|
|
212
|
+
# execution_profile: Symbol?, ?resource_limits: ResourceLimits?,
|
|
213
|
+
# ?limits: SyntaxSessionLimits?, ?cancellation: CancellationToken?, ?blender: bool) -> void
|
|
214
|
+
def initialize(
|
|
215
|
+
parser_class,
|
|
216
|
+
source,
|
|
217
|
+
execution_profile:,
|
|
218
|
+
resource_limits: nil,
|
|
219
|
+
limits: nil,
|
|
220
|
+
cancellation: nil,
|
|
221
|
+
blender: true
|
|
222
|
+
)
|
|
223
|
+
@execution_profile = validate_execution_profile(parser_class, execution_profile)
|
|
224
|
+
@limits = limits || SyntaxSessionLimits.new #: SyntaxSessionLimits
|
|
225
|
+
unless @limits.is_a?(SyntaxSessionLimits)
|
|
226
|
+
raise ArgumentError, "limits must be an Ibex::Runtime::SyntaxSessionLimits"
|
|
227
|
+
end
|
|
228
|
+
unless cancellation.nil? || cancellation.is_a?(CancellationToken)
|
|
229
|
+
raise ArgumentError, "cancellation must be an Ibex::Runtime::CancellationToken or nil"
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
@cancellation = cancellation #: CancellationToken?
|
|
233
|
+
@parser_class = parser_class #: Class
|
|
234
|
+
@resource_limits = resource_limits || ResourceLimits.new #: ResourceLimits
|
|
235
|
+
@mutex = Mutex.new #: Mutex
|
|
236
|
+
@revision = 0 #: Integer
|
|
237
|
+
@operation_expected_tokens = [] #: Array[String]
|
|
238
|
+
@operation_fallback_reasons = [] #: Array[Symbol]
|
|
239
|
+
|
|
240
|
+
source_text = normalize_source(source)
|
|
241
|
+
enforce_limit!(:source_bytes, @limits.max_source_bytes, source_text.bytesize)
|
|
242
|
+
cancellation_checkpoint!
|
|
243
|
+
event_observer = ->(event, parser) { handle_runtime_event(event, parser) } #: Proc
|
|
244
|
+
@incremental = CST::IncrementalParseSession.new(
|
|
245
|
+
parser_class,
|
|
246
|
+
source_text,
|
|
247
|
+
resource_limits: @resource_limits,
|
|
248
|
+
blender: blender,
|
|
249
|
+
event_observer: event_observer
|
|
250
|
+
) #: CST::IncrementalParseSession
|
|
251
|
+
@result = snapshot(@incremental.result) #: SyntaxSessionResult
|
|
252
|
+
rescue ResourceLimitError => e
|
|
253
|
+
raise SyntaxSessionResourceLimitError.new(resource: e.resource, limit: e.limit, observed: e.observed), cause: e
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# @rbs () -> CST::SourceText
|
|
257
|
+
def source_text
|
|
258
|
+
@mutex.synchronize { @incremental.source_text }
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# @rbs () -> SyntaxSessionResult
|
|
262
|
+
def result
|
|
263
|
+
@mutex.synchronize { @result }
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# Apply byte-oriented edits against the current source.
|
|
267
|
+
# @rbs (Array[CST::TextEdit] edits) -> SyntaxSessionResult
|
|
268
|
+
def apply_edits(edits)
|
|
269
|
+
@mutex.synchronize { apply_edits_locked(edits) }
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Propose byte edits through the existing bounded repair search, then
|
|
273
|
+
# validate them with a fresh syntax-only parse. The session is unchanged.
|
|
274
|
+
# @rbs (?policy: RepairPolicy, ?token_text: Hash[String, String]) -> SyntaxRepairResult
|
|
275
|
+
def repair(policy: RepairPolicy.new, token_text: {})
|
|
276
|
+
@mutex.synchronize do
|
|
277
|
+
SyntaxRepairer.new(
|
|
278
|
+
@parser_class, @incremental.source_text, @result,
|
|
279
|
+
execution_profile: @execution_profile, resource_limits: @resource_limits,
|
|
280
|
+
limits: @limits, cancellation: @cancellation, policy: policy, token_text: token_text
|
|
281
|
+
).call
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
private
|
|
286
|
+
|
|
287
|
+
# @rbs (Array[CST::TextEdit] edits) -> SyntaxSessionResult
|
|
288
|
+
def apply_edits_locked(edits)
|
|
289
|
+
raise ArgumentError, "edits must be an Array" unless edits.is_a?(Array)
|
|
290
|
+
|
|
291
|
+
cancellation_checkpoint!
|
|
292
|
+
enforce_limit!(:edits_per_operation, @limits.max_edits_per_operation, edits.length)
|
|
293
|
+
normalized = CST::TextEdit.normalize(edits)
|
|
294
|
+
inserted_bytes = normalized.sum { |edit| edit.insert_text.bytesize }
|
|
295
|
+
enforce_limit!(:inserted_bytes, @limits.max_inserted_bytes, inserted_bytes)
|
|
296
|
+
next_source = @incremental.source_text.apply(normalized)
|
|
297
|
+
enforce_limit!(:source_bytes, @limits.max_source_bytes, next_source.bytesize)
|
|
298
|
+
return @result if normalized.empty?
|
|
299
|
+
|
|
300
|
+
reset_operation_evidence
|
|
301
|
+
parsed = @incremental.edit(normalized)
|
|
302
|
+
@revision += 1
|
|
303
|
+
@result = snapshot(parsed)
|
|
304
|
+
rescue ResourceLimitError => e
|
|
305
|
+
raise SyntaxSessionResourceLimitError.new(resource: e.resource, limit: e.limit, observed: e.observed), cause: e
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# @rbs (Class parser_class, Symbol? acknowledged) -> Symbol
|
|
309
|
+
def validate_execution_profile(parser_class, acknowledged)
|
|
310
|
+
unless parser_class.is_a?(Class) && parser_class <= Parser
|
|
311
|
+
raise ArgumentError, "parser_class must inherit from Ibex::Runtime::Parser"
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
profile = parser_class.syntax_execution_profile
|
|
315
|
+
unless profile == TRUSTED_PROFILE
|
|
316
|
+
raise SyntaxSessionTrustError,
|
|
317
|
+
"declarative syntax artifacts are not available; generated parser profile was #{profile.inspect}"
|
|
318
|
+
end
|
|
319
|
+
return profile if acknowledged == profile
|
|
320
|
+
|
|
321
|
+
raise SyntaxSessionTrustError,
|
|
322
|
+
"syntax_session executes generated lexer actions; " \
|
|
323
|
+
"pass execution_profile: #{profile.inspect} to acknowledge it"
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# @rbs (String | CST::SourceText source) -> CST::SourceText
|
|
327
|
+
def normalize_source(source)
|
|
328
|
+
return source if source.is_a?(CST::SourceText)
|
|
329
|
+
raise ArgumentError, "source must be a String or Ibex::Runtime::CST::SourceText" unless source.is_a?(String)
|
|
330
|
+
unless source.encoding.ascii_compatible?
|
|
331
|
+
raise Encoding::CompatibilityError, "syntax session source must use an ASCII-compatible encoding"
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
CST::SourceText.new(source)
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# @rbs (Symbol resource, Integer limit, Integer observed) -> void
|
|
338
|
+
def enforce_limit!(resource, limit, observed)
|
|
339
|
+
return unless observed > limit
|
|
340
|
+
|
|
341
|
+
raise SyntaxSessionResourceLimitError.new(resource: resource, limit: limit, observed: observed)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# @rbs () -> void
|
|
345
|
+
def cancellation_checkpoint!
|
|
346
|
+
return unless @cancellation&.cancelled?
|
|
347
|
+
|
|
348
|
+
raise SyntaxSessionCancelled, "syntax session operation was cancelled"
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# @rbs (Event event, Parser parser) -> void
|
|
352
|
+
def handle_runtime_event(event, parser)
|
|
353
|
+
cancellation_checkpoint!
|
|
354
|
+
case event.type
|
|
355
|
+
when :error
|
|
356
|
+
parser.expected_tokens.each do |token|
|
|
357
|
+
@operation_expected_tokens << token unless @operation_expected_tokens.include?(token)
|
|
358
|
+
end
|
|
359
|
+
when :cst_fallback
|
|
360
|
+
@operation_fallback_reasons << event.data.fetch("reason").to_s.to_sym
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# @rbs () -> void
|
|
365
|
+
def reset_operation_evidence
|
|
366
|
+
@operation_expected_tokens = [] #: Array[String]
|
|
367
|
+
@operation_fallback_reasons = [] #: Array[Symbol]
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
# @rbs (CST::SyntaxResult parsed) -> SyntaxSessionResult
|
|
371
|
+
def snapshot(parsed)
|
|
372
|
+
token_count = @incremental.token_memo.tokens.length
|
|
373
|
+
reused_tokens = if @incremental.last_full_fallback?
|
|
374
|
+
0
|
|
375
|
+
else
|
|
376
|
+
@incremental.last_relex_result&.reused_count.to_i
|
|
377
|
+
end
|
|
378
|
+
metrics = SyntaxSessionMetrics.new(
|
|
379
|
+
reused_ratio: parsed.reused_ratio,
|
|
380
|
+
reused_tokens: reused_tokens,
|
|
381
|
+
token_count: token_count,
|
|
382
|
+
fallback_reasons: @operation_fallback_reasons
|
|
383
|
+
)
|
|
384
|
+
SyntaxSessionResult.new(
|
|
385
|
+
revision: @revision,
|
|
386
|
+
syntax_root: parsed.syntax_root,
|
|
387
|
+
diagnostics: immutable_diagnostics(parsed.diagnostics),
|
|
388
|
+
expected_tokens: @operation_expected_tokens,
|
|
389
|
+
metrics: metrics,
|
|
390
|
+
status: parsed.diagnostics.empty? ? :ok : :syntax_error
|
|
391
|
+
)
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
# @rbs (Array[Object?] diagnostics) -> Array[SyntaxSessionDiagnostic]
|
|
395
|
+
def immutable_diagnostics(diagnostics)
|
|
396
|
+
diagnostics.map do |diagnostic|
|
|
397
|
+
if diagnostic.is_a?(ParseError)
|
|
398
|
+
SyntaxSessionDiagnostic.new(
|
|
399
|
+
kind: :parse_error,
|
|
400
|
+
data: {
|
|
401
|
+
message: diagnostic.message,
|
|
402
|
+
token_id: diagnostic.token_id,
|
|
403
|
+
token_name: diagnostic.token_name,
|
|
404
|
+
expected_tokens: diagnostic.expected_tokens,
|
|
405
|
+
location: EventSanitizer.location(diagnostic.location)
|
|
406
|
+
}
|
|
407
|
+
)
|
|
408
|
+
else
|
|
409
|
+
data = diagnostic.is_a?(Hash) ? diagnostic : { message: diagnostic.to_s }
|
|
410
|
+
SyntaxSessionDiagnostic.new(kind: :syntax_error, data: data)
|
|
411
|
+
end
|
|
412
|
+
end.freeze
|
|
413
|
+
end
|
|
414
|
+
end
|
|
415
|
+
end
|
|
416
|
+
end
|
|
@@ -6,6 +6,6 @@ module Ibex
|
|
|
6
6
|
# Current parser-table shape emitted by the generator.
|
|
7
7
|
PARSER_TABLE_FORMAT_VERSION = 6 #: Integer
|
|
8
8
|
# Parser-table shapes this runtime can execute.
|
|
9
|
-
SUPPORTED_PARSER_TABLE_FORMAT_VERSIONS = [
|
|
9
|
+
SUPPORTED_PARSER_TABLE_FORMAT_VERSIONS = [PARSER_TABLE_FORMAT_VERSION].freeze #: Array[Integer]
|
|
10
10
|
end
|
|
11
11
|
end
|
data/lib/ibex/runtime/version.rb
CHANGED
data/lib/ibex/runtime.rb
CHANGED
|
@@ -11,14 +11,15 @@ require_relative "runtime/observation"
|
|
|
11
11
|
require_relative "runtime/cst"
|
|
12
12
|
require_relative "runtime/ast_data"
|
|
13
13
|
require_relative "runtime/resource_limits"
|
|
14
|
+
require_relative "runtime/syntax_session"
|
|
14
15
|
require_relative "runtime/repair"
|
|
15
16
|
require_relative "runtime/repair_priority_queue"
|
|
16
17
|
require_relative "runtime/repair_search"
|
|
18
|
+
require_relative "runtime/syntax_repair"
|
|
17
19
|
require_relative "runtime/parser_sync_recovery"
|
|
18
20
|
require_relative "runtime/parser"
|
|
19
21
|
require_relative "runtime/lexer_input"
|
|
20
22
|
require_relative "runtime/generated_lexer"
|
|
21
|
-
require_relative "runtime/jsonl_tracer"
|
|
22
23
|
require_relative "runtime/event_jsonl_tracer"
|
|
23
24
|
|
|
24
25
|
module Ibex
|
data/lib/ibex/tables/compact.rb
CHANGED
|
@@ -6,8 +6,6 @@ module Ibex
|
|
|
6
6
|
# Encodes nonnegative integer arrays into compact generated-source
|
|
7
7
|
# literals. Zero represents nil; positive values are offset by one.
|
|
8
8
|
module PackedIntegers
|
|
9
|
-
module_function
|
|
10
|
-
|
|
11
9
|
# @rbs (Array[Integer?] values) -> String
|
|
12
10
|
def encode(values)
|
|
13
11
|
encoded = values.map do |value|
|
|
@@ -58,6 +56,12 @@ module Ibex
|
|
|
58
56
|
value.even? ? value / 2 : -((value + 1) / 2)
|
|
59
57
|
end
|
|
60
58
|
end
|
|
59
|
+
|
|
60
|
+
module_function :encode
|
|
61
|
+
module_function :decode
|
|
62
|
+
module_function :decode_required
|
|
63
|
+
module_function :encode_signed
|
|
64
|
+
module_function :decode_signed
|
|
61
65
|
end
|
|
62
66
|
|
|
63
67
|
# Sparse table represented by per-row offsets and ownership checks.
|
|
@@ -65,17 +69,17 @@ module Ibex
|
|
|
65
69
|
DENSE_CELL_LIMIT = 1_000_000 #: Integer
|
|
66
70
|
|
|
67
71
|
attr_reader :offsets #: Array[Integer]
|
|
68
|
-
attr_reader :values #: Array[
|
|
72
|
+
attr_reader :values #: Array[Object?]
|
|
69
73
|
attr_reader :checks #: Array[Integer?]
|
|
70
74
|
attr_reader :row_count #: Integer
|
|
71
|
-
attr_reader :dense_values #: Array[
|
|
75
|
+
attr_reader :dense_values #: Array[Object?]?
|
|
72
76
|
attr_reader :dense_width #: Integer?
|
|
73
77
|
|
|
74
78
|
class << self
|
|
75
|
-
# @rbs (Array[Hash[Integer,
|
|
79
|
+
# @rbs (Array[Hash[Integer, Object?]] rows, ?dense: bool) -> Compact
|
|
76
80
|
def build(rows, dense: true)
|
|
77
81
|
offsets = Array.new(rows.length, 0)
|
|
78
|
-
values = [] #: Array[
|
|
82
|
+
values = [] #: Array[Object?]
|
|
79
83
|
checks = [] #: Array[Integer?]
|
|
80
84
|
next_offsets = {} #: Hash[Array[Integer], Integer]
|
|
81
85
|
rows.each_index.sort_by { |row| [-rows[row].length, row] }.each do |row|
|
|
@@ -121,7 +125,7 @@ module Ibex
|
|
|
121
125
|
end
|
|
122
126
|
end
|
|
123
127
|
|
|
124
|
-
# @rbs (offsets: Array[Integer], values: Array[
|
|
128
|
+
# @rbs (offsets: Array[Integer], values: Array[Object?], checks: Array[Integer?],
|
|
125
129
|
# row_count: Integer, ?dense_width: Integer?) -> void
|
|
126
130
|
def initialize(offsets:, values:, checks:, row_count:, dense_width: nil)
|
|
127
131
|
@offsets = offsets.freeze
|
|
@@ -136,7 +140,7 @@ module Ibex
|
|
|
136
140
|
# Keep predicate dispatch out of this lookup because every parser action
|
|
137
141
|
# and goto crosses it.
|
|
138
142
|
# rubocop:disable Style/NumericPredicate
|
|
139
|
-
# @rbs (Integer row, Integer column) ->
|
|
143
|
+
# @rbs (Integer row, Integer column) -> Object?
|
|
140
144
|
def lookup(row, column)
|
|
141
145
|
return nil if row < 0 || row >= @row_count || column < 0
|
|
142
146
|
|
|
@@ -147,11 +151,11 @@ module Ibex
|
|
|
147
151
|
end
|
|
148
152
|
# rubocop:enable Style/NumericPredicate
|
|
149
153
|
|
|
150
|
-
# @rbs (Integer row) -> Hash[Integer,
|
|
154
|
+
# @rbs (Integer row) -> Hash[Integer, Object?]
|
|
151
155
|
def row(row)
|
|
152
156
|
return {} unless row.between?(0, @row_count - 1)
|
|
153
157
|
|
|
154
|
-
result = {} #: Hash[Integer,
|
|
158
|
+
result = {} #: Hash[Integer, Object?]
|
|
155
159
|
@checks.each_index do |index|
|
|
156
160
|
next unless @checks[index] == row
|
|
157
161
|
|
|
@@ -162,13 +166,13 @@ module Ibex
|
|
|
162
166
|
|
|
163
167
|
private
|
|
164
168
|
|
|
165
|
-
# @rbs (Integer? dense_width) -> Array[
|
|
169
|
+
# @rbs (Integer? dense_width) -> Array[Object?]?
|
|
166
170
|
def dense_layout(dense_width)
|
|
167
171
|
return nil unless dense_width
|
|
168
172
|
raise ArgumentError, "compact dense width must be positive" unless dense_width.positive?
|
|
169
173
|
return nil if (@row_count * dense_width) > DENSE_CELL_LIMIT
|
|
170
174
|
|
|
171
|
-
dense = Array.new(@row_count * dense_width) #: Array[
|
|
175
|
+
dense = Array.new(@row_count * dense_width) #: Array[Object?]
|
|
172
176
|
@checks.each_index do |index|
|
|
173
177
|
row = @checks[index]
|
|
174
178
|
next unless row
|
|
@@ -12,27 +12,24 @@ module Ibex
|
|
|
12
12
|
ERROR_CODE = -1 #: Integer
|
|
13
13
|
SHIFT_BASE = 1 #: Integer
|
|
14
14
|
REDUCE_BASE = -2 #: Integer
|
|
15
|
-
LEGACY_ERROR_CODE = 1 #: Integer
|
|
16
|
-
LEGACY_SHIFT_BASE = 2 #: Integer
|
|
17
|
-
LEGACY_REDUCE_BASE = 3 #: Integer
|
|
18
|
-
private_constant :LEGACY_ERROR_CODE, :LEGACY_SHIFT_BASE, :LEGACY_REDUCE_BASE
|
|
19
15
|
|
|
20
16
|
attr_reader :codes #: Array[Integer?]
|
|
21
17
|
attr_reader :dense_codes #: Array[Integer?]?
|
|
22
18
|
attr_reader :column_count #: Integer?
|
|
23
19
|
|
|
24
20
|
class << self
|
|
25
|
-
# @rbs (Array[Hash[Integer,
|
|
21
|
+
# @rbs (Array[Hash[Integer, IR::runtime_action]] rows) -> CompactActions
|
|
26
22
|
def build(rows)
|
|
27
23
|
packed_rows = rows.map do |row|
|
|
28
24
|
row.transform_values { |action| pack(action) }
|
|
29
25
|
end
|
|
30
26
|
layout = Compact.build(packed_rows, dense: false)
|
|
27
|
+
codes = layout.values #: Array[Integer?]
|
|
31
28
|
column_count = rows.flat_map(&:keys).max.to_i + 1
|
|
32
29
|
column_count = nil if (rows.length * column_count) > Compact::DENSE_CELL_LIMIT
|
|
33
30
|
new(
|
|
34
31
|
offsets: layout.offsets,
|
|
35
|
-
codes:
|
|
32
|
+
codes: codes,
|
|
36
33
|
checks: layout.checks,
|
|
37
34
|
row_count: layout.row_count,
|
|
38
35
|
encoding: :signed,
|
|
@@ -42,10 +39,14 @@ module Ibex
|
|
|
42
39
|
|
|
43
40
|
# @rbs (String offsets, String codes, String checks, row_count: Integer,
|
|
44
41
|
# ?encoding: Symbol, ?column_count: Integer?) -> CompactActions
|
|
45
|
-
def packed(offsets, codes, checks, row_count:, encoding: :
|
|
42
|
+
def packed(offsets, codes, checks, row_count:, encoding: :signed, column_count: nil)
|
|
43
|
+
unless encoding == :signed
|
|
44
|
+
raise ArgumentError, "unknown compact action encoding #{encoding.inspect}; expected :signed"
|
|
45
|
+
end
|
|
46
|
+
|
|
46
47
|
new(
|
|
47
48
|
offsets: PackedIntegers.decode_required(offsets),
|
|
48
|
-
codes:
|
|
49
|
+
codes: PackedIntegers.decode_signed(codes),
|
|
49
50
|
checks: PackedIntegers.decode(checks),
|
|
50
51
|
row_count: row_count,
|
|
51
52
|
encoding: encoding,
|
|
@@ -53,60 +54,71 @@ module Ibex
|
|
|
53
54
|
)
|
|
54
55
|
end
|
|
55
56
|
|
|
56
|
-
# @rbs (
|
|
57
|
+
# @rbs (IR::runtime_action? action) -> Integer?
|
|
57
58
|
def pack(action)
|
|
58
59
|
return unless action
|
|
59
60
|
|
|
60
61
|
case action.fetch(0)
|
|
61
62
|
when :accept then ACCEPT_CODE
|
|
62
63
|
when :error then ERROR_CODE
|
|
63
|
-
when :shift
|
|
64
|
-
|
|
64
|
+
when :shift
|
|
65
|
+
state = action.fetch(1) #: Integer
|
|
66
|
+
SHIFT_BASE + state
|
|
67
|
+
when :reduce
|
|
68
|
+
production = action.fetch(1) #: Integer
|
|
69
|
+
REDUCE_BASE - production
|
|
65
70
|
else raise ArgumentError, "unknown compact parser action #{action.inspect}"
|
|
66
71
|
end
|
|
67
72
|
end
|
|
68
73
|
|
|
69
|
-
# @rbs (Integer? code) ->
|
|
74
|
+
# @rbs (Integer? code) -> IR::runtime_action?
|
|
70
75
|
def unpack(code)
|
|
71
76
|
return unless code
|
|
72
|
-
return [:accept].freeze if code == ACCEPT_CODE
|
|
73
|
-
return [:error].freeze if code == ERROR_CODE
|
|
74
|
-
return [:shift, code - SHIFT_BASE].freeze if code.positive?
|
|
75
|
-
|
|
76
|
-
[:reduce, REDUCE_BASE - code].freeze
|
|
77
|
-
end
|
|
78
77
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
78
|
+
if code == ACCEPT_CODE
|
|
79
|
+
accept = [:accept].freeze #: [:accept]
|
|
80
|
+
return accept
|
|
81
|
+
end
|
|
82
|
+
if code == ERROR_CODE
|
|
83
|
+
error = [:error].freeze #: [:error]
|
|
84
|
+
return error
|
|
85
|
+
end
|
|
86
|
+
if code.positive?
|
|
87
|
+
shift = [:shift, code - SHIFT_BASE].freeze #: [:shift, Integer]
|
|
88
|
+
return shift
|
|
89
|
+
end
|
|
87
90
|
|
|
88
|
-
REDUCE_BASE -
|
|
91
|
+
[:reduce, REDUCE_BASE - code].freeze #: [:reduce, Integer]
|
|
89
92
|
end
|
|
90
93
|
end
|
|
91
94
|
|
|
92
95
|
# @rbs (offsets: Array[Integer], codes: Array[Integer?], checks: Array[Integer?],
|
|
93
96
|
# row_count: Integer, ?encoding: Symbol, ?column_count: Integer?) -> void
|
|
94
|
-
def initialize(offsets:, codes:, checks:, row_count:, encoding: :
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
raise ArgumentError, "unknown compact action encoding #{encoding.inspect}"
|
|
97
|
+
def initialize(offsets:, codes:, checks:, row_count:, encoding: :signed, column_count: nil)
|
|
98
|
+
unless encoding == :signed
|
|
99
|
+
raise ArgumentError, "unknown compact action encoding #{encoding.inspect}; expected :signed"
|
|
98
100
|
end
|
|
99
101
|
|
|
100
102
|
@codes = codes.freeze
|
|
101
103
|
@column_count = column_count
|
|
102
104
|
@dense_codes = dense_action_layout(offsets, codes, checks, row_count, column_count)
|
|
103
|
-
decoded_cache = {} #: Hash[Integer,
|
|
105
|
+
decoded_cache = {} #: Hash[Integer, IR::runtime_action]
|
|
104
106
|
decoded = codes.map do |code|
|
|
105
|
-
|
|
107
|
+
next unless code
|
|
108
|
+
|
|
109
|
+
unpacked = self.class.unpack(code) #: IR::runtime_action
|
|
110
|
+
decoded_cache[code] ||= unpacked
|
|
106
111
|
end
|
|
107
112
|
super(offsets: offsets, values: decoded, checks: checks, row_count: row_count)
|
|
108
113
|
end
|
|
109
114
|
|
|
115
|
+
# @rbs (Integer row) -> Hash[Integer, IR::runtime_action]
|
|
116
|
+
# rubocop:disable Lint/UselessMethodDefinition -- narrows the inherited row contract for typed parser actions.
|
|
117
|
+
def row(row)
|
|
118
|
+
super #: Hash[Integer, IR::runtime_action]
|
|
119
|
+
end
|
|
120
|
+
# rubocop:enable Lint/UselessMethodDefinition
|
|
121
|
+
|
|
110
122
|
private
|
|
111
123
|
|
|
112
124
|
# @rbs (Array[Integer] offsets, Array[Integer?] codes, Array[Integer?] checks,
|