ibex-runtime 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +95 -272
- data/lib/ibex/runtime/cst/green/node.rb +1 -1
- data/lib/ibex/runtime/cst/green/token.rb +1 -1
- data/lib/ibex/runtime/cst/green/trivia.rb +1 -1
- data/lib/ibex/runtime/cst/incremental/blender.rb +1 -1
- data/lib/ibex/runtime/cst/incremental/lexed_syntax.rb +2 -2
- data/lib/ibex/runtime/cst/incremental/parse_memo.rb +2 -2
- data/lib/ibex/runtime/cst/incremental/relexer.rb +3 -0
- data/lib/ibex/runtime/cst/incremental/session.rb +26 -2
- data/lib/ibex/runtime/cst/parse_result.rb +7 -7
- data/lib/ibex/runtime/cst/serialize.rb +6 -3
- data/lib/ibex/runtime/cst/syntax_node.rb +4 -4
- data/lib/ibex/runtime/cst/syntax_token.rb +5 -5
- data/lib/ibex/runtime/cst/typed_node.rb +1 -1
- data/lib/ibex/runtime/cst/validator.rb +40 -30
- data/lib/ibex/runtime/embedded_source.rb +98 -0
- data/lib/ibex/runtime/event.rb +7 -4
- data/lib/ibex/runtime/event_jsonl_tracer.rb +1 -1
- data/lib/ibex/runtime/event_sanitizer.rb +32 -26
- data/lib/ibex/runtime/generated_lexer.rb +23 -16
- data/lib/ibex/runtime/lexer_input.rb +1 -1
- data/lib/ibex/runtime/location_span.rb +15 -15
- data/lib/ibex/runtime/observation.rb +1 -1
- data/lib/ibex/runtime/parser.rb +309 -191
- data/lib/ibex/runtime/parser_sync_recovery.rb +19 -15
- data/lib/ibex/runtime/repair.rb +36 -5
- data/lib/ibex/runtime/repair_priority_queue.rb +10 -6
- data/lib/ibex/runtime/repair_search.rb +92 -29
- data/lib/ibex/runtime/syntax_repair.rb +490 -0
- data/lib/ibex/runtime/syntax_session.rb +416 -0
- data/lib/ibex/runtime/table_format.rb +1 -1
- data/lib/ibex/runtime/version.rb +1 -1
- data/lib/ibex/runtime.rb +2 -1
- data/lib/ibex/tables/compact.rb +16 -12
- data/lib/ibex/tables/compact_actions.rb +45 -33
- data/lib/ibex/tables/compact_productions.rb +18 -15
- data/sig/ibex/runtime/cst/green/node.rbs +2 -2
- data/sig/ibex/runtime/cst/green/token.rbs +2 -2
- data/sig/ibex/runtime/cst/green/trivia.rbs +2 -2
- data/sig/ibex/runtime/cst/incremental/blender.rbs +2 -2
- data/sig/ibex/runtime/cst/incremental/lexed_syntax.rbs +3 -3
- data/sig/ibex/runtime/cst/incremental/parse_memo.rbs +4 -4
- data/sig/ibex/runtime/cst/incremental/session.rbs +8 -2
- data/sig/ibex/runtime/cst/parse_result.rbs +9 -9
- data/sig/ibex/runtime/cst/serialize.rbs +8 -6
- data/sig/ibex/runtime/cst/syntax_node.rbs +6 -6
- data/sig/ibex/runtime/cst/syntax_token.rbs +10 -10
- data/sig/ibex/runtime/cst/typed_node.rbs +2 -2
- data/sig/ibex/runtime/cst/validator.rbs +46 -42
- data/sig/ibex/runtime/embedded_source.rbs +21 -0
- data/sig/ibex/runtime/event.rbs +7 -5
- data/sig/ibex/runtime/event_jsonl_tracer.rbs +1 -1
- data/sig/ibex/runtime/event_sanitizer.rbs +50 -44
- data/sig/ibex/runtime/generated_lexer.rbs +31 -24
- data/sig/ibex/runtime/lexer_input.rbs +2 -2
- data/sig/ibex/runtime/location_span.rbs +28 -28
- data/sig/ibex/runtime/observation.rbs +2 -2
- data/sig/ibex/runtime/parser.rbs +266 -252
- data/sig/ibex/runtime/parser_sync_recovery.rbs +15 -15
- data/sig/ibex/runtime/repair.rbs +28 -8
- data/sig/ibex/runtime/repair_priority_queue.rbs +9 -7
- data/sig/ibex/runtime/repair_search.rbs +38 -19
- data/sig/ibex/runtime/syntax_repair.rbs +177 -0
- data/sig/ibex/runtime/syntax_session.rbs +186 -0
- data/sig/ibex/tables/compact.rbs +12 -12
- data/sig/ibex/tables/compact_actions.rbs +10 -17
- data/sig/ibex/tables/compact_productions.rbs +19 -19
- metadata +12 -6
- data/lib/ibex/runtime/jsonl_tracer.rb +0 -75
- data/sig/ibex/runtime/jsonl_tracer.rbs +0 -42
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6086c1dd68cc99d62aa03a04395e07790c29611e86aadf39a27380ed970517c3
|
|
4
|
+
data.tar.gz: 56653912d556177e7126e6075816d72f1183c86d1649e9c760466dd739874097
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: acf1c8076df1e37169b4f8a4597cb4b9bb461b16b8878676032a558d3ba7cf6c7bcffda13e42d4eb6034d47b4cc8c1cfd8263b0b9af7f86429356a83e9a4eb22
|
|
7
|
+
data.tar.gz: 0be19823f4fd459cc60c9fe097d7b278ef2c92309858070aa0911e9032cd13250baa3ed6b9e23f00f85ce56ebbc569cdd655107ba61a50aa2f9511eef44d6a9f
|
data/README.md
CHANGED
|
@@ -1,71 +1,55 @@
|
|
|
1
1
|
# Ibex
|
|
2
2
|
|
|
3
|
-
[](https://rubygems.org/gems/ibex)
|
|
4
4
|
[](https://github.com/ydah/ibex/actions/workflows/main.yml)
|
|
5
|
+
[](https://ydah.github.io/ibex/)
|
|
6
|
+
[](LICENSE.txt)
|
|
5
7
|
|
|
6
8
|
Ibex is a Pure Ruby LR parser generator. It reads racc-compatible grammar
|
|
7
9
|
files, generates parsers with the familiar `do_parse` / `yyparse` API, and
|
|
8
|
-
requires no
|
|
9
|
-
|
|
10
|
-
> [!IMPORTANT]
|
|
11
|
-
> Ibex is pre-1.0. The current v1.0 decision is
|
|
12
|
-
> [**HOLD**](docs/release-readiness.md): compatibility and scale evidence pass,
|
|
13
|
-
> but cold generation and new-instance runtime still trail Racc's Ruby backend,
|
|
14
|
-
> and the error UX evidence still needs independent review. Compatible mode is
|
|
15
|
-
> the stable baseline; opt-in features have the maturity levels documented in
|
|
16
|
-
> the [stability policy](docs/stability.md).
|
|
17
|
-
|
|
18
|
-
- Try grammar analysis in the
|
|
19
|
-
[browser playground](https://ydah.github.io/ibex/playground/).
|
|
20
|
-
- Start with the [grammar reference](docs/grammar-reference.md).
|
|
21
|
-
- Build lossless and incremental tooling with the
|
|
22
|
-
[Red/Green CST guide](docs/cst.md).
|
|
23
|
-
- Migrate an existing parser with the
|
|
24
|
-
[racc migration guide](docs/racc-migration.md).
|
|
25
|
-
- Browse the [API reference](https://ydah.github.io/ibex/api/).
|
|
26
|
-
|
|
27
|
-
The playground runs in a local Web Worker. It does not upload grammar source
|
|
28
|
-
or execute semantic action bodies.
|
|
29
|
-
|
|
30
|
-
## At a glance
|
|
31
|
-
|
|
32
|
-
| Concern | Ibex default |
|
|
33
|
-
| --- | --- |
|
|
34
|
-
| Grammar input | racc-compatible `.y` source |
|
|
35
|
-
| Construction | Direct LALR(1) |
|
|
36
|
-
| Generated tables | Compact, immutable Ruby data |
|
|
37
|
-
| Parser API | Pull parsing with `do_parse` / `yyparse` |
|
|
38
|
-
| Generated dependency | The separately versioned `ibex-runtime` gem |
|
|
39
|
-
| Ruby support | Ruby 3.0 or later |
|
|
40
|
-
| Extensions | Disabled unless `pragma extended` or `--mode=extended` is used |
|
|
41
|
-
| Concrete syntax | Stable format-v6 Red/Green CST with `pragma cst` |
|
|
42
|
-
| Semantic actions | Opaque during generation and analysis; executed only by a running parser |
|
|
43
|
-
|
|
44
|
-
The generator and runtime are deliberately separate:
|
|
10
|
+
requires no native extension.
|
|
45
11
|
|
|
46
|
-
|
|
47
|
-
grammar
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
+--------> optional RBS, IR, reports, diagrams, and docs
|
|
53
|
-
```
|
|
12
|
+
The shortest path is the [Getting Started guide](https://ydah.github.io/ibex/getting-started/):
|
|
13
|
+
install the gem, try the calculator grammar, and then choose the compatible or
|
|
14
|
+
extended contract you need. You can also [inspect a grammar in the browser](https://ydah.github.io/ibex/playground/)
|
|
15
|
+
without uploading its source.
|
|
16
|
+
|
|
17
|
+
## Release status
|
|
54
18
|
|
|
55
|
-
|
|
56
|
-
|
|
19
|
+
Ibex is pre-1.0. The v1.0 publication decision is on hold pending an
|
|
20
|
+
independent review of the published error-experience evidence; feature
|
|
21
|
+
development continues. The default compatible mode and its current IR
|
|
22
|
+
contracts are the conservative adoption baseline. See the [release status](https://ydah.github.io/ibex/project/status/)
|
|
23
|
+
and [stability policy](docs/stability.md) for the human-readable boundaries.
|
|
57
24
|
|
|
58
|
-
|
|
25
|
+
<!-- r001-review-status:start -->
|
|
26
|
+
R001: `HOLD` — [`awaiting_independent_review`](docs/error-ux-review-status-v1.json).
|
|
27
|
+
<!-- r001-review-status:end -->
|
|
59
28
|
|
|
60
|
-
|
|
29
|
+
<!-- type-stats:start -->
|
|
30
|
+
The current whole-library `steep stats` result is 28,836 typed calls and 2,734 untyped calls out of 31,610 (91.2% typed).
|
|
31
|
+
The generated signature tree contains 492 explicit `untyped` occurrences across 39 files.
|
|
32
|
+
<!-- type-stats:end -->
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
Add the generator to an application or tool project:
|
|
61
37
|
|
|
62
38
|
```sh
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
bundle
|
|
39
|
+
gem install ibex
|
|
40
|
+
# or, in a Gemfile:
|
|
41
|
+
bundle add ibex
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Applications that only run generated parsers can depend on the smaller runtime:
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
bundle add ibex-runtime
|
|
66
48
|
```
|
|
67
49
|
|
|
68
|
-
|
|
50
|
+
## First parser
|
|
51
|
+
|
|
52
|
+
Save this as `calculator.y`:
|
|
69
53
|
|
|
70
54
|
<!-- calculator-grammar:start -->
|
|
71
55
|
```text
|
|
@@ -97,241 +81,80 @@ end
|
|
|
97
81
|
```
|
|
98
82
|
<!-- calculator-grammar:end -->
|
|
99
83
|
|
|
100
|
-
Generate and run
|
|
101
|
-
|
|
102
|
-
```sh
|
|
103
|
-
bundle exec ruby -Ilib exe/ibex calculator.y
|
|
104
|
-
bundle exec ruby -Ilib calculator.rb
|
|
105
|
-
# 14
|
|
106
|
-
```
|
|
107
|
-
|
|
108
|
-
With the local gems installed, the equivalent commands are:
|
|
84
|
+
Generate and run it:
|
|
109
85
|
|
|
110
86
|
```sh
|
|
111
|
-
ibex calculator.y
|
|
87
|
+
ibex -o calculator.rb calculator.y
|
|
112
88
|
ruby calculator.rb
|
|
89
|
+
# 14
|
|
113
90
|
```
|
|
114
91
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
embed the runtime.
|
|
119
|
-
|
|
120
|
-
## Generated parser contract
|
|
121
|
-
|
|
122
|
-
### Tokens and lexers
|
|
123
|
-
|
|
124
|
-
A handwritten pull lexer implements `next_token` and returns:
|
|
125
|
-
|
|
126
|
-
```ruby
|
|
127
|
-
[token, value]
|
|
128
|
-
[token, value, location]
|
|
129
|
-
```
|
|
130
|
-
|
|
131
|
-
`false` or `nil` marks EOF. Bare grammar tokens use Ruby symbols such as
|
|
132
|
-
`:NUM`; quoted terminals use strings such as `'+'`. Locations are
|
|
133
|
-
application-owned objects and remain optional.
|
|
134
|
-
|
|
135
|
-
Existing lexers can use the [lexer migration guide](docs/lexer-migration.md).
|
|
136
|
-
Extended grammars may instead generate a streaming lexer from declarative
|
|
137
|
-
regular-expression rules.
|
|
138
|
-
|
|
139
|
-
### Semantic actions and locations
|
|
140
|
-
|
|
141
|
-
Actions read RHS values through `val`. `@1`, `@2`, and so on address parallel
|
|
142
|
-
semantic locations, while `@$` is the span of the current reduction. Action
|
|
143
|
-
source remains opaque Ruby: Ibex preserves its source mapping but does not
|
|
144
|
-
parse, type-check, or sandbox it.
|
|
145
|
-
|
|
146
|
-
### Parse lifecycles and errors
|
|
147
|
-
|
|
148
|
-
Generated parsers support:
|
|
149
|
-
|
|
150
|
-
- `do_parse` and yielding `yyparse` for racc-compatible pull parsing;
|
|
151
|
-
- `push(token, value, location)` and `finish(location:)` for caller-driven
|
|
152
|
-
streaming;
|
|
153
|
-
- structured `Ibex::ParseError` values containing the token, expected tokens,
|
|
154
|
-
state, location, and spelling suggestions;
|
|
155
|
-
- yacc-style recovery through `on_error`, `yyerror`, `yyerrok`, and
|
|
156
|
-
`yyaccept`; and
|
|
157
|
-
- immutable, shareable generated tables.
|
|
158
|
-
|
|
159
|
-
Use one parser instance per concurrent parse. Isolated runtime sessions and
|
|
160
|
-
resource budgets are opt-in when application code needs explicit execution
|
|
161
|
-
boundaries.
|
|
92
|
+
The generated parser uses compact immutable tables and depends on
|
|
93
|
+
`ibex-runtime`. Use `-E` when a single self-contained generated file is more
|
|
94
|
+
convenient. Use `--check` in CI to detect stale generated output.
|
|
162
95
|
|
|
163
|
-
## Choose a
|
|
96
|
+
## Choose a contract
|
|
164
97
|
|
|
165
|
-
|
|
|
98
|
+
| Surface | Use it for | Maturity |
|
|
166
99
|
| --- | --- | --- |
|
|
167
|
-
|
|
|
168
|
-
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
| --- | --- |
|
|
201
|
-
| `--algorithm=lalr` | Default direct LALR(1) construction |
|
|
202
|
-
| `--algorithm=slr` | Grammars whose FOLLOW-set reductions are sufficient |
|
|
203
|
-
| `--algorithm=ielr` | Preview backend for conflicts introduced by LALR state merging |
|
|
204
|
-
| `--algorithm=lr1` | Canonical LR(1) analysis when the larger automaton is acceptable |
|
|
205
|
-
|
|
206
|
-
`--suggest-ielr` checks whether IELR removes an unexpected LALR conflict and
|
|
207
|
-
prints an advisory note when it does.
|
|
208
|
-
That note does not change the selected algorithm or exit status.
|
|
209
|
-
|
|
210
|
-
Conflict freedom is not the same as a proof of unambiguity. `ibex check
|
|
211
|
-
--ambiguity` searches for a concrete sentence with two interpretations, but
|
|
212
|
-
the token and configuration budgets make every result explicitly bounded.
|
|
213
|
-
|
|
214
|
-
## Integrate generation into a build
|
|
215
|
-
|
|
216
|
-
Treat grammar source as the input and generated Ruby as a reproducible build
|
|
217
|
-
artifact:
|
|
218
|
-
|
|
219
|
-
```sh
|
|
220
|
-
ibex --warnings=error -o lib/calculator.rb grammar/calculator.y
|
|
221
|
-
ibex --warnings=error --check -o lib/calculator.rb grammar/calculator.y
|
|
222
|
-
```
|
|
223
|
-
|
|
224
|
-
The first command writes the parser. The second is suitable for CI and fails
|
|
225
|
-
when the committed output is stale without modifying it.
|
|
226
|
-
|
|
227
|
-
Rake projects can declare the dependency directly:
|
|
228
|
-
|
|
229
|
-
```ruby
|
|
230
|
-
require "ibex/rake_task"
|
|
231
|
-
|
|
232
|
-
Ibex::RakeTask.new("lib/calculator.rb") do |task|
|
|
233
|
-
task.grammar = "grammar/calculator.y"
|
|
234
|
-
task.options = ["--warnings=error"]
|
|
235
|
-
end
|
|
236
|
-
```
|
|
237
|
-
|
|
238
|
-
The task follows the grammar's fragment dependency closure. Multi-output
|
|
239
|
-
generation can additionally publish a manifest containing input identities,
|
|
240
|
-
output paths, sizes, and SHA-256 digests. Generation renders all requested
|
|
241
|
-
outputs before publishing them; `--watch` retains the last successful output
|
|
242
|
-
when a later edit is invalid.
|
|
243
|
-
|
|
244
|
-
Applications that only execute generated parsers need `ibex-runtime`, not the
|
|
245
|
-
generator. To build and install both current local gems:
|
|
246
|
-
|
|
247
|
-
```sh
|
|
248
|
-
gem build ibex-runtime.gemspec
|
|
249
|
-
gem build ibex.gemspec
|
|
250
|
-
gem install ./ibex-runtime-0.1.0.gem
|
|
251
|
-
gem install ./ibex-0.1.0.gem
|
|
252
|
-
```
|
|
253
|
-
|
|
254
|
-
## Diagnose, verify, and inspect
|
|
255
|
-
|
|
256
|
-
| Goal | Command |
|
|
257
|
-
| --- | --- |
|
|
258
|
-
| Promote grammar warnings to errors | `ibex --warnings=error -C grammar.y` |
|
|
259
|
-
| Collect structured frontend diagnostics | `ibex diagnose --format=json grammar.y` |
|
|
260
|
-
| Explain one conflict | `ibex explain --state=7 --token=ELSE grammar.y` |
|
|
261
|
-
| Search for a concrete ambiguity | `ibex check --ambiguity --algorithm=lr1 grammar.y` |
|
|
262
|
-
| Run grammar-declared tests | `ibex test --coverage=100 grammar.y` |
|
|
263
|
-
| Check canonical formatting | `ibex fmt --check grammar.y` |
|
|
264
|
-
| Render grammar documentation | `ibex doc --format=html -o grammar.html grammar.y` |
|
|
265
|
-
| Emit Automaton IR | `ibex --emit=automaton-ir grammar.y > automaton.json` |
|
|
266
|
-
| Simulate table behavior without actions | `ibex debug automaton.json NUMBER PLUS NUMBER` |
|
|
267
|
-
| Check a racc migration | `ibex migrate-check grammar.y` |
|
|
268
|
-
| Start the language server | `ibex lsp --stdio` |
|
|
269
|
-
|
|
270
|
-
Grammar IR, Automaton IR, generated table formats, diagnostic JSON, runtime
|
|
271
|
-
events, coverage, and manifests are versioned contracts. Reports can also be
|
|
272
|
-
rendered as text, DOT, Mermaid, HTML, railroad SVG, samples, or conflict
|
|
273
|
-
counterexamples.
|
|
274
|
-
|
|
275
|
-
## Safety boundaries and non-goals
|
|
276
|
-
|
|
277
|
-
- Frontend parsing, diagnosis, formatting, documentation, LSP, static
|
|
278
|
-
migration checks, IR validation, and table simulation do not execute
|
|
279
|
-
semantic actions or user-code sections.
|
|
280
|
-
- Generated parsers, grammar-declared tests, and explicit migration harnesses
|
|
281
|
-
do execute application Ruby. They are not sandboxes.
|
|
282
|
-
- Fragment imports are confined to the declared source root and reject
|
|
283
|
-
traversal, cycles, and symlink escapes.
|
|
284
|
-
- Counterexample, ambiguity, sample, repair, and runtime-budget searches are
|
|
285
|
-
bounded. Exhausting a budget is distinct from finding no witness within it.
|
|
286
|
-
- Ibex generates deterministic LR parsers. It does not provide GLR or
|
|
287
|
-
generalized ambiguity handling. Incremental CST sessions are syntax-only
|
|
288
|
-
and experimental.
|
|
289
|
-
- racc compatibility is a migration surface, not a claim that every generated
|
|
290
|
-
parser is a byte-for-byte or adapter-free replacement.
|
|
291
|
-
- Preview and experimental features may have weaker compatibility guarantees
|
|
292
|
-
than the default mode.
|
|
293
|
-
|
|
294
|
-
## Documentation by task
|
|
295
|
-
|
|
296
|
-
| When you need to... | Start here |
|
|
297
|
-
| --- | --- |
|
|
298
|
-
| Write a grammar or use the runtime | [Grammar reference](docs/grammar-reference.md) |
|
|
299
|
-
| Build, edit, serialize, or incrementally update syntax trees | [Red/Green CST guide](docs/cst.md) and [migration guide](docs/cst-migration.md) |
|
|
300
|
-
| Learn from executable grammars | [Examples](examples/README.md) |
|
|
301
|
-
| Migrate from racc | [racc migration guide](docs/racc-migration.md) |
|
|
302
|
-
| Adapt a handwritten lexer | [Lexer migration guide](docs/lexer-migration.md) |
|
|
303
|
-
| Configure an editor or LSP client | [Editor setup](docs/editor-setup.md) |
|
|
304
|
-
| Understand the pipeline, IR, algorithms, or tables | [Architecture](docs/architecture.md) |
|
|
305
|
-
| Check API maturity and deprecation rules | [Stability policy](docs/stability.md) |
|
|
306
|
-
| Evaluate readiness, performance, or error UX | [Release readiness](docs/release-readiness.md), [benchmarks](benchmark/README.md), and [error UX](docs/error-ux.md) |
|
|
307
|
-
| Contribute or run every quality gate | [Development guide](docs/development.md) |
|
|
308
|
-
| Review implementation decisions | [Architecture decision records](docs/decisions/README.md) |
|
|
309
|
-
|
|
310
|
-
The published documentation separates the
|
|
311
|
-
[compatibility contract](https://ydah.github.io/ibex/compatibility/),
|
|
312
|
-
[opt-in extensions](https://ydah.github.io/ibex/extensions/), and
|
|
313
|
-
[experimental surfaces](https://ydah.github.io/ibex/experimental/). The
|
|
314
|
-
[grammar gallery](https://ydah.github.io/ibex/gallery/) contains executable
|
|
315
|
-
examples whose declared tests reach complete production coverage.
|
|
100
|
+
| Compatible mode | Existing racc grammars and conservative migrations | Stable baseline |
|
|
101
|
+
| Extended mode | EBNF, imports, generated lexers, trees, types, and tooling | Preview; explicit opt-in |
|
|
102
|
+
| Browser playground and incremental CST sessions | Local analysis and evaluation | Preview/Experimental; bounded |
|
|
103
|
+
|
|
104
|
+
The default frontend remains unchanged unless a grammar opts into
|
|
105
|
+
`pragma extended` or the command line uses `--mode=extended`. Read the
|
|
106
|
+
[grammar reference](https://ydah.github.io/ibex/docs/grammar-reference/),
|
|
107
|
+
[racc migration guide](https://ydah.github.io/ibex/docs/racc-migration/), and
|
|
108
|
+
[maturity audit](https://ydah.github.io/ibex/docs/maturity/) before adopting a
|
|
109
|
+
Preview or Experimental surface.
|
|
110
|
+
|
|
111
|
+
IELR is also explicit: `--algorithm=ielr` uses the conservative partition
|
|
112
|
+
strategy by default, while `--ielr-strategy=direct` enables the experimental
|
|
113
|
+
LR(0)-based construction. Direct IELR is profiled and reviewable, but it is not
|
|
114
|
+
the default release strategy; see the [IELR guide](docs/ielr.md).
|
|
115
|
+
|
|
116
|
+
## Trust boundary
|
|
117
|
+
|
|
118
|
+
Static frontend, formatting, documentation, LSP, IR, verification, and
|
|
119
|
+
browser analysis treat grammar actions and user sections as opaque source.
|
|
120
|
+
Generated parsers and semantic parses execute application Ruby and are not
|
|
121
|
+
sandboxes. The playground runs in a local Web Worker, does not upload grammar
|
|
122
|
+
source, and never executes parser actions or user-code sections. Its limits and
|
|
123
|
+
non-goals are documented in the [playground guide](https://ydah.github.io/ibex/playground/).
|
|
124
|
+
|
|
125
|
+
## Where to go next
|
|
126
|
+
|
|
127
|
+
- [Documentation hub](https://ydah.github.io/ibex/docs/) — task-oriented entry points.
|
|
128
|
+
- [Grammar gallery](https://ydah.github.io/ibex/gallery/) — executable examples with declared tests.
|
|
129
|
+
- [API reference](https://ydah.github.io/ibex/api/) — generated Ruby API documentation.
|
|
130
|
+
- [Configuration model](https://ydah.github.io/ibex/docs/configuration-model/) — grammar-owned and invocation-owned settings.
|
|
131
|
+
- [Project status and evidence](https://ydah.github.io/ibex/project/) — release, roadmap, and bounded claims.
|
|
132
|
+
- [Contributing](CONTRIBUTING.md) and [security reporting](SECURITY.md) — community workflow.
|
|
316
133
|
|
|
317
134
|
## Development
|
|
318
135
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
workflow lint commands. The default local gate is:
|
|
136
|
+
Clone the repository only when you need to contribute or run the complete
|
|
137
|
+
quality suite:
|
|
322
138
|
|
|
323
139
|
```sh
|
|
140
|
+
git clone https://github.com/ydah/ibex.git
|
|
141
|
+
cd ibex
|
|
324
142
|
bundle install
|
|
325
143
|
bundle exec rake
|
|
144
|
+
npm ci
|
|
145
|
+
npm run test:site
|
|
326
146
|
```
|
|
327
147
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
148
|
+
The [development guide](https://ydah.github.io/ibex/docs/development/) lists
|
|
149
|
+
the focused frontend, type, evidence, browser, and workflow checks.
|
|
150
|
+
|
|
151
|
+
## Decisions and evidence
|
|
332
152
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
153
|
+
The public reference also indexes the [direct IELR decision](docs/direct-ielr-decision.md),
|
|
154
|
+
[direct multi-entry decision](docs/direct-multi-entry-decision.md),
|
|
155
|
+
[verifier trust boundary](docs/verifier-trust-boundary.md), and the
|
|
156
|
+
[error-experience review status](docs/error-ux-review-status-v1.json). These
|
|
157
|
+
records scope claims to their evidence and revision; they do not change the
|
|
158
|
+
compatible installation path above.
|
|
336
159
|
|
|
337
160
|
Ibex is available under the [MIT License](LICENSE.txt).
|
|
@@ -53,7 +53,7 @@ module Ibex
|
|
|
53
53
|
source
|
|
54
54
|
end
|
|
55
55
|
|
|
56
|
-
# @rbs (
|
|
56
|
+
# @rbs (Object? other) -> bool
|
|
57
57
|
def ==(other)
|
|
58
58
|
other.is_a?(GreenNode) && @kind == other.kind && @intrinsic_flags == other.intrinsic_flags &&
|
|
59
59
|
@annotations == other.annotations && @children == other.children
|
|
@@ -75,7 +75,7 @@ module Ibex
|
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
# Return the next token or a whole nonterminal valid in the current LR state.
|
|
78
|
-
# @rbs (Integer state) -> (Array[
|
|
78
|
+
# @rbs (Integer state) -> (Array[Object?] | ReusableSubtree | false)
|
|
79
79
|
def next_for_state(state)
|
|
80
80
|
candidates = @candidates[@token_index]
|
|
81
81
|
candidate = candidates&.find { |item| item.entry.left_state == state }
|
|
@@ -6,10 +6,10 @@ module Ibex
|
|
|
6
6
|
module CST
|
|
7
7
|
# One complete generated-lexer pass, including the explicit EOF token.
|
|
8
8
|
class LexedSyntax
|
|
9
|
-
attr_reader :raw_tokens #: Array[Array[
|
|
9
|
+
attr_reader :raw_tokens #: Array[Array[Object?]]
|
|
10
10
|
attr_reader :memo #: TokenMemo
|
|
11
11
|
|
|
12
|
-
# @rbs (raw_tokens: Array[Array[
|
|
12
|
+
# @rbs (raw_tokens: Array[Array[Object?]], memo: TokenMemo) -> void
|
|
13
13
|
def initialize(raw_tokens:, memo:)
|
|
14
14
|
raise ArgumentError, "raw and Green token counts differ" unless raw_tokens.length == memo.tokens.length
|
|
15
15
|
|
|
@@ -39,7 +39,7 @@ module Ibex
|
|
|
39
39
|
raise IndexError, "parse memo subtree range is outside the preorder state array"
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
# @rbs (Hash[Symbol,
|
|
42
|
+
# @rbs (Hash[Symbol, String | Integer] tables) -> bool
|
|
43
43
|
def compatible?(tables)
|
|
44
44
|
@grammar_digest == tables[:grammar_digest] &&
|
|
45
45
|
@state_count == tables[:state_count] &&
|
|
@@ -49,7 +49,7 @@ module Ibex
|
|
|
49
49
|
# @rbs () -> Integer
|
|
50
50
|
def estimated_bytes = @left_states.length * ENTRY_BYTES
|
|
51
51
|
|
|
52
|
-
# @rbs () -> Hash[String,
|
|
52
|
+
# @rbs () -> Hash[String, Integer | Array[Integer?]]
|
|
53
53
|
def to_h
|
|
54
54
|
{ "version" => VERSION, "left_states" => @left_states }.freeze
|
|
55
55
|
end
|
|
@@ -123,6 +123,9 @@ module Ibex
|
|
|
123
123
|
|
|
124
124
|
# @rbs (TokenMemo previous, Integer old_index, TokenMemo current, Integer new_index) -> bool
|
|
125
125
|
def self.reusable_token?(previous, old_index, current, new_index)
|
|
126
|
+
return false unless old_index.between?(0, previous.tokens.length - 1)
|
|
127
|
+
return false unless new_index.between?(0, current.tokens.length - 1)
|
|
128
|
+
|
|
126
129
|
previous.states.fetch(old_index) == current.states.fetch(new_index) &&
|
|
127
130
|
previous.tokens.fetch(old_index) == current.tokens.fetch(new_index)
|
|
128
131
|
end
|
|
@@ -22,8 +22,9 @@ module Ibex
|
|
|
22
22
|
# @rbs @blender_enabled: bool
|
|
23
23
|
# @rbs @mutex: Mutex
|
|
24
24
|
|
|
25
|
-
# @rbs (Class parser_class, SourceText source_text, ?resource_limits: ResourceLimits?,
|
|
26
|
-
|
|
25
|
+
# @rbs (Class parser_class, SourceText source_text, ?resource_limits: ResourceLimits?,
|
|
26
|
+
# ?blender: bool, ?event_observer: Proc?) -> void
|
|
27
|
+
def initialize(parser_class, source_text, resource_limits: nil, blender: true, event_observer: nil)
|
|
27
28
|
unless source_text.is_a?(SourceText)
|
|
28
29
|
raise ArgumentError, "incremental_session requires an Ibex::Runtime::CST::SourceText"
|
|
29
30
|
end
|
|
@@ -31,10 +32,12 @@ module Ibex
|
|
|
31
32
|
@resource_limits = resource_limits || ResourceLimits.new #: ResourceLimits
|
|
32
33
|
@parser = parser_class.__send__(:new, resource_limits: @resource_limits) #: Parser
|
|
33
34
|
validate_parser!
|
|
35
|
+
@parser.observe { |event| event_observer.call(event, @parser) } if event_observer
|
|
34
36
|
@cache = NodeCache.new #: NodeCache
|
|
35
37
|
@source_text = source_text
|
|
36
38
|
@last_relex_result = nil #: RelexResult?
|
|
37
39
|
@last_blender = nil #: Blender?
|
|
40
|
+
@last_full_fallback = false #: bool
|
|
38
41
|
@blender_enabled = blender
|
|
39
42
|
@mutex = Mutex.new
|
|
40
43
|
@result, @token_memo, @parse_memo = parse_current(0.0)
|
|
@@ -58,15 +61,25 @@ module Ibex
|
|
|
58
61
|
@parser.unobserve(subscription)
|
|
59
62
|
end
|
|
60
63
|
|
|
64
|
+
# Whether the last completed edit discarded all incremental reuse and
|
|
65
|
+
# reparsed the current source from a fresh token stream.
|
|
66
|
+
# @rbs () -> bool
|
|
67
|
+
def last_full_fallback? = @last_full_fallback
|
|
68
|
+
|
|
61
69
|
private
|
|
62
70
|
|
|
63
71
|
# @rbs (Array[TextEdit] edits) -> SyntaxResult
|
|
64
72
|
def edit_locked(edits)
|
|
65
73
|
return @result if edits.empty?
|
|
66
74
|
|
|
75
|
+
previous_source = @source_text
|
|
67
76
|
previous_memo = @token_memo
|
|
68
77
|
previous_root = @result.syntax_root
|
|
69
78
|
previous_parse_memo = @parse_memo
|
|
79
|
+
previous_result = @result
|
|
80
|
+
previous_relex_result = @last_relex_result
|
|
81
|
+
previous_blender = @last_blender
|
|
82
|
+
previous_full_fallback = @last_full_fallback
|
|
70
83
|
@source_text = @source_text.apply(edits)
|
|
71
84
|
lexed = scan_current
|
|
72
85
|
return finish_full_fallback(previous_memo, edits, :lexical_error) unless lexed
|
|
@@ -74,6 +87,15 @@ module Ibex
|
|
|
74
87
|
relexed = Relexer.reconcile(previous_memo, lexed.memo, edits)
|
|
75
88
|
blender = build_blender(previous_root, previous_parse_memo, lexed.with_memo(relexed.memo), edits)
|
|
76
89
|
finish_blended_edit(relexed, blender)
|
|
90
|
+
rescue StandardError
|
|
91
|
+
@source_text = previous_source
|
|
92
|
+
@token_memo = previous_memo
|
|
93
|
+
@parse_memo = previous_parse_memo
|
|
94
|
+
@result = previous_result
|
|
95
|
+
@last_relex_result = previous_relex_result
|
|
96
|
+
@last_blender = previous_blender
|
|
97
|
+
@last_full_fallback = previous_full_fallback
|
|
98
|
+
raise
|
|
77
99
|
end
|
|
78
100
|
|
|
79
101
|
# @rbs () -> LexedSyntax?
|
|
@@ -99,6 +121,7 @@ module Ibex
|
|
|
99
121
|
def finish_blended_edit(relexed, blender)
|
|
100
122
|
@last_relex_result = relexed
|
|
101
123
|
@last_blender = blender
|
|
124
|
+
@last_full_fallback = false
|
|
102
125
|
parsed = @parser.__send__(:parse_syntax_token_source, blender, @cache)
|
|
103
126
|
reused_ratio = blender_reused_ratio(blender, parsed.syntax_root.green, relexed)
|
|
104
127
|
@token_memo = relexed.memo
|
|
@@ -197,6 +220,7 @@ module Ibex
|
|
|
197
220
|
fresh_result, fresh_memo, fresh_parse_memo = parse_current(0.0)
|
|
198
221
|
@last_relex_result = Relexer.reconcile(previous_memo, fresh_memo, edits)
|
|
199
222
|
@last_blender = nil
|
|
223
|
+
@last_full_fallback = true
|
|
200
224
|
@token_memo = fresh_memo
|
|
201
225
|
@parse_memo = fresh_parse_memo
|
|
202
226
|
@result = fresh_result
|
|
@@ -6,11 +6,11 @@ module Ibex
|
|
|
6
6
|
module CST
|
|
7
7
|
# Immutable result returned by syntax-aware parser entry points.
|
|
8
8
|
class ParseResult
|
|
9
|
-
attr_reader :value #:
|
|
9
|
+
attr_reader :value #: Object
|
|
10
10
|
attr_reader :syntax_root #: SyntaxNode
|
|
11
|
-
attr_reader :diagnostics #: Array[
|
|
11
|
+
attr_reader :diagnostics #: Array[Object]
|
|
12
12
|
|
|
13
|
-
# @rbs (value:
|
|
13
|
+
# @rbs (value: Object, syntax_root: SyntaxNode, diagnostics: Array[Object]) -> void
|
|
14
14
|
def initialize(value:, syntax_root:, diagnostics:)
|
|
15
15
|
@value = value
|
|
16
16
|
@syntax_root = syntax_root
|
|
@@ -22,10 +22,10 @@ module Ibex
|
|
|
22
22
|
# Syntax-only result used by incremental sessions.
|
|
23
23
|
class SyntaxResult
|
|
24
24
|
attr_reader :syntax_root #: SyntaxNode
|
|
25
|
-
attr_reader :diagnostics #: Array[
|
|
25
|
+
attr_reader :diagnostics #: Array[Object]
|
|
26
26
|
attr_reader :reused_ratio #: Float
|
|
27
27
|
|
|
28
|
-
# @rbs (syntax_root: SyntaxNode, diagnostics: Array[
|
|
28
|
+
# @rbs (syntax_root: SyntaxNode, diagnostics: Array[Object], reused_ratio: Float) -> void
|
|
29
29
|
def initialize(syntax_root:, diagnostics:, reused_ratio:)
|
|
30
30
|
@syntax_root = syntax_root
|
|
31
31
|
@diagnostics = diagnostics.dup.freeze
|
|
@@ -35,7 +35,7 @@ module Ibex
|
|
|
35
35
|
end
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
-
ParseResult = CST::ParseResult
|
|
39
|
-
SyntaxResult = CST::SyntaxResult
|
|
38
|
+
ParseResult = CST::ParseResult #: singleton(CST::ParseResult)
|
|
39
|
+
SyntaxResult = CST::SyntaxResult #: singleton(CST::SyntaxResult)
|
|
40
40
|
end
|
|
41
41
|
end
|