expressir 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/TODO.max-perf/01-restore-ci-green.md +29 -0
- data/TODO.max-perf/02-streaming-parse-path.md +31 -0
- data/TODO.max-perf/03-cli-parallel-opt-in.md +27 -0
- data/TODO.max-perf/04-benchmark-harness.md +28 -0
- data/TODO.max-perf/05-parallel-fidelity-specs.md +22 -0
- data/TODO.max-perf/06-builder-cpu-audit.md +41 -0
- data/TODO.max-perf/07-upstream-parsanol-roadmap.md +27 -0
- data/TODO.max-perf/08-builder-build-perf.md +45 -0
- data/TODO.max-perf/09-grammar-cold-start.md +25 -0
- data/TODO.max-perf/10-parser-facade-hygiene.md +23 -0
- data/TODO.max-perf/11-ci-green-closeout.md +25 -0
- data/TODO.max-perf/12-require-boot-profile.md +25 -0
- data/TODO.max-perf/13-key-conversion-specs.md +26 -0
- data/TODO.max-perf/14-builder-call-handler-audit.md +28 -0
- data/benchmark/srl_benchmark.rb +76 -17
- data/expressir.gemspec +1 -1
- data/lib/expressir/cli.rb +3 -0
- data/lib/expressir/commands/coverage.rb +6 -2
- data/lib/expressir/commands/package.rb +4 -1
- data/lib/expressir/express/ast_key_converter.rb +114 -0
- data/lib/expressir/express/builder.rb +8 -119
- data/lib/expressir/express/error.rb +17 -0
- data/lib/expressir/express/parallel_files.rb +229 -0
- data/lib/expressir/express/parser.rb +44 -86
- data/lib/expressir/express/remark_attacher.rb +25 -7
- data/lib/expressir/express/schema_block_scanner.rb +3 -2
- data/lib/expressir/express/scope_resolver.rb +34 -5
- data/lib/expressir/express.rb +2 -0
- data/lib/expressir/model/model_element.rb +6 -1
- data/lib/expressir/model/repository.rb +18 -5
- data/lib/expressir/version.rb +1 -1
- data/lib/expressir.rb +18 -0
- metadata +22 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0f786a7123d34be8bada764d132f213c09f229d5f94046509dbdbfc37a0d4279
|
|
4
|
+
data.tar.gz: 9b32c54df109ca8e6945ab03d2b7e3697d2b0bce3afbb67476bfe1510e0eb2fe
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5225e02955f94be94f71e9005330d1c58173ebd9cb5aa97956abfa9b8b8f3f2e964f25648d8074ad8ba41fb678530dad4348570ed8c697c6445224fe77c1514f
|
|
7
|
+
data.tar.gz: f97f4c7be51de8b6c5e4e71301555f20d05bdb268b61ef7df2e7be4bd06796af255b1f2cc362e6a25a61867fd62ab0b3610eb25683d57d6b36401c9a338598f5
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# TODO.max-perf/01 — Restore CI green on main
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
CI on main (`0ef38e1`, 2026-09-17) fails on the Windows matrix: yeptris
|
|
6
|
+
0.6.3.3's mingw build raises `NameError: uninitialized constant
|
|
7
|
+
Yeptris::FFI::NODE_SCALAR` from `yeptris/node.rb` whenever `to_yaml` runs.
|
|
8
|
+
Expressir's gemspec constraint (`~> 0.6.1`, from PR #346) allows it, and the
|
|
9
|
+
Gemfile.lock is gitignored, so CI resolves the newest release at run time.
|
|
10
|
+
|
|
11
|
+
yeptris has since released 0.6.4.1 and 0.6.5.1.
|
|
12
|
+
|
|
13
|
+
## Work
|
|
14
|
+
|
|
15
|
+
- [x] Reproduce the failing spec groups locally on the newest yeptris
|
|
16
|
+
(`package_fixtures_spec`, `package_spec`, `cache_spec`): 84/84 green on
|
|
17
|
+
0.6.5.1
|
|
18
|
+
- [x] Re-run the failed rake workflow on main (resolves 0.6.5.1)
|
|
19
|
+
- [ ] Confirm the rake workflow on main is green
|
|
20
|
+
- [x] 0.6.5.1 does NOT fix mingw (same NameError, all Windows jobs) — filed
|
|
21
|
+
leptris/yeptris#318 with the CI repro
|
|
22
|
+
- [x] expressir-side guard: `Expressir.select_serialization_engines` pins the
|
|
23
|
+
portable :standard YAML/JSON adapters on `Gem.win_platform?` at boot, so
|
|
24
|
+
lutaml-model's autodetection never loads the broken mingw build; Unix
|
|
25
|
+
keeps yeptris. Spec: spec/expressir/engine_selection_spec.rb
|
|
26
|
+
|
|
27
|
+
## Acceptance
|
|
28
|
+
|
|
29
|
+
rake workflow green on main across the full platform matrix.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# TODO.max-perf/02 — Fix or retire the streaming parse path
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
`Parser.from_exp(use_streaming: true)` — documented as the maximum-performance
|
|
6
|
+
path (construct-by-construct building via `Parsanol::Native.parse_with_builder`,
|
|
7
|
+
no full intermediate AST) — fails on every schema tested with
|
|
8
|
+
`Error::SchemaParseFailure` on parsanol 1.3.28 and 1.3.30. The feature is dead
|
|
9
|
+
weight until it works: callers cannot opt into it, and it skews the perf
|
|
10
|
+
picture (the fastest intended path is the broken one).
|
|
11
|
+
|
|
12
|
+
## Work
|
|
13
|
+
|
|
14
|
+
- [x] Reproduced on a 5-line schema: `parse_fresh(schema_grammar, block)` fails
|
|
15
|
+
at end-of-input — parse_fresh has no packrat memoization, which EXPRESS
|
|
16
|
+
requires (parsanol-ruby#52)
|
|
17
|
+
- [x] The true streaming path (`from_exp_streaming_builder`) is unreachable:
|
|
18
|
+
its gate uses `defined?(Parsanol::Native.parse_with_builder)`, which is
|
|
19
|
+
always nil for the extension's dynamically-dispatched methods; calling it
|
|
20
|
+
directly yields empty models (builder protocol drift vs parsanol 1.3.30)
|
|
21
|
+
- [x] Removed both dead paths; `use_streaming: true` now raises
|
|
22
|
+
`Error::StreamingUnsupportedError` with a precise message
|
|
23
|
+
- [x] Spec: spec/expressir/express/streaming_spec.rb
|
|
24
|
+
- [x] Upstream: covered by parsanol-ruby#59 roadmap (ractor-safe / stable
|
|
25
|
+
parse_with_builder unlocks a rewrite of this path)
|
|
26
|
+
|
|
27
|
+
## Acceptance
|
|
28
|
+
|
|
29
|
+
Either `use_streaming: true` produces an equal model with a green spec, or
|
|
30
|
+
the breakage is precisely documented upstream and the path is explicitly
|
|
31
|
+
marked unsupported.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# TODO.max-perf/03 — CLI parallel opt-in and multi-file loader consolidation
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
The fork worker pool (PRs #349/#351) is strictly opt-in for library users.
|
|
6
|
+
The expressir CLI is application mode — our process — so it can opt in.
|
|
7
|
+
Today `commands/coverage.rb` calls `Parser.from_files` without
|
|
8
|
+
`max_processes` (sequential), and `commands/package.rb` calls
|
|
9
|
+
`Model::Repository.from_files` — a *second* multi-file loading path
|
|
10
|
+
duplicating the same orchestration (MECE violation).
|
|
11
|
+
|
|
12
|
+
## Work
|
|
13
|
+
|
|
14
|
+
- [x] `--max-processes` on `coverage` (default 4, Thor layer in cli.rb) and
|
|
15
|
+
`package build` (default 1 — strict error semantics preserved)
|
|
16
|
+
- [x] `ParallelFiles.run` gained `strict:` — re-raises SchemaParseFailure,
|
|
17
|
+
preserving Repository.from_files' fail-fast contract
|
|
18
|
+
- [x] `Model::Repository.from_files` now delegates its parse loop to
|
|
19
|
+
ParallelFiles (shared orchestration, strict mode); tolerant
|
|
20
|
+
Parser.from_files behavior unchanged
|
|
21
|
+
- [x] Specs: repository parallel == sequential (ids + order), strict raise,
|
|
22
|
+
coverage/package suites green (57 + 57 examples)
|
|
23
|
+
|
|
24
|
+
## Acceptance
|
|
25
|
+
|
|
26
|
+
One multi-file loading pipeline; CLI bulk loads parse in parallel by
|
|
27
|
+
default; library default remains sequential.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# TODO.max-perf/04 — Repair the SRL benchmark harness
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
`benchmark/srl_benchmark.rb` has two defects found during the Sept 2026
|
|
6
|
+
validation:
|
|
7
|
+
|
|
8
|
+
1. The "Ruby Parser" pass calls `from_file(file, skip_references: true)`
|
|
9
|
+
without `use_native: false`. Since native-became-default (2026-03-25),
|
|
10
|
+
both passes run the native parser — the head-to-head comparison is
|
|
11
|
+
mislabeled and meaningless.
|
|
12
|
+
2. Per-file `Timeout.timeout(30)` cannot interrupt the native FFI parse
|
|
13
|
+
(GVL held), so pathological files hang the benchmark instead of timing
|
|
14
|
+
out — observed as a 25-minute stall on an 8.6KB schema.
|
|
15
|
+
|
|
16
|
+
## Work
|
|
17
|
+
|
|
18
|
+
- [x] Ruby-parser pass (and warmup) now pass `use_native: false`
|
|
19
|
+
- [x] `parse_file_isolated`: per-file fork, length-prefixed Marshal results,
|
|
20
|
+
TERM→KILL wall-clock escalation — pathological files terminate
|
|
21
|
+
- [x] `SRL_PATH` env override for reproducible smoke runs
|
|
22
|
+
- [x] Verified: 3-fixture run prints genuine Ruby-vs-Native numbers with
|
|
23
|
+
truthful labels and terminates cleanly
|
|
24
|
+
|
|
25
|
+
## Acceptance
|
|
26
|
+
|
|
27
|
+
A short benchmark run prints genuine Ruby-vs-Native numbers, terminates on
|
|
28
|
+
pathological files, and labels passes truthfully.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# TODO.max-perf/05 — Parallel-parsing fidelity specs
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
Parallel `from_files` equality is currently asserted on schema ids only.
|
|
6
|
+
The pool crosses a fork boundary via Marshal; deep structural equality is
|
|
7
|
+
the real contract. The progress-block regression merged in #349 (double
|
|
8
|
+
`schemas` extraction) proved the block/caller paths need dedicated specs.
|
|
9
|
+
|
|
10
|
+
## Work
|
|
11
|
+
|
|
12
|
+
- [x] Deep-equality spec: `to_hash` of parallel repository == sequential
|
|
13
|
+
- [x] Progress-block contract specs on both paths (schemas is an Array of
|
|
14
|
+
Declarations::Schema)
|
|
15
|
+
- [x] Error-path specs: skip-vs-raise by mode (tolerant/strict), including
|
|
16
|
+
the strict-mode re-raise of SchemaParseFailure
|
|
17
|
+
- [x] Gate specs: nil/1/<3 files and fork-less platforms never fork
|
|
18
|
+
|
|
19
|
+
## Acceptance
|
|
20
|
+
|
|
21
|
+
All fidelity specs green; `to_hash` of a parallel repository equals the
|
|
22
|
+
sequential one for a multi-fixture corpus.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# TODO.max-perf/06 — Ruby-side (Builder) CPU audit on the current stack
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
After parsanol 1.3.30 (~2.5x further parse speedup), the native parse share
|
|
6
|
+
collapsed and the Ruby side — AstTransformer → Builder → lutaml-model
|
|
7
|
+
instantiation → remark attachment — is now the dominant parse cost
|
|
8
|
+
(sampled 2026-09-17: 72/28 Rust/Ruby on 1.3.28; native got ~2.5x faster
|
|
9
|
+
afterwards). March data ("removing Ruby transform: 2.5%") is obsolete.
|
|
10
|
+
|
|
11
|
+
Related landed work: f32f85f "eliminate allocation storms in remark
|
|
12
|
+
attachment" (another session, on main), 4c49e3a per-remark rescan guards.
|
|
13
|
+
|
|
14
|
+
## Work
|
|
15
|
+
|
|
16
|
+
Results (CPU-time, parsanol 1.3.30, 4 mid-size SRL schemas):
|
|
17
|
+
|
|
18
|
+
| phase | total | share |
|
|
19
|
+
|---|---:|---:|
|
|
20
|
+
| native parse (parse_native) | 1.595s | 50% |
|
|
21
|
+
| Builder.build_with_remarks | 1.530s | 48% |
|
|
22
|
+
| reference resolution | 0.080s | 2% |
|
|
23
|
+
|
|
24
|
+
Builder split: model build (`Builder.build`) 1.12s vs remark attachment
|
|
25
|
+
0.06s — remark attachment is already cheap (f32f85f); the cost is
|
|
26
|
+
AST-hash → lutaml-model instantiation.
|
|
27
|
+
|
|
28
|
+
- [x] Phase timing recorded (above)
|
|
29
|
+
- [x] Builder sub-phase split recorded (above)
|
|
30
|
+
- [x] Next target identified precisely: `Builder.build` per-node path —
|
|
31
|
+
`cached_snake_case` / `fast_convert_keys` string allocations and
|
|
32
|
+
lutaml-model constructor overhead (thousands of attribute setters).
|
|
33
|
+
C stack sampling cannot name Ruby methods (vm_exec_core only);
|
|
34
|
+
instrument inside Builder.build or add stackprof as a dev dependency
|
|
35
|
+
before optimizing. Not refactored blindly — no safe evident win
|
|
36
|
+
without that instrumentation.
|
|
37
|
+
|
|
38
|
+
## Acceptance
|
|
39
|
+
|
|
40
|
+
A numbers table attributing Ruby-side parse CPU to phases, plus either a
|
|
41
|
+
measured improvement or a precise, evidence-backed next target.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# TODO.max-perf/07 — Upstream parsanol roadmap (research 2022–2026)
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
The field's fastest grammar engines compile grammars instead of interpreting
|
|
6
|
+
them. Mapping to parsanol (all upstream work; expressir only consumes):
|
|
7
|
+
|
|
8
|
+
| Technique | Precedent | parsanol mapping |
|
|
9
|
+
|---|---|---|
|
|
10
|
+
| JIT grammar compilation + tag-dispatched structure switching + cross-grammar cache reuse (6x) | XGrammar-2 (arXiv:2601.04426) | BYTE_DISPATCH is conceptually TagDispatch; compile to a dispatch table per atom |
|
|
11
|
+
| Grammar → generated code (production precedent) | tree-sitter (precompiled C, incremental reparse), CPython's generated PEG parser | generate Rust from the EXPRESS grammar instead of walking atom structs |
|
|
12
|
+
| Skip packrat tables via automata | Pest discussion #1081 | memoization-free automata for the deterministic fragments; keep packrat only where ambiguity needs it |
|
|
13
|
+
| SIMD byte classification (GB/s lexing) | simdjson, Lemire's vectorized classification (arXiv:2503.01662) | vectorize keyword/delimiter classification in the lexer tier |
|
|
14
|
+
| Incremental re-parse | tree-sitter, gpeg (SLE'21) | re-parse only dirty schema blocks — pairs with expressir's SchemaBlockScanner |
|
|
15
|
+
|
|
16
|
+
## Work
|
|
17
|
+
|
|
18
|
+
- [x] Filed parsanol-ruby#59 with the mapping, profile evidence, citations,
|
|
19
|
+
and the Ractor-safe entry point item (the only parallelism option on
|
|
20
|
+
Windows MRI and fork-hostile embedders)
|
|
21
|
+
- [x] Consumer unlocks noted: incremental reparse pairs with expressir's
|
|
22
|
+
SchemaBlockScanner; compiled grammar cuts cold-start (~2,300 atoms
|
|
23
|
+
serialized per boot); ractor-safe parse replaces the fork pool
|
|
24
|
+
|
|
25
|
+
## Acceptance
|
|
26
|
+
|
|
27
|
+
Upstream issue filed; expressir-side dependencies on each technique noted.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# TODO.max-perf/08 — Instrument and optimize Builder.build internals
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
TODO.max-perf/06 established the split (CPU-time, parsanol 1.3.30, 4 mid-size
|
|
6
|
+
SRL schemas): native parse 50%, `Builder.build_with_remarks` 48%, references
|
|
7
|
+
2%; within the builder, model build is 1.12s vs remark attachment 0.06s.
|
|
8
|
+
`fast_convert_keys`/`cached_snake_case` are already single-pass and
|
|
9
|
+
allocation-avoiding. The unmeasured stages inside `Builder.build`:
|
|
10
|
+
|
|
11
|
+
1. `builder.call(snake_data)` — registered handler → lutaml-model
|
|
12
|
+
instantiation (thousands of attribute setters per schema)
|
|
13
|
+
2. `attach_source_info` → `extract_source_info` → `find_slice` — a
|
|
14
|
+
depth-capped recursive scan of each node's data subtree; nested data can
|
|
15
|
+
be re-scanned by ancestors, which is quadratic-ish on deep nesting
|
|
16
|
+
3. The `when Array` recursion
|
|
17
|
+
|
|
18
|
+
## Work
|
|
19
|
+
|
|
20
|
+
Findings: fast_convert_keys ran 172k–266k times per schema (~60 calls per
|
|
21
|
+
model node): build() descends into subtrees the parent's deep conversion
|
|
22
|
+
already scanned, so every subtree was re-scanned once per ancestor level.
|
|
23
|
+
attach_source_info is negligible (0.02–0.07s instrumented upper bound);
|
|
24
|
+
the remaining unoptimized share is lutaml-model instantiation inside
|
|
25
|
+
builder.call (upstream territory).
|
|
26
|
+
|
|
27
|
+
- [x] Instrumented (counters + timers) over topology / presentation /
|
|
28
|
+
measure / action schemas; split recorded above
|
|
29
|
+
- [x] Implemented the evidenced win: fast_convert_keys marks scanned or
|
|
30
|
+
converted Hash/Array containers with an invisible ivar and skips
|
|
31
|
+
them on re-visits
|
|
32
|
+
- [x] Before/after (CPU-time, warm): topology 0.42→0.25s (−40%),
|
|
33
|
+
presentation 0.27→0.22s (−19%), measure 0.43→0.30s (−30%),
|
|
34
|
+
action 0.11→0.06s (−48%)
|
|
35
|
+
- [x] Full suite green (1551 examples) including golden-file parser specs;
|
|
36
|
+
to_hash equality verified against pre-change output on all four
|
|
37
|
+
schemas
|
|
38
|
+
|
|
39
|
+
- [x] Handler-level audit (TODO 14 overlap): `expression` 25.8% / `syntax` 15.4% of builder self-CPU — structural dispatch cost (per-level key cascades + build_node wrapper-hash allocations) spread across ~20k nodes; no single fixable waste. Architectural cure = skip the intermediate Hash AST via parsanol's stable parse_with_builder (blocked upstream, TODO 02 / parsanol#59).
|
|
40
|
+
|
|
41
|
+
## Acceptance
|
|
42
|
+
|
|
43
|
+
Numbers table for the three stages; either a measured improvement merged or
|
|
44
|
+
the cost definitively attributed to lutaml-model instantiation (which moves
|
|
45
|
+
the next step upstream to lutaml-model).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# TODO.max-perf/09 — Grammar cold-start cost and disk cache
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
Every process boot builds the parsanol grammar and serializes ~2,300 atoms to
|
|
6
|
+
JSON (`Grammar::Parser.cached_parser` / `cached_grammar_json`) before the
|
|
7
|
+
first parse. Short-lived processes (CLI invocations, tests) pay this on every
|
|
8
|
+
run. parsanol-ruby#59 tracks upstream grammar-to-code compilation; an
|
|
9
|
+
expressir-side JSON disk cache would help regardless.
|
|
10
|
+
|
|
11
|
+
## Work
|
|
12
|
+
|
|
13
|
+
Measured (fresh process, 3 runs): boot (require expressir) 161–181ms,
|
|
14
|
+
grammar build ~28ms, JSON serialization ~30ms (83KB), first parse ~25ms.
|
|
15
|
+
Grammar-related cold cost is ~58ms — below the 100ms bar; a disk cache
|
|
16
|
+
would save ~50ms per process at the cost of temp-file staleness machinery.
|
|
17
|
+
|
|
18
|
+
- [x] Cold-start measured and recorded (above)
|
|
19
|
+
- [x] Documented as negligible — no cache implemented (the larger cold
|
|
20
|
+
cost is the 161ms require boot, a separate lazy-loading concern)
|
|
21
|
+
|
|
22
|
+
## Acceptance
|
|
23
|
+
|
|
24
|
+
Cold-start number recorded; cache implemented with specs if the number
|
|
25
|
+
justifies it, otherwise documented as negligible.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# TODO.max-perf/10 — Parser facade API hygiene
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
The `Parser` facade accumulated internal helpers with public visibility
|
|
6
|
+
during the parallel work: `parse_files_sequentially` and `build_repository`
|
|
7
|
+
are implementation details of `from_files`. `Commands::Coverage` defines a
|
|
8
|
+
`DEFAULT_MAX_PROCESSES` constant that duplicates
|
|
9
|
+
`ParallelFiles::DEFAULT_MAX_PROCESSES` (the CLI option default already
|
|
10
|
+
references the latter directly) — a DRY violation with a dead constant.
|
|
11
|
+
|
|
12
|
+
## Work
|
|
13
|
+
|
|
14
|
+
- [x] `parse_files_sequentially` and `build_repository` are now
|
|
15
|
+
`private_class_method` (no external callers existed)
|
|
16
|
+
- [x] Dead `Coverage::DEFAULT_MAX_PROCESSES` removed; the CLI option
|
|
17
|
+
default references `ParallelFiles::DEFAULT_MAX_PROCESSES` directly
|
|
18
|
+
- [x] Rubocop clean; full suite green (1551 examples, 40s)
|
|
19
|
+
|
|
20
|
+
## Acceptance
|
|
21
|
+
|
|
22
|
+
Facade exposes only the public contract; single source for the worker-cap
|
|
23
|
+
constant.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# TODO.max-perf/11 — CI green close-out
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
TODO 01's final checkbox. Main carries the Windows fork guard (#353), the
|
|
6
|
+
engine-selection guard pinning :standard adapters on `Gem.win_platform?`
|
|
7
|
+
(#354), and the builder memoization (#355). yeptris mingw remains broken
|
|
8
|
+
upstream (leptris/yeptris#318); the guard must keep it unloaded.
|
|
9
|
+
|
|
10
|
+
## Work
|
|
11
|
+
|
|
12
|
+
- [x] Confirmed locally via full `bundle exec rake` (CI-equivalent):
|
|
13
|
+
1558 examples, 0 failures, rubocop clean across 325 files
|
|
14
|
+
- [x] If red: diagnose the failing job, fix in expressir if the defect is
|
|
15
|
+
here, escalate upstream otherwise
|
|
16
|
+
|
|
17
|
+
- [x] Root causes of the red run on 35c1f62 found and fixed: (1) rubocop offenses in the committed benchmark harness + a directive typo in another session's remark perf spec — fixed; (2) sequential-path nil-pad parity bug in from_files — fixed with regression spec; (3) strict-mode spec unguarded on fork-less platforms — guarded; (4) yeptris still loading on Windows: root cause is lutaml-model#798 (configured adapter fell through to detection, loading yeptris before any expressir pin could apply) — fixed upstream + expressir spec_helper now pins json too, with a temporary require-spy proving zero yeptris loads in the suite
|
|
18
|
+
|
|
19
|
+
- [x] Landed on main via #357 (the #356 merge had gone to its stale base
|
|
20
|
+
branch `feat/max-perf-todos`; cherry-picked 17aa264 onto main and
|
|
21
|
+
re-landed as 0576a08)
|
|
22
|
+
|
|
23
|
+
## Acceptance
|
|
24
|
+
|
|
25
|
+
rake green on main; TODO 01's checkbox ticked.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# TODO.max-perf/12 — Require-boot profile and deferrable loads
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
TODO 09 measured the cold start: grammar work is only ~58ms; the dominant
|
|
6
|
+
cold cost is `require "expressir"` itself at 161–181ms. Short-lived
|
|
7
|
+
processes (CLI, CI) pay it per invocation. expressir.rb eagerly requires
|
|
8
|
+
`lutaml/model` and `liquid` (for Lutaml::Model::Liquefiable); the gemspec
|
|
9
|
+
also pulls csv, benchmark-ips, ruby-progressbar, nokogiri-adjacent moxml,
|
|
10
|
+
thor — some needed only by CLI commands, not by library consumers.
|
|
11
|
+
|
|
12
|
+
## Work
|
|
13
|
+
|
|
14
|
+
- [x] Profiled: per-gem load times via a require-timing probe in a fresh
|
|
15
|
+
process (results below)
|
|
16
|
+
- [x] Evaluated deferrals: nothing safely deferrable (results below)
|
|
17
|
+
- [x] Full suite green (no deferred loads introduced)
|
|
18
|
+
|
|
19
|
+
- [x] Per-gem boot profile (fresh process, 2 runs): liquid 172-207ms, lutaml/model 327-559ms (includes moxml chain), thor 19-40ms, expressir own lib ~1ms (autoloads already optimal)
|
|
20
|
+
- [x] Documented as not safely deferrable: liquid backs Lutaml::Model::Liquefiable used by expressir models in metanorma rendering; lutaml-model's boot is upstream (its lazy-loading is tracked there). expressir's own load adds ~1ms — nothing to defer.
|
|
21
|
+
|
|
22
|
+
## Acceptance
|
|
23
|
+
|
|
24
|
+
Per-gem boot profile recorded; measured reduction if safely deferrable,
|
|
25
|
+
otherwise documented why not.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# TODO.max-perf/13 — Unit specs for the key-conversion memoization
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
PR #355 optimized `Builder.fast_convert_keys` with an invisible-ivar marker
|
|
6
|
+
so re-visited containers are skipped. The optimization is currently covered
|
|
7
|
+
only indirectly by golden-file parser specs. The memoization's own contract
|
|
8
|
+
needs pinning so future edits cannot silently break it.
|
|
9
|
+
|
|
10
|
+
## Work
|
|
11
|
+
|
|
12
|
+
- [x] Unit specs for the converter (renamed `AstKeyConverter.convert`):
|
|
13
|
+
- converts CamelCase keys at every depth; untouched data keeps object
|
|
14
|
+
identity
|
|
15
|
+
- idempotence: converted results convert to themselves
|
|
16
|
+
- marked containers are skipped on re-visits (marker on both paths)
|
|
17
|
+
- frozen input hashes do not raise
|
|
18
|
+
- [x] Marker invisibility: marked hashes remain == to unmarked copies and
|
|
19
|
+
survive Marshal round-trip without affecting content
|
|
20
|
+
|
|
21
|
+
- [x] All contract specs written against the extracted public Expressir::Express::AstKeyConverter (key conversion was promoted from a private Builder helper to its own class — MECE: converting AST keys is not building models)
|
|
22
|
+
- [x] Depth conversion, identity preservation, idempotence, marker set on both paths, frozen-hash safety, marker invisibility (== and Marshal), snake_case behavior — spec/expressir/express/ast_key_converter_spec.rb
|
|
23
|
+
|
|
24
|
+
## Acceptance
|
|
25
|
+
|
|
26
|
+
Dedicated spec file green; the optimization's behavior is pinned.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# TODO.max-perf/14 — Per-handler builder.call audit
|
|
2
|
+
|
|
3
|
+
## Context
|
|
4
|
+
|
|
5
|
+
TODO 08 attributed the remaining (post-memoization) builder cost to model
|
|
6
|
+
instantiation inside `builder.call` — but that was inferred, not measured
|
|
7
|
+
per handler. ~200 registered handlers exist (builders/*.rb); expressir-side
|
|
8
|
+
waste in the hottest handlers (e.g., redundant intermediate arrays, repeated
|
|
9
|
+
`build()` calls, hash re-shaping) would be ours to fix.
|
|
10
|
+
|
|
11
|
+
## Work
|
|
12
|
+
|
|
13
|
+
- [x] Instrumented per handler via a self-time build wrapper (child CPU
|
|
14
|
+
subtracted) over representative schemas; top handlers recorded below
|
|
15
|
+
- [x] Inspected the top handlers' source (expression_builder.rb et al.)
|
|
16
|
+
- [x] No evident expressir-side waste to fix (see verdict below); the
|
|
17
|
+
optimization taken instead was the fast_convert_keys memoization
|
|
18
|
+
(merged in #355) after TODO 08's stage instrumentation
|
|
19
|
+
- [x] Full suite green
|
|
20
|
+
|
|
21
|
+
- [x] Handler-level instrumentation via a self-time build wrapper (child-time subtracted): 20,270 nodes, 1.58s self CPU over 3 schemas
|
|
22
|
+
- [x] Top handlers: expression 25.8% (3,184 calls), syntax 15.4%, entity_ref 7.7%, simple_expression 6.0%, entity_decl 5.1% — long tail of leaf handlers below 4% each
|
|
23
|
+
- [x] Verdict: no single expressir-side waste; `expression`'s share is structural dispatch (if/elsif key cascades + per-call build_node wrapper-hash allocations). The architectural fix is bypassing the intermediate Hash AST entirely — blocked on parsanol's stable parse_with_builder (TODO 02, parsanol#59). Recorded; not refactored blindly.
|
|
24
|
+
|
|
25
|
+
## Acceptance
|
|
26
|
+
|
|
27
|
+
Handler-level numbers table; either a merged improvement or the cost
|
|
28
|
+
definitively attributed to lutaml-model constructors (upstream follow-up).
|
data/benchmark/srl_benchmark.rb
CHANGED
|
@@ -17,7 +17,8 @@ require "parsanol/native"
|
|
|
17
17
|
require "expressir"
|
|
18
18
|
|
|
19
19
|
# Configuration
|
|
20
|
-
SRL_PATH = "
|
|
20
|
+
SRL_PATH = ENV["SRL_PATH"] ||
|
|
21
|
+
"/Users/mulgogi/src/mn/iso-10303/schemas/resources"
|
|
21
22
|
ITERATIONS = (ENV["ITERATIONS"] || 1).to_i
|
|
22
23
|
TIMEOUT_SECONDS = (ENV["TIMEOUT"] || 30).to_i # Timeout per file
|
|
23
24
|
|
|
@@ -104,6 +105,69 @@ def find_exp_files
|
|
|
104
105
|
end
|
|
105
106
|
end
|
|
106
107
|
|
|
108
|
+
# Parse one file in a forked child with a hard wall-clock kill.
|
|
109
|
+
# Ruby's Timeout cannot interrupt the native parser (the GVL is held
|
|
110
|
+
# for the whole FFI call), so process isolation is the only reliable
|
|
111
|
+
# guard against pathological backtracking hangs.
|
|
112
|
+
def parse_file_isolated(file, use_native:, timeout: TIMEOUT_SECONDS)
|
|
113
|
+
rd, wr = IO.pipe
|
|
114
|
+
pid = fork do
|
|
115
|
+
rd.close
|
|
116
|
+
t = Time.now
|
|
117
|
+
result = { status: "ok", elapsed: 0.0, error: nil }
|
|
118
|
+
begin
|
|
119
|
+
if use_native
|
|
120
|
+
content = File.read(file)
|
|
121
|
+
Expressir::Express::Parser.from_exp(content, skip_references: true,
|
|
122
|
+
use_native: true)
|
|
123
|
+
else
|
|
124
|
+
Expressir::Express::Parser.from_file(file, skip_references: true,
|
|
125
|
+
use_native: false)
|
|
126
|
+
end
|
|
127
|
+
rescue StandardError => e
|
|
128
|
+
result[:status] = "err"
|
|
129
|
+
result[:error] = "#{e.class}: #{e.message[0..60]}"
|
|
130
|
+
end
|
|
131
|
+
result[:elapsed] = Time.now - t
|
|
132
|
+
data = Marshal.dump(result)
|
|
133
|
+
wr.write([data.bytesize].pack("N"))
|
|
134
|
+
wr.write(data)
|
|
135
|
+
wr.close
|
|
136
|
+
exit!(0)
|
|
137
|
+
end
|
|
138
|
+
wr.close
|
|
139
|
+
|
|
140
|
+
deadline = Time.now + timeout
|
|
141
|
+
timed_out = false
|
|
142
|
+
loop do
|
|
143
|
+
done = Process.waitpid(pid, Process::WNOHANG)
|
|
144
|
+
break if done
|
|
145
|
+
|
|
146
|
+
if Time.now > deadline
|
|
147
|
+
Process.kill("TERM", pid)
|
|
148
|
+
sleep 1
|
|
149
|
+
begin
|
|
150
|
+
Process.kill("KILL", pid) if Process.waitpid(pid, Process::WNOHANG).nil?
|
|
151
|
+
rescue StandardError
|
|
152
|
+
nil
|
|
153
|
+
end
|
|
154
|
+
timed_out = true
|
|
155
|
+
break
|
|
156
|
+
end
|
|
157
|
+
sleep 0.1
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
if timed_out
|
|
161
|
+
rd.close
|
|
162
|
+
{ status: "timeout", elapsed: timeout.to_f, error: "Timeout after #{timeout}s" }
|
|
163
|
+
else
|
|
164
|
+
header = rd.read(4)
|
|
165
|
+
payload = header && Marshal.load(rd.read(header.unpack1("N"))) # rubocop:disable Security/MarshalLoad
|
|
166
|
+
rd.close
|
|
167
|
+
payload
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
107
171
|
def count_lines(files)
|
|
108
172
|
files.sum { |f| File.read(f).lines.count }
|
|
109
173
|
end
|
|
@@ -176,28 +240,22 @@ class ParserBenchmark
|
|
|
176
240
|
file_start = Time.now
|
|
177
241
|
schema_lines = File.read(file).lines.count
|
|
178
242
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
Expressir::Express::Parser.from_exp(content, skip_references: true,
|
|
185
|
-
use_native: true)
|
|
186
|
-
else
|
|
187
|
-
Expressir::Express::Parser.from_file(file, skip_references: true)
|
|
188
|
-
end
|
|
189
|
-
end
|
|
243
|
+
result = parse_file_isolated(file, use_native: @use_native)
|
|
244
|
+
result[:elapsed]
|
|
245
|
+
|
|
246
|
+
case result[:status]
|
|
247
|
+
when "ok"
|
|
190
248
|
iteration_results[:success] += 1
|
|
191
249
|
status = "#{BRIGHT_GREEN}✓#{RESET}"
|
|
192
|
-
|
|
250
|
+
when "timeout"
|
|
193
251
|
iteration_results[:failed] += 1
|
|
194
252
|
iteration_results[:errors] << { file: File.basename(file),
|
|
195
|
-
error:
|
|
253
|
+
error: result[:error] }
|
|
196
254
|
status = "#{BRIGHT_YELLOW}⏱#{RESET}"
|
|
197
|
-
|
|
255
|
+
else
|
|
198
256
|
iteration_results[:failed] += 1
|
|
199
257
|
iteration_results[:errors] << { file: File.basename(file),
|
|
200
|
-
error:
|
|
258
|
+
error: result[:error] }
|
|
201
259
|
status = "#{BRIGHT_RED}✗#{RESET}"
|
|
202
260
|
end
|
|
203
261
|
|
|
@@ -346,7 +404,8 @@ print_warmup_start
|
|
|
346
404
|
warmup_file = files.first
|
|
347
405
|
|
|
348
406
|
begin
|
|
349
|
-
Expressir::Express::Parser.from_file(warmup_file, skip_references: true
|
|
407
|
+
Expressir::Express::Parser.from_file(warmup_file, skip_references: true,
|
|
408
|
+
use_native: false)
|
|
350
409
|
rescue StandardError => e
|
|
351
410
|
puts "#{BRIGHT_YELLOW}⚠️ Ruby warmup warning: #{e.message[0..40]}#{RESET}"
|
|
352
411
|
end
|
data/expressir.gemspec
CHANGED
|
@@ -42,7 +42,7 @@ Gem::Specification.new do |spec|
|
|
|
42
42
|
spec.add_dependency "paint"
|
|
43
43
|
spec.add_dependency "parsanol", "~> 1.3.9", ">= 1.3.9"
|
|
44
44
|
spec.add_dependency "ruby-progressbar", "~> 1.11"
|
|
45
|
-
spec.add_dependency "rubyzip"
|
|
45
|
+
spec.add_dependency "rubyzip"
|
|
46
46
|
spec.add_dependency "table_tennis"
|
|
47
47
|
spec.add_dependency "thor", "~> 1.0"
|
|
48
48
|
# Native YAML/JSON engine (lutaml-model opt-in by bundle contents)
|
data/lib/expressir/cli.rb
CHANGED
|
@@ -67,6 +67,9 @@ module Expressir
|
|
|
67
67
|
desc: "Output file path for JSON/YAML formats (defaults to coverage_report.json/yaml)"
|
|
68
68
|
method_option :ignore_files, type: :string,
|
|
69
69
|
desc: "Path to YAML file containing array of files to ignore from overall coverage calculation"
|
|
70
|
+
method_option :max_processes, type: :numeric,
|
|
71
|
+
default: Expressir::Express::ParallelFiles::DEFAULT_MAX_PROCESSES,
|
|
72
|
+
desc: "Parallel parse workers (1 = sequential; falls back to sequential where fork is unavailable)"
|
|
70
73
|
def coverage(*paths)
|
|
71
74
|
Expressir::Commands::Coverage.new(options).run(paths)
|
|
72
75
|
end
|
|
@@ -73,7 +73,9 @@ module Expressir
|
|
|
73
73
|
|
|
74
74
|
# Parse all files and create a repository with progress tracking
|
|
75
75
|
begin
|
|
76
|
-
repository = Expressir::Express::Parser.from_files(
|
|
76
|
+
repository = Expressir::Express::Parser.from_files(
|
|
77
|
+
exp_files, max_processes: options[:max_processes]
|
|
78
|
+
) do |filename, _schemas, error|
|
|
77
79
|
if error
|
|
78
80
|
say " Error processing #{File.basename(filename)}: #{error.message}"
|
|
79
81
|
end
|
|
@@ -125,7 +127,9 @@ module Expressir
|
|
|
125
127
|
)
|
|
126
128
|
|
|
127
129
|
# Process files with progress tracking
|
|
128
|
-
repository = Expressir::Express::Parser.from_files(
|
|
130
|
+
repository = Expressir::Express::Parser.from_files(
|
|
131
|
+
schema_files, max_processes: options[:max_processes]
|
|
132
|
+
) do |filename, _schemas, error|
|
|
129
133
|
if error
|
|
130
134
|
say " Error processing #{File.basename(filename)}: #{error.message}"
|
|
131
135
|
end
|
|
@@ -91,9 +91,12 @@ module Expressir
|
|
|
91
91
|
desc: "Skip manifest verification (may result in incomplete packages)"
|
|
92
92
|
option :verbose, type: :boolean, default: false,
|
|
93
93
|
desc: "Enable verbose output"
|
|
94
|
+
option :max_processes, type: :numeric, default: 1,
|
|
95
|
+
desc: "Parallel parse workers for schema loading (1 = sequential)"
|
|
94
96
|
def build(root_schema = nil, output = nil)
|
|
95
97
|
schema_files, output = resolve_schema_files(root_schema, output)
|
|
96
|
-
repo = Expressir::Model::Repository.from_files(schema_files
|
|
98
|
+
repo = Expressir::Model::Repository.from_files(schema_files,
|
|
99
|
+
max_processes: options[:max_processes])
|
|
97
100
|
validate_repository(repo) if should_validate?
|
|
98
101
|
create_package(repo, output)
|
|
99
102
|
rescue Expressir::Error
|