flexr 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. checksums.yaml +7 -0
  2. data/.rubocop.yml +33 -0
  3. data/CONTRIBUTING.md +39 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +116 -0
  6. data/Rakefile +468 -0
  7. data/benchmark/baselines/json.json +34 -0
  8. data/benchmark/baselines/json_handwritten.rb +43 -0
  9. data/benchmark/baselines/json_rexical.rex +25 -0
  10. data/benchmark/corpora/README.md +11 -0
  11. data/benchmark/corpora/generate_json.rb +26 -0
  12. data/benchmark/golden/calculator_lexer.sha256 +1 -0
  13. data/benchmark/golden/json_lexer.sha256 +1 -0
  14. data/benchmark/golden/regexp_tokenizer.sha256 +1 -0
  15. data/benchmark/golden/ruby_subset_lexer.sha256 +1 -0
  16. data/benchmark/golden/toy_lang_lexer.sha256 +1 -0
  17. data/benchmark/golden/with_lrama_lexer.sha256 +1 -0
  18. data/benchmark/golden/with_racc_lexer.sha256 +1 -0
  19. data/benchmark/run.rb +254 -0
  20. data/docs/README.md +64 -0
  21. data/docs/RELEASING.md +30 -0
  22. data/docs/adr/0001-byte-level-dfa.md +5 -0
  23. data/docs/adr/0003-leftmost-longest.md +4 -0
  24. data/docs/adr/0006-accel-not-scanner.md +4 -0
  25. data/docs/adr/0008-what-pure-ruby-means.md +5 -0
  26. data/docs/adr/0016-spec-is-plain-ruby.md +4 -0
  27. data/docs/adr/0017-static-analysis-by-default.md +5 -0
  28. data/docs/adr/0018-prism-for-generator-only.md +4 -0
  29. data/docs/adr/0019-measured-performance-floor.md +26 -0
  30. data/docs/adr/0020-vendored-unicode-contract.md +21 -0
  31. data/docs/explanation/backends.md +33 -0
  32. data/docs/explanation/matching-semantics.md +20 -0
  33. data/docs/explanation/runtime-vs-generated.md +22 -0
  34. data/docs/explanation/security-model.md +18 -0
  35. data/docs/explanation/unicode-and-encoding.md +20 -0
  36. data/docs/how-to/deploy-a-standalone-lexer.md +23 -0
  37. data/docs/how-to/generate-a-lexer.md +39 -0
  38. data/docs/how-to/handle-errors.md +32 -0
  39. data/docs/how-to/integrate-with-lrama.md +21 -0
  40. data/docs/how-to/integrate-with-racc.md +25 -0
  41. data/docs/how-to/migrate-from-flex.md +21 -0
  42. data/docs/how-to/migrate-from-rexical.md +23 -0
  43. data/docs/how-to/run-a-lexer-at-runtime.md +29 -0
  44. data/docs/how-to/track-token-locations.md +27 -0
  45. data/docs/how-to/tune-performance.md +23 -0
  46. data/docs/how-to/use-states.md +36 -0
  47. data/docs/how-to/use-trailing-context.md +22 -0
  48. data/docs/internals/README.md +14 -0
  49. data/docs/perf-log.md +56 -0
  50. data/docs/reference/README.md +23 -0
  51. data/docs/reference/actions.md +47 -0
  52. data/docs/reference/cli.md +80 -0
  53. data/docs/reference/compatibility.md +38 -0
  54. data/docs/reference/diagnostics.md +41 -0
  55. data/docs/reference/dsl.md +81 -0
  56. data/docs/reference/errors.md +27 -0
  57. data/docs/reference/generated-artifacts.md +50 -0
  58. data/docs/reference/public-api.md +42 -0
  59. data/docs/reference/regexp.md +39 -0
  60. data/docs/reference/runtime.md +49 -0
  61. data/docs/reference/tokens-and-locations.md +33 -0
  62. data/docs/tutorial/build-a-calculator-lexer.md +96 -0
  63. data/examples/calculator/README.md +27 -0
  64. data/examples/calculator/lexer.flexr.rb +17 -0
  65. data/examples/json/README.md +30 -0
  66. data/examples/json/lexer.flexr.rb +24 -0
  67. data/examples/ruby_subset/README.md +17 -0
  68. data/examples/ruby_subset/lexer.flexr.rb +22 -0
  69. data/examples/toy_lang/README.md +17 -0
  70. data/examples/toy_lang/lexer.flexr.rb +18 -0
  71. data/examples/with_lrama/README.md +17 -0
  72. data/examples/with_lrama/lexer.flexr.rb +13 -0
  73. data/examples/with_racc/README.md +17 -0
  74. data/examples/with_racc/lexer.flexr.rb +13 -0
  75. data/exe/flexr +7 -0
  76. data/lib/flexr/automaton/accel.rb +39 -0
  77. data/lib/flexr/automaton/analysis.rb +38 -0
  78. data/lib/flexr/automaton/byte_class_set.rb +29 -0
  79. data/lib/flexr/automaton/compiler.rb +413 -0
  80. data/lib/flexr/automaton/dfa.rb +103 -0
  81. data/lib/flexr/automaton/minimizer.rb +70 -0
  82. data/lib/flexr/automaton/nfa.rb +92 -0
  83. data/lib/flexr/cli.rb +342 -0
  84. data/lib/flexr/codegen/base.rb +17 -0
  85. data/lib/flexr/codegen/direct.rb +52 -0
  86. data/lib/flexr/codegen/firstmatch.rb +17 -0
  87. data/lib/flexr/codegen/table.rb +158 -0
  88. data/lib/flexr/codegen/table_packer.rb +61 -0
  89. data/lib/flexr/diagnostics.rb +94 -0
  90. data/lib/flexr/dsl.rb +182 -0
  91. data/lib/flexr/errors.rb +28 -0
  92. data/lib/flexr/generated.rb +125 -0
  93. data/lib/flexr/generator.rb +400 -0
  94. data/lib/flexr/importer.rb +560 -0
  95. data/lib/flexr/ir.rb +36 -0
  96. data/lib/flexr/lexer.rb +10 -0
  97. data/lib/flexr/options.rb +47 -0
  98. data/lib/flexr/rake_task.rb +27 -0
  99. data/lib/flexr/regexp/ast.rb +45 -0
  100. data/lib/flexr/regexp/char_class.rb +7 -0
  101. data/lib/flexr/regexp/normalizer.rb +117 -0
  102. data/lib/flexr/regexp/parser.rb +517 -0
  103. data/lib/flexr/regexp/tokenizer.flexr.rb +27 -0
  104. data/lib/flexr/regexp/tokenizer.rb +168 -0
  105. data/lib/flexr/regexp/unsupported.rb +7 -0
  106. data/lib/flexr/runtime/buffer.rb +112 -0
  107. data/lib/flexr/runtime/core.rb +388 -0
  108. data/lib/flexr/runtime/errors.rb +22 -0
  109. data/lib/flexr/runtime/interpreter.rb +505 -0
  110. data/lib/flexr/runtime/location.rb +26 -0
  111. data/lib/flexr/runtime/token.rb +7 -0
  112. data/lib/flexr/source/passthrough.rb +31 -0
  113. data/lib/flexr/source/prism_reader.rb +283 -0
  114. data/lib/flexr/source/static_eval.rb +145 -0
  115. data/lib/flexr/unicode/case_fold.rb +45 -0
  116. data/lib/flexr/unicode/data/LICENSE-UNICODE.txt +5 -0
  117. data/lib/flexr/unicode/data/UNICODE_VERSION +1 -0
  118. data/lib/flexr/unicode/data/case_folding.rb +9 -0
  119. data/lib/flexr/unicode/data/properties.rb +10 -0
  120. data/lib/flexr/unicode/property.rb +107 -0
  121. data/lib/flexr/unicode/reference_regexp.rb +102 -0
  122. data/lib/flexr/unicode/utf8_splitter.rb +109 -0
  123. data/lib/flexr/version.rb +5 -0
  124. data/lib/flexr.rb +81 -0
  125. data/site/README.md +22 -0
  126. data/site/astro.config.mjs +57 -0
  127. data/site/package.json +19 -0
  128. data/site/pnpm-lock.yaml +5029 -0
  129. data/site/pnpm-workspace.yaml +6 -0
  130. data/site/public/playground.js +189 -0
  131. data/site/scripts/verify-site.mjs +42 -0
  132. data/site/src/content/docs/benchmarks.md +8 -0
  133. data/site/src/content/docs/concepts/matching-semantics.md +15 -0
  134. data/site/src/content/docs/concepts/regexp-model.md +18 -0
  135. data/site/src/content/docs/concepts/runtime-vs-generated.md +15 -0
  136. data/site/src/content/docs/concepts/security-model.md +15 -0
  137. data/site/src/content/docs/examples.md +17 -0
  138. data/site/src/content/docs/learn/generation.md +29 -0
  139. data/site/src/content/docs/learn/getting-started.md +56 -0
  140. data/site/src/content/docs/learn/parser-integration.md +27 -0
  141. data/site/src/content/docs/learn/runtime-mode.md +32 -0
  142. data/site/src/content/docs/reference/action-context.md +20 -0
  143. data/site/src/content/docs/reference/cli.md +22 -0
  144. data/site/src/content/docs/reference/diagnostics.md +16 -0
  145. data/site/src/content/docs/reference/dsl.md +19 -0
  146. data/site/src/content/docs/reference/public-api.md +18 -0
  147. data/site/src/content/docs/reference/regexp.md +16 -0
  148. data/site/src/content/docs/reference/runtime.md +16 -0
  149. data/site/src/content/docs/reference/tokens-and-locations.md +16 -0
  150. data/site/src/content.config.ts +12 -0
  151. data/site/src/env.d.ts +1 -0
  152. data/site/src/layouts/SiteLayout.astro +39 -0
  153. data/site/src/pages/index.astro +174 -0
  154. data/site/src/pages/playground.astro +64 -0
  155. data/site/src/styles/custom.css +711 -0
  156. data/site/tsconfig.json +5 -0
  157. data/tools/coverage.rb +32 -0
  158. data/tools/docs_verify.rb +116 -0
  159. data/tools/gen_unicode_tables.rb +202 -0
  160. data/tools/regexp_tokenizer_reference.rb +60 -0
  161. metadata +205 -0
data/site/src/layouts/SiteLayout.astro ADDED
@@ -0,0 +1,39 @@
1
+ ---
2
+ import '../styles/custom.css';
3
+
4
+ export interface Props {
5
+ title: string;
6
+ description: string;
7
+ }
8
+
9
+ const { title, description } = Astro.props;
10
+ const siteTitle = `${title} · flexr`;
11
+ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
12
+ ---
13
+ <!doctype html>
14
+ <html lang="en">
15
+ <head>
16
+ <meta charset="UTF-8" />
17
+ <meta name="viewport" content="width=device-width" />
18
+ <meta name="description" content={description} />
19
+ <title>{siteTitle}</title>
20
+ </head>
21
+ <body>
22
+ <a class="skip-link" href="#main-content">Skip to content</a>
23
+ <header class="site-nav">
24
+ <div class="site-nav-inner">
25
+ <a class="site-nav-brand" href={base} aria-label="flexr home">flexr<span>/</span></a>
26
+ <nav class="site-nav-links" aria-label="Primary navigation">
27
+ <a href={`${base}playground/`}>Playground</a>
28
+ <a href={`${base}learn/getting-started/`}>Learn</a>
29
+ <a href={`${base}reference/dsl/`}>Reference</a>
30
+ <a href="https://github.com/ydah/flexr">GitHub</a>
31
+ </nav>
32
+ <a class="site-nav-cta" href={`${base}learn/getting-started/`}>Get started <span aria-hidden="true">↗</span></a>
33
+ </div>
34
+ </header>
35
+ <div id="main-content">
36
+ <slot />
37
+ </div>
38
+ </body>
39
+ </html>
data/site/src/pages/index.astro ADDED
@@ -0,0 +1,174 @@
1
+ ---
2
+ import SiteLayout from '../layouts/SiteLayout.astro';
3
+
4
+ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
5
+ const repo = 'https://github.com/ydah/flexr';
6
+ ---
7
+ <SiteLayout
8
+ title="Ruby-native lexer tooling"
9
+ description="flexr lets parser authors write lexer specifications in ordinary Ruby, inspect matching decisions, and generate deterministic Ruby for deployment."
10
+ >
11
+ <div class="home-shell">
12
+ <header class="home-hero">
13
+ <div>
14
+ <div class="eyebrow">Ruby lexer tooling for parser authors</div>
15
+ <h1>Write lexers in Ruby. See why each token wins.</h1>
16
+ <p class="hero-copy">
17
+ flexr lets you write a lexer specification as ordinary Ruby, run it while developing,
18
+ inspect its matching behavior, and generate deterministic Ruby for deployment.
19
+ </p>
20
+ <div class="hero-actions">
21
+ <a class="button button-primary" href={`${base}playground/`}>Open the playground</a>
22
+ <a class="button button-secondary" href={`${base}learn/getting-started/`}>Get started</a>
23
+ </div>
24
+ </div>
25
+
26
+ <div class="demo-window" aria-label="Longest-match example">
27
+ <div class="window-bar">
28
+ <span class="window-dot"></span>
29
+ <span class="window-dot"></span>
30
+ <span class="window-dot"></span>
31
+ <span>lexer.flexr.rb</span>
32
+ </div>
33
+ <pre class="code-block"><code><span class="code-keyword">class</span> Lexer &lt; Flexr::Lexer
34
+ rule(<span class="code-string">/==/</span>) &#123; emit <span class="code-string">:EQ</span> &#125;
35
+ rule(<span class="code-string">/=/</span>) &#123; emit <span class="code-string">:ASSIGN</span> &#125;
36
+ rule(<span class="code-string">/if/</span>) &#123; emit <span class="code-string">:IF</span> &#125;
37
+ rule(<span class="code-string">/[a-z_][a-z0-9_]*/</span>) &#123; emit <span class="code-string">:IDENT</span> &#125;
38
+ <span class="code-keyword">end</span></code></pre>
39
+ <div class="decision-panel">
40
+ <div class="decision-label"><span>input: <strong>if == total</strong></span><span>step 02</span></div>
41
+ <div class="decision-row winner"><strong>/==/ → EQ</strong><span>longest match · 2 chars</span></div>
42
+ <div class="decision-row"><strong>/=/ → ASSIGN</strong><span>shorter match · 1 char</span></div>
43
+ </div>
44
+ </div>
45
+ </header>
46
+
47
+ <main>
48
+ <section class="home-section" aria-labelledby="why-heading">
49
+ <div class="section-heading">
50
+ <div>
51
+ <div class="eyebrow">Why flexr</div>
52
+ <h2>One specification, two useful ways to run it.</h2>
53
+ </div>
54
+ <p>
55
+ Keep the feedback loop in Ruby. When the lexer is ready to ship, compile the same
56
+ rules into a predictable artifact and keep the behavior easy to reason about.
57
+ </p>
58
+ </div>
59
+ <div class="card-grid">
60
+ <article class="site-card">
61
+ <div class="card-index">01 / RUBY-NATIVE</div>
62
+ <h3>Ordinary Ruby specifications</h3>
63
+ <p>Use classes, constants, interpolation, and familiar Ruby actions without learning a separate lexer language.</p>
64
+ </article>
65
+ <article class="site-card">
66
+ <div class="card-index">02 / DECISIONS</div>
67
+ <h3>Matching you can explain</h3>
68
+ <p>Longest match wins. If matches have the same length, the rule defined first wins. The contract stays visible.</p>
69
+ </article>
70
+ <article class="site-card">
71
+ <div class="card-index">03 / PARITY</div>
72
+ <h3>Runtime and generated parity</h3>
73
+ <p>Develop with the runtime interpreter, then compare its token stream with generated Ruby before deployment.</p>
74
+ </article>
75
+ <article class="site-card">
76
+ <div class="card-index">04 / DIAGNOSTICS</div>
77
+ <h3>Actionable checks</h3>
78
+ <p>Find unsupported regexp constructs, unreachable rules, undeclared tokens, and other problems before they reach a parser.</p>
79
+ </article>
80
+ </div>
81
+ </section>
82
+
83
+ <section class="home-section" aria-labelledby="workflow-heading">
84
+ <div class="section-heading">
85
+ <div>
86
+ <div class="eyebrow">A practical workflow</div>
87
+ <h2>Write. Inspect. Generate. Integrate.</h2>
88
+ </div>
89
+ </div>
90
+ <div class="workflow-grid">
91
+ <article class="workflow-step">
92
+ <div class="card-index">01</div>
93
+ <h3>Write</h3>
94
+ <p>Define rules and actions in a Ruby class that looks like the code around it.</p>
95
+ </article>
96
+ <article class="workflow-step">
97
+ <div class="card-index">02</div>
98
+ <h3>Inspect</h3>
99
+ <p>Check the specification and make a matching decision visible when rules compete.</p>
100
+ </article>
101
+ <article class="workflow-step">
102
+ <div class="card-index">03</div>
103
+ <h3>Generate</h3>
104
+ <p>Produce deterministic Ruby for environments where startup and deployment shape matter.</p>
105
+ </article>
106
+ <article class="workflow-step">
107
+ <div class="card-index">04</div>
108
+ <h3>Integrate</h3>
109
+ <p>Return tokens to Racc, Lrama, or your own parser protocol with explicit locations.</p>
110
+ </article>
111
+ </div>
112
+ </section>
113
+
114
+ <section class="home-section" aria-labelledby="modes-heading">
115
+ <div class="section-heading">
116
+ <div>
117
+ <div class="eyebrow">Choose your mode</div>
118
+ <h2>Fast feedback first. Reproducible output when it matters.</h2>
119
+ </div>
120
+ <p>Both modes consume the same lexer design. The choice is about the deployment boundary, not a second language.</p>
121
+ </div>
122
+ <div class="compare-panel" id="modes-heading">
123
+ <div>
124
+ <h3>Runtime mode</h3>
125
+ <p>Load the specification directly with the <code>flexr</code> runtime. It is a strong fit for iteration, tests, and applications that value a compact source of truth.</p>
126
+ <div class="home-links"><a class="button button-secondary" href={`${base}learn/runtime-mode/`}>Runtime guide</a></div>
127
+ </div>
128
+ <div>
129
+ <h3>Generated mode</h3>
130
+ <p>Compile the specification to Ruby and commit or build the generated artifact. Use standalone output when the deployed process should not load the gem at runtime.</p>
131
+ <div class="home-links"><a class="button button-secondary" href={`${base}learn/generation/`}>Generation guide</a></div>
132
+ </div>
133
+ </div>
134
+ </section>
135
+
136
+ <section class="home-section" aria-labelledby="trust-heading">
137
+ <div class="section-heading">
138
+ <div>
139
+ <div class="eyebrow">Make the boundary explicit</div>
140
+ <h2>Useful power, clear constraints.</h2>
141
+ </div>
142
+ <p>flexr is designed to be inspectable. Know what runs at build time, what remains Ruby code, and what the regexp compiler accepts.</p>
143
+ </div>
144
+ <div class="trust-grid">
145
+ <article class="trust-panel">
146
+ <div class="card-index">SECURITY</div>
147
+ <h3>Actions are Ruby code</h3>
148
+ <p>Only run specifications and generated artifacts you trust. Static checks do not make arbitrary actions safe.</p>
149
+ </article>
150
+ <article class="trust-panel">
151
+ <div class="card-index">COMPATIBILITY</div>
152
+ <h3>Regexp support is documented</h3>
153
+ <p>Check the compatibility matrix before porting lookarounds, backreferences, or open-ended repetition.</p>
154
+ </article>
155
+ <article class="trust-panel">
156
+ <div class="card-index">EXPERIMENTAL</div>
157
+ <h3>Experimental features say so</h3>
158
+ <p><code>firstmatch</code> is experimental because it changes the usual longest-match contract and needs an explicit option.</p>
159
+ </article>
160
+ <article class="trust-panel">
161
+ <div class="card-index">OPEN SOURCE</div>
162
+ <h3>Read the implementation</h3>
163
+ <p>Inspect examples, diagnostics, generated output, and the benchmark log in the repository.</p>
164
+ <div class="home-links"><a class="button button-secondary" href={repo}>View on GitHub</a></div>
165
+ </article>
166
+ </div>
167
+ </section>
168
+ </main>
169
+
170
+ <footer class="site-footer">
171
+ <span class="site-mark">flexr</span> · Ruby-native lexer tooling · <a href={`${base}examples/`}>Examples</a> · <a href={`${base}reference/cli/`}>CLI reference</a> · <a href={repo}>Source</a>
172
+ </footer>
173
+ </div>
174
+ </SiteLayout>
data/site/src/pages/playground.astro ADDED
@@ -0,0 +1,64 @@
1
+ ---
2
+ import SiteLayout from '../layouts/SiteLayout.astro';
3
+
4
+ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
5
+ ---
6
+ <SiteLayout
7
+ title="Playground"
8
+ description="Explore flexr's longest-match decision model with an interactive, fixture-backed lexer preview."
9
+ >
10
+ <main class="playground-shell">
11
+ <a class="back-link" href={base}>← Back to flexr</a>
12
+ <header class="playground-header">
13
+ <div class="eyebrow">Interactive preview · flexr 1.0</div>
14
+ <h1>See the decision, not just the token.</h1>
15
+ <p>
16
+ Choose a fixture, edit its input, and inspect which rule wins at each position. This
17
+ preview is static and fixture-backed: it does not execute arbitrary Ruby in your browser.
18
+ Use the repository examples and CLI for full runtime and generation behavior.
19
+ </p>
20
+ <div class="playground-notice" role="note">
21
+ Safe boundary: this page sends no source or input to a server. Arbitrary actions and
22
+ <code>--eval</code> are intentionally unavailable here.
23
+ </div>
24
+ </header>
25
+
26
+ <section class="workbench" aria-label="flexr playground workbench">
27
+ <div class="workbench-panel">
28
+ <div class="panel-heading"><span>Specification fixture</span><small>read-only model</small></div>
29
+ <select id="preset" aria-label="Choose a playground fixture">
30
+ <option value="longest">Longest match</option>
31
+ <option value="calculator">Calculator</option>
32
+ <option value="json">JSON tokens</option>
33
+ </select>
34
+ <textarea id="spec" aria-label="Lexer specification" readonly></textarea>
35
+ <div class="workbench-actions">
36
+ <button class="button button-secondary" type="button" data-action="check">Check</button>
37
+ <button class="button button-secondary" type="button" data-action="generate">Show generated shape</button>
38
+ </div>
39
+ </div>
40
+
41
+ <div class="workbench-panel">
42
+ <div class="panel-heading"><span>Input</span><small>edit, then run</small></div>
43
+ <textarea id="input" class="input-area" aria-label="Input to tokenize"></textarea>
44
+ <div class="workbench-actions">
45
+ <button class="button button-primary" type="button" data-action="run">Run preview</button>
46
+ <button class="button button-secondary" type="button" data-action="compare">Compare modes</button>
47
+ <button class="button button-secondary" type="button" data-action="share">Copy share link</button>
48
+ </div>
49
+ <p id="status" class="workbench-status" role="status" aria-live="polite"></p>
50
+ </div>
51
+
52
+ <div class="workbench-panel full">
53
+ <div class="panel-heading"><span>Tokens</span><small>output</small></div>
54
+ <div id="tokens" class="workbench-output"><p class="output-empty">Run a fixture to see its token stream.</p></div>
55
+ </div>
56
+
57
+ <div class="workbench-panel full">
58
+ <div class="panel-heading"><span>Match decisions</span><small>why the winner won</small></div>
59
+ <div id="decisions" class="workbench-output"><p class="output-empty">Each input position will show its accepted candidates and the winning rule.</p></div>
60
+ </div>
61
+ </section>
62
+ </main>
63
+ <script is:inline src={`${base}playground.js`} defer></script>
64
+ </SiteLayout>