flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "stringio"
|
|
4
|
+
|
|
5
|
+
module FlexrPlayground
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def run(request)
|
|
9
|
+
path = "/playground.flexr.rb"
|
|
10
|
+
source = request.fetch("source")
|
|
11
|
+
File.write(path, source)
|
|
12
|
+
stdout = $stdout
|
|
13
|
+
stderr = $stderr
|
|
14
|
+
$stdout = StringIO.new
|
|
15
|
+
$stderr = StringIO.new
|
|
16
|
+
options = Flexr::Options.default
|
|
17
|
+
options.set(:max_dfa_states, 1024)
|
|
18
|
+
options.set(:warn_level, :all)
|
|
19
|
+
options.color = :never
|
|
20
|
+
diagnostics = StringIO.new
|
|
21
|
+
Flexr::CLI.check(path, options, diagnostics)
|
|
22
|
+
generated = Flexr::Generator.new(path, options: options.generator_options).generate
|
|
23
|
+
parsed = Flexr::CLI.read_spec(path)
|
|
24
|
+
scope = Module.new
|
|
25
|
+
scope.module_eval(request["mode"] == "generated" ? generated : source, path, 1)
|
|
26
|
+
namespace = parsed.class_name.start_with?("::") ? Object : scope
|
|
27
|
+
klass = parsed.class_name.split("::").reject(&:empty?).inject(namespace) { |parent, name| parent.const_get(name, false) }
|
|
28
|
+
unless request["mode"] == "generated"
|
|
29
|
+
klass.__flexr_config.options[:max_dfa_states] = 1024
|
|
30
|
+
klass.__flexr_config.options[:warn_level] = :all
|
|
31
|
+
end
|
|
32
|
+
input_path = "/input.txt"
|
|
33
|
+
File.write(input_path, request.fetch("input"))
|
|
34
|
+
lexer = klass.new(request.fetch("input"), filename: input_path, max_steps: 1_000_000, max_token_size: 65_536,
|
|
35
|
+
max_lookahead_size: 65_536, max_buffer_size: 1_048_576)
|
|
36
|
+
trace = Flexr::Trace.new(lexer)
|
|
37
|
+
tokens = trace.run
|
|
38
|
+
dot = StringIO.new
|
|
39
|
+
Flexr::CLI.print_dot(path, options, dot)
|
|
40
|
+
JSON.generate(tokens: tokens, steps: trace.steps, diagnostics: diagnostics.string,
|
|
41
|
+
generated: generated, dot: dot.string, ruby_version: RUBY_VERSION,
|
|
42
|
+
prism_version: Prism::VERSION,
|
|
43
|
+
console: ($stdout.string + $stderr.string).byteslice(0, 4096).force_encoding(Encoding::UTF_8).scrub)
|
|
44
|
+
rescue StandardError, SyntaxError, LoadError, SystemExit => e
|
|
45
|
+
if e.is_a?(Flexr::LexError) && !e.diagnostic && e.byte_pos
|
|
46
|
+
diagnostic = Flexr::Diagnostics.error("FLEXR-E000", e.message,
|
|
47
|
+
location: [e.byte_pos, e.byte_pos + e.text.to_s.bytesize])
|
|
48
|
+
e = Flexr::LexError.new(e.message, diagnostic: diagnostic,
|
|
49
|
+
filename: e.filename, byte_pos: e.byte_pos, line: e.line, text: e.text)
|
|
50
|
+
end
|
|
51
|
+
JSON.generate(error: Flexr::CLI.render_error(e, options: options, path: path),
|
|
52
|
+
ruby_version: RUBY_VERSION, prism_version: defined?(Prism::VERSION) && Prism::VERSION)
|
|
53
|
+
ensure
|
|
54
|
+
$stdout = stdout
|
|
55
|
+
$stderr = stderr
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { cp, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
const site = fileURLToPath(new URL('..', import.meta.url));
|
|
6
|
+
const root = path.dirname(site);
|
|
7
|
+
const output = path.join(site, 'public/playground-runtime');
|
|
8
|
+
const files = {};
|
|
9
|
+
async function collect(directory) {
|
|
10
|
+
for (const entry of await readdir(path.join(root, directory), { withFileTypes: true })) {
|
|
11
|
+
const relative = path.posix.join(directory, entry.name);
|
|
12
|
+
if (entry.isDirectory()) await collect(relative);
|
|
13
|
+
else if (entry.name.endsWith('.rb')) files[`/app/${relative}`] = await readFile(path.join(root, relative), 'utf8');
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
await collect('lib');
|
|
17
|
+
files['/app/playground.rb'] = await readFile(path.join(site, 'ruby/playground.rb'), 'utf8');
|
|
18
|
+
await mkdir(output, { recursive: true });
|
|
19
|
+
const payload = JSON.stringify({ files });
|
|
20
|
+
await writeFile(path.join(output, 'flexr.json'), payload);
|
|
21
|
+
const inputs = {
|
|
22
|
+
toml: 'title = "flexr"',
|
|
23
|
+
graphql: 'query { user(id: 12) { name } }',
|
|
24
|
+
sql: 'select name from users where id >= 12;',
|
|
25
|
+
};
|
|
26
|
+
const presets = {};
|
|
27
|
+
for (const [name, input] of Object.entries(inputs)) {
|
|
28
|
+
presets[name] = { source: await readFile(path.join(root, `examples/${name}/lexer.flexr.rb`), 'utf8'), input };
|
|
29
|
+
}
|
|
30
|
+
await writeFile(path.join(output, 'presets.json'), JSON.stringify(presets));
|
|
31
|
+
const wasm = path.join(site, 'node_modules/@ruby/4.0-wasm-wasi/dist');
|
|
32
|
+
await cp(path.join(wasm, 'ruby+stdlib.wasm'), path.join(output, 'ruby.wasm'));
|
|
33
|
+
await cp(path.join(wasm, 'LICENSE'), path.join(output, 'LICENSE'));
|
|
34
|
+
await cp(path.join(wasm, 'NOTICE'), path.join(output, 'NOTICE'));
|
|
35
|
+
console.log(`Playground: packaged ${Object.keys(files).length} Ruby files (${Buffer.byteLength(payload)} bytes).`);
|
|
@@ -10,7 +10,9 @@ const required = [
|
|
|
10
10
|
'src/content/docs/learn/generation.md',
|
|
11
11
|
'src/content/docs/reference/dsl.md',
|
|
12
12
|
'src/content/docs/reference/diagnostics.md',
|
|
13
|
-
'
|
|
13
|
+
'src/playground/client.js',
|
|
14
|
+
'src/playground/worker.mjs',
|
|
15
|
+
'ruby/playground.rb'
|
|
14
16
|
];
|
|
15
17
|
|
|
16
18
|
const missing = required.filter((file) => !existsSync(resolve(root, file)));
|
|
@@ -30,8 +32,8 @@ const requiredPhrases = [
|
|
|
30
32
|
'longest match',
|
|
31
33
|
'Runtime mode',
|
|
32
34
|
'Generated mode',
|
|
33
|
-
'
|
|
34
|
-
'
|
|
35
|
+
'separate worker',
|
|
36
|
+
'source and input stay in your browser'
|
|
35
37
|
];
|
|
36
38
|
const missingPhrases = requiredPhrases.filter((phrase) => !source.toLowerCase().includes(phrase.toLowerCase()));
|
|
37
39
|
if (missingPhrases.length) {
|
|
@@ -3,7 +3,7 @@ title: Matching semantics
|
|
|
3
3
|
description: Understand how flexr chooses a rule when several rules accept the same position.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
With the default longest-match backends, flexr selects the rule that consumes the most bytes at each input position. If multiple rules consume the same number, the rule defined first wins.
|
|
7
7
|
|
|
8
8
|
```ruby
|
|
9
9
|
rule(/==/) { emit :EQ }
|
|
@@ -12,4 +12,6 @@ rule(/=/) { emit :ASSIGN }
|
|
|
12
12
|
|
|
13
13
|
Input `==` produces `EQ`, regardless of the order above. Equal-length alternatives are resolved by definition order, so putting keyword rules before a broad identifier rule is the conventional way to reserve keywords.
|
|
14
14
|
|
|
15
|
-
The
|
|
15
|
+
The stable `firstmatch` backend instead selects the first rule with a match at the current position. Ruby regexp alternation order applies within that rule, so `/a|ab/` matches `a` on input `ab`. Select this behavior explicitly with `backend :firstmatch`.
|
|
16
|
+
|
|
17
|
+
The [playground](/flexr/playground/) shows candidates and the winner for each scan step.
|
|
@@ -12,4 +12,12 @@ flexr specifications are Ruby programs. Rule actions, `--eval`, and generated ac
|
|
|
12
12
|
- Use `flexr check` for diagnostics, not as a sandbox.
|
|
13
13
|
- Keep build-time inputs and generator versions pinned in CI.
|
|
14
14
|
|
|
15
|
-
The
|
|
15
|
+
The Playground executes real Ruby in a separate worker using a virtual filesystem.
|
|
16
|
+
The application does not upload source or input; share links store them in a URL
|
|
17
|
+
fragment and do not run automatically. Review shared source before selecting Run.
|
|
18
|
+
|
|
19
|
+
Runs have a five-second timeout and can be stopped immediately. Input size,
|
|
20
|
+
automaton construction, and output are bounded. The page restricts network access
|
|
21
|
+
and sanitizes output, but a worker is not a complete hostile-code or memory
|
|
22
|
+
sandbox. Ruby actions can modify their VM and access worker JavaScript. Parsing a
|
|
23
|
+
specification does not make its actions safe.
|
|
@@ -10,7 +10,7 @@ bundle exec flexr check lexer.flexr.rb
|
|
|
10
10
|
bundle exec flexr generate lexer.flexr.rb -o lexer.rb
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
-
The generator
|
|
13
|
+
The generator supports the longest-match `table`, `direct`, and `auto` backends. Select `backend :firstmatch` explicitly to use the first matching rule in definition order. This backend is stable and requires no experimental option.
|
|
14
14
|
|
|
15
15
|
## Runtime dependencies
|
|
16
16
|
|
|
@@ -10,10 +10,10 @@ description: The stable lexer specification methods and their role.
|
|
|
10
10
|
| `emits(*kinds)` | Declare token kinds for diagnostics and parser integration |
|
|
11
11
|
| `state(name)` | Define a named start condition |
|
|
12
12
|
| `all_states` | Apply a rule to every state |
|
|
13
|
-
| `backend(name)` | Choose `table`, `direct`, or `auto
|
|
13
|
+
| `backend(name)` | Choose longest-match `table`, `direct`, or `auto`, or rule-order `firstmatch` |
|
|
14
14
|
| `token_kind(name)` | Choose the token return shape |
|
|
15
15
|
| `encoding(name)` | Select the input encoding contract |
|
|
16
16
|
| `option(name)` | Enable an explicit compiler option |
|
|
17
17
|
| `accel(name)` | Configure acceleration where compatible |
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
The default backends use [longest-match semantics](/flexr/concepts/matching-semantics/); `firstmatch` selects the first matching rule. The repository's [DSL reference](https://github.com/ydah/flexr/blob/main/docs/reference/dsl.md) includes option defaults and examples.
|
|
@@ -5,11 +5,13 @@ description: Separate stable user APIs from experimental and internal implementa
|
|
|
5
5
|
|
|
6
6
|
## Public and stable
|
|
7
7
|
|
|
8
|
-
`Flexr::Lexer`, the lexer DSL, runtime token/location
|
|
8
|
+
`Flexr::Lexer`, the lexer DSL, immutable runtime token/location Data records, the documented CLI, and `Flexr::RakeTask` are the supported entry points for normal use.
|
|
9
|
+
|
|
10
|
+
The `firstmatch` backend is stable and uses rule-order semantics in runtime and generated modes. Select it explicitly with `backend :firstmatch`; no experimental option is required.
|
|
9
11
|
|
|
10
12
|
## Experimental
|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
Options explicitly marked experimental may change behavior or API shape between releases.
|
|
13
15
|
|
|
14
16
|
## Internal
|
|
15
17
|
|
|
@@ -13,4 +13,7 @@ description: Return shapes, semantic values, and source positions.
|
|
|
13
13
|
|
|
14
14
|
Locations use byte offsets for the input position and track line/column as the lexer consumes text. Keep the location with the token when a parser needs useful syntax errors.
|
|
15
15
|
|
|
16
|
+
In v2, structured tokens and locations are immutable `Data` records. Read fields
|
|
17
|
+
with attribute readers or `[]`, and use `with` to create a replacement.
|
|
18
|
+
|
|
16
19
|
For exact defaults and the `Location` data structure, use the repository's [tokens and locations reference](https://github.com/ydah/flexr/blob/main/docs/reference/tokens-and-locations.md).
|
|
@@ -4,9 +4,10 @@ import '../styles/custom.css';
|
|
|
4
4
|
export interface Props {
|
|
5
5
|
title: string;
|
|
6
6
|
description: string;
|
|
7
|
+
playground?: boolean;
|
|
7
8
|
}
|
|
8
9
|
|
|
9
|
-
const { title, description } = Astro.props;
|
|
10
|
+
const { title, description, playground = false } = Astro.props;
|
|
10
11
|
const siteTitle = `${title} · flexr`;
|
|
11
12
|
const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
|
|
12
13
|
---
|
|
@@ -16,6 +17,14 @@ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL :
|
|
|
16
17
|
<meta charset="UTF-8" />
|
|
17
18
|
<meta name="viewport" content="width=device-width" />
|
|
18
19
|
<meta name="description" content={description} />
|
|
20
|
+
<meta name="theme-color" content="#161518" />
|
|
21
|
+
{playground && <meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' blob: 'unsafe-eval' 'wasm-unsafe-eval'; worker-src 'self' blob:; connect-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data:; object-src 'none'; base-uri 'self'" />}
|
|
22
|
+
<meta property="og:title" content={siteTitle} />
|
|
23
|
+
<meta property="og:description" content={description} />
|
|
24
|
+
<meta property="og:type" content="website" />
|
|
25
|
+
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
|
26
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
|
27
|
+
<link href="https://fonts.googleapis.com/css2?family=Newsreader:ital,opsz,wght@0,6..72,400..700;1,6..72,400..600&family=Public+Sans:wght@400;500;600;700&display=swap" rel="stylesheet" />
|
|
19
28
|
<title>{siteTitle}</title>
|
|
20
29
|
</head>
|
|
21
30
|
<body>
|
|
@@ -25,11 +34,10 @@ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL :
|
|
|
25
34
|
<a class="site-nav-brand" href={base} aria-label="flexr home">flexr<span>/</span></a>
|
|
26
35
|
<nav class="site-nav-links" aria-label="Primary navigation">
|
|
27
36
|
<a href={`${base}playground/`}>Playground</a>
|
|
28
|
-
<a href={`${base}learn/getting-started/`}>Learn</a>
|
|
29
37
|
<a href={`${base}reference/dsl/`}>Reference</a>
|
|
30
38
|
<a href="https://github.com/ydah/flexr">GitHub</a>
|
|
31
39
|
</nav>
|
|
32
|
-
<a class="site-nav-cta" href={`${base}learn/getting-started/`}>Get started
|
|
40
|
+
<a class="site-nav-cta" href={`${base}learn/getting-started/`}>Get started</a>
|
|
33
41
|
</div>
|
|
34
42
|
</header>
|
|
35
43
|
<div id="main-content">
|
data/site/src/pages/index.astro
CHANGED
|
@@ -1,174 +1,88 @@
|
|
|
1
1
|
---
|
|
2
2
|
import SiteLayout from '../layouts/SiteLayout.astro';
|
|
3
|
+
import '../styles/landing.css';
|
|
3
4
|
|
|
4
5
|
const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
|
|
5
6
|
const repo = 'https://github.com/ydah/flexr';
|
|
6
7
|
---
|
|
7
8
|
<SiteLayout
|
|
8
|
-
title="
|
|
9
|
-
description="
|
|
9
|
+
title="A lexer generator that speaks Ruby"
|
|
10
|
+
description="Define lexer rules in Ruby, understand every match, and generate deterministic Ruby with flexr."
|
|
10
11
|
>
|
|
11
|
-
<
|
|
12
|
-
<
|
|
13
|
-
<div>
|
|
14
|
-
<
|
|
15
|
-
<h1>Write
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
<main class="landing">
|
|
13
|
+
<section class="hero" aria-labelledby="hero-title">
|
|
14
|
+
<div class="hero-heading">
|
|
15
|
+
<p>Lexer generation, in Ruby</p>
|
|
16
|
+
<h1 id="hero-title">Write the rules.<br /><em>Know what wins.</em></h1>
|
|
17
|
+
</div>
|
|
18
|
+
<div class="hero-intro">
|
|
19
|
+
<p>
|
|
20
|
+
flexr keeps lexer specifications in the language they serve. Define readable rules,
|
|
21
|
+
inspect every decision, then generate deterministic Ruby for deployment.
|
|
19
22
|
</p>
|
|
20
23
|
<div class="hero-actions">
|
|
21
|
-
<a class="
|
|
22
|
-
<a class="
|
|
24
|
+
<a class="action action-primary" href={`${base}learn/getting-started/`}>Build your first lexer</a>
|
|
25
|
+
<a class="action action-link" href={`${base}playground/`}>Open the playground</a>
|
|
23
26
|
</div>
|
|
24
27
|
</div>
|
|
28
|
+
</section>
|
|
25
29
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
</
|
|
33
|
-
<
|
|
34
|
-
rule(<span class="code-string">/==/</span>) { emit <span class="code-string">:EQ</span> }
|
|
35
|
-
rule(<span class="code-string">/=/</span>) { emit <span class="code-string">:ASSIGN</span> }
|
|
36
|
-
rule(<span class="code-string">/if/</span>) { emit <span class="code-string">:IF</span> }
|
|
37
|
-
rule(<span class="code-string">/[a-z_][a-z0-9_]*/</span>) { emit <span class="code-string">:IDENT</span> }
|
|
38
|
-
<span class="code-keyword">end</span></code></pre>
|
|
39
|
-
<div class="decision-panel">
|
|
40
|
-
<div class="decision-label"><span>input: <strong>if == total</strong></span><span>step 02</span></div>
|
|
41
|
-
<div class="decision-row winner"><strong>/==/ → EQ</strong><span>longest match · 2 chars</span></div>
|
|
42
|
-
<div class="decision-row"><strong>/=/ → ASSIGN</strong><span>shorter match · 1 char</span></div>
|
|
43
|
-
</div>
|
|
30
|
+
<section class="specimen" aria-labelledby="specimen-title">
|
|
31
|
+
<header>
|
|
32
|
+
<p>One input, four decisions</p>
|
|
33
|
+
<h2 id="specimen-title">A match you can read.</h2>
|
|
34
|
+
</header>
|
|
35
|
+
<div class="source-line">
|
|
36
|
+
<span>source</span>
|
|
37
|
+
<code>if total == 42</code>
|
|
44
38
|
</div>
|
|
45
|
-
|
|
39
|
+
<ol class="token-line" aria-label="Generated token stream">
|
|
40
|
+
<li><span>if</span><strong>IF</strong></li>
|
|
41
|
+
<li><span>total</span><strong>IDENT</strong></li>
|
|
42
|
+
<li class="token-accent"><span>==</span><strong>EQ</strong></li>
|
|
43
|
+
<li><span>42</span><strong>INTEGER</strong></li>
|
|
44
|
+
</ol>
|
|
45
|
+
<p class="decision"><code>/==/</code> wins over <code>/=/</code> because the longest match wins.</p>
|
|
46
|
+
</section>
|
|
46
47
|
|
|
47
|
-
<
|
|
48
|
-
<
|
|
49
|
-
<
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
<
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
<h3>Matching you can explain</h3>
|
|
68
|
-
<p>Longest match wins. If matches have the same length, the rule defined first wins. The contract stays visible.</p>
|
|
69
|
-
</article>
|
|
70
|
-
<article class="site-card">
|
|
71
|
-
<div class="card-index">03 / PARITY</div>
|
|
72
|
-
<h3>Runtime and generated parity</h3>
|
|
73
|
-
<p>Develop with the runtime interpreter, then compare its token stream with generated Ruby before deployment.</p>
|
|
74
|
-
</article>
|
|
75
|
-
<article class="site-card">
|
|
76
|
-
<div class="card-index">04 / DIAGNOSTICS</div>
|
|
77
|
-
<h3>Actionable checks</h3>
|
|
78
|
-
<p>Find unsupported regexp constructs, unreachable rules, undeclared tokens, and other problems before they reach a parser.</p>
|
|
79
|
-
</article>
|
|
80
|
-
</div>
|
|
81
|
-
</section>
|
|
82
|
-
|
|
83
|
-
<section class="home-section" aria-labelledby="workflow-heading">
|
|
84
|
-
<div class="section-heading">
|
|
85
|
-
<div>
|
|
86
|
-
<div class="eyebrow">A practical workflow</div>
|
|
87
|
-
<h2>Write. Inspect. Generate. Integrate.</h2>
|
|
88
|
-
</div>
|
|
89
|
-
</div>
|
|
90
|
-
<div class="workflow-grid">
|
|
91
|
-
<article class="workflow-step">
|
|
92
|
-
<div class="card-index">01</div>
|
|
93
|
-
<h3>Write</h3>
|
|
94
|
-
<p>Define rules and actions in a Ruby class that looks like the code around it.</p>
|
|
95
|
-
</article>
|
|
96
|
-
<article class="workflow-step">
|
|
97
|
-
<div class="card-index">02</div>
|
|
98
|
-
<h3>Inspect</h3>
|
|
99
|
-
<p>Check the specification and make a matching decision visible when rules compete.</p>
|
|
100
|
-
</article>
|
|
101
|
-
<article class="workflow-step">
|
|
102
|
-
<div class="card-index">03</div>
|
|
103
|
-
<h3>Generate</h3>
|
|
104
|
-
<p>Produce deterministic Ruby for environments where startup and deployment shape matter.</p>
|
|
105
|
-
</article>
|
|
106
|
-
<article class="workflow-step">
|
|
107
|
-
<div class="card-index">04</div>
|
|
108
|
-
<h3>Integrate</h3>
|
|
109
|
-
<p>Return tokens to Racc, Lrama, or your own parser protocol with explicit locations.</p>
|
|
110
|
-
</article>
|
|
111
|
-
</div>
|
|
112
|
-
</section>
|
|
113
|
-
|
|
114
|
-
<section class="home-section" aria-labelledby="modes-heading">
|
|
115
|
-
<div class="section-heading">
|
|
116
|
-
<div>
|
|
117
|
-
<div class="eyebrow">Choose your mode</div>
|
|
118
|
-
<h2>Fast feedback first. Reproducible output when it matters.</h2>
|
|
119
|
-
</div>
|
|
120
|
-
<p>Both modes consume the same lexer design. The choice is about the deployment boundary, not a second language.</p>
|
|
121
|
-
</div>
|
|
122
|
-
<div class="compare-panel" id="modes-heading">
|
|
123
|
-
<div>
|
|
124
|
-
<h3>Runtime mode</h3>
|
|
125
|
-
<p>Load the specification directly with the <code>flexr</code> runtime. It is a strong fit for iteration, tests, and applications that value a compact source of truth.</p>
|
|
126
|
-
<div class="home-links"><a class="button button-secondary" href={`${base}learn/runtime-mode/`}>Runtime guide</a></div>
|
|
127
|
-
</div>
|
|
128
|
-
<div>
|
|
129
|
-
<h3>Generated mode</h3>
|
|
130
|
-
<p>Compile the specification to Ruby and commit or build the generated artifact. Use standalone output when the deployed process should not load the gem at runtime.</p>
|
|
131
|
-
<div class="home-links"><a class="button button-secondary" href={`${base}learn/generation/`}>Generation guide</a></div>
|
|
132
|
-
</div>
|
|
133
|
-
</div>
|
|
134
|
-
</section>
|
|
48
|
+
<section class="principles" aria-labelledby="principles-title">
|
|
49
|
+
<header>
|
|
50
|
+
<p>Why flexr</p>
|
|
51
|
+
<h2 id="principles-title">One specification, from first test to release.</h2>
|
|
52
|
+
</header>
|
|
53
|
+
<div class="principle-list">
|
|
54
|
+
<article>
|
|
55
|
+
<h3>Ruby-native</h3>
|
|
56
|
+
<p>Use familiar classes, constants, and actions. There is no separate lexer language to learn.</p>
|
|
57
|
+
</article>
|
|
58
|
+
<article>
|
|
59
|
+
<h3>Explainable</h3>
|
|
60
|
+
<p>Longest match wins. Source order resolves ties. Unsupported patterns surface early.</p>
|
|
61
|
+
</article>
|
|
62
|
+
<article>
|
|
63
|
+
<h3>Ready both ways</h3>
|
|
64
|
+
<p><a href={`${base}learn/runtime-mode/`}>Runtime mode</a> keeps iteration quick. <a href={`${base}learn/generation/`}>Generated mode</a> keeps deployment reproducible.</p>
|
|
65
|
+
</article>
|
|
66
|
+
</div>
|
|
67
|
+
</section>
|
|
135
68
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
<div class="trust-grid">
|
|
145
|
-
<article class="trust-panel">
|
|
146
|
-
<div class="card-index">SECURITY</div>
|
|
147
|
-
<h3>Actions are Ruby code</h3>
|
|
148
|
-
<p>Only run specifications and generated artifacts you trust. Static checks do not make arbitrary actions safe.</p>
|
|
149
|
-
</article>
|
|
150
|
-
<article class="trust-panel">
|
|
151
|
-
<div class="card-index">COMPATIBILITY</div>
|
|
152
|
-
<h3>Regexp support is documented</h3>
|
|
153
|
-
<p>Check the compatibility matrix before porting lookarounds, backreferences, or open-ended repetition.</p>
|
|
154
|
-
</article>
|
|
155
|
-
<article class="trust-panel">
|
|
156
|
-
<div class="card-index">EXPERIMENTAL</div>
|
|
157
|
-
<h3>Experimental features say so</h3>
|
|
158
|
-
<p><code>firstmatch</code> is experimental because it changes the usual longest-match contract and needs an explicit option.</p>
|
|
159
|
-
</article>
|
|
160
|
-
<article class="trust-panel">
|
|
161
|
-
<div class="card-index">OPEN SOURCE</div>
|
|
162
|
-
<h3>Read the implementation</h3>
|
|
163
|
-
<p>Inspect examples, diagnostics, generated output, and the benchmark log in the repository.</p>
|
|
164
|
-
<div class="home-links"><a class="button button-secondary" href={repo}>View on GitHub</a></div>
|
|
165
|
-
</article>
|
|
166
|
-
</div>
|
|
167
|
-
</section>
|
|
168
|
-
</main>
|
|
69
|
+
<section class="start" aria-labelledby="start-title">
|
|
70
|
+
<div>
|
|
71
|
+
<p>Ready when you are</p>
|
|
72
|
+
<h2 id="start-title">Start with one Ruby file.</h2>
|
|
73
|
+
</div>
|
|
74
|
+
<code>gem install flexr</code>
|
|
75
|
+
<a class="action action-primary" href={`${base}learn/getting-started/`}>Read the quick start</a>
|
|
76
|
+
</section>
|
|
169
77
|
|
|
170
|
-
<footer class="
|
|
171
|
-
<
|
|
78
|
+
<footer class="landing-footer">
|
|
79
|
+
<a class="footer-brand" href={base}>flexr/</a>
|
|
80
|
+
<nav aria-label="Footer navigation">
|
|
81
|
+
<a href={`${base}reference/dsl/`}>Reference</a>
|
|
82
|
+
<a href={`${base}examples/`}>Examples</a>
|
|
83
|
+
<a href={repo}>GitHub</a>
|
|
84
|
+
</nav>
|
|
85
|
+
<p>MIT licensed.</p>
|
|
172
86
|
</footer>
|
|
173
|
-
</
|
|
87
|
+
</main>
|
|
174
88
|
</SiteLayout>
|
|
@@ -3,62 +3,64 @@ import SiteLayout from '../layouts/SiteLayout.astro';
|
|
|
3
3
|
|
|
4
4
|
const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
|
|
5
5
|
---
|
|
6
|
-
<SiteLayout
|
|
7
|
-
title="Playground"
|
|
8
|
-
description="Explore flexr's longest-match decision model with an interactive, fixture-backed lexer preview."
|
|
9
|
-
>
|
|
6
|
+
<SiteLayout title="Playground" description="Run real flexr Ruby specifications locally, inspect diagnostics and trace decisions, and generate a scanner." playground>
|
|
10
7
|
<main class="playground-shell">
|
|
11
|
-
<a class="back-link" href={base}
|
|
8
|
+
<a class="back-link" href={base}>Back to flexr</a>
|
|
12
9
|
<header class="playground-header">
|
|
13
|
-
<
|
|
14
|
-
<
|
|
15
|
-
<p>
|
|
16
|
-
Choose a fixture, edit its input, and inspect which rule wins at each position. This
|
|
17
|
-
preview is static and fixture-backed: it does not execute arbitrary Ruby in your browser.
|
|
18
|
-
Use the repository examples and CLI for full runtime and generation behavior.
|
|
19
|
-
</p>
|
|
10
|
+
<h1>Follow every match.</h1>
|
|
11
|
+
<p>Edit Ruby, run the real lexer, and inspect its decisions. Source and input stay in your browser.</p>
|
|
20
12
|
<div class="playground-notice" role="note">
|
|
21
|
-
|
|
22
|
-
|
|
13
|
+
Run executes Ruby and its actions in a separate worker. Review shared source before running it.
|
|
14
|
+
Runs stop after five seconds; Stop cancels the worker immediately. Ruby loads only on your first run.
|
|
23
15
|
</div>
|
|
24
16
|
</header>
|
|
25
|
-
|
|
26
17
|
<section class="workbench" aria-label="flexr playground workbench">
|
|
27
18
|
<div class="workbench-panel">
|
|
28
|
-
<div class="panel-heading"><
|
|
29
|
-
<select id="preset" aria-label="Choose a playground
|
|
30
|
-
<option value="longest">Longest match</option>
|
|
31
|
-
<option value="
|
|
32
|
-
<option value="json">JSON tokens</option>
|
|
19
|
+
<div class="panel-heading"><label for="spec">Ruby specification</label><small>editable</small></div>
|
|
20
|
+
<select id="preset" aria-label="Choose a playground preset">
|
|
21
|
+
<option value="longest">Longest match</option><option value="calculator">Calculator</option><option value="json">JSON tokens</option>
|
|
22
|
+
<option value="toml">TOML subset</option><option value="graphql">GraphQL subset</option><option value="sql">SQL subset</option>
|
|
33
23
|
</select>
|
|
34
|
-
<textarea id="spec" aria-label="Lexer specification"
|
|
35
|
-
<div class="workbench-actions">
|
|
36
|
-
<button class="button button-secondary" type="button" data-action="check">Check</button>
|
|
37
|
-
<button class="button button-secondary" type="button" data-action="generate">Show generated shape</button>
|
|
38
|
-
</div>
|
|
24
|
+
<textarea id="spec" aria-label="Lexer specification" spellcheck="false" maxlength="20000"></textarea>
|
|
39
25
|
</div>
|
|
40
|
-
|
|
41
26
|
<div class="workbench-panel">
|
|
42
|
-
<div class="panel-heading"><
|
|
43
|
-
<textarea id="input" class="input-area" aria-label="Input to tokenize"></textarea>
|
|
27
|
+
<div class="panel-heading"><label for="input">Input</label><small>up to 20,000 characters</small></div>
|
|
28
|
+
<textarea id="input" class="input-area" aria-label="Input to tokenize" spellcheck="false" maxlength="20000"></textarea>
|
|
29
|
+
<select id="mode" aria-label="Execution mode"><option value="runtime">Runtime mode</option><option value="generated">Generated mode</option></select>
|
|
44
30
|
<div class="workbench-actions">
|
|
45
|
-
<button class="button button-primary" type="button" data-action="run">Run
|
|
46
|
-
<button class="button button-secondary" type="button" data-action="
|
|
31
|
+
<button class="button button-primary" type="button" data-action="run">Run</button>
|
|
32
|
+
<button class="button button-secondary" type="button" data-action="stop" disabled>Stop</button>
|
|
47
33
|
<button class="button button-secondary" type="button" data-action="share">Copy share link</button>
|
|
48
34
|
</div>
|
|
49
|
-
<
|
|
35
|
+
<input id="share-link" class="share-link" aria-label="Share link" readonly hidden />
|
|
36
|
+
<p id="status" class="workbench-status" role="status" aria-live="polite">Choose a preset or write a specification, then select Run.</p>
|
|
37
|
+
<p id="metrics" class="workbench-status"></p>
|
|
50
38
|
</div>
|
|
51
|
-
|
|
52
39
|
<div class="workbench-panel full">
|
|
53
|
-
<div class="panel-heading"><span>Tokens</span
|
|
54
|
-
<
|
|
40
|
+
<div class="panel-heading"><span>Tokens</span></div>
|
|
41
|
+
<pre id="tokens" class="workbench-output">Run a specification to see its token stream.</pre>
|
|
42
|
+
</div>
|
|
43
|
+
<div class="workbench-panel full">
|
|
44
|
+
<div class="panel-heading"><span>Diagnostics</span></div>
|
|
45
|
+
<pre id="diagnostics" class="workbench-output">Compiler warnings and source locations appear here.</pre>
|
|
46
|
+
</div>
|
|
47
|
+
<div class="workbench-panel full">
|
|
48
|
+
<div class="panel-heading"><span>Match decisions</span><span id="step-position">No scan steps.</span></div>
|
|
49
|
+
<div class="workbench-actions">
|
|
50
|
+
<button class="button button-secondary" type="button" data-action="previous">Previous step</button>
|
|
51
|
+
<input id="step" type="range" min="0" max="0" value="0" aria-label="Trace step" disabled />
|
|
52
|
+
<button class="button button-secondary" type="button" data-action="next">Next step</button>
|
|
53
|
+
</div>
|
|
54
|
+
<pre id="decisions" class="workbench-output">Run a specification to inspect its decisions.</pre>
|
|
55
55
|
</div>
|
|
56
|
-
|
|
57
56
|
<div class="workbench-panel full">
|
|
58
|
-
<div class="panel-heading"><span>
|
|
59
|
-
<div id="
|
|
57
|
+
<div class="panel-heading"><span>DFA</span><button type="button" data-action="draw">Draw DFA</button></div>
|
|
58
|
+
<div id="graph" class="workbench-output graph-output">Run a specification to inspect its DFA.</div>
|
|
59
|
+
<details><summary>DOT output</summary><pre id="dot" class="workbench-output"></pre></details>
|
|
60
60
|
</div>
|
|
61
|
+
<details class="workbench-panel full"><summary>Generated Ruby</summary><pre id="generated" class="workbench-output">The real generated scanner appears after a run.</pre></details>
|
|
62
|
+
<details class="workbench-panel full"><summary>Console output</summary><pre id="console" class="workbench-output"></pre></details>
|
|
61
63
|
</section>
|
|
62
64
|
</main>
|
|
63
|
-
<script
|
|
65
|
+
<script>import '../playground/client.js';</script>
|
|
64
66
|
</SiteLayout>
|