flexr 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. checksums.yaml +7 -0
  2. data/.rubocop.yml +33 -0
  3. data/CONTRIBUTING.md +39 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +116 -0
  6. data/Rakefile +468 -0
  7. data/benchmark/baselines/json.json +34 -0
  8. data/benchmark/baselines/json_handwritten.rb +43 -0
  9. data/benchmark/baselines/json_rexical.rex +25 -0
  10. data/benchmark/corpora/README.md +11 -0
  11. data/benchmark/corpora/generate_json.rb +26 -0
  12. data/benchmark/golden/calculator_lexer.sha256 +1 -0
  13. data/benchmark/golden/json_lexer.sha256 +1 -0
  14. data/benchmark/golden/regexp_tokenizer.sha256 +1 -0
  15. data/benchmark/golden/ruby_subset_lexer.sha256 +1 -0
  16. data/benchmark/golden/toy_lang_lexer.sha256 +1 -0
  17. data/benchmark/golden/with_lrama_lexer.sha256 +1 -0
  18. data/benchmark/golden/with_racc_lexer.sha256 +1 -0
  19. data/benchmark/run.rb +254 -0
  20. data/docs/README.md +64 -0
  21. data/docs/RELEASING.md +30 -0
  22. data/docs/adr/0001-byte-level-dfa.md +5 -0
  23. data/docs/adr/0003-leftmost-longest.md +4 -0
  24. data/docs/adr/0006-accel-not-scanner.md +4 -0
  25. data/docs/adr/0008-what-pure-ruby-means.md +5 -0
  26. data/docs/adr/0016-spec-is-plain-ruby.md +4 -0
  27. data/docs/adr/0017-static-analysis-by-default.md +5 -0
  28. data/docs/adr/0018-prism-for-generator-only.md +4 -0
  29. data/docs/adr/0019-measured-performance-floor.md +26 -0
  30. data/docs/adr/0020-vendored-unicode-contract.md +21 -0
  31. data/docs/explanation/backends.md +33 -0
  32. data/docs/explanation/matching-semantics.md +20 -0
  33. data/docs/explanation/runtime-vs-generated.md +22 -0
  34. data/docs/explanation/security-model.md +18 -0
  35. data/docs/explanation/unicode-and-encoding.md +20 -0
  36. data/docs/how-to/deploy-a-standalone-lexer.md +23 -0
  37. data/docs/how-to/generate-a-lexer.md +39 -0
  38. data/docs/how-to/handle-errors.md +32 -0
  39. data/docs/how-to/integrate-with-lrama.md +21 -0
  40. data/docs/how-to/integrate-with-racc.md +25 -0
  41. data/docs/how-to/migrate-from-flex.md +21 -0
  42. data/docs/how-to/migrate-from-rexical.md +23 -0
  43. data/docs/how-to/run-a-lexer-at-runtime.md +29 -0
  44. data/docs/how-to/track-token-locations.md +27 -0
  45. data/docs/how-to/tune-performance.md +23 -0
  46. data/docs/how-to/use-states.md +36 -0
  47. data/docs/how-to/use-trailing-context.md +22 -0
  48. data/docs/internals/README.md +14 -0
  49. data/docs/perf-log.md +56 -0
  50. data/docs/reference/README.md +23 -0
  51. data/docs/reference/actions.md +47 -0
  52. data/docs/reference/cli.md +80 -0
  53. data/docs/reference/compatibility.md +38 -0
  54. data/docs/reference/diagnostics.md +41 -0
  55. data/docs/reference/dsl.md +81 -0
  56. data/docs/reference/errors.md +27 -0
  57. data/docs/reference/generated-artifacts.md +50 -0
  58. data/docs/reference/public-api.md +42 -0
  59. data/docs/reference/regexp.md +39 -0
  60. data/docs/reference/runtime.md +49 -0
  61. data/docs/reference/tokens-and-locations.md +33 -0
  62. data/docs/tutorial/build-a-calculator-lexer.md +96 -0
  63. data/examples/calculator/README.md +27 -0
  64. data/examples/calculator/lexer.flexr.rb +17 -0
  65. data/examples/json/README.md +30 -0
  66. data/examples/json/lexer.flexr.rb +24 -0
  67. data/examples/ruby_subset/README.md +17 -0
  68. data/examples/ruby_subset/lexer.flexr.rb +22 -0
  69. data/examples/toy_lang/README.md +17 -0
  70. data/examples/toy_lang/lexer.flexr.rb +18 -0
  71. data/examples/with_lrama/README.md +17 -0
  72. data/examples/with_lrama/lexer.flexr.rb +13 -0
  73. data/examples/with_racc/README.md +17 -0
  74. data/examples/with_racc/lexer.flexr.rb +13 -0
  75. data/exe/flexr +7 -0
  76. data/lib/flexr/automaton/accel.rb +39 -0
  77. data/lib/flexr/automaton/analysis.rb +38 -0
  78. data/lib/flexr/automaton/byte_class_set.rb +29 -0
  79. data/lib/flexr/automaton/compiler.rb +413 -0
  80. data/lib/flexr/automaton/dfa.rb +103 -0
  81. data/lib/flexr/automaton/minimizer.rb +70 -0
  82. data/lib/flexr/automaton/nfa.rb +92 -0
  83. data/lib/flexr/cli.rb +342 -0
  84. data/lib/flexr/codegen/base.rb +17 -0
  85. data/lib/flexr/codegen/direct.rb +52 -0
  86. data/lib/flexr/codegen/firstmatch.rb +17 -0
  87. data/lib/flexr/codegen/table.rb +158 -0
  88. data/lib/flexr/codegen/table_packer.rb +61 -0
  89. data/lib/flexr/diagnostics.rb +94 -0
  90. data/lib/flexr/dsl.rb +182 -0
  91. data/lib/flexr/errors.rb +28 -0
  92. data/lib/flexr/generated.rb +125 -0
  93. data/lib/flexr/generator.rb +400 -0
  94. data/lib/flexr/importer.rb +560 -0
  95. data/lib/flexr/ir.rb +36 -0
  96. data/lib/flexr/lexer.rb +10 -0
  97. data/lib/flexr/options.rb +47 -0
  98. data/lib/flexr/rake_task.rb +27 -0
  99. data/lib/flexr/regexp/ast.rb +45 -0
  100. data/lib/flexr/regexp/char_class.rb +7 -0
  101. data/lib/flexr/regexp/normalizer.rb +117 -0
  102. data/lib/flexr/regexp/parser.rb +517 -0
  103. data/lib/flexr/regexp/tokenizer.flexr.rb +27 -0
  104. data/lib/flexr/regexp/tokenizer.rb +168 -0
  105. data/lib/flexr/regexp/unsupported.rb +7 -0
  106. data/lib/flexr/runtime/buffer.rb +112 -0
  107. data/lib/flexr/runtime/core.rb +388 -0
  108. data/lib/flexr/runtime/errors.rb +22 -0
  109. data/lib/flexr/runtime/interpreter.rb +505 -0
  110. data/lib/flexr/runtime/location.rb +26 -0
  111. data/lib/flexr/runtime/token.rb +7 -0
  112. data/lib/flexr/source/passthrough.rb +31 -0
  113. data/lib/flexr/source/prism_reader.rb +283 -0
  114. data/lib/flexr/source/static_eval.rb +145 -0
  115. data/lib/flexr/unicode/case_fold.rb +45 -0
  116. data/lib/flexr/unicode/data/LICENSE-UNICODE.txt +5 -0
  117. data/lib/flexr/unicode/data/UNICODE_VERSION +1 -0
  118. data/lib/flexr/unicode/data/case_folding.rb +9 -0
  119. data/lib/flexr/unicode/data/properties.rb +10 -0
  120. data/lib/flexr/unicode/property.rb +107 -0
  121. data/lib/flexr/unicode/reference_regexp.rb +102 -0
  122. data/lib/flexr/unicode/utf8_splitter.rb +109 -0
  123. data/lib/flexr/version.rb +5 -0
  124. data/lib/flexr.rb +81 -0
  125. data/site/README.md +22 -0
  126. data/site/astro.config.mjs +57 -0
  127. data/site/package.json +19 -0
  128. data/site/pnpm-lock.yaml +5029 -0
  129. data/site/pnpm-workspace.yaml +6 -0
  130. data/site/public/playground.js +189 -0
  131. data/site/scripts/verify-site.mjs +42 -0
  132. data/site/src/content/docs/benchmarks.md +8 -0
  133. data/site/src/content/docs/concepts/matching-semantics.md +15 -0
  134. data/site/src/content/docs/concepts/regexp-model.md +18 -0
  135. data/site/src/content/docs/concepts/runtime-vs-generated.md +15 -0
  136. data/site/src/content/docs/concepts/security-model.md +15 -0
  137. data/site/src/content/docs/examples.md +17 -0
  138. data/site/src/content/docs/learn/generation.md +29 -0
  139. data/site/src/content/docs/learn/getting-started.md +56 -0
  140. data/site/src/content/docs/learn/parser-integration.md +27 -0
  141. data/site/src/content/docs/learn/runtime-mode.md +32 -0
  142. data/site/src/content/docs/reference/action-context.md +20 -0
  143. data/site/src/content/docs/reference/cli.md +22 -0
  144. data/site/src/content/docs/reference/diagnostics.md +16 -0
  145. data/site/src/content/docs/reference/dsl.md +19 -0
  146. data/site/src/content/docs/reference/public-api.md +18 -0
  147. data/site/src/content/docs/reference/regexp.md +16 -0
  148. data/site/src/content/docs/reference/runtime.md +16 -0
  149. data/site/src/content/docs/reference/tokens-and-locations.md +16 -0
  150. data/site/src/content.config.ts +12 -0
  151. data/site/src/env.d.ts +1 -0
  152. data/site/src/layouts/SiteLayout.astro +39 -0
  153. data/site/src/pages/index.astro +174 -0
  154. data/site/src/pages/playground.astro +64 -0
  155. data/site/src/styles/custom.css +711 -0
  156. data/site/tsconfig.json +5 -0
  157. data/tools/coverage.rb +32 -0
  158. data/tools/docs_verify.rb +116 -0
  159. data/tools/gen_unicode_tables.rb +202 -0
  160. data/tools/regexp_tokenizer_reference.rb +60 -0
  161. metadata +205 -0

There are too many changes on this page to be displayed.

The amount of changes on this page would crash your brower.

You can still verify the content by downloading the gem file manually.