lexer_kit 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/README.md +157 -0
  4. data/exe/lexer_kit +7 -0
  5. data/ext/lexer_kit_rust/Cargo.toml +17 -0
  6. data/ext/lexer_kit_rust/extconf.rb +6 -0
  7. data/ext/lexer_kit_rust/src/deserializer.rs +213 -0
  8. data/ext/lexer_kit_rust/src/dfa.rs +217 -0
  9. data/ext/lexer_kit_rust/src/fast_stream.rs +468 -0
  10. data/ext/lexer_kit_rust/src/lib.rs +248 -0
  11. data/ext/lexer_kit_rust/src/opcodes.rs +718 -0
  12. data/ext/lexer_kit_rust/src/safety_test.rs +498 -0
  13. data/ext/lexer_kit_rust/src/trie.rs +206 -0
  14. data/ext/lexer_kit_rust/src/types.rs +319 -0
  15. data/ext/lexer_kit_rust/src/vm.rs +258 -0
  16. data/lib/lexer_kit/builder/compiler.rb +596 -0
  17. data/lib/lexer_kit/builder/conflict_detector.rb +209 -0
  18. data/lib/lexer_kit/builder/mode_def.rb +36 -0
  19. data/lib/lexer_kit/builder/token_def.rb +65 -0
  20. data/lib/lexer_kit/builder/validator.rb +84 -0
  21. data/lib/lexer_kit/builder.rb +230 -0
  22. data/lib/lexer_kit/cli/commands.rb +389 -0
  23. data/lib/lexer_kit/cli.rb +88 -0
  24. data/lib/lexer_kit/core/diagnostic.rb +103 -0
  25. data/lib/lexer_kit/core/source.rb +154 -0
  26. data/lib/lexer_kit/core/span.rb +80 -0
  27. data/lib/lexer_kit/core/token.rb +120 -0
  28. data/lib/lexer_kit/core.rb +13 -0
  29. data/lib/lexer_kit/debug/disassembler.rb +143 -0
  30. data/lib/lexer_kit/debug/visualizer.rb +203 -0
  31. data/lib/lexer_kit/debug.rb +11 -0
  32. data/lib/lexer_kit/dfa/byte_class_builder.rb +69 -0
  33. data/lib/lexer_kit/dfa/case_folding.rb +45 -0
  34. data/lib/lexer_kit/dfa/char_class_collector.rb +81 -0
  35. data/lib/lexer_kit/dfa/dfa_builder.rb +95 -0
  36. data/lib/lexer_kit/dfa/dfa_minimizer.rb +158 -0
  37. data/lib/lexer_kit/dfa/nfa.rb +304 -0
  38. data/lib/lexer_kit/dfa/regex_ast.rb +64 -0
  39. data/lib/lexer_kit/dfa/regex_parser.rb +385 -0
  40. data/lib/lexer_kit/dfa/utf8_range.rb +175 -0
  41. data/lib/lexer_kit/dfa/utf8_range_pattern.rb +17 -0
  42. data/lib/lexer_kit/dfa.rb +37 -0
  43. data/lib/lexer_kit/errors.rb +76 -0
  44. data/lib/lexer_kit/format/lkb1/decoder.rb +126 -0
  45. data/lib/lexer_kit/format/lkb1.rb +199 -0
  46. data/lib/lexer_kit/format/lkt1.rb +111 -0
  47. data/lib/lexer_kit/format.rb +19 -0
  48. data/lib/lexer_kit/ir/compiled_program.rb +228 -0
  49. data/lib/lexer_kit/ir/constant_pool.rb +107 -0
  50. data/lib/lexer_kit/ir/dfa_table.rb +125 -0
  51. data/lib/lexer_kit/ir/instruction.rb +50 -0
  52. data/lib/lexer_kit/ir/jump_table.rb +94 -0
  53. data/lib/lexer_kit/ir/keyword_table.rb +168 -0
  54. data/lib/lexer_kit/ir/opcode.rb +96 -0
  55. data/lib/lexer_kit/ir/serializer.rb +249 -0
  56. data/lib/lexer_kit/ir.rb +16 -0
  57. data/lib/lexer_kit/runner.rb +114 -0
  58. data/lib/lexer_kit/trie.rb +170 -0
  59. data/lib/lexer_kit/version.rb +5 -0
  60. data/lib/lexer_kit.rb +155 -0
  61. metadata +119 -0
metadata ADDED
@@ -0,0 +1,119 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: lexer_kit
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.5.0
5
+ platform: ruby
6
+ authors:
7
+ - takahashim
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: base64
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - ">="
24
+ - !ruby/object:Gem::Version
25
+ version: '0'
26
+ description: LexerKit is a high performance lexer toolkit for Ruby
27
+ email:
28
+ - takahashimm@gmail.com
29
+ executables:
30
+ - lexer_kit
31
+ extensions:
32
+ - ext/lexer_kit_rust/extconf.rb
33
+ extra_rdoc_files: []
34
+ files:
35
+ - LICENSE.txt
36
+ - README.md
37
+ - exe/lexer_kit
38
+ - ext/lexer_kit_rust/Cargo.toml
39
+ - ext/lexer_kit_rust/extconf.rb
40
+ - ext/lexer_kit_rust/src/deserializer.rs
41
+ - ext/lexer_kit_rust/src/dfa.rs
42
+ - ext/lexer_kit_rust/src/fast_stream.rs
43
+ - ext/lexer_kit_rust/src/lib.rs
44
+ - ext/lexer_kit_rust/src/opcodes.rs
45
+ - ext/lexer_kit_rust/src/safety_test.rs
46
+ - ext/lexer_kit_rust/src/trie.rs
47
+ - ext/lexer_kit_rust/src/types.rs
48
+ - ext/lexer_kit_rust/src/vm.rs
49
+ - lib/lexer_kit.rb
50
+ - lib/lexer_kit/builder.rb
51
+ - lib/lexer_kit/builder/compiler.rb
52
+ - lib/lexer_kit/builder/conflict_detector.rb
53
+ - lib/lexer_kit/builder/mode_def.rb
54
+ - lib/lexer_kit/builder/token_def.rb
55
+ - lib/lexer_kit/builder/validator.rb
56
+ - lib/lexer_kit/cli.rb
57
+ - lib/lexer_kit/cli/commands.rb
58
+ - lib/lexer_kit/core.rb
59
+ - lib/lexer_kit/core/diagnostic.rb
60
+ - lib/lexer_kit/core/source.rb
61
+ - lib/lexer_kit/core/span.rb
62
+ - lib/lexer_kit/core/token.rb
63
+ - lib/lexer_kit/debug.rb
64
+ - lib/lexer_kit/debug/disassembler.rb
65
+ - lib/lexer_kit/debug/visualizer.rb
66
+ - lib/lexer_kit/dfa.rb
67
+ - lib/lexer_kit/dfa/byte_class_builder.rb
68
+ - lib/lexer_kit/dfa/case_folding.rb
69
+ - lib/lexer_kit/dfa/char_class_collector.rb
70
+ - lib/lexer_kit/dfa/dfa_builder.rb
71
+ - lib/lexer_kit/dfa/dfa_minimizer.rb
72
+ - lib/lexer_kit/dfa/nfa.rb
73
+ - lib/lexer_kit/dfa/regex_ast.rb
74
+ - lib/lexer_kit/dfa/regex_parser.rb
75
+ - lib/lexer_kit/dfa/utf8_range.rb
76
+ - lib/lexer_kit/dfa/utf8_range_pattern.rb
77
+ - lib/lexer_kit/errors.rb
78
+ - lib/lexer_kit/format.rb
79
+ - lib/lexer_kit/format/lkb1.rb
80
+ - lib/lexer_kit/format/lkb1/decoder.rb
81
+ - lib/lexer_kit/format/lkt1.rb
82
+ - lib/lexer_kit/ir.rb
83
+ - lib/lexer_kit/ir/compiled_program.rb
84
+ - lib/lexer_kit/ir/constant_pool.rb
85
+ - lib/lexer_kit/ir/dfa_table.rb
86
+ - lib/lexer_kit/ir/instruction.rb
87
+ - lib/lexer_kit/ir/jump_table.rb
88
+ - lib/lexer_kit/ir/keyword_table.rb
89
+ - lib/lexer_kit/ir/opcode.rb
90
+ - lib/lexer_kit/ir/serializer.rb
91
+ - lib/lexer_kit/runner.rb
92
+ - lib/lexer_kit/trie.rb
93
+ - lib/lexer_kit/version.rb
94
+ homepage: https://github.com/takahashim/lexer_kit
95
+ licenses:
96
+ - MIT
97
+ metadata:
98
+ homepage_uri: https://github.com/takahashim/lexer_kit
99
+ source_code_uri: https://github.com/takahashim/lexer_kit
100
+ changelog_uri: https://github.com/takahashim/lexer_kit/blob/main/CHANGELOG.md
101
+ rubygems_mfa_required: 'true'
102
+ rdoc_options: []
103
+ require_paths:
104
+ - lib
105
+ required_ruby_version: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: 3.3.0
110
+ required_rubygems_version: !ruby/object:Gem::Requirement
111
+ requirements:
112
+ - - ">="
113
+ - !ruby/object:Gem::Version
114
+ version: '0'
115
+ requirements: []
116
+ rubygems_version: 3.6.9
117
+ specification_version: 4
118
+ summary: A high-performance lexer toolkit for Ruby
119
+ test_files: []