lexer_kit 0.5.0-x86_64-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9fd860b692d5da23c51450ba74fe6299a46c45d8c2d398e2b788e27358291c33
+  data.tar.gz: 35e0bd59a4117e2d38de220864b8a7e741afe41da15986da4751d45badcaa9df
+SHA512:
+  metadata.gz: 16adcd34af9e6e1af15d9db2759eadac6e4a2bb9b8f9662c64383414503f13495677beead68dfac06ac4fc78cf3d242287f61bce6f1aa05ddb9992d0ce58f9b6
+  data.tar.gz: f96ff94f0211b210a93baacad0ef8691a34384d6b860aaaae68885fa31cde9f4b753f200cf99a8c860ed3d4385f19166e9886f768ced9e70b0cf1869fc461091

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Masayoshi Takahashi (takahashim)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,157 @@
+# LexerKit
+
+A high-performance lexer toolkit for Ruby.
+Define tokenizers with a Ruby DSL and run them through a Rust native extension.
+
+## Features
+
+- DSL-based lexer definition
+- Fast stream lexing with minimal allocation
+- On-demand token object creation for diagnostics
+- Compiled lexer serialization
+- Regex-based token patterns compiled to DFA
+
+## Installation
+
+```ruby
+# Gemfile
+gem "lexer_kit"
+```
+
+```bash
+bundle install
+```
+
+## Quick Start
+
+```ruby
+require "lexer_kit"
+
+lexer = LexerKit.build do
+  token :NUMBER, /[0-9]+/
+  token :PLUS,   "+"
+  token :MINUS,  "-"
+  token :SPACE,  /[ \t\r\n]+/, skip: true
+end.compile
+
+stream = lexer.stream("12 + 34 - 5")
+until stream.eof?
+  puts "#{stream.token_name}: #{stream.text.inspect}"
+  stream.advance
+end
+```
+
+## Core DSL
+
+### `token`
+
+```ruby
+token :IDENT,  /[a-zA-Z_][a-zA-Z0-9_]*/
+token :ARROW,  "->"
+token :SPACE,  /[ \t]+/, skip: true
+token :DQUOTE, '"', push: :string
+token :END_Q,  '"', pop: true
+```
+
+Options:
+
+- `skip: true` skips emitting the token
+- `push: :mode_name` pushes a mode
+- `pop: true` pops the current mode
+
+### `keyword` / `define_keywords`
+
+```ruby
+token :IDENT, /[a-z_]+/
+keyword :IF, "if"
+define_keywords :else, :while, :return
+```
+
+### `mode`
+
+```ruby
+LexerKit.build do
+  token :DQUOTE, '"', push: :string
+  token :IDENT,  /[a-z]+/
+
+  mode :string do
+    token :CONTENT, /[^"\\]+/
+    token :ESCAPE,  /\\./
+    token :DQUOTE,  '"', pop: true
+  end
+end
+```
+
+### `scan_until` / `delimited`
+
+```ruby
+scan_until :BLOCK_COMMENT, open: "/*", close: "*/", skip: true
+
+delimited :TEXT, delimiter: "{{" do
+  token :IDENT, /[a-zA-Z_]+/
+  token :DOT,   "."
+  token :CLOSE, "}}", pop: true
+end
+```
+
+### `utf8_range`
+
+```ruby
+token :HIRAGANA, LexerKit.utf8_range("ぁ".."ん")
+token :CJK,      LexerKit.utf8_range(0x4E00..0x9FFF)
+```
+
+## Regex Notes
+
+- Most common regex syntax is supported (`[]`, quantifiers, groups, alternation, escapes, `/.../i`)
+- Backtracking-dependent features are not supported (lookaround, backreference, etc.)
+- Anchors and word-boundary assertions are not used in lexer matching
+- `*?`, `+?`, `??` are parsed but behave as longest-match (DFA behavior)
+
+## Stream API and Error Handling
+
+`stream.start` and `stream.len` are byte offsets.
+
+```ruby
+stream = lexer.stream(input)
+until stream.eof?
+  if stream.error?
+    token = stream.make_token
+    puts token.render_diagnostic("unexpected character")
+  end
+  stream.advance
+end
+```
+
+`LexerKit` always falls back to `:INVALID` for unmatched input.
+
+## Serialization
+
+Pre-compile lexers for faster startup:
+
+```ruby
+lexer = builder.compile
+LexerKit::Format::LKT1.save(lexer, path: "lexer.lkt1")
+LexerKit::Format::LKB1.save(lexer, path: "lexer.lkb1")
+```
+
+```bash
+lexer_kit compile lexer.rb -o lexer.lkt1
+```
+
+Load later:
+
+```ruby
+lexer = LexerKit.load_lexer(File.expand_path("data/lexer.lkt1", __dir__))
+```
+
+## Performance Snapshot
+
+JSON benchmark (600KB input, project benchmark script):
+
+- LexerKit: `95.2 i/s`
+- StringScanner: `4.8 i/s` (about `20x` slower)
+
+## License
+
+MIT License

data/exe/lexer_kit

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "lexer_kit"
+require "lexer_kit/cli"
+
+exit(LexerKit::CLI.run(ARGV))