tokenzr 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2371c68d11dadc0ca627f29af05d9d4a2f8bbb73d1986d29c554f61d16b1fd65
+  data.tar.gz: 7b03309a452cb7d4aefb575b480c84fd573c7513b869cd377e241bfa0fcf9f30
+SHA512:
+  metadata.gz: 1009de75d1520829b58454d43195481cfd143b769672a8d56f9126f4480089f1ac8f50059cc34a19a05d8b7b8e30a2d4051887f8596b7818226d132fbacbf6ec
+  data.tar.gz: 8a31c6066cd5e3c8b1549d9d30a97a4963f7d5dc2d1c85cedecdeada9a7c88e6091b3cdb48b2588d64f6a79e14a063a94cfd660911c01e9e9ec3d79ecd29e24f

data/Gemfile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tokenzr.gemspec
+gemspec
+
+# Specify development dependencies here and not in the gemspec
+gem 'rake'
+gem 'rspec'
+gem 'rubocop'
+gem 'rubocop-rake'
+gem 'rubocop-rspec'
+gem 'rubocop-yard'
+gem 'yard'
+gem 'yard-rspec'

data/exe/README.md

@@ -0,0 +1,3 @@
+# Executable folder
+
+There should only be one file here which has the same name as the gem, if this gem is a commandline gem.

data/lib/tokenzr/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Tokenzr
+  VERSION = '0.1.0'
+end

data/lib/tokenzr.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'tokenzr/version'
+
+module Tokenzr
+  class Error < StandardError; end
+  class UnknownCharError < Error; end
+  class UnterminatedStringError < Error; end
+
+  class Token
+    attr_reader :content, :type, :line, :column
+
+    def initialize(content = nil, type = nil, line = nil, column = nil)
+      @content = content
+      @type = type
+      @line = line
+      @column = column
+    end
+  end
+
+  class Tokenizer
+    def text_chars
+      @text_chars ||= Set.new('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'.chars)
+    end
+
+    def digit_chars
+      @digit_chars ||= Set.new('0123456789'.chars)
+    end
+
+    def lone_chars
+      @lone_chars ||= Set.new('()[]<>{}!#$%&*+,-./:;=?@\\^`|~'.chars)
+    end
+
+    def space_chars
+      @space_chars ||= Set.new(" \t\n\r\v\f".chars)
+    end
+
+    def string_quotes
+      @string_quotes ||= Set.new(%q{"'}.chars)
+    end
+
+    def parse(content)
+      results = []
+      current_token = nil
+      enum = content.each_char
+      @line = 1
+      @column = 1
+      @cur_line = 1
+      @cur_col = 1
+
+      while (chr = next_char(enum))
+        start_line = @cur_line
+        start_col = @cur_col
+
+        if string_quotes.include?(chr)
+          results << current_token unless current_token.nil?
+          current_token = nil
+          results << read_string(enum, chr, start_line, start_col)
+          next
+        end
+
+        if space_chars.include?(chr)
+          results << current_token unless current_token.nil?
+          current_token = nil
+          next
+        end
+
+        if digit_chars.include?(chr)
+          if !current_token.nil? && current_token.type == :text
+            # digits continue an identifier started by text/underscore
+            current_token = Token.new(current_token.content + chr, :text, current_token.line, current_token.column)
+            next
+          end
+
+          results << current_token unless current_token.nil?
+          current_token = nil
+          result = read_number(enum, chr, start_line, start_col)
+          if result.is_a?(Array)
+            results.concat(result)
+          else
+            results << result
+          end
+          next
+        end
+
+        if text_chars.include?(chr)
+          if !current_token.nil? && current_token.type == :text
+            current_token = Token.new(current_token.content + chr, :text, current_token.line, current_token.column)
+          else
+            results << current_token unless current_token.nil?
+            current_token = Token.new(chr, :text, start_line, start_col)
+          end
+          next
+        end
+
+        if lone_chars.include?(chr)
+          results << current_token unless current_token.nil?
+          current_token = nil
+          results << Token.new(chr, :lone, start_line, start_col)
+          next
+        end
+
+        raise UnknownCharError, "Unknown character: #{chr.inspect}"
+      end
+
+      results << current_token unless current_token.nil?
+      results
+    end
+
+    private
+
+    def next_char(enum)
+      pos_line = @line
+      pos_col = @column
+      chr = enum.next
+      @cur_line = pos_line
+      @cur_col = pos_col
+      if chr == "\n"
+        @line = pos_line + 1
+        @column = 1
+      else
+        @column = pos_col + 1
+      end
+      chr
+    rescue StopIteration
+      nil
+    end
+
+    def peek_char(enum)
+      enum.peek
+    rescue StopIteration
+      nil
+    end
+
+    def read_string(enum, quote, line, column)
+      content = quote
+      loop do
+        chr = next_char(enum)
+        raise UnterminatedStringError, "Unterminated string: #{quote}" if chr.nil?
+
+        content += chr
+        break if chr == quote
+
+        next unless chr == '\\'
+
+        escaped = next_char(enum)
+        raise UnterminatedStringError, "Unterminated string: #{quote}" if escaped.nil?
+
+        content += escaped
+      end
+      Token.new(content, :string, line, column)
+    end
+
+    def read_number(enum, first, line, column)
+      content = first
+
+      # hex: 0x or 0X followed by at least one hex digit
+      if first == '0'
+        peeked = peek_char(enum)
+        if peeked == 'x' || peeked == 'X'
+          next_char(enum) # consume the x/X
+          x_line = @cur_line
+          x_col = @cur_col
+          hex_body = read_hex(enum)
+          return [Token.new(content, :number, line, column), Token.new(peeked, :text, x_line, x_col)] if hex_body.empty?
+
+          content += peeked + hex_body
+          return Token.new(content, :number, line, column)
+        end
+      end
+
+      content += read_digits(enum)
+
+      # fraction: . followed by a digit
+      if peek_char(enum) == '.'
+        next_char(enum) # consume the .
+        dot_line = @cur_line
+        dot_col = @cur_col
+        if digit_char?(peek_char(enum))
+          content += '.' + read_digits(enum)
+        else
+          # . not followed by a digit: put it back as a lone token
+          return [Token.new(content, :number, line, column), Token.new('.', :lone, dot_line, dot_col)]
+        end
+      end
+
+      # exponent: e or E, optional +/-, then at least one digit
+      peeked = peek_char(enum)
+      if peeked == 'e' || peeked == 'E'
+        saved = peeked
+        next_char(enum) # consume the e/E
+        e_line = @cur_line
+        e_col = @cur_col
+        sign = peek_char(enum)
+        consumed_sign = nil
+        sign_line = nil
+        sign_col = nil
+        if sign == '+' || sign == '-'
+          next_char(enum) # consume the sign
+          consumed_sign = sign
+          sign_line = @cur_line
+          sign_col = @cur_col
+        end
+        if digit_char?(peek_char(enum))
+          content += saved + (consumed_sign || '') + read_digits(enum)
+        else
+          # e/E (and optional sign) not followed by a digit: emit number, then the
+          # consumed e/E and sign as separate tokens (already consumed from enum)
+          fallback = [Token.new(content, :number, line, column), Token.new(saved, :text, e_line, e_col)]
+          fallback << Token.new(consumed_sign, :lone, sign_line, sign_col) if consumed_sign
+          return fallback
+        end
+      end
+
+      Token.new(content, :number, line, column)
+    end
+
+    def read_hex(enum)
+      digits = ''
+      while hex_char?(peek_char(enum))
+        digits += next_char(enum)
+      end
+      digits
+    end
+
+    def read_digits(enum)
+      digits = ''
+      while digit_char?(peek_char(enum))
+        digits += next_char(enum)
+      end
+      digits
+    end
+
+    def hex_char?(chr)
+      return false if chr.nil?
+
+      digit_char?(chr) || ('a'..'f').cover?(chr) || ('A'..'F').cover?(chr)
+    end
+
+    def digit_char?(chr)
+      !chr.nil? && digit_chars.include?(chr)
+    end
+  end
+end

data/tokenzr.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'lib/tokenzr/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'tokenzr'
+  spec.version       = Tokenzr::VERSION
+  spec.authors       = ['David Siaw']
+  spec.email         = ['874280+davidsiaw@users.noreply.github.com']
+
+  spec.summary       = 'Tokenzr gem'
+  spec.description   = 'Tokenizer for most common cases of tokenization'
+  spec.homepage      = 'https://github.com/davidsiaw/tokenzr'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 3.0')
+
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/davidsiaw/tokenzr'
+  spec.metadata['changelog_uri'] = 'https://github.com/davidsiaw/tokenzr'
+  spec.metadata['documentation_uri'] = 'https://davidsiaw.github.io/tokenzr'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.files         = Dir['{exe,data,lib}/**/*'] + %w[Gemfile tokenzr.gemspec]
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: tokenzr
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Siaw
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-07-22 00:00:00.000000000 Z
+dependencies: []
+description: Tokenizer for most common cases of tokenization
+email:
+- 874280+davidsiaw@users.noreply.github.com
+executables:
+- README.md
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- exe/README.md
+- lib/tokenzr.rb
+- lib/tokenzr/version.rb
+- tokenzr.gemspec
+homepage: https://github.com/davidsiaw/tokenzr
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/davidsiaw/tokenzr
+  source_code_uri: https://github.com/davidsiaw/tokenzr
+  changelog_uri: https://github.com/davidsiaw/tokenzr
+  documentation_uri: https://davidsiaw.github.io/tokenzr
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Tokenzr gem
+test_files: []