tokenkit 0.1.0.pre.1 → 0.1.0.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76766a917a2c731ac2b6fa223a4f79c2844f0174b13bc5911e7b0e09a1052607
-  data.tar.gz: 791f780ff1f1388ba111a3836e8b4e036b5660417eb3d5278a53c26d8ffb837e
+  metadata.gz: 4f5bf68366a726b7b17fdda101f4fc7019a4990ebe579f765bd9ce7e87f46ae2
+  data.tar.gz: 17e693e29bc188a9bd18813c9f629f5fab484047cef028d6e8f1811294cd31b7
 SHA512:
-  metadata.gz: 97a48ad146e8ac6c1ba1951acc4b485ba1f66e42f3d040d5cb904693cd878464bb7c2f2ea7bfeba652b6c382ddbc4a6b83b3a5b2d10d05347e77537abf73cc01
-  data.tar.gz: 27cb53e54031d9675f78e994cfbb12b202a64eb746b758cd79a7f40be3dc9ac8bdc1bfdeb0bcefa005e418320639cbfad6851d6533be1f1d782d64b2483f46df
+  metadata.gz: f63b7ffbd72a96bf101b14e8da484c0c12c5c47dc912a5cb4b81ae9b372887f72a9925bd28f5506bf95b34f5ca67b9bd155d9f492502423f1e27477cb05995f0
+  data.tar.gz: '026835f1944ab9a7da0ba37a71eee667821be9aa0780239ccd69cf75b9d2199531bc40e3026bfa29a6e79c0394dc56d45bff35d01f5e9cbe79c71558f039b142'

data/README.md

@@ -1,4 +1,4 @@
-# TokenKit
+<img src="/docs/assets/tokenkit-wide.png" alt="tokenkit" height="120px">
 
 Fast, Rust-backed word-level tokenization for Ruby with pattern preservation.
 

data/Rakefile

@@ -1,17 +1,36 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require "rake/extensiontask"
 
-RSpec::Core::RakeTask.new(:spec)
+# rspec and standard are DEVELOPMENT-only deps. The cross-compile build container
+# (rb-sys-dock, via scientist-labs/rust-gem-release) installs the runtime bundle
+# only, so these requires would raise LoadError and abort `rake` before the native
+# build task can run. Guard them so this Rakefile always loads; the spec/standard
+# tasks simply aren't available in a build-only environment.
+begin
+  require "rspec/core/rake_task"
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+  desc "run specs (rspec unavailable in this environment)"
+  task(:spec) { abort "rspec is a development dependency and is not installed here" }
+end
 
-require "standard/rake"
-require "rake/extensiontask"
+begin
+  require "standard/rake"
+rescue LoadError
+  # standard is dev-only; skip the lint task when it isn't installed.
+end
 
 GEMSPEC = Gem::Specification.load("tokenkit.gemspec")
 
 Rake::ExtensionTask.new("tokenkit", GEMSPEC) do |ext|
   ext.lib_dir = "lib/tokenkit"
+  # cross_compile + cross_platform make rake-compiler expose the
+  # `native:<platform>` tasks that rb-sys-dock invokes for each precompiled leg.
+  # Without these, the cross build fails with "Don't know how to build task".
+  ext.cross_compile = true
+  ext.cross_platform = %w[x86_64-linux aarch64-linux arm64-darwin]
 end
 
 task spec: :compile

data/lib/tokenkit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TokenKit
-  VERSION = "0.1.0.pre.1"
+  VERSION = "0.1.0.pre.2"
 end

data/lib/tokenkit.rb

@@ -5,11 +5,17 @@ require_relative "tokenkit/regex_converter"
 require_relative "tokenkit/config_builder"
 require_relative "tokenkit/config_compat"
 
+# Load the compiled Rust extension. Precompiled (platform) gems install it into a
+# Ruby-ABI-versioned subdir (lib/tokenkit/<major.minor>/tokenkit.{so,bundle}) so a single
+# fat gem can carry a binary per Ruby version; source/dev builds place it flat at
+# lib/tokenkit/tokenkit.{so,bundle}. Try the versioned path first, fall back to the flat
+# one. Resolution goes through $LOAD_PATH (`require`, never `require_relative`) because
+# RubyGems installs native extensions outside the gem's lib/ dir.
 begin
   RUBY_VERSION =~ /(\d+\.\d+)/
-  require_relative "tokenkit/#{Regexp.last_match(1)}/tokenkit"
+  require "tokenkit/#{Regexp.last_match(1)}/tokenkit"
 rescue LoadError
-  require_relative "tokenkit/tokenkit"
+  require "tokenkit/tokenkit"
 end
 
 # TokenKit provides fast, Rust-backed tokenization for Ruby with pattern preservation.

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: tokenkit
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.1
+  version: 0.1.0.pre.2
 platform: ruby
 authors:
 - Chris Petersen
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-09-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -116,6 +115,8 @@ files:
 - docs/ARCHITECTURE.md
 - docs/PERFORMANCE.md
 - docs/README.md
+- docs/assets/tokenkit-wide.png
+- docs/assets/tokenkit.png
 - ext/tokenkit/Cargo.toml
 - ext/tokenkit/extconf.rb
 - ext/tokenkit/src/config.rs
@@ -150,7 +151,6 @@ licenses:
 metadata:
   homepage_uri: https://github.com/scientist-labs/tokenkit
   source_code_uri: https://github.com/scientist-labs/tokenkit
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -164,9 +164,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-requirements: []
-rubygems_version: 3.5.3
-signing_key:
+requirements:
+- Rust >= 1.85
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Fast, Rust-backed word-level tokenization for Ruby
 test_files: []