gigatoken 0.1.0

data/lib/gigatoken/cli/support.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Gigatoken
+  module CLI
+    # Helpers shared by the bench and validate commands: tokenizer loading,
+    # byte-size parsing, document splitting, and CPU identification.
+    module Support
+      SIZE_UNITS = {"" => 1, "k" => 10**3, "m" => 10**6, "g" => 10**9, "t" => 10**12}.freeze
+      private_constant :SIZE_UNITS
+
+      SIZE_PATTERN = /\A\s*(\d+(?:\.\d+)?)\s*([kmgt]?)i?b?\s*\z/i
+      private_constant :SIZE_PATTERN
+
+      class << self
+        # Load TOKENIZER: a tokenizer.json path/directory, a HuggingFace
+        # repo id, or a .tiktoken file — see Gigatoken::Tokenizer.load.
+        def load_tokenizer(spec)
+          Gigatoken::Tokenizer.load(spec)
+        end
+
+        # Parse a decimal byte size like "100MB", "2.5GB", or "1000000";
+        # "none"/"unlimited" means no limit.
+        def parse_size(text)
+          return nil if ["none", "unlimited"].include?(text.strip.downcase)
+
+          match = SIZE_PATTERN.match(text)
+          raise Gigatoken::Error, "cannot parse size #{text.inspect}; expected something like 100MB (or 'none')" unless match
+
+          (match[1].to_f * SIZE_UNITS.fetch(match[2].downcase)).to_i
+        end
+
+        # A Native::TextFileSource for FILES, split on `separator` when
+        # given.
+        def text_file_source(files, separator)
+          Gigatoken::Native::TextFileSource.new(files.map(&:to_s), separator: separator)
+        end
+
+        # Whole files as raw bytes, one document per file, or (with a
+        # separator) the separator-split pieces of each file in order, empty
+        # documents skipped.
+        def split_docs(files, separator)
+          raws = files.map { |file| File.binread(file.to_s) }
+          return raws if separator.nil?
+
+          sep = separator.b
+          raws.flat_map { |raw| raw.split(sep).reject(&:empty?) }
+        end
+
+        # The prefix of `docs` totalling at most `limit_bytes`, byte-
+        # truncating the final document to fill the budget. Unlike a
+        # text-comparison tool, gigatoken encodes raw bytes and does not
+        # require the cut to land on a UTF-8 character boundary.
+        def subset_docs(docs, limit_bytes)
+          return docs if limit_bytes.nil?
+
+          subset = []
+          used = 0
+          docs.each do |doc|
+            room = limit_bytes - used
+            if doc.bytesize <= room
+              subset << doc
+              used += doc.bytesize
+            else
+              subset << doc.byteslice(0, room) if room > 0
+              break
+            end
+          end
+          subset
+        end
+
+        # The benchmark machine's CPU as "name, N cores", plus ", M sockets"
+        # when there is more than one socket.
+        def cpu_info
+          name, cores, sockets =
+            case RbConfig::CONFIG["host_os"]
+            when /darwin/ then darwin_cpu_info
+            when /linux/ then linux_cpu_info
+            end
+          name ||= RbConfig::CONFIG["host_cpu"] || "unknown CPU"
+          cores ||= Etc.nprocessors
+          parts = [name, "#{cores} core#{"s" unless cores == 1}"]
+          parts << "#{sockets} sockets" if sockets && sockets > 1
+          parts.join(", ")
+        end
+
+        private
+
+        def darwin_cpu_info
+          name = sysctl("machdep.cpu.brand_string")
+          [name, sysctl_int("hw.physicalcpu"), sysctl_int("hw.packages")]
+        end
+
+        def sysctl(key)
+          output = IO.popen(["sysctl", "-n", key], err: File::NULL, &:read)
+          output.strip unless output.nil? || output.empty? || !$?.success?
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def sysctl_int(key)
+          value = sysctl(key)
+          Integer(value) if value&.match?(/\A\d+\z/)
+        end
+
+        # Within each processor block "physical id" precedes "core id", so
+        # (socket, core) pairs count physical cores across sockets.
+        def linux_cpu_info
+          name = nil
+          physical_ids = Set.new
+          socket_core_ids = Set.new
+          physical_id = ""
+          File.foreach("/proc/cpuinfo") do |line|
+            key, _, value = line.partition(":")
+            key, value = key.strip, value.strip
+            case key
+            when "model name" then name ||= value
+            when "physical id"
+              physical_id = value
+              physical_ids << value
+            when "core id" then socket_core_ids << [physical_id, value]
+            end
+          end
+          [name, (socket_core_ids.size unless socket_core_ids.empty?), (physical_ids.size unless physical_ids.empty?)]
+        rescue Errno::ENOENT
+          [nil, nil, nil]
+        end
+      end
+    end
+  end
+end

data/lib/gigatoken/cli/validate.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "support"
+
+module Gigatoken
+  module CLI
+    # `gigatoken validate TOKENIZER FILES...` — a Ruby-vs-Ruby consistency
+    # check: encode FILES via `Tokenizer#encode_files` (native file loading
+    # and splitting) and via a Ruby-side split plus `Tokenizer#encode_batch`,
+    # and confirm the two paths agree per document. Cross-library validation
+    # against another tokenizer implementation is out of scope for v1
+    # (BRIEF §3.4).
+    class Validate < Dry::CLI::Command
+      desc "Check that encode_files agrees with a Ruby-side split plus encode_batch on FILES"
+
+      argument :tokenizer, required: true, desc: "tokenizer.json path or directory, HuggingFace repo id, or .tiktoken file"
+      argument :files, type: :array, required: true, desc: "UTF-8 text files to encode"
+
+      option :doc_separator, desc: 'document separator to split the files on, e.g. "<|endoftext|>"; whole files are single documents otherwise'
+
+      def call(tokenizer:, files:, doc_separator: nil, **)
+        gt_tokenizer = Support.load_tokenizer(tokenizer)
+
+        via_files = gt_tokenizer.encode_files(Support.text_file_source(files, doc_separator))
+        via_batch = gt_tokenizer.encode_batch(Support.split_docs(files, doc_separator))
+
+        if via_files.length != via_batch.length
+          err.puts "validation FAILED: encode_files produced #{via_files.length} documents, encode_batch produced #{via_batch.length}"
+          exit(1)
+        end
+
+        via_files.each_with_index do |doc, index|
+          next if doc == via_batch[index]
+
+          report_mismatch(index, doc, via_batch[index])
+          exit(1)
+        end
+
+        out.puts "validation OK: #{via_files.length} documents match"
+      rescue Gigatoken::Error => e
+        err.puts "error: #{e.message}"
+        exit(1)
+      end
+
+      private
+
+      def report_mismatch(index, from_files, from_batch)
+        at = from_files.zip(from_batch).index { |a, b| a != b } || [from_files.length, from_batch.length].min
+        err.puts "validation FAILED: document #{index}: first mismatch at token #{at} " \
+          "(encode_files #{from_files[at, 5]}... vs encode_batch #{from_batch[at, 5]}..., " \
+          "lengths #{from_files.length} vs #{from_batch.length})"
+      end
+    end
+  end
+end

data/lib/gigatoken/cli.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "gigatoken"
+require "dry/cli"
+
+require_relative "cli/bench"
+require_relative "cli/validate"
+
+module Gigatoken
+  module CLI
+    module Commands
+      extend Dry::CLI::Registry
+
+      register "bench", Gigatoken::CLI::Bench
+      register "validate", Gigatoken::CLI::Validate
+    end
+  end
+end

data/lib/gigatoken/hub.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/http"
+require "fileutils"
+require "pathname"
+
+module Gigatoken
+  # HuggingFace Hub file fetch, mirroring `huggingface_hub.hf_hub_download`:
+  # same endpoint and URL layout, same token discovery (HF_TOKEN env var,
+  # then the token file written by `hf auth login`), same cache directory
+  # resolution — without requiring huggingface_hub, tokenizers, or
+  # transformers. Files already present in the standard HF cache are served
+  # with a pure-filesystem lookup (no network); on a miss the file is
+  # downloaded straight into the shared cache, so later loads (ours and
+  # huggingface_hub's) are served from it.
+  #
+  # Network I/O runs on async-http, on the reactor: #hub_file wraps its
+  # fetch in `Sync`, so it composes whether the caller is already inside a
+  # reactor or is plain sync code.
+  class Hub
+    # Filename suffixes of local tokenizer files (tokenizer.json contents
+    # and raw sentencepiece models). A name ending in one of these is never
+    # treated as a Hub repo id, so a mistyped local path fails fast instead
+    # of hitting the network. Keep in sync with `src/load_tokenizer/hub.rs`'s
+    # TOKENIZER_FILE_SUFFIXES.
+    TOKENIZER_FILE_SUFFIXES = [".json", ".model"].freeze
+
+    DEFAULT_ENDPOINT = "https://huggingface.co"
+    private_constant :DEFAULT_ENDPOINT
+
+    MAX_REDIRECTS = 10
+    private_constant :MAX_REDIRECTS
+
+    class << self
+      # Whether `name` is shaped like a HuggingFace Hub repo id: `org/name`,
+      # or a bare legacy repo name like `gpt2`. At most one slash, and not
+      # something that is obviously a filesystem path to a local tokenizer
+      # file.
+      def looks_like_repo_id?(name)
+        parts = name.split("/", -1)
+        return false if parts.empty? || parts.size > 2
+
+        org, rest = parts
+        return false unless word_part?(org, first_alnum: true)
+        return false if rest && !word_part?(rest, first_alnum: false)
+
+        TOKENIZER_FILE_SUFFIXES.none? { |suffix| name.end_with?(suffix) }
+      end
+
+      # $HF_HOME, defaulting to $XDG_CACHE_HOME/huggingface then
+      # ~/.cache/huggingface — the root for both the hub cache and the token
+      # file.
+      def hf_home
+        Pathname.new(env("HF_HOME") || File.join(env("XDG_CACHE_HOME") || File.join(Dir.home, ".cache"), "huggingface"))
+      end
+
+      # The standard HuggingFace hub cache directory, resolved like
+      # huggingface_hub does it: HF_HUB_CACHE, then $HF_HOME/hub.
+      def hf_hub_cache_dir
+        Pathname.new(env("HF_HUB_CACHE") || hf_home.join("hub").to_s)
+      end
+
+      # The HuggingFace access token, discovered like huggingface_hub does
+      # it: the HF_TOKEN (or legacy HUGGING_FACE_HUB_TOKEN) environment
+      # variable, then the token file (HF_TOKEN_PATH, default $HF_HOME/token).
+      def hf_token
+        token = env("HF_TOKEN") || env("HUGGING_FACE_HUB_TOKEN")
+        return token.strip if token
+
+        token_path = env("HF_TOKEN_PATH") || hf_home.join("token").to_s
+        return nil unless File.file?(token_path)
+
+        token = File.read(token_path).strip
+        token unless token.empty?
+      end
+
+      # Path of `filename` in the local HF cache, or nil when not cached. A
+      # pure-filesystem lookup — no request is made. `revision` may be a
+      # commit hash (used directly as the snapshot name) or a branch/tag
+      # name (followed through the cached ref).
+      def cached_file(repo_id, filename, revision)
+        repo_dir = repo_cache_dir(repo_id)
+        commit = commit_hash?(revision) ? revision : cached_ref(repo_dir, revision)
+        return nil unless commit
+
+        path = repo_dir.join("snapshots", commit, filename)
+        path if path.file?
+      end
+
+      # The cache directory of a repo (`models--org--name`).
+      def repo_cache_dir(repo_id)
+        hf_hub_cache_dir.join("models--#{repo_id.gsub("/", "--")}")
+      end
+
+      # A full git commit hash: cache snapshot directories are named by
+      # these.
+      def commit_hash?(revision)
+        revision.match?(/\A[0-9a-f]{40}\z/)
+      end
+
+      private
+
+      def env(key)
+        value = ENV[key]
+        value unless value.nil? || value.empty?
+      end
+
+      def word_part?(part, first_alnum:)
+        return false if part.nil? || part.empty?
+
+        first_ok = first_alnum ? part[0].match?(/[A-Za-z0-9]/) : word_char?(part[0])
+        first_ok && part[1..].chars.all? { |c| word_char?(c) }
+      end
+
+      def word_char?(char)
+        char.match?(/[A-Za-z0-9_.-]/)
+      end
+
+      def cached_ref(repo_dir, revision)
+        ref_path = repo_dir.join("refs", revision)
+        File.read(ref_path).strip if ref_path.file?
+      end
+    end
+
+    # @parameter endpoint [String] the Hub endpoint to fetch from — override
+    #   for pointing at a local server in tests (dependency injection, not a
+    #   mock).
+    def initialize(endpoint: DEFAULT_ENDPOINT)
+      @endpoint = endpoint.chomp("/")
+      @internet = Async::HTTP::Internet.new
+    end
+
+    # Path of `filename` from Hub repo `repo_id` at `revision`, served from
+    # the standard HF cache, downloading into it first when absent.
+    def hub_file(repo_id, filename = "tokenizer.json", revision: "main")
+      self.class.cached_file(repo_id, filename, revision) ||
+        Sync { fetch(repo_id, filename, revision) }
+    end
+
+    private
+
+    # GET `endpoint/repo/resolve/revision/filename` and stream the body into
+    # the cache snapshot named by the `x-repo-commit` response header,
+    # recording the branch ref so later lookups (ours and
+    # huggingface_hub's) resolve it.
+    def fetch(repo_id, filename, revision)
+      url = "#{@endpoint}/#{repo_id}/resolve/#{revision}/#{filename}"
+      token = self.class.hf_token
+      response = @internet.get(url, auth_headers(token))
+      # Unlisted headers parse as a Header::Generic (an Array of values);
+      # x-repo-commit is always a single value, so flatten it to a String.
+      commit = response.headers["x-repo-commit"]&.to_s
+
+      # Redirects are followed by hand: resolve/ URLs answer with the
+      # x-repo-commit header and a redirect to a CDN for LFS files, and the
+      # Authorization header must not travel to the other host.
+      hops = 0
+      while (300...400).cover?(response.status)
+        ensure_ok!(url, response.status, !!token)
+        hops += 1
+        raise Error, "#{url}: too many redirects" if hops > MAX_REDIRECTS
+
+        location = response.headers["location"] ||
+          raise(Error, "#{url}: redirect with no Location header")
+        response.close
+        url = absolutize(location, url)
+        response = @internet.get(url, {"user-agent" => "gigatoken"})
+      end
+      ensure_ok!(url, response.status, !!token)
+
+      write_to_cache(repo_id, filename, revision, commit || revision, response)
+    end
+
+    def auth_headers(token)
+      headers = {"user-agent" => "gigatoken"}
+      headers["authorization"] = "Bearer #{token}" if token
+      headers
+    end
+
+    # Stream the response body to a sibling temp file, then rename into
+    # place: concurrent downloaders race benignly and readers never observe
+    # a partial file.
+    def write_to_cache(repo_id, filename, revision, commit, response)
+      repo_dir = self.class.repo_cache_dir(repo_id)
+      target = repo_dir.join("snapshots", commit, filename)
+      FileUtils.mkdir_p(target.dirname)
+
+      tmp = target.dirname.join(".#{target.basename}.#{Process.pid}.tmp")
+      begin
+        response.save(tmp.to_s)
+      rescue
+        FileUtils.rm_f(tmp)
+        raise
+      ensure
+        response.close
+      end
+      File.rename(tmp, target)
+
+      if !self.class.commit_hash?(revision) && revision != commit
+        refs_dir = repo_dir.join("refs")
+        FileUtils.mkdir_p(refs_dir)
+        File.write(refs_dir.join(revision), commit)
+      end
+
+      target
+    end
+
+    def ensure_ok!(url, status, had_token)
+      return if (200...400).cover?(status)
+
+      case status
+      when 404
+        raise Error, "#{url}: HTTP 404 — no such repo with that file, and no such local file either"
+      when 401, 403
+        token_note = had_token ? "the request used the discovered token" : "no token was found"
+        raise Error,
+          "#{url}: HTTP #{status} — the repo may be private or gated (#{token_note}; set HF_TOKEN or run " \
+          "`hf auth login`, and accept the repo's terms on huggingface.co if it is gated)"
+      else
+        raise Error, "#{url}: HTTP #{status}"
+      end
+    end
+
+    # A redirect Location resolved against the request URL: absolute URLs
+    # pass through, host-relative (`/x/y`) and path-relative ones join the
+    # base.
+    def absolutize(location, base)
+      return location if location.include?("://")
+
+      origin_end = base.index("://") ? base.index("://") + 3 : 0
+      origin_end = base.index("/", origin_end) || base.length
+
+      if location.start_with?("/")
+        "#{base[0...origin_end]}#{location}"
+      else
+        dir_end = base.rindex("/") || base.length
+        "#{base[0...[dir_end, origin_end].max]}/#{location}"
+      end
+    end
+  end
+end

data/lib/gigatoken/packed_result.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gigatoken
+  # A packed batch-encode result: one `IO::Buffer` of u32 token ids (native
+  # byte order) for every document, plus `lens`, each document's token count.
+  # Returned by `Tokenizer#encode_batch`/`#encode_files` with `packed: true`
+  # — the Ruby analog of the numpy/awkward-array path the native ext's
+  # header comment says was dropped for lack of a Ruby equivalent. Documents
+  # are unpacked from the buffer on demand via `#[]`/`#each`.
+  class PackedResult
+    include Enumerable
+
+    attr_reader :buffer, :lens
+
+    def initialize(buffer, lens)
+      @buffer = buffer
+      @lens = lens
+      @offsets = lens.inject([0]) { |offsets, len| offsets << offsets.last + len }
+    end
+
+    # Number of documents.
+    def size
+      lens.size
+    end
+
+    # Total token count across every document.
+    def token_count
+      lens.sum
+    end
+
+    # Array of token ids for document `i`, materialized on demand.
+    def [](i)
+      buffer.get_values(Array.new(lens[i], :u32), @offsets[i] * 4)
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+
+      lens.each_index { |i| yield self[i] }
+    end
+
+    # A ragged Array of Arrays, one per document — the same shape
+    # `encode_batch`/`encode_files` return with `packed: false`.
+    def to_a
+      each.to_a
+    end
+  end
+end

data/lib/gigatoken/tokenizer.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Gigatoken
+  # A tokenizer: encode, batch encode, decode, and vocabulary introspection
+  # over a native `Gigatoken::Native::BPETokenizer` or
+  # `Gigatoken::Native::SentencePieceTokenizer`.
+  class Tokenizer
+    TIKTOKEN_ENDOFTEXT = "<|endoftext|>"
+    private_constant :TIKTOKEN_ENDOFTEXT
+
+    FILE_SOURCE_CLASSES = [Native::TextFileSource, Native::JsonlFileSource, Native::ParquetFileSource].freeze
+    private_constant :FILE_SOURCE_CLASSES
+
+    # Load from in-memory tokenizer.json contents (String or bytes). Backed
+    # by a BPETokenizer or a SentencePieceTokenizer, per the model's
+    # byte_fallback flag.
+    def self.from_json(data)
+      native = Native.load_hf_json(data)
+      new(native, special_tokens: special_tokens_from_json(data))
+    end
+
+    # Load from a tokenizer.json path, or a directory containing one.
+    def self.from_file(path)
+      path = File.join(path, "tokenizer.json") if File.directory?(path)
+      from_json(File.binread(path))
+    end
+
+    # Load from a .tiktoken mergeable-ranks file.
+    def self.from_tiktoken(path)
+      native = Native::BPETokenizer.from_tiktoken(path.to_s)
+      new(native, special_tokens: {TIKTOKEN_ENDOFTEXT => native.vocab_size - 1})
+    end
+
+    # Load tokenizer.json from HuggingFace Hub repo `repo_id` at `revision`
+    # (downloaded directly; huggingface_hub is not required).
+    def self.from_hub(repo_id, revision: "main", hub: Hub.new)
+      from_file(hub.hub_file(repo_id, "tokenizer.json", revision: revision))
+    end
+
+    # Load from any of the supported source shapes: an existing file or
+    # directory path (a tokenizer.json, or a directory containing one), a
+    # .tiktoken vocabulary file, or a HuggingFace Hub repo id like
+    # "openai-community/gpt2".
+    def self.load(source, revision: "main", hub: Hub.new)
+      source = source.to_s
+      return from_tiktoken(source) if source.end_with?(".tiktoken")
+      return from_file(source) if File.exist?(source)
+      return from_hub(source, revision: revision, hub: hub) if Hub.looks_like_repo_id?(source)
+
+      raise Error, "#{source.inspect}: no such file or directory, not a .tiktoken path, and doesn't look like a HuggingFace Hub repo id"
+    end
+
+    def self.special_tokens_from_json(data)
+      added = JSON.parse(data.dup.force_encoding(Encoding::UTF_8))["added_tokens"] || []
+      added.each_with_object({}) { |t, h| h[t["content"]] = t["id"] if t["special"] }
+    end
+    private_class_method :special_tokens_from_json
+
+    def initialize(native, special_tokens: {})
+      @native = native
+      @special_tokens = special_tokens
+    end
+
+    def encode(text)
+      @native.encode(text)
+    end
+
+    # Returns a ragged Array of Arrays of token ids, one row per document —
+    # or, with `packed: true`, a Gigatoken::PackedResult (one IO::Buffer of
+    # token ids plus per-document lengths), avoiding the per-token Ruby
+    # array materialization the ragged shape costs.
+    def encode_batch(texts, packed: false)
+      if packed
+        PackedResult.new(*@native.encode_batch_packed(texts))
+      else
+        @native.encode_batch(texts)
+      end
+    end
+
+    # Tokenize whole files in Rust: reads and encodes them in one fused pass
+    # without the documents ever becoming Ruby objects. `source` is a
+    # Native::{Text,Jsonl,Parquet}FileSource, a single path, or an array of
+    # paths; bare path(s) are wrapped in a TextFileSource (with `separator`,
+    # if given). Returns a ragged Array of Arrays of token ids, one row per
+    # document — or, with `packed: true`, a Gigatoken::PackedResult. `parallel:
+    # false` loads and encodes everything on the calling thread instead, with
+    # identical output, never touching the core worker pool.
+    def encode_files(source, separator: nil, parallel: true, packed: false)
+      source = Native::TextFileSource.new(Array(source).map(&:to_s), separator: separator) unless FILE_SOURCE_CLASSES.any? { |klass| source.is_a?(klass) }
+      if packed
+        PackedResult.new(*@native.encode_files_packed(source, parallel: parallel))
+      else
+        @native.encode_files(source, parallel: parallel)
+      end
+    end
+
+    def decode(ids)
+      @native.decode(ids)
+    end
+
+    def vocab_size
+      @native.vocab_size
+    end
+
+    def vocab
+      @native.vocab
+    end
+
+    def merges
+      @native.merges
+    end
+
+    attr_reader :special_tokens
+  end
+end

data/lib/gigatoken/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Gigatoken
+  VERSION = "0.1.0"
+end

data/lib/gigatoken.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "gigatoken/version"
+
+module Gigatoken
+  # Raised for tokenizer load and encode failures surfaced from the native
+  # extension — never a raw Rust panic across the Ruby boundary.
+  class Error < StandardError; end
+
+  NATIVE_EXTENSIONS = %w[.bundle .so .rb].freeze
+
+  # Precompiled native gems ship per-ABI subdirs (`gigatoken/4.0/...`),
+  # the source-gem `rake compile` build lands flat (`gigatoken/...`).
+  # Pick whichever exists for the current Ruby ABI, with the per-ABI path
+  # winning when both are present.
+  def self.locate_native(base, ruby_version: RUBY_VERSION)
+    abi = ruby_version[/\d+\.\d+/]
+    candidates = [File.join(base, abi, "gigatoken_rb"), File.join(base, "gigatoken_rb")]
+    candidates.find { |stem| NATIVE_EXTENSIONS.any? { |ext| File.exist?(stem + ext) } }
+  end
+end
+
+native = Gigatoken.locate_native(File.expand_path("gigatoken", __dir__))
+raise LoadError, "could not locate gigatoken native extension" unless native
+require native
+
+require_relative "gigatoken/hub"
+require_relative "gigatoken/packed_result"
+require_relative "gigatoken/tokenizer"

data/rust-toolchain.toml

@@ -0,0 +1,8 @@
+[toolchain]
+channel = "nightly"
+# Cross triples for the precompiled-gem builds (ruby-gem.yml). rustup's
+# toolchain-file auto-install is the only thing that provisions nightly
+# inside the rb-sys-dock containers, so the targets must be declared here —
+# the cross-gem action's `pre-script` input is dead plumbing (undeclared at
+# v1.4.4, and no released rb-sys-dock reads INPUT_PRE_SCRIPT).
+targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]