gigatoken 0.1.0-aarch64-linux → 0.2.0-aarch64-linux

data/lib/gigatoken/encodings.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Gigatoken
+  # The tiktoken encodings gigatoken vendors ranks for, and the pieces a
+  # .tiktoken file doesn't carry: its pretokenizer scheme and special-token
+  # table (see Tokenizer.from_tiktoken and lib/gigatoken/encodings/
+  # PROVENANCE.md, the source of truth this is transcribed from).
+  module Encodings
+    DATA_DIR = File.expand_path("encodings", __dir__)
+    private_constant :DATA_DIR
+
+    # The non-contiguous head of openai_public.py's o200k_harmony()
+    # special-token table, transcribed verbatim (see PROVENANCE.md): its ten
+    # named control tokens, plus the six reserved slots sitting in the gaps
+    # between them (200000, 200001, 200004, 200009, 200010, 200011). The
+    # reserved range only goes contiguous at 200013.
+    HARMONY_HEAD_TOKENS = {
+      "<|startoftext|>" => 199998,
+      "<|endoftext|>" => 199999,
+      "<|reserved_200000|>" => 200000,
+      "<|reserved_200001|>" => 200001,
+      "<|return|>" => 200002,
+      "<|constrain|>" => 200003,
+      "<|reserved_200004|>" => 200004,
+      "<|channel|>" => 200005,
+      "<|start|>" => 200006,
+      "<|end|>" => 200007,
+      "<|message|>" => 200008,
+      "<|reserved_200009|>" => 200009,
+      "<|reserved_200010|>" => 200010,
+      "<|reserved_200011|>" => 200011,
+      "<|call|>" => 200012,
+      "<|endofprompt|>" => 200018
+    }.freeze
+    private_constant :HARMONY_HEAD_TOKENS
+
+    # The contiguous tail of that reserved range: 200013..201087. With the
+    # head above, 1091 entries total — 10 named, 1081 reserved (see
+    # PROVENANCE.md).
+    HARMONY_RESERVED_TAIL = (200013..201087).to_h { |id| ["<|reserved_#{id}|>", id] }.freeze
+    private_constant :HARMONY_RESERVED_TAIL
+
+    REGISTRY = {
+      "r50k_base" => {
+        rank_file: File.join(DATA_DIR, "r50k_base.tiktoken"),
+        pretokenizer: "gpt2",
+        special_tokens: {"<|endoftext|>" => 50256}
+      },
+      "cl100k_base" => {
+        rank_file: File.join(DATA_DIR, "cl100k_base.tiktoken"),
+        pretokenizer: "gpt4",
+        special_tokens: {
+          "<|endoftext|>" => 100257,
+          "<|fim_prefix|>" => 100258,
+          "<|fim_middle|>" => 100259,
+          "<|fim_suffix|>" => 100260,
+          "<|endofprompt|>" => 100276
+        }
+      },
+      "o200k_base" => {
+        rank_file: File.join(DATA_DIR, "o200k_base.tiktoken"),
+        pretokenizer: "o200k",
+        special_tokens: {"<|endoftext|>" => 199999, "<|endofprompt|>" => 200018}
+      },
+      "o200k_harmony" => {
+        rank_file: File.join(DATA_DIR, "o200k_base.tiktoken"),
+        pretokenizer: "o200k",
+        special_tokens: HARMONY_HEAD_TOKENS.merge(HARMONY_RESERVED_TAIL).freeze
+      }
+    }.freeze
+    private_constant :REGISTRY
+
+    # The packaged encoding names — the single source error messages naming
+    # what's available are built from (the same discipline
+    # Native.pretokenizer_names / PretokenizerType::NAMES applies to
+    # pretokenizer scheme names).
+    NAMES = REGISTRY.keys.freeze
+
+    # Encodings known by name but deliberately not packaged, keyed to the
+    # reason a caller asking for one by name deserves to hear.
+    UNPACKABLE_REASONS = {
+      "p50k_base" => "its ranks are not dense (50256 is left free for <|endoftext|>), " \
+        "and the rank loader rejects non-dense ranks",
+      "p50k_edit" => "it loads the same p50k_base.tiktoken ranks, which are not dense " \
+        "(50256 is left free for <|endoftext|>), and the rank loader rejects non-dense ranks"
+    }.freeze
+    private_constant :UNPACKABLE_REASONS
+
+    class << self
+      # The {rank_file:, pretokenizer:, special_tokens:} registered for a
+      # packaged encoding name, or nil.
+      def [](name)
+        REGISTRY[name]
+      end
+
+      # Why `name` can't be packaged, or nil when there's no reason on
+      # record (it's either packaged, or simply not one gigatoken knows of).
+      def unpackable_reason(name)
+        UNPACKABLE_REASONS[name]
+      end
+    end
+  end
+end

data/lib/gigatoken/tokenizer.rb

@@ -7,9 +7,6 @@ module Gigatoken
   # over a native `Gigatoken::Native::BPETokenizer` or
   # `Gigatoken::Native::SentencePieceTokenizer`.
   class Tokenizer
-    TIKTOKEN_ENDOFTEXT = "<|endoftext|>"
-    private_constant :TIKTOKEN_ENDOFTEXT
-
     FILE_SOURCE_CLASSES = [Native::TextFileSource, Native::JsonlFileSource, Native::ParquetFileSource].freeze
     private_constant :FILE_SOURCE_CLASSES
 
@@ -27,10 +24,27 @@ module Gigatoken
       from_json(File.binread(path))
     end
 
-    # Load from a .tiktoken mergeable-ranks file.
-    def self.from_tiktoken(path)
-      native = Native::BPETokenizer.from_tiktoken(path.to_s)
-      new(native, special_tokens: {TIKTOKEN_ENDOFTEXT => native.vocab_size - 1})
+    # Load from a .tiktoken mergeable-ranks file. The file carries neither a
+    # pretokenization scheme nor special tokens — nothing is guessed here,
+    # so `pretokenizer:` is required: one of the schemes gigatoken ships
+    # (see Native.pretokenizer_names, e.g. "gpt2"/"r50k", "gpt4"/"cl100k",
+    # "o200k", "qwen2", "qwen35", "olmo3", "deepseek_v3", "nemotron", "kimi").
+    # `special_tokens:` maps token content to id (none by default).
+    def self.from_tiktoken(path, pretokenizer:, special_tokens: {})
+      native = Native::BPETokenizer.from_tiktoken(path.to_s, pretokenizer, special_tokens)
+      new(native, special_tokens: special_tokens)
+    end
+
+    # Load one of the tiktoken encodings gigatoken vendors ranks for, by
+    # name — see Gigatoken::Encodings::NAMES — entirely from the vendored
+    # files: no network, no writable cache.
+    def self.from_encoding(name)
+      encoding = Encodings[name]
+      return from_tiktoken(encoding[:rank_file], pretokenizer: encoding[:pretokenizer], special_tokens: encoding[:special_tokens]) if encoding
+
+      reason = Encodings.unpackable_reason(name)
+      detail = reason ? " — #{reason}" : ""
+      raise Error, "#{name.inspect}: not a packaged encoding#{detail} (packaged encodings: #{Encodings::NAMES.join(", ")})"
     end
 
     # Load tokenizer.json from HuggingFace Hub repo `repo_id` at `revision`
@@ -41,12 +55,29 @@ module Gigatoken
 
     # Load from any of the supported source shapes: an existing file or
     # directory path (a tokenizer.json, or a directory containing one), a
-    # .tiktoken vocabulary file, or a HuggingFace Hub repo id like
-    # "openai-community/gpt2".
-    def self.load(source, revision: "main", hub: Hub.new)
+    # .tiktoken vocabulary file, a packaged encoding name (see
+    # Gigatoken::Encodings::NAMES, e.g. "cl100k_base"), or a HuggingFace Hub
+    # repo id like "openai-community/gpt2". A .tiktoken file carries no
+    # pretokenizer scheme of its own, so one must be named explicitly via
+    # `pretokenizer:` — nothing here is guessed. Packaged encoding names are
+    # checked before the Hub-repo-id shape: a bare name like "o200k_base" is
+    # also shaped like a legacy repo id, and must resolve locally rather
+    # than reach the network. Names the registry knows but doesn't package
+    # (see Encodings.unpackable_reason, e.g. "p50k_base") are intercepted
+    # here too, raising the same explanation from_encoding gives rather than
+    # reaching the Hub — but only those; an unrecognized bare name like
+    # "gpt2" still dispatches to the Hub.
+    def self.load(source, pretokenizer: nil, special_tokens: {}, revision: "main", hub: Hub.new)
       source = source.to_s
-      return from_tiktoken(source) if source.end_with?(".tiktoken")
+      if source.end_with?(".tiktoken")
+        unless pretokenizer
+          raise Error, "#{source.inspect}: a .tiktoken file carries no pretokenizer scheme of its own — " \
+            "pass pretokenizer: (one of #{Native.pretokenizer_names.join(", ")})"
+        end
+        return from_tiktoken(source, pretokenizer: pretokenizer, special_tokens: special_tokens)
+      end
       return from_file(source) if File.exist?(source)
+      return from_encoding(source) if Encodings::NAMES.include?(source) || Encodings.unpackable_reason(source)
       return from_hub(source, revision: revision, hub: hub) if Hub.looks_like_repo_id?(source)
 
       raise Error, "#{source.inspect}: no such file or directory, not a .tiktoken path, and doesn't look like a HuggingFace Hub repo id"
@@ -112,6 +143,12 @@ module Gigatoken
       @native.merges
     end
 
+    # Cached pretoken/unit entries on this tokenizer's single-document
+    # encode path — see Gigatoken.max_cache_bytes.
+    def cache_entries
+      @native.cache_entries
+    end
+
     attr_reader :special_tokens
   end
 end

data/lib/gigatoken/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gigatoken
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/gigatoken.rb

@@ -7,6 +7,20 @@ module Gigatoken
   # extension — never a raw Rust panic across the Ruby boundary.
   class Error < StandardError; end
 
+  class << self
+    # The process-global encode-cache budget in bytes per worker (a parallel
+    # batch encode may use up to workers x budget), applied to tokenizers of
+    # either backend constructed *afterward* — changing it never affects an
+    # already-built Tokenizer. nil means unbounded. Default: 512 MiB.
+    def max_cache_bytes
+      Native.get_max_cache_bytes
+    end
+
+    def max_cache_bytes=(bytes)
+      Native.set_max_cache_bytes(bytes)
+    end
+  end
+
   NATIVE_EXTENSIONS = %w[.bundle .so .rb].freeze
 
   # Precompiled native gems ship per-ABI subdirs (`gigatoken/4.0/...`),
@@ -26,4 +40,5 @@ require native
 
 require_relative "gigatoken/hub"
 require_relative "gigatoken/packed_result"
+require_relative "gigatoken/encodings"
 require_relative "gigatoken/tokenizer"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gigatoken
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: aarch64-linux
 authors:
 - Eric Jacobs
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-07-24 00:00:00.000000000 Z
+date: 2026-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -72,6 +72,11 @@ files:
 - lib/gigatoken/cli/bench.rb
 - lib/gigatoken/cli/support.rb
 - lib/gigatoken/cli/validate.rb
+- lib/gigatoken/encodings.rb
+- lib/gigatoken/encodings/PROVENANCE.md
+- lib/gigatoken/encodings/cl100k_base.tiktoken
+- lib/gigatoken/encodings/o200k_base.tiktoken
+- lib/gigatoken/encodings/r50k_base.tiktoken
 - lib/gigatoken/hub.rb
 - lib/gigatoken/packed_result.rb
 - lib/gigatoken/tokenizer.rb
@@ -82,6 +87,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/jetpks/gigatoken-rb/issues
+  changelog_uri: https://github.com/jetpks/gigatoken-rb/blob/main/CHANGELOG.md
   homepage_uri: https://github.com/jetpks/gigatoken-rb
   source_code_uri: https://github.com/jetpks/gigatoken-rb
 post_install_message: