gigatoken 0.3.0 → 0.5.0

data/lib/gigatoken/cli/bench.rb

@@ -20,30 +20,36 @@ module Gigatoken
       option :pretokenizer, desc: "pretokenizer scheme, required when TOKENIZER is a .tiktoken file (one of #{Native.pretokenizer_names.join(", ")}); ignored otherwise"
 
       def call(tokenizer:, files:, doc_separator: nil, limit_bytes: "none", parallel: true, packed: false, pretokenizer: nil, **)
+        Support.check_usage!(files, doc_separator)
         limit = Support.parse_size(limit_bytes)
         out.puts "#{label("cpu")}: #{Support.cpu_info}"
 
         gt_tokenizer = Support.load_tokenizer(tokenizer, pretokenizer: pretokenizer)
 
+        # Only the batch path materializes the documents; the native paths
+        # leave this nil and count their bytes off the clock below.
+        docs = nil
         start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         if packed
           encoded = gt_tokenizer.encode_files(Support.text_file_source(files, doc_separator), parallel: parallel, packed: true)
-          n_bytes = files.sum { |file| File.size(file) }
           n_tokens = encoded.token_count
         elsif parallel
           docs = Support.subset_docs(Support.split_docs(files, doc_separator), limit)
           encoded = gt_tokenizer.encode_batch(docs)
-          n_bytes = docs.sum(&:bytesize)
           n_tokens = encoded.sum(&:length)
         else
           encoded = gt_tokenizer.encode_files(Support.text_file_source(files, doc_separator), parallel: false)
-          n_bytes = files.sum { |file| File.size(file) }
           n_tokens = encoded.sum(&:length)
         end
         seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
 
+        # The throughput is over the bytes encoded, which for a compressed
+        # file are its decompressed bytes. Counting those means reading the
+        # file, so the native paths count off the clock; the batch path
+        # already holds the documents it encoded.
+        n_bytes = docs ? docs.sum(&:bytesize) : Support.input_bytesize(files)
         out.puts report("gigatoken", seconds, n_bytes, n_tokens)
-      rescue Gigatoken::Error => e
+      rescue Gigatoken::Error, SystemCallError => e
         err.puts "error: #{e.message}"
         exit(1)
       end

data/lib/gigatoken/cli/support.rb

@@ -4,16 +4,29 @@ require "etc"
 
 module Gigatoken
   module CLI
-    # Helpers shared by the bench and validate commands: tokenizer loading,
-    # byte-size parsing, document splitting, and CPU identification.
+    # Helpers shared by the bench and validate commands: argument checking,
+    # tokenizer loading, byte-size parsing, document splitting, and CPU
+    # identification.
     module Support
       SIZE_UNITS = {"" => 1, "k" => 10**3, "m" => 10**6, "g" => 10**9, "t" => 10**12}.freeze
       private_constant :SIZE_UNITS
 
-      SIZE_PATTERN = /\A\s*(\d+(?:\.\d+)?)\s*([kmgt]?)i?b?\s*\z/i
+      # "KiB" and friends are binary, as everywhere else that spells the i.
+      BINARY_SIZE_UNITS = {"" => 1, "k" => 2**10, "m" => 2**20, "g" => 2**30, "t" => 2**40}.freeze
+      private_constant :BINARY_SIZE_UNITS
+
+      SIZE_PATTERN = /\A\s*(\d+(?:\.\d+)?)\s*([kmgt]?)(i?)b?\s*\z/i
       private_constant :SIZE_PATTERN
 
       class << self
+        # The argument shapes dry-cli itself lets through: FILES declared
+        # `required: true` still arrives empty, and an empty separator would
+        # split every byte into its own document.
+        def check_usage!(files, separator)
+          raise Gigatoken::Error, "FILES is required: name at least one file to encode" if files.empty?
+          raise Gigatoken::Error, "--doc-separator cannot be empty" if separator == ""
+        end
+
         # Load TOKENIZER: a tokenizer.json path/directory, a packaged
         # tiktoken encoding name, a HuggingFace repo id, or a .tiktoken file
         # — see Gigatoken::Tokenizer.load. `pretokenizer:` is forwarded
@@ -23,15 +36,17 @@ module Gigatoken
           Gigatoken::Tokenizer.load(spec, pretokenizer: pretokenizer)
         end
 
-        # Parse a decimal byte size like "100MB", "2.5GB", or "1000000";
-        # "none"/"unlimited" means no limit.
+        # Parse a byte size like "100MB", "2.5GB", "64KiB" or "1000000" —
+        # decimal units unless the prefix spells the i, which makes it
+        # binary; "none"/"unlimited" means no limit.
         def parse_size(text)
           return nil if ["none", "unlimited"].include?(text.strip.downcase)
 
           match = SIZE_PATTERN.match(text)
           raise Gigatoken::Error, "cannot parse size #{text.inspect}; expected something like 100MB (or 'none')" unless match
 
-          (match[1].to_f * SIZE_UNITS.fetch(match[2].downcase)).to_i
+          units = match[3].empty? ? SIZE_UNITS : BINARY_SIZE_UNITS
+          (match[1].to_f * units.fetch(match[2].downcase)).to_i
         end
 
         # A Native::TextFileSource for FILES, split on `separator` when
@@ -42,15 +57,23 @@ module Gigatoken
 
         # Whole files as raw bytes, one document per file, or (with a
         # separator) the separator-split pieces of each file in order, empty
-        # documents skipped.
+        # documents skipped. Compressed files are decompressed first, as the
+        # native file sources do, so both paths split the same bytes.
         def split_docs(files, separator)
-          raws = files.map { |file| File.binread(file.to_s) }
+          raws = files.map { |file| read_decompressed(file.to_s) }
           return raws if separator.nil?
 
           sep = separator.b
           raws.flat_map { |raw| raw.split(sep).reject(&:empty?) }
         end
 
+        # The bytes the tokenizer actually sees for FILES — a compressed
+        # file's decompressed size, not its size on disk — so throughput is
+        # reported over the input that was tokenized.
+        def input_bytesize(files)
+          files.sum { |file| read_decompressed(file.to_s).bytesize }
+        end
+
         # The prefix of `docs` totalling at most `limit_bytes`, byte-
         # truncating the final document to fill the budget. Unlike a
         # text-comparison tool, gigatoken encodes raw bytes and does not
@@ -90,6 +113,15 @@ module Gigatoken
 
         private
 
+        # One file's bytes, decompressed. `Native.read_input` is the core's
+        # own decoder — the one the native file sources load through — so
+        # `.gz`, `.zst`/`.zstd` and plain files are detected and read here
+        # exactly as they are on the other side of `validate`, and an
+        # unreadable file raises Gigatoken::Error naming it.
+        def read_decompressed(path)
+          Gigatoken::Native.read_input(path)
+        end
+
         def darwin_cpu_info
           name = sysctl("machdep.cpu.brand_string")
           [name, sysctl_int("hw.physicalcpu"), sysctl_int("hw.packages")]

data/lib/gigatoken/cli/validate.rb

@@ -20,6 +20,7 @@ module Gigatoken
       option :pretokenizer, desc: "pretokenizer scheme, required when TOKENIZER is a .tiktoken file (one of #{Native.pretokenizer_names.join(", ")}); ignored otherwise"
 
       def call(tokenizer:, files:, doc_separator: nil, pretokenizer: nil, **)
+        Support.check_usage!(files, doc_separator)
         gt_tokenizer = Support.load_tokenizer(tokenizer, pretokenizer: pretokenizer)
 
         via_files = gt_tokenizer.encode_files(Support.text_file_source(files, doc_separator))
@@ -38,7 +39,7 @@ module Gigatoken
         end
 
         out.puts "validation OK: #{via_files.length} documents match"
-      rescue Gigatoken::Error => e
+      rescue Gigatoken::Error, SystemCallError => e
         err.puts "error: #{e.message}"
         exit(1)
       end

data/lib/gigatoken/encodings/PROVENANCE.md

@@ -1,4 +1,4 @@
-# Vendored tiktoken encodings — provenance
+# Vendored encodings — provenance
 
 The `.tiktoken` files in this directory are OpenAI's published BPE mergeable-rank
 tables, vendored verbatim so `gigatoken` can resolve these encodings by name with
@@ -9,6 +9,10 @@ carries **mergeable ranks only** — the pretokenizer split regex and the specia
 tokens belong to the encoding's *definition*, not to the file, and live in code
 (see the table below).
 
+The three `.json.gz` files are HuggingFace `tokenizer.json` files, not tiktoken
+tables; they are covered in [their own section](#huggingface-tokenizerjson-files)
+at the end.
+
 They live under `lib/` rather than a top-level `data/` directory because this
 repo's `.gitignore` ignores `/data/` ("downloaded test data"); these are shipped
 gem payload, not test fixtures.
@@ -108,3 +112,111 @@ The `tiktoken` project and its published encoding files are MIT licensed,
 Copyright (c) 2022 OpenAI, Shantanu Jain. See
 <https://github.com/openai/tiktoken/blob/main/LICENSE>. The files are vendored
 here unmodified.
+
+## HuggingFace `tokenizer.json` files
+
+`qwen35.json.gz`, `qwen38.json.gz` and `muse_spark.json.gz` are HuggingFace
+`tokenizer.json` files, vendored byte-for-byte and gzipped. Unlike a
+`.tiktoken` table, a `tokenizer.json` carries the encoding's whole definition
+— vocabulary, merges, normalizer, pretokenizer, added tokens — so the registry
+records only the file, and `Tokenizer.from_encoding` loads it with
+`Tokenizer.from_json`, reading the special tokens from the file's
+`added_tokens` (those marked `"special": true`) exactly as `from_json` does.
+
+### Files
+
+| File | Decompressed bytes | sha256 (decompressed) |
+|------|-------------------:|-----------------------|
+| `qwen35.json.gz`     | 12,807,982 | `5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42` |
+| `qwen38.json.gz`     | 12,809,320 | `0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3` |
+| `muse_spark.json.gz` | 28,129,897 | `c9dbee66967b58f31a7c27f723c3760da3526ccd0427578e8905b0abb0031c4d` |
+
+### Source
+
+Retrieved **2026-10-07** over HTTPS from the Hub's `resolve` endpoint, each at
+a pinned commit:
+
+| Encoding | Repo | Revision | URL |
+|---|---|---|---|
+| `qwen35`     | `Qwen/Qwen3.5-27B`             | `fc05daec18b0a78c049392ed2e771dde82bdf654` | <https://huggingface.co/Qwen/Qwen3.5-27B/resolve/fc05daec18b0a78c049392ed2e771dde82bdf654/tokenizer.json> |
+| `qwen38`     | `Qwen/Qwen3.8-27B`             | `1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | <https://huggingface.co/Qwen/Qwen3.8-27B/resolve/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/tokenizer.json> |
+| `muse_spark` | `meta-models/Muse-Glimmer-30B` | `a4e59da52a7bc87ae7251dd5545c0dd437c44b68` | <https://huggingface.co/meta-models/Muse-Glimmer-30B/resolve/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/tokenizer.json> |
+
+**`qwen35` is shared across releases.** The Hub's tree API
+(`https://huggingface.co/api/models/<repo>?blobs=true`) reports the identical
+LFS sha256 `5f9e4d49…` for `tokenizer.json` in `Qwen/Qwen3.5-{0.8B, 2B, 4B, 9B,
+27B, 35B-A3B, 122B-A10B, 397B-A17B}` and `Qwen/Qwen3.6-{27B, 35B-A3B}`
+(queried 2026-10-07), so one file serves Qwen 3.5 and 3.6.
+`Qwen/Qwen3.6-Plus` is gated and was not checked.
+
+**`qwen38` ships its own file**, identical (`0997f410…`) across
+`Qwen/Qwen3.8-{27B, Flash-Next, 2.4T-A95B}` and their `-FP8` twins. It differs
+from `qwen35` only in seven more `"special": true` added tokens, ids
+248070–248076: `<|audio_start|>`, `<|audio_end|>`, `<tts_pad>`,
+`<tts_text_bos>`, `<tts_text_eod>`, `<tts_text_bos_single>`, `<|audio_pad|>`.
+The vocabulary, merges, normalizer, pretokenizer, decoder and post-processor
+are identical, so ordinary text encodes the same under both; only those seven
+literals, `vocab_size` and `special_tokens` differ. It is vendored as a second
+file rather than derived from the first because the engine reads added tokens
+from the JSON, and a JSON-backed tokenizer has no way to add specials after
+the fact. `Qwen3.8-27B` is the repo cited; `Flash-Next` and `2.4T-A95B` carry an
+"other" licence on the *model*, but their tokenizer file is byte-identical to
+the Apache-2.0 repo's.
+
+**`muse_spark` is the Muse-Glimmer-30B file.** Meta publishes no standalone
+Muse Spark tokenizer (a Hub search over `meta-models` finds none, and Meta's
+model-API docs at <https://dev.meta.ai/docs/muse-glimmer> point at none). The
+Muse-Glimmer-30B model card states "Tokenizer: 200,000 BPE tokens + 2,048
+special tokens" (vocabulary 202,048), trained from Muse Spark's outputs, and
+this exact repo and revision is the one the reference application pins. As of
+2026-10-07 that revision is also `main`.
+
+### Authenticity
+
+Verified three independent ways at retrieval time:
+
+1. **Transport** — HTTPS directly from `huggingface.co`, at the pinned
+   commit above.
+2. **Publisher checksum** — each file's sha256 above matches the LFS sha256 the
+   Hub's tree API reports for `tokenizer.json` at that revision, and so does
+   its size. Two independent channels agree on the bytes.
+3. **Behavioral, against the reference implementation** — loaded through
+   `Tokenizer.from_encoding`, every token id matched HuggingFace
+   [`tokenizers`](https://rubygems.org/gems/tokenizers) 0.7.0 loading the same
+   decompressed file with `add_special_tokens: false`, across this repo's own
+   source and docs (`spec/gigatoken/differential_spec.rb`). Resulting
+   `vocab_size`: 248070 / 248077 / 202048; `special_tokens.size`: 14 / 21 / 2048.
+
+### Compression
+
+Each file was compressed with `gzip -9 -n` (no name or timestamp in the gzip
+header, so the bytes are reproducible), and `gunzip -c` of it reproduces the
+published file byte-for-byte — the sha256s above are of the decompressed
+bytes. Compressed sizes: 3,514,616 (`qwen35`), 3,514,699 (`qwen38`), 4,475,666
+(`muse_spark`).
+
+### What gigatoken does not take from the file
+
+- **The post-processor.** gigatoken applies none. `muse_spark`'s is a
+  `TemplateProcessing` that prepends `<|begin_of_text|>`; `Tokenizer#encode`
+  does not, matching `tokenizers`' `add_special_tokens: false`. The Qwen files'
+  is a `ByteLevel` post-processor, a no-op on ids.
+- **Non-special added tokens in `special_tokens`.** Qwen's 12 non-special
+  added tokens (`<tool_call>`, `<think>`, the `<|fim_*|>` tokens,
+  `<|repo_name|>`, `<|file_sep|>`, `<tool_response>` and so on) are matched
+  atomically by the engine, like the special ones, but only `"special": true`
+  tokens appear in `Tokenizer#special_tokens` — 14 for `qwen35`, 21 for
+  `qwen38`.
+- **`tokenizer_config.json`.** Not vendored; gigatoken reads only
+  `tokenizer.json`.
+
+### Licence
+
+All three repos are Apache-2.0, each with its own `LICENSE` at the pinned
+revision:
+
+- <https://huggingface.co/Qwen/Qwen3.5-27B/blob/fc05daec18b0a78c049392ed2e771dde82bdf654/LICENSE>
+- <https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/LICENSE>
+- <https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/a4e59da52a7bc87ae7251dd5545c0dd437c44b68/LICENSE>
+
+The files are vendored here unmodified (gzipped).

data/lib/gigatoken/encodings.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 module Gigatoken
-  # The tiktoken encodings gigatoken vendors ranks for, and the pieces a
-  # .tiktoken file doesn't carry: its pretokenizer scheme and special-token
-  # table (see Tokenizer.from_tiktoken and lib/gigatoken/encodings/
-  # PROVENANCE.md, the source of truth this is transcribed from).
+  # The encodings gigatoken vendors a file for. For the tiktoken ones, the
+  # pieces a .tiktoken file doesn't carry: its pretokenizer scheme and
+  # special-token table (see Tokenizer.from_tiktoken and lib/gigatoken/
+  # encodings/PROVENANCE.md, the source of truth this is transcribed from).
+  # The HuggingFace ones are whole tokenizer.json files, which carry theirs.
   module Encodings
     DATA_DIR = File.expand_path("encodings", __dir__)
     private_constant :DATA_DIR
@@ -40,6 +41,10 @@ module Gigatoken
     HARMONY_RESERVED_TAIL = (200013..201087).to_h { |id| ["<|reserved_#{id}|>", id] }.freeze
     private_constant :HARMONY_RESERVED_TAIL
 
+    # Deep-frozen: entries, their `special_tokens` tables and the `rank_file` /
+    # `json_file` paths. `Tokenizer#special_tokens` hands the registry's own
+    # Hash back to callers, so anything less lets one caller's poke rewrite
+    # what every later `from_encoding` in the process loads.
     REGISTRY = {
       "r50k_base" => {
         rank_file: File.join(DATA_DIR, "r50k_base.tiktoken"),
@@ -66,8 +71,22 @@ module Gigatoken
         rank_file: File.join(DATA_DIR, "o200k_base.tiktoken"),
         pretokenizer: "o200k",
         special_tokens: HARMONY_HEAD_TOKENS.merge(HARMONY_RESERVED_TAIL).freeze
-      }
-    }.freeze
+      },
+      # The three below are gzipped HuggingFace tokenizer.json files, vendored
+      # verbatim (see PROVENANCE.md for repos, revisions and hashes). Nothing
+      # else is registered: the pretokenizer and the special tokens are the
+      # file's own, read at load as Tokenizer.from_json reads them.
+      #
+      # Qwen 3.5 and 3.6 share this one file.
+      "qwen35" => {json_file: File.join(DATA_DIR, "qwen35.json.gz")},
+      # The qwen35 file plus seven audio/TTS special tokens — vendored
+      # separately because a JSON-backed tokenizer takes its added tokens
+      # from the file.
+      "qwen38" => {json_file: File.join(DATA_DIR, "qwen38.json.gz")},
+      # meta-models/Muse-Glimmer-30B's file: Meta publishes no standalone
+      # Muse Spark tokenizer.
+      "muse_spark" => {json_file: File.join(DATA_DIR, "muse_spark.json.gz")}
+    }.each_value { |encoding| encoding.each_value(&:freeze).freeze }.freeze
     private_constant :REGISTRY
 
     # The packaged encoding names — the single source error messages naming
@@ -87,16 +106,17 @@ module Gigatoken
     private_constant :UNPACKABLE_REASONS
 
     class << self
-      # The {rank_file:, pretokenizer:, special_tokens:} registered for a
-      # packaged encoding name, or nil.
+      # The {rank_file:, pretokenizer:, special_tokens:} (tiktoken) or
+      # {json_file:} (HuggingFace) registered for a packaged encoding name, or
+      # nil. Names are Strings or Symbols, as Tokenizer.load accepts both.
       def [](name)
-        REGISTRY[name]
+        REGISTRY[name.to_s]
       end
 
       # Why `name` can't be packaged, or nil when there's no reason on
       # record (it's either packaged, or simply not one gigatoken knows of).
       def unpackable_reason(name)
-        UNPACKABLE_REASONS[name]
+        UNPACKABLE_REASONS[name.to_s]
       end
     end
   end