woods 2.0.1 → 2.1.0

data/lib/woods/embedding/openai.rb

@@ -3,6 +3,7 @@
 require 'net/http'
 require 'json'
 require_relative 'provider'
+require_relative 'input_budget'
 
 module Woods
   module Embedding
@@ -19,18 +20,19 @@ module Woods
       class OpenAI # rubocop:disable Metrics/ClassLength
         include Interface
         include DiscardableClient
+        include VectorConfiguration
 
         ENDPOINT = URI('https://api.openai.com/v1/embeddings')
         DEFAULT_MODEL = 'text-embedding-3-small'
         DIMENSIONS = {
           'text-embedding-3-small' => 1536,
-          'text-embedding-3-large' => 3072
+          'text-embedding-3-large' => 3072,
+          'text-embedding-ada-002' => 1536
         }.freeze
         # Conservatively chunk below OpenAI's 8192-token input limit across
         # text-embedding-3-small / -3-large / ada-002. The chunker uses
-        # 8191 as its ceiling — the actual chunk size lands well
-        # below it once chars-per-token estimation and the prefix
-        # allowance are factored in (see Builder#build_chunker).
+        # 8191 as its ceiling, checked against complete input bytes for known
+        # byte-BPE models (including every context-prefix character).
         MAX_INPUT_TOKENS = 8191
 
         # The API accepts at most 2048 inputs and 300,000 total tokens per
@@ -43,10 +45,12 @@ module Woods
         # @param api_key [String] OpenAI API key
         # @param model [String] OpenAI embedding model name (default: text-embedding-3-small)
         # @param dimensions [Integer, nil] Requested output size for text-embedding-3 models
-        def initialize(api_key:, model: DEFAULT_MODEL, dimensions: nil)
+        # @param expected_dimensions [Integer, nil] Expected width; never sent to the API
+        def initialize(api_key:, model: DEFAULT_MODEL, dimensions: nil, expected_dimensions: nil)
           @api_key = api_key
           @model = model
-          @dimensions = normalize_dimensions(dimensions)
+          configure_dimensions(dimensions: dimensions, expected_dimensions: expected_dimensions)
+          validate_model_dimensions!
         end
 
         # Embed a single text string.
@@ -58,9 +62,10 @@ module Woods
         def embed(text)
           raise ArgumentError, 'embed(text) requires a non-empty string' if text.nil? || text.to_s.strip.empty?
 
+          input_budget.validate!(text)
           response = post_request(request_body(text))
           vectors = Array(response['data']).map { |item| item['embedding'] }
-          VectorValidation.validate!(vectors, expected_count: 1, provider: 'OpenAI')
+          validate_vectors!(vectors, expected_count: 1, provider: 'OpenAI')
           vectors.first
         end
 
@@ -73,16 +78,13 @@ module Woods
         # @raise [Woods::Error] if the API returns an error
         # @raise [ArgumentError] if the array is empty or any element is nil/empty
         def embed_batch(texts)
-          raise ArgumentError, 'embed_batch(texts) requires a non-empty array' if texts.nil? || texts.empty?
-          if texts.any? { |t| t.nil? || t.to_s.strip.empty? }
-            raise ArgumentError, 'embed_batch(texts) rejects nil/empty entries (OpenAI returns 400)'
-          end
+          validate_inputs!(texts)
 
           vectors = texts.each_slice(MAX_REQUEST_INPUTS).flat_map do |slice|
             response = post_request(request_body(slice))
             extract_validated_batch(response, slice.size)
           end
-          VectorValidation.validate!(vectors, expected_count: texts.size, provider: 'OpenAI')
+          validate_vectors!(vectors, expected_count: texts.size, provider: 'OpenAI')
           vectors
         end
 
@@ -93,7 +95,24 @@ module Woods
         #
         # @return [Integer] number of dimensions
         def dimensions
-          @dimensions || DIMENSIONS[@model] || embed('test').length
+          configured_dimensions || @observed_dimensions || embed('test').length
+        end
+
+        # @return [Integer, nil] expected width without a network probe
+        def configured_dimensions
+          super || DIMENSIONS[@model]
+        end
+
+        # API keys do not change embeddings and are deliberately excluded.
+        # @return [Array]
+        def cache_identity
+          [self.class.name, ENDPOINT.to_s, @model, requested_dimensions, configured_dimensions]
+        end
+
+        # Pure constructor settings; credentials are supplied by the reader's host.
+        def configuration_options
+          { model: @model, dimensions: requested_dimensions,
+            expected_dimensions: configured_dimensions || @observed_dimensions }
         end
 
         # Return the model name.
@@ -111,8 +130,25 @@ module Woods
           MAX_INPUT_TOKENS
         end
 
+        # Known cl100k byte-BPE models have a dependency-free UTF-8 byte bound.
+        # Unknown future/custom model names retain honest sizing estimates;
+        # the API remains authoritative and rejects over-limit input.
+        def input_budget
+          @input_budget ||= InputBudget.new(limit: max_input_tokens, model: @model,
+                                            method: DIMENSIONS.key?(@model) ? 'utf8_bytes_bound' : 'estimate')
+        end
+
         private
 
+        def validate_inputs!(texts)
+          raise ArgumentError, 'embed_batch(texts) requires a non-empty array' if texts.nil? || texts.empty?
+          if texts.any? { |t| t.nil? || t.to_s.strip.empty? }
+            raise ArgumentError, 'embed_batch(texts) rejects nil/empty entries (OpenAI returns 400)'
+          end
+
+          texts.each { |text| input_budget.validate!(text) }
+        end
+
         # Validate the batch response's shape/cardinality/indexes and
         # return vectors reordered to match the input order.
         #
@@ -121,7 +157,7 @@ module Woods
         # @return [Array<Array<Float>>]
         def extract_validated_batch(response, expected_count)
           data = Array(response['data'])
-          VectorValidation.validate!(
+          validate_vectors!(
             data.map { |item| item['embedding'] },
             expected_count: expected_count,
             provider: 'OpenAI',
@@ -132,17 +168,31 @@ module Woods
 
         def request_body(input)
           { model: @model, input: input }.tap do |body|
-            body[:dimensions] = @dimensions if @dimensions
+            body[:dimensions] = requested_dimensions if requested_dimensions
           end
         end
 
-        def normalize_dimensions(value)
-          return if value.nil?
+        def validate_model_dimensions!
+          native = DIMENSIONS[@model]
+          return unless native
+
+          width = configured_dimensions
+          raise ArgumentError, "#{@model} supports at most #{native} dimensions" if width > native
+
+          return validate_ada_dimensions!(width, native) if @model == 'text-embedding-ada-002'
+
+          return unless !requested_dimensions && width != native
+
+          raise ArgumentError, "expected dimensions #{width} do not match #{@model} output #{native}; " \
+                               'set dimensions explicitly to request a reduction'
+        end
 
-          dimensions = Integer(value)
-          raise ArgumentError, "dimensions must be positive, got #{value.inspect}" unless dimensions.positive?
+        def validate_ada_dimensions!(width, native)
+          raise ArgumentError, "#{@model} requires dimensions #{native}" unless width == native
 
-          dimensions
+          # ada has a fixed width and does not accept the dimensions parameter,
+          # even when a caller explicitly declares the matching native width.
+          @requested_dimensions = nil
         end
 
         # Cap interpolated response bodies so misconfigured API errors

data/lib/woods/embedding/provider.rb

@@ -2,6 +2,8 @@
 
 require 'net/http'
 require 'json'
+require_relative 'vector_configuration'
+require_relative 'input_budget'
 
 module Woods
   # Standalone-load guard — keeps `require 'woods/embedding/provider'`
@@ -158,7 +160,7 @@ module Woods
         #   check is skipped.
         # @raise [InvalidEmbeddingResponse] on any violation
         # @return [void]
-        def validate!(vectors, expected_count:, provider:, indexes: nil)
+        def validate!(vectors, expected_count:, provider:, indexes: nil, expected_dimensions: nil)
           fail_with = lambda do |msg|
             raise InvalidEmbeddingResponse.new(msg, provider: provider, batch_size: expected_count)
           end
@@ -169,7 +171,7 @@ module Woods
 
           validate_indexes!(indexes, expected_count, fail_with) if indexes
 
-          validate_vector_shapes!(vectors, fail_with)
+          validate_vector_shapes!(vectors, fail_with, expected_dimensions)
         end
 
         # @api private
@@ -187,8 +189,7 @@ module Woods
         private_class_method :validate_indexes!
 
         # @api private
-        def validate_vector_shapes!(vectors, fail_with)
-          dimension = nil
+        def validate_vector_shapes!(vectors, fail_with, dimension)
           vectors.each_with_index do |vector, i|
             check_vector_shape!(vector, i, fail_with)
             dimension ||= vector.size
@@ -220,9 +221,10 @@ module Woods
       #   provider = Woods::Embedding::Provider::Ollama.new
       #   vector = provider.embed("class User < ApplicationRecord; end")
       #   vectors = provider.embed_batch(["text1", "text2"])
-      class Ollama
+      class Ollama # rubocop:disable Metrics/ClassLength
         include Interface
         include DiscardableClient
+        include VectorConfiguration
 
         DEFAULT_MODEL = 'nomic-embed-text'
         DEFAULT_HOST = 'http://localhost:11434'
@@ -270,14 +272,15 @@ module Woods
         #   unknown models. Set explicitly only if running a model with a
         #   known-larger native context that isn't in the registry yet.
         # @param dimensions [Integer, nil] Requested output vector size.
+        # @param expected_dimensions [Integer, nil] Expected width; never sent to the API.
         # @param read_timeout [Integer] HTTP read timeout in seconds.
         #   Bump this for slow / cold-start hosts or very large batches.
-        def initialize(model: DEFAULT_MODEL, host: DEFAULT_HOST, num_ctx: nil,
-                       dimensions: nil, read_timeout: DEFAULT_READ_TIMEOUT)
+        def initialize(model: DEFAULT_MODEL, host: DEFAULT_HOST, num_ctx: nil, # rubocop:disable Metrics/ParameterLists
+                       dimensions: nil, expected_dimensions: nil, read_timeout: DEFAULT_READ_TIMEOUT)
           @model = model
           @host = host
           @num_ctx = num_ctx || MODEL_CONTEXT_LENGTHS.fetch(model, FALLBACK_NUM_CTX)
-          @dimensions = normalize_dimensions(dimensions)
+          configure_dimensions(dimensions: dimensions, expected_dimensions: expected_dimensions)
           @read_timeout = read_timeout
           @uri = URI("#{host}/api/embed")
         end
@@ -293,7 +296,7 @@ module Woods
 
           response = post_request(build_body(text))
           vectors = Array(response['embeddings'])
-          VectorValidation.validate!(vectors, expected_count: 1, provider: 'Ollama')
+          validate_vectors!(vectors, expected_count: 1, provider: 'Ollama')
           vectors.first
         end
 
@@ -311,7 +314,7 @@ module Woods
 
           response = post_request(build_body(texts))
           vectors = Array(response['embeddings'])
-          VectorValidation.validate!(vectors, expected_count: texts.size, provider: 'Ollama')
+          validate_vectors!(vectors, expected_count: texts.size, provider: 'Ollama')
           vectors
         end
 
@@ -321,7 +324,20 @@ module Woods
         #
         # @return [Integer] number of dimensions
         def dimensions
-          @dimensions ||= embed('test').length
+          configured_dimensions || @observed_dimensions || embed('test').length
+        end
+
+        # Pure configuration identity; probes never change request semantics or keys.
+        # @return [Array]
+        def cache_identity
+          [self.class.name, @host, @model, @num_ctx, requested_dimensions, configured_dimensions, 'truncate:false']
+        end
+
+        # Pure constructor settings for ResolvedConfig's allowlisted serialization.
+        # The snapshot layer checks the endpoint before persisting it.
+        def configuration_options
+          { model: @model, host: @host, num_ctx: @num_ctx, read_timeout: @read_timeout,
+            dimensions: requested_dimensions, expected_dimensions: configured_dimensions || @observed_dimensions }
         end
 
         # Return the model name.
@@ -341,17 +357,14 @@ module Woods
           @num_ctx
         end
 
-        private
-
-        def normalize_dimensions(value)
-          return if value.nil?
-
-          dimensions = Integer(value)
-          raise ArgumentError, "dimensions must be positive, got #{value.inspect}" unless dimensions.positive?
-
-          dimensions
+        # A model-specific tokenizer is not available locally. This is a sizing
+        # estimate; truncate:false makes the server reject an actual overflow.
+        def input_budget
+          @input_budget ||= InputBudget.new(limit: max_input_tokens, model: @model)
         end
 
+        private
+
         # Cap interpolated response bodies so misconfigured Ollama responses
         # (e.g. proxied HTML error pages) don't unbounded-leak into logs or
         # re-raised error messages.
@@ -365,12 +378,11 @@ module Woods
           s.length > 500 ? "#{s[0, 500]}... [truncated]" : s
         end
 
-        # Build the JSON body for an `/api/embed` call. Adds `options.num_ctx`
-        # when configured — without it, Ollama silently truncates to 2048
-        # tokens and returns 400 when the input exceeds that default.
+        # Request the configured context and explicitly refuse server truncation.
+        # Unknown model tokenization remains server-authoritative.
         def build_body(input)
-          body = { model: @model, input: input }
-          body[:dimensions] = @dimensions if @dimensions
+          body = { model: @model, input: input, truncate: false }
+          body[:dimensions] = requested_dimensions if requested_dimensions
           body[:options] = { num_ctx: @num_ctx } if @num_ctx
           body
         end

data/lib/woods/embedding/text_preparer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative '../token_utils'
+require_relative 'input_budget'
+require_relative '../chunking/contributor_chunks'
 
 module Woods
   module Embedding
@@ -12,8 +14,8 @@ module Woods
     #   file: ...
     #   dependencies: dep1, dep2, ...
     #
-    # Handles token limit enforcement by truncating text that exceeds the
-    # embedding model's context window.
+    # Refuses oversized direct inputs; the indexing path splits complete inputs
+    # without truncation before submitting them to the provider.
     #
     # @example
     #   preparer = Woods::Embedding::TextPreparer.new(max_tokens: 8192)
@@ -29,9 +31,10 @@ module Woods
 
       # @param max_tokens [Integer] maximum token budget for prepared text
       # @param chars_per_token [Float] tokenizer-calibrated char/token ratio
-      def initialize(max_tokens: DEFAULT_MAX_TOKENS, chars_per_token: DEFAULT_CHARS_PER_TOKEN)
+      def initialize(max_tokens: DEFAULT_MAX_TOKENS, chars_per_token: DEFAULT_CHARS_PER_TOKEN, input_budget: nil)
         @max_tokens = max_tokens
         @chars_per_token = chars_per_token
+        @input_budget = input_budget || InputBudget.new(limit: max_tokens, chars_per_token: chars_per_token)
       end
 
       # @return [Float] configured chars-per-token ratio
@@ -43,15 +46,15 @@ module Woods
       # Prepare text for embedding from an ExtractedUnit.
       #
       # Builds a context prefix and appends the unit's source code (or first
-      # chunk content for chunked units). Enforces token limits via truncation.
+      # chunk content for chunked units). Refuses inputs exceeding the configured limit.
       #
       # @param unit [Woods::ExtractedUnit] the unit to prepare
       # @return [String] context-prefixed text ready for embedding
-      def prepare(unit)
-        prefix = build_prefix(unit)
+      def prepare(unit, budget: @input_budget)
+        prefix = build_prefix(unit, unit.chunks.first)
         content = select_content(unit)
         text = "#{prefix}\n#{content}"
-        enforce_token_limit(text)
+        budget.validate!(text)
       end
 
       # Prepare text for each chunk of an ExtractedUnit.
@@ -62,31 +65,89 @@ module Woods
       #
       # @param unit [Woods::ExtractedUnit] the unit to prepare
       # @return [Array<String>] array of context-prefixed texts
-      def prepare_chunks(unit)
-        return [prepare(unit)] unless unit.chunks&.any?
+      def prepare_chunks(unit, budget: @input_budget)
+        Chunking::ContributorChunks.ensure!(unit)
+        return [prepare(unit, budget: budget)] unless unit.chunks&.any?
 
-        prefix = build_prefix(unit)
         unit.chunks.map do |chunk|
-          text = "#{prefix}\n#{chunk[:content]}"
-          enforce_token_limit(text)
+          text = "#{build_prefix(unit, chunk)}\n#{chunk[:content]}"
+          budget.validate!(text)
         end
       end
 
+      # Fit complete prefixed inputs without removing any source characters.
+      # Existing chunk attributes survive; embedding_slice records byte offsets
+      # relative to the original chunk for downstream physical-source attribution.
+      def prepare_for_embedding(unit, budget: @input_budget)
+        Chunking::ContributorChunks.ensure!(unit)
+        originals = unit.chunks.any? ? unit.chunks : [{ content: unit.source_code || '', chunk_type: :whole }]
+        fitted = originals.flat_map do |chunk|
+          prefix = "#{build_prefix(unit, chunk)}\n"
+          unless budget.fits?(prefix)
+            raise InputLimitError, "Embedding prefix exceeds input limit (#{budget.method}: #{budget.count(prefix)})"
+          end
+
+          fit_chunk(chunk, prefix, budget)
+        end
+        unit.chunks = fitted if unit.chunks.any? || fitted.size > 1
+        prepare_chunks(unit, budget: budget)
+      end
+
+      def preparation_identity
+        { 'class' => self.class.name, 'version' => 2, 'budget' => @input_budget.identity }
+      end
+
       private
 
+      def fit_chunk(chunk, prefix, budget)
+        content = chunk[:content].to_s
+        return [chunk] if budget.fits?(prefix + content)
+
+        offset = SourceContributors.field(chunk[:embedding_slice], :start_byte) || 0
+        initial_offset = offset
+        split_content(content, prefix, budget).map do |part|
+          first = offset
+          offset += part.bytesize
+          metadata = Chunking::ContributorChunks.slice_metadata(chunk[:metadata], content, first - initial_offset,
+                                                                offset - initial_offset)
+          chunk.merge(content: part, metadata: metadata, embedding_slice: { start_byte: first, end_byte: offset })
+        end
+      end
+
+      def split_content(content, prefix, budget)
+        return [content] if budget.fits?(prefix + content)
+        if content.length <= 1
+          raise InputLimitError, 'Embedding input limit leaves no room for one source character after the prefix'
+        end
+
+        middle = content.length / 2
+        split_content(content[0...middle], prefix, budget) + split_content(content[middle..], prefix, budget)
+      end
+
       # Build the context prefix for a unit.
       #
       # @param unit [Woods::ExtractedUnit] the unit
       # @return [String] formatted prefix lines
-      def build_prefix(unit)
+      def build_prefix(unit, chunk = nil)
         lines = []
         lines << "[#{unit.type}] #{unit.identifier}"
         lines << "namespace: #{unit.namespace}" if unit.namespace
-        lines << "file: #{unit.file_path}" if unit.file_path
+        path = chunk&.dig(:metadata, :physical_location, :file_path)
+        if SourceContributors.multiple?(unit)
+          lines.concat(contributor_prefix(unit, path))
+        elsif unit.file_path
+          lines << "file: #{unit.file_path}"
+        end
         append_dependency_line(lines, unit.dependencies)
         lines.join("\n")
       end
 
+      def contributor_prefix(unit, path)
+        return ["file: #{path}"] if path
+
+        ["primary file: #{unit.file_path}", "contributing files: #{SourceContributors.paths(unit).join(', ')}"]
+      end
+
       # Append a formatted dependency line if dependencies exist.
       #
       # @param lines [Array<String>] lines to append to
@@ -100,7 +161,7 @@ module Woods
         # reads JSON and does not symbolize dependency keys, unlike chunks).
         # Read both forms or the whole "dependencies:" prefix silently
         # vanishes from every embedded document on the indexing path.
-        dep_names = dependencies.filter_map { |d| d[:target] || d['target'] }.first(10)
+        dep_names = dependencies.filter_map { |d| d.is_a?(String) ? d : d[:target] || d['target'] }.first(10)
         lines << "dependencies: #{dep_names.join(', ')}" if dep_names.any?
       end
 
@@ -115,23 +176,6 @@ module Woods
           unit.source_code || ''
         end
       end
-
-      # Truncate text to fit within the token budget.
-      #
-      # Uses the configured `chars_per_token` ratio to estimate both the
-      # token count and the safe character cap. Truncation is a last
-      # resort — by the time text reaches here the chunker should have
-      # already split oversize units into pieces that fit.
-      #
-      # @param text [String] the text to truncate
-      # @return [String] text within token limits
-      def enforce_token_limit(text)
-        estimated = (text.length / @chars_per_token).ceil
-        return text if estimated <= @max_tokens
-
-        max_chars = (@max_tokens * @chars_per_token).floor
-        text[0...max_chars]
-      end
     end
   end
 end

data/lib/woods/embedding/token_counter.rb

@@ -4,114 +4,51 @@ require 'set'
 
 module Woods
   module Embedding
-    # Exact or estimated token counts for embedding inputs.
-    #
-    # When the optional `tokenizers` gem (ankane) is installed, loads the
-    # `bert-base-uncased` WordPiece tokenizer that nomic-embed-text is
-    # built on and returns exact token counts. Otherwise falls back to a
-    # conservative chars/token ratio and warns once.
-    #
-    # Exact counting is strictly preferred for the Ollama path — Ollama
-    # v0.13.5+ stopped honouring the `truncate: true` flag on
-    # `/api/embed` (see ollama/ollama#14186), so chunks that exceed
-    # `num_ctx` return a 400 instead of being truncated. Client-side
-    # sizing is the only reliable option until the regression is fixed
-    # upstream, and chars/token ratios vary too widely across Rails
-    # internals to cover every case with a fixed number.
-    #
-    # @example
-    #   counter = Woods::Embedding::TokenCounter.new
-    #   counter.count("ActionController::Metal::ConditionalGet")  # => 13
+    # Counts with an explicitly supplied local tokenizer, otherwise estimates.
+    # The caller owns the tokenizer/model match. No model is downloaded here.
     class TokenCounter
-      # HuggingFace tokenizer id shared by every nomic-embed-text variant.
+      # Legacy constants/keyword remain accepted; tokenizer_id no longer loads
+      # a remote tokenizer or implies that BERT matches the configured model.
       BERT_MODEL = 'bert-base-uncased'
-
-      # Conservative floor for when the tokenizer gem isn't installed.
-      # Lower than any ratio we've observed failing in the testbed
-      # against dense Rails source. Still approximate — install
-      # `tokenizers` for exact counts.
       CONSERVATIVE_CHARS_PER_TOKEN = 1.2
 
-      # @param chars_per_token [Float] fallback ratio when the tokenizer
-      #   is unavailable
-      # @param tokenizer_id [String] HuggingFace model id passed to
-      #   `Tokenizers.from_pretrained`
-      def initialize(chars_per_token: CONSERVATIVE_CHARS_PER_TOKEN, tokenizer_id: BERT_MODEL)
+      attr_reader :chars_per_token
+
+      def initialize(chars_per_token: CONSERVATIVE_CHARS_PER_TOKEN, tokenizer_id: nil, tokenizer: nil)
+        raise ArgumentError, 'chars_per_token must be positive' unless chars_per_token.positive?
+
         @chars_per_token = chars_per_token
+        @tokenizer = tokenizer
         @tokenizer_id = tokenizer_id
-        @load_attempted = false
-        @load_mutex = Mutex.new
       end
 
-      # @return [Float] fallback chars-per-token ratio
-      attr_reader :chars_per_token
-
-      # Exact token count when the tokenizer is loaded, chars/token
-      # estimate otherwise.
-      #
-      # @param text [String, nil]
-      # @return [Integer]
       def count(text)
         return 0 if text.nil? || text.empty?
+        return @tokenizer.encode(text).ids.length if @tokenizer
 
-        tok = tokenizer
-        tok ? tok.encode(text).ids.length : estimate(text)
-      end
-
-      private
-
-      def estimate(text)
+        warn_once('Embedding token counts are estimates: no model-matched local tokenizer was supplied. ' \
+                  'Ollama uses truncate:false and rejects inputs beyond its actual context window.')
         (text.length / @chars_per_token).ceil
       end
 
-      # Lazy-load the tokenizer under a mutex so concurrent first-calls
-      # don't each trigger a separate download. After the first attempt
-      # (successful or not) we memoize the result and skip the load path.
-      def tokenizer
-        @load_mutex.synchronize do
-          return @tokenizer if @load_attempted
-
-          @load_attempted = true
-          @tokenizer = try_load
-        end
-      end
-
-      def try_load
-        require 'tokenizers'
-        Tokenizers.from_pretrained(@tokenizer_id)
-      rescue LoadError
-        warn_once(
-          'Exact token counting disabled: `tokenizers` gem not installed. ' \
-          "Falling back to #{@chars_per_token} chars/token estimation. " \
-          "Add `gem 'tokenizers', '~> 0.5'` to your Gemfile for exact sizing on Ollama."
-        )
-        nil
-      rescue StandardError => e
-        warn_once(
-          "Could not load tokenizer #{@tokenizer_id.inspect} " \
-          "(#{e.class}: #{e.message}). Falling back to chars/token estimate."
-        )
-        nil
+      def exact?
+        !@tokenizer.nil?
       end
 
-      # Per-process dedup so multiple TokenCounter instances (one per
-      # retriever build, plus one per chunker, plus tests) don't each
-      # spam the same fallback warning. The mutex keeps the dedup set
-      # consistent under the same concurrent-first-call pattern that
-      # the per-instance load mutex protects against.
       @warned_messages = Set.new
       @warned_mutex = Mutex.new
 
       class << self
         attr_reader :warned_messages, :warned_mutex
 
-        # Reset the per-process warning dedup. For tests only — production
-        # callers should never need to clear it.
+        # Test seam for process-wide diagnostic deduplication.
         def reset_warned!
           @warned_mutex.synchronize { @warned_messages.clear }
         end
       end
 
+      private
+
       def warn_once(message)
         full = "[woods] #{message}"
         self.class.warned_mutex.synchronize do