woods 2.0.1 → 2.1.0

data/lib/woods/embedding/indexer.rb

@@ -5,6 +5,7 @@ require 'digest'
 require 'fileutils'
 require 'set'
 
+require_relative 'input_budget'
 require_relative '../atomic_file'
 require_relative '../storage_identity'
 require_relative '../generation'
@@ -64,9 +65,10 @@ module Woods
       #   into semantically coherent chunks before embedding. +nil+ disables
       #   chunking — units go to the provider whole (useful in tests).
       # @param checkpoint_interval [Integer] Save checkpoint every N batches (default: 10)
-      # @param metadata_store [#each_entry, #bulk_load, nil] Optional metadata store.
-      #   When present alongside an in-memory vector store, both are persisted
-      #   at the end of a successful {#index_all} run.
+      # @param metadata_store [Storage::MetadataStore::Interface, nil] Optional metadata store.
+      #   Existing identities participate in snapshot-store reconciliation.
+      #   Stores with #each_entry and #bulk_load are persisted alongside vectors;
+      #   SQLite retains its records in the configured database instead.
       # @param resolved_config [Woods::ResolvedConfig, nil] Captured config for
       #   +woods.json+ — written to +output_dir+ on {#index_all} completion.
       # @param dump_retention_count [Integer] Number of completed dump directories
@@ -152,6 +154,7 @@ module Woods
         prepare_run(incremental: incremental)
         checkpoint = incremental ? load_checkpoint : {}
         units = assign_storage_identities(units, checkpoint: checkpoint)
+        preflight_inputs(units, checkpoint, incremental: incremental)
         stats = { processed: 0, skipped: 0, errors: 0 }
 
         embed_batches(units, checkpoint, stats, incremental: incremental)
@@ -170,6 +173,15 @@ module Woods
         stats
       end
 
+      # Reject deterministic input failures before metadata or durable writes.
+      def preflight_inputs(units, checkpoint, incremental:)
+        units.each { |unit| prepared_fingerprint(unit) }
+        return unless reconcilable? && @durable_ids.nil?
+        return if incremental && units.all? { |unit| checkpoint_satisfied?(unit, checkpoint) }
+
+        raise InputLimitError, 'Cannot replace embedding inputs: existing durable vector IDs could not be read'
+      end
+
       # Unambiguous existing keys stay stable. A collision uses reversible typed keys.
       def assign_storage_identities(units, checkpoint:)
         counts = units.group_by { |unit| unit['identifier'] }.transform_values(&:size)
@@ -474,6 +486,11 @@ module Woods
         @metadata_changed = false
         @vectors_changed = false
         @empty_units = {}
+        @prepared_texts = {}
+        @prepared_chunks = {}
+        @prepared_inputs = {}
+        @checkpoint_inputs = {}
+        @unknown_reconciliation_warned = false
         @persisted_metadata = nil
         prepare_snapshot_stores(incremental: incremental)
         retain_metadata_identities
@@ -492,14 +509,17 @@ module Woods
           entries = []
           @vector_store.each_entry { |id, _vector, _metadata| entries << { id: id } }
           @persisted_ids = index_ids_by_identifier(entries)
-          retain_existing_metadata_identities
         end
+        retain_existing_metadata_identities
       end
 
-      def retain_existing_metadata_identities
-        return unless @metadata_store.respond_to?(:each_entry)
-
-        @metadata_store.each_entry { |identifier, _unit| @persisted_ids[identifier] ||= [] }
+      def retain_existing_metadata_identities(identities = @persisted_ids)
+        if implements_own?(@metadata_store, :all_identifiers)
+          @metadata_store.all_identifiers.each { |identifier| identities[identifier] ||= [] }
+        elsif @metadata_store.respond_to?(:each_entry)
+          # Compatibility for custom snapshot-only stores.
+          @metadata_store.each_entry { |identifier, _unit| identities[identifier] ||= [] }
+        end
       end
 
       # Source-empty units retain metadata but intentionally have no vectors.
@@ -524,6 +544,7 @@ module Woods
       def load_durable_store_ids
         @durable_ids = Hash.new { |hash, key| hash[key] = [] }
         @vector_store.each_id { |id| @durable_ids[base_identifier(id)] << id }
+        retain_existing_metadata_identities(@durable_ids)
       rescue StandardError => e
         # A store that cannot be enumerated must not take the embed run down
         # with it. Reconciliation and the presence check both degrade to their
@@ -609,11 +630,9 @@ module Woods
 
       # May this unit's embedding be skipped?
       #
-      # Incremental skip uses `source_hash`, which the extractor derives
-      # from the unit's *source_code string only* (see ExtractedUnit#to_h
-      # and Extractor#dump_units). It is NOT a hash of the serialized
-      # unit_data JSON — so key ordering or whitespace in the _index.json
-      # does not invalidate checkpoints across Ruby-minor upgrades.
+      # Incremental skip requires both the source hash and the complete ordered
+      # prepared-input fingerprint. Dependency/path/namespace/chunk changes
+      # invalidate the latter even if source_code did not change.
       #
       # A matching hash is necessary but not sufficient: the vector must also
       # actually exist. On the dump-backed path that means present in what we
@@ -626,17 +645,17 @@ module Woods
       # unit permanently: checkpoint.json said "embedded", the new store held
       # nothing, and no subsequent incremental run ever disagreed.
       def checkpoint_satisfied?(unit_data, checkpoint)
-        return false unless checkpoint[storage_id(unit_data)] == unit_data['source_hash']
+        identifier = storage_id(unit_data)
+        return false unless checkpoint[identifier] == unit_data['source_hash']
+        return false unless @checkpoint_inputs[identifier] == prepared_fingerprint(unit_data)
 
         known_ids = persistable? ? @persisted_ids : @durable_ids
         # No durable view to check against (an adapter with no #each_id, or an
         # enumeration that failed) — fall back to trusting the checkpoint.
         return true if known_ids.nil?
 
-        return true if known_ids[storage_id(unit_data)]&.any?
-        # A source-empty unit intentionally has no vector. Verify that state
-        # again rather than treating a missing nonempty vector as a cache hit.
-        return true if prepare_texts(unit_data).empty?
+        expected = embedding_ids(identifier, prepared_texts(unit_data).size)
+        return true if Array(known_ids[identifier]).sort == expected.sort
 
         @checkpoint_misses += 1
         false
@@ -654,8 +673,10 @@ module Woods
         return unless @metadata_store
 
         id = storage_id(unit_data)
-        @metadata_changed ||= @persisted_metadata && @persisted_metadata.find(id) != unit_data
-        @metadata_store.store(id, unit_data)
+        chunks = @prepared_chunks[id]
+        data = chunks ? unit_data.merge('embedding_chunks' => JSON.parse(JSON.generate(chunks))) : unit_data
+        @metadata_changed ||= @persisted_metadata && @persisted_metadata.find(id) != data
+        @metadata_store.store(id, data)
       end
 
       # Compare with the promoted artifact, not a fresh metadata store or the
@@ -684,21 +705,22 @@ module Woods
       end
 
       def collect_embed_items(unit_data, items)
-        texts = prepare_texts(unit_data)
+        texts = prepared_texts(unit_data)
         identifier = storage_id(unit_data)
         @empty_units[identifier] = unit_data['source_hash'] if texts.empty?
 
         texts.each_with_index do |text, idx|
           embed_id = texts.length > 1 ? "#{identifier}#chunk_#{idx}" : identifier
           items << { id: embed_id, text: text, unit_data: unit_data,
-                     source_hash: unit_data['source_hash'], identifier: identifier }
+                     source_hash: unit_data['source_hash'], identifier: identifier,
+                     chunk_metadata: @prepared_chunks[identifier]&.[](idx) }
         end
       end
 
       # Defer zero-text deletion until every batch has prepared/embedded
       # successfully. A later provider failure must not retire earlier vectors.
-      # Checkpoints retain their source-hash format; a matching no-vector hash
-      # is trusted only after verifying the current input still prepares empty.
+      # The no-vector checkpoint also records the complete-input fingerprint;
+      # it advances only after obsolete vectors have been reconciled.
       def reconcile_empty_units(checkpoint)
         verify_empty_reconciliation!
         @empty_units.each do |identifier, source_hash|
@@ -706,6 +728,7 @@ module Woods
           prune_identifier(identifier, []) if implements_own?(@vector_store, :delete)
           prune_durable_identifier(identifier, []) if @durable_ids && implements_own?(@vector_store, :delete)
           checkpoint[identifier] = source_hash
+          @checkpoint_inputs[identifier] = @prepared_inputs.fetch(identifier)
         end
       end
 
@@ -715,18 +738,92 @@ module Woods
         raise Woods::Error, 'Cannot reconcile source-empty units: existing vector IDs could not be read'
       end
 
-      def prepare_texts(unit_data) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def input_budget
+        @input_budget ||= InputBudget.for(
+          @provider, limit: [safe_max_input_tokens, preparer_option(:max_tokens, 8192)].compact.min,
+                     chars_per_token: preparer_option(:chars_per_token, 4.0)
+        )
+      end
+
+      def preparer_option(name, fallback)
+        @text_preparer.respond_to?(name) ? @text_preparer.public_send(name) : fallback
+      end
+
+      def prepared_texts(unit_data)
+        @prepared_texts[storage_id(unit_data)] ||= prepare_texts(unit_data)
+      rescue InputLimitError, ArgumentError => e
+        detail = e.is_a?(InputLimitError) ? e.message : 'Cannot split source within the input limit'
+        raise InputLimitError, "#{detail}; #{unit_diagnostic(unit_data)}; #{budget_diagnostic}"
+      end
+
+      def embedding_ids(identifier, count)
+        Array.new(count) { |index| count > 1 ? "#{identifier}#chunk_#{index}" : identifier }
+      end
+
+      def prepared_fingerprint(unit_data)
+        identifier = storage_id(unit_data)
+        @prepared_inputs[identifier] ||= begin
+          texts = prepared_texts(unit_data)
+          Digest::SHA256.hexdigest(JSON.generate(embedding_ids(identifier, texts.size).zip(texts)))
+        end
+      end
+
+      def unit_diagnostic(unit_data)
+        %w[type identifier file_path].map { |key| "#{key}=#{unit_data[key].to_s[0, 160].inspect}" }.join(' ')
+      end
+
+      def budget_diagnostic
+        "model=#{input_budget.model.to_s[0, 100].inspect} counting=#{input_budget.method} limit=#{input_budget.limit}"
+      end
+
+      def prepare_texts(unit_data) # rubocop:disable Metrics/CyclomaticComplexity
         unit = build_unit(unit_data)
+        return [] if unit.chunks.empty? && unit.source_code.to_s.strip.empty?
+
+        Chunking::ContributorChunks.ensure!(unit)
         apply_chunking(unit) if @chunker && unit.chunks.empty? && needs_chunking?(unit)
         # Extraction may have emitted chunks larger than the provider's
         # budget (rails_source in particular). Enforce the ceiling on
         # whatever chunks we have before handing off to the provider.
         @chunker&.enforce_chunk_limits!(unit) if unit.chunks.any?
-        texts = unit.chunks.any? ? @text_preparer.prepare_chunks(unit) : [@text_preparer.prepare(unit)]
+        texts = prepare_unit_texts(unit)
         # Drop empty/whitespace-only texts — embedding providers reject
         # them with 400 and retrying never succeeds. Unit is effectively
         # skipped when every text is empty (zero-source unit).
-        texts.reject { |t| t.nil? || t.strip.empty? || content_portion_empty?(t, unit) }
+        select_prepared_texts(unit_data, unit, texts)
+      end
+
+      def select_prepared_texts(unit_data, unit, texts)
+        selected = texts.each_index.reject do |index|
+          text = texts[index]
+          text.nil? || text.strip.empty? || content_portion_empty?(text, unit)
+        end
+        if SourceContributors.multiple?(unit)
+          @prepared_chunks[storage_id(unit_data)] = selected.map do |index|
+            prepared_chunk_metadata(unit, texts, index)
+          end
+        end
+        selected.map { |index| texts[index] }
+      end
+
+      def prepared_chunk_metadata(unit, texts, index)
+        return {} unless texts.size == unit.chunks.size
+
+        chunk = unit.chunks.fetch(index)
+        content = chunk.fetch(:content).to_s
+        return {} if content.empty? || !texts[index].to_s.end_with?(content)
+
+        chunk.fetch(:metadata, {})
+      end
+
+      def prepare_unit_texts(unit)
+        texts = if @text_preparer.respond_to?(:prepare_for_embedding)
+                  @text_preparer.prepare_for_embedding(unit, budget: input_budget)
+                else
+                  unit.chunks.any? ? @text_preparer.prepare_chunks(unit) : [@text_preparer.prepare(unit)]
+                end
+        texts.compact.each { |text| input_budget.validate!(text) unless text.strip.empty? }
+        texts
       end
 
       # True when a prepared text is just the metadata prefix with no
@@ -777,9 +874,7 @@ module Woods
       # own +max_chars+ safety net is what guarantees each chunk fits,
       # so we pass the same char budget through here.
       def apply_chunking(unit)
-        unit.chunks = @chunker.chunk(unit).map do |chunk|
-          { content: chunk.content, chunk_type: chunk.chunk_type }
-        end
+        unit.chunks = @chunker.chunk(unit).map(&:to_h)
       end
 
       def build_unit(data)
@@ -787,6 +882,7 @@ module Woods
                                  file_path: data['file_path'])
         unit.namespace = data['namespace']
         unit.source_code = data['source_code']
+        unit.metadata = data['metadata'] || {}
         unit.dependencies = data['dependencies'] || []
         unit.chunks = (data['chunks'] || []).map { |c| c.transform_keys(&:to_sym) }
         unit
@@ -801,18 +897,46 @@ module Woods
       def embed_and_store(items, checkpoint, stats)
         return if items.empty?
 
-        vectors = @provider.embed_batch(items.map { |i| i[:text] })
+        warn_unreconciled_adapter
+        vectors = @provider.embed_batch(items.map { |item| item[:text] })
         store_vectors(items, vectors, checkpoint, stats)
       rescue StandardError => e
         stats[:errors] += items.size
-        raise Woods::Error, "Embedding failed: #{e.message}"
+        raise if e.is_a?(InputLimitError)
+
+        raise Woods::Error, embedding_failure(e, items), cause: nil
+      end
+
+      def warn_unreconciled_adapter
+        return if persistable? || reconcilable? || @unknown_reconciliation_warned
+
+        warn '[woods] custom vector store cannot enumerate and reconcile obsolete chunk IDs; cleanup is adapter-owned.'
+        @unknown_reconciliation_warned = true
+      end
+
+      def embedding_failure(error, items)
+        labels = items.lazy.map { |item| unit_diagnostic(item[:unit_data]) }.uniq.first(5).join('; ')
+        status = error.respond_to?(:http_status) ? error.http_status : nil
+        "Embedding failed (#{safe_failure_detail(error)}, HTTP #{status || 'unknown'}); " \
+          "#{labels}; #{items.size} input(s), largest=#{items.map { |item| input_budget.count(item[:text]) }.max}; " \
+          "#{budget_diagnostic}. " \
+          'No failed unit was checkpointed; check provider availability and input limits.'
+      end
+
+      def safe_failure_detail(error)
+        # Preserve numeric shape diagnostics, never source-bearing HTTP bodies
+        # or malformed response indexes (which can contain arbitrary strings).
+        if defined?(Provider::InvalidEmbeddingResponse) && error.is_a?(Provider::InvalidEmbeddingResponse)
+          return error.message[/vector at position \d+ has dimension \d+, expected \d+/] || error.class.name
+        end
+
+        error.class.name
       end
 
       def store_vectors(items, vectors, checkpoint, stats)
         entries = items.each_with_index.map do |item, idx|
           { id: item[:id], vector: vectors[idx],
-            metadata: { type: item[:unit_data]['type'], identifier: item[:unit_data]['identifier'],
-                        file_path: item[:unit_data]['file_path'] } }
+            metadata: vector_metadata(item) }
         end
 
         @vector_store.store_batch(entries)
@@ -821,10 +945,17 @@ module Woods
 
         items.each do |item|
           checkpoint[item[:identifier]] = item[:source_hash]
+          @checkpoint_inputs[item[:identifier]] = @prepared_inputs.fetch(item[:identifier])
           stats[:processed] += 1
         end
       end
 
+      def vector_metadata(item)
+        data = item.fetch(:unit_data)
+        { type: data['type'], identifier: data['identifier'], file_path: data['file_path'] }
+          .merge(Chunking::ContributorChunks.vector_metadata(data, item[:chunk_metadata]))
+      end
+
       # Suffix {#collect_embed_items} appends when a unit is split across
       # several vectors. Mirrors the pattern in {Retriever},
       # {Retrieval::ContextAssembler} and {MCP::Bootstrapper}.
@@ -938,79 +1069,43 @@ module Woods
         AtomicFile.write(File.join(@output_dir, 'checkpoint.json'), JSON.generate(checkpoint_payload(checkpoint)))
       end
 
-      # Schema of the on-disk checkpoint payload when {#resolved_config} is
-      # tracked. Bump only alongside a reader change in {#checkpoint_hashes}.
-      CHECKPOINT_SCHEMA_VERSION = 1
+      # v2 adds a preparation-policy stamp and complete-input fingerprints.
+      # Source hashes alone cannot detect path/namespace/dependency/chunk edits.
+      CHECKPOINT_SCHEMA_VERSION = 2
       private_constant :CHECKPOINT_SCHEMA_VERSION
 
-      # The provider/model/dimension triple checkpoint.json is stamped with,
-      # or +nil+ when this indexer was built without a +resolved_config+ (no
-      # identity to stamp or compare against — see {#checkpoint_payload} and
-      # {#checkpoint_hashes}, both of which treat +nil+ as "skip identity
-      # tracking entirely" for full backward compatibility with callers that
-      # never pass one).
-      #
-      # Reads {ResolvedConfig#to_snapshot_json} rather than calling
-      # +#embedding_provider+/+#dimension+ directly so a test double only
-      # needs to stub the one method the WVF1 header path already requires.
-      #
-      # @return [Hash, nil]
       def current_checkpoint_identity
-        return nil unless @resolved_config
-
-        provider = @resolved_config.to_snapshot_json['embedding_provider'] || {}
+        provider = @resolved_config ? @resolved_config.to_snapshot_json['embedding_provider'] || {} : {}
         provider.transform_keys(&:to_s).slice('class', 'model', 'dimension')
       end
 
-      # Wrap the flat identifier=>source_hash map with its identity stamp for
-      # writing, or leave it flat when this run tracks no identity.
-      def checkpoint_payload(checkpoint)
-        identity = current_checkpoint_identity
-        return checkpoint if identity.nil?
-
-        { 'schema_version' => CHECKPOINT_SCHEMA_VERSION, 'identity' => identity, 'hashes' => checkpoint }
+      def preparation_identity
+        { 'version' => 1, 'budget' => input_budget.identity,
+          'preparer' => collaborator_identity(@text_preparer), 'chunker' => collaborator_identity(@chunker) }
       end
 
-      # Recover the flat identifier=>source_hash map {#checkpoint_satisfied?}
-      # consumes from whichever on-disk shape was parsed. Two shapes:
-      #
-      # - versioned (carries a top-level "hashes" key): written by this gem
-      #   version, stamped with the provider/model/dimension identity that
-      #   produced it (see #checkpoint_payload). A stamped identity that
-      #   disagrees with {#current_checkpoint_identity} — a same-dimension
-      #   model switch, the P1 finding this exists to close — means nothing
-      #   here can say which individual hits are still good, so the *whole*
-      #   checkpoint is discarded rather than trusted per-unit.
-      # - flat (every checkpoint written before this gem version): carries no
-      #   identity at all. When this run tracks identity (a resolved_config
-      #   was given), "no identity recorded" is indistinguishable from "the
-      #   identity that produced this changed" — so it is discarded the same
-      #   way: one full re-embed, after which every checkpoint this gem
-      #   writes is stamped and can be trusted again. When this run has no
-      #   resolved_config either there is nothing to compare against, and the
-      #   flat map is trusted exactly as every prior gem version did.
-      def checkpoint_hashes(data)
-        return data unless data.is_a?(Hash)
-
-        current = current_checkpoint_identity
-        return checkpoint_hashes_versioned(data, current) if data.key?('hashes')
-        return data if current.nil?
+      def collaborator_identity(object)
+        object.respond_to?(:preparation_identity) ? object.preparation_identity : object&.class&.name
+      end
 
-        warn '[woods] checkpoint.json predates embedding-identity tracking and cannot be ' \
-             'verified against the current provider/model — discarding it and re-embedding ' \
-             'every unit once so future checkpoints are stamped and can be trusted safely.'
-        {}
+      def checkpoint_payload(checkpoint)
+        { 'schema_version' => CHECKPOINT_SCHEMA_VERSION, 'identity' => current_checkpoint_identity,
+          'preparation' => preparation_identity, 'hashes' => checkpoint,
+          'prepared_inputs' => @checkpoint_inputs.slice(*checkpoint.keys) }
       end
 
-      def checkpoint_hashes_versioned(data, current)
-        stamped = data['identity']
-        return data['hashes'] || {} if current.nil? || stamped == current
+      def checkpoint_hashes(data)
+        valid = data.is_a?(Hash) && data['schema_version'] == CHECKPOINT_SCHEMA_VERSION &&
+                data['identity'] == current_checkpoint_identity && data['preparation'] == preparation_identity &&
+                data['hashes'].is_a?(Hash) && data['prepared_inputs'].is_a?(Hash)
+        unless valid
+          warn '[woods] checkpoint.json has no matching embedding identity and preparation policy; ' \
+               'discarding the checkpoint and re-embedding every unit once.'
+          return {}
+        end
 
-        warn '[woods] checkpoint.json was stamped for a different embedding identity ' \
-             "(#{stamped.inspect} vs current #{current.inspect}) — the provider or model " \
-             'changed since the last run. Discarding the checkpoint and re-embedding every ' \
-             'unit so no stale-model vector survives.'
-        {}
+        @checkpoint_inputs = data['prepared_inputs']
+        data['hashes']
       end
 
       # Returns true when the vector store can actually be dumped to
@@ -1055,16 +1150,18 @@ module Woods
       end
 
       # Does +object+ define +method_name+ itself, rather than inheriting the
-      # vector-store interface's default stub?
+      # storage interfaces' default stubs?
       #
       # @param object [Object] the adapter under test
       # @param method_name [Symbol]
       # @return [Boolean]
       def implements_own?(object, method_name)
         return false unless object.respond_to?(method_name)
-        return true unless defined?(Storage::VectorStore::Interface)
 
-        object.method(method_name).owner != Storage::VectorStore::Interface
+        interfaces = []
+        interfaces << Storage::VectorStore::Interface if defined?(Storage::VectorStore::Interface)
+        interfaces << Storage::MetadataStore::Interface if defined?(Storage::MetadataStore::Interface)
+        !interfaces.include?(object.method(method_name).owner)
       end
 
       # Persist stores to a timestamped dump directory, write +woods.json+,

data/lib/woods/embedding/input_budget.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Woods
+  class Error < StandardError; end unless defined?(Woods::Error)
+
+  module Embedding
+    # A refusal, never a request for the retry wrapper to repeat the same input.
+    class InputLimitError < Woods::Error; end
+
+    # Counts used for admission are explicitly distinguished from sizing estimates.
+    # Byte BPE starts with UTF-8 bytes and only merges them: its token count cannot
+    # exceed bytesize. This bound applies to the known OpenAI embedding encodings,
+    # not arbitrary tokenizers (whose normalization or special tokens may differ).
+    # Encoding mapping and token-byte examples:
+    # https://developers.openai.com/cookbook/examples/how_to_count_tokens_with_tiktoken
+    class InputBudget
+      attr_reader :limit, :method, :model
+
+      def initialize(limit:, model: nil, method: 'estimate', chars_per_token: 1.2)
+        raise ArgumentError, 'input limit must be positive' unless limit.is_a?(Integer) && limit.positive?
+        raise ArgumentError, 'chars_per_token must be positive' unless chars_per_token.positive?
+
+        @limit = limit
+        @model = model
+        @method = method
+        @chars_per_token = chars_per_token
+      end
+
+      def self.for(provider, limit:, chars_per_token:)
+        supplied = provider.input_budget if provider.respond_to?(:input_budget)
+        return new(limit: limit, chars_per_token: chars_per_token) unless supplied
+        return supplied unless limit < supplied.limit
+
+        supplied.with_limit(limit)
+      end
+
+      # Preserve the provider's counting policy under a stricter caller cap.
+      def with_limit(value)
+        self.class.new(limit: [limit, value].min, model: model, method: method, chars_per_token: @chars_per_token)
+      end
+
+      def count(text)
+        utf8 = text.encode(Encoding::UTF_8)
+        raise InputLimitError, 'Embedding input requires valid UTF-8 text' unless utf8.valid_encoding?
+
+        method == 'utf8_bytes_bound' ? utf8.bytesize : (utf8.length / @chars_per_token).ceil
+      rescue EncodingError
+        raise InputLimitError, 'Embedding input requires valid UTF-8 text'
+      end
+
+      def fits?(text)
+        count(text) <= limit
+      end
+
+      def validate!(text)
+        size = count(text)
+        return text if size <= limit
+
+        raise InputLimitError, "Embedding input limit exceeded (#{method}: #{size}, limit: #{limit}); split the input"
+      end
+
+      def identity
+        { 'method' => method, 'limit' => limit, 'model' => model, 'chars_per_token' => @chars_per_token }
+      end
+    end
+  end
+end