static_embeddings 0.1.5 → 1.5.7

data/lib/static_embeddings/model.rb

@@ -4,39 +4,41 @@ module StaticEmbeddings
   class Model
     attr_reader :path
 
-    def provenance
-      raw = provenance_json
-      return {} if raw.nil?
+    if method_defined?(:max_tokens) && !method_defined?(:max_tokens_limit)
+      alias_method :max_tokens_limit, :max_tokens
 
-      JSON.parse(raw)
-    rescue JSON::ParserError, EncodingError => e
-      raise InvalidModelError, "invalid provenance JSON: #{e.message}"
+      def max_tokens
+        limit = max_tokens_limit
+        limit.zero? ? false : limit
+      end
+    end
+
+    def provenance
+      @provenance ||= parse_provenance.freeze
     end
 
     def model_id
       provenance["source_model_id"]
     end
 
-    def embed_array(text, **opts)
-      format = opts.key?(:format) ? opts[:format] : :f32
-      StaticEmbeddings.unpack(embed(text, **opts), dim, format: format).first
+    def embed_array(text, **options)
+      format = options.fetch(:format, :f32)
+      StaticEmbeddings.unpack(embed(text, **options), dim, format: format).first
     end
 
-    def embed_batch_arrays(texts, **opts)
-      format = opts.key?(:format) ? opts[:format] : :f32
-      StaticEmbeddings.unpack(embed_batch(texts, **opts), dim, format: format)
+    def embed_batch_arrays(texts, **options)
+      format = options.fetch(:format, :f32)
+      StaticEmbeddings.unpack(embed_batch(texts, **options), dim, format: format)
     end
 
-    def cosine_top_k(query_blob, matrix_blob, k, **opts)
-      raise ArgumentError, "dim: is set by the model" if opts.key?(:dim)
-
-      StaticEmbeddings.cosine_top_k(query_blob, matrix_blob, k, **opts.merge(dim: dim))
+    def cosine_top_k(query_blob, matrix_blob, k, **options)
+      reject_runtime_dim!(options)
+      StaticEmbeddings.cosine_top_k(query_blob, matrix_blob, k, **options.merge(dim: dim))
     end
 
-    def dot_top_k(query_blob, matrix_blob, k, **opts)
-      raise ArgumentError, "dim: is set by the model" if opts.key?(:dim)
-
-      StaticEmbeddings.dot_top_k(query_blob, matrix_blob, k, **opts.merge(dim: dim))
+    def dot_top_k(query_blob, matrix_blob, k, **options)
+      reject_runtime_dim!(options)
+      StaticEmbeddings.dot_top_k(query_blob, matrix_blob, k, **options.merge(dim: dim))
     end
 
     def to_s
@@ -44,5 +46,18 @@ module StaticEmbeddings
     end
 
     alias inspect to_s
+
+    private
+
+    def parse_provenance
+      raw = provenance_json
+      raw.nil? ? {} : JSON.parse(raw)
+    rescue JSON::ParserError, EncodingError => e
+      raise InvalidModelError, "invalid provenance JSON: #{e.message}"
+    end
+
+    def reject_runtime_dim!(options)
+      raise InvalidOptionError, "dim: is set by the model" if options.key?(:dim)
+    end
   end
 end

data/lib/static_embeddings/paths.rb

@@ -12,29 +12,25 @@ module StaticEmbeddings
 
     def model_path(model_id, env = ENV)
       id = model_id.to_s
-      raise ArgumentError, "model_id must not be empty" if id.empty?
-      raise ArgumentError, "model_id contains a NUL byte" if id.include?("\0")
+      raise InvalidOptionError, "model_id must not be empty" if id.empty?
+      raise InvalidOptionError, "model_id contains a NUL byte" if id.include?("\0")
 
       normalized = id.tr("\\", "/")
       parts = normalized.split("/", -1)
-      if normalized.start_with?("/") ||
-         parts.any? { |part| part.empty? || part == "." || part == ".." || part.include?(":") }
-        raise ArgumentError, "model_id must be a relative slash-separated identifier"
-      end
+      invalid = normalized.start_with?("/") ||
+                parts.any? { |part| part.empty? || part == "." || part == ".." || part.include?(":") }
+      raise InvalidOptionError, "model_id must be a relative slash-separated identifier" if invalid
 
       base = File.expand_path(File.join(cache_dir(env), "models"))
       path = File.expand_path(File.join(base, "#{normalized}.semb"))
       prefix = base.end_with?(File::SEPARATOR) ? base : "#{base}#{File::SEPARATOR}"
-      unless path.start_with?(prefix)
-        raise ArgumentError, "model_id escapes the model cache"
-      end
+      raise InvalidOptionError, "model_id escapes the model cache" unless path.start_with?(prefix)
+
       path
     end
 
     def builtin_path(name)
-      file = BUILTIN_MODELS.fetch(name) do
-        raise ModelNotFound, "unknown builtin model #{name.inspect}"
-      end
+      file = BUILTIN_MODELS.fetch(name) { raise ModelNotFound, "unknown builtin model #{name.inspect}" }
       File.join(BUILTIN_DIR, file)
     end
 

data/lib/static_embeddings/provenance.rb

@@ -0,0 +1,58 @@
+require "json"
+require "static_embeddings/format/constants"
+require "static_embeddings/bert_wordpiece"
+
+module StaticEmbeddings
+  module Provenance
+    MODEL2VEC_VERSION = "0.9.0"
+    TOKENIZERS_VERSION = "0.23.1"
+    UNICODE_CATEGORIES_VERSION = "0.1.1"
+    NOTES = "Vectors are only reference-compatible if docs/MODEL_AUDIT.md records a passing oracle run for this source revision."
+
+    module_function
+
+    def json(model, unicode_source:)
+      JSON.generate(payload(model, unicode_source: unicode_source).sort.to_h)
+    end
+
+    def payload(model, unicode_source:)
+      dimensions = model.dimensions
+      runtime = model.runtime
+      source = model.source
+      tokenizer = model.tokenizer
+
+      data = {
+        "add_special_tokens" => runtime.add_special_tokens,
+        "config_seq_length" => source.config_seq_length,
+        "converter_version" => StaticEmbeddings::VERSION,
+        "dim" => dimensions.output,
+        "format_version" => Format::VERSION,
+        "native_dim" => dimensions.native,
+        "normalize" => runtime.normalization == Format::NORMALIZATION_L2,
+        "notes" => NOTES,
+        "oracle" => source.oracle,
+        "output_dim" => dimensions.output,
+        "reference_impl" => source.oracle,
+        "reference_max_tokens" => runtime.max_tokens,
+        "reference_tokenizers_version" => TOKENIZERS_VERSION,
+        "source_family" => source.family,
+        "source_files_sha256" => source.files_sha256,
+        "source_model_id" => source.model,
+        "source_revision" => source.revision,
+        "tokenizer_class" => source.tokenizer_class,
+        "tokenizer_profile" => tokenizer.fetch(:tokenizer_profile, BertWordPiece::TOKENIZER_PROFILE),
+        "unicode_source" => unicode_source,
+        "unk_policy" => runtime.unk_policy == Format::UNK_DROP ? "drop" : "include",
+        "vocab_size" => model.tokens.length
+      }
+      data["trained_mrl_dims"] = dimensions.trained if dimensions.trained
+      add_model2vec_versions(data) if source.family == "model2vec"
+      data
+    end
+
+    def add_model2vec_versions(data)
+      data["reference_model2vec_version"] = MODEL2VEC_VERSION
+      data["reference_unicode_categories_version"] = UNICODE_CATEGORIES_VERSION
+    end
+  end
+end

data/lib/static_embeddings/reference.rb

@@ -1,6 +1,5 @@
-require "json"
-require "static_embeddings/converter"
-require "static_embeddings/safetensors"
+require "static_embeddings/importers"
+require "static_embeddings/format/constants"
 
 module StaticEmbeddings
   class Reference
@@ -26,49 +25,52 @@ module StaticEmbeddings
       @dim = meta.fetch(:dim)
     end
 
-    def self.from_source_dir(dir, max_tokens: Converter::REFERENCE_MAX_TOKENS)
-      tokenizer = JSON.parse(File.binread(File.join(dir, "tokenizer.json")))
-      config_path = File.join(dir, "config.json")
-      config = File.file?(config_path) ? JSON.parse(File.binread(config_path)) : {}
-      tokens = tokens_from(tokenizer)
-      tensor = Safetensors.read(File.join(dir, "model.safetensors"))[:tensors].values.first
-      rows, dim = tensor[:shape]
-      floats = Safetensors.f32_bytes(tensor).unpack("e*")
-      normalizer = tokenizer["normalizer"] || {}
+    def self.from_source_dir(dir, max_tokens: nil, dimensions: nil)
+      from_canonical(Importers.import(dir, max_tokens: max_tokens, dimensions: dimensions))
+    end
 
-      new(tokens: tokens,
-          matrix: Array.new(rows) { |i| floats[i * dim, dim] },
-          meta: meta_from(tokenizer, config, normalizer, dim, max_tokens))
+    def self.from_canonical(model)
+      dim = model.dimensions.output
+      floats = matrix_bytes(model.matrix).unpack("e*")
+      rows = model.tokens.length
+      new(
+        tokens: model.tokens,
+        matrix: Array.new(rows) { |index| floats[index * dim, dim] },
+        meta: meta_from_canonical(model)
+      )
     end
 
-    def self.tokens_from(tokenizer)
-      vocab = tokenizer.dig("model", "vocab")
-      vocab.each_with_object(Array.new(vocab.length)) { |(token, id), tokens| tokens[id] = token }
+    def self.matrix_bytes(matrix)
+      return matrix unless matrix.respond_to?(:each_chunk)
+
+      matrix.each_chunk.each_with_object(Format.binary_string) { |chunk, packed| packed << chunk }
     end
 
-    def self.meta_from(tokenizer, config, normalizer, dim, max_tokens)
-      lowercase = normalizer.fetch("lowercase", true)
+    def self.meta_from_canonical(model)
+      tokenizer = model.tokenizer
+      runtime = model.runtime
       {
-        dim: dim,
-        lowercase: lowercase,
-        strip_accents: normalizer["strip_accents"].nil? ? lowercase : normalizer["strip_accents"],
-        clean_text: normalizer.fetch("clean_text", true),
-        handle_chinese_chars: normalizer.fetch("handle_chinese_chars", true),
-        max_input_chars_per_word: tokenizer.dig("model", "max_input_chars_per_word") || 100,
-        unk_token: tokenizer.dig("model", "unk_token") || "[UNK]",
-        normalize: config.key?("normalize") ? config["normalize"] : true,
-        max_tokens: max_tokens,
-        added_tokens: supported_added_tokens(tokenizer)
+        dim: model.dimensions.output,
+        lowercase: tokenizer.fetch(:do_lower_case),
+        strip_accents: tokenizer.fetch(:strip_accents),
+        clean_text: tokenizer.fetch(:clean_text),
+        handle_chinese_chars: tokenizer.fetch(:handle_chinese_chars),
+        max_input_chars_per_word: tokenizer.fetch(:max_input_chars_per_word),
+        unk_token: model.tokens.fetch(tokenizer.fetch(:unk_id)),
+        normalize: runtime.normalization == Format::NORMALIZATION_L2,
+        max_tokens: runtime.max_tokens,
+        unk_policy: runtime.unk_policy,
+        added_tokens: added_tokens(model)
       }
     end
 
-    def self.supported_added_tokens(tokenizer)
-      allowed = Converter::STANDARD_SPECIAL_TOKENS
-      (tokenizer["added_tokens"] || []).each_with_object({}) do |token, out|
-        content = token["content"]
-        next unless token["special"] && allowed.key?(content) && token["normalized"] == false
+    def self.added_tokens(model)
+      mask = model.tokenizer.fetch(:added_token_mask)
+      BertWordPiece::STANDARD_SPECIAL_TOKENS.each_with_object({}) do |(content, bit), tokens|
+        next if (mask & bit).zero?
 
-        out[content] = Integer(token.fetch("id"))
+        id = model.tokens.index(content)
+        tokens[content] = id unless id.nil?
       end
     end
 
@@ -91,14 +93,22 @@ module StaticEmbeddings
     end
 
     def embed(text, max_tokens: @meta[:max_tokens])
-      used = tokenize(text, max_tokens: false).reject { |id| id == unk_id }
-      used = used.first(max_tokens) if max_tokens && max_tokens.positive? && used.length > max_tokens
-      return Array.new(@dim, 0.0) if used.empty?
+      ids = apply_unk_policy(tokenize(text, max_tokens: false))
+      ids = ids.first(max_tokens) if max_tokens && max_tokens.positive? && ids.length > max_tokens
+      return Array.new(@dim, 0.0) if ids.empty?
 
-      vector = pooled(used)
+      vector = pooled(ids)
       @meta[:normalize] ? l2_normalize(vector) : vector
     end
 
+    def apply_unk_policy(ids)
+      case @meta.fetch(:unk_policy)
+      when Format::UNK_DROP then ids.reject { |id| id == unk_id }
+      when Format::UNK_INCLUDE then ids
+      else raise InvalidModelError, "unknown unk_policy #{@meta[:unk_policy].inspect}"
+      end
+    end
+
     private
 
     def tokenize_with_added_tokens(text)

data/lib/static_embeddings/row_prefix_payload.rb

@@ -0,0 +1,59 @@
+require "static_embeddings/errors"
+require "static_embeddings/format/constants"
+
+module StaticEmbeddings
+  class RowPrefixPayload
+    FLOAT_BYTES = 4
+
+    attr_reader :bytesize
+
+    def initialize(inner, native_dim:, output_dim:)
+      raise InvalidOptionError, "native_dim must be positive" unless native_dim.positive?
+      unless output_dim.between?(1, native_dim)
+        raise InvalidOptionError, "output_dim must be between 1 and #{native_dim}"
+      end
+
+      @inner = inner
+      @source_row_bytes = native_dim * FLOAT_BYTES
+      @output_row_bytes = output_dim * FLOAT_BYTES
+      unless (inner.bytesize % @source_row_bytes).zero?
+        raise ConversionError, "embedding payload is not a whole number of #{native_dim}-d rows"
+      end
+      @bytesize = (inner.bytesize / @source_row_bytes) * @output_row_bytes
+    end
+
+    def each_chunk
+      return enum_for(__method__) unless block_given?
+
+      remainder = Format.binary_string
+      @inner.each_chunk do |chunk|
+        data = remainder.empty? ? chunk : remainder + chunk
+        complete_rows, remainder = split_complete_rows(data)
+        yield slice_rows(complete_rows) unless complete_rows.empty?
+      end
+
+      raise ConversionError, "truncated embedding matrix while slicing dimensions" unless remainder.empty?
+    end
+
+    private
+
+    def split_complete_rows(data)
+      bytes = (data.bytesize / @source_row_bytes) * @source_row_bytes
+      complete = data.byteslice(0, bytes) || Format.binary_string
+      remainder = data.byteslice(bytes, data.bytesize - bytes) || Format.binary_string
+      remainder.force_encoding(Encoding::BINARY)
+      [complete, remainder]
+    end
+
+    def slice_rows(data)
+      rows = data.bytesize / @source_row_bytes
+      out = Format.binary_string(rows * @output_row_bytes)
+      offset = 0
+      rows.times do
+        out << data.byteslice(offset, @output_row_bytes)
+        offset += @source_row_bytes
+      end
+      out
+    end
+  end
+end

data/lib/static_embeddings/version.rb

@@ -1,3 +1,3 @@
 module StaticEmbeddings
-  VERSION = "0.1.5"
+  VERSION = "1.5.7"
 end

data/lib/static_embeddings.rb

@@ -1,23 +1,20 @@
-require "json"
 require "static_embeddings/version"
 
 begin
   require "static_embeddings/static_embeddings"
 rescue LoadError
   ext_dir = File.expand_path("static_embeddings", __dir__)
-  so_path = %w[.so .bundle].lazy.map { |ext| File.join(ext_dir, "static_embeddings#{ext}") }
-                                .find { |path| File.file?(path) }
-  unless so_path
-    raise LoadError, "Could not find the compiled StaticEmbeddings extension. " \
-                     "Run: bundle exec rake compile"
+  extension = %w[.so .bundle].lazy.map { |suffix| File.join(ext_dir, "static_embeddings#{suffix}") }
+                              .find { |path| File.file?(path) }
+  unless extension
+    raise LoadError, "Could not find the compiled StaticEmbeddings extension. Run: bundle exec rake compile"
   end
-
-  require so_path
+  require extension
 end
 
 require "static_embeddings/errors"
 require "static_embeddings/paths"
-require "static_embeddings/format"
+require "static_embeddings/format/constants"
 require "static_embeddings/model"
 
 module StaticEmbeddings
@@ -25,11 +22,7 @@ module StaticEmbeddings
     def load(path, verify: false)
       expanded = File.expand_path(path.to_s)
       raise ModelNotFound, "no model at #{expanded}" unless File.file?(expanded)
-
-      if verify
-        result = Format.verify(expanded)
-        raise InvalidModelError, "checksum mismatch for #{expanded}" unless result[:ok]
-      end
+      raise InvalidModelError, "checksum mismatch for #{expanded}" if verify && !self.verify(expanded)[:ok]
 
       Model.new(expanded)
     end
@@ -54,63 +47,44 @@ module StaticEmbeddings
       path = model_path(model_id)
       unless File.file?(path)
         raise ModelNotFound,
-              "model #{model_id.inspect} is not installed. " \
-              "Convert it first: static_embeddings convert <hf-dir> --id #{model_id}"
+              "model #{model_id.inspect} is not installed. Convert it first: " \
+              "static_embeddings convert <hf-dir> --id #{model_id}"
       end
       load(path, verify: verify)
     end
 
-    def convert(source_dir, output_path:, model_id: nil, max_tokens: nil)
-      require "static_embeddings/converter"
-      max_tokens = Converter::REFERENCE_MAX_TOKENS if max_tokens.nil?
-      converter = Converter.new(source_dir)
-      converter.convert(output_path: output_path, model_id: model_id, max_tokens: max_tokens)
-      converter.report
+    def convert(source_dir, output_path:, model_id: nil, max_tokens: nil, dimensions: nil,
+                source_revision: nil, trained_mrl_dims: nil)
+      require "static_embeddings/conversion"
+      Conversion.call(
+        source_dir,
+        output_path: output_path,
+        model_id: model_id,
+        max_tokens: max_tokens,
+        dimensions: dimensions,
+        source_revision: source_revision,
+        trained_mrl_dims: trained_mrl_dims
+      )
     end
 
     def verify(path)
-      Format.verify(File.expand_path(path.to_s))
+      require "static_embeddings/format/verifier"
+      Format::Verifier.call(File.expand_path(path.to_s))
     end
 
     def unpack(blob, dim, format: :f32)
-      raise ArgumentError, "dim must be positive" unless dim.to_i.positive?
-
-      floats =
-        case normalize_format(format)
-        when :f32
-          raise ArgumentError, "f32 blob byte size must be a multiple of 4" unless (blob.bytesize % 4).zero?
-
-          blob.unpack("e*")
-        when :f16
-          raise ArgumentError, "f16 blob byte size must be a multiple of 2" unless (blob.bytesize % 2).zero?
-
-          decode_f16(blob)
-        end
-
-      raise ArgumentError, "blob is not a multiple of dim" unless (floats.length % dim).zero?
-
-      floats.each_slice(dim).to_a
+      require "static_embeddings/codec"
+      Codec.unpack(blob, dim, format: format)
     end
 
     def pack(rows, format: :f32)
-      flat = rows.first.is_a?(Array) ? rows.flatten(1) : rows
-
-      case normalize_format(format)
-      when :f32 then flat.map(&:to_f).pack("e*")
-      when :f16 then encode_f16(flat.map(&:to_f))
-      end
+      require "static_embeddings/codec"
+      Codec.pack(rows, format: format)
     end
 
     def normalize_format(format)
-      case format&.to_sym
-      when nil, :f32, :float32
-        :f32
-      when :f16, :float16
-        :f16
-      else
-        raise ArgumentError, "unsupported embedding format #{format.inspect} (expected :f32 or :f16)"
-      end
+      require "static_embeddings/codec"
+      Codec.normalize_format(format)
     end
-
   end
 end

data/static_embeddings.gemspec

@@ -7,8 +7,8 @@ Gem::Specification.new do |spec|
   spec.email   = ["romnhajdarov@gmail.com"]
 
   spec.summary     = "Fast local text embeddings for Ruby — no ONNX, no Rust, no network"
-  spec.description = "A small C-extension runtime for Model2Vec-style static embedding " \
-                     "models. Models are converted offline into a flat mmap-able .semb " \
+  spec.description = "A small C-extension runtime for Model2Vec and Sentence Transformers static " \
+                     "WordPiece embedding models. Models are converted offline into a flat mmap-able .semb " \
                      "file; at runtime the gem tokenizes (BERT WordPiece), looks up rows " \
                      "and mean-pools them. Releases the GVL on large native work, rejects internal " \
                      "thread fan-out, and links nothing but libc."

data/tools/check_model2vec_parity.rb

@@ -80,7 +80,11 @@ rows.each_with_index do |row, i|
 
   full_raw = model.tokenize(text, max_tokens: false)
   expected_usable = row.fetch("static_usable_token_ids")
-  got_usable = full_raw.reject { |id| id == model.unk_id }.first(max_length)
+  got_usable = if reference["unk_token_id"].nil?
+                 full_raw.first(max_length)
+               else
+                 full_raw.reject { |id| id == model.unk_id }.first(max_length)
+               end
   if got_usable != expected_usable
     usable_failures << i
     first = got_usable.zip(expected_usable).index { |a, b| a != b } || [got_usable.length, expected_usable.length].min

data/tools/check_st_parity.rb

@@ -0,0 +1,125 @@
+require "json"
+require "optparse"
+require "static_embeddings"
+
+options = {
+  min_cosine: 1.0 - 1e-6,
+  max_abs: 1e-5
+}
+
+parser = OptionParser.new do |opts|
+  opts.on("--model PATH") { |value| options[:model] = value }
+  opts.on("--oracle PATH") { |value| options[:oracle] = value }
+  opts.on("--min-cosine N", Float) { |value| options[:min_cosine] = value }
+  opts.on("--max-abs N", Float) { |value| options[:max_abs] = value }
+end
+parser.parse!(ARGV)
+unless options[:model] && options[:oracle]
+  abort "usage: ruby -Ilib tools/check_st_parity.rb --model MODEL.semb --oracle oracle.json"
+end
+
+model = StaticEmbeddings.load(options[:model], verify: true)
+payload = JSON.parse(File.read(options[:oracle], encoding: "UTF-8"))
+abort "unsupported oracle schema #{payload["schema_version"].inspect}" unless payload["schema_version"] == 1
+
+reference = payload.fetch("reference")
+abort "ST oracle must use add_special_tokens=false" unless reference["add_special_tokens"] == false
+abort "ST oracle must use normalize_embeddings=false" unless reference["normalize_embeddings"] == false
+if model.normalized?
+  abort "ST source-faithful .semb must not L2-normalize; got normalized?=true"
+end
+unless model.max_tokens == false
+  abort "ST source-faithful .semb must bake unlimited tokens; got max_tokens=#{model.max_tokens.inspect}"
+end
+if model.provenance["unk_policy"] != "include"
+  abort "ST source-faithful .semb must use UNK_INCLUDE; got #{model.provenance["unk_policy"].inspect}"
+end
+
+def prefix_dims(vector, dim)
+  return vector if vector.length == dim
+  abort "oracle vector shorter than model dim (#{vector.length} < #{dim})" if vector.length < dim
+
+  vector.first(dim)
+end
+
+def dot(a, b)
+  a.zip(b).sum { |x, y| x * y }
+end
+
+def norm(a)
+  Math.sqrt(a.sum { |x| x * x })
+end
+
+def vector_metrics(reference, got)
+  max_abs = reference.zip(got).map { |a, b| (a - b).abs }.max || 0.0
+  ref_zero = reference.all?(&:zero?)
+  got_zero = got.all?(&:zero?)
+  cosine =
+    if ref_zero && got_zero
+      1.0
+    elsif ref_zero || got_zero
+      0.0
+    else
+      dot(reference, got) / (norm(reference) * norm(got))
+    end
+  [cosine, max_abs]
+end
+
+raw_failures = []
+vector_failures = []
+diag_failures = []
+min_cosine = 1.0
+max_abs_all = 0.0
+vectors_checked = 0
+
+payload.fetch("rows").each_with_index do |row, i|
+  text = row.fetch("text")
+  label = row.fetch("label", i.to_s)
+  problems = []
+
+  expected_raw = row.fetch("hf_raw_token_ids")
+  got_raw = model.tokenize(text, max_tokens: false)
+  if got_raw != expected_raw
+    raw_failures << i
+    first = got_raw.zip(expected_raw).index { |a, b| a != b } || [got_raw.length, expected_raw.length].min
+    problems << "raw ids differ at #{first} (got #{got_raw.length}, ref #{expected_raw.length})"
+  end
+
+  got_vector = model.embed_array(text)
+  reference_vector = prefix_dims(row.fetch("st_encode_vector"), got_vector.length)
+  cosine, max_abs = vector_metrics(reference_vector, got_vector)
+  vectors_checked += 1
+  min_cosine = [min_cosine, cosine].min
+  max_abs_all = [max_abs_all, max_abs].max
+  unless cosine >= options[:min_cosine] && max_abs <= options[:max_abs]
+    vector_failures << i
+    problems << format("encode vector out of tolerance cos=%.10f max_abs=%.8g", cosine, max_abs)
+  end
+
+  if row.key?("static_embedding_vector")
+    diag = prefix_dims(row.fetch("static_embedding_vector"), got_vector.length)
+    dcos, dabs = vector_metrics(diag, got_vector)
+    unless dcos >= options[:min_cosine] && dabs <= options[:max_abs]
+      diag_failures << i
+      problems << format("StaticEmbedding diagnostic out of tolerance cos=%.10f max_abs=%.8g", dcos, dabs)
+    end
+  end
+
+  status = problems.empty? ? "ok" : "FAIL"
+  puts "#{status} idx=#{format('%03d', i)} label=#{label.inspect} bytes=#{text.bytesize}" +
+       (problems.empty? ? "" : " [#{problems.join('; ')}]")
+end
+
+puts "rows=#{payload.fetch("rows").length}"
+puts "vectors_checked=#{vectors_checked}"
+puts "min_cosine=#{min_cosine}"
+puts "max_abs_all=#{max_abs_all}"
+puts "raw_token_id_failures=#{raw_failures.inspect}"
+puts "vector_failures=#{vector_failures.inspect}"
+puts "static_embedding_failures=#{diag_failures.inspect}"
+
+unless raw_failures.empty? && vector_failures.empty? && diag_failures.empty?
+  abort "ST parity failed"
+end
+
+puts "ST corpus parity OK (#{payload.fetch("rows").length}/#{payload.fetch("rows").length})"