static_embeddings 0.1.1 → 0.1.2

data/lib/static_embeddings/format.rb

@@ -70,6 +70,8 @@ module StaticEmbeddings
       76 => :clean_text
     }.freeze
 
+    VERIFY_CHUNK_BYTES = 1024 * 1024
+
     module_function
 
     def hash_bytes(str, seed = HASH_SEED)
@@ -120,14 +122,28 @@ module StaticEmbeddings
     end
 
     def verify(path)
-      data = File.binread(path)
-      raise InvalidModelError, "file too small" if data.bytesize < HEADER_SIZE
-      raise InvalidModelError, "bad magic" unless data.byteslice(0, 8) == MAGIC.b
+      raise InvalidModelError, "file too small" if File.size(path) < HEADER_SIZE
+
+      File.open(path, "rb") do |io|
+        header = io.read(HEADER_SIZE)
+        raise InvalidModelError, "bad magic" unless header.byteslice(0, 8) == MAGIC.b
 
-      stored = data.byteslice(CHECKSUM_OFFSET, CHECKSUM_SIZE)
-      actual = checksum_for_verify(data)
+        stored = header.byteslice(CHECKSUM_OFFSET, CHECKSUM_SIZE)
+        actual = streaming_checksum(io, header)
 
-      { ok: stored == actual, expected: actual.unpack1("H*"), stored: stored.unpack1("H*") }
+        { ok: stored == actual, expected: actual.unpack1("H*"), stored: stored.unpack1("H*") }
+      end
+    end
+
+    def streaming_checksum(io, header)
+      digest = Digest::SHA256.new
+      zeroed = header.dup
+      zeroed[CHECKSUM_OFFSET, CHECKSUM_SIZE] = "\0".b * CHECKSUM_SIZE
+      digest << zeroed
+
+      buffer = String.new(capacity: VERIFY_CHUNK_BYTES)
+      digest << buffer while io.read(VERIFY_CHUNK_BYTES, buffer)
+      digest.digest
     end
 
     def binary_string(capacity = nil)
@@ -280,10 +296,5 @@ module StaticEmbeddings
       buffer[offset, 8] = [value & 0xFFFFFFFF, value >> 32].pack("V2")
     end
 
-    def checksum_for_verify(data)
-      zeroed = data.dup
-      zeroed[CHECKSUM_OFFSET, CHECKSUM_SIZE] = "\0".b * CHECKSUM_SIZE
-      Digest::SHA256.digest(zeroed)
-    end
   end
 end

data/lib/static_embeddings/version.rb

@@ -1,3 +1,3 @@
 module StaticEmbeddings
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/tools/benchmark.rb

@@ -12,8 +12,11 @@ CORPUS = (1..5_000).map do |i|
   "локальный поиск по тексту номер #{i} hello world static embedding vector search"
 end
 
-tokens = CORPUS.sum { |t| model.tokenize(t).length }
-bytes = CORPUS.sum(&:bytesize)
+stats = CORPUS.map { |t| model.embed_with_stats(t) }
+tokens = stats.sum { |s| s[:token_count] }
+truncated = stats.count { |s| s[:truncated] }
+logical_bytes = CORPUS.sum(&:bytesize)
+processed_bytes = truncated.zero? ? logical_bytes : nil
 
 def timed(label, iterations)
   elapsed = Benchmark.realtime { iterations.times { yield } }
@@ -28,11 +31,17 @@ per_token_ns = (batch1 / tokens) * 1e9
 per_token_per_100dim = per_token_ns / (model.dim / 100.0)
 
 puts "model            #{model.model_id} dim=#{model.dim} vocab=#{model.vocab_size}"
-puts "corpus           #{CORPUS.length} texts, #{tokens} tokens, #{bytes} bytes"
+puts "corpus           #{CORPUS.length} texts, #{tokens} tokens, #{logical_bytes} logical bytes"
+puts "truncation       #{truncated} of #{CORPUS.length} texts truncated at max_tokens=#{model.max_tokens}"
 puts
 puts "single embed     #{((single / 2_000) * 1e6).round(1)} us/call"
 puts "batch           #{(CORPUS.length / batch1).round} texts/s, #{(tokens / batch1).round} tokens/s"
-puts "tokenize only    #{((tok_only / bytes) * 1e9).round(1)} ns/input byte"
+if processed_bytes
+  puts "tokenize only    #{((tok_only / processed_bytes) * 1e9).round(1)} ns/processed byte"
+else
+  puts "tokenize only    #{(tok_only * 1e3).round(1)} ms total (no ns/byte: truncation makes"
+  puts "                 processed bytes smaller than the #{logical_bytes} logical bytes)"
+end
 puts
 puts "normalised       #{per_token_ns.round(1)} ns/token"
 puts "                 #{per_token_per_100dim.round(1)} ns/token/100dim"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: static_embeddings
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Roman Haydarov
@@ -85,10 +85,13 @@ files:
 - docs/PERFORMANCE.md
 - exe/static_embeddings
 - ext/static_embeddings/extconf.rb
+- ext/static_embeddings/se_alloc_stats.c
 - ext/static_embeddings/se_embed.c
+- ext/static_embeddings/se_f16.c
 - ext/static_embeddings/se_format.c
 - ext/static_embeddings/se_internal.h
 - ext/static_embeddings/se_tokenizer.c
+- ext/static_embeddings/se_topk.c
 - ext/static_embeddings/se_unicode.c
 - ext/static_embeddings/static_embeddings.c
 - lib/static_embeddings.rb