kiribi-multilingual_e5-small 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0a9d42a6343fe87431af5f6b297cc01c4cfa4d48fb7d7e7e94fd21574093a0b3
+  data.tar.gz: b95752db5aa79e0f1949e1a9f01a893ea2d8801154ce24df4b054645c82a979a
+SHA512:
+  metadata.gz: 67c43cefe6faba082a47e5e02e5b1d46a2176067d429c09e4a9073679ff6b0f46eedc1a77b9f1b98ed7f0914fc44020f60ca808a1829e81ff4210b155e0d8f87
+  data.tar.gz: 10f000ad8a5f9a3c23787867037d41aa5472629fe62391ce092075b8ac730cb8c4f7cc58f4940e82996a4eded4a9edb7e6f0845ca5cf65754627880de6623e05

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Katsutoshi Fujihara
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/ext/kiribi-multilingual_e5-small/extconf.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "net/http"
+require "pathname"
+require "rubygems/package"
+require "zlib"
+
+GEM_NAME = "kiribi-multilingual_e5-small"
+MODEL_URL = "https://github.com/matsudai/kiribi-externals/releases/download/intfloat%2Fmultilingual-e5-small%2Fc007d7e/model_qint8_avx512_vnni.tar.gz"
+
+TMP_FILEPATH = File.expand_path(File.join(__dir__, "/../../tmp/model.tar.gz"))
+BUILD_DIRPATH = File.expand_path(File.join(__dir__, "/../../build"))
+
+if Dir.exist?(BUILD_DIRPATH)
+  puts "#{BUILD_DIRPATH} already exists, skipping download and extraction."
+else
+  if File.exist?(TMP_FILEPATH)
+    puts "#{TMP_FILEPATH} already exists, skipping download."
+  else
+    redirect_count = 0
+    url = MODEL_URL
+    loop do
+      raise "Too many redirects" if redirect_count >= 10
+
+      resp = Net::HTTP.get_response(URI.parse(url))
+      case resp
+      when Net::HTTPSuccess
+        FileUtils.mkdir_p(File.dirname(TMP_FILEPATH))
+        File.binwrite(TMP_FILEPATH, resp.body)
+        break
+      when Net::HTTPRedirection
+        url = resp["Location"]
+        redirect_count += 1
+      else
+        raise "HTTP request failed (status code: #{resp.code})"
+      end
+    end
+  end
+
+  Gem::Package::TarReader.new(Zlib::GzipReader.open(TMP_FILEPATH)) do |archive|
+    archive.each do |entry|
+      filepath = File.join(BUILD_DIRPATH, *Pathname(entry.full_name).each_filename.to_a[1..])
+
+      if entry.directory?
+        FileUtils.mkdir_p(filepath)
+      elsif entry.file?
+        FileUtils.mkdir_p(File.dirname(filepath))
+        File.binwrite(filepath, entry.read)
+      end
+    end
+  end
+end
+
+File.write("Makefile", "all install clean:\n\t@echo \"Nothing to do for $(TARGET)\"\n")
+FileUtils.rm_f(TMP_FILEPATH)

data/lib/kiribi/multilingual_e5/small/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Kiribi
+  module MultilingualE5
+    module Small
+      VERSION = "0.0.1"
+    end
+  end
+end

data/lib/kiribi/multilingual_e5/small.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative "small/version"
+require "kiribi"
+require "onnxruntime"
+require "tokenizers"
+
+module Kiribi
+  module MultilingualE5
+    extend Kiribi::Loader
+
+    module Small
+      extend Kiribi::Loader
+
+      TOKENIZER_FILEPATH = File.expand_path(File.join(__dir__, "/../../../build/tokenizer.json"))
+      MODEL_FILEPATH = File.expand_path(File.join(__dir__, "/../../../build/model_qint8_avx512_vnni.onnx"))
+
+      class Model
+        attr_reader :onnx_model, :tokenizer
+
+        def initialize
+          @tokenizer = Tokenizers.from_file(TOKENIZER_FILEPATH)
+          @onnx_model = OnnxRuntime::Model.new(MODEL_FILEPATH)
+        end
+
+        def embedding_query(input)
+          embedding(:query, input)
+        end
+
+        def embedding_passage(input)
+          embedding(:passage, input)
+        end
+
+        def embedding(prefix, input)
+          prefix = prefix.to_s
+
+          raise ArgumentError, "prefix must be :query or :passage" unless %w[query passage].include?(prefix)
+          encoded = tokenizer.encode("#{prefix}: #{input}")
+          batch = {
+            input_ids: [encoded.ids],
+            attention_mask: [encoded.attention_mask],
+            token_type_ids: [[0] * encoded.ids.length]
+          }
+          outputs = onnx_model.predict(batch)
+          last_hidden_state = outputs["last_hidden_state"][0]
+          valid_tokens = encoded.attention_mask.sum
+          last_hidden_state[0...valid_tokens][0..384].transpose.map(&:sum).map { it / valid_tokens }
+        end
+      end
+
+      def self.instantiate
+        Model.new
+      end
+    end
+  end
+end
+
+Kiribi.register(Kiribi::MultilingualE5::Small, order: 100_100_100)

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: kiribi-multilingual_e5-small
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- matsudai
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: kiribi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: onnxruntime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: tokenizers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- Rakefile
+- ext/kiribi-multilingual_e5-small/extconf.rb
+- lib/kiribi/multilingual_e5/small.rb
+- lib/kiribi/multilingual_e5/small/version.rb
+homepage: https://github.com/matsudai/kiribi
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Easy to use some onnx models.
+test_files: []