gte 0.0.1

data/ext/gte/tests/inference_integration_test.rs

@@ -0,0 +1,62 @@
+use gte::embedder::Embedder;
+
+#[test]
+#[ignore = "requires ext/gte/tests/fixtures/e5/tokenizer.json and model.onnx"]
+fn test_e5_single_embedding_shape() {
+    const DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/e5");
+
+    let embedder = Embedder::from_dir(DIR, 0, 3).expect("embedder should initialize");
+    let result = embedder
+        .embed(vec!["query: Hello world".to_string()])
+        .expect("embed should succeed");
+
+    assert_eq!(result.shape()[0], 1);
+    assert!(result.shape()[1] > 0);
+}
+
+#[test]
+#[ignore = "requires ext/gte/tests/fixtures/clip/tokenizer.json and model.onnx"]
+fn test_clip_single_embedding_shape() {
+    const DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/clip");
+
+    let embedder = Embedder::from_dir(DIR, 0, 3).expect("embedder should initialize");
+    let result = embedder
+        .embed(vec!["a photo of a cat".to_string()])
+        .expect("embed should succeed");
+
+    assert_eq!(result.shape()[0], 1);
+    assert!(result.shape()[1] > 0);
+}
+
+#[test]
+#[ignore = "requires ext/gte/tests/fixtures/e5/tokenizer.json and model.onnx"]
+fn test_e5_batch_embedding_shape() {
+    const DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/e5");
+
+    let embedder = Embedder::from_dir(DIR, 0, 3).expect("embedder should initialize");
+    let texts = vec![
+        "query: first sentence".to_string(),
+        "query: second sentence".to_string(),
+        "query: third sentence for batch".to_string(),
+    ];
+
+    let result = embedder.embed(texts).expect("batch embed should succeed");
+
+    assert_eq!(result.shape()[0], 3);
+    assert!(result.shape()[1] > 0);
+}
+
+#[test]
+#[ignore = "requires ext/gte/tests/fixtures/e5/tokenizer.json and model.onnx"]
+fn test_e5_long_input_truncation_no_error() {
+    const DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/e5");
+
+    let embedder = Embedder::from_dir(DIR, 0, 3).expect("embedder should initialize");
+    let very_long_text = "word ".repeat(1000);
+    let result = embedder
+        .embed(vec![very_long_text])
+        .expect("long input should be truncated without error");
+
+    assert_eq!(result.shape()[0], 1);
+    assert!(result.shape()[1] > 0);
+}

data/ext/gte/tests/tokenizer_unit_test.rs

@@ -0,0 +1,44 @@
+use gte::tokenizer::Tokenizer;
+
+#[test]
+#[ignore = "requires ext/gte/tests/fixtures/e5/tokenizer.json"]
+fn test_e5_tokenizer_output_shape() {
+    const TOKENIZER: &str = concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/tests/fixtures/e5/tokenizer.json"
+    );
+
+    let tokenizer = Tokenizer::new(TOKENIZER, 512, true).expect("tokenizer should load");
+    let texts = vec![
+        "Hello, world!".to_string(),
+        "A second, longer sentence to test padding behavior.".to_string(),
+    ];
+
+    let tokenized = tokenizer.tokenize(&texts).expect("tokenize should succeed");
+
+    assert_eq!(tokenized.rows, 2, "batch size should be 2");
+    assert!(tokenized.cols > 0, "sequence length should be non-zero");
+    assert_eq!(tokenized.input_ids.len(), tokenized.rows * tokenized.cols);
+    assert_eq!(tokenized.attn_masks.len(), tokenized.rows * tokenized.cols);
+
+    let type_ids = tokenized.type_ids.as_ref().expect("type_ids should exist");
+    assert_eq!(type_ids.len(), tokenized.rows * tokenized.cols);
+}
+
+#[test]
+#[ignore = "requires ext/gte/tests/fixtures/e5/tokenizer.json"]
+fn test_e5_truncation_at_max_length() {
+    const TOKENIZER: &str = concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/tests/fixtures/e5/tokenizer.json"
+    );
+
+    let tokenizer = Tokenizer::new(TOKENIZER, 16, false).expect("tokenizer should load");
+    let long_text = "word ".repeat(200);
+    let tokenized = tokenizer
+        .tokenize(&[long_text])
+        .expect("tokenize should not error on long input");
+
+    assert_eq!(tokenized.rows, 1);
+    assert_eq!(tokenized.cols, 16, "sequence length should be truncated to max_length");
+}

data/lib/gte.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'gte/gte'
+
+module GTE
+  VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
+
+  class Model
+    def initialize(dir, num_threads: 0, optimization_level: 3)
+      @embedder = GTE::Embedder.new(dir, num_threads, optimization_level)
+    end
+
+    def embed(texts)
+      if texts.is_a?(String)
+        @embedder.embed_one(texts)
+      else
+        @embedder.embed(Array(texts))
+      end
+    end
+
+    def [](input)
+      case input
+      when String then embed(input).row(0)
+      when Array  then embed(input)
+      end
+    end
+  end
+
+  def self.new(dir, num_threads: 0, optimization_level: 3)
+    Model.new(dir, num_threads: num_threads, optimization_level: optimization_level)
+  end
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: gte
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- elcuervo
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- elcuervo@elcuervo.net
+executables: []
+extensions:
+- ext/gte/extconf.rb
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- ext/gte/Cargo.toml
+- ext/gte/benches/hot_path.rs
+- ext/gte/build.rs
+- ext/gte/extconf.rb
+- ext/gte/src/embedder.rs
+- ext/gte/src/error.rs
+- ext/gte/src/lib.rs
+- ext/gte/src/model_config.rs
+- ext/gte/src/postprocess.rs
+- ext/gte/src/ruby_embedder.rs
+- ext/gte/src/session.rs
+- ext/gte/src/tokenizer.rs
+- ext/gte/tests/embedder_unit_test.rs
+- ext/gte/tests/inference_integration_test.rs
+- ext/gte/tests/tokenizer_unit_test.rs
+- lib/gte.rb
+homepage: https://github.com/elcuervo/gte
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: General Text Embeddings
+test_files: []