discourse_ai-tokenizers 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12f6645ef62162c14c3c1c43af8dae6dc890e96853207cf32575106c4b080d7e
-  data.tar.gz: 78c9b157f4a0bd490e4f8ea5f10f3c4793ef8f98baa05d8a773f0b5cd55e0e9c
+  metadata.gz: bc2fd76c9fd338fd19f6d56d4e21c98695d2ff8a2baf5626f36f7df7f98af3d9
+  data.tar.gz: d5e9b1ca74715a0346a3d22e2413834191488167e867d5bec795b6f9c9c25f5d
 SHA512:
-  metadata.gz: 715b68088a90292e82280b0a144a8b3040bd5e4f9720937f876f63d8ba91f711db07c62c9a8be1f48746b1fd38606a2b69b063b712f5949069763e7a63a5f010
-  data.tar.gz: 9e01d44c11d1f88edb838fb876d178b94e69497c73b38c70c4ca215a6296c0bffcf412c59924d1dcf7a64d147db3a30e4cc30d0b864ed02e65d7a0e07a758d03
+  metadata.gz: 4b06f6c801f878f173471a337f0d9d28c3321ba0b7b089876b83296353144257080bc0e1495bdbe020914f4cb6324696f9505a625cd01545b17d6e99624f190f
+  data.tar.gz: f7c37e3d464b16419c7218554a29514c63d84d1d4247969940d2f9d7f3316618ae593aa98df8941a44d2c721471f97073b75b37b2246a18349f43b6a324adb8a

data/lib/discourse_ai/tokenizers/all_mpnet_base_v2_tokenizer.rb

@@ -5,7 +5,10 @@ module DiscourseAi
     # Tokenizer for the mpnet based embeddings models
     class AllMpnetBaseV2Tokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/all-mpnet-base-v2.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("all-mpnet-base-v2.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/anthropic_tokenizer.rb

@@ -6,7 +6,9 @@ module DiscourseAi
     class AnthropicTokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/claude-v1-tokenization.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("claude-v1-tokenization.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/bert_tokenizer.rb

@@ -5,7 +5,10 @@ module DiscourseAi
     # Bert tokenizer, useful for lots of embeddings and small classification models
     class BertTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/bert-base-uncased.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("bert-base-uncased.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/bge_large_en_tokenizer.rb

@@ -5,7 +5,10 @@ module DiscourseAi
     # Tokenizer used in bge-large-en-v1.5, the most common embeddings model used for Discourse
     class BgeLargeEnTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/bge-large-en.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("bge-large-en.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/bge_m3_tokenizer.rb

@@ -5,7 +5,10 @@ module DiscourseAi
     # Tokenizer used in bge-m3, a capable multilingual long context embeddings model.
     class BgeM3Tokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/bge-m3.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("bge-m3.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/gemini_tokenizer.rb

@@ -5,7 +5,10 @@ module DiscourseAi
     # Tokenizer from Gemma3, which is said to be the same for Gemini
     class GeminiTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/gemma3.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("gemma3.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/llama3_tokenizer.rb

@@ -6,7 +6,11 @@ module DiscourseAi
     class Llama3Tokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/Meta-Llama-3-70B-Instruct.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path(
+              "Meta-Llama-3-70B-Instruct.json"
+            )
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/mistral_tokenizer.rb

@@ -6,7 +6,11 @@ module DiscourseAi
     class MistralTokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/mistral-small-3.1-24b-2503.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path(
+              "mistral-small-3.1-24b-2503.json"
+            )
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/multilingual_e5_large_tokenizer.rb

@@ -6,7 +6,9 @@ module DiscourseAi
     class MultilingualE5LargeTokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/multilingual-e5-large.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("multilingual-e5-large.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/qwen_tokenizer.rb

@@ -5,7 +5,10 @@ module DiscourseAi
     # Tokenizer from Qwen3 LLM series. Also compatible with their embedding models
     class QwenTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/qwen3.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("qwen3.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

data/lib/discourse_ai/tokenizers.rb

@@ -20,6 +20,13 @@ module DiscourseAi
   module Tokenizers
     class Error < StandardError
     end
-    # Your code goes here...
+
+    def self.gem_root
+      @gem_root ||= File.expand_path("../../..", __FILE__)
+    end
+
+    def self.vendor_path(filename)
+      File.join(gem_root, "vendor", filename)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Rafael Silva