discourse_ai-tokenizers 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12f6645ef62162c14c3c1c43af8dae6dc890e96853207cf32575106c4b080d7e
-  data.tar.gz: 78c9b157f4a0bd490e4f8ea5f10f3c4793ef8f98baa05d8a773f0b5cd55e0e9c
+  metadata.gz: b9e8c362a4c2c227617258dee4d75e3d5555e1c4fde8575cd060a0efa2eb36b9
+  data.tar.gz: e9b3a6f950399628d6faec4bcf5a472c556a8f936c8fe82d511c63077f2d6c64
 SHA512:
-  metadata.gz: 715b68088a90292e82280b0a144a8b3040bd5e4f9720937f876f63d8ba91f711db07c62c9a8be1f48746b1fd38606a2b69b063b712f5949069763e7a63a5f010
-  data.tar.gz: 9e01d44c11d1f88edb838fb876d178b94e69497c73b38c70c4ca215a6296c0bffcf412c59924d1dcf7a64d147db3a30e4cc30d0b864ed02e65d7a0e07a758d03
+  metadata.gz: 69ef898cca64debd9d89297caf97531a9523e6d1fac769eaf86db05aca5aa3568b1e54b5065e09bda35b67deb447e38a1ef31997e3fb23ac0fa77d0a12af2983
+  data.tar.gz: b5561b0d77c81fb5076adfbd74a4232ba1a0c8b50fbb1de0c1f59de0c0f8753e79890f9439b375146dc28b59db434da3003d49cae789b3cd19ddcfb3f94c3ecb

data/lib/discourse_ai/{tokenizers → tokenizer}/all_mpnet_base_v2_tokenizer.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer for the mpnet based embeddings models
     class AllMpnetBaseV2Tokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/all-mpnet-base-v2.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("all-mpnet-base-v2.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/anthropic_tokenizer.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Extracted from Anthropic's python SDK, compatible with first Claude versions
     class AnthropicTokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/claude-v1-tokenization.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("claude-v1-tokenization.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/basic_tokenizer.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Base class for tokenizers to inherit from
     class BasicTokenizer
       class << self
         def available_llm_tokenizers
           [
-            DiscourseAi::Tokenizers::AnthropicTokenizer,
-            DiscourseAi::Tokenizers::GeminiTokenizer,
-            DiscourseAi::Tokenizers::Llama3Tokenizer,
-            DiscourseAi::Tokenizers::MistralTokenizer,
-            DiscourseAi::Tokenizers::OpenAiTokenizer,
-            DiscourseAi::Tokenizers::QwenTokenizer
+            DiscourseAi::Tokenizer::AnthropicTokenizer,
+            DiscourseAi::Tokenizer::GeminiTokenizer,
+            DiscourseAi::Tokenizer::Llama3Tokenizer,
+            DiscourseAi::Tokenizer::MistralTokenizer,
+            DiscourseAi::Tokenizer::OpenAiTokenizer,
+            DiscourseAi::Tokenizer::QwenTokenizer
           ]
         end
 

data/lib/discourse_ai/{tokenizers → tokenizer}/bert_tokenizer.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Bert tokenizer, useful for lots of embeddings and small classification models
     class BertTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/bert-base-uncased.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("bert-base-uncased.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/bge_large_en_tokenizer.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer used in bge-large-en-v1.5, the most common embeddings model used for Discourse
     class BgeLargeEnTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/bge-large-en.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("bge-large-en.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/bge_m3_tokenizer.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer used in bge-m3, a capable multilingual long context embeddings model.
     class BgeM3Tokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/bge-m3.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("bge-m3.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/gemini_tokenizer.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Gemma3, which is said to be the same for Gemini
     class GeminiTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/gemma3.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("gemma3.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/llama3_tokenizer.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Llama3, popular open weights LLM
     class Llama3Tokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/Meta-Llama-3-70B-Instruct.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path(
+              "Meta-Llama-3-70B-Instruct.json"
+            )
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/mistral_tokenizer.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Mistral Small 2503 LLM
     class MistralTokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/mistral-small-3.1-24b-2503.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path(
+              "mistral-small-3.1-24b-2503.json"
+            )
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/multilingual_e5_large_tokenizer.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from multilingual-e5-large, first multilingual embeddings model used in Discourse
     class MultilingualE5LargeTokenizer < BasicTokenizer
       def self.tokenizer
         @tokenizer ||=
-          ::Tokenizers.from_file("vendor/multilingual-e5-large.json")
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("multilingual-e5-large.json")
+          )
       end
     end
   end

data/lib/discourse_ai/{tokenizers → tokenizer}/open_ai_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Wrapper for OpenAI tokenizer library for compatibility with Discourse AI API
     class OpenAiTokenizer < BasicTokenizer
       class << self

data/lib/discourse_ai/{tokenizers → tokenizer}/qwen_tokenizer.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Qwen3 LLM series. Also compatible with their embedding models
     class QwenTokenizer < BasicTokenizer
       def self.tokenizer
-        @tokenizer ||= ::Tokenizers.from_file("vendor/qwen3.json")
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("qwen3.json")
+          )
       end
     end
   end

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.1.0"
+    VERSION = "0.1.2"
   end
 end

data/lib/discourse_ai/tokenizers.rb

@@ -3,23 +3,31 @@
 require "tokenizers"
 require "tiktoken_ruby"
 require_relative "tokenizers/version"
-require_relative "tokenizers/basic_tokenizer"
-require_relative "tokenizers/bert_tokenizer"
-require_relative "tokenizers/anthropic_tokenizer"
-require_relative "tokenizers/open_ai_tokenizer"
-require_relative "tokenizers/all_mpnet_base_v2_tokenizer"
-require_relative "tokenizers/multilingual_e5_large_tokenizer"
-require_relative "tokenizers/bge_large_en_tokenizer"
-require_relative "tokenizers/bge_m3_tokenizer"
-require_relative "tokenizers/llama3_tokenizer"
-require_relative "tokenizers/gemini_tokenizer"
-require_relative "tokenizers/qwen_tokenizer"
-require_relative "tokenizers/mistral_tokenizer"
+
+require_relative "tokenizer/basic_tokenizer"
+require_relative "tokenizer/bert_tokenizer"
+require_relative "tokenizer/anthropic_tokenizer"
+require_relative "tokenizer/open_ai_tokenizer"
+require_relative "tokenizer/all_mpnet_base_v2_tokenizer"
+require_relative "tokenizer/multilingual_e5_large_tokenizer"
+require_relative "tokenizer/bge_large_en_tokenizer"
+require_relative "tokenizer/bge_m3_tokenizer"
+require_relative "tokenizer/llama3_tokenizer"
+require_relative "tokenizer/gemini_tokenizer"
+require_relative "tokenizer/qwen_tokenizer"
+require_relative "tokenizer/mistral_tokenizer"
 
 module DiscourseAi
   module Tokenizers
     class Error < StandardError
     end
-    # Your code goes here...
+
+    def self.gem_root
+      @gem_root ||= File.expand_path("../../..", __FILE__)
+    end
+
+    def self.vendor_path(filename)
+      File.join(gem_root, "vendor", filename)
+    end
   end
 end

data/sig/discourse_ai/tokenizers.rbs

@@ -1,5 +1,5 @@
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     VERSION: String
     # See the writing guide of rbs: https://github.com/ruby/rbs#guides
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Rafael Silva
@@ -96,19 +96,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/discourse_ai/tokenizer/all_mpnet_base_v2_tokenizer.rb
+- lib/discourse_ai/tokenizer/anthropic_tokenizer.rb
+- lib/discourse_ai/tokenizer/basic_tokenizer.rb
+- lib/discourse_ai/tokenizer/bert_tokenizer.rb
+- lib/discourse_ai/tokenizer/bge_large_en_tokenizer.rb
+- lib/discourse_ai/tokenizer/bge_m3_tokenizer.rb
+- lib/discourse_ai/tokenizer/gemini_tokenizer.rb
+- lib/discourse_ai/tokenizer/llama3_tokenizer.rb
+- lib/discourse_ai/tokenizer/mistral_tokenizer.rb
+- lib/discourse_ai/tokenizer/multilingual_e5_large_tokenizer.rb
+- lib/discourse_ai/tokenizer/open_ai_tokenizer.rb
+- lib/discourse_ai/tokenizer/qwen_tokenizer.rb
 - lib/discourse_ai/tokenizers.rb
-- lib/discourse_ai/tokenizers/all_mpnet_base_v2_tokenizer.rb
-- lib/discourse_ai/tokenizers/anthropic_tokenizer.rb
-- lib/discourse_ai/tokenizers/basic_tokenizer.rb
-- lib/discourse_ai/tokenizers/bert_tokenizer.rb
-- lib/discourse_ai/tokenizers/bge_large_en_tokenizer.rb
-- lib/discourse_ai/tokenizers/bge_m3_tokenizer.rb
-- lib/discourse_ai/tokenizers/gemini_tokenizer.rb
-- lib/discourse_ai/tokenizers/llama3_tokenizer.rb
-- lib/discourse_ai/tokenizers/mistral_tokenizer.rb
-- lib/discourse_ai/tokenizers/multilingual_e5_large_tokenizer.rb
-- lib/discourse_ai/tokenizers/open_ai_tokenizer.rb
-- lib/discourse_ai/tokenizers/qwen_tokenizer.rb
 - lib/discourse_ai/tokenizers/version.rb
 - sig/discourse_ai/tokenizers.rbs
 - vendor/Meta-Llama-3-70B-Instruct.json