discourse_ai-tokenizers 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc2fd76c9fd338fd19f6d56d4e21c98695d2ff8a2baf5626f36f7df7f98af3d9
-  data.tar.gz: d5e9b1ca74715a0346a3d22e2413834191488167e867d5bec795b6f9c9c25f5d
+  metadata.gz: b9e8c362a4c2c227617258dee4d75e3d5555e1c4fde8575cd060a0efa2eb36b9
+  data.tar.gz: e9b3a6f950399628d6faec4bcf5a472c556a8f936c8fe82d511c63077f2d6c64
 SHA512:
-  metadata.gz: 4b06f6c801f878f173471a337f0d9d28c3321ba0b7b089876b83296353144257080bc0e1495bdbe020914f4cb6324696f9505a625cd01545b17d6e99624f190f
-  data.tar.gz: f7c37e3d464b16419c7218554a29514c63d84d1d4247969940d2f9d7f3316618ae593aa98df8941a44d2c721471f97073b75b37b2246a18349f43b6a324adb8a
+  metadata.gz: 69ef898cca64debd9d89297caf97531a9523e6d1fac769eaf86db05aca5aa3568b1e54b5065e09bda35b67deb447e38a1ef31997e3fb23ac0fa77d0a12af2983
+  data.tar.gz: b5561b0d77c81fb5076adfbd74a4232ba1a0c8b50fbb1de0c1f59de0c0f8753e79890f9439b375146dc28b59db434da3003d49cae789b3cd19ddcfb3f94c3ecb

data/lib/discourse_ai/{tokenizers → tokenizer}/all_mpnet_base_v2_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer for the mpnet based embeddings models
     class AllMpnetBaseV2Tokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/anthropic_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Extracted from Anthropic's python SDK, compatible with first Claude versions
     class AnthropicTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/basic_tokenizer.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Base class for tokenizers to inherit from
     class BasicTokenizer
       class << self
         def available_llm_tokenizers
           [
-            DiscourseAi::Tokenizers::AnthropicTokenizer,
-            DiscourseAi::Tokenizers::GeminiTokenizer,
-            DiscourseAi::Tokenizers::Llama3Tokenizer,
-            DiscourseAi::Tokenizers::MistralTokenizer,
-            DiscourseAi::Tokenizers::OpenAiTokenizer,
-            DiscourseAi::Tokenizers::QwenTokenizer
+            DiscourseAi::Tokenizer::AnthropicTokenizer,
+            DiscourseAi::Tokenizer::GeminiTokenizer,
+            DiscourseAi::Tokenizer::Llama3Tokenizer,
+            DiscourseAi::Tokenizer::MistralTokenizer,
+            DiscourseAi::Tokenizer::OpenAiTokenizer,
+            DiscourseAi::Tokenizer::QwenTokenizer
           ]
         end
 

data/lib/discourse_ai/{tokenizers → tokenizer}/bert_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Bert tokenizer, useful for lots of embeddings and small classification models
     class BertTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/bge_large_en_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer used in bge-large-en-v1.5, the most common embeddings model used for Discourse
     class BgeLargeEnTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/bge_m3_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer used in bge-m3, a capable multilingual long context embeddings model.
     class BgeM3Tokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/gemini_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Gemma3, which is said to be the same for Gemini
     class GeminiTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/llama3_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Llama3, popular open weights LLM
     class Llama3Tokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/mistral_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Mistral Small 2503 LLM
     class MistralTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/multilingual_e5_large_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from multilingual-e5-large, first multilingual embeddings model used in Discourse
     class MultilingualE5LargeTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/open_ai_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Wrapper for OpenAI tokenizer library for compatibility with Discourse AI API
     class OpenAiTokenizer < BasicTokenizer
       class << self

data/lib/discourse_ai/{tokenizers → tokenizer}/qwen_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Qwen3 LLM series. Also compatible with their embedding models
     class QwenTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/lib/discourse_ai/tokenizers.rb

@@ -3,18 +3,19 @@
 require "tokenizers"
 require "tiktoken_ruby"
 require_relative "tokenizers/version"
-require_relative "tokenizers/basic_tokenizer"
-require_relative "tokenizers/bert_tokenizer"
-require_relative "tokenizers/anthropic_tokenizer"
-require_relative "tokenizers/open_ai_tokenizer"
-require_relative "tokenizers/all_mpnet_base_v2_tokenizer"
-require_relative "tokenizers/multilingual_e5_large_tokenizer"
-require_relative "tokenizers/bge_large_en_tokenizer"
-require_relative "tokenizers/bge_m3_tokenizer"
-require_relative "tokenizers/llama3_tokenizer"
-require_relative "tokenizers/gemini_tokenizer"
-require_relative "tokenizers/qwen_tokenizer"
-require_relative "tokenizers/mistral_tokenizer"
+
+require_relative "tokenizer/basic_tokenizer"
+require_relative "tokenizer/bert_tokenizer"
+require_relative "tokenizer/anthropic_tokenizer"
+require_relative "tokenizer/open_ai_tokenizer"
+require_relative "tokenizer/all_mpnet_base_v2_tokenizer"
+require_relative "tokenizer/multilingual_e5_large_tokenizer"
+require_relative "tokenizer/bge_large_en_tokenizer"
+require_relative "tokenizer/bge_m3_tokenizer"
+require_relative "tokenizer/llama3_tokenizer"
+require_relative "tokenizer/gemini_tokenizer"
+require_relative "tokenizer/qwen_tokenizer"
+require_relative "tokenizer/mistral_tokenizer"
 
 module DiscourseAi
   module Tokenizers

data/sig/discourse_ai/tokenizers.rbs

@@ -1,5 +1,5 @@
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     VERSION: String
     # See the writing guide of rbs: https://github.com/ruby/rbs#guides
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Rafael Silva
@@ -96,19 +96,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/discourse_ai/tokenizer/all_mpnet_base_v2_tokenizer.rb
+- lib/discourse_ai/tokenizer/anthropic_tokenizer.rb
+- lib/discourse_ai/tokenizer/basic_tokenizer.rb
+- lib/discourse_ai/tokenizer/bert_tokenizer.rb
+- lib/discourse_ai/tokenizer/bge_large_en_tokenizer.rb
+- lib/discourse_ai/tokenizer/bge_m3_tokenizer.rb
+- lib/discourse_ai/tokenizer/gemini_tokenizer.rb
+- lib/discourse_ai/tokenizer/llama3_tokenizer.rb
+- lib/discourse_ai/tokenizer/mistral_tokenizer.rb
+- lib/discourse_ai/tokenizer/multilingual_e5_large_tokenizer.rb
+- lib/discourse_ai/tokenizer/open_ai_tokenizer.rb
+- lib/discourse_ai/tokenizer/qwen_tokenizer.rb
 - lib/discourse_ai/tokenizers.rb
-- lib/discourse_ai/tokenizers/all_mpnet_base_v2_tokenizer.rb
-- lib/discourse_ai/tokenizers/anthropic_tokenizer.rb
-- lib/discourse_ai/tokenizers/basic_tokenizer.rb
-- lib/discourse_ai/tokenizers/bert_tokenizer.rb
-- lib/discourse_ai/tokenizers/bge_large_en_tokenizer.rb
-- lib/discourse_ai/tokenizers/bge_m3_tokenizer.rb
-- lib/discourse_ai/tokenizers/gemini_tokenizer.rb
-- lib/discourse_ai/tokenizers/llama3_tokenizer.rb
-- lib/discourse_ai/tokenizers/mistral_tokenizer.rb
-- lib/discourse_ai/tokenizers/multilingual_e5_large_tokenizer.rb
-- lib/discourse_ai/tokenizers/open_ai_tokenizer.rb
-- lib/discourse_ai/tokenizers/qwen_tokenizer.rb
 - lib/discourse_ai/tokenizers/version.rb
 - sig/discourse_ai/tokenizers.rbs
 - vendor/Meta-Llama-3-70B-Instruct.json