discourse_ai-tokenizers 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc2fd76c9fd338fd19f6d56d4e21c98695d2ff8a2baf5626f36f7df7f98af3d9
-  data.tar.gz: d5e9b1ca74715a0346a3d22e2413834191488167e867d5bec795b6f9c9c25f5d
+  metadata.gz: d23181327ee259c76aa29f86a2b40702b39524705d11afd14239dfe6e3e90009
+  data.tar.gz: 76c2d7bbe1c4ebe97dff5576aec832e8664831ade0b819cf9afd244600f1be38
 SHA512:
-  metadata.gz: 4b06f6c801f878f173471a337f0d9d28c3321ba0b7b089876b83296353144257080bc0e1495bdbe020914f4cb6324696f9505a625cd01545b17d6e99624f190f
-  data.tar.gz: f7c37e3d464b16419c7218554a29514c63d84d1d4247969940d2f9d7f3316618ae593aa98df8941a44d2c721471f97073b75b37b2246a18349f43b6a324adb8a
+  metadata.gz: 02ad662edd31f57b8cba0b1ea221bb7c9f1684d65b9fcd1002739eebf4c1393152ee3f17ea675a8bd596eed4ba9a4fc61cea20b49bd7f3c5b86f64d0e2772bbb
+  data.tar.gz: 9869f1d01ce0388ac2bec619060e7ace28ad5155ff336667072d965a9405217b0ad21e36b258f384ef5456f3e3c3ee35b9c10c536a11e562c4ca5cb948c1ff81

data/CHANGELOG.md

@@ -1,5 +1,5 @@
 ## [Unreleased]
 
-## [0.1.0] - 2025-06-30
+## [0.2.0] - 2025-07-02
 
 - Initial release

data/README.md

@@ -39,27 +39,27 @@ gem install discourse_ai-tokenizers
 require 'discourse_ai/tokenizers'
 
 # Get token count
-DiscourseAi::Tokenizers::OpenAiTokenizer.size("Hello world!")
+DiscourseAi::Tokenizer::OpenAiTokenizer.size("Hello world!")
 # => 3
 
 # Tokenize text
-DiscourseAi::Tokenizers::OpenAiTokenizer.tokenize("Hello world!")
+DiscourseAi::Tokenizer::OpenAiTokenizer.tokenize("Hello world!")
 # => [9906, 1917, 0]
 
 # Encode text to token IDs
-DiscourseAi::Tokenizers::OpenAiTokenizer.encode("Hello world!")
+DiscourseAi::Tokenizer::OpenAiTokenizer.encode("Hello world!")
 # => [9906, 1917, 0]
 
 # Decode token IDs back to text
-DiscourseAi::Tokenizers::OpenAiTokenizer.decode([9906, 1917, 0])
+DiscourseAi::Tokenizer::OpenAiTokenizer.decode([9906, 1917, 0])
 # => "Hello world!"
 
 # Truncate text to token limit
-DiscourseAi::Tokenizers::OpenAiTokenizer.truncate("This is a long sentence", 5)
+DiscourseAi::Tokenizer::OpenAiTokenizer.truncate("This is a long sentence", 5)
 # => "This is a"
 
 # Check if text is within token limit
-DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Short text", 10)
+DiscourseAi::Tokenizer::OpenAiTokenizer.below_limit?("Short text", 10)
 # => true
 ```
 
@@ -67,27 +67,27 @@ DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Short text", 10)
 
 #### LLM Tokenizers
 
-- `DiscourseAi::Tokenizers::AnthropicTokenizer` - Claude models
-- `DiscourseAi::Tokenizers::OpenAiTokenizer` - GPT models
-- `DiscourseAi::Tokenizers::GeminiTokenizer` - Google Gemini
-- `DiscourseAi::Tokenizers::Llama3Tokenizer` - Meta Llama 3
-- `DiscourseAi::Tokenizers::QwenTokenizer` - Alibaba Qwen
-- `DiscourseAi::Tokenizers::MistralTokenizer` - Mistral models
+- `DiscourseAi::Tokenizer::AnthropicTokenizer` - Claude models
+- `DiscourseAi::Tokenizer::OpenAiTokenizer` - GPT models
+- `DiscourseAi::Tokenizer::GeminiTokenizer` - Google Gemini
+- `DiscourseAi::Tokenizer::Llama3Tokenizer` - Meta Llama 3
+- `DiscourseAi::Tokenizer::QwenTokenizer` - Alibaba Qwen
+- `DiscourseAi::Tokenizer::MistralTokenizer` - Mistral models
 
 #### Embedding Tokenizers
 
-- `DiscourseAi::Tokenizers::BertTokenizer` - BERT-based models
-- `DiscourseAi::Tokenizers::AllMpnetBaseV2Tokenizer` - sentence-transformers/all-mpnet-base-v2
-- `DiscourseAi::Tokenizers::BgeLargeEnTokenizer` - BAAI/bge-large-en
-- `DiscourseAi::Tokenizers::BgeM3Tokenizer` - BAAI/bge-m3
-- `DiscourseAi::Tokenizers::MultilingualE5LargeTokenizer` - intfloat/multilingual-e5-large
+- `DiscourseAi::Tokenizer::BertTokenizer` - BERT-based models
+- `DiscourseAi::Tokenizer::AllMpnetBaseV2Tokenizer` - sentence-transformers/all-mpnet-base-v2
+- `DiscourseAi::Tokenizer::BgeLargeEnTokenizer` - BAAI/bge-large-en
+- `DiscourseAi::Tokenizer::BgeM3Tokenizer` - BAAI/bge-m3
+- `DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer` - intfloat/multilingual-e5-large
 
 ### Getting Available LLM Tokenizers
 
 ```ruby
 # Get all available LLM tokenizers dynamically
-DiscourseAi::Tokenizers::BasicTokenizer.available_llm_tokenizers
-# => [DiscourseAi::Tokenizers::AnthropicTokenizer, DiscourseAi::Tokenizers::OpenAiTokenizer, ...]
+DiscourseAi::Tokenizer::BasicTokenizer.available_llm_tokenizers
+# => [DiscourseAi::Tokenizer::AnthropicTokenizer, DiscourseAi::Tokenizer::OpenAiTokenizer, ...]
 ```
 
 ### Advanced Usage
@@ -96,10 +96,10 @@ DiscourseAi::Tokenizers::BasicTokenizer.available_llm_tokenizers
 
 ```ruby
 # Strict mode ensures exact token limit compliance
-DiscourseAi::Tokenizers::OpenAiTokenizer.truncate("Long text here", 5, strict: true)
+DiscourseAi::Tokenizer::OpenAiTokenizer.truncate("Long text here", 5, strict: true)
 
 # Check limits with strict mode
-DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Text", 10, strict: true)
+DiscourseAi::Tokenizer::OpenAiTokenizer.below_limit?("Text", 10, strict: true)
 ```
 
 #### Unicode and Emoji Support
@@ -107,11 +107,11 @@ DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Text", 10, strict: true)
 ```ruby
 # Handles unicode characters properly
 text = "Hello 世界 🌍 👨‍👩‍👧‍👦"
-DiscourseAi::Tokenizers::OpenAiTokenizer.size(text)
+DiscourseAi::Tokenizer::OpenAiTokenizer.size(text)
 # => 8
 
 # Truncation preserves unicode integrity
-truncated = DiscourseAi::Tokenizers::OpenAiTokenizer.truncate(text, 5)
+truncated = DiscourseAi::Tokenizer::OpenAiTokenizer.truncate(text, 5)
 # => "Hello 世界 🌍"
 ```
 
@@ -157,4 +157,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the DiscourseAi::Tokenizers project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](CODE_OF_CONDUCT.md).
+Everyone interacting in the DiscourseAi::Tokenizer project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](CODE_OF_CONDUCT.md).

data/lib/discourse_ai/{tokenizers → tokenizer}/all_mpnet_base_v2_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer for the mpnet based embeddings models
     class AllMpnetBaseV2Tokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/anthropic_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Extracted from Anthropic's python SDK, compatible with first Claude versions
     class AnthropicTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/basic_tokenizer.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Base class for tokenizers to inherit from
     class BasicTokenizer
       class << self
         def available_llm_tokenizers
           [
-            DiscourseAi::Tokenizers::AnthropicTokenizer,
-            DiscourseAi::Tokenizers::GeminiTokenizer,
-            DiscourseAi::Tokenizers::Llama3Tokenizer,
-            DiscourseAi::Tokenizers::MistralTokenizer,
-            DiscourseAi::Tokenizers::OpenAiTokenizer,
-            DiscourseAi::Tokenizers::QwenTokenizer
+            DiscourseAi::Tokenizer::AnthropicTokenizer,
+            DiscourseAi::Tokenizer::GeminiTokenizer,
+            DiscourseAi::Tokenizer::Llama3Tokenizer,
+            DiscourseAi::Tokenizer::MistralTokenizer,
+            DiscourseAi::Tokenizer::OpenAiTokenizer,
+            DiscourseAi::Tokenizer::QwenTokenizer
           ]
         end
 

data/lib/discourse_ai/{tokenizers → tokenizer}/bert_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Bert tokenizer, useful for lots of embeddings and small classification models
     class BertTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/bge_large_en_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer used in bge-large-en-v1.5, the most common embeddings model used for Discourse
     class BgeLargeEnTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/bge_m3_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer used in bge-m3, a capable multilingual long context embeddings model.
     class BgeM3Tokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/gemini_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Gemma3, which is said to be the same for Gemini
     class GeminiTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/llama3_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Llama3, popular open weights LLM
     class Llama3Tokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/mistral_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Mistral Small 2503 LLM
     class MistralTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/multilingual_e5_large_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from multilingual-e5-large, first multilingual embeddings model used in Discourse
     class MultilingualE5LargeTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/{tokenizers → tokenizer}/open_ai_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Wrapper for OpenAI tokenizer library for compatibility with Discourse AI API
     class OpenAiTokenizer < BasicTokenizer
       class << self

data/lib/discourse_ai/{tokenizers → tokenizer}/qwen_tokenizer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     # Tokenizer from Qwen3 LLM series. Also compatible with their embedding models
     class QwenTokenizer < BasicTokenizer
       def self.tokenizer

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.1.1"
+    VERSION = "0.2.0"
   end
 end

data/lib/discourse_ai/tokenizers.rb

@@ -3,18 +3,19 @@
 require "tokenizers"
 require "tiktoken_ruby"
 require_relative "tokenizers/version"
-require_relative "tokenizers/basic_tokenizer"
-require_relative "tokenizers/bert_tokenizer"
-require_relative "tokenizers/anthropic_tokenizer"
-require_relative "tokenizers/open_ai_tokenizer"
-require_relative "tokenizers/all_mpnet_base_v2_tokenizer"
-require_relative "tokenizers/multilingual_e5_large_tokenizer"
-require_relative "tokenizers/bge_large_en_tokenizer"
-require_relative "tokenizers/bge_m3_tokenizer"
-require_relative "tokenizers/llama3_tokenizer"
-require_relative "tokenizers/gemini_tokenizer"
-require_relative "tokenizers/qwen_tokenizer"
-require_relative "tokenizers/mistral_tokenizer"
+
+require_relative "tokenizer/basic_tokenizer"
+require_relative "tokenizer/bert_tokenizer"
+require_relative "tokenizer/anthropic_tokenizer"
+require_relative "tokenizer/open_ai_tokenizer"
+require_relative "tokenizer/all_mpnet_base_v2_tokenizer"
+require_relative "tokenizer/multilingual_e5_large_tokenizer"
+require_relative "tokenizer/bge_large_en_tokenizer"
+require_relative "tokenizer/bge_m3_tokenizer"
+require_relative "tokenizer/llama3_tokenizer"
+require_relative "tokenizer/gemini_tokenizer"
+require_relative "tokenizer/qwen_tokenizer"
+require_relative "tokenizer/mistral_tokenizer"
 
 module DiscourseAi
   module Tokenizers

data/sig/discourse_ai/tokenizers.rbs

@@ -1,5 +1,5 @@
 module DiscourseAi
-  module Tokenizers
+  module Tokenizer
     VERSION: String
     # See the writing guide of rbs: https://github.com/ruby/rbs#guides
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Rafael Silva
@@ -96,19 +96,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/discourse_ai/tokenizer/all_mpnet_base_v2_tokenizer.rb
+- lib/discourse_ai/tokenizer/anthropic_tokenizer.rb
+- lib/discourse_ai/tokenizer/basic_tokenizer.rb
+- lib/discourse_ai/tokenizer/bert_tokenizer.rb
+- lib/discourse_ai/tokenizer/bge_large_en_tokenizer.rb
+- lib/discourse_ai/tokenizer/bge_m3_tokenizer.rb
+- lib/discourse_ai/tokenizer/gemini_tokenizer.rb
+- lib/discourse_ai/tokenizer/llama3_tokenizer.rb
+- lib/discourse_ai/tokenizer/mistral_tokenizer.rb
+- lib/discourse_ai/tokenizer/multilingual_e5_large_tokenizer.rb
+- lib/discourse_ai/tokenizer/open_ai_tokenizer.rb
+- lib/discourse_ai/tokenizer/qwen_tokenizer.rb
 - lib/discourse_ai/tokenizers.rb
-- lib/discourse_ai/tokenizers/all_mpnet_base_v2_tokenizer.rb
-- lib/discourse_ai/tokenizers/anthropic_tokenizer.rb
-- lib/discourse_ai/tokenizers/basic_tokenizer.rb
-- lib/discourse_ai/tokenizers/bert_tokenizer.rb
-- lib/discourse_ai/tokenizers/bge_large_en_tokenizer.rb
-- lib/discourse_ai/tokenizers/bge_m3_tokenizer.rb
-- lib/discourse_ai/tokenizers/gemini_tokenizer.rb
-- lib/discourse_ai/tokenizers/llama3_tokenizer.rb
-- lib/discourse_ai/tokenizers/mistral_tokenizer.rb
-- lib/discourse_ai/tokenizers/multilingual_e5_large_tokenizer.rb
-- lib/discourse_ai/tokenizers/open_ai_tokenizer.rb
-- lib/discourse_ai/tokenizers/qwen_tokenizer.rb
 - lib/discourse_ai/tokenizers/version.rb
 - sig/discourse_ai/tokenizers.rbs
 - vendor/Meta-Llama-3-70B-Instruct.json