discourse_ai-tokenizers 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9e8c362a4c2c227617258dee4d75e3d5555e1c4fde8575cd060a0efa2eb36b9
-  data.tar.gz: e9b3a6f950399628d6faec4bcf5a472c556a8f936c8fe82d511c63077f2d6c64
+  metadata.gz: d23181327ee259c76aa29f86a2b40702b39524705d11afd14239dfe6e3e90009
+  data.tar.gz: 76c2d7bbe1c4ebe97dff5576aec832e8664831ade0b819cf9afd244600f1be38
 SHA512:
-  metadata.gz: 69ef898cca64debd9d89297caf97531a9523e6d1fac769eaf86db05aca5aa3568b1e54b5065e09bda35b67deb447e38a1ef31997e3fb23ac0fa77d0a12af2983
-  data.tar.gz: b5561b0d77c81fb5076adfbd74a4232ba1a0c8b50fbb1de0c1f59de0c0f8753e79890f9439b375146dc28b59db434da3003d49cae789b3cd19ddcfb3f94c3ecb
+  metadata.gz: 02ad662edd31f57b8cba0b1ea221bb7c9f1684d65b9fcd1002739eebf4c1393152ee3f17ea675a8bd596eed4ba9a4fc61cea20b49bd7f3c5b86f64d0e2772bbb
+  data.tar.gz: 9869f1d01ce0388ac2bec619060e7ace28ad5155ff336667072d965a9405217b0ad21e36b258f384ef5456f3e3c3ee35b9c10c536a11e562c4ca5cb948c1ff81

data/CHANGELOG.md

@@ -1,5 +1,5 @@
 ## [Unreleased]
 
-## [0.1.0] - 2025-06-30
+## [0.2.0] - 2025-07-02
 
 - Initial release

data/README.md

@@ -39,27 +39,27 @@ gem install discourse_ai-tokenizers
 require 'discourse_ai/tokenizers'
 
 # Get token count
-DiscourseAi::Tokenizers::OpenAiTokenizer.size("Hello world!")
+DiscourseAi::Tokenizer::OpenAiTokenizer.size("Hello world!")
 # => 3
 
 # Tokenize text
-DiscourseAi::Tokenizers::OpenAiTokenizer.tokenize("Hello world!")
+DiscourseAi::Tokenizer::OpenAiTokenizer.tokenize("Hello world!")
 # => [9906, 1917, 0]
 
 # Encode text to token IDs
-DiscourseAi::Tokenizers::OpenAiTokenizer.encode("Hello world!")
+DiscourseAi::Tokenizer::OpenAiTokenizer.encode("Hello world!")
 # => [9906, 1917, 0]
 
 # Decode token IDs back to text
-DiscourseAi::Tokenizers::OpenAiTokenizer.decode([9906, 1917, 0])
+DiscourseAi::Tokenizer::OpenAiTokenizer.decode([9906, 1917, 0])
 # => "Hello world!"
 
 # Truncate text to token limit
-DiscourseAi::Tokenizers::OpenAiTokenizer.truncate("This is a long sentence", 5)
+DiscourseAi::Tokenizer::OpenAiTokenizer.truncate("This is a long sentence", 5)
 # => "This is a"
 
 # Check if text is within token limit
-DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Short text", 10)
+DiscourseAi::Tokenizer::OpenAiTokenizer.below_limit?("Short text", 10)
 # => true
 ```
 
@@ -67,27 +67,27 @@ DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Short text", 10)
 
 #### LLM Tokenizers
 
-- `DiscourseAi::Tokenizers::AnthropicTokenizer` - Claude models
-- `DiscourseAi::Tokenizers::OpenAiTokenizer` - GPT models
-- `DiscourseAi::Tokenizers::GeminiTokenizer` - Google Gemini
-- `DiscourseAi::Tokenizers::Llama3Tokenizer` - Meta Llama 3
-- `DiscourseAi::Tokenizers::QwenTokenizer` - Alibaba Qwen
-- `DiscourseAi::Tokenizers::MistralTokenizer` - Mistral models
+- `DiscourseAi::Tokenizer::AnthropicTokenizer` - Claude models
+- `DiscourseAi::Tokenizer::OpenAiTokenizer` - GPT models
+- `DiscourseAi::Tokenizer::GeminiTokenizer` - Google Gemini
+- `DiscourseAi::Tokenizer::Llama3Tokenizer` - Meta Llama 3
+- `DiscourseAi::Tokenizer::QwenTokenizer` - Alibaba Qwen
+- `DiscourseAi::Tokenizer::MistralTokenizer` - Mistral models
 
 #### Embedding Tokenizers
 
-- `DiscourseAi::Tokenizers::BertTokenizer` - BERT-based models
-- `DiscourseAi::Tokenizers::AllMpnetBaseV2Tokenizer` - sentence-transformers/all-mpnet-base-v2
-- `DiscourseAi::Tokenizers::BgeLargeEnTokenizer` - BAAI/bge-large-en
-- `DiscourseAi::Tokenizers::BgeM3Tokenizer` - BAAI/bge-m3
-- `DiscourseAi::Tokenizers::MultilingualE5LargeTokenizer` - intfloat/multilingual-e5-large
+- `DiscourseAi::Tokenizer::BertTokenizer` - BERT-based models
+- `DiscourseAi::Tokenizer::AllMpnetBaseV2Tokenizer` - sentence-transformers/all-mpnet-base-v2
+- `DiscourseAi::Tokenizer::BgeLargeEnTokenizer` - BAAI/bge-large-en
+- `DiscourseAi::Tokenizer::BgeM3Tokenizer` - BAAI/bge-m3
+- `DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer` - intfloat/multilingual-e5-large
 
 ### Getting Available LLM Tokenizers
 
 ```ruby
 # Get all available LLM tokenizers dynamically
-DiscourseAi::Tokenizers::BasicTokenizer.available_llm_tokenizers
-# => [DiscourseAi::Tokenizers::AnthropicTokenizer, DiscourseAi::Tokenizers::OpenAiTokenizer, ...]
+DiscourseAi::Tokenizer::BasicTokenizer.available_llm_tokenizers
+# => [DiscourseAi::Tokenizer::AnthropicTokenizer, DiscourseAi::Tokenizer::OpenAiTokenizer, ...]
 ```
 
 ### Advanced Usage
@@ -96,10 +96,10 @@ DiscourseAi::Tokenizers::BasicTokenizer.available_llm_tokenizers
 
 ```ruby
 # Strict mode ensures exact token limit compliance
-DiscourseAi::Tokenizers::OpenAiTokenizer.truncate("Long text here", 5, strict: true)
+DiscourseAi::Tokenizer::OpenAiTokenizer.truncate("Long text here", 5, strict: true)
 
 # Check limits with strict mode
-DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Text", 10, strict: true)
+DiscourseAi::Tokenizer::OpenAiTokenizer.below_limit?("Text", 10, strict: true)
 ```
 
 #### Unicode and Emoji Support
@@ -107,11 +107,11 @@ DiscourseAi::Tokenizers::OpenAiTokenizer.below_limit?("Text", 10, strict: true)
 ```ruby
 # Handles unicode characters properly
 text = "Hello 世界 🌍 👨‍👩‍👧‍👦"
-DiscourseAi::Tokenizers::OpenAiTokenizer.size(text)
+DiscourseAi::Tokenizer::OpenAiTokenizer.size(text)
 # => 8
 
 # Truncation preserves unicode integrity
-truncated = DiscourseAi::Tokenizers::OpenAiTokenizer.truncate(text, 5)
+truncated = DiscourseAi::Tokenizer::OpenAiTokenizer.truncate(text, 5)
 # => "Hello 世界 🌍"
 ```
 
@@ -157,4 +157,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the DiscourseAi::Tokenizers project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](CODE_OF_CONDUCT.md).
+Everyone interacting in the DiscourseAi::Tokenizer project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](CODE_OF_CONDUCT.md).

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.1.2"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Rafael Silva