llm_memory 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85f3330ed767cc28b5a3276a7772678c36726ce04562c184a4e6dcf287d14d1d
-  data.tar.gz: 3e1ae83b2517f7f80bb857de5024b62b886bb6a045428f2d40159e48818cf286
+  metadata.gz: aea28ca9fb65d35a8d98964ac433cb7c445d8137a2672bcf542d35fa31935582
+  data.tar.gz: 02d4d7a619eb3031df0310c2fa9bef9105035f732bc1d9e5381a3dd2d8ded836
 SHA512:
-  metadata.gz: dba158d07c4a97a5b2e6bb1a98987173ba427a18a1a06d0a422f366ab2f7335ac168aea0351657b0c7f957352e6ffe747a2d817deb7c8d5454010c2ec535999a
-  data.tar.gz: 1db3ccb501f3b56d6e6dc00997f3f085b74f395521ffbbc1775bc5799bc388410523f6f209cfc185efbbf1437177c07dd5ee1725a49f6a27b207c6fc296067aa
+  metadata.gz: d30618749a0b4016a2ca9cd2815cb6b7b4971a46c50ea83fa3e5b30d1e0813127053b576882587634f166cff7312f3898c6183f71a32ea724b53deea5d676936
+  data.tar.gz: 4957d9857a4a5b05cd725b45e44c44b9ecfb695034e6c2e6c45ed80c9c819d0b581fdc4f9977d4af9180b15af983f544df62507e5d25001f0e508ff90acfa8b6

data/Gemfile

@@ -11,7 +11,7 @@ gem "standard", "~> 1.3"
 gem "vcr", "~> 6.1.0"
 gem "webmock", "~> 3.18.1"
 gem "ruby-openai"
-gem "tiktoken_ruby"
+gem "tokenizers"
 gem "redis"
 # dev
 gem "dotenv"

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.3)
+    llm_memory (0.1.5)
       redis (~> 4.6.0)
       ruby-openai (~> 3.7.0)
-      tiktoken_ruby (~> 0.0.4)
+      tokenizers (~> 0.3.3)
 
 GEM
   remote: https://rubygems.org/
@@ -81,8 +81,8 @@ GEM
     standard-performance (1.0.1)
       lint_roller (~> 1.0)
       rubocop-performance (~> 1.16.0)
-    tiktoken_ruby (0.0.4-arm64-darwin)
-    tiktoken_ruby (0.0.4-x86_64-linux)
+    tokenizers (0.3.3-arm64-darwin)
+    tokenizers (0.3.3-x86_64-linux)
     unicode-display_width (2.4.2)
     vcr (6.1.0)
     webmock (3.18.1)
@@ -103,7 +103,7 @@ DEPENDENCIES
   rspec (~> 3.0)
   ruby-openai
   standard (~> 1.3)
-  tiktoken_ruby
+  tokenizers
   vcr (~> 6.1.0)
   webmock (~> 3.18.1)
 

data/lib/llm_memory/broca.rb

@@ -1,5 +1,5 @@
 require "erb"
-require "tiktoken_ruby"
+require "tokenizers"
 
 module LlmMemory
   class Broca
@@ -51,9 +51,9 @@ module LlmMemory
       count = 0
       new_messages = []
       @messages.reverse_each do |message|
-        encoded = tokenizer.encode(message[:content])
+        encoded = tokenizer.encode(message[:content], add_special_tokens: true)
         if count < @max_token
-          count += encoded.length
+          count += encoded.tokens.length
           new_messages.push(message)
         else
           break
@@ -63,7 +63,7 @@ module LlmMemory
     end
 
     def tokenizer
-      @tokenizer ||= Tiktoken.encoding_for_model("gpt-4")
+      @tokenizer ||= Tokenizers.from_pretrained("gpt2")
     end
   end
 end

data/lib/llm_memory/hippocampus.rb

@@ -23,7 +23,7 @@ module LlmMemory
       raise "Store '#{store_name}' not found." unless store_class
       @store = store_class.new(index_name: index_name)
 
-      # word count, not char count
+      # char count, not word count
       @chunk_size = chunk_size
       @chunk_overlap = chunk_overlap
     end
@@ -87,18 +87,14 @@ module LlmMemory
       docs.each do |item|
         content = item[:content]
         metadata = item[:metadata]
-        words = content.split
-
-        if words.length > @chunk_size
+        if content.length > @chunk_size
           start_index = 0
-
-          while start_index < words.length
-            end_index = [start_index + @chunk_size, words.length].min
-            chunk_words = words[start_index...end_index]
-            chunk = chunk_words.join(" ")
+          while start_index < content.length
+            end_index = [start_index + @chunk_size, content.length].min
+            chunk = content[start_index...end_index]
             result << {content: chunk, metadata: metadata}
-
-            start_index += @chunk_size - @chunk_overlap # Move index to create a overlap
+            break if end_index == content.length
+            start_index += @chunk_size - @chunk_overlap
           end
         else
           result << {content: content, metadata: metadata}

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

data/llm_memory.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "example-gem", "~> 1.0"
-  spec.add_dependency "tiktoken_ruby", "~> 0.0.4"
+  spec.add_dependency "tokenizers", "~> 0.3.3"
   spec.add_dependency "ruby-openai", "~> 3.7.0"
   spec.add_dependency "redis", "~> 4.6.0"
 

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-10 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: tiktoken_ruby
+  name: tokenizers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: 0.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: 0.3.3
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement