llm_memory 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 072b5fa983000a18bccb6e5d0185663cbb30547a0fd6cc6194e1d4235ae833fb
-  data.tar.gz: 7233ecf0cbbb254b08cbf7d1fe747b8374e1e3baa3b6214a1ba5fa64ecd0360d
+  metadata.gz: 72e01db010504ae3ec7877d5122ecba01bf7c1f15140e7ed94d9e66c62e2ea40
+  data.tar.gz: 8d170f4f7fda8c1af72fe62bb7873f3758fba9960babb1dc562b048bb4201f09
 SHA512:
-  metadata.gz: 9e670ee60c3e780343317d2b2175ea5cf9fefdaa8b4ce81b13ec1b32caa3cbd13ad4363dec63ebaabf803e434897eb34a21d4b4e0b37b60bb4b3ff22b6a9a80c
-  data.tar.gz: e4defb25011469c10b36c6d4b98763dcb56a673eaedb2709741769de0fc888a8d30e4301f5444c88dbb07c31b8a290ae0fbfae584d9d93a21c6c7ce34d96c40b
+  metadata.gz: be7ac41f195ae267c60b917f5640be23bf3a458fea0d6fd9de1084367db6b89bc517a6285fc4459ad5f0a36fb82257bf7cb0dc925233910a567f15fb74e1fb7e
+  data.tar.gz: '0888eea1d469818a6ea2272b164218d934b3a1591e7e7179d74d5748e4e3765082a0ea40a3177793a8be60b82f702341f8eb4ac387639c8d219cbb09de263908'

data/Gemfile

@@ -11,7 +11,7 @@ gem "standard", "~> 1.3"
 gem "vcr", "~> 6.1.0"
 gem "webmock", "~> 3.18.1"
 gem "ruby-openai"
-gem "tiktoken_ruby"
+gem "tokenizers"
 gem "redis"
 # dev
 gem "dotenv"

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.4)
+    llm_memory (0.1.6)
       redis (~> 4.6.0)
       ruby-openai (~> 3.7.0)
-      tiktoken_ruby (~> 0.0.4)
+      tokenizers (~> 0.3.3)
 
 GEM
   remote: https://rubygems.org/
@@ -81,8 +81,8 @@ GEM
     standard-performance (1.0.1)
       lint_roller (~> 1.0)
       rubocop-performance (~> 1.16.0)
-    tiktoken_ruby (0.0.4-arm64-darwin)
-    tiktoken_ruby (0.0.4-x86_64-linux)
+    tokenizers (0.3.3-arm64-darwin)
+    tokenizers (0.3.3-x86_64-linux)
     unicode-display_width (2.4.2)
     vcr (6.1.0)
     webmock (3.18.1)
@@ -103,7 +103,7 @@ DEPENDENCIES
   rspec (~> 3.0)
   ruby-openai
   standard (~> 1.3)
-  tiktoken_ruby
+  tokenizers
   vcr (~> 6.1.0)
   webmock (~> 3.18.1)
 

data/lib/llm_memory/broca.rb

@@ -1,5 +1,5 @@
 require "erb"
-require "tiktoken_ruby"
+require "tokenizers"
 
 module LlmMemory
   class Broca
@@ -32,11 +32,12 @@ module LlmMemory
             temperature: @temperature
           }
         )
+        LlmMemory.logger.debug(response)
         response_content = response.dig("choices", 0, "message", "content")
-        @messages.push({role: "system", content: response_content})
+        @messages.push({role: "system", content: response_content}) unless response_content.nil?
         response_content
       rescue => e
-        puts e.inspect
+        LlmMemory.logger.info(e.inspect)
         # @messages = []
         nil
       end
@@ -51,9 +52,10 @@ module LlmMemory
       count = 0
       new_messages = []
       @messages.reverse_each do |message|
-        encoded = tokenizer.encode(message[:content])
-        if count < @max_token
-          count += encoded.length
+        encoded = tokenizer.encode(message[:content], add_special_tokens: true)
+        token_count = encoded.tokens.length
+        count += token_count
+        if count <= @max_token
           new_messages.push(message)
         else
           break
@@ -63,7 +65,7 @@ module LlmMemory
     end
 
     def tokenizer
-      @tokenizer ||= Tiktoken.encoding_for_model("gpt-4")
+      @tokenizer ||= Tokenizers.from_pretrained("gpt2")
     end
   end
 end

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.4"
+  VERSION = "0.1.6"
 end

data/lib/llm_memory.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
+require "logger"
 # config
 require_relative "llm_memory/configuration"
-
 require_relative "llm_memory/hippocampus"
 require_relative "llm_memory/broca"
 require_relative "llm_memory/wernicke"
-
 require_relative "llm_memory/version"
 
 module LlmMemory
   class Error < StandardError; end
 
   class << self
-    attr_accessor :configuration
+    attr_accessor :configuration, :log_level
+
+    def logger
+      @logger ||= Logger.new($stdout).tap do |logger|
+        logger.level = log_level || Logger::INFO
+      end
+    end
   end
 
   def self.configure

data/llm_memory.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "example-gem", "~> 1.0"
-  spec.add_dependency "tiktoken_ruby", "~> 0.0.4"
+  spec.add_dependency "tokenizers", "~> 0.3.3"
   spec.add_dependency "ruby-openai", "~> 3.7.0"
   spec.add_dependency "redis", "~> 4.6.0"
 

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-10 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: tiktoken_ruby
+  name: tokenizers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: 0.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: 0.3.3
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement