llm_memory 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3f9540b8e3ca30d11eacd2e3100b4aa4ef985d041017b2b09bd5fad52d2f776
-  data.tar.gz: 9998bca0ea546d93b3a051ac7a6c62a9597ae2da9e85d8d5b36aab4212c59641
+  metadata.gz: 0615d07f9e76aa0b4c762d027e757bb0cb03cc1448b11d5b5cf684d5a75b381a
+  data.tar.gz: 6aa24ea5ddf2476876ca2d873400018785c3bcc6c385faf35cac3f59de94049a
 SHA512:
-  metadata.gz: 9923ca576cf858eac89facf8da2f88d6284f7577366b252bb4e3ab197879a78035dfacba62e7890ab097dc193ee922f2b6abf16b321d4f1b09e6f1d30e4e9184
-  data.tar.gz: 44ff089feebead7cc9e1ef8b555212d20a938fcd6a4c72c13cfb55ae3029ebf6699722ea42e977ac108375e5d91c83aad450177ec2b9ad1037f87a83b83eadf7
+  metadata.gz: 7f6ac569b18f28502e3b0e34614498b559dff4b6e085988f3bae2f5160aef9eb6491a7e61f78edc3d5e0c9be0ac8878f4e0747cc2d02358aa389071eea393c0c
+  data.tar.gz: b818a177d945d9e2544f1079f214231d792419772688b4907a50a8ceb2fde2435d5571e4e03307fa1bee8f8d22499e7ddf5821363203b1a1ecbd473d4edbdc2d

data/Gemfile

@@ -13,3 +13,6 @@ gem "webmock", "~> 3.18.1"
 gem "ruby-openai"
 gem "tiktoken_ruby"
 gem "redis"
+# dev
+gem "dotenv"
+gem "pry"

data/Gemfile.lock

@@ -1,7 +1,10 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.0)
+    llm_memory (0.1.2)
+      redis (~> 4.6.0)
+      ruby-openai (~> 3.7.0)
+      tiktoken_ruby (~> 0.0.4)
 
 GEM
   remote: https://rubygems.org/
@@ -9,31 +12,31 @@ GEM
     addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
-    connection_pool (2.4.0)
+    coderay (1.1.3)
     crack (0.4.5)
       rexml
     diff-lcs (1.5.0)
-    faraday (2.7.4)
-      faraday-net_http (>= 2.0, < 3.1)
-      ruby2_keywords (>= 0.0.4)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (3.0.2)
+    dotenv (2.8.1)
     hashdiff (1.0.1)
+    httparty (0.21.0)
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
     lint_roller (1.0.0)
-    multipart-post (2.3.0)
+    method_source (1.0.0)
+    mini_mime (1.1.2)
+    multi_xml (0.6.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (5.0.1)
     rainbow (3.1.1)
     rake (13.0.6)
-    redis (5.0.6)
-      redis-client (>= 0.9.0)
-    redis-client (0.14.1)
-      connection_pool
+    redis (4.6.0)
     regexp_parser (2.8.0)
     rexml (3.2.5)
     rspec (3.12.0)
@@ -64,11 +67,9 @@ GEM
     rubocop-performance (1.16.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    ruby-openai (4.0.0)
-      faraday (>= 1)
-      faraday-multipart (>= 1)
+    ruby-openai (3.7.0)
+      httparty (>= 0.18.1)
     ruby-progressbar (1.13.0)
-    ruby2_keywords (0.0.5)
     standard (1.28.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
@@ -81,6 +82,7 @@ GEM
       lint_roller (~> 1.0)
       rubocop-performance (~> 1.16.0)
     tiktoken_ruby (0.0.4-arm64-darwin)
+    tiktoken_ruby (0.0.4-x86_64-linux)
     unicode-display_width (2.4.2)
     vcr (6.1.0)
     webmock (3.18.1)
@@ -93,7 +95,9 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  dotenv
   llm_memory!
+  pry
   rake (~> 13.0)
   redis
   rspec (~> 3.0)

data/README.md

@@ -14,6 +14,8 @@ This enables better integration with systems such as Rails and web services whil
 
 ## LLM Memory Components
 
+![image](https://user-images.githubusercontent.com/1880965/236099477-421b2003-79d2-4a7c-8f80-1afac4fd616d.png)
+
 1. LlmMemory::Wernicke: Responsible for loading external data (currently from files). More loader types are planned for future development.
 
 > Wernicke's area in brain is involved in the comprehension of written and spoken language
@@ -28,15 +30,13 @@ This enables better integration with systems such as Rails and web services whil
 
 ## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
-
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ bundle add llm_memory
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ gem install llm_memory
 
 ### Setup
 
@@ -55,7 +55,7 @@ end
 To use LLM Memory, follow these steps:
 
 1. Install the gem: gem install llm_memory
-2. Set up Redis with Redisearch module enabled
+2. Set up Redis with Redisearch module enabled - Go to [Reids Cloud](https://redis.com/redis-enterprise-cloud/overview/) and get the redis url
 3. Configure LLM Memory to connect to your Redis instance
 4. Use LlmMemory::Wernicke to load data from your external sources
 5. Use LlmMemory::Hippocampus to search for relevant information based on user queries
@@ -107,6 +107,14 @@ related_docs = hippocampus.query(query_str2, limit: 3)
 message2 = broca.respond(query_str: query_str2, related_docs: related_docs)
 ```
 
+## Plugins
+
+The table below provides a list of plugins utilized by llm_memory. The aim is to keep the core llm_memory lightweight while allowing for easy extensibility through the use of plugins.
+
+| Plugin Name             | Type   | Module   | Link                                                          |
+| ----------------------- | ------ | -------- | ------------------------------------------------------------- |
+| llm_memory_gmail_loader | Loader | Wernicke | [link](https://github.com/shohey1226/llm_memory_gmail_loader) |
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/llm_memory/broca.rb

@@ -12,6 +12,7 @@ module LlmMemory
       temperature: 0.7,
       max_token: 4096
     )
+      LlmMemory.configure
       @prompt = prompt
       @model = model
       @messages = []

data/lib/llm_memory/hippocampus.rb

@@ -13,6 +13,8 @@ module LlmMemory
       store_name: :redis,
       index_name: "llm_memory"
     )
+      LlmMemory.configure
+
       embedding_class = EmbeddingManager.embeddings[embedding_name]
       raise "Embedding '#{embedding_name}' not found." unless embedding_class
       @embedding_instance = embedding_class.new
@@ -26,7 +28,20 @@ module LlmMemory
       @chunk_overlap = chunk_overlap
     end
 
+    # validate the document format
+    def validate_documents(documents)
+      is_valid = documents.all? do |hash|
+        hash.is_a?(Hash) &&
+          hash.key?(:content) && hash[:content].is_a?(String) &&
+          hash.key?(:metadata) && hash[:metadata].is_a?(Hash)
+      end
+      unless is_valid
+        raise "Your documents need to have an array of hashes (content: string and metadata: hash)"
+      end
+    end
+
     def memorize(docs)
+      validate_documents(docs)
       docs = make_chunks(docs)
       docs = add_vectors(docs)
       @store.create_index unless @store.index_exists?

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/llm_memory.rb

@@ -20,5 +20,4 @@ module LlmMemory
     self.configuration ||= Configuration.new
     yield(configuration) if block_given?
   end
-  configure # init for default values
 end

data/llm_memory.gemspec

@@ -31,6 +31,9 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_dependency "tiktoken_ruby", "~> 0.0.4"
+  spec.add_dependency "ruby-openai", "~> 3.7.0"
+  spec.add_dependency "redis", "~> 4.6.0"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-04 00:00:00.000000000 Z
-dependencies: []
+date: 2023-05-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: tiktoken_ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.7.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.6.0
 description: LLM Memory is a Ruby gem designed to provide large language models (LLMs)
   like ChatGPT with memory using in-context learning. This enables better integration
   with systems such as Rails and web services while providing a more user-friendly