llm_memory 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3f9540b8e3ca30d11eacd2e3100b4aa4ef985d041017b2b09bd5fad52d2f776
-  data.tar.gz: 9998bca0ea546d93b3a051ac7a6c62a9597ae2da9e85d8d5b36aab4212c59641
+  metadata.gz: b81b1033f4e0ade103fd1c73317e0ef575244225ccce51fab50f232f2d138e37
+  data.tar.gz: 945bd74694fc40c020df96763dfe96df2b21d402648b2b7f948c526a49532b7c
 SHA512:
-  metadata.gz: 9923ca576cf858eac89facf8da2f88d6284f7577366b252bb4e3ab197879a78035dfacba62e7890ab097dc193ee922f2b6abf16b321d4f1b09e6f1d30e4e9184
-  data.tar.gz: 44ff089feebead7cc9e1ef8b555212d20a938fcd6a4c72c13cfb55ae3029ebf6699722ea42e977ac108375e5d91c83aad450177ec2b9ad1037f87a83b83eadf7
+  metadata.gz: 357abfa9fcc316129c40e3fc869cab7460812e32ea1005efa2b039b84d537f6a307f6fb80f112920bc0c3e4be2fe02fa0af94e8451b9f1e6280658e79513133b
+  data.tar.gz: 818c51b4c18f0277d99ec85dbf0e80c88d05caea894035cd0cbb90cab9e3853b2f760b4a1ff8690afa00172d1ae7656df1db2a7c24618d73fc5fafa39b855458

data/Gemfile

@@ -13,3 +13,6 @@ gem "webmock", "~> 3.18.1"
 gem "ruby-openai"
 gem "tiktoken_ruby"
 gem "redis"
+# dev
+gem "dotenv"
+gem "pry"

data/Gemfile.lock

@@ -9,10 +9,12 @@ GEM
     addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
+    coderay (1.1.3)
     connection_pool (2.4.0)
     crack (0.4.5)
       rexml
     diff-lcs (1.5.0)
+    dotenv (2.8.1)
     faraday (2.7.4)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
@@ -23,10 +25,14 @@ GEM
     json (2.6.3)
     language_server-protocol (3.17.0.3)
     lint_roller (1.0.0)
+    method_source (1.0.0)
     multipart-post (2.3.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (5.0.1)
     rainbow (3.1.1)
     rake (13.0.6)
@@ -81,6 +87,7 @@ GEM
       lint_roller (~> 1.0)
       rubocop-performance (~> 1.16.0)
     tiktoken_ruby (0.0.4-arm64-darwin)
+    tiktoken_ruby (0.0.4-x86_64-linux)
     unicode-display_width (2.4.2)
     vcr (6.1.0)
     webmock (3.18.1)
@@ -93,7 +100,9 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  dotenv
   llm_memory!
+  pry
   rake (~> 13.0)
   redis
   rspec (~> 3.0)

data/README.md

@@ -14,6 +14,9 @@ This enables better integration with systems such as Rails and web services whil
 
 ## LLM Memory Components
 
+![image](https://user-images.githubusercontent.com/1880965/236099477-421b2003-79d2-4a7c-8f80-1afac4fd616d.png)
+
+
 1. LlmMemory::Wernicke: Responsible for loading external data (currently from files). More loader types are planned for future development.
 
 > Wernicke's area in brain is involved in the comprehension of written and spoken language
@@ -28,15 +31,13 @@ This enables better integration with systems such as Rails and web services whil
 
 ## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
-
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ bundle add llm_memory
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ gem install llm_memory
 
 ### Setup
 
@@ -55,7 +56,7 @@ end
 To use LLM Memory, follow these steps:
 
 1. Install the gem: gem install llm_memory
-2. Set up Redis with Redisearch module enabled
+2. Set up Redis with Redisearch module enabled - Go to [Reids Cloud](https://redis.com/redis-enterprise-cloud/overview/) and get the redis url
 3. Configure LLM Memory to connect to your Redis instance
 4. Use LlmMemory::Wernicke to load data from your external sources
 5. Use LlmMemory::Hippocampus to search for relevant information based on user queries

data/lib/llm_memory/broca.rb

@@ -12,6 +12,7 @@ module LlmMemory
       temperature: 0.7,
       max_token: 4096
     )
+      LlmMemory.configure
       @prompt = prompt
       @model = model
       @messages = []

data/lib/llm_memory/hippocampus.rb

@@ -13,6 +13,8 @@ module LlmMemory
       store_name: :redis,
       index_name: "llm_memory"
     )
+      LlmMemory.configure
+
       embedding_class = EmbeddingManager.embeddings[embedding_name]
       raise "Embedding '#{embedding_name}' not found." unless embedding_class
       @embedding_instance = embedding_class.new
@@ -26,7 +28,20 @@ module LlmMemory
       @chunk_overlap = chunk_overlap
     end
 
+    # validate the document format
+    def validate_documents(documents)
+      is_valid = documents.all? do |hash|
+        hash.is_a?(Hash) &&
+          hash.key?(:content) && hash[:content].is_a?(String) &&
+          hash.key?(:metadata) && hash[:metadata].is_a?(Hash)
+      end
+      unless is_valid
+        raise "Your documents need to have an array of hashes (content: string and metadata: hash)"
+      end
+    end
+
     def memorize(docs)
+      validate_documents(docs)
       docs = make_chunks(docs)
       docs = add_vectors(docs)
       @store.create_index unless @store.index_exists?

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/llm_memory.rb

@@ -20,5 +20,4 @@ module LlmMemory
     self.configuration ||= Configuration.new
     yield(configuration) if block_given?
   end
-  configure # init for default values
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-04 00:00:00.000000000 Z
+date: 2023-05-09 00:00:00.000000000 Z
 dependencies: []
 description: LLM Memory is a Ruby gem designed to provide large language models (LLMs)
   like ChatGPT with memory using in-context learning. This enables better integration