llm_memory 0.1.7 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c20af963efe2764e2360a0e7c5149afc18b6c757b7c24dc178b3efe190330c20
-  data.tar.gz: 90d3ee12fffc237846e9bd6342ad9863e257bed7ae573c378673cb6c1b4a42f5
+  metadata.gz: 723ea7f6c27fd43b5cb447f476e802af917b678c68e3aa999bbad1807a950b20
+  data.tar.gz: e65bf3f2d0555d0c101f5fe550f436760dcecd28b57e67f54d89a1bcecde3ab3
 SHA512:
-  metadata.gz: 2240230ab5c96a4f8642f01114da452207d086e8a35432bf69c8652a170307b200fdacb8f48713c3027074d712bb33628da905a5aec1d5afb2c6db9659723d8f
-  data.tar.gz: 907c03f50e230d7fccbd60244ff6a9c412c5dc7962b1202585c3e56619203c7e3b5dc57be2881d49228167d8296827958315b6e823b69a3e35c32dd08db3be36
+  metadata.gz: 975b970c1f2a8c447470755b74b115f3411b072f9bb9e2536c52462cdcedd120c93a342df169dd955ae4cbc9267569393039c5b319fea9487e1c49105a5cbf91
+  data.tar.gz: feb02279f9f41132237ef2649d0f3582a83d85c16240d8a79cbc0e8d85c485b659f2d7d2458367af5446ba08a856992bfd12f9bbd9c97510ccbee99bbf1140c6

data/Gemfile

@@ -13,6 +13,7 @@ gem "webmock", "~> 3.18.1"
 gem "ruby-openai"
 gem "tokenizers"
 gem "redis"
+gem "llm_memory_pgvector"
 # dev
 gem "dotenv"
 gem "pry"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.7)
+    llm_memory (0.1.9)
       redis (~> 4.6.0)
       ruby-openai (~> 3.7.0)
       tokenizers (~> 0.3.3)
@@ -24,12 +24,18 @@ GEM
     json (2.6.3)
     language_server-protocol (3.17.0.3)
     lint_roller (1.0.0)
+    llm_memory_pgvector (0.1.2)
+      llm_memory (~> 0.1.7)
+      pg (~> 1.5.3)
+      pgvector (~> 0.2.0)
     method_source (1.0.0)
     mini_mime (1.1.2)
     multi_xml (0.6.0)
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
+    pg (1.5.3)
+    pgvector (0.2.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -97,6 +103,7 @@ PLATFORMS
 DEPENDENCIES
   dotenv
   llm_memory!
+  llm_memory_pgvector
   pry
   rake (~> 13.0)
   redis

data/README.md

@@ -14,7 +14,7 @@ This enables better integration with systems such as Rails and web services whil
 
 ## LLM Memory Components
 
-![image](https://user-images.githubusercontent.com/1880965/236099477-421b2003-79d2-4a7c-8f80-1afac4fd616d.png)
+![llm_memory_diagram](https://github.com/shohey1226/llm_memory/assets/1880965/b77d0efa-3fec-4549-b98a-eae510de5c3d)
 
 1. LlmMemory::Wernicke: Responsible for loading external data (currently from files). More loader types are planned for future development.
 
@@ -61,6 +61,8 @@ To use LLM Memory, follow these steps:
 5. Use LlmMemory::Hippocampus to search for relevant information based on user queries
 6. Create and use ERB templates with LlmMemory::Broca to generate responses based on the information retrieved
 
+For the details of each class, please refer to [API reference document](https://github.com/shohey1226/llm_memory/wiki/API-Reference).
+
 ```ruby
 docs = LlmMemory::Wernicke.load(:file, "/tmp/a_directory")
 # docs is just an array of hash.
@@ -111,9 +113,28 @@ message2 = broca.respond(query_str: query_str2, related_docs: related_docs)
 
 The table below provides a list of plugins utilized by llm_memory. The aim is to keep the core llm_memory lightweight while allowing for easy extensibility through the use of plugins.
 
-| Plugin Name             | Type   | Module   | Link                                                          |
-| ----------------------- | ------ | -------- | ------------------------------------------------------------- |
-| llm_memory_gmail_loader | Loader | Wernicke | [link](https://github.com/shohey1226/llm_memory_gmail_loader) |
+Install the plugin and update the method.
+
+For example, if you wan to use pgvector. then,
+
+```
+$ bundle add llm_memory_pgvector
+```
+
+Then, load it instead of `:redis` (default is redis).
+
+```ruby
+# may need to have require depending on the project
+# require llm_memory_pgvector
+hippocamups = LlmMemory::Hippocampus.new(store: :pgvector)`
+```
+
+Please refer to the links for the details.
+
+| Plugin Name             | Type   | Module      | Link                                                          |
+| ----------------------- | ------ | ----------- | ------------------------------------------------------------- |
+| llm_memory_gmail_loader | Loader | Wernicke    | [link](https://github.com/shohey1226/llm_memory_gmail_loader) |
+| llm_memory_pgvector     | Store  | Hippocampus | [link](https://github.com/shohey1226/llm_memory_pgvector)     |
 
 ## Development
 
@@ -123,7 +144,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/llm_memory. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/llm_memory/blob/master/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/shohey1226/llm_memory. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/shohey1226/llm_memory/blob/master/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -131,4 +152,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the LlmMemory project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/llm_memory/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the LlmMemory project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shohey1226/llm_memory/blob/master/CODE_OF_CONDUCT.md).

data/lib/llm_memory/hippocampus.rb

@@ -10,7 +10,7 @@ module LlmMemory
       embedding_name: :openai,
       chunk_size: 1024,
       chunk_overlap: 50,
-      store_name: :redis,
+      store: :redis,
       index_name: "llm_memory"
     )
       LlmMemory.configure
@@ -19,8 +19,8 @@ module LlmMemory
       raise "Embedding '#{embedding_name}' not found." unless embedding_class
       @embedding_instance = embedding_class.new
 
-      store_class = StoreManager.stores[store_name]
-      raise "Store '#{store_name}' not found." unless store_class
+      store_class = StoreManager.stores[store]
+      raise "Store '#{store}' not found." unless store_class
       @store = store_class.new(index_name: index_name)
 
       # char count, not word count
@@ -50,17 +50,7 @@ module LlmMemory
 
     def query(query_str, limit: 3)
       vector = @embedding_instance.embed_document(query_str)
-      response_list = @store.search(query: vector, k: limit)
-      response_list.shift # the first one is the size
-      # now [redis_key1, [],,, ]
-      result = response_list.each_slice(2).to_h.values.map { |v|
-        v.each_slice(2).to_h.transform_keys(&:to_sym)
-      }
-      result.each do |item|
-        hash = JSON.parse(item[:metadata])
-        item[:metadata] = hash.transform_keys(&:to_sym)
-      end
-      result
+      @store.search(query: vector, k: limit)
     end
 
     def forget_all

data/lib/llm_memory/stores/redis_store.rb

@@ -148,7 +148,17 @@ module LlmMemory
         "DIALECT",
         2
       ]
-      @client.call(command)
+      response_list = @client.call(command)
+      response_list.shift # the first one is the size
+      # now [redis_key1, [],,, ]
+      result = response_list.each_slice(2).to_h.values.map { |v|
+        v.each_slice(2).to_h.transform_keys(&:to_sym)
+      }
+      result.each do |item|
+        hash = JSON.parse(item[:metadata])
+        item[:metadata] = hash.transform_keys(&:to_sym)
+      end
+      result
     end
   end
 end

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.7"
+  VERSION = "0.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.9
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-11 00:00:00.000000000 Z
+date: 2023-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tokenizers