llm_memory 0.1.11 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c72d46a390a534d27ea5ee90bfb2950b75fd16f5320af2a431a314bcee13be30
-  data.tar.gz: 974f529b0158cf5b4a8d4330dca59fe2fd3c2ee37cdca6465d98abee0f84db18
+  metadata.gz: 783f2bf830c8e3bf6ac8f033c200bcdc068ac51e23c444a15a28ce4c3fdb3faf
+  data.tar.gz: 0705cd834c8877f368bb8827e7916f72ec81d1e70839b2455fd27f030b1d9346
 SHA512:
-  metadata.gz: 697d7a93bc637ec00004e68cd83dac1077e9705706e6b0123131acfaac201ce7979eeef63772bae35c53fa1b95ea7123f23399e5ee0ea1b454c689b6aa0f6a33
-  data.tar.gz: ab67fd9ec6c1297b5308dfc28b9475bab46e28aa3b4f2af1694a9053ed7203c80e478320c11663a49583e43762814b85e0bdc52d2ea7002274a282609b1db9bd
+  metadata.gz: 6adf9915342427085ec647c1475f4169159d8670731b76100820bd13780d43ab7e74e31e36f8eed48453ea7f3a63dd1d4da11e784ab6124a16dfe0a853c17199
+  data.tar.gz: fecd053b1881b4d52a6f587469b7ffb50501d1e75b100eedc27a33039805dfdccb33b2904cb0482754bcbce9c5d034369364db28b7749bcd511eac51176c01a2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm_memory (0.1.11)
+    llm_memory (0.1.13)
       redis (~> 4.6.0)
       ruby-openai (~> 3.7.0)
       tokenizers (~> 0.3.3)

data/README.md

@@ -54,8 +54,10 @@ end
 
 To use LLM Memory, follow these steps:
 
+If you want to use pgvector instead of Redisearch. You can use the plugin. Please check the doc and change the setup steps(2&3)
+
 1. Install the gem: gem install llm_memory
-2. Set up Redis with Redisearch module enabled - Go to [Reids Cloud](https://redis.com/redis-enterprise-cloud/overview/) and get the redis url
+2. Set up Redis with Redisearch module enabled - Go to [Reids Cloud](https://redis.com/redis-enterprise-cloud/overview/) and get the redis url.
 3. Configure LLM Memory to connect to your Redis instance
 4. Use LlmMemory::Wernicke to load data from your external sources
 5. Use LlmMemory::Hippocampus to search for relevant information based on user queries
@@ -71,12 +73,13 @@ docs = LlmMemory::Wernicke.load(:file, "/tmp/a_directory")
 # docs = [{
 #   content: "Hi there",
 #   metadata: {
-#     file_name: "a.txt"
+#     file_name: "a.txt",
+#     timestamp: "20201231235959"
 #   }
 # },,,]
 
 hippocampus = LlmMemory::Hippocampus.new
-hippocampus.memorize(docs)
+res = hippocampus.memorize(docs)
 
 query_str = "What is my name?"
 related_docs = hippocampus.query(query_str, limit: 3)

data/lib/llm_memory/hippocampus.rb

@@ -57,6 +57,14 @@ module LlmMemory
       @store.drop_index if @store.index_exists?
     end
 
+    def forget(key)
+      @store.delete(key)
+    end
+
+    def list(array = [])
+      @store.list(array)
+    end
+
     def add_vectors(docs)
       # embed documents and add vector
       result = []

data/lib/llm_memory/loaders/file_loader.rb

@@ -14,11 +14,13 @@ module LlmMemory
 
         file_name = File.basename(file_path)
         file_content = File.read(file_path)
+        ctime = File.ctime(file_path)
 
         files_array << {
           content: file_content,
           metadata: {
-            file_name: file_name
+            file_name: file_name,
+            timestamp: ctime.strftime("%Y%m%d%H%M%S") # YYMMDDHHmmss
           }
         }
       end

data/lib/llm_memory/store.rb

@@ -27,6 +27,14 @@ module LlmMemory
       raise NotImplementedError, "Each store must implement the 'add' method."
     end
 
+    def list
+      raise NotImplementedError, "Each store must implement the 'list' method."
+    end
+
+    def delete
+      raise NotImplementedError, "Each store must implement the 'delete' method."
+    end
+
     def search
       raise NotImplementedError, "Each store must implement the 'search' method."
     end

data/lib/llm_memory/stores/redis_store.rb

@@ -79,7 +79,12 @@ module LlmMemory
       result = {}
       @client.pipelined do |pipeline|
         data.each_with_index do |d, i|
-          key = "#{@index_name}:#{SecureRandom.uuid.delete("-")}"
+          key = @index_name # index_name:create_time:metadata_timestamp:uuid
+          timestamp = d.dig(:metadata, :timestamp)
+          key += ":#{Time.now.strftime("%Y%m%d%H%M%S")}"
+          key += ":#{timestamp}"
+          key += ":#{SecureRandom.hex(8)}"
+
           meta_json = d[:metadata].nil? ? "" : d[:metadata].to_json # serialize
           vector_value = d[:vector].map(&:to_f).pack("f*")
           pipeline.hset(
@@ -118,7 +123,33 @@ module LlmMemory
       puts "Unexpected Error: #{e.message}"
     end
 
-    def delete
+    def delete(key)
+      @client.del(key) if @client.exists?(key)
+    end
+
+    def delete_all
+      list.keys.each do |key|
+        delete(key)
+      end
+    end
+
+    def list(patterns = [])
+      patterns = if patterns.empty?
+        ["#{@index_name}:*"]
+      else
+        patterns.map { |pattern| "#{@index_name}:#{pattern}" }
+      end
+      data = {}
+      patterns.each do |pattern|
+        @client.keys(pattern).each do |key|
+          obj = @client.hgetall(key)
+          data[key] = {
+            content: obj.dig("content"),
+            metadata: JSON.parse(obj.dig("metadata"))
+          }
+        end
+      end
+      data
     end
 
     def update

data/lib/llm_memory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmMemory
-  VERSION = "0.1.11"
+  VERSION = "0.1.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llm_memory
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.13
 platform: ruby
 authors:
 - Shohei Kameda
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-02 00:00:00.000000000 Z
+date: 2023-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tokenizers