llmemory 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +31 -0
- data/lib/llmemory/configuration.rb +22 -2
- data/lib/llmemory/crypto/cipher.rb +147 -0
- data/lib/llmemory/crypto/field_helpers.rb +110 -0
- data/lib/llmemory/llm/anthropic.rb +2 -1
- data/lib/llmemory/llm/openai.rb +2 -1
- data/lib/llmemory/long_term/episodic/memory.rb +4 -3
- data/lib/llmemory/long_term/episodic/storage.rb +11 -4
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +19 -6
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +25 -3
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +22 -5
- data/lib/llmemory/long_term/file_based/storage.rb +11 -4
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +16 -10
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +24 -8
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +28 -14
- data/lib/llmemory/long_term/graph_based/memory.rb +4 -3
- data/lib/llmemory/long_term/graph_based/storage.rb +3 -2
- data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +47 -21
- data/lib/llmemory/long_term/procedural/memory.rb +4 -3
- data/lib/llmemory/long_term/procedural/storage.rb +11 -4
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +33 -13
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +25 -4
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +23 -6
- data/lib/llmemory/memory.rb +40 -8
- data/lib/llmemory/short_term/checkpoint.rb +3 -2
- data/lib/llmemory/short_term/stores/active_record_store.rb +12 -10
- data/lib/llmemory/short_term/stores/memory_store.rb +1 -1
- data/lib/llmemory/short_term/stores/postgres_store.rb +11 -5
- data/lib/llmemory/short_term/stores/redis_store.rb +7 -5
- data/lib/llmemory/short_term/stores.rb +7 -6
- data/lib/llmemory/vector_store/active_record_store.rb +24 -3
- data/lib/llmemory/vector_store/memory_store.rb +23 -3
- data/lib/llmemory/vector_store.rb +4 -3
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +2 -0
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8e44ccb1c23fc659d9607e1eb3181e598a57edb95f47b94df4060ad46bfe7c31
|
|
4
|
+
data.tar.gz: 5d80e1fefb1dd77cbdc8c0b4d24e588abaf6f18c570db10e3fc0b2c808b09461
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: edb2f849efc9a6d1dbabc0be0a1210bf21e77f33174b965e3b22e9df3ca90aa536e47152c75891e548d6b43a08d82cfda29afa59d3b2e777aa144f4a3900570f
|
|
7
|
+
data.tar.gz: a8c84411d9c262dee5b5a0c07835c05816283a17749c1b6b282c75694dc56a497d8fb9f2b18eeccd242a82d5dce36234342ede75a13e2739ad11f99fb23ae005
|
data/README.md
CHANGED
|
@@ -65,6 +65,12 @@ Llmemory.configure do |config|
|
|
|
65
65
|
config.long_term_store = :memory # or :file, :postgres, :active_record
|
|
66
66
|
config.long_term_storage_path = "./llmemory_data" # for :file
|
|
67
67
|
config.database_url = ENV["DATABASE_URL"] # for :postgres
|
|
68
|
+
|
|
69
|
+
# Optional encryption at rest (AES-256-GCM). Requires a key; isolates data
|
|
70
|
+
# cryptographically per key (e.g. per agent/user). See "Encryption at rest".
|
|
71
|
+
config.encryption_enabled = false
|
|
72
|
+
config.encryption_key = ENV["LLMEMORY_ENCRYPTION_KEY"]
|
|
73
|
+
|
|
68
74
|
config.time_decay_half_life_days = 30
|
|
69
75
|
config.max_retrieval_tokens = 2000
|
|
70
76
|
config.prune_after_days = 90
|
|
@@ -112,6 +118,31 @@ Llmemory.configure do |config|
|
|
|
112
118
|
end
|
|
113
119
|
```
|
|
114
120
|
|
|
121
|
+
## Encryption at rest
|
|
122
|
+
|
|
123
|
+
Optional AES-256-GCM encryption protects persisted memory. Without the key, stored data is unreadable — useful for isolating agents or tenants.
|
|
124
|
+
|
|
125
|
+
```ruby
|
|
126
|
+
# Global default key (applies to all Memory instances)
|
|
127
|
+
Llmemory.configure do |config|
|
|
128
|
+
config.encryption_enabled = true
|
|
129
|
+
config.encryption_key = ENV["LLMEMORY_ENCRYPTION_KEY"]
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
memory = Llmemory::Memory.new(user_id: "agent-1")
|
|
133
|
+
|
|
134
|
+
# Per-instance key override (isolates this agent even if global config differs)
|
|
135
|
+
memory = Llmemory::Memory.new(user_id: "agent-1", encryption_key: "tenant-specific-secret")
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**What is encrypted:** conversation checkpoints (redis/postgres/active_record), file-based facts/resources/categories, episodic/procedural documents, graph node names/types/predicates (deterministic) and properties (random IV). **Vector embeddings are not encrypted** (required for pgvector search); associated `text_content` metadata is encrypted.
|
|
139
|
+
|
|
140
|
+
**Trade-offs:**
|
|
141
|
+
- Database keyword search (`LIKE`, BM25 on encrypted columns) no longer works on ciphertext; file backends still search in memory after decrypt.
|
|
142
|
+
- `:memory` backends are in-process only and are **not** encrypted at rest.
|
|
143
|
+
- Existing plaintext data remains readable (markers `enc:v1:` / `encd:v1:`); new writes are encrypted when enabled.
|
|
144
|
+
- Deterministic encryption on graph identifiers leaks equality (same name ⇒ same ciphertext) but keeps graph traversal working.
|
|
145
|
+
|
|
115
146
|
## Long-Term Storage
|
|
116
147
|
|
|
117
148
|
Long-term memory can use different backends:
|
|
@@ -48,12 +48,14 @@ module Llmemory
|
|
|
48
48
|
:message_sanitizer_enabled,
|
|
49
49
|
:ttl_episodic_days,
|
|
50
50
|
:ttl_procedural_days,
|
|
51
|
-
:skill_mining_enabled
|
|
51
|
+
:skill_mining_enabled,
|
|
52
|
+
:encryption_enabled,
|
|
53
|
+
:encryption_key
|
|
52
54
|
|
|
53
55
|
def initialize
|
|
54
56
|
@llm_provider = :openai
|
|
55
57
|
@llm_api_key = ENV["OPENAI_API_KEY"]
|
|
56
|
-
@llm_model =
|
|
58
|
+
@llm_model = nil # falls back to the active provider's DEFAULT_MODEL
|
|
57
59
|
@llm_base_url = nil
|
|
58
60
|
@short_term_store = :memory
|
|
59
61
|
@redis_url = ENV["REDIS_URL"] || "redis://localhost:6379/0"
|
|
@@ -98,6 +100,8 @@ module Llmemory
|
|
|
98
100
|
@embedding_cache_max_entries = 10_000
|
|
99
101
|
@max_message_chars = 32_000
|
|
100
102
|
@message_sanitizer_enabled = false
|
|
103
|
+
@encryption_enabled = false
|
|
104
|
+
@encryption_key = ENV["LLMEMORY_ENCRYPTION_KEY"]
|
|
101
105
|
end
|
|
102
106
|
end
|
|
103
107
|
|
|
@@ -113,5 +117,21 @@ module Llmemory
|
|
|
113
117
|
def reset_configuration!
|
|
114
118
|
@configuration = Configuration.new
|
|
115
119
|
end
|
|
120
|
+
|
|
121
|
+
# Builds a Crypto::Cipher when encryption is enabled and a key is present;
|
|
122
|
+
# otherwise returns Crypto::NullCipher. An explicit non-empty instance key
|
|
123
|
+
# enables encryption even when the global flag is off.
|
|
124
|
+
def build_cipher(key = nil)
|
|
125
|
+
explicit_key = !key.nil? && !key.to_s.empty?
|
|
126
|
+
resolved = key.nil? ? configuration.encryption_key : key
|
|
127
|
+
enabled = configuration.encryption_enabled || explicit_key
|
|
128
|
+
if enabled && !resolved.to_s.empty?
|
|
129
|
+
require_relative "crypto/cipher"
|
|
130
|
+
Crypto::Cipher.new(resolved)
|
|
131
|
+
else
|
|
132
|
+
require_relative "crypto/cipher"
|
|
133
|
+
Crypto::NullCipher.new
|
|
134
|
+
end
|
|
135
|
+
end
|
|
116
136
|
end
|
|
117
137
|
end
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module Crypto
|
|
8
|
+
class DecryptionError < Llmemory::Error; end
|
|
9
|
+
|
|
10
|
+
# No-op cipher when encryption is disabled or no key is configured.
|
|
11
|
+
class NullCipher
|
|
12
|
+
def enabled?
|
|
13
|
+
false
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def encrypt(str)
|
|
17
|
+
str.to_s
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def encrypt_deterministic(str)
|
|
21
|
+
str.to_s
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def decrypt(str)
|
|
25
|
+
str.to_s
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def encrypt_json(obj)
|
|
29
|
+
JSON.generate(obj)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def decrypt_json(str)
|
|
33
|
+
JSON.parse(str.to_s, symbolize_names: true)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def encrypted?(str)
|
|
37
|
+
false
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# AES-256-GCM encryption with separate content (random IV) and index
|
|
42
|
+
# (deterministic IV) subkeys derived from the master key via HMAC-SHA256.
|
|
43
|
+
class Cipher
|
|
44
|
+
MARKER = "enc:v1:"
|
|
45
|
+
DETERMINISTIC_MARKER = "encd:v1:"
|
|
46
|
+
IV_LENGTH = 12
|
|
47
|
+
TAG_LENGTH = 16
|
|
48
|
+
|
|
49
|
+
def initialize(key)
|
|
50
|
+
@master_key = derive_master_key(key)
|
|
51
|
+
@content_key = derive_subkey("content")
|
|
52
|
+
@index_key = derive_subkey("index")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def enabled?
|
|
56
|
+
true
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def encrypt(plaintext)
|
|
60
|
+
str = plaintext.to_s
|
|
61
|
+
return str if str.empty?
|
|
62
|
+
|
|
63
|
+
encrypt_with_key(str, @content_key, iv: OpenSSL::Random.random_bytes(IV_LENGTH), marker: MARKER)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def encrypt_deterministic(plaintext)
|
|
67
|
+
str = plaintext.to_s
|
|
68
|
+
return str if str.empty?
|
|
69
|
+
|
|
70
|
+
iv = OpenSSL::HMAC.digest("SHA256", @index_key, str)[0, IV_LENGTH]
|
|
71
|
+
encrypt_with_key(str, @index_key, iv: iv, marker: DETERMINISTIC_MARKER)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def decrypt(ciphertext)
|
|
75
|
+
str = ciphertext.to_s
|
|
76
|
+
return str if str.empty?
|
|
77
|
+
return str unless encrypted?(str)
|
|
78
|
+
|
|
79
|
+
marker, key = if str.start_with?(DETERMINISTIC_MARKER)
|
|
80
|
+
[DETERMINISTIC_MARKER, @index_key]
|
|
81
|
+
else
|
|
82
|
+
[MARKER, @content_key]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
payload = decode64(str.delete_prefix(marker))
|
|
86
|
+
iv = payload[0, IV_LENGTH]
|
|
87
|
+
tag = payload[IV_LENGTH, TAG_LENGTH]
|
|
88
|
+
ct = payload[(IV_LENGTH + TAG_LENGTH)..]
|
|
89
|
+
|
|
90
|
+
cipher = OpenSSL::Cipher.new("aes-256-gcm")
|
|
91
|
+
cipher.decrypt
|
|
92
|
+
cipher.key = key
|
|
93
|
+
cipher.iv = iv
|
|
94
|
+
cipher.auth_tag = tag
|
|
95
|
+
cipher.auth_data = ""
|
|
96
|
+
cipher.update(ct) + cipher.final
|
|
97
|
+
rescue OpenSSL::Cipher::CipherError, ArgumentError => e
|
|
98
|
+
raise DecryptionError, "Failed to decrypt data: #{e.message}"
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def encrypt_json(obj)
|
|
102
|
+
encrypt(JSON.generate(obj))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def decrypt_json(str)
|
|
106
|
+
JSON.parse(decrypt(str), symbolize_names: true)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def encrypted?(str)
|
|
110
|
+
s = str.to_s
|
|
111
|
+
s.start_with?(MARKER) || s.start_with?(DETERMINISTIC_MARKER)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private
|
|
115
|
+
|
|
116
|
+
def encrypt_with_key(plaintext, key, iv:, marker:)
|
|
117
|
+
cipher = OpenSSL::Cipher.new("aes-256-gcm")
|
|
118
|
+
cipher.encrypt
|
|
119
|
+
cipher.key = key
|
|
120
|
+
cipher.iv = iv
|
|
121
|
+
cipher.auth_data = ""
|
|
122
|
+
ct = cipher.update(plaintext) + cipher.final
|
|
123
|
+
tag = cipher.auth_tag
|
|
124
|
+
marker + encode64(iv + tag + ct)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def encode64(bin)
|
|
128
|
+
[bin].pack("m0")
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def decode64(str)
|
|
132
|
+
str.unpack1("m0")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def derive_master_key(key)
|
|
136
|
+
raw = key.to_s
|
|
137
|
+
raise ConfigurationError, "encryption_key cannot be empty when encryption is enabled" if raw.empty?
|
|
138
|
+
|
|
139
|
+
OpenSSL::Digest::SHA256.digest(raw)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def derive_subkey(label)
|
|
143
|
+
OpenSSL::HMAC.digest("SHA256", @master_key, "llmemory:#{label}")[0, 32]
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Crypto
|
|
7
|
+
# Shared encrypt/decrypt helpers for storage backends.
|
|
8
|
+
module FieldHelpers
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def cipher
|
|
12
|
+
@cipher || Llmemory.build_cipher
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def enc(str)
|
|
16
|
+
return str if str.nil?
|
|
17
|
+
return str.to_s unless cipher.enabled?
|
|
18
|
+
|
|
19
|
+
cipher.encrypt(str.to_s)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def dec(str)
|
|
23
|
+
return str if str.nil?
|
|
24
|
+
return str unless str.is_a?(String) && cipher.encrypted?(str)
|
|
25
|
+
|
|
26
|
+
cipher.decrypt(str)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def enc_det(str)
|
|
30
|
+
return str if str.nil?
|
|
31
|
+
return str.to_s unless cipher.enabled?
|
|
32
|
+
|
|
33
|
+
cipher.encrypt_deterministic(str.to_s)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def enc_json(obj)
|
|
37
|
+
return obj if obj.nil?
|
|
38
|
+
return obj unless cipher.enabled?
|
|
39
|
+
|
|
40
|
+
cipher.encrypt_json(obj)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def dec_json(value)
|
|
44
|
+
return value if value.nil?
|
|
45
|
+
return value.transform_keys(&:to_sym) if value.is_a?(Hash)
|
|
46
|
+
return value unless value.is_a?(String) && cipher.encrypted?(value)
|
|
47
|
+
|
|
48
|
+
cipher.decrypt_json(value)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def write_encrypted_file(path, data)
|
|
52
|
+
payload = JSON.generate(data)
|
|
53
|
+
File.write(path, cipher.enabled? ? cipher.encrypt(payload) : payload)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def read_encrypted_file(path)
|
|
57
|
+
raw = File.read(path)
|
|
58
|
+
json = cipher.enabled? && cipher.encrypted?(raw) ? cipher.decrypt(raw) : raw
|
|
59
|
+
JSON.parse(json, symbolize_names: true)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def write_encrypted_text_file(path, content, append: false)
|
|
63
|
+
text = content.to_s
|
|
64
|
+
if cipher.enabled?
|
|
65
|
+
if append && File.file?(path)
|
|
66
|
+
existing = read_encrypted_text_file(path)
|
|
67
|
+
text = existing + text
|
|
68
|
+
end
|
|
69
|
+
File.write(path, cipher.encrypt(text))
|
|
70
|
+
elsif append && File.file?(path)
|
|
71
|
+
File.write(path, File.read(path) + text)
|
|
72
|
+
else
|
|
73
|
+
File.write(path, text)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def read_encrypted_text_file(path)
|
|
78
|
+
raw = File.read(path)
|
|
79
|
+
cipher.enabled? && cipher.encrypted?(raw) ? cipher.decrypt(raw) : raw
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def serialize_state(state)
|
|
83
|
+
json = JSON.generate(state)
|
|
84
|
+
return json unless cipher.enabled?
|
|
85
|
+
|
|
86
|
+
cipher.encrypt(json)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def deserialize_state(data)
|
|
90
|
+
if data.is_a?(Hash)
|
|
91
|
+
return data.transform_keys(&:to_sym)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
str = data.to_s
|
|
95
|
+
json = cipher.enabled? && cipher.encrypted?(str) ? cipher.decrypt(str) : str
|
|
96
|
+
JSON.parse(json, symbolize_names: true)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def parse_provenance(value)
|
|
100
|
+
return nil if value.nil?
|
|
101
|
+
return value.transform_keys(&:to_sym) if value.is_a?(Hash)
|
|
102
|
+
return dec_json(value) if value.is_a?(String) && cipher.encrypted?(value)
|
|
103
|
+
|
|
104
|
+
JSON.parse(value.to_s, symbolize_names: true)
|
|
105
|
+
rescue JSON::ParserError
|
|
106
|
+
nil
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -8,10 +8,11 @@ module Llmemory
|
|
|
8
8
|
module LLM
|
|
9
9
|
class Anthropic < Base
|
|
10
10
|
DEFAULT_BASE_URL = "https://api.anthropic.com"
|
|
11
|
+
DEFAULT_MODEL = "claude-sonnet-4-6"
|
|
11
12
|
|
|
12
13
|
def initialize(api_key: nil, model: nil, base_url: nil)
|
|
13
14
|
@api_key = api_key || config.llm_api_key || ENV["ANTHROPIC_API_KEY"]
|
|
14
|
-
@model = model || config.llm_model ||
|
|
15
|
+
@model = model || config.llm_model || DEFAULT_MODEL
|
|
15
16
|
@base_url = base_url || config.llm_base_url || DEFAULT_BASE_URL
|
|
16
17
|
end
|
|
17
18
|
|
data/lib/llmemory/llm/openai.rb
CHANGED
|
@@ -8,10 +8,11 @@ module Llmemory
|
|
|
8
8
|
module LLM
|
|
9
9
|
class OpenAI < Base
|
|
10
10
|
DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
|
11
|
+
DEFAULT_MODEL = "gpt-4"
|
|
11
12
|
|
|
12
13
|
def initialize(api_key: nil, model: nil, base_url: nil)
|
|
13
14
|
@api_key = api_key || config.llm_api_key
|
|
14
|
-
@model = model || config.llm_model
|
|
15
|
+
@model = model || config.llm_model || DEFAULT_MODEL
|
|
15
16
|
@base_url = base_url || config.llm_base_url || DEFAULT_BASE_URL
|
|
16
17
|
end
|
|
17
18
|
|
|
@@ -21,9 +21,10 @@ module Llmemory
|
|
|
21
21
|
|
|
22
22
|
attr_reader :user_id, :storage
|
|
23
23
|
|
|
24
|
-
def initialize(user_id:, storage: nil, vector_store: nil)
|
|
24
|
+
def initialize(user_id:, storage: nil, vector_store: nil, cipher: nil)
|
|
25
25
|
@user_id = user_id
|
|
26
|
-
@
|
|
26
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
27
|
+
@storage = storage || Storages.build(cipher: @cipher)
|
|
27
28
|
@vector_store = vector_store
|
|
28
29
|
@vector_explicit = !vector_store.nil?
|
|
29
30
|
end
|
|
@@ -125,7 +126,7 @@ module Llmemory
|
|
|
125
126
|
if @vector_explicit
|
|
126
127
|
@vector_store
|
|
127
128
|
elsif Llmemory.configuration.episodic_vector_enabled
|
|
128
|
-
@vector_store ||= Llmemory::VectorStore.build(source_type: "episode")
|
|
129
|
+
@vector_store ||= Llmemory::VectorStore.build(source_type: "episode", cipher: @cipher)
|
|
129
130
|
end
|
|
130
131
|
end
|
|
131
132
|
|
|
@@ -12,17 +12,24 @@ module Llmemory
|
|
|
12
12
|
Storage = Storages::MemoryStorage
|
|
13
13
|
|
|
14
14
|
module Storages
|
|
15
|
-
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
15
|
+
def self.build(store: nil, base_path: nil, database_url: nil, cipher: nil)
|
|
16
|
+
resolved_cipher = cipher || Llmemory.build_cipher
|
|
16
17
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
17
18
|
when :memory
|
|
18
19
|
MemoryStorage.new
|
|
19
20
|
when :file
|
|
20
|
-
FileStorage.new(
|
|
21
|
+
FileStorage.new(
|
|
22
|
+
base_path: base_path || Llmemory.configuration.long_term_storage_path,
|
|
23
|
+
cipher: resolved_cipher
|
|
24
|
+
)
|
|
21
25
|
when :postgres, :database
|
|
22
|
-
DatabaseStorage.new(
|
|
26
|
+
DatabaseStorage.new(
|
|
27
|
+
database_url: database_url || Llmemory.configuration.database_url,
|
|
28
|
+
cipher: resolved_cipher
|
|
29
|
+
)
|
|
23
30
|
when :active_record, :activerecord
|
|
24
31
|
require_relative "storages/active_record_storage"
|
|
25
|
-
ActiveRecordStorage.new
|
|
32
|
+
ActiveRecordStorage.new(cipher: resolved_cipher)
|
|
26
33
|
else
|
|
27
34
|
MemoryStorage.new
|
|
28
35
|
end
|
|
@@ -4,6 +4,7 @@ require "json"
|
|
|
4
4
|
require "securerandom"
|
|
5
5
|
require "time"
|
|
6
6
|
require_relative "base"
|
|
7
|
+
require_relative "../../../crypto/field_helpers"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module LongTerm
|
|
@@ -13,7 +14,10 @@ module Llmemory
|
|
|
13
14
|
# AR auto-deserializes jsonb to a Hash (string keys), which Episode.from_h
|
|
14
15
|
# handles. Mirrors the file-based ActiveRecordStorage pattern.
|
|
15
16
|
class ActiveRecordStorage < Base
|
|
16
|
-
|
|
17
|
+
include Llmemory::Crypto::FieldHelpers
|
|
18
|
+
|
|
19
|
+
def initialize(cipher: nil)
|
|
20
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
17
21
|
self.class.load_models!
|
|
18
22
|
end
|
|
19
23
|
|
|
@@ -30,8 +34,8 @@ module Llmemory
|
|
|
30
34
|
data["created_at"] ||= Time.now.utc.iso8601
|
|
31
35
|
rec = LlmemoryEpisode.find_or_initialize_by(id: id)
|
|
32
36
|
rec.user_id = user_id
|
|
33
|
-
rec.data = data
|
|
34
|
-
rec.search_text = searchable_text(data)
|
|
37
|
+
rec.data = cipher.enabled? ? enc_json(data) : data
|
|
38
|
+
rec.search_text = enc(searchable_text(data))
|
|
35
39
|
rec.created_at ||= Time.current
|
|
36
40
|
rec.save!
|
|
37
41
|
id
|
|
@@ -39,19 +43,21 @@ module Llmemory
|
|
|
39
43
|
|
|
40
44
|
def get_episode(user_id, id)
|
|
41
45
|
rec = LlmemoryEpisode.find_by(user_id: user_id, id: id)
|
|
42
|
-
rec
|
|
46
|
+
return nil unless rec
|
|
47
|
+
|
|
48
|
+
decode_data(rec.data)
|
|
43
49
|
end
|
|
44
50
|
|
|
45
51
|
def list_episodes(user_id, limit: nil, offset: nil)
|
|
46
52
|
scope = LlmemoryEpisode.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
|
|
47
53
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
48
54
|
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
49
|
-
scope.map(
|
|
55
|
+
scope.map { |r| decode_data(r.data) }
|
|
50
56
|
end
|
|
51
57
|
|
|
52
58
|
def search_episodes(user_id, query)
|
|
53
59
|
token_scope(LlmemoryEpisode.where(user_id: user_id, archived_at: nil), "search_text", query)
|
|
54
|
-
.order(created_at: :desc).map(
|
|
60
|
+
.order(created_at: :desc).map { |r| decode_data(r.data) }
|
|
55
61
|
end
|
|
56
62
|
|
|
57
63
|
def count_episodes(user_id)
|
|
@@ -96,6 +102,13 @@ module Llmemory
|
|
|
96
102
|
end
|
|
97
103
|
parts.compact.join("\n")
|
|
98
104
|
end
|
|
105
|
+
|
|
106
|
+
def decode_data(raw)
|
|
107
|
+
return raw.transform_keys(&:to_sym) if raw.is_a?(Hash)
|
|
108
|
+
return dec_json(raw) if raw.is_a?(String) && cipher.encrypted?(raw)
|
|
109
|
+
|
|
110
|
+
raw
|
|
111
|
+
end
|
|
99
112
|
end
|
|
100
113
|
end
|
|
101
114
|
end
|
|
@@ -4,6 +4,7 @@ require "json"
|
|
|
4
4
|
require "securerandom"
|
|
5
5
|
require "time"
|
|
6
6
|
require_relative "base"
|
|
7
|
+
require_relative "../../../crypto/field_helpers"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module LongTerm
|
|
@@ -13,9 +14,12 @@ module Llmemory
|
|
|
13
14
|
# (plus id/user_id/created_at and a denormalized search_text for keyword
|
|
14
15
|
# search), mirroring the file-based DatabaseStorage pattern.
|
|
15
16
|
class DatabaseStorage < Base
|
|
16
|
-
|
|
17
|
+
include Llmemory::Crypto::FieldHelpers
|
|
18
|
+
|
|
19
|
+
def initialize(database_url: nil, cipher: nil)
|
|
17
20
|
@database_url = database_url || Llmemory.configuration.database_url
|
|
18
21
|
@connection = nil
|
|
22
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
19
23
|
end
|
|
20
24
|
|
|
21
25
|
def save_episode(user_id, episode)
|
|
@@ -23,11 +27,12 @@ module Llmemory
|
|
|
23
27
|
id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
|
|
24
28
|
data = symbolize(episode).merge(id: id, user_id: user_id)
|
|
25
29
|
data[:created_at] ||= Time.now.utc.iso8601
|
|
30
|
+
search = searchable_text(data)
|
|
26
31
|
conn.exec_params(
|
|
27
32
|
"INSERT INTO llmemory_episodes (id, user_id, data, search_text, created_at) " \
|
|
28
33
|
"VALUES ($1, $2, $3::jsonb, $4, $5) " \
|
|
29
34
|
"ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
|
|
30
|
-
[id, user_id,
|
|
35
|
+
[id, user_id, store_data(data), enc(search), created_at_value(data)]
|
|
31
36
|
)
|
|
32
37
|
id
|
|
33
38
|
end
|
|
@@ -124,11 +129,28 @@ module Llmemory
|
|
|
124
129
|
end
|
|
125
130
|
|
|
126
131
|
def parse_data(value)
|
|
127
|
-
|
|
132
|
+
if value.is_a?(Hash)
|
|
133
|
+
return value.transform_keys(&:to_sym)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
str = value.to_s
|
|
137
|
+
if cipher.encrypted?(str)
|
|
138
|
+
cipher.decrypt_json(str)
|
|
139
|
+
else
|
|
140
|
+
JSON.parse(str, symbolize_names: true)
|
|
141
|
+
end
|
|
128
142
|
rescue JSON::ParserError
|
|
129
143
|
{}
|
|
130
144
|
end
|
|
131
145
|
|
|
146
|
+
def store_data(data)
|
|
147
|
+
if cipher.enabled?
|
|
148
|
+
JSON.generate(enc_json(data))
|
|
149
|
+
else
|
|
150
|
+
JSON.generate(data)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
132
154
|
def symbolize(hash)
|
|
133
155
|
hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
134
156
|
end
|
|
@@ -4,22 +4,26 @@ require "fileutils"
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "time"
|
|
6
6
|
require_relative "base"
|
|
7
|
+
require_relative "../../../crypto/field_helpers"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module LongTerm
|
|
10
11
|
module Episodic
|
|
11
12
|
module Storages
|
|
12
13
|
class FileStorage < Base
|
|
13
|
-
|
|
14
|
+
include Llmemory::Crypto::FieldHelpers
|
|
15
|
+
|
|
16
|
+
def initialize(base_path: nil, cipher: nil)
|
|
14
17
|
@base_path = base_path || Llmemory.configuration.long_term_storage_path || "./llmemory_data"
|
|
15
18
|
@base_path = File.expand_path(@base_path)
|
|
19
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
16
20
|
end
|
|
17
21
|
|
|
18
22
|
def save_episode(user_id, episode)
|
|
19
23
|
id = episode[:id] || episode["id"] || "ep_#{next_seq(user_id)}"
|
|
20
24
|
data = stringify_for_json(episode).merge("id" => id, "user_id" => user_id)
|
|
21
25
|
data["created_at"] ||= Time.now.iso8601
|
|
22
|
-
|
|
26
|
+
write_episode_file(episode_path(user_id, id), data)
|
|
23
27
|
id
|
|
24
28
|
end
|
|
25
29
|
|
|
@@ -57,10 +61,10 @@ module Llmemory
|
|
|
57
61
|
Array(ids).map(&:to_s).count do |id|
|
|
58
62
|
path = episode_path(user_id, id)
|
|
59
63
|
next false unless File.file?(path)
|
|
60
|
-
data =
|
|
64
|
+
data = load_episode_raw(path)
|
|
61
65
|
next false if data["archived_at"]
|
|
62
66
|
data["archived_at"] = Time.now.iso8601
|
|
63
|
-
|
|
67
|
+
write_episode_file(path, data)
|
|
64
68
|
true
|
|
65
69
|
end
|
|
66
70
|
end
|
|
@@ -89,13 +93,26 @@ module Llmemory
|
|
|
89
93
|
end
|
|
90
94
|
|
|
91
95
|
def load_episode(path)
|
|
92
|
-
data =
|
|
96
|
+
data = load_episode_raw(path)
|
|
97
|
+
return nil unless data
|
|
98
|
+
|
|
93
99
|
data[:created_at] = parse_time(data[:created_at])
|
|
94
100
|
data
|
|
95
101
|
rescue JSON::ParserError
|
|
96
102
|
nil
|
|
97
103
|
end
|
|
98
104
|
|
|
105
|
+
def load_episode_raw(path)
|
|
106
|
+
raw = File.read(path)
|
|
107
|
+
json = cipher.enabled? && cipher.encrypted?(raw) ? cipher.decrypt(raw) : raw
|
|
108
|
+
JSON.parse(json, symbolize_names: true)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def write_episode_file(path, data)
|
|
112
|
+
payload = JSON.generate(data)
|
|
113
|
+
File.write(path, cipher.enabled? ? cipher.encrypt(payload) : payload)
|
|
114
|
+
end
|
|
115
|
+
|
|
99
116
|
def episode_text(episode)
|
|
100
117
|
parts = [episode[:summary], episode[:outcome]]
|
|
101
118
|
Array(episode[:steps]).each do |s|
|
|
@@ -14,17 +14,24 @@ module Llmemory
|
|
|
14
14
|
Storage = Storages::MemoryStorage
|
|
15
15
|
|
|
16
16
|
module Storages
|
|
17
|
-
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
17
|
+
def self.build(store: nil, base_path: nil, database_url: nil, cipher: nil)
|
|
18
|
+
resolved_cipher = cipher || Llmemory.build_cipher
|
|
18
19
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
19
20
|
when :memory
|
|
20
21
|
MemoryStorage.new
|
|
21
22
|
when :file
|
|
22
|
-
FileStorage.new(
|
|
23
|
+
FileStorage.new(
|
|
24
|
+
base_path: base_path || Llmemory.configuration.long_term_storage_path,
|
|
25
|
+
cipher: resolved_cipher
|
|
26
|
+
)
|
|
23
27
|
when :postgres, :database
|
|
24
|
-
DatabaseStorage.new(
|
|
28
|
+
DatabaseStorage.new(
|
|
29
|
+
database_url: database_url || Llmemory.configuration.database_url,
|
|
30
|
+
cipher: resolved_cipher
|
|
31
|
+
)
|
|
25
32
|
when :active_record, :activerecord
|
|
26
33
|
require_relative "storages/active_record_storage"
|
|
27
|
-
ActiveRecordStorage.new
|
|
34
|
+
ActiveRecordStorage.new(cipher: resolved_cipher)
|
|
28
35
|
else
|
|
29
36
|
MemoryStorage.new
|
|
30
37
|
end
|