llmemory 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +78 -1
- data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
- data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
- data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
- data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
- data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
- data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
- data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
- data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
- data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
- data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
- data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
- data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
- data/config/routes.rb +14 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +2 -0
- data/lib/llmemory/cli/commands/maintain.rb +62 -0
- data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
- data/lib/llmemory/cli.rb +6 -0
- data/lib/llmemory/configuration.rb +28 -2
- data/lib/llmemory/crypto/cipher.rb +147 -0
- data/lib/llmemory/crypto/field_helpers.rb +110 -0
- data/lib/llmemory/instrumentation.rb +33 -0
- data/lib/llmemory/llm/anthropic.rb +21 -16
- data/lib/llmemory/llm/openai.rb +18 -13
- data/lib/llmemory/long_term/episodic/memory.rb +27 -13
- data/lib/llmemory/long_term/episodic/storage.rb +11 -4
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +33 -10
- data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +51 -8
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +47 -9
- data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +35 -4
- data/lib/llmemory/long_term/file_based/memory.rb +12 -4
- data/lib/llmemory/long_term/file_based/storage.rb +11 -4
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +20 -12
- data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +28 -10
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +32 -16
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/graph_based/memory.rb +16 -7
- data/lib/llmemory/long_term/graph_based/storage.rb +3 -2
- data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +51 -23
- data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/procedural/memory.rb +30 -16
- data/lib/llmemory/long_term/procedural/skill.rb +6 -2
- data/lib/llmemory/long_term/procedural/storage.rb +11 -4
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +47 -17
- data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +52 -10
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +49 -11
- data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +36 -5
- data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
- data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +5 -1
- data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
- data/lib/llmemory/memory.rb +60 -8
- data/lib/llmemory/memory_module.rb +13 -6
- data/lib/llmemory/reflection/reflector.rb +24 -20
- data/lib/llmemory/retrieval/engine.rb +25 -16
- data/lib/llmemory/short_term/checkpoint.rb +3 -2
- data/lib/llmemory/short_term/stores/active_record_store.rb +12 -10
- data/lib/llmemory/short_term/stores/memory_store.rb +1 -1
- data/lib/llmemory/short_term/stores/postgres_store.rb +11 -5
- data/lib/llmemory/short_term/stores/redis_store.rb +7 -5
- data/lib/llmemory/short_term/stores.rb +7 -6
- data/lib/llmemory/skill_mining/miner.rb +163 -0
- data/lib/llmemory/skill_mining.rb +8 -0
- data/lib/llmemory/vector_store/active_record_store.rb +24 -3
- data/lib/llmemory/vector_store/memory_store.rb +23 -3
- data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
- data/lib/llmemory/vector_store.rb +4 -3
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +4 -0
- metadata +24 -1
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
# Runs the cognitive maintenance pass for a user: reflect -> mine skills ->
|
|
9
|
+
# expire. Each step is isolated; failures are reported, not fatal.
|
|
10
|
+
class Maintain < Commands::Base
|
|
11
|
+
def option_parser(parser)
|
|
12
|
+
parser.on("--[no-]reflect", "Distill insights from recent episodes (default: on)") { |v| @reflect = v }
|
|
13
|
+
parser.on("--mine-skills", "Mine and register skills from episodes (default: config)") { @mine = true }
|
|
14
|
+
parser.on("--[no-]expire", "Soft-archive entries past their TTL (default: on)") { |v| @expire = v }
|
|
15
|
+
parser.on("--window N", Integer, "Episodes to reflect over (default 10)") { |v| @window = v }
|
|
16
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def execute(argv, _opts)
|
|
20
|
+
user_id = argv.first
|
|
21
|
+
unless user_id
|
|
22
|
+
$stderr.puts "Usage: llmemory maintain USER_ID [--[no-]reflect] [--mine-skills] [--[no-]expire] [--window N] [--store TYPE]"
|
|
23
|
+
exit 1
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
opts = {
|
|
27
|
+
episodic: Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type)),
|
|
28
|
+
procedural: Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type)),
|
|
29
|
+
semantic: build_semantic(user_id),
|
|
30
|
+
reflect: @reflect.nil? ? true : @reflect,
|
|
31
|
+
expire: @expire.nil? ? true : @expire
|
|
32
|
+
}
|
|
33
|
+
opts[:mine_skills] = @mine unless @mine.nil?
|
|
34
|
+
opts[:reflection_window] = @window if @window
|
|
35
|
+
|
|
36
|
+
report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
|
|
37
|
+
print_report(user_id, report)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def build_semantic(user_id)
|
|
43
|
+
if Llmemory.configuration.long_term_type.to_s == "graph_based"
|
|
44
|
+
Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id, storage: graph_based_storage(@store_type))
|
|
45
|
+
else
|
|
46
|
+
Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id, storage: file_based_storage(@store_type))
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def print_report(user_id, report)
|
|
51
|
+
expired = report[:expired] || {}
|
|
52
|
+
puts "Cognitive pass for #{user_id}:"
|
|
53
|
+
puts " insights: #{Array(report[:insights]).size}"
|
|
54
|
+
puts " skills mined: #{Array(report[:mined]).size}"
|
|
55
|
+
puts " expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
|
|
56
|
+
errors = report[:errors] || {}
|
|
57
|
+
errors.each { |step, msg| puts " error (#{step}): #{msg}" }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
# Mines reusable skills from a user's successful episodes. Human-in-the-loop
|
|
9
|
+
# by default: prints proposals and writes nothing unless --register is given.
|
|
10
|
+
class MineSkills < Commands::Base
|
|
11
|
+
def option_parser(parser)
|
|
12
|
+
parser.on("--window N", Integer, "Episodes to mine (default 20)") { |v| @window = v }
|
|
13
|
+
parser.on("--outcomes LIST", "Comma-separated outcome allowlist (e.g. success,recovered)") { |v| @outcomes = v.split(",").map(&:strip) }
|
|
14
|
+
parser.on("--register", "Register the proposals instead of only printing them") { @register = true }
|
|
15
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def execute(argv, _opts)
|
|
19
|
+
user_id = argv.first
|
|
20
|
+
unless user_id
|
|
21
|
+
$stderr.puts "Usage: llmemory mine-skills USER_ID [--window N] [--outcomes LIST] [--register] [--store TYPE]"
|
|
22
|
+
exit 1
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
episodic = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type))
|
|
26
|
+
procedural = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type))
|
|
27
|
+
result = Llmemory::SkillMining::Miner.new(episodic: episodic, procedural: procedural).mine(
|
|
28
|
+
window: @window || Llmemory::SkillMining::Miner::DEFAULT_WINDOW,
|
|
29
|
+
outcomes: @outcomes,
|
|
30
|
+
auto_register: @register
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
if result.empty?
|
|
34
|
+
puts "No skills could be mined for user #{user_id}."
|
|
35
|
+
return
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
if @register
|
|
39
|
+
puts "Registered #{result.size} mined skill(s): #{result.join(', ')}"
|
|
40
|
+
else
|
|
41
|
+
puts "#{result.size} skill proposal(s) for user #{user_id} (not registered):"
|
|
42
|
+
result.each do |p|
|
|
43
|
+
puts " - #{p[:name]} (#{p[:kind]}, confidence: #{p[:confidence]}): #{p[:description] || p[:body]}"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
data/lib/llmemory/cli.rb
CHANGED
|
@@ -9,6 +9,8 @@ require_relative "cli/commands/episodic"
|
|
|
9
9
|
require_relative "cli/commands/procedural"
|
|
10
10
|
require_relative "cli/commands/working"
|
|
11
11
|
require_relative "cli/commands/forget_log"
|
|
12
|
+
require_relative "cli/commands/mine_skills"
|
|
13
|
+
require_relative "cli/commands/maintain"
|
|
12
14
|
require_relative "cli/commands/stats"
|
|
13
15
|
require_relative "cli/commands/search"
|
|
14
16
|
require_relative "cli/commands/mcp"
|
|
@@ -55,6 +57,8 @@ module Llmemory
|
|
|
55
57
|
"skills" => Cli::Commands::Procedural,
|
|
56
58
|
"working" => Cli::Commands::Working,
|
|
57
59
|
"forget_log" => Cli::Commands::ForgetLog,
|
|
60
|
+
"mine_skills" => Cli::Commands::MineSkills,
|
|
61
|
+
"maintain" => Cli::Commands::Maintain,
|
|
58
62
|
"search" => Cli::Commands::Search,
|
|
59
63
|
"stats" => Cli::Commands::Stats,
|
|
60
64
|
"mcp" => Cli::Commands::Mcp
|
|
@@ -80,6 +84,8 @@ module Llmemory
|
|
|
80
84
|
skills USER_ID List registered skills (procedural memory)
|
|
81
85
|
working USER_ID SESSION Show working-memory slots for a session
|
|
82
86
|
forget-log USER_ID Show audit of forgotten entries
|
|
87
|
+
mine-skills USER_ID Mine reusable skills from episodes (--register to save)
|
|
88
|
+
maintain USER_ID Run the cognitive maintenance pass (reflect/mine/expire)
|
|
83
89
|
search USER_ID "query" Search in memory
|
|
84
90
|
stats [USER_ID] Show statistics
|
|
85
91
|
mcp [serve] Start MCP server for LLM agents
|
|
@@ -45,12 +45,17 @@ module Llmemory
|
|
|
45
45
|
:embedding_cache_enabled,
|
|
46
46
|
:embedding_cache_max_entries,
|
|
47
47
|
:max_message_chars,
|
|
48
|
-
:message_sanitizer_enabled
|
|
48
|
+
:message_sanitizer_enabled,
|
|
49
|
+
:ttl_episodic_days,
|
|
50
|
+
:ttl_procedural_days,
|
|
51
|
+
:skill_mining_enabled,
|
|
52
|
+
:encryption_enabled,
|
|
53
|
+
:encryption_key
|
|
49
54
|
|
|
50
55
|
def initialize
|
|
51
56
|
@llm_provider = :openai
|
|
52
57
|
@llm_api_key = ENV["OPENAI_API_KEY"]
|
|
53
|
-
@llm_model =
|
|
58
|
+
@llm_model = nil # falls back to the active provider's DEFAULT_MODEL
|
|
54
59
|
@llm_base_url = nil
|
|
55
60
|
@short_term_store = :memory
|
|
56
61
|
@redis_url = ENV["REDIS_URL"] || "redis://localhost:6379/0"
|
|
@@ -59,6 +64,9 @@ module Llmemory
|
|
|
59
64
|
@long_term_storage_path = ENV["LLMEMORY_STORAGE_PATH"] || "./llmemory_data"
|
|
60
65
|
@episodic_vector_enabled = false
|
|
61
66
|
@procedural_vector_enabled = false
|
|
67
|
+
@ttl_episodic_days = nil
|
|
68
|
+
@ttl_procedural_days = nil
|
|
69
|
+
@skill_mining_enabled = false
|
|
62
70
|
@database_url = ENV["DATABASE_URL"]
|
|
63
71
|
@vector_store = nil
|
|
64
72
|
@time_decay_half_life_days = 30
|
|
@@ -92,6 +100,8 @@ module Llmemory
|
|
|
92
100
|
@embedding_cache_max_entries = 10_000
|
|
93
101
|
@max_message_chars = 32_000
|
|
94
102
|
@message_sanitizer_enabled = false
|
|
103
|
+
@encryption_enabled = false
|
|
104
|
+
@encryption_key = ENV["LLMEMORY_ENCRYPTION_KEY"]
|
|
95
105
|
end
|
|
96
106
|
end
|
|
97
107
|
|
|
@@ -107,5 +117,21 @@ module Llmemory
|
|
|
107
117
|
def reset_configuration!
|
|
108
118
|
@configuration = Configuration.new
|
|
109
119
|
end
|
|
120
|
+
|
|
121
|
+
# Builds a Crypto::Cipher when encryption is enabled and a key is present;
|
|
122
|
+
# otherwise returns Crypto::NullCipher. An explicit non-empty instance key
|
|
123
|
+
# enables encryption even when the global flag is off.
|
|
124
|
+
def build_cipher(key = nil)
|
|
125
|
+
explicit_key = !key.nil? && !key.to_s.empty?
|
|
126
|
+
resolved = key.nil? ? configuration.encryption_key : key
|
|
127
|
+
enabled = configuration.encryption_enabled || explicit_key
|
|
128
|
+
if enabled && !resolved.to_s.empty?
|
|
129
|
+
require_relative "crypto/cipher"
|
|
130
|
+
Crypto::Cipher.new(resolved)
|
|
131
|
+
else
|
|
132
|
+
require_relative "crypto/cipher"
|
|
133
|
+
Crypto::NullCipher.new
|
|
134
|
+
end
|
|
135
|
+
end
|
|
110
136
|
end
|
|
111
137
|
end
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module Crypto
|
|
8
|
+
class DecryptionError < Llmemory::Error; end
|
|
9
|
+
|
|
10
|
+
# No-op cipher when encryption is disabled or no key is configured.
|
|
11
|
+
class NullCipher
|
|
12
|
+
def enabled?
|
|
13
|
+
false
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def encrypt(str)
|
|
17
|
+
str.to_s
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def encrypt_deterministic(str)
|
|
21
|
+
str.to_s
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def decrypt(str)
|
|
25
|
+
str.to_s
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def encrypt_json(obj)
|
|
29
|
+
JSON.generate(obj)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def decrypt_json(str)
|
|
33
|
+
JSON.parse(str.to_s, symbolize_names: true)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def encrypted?(str)
|
|
37
|
+
false
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# AES-256-GCM encryption with separate content (random IV) and index
|
|
42
|
+
# (deterministic IV) subkeys derived from the master key via HMAC-SHA256.
|
|
43
|
+
class Cipher
|
|
44
|
+
MARKER = "enc:v1:"
|
|
45
|
+
DETERMINISTIC_MARKER = "encd:v1:"
|
|
46
|
+
IV_LENGTH = 12
|
|
47
|
+
TAG_LENGTH = 16
|
|
48
|
+
|
|
49
|
+
def initialize(key)
|
|
50
|
+
@master_key = derive_master_key(key)
|
|
51
|
+
@content_key = derive_subkey("content")
|
|
52
|
+
@index_key = derive_subkey("index")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def enabled?
|
|
56
|
+
true
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def encrypt(plaintext)
|
|
60
|
+
str = plaintext.to_s
|
|
61
|
+
return str if str.empty?
|
|
62
|
+
|
|
63
|
+
encrypt_with_key(str, @content_key, iv: OpenSSL::Random.random_bytes(IV_LENGTH), marker: MARKER)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def encrypt_deterministic(plaintext)
|
|
67
|
+
str = plaintext.to_s
|
|
68
|
+
return str if str.empty?
|
|
69
|
+
|
|
70
|
+
iv = OpenSSL::HMAC.digest("SHA256", @index_key, str)[0, IV_LENGTH]
|
|
71
|
+
encrypt_with_key(str, @index_key, iv: iv, marker: DETERMINISTIC_MARKER)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def decrypt(ciphertext)
|
|
75
|
+
str = ciphertext.to_s
|
|
76
|
+
return str if str.empty?
|
|
77
|
+
return str unless encrypted?(str)
|
|
78
|
+
|
|
79
|
+
marker, key = if str.start_with?(DETERMINISTIC_MARKER)
|
|
80
|
+
[DETERMINISTIC_MARKER, @index_key]
|
|
81
|
+
else
|
|
82
|
+
[MARKER, @content_key]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
payload = decode64(str.delete_prefix(marker))
|
|
86
|
+
iv = payload[0, IV_LENGTH]
|
|
87
|
+
tag = payload[IV_LENGTH, TAG_LENGTH]
|
|
88
|
+
ct = payload[(IV_LENGTH + TAG_LENGTH)..]
|
|
89
|
+
|
|
90
|
+
cipher = OpenSSL::Cipher.new("aes-256-gcm")
|
|
91
|
+
cipher.decrypt
|
|
92
|
+
cipher.key = key
|
|
93
|
+
cipher.iv = iv
|
|
94
|
+
cipher.auth_tag = tag
|
|
95
|
+
cipher.auth_data = ""
|
|
96
|
+
cipher.update(ct) + cipher.final
|
|
97
|
+
rescue OpenSSL::Cipher::CipherError, ArgumentError => e
|
|
98
|
+
raise DecryptionError, "Failed to decrypt data: #{e.message}"
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def encrypt_json(obj)
|
|
102
|
+
encrypt(JSON.generate(obj))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def decrypt_json(str)
|
|
106
|
+
JSON.parse(decrypt(str), symbolize_names: true)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def encrypted?(str)
|
|
110
|
+
s = str.to_s
|
|
111
|
+
s.start_with?(MARKER) || s.start_with?(DETERMINISTIC_MARKER)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private
|
|
115
|
+
|
|
116
|
+
def encrypt_with_key(plaintext, key, iv:, marker:)
|
|
117
|
+
cipher = OpenSSL::Cipher.new("aes-256-gcm")
|
|
118
|
+
cipher.encrypt
|
|
119
|
+
cipher.key = key
|
|
120
|
+
cipher.iv = iv
|
|
121
|
+
cipher.auth_data = ""
|
|
122
|
+
ct = cipher.update(plaintext) + cipher.final
|
|
123
|
+
tag = cipher.auth_tag
|
|
124
|
+
marker + encode64(iv + tag + ct)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def encode64(bin)
|
|
128
|
+
[bin].pack("m0")
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def decode64(str)
|
|
132
|
+
str.unpack1("m0")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def derive_master_key(key)
|
|
136
|
+
raw = key.to_s
|
|
137
|
+
raise ConfigurationError, "encryption_key cannot be empty when encryption is enabled" if raw.empty?
|
|
138
|
+
|
|
139
|
+
OpenSSL::Digest::SHA256.digest(raw)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def derive_subkey(label)
|
|
143
|
+
OpenSSL::HMAC.digest("SHA256", @master_key, "llmemory:#{label}")[0, 32]
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Crypto
|
|
7
|
+
# Shared encrypt/decrypt helpers for storage backends.
|
|
8
|
+
module FieldHelpers
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def cipher
|
|
12
|
+
@cipher || Llmemory.build_cipher
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def enc(str)
|
|
16
|
+
return str if str.nil?
|
|
17
|
+
return str.to_s unless cipher.enabled?
|
|
18
|
+
|
|
19
|
+
cipher.encrypt(str.to_s)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def dec(str)
|
|
23
|
+
return str if str.nil?
|
|
24
|
+
return str unless str.is_a?(String) && cipher.encrypted?(str)
|
|
25
|
+
|
|
26
|
+
cipher.decrypt(str)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def enc_det(str)
|
|
30
|
+
return str if str.nil?
|
|
31
|
+
return str.to_s unless cipher.enabled?
|
|
32
|
+
|
|
33
|
+
cipher.encrypt_deterministic(str.to_s)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def enc_json(obj)
|
|
37
|
+
return obj if obj.nil?
|
|
38
|
+
return obj unless cipher.enabled?
|
|
39
|
+
|
|
40
|
+
cipher.encrypt_json(obj)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def dec_json(value)
|
|
44
|
+
return value if value.nil?
|
|
45
|
+
return value.transform_keys(&:to_sym) if value.is_a?(Hash)
|
|
46
|
+
return value unless value.is_a?(String) && cipher.encrypted?(value)
|
|
47
|
+
|
|
48
|
+
cipher.decrypt_json(value)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def write_encrypted_file(path, data)
|
|
52
|
+
payload = JSON.generate(data)
|
|
53
|
+
File.write(path, cipher.enabled? ? cipher.encrypt(payload) : payload)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def read_encrypted_file(path)
|
|
57
|
+
raw = File.read(path)
|
|
58
|
+
json = cipher.enabled? && cipher.encrypted?(raw) ? cipher.decrypt(raw) : raw
|
|
59
|
+
JSON.parse(json, symbolize_names: true)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def write_encrypted_text_file(path, content, append: false)
|
|
63
|
+
text = content.to_s
|
|
64
|
+
if cipher.enabled?
|
|
65
|
+
if append && File.file?(path)
|
|
66
|
+
existing = read_encrypted_text_file(path)
|
|
67
|
+
text = existing + text
|
|
68
|
+
end
|
|
69
|
+
File.write(path, cipher.encrypt(text))
|
|
70
|
+
elsif append && File.file?(path)
|
|
71
|
+
File.write(path, File.read(path) + text)
|
|
72
|
+
else
|
|
73
|
+
File.write(path, text)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def read_encrypted_text_file(path)
|
|
78
|
+
raw = File.read(path)
|
|
79
|
+
cipher.enabled? && cipher.encrypted?(raw) ? cipher.decrypt(raw) : raw
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def serialize_state(state)
|
|
83
|
+
json = JSON.generate(state)
|
|
84
|
+
return json unless cipher.enabled?
|
|
85
|
+
|
|
86
|
+
cipher.encrypt(json)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def deserialize_state(data)
|
|
90
|
+
if data.is_a?(Hash)
|
|
91
|
+
return data.transform_keys(&:to_sym)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
str = data.to_s
|
|
95
|
+
json = cipher.enabled? && cipher.encrypted?(str) ? cipher.decrypt(str) : str
|
|
96
|
+
JSON.parse(json, symbolize_names: true)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def parse_provenance(value)
|
|
100
|
+
return nil if value.nil?
|
|
101
|
+
return value.transform_keys(&:to_sym) if value.is_a?(Hash)
|
|
102
|
+
return dec_json(value) if value.is_a?(String) && cipher.encrypted?(value)
|
|
103
|
+
|
|
104
|
+
JSON.parse(value.to_s, symbolize_names: true)
|
|
105
|
+
rescue JSON::ParserError
|
|
106
|
+
nil
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
# Lightweight instrumentation seam. When ActiveSupport::Notifications is
|
|
5
|
+
# available (Rails apps, opt-in elsewhere), events are published with the
|
|
6
|
+
# `.llmemory` suffix so subscribers can hook into LLM calls, retrieval,
|
|
7
|
+
# writes and forgets for metrics, traces or cost dashboards. When AS is not
|
|
8
|
+
# loaded, the block is yielded transparently and no event is emitted.
|
|
9
|
+
#
|
|
10
|
+
# Events (payload keys are best-effort; subscribers should treat them as
|
|
11
|
+
# optional):
|
|
12
|
+
#
|
|
13
|
+
# llm_invoke.llmemory provider:, model:, prompt_chars:, response_chars:
|
|
14
|
+
# llm_embed.llmemory provider:, model:, text_chars:, dimensions:
|
|
15
|
+
# memory_write.llmemory memory_type:, user_id:
|
|
16
|
+
# memory_forget.llmemory memory_type:, user_id:, count:
|
|
17
|
+
# retrieve.llmemory query_chars:, candidates:, results:
|
|
18
|
+
# iterative_retrieve.llmemory hops:, total_results:
|
|
19
|
+
# reflect.llmemory window:, insights:
|
|
20
|
+
# mine_skills.llmemory window:, auto_register:
|
|
21
|
+
module Instrumentation
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
def instrument(event, payload = {})
|
|
25
|
+
name = "#{event}.llmemory"
|
|
26
|
+
if defined?(ActiveSupport::Notifications)
|
|
27
|
+
ActiveSupport::Notifications.instrument(name, payload) { yield if block_given? }
|
|
28
|
+
else
|
|
29
|
+
yield if block_given?
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -8,30 +8,35 @@ module Llmemory
|
|
|
8
8
|
module LLM
|
|
9
9
|
class Anthropic < Base
|
|
10
10
|
DEFAULT_BASE_URL = "https://api.anthropic.com"
|
|
11
|
+
DEFAULT_MODEL = "claude-sonnet-4-6"
|
|
11
12
|
|
|
12
13
|
def initialize(api_key: nil, model: nil, base_url: nil)
|
|
13
14
|
@api_key = api_key || config.llm_api_key || ENV["ANTHROPIC_API_KEY"]
|
|
14
|
-
@model = model || config.llm_model ||
|
|
15
|
+
@model = model || config.llm_model || DEFAULT_MODEL
|
|
15
16
|
@base_url = base_url || config.llm_base_url || DEFAULT_BASE_URL
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
def invoke(prompt)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
20
|
+
result = nil
|
|
21
|
+
Llmemory::Instrumentation.instrument(:llm_invoke, provider: :anthropic, model: @model, prompt_chars: prompt.to_s.length) do
|
|
22
|
+
response = connection.post("v1/messages") do |req|
|
|
23
|
+
req.body = {
|
|
24
|
+
model: @model,
|
|
25
|
+
max_tokens: 1024,
|
|
26
|
+
messages: [{ role: "user", content: prompt }]
|
|
27
|
+
}.to_json
|
|
28
|
+
req.headers["Content-Type"] = "application/json"
|
|
29
|
+
req.headers["x-api-key"] = @api_key
|
|
30
|
+
req.headers["anthropic-version"] = "2023-06-01"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
|
|
34
|
+
|
|
35
|
+
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
36
|
+
content = body.dig("content", 0, "text")
|
|
37
|
+
result = content&.strip || ""
|
|
28
38
|
end
|
|
29
|
-
|
|
30
|
-
raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
|
|
31
|
-
|
|
32
|
-
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
33
|
-
content = body.dig("content", 0, "text")
|
|
34
|
-
content&.strip || ""
|
|
39
|
+
result
|
|
35
40
|
end
|
|
36
41
|
|
|
37
42
|
private
|
data/lib/llmemory/llm/openai.rb
CHANGED
|
@@ -8,28 +8,33 @@ module Llmemory
|
|
|
8
8
|
module LLM
|
|
9
9
|
class OpenAI < Base
|
|
10
10
|
DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
|
11
|
+
DEFAULT_MODEL = "gpt-4"
|
|
11
12
|
|
|
12
13
|
def initialize(api_key: nil, model: nil, base_url: nil)
|
|
13
14
|
@api_key = api_key || config.llm_api_key
|
|
14
|
-
@model = model || config.llm_model
|
|
15
|
+
@model = model || config.llm_model || DEFAULT_MODEL
|
|
15
16
|
@base_url = base_url || config.llm_base_url || DEFAULT_BASE_URL
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
def invoke(prompt)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
20
|
+
result = nil
|
|
21
|
+
Llmemory::Instrumentation.instrument(:llm_invoke, provider: :openai, model: @model, prompt_chars: prompt.to_s.length) do
|
|
22
|
+
response = connection.post("chat/completions") do |req|
|
|
23
|
+
req.body = {
|
|
24
|
+
model: @model,
|
|
25
|
+
messages: [{ role: "user", content: prompt }],
|
|
26
|
+
temperature: 0.3
|
|
27
|
+
}.to_json
|
|
28
|
+
req.headers["Content-Type"] = "application/json"
|
|
29
|
+
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
30
|
+
end
|
|
28
31
|
|
|
29
|
-
|
|
32
|
+
raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
|
|
34
|
+
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
35
|
+
result = body.dig("choices", 0, "message", "content")&.strip || ""
|
|
36
|
+
end
|
|
37
|
+
result
|
|
33
38
|
end
|
|
34
39
|
|
|
35
40
|
# Calls the model with response_format json_schema (Structured Outputs).
|
|
@@ -21,9 +21,10 @@ module Llmemory
|
|
|
21
21
|
|
|
22
22
|
attr_reader :user_id, :storage
|
|
23
23
|
|
|
24
|
-
def initialize(user_id:, storage: nil, vector_store: nil)
|
|
24
|
+
def initialize(user_id:, storage: nil, vector_store: nil, cipher: nil)
|
|
25
25
|
@user_id = user_id
|
|
26
|
-
@
|
|
26
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
27
|
+
@storage = storage || Storages.build(cipher: @cipher)
|
|
27
28
|
@vector_store = vector_store
|
|
28
29
|
@vector_explicit = !vector_store.nil?
|
|
29
30
|
end
|
|
@@ -52,8 +53,8 @@ module Llmemory
|
|
|
52
53
|
@storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
|
|
53
54
|
end
|
|
54
55
|
|
|
55
|
-
def episodes(limit: nil)
|
|
56
|
-
@storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
|
|
56
|
+
def episodes(limit: nil, offset: nil)
|
|
57
|
+
@storage.list_episodes(@user_id, limit: limit, offset: offset).map { |e| Episode.from_h(e) }
|
|
57
58
|
end
|
|
58
59
|
|
|
59
60
|
def find_episode(id)
|
|
@@ -84,24 +85,37 @@ module Llmemory
|
|
|
84
85
|
# --- MemoryModule uniform interface ---
|
|
85
86
|
|
|
86
87
|
def write(steps:, summary: nil, outcome: nil, importance: 0.5, **_meta)
|
|
87
|
-
|
|
88
|
+
result = nil
|
|
89
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "episodic", user_id: @user_id) do
|
|
90
|
+
result = record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
|
|
91
|
+
end
|
|
92
|
+
result
|
|
88
93
|
end
|
|
89
94
|
|
|
90
|
-
def list(user_id: nil, limit: nil)
|
|
91
|
-
episodes(limit: limit)
|
|
95
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
96
|
+
episodes(limit: limit, offset: offset)
|
|
92
97
|
end
|
|
93
98
|
|
|
94
99
|
def stats(user_id: nil)
|
|
95
100
|
{ episodes: count }
|
|
96
101
|
end
|
|
97
102
|
|
|
98
|
-
def forget(ids:, reason: nil)
|
|
103
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
99
104
|
requested = Array(ids).map(&:to_s)
|
|
100
105
|
existing = @storage.list_episodes(@user_id).map { |e| (e[:id] || e["id"]).to_s }
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
106
|
+
targeted = requested & existing
|
|
107
|
+
count = case mode
|
|
108
|
+
when :hard then @storage.delete_episodes(@user_id, targeted).to_i
|
|
109
|
+
else @storage.archive_episodes(@user_id, targeted).to_i
|
|
110
|
+
end
|
|
111
|
+
forget_log.record(@user_id, memory_type: "episodic", ids: targeted, reason: reason)
|
|
112
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "episodic", user_id: @user_id, count: count, mode: mode)
|
|
113
|
+
count
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Storage accessor for the TTL maintenance job.
|
|
117
|
+
def expired_ids(cutoff:)
|
|
118
|
+
@storage.expired_episode_ids(@user_id, cutoff: cutoff)
|
|
105
119
|
end
|
|
106
120
|
|
|
107
121
|
private
|
|
@@ -112,7 +126,7 @@ module Llmemory
|
|
|
112
126
|
if @vector_explicit
|
|
113
127
|
@vector_store
|
|
114
128
|
elsif Llmemory.configuration.episodic_vector_enabled
|
|
115
|
-
@vector_store ||= Llmemory::VectorStore.build(source_type: "episode")
|
|
129
|
+
@vector_store ||= Llmemory::VectorStore.build(source_type: "episode", cipher: @cipher)
|
|
116
130
|
end
|
|
117
131
|
end
|
|
118
132
|
|