llmemory 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -3
- data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
- data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
- data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
- data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
- data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
- data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
- data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
- data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
- data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
- data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
- data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
- data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
- data/config/routes.rb +14 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +22 -0
- data/lib/llmemory/cli/commands/base.rb +8 -0
- data/lib/llmemory/cli/commands/episodic.rb +42 -0
- data/lib/llmemory/cli/commands/forget_log.rb +36 -0
- data/lib/llmemory/cli/commands/maintain.rb +62 -0
- data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
- data/lib/llmemory/cli/commands/procedural.rb +44 -0
- data/lib/llmemory/cli/commands/working.rb +31 -0
- data/lib/llmemory/cli.rb +18 -0
- data/lib/llmemory/configuration.rb +11 -1
- data/lib/llmemory/instrumentation.rb +33 -0
- data/lib/llmemory/llm/anthropic.rb +19 -15
- data/lib/llmemory/llm/openai.rb +16 -12
- data/lib/llmemory/long_term/episodic/memory.rb +94 -26
- data/lib/llmemory/long_term/episodic/storage.rb +7 -5
- data/lib/llmemory/long_term/episodic/storages/active_record_models.rb +17 -0
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +103 -0
- data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +156 -0
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +28 -8
- data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +36 -6
- data/lib/llmemory/long_term/file_based/memory.rb +12 -4
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +15 -6
- data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +20 -8
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +6 -6
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +6 -6
- data/lib/llmemory/long_term/graph_based/memory.rb +89 -64
- data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +4 -2
- data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/procedural/memory.rb +97 -30
- data/lib/llmemory/long_term/procedural/skill.rb +6 -2
- data/lib/llmemory/long_term/procedural/storage.rb +7 -5
- data/lib/llmemory/long_term/procedural/storages/active_record_models.rb +17 -0
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +114 -0
- data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +169 -0
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +29 -9
- data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +37 -7
- data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
- data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +17 -1
- data/lib/llmemory/mcp/tools/memory_episode_record.rb +48 -0
- data/lib/llmemory/mcp/tools/memory_episodes.rb +43 -0
- data/lib/llmemory/mcp/tools/memory_forget.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_retrieve.rb +10 -2
- data/lib/llmemory/mcp/tools/memory_skill_register.rb +35 -0
- data/lib/llmemory/mcp/tools/memory_skill_report.rb +35 -0
- data/lib/llmemory/mcp/tools/memory_skills.rb +43 -0
- data/lib/llmemory/memory.rb +48 -3
- data/lib/llmemory/memory_module.rb +13 -6
- data/lib/llmemory/reflection/reflector.rb +24 -20
- data/lib/llmemory/retrieval/bm25_scorer.rb +1 -1
- data/lib/llmemory/retrieval/engine.rb +25 -16
- data/lib/llmemory/retrieval/mmr_reranker.rb +1 -1
- data/lib/llmemory/short_term/session_lifecycle.rb +19 -3
- data/lib/llmemory/skill_mining/miner.rb +163 -0
- data/lib/llmemory/skill_mining.rb +8 -0
- data/lib/llmemory/tokenizer.rb +27 -0
- data/lib/llmemory/vector_store/active_record_store.rb +4 -3
- data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
- data/lib/llmemory/vector_store.rb +14 -0
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +3 -0
- metadata +39 -1
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
class Episodic < Commands::Base
|
|
9
|
+
def option_parser(parser)
|
|
10
|
+
parser.on("--limit N", Integer, "Max number of episodes (newest first)") { |v| @limit = v }
|
|
11
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def execute(argv, _opts)
|
|
15
|
+
user_id = argv.first
|
|
16
|
+
unless user_id
|
|
17
|
+
$stderr.puts "Usage: llmemory episodes USER_ID [--limit N] [--store TYPE]"
|
|
18
|
+
exit 1
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
storage = episodic_storage(@store_type)
|
|
22
|
+
episodes = storage.list_episodes(user_id, limit: @limit)
|
|
23
|
+
|
|
24
|
+
if episodes.empty?
|
|
25
|
+
puts "No episodes for user #{user_id}."
|
|
26
|
+
return
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
episodes.each do |e|
|
|
30
|
+
id = e[:id] || e["id"]
|
|
31
|
+
summary = e[:summary] || e["summary"]
|
|
32
|
+
outcome = e[:outcome] || e["outcome"]
|
|
33
|
+
importance = e[:importance] || e["importance"]
|
|
34
|
+
steps = e[:steps] || e["steps"] || []
|
|
35
|
+
puts "[#{id}] (importance: #{importance}; outcome: #{outcome || 'n/a'}) #{summary}"
|
|
36
|
+
puts " steps: #{Array(steps).size}"
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
class ForgetLog < Commands::Base
|
|
9
|
+
def execute(argv, _opts)
|
|
10
|
+
user_id = argv.first
|
|
11
|
+
unless user_id
|
|
12
|
+
$stderr.puts "Usage: llmemory forget-log USER_ID"
|
|
13
|
+
exit 1
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
entries = Llmemory::ForgetLog.new(store: short_term_store).entries(user_id)
|
|
17
|
+
|
|
18
|
+
if entries.empty?
|
|
19
|
+
puts "No forget audit entries for user #{user_id}."
|
|
20
|
+
return
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
entries.each do |e|
|
|
24
|
+
type = e[:memory_type] || e["memory_type"]
|
|
25
|
+
count = e[:count] || e["count"]
|
|
26
|
+
reason = e[:reason] || e["reason"]
|
|
27
|
+
at = e[:at] || e["at"]
|
|
28
|
+
ids = e[:ids] || e["ids"] || []
|
|
29
|
+
reason_str = reason ? " — #{reason}" : ""
|
|
30
|
+
puts "[#{at}] #{type}: removed #{count} (#{ids.join(', ')})#{reason_str}"
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
# Runs the cognitive maintenance pass for a user: reflect -> mine skills ->
|
|
9
|
+
# expire. Each step is isolated; failures are reported, not fatal.
|
|
10
|
+
class Maintain < Commands::Base
|
|
11
|
+
def option_parser(parser)
|
|
12
|
+
parser.on("--[no-]reflect", "Distill insights from recent episodes (default: on)") { |v| @reflect = v }
|
|
13
|
+
parser.on("--mine-skills", "Mine and register skills from episodes (default: config)") { @mine = true }
|
|
14
|
+
parser.on("--[no-]expire", "Soft-archive entries past their TTL (default: on)") { |v| @expire = v }
|
|
15
|
+
parser.on("--window N", Integer, "Episodes to reflect over (default 10)") { |v| @window = v }
|
|
16
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def execute(argv, _opts)
|
|
20
|
+
user_id = argv.first
|
|
21
|
+
unless user_id
|
|
22
|
+
$stderr.puts "Usage: llmemory maintain USER_ID [--[no-]reflect] [--mine-skills] [--[no-]expire] [--window N] [--store TYPE]"
|
|
23
|
+
exit 1
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
opts = {
|
|
27
|
+
episodic: Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type)),
|
|
28
|
+
procedural: Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type)),
|
|
29
|
+
semantic: build_semantic(user_id),
|
|
30
|
+
reflect: @reflect.nil? ? true : @reflect,
|
|
31
|
+
expire: @expire.nil? ? true : @expire
|
|
32
|
+
}
|
|
33
|
+
opts[:mine_skills] = @mine unless @mine.nil?
|
|
34
|
+
opts[:reflection_window] = @window if @window
|
|
35
|
+
|
|
36
|
+
report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
|
|
37
|
+
print_report(user_id, report)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def build_semantic(user_id)
|
|
43
|
+
if Llmemory.configuration.long_term_type.to_s == "graph_based"
|
|
44
|
+
Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id, storage: graph_based_storage(@store_type))
|
|
45
|
+
else
|
|
46
|
+
Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id, storage: file_based_storage(@store_type))
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def print_report(user_id, report)
|
|
51
|
+
expired = report[:expired] || {}
|
|
52
|
+
puts "Cognitive pass for #{user_id}:"
|
|
53
|
+
puts " insights: #{Array(report[:insights]).size}"
|
|
54
|
+
puts " skills mined: #{Array(report[:mined]).size}"
|
|
55
|
+
puts " expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
|
|
56
|
+
errors = report[:errors] || {}
|
|
57
|
+
errors.each { |step, msg| puts " error (#{step}): #{msg}" }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
# Mines reusable skills from a user's successful episodes. Human-in-the-loop
|
|
9
|
+
# by default: prints proposals and writes nothing unless --register is given.
|
|
10
|
+
class MineSkills < Commands::Base
|
|
11
|
+
def option_parser(parser)
|
|
12
|
+
parser.on("--window N", Integer, "Episodes to mine (default 20)") { |v| @window = v }
|
|
13
|
+
parser.on("--outcomes LIST", "Comma-separated outcome allowlist (e.g. success,recovered)") { |v| @outcomes = v.split(",").map(&:strip) }
|
|
14
|
+
parser.on("--register", "Register the proposals instead of only printing them") { @register = true }
|
|
15
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def execute(argv, _opts)
|
|
19
|
+
user_id = argv.first
|
|
20
|
+
unless user_id
|
|
21
|
+
$stderr.puts "Usage: llmemory mine-skills USER_ID [--window N] [--outcomes LIST] [--register] [--store TYPE]"
|
|
22
|
+
exit 1
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
episodic = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type))
|
|
26
|
+
procedural = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type))
|
|
27
|
+
result = Llmemory::SkillMining::Miner.new(episodic: episodic, procedural: procedural).mine(
|
|
28
|
+
window: @window || Llmemory::SkillMining::Miner::DEFAULT_WINDOW,
|
|
29
|
+
outcomes: @outcomes,
|
|
30
|
+
auto_register: @register
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
if result.empty?
|
|
34
|
+
puts "No skills could be mined for user #{user_id}."
|
|
35
|
+
return
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
if @register
|
|
39
|
+
puts "Registered #{result.size} mined skill(s): #{result.join(', ')}"
|
|
40
|
+
else
|
|
41
|
+
puts "#{result.size} skill proposal(s) for user #{user_id} (not registered):"
|
|
42
|
+
result.each do |p|
|
|
43
|
+
puts " - #{p[:name]} (#{p[:kind]}, confidence: #{p[:confidence]}): #{p[:description] || p[:body]}"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
class Procedural < Commands::Base
|
|
9
|
+
def option_parser(parser)
|
|
10
|
+
parser.on("--limit N", Integer, "Max number of skills (newest first)") { |v| @limit = v }
|
|
11
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def execute(argv, _opts)
|
|
15
|
+
user_id = argv.first
|
|
16
|
+
unless user_id
|
|
17
|
+
$stderr.puts "Usage: llmemory skills USER_ID [--limit N] [--store TYPE]"
|
|
18
|
+
exit 1
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
storage = procedural_storage(@store_type)
|
|
22
|
+
skills = storage.list_skills(user_id, limit: @limit)
|
|
23
|
+
|
|
24
|
+
if skills.empty?
|
|
25
|
+
puts "No skills for user #{user_id}."
|
|
26
|
+
return
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
skills.each do |s|
|
|
30
|
+
id = s[:id] || s["id"]
|
|
31
|
+
name = s[:name] || s["name"]
|
|
32
|
+
kind = s[:kind] || s["kind"]
|
|
33
|
+
version = s[:version] || s["version"]
|
|
34
|
+
succ = (s[:success_count] || s["success_count"] || 0).to_i
|
|
35
|
+
fail = (s[:failure_count] || s["failure_count"] || 0).to_i
|
|
36
|
+
total = succ + fail
|
|
37
|
+
rate = total.zero? ? "n/a" : format("%.2f", succ.to_f / total)
|
|
38
|
+
puts "[#{id}] #{name} v#{version} (#{kind}) — success rate: #{rate} (#{succ}/#{total})"
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
class Working < Commands::Base
|
|
9
|
+
def execute(argv, _opts)
|
|
10
|
+
user_id, session_id = argv
|
|
11
|
+
unless user_id && session_id
|
|
12
|
+
$stderr.puts "Usage: llmemory working USER_ID SESSION_ID"
|
|
13
|
+
exit 1
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
wm = Llmemory::WorkingMemory.new(user_id: user_id, session_id: session_id, store: short_term_store)
|
|
17
|
+
state = wm.to_h
|
|
18
|
+
|
|
19
|
+
if state.empty?
|
|
20
|
+
puts "Empty working memory for user #{user_id}, session #{session_id}."
|
|
21
|
+
return
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
state.each do |slot, value|
|
|
25
|
+
puts "#{slot}: #{value.inspect}"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/llmemory/cli.rb
CHANGED
|
@@ -5,6 +5,12 @@ require_relative "cli/commands/base"
|
|
|
5
5
|
require_relative "cli/commands/users"
|
|
6
6
|
require_relative "cli/commands/short_term"
|
|
7
7
|
require_relative "cli/commands/long_term"
|
|
8
|
+
require_relative "cli/commands/episodic"
|
|
9
|
+
require_relative "cli/commands/procedural"
|
|
10
|
+
require_relative "cli/commands/working"
|
|
11
|
+
require_relative "cli/commands/forget_log"
|
|
12
|
+
require_relative "cli/commands/mine_skills"
|
|
13
|
+
require_relative "cli/commands/maintain"
|
|
8
14
|
require_relative "cli/commands/stats"
|
|
9
15
|
require_relative "cli/commands/search"
|
|
10
16
|
require_relative "cli/commands/mcp"
|
|
@@ -47,6 +53,12 @@ module Llmemory
|
|
|
47
53
|
"nodes" => Cli::Commands::LongTerm::Nodes,
|
|
48
54
|
"edges" => Cli::Commands::LongTerm::Edges,
|
|
49
55
|
"graph" => Cli::Commands::LongTerm::Graph,
|
|
56
|
+
"episodes" => Cli::Commands::Episodic,
|
|
57
|
+
"skills" => Cli::Commands::Procedural,
|
|
58
|
+
"working" => Cli::Commands::Working,
|
|
59
|
+
"forget_log" => Cli::Commands::ForgetLog,
|
|
60
|
+
"mine_skills" => Cli::Commands::MineSkills,
|
|
61
|
+
"maintain" => Cli::Commands::Maintain,
|
|
50
62
|
"search" => Cli::Commands::Search,
|
|
51
63
|
"stats" => Cli::Commands::Stats,
|
|
52
64
|
"mcp" => Cli::Commands::Mcp
|
|
@@ -68,6 +80,12 @@ module Llmemory
|
|
|
68
80
|
nodes USER_ID List graph nodes (graph-based)
|
|
69
81
|
edges USER_ID List graph edges (graph-based)
|
|
70
82
|
graph USER_ID Export graph (--format dot|json)
|
|
83
|
+
episodes USER_ID List recorded episodes (episodic memory)
|
|
84
|
+
skills USER_ID List registered skills (procedural memory)
|
|
85
|
+
working USER_ID SESSION Show working-memory slots for a session
|
|
86
|
+
forget-log USER_ID Show audit of forgotten entries
|
|
87
|
+
mine-skills USER_ID Mine reusable skills from episodes (--register to save)
|
|
88
|
+
maintain USER_ID Run the cognitive maintenance pass (reflect/mine/expire)
|
|
71
89
|
search USER_ID "query" Search in memory
|
|
72
90
|
stats [USER_ID] Show statistics
|
|
73
91
|
mcp [serve] Start MCP server for LLM agents
|
|
@@ -11,6 +11,8 @@ module Llmemory
|
|
|
11
11
|
:long_term_type,
|
|
12
12
|
:long_term_store,
|
|
13
13
|
:long_term_storage_path,
|
|
14
|
+
:episodic_vector_enabled,
|
|
15
|
+
:procedural_vector_enabled,
|
|
14
16
|
:database_url,
|
|
15
17
|
:vector_store,
|
|
16
18
|
:time_decay_half_life_days,
|
|
@@ -43,7 +45,10 @@ module Llmemory
|
|
|
43
45
|
:embedding_cache_enabled,
|
|
44
46
|
:embedding_cache_max_entries,
|
|
45
47
|
:max_message_chars,
|
|
46
|
-
:message_sanitizer_enabled
|
|
48
|
+
:message_sanitizer_enabled,
|
|
49
|
+
:ttl_episodic_days,
|
|
50
|
+
:ttl_procedural_days,
|
|
51
|
+
:skill_mining_enabled
|
|
47
52
|
|
|
48
53
|
def initialize
|
|
49
54
|
@llm_provider = :openai
|
|
@@ -55,6 +60,11 @@ module Llmemory
|
|
|
55
60
|
@long_term_type = :file_based
|
|
56
61
|
@long_term_store = :memory
|
|
57
62
|
@long_term_storage_path = ENV["LLMEMORY_STORAGE_PATH"] || "./llmemory_data"
|
|
63
|
+
@episodic_vector_enabled = false
|
|
64
|
+
@procedural_vector_enabled = false
|
|
65
|
+
@ttl_episodic_days = nil
|
|
66
|
+
@ttl_procedural_days = nil
|
|
67
|
+
@skill_mining_enabled = false
|
|
58
68
|
@database_url = ENV["DATABASE_URL"]
|
|
59
69
|
@vector_store = nil
|
|
60
70
|
@time_decay_half_life_days = 30
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
# Lightweight instrumentation seam. When ActiveSupport::Notifications is
|
|
5
|
+
# available (Rails apps, opt-in elsewhere), events are published with the
|
|
6
|
+
# `.llmemory` suffix so subscribers can hook into LLM calls, retrieval,
|
|
7
|
+
# writes and forgets for metrics, traces or cost dashboards. When AS is not
|
|
8
|
+
# loaded, the block is yielded transparently and no event is emitted.
|
|
9
|
+
#
|
|
10
|
+
# Events (payload keys are best-effort; subscribers should treat them as
|
|
11
|
+
# optional):
|
|
12
|
+
#
|
|
13
|
+
# llm_invoke.llmemory provider:, model:, prompt_chars:, response_chars:
|
|
14
|
+
# llm_embed.llmemory provider:, model:, text_chars:, dimensions:
|
|
15
|
+
# memory_write.llmemory memory_type:, user_id:
|
|
16
|
+
# memory_forget.llmemory memory_type:, user_id:, count:
|
|
17
|
+
# retrieve.llmemory query_chars:, candidates:, results:
|
|
18
|
+
# iterative_retrieve.llmemory hops:, total_results:
|
|
19
|
+
# reflect.llmemory window:, insights:
|
|
20
|
+
# mine_skills.llmemory window:, auto_register:
|
|
21
|
+
module Instrumentation
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
def instrument(event, payload = {})
|
|
25
|
+
name = "#{event}.llmemory"
|
|
26
|
+
if defined?(ActiveSupport::Notifications)
|
|
27
|
+
ActiveSupport::Notifications.instrument(name, payload) { yield if block_given? }
|
|
28
|
+
else
|
|
29
|
+
yield if block_given?
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -16,22 +16,26 @@ module Llmemory
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def invoke(prompt)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
19
|
+
result = nil
|
|
20
|
+
Llmemory::Instrumentation.instrument(:llm_invoke, provider: :anthropic, model: @model, prompt_chars: prompt.to_s.length) do
|
|
21
|
+
response = connection.post("v1/messages") do |req|
|
|
22
|
+
req.body = {
|
|
23
|
+
model: @model,
|
|
24
|
+
max_tokens: 1024,
|
|
25
|
+
messages: [{ role: "user", content: prompt }]
|
|
26
|
+
}.to_json
|
|
27
|
+
req.headers["Content-Type"] = "application/json"
|
|
28
|
+
req.headers["x-api-key"] = @api_key
|
|
29
|
+
req.headers["anthropic-version"] = "2023-06-01"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
|
|
33
|
+
|
|
34
|
+
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
35
|
+
content = body.dig("content", 0, "text")
|
|
36
|
+
result = content&.strip || ""
|
|
28
37
|
end
|
|
29
|
-
|
|
30
|
-
raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
|
|
31
|
-
|
|
32
|
-
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
33
|
-
content = body.dig("content", 0, "text")
|
|
34
|
-
content&.strip || ""
|
|
38
|
+
result
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
private
|
data/lib/llmemory/llm/openai.rb
CHANGED
|
@@ -16,20 +16,24 @@ module Llmemory
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def invoke(prompt)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
19
|
+
result = nil
|
|
20
|
+
Llmemory::Instrumentation.instrument(:llm_invoke, provider: :openai, model: @model, prompt_chars: prompt.to_s.length) do
|
|
21
|
+
response = connection.post("chat/completions") do |req|
|
|
22
|
+
req.body = {
|
|
23
|
+
model: @model,
|
|
24
|
+
messages: [{ role: "user", content: prompt }],
|
|
25
|
+
temperature: 0.3
|
|
26
|
+
}.to_json
|
|
27
|
+
req.headers["Content-Type"] = "application/json"
|
|
28
|
+
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
29
|
+
end
|
|
28
30
|
|
|
29
|
-
|
|
31
|
+
raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
|
|
30
32
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
34
|
+
result = body.dig("choices", 0, "message", "content")&.strip || ""
|
|
35
|
+
end
|
|
36
|
+
result
|
|
33
37
|
end
|
|
34
38
|
|
|
35
39
|
# Calls the model with response_format json_schema (Structured Outputs).
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "episode"
|
|
4
4
|
require_relative "storage"
|
|
5
5
|
require_relative "../../memory_module"
|
|
6
|
+
require_relative "../../vector_store"
|
|
6
7
|
|
|
7
8
|
module Llmemory
|
|
8
9
|
module LongTerm
|
|
@@ -12,16 +13,19 @@ module Llmemory
|
|
|
12
13
|
# memory (file/graph), not replace it, and to feed reflection (P2), which
|
|
13
14
|
# distills episodes into semantic knowledge.
|
|
14
15
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
16
|
+
# Recording/retrieval are deterministic and LLM-free by default. Semantic
|
|
17
|
+
# (embedding) retrieval is opt-in via `config.episodic_vector_enabled` or by
|
|
18
|
+
# injecting a `vector_store:`; when off, search is keyword-only (unchanged).
|
|
17
19
|
class Memory
|
|
18
20
|
include Llmemory::MemoryModule
|
|
19
21
|
|
|
20
22
|
attr_reader :user_id, :storage
|
|
21
23
|
|
|
22
|
-
def initialize(user_id:, storage: nil)
|
|
24
|
+
def initialize(user_id:, storage: nil, vector_store: nil)
|
|
23
25
|
@user_id = user_id
|
|
24
26
|
@storage = storage || Storages.build
|
|
27
|
+
@vector_store = vector_store
|
|
28
|
+
@vector_explicit = !vector_store.nil?
|
|
25
29
|
end
|
|
26
30
|
|
|
27
31
|
# Records a trajectory. `steps` is an array of hashes with any of
|
|
@@ -39,15 +43,17 @@ module Llmemory
|
|
|
39
43
|
episode.searchable_text, method: "episode_recording", confidence: episode.importance
|
|
40
44
|
)
|
|
41
45
|
record = episode.to_h.merge(provenance: provenance)
|
|
42
|
-
@storage.save_episode(@user_id, record)
|
|
46
|
+
id = @storage.save_episode(@user_id, record)
|
|
47
|
+
index_vector(id, episode.searchable_text)
|
|
48
|
+
id
|
|
43
49
|
end
|
|
44
50
|
|
|
45
51
|
def recent_episodes(limit: 10)
|
|
46
52
|
@storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
|
|
47
53
|
end
|
|
48
54
|
|
|
49
|
-
def episodes(limit: nil)
|
|
50
|
-
@storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
|
|
55
|
+
def episodes(limit: nil, offset: nil)
|
|
56
|
+
@storage.list_episodes(@user_id, limit: limit, offset: offset).map { |e| Episode.from_h(e) }
|
|
51
57
|
end
|
|
52
58
|
|
|
53
59
|
def find_episode(id)
|
|
@@ -62,49 +68,111 @@ module Llmemory
|
|
|
62
68
|
# Retrieval Engine integration. Returns candidates shaped like the other
|
|
63
69
|
# long-term memories so the Engine can rank episodes by relevance,
|
|
64
70
|
# recency (temporal decay) and importance (P3), with provenance (P10).
|
|
71
|
+
# Hybrid (vector + keyword) when a vector store is active; otherwise
|
|
72
|
+
# keyword-only.
|
|
65
73
|
def search_candidates(query, user_id: nil, top_k: 20)
|
|
66
74
|
uid = user_id || @user_id
|
|
67
75
|
return [] unless uid == @user_id
|
|
68
76
|
|
|
69
|
-
@storage.search_episodes(uid, query).first(top_k).map
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
timestamp: episode.created_at,
|
|
75
|
-
score: 1.0,
|
|
76
|
-
importance: episode.importance,
|
|
77
|
-
evergreen: false,
|
|
78
|
-
provenance: e[:provenance] || e["provenance"]
|
|
79
|
-
}
|
|
80
|
-
end
|
|
77
|
+
keyword = @storage.search_episodes(uid, query).first(top_k).map { |e| candidate_for(e, 1.0) }
|
|
78
|
+
vs = vector_store
|
|
79
|
+
return keyword unless vs
|
|
80
|
+
|
|
81
|
+
merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
|
|
81
82
|
end
|
|
82
83
|
|
|
83
84
|
# --- MemoryModule uniform interface ---
|
|
84
85
|
|
|
85
86
|
def write(steps:, summary: nil, outcome: nil, importance: 0.5, **_meta)
|
|
86
|
-
|
|
87
|
+
result = nil
|
|
88
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "episodic", user_id: @user_id) do
|
|
89
|
+
result = record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
|
|
90
|
+
end
|
|
91
|
+
result
|
|
87
92
|
end
|
|
88
93
|
|
|
89
|
-
def list(user_id: nil, limit: nil)
|
|
90
|
-
episodes(limit: limit)
|
|
94
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
95
|
+
episodes(limit: limit, offset: offset)
|
|
91
96
|
end
|
|
92
97
|
|
|
93
98
|
def stats(user_id: nil)
|
|
94
99
|
{ episodes: count }
|
|
95
100
|
end
|
|
96
101
|
|
|
97
|
-
def forget(ids:, reason: nil)
|
|
102
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
98
103
|
requested = Array(ids).map(&:to_s)
|
|
99
104
|
existing = @storage.list_episodes(@user_id).map { |e| (e[:id] || e["id"]).to_s }
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
targeted = requested & existing
|
|
106
|
+
count = case mode
|
|
107
|
+
when :hard then @storage.delete_episodes(@user_id, targeted).to_i
|
|
108
|
+
else @storage.archive_episodes(@user_id, targeted).to_i
|
|
109
|
+
end
|
|
110
|
+
forget_log.record(@user_id, memory_type: "episodic", ids: targeted, reason: reason)
|
|
111
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "episodic", user_id: @user_id, count: count, mode: mode)
|
|
112
|
+
count
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Storage accessor for the TTL maintenance job.
|
|
116
|
+
def expired_ids(cutoff:)
|
|
117
|
+
@storage.expired_episode_ids(@user_id, cutoff: cutoff)
|
|
104
118
|
end
|
|
105
119
|
|
|
106
120
|
private
|
|
107
121
|
|
|
122
|
+
# Active vector store: the injected one, or a config-gated lazy build.
|
|
123
|
+
# Returns nil when semantic search is disabled (default).
|
|
124
|
+
def vector_store
|
|
125
|
+
if @vector_explicit
|
|
126
|
+
@vector_store
|
|
127
|
+
elsif Llmemory.configuration.episodic_vector_enabled
|
|
128
|
+
@vector_store ||= Llmemory::VectorStore.build(source_type: "episode")
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Best-effort embedding indexing; a failure must never break recording.
|
|
133
|
+
def index_vector(id, text)
|
|
134
|
+
vs = vector_store
|
|
135
|
+
return if vs.nil? || text.to_s.strip.empty?
|
|
136
|
+
embedding = vs.embed(text)
|
|
137
|
+
return unless embedding
|
|
138
|
+
vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
|
|
139
|
+
rescue StandardError
|
|
140
|
+
nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def vector_candidates(query, top_k, vs)
|
|
144
|
+
vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
|
|
145
|
+
raw = @storage.get_episode(@user_id, r[:id] || r["id"])
|
|
146
|
+
raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
|
|
147
|
+
end
|
|
148
|
+
rescue StandardError
|
|
149
|
+
[]
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def candidate_for(raw, score)
|
|
153
|
+
episode = Episode.from_h(raw)
|
|
154
|
+
{
|
|
155
|
+
id: episode.id,
|
|
156
|
+
text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
|
|
157
|
+
timestamp: episode.created_at,
|
|
158
|
+
score: score,
|
|
159
|
+
importance: episode.importance,
|
|
160
|
+
evergreen: false,
|
|
161
|
+
provenance: raw[:provenance] || raw["provenance"]
|
|
162
|
+
}
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Dedup by id keeping the higher score; highest score first, capped.
|
|
166
|
+
def merge_candidates(primary, secondary, top_k)
|
|
167
|
+
by_id = {}
|
|
168
|
+
(primary + secondary).each do |c|
|
|
169
|
+
key = c[:id] || c[:text]
|
|
170
|
+
existing = by_id[key]
|
|
171
|
+
by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
|
|
172
|
+
end
|
|
173
|
+
by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
|
|
174
|
+
end
|
|
175
|
+
|
|
108
176
|
# Cheap, deterministic summary when the caller does not provide one.
|
|
109
177
|
# LLM-based summarization is reflection's job (P2).
|
|
110
178
|
def derive_summary(steps)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "storages/base"
|
|
4
4
|
require_relative "storages/memory_storage"
|
|
5
5
|
require_relative "storages/file_storage"
|
|
6
|
+
require_relative "storages/database_storage"
|
|
6
7
|
|
|
7
8
|
module Llmemory
|
|
8
9
|
module LongTerm
|
|
@@ -11,16 +12,17 @@ module Llmemory
|
|
|
11
12
|
Storage = Storages::MemoryStorage
|
|
12
13
|
|
|
13
14
|
module Storages
|
|
14
|
-
def self.build(store: nil, base_path: nil)
|
|
15
|
+
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
15
16
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
16
17
|
when :memory
|
|
17
18
|
MemoryStorage.new
|
|
18
19
|
when :file
|
|
19
20
|
FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
|
|
20
|
-
when :postgres, :database
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
when :postgres, :database
|
|
22
|
+
DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
|
|
23
|
+
when :active_record, :activerecord
|
|
24
|
+
require_relative "storages/active_record_storage"
|
|
25
|
+
ActiveRecordStorage.new
|
|
24
26
|
else
|
|
25
27
|
MemoryStorage.new
|
|
26
28
|
end
|