llmemory 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +54 -3
  3. data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
  4. data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
  5. data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
  6. data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
  7. data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
  8. data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
  9. data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
  10. data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
  11. data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
  12. data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
  13. data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
  14. data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
  15. data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
  16. data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
  17. data/config/routes.rb +14 -0
  18. data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +22 -0
  19. data/lib/llmemory/cli/commands/base.rb +8 -0
  20. data/lib/llmemory/cli/commands/episodic.rb +42 -0
  21. data/lib/llmemory/cli/commands/forget_log.rb +36 -0
  22. data/lib/llmemory/cli/commands/maintain.rb +62 -0
  23. data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
  24. data/lib/llmemory/cli/commands/procedural.rb +44 -0
  25. data/lib/llmemory/cli/commands/working.rb +31 -0
  26. data/lib/llmemory/cli.rb +18 -0
  27. data/lib/llmemory/configuration.rb +11 -1
  28. data/lib/llmemory/instrumentation.rb +33 -0
  29. data/lib/llmemory/llm/anthropic.rb +19 -15
  30. data/lib/llmemory/llm/openai.rb +16 -12
  31. data/lib/llmemory/long_term/episodic/memory.rb +94 -26
  32. data/lib/llmemory/long_term/episodic/storage.rb +7 -5
  33. data/lib/llmemory/long_term/episodic/storages/active_record_models.rb +17 -0
  34. data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +103 -0
  35. data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
  36. data/lib/llmemory/long_term/episodic/storages/database_storage.rb +156 -0
  37. data/lib/llmemory/long_term/episodic/storages/file_storage.rb +28 -8
  38. data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +36 -6
  39. data/lib/llmemory/long_term/file_based/memory.rb +12 -4
  40. data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +15 -6
  41. data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
  42. data/lib/llmemory/long_term/file_based/storages/database_storage.rb +20 -8
  43. data/lib/llmemory/long_term/file_based/storages/file_storage.rb +6 -6
  44. data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +6 -6
  45. data/lib/llmemory/long_term/graph_based/memory.rb +89 -64
  46. data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +4 -2
  47. data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
  48. data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
  49. data/lib/llmemory/long_term/procedural/memory.rb +97 -30
  50. data/lib/llmemory/long_term/procedural/skill.rb +6 -2
  51. data/lib/llmemory/long_term/procedural/storage.rb +7 -5
  52. data/lib/llmemory/long_term/procedural/storages/active_record_models.rb +17 -0
  53. data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +114 -0
  54. data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
  55. data/lib/llmemory/long_term/procedural/storages/database_storage.rb +169 -0
  56. data/lib/llmemory/long_term/procedural/storages/file_storage.rb +29 -9
  57. data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +37 -7
  58. data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
  59. data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
  60. data/lib/llmemory/maintenance.rb +2 -0
  61. data/lib/llmemory/mcp/server.rb +17 -1
  62. data/lib/llmemory/mcp/tools/memory_episode_record.rb +48 -0
  63. data/lib/llmemory/mcp/tools/memory_episodes.rb +43 -0
  64. data/lib/llmemory/mcp/tools/memory_forget.rb +53 -0
  65. data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
  66. data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
  67. data/lib/llmemory/mcp/tools/memory_retrieve.rb +10 -2
  68. data/lib/llmemory/mcp/tools/memory_skill_register.rb +35 -0
  69. data/lib/llmemory/mcp/tools/memory_skill_report.rb +35 -0
  70. data/lib/llmemory/mcp/tools/memory_skills.rb +43 -0
  71. data/lib/llmemory/memory.rb +48 -3
  72. data/lib/llmemory/memory_module.rb +13 -6
  73. data/lib/llmemory/reflection/reflector.rb +24 -20
  74. data/lib/llmemory/retrieval/bm25_scorer.rb +1 -1
  75. data/lib/llmemory/retrieval/engine.rb +25 -16
  76. data/lib/llmemory/retrieval/mmr_reranker.rb +1 -1
  77. data/lib/llmemory/short_term/session_lifecycle.rb +19 -3
  78. data/lib/llmemory/skill_mining/miner.rb +163 -0
  79. data/lib/llmemory/skill_mining.rb +8 -0
  80. data/lib/llmemory/tokenizer.rb +27 -0
  81. data/lib/llmemory/vector_store/active_record_store.rb +4 -3
  82. data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
  83. data/lib/llmemory/vector_store.rb +14 -0
  84. data/lib/llmemory/version.rb +1 -1
  85. data/lib/llmemory.rb +3 -0
  86. metadata +39 -1
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ class Episodic < Commands::Base
9
+ def option_parser(parser)
10
+ parser.on("--limit N", Integer, "Max number of episodes (newest first)") { |v| @limit = v }
11
+ parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
12
+ end
13
+
14
+ def execute(argv, _opts)
15
+ user_id = argv.first
16
+ unless user_id
17
+ $stderr.puts "Usage: llmemory episodes USER_ID [--limit N] [--store TYPE]"
18
+ exit 1
19
+ end
20
+
21
+ storage = episodic_storage(@store_type)
22
+ episodes = storage.list_episodes(user_id, limit: @limit)
23
+
24
+ if episodes.empty?
25
+ puts "No episodes for user #{user_id}."
26
+ return
27
+ end
28
+
29
+ episodes.each do |e|
30
+ id = e[:id] || e["id"]
31
+ summary = e[:summary] || e["summary"]
32
+ outcome = e[:outcome] || e["outcome"]
33
+ importance = e[:importance] || e["importance"]
34
+ steps = e[:steps] || e["steps"] || []
35
+ puts "[#{id}] (importance: #{importance}; outcome: #{outcome || 'n/a'}) #{summary}"
36
+ puts " steps: #{Array(steps).size}"
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ class ForgetLog < Commands::Base
9
+ def execute(argv, _opts)
10
+ user_id = argv.first
11
+ unless user_id
12
+ $stderr.puts "Usage: llmemory forget-log USER_ID"
13
+ exit 1
14
+ end
15
+
16
+ entries = Llmemory::ForgetLog.new(store: short_term_store).entries(user_id)
17
+
18
+ if entries.empty?
19
+ puts "No forget audit entries for user #{user_id}."
20
+ return
21
+ end
22
+
23
+ entries.each do |e|
24
+ type = e[:memory_type] || e["memory_type"]
25
+ count = e[:count] || e["count"]
26
+ reason = e[:reason] || e["reason"]
27
+ at = e[:at] || e["at"]
28
+ ids = e[:ids] || e["ids"] || []
29
+ reason_str = reason ? " — #{reason}" : ""
30
+ puts "[#{at}] #{type}: removed #{count} (#{ids.join(', ')})#{reason_str}"
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ # Runs the cognitive maintenance pass for a user: reflect -> mine skills ->
9
+ # expire. Each step is isolated; failures are reported, not fatal.
10
+ class Maintain < Commands::Base
11
+ def option_parser(parser)
12
+ parser.on("--[no-]reflect", "Distill insights from recent episodes (default: on)") { |v| @reflect = v }
13
+ parser.on("--mine-skills", "Mine and register skills from episodes (default: config)") { @mine = true }
14
+ parser.on("--[no-]expire", "Soft-archive entries past their TTL (default: on)") { |v| @expire = v }
15
+ parser.on("--window N", Integer, "Episodes to reflect over (default 10)") { |v| @window = v }
16
+ parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
17
+ end
18
+
19
+ def execute(argv, _opts)
20
+ user_id = argv.first
21
+ unless user_id
22
+ $stderr.puts "Usage: llmemory maintain USER_ID [--[no-]reflect] [--mine-skills] [--[no-]expire] [--window N] [--store TYPE]"
23
+ exit 1
24
+ end
25
+
26
+ opts = {
27
+ episodic: Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type)),
28
+ procedural: Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type)),
29
+ semantic: build_semantic(user_id),
30
+ reflect: @reflect.nil? ? true : @reflect,
31
+ expire: @expire.nil? ? true : @expire
32
+ }
33
+ opts[:mine_skills] = @mine unless @mine.nil?
34
+ opts[:reflection_window] = @window if @window
35
+
36
+ report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
37
+ print_report(user_id, report)
38
+ end
39
+
40
+ private
41
+
42
+ def build_semantic(user_id)
43
+ if Llmemory.configuration.long_term_type.to_s == "graph_based"
44
+ Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id, storage: graph_based_storage(@store_type))
45
+ else
46
+ Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id, storage: file_based_storage(@store_type))
47
+ end
48
+ end
49
+
50
+ def print_report(user_id, report)
51
+ expired = report[:expired] || {}
52
+ puts "Cognitive pass for #{user_id}:"
53
+ puts " insights: #{Array(report[:insights]).size}"
54
+ puts " skills mined: #{Array(report[:mined]).size}"
55
+ puts " expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
56
+ errors = report[:errors] || {}
57
+ errors.each { |step, msg| puts " error (#{step}): #{msg}" }
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ # Mines reusable skills from a user's successful episodes. Human-in-the-loop
9
+ # by default: prints proposals and writes nothing unless --register is given.
10
+ class MineSkills < Commands::Base
11
+ def option_parser(parser)
12
+ parser.on("--window N", Integer, "Episodes to mine (default 20)") { |v| @window = v }
13
+ parser.on("--outcomes LIST", "Comma-separated outcome allowlist (e.g. success,recovered)") { |v| @outcomes = v.split(",").map(&:strip) }
14
+ parser.on("--register", "Register the proposals instead of only printing them") { @register = true }
15
+ parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
16
+ end
17
+
18
+ def execute(argv, _opts)
19
+ user_id = argv.first
20
+ unless user_id
21
+ $stderr.puts "Usage: llmemory mine-skills USER_ID [--window N] [--outcomes LIST] [--register] [--store TYPE]"
22
+ exit 1
23
+ end
24
+
25
+ episodic = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type))
26
+ procedural = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type))
27
+ result = Llmemory::SkillMining::Miner.new(episodic: episodic, procedural: procedural).mine(
28
+ window: @window || Llmemory::SkillMining::Miner::DEFAULT_WINDOW,
29
+ outcomes: @outcomes,
30
+ auto_register: @register
31
+ )
32
+
33
+ if result.empty?
34
+ puts "No skills could be mined for user #{user_id}."
35
+ return
36
+ end
37
+
38
+ if @register
39
+ puts "Registered #{result.size} mined skill(s): #{result.join(', ')}"
40
+ else
41
+ puts "#{result.size} skill proposal(s) for user #{user_id} (not registered):"
42
+ result.each do |p|
43
+ puts " - #{p[:name]} (#{p[:kind]}, confidence: #{p[:confidence]}): #{p[:description] || p[:body]}"
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ class Procedural < Commands::Base
9
+ def option_parser(parser)
10
+ parser.on("--limit N", Integer, "Max number of skills (newest first)") { |v| @limit = v }
11
+ parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
12
+ end
13
+
14
+ def execute(argv, _opts)
15
+ user_id = argv.first
16
+ unless user_id
17
+ $stderr.puts "Usage: llmemory skills USER_ID [--limit N] [--store TYPE]"
18
+ exit 1
19
+ end
20
+
21
+ storage = procedural_storage(@store_type)
22
+ skills = storage.list_skills(user_id, limit: @limit)
23
+
24
+ if skills.empty?
25
+ puts "No skills for user #{user_id}."
26
+ return
27
+ end
28
+
29
+ skills.each do |s|
30
+ id = s[:id] || s["id"]
31
+ name = s[:name] || s["name"]
32
+ kind = s[:kind] || s["kind"]
33
+ version = s[:version] || s["version"]
34
+ succ = (s[:success_count] || s["success_count"] || 0).to_i
35
+ fail = (s[:failure_count] || s["failure_count"] || 0).to_i
36
+ total = succ + fail
37
+ rate = total.zero? ? "n/a" : format("%.2f", succ.to_f / total)
38
+ puts "[#{id}] #{name} v#{version} (#{kind}) — success rate: #{rate} (#{succ}/#{total})"
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ class Working < Commands::Base
9
+ def execute(argv, _opts)
10
+ user_id, session_id = argv
11
+ unless user_id && session_id
12
+ $stderr.puts "Usage: llmemory working USER_ID SESSION_ID"
13
+ exit 1
14
+ end
15
+
16
+ wm = Llmemory::WorkingMemory.new(user_id: user_id, session_id: session_id, store: short_term_store)
17
+ state = wm.to_h
18
+
19
+ if state.empty?
20
+ puts "Empty working memory for user #{user_id}, session #{session_id}."
21
+ return
22
+ end
23
+
24
+ state.each do |slot, value|
25
+ puts "#{slot}: #{value.inspect}"
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
data/lib/llmemory/cli.rb CHANGED
@@ -5,6 +5,12 @@ require_relative "cli/commands/base"
5
5
  require_relative "cli/commands/users"
6
6
  require_relative "cli/commands/short_term"
7
7
  require_relative "cli/commands/long_term"
8
+ require_relative "cli/commands/episodic"
9
+ require_relative "cli/commands/procedural"
10
+ require_relative "cli/commands/working"
11
+ require_relative "cli/commands/forget_log"
12
+ require_relative "cli/commands/mine_skills"
13
+ require_relative "cli/commands/maintain"
8
14
  require_relative "cli/commands/stats"
9
15
  require_relative "cli/commands/search"
10
16
  require_relative "cli/commands/mcp"
@@ -47,6 +53,12 @@ module Llmemory
47
53
  "nodes" => Cli::Commands::LongTerm::Nodes,
48
54
  "edges" => Cli::Commands::LongTerm::Edges,
49
55
  "graph" => Cli::Commands::LongTerm::Graph,
56
+ "episodes" => Cli::Commands::Episodic,
57
+ "skills" => Cli::Commands::Procedural,
58
+ "working" => Cli::Commands::Working,
59
+ "forget_log" => Cli::Commands::ForgetLog,
60
+ "mine_skills" => Cli::Commands::MineSkills,
61
+ "maintain" => Cli::Commands::Maintain,
50
62
  "search" => Cli::Commands::Search,
51
63
  "stats" => Cli::Commands::Stats,
52
64
  "mcp" => Cli::Commands::Mcp
@@ -68,6 +80,12 @@ module Llmemory
68
80
  nodes USER_ID List graph nodes (graph-based)
69
81
  edges USER_ID List graph edges (graph-based)
70
82
  graph USER_ID Export graph (--format dot|json)
83
+ episodes USER_ID List recorded episodes (episodic memory)
84
+ skills USER_ID List registered skills (procedural memory)
85
+ working USER_ID SESSION Show working-memory slots for a session
86
+ forget-log USER_ID Show audit of forgotten entries
87
+ mine-skills USER_ID Mine reusable skills from episodes (--register to save)
88
+ maintain USER_ID Run the cognitive maintenance pass (reflect/mine/expire)
71
89
  search USER_ID "query" Search in memory
72
90
  stats [USER_ID] Show statistics
73
91
  mcp [serve] Start MCP server for LLM agents
@@ -11,6 +11,8 @@ module Llmemory
11
11
  :long_term_type,
12
12
  :long_term_store,
13
13
  :long_term_storage_path,
14
+ :episodic_vector_enabled,
15
+ :procedural_vector_enabled,
14
16
  :database_url,
15
17
  :vector_store,
16
18
  :time_decay_half_life_days,
@@ -43,7 +45,10 @@ module Llmemory
43
45
  :embedding_cache_enabled,
44
46
  :embedding_cache_max_entries,
45
47
  :max_message_chars,
46
- :message_sanitizer_enabled
48
+ :message_sanitizer_enabled,
49
+ :ttl_episodic_days,
50
+ :ttl_procedural_days,
51
+ :skill_mining_enabled
47
52
 
48
53
  def initialize
49
54
  @llm_provider = :openai
@@ -55,6 +60,11 @@ module Llmemory
55
60
  @long_term_type = :file_based
56
61
  @long_term_store = :memory
57
62
  @long_term_storage_path = ENV["LLMEMORY_STORAGE_PATH"] || "./llmemory_data"
63
+ @episodic_vector_enabled = false
64
+ @procedural_vector_enabled = false
65
+ @ttl_episodic_days = nil
66
+ @ttl_procedural_days = nil
67
+ @skill_mining_enabled = false
58
68
  @database_url = ENV["DATABASE_URL"]
59
69
  @vector_store = nil
60
70
  @time_decay_half_life_days = 30
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Llmemory
4
+ # Lightweight instrumentation seam. When ActiveSupport::Notifications is
5
+ # available (Rails apps, opt-in elsewhere), events are published with the
6
+ # `.llmemory` suffix so subscribers can hook into LLM calls, retrieval,
7
+ # writes and forgets for metrics, traces or cost dashboards. When AS is not
8
+ # loaded, the block is yielded transparently and no event is emitted.
9
+ #
10
+ # Events (payload keys are best-effort; subscribers should treat them as
11
+ # optional):
12
+ #
13
+ # llm_invoke.llmemory provider:, model:, prompt_chars:, response_chars:
14
+ # llm_embed.llmemory provider:, model:, text_chars:, dimensions:
15
+ # memory_write.llmemory memory_type:, user_id:
16
+ # memory_forget.llmemory memory_type:, user_id:, count:
17
+ # retrieve.llmemory query_chars:, candidates:, results:
18
+ # iterative_retrieve.llmemory hops:, total_results:
19
+ # reflect.llmemory window:, insights:
20
+ # mine_skills.llmemory window:, auto_register:
21
+ module Instrumentation
22
+ module_function
23
+
24
+ def instrument(event, payload = {})
25
+ name = "#{event}.llmemory"
26
+ if defined?(ActiveSupport::Notifications)
27
+ ActiveSupport::Notifications.instrument(name, payload) { yield if block_given? }
28
+ else
29
+ yield if block_given?
30
+ end
31
+ end
32
+ end
33
+ end
@@ -16,22 +16,26 @@ module Llmemory
16
16
  end
17
17
 
18
18
  def invoke(prompt)
19
- response = connection.post("v1/messages") do |req|
20
- req.body = {
21
- model: @model,
22
- max_tokens: 1024,
23
- messages: [{ role: "user", content: prompt }]
24
- }.to_json
25
- req.headers["Content-Type"] = "application/json"
26
- req.headers["x-api-key"] = @api_key
27
- req.headers["anthropic-version"] = "2023-06-01"
19
+ result = nil
20
+ Llmemory::Instrumentation.instrument(:llm_invoke, provider: :anthropic, model: @model, prompt_chars: prompt.to_s.length) do
21
+ response = connection.post("v1/messages") do |req|
22
+ req.body = {
23
+ model: @model,
24
+ max_tokens: 1024,
25
+ messages: [{ role: "user", content: prompt }]
26
+ }.to_json
27
+ req.headers["Content-Type"] = "application/json"
28
+ req.headers["x-api-key"] = @api_key
29
+ req.headers["anthropic-version"] = "2023-06-01"
30
+ end
31
+
32
+ raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
33
+
34
+ body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
35
+ content = body.dig("content", 0, "text")
36
+ result = content&.strip || ""
28
37
  end
29
-
30
- raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
31
-
32
- body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
33
- content = body.dig("content", 0, "text")
34
- content&.strip || ""
38
+ result
35
39
  end
36
40
 
37
41
  private
@@ -16,20 +16,24 @@ module Llmemory
16
16
  end
17
17
 
18
18
  def invoke(prompt)
19
- response = connection.post("chat/completions") do |req|
20
- req.body = {
21
- model: @model,
22
- messages: [{ role: "user", content: prompt }],
23
- temperature: 0.3
24
- }.to_json
25
- req.headers["Content-Type"] = "application/json"
26
- req.headers["Authorization"] = "Bearer #{@api_key}"
27
- end
19
+ result = nil
20
+ Llmemory::Instrumentation.instrument(:llm_invoke, provider: :openai, model: @model, prompt_chars: prompt.to_s.length) do
21
+ response = connection.post("chat/completions") do |req|
22
+ req.body = {
23
+ model: @model,
24
+ messages: [{ role: "user", content: prompt }],
25
+ temperature: 0.3
26
+ }.to_json
27
+ req.headers["Content-Type"] = "application/json"
28
+ req.headers["Authorization"] = "Bearer #{@api_key}"
29
+ end
28
30
 
29
- raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
31
+ raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
30
32
 
31
- body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
32
- body.dig("choices", 0, "message", "content")&.strip || ""
33
+ body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
34
+ result = body.dig("choices", 0, "message", "content")&.strip || ""
35
+ end
36
+ result
33
37
  end
34
38
 
35
39
  # Calls the model with response_format json_schema (Structured Outputs).
@@ -3,6 +3,7 @@
3
3
  require_relative "episode"
4
4
  require_relative "storage"
5
5
  require_relative "../../memory_module"
6
+ require_relative "../../vector_store"
6
7
 
7
8
  module Llmemory
8
9
  module LongTerm
@@ -12,16 +13,19 @@ module Llmemory
12
13
  # memory (file/graph), not replace it, and to feed reflection (P2), which
13
14
  # distills episodes into semantic knowledge.
14
15
  #
15
- # Deliberately LLM-free: recording and retrieval are deterministic. Higher
16
- # order summarization belongs to reflection.
16
+ # Recording/retrieval are deterministic and LLM-free by default. Semantic
17
+ # (embedding) retrieval is opt-in via `config.episodic_vector_enabled` or by
18
+ # injecting a `vector_store:`; when off, search is keyword-only (unchanged).
17
19
  class Memory
18
20
  include Llmemory::MemoryModule
19
21
 
20
22
  attr_reader :user_id, :storage
21
23
 
22
- def initialize(user_id:, storage: nil)
24
+ def initialize(user_id:, storage: nil, vector_store: nil)
23
25
  @user_id = user_id
24
26
  @storage = storage || Storages.build
27
+ @vector_store = vector_store
28
+ @vector_explicit = !vector_store.nil?
25
29
  end
26
30
 
27
31
  # Records a trajectory. `steps` is an array of hashes with any of
@@ -39,15 +43,17 @@ module Llmemory
39
43
  episode.searchable_text, method: "episode_recording", confidence: episode.importance
40
44
  )
41
45
  record = episode.to_h.merge(provenance: provenance)
42
- @storage.save_episode(@user_id, record)
46
+ id = @storage.save_episode(@user_id, record)
47
+ index_vector(id, episode.searchable_text)
48
+ id
43
49
  end
44
50
 
45
51
  def recent_episodes(limit: 10)
46
52
  @storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
47
53
  end
48
54
 
49
- def episodes(limit: nil)
50
- @storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
55
+ def episodes(limit: nil, offset: nil)
56
+ @storage.list_episodes(@user_id, limit: limit, offset: offset).map { |e| Episode.from_h(e) }
51
57
  end
52
58
 
53
59
  def find_episode(id)
@@ -62,49 +68,111 @@ module Llmemory
62
68
  # Retrieval Engine integration. Returns candidates shaped like the other
63
69
  # long-term memories so the Engine can rank episodes by relevance,
64
70
  # recency (temporal decay) and importance (P3), with provenance (P10).
71
+ # Hybrid (vector + keyword) when a vector store is active; otherwise
72
+ # keyword-only.
65
73
  def search_candidates(query, user_id: nil, top_k: 20)
66
74
  uid = user_id || @user_id
67
75
  return [] unless uid == @user_id
68
76
 
69
- @storage.search_episodes(uid, query).first(top_k).map do |e|
70
- episode = Episode.from_h(e)
71
- {
72
- id: episode.id,
73
- text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
74
- timestamp: episode.created_at,
75
- score: 1.0,
76
- importance: episode.importance,
77
- evergreen: false,
78
- provenance: e[:provenance] || e["provenance"]
79
- }
80
- end
77
+ keyword = @storage.search_episodes(uid, query).first(top_k).map { |e| candidate_for(e, 1.0) }
78
+ vs = vector_store
79
+ return keyword unless vs
80
+
81
+ merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
81
82
  end
82
83
 
83
84
  # --- MemoryModule uniform interface ---
84
85
 
85
86
  def write(steps:, summary: nil, outcome: nil, importance: 0.5, **_meta)
86
- record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
87
+ result = nil
88
+ Llmemory::Instrumentation.instrument(:memory_write, memory_type: "episodic", user_id: @user_id) do
89
+ result = record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
90
+ end
91
+ result
87
92
  end
88
93
 
89
- def list(user_id: nil, limit: nil)
90
- episodes(limit: limit)
94
+ def list(user_id: nil, limit: nil, offset: nil)
95
+ episodes(limit: limit, offset: offset)
91
96
  end
92
97
 
93
98
  def stats(user_id: nil)
94
99
  { episodes: count }
95
100
  end
96
101
 
97
- def forget(ids:, reason: nil)
102
+ def forget(ids:, reason: nil, mode: :soft)
98
103
  requested = Array(ids).map(&:to_s)
99
104
  existing = @storage.list_episodes(@user_id).map { |e| (e[:id] || e["id"]).to_s }
100
- removed = requested & existing
101
- @storage.delete_episodes(@user_id, removed)
102
- forget_log.record(@user_id, memory_type: "episodic", ids: removed, reason: reason)
103
- removed.size
105
+ targeted = requested & existing
106
+ count = case mode
107
+ when :hard then @storage.delete_episodes(@user_id, targeted).to_i
108
+ else @storage.archive_episodes(@user_id, targeted).to_i
109
+ end
110
+ forget_log.record(@user_id, memory_type: "episodic", ids: targeted, reason: reason)
111
+ Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "episodic", user_id: @user_id, count: count, mode: mode)
112
+ count
113
+ end
114
+
115
+ # Storage accessor for the TTL maintenance job.
116
+ def expired_ids(cutoff:)
117
+ @storage.expired_episode_ids(@user_id, cutoff: cutoff)
104
118
  end
105
119
 
106
120
  private
107
121
 
122
+ # Active vector store: the injected one, or a config-gated lazy build.
123
+ # Returns nil when semantic search is disabled (default).
124
+ def vector_store
125
+ if @vector_explicit
126
+ @vector_store
127
+ elsif Llmemory.configuration.episodic_vector_enabled
128
+ @vector_store ||= Llmemory::VectorStore.build(source_type: "episode")
129
+ end
130
+ end
131
+
132
+ # Best-effort embedding indexing; a failure must never break recording.
133
+ def index_vector(id, text)
134
+ vs = vector_store
135
+ return if vs.nil? || text.to_s.strip.empty?
136
+ embedding = vs.embed(text)
137
+ return unless embedding
138
+ vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
139
+ rescue StandardError
140
+ nil
141
+ end
142
+
143
+ def vector_candidates(query, top_k, vs)
144
+ vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
145
+ raw = @storage.get_episode(@user_id, r[:id] || r["id"])
146
+ raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
147
+ end
148
+ rescue StandardError
149
+ []
150
+ end
151
+
152
+ def candidate_for(raw, score)
153
+ episode = Episode.from_h(raw)
154
+ {
155
+ id: episode.id,
156
+ text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
157
+ timestamp: episode.created_at,
158
+ score: score,
159
+ importance: episode.importance,
160
+ evergreen: false,
161
+ provenance: raw[:provenance] || raw["provenance"]
162
+ }
163
+ end
164
+
165
+ # Dedup by id keeping the higher score; highest score first, capped.
166
+ def merge_candidates(primary, secondary, top_k)
167
+ by_id = {}
168
+ (primary + secondary).each do |c|
169
+ key = c[:id] || c[:text]
170
+ existing = by_id[key]
171
+ by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
172
+ end
173
+ by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
174
+ end
175
+
108
176
  # Cheap, deterministic summary when the caller does not provide one.
109
177
  # LLM-based summarization is reflection's job (P2).
110
178
  def derive_summary(steps)
@@ -3,6 +3,7 @@
3
3
  require_relative "storages/base"
4
4
  require_relative "storages/memory_storage"
5
5
  require_relative "storages/file_storage"
6
+ require_relative "storages/database_storage"
6
7
 
7
8
  module Llmemory
8
9
  module LongTerm
@@ -11,16 +12,17 @@ module Llmemory
11
12
  Storage = Storages::MemoryStorage
12
13
 
13
14
  module Storages
14
- def self.build(store: nil, base_path: nil)
15
+ def self.build(store: nil, base_path: nil, database_url: nil)
15
16
  case (store || Llmemory.configuration.long_term_store).to_s.to_sym
16
17
  when :memory
17
18
  MemoryStorage.new
18
19
  when :file
19
20
  FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
20
- when :postgres, :database, :active_record, :activerecord
21
- raise NotImplementedError,
22
- "Episodic SQL/ActiveRecord storage is not implemented yet; use :memory or :file " \
23
- "(or pass an explicit storage instance)."
21
+ when :postgres, :database
22
+ DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
23
+ when :active_record, :activerecord
24
+ require_relative "storages/active_record_storage"
25
+ ActiveRecordStorage.new
24
26
  else
25
27
  MemoryStorage.new
26
28
  end