llmemory 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +47 -1
  3. data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
  4. data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
  5. data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
  6. data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
  7. data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
  8. data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
  9. data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
  10. data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
  11. data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
  12. data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
  13. data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
  14. data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
  15. data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
  16. data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
  17. data/config/routes.rb +14 -0
  18. data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +2 -0
  19. data/lib/llmemory/cli/commands/maintain.rb +62 -0
  20. data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
  21. data/lib/llmemory/cli.rb +6 -0
  22. data/lib/llmemory/configuration.rb +7 -1
  23. data/lib/llmemory/instrumentation.rb +33 -0
  24. data/lib/llmemory/llm/anthropic.rb +19 -15
  25. data/lib/llmemory/llm/openai.rb +16 -12
  26. data/lib/llmemory/long_term/episodic/memory.rb +23 -10
  27. data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +14 -4
  28. data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
  29. data/lib/llmemory/long_term/episodic/storages/database_storage.rb +26 -5
  30. data/lib/llmemory/long_term/episodic/storages/file_storage.rb +27 -6
  31. data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +35 -4
  32. data/lib/llmemory/long_term/file_based/memory.rb +12 -4
  33. data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +4 -2
  34. data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
  35. data/lib/llmemory/long_term/file_based/storages/database_storage.rb +4 -2
  36. data/lib/llmemory/long_term/file_based/storages/file_storage.rb +4 -2
  37. data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +4 -2
  38. data/lib/llmemory/long_term/graph_based/memory.rb +12 -4
  39. data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +4 -2
  40. data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
  41. data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
  42. data/lib/llmemory/long_term/procedural/memory.rb +26 -13
  43. data/lib/llmemory/long_term/procedural/skill.rb +6 -2
  44. data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +15 -5
  45. data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
  46. data/lib/llmemory/long_term/procedural/storages/database_storage.rb +27 -6
  47. data/lib/llmemory/long_term/procedural/storages/file_storage.rb +28 -7
  48. data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +36 -5
  49. data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
  50. data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
  51. data/lib/llmemory/maintenance.rb +2 -0
  52. data/lib/llmemory/mcp/server.rb +5 -1
  53. data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
  54. data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
  55. data/lib/llmemory/memory.rb +20 -0
  56. data/lib/llmemory/memory_module.rb +13 -6
  57. data/lib/llmemory/reflection/reflector.rb +24 -20
  58. data/lib/llmemory/retrieval/engine.rb +25 -16
  59. data/lib/llmemory/skill_mining/miner.rb +163 -0
  60. data/lib/llmemory/skill_mining.rb +8 -0
  61. data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
  62. data/lib/llmemory/version.rb +1 -1
  63. data/lib/llmemory.rb +2 -0
  64. metadata +22 -1
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ # Runs the cognitive maintenance pass for a user: reflect -> mine skills ->
9
+ # expire. Each step is isolated; failures are reported, not fatal.
10
+ class Maintain < Commands::Base
11
+ def option_parser(parser)
12
+ parser.on("--[no-]reflect", "Distill insights from recent episodes (default: on)") { |v| @reflect = v }
13
+ parser.on("--mine-skills", "Mine and register skills from episodes (default: config)") { @mine = true }
14
+ parser.on("--[no-]expire", "Soft-archive entries past their TTL (default: on)") { |v| @expire = v }
15
+ parser.on("--window N", Integer, "Episodes to reflect over (default 10)") { |v| @window = v }
16
+ parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
17
+ end
18
+
19
+ def execute(argv, _opts)
20
+ user_id = argv.first
21
+ unless user_id
22
+ $stderr.puts "Usage: llmemory maintain USER_ID [--[no-]reflect] [--mine-skills] [--[no-]expire] [--window N] [--store TYPE]"
23
+ exit 1
24
+ end
25
+
26
+ opts = {
27
+ episodic: Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type)),
28
+ procedural: Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type)),
29
+ semantic: build_semantic(user_id),
30
+ reflect: @reflect.nil? ? true : @reflect,
31
+ expire: @expire.nil? ? true : @expire
32
+ }
33
+ opts[:mine_skills] = @mine unless @mine.nil?
34
+ opts[:reflection_window] = @window if @window
35
+
36
+ report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
37
+ print_report(user_id, report)
38
+ end
39
+
40
+ private
41
+
42
+ def build_semantic(user_id)
43
+ if Llmemory.configuration.long_term_type.to_s == "graph_based"
44
+ Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id, storage: graph_based_storage(@store_type))
45
+ else
46
+ Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id, storage: file_based_storage(@store_type))
47
+ end
48
+ end
49
+
50
+ def print_report(user_id, report)
51
+ expired = report[:expired] || {}
52
+ puts "Cognitive pass for #{user_id}:"
53
+ puts " insights: #{Array(report[:insights]).size}"
54
+ puts " skills mined: #{Array(report[:mined]).size}"
55
+ puts " expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
56
+ errors = report[:errors] || {}
57
+ errors.each { |step, msg| puts " error (#{step}): #{msg}" }
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Llmemory
6
+ module Cli
7
+ module Commands
8
+ # Mines reusable skills from a user's successful episodes. Human-in-the-loop
9
+ # by default: prints proposals and writes nothing unless --register is given.
10
+ class MineSkills < Commands::Base
11
+ def option_parser(parser)
12
+ parser.on("--window N", Integer, "Episodes to mine (default 20)") { |v| @window = v }
13
+ parser.on("--outcomes LIST", "Comma-separated outcome allowlist (e.g. success,recovered)") { |v| @outcomes = v.split(",").map(&:strip) }
14
+ parser.on("--register", "Register the proposals instead of only printing them") { @register = true }
15
+ parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
16
+ end
17
+
18
+ def execute(argv, _opts)
19
+ user_id = argv.first
20
+ unless user_id
21
+ $stderr.puts "Usage: llmemory mine-skills USER_ID [--window N] [--outcomes LIST] [--register] [--store TYPE]"
22
+ exit 1
23
+ end
24
+
25
+ episodic = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type))
26
+ procedural = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type))
27
+ result = Llmemory::SkillMining::Miner.new(episodic: episodic, procedural: procedural).mine(
28
+ window: @window || Llmemory::SkillMining::Miner::DEFAULT_WINDOW,
29
+ outcomes: @outcomes,
30
+ auto_register: @register
31
+ )
32
+
33
+ if result.empty?
34
+ puts "No skills could be mined for user #{user_id}."
35
+ return
36
+ end
37
+
38
+ if @register
39
+ puts "Registered #{result.size} mined skill(s): #{result.join(', ')}"
40
+ else
41
+ puts "#{result.size} skill proposal(s) for user #{user_id} (not registered):"
42
+ result.each do |p|
43
+ puts " - #{p[:name]} (#{p[:kind]}, confidence: #{p[:confidence]}): #{p[:description] || p[:body]}"
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
data/lib/llmemory/cli.rb CHANGED
@@ -9,6 +9,8 @@ require_relative "cli/commands/episodic"
9
9
  require_relative "cli/commands/procedural"
10
10
  require_relative "cli/commands/working"
11
11
  require_relative "cli/commands/forget_log"
12
+ require_relative "cli/commands/mine_skills"
13
+ require_relative "cli/commands/maintain"
12
14
  require_relative "cli/commands/stats"
13
15
  require_relative "cli/commands/search"
14
16
  require_relative "cli/commands/mcp"
@@ -55,6 +57,8 @@ module Llmemory
55
57
  "skills" => Cli::Commands::Procedural,
56
58
  "working" => Cli::Commands::Working,
57
59
  "forget_log" => Cli::Commands::ForgetLog,
60
+ "mine_skills" => Cli::Commands::MineSkills,
61
+ "maintain" => Cli::Commands::Maintain,
58
62
  "search" => Cli::Commands::Search,
59
63
  "stats" => Cli::Commands::Stats,
60
64
  "mcp" => Cli::Commands::Mcp
@@ -80,6 +84,8 @@ module Llmemory
80
84
  skills USER_ID List registered skills (procedural memory)
81
85
  working USER_ID SESSION Show working-memory slots for a session
82
86
  forget-log USER_ID Show audit of forgotten entries
87
+ mine-skills USER_ID Mine reusable skills from episodes (--register to save)
88
+ maintain USER_ID Run the cognitive maintenance pass (reflect/mine/expire)
83
89
  search USER_ID "query" Search in memory
84
90
  stats [USER_ID] Show statistics
85
91
  mcp [serve] Start MCP server for LLM agents
@@ -45,7 +45,10 @@ module Llmemory
45
45
  :embedding_cache_enabled,
46
46
  :embedding_cache_max_entries,
47
47
  :max_message_chars,
48
- :message_sanitizer_enabled
48
+ :message_sanitizer_enabled,
49
+ :ttl_episodic_days,
50
+ :ttl_procedural_days,
51
+ :skill_mining_enabled
49
52
 
50
53
  def initialize
51
54
  @llm_provider = :openai
@@ -59,6 +62,9 @@ module Llmemory
59
62
  @long_term_storage_path = ENV["LLMEMORY_STORAGE_PATH"] || "./llmemory_data"
60
63
  @episodic_vector_enabled = false
61
64
  @procedural_vector_enabled = false
65
+ @ttl_episodic_days = nil
66
+ @ttl_procedural_days = nil
67
+ @skill_mining_enabled = false
62
68
  @database_url = ENV["DATABASE_URL"]
63
69
  @vector_store = nil
64
70
  @time_decay_half_life_days = 30
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Llmemory
4
+ # Lightweight instrumentation seam. When ActiveSupport::Notifications is
5
+ # available (Rails apps, opt-in elsewhere), events are published with the
6
+ # `.llmemory` suffix so subscribers can hook into LLM calls, retrieval,
7
+ # writes and forgets for metrics, traces or cost dashboards. When AS is not
8
+ # loaded, the block is yielded transparently and no event is emitted.
9
+ #
10
+ # Events (payload keys are best-effort; subscribers should treat them as
11
+ # optional):
12
+ #
13
+ # llm_invoke.llmemory provider:, model:, prompt_chars:, response_chars:
14
+ # llm_embed.llmemory provider:, model:, text_chars:, dimensions:
15
+ # memory_write.llmemory memory_type:, user_id:
16
+ # memory_forget.llmemory memory_type:, user_id:, count:
17
+ # retrieve.llmemory query_chars:, candidates:, results:
18
+ # iterative_retrieve.llmemory hops:, total_results:
19
+ # reflect.llmemory window:, insights:
20
+ # mine_skills.llmemory window:, auto_register:
21
+ module Instrumentation
22
+ module_function
23
+
24
+ def instrument(event, payload = {})
25
+ name = "#{event}.llmemory"
26
+ if defined?(ActiveSupport::Notifications)
27
+ ActiveSupport::Notifications.instrument(name, payload) { yield if block_given? }
28
+ else
29
+ yield if block_given?
30
+ end
31
+ end
32
+ end
33
+ end
@@ -16,22 +16,26 @@ module Llmemory
16
16
  end
17
17
 
18
18
  def invoke(prompt)
19
- response = connection.post("v1/messages") do |req|
20
- req.body = {
21
- model: @model,
22
- max_tokens: 1024,
23
- messages: [{ role: "user", content: prompt }]
24
- }.to_json
25
- req.headers["Content-Type"] = "application/json"
26
- req.headers["x-api-key"] = @api_key
27
- req.headers["anthropic-version"] = "2023-06-01"
19
+ result = nil
20
+ Llmemory::Instrumentation.instrument(:llm_invoke, provider: :anthropic, model: @model, prompt_chars: prompt.to_s.length) do
21
+ response = connection.post("v1/messages") do |req|
22
+ req.body = {
23
+ model: @model,
24
+ max_tokens: 1024,
25
+ messages: [{ role: "user", content: prompt }]
26
+ }.to_json
27
+ req.headers["Content-Type"] = "application/json"
28
+ req.headers["x-api-key"] = @api_key
29
+ req.headers["anthropic-version"] = "2023-06-01"
30
+ end
31
+
32
+ raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
33
+
34
+ body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
35
+ content = body.dig("content", 0, "text")
36
+ result = content&.strip || ""
28
37
  end
29
-
30
- raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
31
-
32
- body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
33
- content = body.dig("content", 0, "text")
34
- content&.strip || ""
38
+ result
35
39
  end
36
40
 
37
41
  private
@@ -16,20 +16,24 @@ module Llmemory
16
16
  end
17
17
 
18
18
  def invoke(prompt)
19
- response = connection.post("chat/completions") do |req|
20
- req.body = {
21
- model: @model,
22
- messages: [{ role: "user", content: prompt }],
23
- temperature: 0.3
24
- }.to_json
25
- req.headers["Content-Type"] = "application/json"
26
- req.headers["Authorization"] = "Bearer #{@api_key}"
27
- end
19
+ result = nil
20
+ Llmemory::Instrumentation.instrument(:llm_invoke, provider: :openai, model: @model, prompt_chars: prompt.to_s.length) do
21
+ response = connection.post("chat/completions") do |req|
22
+ req.body = {
23
+ model: @model,
24
+ messages: [{ role: "user", content: prompt }],
25
+ temperature: 0.3
26
+ }.to_json
27
+ req.headers["Content-Type"] = "application/json"
28
+ req.headers["Authorization"] = "Bearer #{@api_key}"
29
+ end
28
30
 
29
- raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
31
+ raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
30
32
 
31
- body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
32
- body.dig("choices", 0, "message", "content")&.strip || ""
33
+ body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
34
+ result = body.dig("choices", 0, "message", "content")&.strip || ""
35
+ end
36
+ result
33
37
  end
34
38
 
35
39
  # Calls the model with response_format json_schema (Structured Outputs).
@@ -52,8 +52,8 @@ module Llmemory
52
52
  @storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
53
53
  end
54
54
 
55
- def episodes(limit: nil)
56
- @storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
55
+ def episodes(limit: nil, offset: nil)
56
+ @storage.list_episodes(@user_id, limit: limit, offset: offset).map { |e| Episode.from_h(e) }
57
57
  end
58
58
 
59
59
  def find_episode(id)
@@ -84,24 +84,37 @@ module Llmemory
84
84
  # --- MemoryModule uniform interface ---
85
85
 
86
86
  def write(steps:, summary: nil, outcome: nil, importance: 0.5, **_meta)
87
- record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
87
+ result = nil
88
+ Llmemory::Instrumentation.instrument(:memory_write, memory_type: "episodic", user_id: @user_id) do
89
+ result = record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
90
+ end
91
+ result
88
92
  end
89
93
 
90
- def list(user_id: nil, limit: nil)
91
- episodes(limit: limit)
94
+ def list(user_id: nil, limit: nil, offset: nil)
95
+ episodes(limit: limit, offset: offset)
92
96
  end
93
97
 
94
98
  def stats(user_id: nil)
95
99
  { episodes: count }
96
100
  end
97
101
 
98
- def forget(ids:, reason: nil)
102
+ def forget(ids:, reason: nil, mode: :soft)
99
103
  requested = Array(ids).map(&:to_s)
100
104
  existing = @storage.list_episodes(@user_id).map { |e| (e[:id] || e["id"]).to_s }
101
- removed = requested & existing
102
- @storage.delete_episodes(@user_id, removed)
103
- forget_log.record(@user_id, memory_type: "episodic", ids: removed, reason: reason)
104
- removed.size
105
+ targeted = requested & existing
106
+ count = case mode
107
+ when :hard then @storage.delete_episodes(@user_id, targeted).to_i
108
+ else @storage.archive_episodes(@user_id, targeted).to_i
109
+ end
110
+ forget_log.record(@user_id, memory_type: "episodic", ids: targeted, reason: reason)
111
+ Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "episodic", user_id: @user_id, count: count, mode: mode)
112
+ count
113
+ end
114
+
115
+ # Storage accessor for the TTL maintenance job.
116
+ def expired_ids(cutoff:)
117
+ @storage.expired_episode_ids(@user_id, cutoff: cutoff)
105
118
  end
106
119
 
107
120
  private
@@ -42,25 +42,35 @@ module Llmemory
42
42
  rec&.data
43
43
  end
44
44
 
45
- def list_episodes(user_id, limit: nil)
46
- scope = LlmemoryEpisode.where(user_id: user_id).order(created_at: :desc)
45
+ def list_episodes(user_id, limit: nil, offset: nil)
46
+ scope = LlmemoryEpisode.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
47
47
  scope = scope.limit(limit) if limit && limit.to_i.positive?
48
+ scope = scope.offset(offset) if offset && offset.to_i.positive?
48
49
  scope.map(&:data)
49
50
  end
50
51
 
51
52
  def search_episodes(user_id, query)
52
- token_scope(LlmemoryEpisode.where(user_id: user_id), "search_text", query)
53
+ token_scope(LlmemoryEpisode.where(user_id: user_id, archived_at: nil), "search_text", query)
53
54
  .order(created_at: :desc).map(&:data)
54
55
  end
55
56
 
56
57
  def count_episodes(user_id)
57
- LlmemoryEpisode.where(user_id: user_id).count
58
+ LlmemoryEpisode.where(user_id: user_id, archived_at: nil).count
58
59
  end
59
60
 
60
61
  def delete_episodes(user_id, ids)
61
62
  LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
62
63
  end
63
64
 
65
+ def archive_episodes(user_id, ids)
66
+ LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s), archived_at: nil)
67
+ .update_all(archived_at: Time.current)
68
+ end
69
+
70
+ def expired_episode_ids(user_id, cutoff:)
71
+ LlmemoryEpisode.where(user_id: user_id, archived_at: nil).where("created_at < ?", cutoff).pluck(:id)
72
+ end
73
+
64
74
  def list_users
65
75
  LlmemoryEpisode.distinct.pluck(:user_id)
66
76
  end
@@ -16,8 +16,8 @@ module Llmemory
16
16
  raise NotImplementedError, "#{self.class}#get_episode must be implemented"
17
17
  end
18
18
 
19
- # Newest first. Optionally capped by limit.
20
- def list_episodes(user_id, limit: nil)
19
+ # Newest first. Optionally paginated with offset/limit.
20
+ def list_episodes(user_id, limit: nil, offset: nil)
21
21
  raise NotImplementedError, "#{self.class}#list_episodes must be implemented"
22
22
  end
23
23
 
@@ -34,6 +34,19 @@ module Llmemory
34
34
  raise NotImplementedError, "#{self.class}#delete_episodes must be implemented"
35
35
  end
36
36
 
37
+ # Soft-archives episodes by id (sets archived_at on the record). Archived
38
+ # episodes are excluded from list_episodes / search_episodes / count_episodes
39
+ # but remain accessible via get_episode. Returns the number archived.
40
+ def archive_episodes(user_id, ids)
41
+ raise NotImplementedError, "#{self.class}#archive_episodes must be implemented"
42
+ end
43
+
44
+ # Returns episodes whose created_at is older than the cutoff and that are
45
+ # not already archived. Used by the TTL maintenance job.
46
+ def expired_episode_ids(user_id, cutoff:)
47
+ raise NotImplementedError, "#{self.class}#expired_episode_ids must be implemented"
48
+ end
49
+
37
50
  def list_users
38
51
  raise NotImplementedError, "#{self.class}#list_users must be implemented"
39
52
  end
@@ -38,10 +38,11 @@ module Llmemory
38
38
  rows.any? ? parse_data(rows.first["data"]) : nil
39
39
  end
40
40
 
41
- def list_episodes(user_id, limit: nil)
41
+ def list_episodes(user_id, limit: nil, offset: nil)
42
42
  ensure_tables!
43
- sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 ORDER BY created_at DESC"
43
+ sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC"
44
44
  sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
45
+ sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
45
46
  conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
46
47
  end
47
48
 
@@ -49,14 +50,14 @@ module Llmemory
49
50
  ensure_tables!
50
51
  suffix, params = token_filter("search_text", query, 2)
51
52
  conn.exec_params(
52
- "SELECT data FROM llmemory_episodes WHERE user_id = $1#{suffix} ORDER BY created_at DESC",
53
+ "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
53
54
  [user_id, *params]
54
55
  ).map { |r| parse_data(r["data"]) }
55
56
  end
56
57
 
57
58
  def count_episodes(user_id)
58
59
  ensure_tables!
59
- conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1", [user_id]).first["c"].to_i
60
+ conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL", [user_id]).first["c"].to_i
60
61
  end
61
62
 
62
63
  def delete_episodes(user_id, ids)
@@ -66,6 +67,24 @@ module Llmemory
66
67
  end
67
68
  end
68
69
 
70
+ def archive_episodes(user_id, ids)
71
+ ensure_tables!
72
+ Array(ids).sum do |id|
73
+ conn.exec_params(
74
+ "UPDATE llmemory_episodes SET archived_at = NOW() WHERE user_id = $1 AND id = $2 AND archived_at IS NULL",
75
+ [user_id, id]
76
+ ).cmd_tuples
77
+ end
78
+ end
79
+
80
+ def expired_episode_ids(user_id, cutoff:)
81
+ ensure_tables!
82
+ conn.exec_params(
83
+ "SELECT id FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL AND created_at < $2",
84
+ [user_id, cutoff.iso8601]
85
+ ).map { |r| r["id"] }
86
+ end
87
+
69
88
  def list_users
70
89
  ensure_tables!
71
90
  conn.exec("SELECT DISTINCT user_id FROM llmemory_episodes").map { |r| r["user_id"] }
@@ -87,9 +106,11 @@ module Llmemory
87
106
  user_id TEXT NOT NULL,
88
107
  data JSONB NOT NULL DEFAULT '{}'::jsonb,
89
108
  search_text TEXT,
90
- created_at TIMESTAMPTZ NOT NULL
109
+ created_at TIMESTAMPTZ NOT NULL,
110
+ archived_at TIMESTAMPTZ
91
111
  );
92
112
  CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
113
+ ALTER TABLE llmemory_episodes ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
93
114
  SQL
94
115
  end
95
116
 
@@ -29,20 +29,19 @@ module Llmemory
29
29
  load_episode(path)
30
30
  end
31
31
 
32
- def list_episodes(user_id, limit: nil)
33
- sorted = all_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
32
+ def list_episodes(user_id, limit: nil, offset: nil)
33
+ sorted = active_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
34
+ sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
34
35
  limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
35
36
  end
36
37
 
37
38
  def search_episodes(user_id, query)
38
39
  return list_episodes(user_id) if query.to_s.strip.empty?
39
- all_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
40
+ active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
40
41
  end
41
42
 
42
43
  def count_episodes(user_id)
43
- dir = user_path(user_id, "episodes")
44
- return 0 unless Dir.exist?(dir)
45
- Dir.children(dir).count { |f| f.end_with?(".json") }
44
+ active_episodes(user_id).size
46
45
  end
47
46
 
48
47
  def delete_episodes(user_id, ids)
@@ -54,6 +53,24 @@ module Llmemory
54
53
  end
55
54
  end
56
55
 
56
+ def archive_episodes(user_id, ids)
57
+ Array(ids).map(&:to_s).count do |id|
58
+ path = episode_path(user_id, id)
59
+ next false unless File.file?(path)
60
+ data = JSON.parse(File.read(path))
61
+ next false if data["archived_at"]
62
+ data["archived_at"] = Time.now.iso8601
63
+ File.write(path, JSON.generate(data))
64
+ true
65
+ end
66
+ end
67
+
68
+ def expired_episode_ids(user_id, cutoff:)
69
+ active_episodes(user_id)
70
+ .select { |e| (e[:created_at] || Time.now) < cutoff }
71
+ .map { |e| e[:id].to_s }
72
+ end
73
+
57
74
  def list_users
58
75
  return [] unless Dir.exist?(@base_path)
59
76
  Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }
@@ -61,6 +78,10 @@ module Llmemory
61
78
 
62
79
  private
63
80
 
81
+ def active_episodes(user_id)
82
+ all_episodes(user_id).reject { |e| e[:archived_at] }
83
+ end
84
+
64
85
  def all_episodes(user_id)
65
86
  dir = user_path(user_id, "episodes")
66
87
  return [] unless Dir.exist?(dir)
@@ -25,18 +25,19 @@ module Llmemory
25
25
  @episodes[user_id].find { |e| e[:id] == id }
26
26
  end
27
27
 
28
- def list_episodes(user_id, limit: nil)
29
- sorted = @episodes[user_id].sort_by { |e| e[:created_at] }.reverse
28
+ def list_episodes(user_id, limit: nil, offset: nil)
29
+ sorted = active_episodes(user_id).sort_by { |e| as_time(e[:created_at]) }.reverse
30
+ sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
30
31
  limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
31
32
  end
32
33
 
33
34
  def search_episodes(user_id, query)
34
35
  return list_episodes(user_id) if query.to_s.strip.empty?
35
- @episodes[user_id].select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
36
+ active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
36
37
  end
37
38
 
38
39
  def count_episodes(user_id)
39
- @episodes[user_id].size
40
+ active_episodes(user_id).size
40
41
  end
41
42
 
42
43
  def delete_episodes(user_id, ids)
@@ -46,12 +47,42 @@ module Llmemory
46
47
  before - @episodes[user_id].size
47
48
  end
48
49
 
50
+ def archive_episodes(user_id, ids)
51
+ ids = Array(ids).map(&:to_s)
52
+ count = 0
53
+ @episodes[user_id].each do |e|
54
+ next unless ids.include?(e[:id].to_s)
55
+ next if e[:archived_at]
56
+ e[:archived_at] = Time.now
57
+ count += 1
58
+ end
59
+ count
60
+ end
61
+
62
+ def expired_episode_ids(user_id, cutoff:)
63
+ active_episodes(user_id)
64
+ .select { |e| as_time(e[:created_at]) < cutoff }
65
+ .map { |e| e[:id].to_s }
66
+ end
67
+
49
68
  def list_users
50
69
  @episodes.keys
51
70
  end
52
71
 
53
72
  private
54
73
 
74
+ def active_episodes(user_id)
75
+ @episodes[user_id].reject { |e| e[:archived_at] }
76
+ end
77
+
78
+ def as_time(value)
79
+ return Time.now if value.nil?
80
+ return value if value.is_a?(Time)
81
+ Time.parse(value.to_s)
82
+ rescue ArgumentError
83
+ Time.now
84
+ end
85
+
55
86
  def symbolize(hash)
56
87
  hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
57
88
  end
@@ -108,11 +108,15 @@ module Llmemory
108
108
  # --- MemoryModule uniform interface ---
109
109
 
110
110
  def write(payload, **_meta)
111
- memorize(payload)
111
+ result = nil
112
+ Llmemory::Instrumentation.instrument(:memory_write, memory_type: "file_based", user_id: @user_id) do
113
+ result = memorize(payload)
114
+ end
115
+ result
112
116
  end
113
117
 
114
- def list(user_id: nil, limit: nil)
115
- @storage.list_items(user_id: user_id || @user_id, limit: limit)
118
+ def list(user_id: nil, limit: nil, offset: nil)
119
+ @storage.list_items(user_id: user_id || @user_id, limit: limit, offset: offset)
116
120
  end
117
121
 
118
122
  def stats(user_id: nil)
@@ -120,7 +124,10 @@ module Llmemory
120
124
  end
121
125
 
122
126
  # Removes items/resources by id and records the removal in the audit log.
123
- def forget(ids:, reason: nil)
127
+ # Note: file-based storages currently implement `archive_*` as physical
128
+ # removal — `mode: :soft` and `mode: :hard` are functionally equivalent
129
+ # here. Kept for API uniformity.
130
+ def forget(ids:, reason: nil, mode: :soft)
124
131
  requested = Array(ids).map(&:to_s)
125
132
  existing = (@storage.get_all_items(@user_id) + @storage.get_all_resources(@user_id))
126
133
  .map { |r| (r[:id] || r["id"]).to_s }
@@ -128,6 +135,7 @@ module Llmemory
128
135
  @storage.archive_items(@user_id, removed)
129
136
  @storage.archive_resources(@user_id, removed)
130
137
  forget_log.record(@user_id, memory_type: "file_based", ids: removed, reason: reason)
138
+ Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "file_based", user_id: @user_id, count: removed.size, mode: mode)
131
139
  removed.size
132
140
  end
133
141