llmemory 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +47 -1
- data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
- data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
- data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
- data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
- data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
- data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
- data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
- data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
- data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
- data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
- data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
- data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
- data/config/routes.rb +14 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +2 -0
- data/lib/llmemory/cli/commands/maintain.rb +62 -0
- data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
- data/lib/llmemory/cli.rb +6 -0
- data/lib/llmemory/configuration.rb +7 -1
- data/lib/llmemory/instrumentation.rb +33 -0
- data/lib/llmemory/llm/anthropic.rb +19 -15
- data/lib/llmemory/llm/openai.rb +16 -12
- data/lib/llmemory/long_term/episodic/memory.rb +23 -10
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +14 -4
- data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +26 -5
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +27 -6
- data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +35 -4
- data/lib/llmemory/long_term/file_based/memory.rb +12 -4
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +4 -2
- data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +4 -2
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +4 -2
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/graph_based/memory.rb +12 -4
- data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +4 -2
- data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/procedural/memory.rb +26 -13
- data/lib/llmemory/long_term/procedural/skill.rb +6 -2
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +15 -5
- data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +27 -6
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +28 -7
- data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +36 -5
- data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
- data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +5 -1
- data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
- data/lib/llmemory/memory.rb +20 -0
- data/lib/llmemory/memory_module.rb +13 -6
- data/lib/llmemory/reflection/reflector.rb +24 -20
- data/lib/llmemory/retrieval/engine.rb +25 -16
- data/lib/llmemory/skill_mining/miner.rb +163 -0
- data/lib/llmemory/skill_mining.rb +8 -0
- data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +2 -0
- metadata +22 -1
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
# Runs the cognitive maintenance pass for a user: reflect -> mine skills ->
|
|
9
|
+
# expire. Each step is isolated; failures are reported, not fatal.
|
|
10
|
+
class Maintain < Commands::Base
|
|
11
|
+
def option_parser(parser)
|
|
12
|
+
parser.on("--[no-]reflect", "Distill insights from recent episodes (default: on)") { |v| @reflect = v }
|
|
13
|
+
parser.on("--mine-skills", "Mine and register skills from episodes (default: config)") { @mine = true }
|
|
14
|
+
parser.on("--[no-]expire", "Soft-archive entries past their TTL (default: on)") { |v| @expire = v }
|
|
15
|
+
parser.on("--window N", Integer, "Episodes to reflect over (default 10)") { |v| @window = v }
|
|
16
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def execute(argv, _opts)
|
|
20
|
+
user_id = argv.first
|
|
21
|
+
unless user_id
|
|
22
|
+
$stderr.puts "Usage: llmemory maintain USER_ID [--[no-]reflect] [--mine-skills] [--[no-]expire] [--window N] [--store TYPE]"
|
|
23
|
+
exit 1
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
opts = {
|
|
27
|
+
episodic: Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type)),
|
|
28
|
+
procedural: Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type)),
|
|
29
|
+
semantic: build_semantic(user_id),
|
|
30
|
+
reflect: @reflect.nil? ? true : @reflect,
|
|
31
|
+
expire: @expire.nil? ? true : @expire
|
|
32
|
+
}
|
|
33
|
+
opts[:mine_skills] = @mine unless @mine.nil?
|
|
34
|
+
opts[:reflection_window] = @window if @window
|
|
35
|
+
|
|
36
|
+
report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
|
|
37
|
+
print_report(user_id, report)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def build_semantic(user_id)
|
|
43
|
+
if Llmemory.configuration.long_term_type.to_s == "graph_based"
|
|
44
|
+
Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id, storage: graph_based_storage(@store_type))
|
|
45
|
+
else
|
|
46
|
+
Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id, storage: file_based_storage(@store_type))
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def print_report(user_id, report)
|
|
51
|
+
expired = report[:expired] || {}
|
|
52
|
+
puts "Cognitive pass for #{user_id}:"
|
|
53
|
+
puts " insights: #{Array(report[:insights]).size}"
|
|
54
|
+
puts " skills mined: #{Array(report[:mined]).size}"
|
|
55
|
+
puts " expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
|
|
56
|
+
errors = report[:errors] || {}
|
|
57
|
+
errors.each { |step, msg| puts " error (#{step}): #{msg}" }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module Cli
|
|
7
|
+
module Commands
|
|
8
|
+
# Mines reusable skills from a user's successful episodes. Human-in-the-loop
|
|
9
|
+
# by default: prints proposals and writes nothing unless --register is given.
|
|
10
|
+
class MineSkills < Commands::Base
|
|
11
|
+
def option_parser(parser)
|
|
12
|
+
parser.on("--window N", Integer, "Episodes to mine (default 20)") { |v| @window = v }
|
|
13
|
+
parser.on("--outcomes LIST", "Comma-separated outcome allowlist (e.g. success,recovered)") { |v| @outcomes = v.split(",").map(&:strip) }
|
|
14
|
+
parser.on("--register", "Register the proposals instead of only printing them") { @register = true }
|
|
15
|
+
parser.on("--store TYPE", "Storage type (memory|file|postgres|active_record)") { |v| @store_type = v }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def execute(argv, _opts)
|
|
19
|
+
user_id = argv.first
|
|
20
|
+
unless user_id
|
|
21
|
+
$stderr.puts "Usage: llmemory mine-skills USER_ID [--window N] [--outcomes LIST] [--register] [--store TYPE]"
|
|
22
|
+
exit 1
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
episodic = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id, storage: episodic_storage(@store_type))
|
|
26
|
+
procedural = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id, storage: procedural_storage(@store_type))
|
|
27
|
+
result = Llmemory::SkillMining::Miner.new(episodic: episodic, procedural: procedural).mine(
|
|
28
|
+
window: @window || Llmemory::SkillMining::Miner::DEFAULT_WINDOW,
|
|
29
|
+
outcomes: @outcomes,
|
|
30
|
+
auto_register: @register
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
if result.empty?
|
|
34
|
+
puts "No skills could be mined for user #{user_id}."
|
|
35
|
+
return
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
if @register
|
|
39
|
+
puts "Registered #{result.size} mined skill(s): #{result.join(', ')}"
|
|
40
|
+
else
|
|
41
|
+
puts "#{result.size} skill proposal(s) for user #{user_id} (not registered):"
|
|
42
|
+
result.each do |p|
|
|
43
|
+
puts " - #{p[:name]} (#{p[:kind]}, confidence: #{p[:confidence]}): #{p[:description] || p[:body]}"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
data/lib/llmemory/cli.rb
CHANGED
|
@@ -9,6 +9,8 @@ require_relative "cli/commands/episodic"
|
|
|
9
9
|
require_relative "cli/commands/procedural"
|
|
10
10
|
require_relative "cli/commands/working"
|
|
11
11
|
require_relative "cli/commands/forget_log"
|
|
12
|
+
require_relative "cli/commands/mine_skills"
|
|
13
|
+
require_relative "cli/commands/maintain"
|
|
12
14
|
require_relative "cli/commands/stats"
|
|
13
15
|
require_relative "cli/commands/search"
|
|
14
16
|
require_relative "cli/commands/mcp"
|
|
@@ -55,6 +57,8 @@ module Llmemory
|
|
|
55
57
|
"skills" => Cli::Commands::Procedural,
|
|
56
58
|
"working" => Cli::Commands::Working,
|
|
57
59
|
"forget_log" => Cli::Commands::ForgetLog,
|
|
60
|
+
"mine_skills" => Cli::Commands::MineSkills,
|
|
61
|
+
"maintain" => Cli::Commands::Maintain,
|
|
58
62
|
"search" => Cli::Commands::Search,
|
|
59
63
|
"stats" => Cli::Commands::Stats,
|
|
60
64
|
"mcp" => Cli::Commands::Mcp
|
|
@@ -80,6 +84,8 @@ module Llmemory
|
|
|
80
84
|
skills USER_ID List registered skills (procedural memory)
|
|
81
85
|
working USER_ID SESSION Show working-memory slots for a session
|
|
82
86
|
forget-log USER_ID Show audit of forgotten entries
|
|
87
|
+
mine-skills USER_ID Mine reusable skills from episodes (--register to save)
|
|
88
|
+
maintain USER_ID Run the cognitive maintenance pass (reflect/mine/expire)
|
|
83
89
|
search USER_ID "query" Search in memory
|
|
84
90
|
stats [USER_ID] Show statistics
|
|
85
91
|
mcp [serve] Start MCP server for LLM agents
|
|
@@ -45,7 +45,10 @@ module Llmemory
|
|
|
45
45
|
:embedding_cache_enabled,
|
|
46
46
|
:embedding_cache_max_entries,
|
|
47
47
|
:max_message_chars,
|
|
48
|
-
:message_sanitizer_enabled
|
|
48
|
+
:message_sanitizer_enabled,
|
|
49
|
+
:ttl_episodic_days,
|
|
50
|
+
:ttl_procedural_days,
|
|
51
|
+
:skill_mining_enabled
|
|
49
52
|
|
|
50
53
|
def initialize
|
|
51
54
|
@llm_provider = :openai
|
|
@@ -59,6 +62,9 @@ module Llmemory
|
|
|
59
62
|
@long_term_storage_path = ENV["LLMEMORY_STORAGE_PATH"] || "./llmemory_data"
|
|
60
63
|
@episodic_vector_enabled = false
|
|
61
64
|
@procedural_vector_enabled = false
|
|
65
|
+
@ttl_episodic_days = nil
|
|
66
|
+
@ttl_procedural_days = nil
|
|
67
|
+
@skill_mining_enabled = false
|
|
62
68
|
@database_url = ENV["DATABASE_URL"]
|
|
63
69
|
@vector_store = nil
|
|
64
70
|
@time_decay_half_life_days = 30
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
# Lightweight instrumentation seam. When ActiveSupport::Notifications is
|
|
5
|
+
# available (Rails apps, opt-in elsewhere), events are published with the
|
|
6
|
+
# `.llmemory` suffix so subscribers can hook into LLM calls, retrieval,
|
|
7
|
+
# writes and forgets for metrics, traces or cost dashboards. When AS is not
|
|
8
|
+
# loaded, the block is yielded transparently and no event is emitted.
|
|
9
|
+
#
|
|
10
|
+
# Events (payload keys are best-effort; subscribers should treat them as
|
|
11
|
+
# optional):
|
|
12
|
+
#
|
|
13
|
+
# llm_invoke.llmemory provider:, model:, prompt_chars:, response_chars:
|
|
14
|
+
# llm_embed.llmemory provider:, model:, text_chars:, dimensions:
|
|
15
|
+
# memory_write.llmemory memory_type:, user_id:
|
|
16
|
+
# memory_forget.llmemory memory_type:, user_id:, count:
|
|
17
|
+
# retrieve.llmemory query_chars:, candidates:, results:
|
|
18
|
+
# iterative_retrieve.llmemory hops:, total_results:
|
|
19
|
+
# reflect.llmemory window:, insights:
|
|
20
|
+
# mine_skills.llmemory window:, auto_register:
|
|
21
|
+
module Instrumentation
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
def instrument(event, payload = {})
|
|
25
|
+
name = "#{event}.llmemory"
|
|
26
|
+
if defined?(ActiveSupport::Notifications)
|
|
27
|
+
ActiveSupport::Notifications.instrument(name, payload) { yield if block_given? }
|
|
28
|
+
else
|
|
29
|
+
yield if block_given?
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -16,22 +16,26 @@ module Llmemory
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def invoke(prompt)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
19
|
+
result = nil
|
|
20
|
+
Llmemory::Instrumentation.instrument(:llm_invoke, provider: :anthropic, model: @model, prompt_chars: prompt.to_s.length) do
|
|
21
|
+
response = connection.post("v1/messages") do |req|
|
|
22
|
+
req.body = {
|
|
23
|
+
model: @model,
|
|
24
|
+
max_tokens: 1024,
|
|
25
|
+
messages: [{ role: "user", content: prompt }]
|
|
26
|
+
}.to_json
|
|
27
|
+
req.headers["Content-Type"] = "application/json"
|
|
28
|
+
req.headers["x-api-key"] = @api_key
|
|
29
|
+
req.headers["anthropic-version"] = "2023-06-01"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
|
|
33
|
+
|
|
34
|
+
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
35
|
+
content = body.dig("content", 0, "text")
|
|
36
|
+
result = content&.strip || ""
|
|
28
37
|
end
|
|
29
|
-
|
|
30
|
-
raise Llmemory::LLMError, "Anthropic API error: #{response.body}" unless response.success?
|
|
31
|
-
|
|
32
|
-
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
33
|
-
content = body.dig("content", 0, "text")
|
|
34
|
-
content&.strip || ""
|
|
38
|
+
result
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
private
|
data/lib/llmemory/llm/openai.rb
CHANGED
|
@@ -16,20 +16,24 @@ module Llmemory
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def invoke(prompt)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
19
|
+
result = nil
|
|
20
|
+
Llmemory::Instrumentation.instrument(:llm_invoke, provider: :openai, model: @model, prompt_chars: prompt.to_s.length) do
|
|
21
|
+
response = connection.post("chat/completions") do |req|
|
|
22
|
+
req.body = {
|
|
23
|
+
model: @model,
|
|
24
|
+
messages: [{ role: "user", content: prompt }],
|
|
25
|
+
temperature: 0.3
|
|
26
|
+
}.to_json
|
|
27
|
+
req.headers["Content-Type"] = "application/json"
|
|
28
|
+
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
29
|
+
end
|
|
28
30
|
|
|
29
|
-
|
|
31
|
+
raise Llmemory::LLMError, "OpenAI API error: #{response.body}" unless response.success?
|
|
30
32
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
|
|
34
|
+
result = body.dig("choices", 0, "message", "content")&.strip || ""
|
|
35
|
+
end
|
|
36
|
+
result
|
|
33
37
|
end
|
|
34
38
|
|
|
35
39
|
# Calls the model with response_format json_schema (Structured Outputs).
|
|
@@ -52,8 +52,8 @@ module Llmemory
|
|
|
52
52
|
@storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
-
def episodes(limit: nil)
|
|
56
|
-
@storage.list_episodes(@user_id, limit: limit).map { |e| Episode.from_h(e) }
|
|
55
|
+
def episodes(limit: nil, offset: nil)
|
|
56
|
+
@storage.list_episodes(@user_id, limit: limit, offset: offset).map { |e| Episode.from_h(e) }
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
def find_episode(id)
|
|
@@ -84,24 +84,37 @@ module Llmemory
|
|
|
84
84
|
# --- MemoryModule uniform interface ---
|
|
85
85
|
|
|
86
86
|
def write(steps:, summary: nil, outcome: nil, importance: 0.5, **_meta)
|
|
87
|
-
|
|
87
|
+
result = nil
|
|
88
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "episodic", user_id: @user_id) do
|
|
89
|
+
result = record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
|
|
90
|
+
end
|
|
91
|
+
result
|
|
88
92
|
end
|
|
89
93
|
|
|
90
|
-
def list(user_id: nil, limit: nil)
|
|
91
|
-
episodes(limit: limit)
|
|
94
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
95
|
+
episodes(limit: limit, offset: offset)
|
|
92
96
|
end
|
|
93
97
|
|
|
94
98
|
def stats(user_id: nil)
|
|
95
99
|
{ episodes: count }
|
|
96
100
|
end
|
|
97
101
|
|
|
98
|
-
def forget(ids:, reason: nil)
|
|
102
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
99
103
|
requested = Array(ids).map(&:to_s)
|
|
100
104
|
existing = @storage.list_episodes(@user_id).map { |e| (e[:id] || e["id"]).to_s }
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
+
targeted = requested & existing
|
|
106
|
+
count = case mode
|
|
107
|
+
when :hard then @storage.delete_episodes(@user_id, targeted).to_i
|
|
108
|
+
else @storage.archive_episodes(@user_id, targeted).to_i
|
|
109
|
+
end
|
|
110
|
+
forget_log.record(@user_id, memory_type: "episodic", ids: targeted, reason: reason)
|
|
111
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "episodic", user_id: @user_id, count: count, mode: mode)
|
|
112
|
+
count
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Storage accessor for the TTL maintenance job.
|
|
116
|
+
def expired_ids(cutoff:)
|
|
117
|
+
@storage.expired_episode_ids(@user_id, cutoff: cutoff)
|
|
105
118
|
end
|
|
106
119
|
|
|
107
120
|
private
|
|
@@ -42,25 +42,35 @@ module Llmemory
|
|
|
42
42
|
rec&.data
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
def list_episodes(user_id, limit: nil)
|
|
46
|
-
scope = LlmemoryEpisode.where(user_id: user_id).order(created_at: :desc)
|
|
45
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
46
|
+
scope = LlmemoryEpisode.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
|
|
47
47
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
48
|
+
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
48
49
|
scope.map(&:data)
|
|
49
50
|
end
|
|
50
51
|
|
|
51
52
|
def search_episodes(user_id, query)
|
|
52
|
-
token_scope(LlmemoryEpisode.where(user_id: user_id), "search_text", query)
|
|
53
|
+
token_scope(LlmemoryEpisode.where(user_id: user_id, archived_at: nil), "search_text", query)
|
|
53
54
|
.order(created_at: :desc).map(&:data)
|
|
54
55
|
end
|
|
55
56
|
|
|
56
57
|
def count_episodes(user_id)
|
|
57
|
-
LlmemoryEpisode.where(user_id: user_id).count
|
|
58
|
+
LlmemoryEpisode.where(user_id: user_id, archived_at: nil).count
|
|
58
59
|
end
|
|
59
60
|
|
|
60
61
|
def delete_episodes(user_id, ids)
|
|
61
62
|
LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
|
|
62
63
|
end
|
|
63
64
|
|
|
65
|
+
def archive_episodes(user_id, ids)
|
|
66
|
+
LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s), archived_at: nil)
|
|
67
|
+
.update_all(archived_at: Time.current)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
71
|
+
LlmemoryEpisode.where(user_id: user_id, archived_at: nil).where("created_at < ?", cutoff).pluck(:id)
|
|
72
|
+
end
|
|
73
|
+
|
|
64
74
|
def list_users
|
|
65
75
|
LlmemoryEpisode.distinct.pluck(:user_id)
|
|
66
76
|
end
|
|
@@ -16,8 +16,8 @@ module Llmemory
|
|
|
16
16
|
raise NotImplementedError, "#{self.class}#get_episode must be implemented"
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
# Newest first. Optionally
|
|
20
|
-
def list_episodes(user_id, limit: nil)
|
|
19
|
+
# Newest first. Optionally paginated with offset/limit.
|
|
20
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
21
21
|
raise NotImplementedError, "#{self.class}#list_episodes must be implemented"
|
|
22
22
|
end
|
|
23
23
|
|
|
@@ -34,6 +34,19 @@ module Llmemory
|
|
|
34
34
|
raise NotImplementedError, "#{self.class}#delete_episodes must be implemented"
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
# Soft-archives episodes by id (sets archived_at on the record). Archived
|
|
38
|
+
# episodes are excluded from list_episodes / search_episodes / count_episodes
|
|
39
|
+
# but remain accessible via get_episode. Returns the number archived.
|
|
40
|
+
def archive_episodes(user_id, ids)
|
|
41
|
+
raise NotImplementedError, "#{self.class}#archive_episodes must be implemented"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Returns episodes whose created_at is older than the cutoff and that are
|
|
45
|
+
# not already archived. Used by the TTL maintenance job.
|
|
46
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
47
|
+
raise NotImplementedError, "#{self.class}#expired_episode_ids must be implemented"
|
|
48
|
+
end
|
|
49
|
+
|
|
37
50
|
def list_users
|
|
38
51
|
raise NotImplementedError, "#{self.class}#list_users must be implemented"
|
|
39
52
|
end
|
|
@@ -38,10 +38,11 @@ module Llmemory
|
|
|
38
38
|
rows.any? ? parse_data(rows.first["data"]) : nil
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
def list_episodes(user_id, limit: nil)
|
|
41
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
42
42
|
ensure_tables!
|
|
43
|
-
sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 ORDER BY created_at DESC"
|
|
43
|
+
sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC"
|
|
44
44
|
sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
|
|
45
|
+
sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
|
|
45
46
|
conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
|
|
46
47
|
end
|
|
47
48
|
|
|
@@ -49,14 +50,14 @@ module Llmemory
|
|
|
49
50
|
ensure_tables!
|
|
50
51
|
suffix, params = token_filter("search_text", query, 2)
|
|
51
52
|
conn.exec_params(
|
|
52
|
-
"SELECT data FROM llmemory_episodes WHERE user_id = $1#{suffix} ORDER BY created_at DESC",
|
|
53
|
+
"SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
|
|
53
54
|
[user_id, *params]
|
|
54
55
|
).map { |r| parse_data(r["data"]) }
|
|
55
56
|
end
|
|
56
57
|
|
|
57
58
|
def count_episodes(user_id)
|
|
58
59
|
ensure_tables!
|
|
59
|
-
conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1", [user_id]).first["c"].to_i
|
|
60
|
+
conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL", [user_id]).first["c"].to_i
|
|
60
61
|
end
|
|
61
62
|
|
|
62
63
|
def delete_episodes(user_id, ids)
|
|
@@ -66,6 +67,24 @@ module Llmemory
|
|
|
66
67
|
end
|
|
67
68
|
end
|
|
68
69
|
|
|
70
|
+
def archive_episodes(user_id, ids)
|
|
71
|
+
ensure_tables!
|
|
72
|
+
Array(ids).sum do |id|
|
|
73
|
+
conn.exec_params(
|
|
74
|
+
"UPDATE llmemory_episodes SET archived_at = NOW() WHERE user_id = $1 AND id = $2 AND archived_at IS NULL",
|
|
75
|
+
[user_id, id]
|
|
76
|
+
).cmd_tuples
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
81
|
+
ensure_tables!
|
|
82
|
+
conn.exec_params(
|
|
83
|
+
"SELECT id FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL AND created_at < $2",
|
|
84
|
+
[user_id, cutoff.iso8601]
|
|
85
|
+
).map { |r| r["id"] }
|
|
86
|
+
end
|
|
87
|
+
|
|
69
88
|
def list_users
|
|
70
89
|
ensure_tables!
|
|
71
90
|
conn.exec("SELECT DISTINCT user_id FROM llmemory_episodes").map { |r| r["user_id"] }
|
|
@@ -87,9 +106,11 @@ module Llmemory
|
|
|
87
106
|
user_id TEXT NOT NULL,
|
|
88
107
|
data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
89
108
|
search_text TEXT,
|
|
90
|
-
created_at TIMESTAMPTZ NOT NULL
|
|
109
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
110
|
+
archived_at TIMESTAMPTZ
|
|
91
111
|
);
|
|
92
112
|
CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
|
|
113
|
+
ALTER TABLE llmemory_episodes ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
|
|
93
114
|
SQL
|
|
94
115
|
end
|
|
95
116
|
|
|
@@ -29,20 +29,19 @@ module Llmemory
|
|
|
29
29
|
load_episode(path)
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
-
def list_episodes(user_id, limit: nil)
|
|
33
|
-
sorted =
|
|
32
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
33
|
+
sorted = active_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
|
|
34
|
+
sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
34
35
|
limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
|
|
35
36
|
end
|
|
36
37
|
|
|
37
38
|
def search_episodes(user_id, query)
|
|
38
39
|
return list_episodes(user_id) if query.to_s.strip.empty?
|
|
39
|
-
|
|
40
|
+
active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
|
|
40
41
|
end
|
|
41
42
|
|
|
42
43
|
def count_episodes(user_id)
|
|
43
|
-
|
|
44
|
-
return 0 unless Dir.exist?(dir)
|
|
45
|
-
Dir.children(dir).count { |f| f.end_with?(".json") }
|
|
44
|
+
active_episodes(user_id).size
|
|
46
45
|
end
|
|
47
46
|
|
|
48
47
|
def delete_episodes(user_id, ids)
|
|
@@ -54,6 +53,24 @@ module Llmemory
|
|
|
54
53
|
end
|
|
55
54
|
end
|
|
56
55
|
|
|
56
|
+
def archive_episodes(user_id, ids)
|
|
57
|
+
Array(ids).map(&:to_s).count do |id|
|
|
58
|
+
path = episode_path(user_id, id)
|
|
59
|
+
next false unless File.file?(path)
|
|
60
|
+
data = JSON.parse(File.read(path))
|
|
61
|
+
next false if data["archived_at"]
|
|
62
|
+
data["archived_at"] = Time.now.iso8601
|
|
63
|
+
File.write(path, JSON.generate(data))
|
|
64
|
+
true
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
69
|
+
active_episodes(user_id)
|
|
70
|
+
.select { |e| (e[:created_at] || Time.now) < cutoff }
|
|
71
|
+
.map { |e| e[:id].to_s }
|
|
72
|
+
end
|
|
73
|
+
|
|
57
74
|
def list_users
|
|
58
75
|
return [] unless Dir.exist?(@base_path)
|
|
59
76
|
Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }
|
|
@@ -61,6 +78,10 @@ module Llmemory
|
|
|
61
78
|
|
|
62
79
|
private
|
|
63
80
|
|
|
81
|
+
def active_episodes(user_id)
|
|
82
|
+
all_episodes(user_id).reject { |e| e[:archived_at] }
|
|
83
|
+
end
|
|
84
|
+
|
|
64
85
|
def all_episodes(user_id)
|
|
65
86
|
dir = user_path(user_id, "episodes")
|
|
66
87
|
return [] unless Dir.exist?(dir)
|
|
@@ -25,18 +25,19 @@ module Llmemory
|
|
|
25
25
|
@episodes[user_id].find { |e| e[:id] == id }
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
def list_episodes(user_id, limit: nil)
|
|
29
|
-
sorted =
|
|
28
|
+
def list_episodes(user_id, limit: nil, offset: nil)
|
|
29
|
+
sorted = active_episodes(user_id).sort_by { |e| as_time(e[:created_at]) }.reverse
|
|
30
|
+
sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
30
31
|
limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
|
|
31
32
|
end
|
|
32
33
|
|
|
33
34
|
def search_episodes(user_id, query)
|
|
34
35
|
return list_episodes(user_id) if query.to_s.strip.empty?
|
|
35
|
-
|
|
36
|
+
active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
|
|
36
37
|
end
|
|
37
38
|
|
|
38
39
|
def count_episodes(user_id)
|
|
39
|
-
|
|
40
|
+
active_episodes(user_id).size
|
|
40
41
|
end
|
|
41
42
|
|
|
42
43
|
def delete_episodes(user_id, ids)
|
|
@@ -46,12 +47,42 @@ module Llmemory
|
|
|
46
47
|
before - @episodes[user_id].size
|
|
47
48
|
end
|
|
48
49
|
|
|
50
|
+
def archive_episodes(user_id, ids)
|
|
51
|
+
ids = Array(ids).map(&:to_s)
|
|
52
|
+
count = 0
|
|
53
|
+
@episodes[user_id].each do |e|
|
|
54
|
+
next unless ids.include?(e[:id].to_s)
|
|
55
|
+
next if e[:archived_at]
|
|
56
|
+
e[:archived_at] = Time.now
|
|
57
|
+
count += 1
|
|
58
|
+
end
|
|
59
|
+
count
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def expired_episode_ids(user_id, cutoff:)
|
|
63
|
+
active_episodes(user_id)
|
|
64
|
+
.select { |e| as_time(e[:created_at]) < cutoff }
|
|
65
|
+
.map { |e| e[:id].to_s }
|
|
66
|
+
end
|
|
67
|
+
|
|
49
68
|
def list_users
|
|
50
69
|
@episodes.keys
|
|
51
70
|
end
|
|
52
71
|
|
|
53
72
|
private
|
|
54
73
|
|
|
74
|
+
def active_episodes(user_id)
|
|
75
|
+
@episodes[user_id].reject { |e| e[:archived_at] }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def as_time(value)
|
|
79
|
+
return Time.now if value.nil?
|
|
80
|
+
return value if value.is_a?(Time)
|
|
81
|
+
Time.parse(value.to_s)
|
|
82
|
+
rescue ArgumentError
|
|
83
|
+
Time.now
|
|
84
|
+
end
|
|
85
|
+
|
|
55
86
|
def symbolize(hash)
|
|
56
87
|
hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
57
88
|
end
|
|
@@ -108,11 +108,15 @@ module Llmemory
|
|
|
108
108
|
# --- MemoryModule uniform interface ---
|
|
109
109
|
|
|
110
110
|
def write(payload, **_meta)
|
|
111
|
-
|
|
111
|
+
result = nil
|
|
112
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "file_based", user_id: @user_id) do
|
|
113
|
+
result = memorize(payload)
|
|
114
|
+
end
|
|
115
|
+
result
|
|
112
116
|
end
|
|
113
117
|
|
|
114
|
-
def list(user_id: nil, limit: nil)
|
|
115
|
-
@storage.list_items(user_id: user_id || @user_id, limit: limit)
|
|
118
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
119
|
+
@storage.list_items(user_id: user_id || @user_id, limit: limit, offset: offset)
|
|
116
120
|
end
|
|
117
121
|
|
|
118
122
|
def stats(user_id: nil)
|
|
@@ -120,7 +124,10 @@ module Llmemory
|
|
|
120
124
|
end
|
|
121
125
|
|
|
122
126
|
# Removes items/resources by id and records the removal in the audit log.
|
|
123
|
-
|
|
127
|
+
# Note: file-based storages currently implement `archive_*` as physical
|
|
128
|
+
# removal — `mode: :soft` and `mode: :hard` are functionally equivalent
|
|
129
|
+
# here. Kept for API uniformity.
|
|
130
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
124
131
|
requested = Array(ids).map(&:to_s)
|
|
125
132
|
existing = (@storage.get_all_items(@user_id) + @storage.get_all_resources(@user_id))
|
|
126
133
|
.map { |r| (r[:id] || r["id"]).to_s }
|
|
@@ -128,6 +135,7 @@ module Llmemory
|
|
|
128
135
|
@storage.archive_items(@user_id, removed)
|
|
129
136
|
@storage.archive_resources(@user_id, removed)
|
|
130
137
|
forget_log.record(@user_id, memory_type: "file_based", ids: removed, reason: reason)
|
|
138
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "file_based", user_id: @user_id, count: removed.size, mode: mode)
|
|
131
139
|
removed.size
|
|
132
140
|
end
|
|
133
141
|
|