llmemory 0.2.10 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +48 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +71 -0
- data/lib/generators/llmemory/upgrade_zero_mem/templates/create_llmemory_zero_mem_tables.rb +77 -0
- data/lib/generators/llmemory/upgrade_zero_mem/upgrade_zero_mem_generator.rb +19 -0
- data/lib/llmemory/cli/commands/zero_mem.rb +107 -0
- data/lib/llmemory/cli.rb +3 -0
- data/lib/llmemory/configuration.rb +67 -1
- data/lib/llmemory/consolidation/filter.rb +74 -0
- data/lib/llmemory/consolidation/policy.rb +17 -0
- data/lib/llmemory/consolidation/rule_based_policy.rb +63 -0
- data/lib/llmemory/consolidation/unit.rb +54 -0
- data/lib/llmemory/consolidation.rb +11 -0
- data/lib/llmemory/instrumentation.rb +10 -1
- data/lib/llmemory/long_term/file_based/memory.rb +18 -7
- data/lib/llmemory/long_term/graph_based/memory.rb +31 -8
- data/lib/llmemory/maintenance/cognitive_pass.rb +29 -1
- data/lib/llmemory/maintenance/policy_prune.rb +85 -0
- data/lib/llmemory/maintenance/zero_mem_ttl.rb +26 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +2 -0
- data/lib/llmemory/mcp/store_helpers.rb +16 -0
- data/lib/llmemory/mcp/tools/memory_add_message.rb +17 -4
- data/lib/llmemory/mcp/tools/memory_consolidate.rb +9 -4
- data/lib/llmemory/mcp/tools/memory_maintain.rb +6 -1
- data/lib/llmemory/mcp/tools/memory_retrieve.rb +3 -1
- data/lib/llmemory/mcp/tools/memory_retrieve_evidence.rb +70 -0
- data/lib/llmemory/mcp/tools/memory_stats.rb +22 -0
- data/lib/llmemory/memory.rb +257 -9
- data/lib/llmemory/retrieval/bm25_scorer.rb +5 -0
- data/lib/llmemory/retrieval/context_assembler.rb +17 -1
- data/lib/llmemory/tokenizer.rb +18 -8
- data/lib/llmemory/vector_store/memory_store.rb +23 -1
- data/lib/llmemory/vector_store/openai_embeddings.rb +47 -21
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory/zero_mem/answer_calibrator.rb +63 -0
- data/lib/llmemory/zero_mem/backfill.rb +43 -0
- data/lib/llmemory/zero_mem/budget_assembler.rb +62 -0
- data/lib/llmemory/zero_mem/circuit_breaker.rb +37 -0
- data/lib/llmemory/zero_mem/embedding_provider.rb +73 -0
- data/lib/llmemory/zero_mem/engine.rb +247 -0
- data/lib/llmemory/zero_mem/entity.rb +31 -0
- data/lib/llmemory/zero_mem/entity_extractor.rb +23 -0
- data/lib/llmemory/zero_mem/entity_index.rb +65 -0
- data/lib/llmemory/zero_mem/entity_mention.rb +35 -0
- data/lib/llmemory/zero_mem/errors.rb +7 -0
- data/lib/llmemory/zero_mem/evidence.rb +43 -0
- data/lib/llmemory/zero_mem/evidence_calibrator.rb +158 -0
- data/lib/llmemory/zero_mem/evidence_closure.rb +98 -0
- data/lib/llmemory/zero_mem/evidence_fusion.rb +57 -0
- data/lib/llmemory/zero_mem/evidence_set.rb +54 -0
- data/lib/llmemory/zero_mem/extractors/heuristic.rb +44 -0
- data/lib/llmemory/zero_mem/extractors/http.rb +58 -0
- data/lib/llmemory/zero_mem/graph_retriever.rb +107 -0
- data/lib/llmemory/zero_mem/hierarchy_builder.rb +123 -0
- data/lib/llmemory/zero_mem/hierarchy_retriever.rb +133 -0
- data/lib/llmemory/zero_mem/indexer.rb +73 -0
- data/lib/llmemory/zero_mem/mode.rb +30 -0
- data/lib/llmemory/zero_mem/pagerank.rb +42 -0
- data/lib/llmemory/zero_mem/query_profile.rb +48 -0
- data/lib/llmemory/zero_mem/query_profiler.rb +138 -0
- data/lib/llmemory/zero_mem/repair.rb +31 -0
- data/lib/llmemory/zero_mem/router.rb +69 -0
- data/lib/llmemory/zero_mem/sidecar_client.rb +82 -0
- data/lib/llmemory/zero_mem/storage.rb +124 -0
- data/lib/llmemory/zero_mem/storages/active_record.rb +311 -0
- data/lib/llmemory/zero_mem/storages/active_record_models.rb +21 -0
- data/lib/llmemory/zero_mem/storages/file.rb +67 -0
- data/lib/llmemory/zero_mem/storages/memory.rb +294 -0
- data/lib/llmemory/zero_mem/storages/postgres.rb +85 -0
- data/lib/llmemory/zero_mem/storages/snapshot_codec.rb +180 -0
- data/lib/llmemory/zero_mem/storages.rb +43 -0
- data/lib/llmemory/zero_mem/trace.rb +100 -0
- data/lib/llmemory/zero_mem/trace_state_link.rb +64 -0
- data/lib/llmemory/zero_mem/trace_unit.rb +83 -0
- data/lib/llmemory/zero_mem.rb +42 -0
- data/lib/llmemory.rb +8 -0
- data/lib/tasks/llmemory.rake +84 -2
- metadata +54 -2
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "consolidation/unit"
|
|
4
|
+
require_relative "consolidation/policy"
|
|
5
|
+
require_relative "consolidation/rule_based_policy"
|
|
6
|
+
require_relative "consolidation/filter"
|
|
7
|
+
|
|
8
|
+
module Llmemory
|
|
9
|
+
module Consolidation
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -16,10 +16,19 @@ module Llmemory
|
|
|
16
16
|
# output_tokens:, total_tokens:
|
|
17
17
|
# memory_write.llmemory memory_type:, user_id:
|
|
18
18
|
# memory_forget.llmemory memory_type:, user_id:, count:
|
|
19
|
-
# retrieve.llmemory query_chars:, candidates:, results:
|
|
19
|
+
# retrieve.llmemory query_chars:, candidates:, results:, zero_mem_compliant:
|
|
20
20
|
# iterative_retrieve.llmemory hops:, total_results:
|
|
21
21
|
# reflect.llmemory window:, insights:
|
|
22
22
|
# mine_skills.llmemory window:, auto_register:
|
|
23
|
+
# trace_write.llmemory user_id:, session_id:, trace_id:, sequence:, maintenance_fanout:
|
|
24
|
+
# zero_mem_state_update.llmemory user_id:, state_key:, trace_id:, supersedes_trace_id:
|
|
25
|
+
# zero_mem_profile.llmemory user_id:, query_chars:
|
|
26
|
+
# zero_mem_hierarchy_retrieve.llmemory user_id:, workload:
|
|
27
|
+
# zero_mem_retrieve.llmemory user_id:, evidence_count:, degraded:
|
|
28
|
+
# zero_mem_graph_retrieve.llmemory user_id:
|
|
29
|
+
# zero_mem_closure.llmemory user_id:, seeds:
|
|
30
|
+
# zero_mem_calibrate.llmemory user_id:, trace_count:
|
|
31
|
+
# encoder_embed.llmemory user_id:, trace_id:, model:, dimensions:, duration_ms:
|
|
23
32
|
module Instrumentation
|
|
24
33
|
module_function
|
|
25
34
|
|
|
@@ -6,6 +6,7 @@ require_relative "category"
|
|
|
6
6
|
require_relative "storage"
|
|
7
7
|
require_relative "../../noise_filter"
|
|
8
8
|
require_relative "../../memory_module"
|
|
9
|
+
require_relative "../../consolidation/filter"
|
|
9
10
|
|
|
10
11
|
module Llmemory
|
|
11
12
|
module LongTerm
|
|
@@ -28,19 +29,26 @@ module Llmemory
|
|
|
28
29
|
resource_id = save_resource(text)
|
|
29
30
|
append_to_daily_log(text) if Llmemory.configuration.daily_logs_enabled && @storage.respond_to?(:save_daily_log_entry)
|
|
30
31
|
items = @extractor.extract_items(text)
|
|
31
|
-
|
|
32
|
-
item.is_a?(Hash) ? (item["content"] || item[:content]).to_s : item.to_s
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
classifications = @extractor.classify_items(
|
|
33
|
+
items.map { |item| item.is_a?(Hash) ? (item["content"] || item[:content]).to_s : item.to_s }
|
|
34
|
+
)
|
|
35
|
+
items = Consolidation::Filter.apply_items(items, classifications)
|
|
35
36
|
updates_by_category = {}
|
|
36
37
|
|
|
37
38
|
items.each do |item|
|
|
38
39
|
content = item.is_a?(Hash) ? (item["content"] || item[:content]) : item.to_s
|
|
39
40
|
importance = (item["importance"] || item[:importance] || 0.7).to_f
|
|
40
41
|
cat = classifications[content] || @extractor.classify_item(content)
|
|
42
|
+
volatile = item[:volatile] == true
|
|
41
43
|
updates_by_category[cat] ||= []
|
|
42
44
|
updates_by_category[cat] << content.to_s
|
|
43
|
-
save_item(
|
|
45
|
+
save_item(
|
|
46
|
+
category: cat,
|
|
47
|
+
item: item,
|
|
48
|
+
source_resource_id: resource_id,
|
|
49
|
+
importance: importance,
|
|
50
|
+
volatile: volatile
|
|
51
|
+
)
|
|
44
52
|
end
|
|
45
53
|
|
|
46
54
|
updates_by_category.each do |category, new_memories|
|
|
@@ -70,13 +78,15 @@ module Llmemory
|
|
|
70
78
|
end
|
|
71
79
|
|
|
72
80
|
items.first(top_k).each do |i|
|
|
81
|
+
prov = i[:provenance] || i["provenance"]
|
|
73
82
|
out << {
|
|
74
83
|
id: i[:id] || i["id"],
|
|
75
84
|
text: i[:content] || i["content"],
|
|
76
85
|
timestamp: i[:created_at] || i["created_at"],
|
|
77
86
|
score: 1.0,
|
|
78
87
|
importance: (i[:importance] || i["importance"] || 1.0).to_f,
|
|
79
|
-
evergreen: i[:evergreen] || i["evergreen"]
|
|
88
|
+
evergreen: i[:evergreen] || i["evergreen"],
|
|
89
|
+
volatile: Consolidation::Filter.volatile_marked?(prov)
|
|
80
90
|
}
|
|
81
91
|
end
|
|
82
92
|
resources.first([top_k - out.size, 0].max).each do |r|
|
|
@@ -152,11 +162,12 @@ module Llmemory
|
|
|
152
162
|
@storage.save_resource(@user_id, text)
|
|
153
163
|
end
|
|
154
164
|
|
|
155
|
-
def save_item(category:, item:, source_resource_id:, importance: 0.7)
|
|
165
|
+
def save_item(category:, item:, source_resource_id:, importance: 0.7, volatile: false)
|
|
156
166
|
content = item.is_a?(Hash) ? item["content"] || item[:content] : item.to_s
|
|
157
167
|
provenance = Llmemory::Provenance.from_resource(
|
|
158
168
|
source_resource_id, method: "fact_extraction", confidence: importance
|
|
159
169
|
)
|
|
170
|
+
provenance = Consolidation::Filter.volatile_provenance(provenance) if volatile
|
|
160
171
|
@storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance, provenance: provenance)
|
|
161
172
|
end
|
|
162
173
|
|
|
@@ -8,6 +8,7 @@ require_relative "conflict_resolver"
|
|
|
8
8
|
require_relative "storage"
|
|
9
9
|
require_relative "../../noise_filter"
|
|
10
10
|
require_relative "../../memory_module"
|
|
11
|
+
require_relative "../../consolidation/filter"
|
|
11
12
|
|
|
12
13
|
module Llmemory
|
|
13
14
|
module LongTerm
|
|
@@ -32,6 +33,7 @@ module Llmemory
|
|
|
32
33
|
return true if text.strip.empty?
|
|
33
34
|
|
|
34
35
|
entities, relations = extract_graph(text)
|
|
36
|
+
relations = Consolidation::Filter.apply_relations(relations)
|
|
35
37
|
return true if entities.empty? && relations.empty?
|
|
36
38
|
|
|
37
39
|
provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
|
|
@@ -54,7 +56,8 @@ module Llmemory
|
|
|
54
56
|
text: r[:text],
|
|
55
57
|
timestamp: r[:created_at] || r[:timestamp],
|
|
56
58
|
score: r[:score] || 1.0,
|
|
57
|
-
importance: r[:importance]
|
|
59
|
+
importance: r[:importance],
|
|
60
|
+
volatile: r[:volatile]
|
|
58
61
|
}
|
|
59
62
|
end
|
|
60
63
|
end
|
|
@@ -147,8 +150,12 @@ module Llmemory
|
|
|
147
150
|
object = (r[:object] || r["object"]).to_s.strip
|
|
148
151
|
next if subject.empty? || predicate.empty? || object.empty?
|
|
149
152
|
|
|
150
|
-
|
|
151
|
-
|
|
153
|
+
volatile = r[:volatile] == true
|
|
154
|
+
relation_provenance = volatile ? Consolidation::Filter.volatile_provenance(provenance) : provenance
|
|
155
|
+
edge_properties = volatile ? Consolidation::Filter.volatile_properties("provenance" => relation_provenance) : {"provenance" => relation_provenance}
|
|
156
|
+
|
|
157
|
+
subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => relation_provenance })
|
|
158
|
+
object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => relation_provenance })
|
|
152
159
|
|
|
153
160
|
existing = @kg.find_edges(
|
|
154
161
|
subject: subject_id,
|
|
@@ -164,13 +171,13 @@ module Llmemory
|
|
|
164
171
|
subject_id: subject_id,
|
|
165
172
|
predicate: predicate,
|
|
166
173
|
target_id: object_id,
|
|
167
|
-
properties:
|
|
174
|
+
properties: edge_properties,
|
|
168
175
|
created_at: Time.now,
|
|
169
176
|
archived_at: nil
|
|
170
177
|
)
|
|
171
178
|
archived_ids = @conflict_resolver.resolve(edge)
|
|
172
179
|
remove_edge_embeddings(archived_ids)
|
|
173
|
-
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties:
|
|
180
|
+
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: edge_properties)
|
|
174
181
|
|
|
175
182
|
edge_text = "#{subject} #{predicate} #{object}"
|
|
176
183
|
embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(edge_text) : nil
|
|
@@ -197,7 +204,13 @@ module Llmemory
|
|
|
197
204
|
next unless active_edge?(id)
|
|
198
205
|
|
|
199
206
|
meta = v[:metadata] || v["metadata"] || {}
|
|
200
|
-
{
|
|
207
|
+
{
|
|
208
|
+
id: id,
|
|
209
|
+
text: meta["text"] || meta[:text] || id.to_s,
|
|
210
|
+
score: v[:score] || v["score"] || 1.0,
|
|
211
|
+
created_at: meta["created_at"] || meta[:created_at],
|
|
212
|
+
volatile: volatile_candidate?(edge_properties(id))
|
|
213
|
+
}
|
|
201
214
|
end
|
|
202
215
|
|
|
203
216
|
node_ids = out.flat_map { |r| extract_node_ids_from_text(r[:text]) }.compact.uniq
|
|
@@ -209,7 +222,7 @@ module Llmemory
|
|
|
209
222
|
subj = @kg.find_node_by_id(e.subject_id)
|
|
210
223
|
obj = @kg.find_node_by_id(e.target_id)
|
|
211
224
|
edge_text = "#{subj&.name} #{e.predicate} #{obj&.name}"
|
|
212
|
-
out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
|
|
225
|
+
out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at, volatile: volatile_candidate?(e.properties) } unless out.any? { |o| o[:text] == edge_text }
|
|
213
226
|
end
|
|
214
227
|
end
|
|
215
228
|
|
|
@@ -222,7 +235,7 @@ module Llmemory
|
|
|
222
235
|
obj = @kg.find_node_by_id(e.target_id)
|
|
223
236
|
next unless subj && obj
|
|
224
237
|
edge_text = "#{subj.name} #{e.predicate} #{obj.name}"
|
|
225
|
-
out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at }
|
|
238
|
+
out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at, volatile: volatile_candidate?(e.properties) }
|
|
226
239
|
end
|
|
227
240
|
end
|
|
228
241
|
|
|
@@ -278,6 +291,16 @@ module Llmemory
|
|
|
278
291
|
@kg.find_edges(subject: nil, predicate: nil, object: nil, include_archived: false)
|
|
279
292
|
.any? { |edge| edge.id.to_s == edge_id.to_s }
|
|
280
293
|
end
|
|
294
|
+
|
|
295
|
+
def volatile_candidate?(properties_or_provenance)
|
|
296
|
+
Consolidation::Filter.volatile_marked?(properties_or_provenance)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def edge_properties(edge_id)
|
|
300
|
+
@kg.find_edges(subject: nil, predicate: nil, object: nil, include_archived: false)
|
|
301
|
+
.find { |edge| edge.id.to_s == edge_id.to_s }
|
|
302
|
+
&.properties
|
|
303
|
+
end
|
|
281
304
|
end
|
|
282
305
|
end
|
|
283
306
|
end
|
|
@@ -43,7 +43,22 @@ module Llmemory
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
def run!
|
|
46
|
-
report = {
|
|
46
|
+
report = {
|
|
47
|
+
consolidated: nil,
|
|
48
|
+
insights: [],
|
|
49
|
+
mined: [],
|
|
50
|
+
expired: { episodic: 0, procedural: 0 },
|
|
51
|
+
errors: {},
|
|
52
|
+
disabled: []
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if @memory&.zero_mem_strict?
|
|
56
|
+
report[:disabled] = %i[consolidate reflect mine]
|
|
57
|
+
report[:consolidated] = false
|
|
58
|
+
step(report, :zero_mem_repair) { report[:zero_mem] = zero_mem_repair } if @memory.trace_store
|
|
59
|
+
step(report, :expire) { report[:expired] = expire } if @expire
|
|
60
|
+
return report
|
|
61
|
+
end
|
|
47
62
|
|
|
48
63
|
step(report, :consolidate) { report[:consolidated] = consolidate } if @memory
|
|
49
64
|
step(report, :reflect) { report[:insights] = reflect } if @reflect
|
|
@@ -79,6 +94,19 @@ module Llmemory
|
|
|
79
94
|
TTLExpiry.run!(@user_id, episodic: episodic, procedural: procedural)
|
|
80
95
|
end
|
|
81
96
|
|
|
97
|
+
def zero_mem_repair
|
|
98
|
+
store = @memory.trace_store
|
|
99
|
+
repair = ZeroMem::Repair.new(storage: store)
|
|
100
|
+
sessions = store.list_traces(@user_id).map(&:session_id).uniq
|
|
101
|
+
sessions.map do |sid|
|
|
102
|
+
repair.run!(
|
|
103
|
+
user_id: @user_id,
|
|
104
|
+
session_id: sid,
|
|
105
|
+
checkpoint_messages: @memory.messages
|
|
106
|
+
)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
82
110
|
def episodic
|
|
83
111
|
@episodic ||= @memory&.episodic || Llmemory::LongTerm::Episodic::Memory.new(user_id: @user_id)
|
|
84
112
|
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../consolidation/unit"
|
|
4
|
+
require_relative "../consolidation/filter"
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module Maintenance
|
|
8
|
+
# Archives long-term facts that the current consolidation policy would drop.
|
|
9
|
+
# Useful for cleaning legacy data consolidated before a policy was configured.
|
|
10
|
+
class PolicyPrune
|
|
11
|
+
Result = Struct.new(:graph_edges, :file_items, :dry_run, keyword_init: true) do
|
|
12
|
+
def total
|
|
13
|
+
graph_edges + file_items
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def to_h
|
|
17
|
+
{ graph_edges: graph_edges, file_items: file_items, dry_run: dry_run, total: total }
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(dry_run: false, policy: nil)
|
|
22
|
+
@dry_run = dry_run
|
|
23
|
+
@policy = policy || Llmemory.configuration.consolidation_policy
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def run(memory:)
|
|
27
|
+
case memory
|
|
28
|
+
when LongTerm::GraphBased::Memory
|
|
29
|
+
Result.new(graph_edges: prune_graph(memory), file_items: 0, dry_run: @dry_run)
|
|
30
|
+
when LongTerm::FileBased::Memory
|
|
31
|
+
Result.new(graph_edges: 0, file_items: prune_file(memory), dry_run: @dry_run)
|
|
32
|
+
else
|
|
33
|
+
raise ArgumentError, "Unsupported memory type: #{memory.class}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def prune_graph(memory)
|
|
40
|
+
storage = memory.instance_variable_get(:@graph_storage)
|
|
41
|
+
kg = memory.instance_variable_get(:@kg)
|
|
42
|
+
user_id = memory.user_id
|
|
43
|
+
return 0 unless storage.respond_to?(:list_edges)
|
|
44
|
+
|
|
45
|
+
removed = 0
|
|
46
|
+
storage.list_edges(user_id).each do |edge|
|
|
47
|
+
next if edge.archived?
|
|
48
|
+
|
|
49
|
+
subject = kg.find_node_by_id(edge.subject_id)
|
|
50
|
+
object = kg.find_node_by_id(edge.target_id)
|
|
51
|
+
unit = Consolidation::Unit.from_relation(
|
|
52
|
+
subject: subject&.name,
|
|
53
|
+
predicate: edge.predicate,
|
|
54
|
+
object: object&.name
|
|
55
|
+
)
|
|
56
|
+
next unless @policy.decide(unit) == :drop
|
|
57
|
+
|
|
58
|
+
removed += 1
|
|
59
|
+
next if @dry_run
|
|
60
|
+
|
|
61
|
+
memory.forget(ids: [edge.id], reason: "consolidation_policy_prune")
|
|
62
|
+
end
|
|
63
|
+
removed
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def prune_file(memory)
|
|
67
|
+
storage = memory.instance_variable_get(:@storage)
|
|
68
|
+
user_id = memory.user_id
|
|
69
|
+
removed = 0
|
|
70
|
+
|
|
71
|
+
storage.get_all_items(user_id).each do |item|
|
|
72
|
+
unit = Consolidation::Unit.from_item(item, category: item[:category] || item["category"])
|
|
73
|
+
next unless @policy.decide(unit) == :drop
|
|
74
|
+
|
|
75
|
+
item_id = item[:id] || item["id"]
|
|
76
|
+
removed += 1
|
|
77
|
+
next if @dry_run
|
|
78
|
+
|
|
79
|
+
memory.forget(ids: [item_id], reason: "consolidation_policy_prune")
|
|
80
|
+
end
|
|
81
|
+
removed
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
module Maintenance
|
|
5
|
+
class ZeroMemTTL
|
|
6
|
+
def initialize(storage:, ttl_days: nil)
|
|
7
|
+
@storage = storage
|
|
8
|
+
@ttl_days = ttl_days || Llmemory.configuration.zero_mem_ttl_days
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run!(user_id:, dry_run: false)
|
|
12
|
+
return { archived: 0, skipped: true } if @ttl_days.nil? || @ttl_days.to_i <= 0
|
|
13
|
+
|
|
14
|
+
cutoff = Time.now - (@ttl_days.to_i * 86_400)
|
|
15
|
+
archived = 0
|
|
16
|
+
@storage.list_traces(user_id, include_archived: false).each do |trace|
|
|
17
|
+
next unless trace.occurred_at < cutoff
|
|
18
|
+
|
|
19
|
+
archived += 1
|
|
20
|
+
@storage.archive_trace(user_id, trace.id) unless dry_run
|
|
21
|
+
end
|
|
22
|
+
{ archived: archived, dry_run: dry_run, cutoff: cutoff.utc.iso8601 }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
data/lib/llmemory/maintenance.rb
CHANGED
|
@@ -4,6 +4,8 @@ require_relative "maintenance/runner"
|
|
|
4
4
|
require_relative "maintenance/ttl_expiry"
|
|
5
5
|
require_relative "maintenance/cognitive_pass"
|
|
6
6
|
require_relative "maintenance/search_tokens_backfill"
|
|
7
|
+
require_relative "maintenance/policy_prune"
|
|
8
|
+
require_relative "maintenance/zero_mem_ttl"
|
|
7
9
|
|
|
8
10
|
module Llmemory
|
|
9
11
|
module Maintenance
|
data/lib/llmemory/mcp/server.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "authentication"
|
|
|
5
5
|
require_relative "tools/memory_search"
|
|
6
6
|
require_relative "tools/memory_save"
|
|
7
7
|
require_relative "tools/memory_retrieve"
|
|
8
|
+
require_relative "tools/memory_retrieve_evidence"
|
|
8
9
|
require_relative "tools/memory_timeline"
|
|
9
10
|
require_relative "tools/memory_add_message"
|
|
10
11
|
require_relative "tools/memory_consolidate"
|
|
@@ -175,6 +176,7 @@ module Llmemory
|
|
|
175
176
|
Tools::MemorySearch,
|
|
176
177
|
Tools::MemorySave,
|
|
177
178
|
Tools::MemoryRetrieve,
|
|
179
|
+
Tools::MemoryRetrieveEvidence,
|
|
178
180
|
Tools::MemoryTimeline,
|
|
179
181
|
Tools::MemoryTimelineContext,
|
|
180
182
|
Tools::MemoryAddMessage,
|
|
@@ -30,6 +30,22 @@ module Llmemory
|
|
|
30
30
|
def graph_based?
|
|
31
31
|
Llmemory.configuration.long_term_type.to_sym == :graph_based
|
|
32
32
|
end
|
|
33
|
+
|
|
34
|
+
def trace_store_instance
|
|
35
|
+
@trace_store_instance ||= ZeroMem::Storages.build
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def build_memory(user_id:, session_id: "default", trace_store: nil, memory_mode: nil)
|
|
39
|
+
mode = memory_mode || Llmemory.configuration.memory_mode
|
|
40
|
+
store = trace_store
|
|
41
|
+
store = trace_store_instance if store.nil? && ZeroMem::Mode.zero_mem_enabled?(mode)
|
|
42
|
+
Llmemory::Memory.new(
|
|
43
|
+
user_id: user_id,
|
|
44
|
+
session_id: session_id,
|
|
45
|
+
trace_store: store,
|
|
46
|
+
memory_mode: mode
|
|
47
|
+
)
|
|
48
|
+
end
|
|
33
49
|
end
|
|
34
50
|
end
|
|
35
51
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../store_helpers"
|
|
4
|
+
|
|
3
5
|
module Llmemory
|
|
4
6
|
module MCP
|
|
5
7
|
module Tools
|
|
@@ -11,17 +13,28 @@ module Llmemory
|
|
|
11
13
|
user_id: { type: "string", description: "User identifier" },
|
|
12
14
|
session_id: { type: "string", description: "Session identifier (default: 'default')" },
|
|
13
15
|
role: { type: "string", enum: ["user", "assistant", "system", "tool", "tool_result"], description: "Message role" },
|
|
14
|
-
content: { type: "string", description: "Message content" }
|
|
16
|
+
content: { type: "string", description: "Message content" },
|
|
17
|
+
occurred_at: { type: "string", description: "Optional ISO8601 timestamp for trace ingestion" },
|
|
18
|
+
boundary_id: { type: "string", description: "Optional boundary identifier (Zero-Mem)" },
|
|
19
|
+
idempotency_key: { type: "string", description: "Optional idempotency key for trace writes" }
|
|
15
20
|
},
|
|
16
21
|
required: ["user_id", "role", "content"]
|
|
17
22
|
)
|
|
18
23
|
|
|
19
24
|
class << self
|
|
20
|
-
def call(user_id:, role:, content:, session_id: nil,
|
|
25
|
+
def call(user_id:, role:, content:, session_id: nil, occurred_at: nil, boundary_id: nil,
|
|
26
|
+
idempotency_key: nil, server_context: nil)
|
|
21
27
|
session = session_id || "default"
|
|
22
28
|
|
|
23
|
-
memory =
|
|
24
|
-
|
|
29
|
+
memory = StoreHelpers.build_memory(user_id: user_id, session_id: session)
|
|
30
|
+
parsed_time = occurred_at ? Time.parse(occurred_at.to_s) : nil
|
|
31
|
+
memory.add_message(
|
|
32
|
+
role: role.to_sym,
|
|
33
|
+
content: content,
|
|
34
|
+
occurred_at: parsed_time,
|
|
35
|
+
boundary_id: boundary_id,
|
|
36
|
+
idempotency_key: idempotency_key
|
|
37
|
+
)
|
|
25
38
|
|
|
26
39
|
::MCP::Tool::Response.new([{
|
|
27
40
|
type: "text",
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../store_helpers"
|
|
4
|
+
|
|
3
5
|
module Llmemory
|
|
4
6
|
module MCP
|
|
5
7
|
module Tools
|
|
@@ -20,7 +22,7 @@ module Llmemory
|
|
|
20
22
|
session = session_id || "default"
|
|
21
23
|
should_clear = clear_session || false
|
|
22
24
|
|
|
23
|
-
memory =
|
|
25
|
+
memory = StoreHelpers.build_memory(user_id: user_id, session_id: session)
|
|
24
26
|
|
|
25
27
|
messages = memory.messages
|
|
26
28
|
if messages.empty?
|
|
@@ -33,9 +35,7 @@ module Llmemory
|
|
|
33
35
|
message_count = messages.size
|
|
34
36
|
memory.consolidate!
|
|
35
37
|
|
|
36
|
-
if should_clear
|
|
37
|
-
memory.clear_session!
|
|
38
|
-
end
|
|
38
|
+
memory.clear_session! if should_clear
|
|
39
39
|
|
|
40
40
|
response_text = "Consolidated #{message_count} messages from session '#{session}' into long-term memory."
|
|
41
41
|
response_text += "\nSession cleared." if should_clear
|
|
@@ -44,6 +44,11 @@ module Llmemory
|
|
|
44
44
|
type: "text",
|
|
45
45
|
text: response_text
|
|
46
46
|
}])
|
|
47
|
+
rescue Llmemory::GenerativeOperationDisabled => e
|
|
48
|
+
::MCP::Tool::Response.new([{
|
|
49
|
+
type: "text",
|
|
50
|
+
text: e.message
|
|
51
|
+
}], error: true)
|
|
47
52
|
rescue => e
|
|
48
53
|
::MCP::Tool::Response.new([{
|
|
49
54
|
type: "text",
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../store_helpers"
|
|
4
|
+
|
|
3
5
|
module Llmemory
|
|
4
6
|
module MCP
|
|
5
7
|
module Tools
|
|
@@ -26,7 +28,8 @@ module Llmemory
|
|
|
26
28
|
opts[:reflection_window] = reflection_window.to_i unless reflection_window.nil?
|
|
27
29
|
opts[:mining_window] = mining_window.to_i unless mining_window.nil?
|
|
28
30
|
|
|
29
|
-
|
|
31
|
+
memory = StoreHelpers.build_memory(user_id: user_id)
|
|
32
|
+
report = Llmemory::Maintenance::CognitivePass.run!(user_id, memory: memory, **opts)
|
|
30
33
|
::MCP::Tool::Response.new([{ type: "text", text: format_report(user_id, report) }])
|
|
31
34
|
rescue => e
|
|
32
35
|
::MCP::Tool::Response.new([{ type: "text", text: "Error running maintenance pass: #{e.message}" }], error: true)
|
|
@@ -42,6 +45,8 @@ module Llmemory
|
|
|
42
45
|
" skills mined: #{Array(report[:mined]).size}",
|
|
43
46
|
" expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
|
|
44
47
|
]
|
|
48
|
+
disabled = Array(report[:disabled])
|
|
49
|
+
lines << " disabled: #{disabled.join(', ')}" if disabled.any?
|
|
45
50
|
errors = report[:errors] || {}
|
|
46
51
|
lines << " errors: #{errors.map { |k, v| "#{k}: #{v}" }.join('; ')}" unless errors.empty?
|
|
47
52
|
lines.join("\n")
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../store_helpers"
|
|
4
|
+
|
|
3
5
|
module Llmemory
|
|
4
6
|
module MCP
|
|
5
7
|
module Tools
|
|
@@ -25,7 +27,7 @@ module Llmemory
|
|
|
25
27
|
include_timeline = include_timeline_context == true
|
|
26
28
|
window = timeline_window || 3
|
|
27
29
|
|
|
28
|
-
memory =
|
|
30
|
+
memory = StoreHelpers.build_memory(user_id: user_id, session_id: session)
|
|
29
31
|
context = memory.retrieve(query, max_tokens: tokens)
|
|
30
32
|
|
|
31
33
|
# Add timeline context if requested
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../store_helpers"
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module MCP
|
|
8
|
+
module Tools
|
|
9
|
+
class MemoryRetrieveEvidence < ::MCP::Tool
|
|
10
|
+
description "Retrieve structured Zero-Mem evidence (traces) for a query without generative LLM calls. Returns JSON plus a text context block."
|
|
11
|
+
|
|
12
|
+
input_schema(
|
|
13
|
+
properties: {
|
|
14
|
+
query: { type: "string", description: "Evidence query" },
|
|
15
|
+
user_id: { type: "string", description: "User identifier" },
|
|
16
|
+
session_id: { type: "string", description: "Session identifier (default: 'default')" },
|
|
17
|
+
top_k: { type: "integer", description: "Maximum seed traces (default from config)" },
|
|
18
|
+
max_tokens: { type: "integer", description: "Token budget for assembled evidence" },
|
|
19
|
+
boundary: { type: "object", description: "Optional session/boundary filter" },
|
|
20
|
+
explain: { type: "boolean", description: "Include routing/fusion explain payload (default false)" }
|
|
21
|
+
},
|
|
22
|
+
required: ["query", "user_id"]
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
def call(query:, user_id:, session_id: nil, top_k: nil, max_tokens: nil, boundary: nil,
|
|
27
|
+
explain: false, server_context: nil)
|
|
28
|
+
unless ZeroMem::Mode.zero_mem_enabled?(Llmemory.configuration.memory_mode)
|
|
29
|
+
return error_response("memory_retrieve_evidence requires memory_mode :zero_mem or :hybrid")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
session = session_id || "default"
|
|
33
|
+
memory = StoreHelpers.build_memory(user_id: user_id, session_id: session)
|
|
34
|
+
result = memory.retrieve_evidence(
|
|
35
|
+
query,
|
|
36
|
+
top_k: top_k,
|
|
37
|
+
max_tokens: max_tokens,
|
|
38
|
+
boundary: boundary,
|
|
39
|
+
explain: explain == true
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
payload = {
|
|
43
|
+
route: result.route,
|
|
44
|
+
profile: result.profile.to_h,
|
|
45
|
+
evidence: result.evidence.map(&:to_h),
|
|
46
|
+
seed_evidence: result.seed_evidence.map(&:to_h),
|
|
47
|
+
closure_evidence: result.closure_evidence.map(&:to_h),
|
|
48
|
+
metrics: result.metrics,
|
|
49
|
+
degraded: result.degraded,
|
|
50
|
+
warnings: result.warnings,
|
|
51
|
+
explain: result.explain,
|
|
52
|
+
context: result.to_context
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
::MCP::Tool::Response.new([{
|
|
56
|
+
type: "text",
|
|
57
|
+
text: JSON.pretty_generate(payload)
|
|
58
|
+
}])
|
|
59
|
+
rescue => e
|
|
60
|
+
error_response(e.message)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def error_response(message)
|
|
64
|
+
::MCP::Tool::Response.new([{ type: "text", text: "Error retrieving evidence: #{message}" }], error: true)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../store_helpers"
|
|
4
|
+
|
|
3
5
|
module Llmemory
|
|
4
6
|
module MCP
|
|
5
7
|
module Tools
|
|
@@ -53,6 +55,16 @@ module Llmemory
|
|
|
53
55
|
end
|
|
54
56
|
|
|
55
57
|
stats[:llm_usage] = Llmemory::LLM::UsageLedger.new(store: store).totals(user_id)
|
|
58
|
+
stats[:memory_mode] = Llmemory.configuration.memory_mode
|
|
59
|
+
if ZeroMem::Mode.zero_mem_enabled?(Llmemory.configuration.memory_mode)
|
|
60
|
+
memory = StoreHelpers.build_memory(user_id: user_id)
|
|
61
|
+
stats[:zero_mem] = memory.zero_mem_status
|
|
62
|
+
invoke_calls = stats[:llm_usage].dig(:invoke, :calls).to_i
|
|
63
|
+
stats[:zero_mem_compliant] = ZeroMem::Mode.zero_mem_strict?(Llmemory.configuration.memory_mode) &&
|
|
64
|
+
invoke_calls.zero?
|
|
65
|
+
else
|
|
66
|
+
stats[:zero_mem_compliant] = false
|
|
67
|
+
end
|
|
56
68
|
|
|
57
69
|
::MCP::Tool::Response.new([{
|
|
58
70
|
type: "text",
|
|
@@ -106,6 +118,16 @@ module Llmemory
|
|
|
106
118
|
|
|
107
119
|
output << ""
|
|
108
120
|
output << Llmemory::LLM::UsageLedger.format_text(stats[:llm_usage] || default_llm_usage)
|
|
121
|
+
output << ""
|
|
122
|
+
output << "MEMORY MODE: #{stats[:memory_mode]}"
|
|
123
|
+
if stats[:zero_mem]
|
|
124
|
+
zm = stats[:zero_mem]
|
|
125
|
+
output << "ZERO-MEM:"
|
|
126
|
+
output << " traces: #{zm[:trace_count]}"
|
|
127
|
+
output << " index_lag: #{zm[:index_lag]}"
|
|
128
|
+
output << " index_version: #{zm[:index_version]}"
|
|
129
|
+
end
|
|
130
|
+
output << " zero_mem_compliant: #{stats[:zero_mem_compliant]}"
|
|
109
131
|
|
|
110
132
|
output.join("\n")
|
|
111
133
|
end
|