llmemory 0.2.10 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -0
- data/lib/llmemory/configuration.rb +9 -1
- data/lib/llmemory/consolidation/filter.rb +74 -0
- data/lib/llmemory/consolidation/policy.rb +17 -0
- data/lib/llmemory/consolidation/rule_based_policy.rb +63 -0
- data/lib/llmemory/consolidation/unit.rb +54 -0
- data/lib/llmemory/consolidation.rb +11 -0
- data/lib/llmemory/long_term/file_based/memory.rb +18 -7
- data/lib/llmemory/long_term/graph_based/memory.rb +31 -8
- data/lib/llmemory/maintenance/policy_prune.rb +85 -0
- data/lib/llmemory/maintenance.rb +1 -0
- data/lib/llmemory/retrieval/context_assembler.rb +17 -1
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +1 -0
- data/lib/tasks/llmemory.rake +32 -2
- metadata +7 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c52d2f218d44057a93d247a535d06a4f4d0877d3ff67fdc48bf9011886ac9199
|
|
4
|
+
data.tar.gz: 1934ecd2e82c9518cbb9806c297c172fa5d3c2baa2c676da81943646309cf4bc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5879fda1878fe865b8e9391f36270cbb85b74b41461a06f66e2c86f3c33475fb383f98ae6dc06391764ad9ec65055b97b146908d3b96d39cff39e4971aeb134b
|
|
7
|
+
data.tar.gz: e7f3f99792aa601b49b6c96f3d960acbf4969b5df6b7341aa8074a46e87ab6a96245752a758902300a8c40b7b2683d5bcc4ba8d0517b978169526647a3d4e64c
|
data/README.md
CHANGED
|
@@ -153,6 +153,31 @@ Llmemory.configure do |config|
|
|
|
153
153
|
end
|
|
154
154
|
```
|
|
155
155
|
|
|
156
|
+
## Consolidation policy
|
|
157
|
+
|
|
158
|
+
By default, llmemory consolidates every extracted fact into long-term memory. For agents with **authoritative live data** (tools, DB snapshots), configure a `Consolidation::Policy` to **drop** or **mark volatile** facts that should not be treated as stable memory (e.g. current job, plan status, task lists).
|
|
159
|
+
|
|
160
|
+
```ruby
|
|
161
|
+
Llmemory.configure do |config|
|
|
162
|
+
config.consolidation_policy = Llmemory::Consolidation::RuleBasedPolicy.new(
|
|
163
|
+
drop_predicates: %w[current_job works_at job_title plan_status has_task],
|
|
164
|
+
drop_categories: %w[work_life]
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# Optional: prefix volatile memories at retrieval time with "[verify live]"
|
|
168
|
+
config.retrieval_label_volatile = true
|
|
169
|
+
end
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Policy decisions per extracted unit: `:keep` (default), `:drop` (not persisted), or `:volatile` (persisted with a volatile marker in provenance/properties).
|
|
173
|
+
|
|
174
|
+
To clean **legacy** data consolidated before a policy was enabled:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
USER_ID=user_123 LONG_TERM_TYPE=graph_based bundle exec rake llmemory:prune_by_policy
|
|
178
|
+
DRY_RUN=1 bundle exec rake llmemory:prune_by_policy
|
|
179
|
+
```
|
|
180
|
+
|
|
156
181
|
## Encryption at rest
|
|
157
182
|
|
|
158
183
|
Optional AES-256-GCM encryption protects persisted memory. Without the key, stored data is unreadable — useful for isolating agents or tenants.
|
|
@@ -57,7 +57,9 @@ module Llmemory
|
|
|
57
57
|
:encryption_key,
|
|
58
58
|
:shared_memory_stores,
|
|
59
59
|
:dashboard_auth,
|
|
60
|
-
:dashboard_require_auth
|
|
60
|
+
:dashboard_require_auth,
|
|
61
|
+
:consolidation_policy,
|
|
62
|
+
:retrieval_label_volatile
|
|
61
63
|
|
|
62
64
|
def initialize
|
|
63
65
|
@llm_provider = :openai
|
|
@@ -116,6 +118,12 @@ module Llmemory
|
|
|
116
118
|
@shared_memory_stores = false
|
|
117
119
|
@dashboard_auth = nil
|
|
118
120
|
@dashboard_require_auth = nil
|
|
121
|
+
@consolidation_policy = nil
|
|
122
|
+
@retrieval_label_volatile = false
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def consolidation_policy
|
|
126
|
+
@consolidation_policy ||= Consolidation::NullPolicy.new
|
|
119
127
|
end
|
|
120
128
|
|
|
121
129
|
def dashboard_require_auth?
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
module Consolidation
|
|
5
|
+
module Filter
|
|
6
|
+
VOLATILE_PROPERTY = "consolidation_volatile"
|
|
7
|
+
VOLATILE_PROVENANCE_FLAG = "volatile"
|
|
8
|
+
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def policy
|
|
12
|
+
Llmemory.configuration.consolidation_policy || NullPolicy.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# @return [Array<Hash>] relations with :volatile key
|
|
16
|
+
def apply_relations(relations)
|
|
17
|
+
Array(relations).filter_map do |relation|
|
|
18
|
+
next unless relation.is_a?(Hash)
|
|
19
|
+
|
|
20
|
+
unit = Unit.from_relation(relation)
|
|
21
|
+
case policy.decide(unit)
|
|
22
|
+
when :drop
|
|
23
|
+
next
|
|
24
|
+
when :volatile
|
|
25
|
+
relation.merge(volatile: true)
|
|
26
|
+
else
|
|
27
|
+
relation.merge(volatile: false)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @return [Array<Hash>] item hashes with :volatile key
|
|
33
|
+
def apply_items(items, classifications = {})
|
|
34
|
+
Array(items).filter_map do |item|
|
|
35
|
+
content = item.is_a?(Hash) ? (item["content"] || item[:content]).to_s : item.to_s
|
|
36
|
+
category = classifications[content] || classifications[content.to_sym]
|
|
37
|
+
unit = Unit.from_item(item, category: category)
|
|
38
|
+
|
|
39
|
+
case policy.decide(unit)
|
|
40
|
+
when :drop
|
|
41
|
+
next
|
|
42
|
+
when :volatile
|
|
43
|
+
item_hash = item.is_a?(Hash) ? item.dup : {"content" => content}
|
|
44
|
+
item_hash.merge(volatile: true)
|
|
45
|
+
else
|
|
46
|
+
item_hash = item.is_a?(Hash) ? item.dup : {"content" => content}
|
|
47
|
+
item_hash.merge(volatile: false)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def volatile_provenance(base_provenance)
|
|
53
|
+
prov = base_provenance.is_a?(Hash) ? base_provenance.dup : {}
|
|
54
|
+
prov[:consolidation] = VOLATILE_PROVENANCE_FLAG
|
|
55
|
+
prov["consolidation"] = VOLATILE_PROVENANCE_FLAG unless prov.key?(:consolidation)
|
|
56
|
+
prov
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def volatile_properties(base_properties)
|
|
60
|
+
props = base_properties.is_a?(Hash) ? base_properties.dup : {}
|
|
61
|
+
props[VOLATILE_PROPERTY] = true
|
|
62
|
+
props
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def volatile_marked?(properties_or_provenance)
|
|
66
|
+
hash = properties_or_provenance || {}
|
|
67
|
+
hash[VOLATILE_PROPERTY] == true ||
|
|
68
|
+
hash["consolidation_volatile"] == true ||
|
|
69
|
+
hash[:consolidation] == VOLATILE_PROVENANCE_FLAG ||
|
|
70
|
+
hash["consolidation"] == VOLATILE_PROVENANCE_FLAG
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
module Consolidation
|
|
5
|
+
# Base policy: all extracted units are kept in long-term memory.
|
|
6
|
+
class Policy
|
|
7
|
+
DECISIONS = %i[keep drop volatile].freeze
|
|
8
|
+
|
|
9
|
+
def decide(_unit)
|
|
10
|
+
:keep
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
class NullPolicy < Policy
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
module Consolidation
|
|
5
|
+
# Rule-based consolidation policy for agents with authoritative live data sources
|
|
6
|
+
# (tools, DB snapshots). Drops or marks volatile facts that should not be treated
|
|
7
|
+
# as stable long-term memory.
|
|
8
|
+
#
|
|
9
|
+
# Precedence (first match wins):
|
|
10
|
+
# 1. drop_if callable
|
|
11
|
+
# 2. volatile_if callable
|
|
12
|
+
# 3. drop_predicates (graph relations)
|
|
13
|
+
# 4. volatile_predicates (graph relations)
|
|
14
|
+
# 5. drop_categories (file-based items)
|
|
15
|
+
# 6. volatile_categories (file-based items)
|
|
16
|
+
# 7. :keep
|
|
17
|
+
class RuleBasedPolicy < Policy
|
|
18
|
+
def initialize(
|
|
19
|
+
drop_predicates: [],
|
|
20
|
+
volatile_predicates: [],
|
|
21
|
+
drop_categories: [],
|
|
22
|
+
volatile_categories: [],
|
|
23
|
+
drop_if: nil,
|
|
24
|
+
volatile_if: nil
|
|
25
|
+
)
|
|
26
|
+
@drop_predicates = normalize_list(drop_predicates)
|
|
27
|
+
@volatile_predicates = normalize_list(volatile_predicates)
|
|
28
|
+
@drop_categories = normalize_list(drop_categories)
|
|
29
|
+
@volatile_categories = normalize_list(volatile_categories)
|
|
30
|
+
@drop_if = drop_if
|
|
31
|
+
@volatile_if = volatile_if
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def decide(unit)
|
|
35
|
+
return :drop if callable_true?(@drop_if, unit)
|
|
36
|
+
return :volatile if callable_true?(@volatile_if, unit)
|
|
37
|
+
|
|
38
|
+
case unit.kind
|
|
39
|
+
when :relation
|
|
40
|
+
predicate = Unit.normalize_predicate(unit.predicate)
|
|
41
|
+
return :drop if @drop_predicates.include?(predicate)
|
|
42
|
+
return :volatile if @volatile_predicates.include?(predicate)
|
|
43
|
+
when :item
|
|
44
|
+
category = Unit.normalize_category(unit.category)
|
|
45
|
+
return :drop if @drop_categories.include?(category)
|
|
46
|
+
return :volatile if @volatile_categories.include?(category)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
:keep
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def normalize_list(values)
|
|
55
|
+
Array(values).map { |value| value.to_s.strip.downcase.gsub(/\s+/, "_") }.reject(&:empty?)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def callable_true?(callable, unit)
|
|
59
|
+
callable.respond_to?(:call) && callable.call(unit) == true
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
module Consolidation
|
|
5
|
+
# Normalized fact unit passed to Consolidation::Policy.
|
|
6
|
+
# kind: :relation (graph) | :item (file-based)
|
|
7
|
+
Unit = Data.define(:kind, :subject, :predicate, :object, :content, :category, :importance) do
|
|
8
|
+
def self.from_relation(relation)
|
|
9
|
+
new(
|
|
10
|
+
kind: :relation,
|
|
11
|
+
subject: normalize_string(relation[:subject] || relation["subject"]),
|
|
12
|
+
predicate: normalize_predicate(relation[:predicate] || relation["predicate"]),
|
|
13
|
+
object: normalize_string(relation[:object] || relation["object"]),
|
|
14
|
+
content: nil,
|
|
15
|
+
category: nil,
|
|
16
|
+
importance: nil
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.from_item(item, category: nil)
|
|
21
|
+
content = if item.is_a?(Hash)
|
|
22
|
+
(item["content"] || item[:content]).to_s
|
|
23
|
+
else
|
|
24
|
+
item.to_s
|
|
25
|
+
end
|
|
26
|
+
importance = if item.is_a?(Hash)
|
|
27
|
+
item["importance"] || item[:importance]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
new(
|
|
31
|
+
kind: :item,
|
|
32
|
+
subject: nil,
|
|
33
|
+
predicate: nil,
|
|
34
|
+
object: nil,
|
|
35
|
+
content: content,
|
|
36
|
+
category: normalize_category(category),
|
|
37
|
+
importance: importance
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.normalize_predicate(value)
|
|
42
|
+
normalize_string(value).downcase.gsub(/\s+/, "_")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.normalize_category(value)
|
|
46
|
+
normalize_string(value).downcase.gsub(/\s+/, "_")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.normalize_string(value)
|
|
50
|
+
value.to_s.strip
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "consolidation/unit"
|
|
4
|
+
require_relative "consolidation/policy"
|
|
5
|
+
require_relative "consolidation/rule_based_policy"
|
|
6
|
+
require_relative "consolidation/filter"
|
|
7
|
+
|
|
8
|
+
module Llmemory
|
|
9
|
+
module Consolidation
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -6,6 +6,7 @@ require_relative "category"
|
|
|
6
6
|
require_relative "storage"
|
|
7
7
|
require_relative "../../noise_filter"
|
|
8
8
|
require_relative "../../memory_module"
|
|
9
|
+
require_relative "../../consolidation/filter"
|
|
9
10
|
|
|
10
11
|
module Llmemory
|
|
11
12
|
module LongTerm
|
|
@@ -28,19 +29,26 @@ module Llmemory
|
|
|
28
29
|
resource_id = save_resource(text)
|
|
29
30
|
append_to_daily_log(text) if Llmemory.configuration.daily_logs_enabled && @storage.respond_to?(:save_daily_log_entry)
|
|
30
31
|
items = @extractor.extract_items(text)
|
|
31
|
-
|
|
32
|
-
item.is_a?(Hash) ? (item["content"] || item[:content]).to_s : item.to_s
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
classifications = @extractor.classify_items(
|
|
33
|
+
items.map { |item| item.is_a?(Hash) ? (item["content"] || item[:content]).to_s : item.to_s }
|
|
34
|
+
)
|
|
35
|
+
items = Consolidation::Filter.apply_items(items, classifications)
|
|
35
36
|
updates_by_category = {}
|
|
36
37
|
|
|
37
38
|
items.each do |item|
|
|
38
39
|
content = item.is_a?(Hash) ? (item["content"] || item[:content]) : item.to_s
|
|
39
40
|
importance = (item["importance"] || item[:importance] || 0.7).to_f
|
|
40
41
|
cat = classifications[content] || @extractor.classify_item(content)
|
|
42
|
+
volatile = item[:volatile] == true
|
|
41
43
|
updates_by_category[cat] ||= []
|
|
42
44
|
updates_by_category[cat] << content.to_s
|
|
43
|
-
save_item(
|
|
45
|
+
save_item(
|
|
46
|
+
category: cat,
|
|
47
|
+
item: item,
|
|
48
|
+
source_resource_id: resource_id,
|
|
49
|
+
importance: importance,
|
|
50
|
+
volatile: volatile
|
|
51
|
+
)
|
|
44
52
|
end
|
|
45
53
|
|
|
46
54
|
updates_by_category.each do |category, new_memories|
|
|
@@ -70,13 +78,15 @@ module Llmemory
|
|
|
70
78
|
end
|
|
71
79
|
|
|
72
80
|
items.first(top_k).each do |i|
|
|
81
|
+
prov = i[:provenance] || i["provenance"]
|
|
73
82
|
out << {
|
|
74
83
|
id: i[:id] || i["id"],
|
|
75
84
|
text: i[:content] || i["content"],
|
|
76
85
|
timestamp: i[:created_at] || i["created_at"],
|
|
77
86
|
score: 1.0,
|
|
78
87
|
importance: (i[:importance] || i["importance"] || 1.0).to_f,
|
|
79
|
-
evergreen: i[:evergreen] || i["evergreen"]
|
|
88
|
+
evergreen: i[:evergreen] || i["evergreen"],
|
|
89
|
+
volatile: Consolidation::Filter.volatile_marked?(prov)
|
|
80
90
|
}
|
|
81
91
|
end
|
|
82
92
|
resources.first([top_k - out.size, 0].max).each do |r|
|
|
@@ -152,11 +162,12 @@ module Llmemory
|
|
|
152
162
|
@storage.save_resource(@user_id, text)
|
|
153
163
|
end
|
|
154
164
|
|
|
155
|
-
def save_item(category:, item:, source_resource_id:, importance: 0.7)
|
|
165
|
+
def save_item(category:, item:, source_resource_id:, importance: 0.7, volatile: false)
|
|
156
166
|
content = item.is_a?(Hash) ? item["content"] || item[:content] : item.to_s
|
|
157
167
|
provenance = Llmemory::Provenance.from_resource(
|
|
158
168
|
source_resource_id, method: "fact_extraction", confidence: importance
|
|
159
169
|
)
|
|
170
|
+
provenance = Consolidation::Filter.volatile_provenance(provenance) if volatile
|
|
160
171
|
@storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance, provenance: provenance)
|
|
161
172
|
end
|
|
162
173
|
|
|
@@ -8,6 +8,7 @@ require_relative "conflict_resolver"
|
|
|
8
8
|
require_relative "storage"
|
|
9
9
|
require_relative "../../noise_filter"
|
|
10
10
|
require_relative "../../memory_module"
|
|
11
|
+
require_relative "../../consolidation/filter"
|
|
11
12
|
|
|
12
13
|
module Llmemory
|
|
13
14
|
module LongTerm
|
|
@@ -32,6 +33,7 @@ module Llmemory
|
|
|
32
33
|
return true if text.strip.empty?
|
|
33
34
|
|
|
34
35
|
entities, relations = extract_graph(text)
|
|
36
|
+
relations = Consolidation::Filter.apply_relations(relations)
|
|
35
37
|
return true if entities.empty? && relations.empty?
|
|
36
38
|
|
|
37
39
|
provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
|
|
@@ -54,7 +56,8 @@ module Llmemory
|
|
|
54
56
|
text: r[:text],
|
|
55
57
|
timestamp: r[:created_at] || r[:timestamp],
|
|
56
58
|
score: r[:score] || 1.0,
|
|
57
|
-
importance: r[:importance]
|
|
59
|
+
importance: r[:importance],
|
|
60
|
+
volatile: r[:volatile]
|
|
58
61
|
}
|
|
59
62
|
end
|
|
60
63
|
end
|
|
@@ -147,8 +150,12 @@ module Llmemory
|
|
|
147
150
|
object = (r[:object] || r["object"]).to_s.strip
|
|
148
151
|
next if subject.empty? || predicate.empty? || object.empty?
|
|
149
152
|
|
|
150
|
-
|
|
151
|
-
|
|
153
|
+
volatile = r[:volatile] == true
|
|
154
|
+
relation_provenance = volatile ? Consolidation::Filter.volatile_provenance(provenance) : provenance
|
|
155
|
+
edge_properties = volatile ? Consolidation::Filter.volatile_properties("provenance" => relation_provenance) : {"provenance" => relation_provenance}
|
|
156
|
+
|
|
157
|
+
subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => relation_provenance })
|
|
158
|
+
object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => relation_provenance })
|
|
152
159
|
|
|
153
160
|
existing = @kg.find_edges(
|
|
154
161
|
subject: subject_id,
|
|
@@ -164,13 +171,13 @@ module Llmemory
|
|
|
164
171
|
subject_id: subject_id,
|
|
165
172
|
predicate: predicate,
|
|
166
173
|
target_id: object_id,
|
|
167
|
-
properties:
|
|
174
|
+
properties: edge_properties,
|
|
168
175
|
created_at: Time.now,
|
|
169
176
|
archived_at: nil
|
|
170
177
|
)
|
|
171
178
|
archived_ids = @conflict_resolver.resolve(edge)
|
|
172
179
|
remove_edge_embeddings(archived_ids)
|
|
173
|
-
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties:
|
|
180
|
+
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: edge_properties)
|
|
174
181
|
|
|
175
182
|
edge_text = "#{subject} #{predicate} #{object}"
|
|
176
183
|
embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(edge_text) : nil
|
|
@@ -197,7 +204,13 @@ module Llmemory
|
|
|
197
204
|
next unless active_edge?(id)
|
|
198
205
|
|
|
199
206
|
meta = v[:metadata] || v["metadata"] || {}
|
|
200
|
-
{
|
|
207
|
+
{
|
|
208
|
+
id: id,
|
|
209
|
+
text: meta["text"] || meta[:text] || id.to_s,
|
|
210
|
+
score: v[:score] || v["score"] || 1.0,
|
|
211
|
+
created_at: meta["created_at"] || meta[:created_at],
|
|
212
|
+
volatile: volatile_candidate?(edge_properties(id))
|
|
213
|
+
}
|
|
201
214
|
end
|
|
202
215
|
|
|
203
216
|
node_ids = out.flat_map { |r| extract_node_ids_from_text(r[:text]) }.compact.uniq
|
|
@@ -209,7 +222,7 @@ module Llmemory
|
|
|
209
222
|
subj = @kg.find_node_by_id(e.subject_id)
|
|
210
223
|
obj = @kg.find_node_by_id(e.target_id)
|
|
211
224
|
edge_text = "#{subj&.name} #{e.predicate} #{obj&.name}"
|
|
212
|
-
out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
|
|
225
|
+
out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at, volatile: volatile_candidate?(e.properties) } unless out.any? { |o| o[:text] == edge_text }
|
|
213
226
|
end
|
|
214
227
|
end
|
|
215
228
|
|
|
@@ -222,7 +235,7 @@ module Llmemory
|
|
|
222
235
|
obj = @kg.find_node_by_id(e.target_id)
|
|
223
236
|
next unless subj && obj
|
|
224
237
|
edge_text = "#{subj.name} #{e.predicate} #{obj.name}"
|
|
225
|
-
out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at }
|
|
238
|
+
out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at, volatile: volatile_candidate?(e.properties) }
|
|
226
239
|
end
|
|
227
240
|
end
|
|
228
241
|
|
|
@@ -278,6 +291,16 @@ module Llmemory
|
|
|
278
291
|
@kg.find_edges(subject: nil, predicate: nil, object: nil, include_archived: false)
|
|
279
292
|
.any? { |edge| edge.id.to_s == edge_id.to_s }
|
|
280
293
|
end
|
|
294
|
+
|
|
295
|
+
def volatile_candidate?(properties_or_provenance)
|
|
296
|
+
Consolidation::Filter.volatile_marked?(properties_or_provenance)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def edge_properties(edge_id)
|
|
300
|
+
@kg.find_edges(subject: nil, predicate: nil, object: nil, include_archived: false)
|
|
301
|
+
.find { |edge| edge.id.to_s == edge_id.to_s }
|
|
302
|
+
&.properties
|
|
303
|
+
end
|
|
281
304
|
end
|
|
282
305
|
end
|
|
283
306
|
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../consolidation/unit"
|
|
4
|
+
require_relative "../consolidation/filter"
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module Maintenance
|
|
8
|
+
# Archives long-term facts that the current consolidation policy would drop.
|
|
9
|
+
# Useful for cleaning legacy data consolidated before a policy was configured.
|
|
10
|
+
class PolicyPrune
|
|
11
|
+
Result = Struct.new(:graph_edges, :file_items, :dry_run, keyword_init: true) do
|
|
12
|
+
def total
|
|
13
|
+
graph_edges + file_items
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def to_h
|
|
17
|
+
{ graph_edges: graph_edges, file_items: file_items, dry_run: dry_run, total: total }
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(dry_run: false, policy: nil)
|
|
22
|
+
@dry_run = dry_run
|
|
23
|
+
@policy = policy || Llmemory.configuration.consolidation_policy
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def run(memory:)
|
|
27
|
+
case memory
|
|
28
|
+
when LongTerm::GraphBased::Memory
|
|
29
|
+
Result.new(graph_edges: prune_graph(memory), file_items: 0, dry_run: @dry_run)
|
|
30
|
+
when LongTerm::FileBased::Memory
|
|
31
|
+
Result.new(graph_edges: 0, file_items: prune_file(memory), dry_run: @dry_run)
|
|
32
|
+
else
|
|
33
|
+
raise ArgumentError, "Unsupported memory type: #{memory.class}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def prune_graph(memory)
|
|
40
|
+
storage = memory.instance_variable_get(:@graph_storage)
|
|
41
|
+
kg = memory.instance_variable_get(:@kg)
|
|
42
|
+
user_id = memory.user_id
|
|
43
|
+
return 0 unless storage.respond_to?(:list_edges)
|
|
44
|
+
|
|
45
|
+
removed = 0
|
|
46
|
+
storage.list_edges(user_id).each do |edge|
|
|
47
|
+
next if edge.archived?
|
|
48
|
+
|
|
49
|
+
subject = kg.find_node_by_id(edge.subject_id)
|
|
50
|
+
object = kg.find_node_by_id(edge.target_id)
|
|
51
|
+
unit = Consolidation::Unit.from_relation(
|
|
52
|
+
subject: subject&.name,
|
|
53
|
+
predicate: edge.predicate,
|
|
54
|
+
object: object&.name
|
|
55
|
+
)
|
|
56
|
+
next unless @policy.decide(unit) == :drop
|
|
57
|
+
|
|
58
|
+
removed += 1
|
|
59
|
+
next if @dry_run
|
|
60
|
+
|
|
61
|
+
memory.forget(ids: [edge.id], reason: "consolidation_policy_prune")
|
|
62
|
+
end
|
|
63
|
+
removed
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def prune_file(memory)
|
|
67
|
+
storage = memory.instance_variable_get(:@storage)
|
|
68
|
+
user_id = memory.user_id
|
|
69
|
+
removed = 0
|
|
70
|
+
|
|
71
|
+
storage.get_all_items(user_id).each do |item|
|
|
72
|
+
unit = Consolidation::Unit.from_item(item, category: item[:category] || item["category"])
|
|
73
|
+
next unless @policy.decide(unit) == :drop
|
|
74
|
+
|
|
75
|
+
item_id = item[:id] || item["id"]
|
|
76
|
+
removed += 1
|
|
77
|
+
next if @dry_run
|
|
78
|
+
|
|
79
|
+
memory.forget(ids: [item_id], reason: "consolidation_policy_prune")
|
|
80
|
+
end
|
|
81
|
+
removed
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
data/lib/llmemory/maintenance.rb
CHANGED
|
@@ -4,6 +4,7 @@ require_relative "maintenance/runner"
|
|
|
4
4
|
require_relative "maintenance/ttl_expiry"
|
|
5
5
|
require_relative "maintenance/cognitive_pass"
|
|
6
6
|
require_relative "maintenance/search_tokens_backfill"
|
|
7
|
+
require_relative "maintenance/policy_prune"
|
|
7
8
|
|
|
8
9
|
module Llmemory
|
|
9
10
|
module Maintenance
|
|
@@ -14,13 +14,15 @@ module Llmemory
|
|
|
14
14
|
|
|
15
15
|
ranked_memories.each do |memory|
|
|
16
16
|
text = memory[:text] || memory["text"] || ""
|
|
17
|
+
text = label_volatile(text) if volatile_memory?(memory)
|
|
17
18
|
memory_tokens = count_tokens(text)
|
|
18
19
|
break if token_count + memory_tokens > max_tokens
|
|
19
20
|
|
|
20
21
|
selected << {
|
|
21
22
|
text: text,
|
|
22
23
|
timestamp: memory[:timestamp] || memory["timestamp"],
|
|
23
|
-
confidence: memory[:temporal_score] || memory[:score] || memory["score"]
|
|
24
|
+
confidence: memory[:temporal_score] || memory[:score] || memory["score"],
|
|
25
|
+
volatile: volatile_memory?(memory)
|
|
24
26
|
}
|
|
25
27
|
token_count += memory_tokens
|
|
26
28
|
end
|
|
@@ -48,6 +50,20 @@ module Llmemory
|
|
|
48
50
|
lines << "=== END MEMORIES ==="
|
|
49
51
|
lines.join("\n")
|
|
50
52
|
end
|
|
53
|
+
|
|
54
|
+
def volatile_memory?(memory)
|
|
55
|
+
memory[:volatile] == true || memory["volatile"] == true
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def label_volatile_enabled?
|
|
59
|
+
Llmemory.configuration.retrieval_label_volatile
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def label_volatile(text)
|
|
63
|
+
return text unless label_volatile_enabled?
|
|
64
|
+
|
|
65
|
+
"[verify live] #{text}"
|
|
66
|
+
end
|
|
51
67
|
end
|
|
52
68
|
end
|
|
53
69
|
end
|
data/lib/llmemory/version.rb
CHANGED
data/lib/llmemory.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "llmemory/version"
|
|
4
4
|
require_relative "llmemory/configuration"
|
|
5
5
|
require_relative "llmemory/provenance"
|
|
6
|
+
require_relative "llmemory/consolidation"
|
|
6
7
|
require_relative "llmemory/tokenizer"
|
|
7
8
|
require_relative "llmemory/instrumentation"
|
|
8
9
|
require_relative "llmemory/memory_module"
|
data/lib/tasks/llmemory.rake
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
namespace :llmemory do
|
|
4
4
|
desc "Backfill search_tokens blind index (and name_det on skills) for encrypted keyword search. " \
|
|
5
5
|
"Env: FORCE=1 (re-backfill all rows), DRY_RUN=1 (count only), STORE=active_record|postgres"
|
|
6
|
-
|
|
6
|
+
task :backfill_search_tokens, [:user_id] do |_t, args|
|
|
7
|
+
Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
|
|
7
8
|
|
|
8
|
-
task :backfill_search_tokens, [:user_id] => task_deps do |_t, args|
|
|
9
9
|
require "llmemory"
|
|
10
10
|
require_relative "../llmemory/maintenance/search_tokens_backfill"
|
|
11
11
|
|
|
@@ -36,4 +36,34 @@ namespace :llmemory do
|
|
|
36
36
|
"(total=#{summary[:total]})"
|
|
37
37
|
puts "Dry run — no rows written." if dry_run
|
|
38
38
|
end
|
|
39
|
+
|
|
40
|
+
desc "Archive long-term facts that the current consolidation policy would drop. " \
|
|
41
|
+
"Env: DRY_RUN=1 (count only), USER_ID=..., LONG_TERM_TYPE=graph_based|file_based"
|
|
42
|
+
task :prune_by_policy do
|
|
43
|
+
Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
|
|
44
|
+
|
|
45
|
+
require "llmemory"
|
|
46
|
+
require_relative "../llmemory/maintenance/policy_prune"
|
|
47
|
+
|
|
48
|
+
dry_run = ENV["DRY_RUN"].to_s == "1"
|
|
49
|
+
user_id = ENV["USER_ID"].presence || "default"
|
|
50
|
+
long_term_type = (ENV["LONG_TERM_TYPE"].presence || Llmemory.configuration.long_term_type).to_s
|
|
51
|
+
|
|
52
|
+
memory = case long_term_type
|
|
53
|
+
when "graph_based"
|
|
54
|
+
Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id)
|
|
55
|
+
when "file_based"
|
|
56
|
+
Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id)
|
|
57
|
+
else
|
|
58
|
+
abort "Unsupported LONG_TERM_TYPE=#{long_term_type.inspect}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
prune = Llmemory::Maintenance::PolicyPrune.new(dry_run: dry_run)
|
|
62
|
+
result = prune.run(memory: memory)
|
|
63
|
+
|
|
64
|
+
puts "Policy prune for user_id=#{user_id} type=#{long_term_type}" \
|
|
65
|
+
"#{dry_run ? " [DRY RUN]" : ""}"
|
|
66
|
+
puts "Archived: graph_edges=#{result.graph_edges} file_items=#{result.file_items} " \
|
|
67
|
+
"(total=#{result.total})"
|
|
68
|
+
end
|
|
39
69
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llmemory
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.11
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- llmemory
|
|
@@ -149,6 +149,11 @@ files:
|
|
|
149
149
|
- lib/llmemory/cli/commands/users.rb
|
|
150
150
|
- lib/llmemory/cli/commands/working.rb
|
|
151
151
|
- lib/llmemory/configuration.rb
|
|
152
|
+
- lib/llmemory/consolidation.rb
|
|
153
|
+
- lib/llmemory/consolidation/filter.rb
|
|
154
|
+
- lib/llmemory/consolidation/policy.rb
|
|
155
|
+
- lib/llmemory/consolidation/rule_based_policy.rb
|
|
156
|
+
- lib/llmemory/consolidation/unit.rb
|
|
152
157
|
- lib/llmemory/crypto/cipher.rb
|
|
153
158
|
- lib/llmemory/crypto/field_helpers.rb
|
|
154
159
|
- lib/llmemory/dashboard.rb
|
|
@@ -216,6 +221,7 @@ files:
|
|
|
216
221
|
- lib/llmemory/maintenance.rb
|
|
217
222
|
- lib/llmemory/maintenance/cognitive_pass.rb
|
|
218
223
|
- lib/llmemory/maintenance/consolidator.rb
|
|
224
|
+
- lib/llmemory/maintenance/policy_prune.rb
|
|
219
225
|
- lib/llmemory/maintenance/reindexer.rb
|
|
220
226
|
- lib/llmemory/maintenance/runner.rb
|
|
221
227
|
- lib/llmemory/maintenance/search_tokens_backfill.rb
|