llmemory 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +54 -3
- data/app/controllers/llmemory/dashboard/application_controller.rb +15 -1
- data/app/controllers/llmemory/dashboard/episodic_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/forget_log_controller.rb +12 -0
- data/app/controllers/llmemory/dashboard/maintenance_controller.rb +92 -0
- data/app/controllers/llmemory/dashboard/procedural_controller.rb +22 -0
- data/app/controllers/llmemory/dashboard/reflection_controller.rb +37 -0
- data/app/controllers/llmemory/dashboard/working_controller.rb +14 -0
- data/app/views/llmemory/dashboard/episodic/index.html.erb +37 -0
- data/app/views/llmemory/dashboard/forget_log/show.html.erb +23 -0
- data/app/views/llmemory/dashboard/maintenance/show.html.erb +65 -0
- data/app/views/llmemory/dashboard/procedural/index.html.erb +38 -0
- data/app/views/llmemory/dashboard/reflection/show.html.erb +29 -0
- data/app/views/llmemory/dashboard/users/show.html.erb +16 -0
- data/app/views/llmemory/dashboard/working/show.html.erb +20 -0
- data/config/routes.rb +14 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +22 -0
- data/lib/llmemory/cli/commands/base.rb +8 -0
- data/lib/llmemory/cli/commands/episodic.rb +42 -0
- data/lib/llmemory/cli/commands/forget_log.rb +36 -0
- data/lib/llmemory/cli/commands/maintain.rb +62 -0
- data/lib/llmemory/cli/commands/mine_skills.rb +50 -0
- data/lib/llmemory/cli/commands/procedural.rb +44 -0
- data/lib/llmemory/cli/commands/working.rb +31 -0
- data/lib/llmemory/cli.rb +18 -0
- data/lib/llmemory/configuration.rb +11 -1
- data/lib/llmemory/instrumentation.rb +33 -0
- data/lib/llmemory/llm/anthropic.rb +19 -15
- data/lib/llmemory/llm/openai.rb +16 -12
- data/lib/llmemory/long_term/episodic/memory.rb +94 -26
- data/lib/llmemory/long_term/episodic/storage.rb +7 -5
- data/lib/llmemory/long_term/episodic/storages/active_record_models.rb +17 -0
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +103 -0
- data/lib/llmemory/long_term/episodic/storages/base.rb +15 -2
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +156 -0
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +28 -8
- data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +36 -6
- data/lib/llmemory/long_term/file_based/memory.rb +12 -4
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +15 -6
- data/lib/llmemory/long_term/file_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +20 -8
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +6 -6
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +6 -6
- data/lib/llmemory/long_term/graph_based/memory.rb +89 -64
- data/lib/llmemory/long_term/graph_based/storages/active_record_storage.rb +4 -2
- data/lib/llmemory/long_term/graph_based/storages/base.rb +2 -2
- data/lib/llmemory/long_term/graph_based/storages/memory_storage.rb +4 -2
- data/lib/llmemory/long_term/procedural/memory.rb +97 -30
- data/lib/llmemory/long_term/procedural/skill.rb +6 -2
- data/lib/llmemory/long_term/procedural/storage.rb +7 -5
- data/lib/llmemory/long_term/procedural/storages/active_record_models.rb +17 -0
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +114 -0
- data/lib/llmemory/long_term/procedural/storages/base.rb +14 -1
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +169 -0
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +29 -9
- data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +37 -7
- data/lib/llmemory/maintenance/cognitive_pass.rb +109 -0
- data/lib/llmemory/maintenance/ttl_expiry.rb +50 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +17 -1
- data/lib/llmemory/mcp/tools/memory_episode_record.rb +48 -0
- data/lib/llmemory/mcp/tools/memory_episodes.rb +43 -0
- data/lib/llmemory/mcp/tools/memory_forget.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_maintain.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_mine_skills.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_retrieve.rb +10 -2
- data/lib/llmemory/mcp/tools/memory_skill_register.rb +35 -0
- data/lib/llmemory/mcp/tools/memory_skill_report.rb +35 -0
- data/lib/llmemory/mcp/tools/memory_skills.rb +43 -0
- data/lib/llmemory/memory.rb +48 -3
- data/lib/llmemory/memory_module.rb +13 -6
- data/lib/llmemory/reflection/reflector.rb +24 -20
- data/lib/llmemory/retrieval/bm25_scorer.rb +1 -1
- data/lib/llmemory/retrieval/engine.rb +25 -16
- data/lib/llmemory/retrieval/mmr_reranker.rb +1 -1
- data/lib/llmemory/short_term/session_lifecycle.rb +19 -3
- data/lib/llmemory/skill_mining/miner.rb +163 -0
- data/lib/llmemory/skill_mining.rb +8 -0
- data/lib/llmemory/tokenizer.rb +27 -0
- data/lib/llmemory/vector_store/active_record_store.rb +4 -3
- data/lib/llmemory/vector_store/openai_embeddings.rb +11 -7
- data/lib/llmemory/vector_store.rb +14 -0
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +3 -0
- metadata +39 -1
|
@@ -51,13 +51,11 @@ module Llmemory
|
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
def search_items(user_id, query)
|
|
54
|
-
|
|
55
|
-
@items[user_id].select { |i| i[:content].to_s.downcase.include?(query_lower) }
|
|
54
|
+
@items[user_id].select { |i| Llmemory::Tokenizer.matches?(i[:content], query) }
|
|
56
55
|
end
|
|
57
56
|
|
|
58
57
|
def search_resources(user_id, query)
|
|
59
|
-
|
|
60
|
-
@resources[user_id].select { |r| r[:text].to_s.downcase.include?(query_lower) }
|
|
58
|
+
@resources[user_id].select { |r| Llmemory::Tokenizer.matches?(r[:text], query) }
|
|
61
59
|
end
|
|
62
60
|
|
|
63
61
|
def get_resources_since(user_id, hours:)
|
|
@@ -100,17 +98,19 @@ module Llmemory
|
|
|
100
98
|
(@resources.keys + @items.keys + @categories.keys).uniq
|
|
101
99
|
end
|
|
102
100
|
|
|
103
|
-
def list_resources(user_id:, limit: nil)
|
|
101
|
+
def list_resources(user_id:, limit: nil, offset: nil)
|
|
104
102
|
list = @resources[user_id].dup
|
|
103
|
+
list = list.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
105
104
|
limit ? list.take(limit) : list
|
|
106
105
|
end
|
|
107
106
|
|
|
108
|
-
def list_items(user_id:, category: nil, limit: nil)
|
|
107
|
+
def list_items(user_id:, category: nil, limit: nil, offset: nil)
|
|
109
108
|
list = if category
|
|
110
109
|
@items[user_id].select { |i| i[:category].to_s == category.to_s }
|
|
111
110
|
else
|
|
112
111
|
@items[user_id].dup
|
|
113
112
|
end
|
|
113
|
+
list = list.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
114
114
|
list = list.take(limit) if limit
|
|
115
115
|
list
|
|
116
116
|
end
|
|
@@ -28,55 +28,11 @@ module Llmemory
|
|
|
28
28
|
text = Llmemory.configuration.noise_filter_enabled ? NoiseFilter.filter?(conversation_text) : conversation_text.to_s
|
|
29
29
|
return true if text.strip.empty?
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
data = { entities: [], relations: [] } unless data.is_a?(Hash)
|
|
33
|
-
entities = Array(data[:entities] || data["entities"])
|
|
34
|
-
relations = Array(data[:relations] || data["relations"])
|
|
35
|
-
|
|
31
|
+
entities, relations = extract_graph(text)
|
|
36
32
|
return true if entities.empty? && relations.empty?
|
|
37
33
|
|
|
38
34
|
provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
entities.each do |e|
|
|
42
|
-
next unless e.is_a?(Hash)
|
|
43
|
-
entity_type = e[:type] || e["type"] || "concept"
|
|
44
|
-
name = e[:name] || e["name"]
|
|
45
|
-
next if name.nil? || name.to_s.strip.empty?
|
|
46
|
-
id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: { "provenance" => provenance })
|
|
47
|
-
name_to_id[name.to_s.strip] ||= id
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
relations.each do |r|
|
|
51
|
-
next unless r.is_a?(Hash)
|
|
52
|
-
subject = (r[:subject] || r["subject"]).to_s.strip
|
|
53
|
-
predicate = (r[:predicate] || r["predicate"]).to_s.strip
|
|
54
|
-
object = (r[:object] || r["object"]).to_s.strip
|
|
55
|
-
next if subject.empty? || predicate.empty? || object.empty?
|
|
56
|
-
|
|
57
|
-
subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => provenance })
|
|
58
|
-
object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => provenance })
|
|
59
|
-
|
|
60
|
-
edge = Edge.new(
|
|
61
|
-
id: nil,
|
|
62
|
-
user_id: @user_id,
|
|
63
|
-
subject_id: subject_id,
|
|
64
|
-
predicate: predicate,
|
|
65
|
-
target_id: object_id,
|
|
66
|
-
properties: { "provenance" => provenance },
|
|
67
|
-
created_at: Time.now,
|
|
68
|
-
archived_at: nil
|
|
69
|
-
)
|
|
70
|
-
@conflict_resolver.resolve(edge)
|
|
71
|
-
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
|
|
72
|
-
|
|
73
|
-
text = "#{subject} #{predicate} #{object}"
|
|
74
|
-
embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(text) : nil
|
|
75
|
-
if embedding && @vector_store.respond_to?(:store)
|
|
76
|
-
@vector_store.store(id: "edge_#{edge_id}", embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
35
|
+
ingest(entities, relations, provenance)
|
|
80
36
|
true
|
|
81
37
|
end
|
|
82
38
|
|
|
@@ -106,14 +62,31 @@ module Llmemory
|
|
|
106
62
|
@graph_storage
|
|
107
63
|
end
|
|
108
64
|
|
|
65
|
+
# Stores a fact produced outside the conversational flow (e.g. a
|
|
66
|
+
# reflection insight) by extracting entities/relations from `content` and
|
|
67
|
+
# adding them to the graph, preserving caller-supplied provenance. Lets
|
|
68
|
+
# the Reflector target graph-based semantic memory.
|
|
69
|
+
def remember_fact(content:, category: nil, importance: nil, provenance: nil)
|
|
70
|
+
return nil if content.to_s.strip.empty?
|
|
71
|
+
entities, relations = extract_graph(content)
|
|
72
|
+
return nil if entities.empty? && relations.empty?
|
|
73
|
+
prov = provenance || Llmemory::Provenance.from_text_fingerprint(content, method: "reflection")
|
|
74
|
+
ingest(entities, relations, prov)
|
|
75
|
+
true
|
|
76
|
+
end
|
|
77
|
+
|
|
109
78
|
# --- MemoryModule uniform interface ---
|
|
110
79
|
|
|
111
80
|
def write(payload, **_meta)
|
|
112
|
-
|
|
81
|
+
result = nil
|
|
82
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "graph_based", user_id: @user_id) do
|
|
83
|
+
result = memorize(payload)
|
|
84
|
+
end
|
|
85
|
+
result
|
|
113
86
|
end
|
|
114
87
|
|
|
115
|
-
def list(user_id: nil, limit: nil)
|
|
116
|
-
@graph_storage.list_nodes(user_id || @user_id, limit: limit)
|
|
88
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
89
|
+
@graph_storage.list_nodes(user_id || @user_id, limit: limit, offset: offset)
|
|
117
90
|
end
|
|
118
91
|
|
|
119
92
|
def stats(user_id: nil)
|
|
@@ -121,24 +94,76 @@ module Llmemory
|
|
|
121
94
|
{ nodes: @graph_storage.count_nodes(uid), edges: @graph_storage.count_edges(uid) }
|
|
122
95
|
end
|
|
123
96
|
|
|
124
|
-
#
|
|
125
|
-
#
|
|
126
|
-
#
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
97
|
+
# Forgets relations by archiving the edges identified by the candidate
|
|
98
|
+
# ids returned from #read/#search_candidates (edge ids), recording the
|
|
99
|
+
# removal in the audit log. Edges are soft-archived (archived_at) so they
|
|
100
|
+
# no longer appear in retrieval; nodes are left in place (a node may still
|
|
101
|
+
# be referenced by other active edges). Returns the number archived.
|
|
102
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
103
|
+
# `:hard` would physically delete edge rows; not yet wired (the graph
|
|
104
|
+
# store only exposes soft archive_edge). Both modes route to archive
|
|
105
|
+
# for now; behavior is the same — kept for API uniformity.
|
|
106
|
+
archived = Array(ids).map(&:to_s).select { |edge_id| @kg.archive_edge(edge_id) }
|
|
107
|
+
forget_log.record(@user_id, memory_type: "graph_based", ids: archived, reason: reason)
|
|
108
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "graph_based", user_id: @user_id, count: archived.size, mode: mode)
|
|
109
|
+
archived.size
|
|
130
110
|
end
|
|
131
111
|
|
|
132
112
|
private
|
|
133
113
|
|
|
134
114
|
def build_vector_store
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
115
|
+
Llmemory::VectorStore.build(source_type: "edge")
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def extract_graph(text)
|
|
119
|
+
data = @extractor.extract(text) rescue { entities: [], relations: [] }
|
|
120
|
+
data = { entities: [], relations: [] } unless data.is_a?(Hash)
|
|
121
|
+
[Array(data[:entities] || data["entities"]), Array(data[:relations] || data["relations"])]
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Adds entities and relations to the graph (nodes, edges, embeddings) with
|
|
125
|
+
# the given provenance. Shared by memorize (conversation) and
|
|
126
|
+
# remember_fact (reflection).
|
|
127
|
+
def ingest(entities, relations, provenance)
|
|
128
|
+
name_to_id = {}
|
|
129
|
+
|
|
130
|
+
entities.each do |e|
|
|
131
|
+
next unless e.is_a?(Hash)
|
|
132
|
+
entity_type = e[:type] || e["type"] || "concept"
|
|
133
|
+
name = e[:name] || e["name"]
|
|
134
|
+
next if name.nil? || name.to_s.strip.empty?
|
|
135
|
+
id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: { "provenance" => provenance })
|
|
136
|
+
name_to_id[name.to_s.strip] ||= id
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
relations.each do |r|
|
|
140
|
+
next unless r.is_a?(Hash)
|
|
141
|
+
subject = (r[:subject] || r["subject"]).to_s.strip
|
|
142
|
+
predicate = (r[:predicate] || r["predicate"]).to_s.strip
|
|
143
|
+
object = (r[:object] || r["object"]).to_s.strip
|
|
144
|
+
next if subject.empty? || predicate.empty? || object.empty?
|
|
145
|
+
|
|
146
|
+
subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => provenance })
|
|
147
|
+
object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => provenance })
|
|
148
|
+
|
|
149
|
+
edge = Edge.new(
|
|
150
|
+
id: nil,
|
|
151
|
+
user_id: @user_id,
|
|
152
|
+
subject_id: subject_id,
|
|
153
|
+
predicate: predicate,
|
|
154
|
+
target_id: object_id,
|
|
155
|
+
properties: { "provenance" => provenance },
|
|
156
|
+
created_at: Time.now,
|
|
157
|
+
archived_at: nil
|
|
158
|
+
)
|
|
159
|
+
@conflict_resolver.resolve(edge)
|
|
160
|
+
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
|
|
161
|
+
|
|
162
|
+
edge_text = "#{subject} #{predicate} #{object}"
|
|
163
|
+
embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(edge_text) : nil
|
|
164
|
+
if embedding && @vector_store.respond_to?(:store)
|
|
165
|
+
@vector_store.store(id: edge_id, embedding: embedding, metadata: { text: edge_text, created_at: Time.now }, user_id: @user_id)
|
|
166
|
+
end
|
|
142
167
|
end
|
|
143
168
|
end
|
|
144
169
|
|
|
@@ -166,7 +191,7 @@ module Llmemory
|
|
|
166
191
|
subj = @kg.find_node_by_id(e.subject_id)
|
|
167
192
|
obj = @kg.find_node_by_id(e.target_id)
|
|
168
193
|
edge_text = "#{subj&.name} #{e.predicate} #{obj&.name}"
|
|
169
|
-
out << { id:
|
|
194
|
+
out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
|
|
170
195
|
end
|
|
171
196
|
end
|
|
172
197
|
|
|
@@ -179,7 +204,7 @@ module Llmemory
|
|
|
179
204
|
obj = @kg.find_node_by_id(e.target_id)
|
|
180
205
|
next unless subj && obj
|
|
181
206
|
edge_text = "#{subj.name} #{e.predicate} #{obj.name}"
|
|
182
|
-
out << { id:
|
|
207
|
+
out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at }
|
|
183
208
|
end
|
|
184
209
|
end
|
|
185
210
|
|
|
@@ -52,10 +52,11 @@ module Llmemory
|
|
|
52
52
|
record_to_node(rec) if rec
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
-
def list_nodes(user_id, entity_type: nil, limit: nil)
|
|
55
|
+
def list_nodes(user_id, entity_type: nil, limit: nil, offset: nil)
|
|
56
56
|
scope = LlmemoryGraphNode.where(user_id: user_id)
|
|
57
57
|
scope = scope.where(entity_type: entity_type) if entity_type
|
|
58
58
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
59
|
+
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
59
60
|
scope.map { |r| record_to_node(r) }
|
|
60
61
|
end
|
|
61
62
|
|
|
@@ -106,12 +107,13 @@ module Llmemory
|
|
|
106
107
|
(LlmemoryGraphNode.distinct.pluck(:user_id) + LlmemoryGraphEdge.distinct.pluck(:user_id)).uniq
|
|
107
108
|
end
|
|
108
109
|
|
|
109
|
-
def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil)
|
|
110
|
+
def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil, offset: nil)
|
|
110
111
|
scope = LlmemoryGraphEdge.where(user_id: user_id, archived_at: nil)
|
|
111
112
|
scope = scope.where(subject_id: subject_id) if subject_id
|
|
112
113
|
scope = scope.where(predicate: predicate) if predicate
|
|
113
114
|
scope = scope.order(created_at: :desc) if limit && limit.to_i.positive?
|
|
114
115
|
scope = scope.limit(limit) if limit && limit.to_i.positive?
|
|
116
|
+
scope = scope.offset(offset) if offset && offset.to_i.positive?
|
|
115
117
|
scope.map { |r| record_to_edge(r) }
|
|
116
118
|
end
|
|
117
119
|
|
|
@@ -17,7 +17,7 @@ module Llmemory
|
|
|
17
17
|
raise NotImplementedError, "#{self.class}#find_node_by_name must be implemented"
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
def list_nodes(user_id, entity_type: nil, limit: nil)
|
|
20
|
+
def list_nodes(user_id, entity_type: nil, limit: nil, offset: nil)
|
|
21
21
|
raise NotImplementedError, "#{self.class}#list_nodes must be implemented"
|
|
22
22
|
end
|
|
23
23
|
|
|
@@ -37,7 +37,7 @@ module Llmemory
|
|
|
37
37
|
raise NotImplementedError, "#{self.class}#list_users must be implemented"
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
-
def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil)
|
|
40
|
+
def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil, offset: nil)
|
|
41
41
|
raise NotImplementedError, "#{self.class}#list_edges must be implemented"
|
|
42
42
|
end
|
|
43
43
|
|
|
@@ -43,9 +43,10 @@ module Llmemory
|
|
|
43
43
|
@nodes[user_id].values.find { |n| n.entity_type == entity_type.to_s && n.name.to_s == name.to_s }
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
-
def list_nodes(user_id, entity_type: nil, limit: nil)
|
|
46
|
+
def list_nodes(user_id, entity_type: nil, limit: nil, offset: nil)
|
|
47
47
|
list = @nodes[user_id].values
|
|
48
48
|
list = list.select { |n| n.entity_type.to_s == entity_type.to_s } if entity_type
|
|
49
|
+
list = list.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
49
50
|
limit ? list.take(limit) : list
|
|
50
51
|
end
|
|
51
52
|
|
|
@@ -106,8 +107,9 @@ module Llmemory
|
|
|
106
107
|
(@nodes.keys + @edges.keys).uniq
|
|
107
108
|
end
|
|
108
109
|
|
|
109
|
-
def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil)
|
|
110
|
+
def list_edges(user_id, subject_id: nil, predicate: nil, limit: nil, offset: nil)
|
|
110
111
|
list = find_edges(user_id, subject_id: subject_id, predicate: predicate, object_id: nil, include_archived: false)
|
|
112
|
+
list = list.drop(offset.to_i) if offset && offset.to_i.positive?
|
|
111
113
|
limit ? list.take(limit) : list
|
|
112
114
|
end
|
|
113
115
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "skill"
|
|
4
4
|
require_relative "storage"
|
|
5
5
|
require_relative "../../memory_module"
|
|
6
|
+
require_relative "../../vector_store"
|
|
6
7
|
|
|
7
8
|
module Llmemory
|
|
8
9
|
module LongTerm
|
|
@@ -12,28 +13,33 @@ module Llmemory
|
|
|
12
13
|
# relevance to the current task, and report outcomes so proven skills are
|
|
13
14
|
# preferred over unproven ones.
|
|
14
15
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
16
|
+
# The success rate of each skill is surfaced as `importance`, so the
|
|
17
|
+
# retrieval Engine ranks battle-tested skills higher (P3). Semantic
|
|
18
|
+
# (embedding) retrieval is opt-in via `config.procedural_vector_enabled` or
|
|
19
|
+
# by injecting a `vector_store:`; when off, search is keyword-only.
|
|
18
20
|
class Memory
|
|
19
21
|
include Llmemory::MemoryModule
|
|
20
22
|
|
|
21
23
|
attr_reader :user_id, :storage
|
|
22
24
|
|
|
23
|
-
def initialize(user_id:, storage: nil)
|
|
25
|
+
def initialize(user_id:, storage: nil, vector_store: nil)
|
|
24
26
|
@user_id = user_id
|
|
25
27
|
@storage = storage || Storages.build
|
|
28
|
+
@vector_store = vector_store
|
|
29
|
+
@vector_explicit = !vector_store.nil?
|
|
26
30
|
end
|
|
27
31
|
|
|
28
32
|
# Registers a skill. If `version` is omitted and a skill with the same
|
|
29
33
|
# name exists, the version auto-increments (skill evolution).
|
|
30
|
-
def register_skill(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil)
|
|
34
|
+
def register_skill(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil, provenance: nil)
|
|
31
35
|
version ||= next_version_for(name)
|
|
32
36
|
skill = Skill.new(
|
|
33
37
|
id: nil, user_id: @user_id, name: name, body: body,
|
|
34
|
-
description: description, kind: kind, version: version
|
|
38
|
+
description: description, kind: kind, version: version, provenance: provenance
|
|
35
39
|
)
|
|
36
|
-
@storage.save_skill(@user_id, skill.to_h)
|
|
40
|
+
id = @storage.save_skill(@user_id, skill.to_h)
|
|
41
|
+
index_vector(id, skill.searchable_text)
|
|
42
|
+
id
|
|
37
43
|
end
|
|
38
44
|
|
|
39
45
|
def find_skill(query)
|
|
@@ -46,8 +52,8 @@ module Llmemory
|
|
|
46
52
|
raw && Skill.from_h(raw)
|
|
47
53
|
end
|
|
48
54
|
|
|
49
|
-
def skills(limit: nil)
|
|
50
|
-
@storage.list_skills(@user_id, limit: limit).map { |s| Skill.from_h(s) }
|
|
55
|
+
def skills(limit: nil, offset: nil)
|
|
56
|
+
@storage.list_skills(@user_id, limit: limit, offset: offset).map { |s| Skill.from_h(s) }
|
|
51
57
|
end
|
|
52
58
|
|
|
53
59
|
def count
|
|
@@ -62,45 +68,53 @@ module Llmemory
|
|
|
62
68
|
end
|
|
63
69
|
|
|
64
70
|
# Retrieval Engine integration: skills ranked by relevance, recency and
|
|
65
|
-
# proven utility (success rate exposed as importance).
|
|
71
|
+
# proven utility (success rate exposed as importance). Hybrid (vector +
|
|
72
|
+
# keyword) when a vector store is active; otherwise keyword-only.
|
|
66
73
|
def search_candidates(query, user_id: nil, top_k: 20)
|
|
67
74
|
uid = user_id || @user_id
|
|
68
75
|
return [] unless uid == @user_id
|
|
69
76
|
|
|
70
|
-
@storage.search_skills(uid, query).first(top_k).map
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
timestamp: skill.created_at,
|
|
76
|
-
score: 1.0,
|
|
77
|
-
importance: skill.success_rate,
|
|
78
|
-
evergreen: false
|
|
79
|
-
}
|
|
80
|
-
end
|
|
77
|
+
keyword = @storage.search_skills(uid, query).first(top_k).map { |raw| candidate_for(raw, 1.0) }
|
|
78
|
+
vs = vector_store
|
|
79
|
+
return keyword unless vs
|
|
80
|
+
|
|
81
|
+
merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
|
|
81
82
|
end
|
|
82
83
|
|
|
83
84
|
# --- MemoryModule uniform interface ---
|
|
84
85
|
|
|
85
|
-
def write(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil, **_meta)
|
|
86
|
-
|
|
86
|
+
def write(name:, body:, description: nil, kind: Skill::DEFAULT_KIND, version: nil, provenance: nil, **_meta)
|
|
87
|
+
result = nil
|
|
88
|
+
Llmemory::Instrumentation.instrument(:memory_write, memory_type: "procedural", user_id: @user_id) do
|
|
89
|
+
result = register_skill(name: name, body: body, description: description, kind: kind, version: version, provenance: provenance)
|
|
90
|
+
end
|
|
91
|
+
result
|
|
87
92
|
end
|
|
88
93
|
|
|
89
|
-
def list(user_id: nil, limit: nil)
|
|
90
|
-
skills(limit: limit)
|
|
94
|
+
def list(user_id: nil, limit: nil, offset: nil)
|
|
95
|
+
skills(limit: limit, offset: offset)
|
|
91
96
|
end
|
|
92
97
|
|
|
93
98
|
def stats(user_id: nil)
|
|
94
99
|
{ skills: count }
|
|
95
100
|
end
|
|
96
101
|
|
|
97
|
-
def forget(ids:, reason: nil)
|
|
102
|
+
def forget(ids:, reason: nil, mode: :soft)
|
|
98
103
|
requested = Array(ids).map(&:to_s)
|
|
99
104
|
existing = @storage.list_skills(@user_id).map { |s| (s[:id] || s["id"]).to_s }
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
targeted = requested & existing
|
|
106
|
+
count = case mode
|
|
107
|
+
when :hard then @storage.delete_skills(@user_id, targeted).to_i
|
|
108
|
+
else @storage.archive_skills(@user_id, targeted).to_i
|
|
109
|
+
end
|
|
110
|
+
forget_log.record(@user_id, memory_type: "procedural", ids: targeted, reason: reason)
|
|
111
|
+
Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "procedural", user_id: @user_id, count: count, mode: mode)
|
|
112
|
+
count
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Storage accessor for the TTL maintenance job.
|
|
116
|
+
def expired_ids(cutoff:)
|
|
117
|
+
@storage.expired_skill_ids(@user_id, cutoff: cutoff)
|
|
104
118
|
end
|
|
105
119
|
|
|
106
120
|
private
|
|
@@ -110,6 +124,59 @@ module Llmemory
|
|
|
110
124
|
return 1 if existing.empty?
|
|
111
125
|
existing.map { |s| (s[:version] || s["version"] || 1).to_i }.max + 1
|
|
112
126
|
end
|
|
127
|
+
|
|
128
|
+
# Active vector store: injected, or a config-gated lazy build; nil when
|
|
129
|
+
# semantic search is disabled (default).
|
|
130
|
+
def vector_store
|
|
131
|
+
if @vector_explicit
|
|
132
|
+
@vector_store
|
|
133
|
+
elsif Llmemory.configuration.procedural_vector_enabled
|
|
134
|
+
@vector_store ||= Llmemory::VectorStore.build(source_type: "skill")
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Best-effort embedding indexing; failures never break registration.
|
|
139
|
+
def index_vector(id, text)
|
|
140
|
+
vs = vector_store
|
|
141
|
+
return if vs.nil? || text.to_s.strip.empty?
|
|
142
|
+
embedding = vs.embed(text)
|
|
143
|
+
return unless embedding
|
|
144
|
+
vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
|
|
145
|
+
rescue StandardError
|
|
146
|
+
nil
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def vector_candidates(query, top_k, vs)
|
|
150
|
+
vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
|
|
151
|
+
raw = @storage.get_skill(@user_id, r[:id] || r["id"])
|
|
152
|
+
raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
|
|
153
|
+
end
|
|
154
|
+
rescue StandardError
|
|
155
|
+
[]
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def candidate_for(raw, score)
|
|
159
|
+
skill = Skill.from_h(raw)
|
|
160
|
+
{
|
|
161
|
+
id: skill.id,
|
|
162
|
+
text: skill.searchable_text,
|
|
163
|
+
timestamp: skill.created_at,
|
|
164
|
+
score: score,
|
|
165
|
+
importance: skill.success_rate,
|
|
166
|
+
evergreen: false
|
|
167
|
+
}
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Dedup by id keeping the higher score; highest score first, capped.
|
|
171
|
+
def merge_candidates(primary, secondary, top_k)
|
|
172
|
+
by_id = {}
|
|
173
|
+
(primary + secondary).each do |c|
|
|
174
|
+
key = c[:id] || c[:text]
|
|
175
|
+
existing = by_id[key]
|
|
176
|
+
by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
|
|
177
|
+
end
|
|
178
|
+
by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
|
|
179
|
+
end
|
|
113
180
|
end
|
|
114
181
|
end
|
|
115
182
|
end
|
|
@@ -17,10 +17,11 @@ module Llmemory
|
|
|
17
17
|
DEFAULT_KIND = "prompt"
|
|
18
18
|
|
|
19
19
|
attr_reader :id, :user_id, :name, :description, :body, :kind, :version,
|
|
20
|
-
:success_count, :failure_count, :created_at, :updated_at
|
|
20
|
+
:success_count, :failure_count, :provenance, :created_at, :updated_at
|
|
21
21
|
|
|
22
22
|
def initialize(id:, user_id:, name:, body:, description: nil, kind: DEFAULT_KIND,
|
|
23
|
-
version: 1, success_count: 0, failure_count: 0,
|
|
23
|
+
version: 1, success_count: 0, failure_count: 0, provenance: nil,
|
|
24
|
+
created_at: nil, updated_at: nil)
|
|
24
25
|
@id = id
|
|
25
26
|
@user_id = user_id
|
|
26
27
|
@name = name.to_s
|
|
@@ -30,6 +31,7 @@ module Llmemory
|
|
|
30
31
|
@version = version.to_i
|
|
31
32
|
@success_count = success_count.to_i
|
|
32
33
|
@failure_count = failure_count.to_i
|
|
34
|
+
@provenance = provenance
|
|
33
35
|
@created_at = created_at || Time.now
|
|
34
36
|
@updated_at = updated_at || @created_at
|
|
35
37
|
end
|
|
@@ -60,6 +62,7 @@ module Llmemory
|
|
|
60
62
|
version: hash[:version] || hash["version"] || 1,
|
|
61
63
|
success_count: hash[:success_count] || hash["success_count"] || 0,
|
|
62
64
|
failure_count: hash[:failure_count] || hash["failure_count"] || 0,
|
|
65
|
+
provenance: hash[:provenance] || hash["provenance"],
|
|
63
66
|
created_at: parse_time(hash[:created_at] || hash["created_at"]),
|
|
64
67
|
updated_at: parse_time(hash[:updated_at] || hash["updated_at"])
|
|
65
68
|
)
|
|
@@ -83,6 +86,7 @@ module Llmemory
|
|
|
83
86
|
version: version,
|
|
84
87
|
success_count: success_count,
|
|
85
88
|
failure_count: failure_count,
|
|
89
|
+
provenance: provenance,
|
|
86
90
|
created_at: created_at.respond_to?(:iso8601) ? created_at.iso8601(6) : created_at,
|
|
87
91
|
updated_at: updated_at.respond_to?(:iso8601) ? updated_at.iso8601(6) : updated_at
|
|
88
92
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "storages/base"
|
|
4
4
|
require_relative "storages/memory_storage"
|
|
5
5
|
require_relative "storages/file_storage"
|
|
6
|
+
require_relative "storages/database_storage"
|
|
6
7
|
|
|
7
8
|
module Llmemory
|
|
8
9
|
module LongTerm
|
|
@@ -11,16 +12,17 @@ module Llmemory
|
|
|
11
12
|
Storage = Storages::MemoryStorage
|
|
12
13
|
|
|
13
14
|
module Storages
|
|
14
|
-
def self.build(store: nil, base_path: nil)
|
|
15
|
+
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
15
16
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
16
17
|
when :memory
|
|
17
18
|
MemoryStorage.new
|
|
18
19
|
when :file
|
|
19
20
|
FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
|
|
20
|
-
when :postgres, :database
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
when :postgres, :database
|
|
22
|
+
DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
|
|
23
|
+
when :active_record, :activerecord
|
|
24
|
+
require_relative "storages/active_record_storage"
|
|
25
|
+
ActiveRecordStorage.new
|
|
24
26
|
else
|
|
25
27
|
MemoryStorage.new
|
|
26
28
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Model for Procedural ActiveRecordStorage. Loaded only when using
|
|
4
|
+
# store: :active_record. JSONB `data` auto-deserializes to a Hash in Rails 5+.
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module LongTerm
|
|
8
|
+
module Procedural
|
|
9
|
+
module Storages
|
|
10
|
+
class LlmemorySkill < ::ActiveRecord::Base
|
|
11
|
+
self.table_name = "llmemory_skills"
|
|
12
|
+
self.primary_key = "id"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|