llmemory 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -2
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +20 -0
- data/lib/llmemory/cli/commands/base.rb +8 -0
- data/lib/llmemory/cli/commands/episodic.rb +42 -0
- data/lib/llmemory/cli/commands/forget_log.rb +36 -0
- data/lib/llmemory/cli/commands/procedural.rb +44 -0
- data/lib/llmemory/cli/commands/working.rb +31 -0
- data/lib/llmemory/cli.rb +12 -0
- data/lib/llmemory/configuration.rb +4 -0
- data/lib/llmemory/long_term/episodic/memory.rb +71 -16
- data/lib/llmemory/long_term/episodic/storage.rb +7 -5
- data/lib/llmemory/long_term/episodic/storages/active_record_models.rb +17 -0
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +93 -0
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +135 -0
- data/lib/llmemory/long_term/episodic/storages/file_storage.rb +2 -3
- data/lib/llmemory/long_term/episodic/storages/memory_storage.rb +2 -3
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +11 -4
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +16 -6
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +2 -4
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +2 -4
- data/lib/llmemory/long_term/graph_based/memory.rb +77 -60
- data/lib/llmemory/long_term/procedural/memory.rb +71 -17
- data/lib/llmemory/long_term/procedural/storage.rb +7 -5
- data/lib/llmemory/long_term/procedural/storages/active_record_models.rb +17 -0
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +104 -0
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +148 -0
- data/lib/llmemory/long_term/procedural/storages/file_storage.rb +2 -3
- data/lib/llmemory/long_term/procedural/storages/memory_storage.rb +2 -3
- data/lib/llmemory/mcp/server.rb +13 -1
- data/lib/llmemory/mcp/tools/memory_episode_record.rb +48 -0
- data/lib/llmemory/mcp/tools/memory_episodes.rb +43 -0
- data/lib/llmemory/mcp/tools/memory_forget.rb +53 -0
- data/lib/llmemory/mcp/tools/memory_retrieve.rb +10 -2
- data/lib/llmemory/mcp/tools/memory_skill_register.rb +35 -0
- data/lib/llmemory/mcp/tools/memory_skill_report.rb +35 -0
- data/lib/llmemory/mcp/tools/memory_skills.rb +43 -0
- data/lib/llmemory/memory.rb +28 -3
- data/lib/llmemory/retrieval/bm25_scorer.rb +1 -1
- data/lib/llmemory/retrieval/mmr_reranker.rb +1 -1
- data/lib/llmemory/short_term/session_lifecycle.rb +19 -3
- data/lib/llmemory/tokenizer.rb +27 -0
- data/lib/llmemory/vector_store/active_record_store.rb +4 -3
- data/lib/llmemory/vector_store.rb +14 -0
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory.rb +1 -0
- metadata +18 -1
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "time"
|
|
6
|
+
require_relative "base"
|
|
7
|
+
|
|
8
|
+
module Llmemory
|
|
9
|
+
module LongTerm
|
|
10
|
+
module Episodic
|
|
11
|
+
module Storages
|
|
12
|
+
# PostgreSQL backend. Each episode is stored as a JSONB `data` document
|
|
13
|
+
# (plus id/user_id/created_at and a denormalized search_text for keyword
|
|
14
|
+
# search), mirroring the file-based DatabaseStorage pattern.
|
|
15
|
+
class DatabaseStorage < Base
|
|
16
|
+
def initialize(database_url: nil)
|
|
17
|
+
@database_url = database_url || Llmemory.configuration.database_url
|
|
18
|
+
@connection = nil
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def save_episode(user_id, episode)
|
|
22
|
+
ensure_tables!
|
|
23
|
+
id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
|
|
24
|
+
data = symbolize(episode).merge(id: id, user_id: user_id)
|
|
25
|
+
data[:created_at] ||= Time.now.utc.iso8601
|
|
26
|
+
conn.exec_params(
|
|
27
|
+
"INSERT INTO llmemory_episodes (id, user_id, data, search_text, created_at) " \
|
|
28
|
+
"VALUES ($1, $2, $3::jsonb, $4, $5) " \
|
|
29
|
+
"ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
|
|
30
|
+
[id, user_id, JSON.generate(data), searchable_text(data), created_at_value(data)]
|
|
31
|
+
)
|
|
32
|
+
id
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def get_episode(user_id, id)
|
|
36
|
+
ensure_tables!
|
|
37
|
+
rows = conn.exec_params("SELECT data FROM llmemory_episodes WHERE user_id = $1 AND id = $2", [user_id, id])
|
|
38
|
+
rows.any? ? parse_data(rows.first["data"]) : nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def list_episodes(user_id, limit: nil)
|
|
42
|
+
ensure_tables!
|
|
43
|
+
sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 ORDER BY created_at DESC"
|
|
44
|
+
sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
|
|
45
|
+
conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def search_episodes(user_id, query)
|
|
49
|
+
ensure_tables!
|
|
50
|
+
suffix, params = token_filter("search_text", query, 2)
|
|
51
|
+
conn.exec_params(
|
|
52
|
+
"SELECT data FROM llmemory_episodes WHERE user_id = $1#{suffix} ORDER BY created_at DESC",
|
|
53
|
+
[user_id, *params]
|
|
54
|
+
).map { |r| parse_data(r["data"]) }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def count_episodes(user_id)
|
|
58
|
+
ensure_tables!
|
|
59
|
+
conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1", [user_id]).first["c"].to_i
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def delete_episodes(user_id, ids)
|
|
63
|
+
ensure_tables!
|
|
64
|
+
Array(ids).sum do |id|
|
|
65
|
+
conn.exec_params("DELETE FROM llmemory_episodes WHERE user_id = $1 AND id = $2", [user_id, id]).cmd_tuples
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def list_users
|
|
70
|
+
ensure_tables!
|
|
71
|
+
conn.exec("SELECT DISTINCT user_id FROM llmemory_episodes").map { |r| r["user_id"] }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
def conn
|
|
77
|
+
@connection ||= begin
|
|
78
|
+
require "pg"
|
|
79
|
+
PG.connect(@database_url)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def ensure_tables!
|
|
84
|
+
conn.exec(<<~SQL)
|
|
85
|
+
CREATE TABLE IF NOT EXISTS llmemory_episodes (
|
|
86
|
+
id TEXT NOT NULL PRIMARY KEY,
|
|
87
|
+
user_id TEXT NOT NULL,
|
|
88
|
+
data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
89
|
+
search_text TEXT,
|
|
90
|
+
created_at TIMESTAMPTZ NOT NULL
|
|
91
|
+
);
|
|
92
|
+
CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
|
|
93
|
+
SQL
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# OR-of-token LIKE filter (see file-based DatabaseStorage). [""] for an
|
|
97
|
+
# empty query => match all.
|
|
98
|
+
def token_filter(column, query, start_index)
|
|
99
|
+
tokens = Llmemory::Tokenizer.tokenize(query)
|
|
100
|
+
return ["", []] if tokens.empty?
|
|
101
|
+
likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
|
|
102
|
+
[" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def parse_data(value)
|
|
106
|
+
JSON.parse(value.to_s, symbolize_names: true)
|
|
107
|
+
rescue JSON::ParserError
|
|
108
|
+
{}
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def symbolize(hash)
|
|
112
|
+
hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def searchable_text(data)
|
|
116
|
+
parts = [data[:summary], data[:outcome]]
|
|
117
|
+
Array(data[:steps]).each do |s|
|
|
118
|
+
next unless s.is_a?(Hash)
|
|
119
|
+
parts << (s[:observation] || s["observation"])
|
|
120
|
+
parts << (s[:action] || s["action"])
|
|
121
|
+
parts << (s[:result] || s["result"])
|
|
122
|
+
end
|
|
123
|
+
parts.compact.join("\n")
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def created_at_value(data)
|
|
127
|
+
ca = data[:created_at]
|
|
128
|
+
return Time.now.utc.iso8601 if ca.nil?
|
|
129
|
+
ca.respond_to?(:iso8601) ? ca.iso8601 : ca.to_s
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
@@ -35,9 +35,8 @@ module Llmemory
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def search_episodes(user_id, query)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
all_episodes(user_id).select { |e| episode_text(e).downcase.include?(q) }
|
|
38
|
+
return list_episodes(user_id) if query.to_s.strip.empty?
|
|
39
|
+
all_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
|
|
41
40
|
end
|
|
42
41
|
|
|
43
42
|
def count_episodes(user_id)
|
|
@@ -31,9 +31,8 @@ module Llmemory
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def search_episodes(user_id, query)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@episodes[user_id].select { |e| episode_text(e).downcase.include?(q) }
|
|
34
|
+
return list_episodes(user_id) if query.to_s.strip.empty?
|
|
35
|
+
@episodes[user_id].select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
def count_episodes(user_id)
|
|
@@ -64,13 +64,11 @@ module Llmemory
|
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
def search_items(user_id, query)
|
|
67
|
-
|
|
68
|
-
LlmemoryItem.where(user_id: user_id).where("LOWER(content) LIKE LOWER(?)", q).map { |r| row_to_item(r) }
|
|
67
|
+
token_scope(LlmemoryItem.where(user_id: user_id), "content", query).map { |r| row_to_item(r) }
|
|
69
68
|
end
|
|
70
69
|
|
|
71
70
|
def search_resources(user_id, query)
|
|
72
|
-
|
|
73
|
-
LlmemoryResource.where(user_id: user_id).where("LOWER(text) LIKE LOWER(?)", q).map { |r| row_to_resource(r) }
|
|
71
|
+
token_scope(LlmemoryResource.where(user_id: user_id), "text", query).map { |r| row_to_resource(r) }
|
|
74
72
|
end
|
|
75
73
|
|
|
76
74
|
def get_resources_since(user_id, hours:)
|
|
@@ -181,6 +179,15 @@ module Llmemory
|
|
|
181
179
|
(str || "").to_s.gsub(/[%_\\]/) { |c| "\\#{c}" }
|
|
182
180
|
end
|
|
183
181
|
|
|
182
|
+
# OR-of-token LIKE scope for keyword search; unchanged scope (match all)
|
|
183
|
+
# when the query has no tokens.
|
|
184
|
+
def token_scope(scope, column, query)
|
|
185
|
+
tokens = Llmemory::Tokenizer.tokenize(query)
|
|
186
|
+
return scope if tokens.empty?
|
|
187
|
+
clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
|
|
188
|
+
scope.where(clause, *tokens.map { |t| "%#{sanitize_like(t)}%" })
|
|
189
|
+
end
|
|
190
|
+
|
|
184
191
|
def row_to_item(r)
|
|
185
192
|
h = {
|
|
186
193
|
id: r.id,
|
|
@@ -65,20 +65,20 @@ module Llmemory
|
|
|
65
65
|
|
|
66
66
|
def search_items(user_id, query)
|
|
67
67
|
ensure_tables!
|
|
68
|
-
|
|
68
|
+
suffix, params = token_filter("content", query, 2)
|
|
69
69
|
rows = conn.exec_params(
|
|
70
|
-
"SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1
|
|
71
|
-
[user_id,
|
|
70
|
+
"SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1#{suffix}",
|
|
71
|
+
[user_id, *params]
|
|
72
72
|
)
|
|
73
73
|
rows_to_items(rows)
|
|
74
74
|
end
|
|
75
75
|
|
|
76
76
|
def search_resources(user_id, query)
|
|
77
77
|
ensure_tables!
|
|
78
|
-
|
|
78
|
+
suffix, params = token_filter("text", query, 2)
|
|
79
79
|
rows = conn.exec_params(
|
|
80
|
-
"SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1
|
|
81
|
-
[user_id,
|
|
80
|
+
"SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1#{suffix}",
|
|
81
|
+
[user_id, *params]
|
|
82
82
|
)
|
|
83
83
|
rows_to_resources(rows)
|
|
84
84
|
end
|
|
@@ -290,6 +290,16 @@ module Llmemory
|
|
|
290
290
|
end
|
|
291
291
|
end
|
|
292
292
|
|
|
293
|
+
# Builds an OR-of-token LIKE filter for keyword search. Returns
|
|
294
|
+
# ["" , []] for an empty query (match all). Tokens are [a-z0-9]{2,} so
|
|
295
|
+
# they carry no LIKE wildcards.
|
|
296
|
+
def token_filter(column, query, start_index)
|
|
297
|
+
tokens = Llmemory::Tokenizer.tokenize(query)
|
|
298
|
+
return ["", []] if tokens.empty?
|
|
299
|
+
likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
|
|
300
|
+
[" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
|
|
301
|
+
end
|
|
302
|
+
|
|
293
303
|
def parse_provenance(value)
|
|
294
304
|
return nil if value.nil? || value.to_s.strip.empty?
|
|
295
305
|
JSON.parse(value, symbolize_names: true)
|
|
@@ -62,13 +62,11 @@ module Llmemory
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def search_items(user_id, query)
|
|
65
|
-
|
|
66
|
-
get_all_items(user_id).select { |i| (i[:content] || i["content"]).to_s.downcase.include?(query_lower) }
|
|
65
|
+
get_all_items(user_id).select { |i| Llmemory::Tokenizer.matches?(i[:content] || i["content"], query) }
|
|
67
66
|
end
|
|
68
67
|
|
|
69
68
|
def search_resources(user_id, query)
|
|
70
|
-
|
|
71
|
-
get_all_resources(user_id).select { |r| (r[:text] || r["text"]).to_s.downcase.include?(query_lower) }
|
|
69
|
+
get_all_resources(user_id).select { |r| Llmemory::Tokenizer.matches?(r[:text] || r["text"], query) }
|
|
72
70
|
end
|
|
73
71
|
|
|
74
72
|
def get_resources_since(user_id, hours:)
|
|
@@ -51,13 +51,11 @@ module Llmemory
|
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
def search_items(user_id, query)
|
|
54
|
-
|
|
55
|
-
@items[user_id].select { |i| i[:content].to_s.downcase.include?(query_lower) }
|
|
54
|
+
@items[user_id].select { |i| Llmemory::Tokenizer.matches?(i[:content], query) }
|
|
56
55
|
end
|
|
57
56
|
|
|
58
57
|
def search_resources(user_id, query)
|
|
59
|
-
|
|
60
|
-
@resources[user_id].select { |r| r[:text].to_s.downcase.include?(query_lower) }
|
|
58
|
+
@resources[user_id].select { |r| Llmemory::Tokenizer.matches?(r[:text], query) }
|
|
61
59
|
end
|
|
62
60
|
|
|
63
61
|
def get_resources_since(user_id, hours:)
|
|
@@ -28,55 +28,11 @@ module Llmemory
|
|
|
28
28
|
text = Llmemory.configuration.noise_filter_enabled ? NoiseFilter.filter?(conversation_text) : conversation_text.to_s
|
|
29
29
|
return true if text.strip.empty?
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
data = { entities: [], relations: [] } unless data.is_a?(Hash)
|
|
33
|
-
entities = Array(data[:entities] || data["entities"])
|
|
34
|
-
relations = Array(data[:relations] || data["relations"])
|
|
35
|
-
|
|
31
|
+
entities, relations = extract_graph(text)
|
|
36
32
|
return true if entities.empty? && relations.empty?
|
|
37
33
|
|
|
38
34
|
provenance = Llmemory::Provenance.from_text_fingerprint(text, method: "entity_relation_extraction")
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
entities.each do |e|
|
|
42
|
-
next unless e.is_a?(Hash)
|
|
43
|
-
entity_type = e[:type] || e["type"] || "concept"
|
|
44
|
-
name = e[:name] || e["name"]
|
|
45
|
-
next if name.nil? || name.to_s.strip.empty?
|
|
46
|
-
id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: { "provenance" => provenance })
|
|
47
|
-
name_to_id[name.to_s.strip] ||= id
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
relations.each do |r|
|
|
51
|
-
next unless r.is_a?(Hash)
|
|
52
|
-
subject = (r[:subject] || r["subject"]).to_s.strip
|
|
53
|
-
predicate = (r[:predicate] || r["predicate"]).to_s.strip
|
|
54
|
-
object = (r[:object] || r["object"]).to_s.strip
|
|
55
|
-
next if subject.empty? || predicate.empty? || object.empty?
|
|
56
|
-
|
|
57
|
-
subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => provenance })
|
|
58
|
-
object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => provenance })
|
|
59
|
-
|
|
60
|
-
edge = Edge.new(
|
|
61
|
-
id: nil,
|
|
62
|
-
user_id: @user_id,
|
|
63
|
-
subject_id: subject_id,
|
|
64
|
-
predicate: predicate,
|
|
65
|
-
target_id: object_id,
|
|
66
|
-
properties: { "provenance" => provenance },
|
|
67
|
-
created_at: Time.now,
|
|
68
|
-
archived_at: nil
|
|
69
|
-
)
|
|
70
|
-
@conflict_resolver.resolve(edge)
|
|
71
|
-
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
|
|
72
|
-
|
|
73
|
-
text = "#{subject} #{predicate} #{object}"
|
|
74
|
-
embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(text) : nil
|
|
75
|
-
if embedding && @vector_store.respond_to?(:store)
|
|
76
|
-
@vector_store.store(id: "edge_#{edge_id}", embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
35
|
+
ingest(entities, relations, provenance)
|
|
80
36
|
true
|
|
81
37
|
end
|
|
82
38
|
|
|
@@ -106,6 +62,19 @@ module Llmemory
|
|
|
106
62
|
@graph_storage
|
|
107
63
|
end
|
|
108
64
|
|
|
65
|
+
# Stores a fact produced outside the conversational flow (e.g. a
|
|
66
|
+
# reflection insight) by extracting entities/relations from `content` and
|
|
67
|
+
# adding them to the graph, preserving caller-supplied provenance. Lets
|
|
68
|
+
# the Reflector target graph-based semantic memory.
|
|
69
|
+
def remember_fact(content:, category: nil, importance: nil, provenance: nil)
|
|
70
|
+
return nil if content.to_s.strip.empty?
|
|
71
|
+
entities, relations = extract_graph(content)
|
|
72
|
+
return nil if entities.empty? && relations.empty?
|
|
73
|
+
prov = provenance || Llmemory::Provenance.from_text_fingerprint(content, method: "reflection")
|
|
74
|
+
ingest(entities, relations, prov)
|
|
75
|
+
true
|
|
76
|
+
end
|
|
77
|
+
|
|
109
78
|
# --- MemoryModule uniform interface ---
|
|
110
79
|
|
|
111
80
|
def write(payload, **_meta)
|
|
@@ -121,24 +90,72 @@ module Llmemory
|
|
|
121
90
|
{ nodes: @graph_storage.count_nodes(uid), edges: @graph_storage.count_edges(uid) }
|
|
122
91
|
end
|
|
123
92
|
|
|
124
|
-
#
|
|
125
|
-
#
|
|
126
|
-
#
|
|
93
|
+
# Forgets relations by archiving the edges identified by the candidate
|
|
94
|
+
# ids returned from #read/#search_candidates (edge ids), recording the
|
|
95
|
+
# removal in the audit log. Edges are soft-archived (archived_at) so they
|
|
96
|
+
# no longer appear in retrieval; nodes are left in place (a node may still
|
|
97
|
+
# be referenced by other active edges). Returns the number archived.
|
|
127
98
|
def forget(ids:, reason: nil)
|
|
128
|
-
|
|
129
|
-
|
|
99
|
+
archived = Array(ids).map(&:to_s).select { |edge_id| @kg.archive_edge(edge_id) }
|
|
100
|
+
forget_log.record(@user_id, memory_type: "graph_based", ids: archived, reason: reason)
|
|
101
|
+
archived.size
|
|
130
102
|
end
|
|
131
103
|
|
|
132
104
|
private
|
|
133
105
|
|
|
134
106
|
def build_vector_store
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
107
|
+
Llmemory::VectorStore.build(source_type: "edge")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def extract_graph(text)
|
|
111
|
+
data = @extractor.extract(text) rescue { entities: [], relations: [] }
|
|
112
|
+
data = { entities: [], relations: [] } unless data.is_a?(Hash)
|
|
113
|
+
[Array(data[:entities] || data["entities"]), Array(data[:relations] || data["relations"])]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Adds entities and relations to the graph (nodes, edges, embeddings) with
|
|
117
|
+
# the given provenance. Shared by memorize (conversation) and
|
|
118
|
+
# remember_fact (reflection).
|
|
119
|
+
def ingest(entities, relations, provenance)
|
|
120
|
+
name_to_id = {}
|
|
121
|
+
|
|
122
|
+
entities.each do |e|
|
|
123
|
+
next unless e.is_a?(Hash)
|
|
124
|
+
entity_type = e[:type] || e["type"] || "concept"
|
|
125
|
+
name = e[:name] || e["name"]
|
|
126
|
+
next if name.nil? || name.to_s.strip.empty?
|
|
127
|
+
id = @kg.add_node(entity_type: entity_type, name: name.to_s.strip, properties: { "provenance" => provenance })
|
|
128
|
+
name_to_id[name.to_s.strip] ||= id
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
relations.each do |r|
|
|
132
|
+
next unless r.is_a?(Hash)
|
|
133
|
+
subject = (r[:subject] || r["subject"]).to_s.strip
|
|
134
|
+
predicate = (r[:predicate] || r["predicate"]).to_s.strip
|
|
135
|
+
object = (r[:object] || r["object"]).to_s.strip
|
|
136
|
+
next if subject.empty? || predicate.empty? || object.empty?
|
|
137
|
+
|
|
138
|
+
subject_id = name_to_id[subject] || @kg.add_node(entity_type: "concept", name: subject, properties: { "provenance" => provenance })
|
|
139
|
+
object_id = name_to_id[object] || @kg.add_node(entity_type: "concept", name: object, properties: { "provenance" => provenance })
|
|
140
|
+
|
|
141
|
+
edge = Edge.new(
|
|
142
|
+
id: nil,
|
|
143
|
+
user_id: @user_id,
|
|
144
|
+
subject_id: subject_id,
|
|
145
|
+
predicate: predicate,
|
|
146
|
+
target_id: object_id,
|
|
147
|
+
properties: { "provenance" => provenance },
|
|
148
|
+
created_at: Time.now,
|
|
149
|
+
archived_at: nil
|
|
150
|
+
)
|
|
151
|
+
@conflict_resolver.resolve(edge)
|
|
152
|
+
edge_id = @kg.add_edge(subject: subject_id, predicate: predicate, object: object_id, properties: { "provenance" => provenance })
|
|
153
|
+
|
|
154
|
+
edge_text = "#{subject} #{predicate} #{object}"
|
|
155
|
+
embedding = @vector_store.respond_to?(:embed) ? @vector_store.embed(edge_text) : nil
|
|
156
|
+
if embedding && @vector_store.respond_to?(:store)
|
|
157
|
+
@vector_store.store(id: edge_id, embedding: embedding, metadata: { text: edge_text, created_at: Time.now }, user_id: @user_id)
|
|
158
|
+
end
|
|
142
159
|
end
|
|
143
160
|
end
|
|
144
161
|
|
|
@@ -166,7 +183,7 @@ module Llmemory
|
|
|
166
183
|
subj = @kg.find_node_by_id(e.subject_id)
|
|
167
184
|
obj = @kg.find_node_by_id(e.target_id)
|
|
168
185
|
edge_text = "#{subj&.name} #{e.predicate} #{obj&.name}"
|
|
169
|
-
out << { id:
|
|
186
|
+
out << { id: e.id, text: edge_text, score: 0.85, created_at: e.created_at } unless out.any? { |o| o[:text] == edge_text }
|
|
170
187
|
end
|
|
171
188
|
end
|
|
172
189
|
|
|
@@ -179,7 +196,7 @@ module Llmemory
|
|
|
179
196
|
obj = @kg.find_node_by_id(e.target_id)
|
|
180
197
|
next unless subj && obj
|
|
181
198
|
edge_text = "#{subj.name} #{e.predicate} #{obj.name}"
|
|
182
|
-
out << { id:
|
|
199
|
+
out << { id: e.id, text: edge_text, score: 0.7, created_at: e.created_at }
|
|
183
200
|
end
|
|
184
201
|
end
|
|
185
202
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "skill"
|
|
4
4
|
require_relative "storage"
|
|
5
5
|
require_relative "../../memory_module"
|
|
6
|
+
require_relative "../../vector_store"
|
|
6
7
|
|
|
7
8
|
module Llmemory
|
|
8
9
|
module LongTerm
|
|
@@ -12,17 +13,20 @@ module Llmemory
|
|
|
12
13
|
# relevance to the current task, and report outcomes so proven skills are
|
|
13
14
|
# preferred over unproven ones.
|
|
14
15
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
16
|
+
# The success rate of each skill is surfaced as `importance`, so the
|
|
17
|
+
# retrieval Engine ranks battle-tested skills higher (P3). Semantic
|
|
18
|
+
# (embedding) retrieval is opt-in via `config.procedural_vector_enabled` or
|
|
19
|
+
# by injecting a `vector_store:`; when off, search is keyword-only.
|
|
18
20
|
class Memory
|
|
19
21
|
include Llmemory::MemoryModule
|
|
20
22
|
|
|
21
23
|
attr_reader :user_id, :storage
|
|
22
24
|
|
|
23
|
-
def initialize(user_id:, storage: nil)
|
|
25
|
+
def initialize(user_id:, storage: nil, vector_store: nil)
|
|
24
26
|
@user_id = user_id
|
|
25
27
|
@storage = storage || Storages.build
|
|
28
|
+
@vector_store = vector_store
|
|
29
|
+
@vector_explicit = !vector_store.nil?
|
|
26
30
|
end
|
|
27
31
|
|
|
28
32
|
# Registers a skill. If `version` is omitted and a skill with the same
|
|
@@ -33,7 +37,9 @@ module Llmemory
|
|
|
33
37
|
id: nil, user_id: @user_id, name: name, body: body,
|
|
34
38
|
description: description, kind: kind, version: version
|
|
35
39
|
)
|
|
36
|
-
@storage.save_skill(@user_id, skill.to_h)
|
|
40
|
+
id = @storage.save_skill(@user_id, skill.to_h)
|
|
41
|
+
index_vector(id, skill.searchable_text)
|
|
42
|
+
id
|
|
37
43
|
end
|
|
38
44
|
|
|
39
45
|
def find_skill(query)
|
|
@@ -62,22 +68,17 @@ module Llmemory
|
|
|
62
68
|
end
|
|
63
69
|
|
|
64
70
|
# Retrieval Engine integration: skills ranked by relevance, recency and
|
|
65
|
-
# proven utility (success rate exposed as importance).
|
|
71
|
+
# proven utility (success rate exposed as importance). Hybrid (vector +
|
|
72
|
+
# keyword) when a vector store is active; otherwise keyword-only.
|
|
66
73
|
def search_candidates(query, user_id: nil, top_k: 20)
|
|
67
74
|
uid = user_id || @user_id
|
|
68
75
|
return [] unless uid == @user_id
|
|
69
76
|
|
|
70
|
-
@storage.search_skills(uid, query).first(top_k).map
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
timestamp: skill.created_at,
|
|
76
|
-
score: 1.0,
|
|
77
|
-
importance: skill.success_rate,
|
|
78
|
-
evergreen: false
|
|
79
|
-
}
|
|
80
|
-
end
|
|
77
|
+
keyword = @storage.search_skills(uid, query).first(top_k).map { |raw| candidate_for(raw, 1.0) }
|
|
78
|
+
vs = vector_store
|
|
79
|
+
return keyword unless vs
|
|
80
|
+
|
|
81
|
+
merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
|
|
81
82
|
end
|
|
82
83
|
|
|
83
84
|
# --- MemoryModule uniform interface ---
|
|
@@ -110,6 +111,59 @@ module Llmemory
|
|
|
110
111
|
return 1 if existing.empty?
|
|
111
112
|
existing.map { |s| (s[:version] || s["version"] || 1).to_i }.max + 1
|
|
112
113
|
end
|
|
114
|
+
|
|
115
|
+
# Active vector store: injected, or a config-gated lazy build; nil when
|
|
116
|
+
# semantic search is disabled (default).
|
|
117
|
+
def vector_store
|
|
118
|
+
if @vector_explicit
|
|
119
|
+
@vector_store
|
|
120
|
+
elsif Llmemory.configuration.procedural_vector_enabled
|
|
121
|
+
@vector_store ||= Llmemory::VectorStore.build(source_type: "skill")
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Best-effort embedding indexing; failures never break registration.
|
|
126
|
+
def index_vector(id, text)
|
|
127
|
+
vs = vector_store
|
|
128
|
+
return if vs.nil? || text.to_s.strip.empty?
|
|
129
|
+
embedding = vs.embed(text)
|
|
130
|
+
return unless embedding
|
|
131
|
+
vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
|
|
132
|
+
rescue StandardError
|
|
133
|
+
nil
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def vector_candidates(query, top_k, vs)
|
|
137
|
+
vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
|
|
138
|
+
raw = @storage.get_skill(@user_id, r[:id] || r["id"])
|
|
139
|
+
raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
|
|
140
|
+
end
|
|
141
|
+
rescue StandardError
|
|
142
|
+
[]
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def candidate_for(raw, score)
|
|
146
|
+
skill = Skill.from_h(raw)
|
|
147
|
+
{
|
|
148
|
+
id: skill.id,
|
|
149
|
+
text: skill.searchable_text,
|
|
150
|
+
timestamp: skill.created_at,
|
|
151
|
+
score: score,
|
|
152
|
+
importance: skill.success_rate,
|
|
153
|
+
evergreen: false
|
|
154
|
+
}
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Dedup by id keeping the higher score; highest score first, capped.
|
|
158
|
+
def merge_candidates(primary, secondary, top_k)
|
|
159
|
+
by_id = {}
|
|
160
|
+
(primary + secondary).each do |c|
|
|
161
|
+
key = c[:id] || c[:text]
|
|
162
|
+
existing = by_id[key]
|
|
163
|
+
by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
|
|
164
|
+
end
|
|
165
|
+
by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
|
|
166
|
+
end
|
|
113
167
|
end
|
|
114
168
|
end
|
|
115
169
|
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "storages/base"
|
|
4
4
|
require_relative "storages/memory_storage"
|
|
5
5
|
require_relative "storages/file_storage"
|
|
6
|
+
require_relative "storages/database_storage"
|
|
6
7
|
|
|
7
8
|
module Llmemory
|
|
8
9
|
module LongTerm
|
|
@@ -11,16 +12,17 @@ module Llmemory
|
|
|
11
12
|
Storage = Storages::MemoryStorage
|
|
12
13
|
|
|
13
14
|
module Storages
|
|
14
|
-
def self.build(store: nil, base_path: nil)
|
|
15
|
+
def self.build(store: nil, base_path: nil, database_url: nil)
|
|
15
16
|
case (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
16
17
|
when :memory
|
|
17
18
|
MemoryStorage.new
|
|
18
19
|
when :file
|
|
19
20
|
FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
|
|
20
|
-
when :postgres, :database
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
when :postgres, :database
|
|
22
|
+
DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
|
|
23
|
+
when :active_record, :activerecord
|
|
24
|
+
require_relative "storages/active_record_storage"
|
|
25
|
+
ActiveRecordStorage.new
|
|
24
26
|
else
|
|
25
27
|
MemoryStorage.new
|
|
26
28
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Model for Procedural ActiveRecordStorage. Loaded only when using
|
|
4
|
+
# store: :active_record. JSONB `data` auto-deserializes to a Hash in Rails 5+.
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module LongTerm
|
|
8
|
+
module Procedural
|
|
9
|
+
module Storages
|
|
10
|
+
class LlmemorySkill < ::ActiveRecord::Base
|
|
11
|
+
self.table_name = "llmemory_skills"
|
|
12
|
+
self.primary_key = "id"
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|