llmemory 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +5 -0
- data/lib/llmemory/crypto/cipher.rb +12 -0
- data/lib/llmemory/crypto/field_helpers.rb +25 -0
- data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +25 -7
- data/lib/llmemory/long_term/episodic/storages/database_storage.rb +28 -11
- data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +28 -34
- data/lib/llmemory/long_term/file_based/storages/base.rb +24 -0
- data/lib/llmemory/long_term/file_based/storages/database_storage.rb +50 -34
- data/lib/llmemory/long_term/file_based/storages/file_storage.rb +10 -2
- data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +2 -28
- data/lib/llmemory/long_term/graph_based/storage.rb +3 -1
- data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +40 -9
- data/lib/llmemory/long_term/procedural/storages/database_storage.rb +57 -17
- data/lib/llmemory/maintenance/search_tokens_backfill.rb +339 -0
- data/lib/llmemory/maintenance.rb +1 -0
- data/lib/llmemory/mcp/tools/memory_retrieve.rb +2 -0
- data/lib/llmemory/mcp/tools/memory_timeline.rb +48 -14
- data/lib/llmemory/mcp/tools/memory_timeline_context.rb +15 -5
- data/lib/llmemory/vector_store/memory_store.rb +7 -3
- data/lib/llmemory/version.rb +1 -1
- data/lib/tasks/llmemory.rake +37 -0
- metadata +4 -2
|
@@ -35,7 +35,11 @@ module Llmemory
|
|
|
35
35
|
rec = LlmemorySkill.find_or_initialize_by(id: id)
|
|
36
36
|
rec.user_id = user_id
|
|
37
37
|
rec.data = cipher.enabled? ? enc_json(data) : data
|
|
38
|
-
|
|
38
|
+
text = searchable_text(data)
|
|
39
|
+
rec.search_text = enc(text)
|
|
40
|
+
rec.search_tokens = search_tokens_for(text) if LlmemorySkill.column_names.include?("search_tokens")
|
|
41
|
+
name = (data["name"] || data[:name]).to_s
|
|
42
|
+
rec.name_det = enc_det(name) if LlmemorySkill.column_names.include?("name_det")
|
|
39
43
|
rec.created_at ||= Time.current
|
|
40
44
|
rec.save!
|
|
41
45
|
id
|
|
@@ -56,12 +60,21 @@ module Llmemory
|
|
|
56
60
|
end
|
|
57
61
|
|
|
58
62
|
def search_skills(user_id, query)
|
|
59
|
-
|
|
60
|
-
|
|
63
|
+
scope = LlmemorySkill.where(user_id: user_id, archived_at: nil)
|
|
64
|
+
token_scope(scope, "search_text", query, model: LlmemorySkill)
|
|
65
|
+
.sort_by { |r| -r.created_at.to_i }
|
|
66
|
+
.map { |r| decode_data(r.data) }
|
|
61
67
|
end
|
|
62
68
|
|
|
63
69
|
def find_skills_by_name(user_id, name)
|
|
64
|
-
if cipher.enabled?
|
|
70
|
+
if cipher.enabled? && LlmemorySkill.column_names.include?("name_det")
|
|
71
|
+
indexed = LlmemorySkill.where(user_id: user_id, archived_at: nil, name_det: enc_det(name.to_s))
|
|
72
|
+
.map { |r| decode_data(r.data) }
|
|
73
|
+
legacy = LlmemorySkill.where(user_id: user_id, archived_at: nil, name_det: nil)
|
|
74
|
+
.map { |r| decode_data(r.data) }
|
|
75
|
+
.select { |s| (s[:name] || s["name"]).to_s == name.to_s }
|
|
76
|
+
(indexed + legacy).uniq { |s| s[:id] || s["id"] }
|
|
77
|
+
elsif cipher.enabled?
|
|
65
78
|
LlmemorySkill.where(user_id: user_id, archived_at: nil).map { |r| decode_data(r.data) }
|
|
66
79
|
.select { |s| (s[:name] || s["name"]).to_s == name.to_s }
|
|
67
80
|
else
|
|
@@ -77,7 +90,11 @@ module Llmemory
|
|
|
77
90
|
data[key] = (data[key] || 0).to_i + 1
|
|
78
91
|
data[:updated_at] = Time.now.utc.iso8601
|
|
79
92
|
rec.data = cipher.enabled? ? enc_json(data) : data
|
|
80
|
-
|
|
93
|
+
text = searchable_text(data)
|
|
94
|
+
rec.search_text = enc(text)
|
|
95
|
+
rec.search_tokens = search_tokens_for(text) if LlmemorySkill.column_names.include?("search_tokens")
|
|
96
|
+
name = (data[:name] || data["name"]).to_s
|
|
97
|
+
rec.name_det = enc_det(name) if LlmemorySkill.column_names.include?("name_det")
|
|
81
98
|
rec.save!
|
|
82
99
|
data
|
|
83
100
|
end
|
|
@@ -105,11 +122,25 @@ module Llmemory
|
|
|
105
122
|
|
|
106
123
|
private
|
|
107
124
|
|
|
108
|
-
def token_scope(scope, column, query)
|
|
125
|
+
def token_scope(scope, column, query, model:)
|
|
109
126
|
tokens = Llmemory::Tokenizer.tokenize(query)
|
|
110
|
-
return scope if tokens.empty?
|
|
111
|
-
|
|
112
|
-
|
|
127
|
+
return scope.to_a if tokens.empty?
|
|
128
|
+
|
|
129
|
+
if cipher.enabled? && model.column_names.include?("search_tokens")
|
|
130
|
+
digests = tokens.map { |t| cipher.blind_index(t) }
|
|
131
|
+
clause = digests.map { "search_tokens LIKE ?" }.join(" OR ")
|
|
132
|
+
indexed = scope.where(clause, *digests.map { |d| "% #{d} %" })
|
|
133
|
+
legacy_scope = scope.where(search_tokens: nil)
|
|
134
|
+
indexed_records = indexed.to_a
|
|
135
|
+
legacy_records = legacy_scope.to_a.select do |record|
|
|
136
|
+
plaintext = dec(record.public_send(column))
|
|
137
|
+
Llmemory::Tokenizer.matches?(plaintext, query)
|
|
138
|
+
end
|
|
139
|
+
(indexed_records + legacy_records).uniq { |r| r.id }
|
|
140
|
+
else
|
|
141
|
+
clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
|
|
142
|
+
scope.where(clause, *tokens.map { |t| "%#{t}%" }).to_a
|
|
143
|
+
end
|
|
113
144
|
end
|
|
114
145
|
|
|
115
146
|
def stringify(hash)
|
|
@@ -27,11 +27,13 @@ module Llmemory
|
|
|
27
27
|
id = skill[:id] || skill["id"] || "skill_#{SecureRandom.hex(8)}"
|
|
28
28
|
data = symbolize(skill).merge(id: id, user_id: user_id)
|
|
29
29
|
data[:created_at] ||= Time.now.utc.iso8601
|
|
30
|
+
search = searchable_text(data)
|
|
31
|
+
name = (data[:name] || data["name"]).to_s
|
|
30
32
|
conn.exec_params(
|
|
31
|
-
"INSERT INTO llmemory_skills (id, user_id, data, search_text, created_at) " \
|
|
32
|
-
"VALUES ($1, $2, $3::jsonb, $4, $5) " \
|
|
33
|
-
"ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
|
|
34
|
-
[id, user_id, store_data(data), enc(
|
|
33
|
+
"INSERT INTO llmemory_skills (id, user_id, data, search_text, search_tokens, name_det, created_at) " \
|
|
34
|
+
"VALUES ($1, $2, $3::jsonb, $4, $5, $6, $7) " \
|
|
35
|
+
"ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4, search_tokens = $5, name_det = $6",
|
|
36
|
+
[id, user_id, store_data(data), enc(search), search_tokens_for(search), enc_det(name), created_at_value(data)]
|
|
35
37
|
)
|
|
36
38
|
id
|
|
37
39
|
end
|
|
@@ -52,19 +54,56 @@ module Llmemory
|
|
|
52
54
|
|
|
53
55
|
def search_skills(user_id, query)
|
|
54
56
|
ensure_tables!
|
|
55
|
-
|
|
56
|
-
conn.exec_params(
|
|
57
|
+
tokens = Llmemory::Tokenizer.tokenize(query)
|
|
58
|
+
return conn.exec_params(
|
|
59
|
+
"SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC",
|
|
60
|
+
[user_id]
|
|
61
|
+
).map { |r| parse_data(r["data"]) } if tokens.empty?
|
|
62
|
+
|
|
63
|
+
suffix, params = blind_token_filter("search_text", query, 2, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
|
|
64
|
+
rows = conn.exec_params(
|
|
57
65
|
"SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
|
|
58
66
|
[user_id, *params]
|
|
59
|
-
)
|
|
67
|
+
)
|
|
68
|
+
skills = rows.map { |r| parse_data(r["data"]) }
|
|
69
|
+
return skills unless cipher.enabled?
|
|
70
|
+
|
|
71
|
+
legacy_rows = conn.exec_params(
|
|
72
|
+
"SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND search_tokens IS NULL",
|
|
73
|
+
[user_id]
|
|
74
|
+
)
|
|
75
|
+
legacy = legacy_rows.map { |r| parse_data(r["data"]) }.select do |skill|
|
|
76
|
+
Llmemory::Tokenizer.matches?(searchable_text(skill), query)
|
|
77
|
+
end
|
|
78
|
+
by_id = {}
|
|
79
|
+
(skills + legacy).each { |skill| by_id[skill[:id] || skill["id"]] = skill }
|
|
80
|
+
by_id.values
|
|
60
81
|
end
|
|
61
82
|
|
|
62
83
|
def find_skills_by_name(user_id, name)
|
|
63
84
|
ensure_tables!
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
85
|
+
if cipher.enabled?
|
|
86
|
+
rows = conn.exec_params(
|
|
87
|
+
"SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND name_det = $2",
|
|
88
|
+
[user_id, enc_det(name.to_s)]
|
|
89
|
+
)
|
|
90
|
+
indexed = rows.map { |r| parse_data(r["data"]) }
|
|
91
|
+
legacy_rows = conn.exec_params(
|
|
92
|
+
"SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND name_det IS NULL",
|
|
93
|
+
[user_id]
|
|
94
|
+
)
|
|
95
|
+
legacy = legacy_rows.map { |r| parse_data(r["data"]) }.select do |skill|
|
|
96
|
+
(skill[:name] || skill["name"]).to_s == name.to_s
|
|
97
|
+
end
|
|
98
|
+
by_id = {}
|
|
99
|
+
(indexed + legacy).each { |skill| by_id[skill[:id] || skill["id"]] = skill }
|
|
100
|
+
by_id.values
|
|
101
|
+
else
|
|
102
|
+
conn.exec_params(
|
|
103
|
+
"SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND data->>'name' = $2",
|
|
104
|
+
[user_id, name.to_s]
|
|
105
|
+
).map { |r| parse_data(r["data"]) }
|
|
106
|
+
end
|
|
68
107
|
end
|
|
69
108
|
|
|
70
109
|
def record_outcome(user_id, skill_id, success:)
|
|
@@ -74,9 +113,11 @@ module Llmemory
|
|
|
74
113
|
key = success ? :success_count : :failure_count
|
|
75
114
|
data[key] = (data[key] || 0).to_i + 1
|
|
76
115
|
data[:updated_at] = Time.now.utc.iso8601
|
|
116
|
+
search = searchable_text(data)
|
|
117
|
+
name = (data[:name] || data["name"]).to_s
|
|
77
118
|
conn.exec_params(
|
|
78
|
-
"UPDATE llmemory_skills SET data = $3::jsonb, search_text = $4 WHERE user_id = $1 AND id = $2",
|
|
79
|
-
[user_id, skill_id, store_data(data), enc(
|
|
119
|
+
"UPDATE llmemory_skills SET data = $3::jsonb, search_text = $4, search_tokens = $5, name_det = $6 WHERE user_id = $1 AND id = $2",
|
|
120
|
+
[user_id, skill_id, store_data(data), enc(search), search_tokens_for(search), enc_det(name)]
|
|
80
121
|
)
|
|
81
122
|
data
|
|
82
123
|
end
|
|
@@ -137,14 +178,13 @@ module Llmemory
|
|
|
137
178
|
);
|
|
138
179
|
CREATE INDEX IF NOT EXISTS idx_llmemory_skills_user_id ON llmemory_skills(user_id);
|
|
139
180
|
ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
|
|
181
|
+
ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS search_tokens TEXT;
|
|
182
|
+
ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS name_det TEXT;
|
|
140
183
|
SQL
|
|
141
184
|
end
|
|
142
185
|
|
|
143
186
|
def token_filter(column, query, start_index)
|
|
144
|
-
|
|
145
|
-
return ["", []] if tokens.empty?
|
|
146
|
-
likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
|
|
147
|
-
[" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
|
|
187
|
+
blind_token_filter(column, query, start_index, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
|
|
148
188
|
end
|
|
149
189
|
|
|
150
190
|
def parse_data(value)
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../crypto/field_helpers"
|
|
5
|
+
|
|
6
|
+
module Llmemory
|
|
7
|
+
module Maintenance
|
|
8
|
+
# Populates search_tokens (blind index) and procedural name_det for rows
|
|
9
|
+
# written before encrypted keyword search support. In-place UPDATE only.
|
|
10
|
+
class SearchTokensBackfill
|
|
11
|
+
include Crypto::FieldHelpers
|
|
12
|
+
|
|
13
|
+
Result = Struct.new(
|
|
14
|
+
:items, :resources, :episodes, :skills, :skipped, :dry_run,
|
|
15
|
+
keyword_init: true
|
|
16
|
+
) do
|
|
17
|
+
def total
|
|
18
|
+
items + resources + episodes + skills
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def to_h
|
|
22
|
+
{ items: items, resources: resources, episodes: episodes, skills: skills,
|
|
23
|
+
skipped: skipped, dry_run: dry_run, total: total }
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def initialize(cipher: nil, store: nil, dry_run: false, force: false)
|
|
28
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
29
|
+
@store = (store || Llmemory.configuration.long_term_store).to_s.to_sym
|
|
30
|
+
@dry_run = dry_run
|
|
31
|
+
@force = force
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def run(user_id: nil)
|
|
35
|
+
case @store
|
|
36
|
+
when :active_record, :activerecord
|
|
37
|
+
run_active_record(user_id: user_id)
|
|
38
|
+
when :postgres
|
|
39
|
+
run_postgres(user_id: user_id)
|
|
40
|
+
else
|
|
41
|
+
raise ConfigurationError,
|
|
42
|
+
"backfill_search_tokens requires long_term_store :active_record or :postgres (got #{@store.inspect})"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def run_active_record(user_id: nil)
|
|
49
|
+
require "active_record"
|
|
50
|
+
load_active_record_models!
|
|
51
|
+
|
|
52
|
+
result = empty_result
|
|
53
|
+
result.items = backfill_ar_items(user_id)
|
|
54
|
+
result.resources = backfill_ar_resources(user_id)
|
|
55
|
+
result.episodes = backfill_ar_episodes(user_id)
|
|
56
|
+
result.skills = backfill_ar_skills(user_id)
|
|
57
|
+
result
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def run_postgres(user_id: nil)
|
|
61
|
+
result = empty_result
|
|
62
|
+
result.items = backfill_pg_items(user_id)
|
|
63
|
+
result.resources = backfill_pg_resources(user_id)
|
|
64
|
+
result.episodes = backfill_pg_episodes(user_id)
|
|
65
|
+
result.skills = backfill_pg_skills(user_id)
|
|
66
|
+
result
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def empty_result
|
|
70
|
+
Result.new(items: 0, resources: 0, episodes: 0, skills: 0, skipped: 0, dry_run: @dry_run)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def load_active_record_models!
|
|
74
|
+
LongTerm::FileBased::Storages::ActiveRecordStorage.load_models!
|
|
75
|
+
LongTerm::Episodic::Storages::ActiveRecordStorage.load_models!
|
|
76
|
+
LongTerm::Procedural::Storages::ActiveRecordStorage.load_models!
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# --- ActiveRecord ---
|
|
80
|
+
|
|
81
|
+
def backfill_ar_items(user_id)
|
|
82
|
+
model = LongTerm::FileBased::Storages::LlmemoryItem
|
|
83
|
+
return 0 unless model.column_names.include?("search_tokens")
|
|
84
|
+
|
|
85
|
+
count = 0
|
|
86
|
+
ar_scope(model, user_id).find_each do |rec|
|
|
87
|
+
tokens = search_tokens_for(dec(rec.content))
|
|
88
|
+
next if tokens.nil?
|
|
89
|
+
|
|
90
|
+
rec.update_column(:search_tokens, tokens) unless @dry_run
|
|
91
|
+
count += 1
|
|
92
|
+
end
|
|
93
|
+
count
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def backfill_ar_resources(user_id)
|
|
97
|
+
model = LongTerm::FileBased::Storages::LlmemoryResource
|
|
98
|
+
return 0 unless model.column_names.include?("search_tokens")
|
|
99
|
+
|
|
100
|
+
count = 0
|
|
101
|
+
ar_scope(model, user_id).find_each do |rec|
|
|
102
|
+
tokens = search_tokens_for(dec(rec.text))
|
|
103
|
+
next if tokens.nil?
|
|
104
|
+
|
|
105
|
+
rec.update_column(:search_tokens, tokens) unless @dry_run
|
|
106
|
+
count += 1
|
|
107
|
+
end
|
|
108
|
+
count
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def backfill_ar_episodes(user_id)
|
|
112
|
+
model = LongTerm::Episodic::Storages::LlmemoryEpisode
|
|
113
|
+
return 0 unless model.column_names.include?("search_tokens")
|
|
114
|
+
|
|
115
|
+
count = 0
|
|
116
|
+
ar_scope(model, user_id).find_each do |rec|
|
|
117
|
+
plain = dec(rec.search_text.to_s)
|
|
118
|
+
tokens = search_tokens_for(plain)
|
|
119
|
+
next if tokens.nil?
|
|
120
|
+
|
|
121
|
+
rec.update_column(:search_tokens, tokens) unless @dry_run
|
|
122
|
+
count += 1
|
|
123
|
+
end
|
|
124
|
+
count
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def backfill_ar_skills(user_id)
|
|
128
|
+
model = LongTerm::Procedural::Storages::LlmemorySkill
|
|
129
|
+
has_tokens = model.column_names.include?("search_tokens")
|
|
130
|
+
has_name_det = model.column_names.include?("name_det")
|
|
131
|
+
return 0 unless has_tokens || has_name_det
|
|
132
|
+
|
|
133
|
+
count = 0
|
|
134
|
+
ar_scope_skills(model, user_id, has_tokens: has_tokens, has_name_det: has_name_det).find_each do |rec|
|
|
135
|
+
updates = {}
|
|
136
|
+
if has_tokens
|
|
137
|
+
plain = dec(rec.search_text.to_s)
|
|
138
|
+
tokens = search_tokens_for(plain)
|
|
139
|
+
updates[:search_tokens] = tokens if tokens
|
|
140
|
+
end
|
|
141
|
+
if has_name_det
|
|
142
|
+
data = decode_json_document(rec.data)
|
|
143
|
+
name = (data[:name] || data["name"]).to_s
|
|
144
|
+
updates[:name_det] = enc_det(name) unless name.empty?
|
|
145
|
+
end
|
|
146
|
+
next if updates.empty?
|
|
147
|
+
|
|
148
|
+
if @dry_run
|
|
149
|
+
count += 1
|
|
150
|
+
else
|
|
151
|
+
rec.update_columns(updates)
|
|
152
|
+
count += 1
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
count
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def ar_scope(model, user_id)
|
|
159
|
+
scope = model.all
|
|
160
|
+
scope = scope.where(user_id: user_id) if user_id
|
|
161
|
+
scope = scope.where(search_tokens: nil) unless @force
|
|
162
|
+
scope
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def ar_scope_skills(model, user_id, has_tokens:, has_name_det:)
|
|
166
|
+
scope = model.all
|
|
167
|
+
scope = scope.where(user_id: user_id) if user_id
|
|
168
|
+
return scope if @force
|
|
169
|
+
|
|
170
|
+
if has_tokens && has_name_det
|
|
171
|
+
scope.where("search_tokens IS NULL OR name_det IS NULL")
|
|
172
|
+
elsif has_tokens
|
|
173
|
+
scope.where(search_tokens: nil)
|
|
174
|
+
elsif has_name_det
|
|
175
|
+
scope.where(name_det: nil)
|
|
176
|
+
else
|
|
177
|
+
scope
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# --- Postgres ---
|
|
182
|
+
|
|
183
|
+
def backfill_pg_items(user_id)
|
|
184
|
+
return 0 unless pg_column_exists?("llmemory_items", "search_tokens")
|
|
185
|
+
|
|
186
|
+
count = 0
|
|
187
|
+
pg_each_row("llmemory_items", "id, user_id, content", user_id) do |row|
|
|
188
|
+
tokens = search_tokens_for(dec(row["content"]))
|
|
189
|
+
next if tokens.nil?
|
|
190
|
+
|
|
191
|
+
pg_update("llmemory_items", row["id"], search_tokens: tokens) unless @dry_run
|
|
192
|
+
count += 1
|
|
193
|
+
end
|
|
194
|
+
count
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def backfill_pg_resources(user_id)
|
|
198
|
+
return 0 unless pg_column_exists?("llmemory_resources", "search_tokens")
|
|
199
|
+
|
|
200
|
+
count = 0
|
|
201
|
+
pg_each_row("llmemory_resources", "id, user_id, text", user_id) do |row|
|
|
202
|
+
tokens = search_tokens_for(dec(row["text"]))
|
|
203
|
+
next if tokens.nil?
|
|
204
|
+
|
|
205
|
+
pg_update("llmemory_resources", row["id"], search_tokens: tokens) unless @dry_run
|
|
206
|
+
count += 1
|
|
207
|
+
end
|
|
208
|
+
count
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def backfill_pg_episodes(user_id)
|
|
212
|
+
return 0 unless pg_column_exists?("llmemory_episodes", "search_tokens")
|
|
213
|
+
|
|
214
|
+
count = 0
|
|
215
|
+
pg_each_row("llmemory_episodes", "id, user_id, search_text", user_id) do |row|
|
|
216
|
+
tokens = search_tokens_for(dec(row["search_text"].to_s))
|
|
217
|
+
next if tokens.nil?
|
|
218
|
+
|
|
219
|
+
pg_update("llmemory_episodes", row["id"], search_tokens: tokens) unless @dry_run
|
|
220
|
+
count += 1
|
|
221
|
+
end
|
|
222
|
+
count
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def backfill_pg_skills(user_id)
|
|
226
|
+
has_tokens = pg_column_exists?("llmemory_skills", "search_tokens")
|
|
227
|
+
has_name_det = pg_column_exists?("llmemory_skills", "name_det")
|
|
228
|
+
return 0 unless has_tokens || has_name_det
|
|
229
|
+
|
|
230
|
+
count = 0
|
|
231
|
+
sql = "SELECT id, user_id, data, search_text FROM llmemory_skills"
|
|
232
|
+
params = []
|
|
233
|
+
clauses = []
|
|
234
|
+
if user_id
|
|
235
|
+
params << user_id
|
|
236
|
+
clauses << "user_id = $#{params.size}"
|
|
237
|
+
end
|
|
238
|
+
unless @force
|
|
239
|
+
parts = []
|
|
240
|
+
parts << "search_tokens IS NULL" if has_tokens
|
|
241
|
+
parts << "name_det IS NULL" if has_name_det
|
|
242
|
+
clauses << "(#{parts.join(' OR ')})" if parts.any?
|
|
243
|
+
end
|
|
244
|
+
sql += " WHERE #{clauses.join(' AND ')}" unless clauses.empty?
|
|
245
|
+
|
|
246
|
+
pg_conn.exec_params(sql, params).each do |row|
|
|
247
|
+
sets = {}
|
|
248
|
+
if has_tokens
|
|
249
|
+
tokens = search_tokens_for(dec(row["search_text"].to_s))
|
|
250
|
+
sets["search_tokens"] = tokens if tokens
|
|
251
|
+
end
|
|
252
|
+
if has_name_det
|
|
253
|
+
data = decode_json_document(row["data"])
|
|
254
|
+
name = (data[:name] || data["name"]).to_s
|
|
255
|
+
sets["name_det"] = enc_det(name) unless name.empty?
|
|
256
|
+
end
|
|
257
|
+
next if sets.empty?
|
|
258
|
+
|
|
259
|
+
pg_update_columns("llmemory_skills", row["id"], sets) unless @dry_run
|
|
260
|
+
count += 1
|
|
261
|
+
end
|
|
262
|
+
count
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def pg_each_row(table, columns, user_id)
|
|
266
|
+
sql = "SELECT #{columns} FROM #{table}"
|
|
267
|
+
params = []
|
|
268
|
+
clauses = []
|
|
269
|
+
if user_id
|
|
270
|
+
params << user_id
|
|
271
|
+
clauses << "user_id = $#{params.size}"
|
|
272
|
+
end
|
|
273
|
+
unless @force
|
|
274
|
+
clauses << "search_tokens IS NULL" if pg_column_exists?(table, "search_tokens")
|
|
275
|
+
end
|
|
276
|
+
sql += " WHERE #{clauses.join(' AND ')}" unless clauses.empty?
|
|
277
|
+
|
|
278
|
+
pg_conn.exec_params(sql, params).each { |row| yield row }
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def pg_update(table, id, search_tokens:)
|
|
282
|
+
pg_conn.exec_params(
|
|
283
|
+
"UPDATE #{table} SET search_tokens = $1 WHERE id = $2",
|
|
284
|
+
[search_tokens, id]
|
|
285
|
+
)
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def pg_update_columns(table, id, sets)
|
|
289
|
+
return if sets.empty?
|
|
290
|
+
|
|
291
|
+
idx = 1
|
|
292
|
+
params = []
|
|
293
|
+
assignments = sets.each_key.map do |col|
|
|
294
|
+
params << sets[col]
|
|
295
|
+
clause = "#{col} = $#{idx}"
|
|
296
|
+
idx += 1
|
|
297
|
+
clause
|
|
298
|
+
end
|
|
299
|
+
params << id
|
|
300
|
+
pg_conn.exec_params(
|
|
301
|
+
"UPDATE #{table} SET #{assignments.join(', ')} WHERE id = $#{idx}",
|
|
302
|
+
params
|
|
303
|
+
)
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def pg_column_exists?(table, column)
|
|
307
|
+
@pg_columns ||= {}
|
|
308
|
+
cache_key = "#{table}.#{column}"
|
|
309
|
+
return @pg_columns[cache_key] if @pg_columns.key?(cache_key)
|
|
310
|
+
|
|
311
|
+
rows = pg_conn.exec_params(
|
|
312
|
+
"SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = $2 LIMIT 1",
|
|
313
|
+
[table, column]
|
|
314
|
+
)
|
|
315
|
+
@pg_columns[cache_key] = rows.any?
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def pg_conn
|
|
319
|
+
@pg_conn ||= begin
|
|
320
|
+
require "pg"
|
|
321
|
+
PG.connect(Llmemory.configuration.database_url)
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def decode_json_document(raw)
|
|
326
|
+
return raw.transform_keys(&:to_sym) if raw.is_a?(Hash)
|
|
327
|
+
|
|
328
|
+
str = raw.to_s
|
|
329
|
+
if cipher.encrypted?(str)
|
|
330
|
+
dec_json(str)
|
|
331
|
+
else
|
|
332
|
+
JSON.parse(str, symbolize_names: true)
|
|
333
|
+
end
|
|
334
|
+
rescue JSON::ParserError
|
|
335
|
+
{}
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
end
|
data/lib/llmemory/maintenance.rb
CHANGED
|
@@ -55,6 +55,8 @@ module Llmemory
|
|
|
55
55
|
private
|
|
56
56
|
|
|
57
57
|
def fetch_timeline_context(user_id, query, window)
|
|
58
|
+
return "" if Llmemory.configuration.long_term_type.to_s == "graph_based"
|
|
59
|
+
|
|
58
60
|
storage = build_storage
|
|
59
61
|
items = storage.search_items(user_id, query)
|
|
60
62
|
return "" if items.empty?
|
|
@@ -23,19 +23,10 @@ module Llmemory
|
|
|
23
23
|
timeline = []
|
|
24
24
|
|
|
25
25
|
# Get recent items from long-term memory
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
timeline << {
|
|
31
|
-
type: "fact",
|
|
32
|
-
timestamp: item[:created_at] || item["created_at"],
|
|
33
|
-
category: item[:category] || item["category"],
|
|
34
|
-
content: item[:content] || item["content"]
|
|
35
|
-
}
|
|
36
|
-
end
|
|
37
|
-
rescue NotImplementedError
|
|
38
|
-
# Some storages may not implement get_items_since
|
|
26
|
+
if graph_based?
|
|
27
|
+
append_graph_timeline(timeline, user_id, hours)
|
|
28
|
+
else
|
|
29
|
+
append_file_timeline(timeline, user_id, hours)
|
|
39
30
|
end
|
|
40
31
|
|
|
41
32
|
# Get recent messages from short-term
|
|
@@ -89,7 +80,48 @@ module Llmemory
|
|
|
89
80
|
end
|
|
90
81
|
|
|
91
82
|
def build_long_term_storage
|
|
92
|
-
|
|
83
|
+
if graph_based?
|
|
84
|
+
LongTerm::GraphBased::Storages.build
|
|
85
|
+
else
|
|
86
|
+
LongTerm::FileBased::Storages.build
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def graph_based?
|
|
91
|
+
Llmemory.configuration.long_term_type.to_s == "graph_based"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def append_file_timeline(timeline, user_id, hours)
|
|
95
|
+
storage = build_long_term_storage
|
|
96
|
+
items = storage.get_items_since(user_id, hours: hours)
|
|
97
|
+
items.each do |item|
|
|
98
|
+
timeline << {
|
|
99
|
+
type: "fact",
|
|
100
|
+
timestamp: item[:created_at] || item["created_at"],
|
|
101
|
+
category: item[:category] || item["category"],
|
|
102
|
+
content: item[:content] || item["content"]
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
rescue NotImplementedError
|
|
106
|
+
# Some storages may not implement get_items_since
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def append_graph_timeline(timeline, user_id, hours)
|
|
110
|
+
storage = build_long_term_storage
|
|
111
|
+
cutoff = Time.now - (hours * 3600)
|
|
112
|
+
storage.list_edges(user_id).each do |edge|
|
|
113
|
+
created_at = edge.respond_to?(:created_at) ? edge.created_at : edge[:created_at]
|
|
114
|
+
next unless created_at && created_at >= cutoff
|
|
115
|
+
|
|
116
|
+
subject = edge.respond_to?(:subject_id) ? edge.subject_id : edge[:subject_id]
|
|
117
|
+
predicate = edge.respond_to?(:predicate) ? edge.predicate : edge[:predicate]
|
|
118
|
+
target = edge.respond_to?(:target_id) ? edge.target_id : edge[:target_id]
|
|
119
|
+
timeline << {
|
|
120
|
+
type: "relation",
|
|
121
|
+
timestamp: created_at,
|
|
122
|
+
content: "#{subject} -> #{predicate} -> #{target}"
|
|
123
|
+
}
|
|
124
|
+
end
|
|
93
125
|
end
|
|
94
126
|
|
|
95
127
|
def format_timeline(timeline, hours)
|
|
@@ -101,6 +133,8 @@ module Llmemory
|
|
|
101
133
|
when "fact"
|
|
102
134
|
cat_info = entry[:category] ? "[#{entry[:category]}]" : ""
|
|
103
135
|
output << "- [FACT] #{cat_info} #{truncate(entry[:content], 150)}"
|
|
136
|
+
when "relation"
|
|
137
|
+
output << "- [REL] #{truncate(entry[:content], 150)}"
|
|
104
138
|
when "message"
|
|
105
139
|
output << "- [MSG #{entry[:session_id]}] [#{entry[:role]}] #{truncate(entry[:content], 150)}"
|
|
106
140
|
end
|
|
@@ -31,7 +31,11 @@ module Llmemory
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
storage = build_storage
|
|
34
|
-
result =
|
|
34
|
+
result = if graph_based?
|
|
35
|
+
storage.get_edges_around(user_id, reference, before: before_count, after: after_count)
|
|
36
|
+
else
|
|
37
|
+
storage.get_items_around(user_id, reference, before: before_count, after: after_count)
|
|
38
|
+
end
|
|
35
39
|
|
|
36
40
|
::MCP::Tool::Response.new([{
|
|
37
41
|
type: "text",
|
|
@@ -47,13 +51,17 @@ module Llmemory
|
|
|
47
51
|
private
|
|
48
52
|
|
|
49
53
|
def build_storage
|
|
50
|
-
if
|
|
54
|
+
if graph_based?
|
|
51
55
|
LongTerm::GraphBased::Storages.build
|
|
52
56
|
else
|
|
53
57
|
LongTerm::FileBased::Storages.build
|
|
54
58
|
end
|
|
55
59
|
end
|
|
56
60
|
|
|
61
|
+
def graph_based?
|
|
62
|
+
Llmemory.configuration.long_term_type.to_s == "graph_based"
|
|
63
|
+
end
|
|
64
|
+
|
|
57
65
|
def format_context(result, reference)
|
|
58
66
|
output = []
|
|
59
67
|
output << "Timeline Context around '#{reference}':\n"
|
|
@@ -122,10 +130,12 @@ module Llmemory
|
|
|
122
130
|
end
|
|
123
131
|
|
|
124
132
|
def extract_category(item)
|
|
125
|
-
if item.
|
|
126
|
-
item.category
|
|
127
|
-
else
|
|
133
|
+
if item.is_a?(Hash)
|
|
128
134
|
item[:category] || item["category"]
|
|
135
|
+
elsif item.respond_to?(:members) && item.members.include?(:category)
|
|
136
|
+
item.category
|
|
137
|
+
elsif item.respond_to?(:category)
|
|
138
|
+
item.category
|
|
129
139
|
end
|
|
130
140
|
end
|
|
131
141
|
|