llmemory 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +37 -2
  3. data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +5 -0
  4. data/lib/llmemory/cli/commands/stats.rb +5 -0
  5. data/lib/llmemory/crypto/cipher.rb +12 -0
  6. data/lib/llmemory/crypto/field_helpers.rb +25 -0
  7. data/lib/llmemory/instrumentation.rb +4 -2
  8. data/lib/llmemory/llm/anthropic.rb +8 -3
  9. data/lib/llmemory/llm/base.rb +42 -0
  10. data/lib/llmemory/llm/openai.rb +27 -12
  11. data/lib/llmemory/llm/response.rb +18 -0
  12. data/lib/llmemory/llm/tracking_client.rb +61 -0
  13. data/lib/llmemory/llm/usage.rb +31 -0
  14. data/lib/llmemory/llm/usage_ledger.rb +118 -0
  15. data/lib/llmemory/llm/usage_recorder.rb +37 -0
  16. data/lib/llmemory/llm.rb +5 -0
  17. data/lib/llmemory/long_term/episodic/memory.rb +12 -1
  18. data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb +25 -7
  19. data/lib/llmemory/long_term/episodic/storages/database_storage.rb +28 -11
  20. data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb +28 -34
  21. data/lib/llmemory/long_term/file_based/storages/base.rb +24 -0
  22. data/lib/llmemory/long_term/file_based/storages/database_storage.rb +50 -34
  23. data/lib/llmemory/long_term/file_based/storages/file_storage.rb +10 -2
  24. data/lib/llmemory/long_term/file_based/storages/memory_storage.rb +2 -28
  25. data/lib/llmemory/long_term/graph_based/memory.rb +13 -0
  26. data/lib/llmemory/long_term/graph_based/storage.rb +3 -1
  27. data/lib/llmemory/long_term/procedural/memory.rb +12 -1
  28. data/lib/llmemory/long_term/procedural/storages/active_record_storage.rb +40 -9
  29. data/lib/llmemory/long_term/procedural/storages/database_storage.rb +57 -17
  30. data/lib/llmemory/maintenance/search_tokens_backfill.rb +339 -0
  31. data/lib/llmemory/maintenance.rb +1 -0
  32. data/lib/llmemory/mcp/tools/memory_retrieve.rb +2 -0
  33. data/lib/llmemory/mcp/tools/memory_stats.rb +13 -0
  34. data/lib/llmemory/mcp/tools/memory_timeline.rb +48 -14
  35. data/lib/llmemory/mcp/tools/memory_timeline_context.rb +15 -5
  36. data/lib/llmemory/memory.rb +34 -15
  37. data/lib/llmemory/short_term/checkpoint.rb +2 -0
  38. data/lib/llmemory/vector_store/active_record_store.rb +6 -0
  39. data/lib/llmemory/vector_store/memory_store.rb +13 -3
  40. data/lib/llmemory/vector_store/openai_embeddings.rb +23 -2
  41. data/lib/llmemory/version.rb +1 -1
  42. data/lib/tasks/llmemory.rake +37 -0
  43. metadata +9 -2
@@ -35,7 +35,11 @@ module Llmemory
35
35
  rec = LlmemorySkill.find_or_initialize_by(id: id)
36
36
  rec.user_id = user_id
37
37
  rec.data = cipher.enabled? ? enc_json(data) : data
38
- rec.search_text = enc(searchable_text(data))
38
+ text = searchable_text(data)
39
+ rec.search_text = enc(text)
40
+ rec.search_tokens = search_tokens_for(text) if LlmemorySkill.column_names.include?("search_tokens")
41
+ name = (data["name"] || data[:name]).to_s
42
+ rec.name_det = enc_det(name) if LlmemorySkill.column_names.include?("name_det")
39
43
  rec.created_at ||= Time.current
40
44
  rec.save!
41
45
  id
@@ -56,12 +60,21 @@ module Llmemory
56
60
  end
57
61
 
58
62
  def search_skills(user_id, query)
59
- token_scope(LlmemorySkill.where(user_id: user_id, archived_at: nil), "search_text", query)
60
- .order(created_at: :desc).map { |r| decode_data(r.data) }
63
+ scope = LlmemorySkill.where(user_id: user_id, archived_at: nil)
64
+ token_scope(scope, "search_text", query, model: LlmemorySkill)
65
+ .sort_by { |r| -r.created_at.to_i }
66
+ .map { |r| decode_data(r.data) }
61
67
  end
62
68
 
63
69
  def find_skills_by_name(user_id, name)
64
- if cipher.enabled?
70
+ if cipher.enabled? && LlmemorySkill.column_names.include?("name_det")
71
+ indexed = LlmemorySkill.where(user_id: user_id, archived_at: nil, name_det: enc_det(name.to_s))
72
+ .map { |r| decode_data(r.data) }
73
+ legacy = LlmemorySkill.where(user_id: user_id, archived_at: nil, name_det: nil)
74
+ .map { |r| decode_data(r.data) }
75
+ .select { |s| (s[:name] || s["name"]).to_s == name.to_s }
76
+ (indexed + legacy).uniq { |s| s[:id] || s["id"] }
77
+ elsif cipher.enabled?
65
78
  LlmemorySkill.where(user_id: user_id, archived_at: nil).map { |r| decode_data(r.data) }
66
79
  .select { |s| (s[:name] || s["name"]).to_s == name.to_s }
67
80
  else
@@ -77,7 +90,11 @@ module Llmemory
77
90
  data[key] = (data[key] || 0).to_i + 1
78
91
  data[:updated_at] = Time.now.utc.iso8601
79
92
  rec.data = cipher.enabled? ? enc_json(data) : data
80
- rec.search_text = enc(searchable_text(data))
93
+ text = searchable_text(data)
94
+ rec.search_text = enc(text)
95
+ rec.search_tokens = search_tokens_for(text) if LlmemorySkill.column_names.include?("search_tokens")
96
+ name = (data[:name] || data["name"]).to_s
97
+ rec.name_det = enc_det(name) if LlmemorySkill.column_names.include?("name_det")
81
98
  rec.save!
82
99
  data
83
100
  end
@@ -105,11 +122,25 @@ module Llmemory
105
122
 
106
123
  private
107
124
 
108
- def token_scope(scope, column, query)
125
+ def token_scope(scope, column, query, model:)
109
126
  tokens = Llmemory::Tokenizer.tokenize(query)
110
- return scope if tokens.empty?
111
- clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
112
- scope.where(clause, *tokens.map { |t| "%#{t}%" })
127
+ return scope.to_a if tokens.empty?
128
+
129
+ if cipher.enabled? && model.column_names.include?("search_tokens")
130
+ digests = tokens.map { |t| cipher.blind_index(t) }
131
+ clause = digests.map { "search_tokens LIKE ?" }.join(" OR ")
132
+ indexed = scope.where(clause, *digests.map { |d| "% #{d} %" })
133
+ legacy_scope = scope.where(search_tokens: nil)
134
+ indexed_records = indexed.to_a
135
+ legacy_records = legacy_scope.to_a.select do |record|
136
+ plaintext = dec(record.public_send(column))
137
+ Llmemory::Tokenizer.matches?(plaintext, query)
138
+ end
139
+ (indexed_records + legacy_records).uniq { |r| r.id }
140
+ else
141
+ clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
142
+ scope.where(clause, *tokens.map { |t| "%#{t}%" }).to_a
143
+ end
113
144
  end
114
145
 
115
146
  def stringify(hash)
@@ -27,11 +27,13 @@ module Llmemory
27
27
  id = skill[:id] || skill["id"] || "skill_#{SecureRandom.hex(8)}"
28
28
  data = symbolize(skill).merge(id: id, user_id: user_id)
29
29
  data[:created_at] ||= Time.now.utc.iso8601
30
+ search = searchable_text(data)
31
+ name = (data[:name] || data["name"]).to_s
30
32
  conn.exec_params(
31
- "INSERT INTO llmemory_skills (id, user_id, data, search_text, created_at) " \
32
- "VALUES ($1, $2, $3::jsonb, $4, $5) " \
33
- "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
34
- [id, user_id, store_data(data), enc(searchable_text(data)), created_at_value(data)]
33
+ "INSERT INTO llmemory_skills (id, user_id, data, search_text, search_tokens, name_det, created_at) " \
34
+ "VALUES ($1, $2, $3::jsonb, $4, $5, $6, $7) " \
35
+ "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4, search_tokens = $5, name_det = $6",
36
+ [id, user_id, store_data(data), enc(search), search_tokens_for(search), enc_det(name), created_at_value(data)]
35
37
  )
36
38
  id
37
39
  end
@@ -52,19 +54,56 @@ module Llmemory
52
54
 
53
55
  def search_skills(user_id, query)
54
56
  ensure_tables!
55
- suffix, params = token_filter("search_text", query, 2)
56
- conn.exec_params(
57
+ tokens = Llmemory::Tokenizer.tokenize(query)
58
+ return conn.exec_params(
59
+ "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC",
60
+ [user_id]
61
+ ).map { |r| parse_data(r["data"]) } if tokens.empty?
62
+
63
+ suffix, params = blind_token_filter("search_text", query, 2, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
64
+ rows = conn.exec_params(
57
65
  "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
58
66
  [user_id, *params]
59
- ).map { |r| parse_data(r["data"]) }
67
+ )
68
+ skills = rows.map { |r| parse_data(r["data"]) }
69
+ return skills unless cipher.enabled?
70
+
71
+ legacy_rows = conn.exec_params(
72
+ "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND search_tokens IS NULL",
73
+ [user_id]
74
+ )
75
+ legacy = legacy_rows.map { |r| parse_data(r["data"]) }.select do |skill|
76
+ Llmemory::Tokenizer.matches?(searchable_text(skill), query)
77
+ end
78
+ by_id = {}
79
+ (skills + legacy).each { |skill| by_id[skill[:id] || skill["id"]] = skill }
80
+ by_id.values
60
81
  end
61
82
 
62
83
  def find_skills_by_name(user_id, name)
63
84
  ensure_tables!
64
- conn.exec_params(
65
- "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND data->>'name' = $2",
66
- [user_id, name.to_s]
67
- ).map { |r| parse_data(r["data"]) }
85
+ if cipher.enabled?
86
+ rows = conn.exec_params(
87
+ "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND name_det = $2",
88
+ [user_id, enc_det(name.to_s)]
89
+ )
90
+ indexed = rows.map { |r| parse_data(r["data"]) }
91
+ legacy_rows = conn.exec_params(
92
+ "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND name_det IS NULL",
93
+ [user_id]
94
+ )
95
+ legacy = legacy_rows.map { |r| parse_data(r["data"]) }.select do |skill|
96
+ (skill[:name] || skill["name"]).to_s == name.to_s
97
+ end
98
+ by_id = {}
99
+ (indexed + legacy).each { |skill| by_id[skill[:id] || skill["id"]] = skill }
100
+ by_id.values
101
+ else
102
+ conn.exec_params(
103
+ "SELECT data FROM llmemory_skills WHERE user_id = $1 AND archived_at IS NULL AND data->>'name' = $2",
104
+ [user_id, name.to_s]
105
+ ).map { |r| parse_data(r["data"]) }
106
+ end
68
107
  end
69
108
 
70
109
  def record_outcome(user_id, skill_id, success:)
@@ -74,9 +113,11 @@ module Llmemory
74
113
  key = success ? :success_count : :failure_count
75
114
  data[key] = (data[key] || 0).to_i + 1
76
115
  data[:updated_at] = Time.now.utc.iso8601
116
+ search = searchable_text(data)
117
+ name = (data[:name] || data["name"]).to_s
77
118
  conn.exec_params(
78
- "UPDATE llmemory_skills SET data = $3::jsonb, search_text = $4 WHERE user_id = $1 AND id = $2",
79
- [user_id, skill_id, store_data(data), enc(searchable_text(data))]
119
+ "UPDATE llmemory_skills SET data = $3::jsonb, search_text = $4, search_tokens = $5, name_det = $6 WHERE user_id = $1 AND id = $2",
120
+ [user_id, skill_id, store_data(data), enc(search), search_tokens_for(search), enc_det(name)]
80
121
  )
81
122
  data
82
123
  end
@@ -137,14 +178,13 @@ module Llmemory
137
178
  );
138
179
  CREATE INDEX IF NOT EXISTS idx_llmemory_skills_user_id ON llmemory_skills(user_id);
139
180
  ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
181
+ ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS search_tokens TEXT;
182
+ ALTER TABLE llmemory_skills ADD COLUMN IF NOT EXISTS name_det TEXT;
140
183
  SQL
141
184
  end
142
185
 
143
186
  def token_filter(column, query, start_index)
144
- tokens = Llmemory::Tokenizer.tokenize(query)
145
- return ["", []] if tokens.empty?
146
- likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
147
- [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
187
+ blind_token_filter(column, query, start_index, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
148
188
  end
149
189
 
150
190
  def parse_data(value)
@@ -0,0 +1,339 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require_relative "../crypto/field_helpers"
5
+
6
+ module Llmemory
7
+ module Maintenance
8
+ # Populates search_tokens (blind index) and procedural name_det for rows
9
+ # written before encrypted keyword search support. In-place UPDATE only.
10
+ class SearchTokensBackfill
11
+ include Crypto::FieldHelpers
12
+
13
+ Result = Struct.new(
14
+ :items, :resources, :episodes, :skills, :skipped, :dry_run,
15
+ keyword_init: true
16
+ ) do
17
+ def total
18
+ items + resources + episodes + skills
19
+ end
20
+
21
+ def to_h
22
+ { items: items, resources: resources, episodes: episodes, skills: skills,
23
+ skipped: skipped, dry_run: dry_run, total: total }
24
+ end
25
+ end
26
+
27
+ def initialize(cipher: nil, store: nil, dry_run: false, force: false)
28
+ @cipher = cipher || Llmemory.build_cipher
29
+ @store = (store || Llmemory.configuration.long_term_store).to_s.to_sym
30
+ @dry_run = dry_run
31
+ @force = force
32
+ end
33
+
34
+ def run(user_id: nil)
35
+ case @store
36
+ when :active_record, :activerecord
37
+ run_active_record(user_id: user_id)
38
+ when :postgres
39
+ run_postgres(user_id: user_id)
40
+ else
41
+ raise ConfigurationError,
42
+ "backfill_search_tokens requires long_term_store :active_record or :postgres (got #{@store.inspect})"
43
+ end
44
+ end
45
+
46
+ private
47
+
48
+ def run_active_record(user_id: nil)
49
+ require "active_record"
50
+ load_active_record_models!
51
+
52
+ result = empty_result
53
+ result.items = backfill_ar_items(user_id)
54
+ result.resources = backfill_ar_resources(user_id)
55
+ result.episodes = backfill_ar_episodes(user_id)
56
+ result.skills = backfill_ar_skills(user_id)
57
+ result
58
+ end
59
+
60
+ def run_postgres(user_id: nil)
61
+ result = empty_result
62
+ result.items = backfill_pg_items(user_id)
63
+ result.resources = backfill_pg_resources(user_id)
64
+ result.episodes = backfill_pg_episodes(user_id)
65
+ result.skills = backfill_pg_skills(user_id)
66
+ result
67
+ end
68
+
69
+ def empty_result
70
+ Result.new(items: 0, resources: 0, episodes: 0, skills: 0, skipped: 0, dry_run: @dry_run)
71
+ end
72
+
73
+ def load_active_record_models!
74
+ LongTerm::FileBased::Storages::ActiveRecordStorage.load_models!
75
+ LongTerm::Episodic::Storages::ActiveRecordStorage.load_models!
76
+ LongTerm::Procedural::Storages::ActiveRecordStorage.load_models!
77
+ end
78
+
79
+ # --- ActiveRecord ---
80
+
81
+ def backfill_ar_items(user_id)
82
+ model = LongTerm::FileBased::Storages::LlmemoryItem
83
+ return 0 unless model.column_names.include?("search_tokens")
84
+
85
+ count = 0
86
+ ar_scope(model, user_id).find_each do |rec|
87
+ tokens = search_tokens_for(dec(rec.content))
88
+ next if tokens.nil?
89
+
90
+ rec.update_column(:search_tokens, tokens) unless @dry_run
91
+ count += 1
92
+ end
93
+ count
94
+ end
95
+
96
+ def backfill_ar_resources(user_id)
97
+ model = LongTerm::FileBased::Storages::LlmemoryResource
98
+ return 0 unless model.column_names.include?("search_tokens")
99
+
100
+ count = 0
101
+ ar_scope(model, user_id).find_each do |rec|
102
+ tokens = search_tokens_for(dec(rec.text))
103
+ next if tokens.nil?
104
+
105
+ rec.update_column(:search_tokens, tokens) unless @dry_run
106
+ count += 1
107
+ end
108
+ count
109
+ end
110
+
111
+ def backfill_ar_episodes(user_id)
112
+ model = LongTerm::Episodic::Storages::LlmemoryEpisode
113
+ return 0 unless model.column_names.include?("search_tokens")
114
+
115
+ count = 0
116
+ ar_scope(model, user_id).find_each do |rec|
117
+ plain = dec(rec.search_text.to_s)
118
+ tokens = search_tokens_for(plain)
119
+ next if tokens.nil?
120
+
121
+ rec.update_column(:search_tokens, tokens) unless @dry_run
122
+ count += 1
123
+ end
124
+ count
125
+ end
126
+
127
+ def backfill_ar_skills(user_id)
128
+ model = LongTerm::Procedural::Storages::LlmemorySkill
129
+ has_tokens = model.column_names.include?("search_tokens")
130
+ has_name_det = model.column_names.include?("name_det")
131
+ return 0 unless has_tokens || has_name_det
132
+
133
+ count = 0
134
+ ar_scope_skills(model, user_id, has_tokens: has_tokens, has_name_det: has_name_det).find_each do |rec|
135
+ updates = {}
136
+ if has_tokens
137
+ plain = dec(rec.search_text.to_s)
138
+ tokens = search_tokens_for(plain)
139
+ updates[:search_tokens] = tokens if tokens
140
+ end
141
+ if has_name_det
142
+ data = decode_json_document(rec.data)
143
+ name = (data[:name] || data["name"]).to_s
144
+ updates[:name_det] = enc_det(name) unless name.empty?
145
+ end
146
+ next if updates.empty?
147
+
148
+ if @dry_run
149
+ count += 1
150
+ else
151
+ rec.update_columns(updates)
152
+ count += 1
153
+ end
154
+ end
155
+ count
156
+ end
157
+
158
+ def ar_scope(model, user_id)
159
+ scope = model.all
160
+ scope = scope.where(user_id: user_id) if user_id
161
+ scope = scope.where(search_tokens: nil) unless @force
162
+ scope
163
+ end
164
+
165
+ def ar_scope_skills(model, user_id, has_tokens:, has_name_det:)
166
+ scope = model.all
167
+ scope = scope.where(user_id: user_id) if user_id
168
+ return scope if @force
169
+
170
+ if has_tokens && has_name_det
171
+ scope.where("search_tokens IS NULL OR name_det IS NULL")
172
+ elsif has_tokens
173
+ scope.where(search_tokens: nil)
174
+ elsif has_name_det
175
+ scope.where(name_det: nil)
176
+ else
177
+ scope
178
+ end
179
+ end
180
+
181
+ # --- Postgres ---
182
+
183
+ def backfill_pg_items(user_id)
184
+ return 0 unless pg_column_exists?("llmemory_items", "search_tokens")
185
+
186
+ count = 0
187
+ pg_each_row("llmemory_items", "id, user_id, content", user_id) do |row|
188
+ tokens = search_tokens_for(dec(row["content"]))
189
+ next if tokens.nil?
190
+
191
+ pg_update("llmemory_items", row["id"], search_tokens: tokens) unless @dry_run
192
+ count += 1
193
+ end
194
+ count
195
+ end
196
+
197
+ def backfill_pg_resources(user_id)
198
+ return 0 unless pg_column_exists?("llmemory_resources", "search_tokens")
199
+
200
+ count = 0
201
+ pg_each_row("llmemory_resources", "id, user_id, text", user_id) do |row|
202
+ tokens = search_tokens_for(dec(row["text"]))
203
+ next if tokens.nil?
204
+
205
+ pg_update("llmemory_resources", row["id"], search_tokens: tokens) unless @dry_run
206
+ count += 1
207
+ end
208
+ count
209
+ end
210
+
211
+ def backfill_pg_episodes(user_id)
212
+ return 0 unless pg_column_exists?("llmemory_episodes", "search_tokens")
213
+
214
+ count = 0
215
+ pg_each_row("llmemory_episodes", "id, user_id, search_text", user_id) do |row|
216
+ tokens = search_tokens_for(dec(row["search_text"].to_s))
217
+ next if tokens.nil?
218
+
219
+ pg_update("llmemory_episodes", row["id"], search_tokens: tokens) unless @dry_run
220
+ count += 1
221
+ end
222
+ count
223
+ end
224
+
225
+ def backfill_pg_skills(user_id)
226
+ has_tokens = pg_column_exists?("llmemory_skills", "search_tokens")
227
+ has_name_det = pg_column_exists?("llmemory_skills", "name_det")
228
+ return 0 unless has_tokens || has_name_det
229
+
230
+ count = 0
231
+ sql = "SELECT id, user_id, data, search_text FROM llmemory_skills"
232
+ params = []
233
+ clauses = []
234
+ if user_id
235
+ params << user_id
236
+ clauses << "user_id = $#{params.size}"
237
+ end
238
+ unless @force
239
+ parts = []
240
+ parts << "search_tokens IS NULL" if has_tokens
241
+ parts << "name_det IS NULL" if has_name_det
242
+ clauses << "(#{parts.join(' OR ')})" if parts.any?
243
+ end
244
+ sql += " WHERE #{clauses.join(' AND ')}" unless clauses.empty?
245
+
246
+ pg_conn.exec_params(sql, params).each do |row|
247
+ sets = {}
248
+ if has_tokens
249
+ tokens = search_tokens_for(dec(row["search_text"].to_s))
250
+ sets["search_tokens"] = tokens if tokens
251
+ end
252
+ if has_name_det
253
+ data = decode_json_document(row["data"])
254
+ name = (data[:name] || data["name"]).to_s
255
+ sets["name_det"] = enc_det(name) unless name.empty?
256
+ end
257
+ next if sets.empty?
258
+
259
+ pg_update_columns("llmemory_skills", row["id"], sets) unless @dry_run
260
+ count += 1
261
+ end
262
+ count
263
+ end
264
+
265
+ def pg_each_row(table, columns, user_id)
266
+ sql = "SELECT #{columns} FROM #{table}"
267
+ params = []
268
+ clauses = []
269
+ if user_id
270
+ params << user_id
271
+ clauses << "user_id = $#{params.size}"
272
+ end
273
+ unless @force
274
+ clauses << "search_tokens IS NULL" if pg_column_exists?(table, "search_tokens")
275
+ end
276
+ sql += " WHERE #{clauses.join(' AND ')}" unless clauses.empty?
277
+
278
+ pg_conn.exec_params(sql, params).each { |row| yield row }
279
+ end
280
+
281
+ def pg_update(table, id, search_tokens:)
282
+ pg_conn.exec_params(
283
+ "UPDATE #{table} SET search_tokens = $1 WHERE id = $2",
284
+ [search_tokens, id]
285
+ )
286
+ end
287
+
288
+ def pg_update_columns(table, id, sets)
289
+ return if sets.empty?
290
+
291
+ idx = 1
292
+ params = []
293
+ assignments = sets.each_key.map do |col|
294
+ params << sets[col]
295
+ clause = "#{col} = $#{idx}"
296
+ idx += 1
297
+ clause
298
+ end
299
+ params << id
300
+ pg_conn.exec_params(
301
+ "UPDATE #{table} SET #{assignments.join(', ')} WHERE id = $#{idx}",
302
+ params
303
+ )
304
+ end
305
+
306
+ def pg_column_exists?(table, column)
307
+ @pg_columns ||= {}
308
+ cache_key = "#{table}.#{column}"
309
+ return @pg_columns[cache_key] if @pg_columns.key?(cache_key)
310
+
311
+ rows = pg_conn.exec_params(
312
+ "SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = $2 LIMIT 1",
313
+ [table, column]
314
+ )
315
+ @pg_columns[cache_key] = rows.any?
316
+ end
317
+
318
+ def pg_conn
319
+ @pg_conn ||= begin
320
+ require "pg"
321
+ PG.connect(Llmemory.configuration.database_url)
322
+ end
323
+ end
324
+
325
+ def decode_json_document(raw)
326
+ return raw.transform_keys(&:to_sym) if raw.is_a?(Hash)
327
+
328
+ str = raw.to_s
329
+ if cipher.encrypted?(str)
330
+ dec_json(str)
331
+ else
332
+ JSON.parse(str, symbolize_names: true)
333
+ end
334
+ rescue JSON::ParserError
335
+ {}
336
+ end
337
+ end
338
+ end
339
+ end
@@ -3,6 +3,7 @@
3
3
  require_relative "maintenance/runner"
4
4
  require_relative "maintenance/ttl_expiry"
5
5
  require_relative "maintenance/cognitive_pass"
6
+ require_relative "maintenance/search_tokens_backfill"
6
7
 
7
8
  module Llmemory
8
9
  module Maintenance
@@ -55,6 +55,8 @@ module Llmemory
55
55
  private
56
56
 
57
57
  def fetch_timeline_context(user_id, query, window)
58
+ return "" if Llmemory.configuration.long_term_type.to_s == "graph_based"
59
+
58
60
  storage = build_storage
59
61
  items = storage.search_items(user_id, query)
60
62
  return "" if items.empty?
@@ -52,6 +52,8 @@ module Llmemory
52
52
  stats[:long_term] = { error: e.message }
53
53
  end
54
54
 
55
+ stats[:llm_usage] = Llmemory::LLM::UsageLedger.new(store: store).totals(user_id)
56
+
55
57
  ::MCP::Tool::Response.new([{
56
58
  type: "text",
57
59
  text: format_stats(stats)
@@ -102,8 +104,19 @@ module Llmemory
102
104
  output << " Resources: #{stats[:long_term][:resources]}"
103
105
  end
104
106
 
107
+ output << ""
108
+ output << Llmemory::LLM::UsageLedger.format_text(stats[:llm_usage] || default_llm_usage)
109
+
105
110
  output.join("\n")
106
111
  end
112
+
113
+ def default_llm_usage
114
+ {
115
+ invoke: { input_tokens: 0, output_tokens: 0, total_tokens: 0, calls: 0 },
116
+ embed: { total_tokens: 0, calls: 0 },
117
+ updated_at: nil
118
+ }
119
+ end
107
120
  end
108
121
  end
109
122
  end
@@ -23,19 +23,10 @@ module Llmemory
23
23
  timeline = []
24
24
 
25
25
  # Get recent items from long-term memory
26
- storage = build_long_term_storage
27
- begin
28
- items = storage.get_items_since(user_id, hours: hours)
29
- items.each do |item|
30
- timeline << {
31
- type: "fact",
32
- timestamp: item[:created_at] || item["created_at"],
33
- category: item[:category] || item["category"],
34
- content: item[:content] || item["content"]
35
- }
36
- end
37
- rescue NotImplementedError
38
- # Some storages may not implement get_items_since
26
+ if graph_based?
27
+ append_graph_timeline(timeline, user_id, hours)
28
+ else
29
+ append_file_timeline(timeline, user_id, hours)
39
30
  end
40
31
 
41
32
  # Get recent messages from short-term
@@ -89,7 +80,48 @@ module Llmemory
89
80
  end
90
81
 
91
82
  def build_long_term_storage
92
- LongTerm::FileBased::Storages.build
83
+ if graph_based?
84
+ LongTerm::GraphBased::Storages.build
85
+ else
86
+ LongTerm::FileBased::Storages.build
87
+ end
88
+ end
89
+
90
+ def graph_based?
91
+ Llmemory.configuration.long_term_type.to_s == "graph_based"
92
+ end
93
+
94
+ def append_file_timeline(timeline, user_id, hours)
95
+ storage = build_long_term_storage
96
+ items = storage.get_items_since(user_id, hours: hours)
97
+ items.each do |item|
98
+ timeline << {
99
+ type: "fact",
100
+ timestamp: item[:created_at] || item["created_at"],
101
+ category: item[:category] || item["category"],
102
+ content: item[:content] || item["content"]
103
+ }
104
+ end
105
+ rescue NotImplementedError
106
+ # Some storages may not implement get_items_since
107
+ end
108
+
109
+ def append_graph_timeline(timeline, user_id, hours)
110
+ storage = build_long_term_storage
111
+ cutoff = Time.now - (hours * 3600)
112
+ storage.list_edges(user_id).each do |edge|
113
+ created_at = edge.respond_to?(:created_at) ? edge.created_at : edge[:created_at]
114
+ next unless created_at && created_at >= cutoff
115
+
116
+ subject = edge.respond_to?(:subject_id) ? edge.subject_id : edge[:subject_id]
117
+ predicate = edge.respond_to?(:predicate) ? edge.predicate : edge[:predicate]
118
+ target = edge.respond_to?(:target_id) ? edge.target_id : edge[:target_id]
119
+ timeline << {
120
+ type: "relation",
121
+ timestamp: created_at,
122
+ content: "#{subject} -> #{predicate} -> #{target}"
123
+ }
124
+ end
93
125
  end
94
126
 
95
127
  def format_timeline(timeline, hours)
@@ -101,6 +133,8 @@ module Llmemory
101
133
  when "fact"
102
134
  cat_info = entry[:category] ? "[#{entry[:category]}]" : ""
103
135
  output << "- [FACT] #{cat_info} #{truncate(entry[:content], 150)}"
136
+ when "relation"
137
+ output << "- [REL] #{truncate(entry[:content], 150)}"
104
138
  when "message"
105
139
  output << "- [MSG #{entry[:session_id]}] [#{entry[:role]}] #{truncate(entry[:content], 150)}"
106
140
  end