llmemory 0.2.10 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +48 -0
- data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb +71 -0
- data/lib/generators/llmemory/upgrade_zero_mem/templates/create_llmemory_zero_mem_tables.rb +77 -0
- data/lib/generators/llmemory/upgrade_zero_mem/upgrade_zero_mem_generator.rb +19 -0
- data/lib/llmemory/cli/commands/zero_mem.rb +107 -0
- data/lib/llmemory/cli.rb +3 -0
- data/lib/llmemory/configuration.rb +67 -1
- data/lib/llmemory/consolidation/filter.rb +74 -0
- data/lib/llmemory/consolidation/policy.rb +17 -0
- data/lib/llmemory/consolidation/rule_based_policy.rb +63 -0
- data/lib/llmemory/consolidation/unit.rb +54 -0
- data/lib/llmemory/consolidation.rb +11 -0
- data/lib/llmemory/instrumentation.rb +10 -1
- data/lib/llmemory/long_term/file_based/memory.rb +18 -7
- data/lib/llmemory/long_term/graph_based/memory.rb +31 -8
- data/lib/llmemory/maintenance/cognitive_pass.rb +29 -1
- data/lib/llmemory/maintenance/policy_prune.rb +85 -0
- data/lib/llmemory/maintenance/zero_mem_ttl.rb +26 -0
- data/lib/llmemory/maintenance.rb +2 -0
- data/lib/llmemory/mcp/server.rb +2 -0
- data/lib/llmemory/mcp/store_helpers.rb +16 -0
- data/lib/llmemory/mcp/tools/memory_add_message.rb +17 -4
- data/lib/llmemory/mcp/tools/memory_consolidate.rb +9 -4
- data/lib/llmemory/mcp/tools/memory_maintain.rb +6 -1
- data/lib/llmemory/mcp/tools/memory_retrieve.rb +3 -1
- data/lib/llmemory/mcp/tools/memory_retrieve_evidence.rb +70 -0
- data/lib/llmemory/mcp/tools/memory_stats.rb +22 -0
- data/lib/llmemory/memory.rb +257 -9
- data/lib/llmemory/retrieval/bm25_scorer.rb +5 -0
- data/lib/llmemory/retrieval/context_assembler.rb +17 -1
- data/lib/llmemory/tokenizer.rb +18 -8
- data/lib/llmemory/vector_store/memory_store.rb +23 -1
- data/lib/llmemory/vector_store/openai_embeddings.rb +47 -21
- data/lib/llmemory/version.rb +1 -1
- data/lib/llmemory/zero_mem/answer_calibrator.rb +63 -0
- data/lib/llmemory/zero_mem/backfill.rb +43 -0
- data/lib/llmemory/zero_mem/budget_assembler.rb +62 -0
- data/lib/llmemory/zero_mem/circuit_breaker.rb +37 -0
- data/lib/llmemory/zero_mem/embedding_provider.rb +73 -0
- data/lib/llmemory/zero_mem/engine.rb +247 -0
- data/lib/llmemory/zero_mem/entity.rb +31 -0
- data/lib/llmemory/zero_mem/entity_extractor.rb +23 -0
- data/lib/llmemory/zero_mem/entity_index.rb +65 -0
- data/lib/llmemory/zero_mem/entity_mention.rb +35 -0
- data/lib/llmemory/zero_mem/errors.rb +7 -0
- data/lib/llmemory/zero_mem/evidence.rb +43 -0
- data/lib/llmemory/zero_mem/evidence_calibrator.rb +158 -0
- data/lib/llmemory/zero_mem/evidence_closure.rb +98 -0
- data/lib/llmemory/zero_mem/evidence_fusion.rb +57 -0
- data/lib/llmemory/zero_mem/evidence_set.rb +54 -0
- data/lib/llmemory/zero_mem/extractors/heuristic.rb +44 -0
- data/lib/llmemory/zero_mem/extractors/http.rb +58 -0
- data/lib/llmemory/zero_mem/graph_retriever.rb +107 -0
- data/lib/llmemory/zero_mem/hierarchy_builder.rb +123 -0
- data/lib/llmemory/zero_mem/hierarchy_retriever.rb +133 -0
- data/lib/llmemory/zero_mem/indexer.rb +73 -0
- data/lib/llmemory/zero_mem/mode.rb +30 -0
- data/lib/llmemory/zero_mem/pagerank.rb +42 -0
- data/lib/llmemory/zero_mem/query_profile.rb +48 -0
- data/lib/llmemory/zero_mem/query_profiler.rb +138 -0
- data/lib/llmemory/zero_mem/repair.rb +31 -0
- data/lib/llmemory/zero_mem/router.rb +69 -0
- data/lib/llmemory/zero_mem/sidecar_client.rb +82 -0
- data/lib/llmemory/zero_mem/storage.rb +124 -0
- data/lib/llmemory/zero_mem/storages/active_record.rb +311 -0
- data/lib/llmemory/zero_mem/storages/active_record_models.rb +21 -0
- data/lib/llmemory/zero_mem/storages/file.rb +67 -0
- data/lib/llmemory/zero_mem/storages/memory.rb +294 -0
- data/lib/llmemory/zero_mem/storages/postgres.rb +85 -0
- data/lib/llmemory/zero_mem/storages/snapshot_codec.rb +180 -0
- data/lib/llmemory/zero_mem/storages.rb +43 -0
- data/lib/llmemory/zero_mem/trace.rb +100 -0
- data/lib/llmemory/zero_mem/trace_state_link.rb +64 -0
- data/lib/llmemory/zero_mem/trace_unit.rb +83 -0
- data/lib/llmemory/zero_mem.rb +42 -0
- data/lib/llmemory.rb +8 -0
- data/lib/tasks/llmemory.rake +84 -2
- metadata +54 -2
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../storage"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module ZeroMem
|
|
7
|
+
module Storages
|
|
8
|
+
class Memory
|
|
9
|
+
include Storage
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
@traces = {}
|
|
13
|
+
@idempotency = {}
|
|
14
|
+
@sequences = Hash.new(0)
|
|
15
|
+
@state_links = []
|
|
16
|
+
@watermarks = {}
|
|
17
|
+
@units = {}
|
|
18
|
+
@open_episodes = {}
|
|
19
|
+
@entities = {}
|
|
20
|
+
@mentions = []
|
|
21
|
+
@entity_traces = Hash.new { |h, k| h[k] = [] }
|
|
22
|
+
@trace_entity_keys = Hash.new { |h, k| h[k] = [] }
|
|
23
|
+
@adjacency = {}
|
|
24
|
+
@turn_embeddings = {}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def write_trace(trace)
|
|
28
|
+
t = trace.is_a?(Trace) ? trace : Trace.new(**trace)
|
|
29
|
+
key = idempotency_key(t.user_id, t.idempotency_key)
|
|
30
|
+
if key && @idempotency[key]
|
|
31
|
+
return @idempotency[key]
|
|
32
|
+
end
|
|
33
|
+
if t.idempotency_key && (existing = find_by_idempotency_key(t.user_id, t.idempotency_key))
|
|
34
|
+
return existing.id
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
@traces[t.id] = t
|
|
38
|
+
@idempotency[key] = t.id if key
|
|
39
|
+
@sequences[[t.user_id, t.session_id]] = t.sequence if t.sequence > @sequences[[t.user_id, t.session_id]]
|
|
40
|
+
t.id
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def get_trace(user_id, trace_id)
|
|
44
|
+
t = @traces[trace_id.to_s]
|
|
45
|
+
return nil unless t
|
|
46
|
+
return nil unless t.user_id == user_id.to_s
|
|
47
|
+
|
|
48
|
+
t
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def find_by_idempotency_key(user_id, idempotency_key)
|
|
52
|
+
id = @idempotency[idempotency_key(user_id, idempotency_key)]
|
|
53
|
+
id ? get_trace(user_id, id) : nil
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def next_sequence(user_id, session_id)
|
|
57
|
+
@sequences[[user_id.to_s, session_id.to_s]] + 1
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def list_traces(user_id, session_id: nil, include_archived: false, limit: nil, offset: nil)
|
|
61
|
+
uid = user_id.to_s
|
|
62
|
+
list = @traces.values.select do |t|
|
|
63
|
+
next false unless t.user_id == uid
|
|
64
|
+
next false if session_id && t.session_id != session_id.to_s
|
|
65
|
+
next false if !include_archived && !t.active?
|
|
66
|
+
|
|
67
|
+
true
|
|
68
|
+
end
|
|
69
|
+
list.sort_by! { |t| [t.session_id, t.sequence, t.ingested_at.to_f] }
|
|
70
|
+
list = list.drop(offset.to_i) if offset
|
|
71
|
+
list = list.first(limit.to_i) if limit && limit.to_i.positive?
|
|
72
|
+
list
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def archive_trace(user_id, trace_id, archived_at: Time.now)
|
|
76
|
+
t = get_trace(user_id, trace_id)
|
|
77
|
+
return false unless t
|
|
78
|
+
|
|
79
|
+
@traces[t.id] = t.with(archived_at: archived_at)
|
|
80
|
+
cascade_archive_derivatives(user_id, t.id)
|
|
81
|
+
true
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def write_state_link(link)
|
|
85
|
+
@state_links << link
|
|
86
|
+
link.trace_id
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def close_open_state_links(user_id, state_key, valid_to:, except_trace_id: nil)
|
|
90
|
+
uid = user_id.to_s
|
|
91
|
+
sk = state_key.to_s
|
|
92
|
+
closed = 0
|
|
93
|
+
@state_links.each_with_index do |link, idx|
|
|
94
|
+
next unless link.user_id == uid && link.state_key == sk
|
|
95
|
+
next unless link.valid_to.nil?
|
|
96
|
+
next if except_trace_id && link.trace_id == except_trace_id.to_s
|
|
97
|
+
|
|
98
|
+
@state_links[idx] = TraceStateLink.new(
|
|
99
|
+
user_id: link.user_id,
|
|
100
|
+
state_key: link.state_key,
|
|
101
|
+
trace_id: link.trace_id,
|
|
102
|
+
valid_from: link.valid_from,
|
|
103
|
+
valid_to: valid_to,
|
|
104
|
+
supersedes_trace_id: link.supersedes_trace_id,
|
|
105
|
+
source: link.source,
|
|
106
|
+
created_at: link.created_at
|
|
107
|
+
)
|
|
108
|
+
closed += 1
|
|
109
|
+
end
|
|
110
|
+
closed
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def state_links_for(user_id, state_key)
|
|
114
|
+
@state_links.select { |l| l.user_id == user_id.to_s && l.state_key == state_key.to_s }
|
|
115
|
+
.sort_by { |l| [l.valid_from.to_f, l.created_at.to_f] }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def current_state_trace_id(user_id, state_key, as_of: Time.now)
|
|
119
|
+
state_links_for(user_id, state_key).reverse.find { |l| l.active_at?(as_of) }&.trace_id
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def get_watermark(user_id, session_id)
|
|
123
|
+
@watermarks[[user_id.to_s, session_id.to_s]] || { last_sequence: 0, index_version: 0 }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def set_watermark(user_id, session_id, last_sequence:, index_version:)
|
|
127
|
+
@watermarks[[user_id.to_s, session_id.to_s]] = {
|
|
128
|
+
last_sequence: last_sequence.to_i,
|
|
129
|
+
index_version: index_version.to_i
|
|
130
|
+
}
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def write_unit(unit)
|
|
134
|
+
u = unit.is_a?(TraceUnit) ? unit : TraceUnit.new(**unit)
|
|
135
|
+
@units[u.id] = u
|
|
136
|
+
u.id
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def list_units(user_id, kind: nil, session_id: nil, boundary_id: nil)
|
|
140
|
+
uid = user_id.to_s
|
|
141
|
+
@units.values.select do |u|
|
|
142
|
+
next false unless u.user_id == uid
|
|
143
|
+
next false if kind && u.kind != kind.to_sym
|
|
144
|
+
next false if session_id && u.session_id != session_id.to_s
|
|
145
|
+
next false if boundary_id && u.boundary_id != boundary_id.to_s
|
|
146
|
+
next false if unit_references_archived_trace?(u)
|
|
147
|
+
|
|
148
|
+
true
|
|
149
|
+
end.sort_by { |u| [u.kind.to_s, u.start_sequence, u.end_sequence] }
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def get_unit(user_id, unit_id)
|
|
153
|
+
u = @units[unit_id.to_s]
|
|
154
|
+
return nil unless u
|
|
155
|
+
return nil unless u.user_id == user_id.to_s
|
|
156
|
+
|
|
157
|
+
u
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def open_episode(user_id, episode_key)
|
|
161
|
+
id = @open_episodes[[user_id.to_s, episode_key.to_s]]
|
|
162
|
+
id ? get_unit(user_id, id) : nil
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def set_open_episode(user_id, episode_key, unit_id)
|
|
166
|
+
@open_episodes[[user_id.to_s, episode_key.to_s]] = unit_id.to_s
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def close_episode(unit_id)
|
|
170
|
+
@open_episodes.delete_if { |_, v| v == unit_id.to_s }
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def upsert_entity(entity)
|
|
174
|
+
ent = entity.is_a?(Entity) ? entity : Entity.new(**entity)
|
|
175
|
+
key = [ent.user_id, ent.normalized_key]
|
|
176
|
+
@entities[key] = ent
|
|
177
|
+
ent.id
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def write_mention(mention)
|
|
181
|
+
m = mention.is_a?(EntityMention) ? mention : EntityMention.new(**mention)
|
|
182
|
+
@mentions << m
|
|
183
|
+
@entity_traces[[m.user_id, m.normalized_key]] << m.trace_id unless @entity_traces[[m.user_id, m.normalized_key]].include?(m.trace_id)
|
|
184
|
+
@trace_entity_keys[[m.user_id, m.trace_id]] << m.normalized_key unless @trace_entity_keys[[m.user_id, m.trace_id]].include?(m.normalized_key)
|
|
185
|
+
m.id
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def entity_exists?(user_id, normalized_key)
|
|
189
|
+
@entities.key?([user_id.to_s, normalized_key.to_s])
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def trace_ids_for_entity_key(user_id, normalized_key)
|
|
193
|
+
@entity_traces[[user_id.to_s, normalized_key.to_s]].select do |tid|
|
|
194
|
+
active_trace?(user_id, tid)
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def entity_keys_for_trace(user_id, trace_id)
|
|
199
|
+
@trace_entity_keys[[user_id.to_s, trace_id.to_s]].dup
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def previous_trace(user_id, session_id, sequence)
|
|
203
|
+
list_traces(user_id, session_id: session_id).reverse.find { |t| t.sequence < sequence.to_i }
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def link_adjacent_traces(user_id, prev_trace_id, trace_id)
|
|
207
|
+
uid = user_id.to_s
|
|
208
|
+
@adjacency[prev_trace_id.to_s] ||= {}
|
|
209
|
+
@adjacency[prev_trace_id.to_s][:next] = trace_id.to_s
|
|
210
|
+
@adjacency[trace_id.to_s] ||= {}
|
|
211
|
+
@adjacency[trace_id.to_s][:prev] = prev_trace_id.to_s
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def adjacent_traces(user_id, trace_id)
|
|
215
|
+
_uid = user_id.to_s
|
|
216
|
+
adj = @adjacency[trace_id.to_s] || {}
|
|
217
|
+
[adj[:prev], adj[:next]]
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def all_entity_keys(user_id)
|
|
221
|
+
uid = user_id.to_s
|
|
222
|
+
@entities.select { |(u, _), _| u == uid }.map { |(_, key), _| key }
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def store_turn_embedding(user_id, trace_id, vector, model:, dimensions:)
|
|
226
|
+
@turn_embeddings[[user_id.to_s, trace_id.to_s]] = {
|
|
227
|
+
vector: vector,
|
|
228
|
+
model: model,
|
|
229
|
+
dimensions: dimensions
|
|
230
|
+
}
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def search_traces_by_tokens(user_id, query, limit: 20)
|
|
234
|
+
tokens = Llmemory::Tokenizer.tokenize(query)
|
|
235
|
+
return [] if tokens.empty?
|
|
236
|
+
|
|
237
|
+
list = list_traces(user_id)
|
|
238
|
+
matched = list.select do |t|
|
|
239
|
+
down = t.content.downcase
|
|
240
|
+
tokens.all? { |tok| down.include?(tok) }
|
|
241
|
+
end
|
|
242
|
+
limit.to_i.positive? ? matched.first(limit.to_i) : matched
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def reset_state!
|
|
246
|
+
@traces = {}
|
|
247
|
+
@idempotency = {}
|
|
248
|
+
@sequences = Hash.new(0)
|
|
249
|
+
@state_links = []
|
|
250
|
+
@watermarks = {}
|
|
251
|
+
@units = {}
|
|
252
|
+
@open_episodes = {}
|
|
253
|
+
@entities = {}
|
|
254
|
+
@mentions = []
|
|
255
|
+
@entity_traces = Hash.new { |h, k| h[k] = [] }
|
|
256
|
+
@trace_entity_keys = Hash.new { |h, k| h[k] = [] }
|
|
257
|
+
@adjacency = {}
|
|
258
|
+
@turn_embeddings = {}
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
private
|
|
262
|
+
|
|
263
|
+
def cascade_archive_derivatives(user_id, trace_id)
|
|
264
|
+
uid = user_id.to_s
|
|
265
|
+
tid = trace_id.to_s
|
|
266
|
+
@mentions.reject! { |m| m.user_id == uid && m.trace_id == tid }
|
|
267
|
+
@trace_entity_keys.delete([uid, tid])
|
|
268
|
+
@entity_traces.each_value { |ids| ids.delete(tid) }
|
|
269
|
+
@turn_embeddings.delete([uid, tid])
|
|
270
|
+
@adjacency.delete(tid)
|
|
271
|
+
@adjacency.each_value do |adj|
|
|
272
|
+
adj.delete(:prev) if adj[:prev] == tid
|
|
273
|
+
adj.delete(:next) if adj[:next] == tid
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def unit_references_archived_trace?(unit)
|
|
278
|
+
unit.member_trace_ids.any? { |tid| !active_trace?(unit.user_id, tid) }
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def active_trace?(user_id, trace_id)
|
|
282
|
+
t = get_trace(user_id, trace_id)
|
|
283
|
+
t&.active?
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def idempotency_key(user_id, key)
|
|
287
|
+
return nil if key.nil? || key.to_s.empty?
|
|
288
|
+
|
|
289
|
+
[user_id.to_s, key.to_s]
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../storage"
|
|
5
|
+
require_relative "memory"
|
|
6
|
+
require_relative "snapshot_codec"
|
|
7
|
+
require_relative "../../crypto/field_helpers"
|
|
8
|
+
|
|
9
|
+
module Llmemory
|
|
10
|
+
module ZeroMem
|
|
11
|
+
module Storages
|
|
12
|
+
class PostgresStorage
|
|
13
|
+
include Storage
|
|
14
|
+
include Llmemory::Crypto::FieldHelpers
|
|
15
|
+
|
|
16
|
+
SNAPSHOT_ID = "default"
|
|
17
|
+
MUTATORS = Storages::FileStorage::MUTATORS
|
|
18
|
+
|
|
19
|
+
def initialize(database_url: nil, cipher: nil)
|
|
20
|
+
@database_url = database_url || Llmemory.configuration.database_url || ENV["DATABASE_URL"]
|
|
21
|
+
raise ConfigurationError, "DATABASE_URL required for zero_mem postgres store" if @database_url.to_s.empty?
|
|
22
|
+
|
|
23
|
+
require "pg"
|
|
24
|
+
@conn = PG.connect(@database_url)
|
|
25
|
+
@cipher = cipher || Llmemory.build_cipher
|
|
26
|
+
@inner = Memory.new
|
|
27
|
+
ensure_tables!
|
|
28
|
+
load!
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
Storage.instance_methods(false).each do |method_name|
|
|
32
|
+
next if MUTATORS.include?(method_name)
|
|
33
|
+
|
|
34
|
+
define_method(method_name) do |*args, **kwargs, &block|
|
|
35
|
+
@inner.public_send(method_name, *args, **kwargs, &block)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
MUTATORS.each do |method_name|
|
|
40
|
+
define_method(method_name) do |*args, **kwargs, &block|
|
|
41
|
+
result = @inner.public_send(method_name, *args, **kwargs, &block)
|
|
42
|
+
save!
|
|
43
|
+
result
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def ensure_tables!
|
|
50
|
+
@conn.exec(<<~SQL)
|
|
51
|
+
CREATE TABLE IF NOT EXISTS llmemory_zero_mem_snapshots (
|
|
52
|
+
id TEXT PRIMARY KEY,
|
|
53
|
+
payload TEXT NOT NULL,
|
|
54
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
55
|
+
);
|
|
56
|
+
SQL
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def load!
|
|
60
|
+
result = @conn.exec_params(
|
|
61
|
+
"SELECT payload FROM llmemory_zero_mem_snapshots WHERE id = $1",
|
|
62
|
+
[SNAPSHOT_ID]
|
|
63
|
+
)
|
|
64
|
+
return if result.ntuples.zero?
|
|
65
|
+
|
|
66
|
+
raw = dec(result.first["payload"])
|
|
67
|
+
data = JSON.parse(raw, symbolize_names: true)
|
|
68
|
+
SnapshotCodec.load!(@inner, data)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def save!
|
|
72
|
+
payload = enc(JSON.generate(SnapshotCodec.dump(@inner)))
|
|
73
|
+
@conn.exec_params(
|
|
74
|
+
<<~SQL,
|
|
75
|
+
INSERT INTO llmemory_zero_mem_snapshots (id, payload, updated_at)
|
|
76
|
+
VALUES ($1, $2, NOW())
|
|
77
|
+
ON CONFLICT (id) DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW()
|
|
78
|
+
SQL
|
|
79
|
+
[SNAPSHOT_ID, payload]
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Llmemory
|
|
4
|
+
module ZeroMem
|
|
5
|
+
module Storages
|
|
6
|
+
module SnapshotCodec
|
|
7
|
+
VERSION = 1
|
|
8
|
+
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def dump(store)
|
|
12
|
+
{
|
|
13
|
+
version: VERSION,
|
|
14
|
+
traces: store.instance_variable_get(:@traces).transform_values(&:to_h),
|
|
15
|
+
idempotency: encode_tuple_keys(store.instance_variable_get(:@idempotency)),
|
|
16
|
+
sequences: encode_tuple_keys(store.instance_variable_get(:@sequences)),
|
|
17
|
+
state_links: store.instance_variable_get(:@state_links).map(&:to_h),
|
|
18
|
+
watermarks: encode_tuple_keys(store.instance_variable_get(:@watermarks)),
|
|
19
|
+
units: store.instance_variable_get(:@units).transform_values { |u| unit_to_h(u) },
|
|
20
|
+
open_episodes: encode_tuple_keys(store.instance_variable_get(:@open_episodes)),
|
|
21
|
+
entities: encode_tuple_keys(
|
|
22
|
+
store.instance_variable_get(:@entities).transform_values { |e| entity_to_h(e) }
|
|
23
|
+
),
|
|
24
|
+
mentions: store.instance_variable_get(:@mentions).map { |m| mention_to_h(m) },
|
|
25
|
+
entity_traces: encode_tuple_keys(store.instance_variable_get(:@entity_traces)),
|
|
26
|
+
trace_entity_keys: encode_tuple_keys(store.instance_variable_get(:@trace_entity_keys)),
|
|
27
|
+
adjacency: store.instance_variable_get(:@adjacency),
|
|
28
|
+
turn_embeddings: encode_tuple_keys(store.instance_variable_get(:@turn_embeddings))
|
|
29
|
+
}
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def encode_tuple_keys(hash)
|
|
33
|
+
hash.each_with_object({}) do |(key, value), acc|
|
|
34
|
+
encoded = key.is_a?(Array) ? key.join("|") : key.to_s
|
|
35
|
+
acc[encoded] = value
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def decode_tuple_keys(hash, default_factory: nil)
|
|
40
|
+
out = default_factory ? Hash.new { |h, k| h[k] = default_factory.call } : {}
|
|
41
|
+
hash.each do |key, value|
|
|
42
|
+
parts = key.to_s.split("|", 2)
|
|
43
|
+
out_key = parts.size == 2 ? [parts[0], parts[1]] : key
|
|
44
|
+
out[out_key] = value
|
|
45
|
+
end
|
|
46
|
+
out
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def load!(store, data)
|
|
50
|
+
store.send(:reset_state!)
|
|
51
|
+
data = data.transform_keys(&:to_sym)
|
|
52
|
+
traces_h = data[:traces]
|
|
53
|
+
trace_rows = traces_h.is_a?(Hash) ? traces_h.values : Array(traces_h)
|
|
54
|
+
trace_rows.each do |h|
|
|
55
|
+
trace = Trace.new(**symbolize_times(h))
|
|
56
|
+
store.instance_variable_get(:@traces)[trace.id] = trace
|
|
57
|
+
end
|
|
58
|
+
store.instance_variable_set(:@idempotency, decode_tuple_keys(data[:idempotency] || {}))
|
|
59
|
+
sequences = decode_tuple_keys(data[:sequences] || {})
|
|
60
|
+
sequences.default = 0
|
|
61
|
+
store.instance_variable_set(:@sequences, sequences)
|
|
62
|
+
store.instance_variable_set(:@state_links, Array(data[:state_links]).map { |h| TraceStateLink.new(**symbolize_times(h)) })
|
|
63
|
+
store.instance_variable_set(:@watermarks, decode_tuple_keys(data[:watermarks] || {}))
|
|
64
|
+
units_h = data[:units]
|
|
65
|
+
unit_rows = units_h.is_a?(Hash) ? units_h.values : Array(units_h)
|
|
66
|
+
unit_rows.each do |h|
|
|
67
|
+
unit = unit_from_h(h)
|
|
68
|
+
store.instance_variable_get(:@units)[unit.id] = unit
|
|
69
|
+
end
|
|
70
|
+
store.instance_variable_set(:@open_episodes, decode_tuple_keys(data[:open_episodes] || {}))
|
|
71
|
+
decode_tuple_keys(data[:entities] || {}).each_value do |h|
|
|
72
|
+
ent = entity_from_h(h)
|
|
73
|
+
store.instance_variable_get(:@entities)[[ent.user_id, ent.normalized_key]] = ent
|
|
74
|
+
end
|
|
75
|
+
store.instance_variable_set(:@mentions, Array(data[:mentions]).map { |h| mention_from_h(h) })
|
|
76
|
+
store.instance_variable_set(
|
|
77
|
+
:@entity_traces,
|
|
78
|
+
decode_tuple_keys(data[:entity_traces] || {}, default_factory: -> { [] })
|
|
79
|
+
)
|
|
80
|
+
store.instance_variable_get(:@entity_traces).default_proc = proc { |h, k| h[k] = [] }
|
|
81
|
+
store.instance_variable_set(
|
|
82
|
+
:@trace_entity_keys,
|
|
83
|
+
decode_tuple_keys(data[:trace_entity_keys] || {}, default_factory: -> { [] })
|
|
84
|
+
)
|
|
85
|
+
store.instance_variable_get(:@trace_entity_keys).default_proc = proc { |h, k| h[k] = [] }
|
|
86
|
+
store.instance_variable_set(:@adjacency, data[:adjacency] || {})
|
|
87
|
+
store.instance_variable_set(:@turn_embeddings, decode_tuple_keys(data[:turn_embeddings] || {}))
|
|
88
|
+
store
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def unit_to_h(unit)
|
|
92
|
+
{
|
|
93
|
+
id: unit.id,
|
|
94
|
+
user_id: unit.user_id,
|
|
95
|
+
kind: unit.kind,
|
|
96
|
+
session_id: unit.session_id,
|
|
97
|
+
boundary_id: unit.boundary_id,
|
|
98
|
+
start_sequence: unit.start_sequence,
|
|
99
|
+
end_sequence: unit.end_sequence,
|
|
100
|
+
occurred_from: unit.occurred_from.utc.iso8601(6),
|
|
101
|
+
occurred_to: unit.occurred_to.utc.iso8601(6),
|
|
102
|
+
member_trace_ids: unit.member_trace_ids,
|
|
103
|
+
embedding_ref: unit.embedding_ref,
|
|
104
|
+
index_version: unit.index_version
|
|
105
|
+
}
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def unit_from_h(h)
|
|
109
|
+
h = symbolize_times(h)
|
|
110
|
+
TraceUnit.new(
|
|
111
|
+
id: h[:id],
|
|
112
|
+
user_id: h[:user_id],
|
|
113
|
+
kind: h[:kind].to_sym,
|
|
114
|
+
session_id: h[:session_id],
|
|
115
|
+
boundary_id: h[:boundary_id],
|
|
116
|
+
start_sequence: h[:start_sequence],
|
|
117
|
+
end_sequence: h[:end_sequence],
|
|
118
|
+
occurred_from: h[:occurred_from],
|
|
119
|
+
occurred_to: h[:occurred_to],
|
|
120
|
+
member_trace_ids: h[:member_trace_ids] || [],
|
|
121
|
+
embedding_ref: h[:embedding_ref],
|
|
122
|
+
index_version: h[:index_version] || 0
|
|
123
|
+
)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def entity_to_h(entity)
|
|
127
|
+
{
|
|
128
|
+
id: entity.id,
|
|
129
|
+
user_id: entity.user_id,
|
|
130
|
+
normalized_key: entity.normalized_key,
|
|
131
|
+
display_name: entity.display_name,
|
|
132
|
+
entity_type: entity.entity_type,
|
|
133
|
+
ambiguous: entity.ambiguous
|
|
134
|
+
}
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def entity_from_h(h)
|
|
138
|
+
h = h.transform_keys(&:to_sym)
|
|
139
|
+
Entity.new(
|
|
140
|
+
id: h[:id],
|
|
141
|
+
user_id: h[:user_id],
|
|
142
|
+
normalized_key: h[:normalized_key],
|
|
143
|
+
display_name: h[:display_name],
|
|
144
|
+
entity_type: (h[:entity_type] || :entity).to_sym,
|
|
145
|
+
ambiguous: h[:ambiguous] || false
|
|
146
|
+
)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def mention_to_h(mention)
|
|
150
|
+
{
|
|
151
|
+
id: mention.id,
|
|
152
|
+
user_id: mention.user_id,
|
|
153
|
+
trace_id: mention.trace_id,
|
|
154
|
+
entity_id: mention.entity_id,
|
|
155
|
+
normalized_key: mention.normalized_key,
|
|
156
|
+
text: mention.text,
|
|
157
|
+
offset_start: mention.offset_start,
|
|
158
|
+
offset_end: mention.offset_end
|
|
159
|
+
}
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def mention_from_h(h)
|
|
163
|
+
EntityMention.new(**h.transform_keys(&:to_sym))
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def symbolize_times(h)
|
|
167
|
+
out = h.transform_keys(&:to_sym)
|
|
168
|
+
%i[occurred_at ingested_at archived_at valid_from valid_to created_at occurred_from occurred_to].each do |key|
|
|
169
|
+
next unless out[key]
|
|
170
|
+
|
|
171
|
+
out[key] = Time.parse(out[key].to_s) unless out[key].is_a?(Time)
|
|
172
|
+
end
|
|
173
|
+
out[:role] = out[:role].to_sym if out[:role]
|
|
174
|
+
out[:source] = out[:source].to_sym if out[:source]
|
|
175
|
+
out
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "storages/memory"
|
|
4
|
+
|
|
5
|
+
module Llmemory
|
|
6
|
+
module ZeroMem
|
|
7
|
+
module Storages
|
|
8
|
+
def self.build(store: nil, cipher: nil)
|
|
9
|
+
resolved = store || resolve_store
|
|
10
|
+
case resolved.to_sym
|
|
11
|
+
when :memory
|
|
12
|
+
Memory.new
|
|
13
|
+
when :file
|
|
14
|
+
require_relative "storages/file"
|
|
15
|
+
FileStorage.new(cipher: cipher)
|
|
16
|
+
when :postgres
|
|
17
|
+
require_relative "storages/postgres"
|
|
18
|
+
PostgresStorage.new(cipher: cipher)
|
|
19
|
+
when :active_record
|
|
20
|
+
require_relative "storages/active_record"
|
|
21
|
+
ActiveRecordStorage.new(cipher: cipher)
|
|
22
|
+
else
|
|
23
|
+
raise Llmemory::ConfigurationError,
|
|
24
|
+
"Unsupported zero_mem trace store: #{resolved.inspect} " \
|
|
25
|
+
"(use :memory, :file, :postgres, or :active_record)"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.resolve_store
|
|
30
|
+
explicit = Llmemory.configuration.zero_mem_trace_store
|
|
31
|
+
return explicit if explicit && explicit.to_sym != :memory
|
|
32
|
+
|
|
33
|
+
case Llmemory.configuration.long_term_store.to_sym
|
|
34
|
+
when :file then :file
|
|
35
|
+
when :postgres then :postgres
|
|
36
|
+
when :active_record, :activerecord then :active_record
|
|
37
|
+
else
|
|
38
|
+
Llmemory.configuration.zero_mem_trace_store || :memory
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "time"
|
|
6
|
+
|
|
7
|
+
module Llmemory
|
|
8
|
+
module ZeroMem
|
|
9
|
+
class Trace
|
|
10
|
+
ATTRS = %i[
|
|
11
|
+
id user_id session_id boundary_id sequence role content occurred_at ingested_at
|
|
12
|
+
metadata content_sha256 idempotency_key archived_at
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
def self.build(user_id:, session_id:, role:, content:, sequence:, id: nil,
|
|
16
|
+
boundary_id: nil, occurred_at: nil, ingested_at: nil, metadata: nil,
|
|
17
|
+
idempotency_key: nil, archived_at: nil)
|
|
18
|
+
accepted = normalize_content!(content)
|
|
19
|
+
ingested = ingested_at || Time.now
|
|
20
|
+
occurred = occurred_at || ingested
|
|
21
|
+
meta = normalize_metadata(metadata)
|
|
22
|
+
|
|
23
|
+
new(
|
|
24
|
+
id: id || "tr_#{SecureRandom.hex(12)}",
|
|
25
|
+
user_id: user_id.to_s,
|
|
26
|
+
session_id: session_id.to_s,
|
|
27
|
+
boundary_id: boundary_id&.to_s,
|
|
28
|
+
sequence: sequence.to_i,
|
|
29
|
+
role: role.to_sym,
|
|
30
|
+
content: accepted,
|
|
31
|
+
occurred_at: coerce_time(occurred),
|
|
32
|
+
ingested_at: coerce_time(ingested),
|
|
33
|
+
metadata: meta,
|
|
34
|
+
content_sha256: Digest::SHA256.hexdigest(accepted),
|
|
35
|
+
idempotency_key: idempotency_key&.to_s,
|
|
36
|
+
archived_at: archived_at ? coerce_time(archived_at) : nil
|
|
37
|
+
)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.normalize_content!(content)
|
|
41
|
+
text = content.to_s
|
|
42
|
+
unless text.dup.force_encoding(Encoding::UTF_8).valid_encoding?
|
|
43
|
+
raise ValidationError, "trace content must be valid UTF-8"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
text = text.encode(Encoding::UTF_8)
|
|
47
|
+
max = Llmemory.configuration.max_message_chars.to_i
|
|
48
|
+
if max.positive? && text.length > max
|
|
49
|
+
raise ValidationError, "trace content exceeds max_message_chars (#{max})"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
if Llmemory.configuration.message_sanitizer_enabled
|
|
53
|
+
sanitizer = Llmemory::ShortTerm::MessageSanitizer.new
|
|
54
|
+
sanitized = sanitizer.sanitize!([{ role: :user, content: text }])
|
|
55
|
+
text = sanitized.first[:content].to_s
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
text
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def self.normalize_metadata(metadata)
|
|
62
|
+
return {} if metadata.nil?
|
|
63
|
+
raise ValidationError, "metadata must be a Hash" unless metadata.is_a?(Hash)
|
|
64
|
+
|
|
65
|
+
metadata.transform_keys(&:to_sym)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def self.coerce_time(value)
|
|
69
|
+
return value if value.is_a?(Time)
|
|
70
|
+
return Time.parse(value.to_s) if value
|
|
71
|
+
|
|
72
|
+
Time.now
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
attr_reader(*ATTRS)
|
|
76
|
+
|
|
77
|
+
def initialize(**attrs)
|
|
78
|
+
ATTRS.each do |key|
|
|
79
|
+
instance_variable_set(:"@#{key}", attrs[key])
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def to_h
|
|
84
|
+
ATTRS.each_with_object({}) do |key, acc|
|
|
85
|
+
val = public_send(key)
|
|
86
|
+
acc[key] = val.is_a?(Time) ? val.utc.iso8601(6) : val
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def active?
|
|
91
|
+
archived_at.nil?
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def with(**changes)
|
|
95
|
+
attrs = ATTRS.to_h { |key| [key, public_send(key)] }.merge(changes)
|
|
96
|
+
self.class.new(**attrs)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|