dspy 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/lib/dspy/code_act.rb +463 -0
- data/lib/dspy/instrumentation.rb +15 -0
- data/lib/dspy/lm/adapters/anthropic_adapter.rb +106 -0
- data/lib/dspy/lm.rb +21 -7
- data/lib/dspy/memory/embedding_engine.rb +68 -0
- data/lib/dspy/memory/in_memory_store.rb +216 -0
- data/lib/dspy/memory/local_embedding_engine.rb +241 -0
- data/lib/dspy/memory/memory_compactor.rb +299 -0
- data/lib/dspy/memory/memory_manager.rb +248 -0
- data/lib/dspy/memory/memory_record.rb +163 -0
- data/lib/dspy/memory/memory_store.rb +90 -0
- data/lib/dspy/memory.rb +30 -0
- data/lib/dspy/mixins/instrumentation_helpers.rb +3 -5
- data/lib/dspy/mixins/type_coercion.rb +3 -0
- data/lib/dspy/prompt.rb +48 -1
- data/lib/dspy/subscribers/logger_subscriber.rb +91 -1
- data/lib/dspy/tools/base.rb +1 -1
- data/lib/dspy/tools/memory_toolset.rb +117 -0
- data/lib/dspy/tools/text_processing_toolset.rb +186 -0
- data/lib/dspy/tools/toolset.rb +223 -0
- data/lib/dspy/tools.rb +1 -0
- data/lib/dspy/version.rb +1 -1
- data/lib/dspy.rb +2 -0
- metadata +28 -2
@@ -0,0 +1,299 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'sorbet-runtime'
|
4
|
+
require_relative '../instrumentation'
|
5
|
+
|
6
|
+
module DSPy
|
7
|
+
module Memory
|
8
|
+
# Simple memory compaction system with inline triggers
|
9
|
+
# Handles deduplication, relevance pruning, and conflict resolution
|
10
|
+
class MemoryCompactor
|
11
|
+
extend T::Sig
|
12
|
+
|
13
|
+
# Compaction thresholds
|
14
|
+
DEFAULT_MAX_MEMORIES = 1000
|
15
|
+
DEFAULT_MAX_AGE_DAYS = 90
|
16
|
+
DEFAULT_SIMILARITY_THRESHOLD = 0.95
|
17
|
+
DEFAULT_LOW_ACCESS_THRESHOLD = 0.1
|
18
|
+
|
19
|
+
sig { returns(Integer) }
|
20
|
+
attr_reader :max_memories
|
21
|
+
|
22
|
+
sig { returns(Integer) }
|
23
|
+
attr_reader :max_age_days
|
24
|
+
|
25
|
+
sig { returns(Float) }
|
26
|
+
attr_reader :similarity_threshold
|
27
|
+
|
28
|
+
sig { returns(Float) }
|
29
|
+
attr_reader :low_access_threshold
|
30
|
+
|
31
|
+
sig do
|
32
|
+
params(
|
33
|
+
max_memories: Integer,
|
34
|
+
max_age_days: Integer,
|
35
|
+
similarity_threshold: Float,
|
36
|
+
low_access_threshold: Float
|
37
|
+
).void
|
38
|
+
end
|
39
|
+
def initialize(
|
40
|
+
max_memories: DEFAULT_MAX_MEMORIES,
|
41
|
+
max_age_days: DEFAULT_MAX_AGE_DAYS,
|
42
|
+
similarity_threshold: DEFAULT_SIMILARITY_THRESHOLD,
|
43
|
+
low_access_threshold: DEFAULT_LOW_ACCESS_THRESHOLD
|
44
|
+
)
|
45
|
+
@max_memories = max_memories
|
46
|
+
@max_age_days = max_age_days
|
47
|
+
@similarity_threshold = similarity_threshold
|
48
|
+
@low_access_threshold = low_access_threshold
|
49
|
+
end
|
50
|
+
|
51
|
+
# Main compaction entry point - checks all triggers and compacts if needed
|
52
|
+
sig { params(store: MemoryStore, embedding_engine: EmbeddingEngine, user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
53
|
+
def compact_if_needed!(store, embedding_engine, user_id: nil)
|
54
|
+
DSPy::Instrumentation.instrument('dspy.memory.compaction_check', { user_id: user_id }) do
|
55
|
+
results = {}
|
56
|
+
|
57
|
+
# Check triggers in order of impact
|
58
|
+
if size_compaction_needed?(store, user_id)
|
59
|
+
results[:size_compaction] = perform_size_compaction!(store, user_id)
|
60
|
+
end
|
61
|
+
|
62
|
+
if age_compaction_needed?(store, user_id)
|
63
|
+
results[:age_compaction] = perform_age_compaction!(store, user_id)
|
64
|
+
end
|
65
|
+
|
66
|
+
if duplication_compaction_needed?(store, embedding_engine, user_id)
|
67
|
+
results[:deduplication] = perform_deduplication!(store, embedding_engine, user_id)
|
68
|
+
end
|
69
|
+
|
70
|
+
if relevance_compaction_needed?(store, user_id)
|
71
|
+
results[:relevance_pruning] = perform_relevance_pruning!(store, user_id)
|
72
|
+
end
|
73
|
+
|
74
|
+
results[:total_compacted] = results.values.sum { |r| r.is_a?(Hash) ? r[:removed_count] || 0 : 0 }
|
75
|
+
results
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# Check if size-based compaction is needed
|
80
|
+
sig { params(store: MemoryStore, user_id: T.nilable(String)).returns(T::Boolean) }
|
81
|
+
def size_compaction_needed?(store, user_id)
|
82
|
+
store.count(user_id: user_id) > @max_memories
|
83
|
+
end
|
84
|
+
|
85
|
+
# Check if age-based compaction is needed
|
86
|
+
sig { params(store: MemoryStore, user_id: T.nilable(String)).returns(T::Boolean) }
|
87
|
+
def age_compaction_needed?(store, user_id)
|
88
|
+
memories = store.list(user_id: user_id)
|
89
|
+
return false if memories.empty?
|
90
|
+
|
91
|
+
# Check if any memory exceeds the age limit
|
92
|
+
memories.any? { |memory| memory.age_in_days > @max_age_days }
|
93
|
+
end
|
94
|
+
|
95
|
+
# Check if deduplication is needed (simple heuristic)
|
96
|
+
sig { params(store: MemoryStore, embedding_engine: EmbeddingEngine, user_id: T.nilable(String)).returns(T::Boolean) }
|
97
|
+
def duplication_compaction_needed?(store, embedding_engine, user_id)
|
98
|
+
# Sample recent memories to check for duplicates
|
99
|
+
recent_memories = store.list(user_id: user_id, limit: 50)
|
100
|
+
return false if recent_memories.length < 10
|
101
|
+
|
102
|
+
# Quick duplicate check on a sample
|
103
|
+
sample_size = [recent_memories.length / 4, 10].max
|
104
|
+
sample = recent_memories.sample(sample_size)
|
105
|
+
|
106
|
+
duplicate_count = 0
|
107
|
+
sample.each_with_index do |memory1, i|
|
108
|
+
sample[(i+1)..-1].each do |memory2|
|
109
|
+
next unless memory1.embedding && memory2.embedding
|
110
|
+
|
111
|
+
similarity = embedding_engine.cosine_similarity(memory1.embedding, memory2.embedding)
|
112
|
+
duplicate_count += 1 if similarity > @similarity_threshold
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
# Need deduplication if > 20% of sample has duplicates
|
117
|
+
(duplicate_count.to_f / sample_size) > 0.2
|
118
|
+
end
|
119
|
+
|
120
|
+
# Check if relevance-based pruning is needed
|
121
|
+
sig { params(store: MemoryStore, user_id: T.nilable(String)).returns(T::Boolean) }
|
122
|
+
def relevance_compaction_needed?(store, user_id)
|
123
|
+
memories = store.list(user_id: user_id, limit: 100)
|
124
|
+
return false if memories.length < 50
|
125
|
+
|
126
|
+
# Check if many memories have low access counts
|
127
|
+
total_access = memories.sum(&:access_count)
|
128
|
+
return false if total_access == 0
|
129
|
+
|
130
|
+
# Calculate relative access for each memory
|
131
|
+
low_access_count = memories.count do |memory|
|
132
|
+
relative_access = memory.access_count.to_f / total_access
|
133
|
+
relative_access < @low_access_threshold
|
134
|
+
end
|
135
|
+
|
136
|
+
# Need pruning if > 30% of memories have low relative access
|
137
|
+
low_access_ratio = low_access_count.to_f / memories.length
|
138
|
+
low_access_ratio > 0.3
|
139
|
+
end
|
140
|
+
|
141
|
+
private
|
142
|
+
|
143
|
+
# Remove oldest memories when over size limit
|
144
|
+
sig { params(store: MemoryStore, user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
145
|
+
def perform_size_compaction!(store, user_id)
|
146
|
+
DSPy::Instrumentation.instrument('dspy.memory.size_compaction', { user_id: user_id }) do
|
147
|
+
current_count = store.count(user_id: user_id)
|
148
|
+
target_count = (@max_memories * 0.8).to_i # Remove to 80% of limit
|
149
|
+
remove_count = current_count - target_count
|
150
|
+
|
151
|
+
# Don't remove if already under target
|
152
|
+
if remove_count <= 0
|
153
|
+
return {
|
154
|
+
trigger: 'size_limit_exceeded',
|
155
|
+
removed_count: 0,
|
156
|
+
before_count: current_count,
|
157
|
+
after_count: current_count,
|
158
|
+
note: 'already_under_target'
|
159
|
+
}
|
160
|
+
end
|
161
|
+
|
162
|
+
# Get oldest memories
|
163
|
+
all_memories = store.list(user_id: user_id)
|
164
|
+
oldest_memories = all_memories.sort_by(&:created_at).first(remove_count)
|
165
|
+
|
166
|
+
removed_count = 0
|
167
|
+
oldest_memories.each do |memory|
|
168
|
+
if store.delete(memory.id)
|
169
|
+
removed_count += 1
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
{
|
174
|
+
trigger: 'size_limit_exceeded',
|
175
|
+
removed_count: removed_count,
|
176
|
+
before_count: current_count,
|
177
|
+
after_count: current_count - removed_count
|
178
|
+
}
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
# Remove memories older than age limit
|
183
|
+
sig { params(store: MemoryStore, user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
184
|
+
def perform_age_compaction!(store, user_id)
|
185
|
+
DSPy::Instrumentation.instrument('dspy.memory.age_compaction', { user_id: user_id }) do
|
186
|
+
cutoff_time = Time.now - (@max_age_days * 24 * 60 * 60)
|
187
|
+
all_memories = store.list(user_id: user_id)
|
188
|
+
old_memories = all_memories.select { |m| m.created_at < cutoff_time }
|
189
|
+
|
190
|
+
removed_count = 0
|
191
|
+
old_memories.each do |memory|
|
192
|
+
if store.delete(memory.id)
|
193
|
+
removed_count += 1
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
{
|
198
|
+
trigger: 'age_limit_exceeded',
|
199
|
+
removed_count: removed_count,
|
200
|
+
cutoff_age_days: @max_age_days,
|
201
|
+
oldest_removed_age: old_memories.empty? ? nil : old_memories.max_by(&:created_at).age_in_days
|
202
|
+
}
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
# Remove near-duplicate memories using embedding similarity
|
207
|
+
sig { params(store: MemoryStore, embedding_engine: EmbeddingEngine, user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
208
|
+
def perform_deduplication!(store, embedding_engine, user_id)
|
209
|
+
DSPy::Instrumentation.instrument('dspy.memory.deduplication', { user_id: user_id }) do
|
210
|
+
memories = store.list(user_id: user_id)
|
211
|
+
memories_with_embeddings = memories.select(&:embedding)
|
212
|
+
|
213
|
+
duplicates_to_remove = []
|
214
|
+
processed = Set.new
|
215
|
+
|
216
|
+
memories_with_embeddings.each_with_index do |memory1, i|
|
217
|
+
next if processed.include?(memory1.id)
|
218
|
+
|
219
|
+
memories_with_embeddings[(i+1)..-1].each do |memory2|
|
220
|
+
next if processed.include?(memory2.id)
|
221
|
+
|
222
|
+
similarity = embedding_engine.cosine_similarity(memory1.embedding, memory2.embedding)
|
223
|
+
|
224
|
+
if similarity > @similarity_threshold
|
225
|
+
# Keep the one with higher access count, or newer if tied
|
226
|
+
keeper, duplicate = if memory1.access_count > memory2.access_count
|
227
|
+
[memory1, memory2]
|
228
|
+
elsif memory1.access_count < memory2.access_count
|
229
|
+
[memory2, memory1]
|
230
|
+
else
|
231
|
+
# Tie: keep newer one
|
232
|
+
memory1.created_at > memory2.created_at ? [memory1, memory2] : [memory2, memory1]
|
233
|
+
end
|
234
|
+
|
235
|
+
duplicates_to_remove << duplicate
|
236
|
+
processed.add(duplicate.id)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
processed.add(memory1.id)
|
241
|
+
end
|
242
|
+
|
243
|
+
removed_count = 0
|
244
|
+
duplicates_to_remove.uniq.each do |memory|
|
245
|
+
if store.delete(memory.id)
|
246
|
+
removed_count += 1
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
{
|
251
|
+
trigger: 'duplicate_similarity_detected',
|
252
|
+
removed_count: removed_count,
|
253
|
+
similarity_threshold: @similarity_threshold,
|
254
|
+
total_checked: memories_with_embeddings.length
|
255
|
+
}
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
# Remove memories with low relevance (low access patterns)
|
260
|
+
sig { params(store: MemoryStore, user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
261
|
+
def perform_relevance_pruning!(store, user_id)
|
262
|
+
DSPy::Instrumentation.instrument('dspy.memory.relevance_pruning', { user_id: user_id }) do
|
263
|
+
memories = store.list(user_id: user_id)
|
264
|
+
total_access = memories.sum(&:access_count)
|
265
|
+
return { removed_count: 0, trigger: 'no_access_data' } if total_access == 0
|
266
|
+
|
267
|
+
# Calculate relevance scores
|
268
|
+
scored_memories = memories.map do |memory|
|
269
|
+
# Combine access frequency with recency
|
270
|
+
access_score = memory.access_count.to_f / total_access
|
271
|
+
recency_score = 1.0 / (memory.age_in_days + 1) # Avoid division by zero
|
272
|
+
relevance_score = (access_score * 0.7) + (recency_score * 0.3)
|
273
|
+
|
274
|
+
{ memory: memory, score: relevance_score }
|
275
|
+
end
|
276
|
+
|
277
|
+
# Remove bottom 20% by relevance
|
278
|
+
sorted_by_relevance = scored_memories.sort_by { |item| item[:score] }
|
279
|
+
remove_count = (memories.length * 0.2).to_i
|
280
|
+
to_remove = sorted_by_relevance.first(remove_count)
|
281
|
+
|
282
|
+
removed_count = 0
|
283
|
+
to_remove.each do |item|
|
284
|
+
if store.delete(item[:memory].id)
|
285
|
+
removed_count += 1
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
{
|
290
|
+
trigger: 'low_relevance_detected',
|
291
|
+
removed_count: removed_count,
|
292
|
+
lowest_score: to_remove.first&.dig(:score),
|
293
|
+
highest_score: sorted_by_relevance.last&.dig(:score)
|
294
|
+
}
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
@@ -0,0 +1,248 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'sorbet-runtime'
|
4
|
+
require_relative 'memory_record'
|
5
|
+
require_relative 'memory_store'
|
6
|
+
require_relative 'in_memory_store'
|
7
|
+
require_relative 'embedding_engine'
|
8
|
+
require_relative 'local_embedding_engine'
|
9
|
+
require_relative 'memory_compactor'
|
10
|
+
|
11
|
+
module DSPy
|
12
|
+
module Memory
|
13
|
+
# High-level memory management interface implementing MemoryTools API
|
14
|
+
class MemoryManager
|
15
|
+
extend T::Sig
|
16
|
+
|
17
|
+
sig { returns(MemoryStore) }
|
18
|
+
attr_reader :store
|
19
|
+
|
20
|
+
sig { returns(EmbeddingEngine) }
|
21
|
+
attr_reader :embedding_engine
|
22
|
+
|
23
|
+
sig { returns(MemoryCompactor) }
|
24
|
+
attr_reader :compactor
|
25
|
+
|
26
|
+
sig { params(store: T.nilable(MemoryStore), embedding_engine: T.nilable(EmbeddingEngine), compactor: T.nilable(MemoryCompactor)).void }
|
27
|
+
def initialize(store: nil, embedding_engine: nil, compactor: nil)
|
28
|
+
@store = store || InMemoryStore.new
|
29
|
+
@embedding_engine = embedding_engine || create_default_embedding_engine
|
30
|
+
@compactor = compactor || MemoryCompactor.new
|
31
|
+
end
|
32
|
+
|
33
|
+
# Store a memory with automatic embedding generation
|
34
|
+
sig { params(content: String, user_id: T.nilable(String), tags: T::Array[String], metadata: T::Hash[String, T.untyped]).returns(MemoryRecord) }
|
35
|
+
def store_memory(content, user_id: nil, tags: [], metadata: {})
|
36
|
+
# Generate embedding for the content
|
37
|
+
embedding = @embedding_engine.embed(content)
|
38
|
+
|
39
|
+
# Create memory record
|
40
|
+
record = MemoryRecord.new(
|
41
|
+
content: content,
|
42
|
+
user_id: user_id,
|
43
|
+
tags: tags,
|
44
|
+
embedding: embedding,
|
45
|
+
metadata: metadata
|
46
|
+
)
|
47
|
+
|
48
|
+
# Store in backend
|
49
|
+
success = @store.store(record)
|
50
|
+
raise "Failed to store memory" unless success
|
51
|
+
|
52
|
+
# Check if compaction is needed after storing
|
53
|
+
compact_if_needed!(user_id)
|
54
|
+
|
55
|
+
record
|
56
|
+
end
|
57
|
+
|
58
|
+
# Retrieve a memory by ID
|
59
|
+
sig { params(memory_id: String).returns(T.nilable(MemoryRecord)) }
|
60
|
+
def get_memory(memory_id)
|
61
|
+
@store.retrieve(memory_id)
|
62
|
+
end
|
63
|
+
|
64
|
+
# Update an existing memory
|
65
|
+
sig { params(memory_id: String, new_content: String, tags: T.nilable(T::Array[String]), metadata: T.nilable(T::Hash[String, T.untyped])).returns(T::Boolean) }
|
66
|
+
def update_memory(memory_id, new_content, tags: nil, metadata: nil)
|
67
|
+
record = @store.retrieve(memory_id)
|
68
|
+
return false unless record
|
69
|
+
|
70
|
+
# Update content and regenerate embedding
|
71
|
+
record.update_content!(new_content)
|
72
|
+
record.embedding = @embedding_engine.embed(new_content)
|
73
|
+
|
74
|
+
# Update tags if provided
|
75
|
+
record.tags = tags if tags
|
76
|
+
|
77
|
+
# Update metadata if provided
|
78
|
+
record.metadata.merge!(metadata) if metadata
|
79
|
+
|
80
|
+
@store.update(record)
|
81
|
+
end
|
82
|
+
|
83
|
+
# Delete a memory
|
84
|
+
sig { params(memory_id: String).returns(T::Boolean) }
|
85
|
+
def delete_memory(memory_id)
|
86
|
+
@store.delete(memory_id)
|
87
|
+
end
|
88
|
+
|
89
|
+
# Get all memories for a user
|
90
|
+
sig { params(user_id: T.nilable(String), limit: T.nilable(Integer), offset: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
|
91
|
+
def get_all_memories(user_id: nil, limit: nil, offset: nil)
|
92
|
+
@store.list(user_id: user_id, limit: limit, offset: offset)
|
93
|
+
end
|
94
|
+
|
95
|
+
# Semantic search using embeddings
|
96
|
+
sig { params(query: String, user_id: T.nilable(String), limit: T.nilable(Integer), threshold: T.nilable(Float)).returns(T::Array[MemoryRecord]) }
|
97
|
+
def search_memories(query, user_id: nil, limit: 10, threshold: 0.5)
|
98
|
+
# Generate embedding for the query
|
99
|
+
query_embedding = @embedding_engine.embed(query)
|
100
|
+
|
101
|
+
# Perform vector search if supported
|
102
|
+
if @store.supports_vector_search?
|
103
|
+
@store.vector_search(query_embedding, user_id: user_id, limit: limit, threshold: threshold)
|
104
|
+
else
|
105
|
+
# Fallback to text search
|
106
|
+
@store.search(query, user_id: user_id, limit: limit)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
# Search by tags
|
111
|
+
sig { params(tags: T::Array[String], user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
|
112
|
+
def search_by_tags(tags, user_id: nil, limit: nil)
|
113
|
+
@store.search_by_tags(tags, user_id: user_id, limit: limit)
|
114
|
+
end
|
115
|
+
|
116
|
+
# Text-based search (fallback when embeddings not available)
|
117
|
+
sig { params(query: String, user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
|
118
|
+
def search_text(query, user_id: nil, limit: nil)
|
119
|
+
@store.search(query, user_id: user_id, limit: limit)
|
120
|
+
end
|
121
|
+
|
122
|
+
# Count memories
|
123
|
+
sig { params(user_id: T.nilable(String)).returns(Integer) }
|
124
|
+
def count_memories(user_id: nil)
|
125
|
+
@store.count(user_id: user_id)
|
126
|
+
end
|
127
|
+
|
128
|
+
# Clear all memories for a user
|
129
|
+
sig { params(user_id: T.nilable(String)).returns(Integer) }
|
130
|
+
def clear_memories(user_id: nil)
|
131
|
+
@store.clear(user_id: user_id)
|
132
|
+
end
|
133
|
+
|
134
|
+
# Find similar memories to a given memory
|
135
|
+
sig { params(memory_id: String, limit: T.nilable(Integer), threshold: T.nilable(Float)).returns(T::Array[MemoryRecord]) }
|
136
|
+
def find_similar(memory_id, limit: 5, threshold: 0.7)
|
137
|
+
record = @store.retrieve(memory_id)
|
138
|
+
return [] unless record&.embedding
|
139
|
+
|
140
|
+
results = @store.vector_search(record.embedding, user_id: record.user_id, limit: limit + 1, threshold: threshold)
|
141
|
+
|
142
|
+
# Remove the original record from results
|
143
|
+
results.reject { |r| r.id == memory_id }
|
144
|
+
end
|
145
|
+
|
146
|
+
# Batch operations
|
147
|
+
sig { params(contents: T::Array[String], user_id: T.nilable(String), tags: T::Array[String]).returns(T::Array[MemoryRecord]) }
|
148
|
+
def store_memories_batch(contents, user_id: nil, tags: [])
|
149
|
+
# Generate embeddings in batch for efficiency
|
150
|
+
embeddings = @embedding_engine.embed_batch(contents)
|
151
|
+
|
152
|
+
records = contents.zip(embeddings).map do |content, embedding|
|
153
|
+
MemoryRecord.new(
|
154
|
+
content: content,
|
155
|
+
user_id: user_id,
|
156
|
+
tags: tags,
|
157
|
+
embedding: embedding
|
158
|
+
)
|
159
|
+
end
|
160
|
+
|
161
|
+
# Store all records
|
162
|
+
results = @store.store_batch(records)
|
163
|
+
|
164
|
+
# Compact after batch operation
|
165
|
+
compact_if_needed!(user_id)
|
166
|
+
|
167
|
+
# Return only successfully stored records
|
168
|
+
records.select.with_index { |_, idx| results[idx] }
|
169
|
+
end
|
170
|
+
|
171
|
+
# Get memory statistics
|
172
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
173
|
+
def stats
|
174
|
+
store_stats = @store.stats
|
175
|
+
engine_stats = @embedding_engine.stats
|
176
|
+
|
177
|
+
{
|
178
|
+
store: store_stats,
|
179
|
+
embedding_engine: engine_stats,
|
180
|
+
total_memories: store_stats[:total_memories] || 0
|
181
|
+
}
|
182
|
+
end
|
183
|
+
|
184
|
+
# Health check
|
185
|
+
sig { returns(T::Boolean) }
|
186
|
+
def healthy?
|
187
|
+
@embedding_engine.ready? && @store.respond_to?(:count)
|
188
|
+
end
|
189
|
+
|
190
|
+
# Export memories to hash format
|
191
|
+
sig { params(user_id: T.nilable(String)).returns(T::Array[T::Hash[String, T.untyped]]) }
|
192
|
+
def export_memories(user_id: nil)
|
193
|
+
memories = get_all_memories(user_id: user_id)
|
194
|
+
memories.map(&:to_h)
|
195
|
+
end
|
196
|
+
|
197
|
+
# Import memories from hash format
|
198
|
+
sig { params(memories_data: T::Array[T::Hash[String, T.untyped]]).returns(Integer) }
|
199
|
+
def import_memories(memories_data)
|
200
|
+
records = memories_data.map { |data| MemoryRecord.from_h(data) }
|
201
|
+
results = @store.store_batch(records)
|
202
|
+
|
203
|
+
# Compact after batch import
|
204
|
+
user_ids = records.map(&:user_id).compact.uniq
|
205
|
+
user_ids.each { |user_id| compact_if_needed!(user_id) }
|
206
|
+
|
207
|
+
results.count(true)
|
208
|
+
end
|
209
|
+
|
210
|
+
# Trigger memory compaction if needed
|
211
|
+
sig { params(user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
212
|
+
def compact_if_needed!(user_id = nil)
|
213
|
+
@compactor.compact_if_needed!(@store, @embedding_engine, user_id: user_id)
|
214
|
+
end
|
215
|
+
|
216
|
+
# Force memory compaction (useful for testing or manual cleanup)
|
217
|
+
sig { params(user_id: T.nilable(String)).returns(T::Hash[Symbol, T.untyped]) }
|
218
|
+
def force_compact!(user_id = nil)
|
219
|
+
DSPy::Instrumentation.instrument('dspy.memory.compaction_complete', {
|
220
|
+
user_id: user_id,
|
221
|
+
forced: true
|
222
|
+
}) do
|
223
|
+
results = {}
|
224
|
+
|
225
|
+
# Run all compaction strategies regardless of thresholds
|
226
|
+
results[:size_compaction] = @compactor.send(:perform_size_compaction!, @store, user_id)
|
227
|
+
results[:age_compaction] = @compactor.send(:perform_age_compaction!, @store, user_id)
|
228
|
+
results[:deduplication] = @compactor.send(:perform_deduplication!, @store, @embedding_engine, user_id)
|
229
|
+
results[:relevance_pruning] = @compactor.send(:perform_relevance_pruning!, @store, user_id)
|
230
|
+
|
231
|
+
results[:total_compacted] = results.values.sum { |r| r.is_a?(Hash) ? r[:removed_count] || 0 : 0 }
|
232
|
+
results
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
private
|
237
|
+
|
238
|
+
# Create default embedding engine
|
239
|
+
sig { returns(EmbeddingEngine) }
|
240
|
+
def create_default_embedding_engine
|
241
|
+
LocalEmbeddingEngine.new
|
242
|
+
rescue => e
|
243
|
+
# Fallback to no-op engine if local engine fails
|
244
|
+
NoOpEmbeddingEngine.new
|
245
|
+
end
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
@@ -0,0 +1,163 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'sorbet-runtime'
|
4
|
+
require 'securerandom'
|
5
|
+
|
6
|
+
module DSPy
|
7
|
+
module Memory
|
8
|
+
# Represents a single memory entry with metadata and embeddings
|
9
|
+
class MemoryRecord
|
10
|
+
extend T::Sig
|
11
|
+
|
12
|
+
sig { returns(String) }
|
13
|
+
attr_reader :id
|
14
|
+
|
15
|
+
sig { returns(String) }
|
16
|
+
attr_accessor :content
|
17
|
+
|
18
|
+
sig { returns(T.nilable(String)) }
|
19
|
+
attr_accessor :user_id
|
20
|
+
|
21
|
+
sig { returns(T::Array[String]) }
|
22
|
+
attr_accessor :tags
|
23
|
+
|
24
|
+
sig { returns(T.nilable(T::Array[Float])) }
|
25
|
+
attr_accessor :embedding
|
26
|
+
|
27
|
+
sig { returns(Time) }
|
28
|
+
attr_reader :created_at
|
29
|
+
|
30
|
+
sig { returns(Time) }
|
31
|
+
attr_accessor :updated_at
|
32
|
+
|
33
|
+
sig { returns(Integer) }
|
34
|
+
attr_accessor :access_count
|
35
|
+
|
36
|
+
sig { returns(T.nilable(Time)) }
|
37
|
+
attr_accessor :last_accessed_at
|
38
|
+
|
39
|
+
sig { returns(T::Hash[String, T.untyped]) }
|
40
|
+
attr_accessor :metadata
|
41
|
+
|
42
|
+
sig do
|
43
|
+
params(
|
44
|
+
content: String,
|
45
|
+
user_id: T.nilable(String),
|
46
|
+
tags: T::Array[String],
|
47
|
+
embedding: T.nilable(T::Array[Float]),
|
48
|
+
id: T.nilable(String),
|
49
|
+
metadata: T::Hash[String, T.untyped]
|
50
|
+
).void
|
51
|
+
end
|
52
|
+
def initialize(content:, user_id: nil, tags: [], embedding: nil, id: nil, metadata: {})
|
53
|
+
@id = id || SecureRandom.uuid
|
54
|
+
@content = content
|
55
|
+
@user_id = user_id
|
56
|
+
@tags = tags
|
57
|
+
@embedding = embedding
|
58
|
+
@created_at = Time.now
|
59
|
+
@updated_at = Time.now
|
60
|
+
@access_count = 0
|
61
|
+
@last_accessed_at = nil
|
62
|
+
@metadata = metadata
|
63
|
+
end
|
64
|
+
|
65
|
+
# Record an access to this memory
|
66
|
+
sig { void }
|
67
|
+
def record_access!
|
68
|
+
@access_count += 1
|
69
|
+
@last_accessed_at = Time.now
|
70
|
+
end
|
71
|
+
|
72
|
+
# Update the content and timestamp
|
73
|
+
sig { params(new_content: String).void }
|
74
|
+
def update_content!(new_content)
|
75
|
+
@content = new_content
|
76
|
+
@updated_at = Time.now
|
77
|
+
end
|
78
|
+
|
79
|
+
# Calculate age in seconds
|
80
|
+
sig { returns(Float) }
|
81
|
+
def age_in_seconds
|
82
|
+
Time.now - @created_at
|
83
|
+
end
|
84
|
+
|
85
|
+
# Calculate age in days
|
86
|
+
sig { returns(Float) }
|
87
|
+
def age_in_days
|
88
|
+
age_in_seconds / 86400.0
|
89
|
+
end
|
90
|
+
|
91
|
+
# Check if memory has been accessed recently (within last N seconds)
|
92
|
+
sig { params(seconds: Integer).returns(T::Boolean) }
|
93
|
+
def accessed_recently?(seconds = 3600)
|
94
|
+
return false if @last_accessed_at.nil?
|
95
|
+
(Time.now - @last_accessed_at) <= seconds
|
96
|
+
end
|
97
|
+
|
98
|
+
# Check if memory matches a tag
|
99
|
+
sig { params(tag: String).returns(T::Boolean) }
|
100
|
+
def has_tag?(tag)
|
101
|
+
@tags.include?(tag)
|
102
|
+
end
|
103
|
+
|
104
|
+
# Add a tag if not already present
|
105
|
+
sig { params(tag: String).void }
|
106
|
+
def add_tag(tag)
|
107
|
+
@tags << tag unless @tags.include?(tag)
|
108
|
+
end
|
109
|
+
|
110
|
+
# Remove a tag
|
111
|
+
sig { params(tag: String).void }
|
112
|
+
def remove_tag(tag)
|
113
|
+
@tags.delete(tag)
|
114
|
+
end
|
115
|
+
|
116
|
+
# Convert to hash for serialization
|
117
|
+
sig { returns(T::Hash[String, T.untyped]) }
|
118
|
+
def to_h
|
119
|
+
{
|
120
|
+
'id' => @id,
|
121
|
+
'content' => @content,
|
122
|
+
'user_id' => @user_id,
|
123
|
+
'tags' => @tags,
|
124
|
+
'embedding' => @embedding,
|
125
|
+
'created_at' => @created_at.iso8601,
|
126
|
+
'updated_at' => @updated_at.iso8601,
|
127
|
+
'access_count' => @access_count,
|
128
|
+
'last_accessed_at' => @last_accessed_at&.iso8601,
|
129
|
+
'metadata' => @metadata
|
130
|
+
}
|
131
|
+
end
|
132
|
+
|
133
|
+
# Create from hash (for deserialization)
|
134
|
+
sig { params(hash: T::Hash[String, T.untyped]).returns(MemoryRecord) }
|
135
|
+
def self.from_h(hash)
|
136
|
+
record = allocate
|
137
|
+
record.instance_variable_set(:@id, hash['id'])
|
138
|
+
record.instance_variable_set(:@content, hash['content'])
|
139
|
+
record.instance_variable_set(:@user_id, hash['user_id'])
|
140
|
+
record.instance_variable_set(:@tags, hash['tags'] || [])
|
141
|
+
record.instance_variable_set(:@embedding, hash['embedding'])
|
142
|
+
record.instance_variable_set(:@created_at, Time.parse(hash['created_at']))
|
143
|
+
record.instance_variable_set(:@updated_at, Time.parse(hash['updated_at']))
|
144
|
+
record.instance_variable_set(:@access_count, hash['access_count'] || 0)
|
145
|
+
record.instance_variable_set(:@last_accessed_at,
|
146
|
+
hash['last_accessed_at'] ? Time.parse(hash['last_accessed_at']) : nil)
|
147
|
+
record.instance_variable_set(:@metadata, hash['metadata'] || {})
|
148
|
+
record
|
149
|
+
end
|
150
|
+
|
151
|
+
# String representation
|
152
|
+
sig { returns(String) }
|
153
|
+
def to_s
|
154
|
+
"#<MemoryRecord id=#{@id[0..7]}... content=\"#{@content[0..50]}...\" tags=#{@tags}>"
|
155
|
+
end
|
156
|
+
|
157
|
+
sig { returns(String) }
|
158
|
+
def inspect
|
159
|
+
to_s
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|