htm 0.0.2 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.aigcm_msg +1 -0
- data/.architecture/reviews/comprehensive-codebase-review.md +577 -0
- data/.claude/settings.local.json +95 -0
- data/.irbrc +283 -80
- data/.tbls.yml +2 -1
- data/CHANGELOG.md +327 -26
- data/CLAUDE.md +603 -0
- data/README.md +83 -12
- data/Rakefile +5 -0
- data/bin/htm_mcp.rb +527 -0
- data/db/migrate/{20250101000001_enable_extensions.rb → 00001_enable_extensions.rb} +0 -1
- data/db/migrate/00002_create_robots.rb +11 -0
- data/db/migrate/00003_create_file_sources.rb +20 -0
- data/db/migrate/00004_create_nodes.rb +65 -0
- data/db/migrate/00005_create_tags.rb +13 -0
- data/db/migrate/00006_create_node_tags.rb +18 -0
- data/db/migrate/00007_create_robot_nodes.rb +26 -0
- data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +12 -0
- data/db/schema.sql +172 -1
- data/docs/api/database.md +1 -2
- data/docs/api/htm.md +197 -2
- data/docs/api/yard/HTM/ActiveRecordConfig.md +23 -0
- data/docs/api/yard/HTM/AuthorizationError.md +11 -0
- data/docs/api/yard/HTM/CircuitBreaker.md +92 -0
- data/docs/api/yard/HTM/CircuitBreakerOpenError.md +34 -0
- data/docs/api/yard/HTM/Configuration.md +175 -0
- data/docs/api/yard/HTM/Database.md +99 -0
- data/docs/api/yard/HTM/DatabaseError.md +14 -0
- data/docs/api/yard/HTM/EmbeddingError.md +18 -0
- data/docs/api/yard/HTM/EmbeddingService.md +58 -0
- data/docs/api/yard/HTM/Error.md +11 -0
- data/docs/api/yard/HTM/JobAdapter.md +39 -0
- data/docs/api/yard/HTM/LongTermMemory.md +342 -0
- data/docs/api/yard/HTM/NotFoundError.md +17 -0
- data/docs/api/yard/HTM/Observability.md +107 -0
- data/docs/api/yard/HTM/QueryTimeoutError.md +19 -0
- data/docs/api/yard/HTM/Railtie.md +27 -0
- data/docs/api/yard/HTM/ResourceExhaustedError.md +13 -0
- data/docs/api/yard/HTM/TagError.md +18 -0
- data/docs/api/yard/HTM/TagService.md +67 -0
- data/docs/api/yard/HTM/Timeframe/Result.md +24 -0
- data/docs/api/yard/HTM/Timeframe.md +40 -0
- data/docs/api/yard/HTM/TimeframeExtractor/Result.md +24 -0
- data/docs/api/yard/HTM/TimeframeExtractor.md +45 -0
- data/docs/api/yard/HTM/ValidationError.md +20 -0
- data/docs/api/yard/HTM/WorkingMemory.md +131 -0
- data/docs/api/yard/HTM.md +80 -0
- data/docs/api/yard/index.csv +179 -0
- data/docs/api/yard-reference.md +51 -0
- data/docs/database/README.md +128 -128
- data/docs/database/public.file_sources.md +42 -0
- data/docs/database/public.file_sources.svg +211 -0
- data/docs/database/public.node_tags.md +4 -4
- data/docs/database/public.node_tags.svg +212 -79
- data/docs/database/public.nodes.md +22 -12
- data/docs/database/public.nodes.svg +246 -127
- data/docs/database/public.robot_nodes.md +11 -9
- data/docs/database/public.robot_nodes.svg +220 -98
- data/docs/database/public.robots.md +2 -2
- data/docs/database/public.robots.svg +136 -81
- data/docs/database/public.tags.md +3 -3
- data/docs/database/public.tags.svg +118 -39
- data/docs/database/schema.json +850 -771
- data/docs/database/schema.svg +256 -197
- data/docs/development/schema.md +67 -2
- data/docs/guides/adding-memories.md +93 -7
- data/docs/guides/recalling-memories.md +36 -1
- data/examples/README.md +405 -0
- data/examples/cli_app/htm_cli.rb +65 -5
- data/examples/cli_app/temp.log +93 -0
- data/examples/file_loader_usage.rb +177 -0
- data/examples/mcp_client.rb +529 -0
- data/examples/robot_groups/lib/robot_group.rb +419 -0
- data/examples/robot_groups/lib/working_memory_channel.rb +140 -0
- data/examples/robot_groups/multi_process.rb +286 -0
- data/examples/robot_groups/robot_worker.rb +136 -0
- data/examples/robot_groups/same_process.rb +229 -0
- data/examples/timeframe_demo.rb +276 -0
- data/lib/htm/active_record_config.rb +1 -1
- data/lib/htm/circuit_breaker.rb +202 -0
- data/lib/htm/configuration.rb +59 -13
- data/lib/htm/database.rb +67 -36
- data/lib/htm/embedding_service.rb +39 -2
- data/lib/htm/errors.rb +131 -11
- data/lib/htm/jobs/generate_embedding_job.rb +5 -4
- data/lib/htm/jobs/generate_tags_job.rb +4 -0
- data/lib/htm/loaders/markdown_loader.rb +263 -0
- data/lib/htm/loaders/paragraph_chunker.rb +112 -0
- data/lib/htm/long_term_memory.rb +460 -343
- data/lib/htm/models/file_source.rb +99 -0
- data/lib/htm/models/node.rb +80 -5
- data/lib/htm/models/robot.rb +24 -1
- data/lib/htm/models/robot_node.rb +1 -0
- data/lib/htm/models/tag.rb +254 -4
- data/lib/htm/observability.rb +395 -0
- data/lib/htm/tag_service.rb +60 -3
- data/lib/htm/tasks.rb +26 -1
- data/lib/htm/timeframe.rb +194 -0
- data/lib/htm/timeframe_extractor.rb +307 -0
- data/lib/htm/version.rb +1 -1
- data/lib/htm/working_memory.rb +165 -70
- data/lib/htm.rb +328 -130
- data/lib/tasks/doc.rake +300 -0
- data/lib/tasks/files.rake +299 -0
- data/lib/tasks/htm.rake +158 -3
- data/lib/tasks/jobs.rake +3 -9
- data/lib/tasks/tags.rake +166 -6
- data/mkdocs.yml +36 -1
- data/notes/ARCHITECTURE_REVIEW.md +1167 -0
- data/notes/IMPLEMENTATION_SUMMARY.md +606 -0
- data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +451 -0
- data/notes/next_steps.md +100 -0
- data/notes/plan.md +627 -0
- data/notes/tag_ontology_enhancement_ideas.md +222 -0
- data/notes/timescaledb_removal_summary.md +200 -0
- metadata +158 -17
- data/db/migrate/20250101000002_create_robots.rb +0 -14
- data/db/migrate/20250101000003_create_nodes.rb +0 -42
- data/db/migrate/20250101000005_create_tags.rb +0 -38
- data/db/migrate/20250101000007_add_node_vector_indexes.rb +0 -30
- data/db/migrate/20250125000001_add_content_hash_to_nodes.rb +0 -14
- data/db/migrate/20250125000002_create_robot_nodes.rb +0 -35
- data/db/migrate/20250125000003_remove_source_and_robot_id_from_nodes.rb +0 -28
- data/db/migrate/20250126000001_create_working_memories.rb +0 -19
- data/db/migrate/20250126000002_remove_unused_columns.rb +0 -12
- data/docs/database/public.working_memories.md +0 -40
- data/docs/database/public.working_memories.svg +0 -112
- data/lib/htm/models/working_memory_entry.rb +0 -88
data/lib/htm/working_memory.rb
CHANGED
|
@@ -6,6 +6,9 @@ class HTM
|
|
|
6
6
|
# WorkingMemory manages the active conversation context within token limits.
|
|
7
7
|
# When full, it evicts less important or older nodes back to long-term storage.
|
|
8
8
|
#
|
|
9
|
+
# Thread Safety: All public methods are protected by a mutex to ensure
|
|
10
|
+
# safe concurrent access from multiple threads.
|
|
11
|
+
#
|
|
9
12
|
class WorkingMemory
|
|
10
13
|
attr_reader :max_tokens
|
|
11
14
|
|
|
@@ -17,6 +20,7 @@ class HTM
|
|
|
17
20
|
@max_tokens = max_tokens
|
|
18
21
|
@nodes = {}
|
|
19
22
|
@access_order = []
|
|
23
|
+
@mutex = Mutex.new
|
|
20
24
|
end
|
|
21
25
|
|
|
22
26
|
# Add a node to working memory
|
|
@@ -30,15 +34,17 @@ class HTM
|
|
|
30
34
|
# @return [void]
|
|
31
35
|
#
|
|
32
36
|
def add(key, value, token_count:, access_count: 0, last_accessed: nil, from_recall: false)
|
|
33
|
-
@
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
37
|
+
@mutex.synchronize do
|
|
38
|
+
@nodes[key] = {
|
|
39
|
+
value: value,
|
|
40
|
+
token_count: token_count,
|
|
41
|
+
access_count: access_count,
|
|
42
|
+
last_accessed: last_accessed || Time.now,
|
|
43
|
+
added_at: Time.now,
|
|
44
|
+
from_recall: from_recall
|
|
45
|
+
}
|
|
46
|
+
update_access_unlocked(key)
|
|
47
|
+
end
|
|
42
48
|
end
|
|
43
49
|
|
|
44
50
|
# Remove a node from working memory
|
|
@@ -47,8 +53,10 @@ class HTM
|
|
|
47
53
|
# @return [void]
|
|
48
54
|
#
|
|
49
55
|
def remove(key)
|
|
50
|
-
@
|
|
51
|
-
|
|
56
|
+
@mutex.synchronize do
|
|
57
|
+
@nodes.delete(key)
|
|
58
|
+
@access_order.delete(key)
|
|
59
|
+
end
|
|
52
60
|
end
|
|
53
61
|
|
|
54
62
|
# Check if there's space for a node
|
|
@@ -57,7 +65,9 @@ class HTM
|
|
|
57
65
|
# @return [Boolean] true if space available
|
|
58
66
|
#
|
|
59
67
|
def has_space?(token_count)
|
|
60
|
-
|
|
68
|
+
@mutex.synchronize do
|
|
69
|
+
current_tokens_unlocked + token_count <= @max_tokens
|
|
70
|
+
end
|
|
61
71
|
end
|
|
62
72
|
|
|
63
73
|
# Evict nodes to make space
|
|
@@ -69,33 +79,35 @@ class HTM
|
|
|
69
79
|
# @return [Array<Hash>] Evicted nodes
|
|
70
80
|
#
|
|
71
81
|
def evict_to_make_space(needed_tokens)
|
|
72
|
-
|
|
73
|
-
|
|
82
|
+
@mutex.synchronize do
|
|
83
|
+
evicted = []
|
|
84
|
+
tokens_freed = 0
|
|
74
85
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
86
|
+
# Sort by access frequency + recency (lower score = more evictable)
|
|
87
|
+
candidates = @nodes.sort_by do |key, node|
|
|
88
|
+
access_frequency = node[:access_count] || 0
|
|
89
|
+
time_since_accessed = Time.now - (node[:last_accessed] || node[:added_at])
|
|
79
90
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
# Combined score: lower is more evictable
|
|
92
|
+
# Frequently accessed = higher score (keep)
|
|
93
|
+
# Recently accessed = higher score (keep)
|
|
94
|
+
access_score = Math.log(1 + access_frequency)
|
|
95
|
+
recency_score = 1.0 / (1 + time_since_accessed / 3600.0)
|
|
85
96
|
|
|
86
|
-
|
|
87
|
-
|
|
97
|
+
-(access_score + recency_score) # Negative for ascending sort
|
|
98
|
+
end
|
|
88
99
|
|
|
89
|
-
|
|
90
|
-
|
|
100
|
+
candidates.each do |key, node|
|
|
101
|
+
break if tokens_freed >= needed_tokens
|
|
91
102
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
103
|
+
evicted << { key: key, value: node[:value] }
|
|
104
|
+
tokens_freed += node[:token_count]
|
|
105
|
+
@nodes.delete(key)
|
|
106
|
+
@access_order.delete(key)
|
|
107
|
+
end
|
|
97
108
|
|
|
98
|
-
|
|
109
|
+
evicted
|
|
110
|
+
end
|
|
99
111
|
end
|
|
100
112
|
|
|
101
113
|
# Assemble context string for LLM
|
|
@@ -108,40 +120,43 @@ class HTM
|
|
|
108
120
|
# @return [String] Assembled context
|
|
109
121
|
#
|
|
110
122
|
def assemble_context(strategy:, max_tokens: nil)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
nodes = case strategy
|
|
114
|
-
when :recent
|
|
115
|
-
# Most recently accessed (LRU)
|
|
116
|
-
@access_order.reverse.map { |k| @nodes[k] }
|
|
117
|
-
when :frequent
|
|
118
|
-
# Most frequently accessed (LFU)
|
|
119
|
-
@nodes.sort_by { |k, v| -(v[:access_count] || 0) }.map(&:last)
|
|
120
|
-
when :balanced
|
|
121
|
-
# Combined frequency × recency
|
|
122
|
-
@nodes.sort_by { |k, v|
|
|
123
|
-
access_frequency = v[:access_count] || 0
|
|
124
|
-
time_since_accessed = Time.now - (v[:last_accessed] || v[:added_at])
|
|
125
|
-
recency_factor = 1.0 / (1 + time_since_accessed / 3600.0)
|
|
126
|
-
|
|
127
|
-
# Higher score = more relevant
|
|
128
|
-
-(Math.log(1 + access_frequency) * recency_factor)
|
|
129
|
-
}.map(&:last)
|
|
130
|
-
else
|
|
131
|
-
raise ArgumentError, "Unknown strategy: #{strategy}. Use :recent, :frequent, or :balanced"
|
|
132
|
-
end
|
|
123
|
+
@mutex.synchronize do
|
|
124
|
+
max = max_tokens || @max_tokens
|
|
133
125
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
126
|
+
# Make defensive copies of nodes to prevent external mutation of internal state
|
|
127
|
+
nodes = case strategy
|
|
128
|
+
when :recent
|
|
129
|
+
# Most recently accessed (LRU)
|
|
130
|
+
@access_order.reverse.map { |k| @nodes[k]&.dup }.compact
|
|
131
|
+
when :frequent
|
|
132
|
+
# Most frequently accessed (LFU)
|
|
133
|
+
@nodes.sort_by { |k, v| -(v[:access_count] || 0) }.map { |_, v| v.dup }
|
|
134
|
+
when :balanced
|
|
135
|
+
# Combined frequency × recency
|
|
136
|
+
@nodes.sort_by { |k, v|
|
|
137
|
+
access_frequency = v[:access_count] || 0
|
|
138
|
+
time_since_accessed = Time.now - (v[:last_accessed] || v[:added_at])
|
|
139
|
+
recency_factor = 1.0 / (1 + time_since_accessed / 3600.0)
|
|
137
140
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
141
|
+
# Higher score = more relevant
|
|
142
|
+
-(Math.log(1 + access_frequency) * recency_factor)
|
|
143
|
+
}.map { |_, v| v.dup }
|
|
144
|
+
else
|
|
145
|
+
raise ArgumentError, "Unknown strategy: #{strategy}. Use :recent, :frequent, or :balanced"
|
|
146
|
+
end
|
|
143
147
|
|
|
144
|
-
|
|
148
|
+
# Build context up to token limit
|
|
149
|
+
context_parts = []
|
|
150
|
+
current_tokens = 0
|
|
151
|
+
|
|
152
|
+
nodes.each do |node|
|
|
153
|
+
break if current_tokens + node[:token_count] > max
|
|
154
|
+
context_parts << node[:value]
|
|
155
|
+
current_tokens += node[:token_count]
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
context_parts.join("\n\n")
|
|
159
|
+
end
|
|
145
160
|
end
|
|
146
161
|
|
|
147
162
|
# Get current token count
|
|
@@ -149,7 +164,9 @@ class HTM
|
|
|
149
164
|
# @return [Integer] Total tokens in working memory
|
|
150
165
|
#
|
|
151
166
|
def token_count
|
|
152
|
-
@
|
|
167
|
+
@mutex.synchronize do
|
|
168
|
+
current_tokens_unlocked
|
|
169
|
+
end
|
|
153
170
|
end
|
|
154
171
|
|
|
155
172
|
# Get utilization percentage
|
|
@@ -157,7 +174,9 @@ class HTM
|
|
|
157
174
|
# @return [Float] Percentage of working memory used
|
|
158
175
|
#
|
|
159
176
|
def utilization_percentage
|
|
160
|
-
|
|
177
|
+
@mutex.synchronize do
|
|
178
|
+
(current_tokens_unlocked.to_f / @max_tokens * 100).round(2)
|
|
179
|
+
end
|
|
161
180
|
end
|
|
162
181
|
|
|
163
182
|
# Get node count
|
|
@@ -165,16 +184,92 @@ class HTM
|
|
|
165
184
|
# @return [Integer] Number of nodes in working memory
|
|
166
185
|
#
|
|
167
186
|
def node_count
|
|
168
|
-
@
|
|
187
|
+
@mutex.synchronize do
|
|
188
|
+
@nodes.size
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Clear all nodes from working memory
|
|
193
|
+
#
|
|
194
|
+
# @return [void]
|
|
195
|
+
#
|
|
196
|
+
def clear
|
|
197
|
+
@mutex.synchronize do
|
|
198
|
+
@nodes.clear
|
|
199
|
+
@access_order.clear
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# ===========================================================================
|
|
204
|
+
# Sync Methods (for inter-robot coordination via LISTEN/NOTIFY)
|
|
205
|
+
# ===========================================================================
|
|
206
|
+
|
|
207
|
+
# Add a node from sync notification (bypasses normal add flow)
|
|
208
|
+
#
|
|
209
|
+
# Called by RobotGroup when another robot adds to working memory.
|
|
210
|
+
# Does not trigger notifications to avoid infinite loops.
|
|
211
|
+
#
|
|
212
|
+
# @param id [Integer] Node database ID
|
|
213
|
+
# @param content [String] Node content
|
|
214
|
+
# @param token_count [Integer] Token count
|
|
215
|
+
# @param created_at [Time] When node was created
|
|
216
|
+
# @return [void]
|
|
217
|
+
#
|
|
218
|
+
def add_from_sync(id:, content:, token_count:, created_at:)
|
|
219
|
+
@mutex.synchronize do
|
|
220
|
+
key = id.to_s
|
|
221
|
+
return if @nodes.key?(key) # Already have this node
|
|
222
|
+
|
|
223
|
+
@nodes[key] = {
|
|
224
|
+
value: content,
|
|
225
|
+
token_count: token_count,
|
|
226
|
+
access_count: 0,
|
|
227
|
+
last_accessed: Time.now,
|
|
228
|
+
added_at: created_at,
|
|
229
|
+
from_recall: false,
|
|
230
|
+
from_sync: true
|
|
231
|
+
}
|
|
232
|
+
update_access_unlocked(key)
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Remove a node from sync notification
|
|
237
|
+
#
|
|
238
|
+
# Called by RobotGroup when another robot evicts from working memory.
|
|
239
|
+
#
|
|
240
|
+
# @param node_id [Integer] Node database ID
|
|
241
|
+
# @return [void]
|
|
242
|
+
#
|
|
243
|
+
def remove_from_sync(node_id)
|
|
244
|
+
@mutex.synchronize do
|
|
245
|
+
key = node_id.to_s
|
|
246
|
+
@nodes.delete(key)
|
|
247
|
+
@access_order.delete(key)
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Clear all nodes from sync notification
|
|
252
|
+
#
|
|
253
|
+
# Called by RobotGroup when another robot clears working memory.
|
|
254
|
+
#
|
|
255
|
+
# @return [void]
|
|
256
|
+
#
|
|
257
|
+
def clear_from_sync
|
|
258
|
+
@mutex.synchronize do
|
|
259
|
+
@nodes.clear
|
|
260
|
+
@access_order.clear
|
|
261
|
+
end
|
|
169
262
|
end
|
|
170
263
|
|
|
171
264
|
private
|
|
172
265
|
|
|
173
|
-
|
|
174
|
-
|
|
266
|
+
# Internal unlocked version - must be called within @mutex.synchronize
|
|
267
|
+
def current_tokens_unlocked
|
|
268
|
+
@nodes.values.sum { |n| n[:token_count] }
|
|
175
269
|
end
|
|
176
270
|
|
|
177
|
-
|
|
271
|
+
# Internal unlocked version - must be called within @mutex.synchronize
|
|
272
|
+
def update_access_unlocked(key)
|
|
178
273
|
@access_order.delete(key)
|
|
179
274
|
@access_order << key
|
|
180
275
|
end
|