htm 0.0.1 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. checksums.yaml +4 -4
  2. data/.aigcm_msg +1 -0
  3. data/.architecture/reviews/comprehensive-codebase-review.md +577 -0
  4. data/.claude/settings.local.json +92 -0
  5. data/.envrc +1 -0
  6. data/.irbrc +283 -80
  7. data/.tbls.yml +31 -0
  8. data/CHANGELOG.md +314 -16
  9. data/CLAUDE.md +603 -0
  10. data/README.md +76 -5
  11. data/Rakefile +5 -0
  12. data/SETUP.md +132 -101
  13. data/db/migrate/{20250101000001_enable_extensions.rb → 00001_enable_extensions.rb} +0 -1
  14. data/db/migrate/00002_create_robots.rb +11 -0
  15. data/db/migrate/00003_create_file_sources.rb +20 -0
  16. data/db/migrate/00004_create_nodes.rb +65 -0
  17. data/db/migrate/00005_create_tags.rb +13 -0
  18. data/db/migrate/00006_create_node_tags.rb +18 -0
  19. data/db/migrate/00007_create_robot_nodes.rb +26 -0
  20. data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +12 -0
  21. data/db/schema.sql +390 -36
  22. data/docs/api/database.md +19 -232
  23. data/docs/api/embedding-service.md +1 -7
  24. data/docs/api/htm.md +305 -364
  25. data/docs/api/index.md +1 -7
  26. data/docs/api/long-term-memory.md +342 -590
  27. data/docs/api/yard/HTM/ActiveRecordConfig.md +23 -0
  28. data/docs/api/yard/HTM/AuthorizationError.md +11 -0
  29. data/docs/api/yard/HTM/CircuitBreaker.md +92 -0
  30. data/docs/api/yard/HTM/CircuitBreakerOpenError.md +34 -0
  31. data/docs/api/yard/HTM/Configuration.md +175 -0
  32. data/docs/api/yard/HTM/Database.md +99 -0
  33. data/docs/api/yard/HTM/DatabaseError.md +14 -0
  34. data/docs/api/yard/HTM/EmbeddingError.md +18 -0
  35. data/docs/api/yard/HTM/EmbeddingService.md +58 -0
  36. data/docs/api/yard/HTM/Error.md +11 -0
  37. data/docs/api/yard/HTM/JobAdapter.md +39 -0
  38. data/docs/api/yard/HTM/LongTermMemory.md +342 -0
  39. data/docs/api/yard/HTM/NotFoundError.md +17 -0
  40. data/docs/api/yard/HTM/Observability.md +107 -0
  41. data/docs/api/yard/HTM/QueryTimeoutError.md +19 -0
  42. data/docs/api/yard/HTM/Railtie.md +27 -0
  43. data/docs/api/yard/HTM/ResourceExhaustedError.md +13 -0
  44. data/docs/api/yard/HTM/TagError.md +18 -0
  45. data/docs/api/yard/HTM/TagService.md +67 -0
  46. data/docs/api/yard/HTM/Timeframe/Result.md +24 -0
  47. data/docs/api/yard/HTM/Timeframe.md +40 -0
  48. data/docs/api/yard/HTM/TimeframeExtractor/Result.md +24 -0
  49. data/docs/api/yard/HTM/TimeframeExtractor.md +45 -0
  50. data/docs/api/yard/HTM/ValidationError.md +20 -0
  51. data/docs/api/yard/HTM/WorkingMemory.md +131 -0
  52. data/docs/api/yard/HTM.md +80 -0
  53. data/docs/api/yard/index.csv +179 -0
  54. data/docs/api/yard-reference.md +51 -0
  55. data/docs/architecture/adrs/001-postgresql-timescaledb.md +1 -1
  56. data/docs/architecture/adrs/003-ollama-embeddings.md +1 -1
  57. data/docs/architecture/adrs/010-redis-working-memory-rejected.md +2 -27
  58. data/docs/architecture/adrs/index.md +2 -13
  59. data/docs/architecture/hive-mind.md +165 -166
  60. data/docs/architecture/index.md +2 -2
  61. data/docs/architecture/overview.md +5 -171
  62. data/docs/architecture/two-tier-memory.md +1 -35
  63. data/docs/assets/images/adr-010-current-architecture.svg +37 -0
  64. data/docs/assets/images/adr-010-proposed-architecture.svg +48 -0
  65. data/docs/assets/images/adr-dependency-tree.svg +93 -0
  66. data/docs/assets/images/class-hierarchy.svg +55 -0
  67. data/docs/assets/images/exception-hierarchy.svg +45 -0
  68. data/docs/assets/images/htm-architecture-overview.svg +83 -0
  69. data/docs/assets/images/htm-complete-memory-flow.svg +160 -0
  70. data/docs/assets/images/htm-context-assembly-flow.svg +148 -0
  71. data/docs/assets/images/htm-eviction-process.svg +141 -0
  72. data/docs/assets/images/htm-memory-addition-flow.svg +138 -0
  73. data/docs/assets/images/htm-memory-recall-flow.svg +152 -0
  74. data/docs/assets/images/htm-node-states.svg +123 -0
  75. data/docs/assets/images/project-structure.svg +78 -0
  76. data/docs/assets/images/test-directory-structure.svg +38 -0
  77. data/{dbdoc → docs/database}/README.md +127 -125
  78. data/docs/database/public.file_sources.md +42 -0
  79. data/docs/database/public.file_sources.svg +211 -0
  80. data/{dbdoc → docs/database}/public.node_tags.md +7 -8
  81. data/docs/database/public.node_tags.svg +239 -0
  82. data/{dbdoc → docs/database}/public.nodes.md +22 -17
  83. data/docs/database/public.nodes.svg +271 -0
  84. data/docs/database/public.robot_nodes.md +46 -0
  85. data/docs/database/public.robot_nodes.svg +243 -0
  86. data/{dbdoc → docs/database}/public.robots.md +2 -3
  87. data/docs/database/public.robots.svg +161 -0
  88. data/docs/database/public.tags.svg +139 -0
  89. data/{dbdoc → docs/database}/schema.json +941 -630
  90. data/docs/database/schema.svg +282 -0
  91. data/docs/development/index.md +1 -29
  92. data/docs/development/schema.md +134 -309
  93. data/docs/development/testing.md +1 -9
  94. data/docs/getting-started/index.md +47 -0
  95. data/docs/{installation.md → getting-started/installation.md} +2 -2
  96. data/docs/{quick-start.md → getting-started/quick-start.md} +5 -5
  97. data/docs/guides/adding-memories.md +295 -643
  98. data/docs/guides/recalling-memories.md +36 -1
  99. data/docs/guides/search-strategies.md +85 -51
  100. data/docs/images/htm-er-diagram.svg +156 -0
  101. data/docs/index.md +16 -31
  102. data/docs/multi_framework_support.md +4 -4
  103. data/examples/README.md +280 -0
  104. data/examples/basic_usage.rb +18 -16
  105. data/examples/cli_app/htm_cli.rb +146 -8
  106. data/examples/cli_app/temp.log +93 -0
  107. data/examples/custom_llm_configuration.rb +1 -2
  108. data/examples/example_app/app.rb +11 -14
  109. data/examples/file_loader_usage.rb +177 -0
  110. data/examples/robot_groups/lib/robot_group.rb +419 -0
  111. data/examples/robot_groups/lib/working_memory_channel.rb +140 -0
  112. data/examples/robot_groups/multi_process.rb +286 -0
  113. data/examples/robot_groups/robot_worker.rb +136 -0
  114. data/examples/robot_groups/same_process.rb +229 -0
  115. data/examples/sinatra_app/Gemfile +1 -0
  116. data/examples/sinatra_app/Gemfile.lock +166 -0
  117. data/examples/sinatra_app/app.rb +219 -24
  118. data/examples/timeframe_demo.rb +276 -0
  119. data/lib/htm/active_record_config.rb +10 -3
  120. data/lib/htm/circuit_breaker.rb +202 -0
  121. data/lib/htm/configuration.rb +313 -80
  122. data/lib/htm/database.rb +67 -36
  123. data/lib/htm/embedding_service.rb +39 -2
  124. data/lib/htm/errors.rb +131 -11
  125. data/lib/htm/{sinatra.rb → integrations/sinatra.rb} +87 -12
  126. data/lib/htm/job_adapter.rb +10 -3
  127. data/lib/htm/jobs/generate_embedding_job.rb +5 -4
  128. data/lib/htm/jobs/generate_tags_job.rb +4 -0
  129. data/lib/htm/loaders/markdown_loader.rb +263 -0
  130. data/lib/htm/loaders/paragraph_chunker.rb +112 -0
  131. data/lib/htm/long_term_memory.rb +601 -321
  132. data/lib/htm/models/file_source.rb +99 -0
  133. data/lib/htm/models/node.rb +116 -12
  134. data/lib/htm/models/robot.rb +53 -4
  135. data/lib/htm/models/robot_node.rb +51 -0
  136. data/lib/htm/models/tag.rb +302 -0
  137. data/lib/htm/observability.rb +395 -0
  138. data/lib/htm/tag_service.rb +60 -3
  139. data/lib/htm/tasks.rb +29 -0
  140. data/lib/htm/timeframe.rb +194 -0
  141. data/lib/htm/timeframe_extractor.rb +307 -0
  142. data/lib/htm/version.rb +1 -1
  143. data/lib/htm/working_memory.rb +165 -70
  144. data/lib/htm.rb +352 -133
  145. data/lib/tasks/doc.rake +300 -0
  146. data/lib/tasks/files.rake +299 -0
  147. data/lib/tasks/htm.rake +188 -2
  148. data/lib/tasks/jobs.rake +10 -12
  149. data/lib/tasks/tags.rake +194 -0
  150. data/mkdocs.yml +91 -9
  151. data/notes/ARCHITECTURE_REVIEW.md +1167 -0
  152. data/notes/IMPLEMENTATION_SUMMARY.md +606 -0
  153. data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +451 -0
  154. data/notes/next_steps.md +100 -0
  155. data/notes/plan.md +627 -0
  156. data/notes/tag_ontology_enhancement_ideas.md +222 -0
  157. data/notes/timescaledb_removal_summary.md +200 -0
  158. metadata +177 -37
  159. data/db/migrate/20250101000002_create_robots.rb +0 -14
  160. data/db/migrate/20250101000003_create_nodes.rb +0 -42
  161. data/db/migrate/20250101000005_create_tags.rb +0 -38
  162. data/db/migrate/20250101000007_add_node_vector_indexes.rb +0 -30
  163. data/dbdoc/public.node_tags.svg +0 -112
  164. data/dbdoc/public.nodes.svg +0 -118
  165. data/dbdoc/public.robots.svg +0 -90
  166. data/dbdoc/public.tags.svg +0 -60
  167. data/dbdoc/schema.svg +0 -154
  168. data/{dbdoc → docs/database}/public.node_stats.md +0 -0
  169. data/{dbdoc → docs/database}/public.node_stats.svg +0 -0
  170. data/{dbdoc → docs/database}/public.nodes_tags.md +0 -0
  171. data/{dbdoc → docs/database}/public.nodes_tags.svg +0 -0
  172. data/{dbdoc → docs/database}/public.ontology_structure.md +0 -0
  173. data/{dbdoc → docs/database}/public.ontology_structure.svg +0 -0
  174. data/{dbdoc → docs/database}/public.operations_log.md +0 -0
  175. data/{dbdoc → docs/database}/public.operations_log.svg +0 -0
  176. data/{dbdoc → docs/database}/public.relationships.md +0 -0
  177. data/{dbdoc → docs/database}/public.relationships.svg +0 -0
  178. data/{dbdoc → docs/database}/public.robot_activity.md +0 -0
  179. data/{dbdoc → docs/database}/public.robot_activity.svg +0 -0
  180. data/{dbdoc → docs/database}/public.schema_migrations.md +0 -0
  181. data/{dbdoc → docs/database}/public.schema_migrations.svg +0 -0
  182. data/{dbdoc → docs/database}/public.tags.md +3 -3
  183. /data/{dbdoc → docs/database}/public.topic_relationships.md +0 -0
  184. /data/{dbdoc → docs/database}/public.topic_relationships.svg +0 -0
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: htm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dewayne VanHoozer
@@ -93,6 +93,34 @@ dependencies:
93
93
  - - ">="
94
94
  - !ruby/object:Gem::Version
95
95
  version: '0'
96
+ - !ruby/object:Gem::Dependency
97
+ name: ruby-progressbar
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ type: :runtime
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ - !ruby/object:Gem::Dependency
111
+ name: chronic
112
+ requirement: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: '0'
117
+ type: :runtime
118
+ prerelease: false
119
+ version_requirements: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
96
124
  - !ruby/object:Gem::Dependency
97
125
  name: rake
98
126
  requirement: !ruby/object:Gem::Requirement
@@ -149,6 +177,34 @@ dependencies:
149
177
  - - ">="
150
178
  - !ruby/object:Gem::Version
151
179
  version: '0'
180
+ - !ruby/object:Gem::Dependency
181
+ name: yard
182
+ requirement: !ruby/object:Gem::Requirement
183
+ requirements:
184
+ - - ">="
185
+ - !ruby/object:Gem::Version
186
+ version: '0'
187
+ type: :development
188
+ prerelease: false
189
+ version_requirements: !ruby/object:Gem::Requirement
190
+ requirements:
191
+ - - ">="
192
+ - !ruby/object:Gem::Version
193
+ version: '0'
194
+ - !ruby/object:Gem::Dependency
195
+ name: yard-markdown
196
+ requirement: !ruby/object:Gem::Requirement
197
+ requirements:
198
+ - - ">="
199
+ - !ruby/object:Gem::Version
200
+ version: '0'
201
+ type: :development
202
+ prerelease: false
203
+ version_requirements: !ruby/object:Gem::Requirement
204
+ requirements:
205
+ - - ">="
206
+ - !ruby/object:Gem::Version
207
+ version: '0'
152
208
  description: |
153
209
  HTM (Hierarchical Temporary Memory) provides intelligent memory management for
154
210
  LLM-based applications. It implements a two-tier memory system with
@@ -161,6 +217,7 @@ executables: []
161
217
  extensions: []
162
218
  extra_rdoc_files: []
163
219
  files:
220
+ - ".aigcm_msg"
164
221
  - ".architecture/decisions/adrs/001-use-postgresql-timescaledb-storage.md"
165
222
  - ".architecture/decisions/adrs/002-two-tier-memory-architecture.md"
166
223
  - ".architecture/decisions/adrs/003-ollama-default-embedding-provider.md"
@@ -179,59 +236,67 @@ files:
179
236
  - ".architecture/decisions/adrs/016-async-embedding-and-tag-generation.md"
180
237
  - ".architecture/members.yml"
181
238
  - ".architecture/reviews/2025-10-29-llm-configuration-and-async-processing-review.md"
239
+ - ".architecture/reviews/comprehensive-codebase-review.md"
182
240
  - ".architecture/reviews/initial-system-analysis.md"
241
+ - ".claude/settings.local.json"
183
242
  - ".envrc"
184
243
  - ".irbrc"
244
+ - ".tbls.yml"
185
245
  - CHANGELOG.md
246
+ - CLAUDE.md
186
247
  - COMMITS.md
187
248
  - LICENSE
188
249
  - README.md
189
250
  - Rakefile
190
251
  - SETUP.md
191
252
  - config/database.yml
192
- - db/migrate/20250101000001_enable_extensions.rb
193
- - db/migrate/20250101000002_create_robots.rb
194
- - db/migrate/20250101000003_create_nodes.rb
195
- - db/migrate/20250101000005_create_tags.rb
196
- - db/migrate/20250101000007_add_node_vector_indexes.rb
253
+ - db/migrate/00001_enable_extensions.rb
254
+ - db/migrate/00002_create_robots.rb
255
+ - db/migrate/00003_create_file_sources.rb
256
+ - db/migrate/00004_create_nodes.rb
257
+ - db/migrate/00005_create_tags.rb
258
+ - db/migrate/00006_create_node_tags.rb
259
+ - db/migrate/00007_create_robot_nodes.rb
260
+ - db/migrate/00009_add_working_memory_to_robot_nodes.rb
197
261
  - db/schema.sql
198
262
  - db/seed_data/README.md
199
263
  - db/seed_data/presidents.md
200
264
  - db/seed_data/states.md
201
265
  - db/seeds.rb
202
- - dbdoc/README.md
203
- - dbdoc/public.node_stats.md
204
- - dbdoc/public.node_stats.svg
205
- - dbdoc/public.node_tags.md
206
- - dbdoc/public.node_tags.svg
207
- - dbdoc/public.nodes.md
208
- - dbdoc/public.nodes.svg
209
- - dbdoc/public.nodes_tags.md
210
- - dbdoc/public.nodes_tags.svg
211
- - dbdoc/public.ontology_structure.md
212
- - dbdoc/public.ontology_structure.svg
213
- - dbdoc/public.operations_log.md
214
- - dbdoc/public.operations_log.svg
215
- - dbdoc/public.relationships.md
216
- - dbdoc/public.relationships.svg
217
- - dbdoc/public.robot_activity.md
218
- - dbdoc/public.robot_activity.svg
219
- - dbdoc/public.robots.md
220
- - dbdoc/public.robots.svg
221
- - dbdoc/public.schema_migrations.md
222
- - dbdoc/public.schema_migrations.svg
223
- - dbdoc/public.tags.md
224
- - dbdoc/public.tags.svg
225
- - dbdoc/public.topic_relationships.md
226
- - dbdoc/public.topic_relationships.svg
227
- - dbdoc/schema.json
228
- - dbdoc/schema.svg
229
266
  - docs/api/database.md
230
267
  - docs/api/embedding-service.md
231
268
  - docs/api/htm.md
232
269
  - docs/api/index.md
233
270
  - docs/api/long-term-memory.md
234
271
  - docs/api/working-memory.md
272
+ - docs/api/yard-reference.md
273
+ - docs/api/yard/HTM.md
274
+ - docs/api/yard/HTM/ActiveRecordConfig.md
275
+ - docs/api/yard/HTM/AuthorizationError.md
276
+ - docs/api/yard/HTM/CircuitBreaker.md
277
+ - docs/api/yard/HTM/CircuitBreakerOpenError.md
278
+ - docs/api/yard/HTM/Configuration.md
279
+ - docs/api/yard/HTM/Database.md
280
+ - docs/api/yard/HTM/DatabaseError.md
281
+ - docs/api/yard/HTM/EmbeddingError.md
282
+ - docs/api/yard/HTM/EmbeddingService.md
283
+ - docs/api/yard/HTM/Error.md
284
+ - docs/api/yard/HTM/JobAdapter.md
285
+ - docs/api/yard/HTM/LongTermMemory.md
286
+ - docs/api/yard/HTM/NotFoundError.md
287
+ - docs/api/yard/HTM/Observability.md
288
+ - docs/api/yard/HTM/QueryTimeoutError.md
289
+ - docs/api/yard/HTM/Railtie.md
290
+ - docs/api/yard/HTM/ResourceExhaustedError.md
291
+ - docs/api/yard/HTM/TagError.md
292
+ - docs/api/yard/HTM/TagService.md
293
+ - docs/api/yard/HTM/Timeframe.md
294
+ - docs/api/yard/HTM/Timeframe/Result.md
295
+ - docs/api/yard/HTM/TimeframeExtractor.md
296
+ - docs/api/yard/HTM/TimeframeExtractor/Result.md
297
+ - docs/api/yard/HTM/ValidationError.md
298
+ - docs/api/yard/HTM/WorkingMemory.md
299
+ - docs/api/yard/index.csv
235
300
  - docs/architecture/adrs/001-postgresql-timescaledb.md
236
301
  - docs/architecture/adrs/002-two-tier-memory.md
237
302
  - docs/architecture/adrs/003-ollama-embeddings.md
@@ -249,23 +314,71 @@ files:
249
314
  - docs/architecture/overview.md
250
315
  - docs/architecture/two-tier-memory.md
251
316
  - docs/assets/css/custom.css
317
+ - docs/assets/images/adr-010-current-architecture.svg
318
+ - docs/assets/images/adr-010-proposed-architecture.svg
319
+ - docs/assets/images/adr-dependency-tree.svg
320
+ - docs/assets/images/class-hierarchy.svg
321
+ - docs/assets/images/exception-hierarchy.svg
322
+ - docs/assets/images/htm-architecture-overview.svg
323
+ - docs/assets/images/htm-complete-memory-flow.svg
324
+ - docs/assets/images/htm-context-assembly-flow.svg
252
325
  - docs/assets/images/htm-core-components.svg
253
326
  - docs/assets/images/htm-database-schema.svg
327
+ - docs/assets/images/htm-eviction-process.svg
254
328
  - docs/assets/images/htm-hive-mind-architecture.svg
255
329
  - docs/assets/images/htm-importance-scoring-framework.svg
256
330
  - docs/assets/images/htm-layered-architecture.svg
257
331
  - docs/assets/images/htm-long-term-memory-architecture.svg
332
+ - docs/assets/images/htm-memory-addition-flow.svg
333
+ - docs/assets/images/htm-memory-recall-flow.svg
334
+ - docs/assets/images/htm-node-states.svg
258
335
  - docs/assets/images/htm-working-memory-architecture.svg
259
336
  - docs/assets/images/htm.jpg
260
337
  - docs/assets/images/htm_demo.gif
338
+ - docs/assets/images/project-structure.svg
339
+ - docs/assets/images/test-directory-structure.svg
261
340
  - docs/assets/js/mathjax.js
262
341
  - docs/assets/videos/htm_video.mp4
342
+ - docs/database/README.md
343
+ - docs/database/public.file_sources.md
344
+ - docs/database/public.file_sources.svg
345
+ - docs/database/public.node_stats.md
346
+ - docs/database/public.node_stats.svg
347
+ - docs/database/public.node_tags.md
348
+ - docs/database/public.node_tags.svg
349
+ - docs/database/public.nodes.md
350
+ - docs/database/public.nodes.svg
351
+ - docs/database/public.nodes_tags.md
352
+ - docs/database/public.nodes_tags.svg
353
+ - docs/database/public.ontology_structure.md
354
+ - docs/database/public.ontology_structure.svg
355
+ - docs/database/public.operations_log.md
356
+ - docs/database/public.operations_log.svg
357
+ - docs/database/public.relationships.md
358
+ - docs/database/public.relationships.svg
359
+ - docs/database/public.robot_activity.md
360
+ - docs/database/public.robot_activity.svg
361
+ - docs/database/public.robot_nodes.md
362
+ - docs/database/public.robot_nodes.svg
363
+ - docs/database/public.robots.md
364
+ - docs/database/public.robots.svg
365
+ - docs/database/public.schema_migrations.md
366
+ - docs/database/public.schema_migrations.svg
367
+ - docs/database/public.tags.md
368
+ - docs/database/public.tags.svg
369
+ - docs/database/public.topic_relationships.md
370
+ - docs/database/public.topic_relationships.svg
371
+ - docs/database/schema.json
372
+ - docs/database/schema.svg
263
373
  - docs/database_rake_tasks.md
264
374
  - docs/development/contributing.md
265
375
  - docs/development/index.md
266
376
  - docs/development/schema.md
267
377
  - docs/development/setup.md
268
378
  - docs/development/testing.md
379
+ - docs/getting-started/index.md
380
+ - docs/getting-started/installation.md
381
+ - docs/getting-started/quick-start.md
269
382
  - docs/guides/adding-memories.md
270
383
  - docs/guides/context-assembly.md
271
384
  - docs/guides/getting-started.md
@@ -275,44 +388,71 @@ files:
275
388
  - docs/guides/recalling-memories.md
276
389
  - docs/guides/search-strategies.md
277
390
  - docs/guides/working-memory.md
391
+ - docs/images/htm-er-diagram.svg
278
392
  - docs/index.md
279
- - docs/installation.md
280
393
  - docs/multi_framework_support.md
281
- - docs/quick-start.md
282
394
  - docs/setup_local_database.md
283
395
  - docs/using_rake_tasks_in_your_app.md
396
+ - examples/README.md
284
397
  - examples/basic_usage.rb
285
398
  - examples/cli_app/README.md
286
399
  - examples/cli_app/htm_cli.rb
400
+ - examples/cli_app/temp.log
287
401
  - examples/custom_llm_configuration.rb
288
402
  - examples/example_app/Rakefile
289
403
  - examples/example_app/app.rb
404
+ - examples/file_loader_usage.rb
405
+ - examples/robot_groups/lib/robot_group.rb
406
+ - examples/robot_groups/lib/working_memory_channel.rb
407
+ - examples/robot_groups/multi_process.rb
408
+ - examples/robot_groups/robot_worker.rb
409
+ - examples/robot_groups/same_process.rb
290
410
  - examples/sinatra_app/Gemfile
411
+ - examples/sinatra_app/Gemfile.lock
291
412
  - examples/sinatra_app/app.rb
413
+ - examples/timeframe_demo.rb
292
414
  - lib/htm.rb
293
415
  - lib/htm/active_record_config.rb
416
+ - lib/htm/circuit_breaker.rb
294
417
  - lib/htm/configuration.rb
295
418
  - lib/htm/database.rb
296
419
  - lib/htm/embedding_service.rb
297
420
  - lib/htm/errors.rb
421
+ - lib/htm/integrations/sinatra.rb
298
422
  - lib/htm/job_adapter.rb
299
423
  - lib/htm/jobs/generate_embedding_job.rb
300
424
  - lib/htm/jobs/generate_tags_job.rb
425
+ - lib/htm/loaders/markdown_loader.rb
426
+ - lib/htm/loaders/paragraph_chunker.rb
301
427
  - lib/htm/long_term_memory.rb
428
+ - lib/htm/models/file_source.rb
302
429
  - lib/htm/models/node.rb
303
430
  - lib/htm/models/node_tag.rb
304
431
  - lib/htm/models/robot.rb
432
+ - lib/htm/models/robot_node.rb
305
433
  - lib/htm/models/tag.rb
434
+ - lib/htm/observability.rb
306
435
  - lib/htm/railtie.rb
307
- - lib/htm/sinatra.rb
308
436
  - lib/htm/tag_service.rb
309
437
  - lib/htm/tasks.rb
438
+ - lib/htm/timeframe.rb
439
+ - lib/htm/timeframe_extractor.rb
310
440
  - lib/htm/version.rb
311
441
  - lib/htm/working_memory.rb
312
442
  - lib/tasks/db.rake
443
+ - lib/tasks/doc.rake
444
+ - lib/tasks/files.rake
313
445
  - lib/tasks/htm.rake
314
446
  - lib/tasks/jobs.rake
447
+ - lib/tasks/tags.rake
315
448
  - mkdocs.yml
449
+ - notes/ARCHITECTURE_REVIEW.md
450
+ - notes/IMPLEMENTATION_SUMMARY.md
451
+ - notes/MULTI_FRAMEWORK_IMPLEMENTATION.md
452
+ - notes/next_steps.md
453
+ - notes/plan.md
454
+ - notes/tag_ontology_enhancement_ideas.md
455
+ - notes/timescaledb_removal_summary.md
316
456
  - scripts/install_local_database.sh
317
457
  homepage: https://github.com/madbomber/htm
318
458
  licenses:
@@ -335,7 +475,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
335
475
  - !ruby/object:Gem::Version
336
476
  version: '0'
337
477
  requirements: []
338
- rubygems_version: 3.7.2
478
+ rubygems_version: 4.0.0.dev
339
479
  specification_version: 4
340
480
  summary: Hierarchical Temporary Memory for LLM robots
341
481
  test_files: []
@@ -1,14 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class CreateRobots < ActiveRecord::Migration[7.1]
4
- def change
5
- unless table_exists?(:robots)
6
- create_table :robots, comment: 'Registry of all LLM robots using the HTM system' do |t|
7
- t.text :name, comment: 'Human-readable name for the robot'
8
- t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When the robot was first registered'
9
- t.timestamptz :last_active, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'Last time the robot accessed the system'
10
- t.jsonb :metadata, comment: 'Robot-specific configuration and metadata'
11
- end
12
- end
13
- end
14
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class CreateNodes < ActiveRecord::Migration[7.1]
4
- def change
5
- unless table_exists?(:nodes)
6
- create_table :nodes, comment: 'Core memory storage for conversation messages and context' do |t|
7
- t.text :content, null: false, comment: 'The conversation message/utterance content'
8
- t.text :source, default: '', comment: 'From where the content came (empty string if unknown)'
9
- t.integer :access_count, default: 0, null: false, comment: 'Number of times this node has been accessed/retrieved'
10
- t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this memory was created'
11
- t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this memory was last modified'
12
- t.timestamptz :last_accessed, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this memory was last accessed'
13
- t.integer :token_count, comment: 'Number of tokens in the content (for context budget management)'
14
- t.boolean :in_working_memory, default: false, comment: 'Whether this memory is currently in working memory'
15
- t.bigint :robot_id, null: false, comment: 'ID of the robot that owns this memory'
16
- t.vector :embedding, limit: 2000, comment: 'Vector embedding (max 2000 dimensions) for semantic search'
17
- t.integer :embedding_dimension, comment: 'Actual number of dimensions used in the embedding vector (max 2000)'
18
- end
19
-
20
- # Basic indexes for common queries
21
- add_index :nodes, :created_at, name: 'idx_nodes_created_at'
22
- add_index :nodes, :updated_at, name: 'idx_nodes_updated_at'
23
- add_index :nodes, :last_accessed, name: 'idx_nodes_last_accessed'
24
- add_index :nodes, :access_count, name: 'idx_nodes_access_count'
25
- add_index :nodes, :robot_id, name: 'idx_nodes_robot_id'
26
- add_index :nodes, :source, name: 'idx_nodes_source'
27
- add_index :nodes, :in_working_memory, name: 'idx_nodes_in_working_memory'
28
-
29
- # Add check constraint for embedding dimensions
30
- # Only validates when embedding_dimension is provided (allows NULL for nodes without embeddings)
31
- execute <<-SQL
32
- ALTER TABLE nodes ADD CONSTRAINT check_embedding_dimension
33
- CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
34
- SQL
35
- end
36
-
37
- # Foreign key to robots table (outside table_exists check so it gets added even if table already exists)
38
- unless foreign_key_exists?(:nodes, :robots, column: :robot_id)
39
- add_foreign_key :nodes, :robots, column: :robot_id, primary_key: :id, on_delete: :cascade
40
- end
41
- end
42
- end
@@ -1,38 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class CreateTags < ActiveRecord::Migration[7.1]
4
- def change
5
- # Create tags table with unique tag names
6
- unless table_exists?(:tags)
7
- create_table :tags, comment: 'Unique tag names for categorization' do |t|
8
- t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
9
- t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
10
- end
11
-
12
- add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
13
- add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
14
- end
15
-
16
- # Create join table for many-to-many relationship
17
- unless table_exists?(:node_tags)
18
- create_table :node_tags, comment: 'Join table connecting nodes to tags (many-to-many)' do |t|
19
- t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
20
- t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
21
- t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
22
- end
23
-
24
- add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
25
- add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
26
- add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
27
- end
28
-
29
- # Add foreign keys (outside table_exists check so they get added even if table already exists)
30
- unless foreign_key_exists?(:node_tags, :nodes, column: :node_id)
31
- add_foreign_key :node_tags, :nodes, column: :node_id, primary_key: :id, on_delete: :cascade
32
- end
33
-
34
- unless foreign_key_exists?(:node_tags, :tags, column: :tag_id)
35
- add_foreign_key :node_tags, :tags, column: :tag_id, primary_key: :id, on_delete: :cascade
36
- end
37
- end
38
- end
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class AddNodeVectorIndexes < ActiveRecord::Migration[7.1]
4
- def up
5
- # Vector similarity search index (HNSW for better performance)
6
- execute <<-SQL
7
- CREATE INDEX IF NOT EXISTS idx_nodes_embedding ON nodes
8
- USING hnsw (embedding vector_cosine_ops)
9
- WITH (m = 16, ef_construction = 64)
10
- SQL
11
-
12
- # Full-text search on conversation content
13
- execute <<-SQL
14
- CREATE INDEX IF NOT EXISTS idx_nodes_content_gin ON nodes
15
- USING gin(to_tsvector('english', content))
16
- SQL
17
-
18
- # Trigram indexes for fuzzy matching on conversation content
19
- execute <<-SQL
20
- CREATE INDEX IF NOT EXISTS idx_nodes_content_trgm ON nodes
21
- USING gin(content gin_trgm_ops)
22
- SQL
23
- end
24
-
25
- def down
26
- remove_index :nodes, name: 'idx_nodes_embedding'
27
- remove_index :nodes, name: 'idx_nodes_content_gin'
28
- remove_index :nodes, name: 'idx_nodes_content_trgm'
29
- end
30
- end
@@ -1,112 +0,0 @@
1
- <?xml version="1.0" encoding="UTF-8" standalone="no"?>
2
- <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
3
- "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
4
- <!-- Generated by graphviz version 12.1.2 (20240928.0832)
5
- -->
6
- <!-- Title: public.node_tags Pages: 1 -->
7
- <svg width="961pt" height="787pt"
8
- viewBox="0.00 0.00 961.37 786.80" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
9
- <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 782.8)">
10
- <title>public.node_tags</title>
11
- <polygon fill="white" stroke="none" points="-4,4 -4,-782.8 957.37,-782.8 957.37,4 -4,4"/>
12
- <!-- public.node_tags -->
13
- <g id="node1" class="node">
14
- <title>public.node_tags</title>
15
- <polygon fill="#efefef" stroke="none" points="354.55,-697 354.55,-732.6 615.51,-732.6 615.51,-697 354.55,-697"/>
16
- <polygon fill="none" stroke="black" points="354.55,-697 354.55,-732.6 615.51,-732.6 615.51,-697 354.55,-697"/>
17
- <text text-anchor="start" x="361.55" y="-710.4" font-family="Arial Bold" font-size="18.00">public.node_tags</text>
18
- <text text-anchor="start" x="484.02" y="-710.4" font-family="Arial" font-size="14.00">    </text>
19
- <text text-anchor="start" x="515.14" y="-710.4" font-family="Arial" font-size="14.00" fill="#666666">[BASE TABLE]</text>
20
- <polygon fill="none" stroke="black" points="354.55,-666.2 354.55,-697 615.51,-697 615.51,-666.2 354.55,-666.2"/>
21
- <text text-anchor="start" x="361.55" y="-678.4" font-family="Arial" font-size="14.00">id </text>
22
- <text text-anchor="start" x="376.33" y="-678.4" font-family="Arial" font-size="14.00" fill="#666666">[bigint]</text>
23
- <polygon fill="none" stroke="black" points="354.55,-635.4 354.55,-666.2 615.51,-666.2 615.51,-635.4 354.55,-635.4"/>
24
- <text text-anchor="start" x="361.55" y="-647.6" font-family="Arial" font-size="14.00">node_id </text>
25
- <text text-anchor="start" x="415.26" y="-647.6" font-family="Arial" font-size="14.00" fill="#666666">[bigint]</text>
26
- <polygon fill="none" stroke="black" points="354.55,-604.6 354.55,-635.4 615.51,-635.4 615.51,-604.6 354.55,-604.6"/>
27
- <text text-anchor="start" x="361.55" y="-616.8" font-family="Arial" font-size="14.00">tag_id </text>
28
- <text text-anchor="start" x="403.58" y="-616.8" font-family="Arial" font-size="14.00" fill="#666666">[bigint]</text>
29
- <polygon fill="none" stroke="black" points="354.55,-573.8 354.55,-604.6 615.51,-604.6 615.51,-573.8 354.55,-573.8"/>
30
- <text text-anchor="start" x="361.55" y="-586" font-family="Arial" font-size="14.00">created_at </text>
31
- <text text-anchor="start" x="431.59" y="-586" font-family="Arial" font-size="14.00" fill="#666666">[timestamp with time zone]</text>
32
- <polygon fill="none" stroke="black" stroke-width="3" points="353.05,-572.3 353.05,-734.1 617.01,-734.1 617.01,-572.3 353.05,-572.3"/>
33
- </g>
34
- <!-- public.nodes -->
35
- <g id="node2" class="node">
36
- <title>public.nodes</title>
37
- <polygon fill="#efefef" stroke="none" points="166.86,-412.8 166.86,-448.4 439.19,-448.4 439.19,-412.8 166.86,-412.8"/>
38
- <polygon fill="none" stroke="black" points="166.86,-412.8 166.86,-448.4 439.19,-448.4 439.19,-412.8 166.86,-412.8"/>
39
- <text text-anchor="start" x="195.04" y="-426.2" font-family="Arial Bold" font-size="18.00">public.nodes</text>
40
- <text text-anchor="start" x="286.53" y="-426.2" font-family="Arial" font-size="14.00">    </text>
41
- <text text-anchor="start" x="317.64" y="-426.2" font-family="Arial" font-size="14.00" fill="#666666">[BASE TABLE]</text>
42
- <polygon fill="none" stroke="black" points="166.86,-382 166.86,-412.8 439.19,-412.8 439.19,-382 166.86,-382"/>
43
- <text text-anchor="start" x="173.86" y="-394.2" font-family="Arial" font-size="14.00">id </text>
44
- <text text-anchor="start" x="188.65" y="-394.2" font-family="Arial" font-size="14.00" fill="#666666">[bigint]</text>
45
- <polygon fill="none" stroke="black" points="166.86,-351.2 166.86,-382 439.19,-382 439.19,-351.2 166.86,-351.2"/>
46
- <text text-anchor="start" x="173.86" y="-363.4" font-family="Arial" font-size="14.00">content </text>
47
- <text text-anchor="start" x="223.68" y="-363.4" font-family="Arial" font-size="14.00" fill="#666666">[text]</text>
48
- <polygon fill="none" stroke="black" points="166.86,-320.4 166.86,-351.2 439.19,-351.2 439.19,-320.4 166.86,-320.4"/>
49
- <text text-anchor="start" x="173.86" y="-332.6" font-family="Arial" font-size="14.00">source </text>
50
- <text text-anchor="start" x="219.77" y="-332.6" font-family="Arial" font-size="14.00" fill="#666666">[text]</text>
51
- <polygon fill="none" stroke="black" points="166.86,-289.6 166.86,-320.4 439.19,-320.4 439.19,-289.6 166.86,-289.6"/>
52
- <text text-anchor="start" x="173.86" y="-301.8" font-family="Arial" font-size="14.00">access_count </text>
53
- <text text-anchor="start" x="263.36" y="-301.8" font-family="Arial" font-size="14.00" fill="#666666">[integer]</text>
54
- <polygon fill="none" stroke="black" points="166.86,-258.8 166.86,-289.6 439.19,-289.6 439.19,-258.8 166.86,-258.8"/>
55
- <text text-anchor="start" x="173.86" y="-271" font-family="Arial" font-size="14.00">created_at </text>
56
- <text text-anchor="start" x="243.91" y="-271" font-family="Arial" font-size="14.00" fill="#666666">[timestamp with time zone]</text>
57
- <polygon fill="none" stroke="black" points="166.86,-228 166.86,-258.8 439.19,-258.8 439.19,-228 166.86,-228"/>
58
- <text text-anchor="start" x="173.86" y="-240.2" font-family="Arial" font-size="14.00">updated_at </text>
59
- <text text-anchor="start" x="247.82" y="-240.2" font-family="Arial" font-size="14.00" fill="#666666">[timestamp with time zone]</text>
60
- <polygon fill="none" stroke="black" points="166.86,-197.2 166.86,-228 439.19,-228 439.19,-197.2 166.86,-197.2"/>
61
- <text text-anchor="start" x="173.86" y="-209.4" font-family="Arial" font-size="14.00">last_accessed </text>
62
- <text text-anchor="start" x="266.47" y="-209.4" font-family="Arial" font-size="14.00" fill="#666666">[timestamp with time zone]</text>
63
- <polygon fill="none" stroke="black" points="166.86,-166.4 166.86,-197.2 439.19,-197.2 439.19,-166.4 166.86,-166.4"/>
64
- <text text-anchor="start" x="173.86" y="-178.6" font-family="Arial" font-size="14.00">token_count </text>
65
- <text text-anchor="start" x="254.04" y="-178.6" font-family="Arial" font-size="14.00" fill="#666666">[integer]</text>
66
- <polygon fill="none" stroke="black" points="166.86,-135.6 166.86,-166.4 439.19,-166.4 439.19,-135.6 166.86,-135.6"/>
67
- <text text-anchor="start" x="173.86" y="-147.8" font-family="Arial" font-size="14.00">in_working_memory </text>
68
- <text text-anchor="start" x="303.02" y="-147.8" font-family="Arial" font-size="14.00" fill="#666666">[boolean]</text>
69
- <polygon fill="none" stroke="black" points="166.86,-104.8 166.86,-135.6 439.19,-135.6 439.19,-104.8 166.86,-104.8"/>
70
- <text text-anchor="start" x="173.86" y="-117" font-family="Arial" font-size="14.00">robot_id </text>
71
- <text text-anchor="start" x="228.35" y="-117" font-family="Arial" font-size="14.00" fill="#666666">[bigint]</text>
72
- <polygon fill="none" stroke="black" points="166.86,-74 166.86,-104.8 439.19,-104.8 439.19,-74 166.86,-74"/>
73
- <text text-anchor="start" x="173.86" y="-86.2" font-family="Arial" font-size="14.00">embedding </text>
74
- <text text-anchor="start" x="247.03" y="-86.2" font-family="Arial" font-size="14.00" fill="#666666">[vector(2000)]</text>
75
- <polygon fill="none" stroke="black" points="166.86,-43.2 166.86,-74 439.19,-74 439.19,-43.2 166.86,-43.2"/>
76
- <text text-anchor="start" x="173.86" y="-55.4" font-family="Arial" font-size="14.00">embedding_dimension </text>
77
- <text text-anchor="start" x="318.63" y="-55.4" font-family="Arial" font-size="14.00" fill="#666666">[integer]</text>
78
- </g>
79
- <!-- public.node_tags&#45;&gt;public.nodes -->
80
- <g id="edge1" class="edge">
81
- <title>public.node_tags:node_id&#45;&gt;public.nodes:id</title>
82
- <path fill="none" stroke="black" d="M343.42,-649.75C302.28,-640.33 326.23,-570.04 354.55,-527.6 377.24,-493.59 416.46,-525.58 439.19,-491.6 462.48,-456.8 482.06,-397.4 440.19,-397.4"/>
83
- <polygon fill="black" stroke="black" points="343.27,-649.73 352.75,-655.24 348.57,-650.28 352.88,-650.73 352.88,-650.73 352.88,-650.73 348.57,-650.28 353.68,-646.29 343.27,-649.73"/>
84
- <text text-anchor="start" x="7" y="-660.8" font-family="Arial" font-size="10.00">FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE</text>
85
- </g>
86
- <!-- public.tags -->
87
- <g id="node3" class="node">
88
- <title>public.tags</title>
89
- <polygon fill="#efefef" stroke="none" points="543.14,-274.2 543.14,-309.8 792.91,-309.8 792.91,-274.2 543.14,-274.2"/>
90
- <polygon fill="none" stroke="black" points="543.14,-274.2 543.14,-309.8 792.91,-309.8 792.91,-274.2 543.14,-274.2"/>
91
- <text text-anchor="start" x="566.54" y="-287.6" font-family="Arial Bold" font-size="18.00">public.tags</text>
92
- <text text-anchor="start" x="645.03" y="-287.6" font-family="Arial" font-size="14.00">    </text>
93
- <text text-anchor="start" x="676.14" y="-287.6" font-family="Arial" font-size="14.00" fill="#666666">[BASE TABLE]</text>
94
- <polygon fill="none" stroke="black" points="543.14,-243.4 543.14,-274.2 792.91,-274.2 792.91,-243.4 543.14,-243.4"/>
95
- <text text-anchor="start" x="550.14" y="-255.6" font-family="Arial" font-size="14.00">id </text>
96
- <text text-anchor="start" x="564.93" y="-255.6" font-family="Arial" font-size="14.00" fill="#666666">[bigint]</text>
97
- <polygon fill="none" stroke="black" points="543.14,-212.6 543.14,-243.4 792.91,-243.4 792.91,-212.6 543.14,-212.6"/>
98
- <text text-anchor="start" x="550.14" y="-224.8" font-family="Arial" font-size="14.00">name </text>
99
- <text text-anchor="start" x="589.05" y="-224.8" font-family="Arial" font-size="14.00" fill="#666666">[text]</text>
100
- <polygon fill="none" stroke="black" points="543.14,-181.8 543.14,-212.6 792.91,-212.6 792.91,-181.8 543.14,-181.8"/>
101
- <text text-anchor="start" x="550.14" y="-194" font-family="Arial" font-size="14.00">created_at </text>
102
- <text text-anchor="start" x="620.19" y="-194" font-family="Arial" font-size="14.00" fill="#666666">[timestamp with time zone]</text>
103
- </g>
104
- <!-- public.node_tags&#45;&gt;public.tags -->
105
- <g id="edge2" class="edge">
106
- <title>public.node_tags:tag_id&#45;&gt;public.tags:id</title>
107
- <path fill="none" stroke="black" d="M626.71,-619.46C762.63,-603.96 381.92,-258.8 542.14,-258.8"/>
108
- <polygon fill="black" stroke="black" points="626.83,-619.46 616.61,-615.49 621.5,-619.74 617.18,-619.96 617.18,-619.96 617.18,-619.96 621.5,-619.74 617.08,-624.48 626.83,-619.46"/>
109
- <text text-anchor="start" x="623.51" y="-630" font-family="Arial" font-size="10.00">FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE</text>
110
- </g>
111
- </g>
112
- </svg>