htm 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +7 -0
  2. data/.architecture/decisions/adrs/001-use-postgresql-timescaledb-storage.md +227 -0
  3. data/.architecture/decisions/adrs/002-two-tier-memory-architecture.md +322 -0
  4. data/.architecture/decisions/adrs/003-ollama-default-embedding-provider.md +339 -0
  5. data/.architecture/decisions/adrs/004-multi-robot-shared-memory-hive-mind.md +374 -0
  6. data/.architecture/decisions/adrs/005-rag-based-retrieval-with-hybrid-search.md +443 -0
  7. data/.architecture/decisions/adrs/006-context-assembly-strategies.md +444 -0
  8. data/.architecture/decisions/adrs/007-working-memory-eviction-strategy.md +461 -0
  9. data/.architecture/decisions/adrs/008-robot-identification-system.md +550 -0
  10. data/.architecture/decisions/adrs/009-never-forget-explicit-deletion-only.md +570 -0
  11. data/.architecture/decisions/adrs/010-redis-working-memory-rejected.md +323 -0
  12. data/.architecture/decisions/adrs/011-database-side-embedding-generation-with-pgai.md +585 -0
  13. data/.architecture/decisions/adrs/012-llm-driven-ontology-topic-extraction.md +583 -0
  14. data/.architecture/decisions/adrs/013-activerecord-orm-and-many-to-many-tagging.md +299 -0
  15. data/.architecture/decisions/adrs/014-client-side-embedding-generation-workflow.md +569 -0
  16. data/.architecture/decisions/adrs/015-hierarchical-tag-ontology-and-llm-extraction.md +701 -0
  17. data/.architecture/decisions/adrs/016-async-embedding-and-tag-generation.md +694 -0
  18. data/.architecture/members.yml +144 -0
  19. data/.architecture/reviews/2025-10-29-llm-configuration-and-async-processing-review.md +1137 -0
  20. data/.architecture/reviews/initial-system-analysis.md +330 -0
  21. data/.envrc +32 -0
  22. data/.irbrc +145 -0
  23. data/CHANGELOG.md +150 -0
  24. data/COMMITS.md +196 -0
  25. data/LICENSE +21 -0
  26. data/README.md +1347 -0
  27. data/Rakefile +51 -0
  28. data/SETUP.md +268 -0
  29. data/config/database.yml +67 -0
  30. data/db/migrate/20250101000001_enable_extensions.rb +14 -0
  31. data/db/migrate/20250101000002_create_robots.rb +14 -0
  32. data/db/migrate/20250101000003_create_nodes.rb +42 -0
  33. data/db/migrate/20250101000005_create_tags.rb +38 -0
  34. data/db/migrate/20250101000007_add_node_vector_indexes.rb +30 -0
  35. data/db/schema.sql +473 -0
  36. data/db/seed_data/README.md +100 -0
  37. data/db/seed_data/presidents.md +136 -0
  38. data/db/seed_data/states.md +151 -0
  39. data/db/seeds.rb +208 -0
  40. data/dbdoc/README.md +173 -0
  41. data/dbdoc/public.node_stats.md +48 -0
  42. data/dbdoc/public.node_stats.svg +41 -0
  43. data/dbdoc/public.node_tags.md +40 -0
  44. data/dbdoc/public.node_tags.svg +112 -0
  45. data/dbdoc/public.nodes.md +54 -0
  46. data/dbdoc/public.nodes.svg +118 -0
  47. data/dbdoc/public.nodes_tags.md +39 -0
  48. data/dbdoc/public.nodes_tags.svg +112 -0
  49. data/dbdoc/public.ontology_structure.md +48 -0
  50. data/dbdoc/public.ontology_structure.svg +38 -0
  51. data/dbdoc/public.operations_log.md +42 -0
  52. data/dbdoc/public.operations_log.svg +130 -0
  53. data/dbdoc/public.relationships.md +39 -0
  54. data/dbdoc/public.relationships.svg +41 -0
  55. data/dbdoc/public.robot_activity.md +46 -0
  56. data/dbdoc/public.robot_activity.svg +35 -0
  57. data/dbdoc/public.robots.md +35 -0
  58. data/dbdoc/public.robots.svg +90 -0
  59. data/dbdoc/public.schema_migrations.md +29 -0
  60. data/dbdoc/public.schema_migrations.svg +26 -0
  61. data/dbdoc/public.tags.md +35 -0
  62. data/dbdoc/public.tags.svg +60 -0
  63. data/dbdoc/public.topic_relationships.md +45 -0
  64. data/dbdoc/public.topic_relationships.svg +32 -0
  65. data/dbdoc/schema.json +1437 -0
  66. data/dbdoc/schema.svg +154 -0
  67. data/docs/api/database.md +806 -0
  68. data/docs/api/embedding-service.md +532 -0
  69. data/docs/api/htm.md +797 -0
  70. data/docs/api/index.md +259 -0
  71. data/docs/api/long-term-memory.md +1096 -0
  72. data/docs/api/working-memory.md +665 -0
  73. data/docs/architecture/adrs/001-postgresql-timescaledb.md +314 -0
  74. data/docs/architecture/adrs/002-two-tier-memory.md +411 -0
  75. data/docs/architecture/adrs/003-ollama-embeddings.md +421 -0
  76. data/docs/architecture/adrs/004-hive-mind.md +437 -0
  77. data/docs/architecture/adrs/005-rag-retrieval.md +531 -0
  78. data/docs/architecture/adrs/006-context-assembly.md +496 -0
  79. data/docs/architecture/adrs/007-eviction-strategy.md +645 -0
  80. data/docs/architecture/adrs/008-robot-identification.md +625 -0
  81. data/docs/architecture/adrs/009-never-forget.md +648 -0
  82. data/docs/architecture/adrs/010-redis-working-memory-rejected.md +323 -0
  83. data/docs/architecture/adrs/011-pgai-integration.md +494 -0
  84. data/docs/architecture/adrs/index.md +215 -0
  85. data/docs/architecture/hive-mind.md +736 -0
  86. data/docs/architecture/index.md +351 -0
  87. data/docs/architecture/overview.md +538 -0
  88. data/docs/architecture/two-tier-memory.md +873 -0
  89. data/docs/assets/css/custom.css +83 -0
  90. data/docs/assets/images/htm-core-components.svg +63 -0
  91. data/docs/assets/images/htm-database-schema.svg +93 -0
  92. data/docs/assets/images/htm-hive-mind-architecture.svg +125 -0
  93. data/docs/assets/images/htm-importance-scoring-framework.svg +83 -0
  94. data/docs/assets/images/htm-layered-architecture.svg +71 -0
  95. data/docs/assets/images/htm-long-term-memory-architecture.svg +115 -0
  96. data/docs/assets/images/htm-working-memory-architecture.svg +120 -0
  97. data/docs/assets/images/htm.jpg +0 -0
  98. data/docs/assets/images/htm_demo.gif +0 -0
  99. data/docs/assets/js/mathjax.js +18 -0
  100. data/docs/assets/videos/htm_video.mp4 +0 -0
  101. data/docs/database_rake_tasks.md +322 -0
  102. data/docs/development/contributing.md +787 -0
  103. data/docs/development/index.md +336 -0
  104. data/docs/development/schema.md +596 -0
  105. data/docs/development/setup.md +719 -0
  106. data/docs/development/testing.md +819 -0
  107. data/docs/guides/adding-memories.md +824 -0
  108. data/docs/guides/context-assembly.md +1009 -0
  109. data/docs/guides/getting-started.md +577 -0
  110. data/docs/guides/index.md +118 -0
  111. data/docs/guides/long-term-memory.md +941 -0
  112. data/docs/guides/multi-robot.md +866 -0
  113. data/docs/guides/recalling-memories.md +927 -0
  114. data/docs/guides/search-strategies.md +953 -0
  115. data/docs/guides/working-memory.md +717 -0
  116. data/docs/index.md +214 -0
  117. data/docs/installation.md +477 -0
  118. data/docs/multi_framework_support.md +519 -0
  119. data/docs/quick-start.md +655 -0
  120. data/docs/setup_local_database.md +302 -0
  121. data/docs/using_rake_tasks_in_your_app.md +383 -0
  122. data/examples/basic_usage.rb +93 -0
  123. data/examples/cli_app/README.md +317 -0
  124. data/examples/cli_app/htm_cli.rb +270 -0
  125. data/examples/custom_llm_configuration.rb +183 -0
  126. data/examples/example_app/Rakefile +71 -0
  127. data/examples/example_app/app.rb +206 -0
  128. data/examples/sinatra_app/Gemfile +21 -0
  129. data/examples/sinatra_app/app.rb +335 -0
  130. data/lib/htm/active_record_config.rb +113 -0
  131. data/lib/htm/configuration.rb +342 -0
  132. data/lib/htm/database.rb +594 -0
  133. data/lib/htm/embedding_service.rb +115 -0
  134. data/lib/htm/errors.rb +34 -0
  135. data/lib/htm/job_adapter.rb +154 -0
  136. data/lib/htm/jobs/generate_embedding_job.rb +65 -0
  137. data/lib/htm/jobs/generate_tags_job.rb +82 -0
  138. data/lib/htm/long_term_memory.rb +965 -0
  139. data/lib/htm/models/node.rb +109 -0
  140. data/lib/htm/models/node_tag.rb +33 -0
  141. data/lib/htm/models/robot.rb +52 -0
  142. data/lib/htm/models/tag.rb +76 -0
  143. data/lib/htm/railtie.rb +76 -0
  144. data/lib/htm/sinatra.rb +157 -0
  145. data/lib/htm/tag_service.rb +135 -0
  146. data/lib/htm/tasks.rb +38 -0
  147. data/lib/htm/version.rb +5 -0
  148. data/lib/htm/working_memory.rb +182 -0
  149. data/lib/htm.rb +400 -0
  150. data/lib/tasks/db.rake +19 -0
  151. data/lib/tasks/htm.rake +147 -0
  152. data/lib/tasks/jobs.rake +312 -0
  153. data/mkdocs.yml +190 -0
  154. data/scripts/install_local_database.sh +309 -0
  155. metadata +341 -0
@@ -0,0 +1,927 @@
1
+ # Recalling Memories from HTM
2
+
3
+ This guide covers HTM's powerful RAG-based retrieval system for finding relevant memories from your knowledge base.
4
+
5
+ ## Basic Recall
6
+
7
+ The `recall` method searches long-term memory using timeframe and topic:
8
+
9
+ ```ruby
10
+ memories = htm.recall(
11
+ timeframe: "last week", # Time range to search
12
+ topic: "database design", # What to search for
13
+ limit: 20, # Max results (default: 20)
14
+ strategy: :vector # Search strategy (default: :vector)
15
+ )
16
+
17
+ memories.each do |memory|
18
+ puts memory['value']
19
+ puts "Similarity: #{memory['similarity']}"
20
+ puts "Importance: #{memory['importance']}"
21
+ puts "Created: #{memory['created_at']}"
22
+ puts
23
+ end
24
+ ```
25
+
26
+ <svg viewBox="0 0 900 700" xmlns="http://www.w3.org/2000/svg" style="background: transparent;">
27
+ <!-- Title -->
28
+ <text x="450" y="30" text-anchor="middle" fill="#E0E0E0" font-size="18" font-weight="bold">HTM RAG-Based Recall Process</text>
29
+
30
+ <!-- Step 1: User Query -->
31
+ <rect x="50" y="70" width="200" height="80" fill="rgba(76, 175, 80, 0.2)" stroke="#4CAF50" stroke-width="3" rx="5"/>
32
+ <text x="150" y="95" text-anchor="middle" fill="#4CAF50" font-size="14" font-weight="bold">1. User Query</text>
33
+ <text x="150" y="120" text-anchor="middle" fill="#B0B0B0" font-size="11">recall(</text>
34
+ <text x="150" y="135" text-anchor="middle" fill="#B0B0B0" font-size="11"> "database design")</text>
35
+
36
+ <!-- Arrow 1 to 2 -->
37
+ <line x1="250" y1="110" x2="290" y2="110" stroke="#4CAF50" stroke-width="2" marker-end="url(#arrow-g)"/>
38
+
39
+ <!-- Step 2: Generate Embedding -->
40
+ <rect x="290" y="70" width="200" height="80" fill="rgba(33, 150, 243, 0.2)" stroke="#2196F3" stroke-width="2" rx="5"/>
41
+ <text x="390" y="95" text-anchor="middle" fill="#2196F3" font-size="14" font-weight="bold">2. Generate Embedding</text>
42
+ <text x="390" y="120" text-anchor="middle" fill="#B0B0B0" font-size="10">Ollama/OpenAI</text>
43
+ <text x="390" y="135" text-anchor="middle" fill="#B0B0B0" font-size="10">[0.23, -0.57, ...]</text>
44
+
45
+ <!-- Arrow 2 to 3 -->
46
+ <line x1="490" y1="110" x2="530" y2="110" stroke="#2196F3" stroke-width="2" marker-end="url(#arrow-b)"/>
47
+
48
+ <!-- Step 3: Database Search -->
49
+ <rect x="530" y="70" width="200" height="80" fill="rgba(156, 39, 176, 0.2)" stroke="#9C27B0" stroke-width="2" rx="5"/>
50
+ <text x="630" y="95" text-anchor="middle" fill="#9C27B0" font-size="14" font-weight="bold">3. Search Database</text>
51
+ <text x="630" y="120" text-anchor="middle" fill="#B0B0B0" font-size="10">Vector + Temporal</text>
52
+ <text x="630" y="135" text-anchor="middle" fill="#B0B0B0" font-size="10">+ Full-text</text>
53
+
54
+ <!-- Search Strategy Branches -->
55
+ <line x1="630" y1="150" x2="200" y2="200" stroke="#2196F3" stroke-width="2" marker-end="url(#arrow-b2)"/>
56
+ <line x1="630" y1="150" x2="450" y2="200" stroke="#4CAF50" stroke-width="2" marker-end="url(#arrow-g2)"/>
57
+ <line x1="630" y1="150" x2="680" y2="200" stroke="#9C27B0" stroke-width="2" marker-end="url(#arrow-p)"/>
58
+
59
+ <!-- Vector Search -->
60
+ <rect x="100" y="210" width="200" height="120" fill="rgba(33, 150, 243, 0.15)" stroke="#2196F3" stroke-width="2" rx="3"/>
61
+ <text x="200" y="235" text-anchor="middle" fill="#2196F3" font-size="12" font-weight="bold">Vector Search</text>
62
+ <text x="120" y="260" fill="#B0B0B0" font-size="10">pgvector HNSW</text>
63
+ <text x="120" y="280" fill="#B0B0B0" font-size="10">Cosine similarity</text>
64
+ <text x="120" y="300" fill="#B0B0B0" font-size="10">Semantic matching</text>
65
+ <text x="200" y="320" text-anchor="middle" fill="#4CAF50" font-size="9">~80ms</text>
66
+
67
+ <!-- Full-text Search -->
68
+ <rect x="350" y="210" width="200" height="120" fill="rgba(76, 175, 80, 0.15)" stroke="#4CAF50" stroke-width="2" rx="3"/>
69
+ <text x="450" y="235" text-anchor="middle" fill="#4CAF50" font-size="12" font-weight="bold">Full-Text Search</text>
70
+ <text x="370" y="260" fill="#B0B0B0" font-size="10">PostgreSQL GIN</text>
71
+ <text x="370" y="280" fill="#B0B0B0" font-size="10">ts_query matching</text>
72
+ <text x="370" y="300" fill="#B0B0B0" font-size="10">Keyword matching</text>
73
+ <text x="450" y="320" text-anchor="middle" fill="#4CAF50" font-size="9">~30ms</text>
74
+
75
+ <!-- Hybrid Search -->
76
+ <rect x="600" y="210" width="200" height="120" fill="rgba(156, 39, 176, 0.15)" stroke="#9C27B0" stroke-width="2" rx="3"/>
77
+ <text x="700" y="235" text-anchor="middle" fill="#9C27B0" font-size="12" font-weight="bold">Hybrid Search</text>
78
+ <text x="620" y="260" fill="#B0B0B0" font-size="10">Both searches</text>
79
+ <text x="620" y="280" fill="#B0B0B0" font-size="10">RRF scoring</text>
80
+ <text x="620" y="300" fill="#B0B0B0" font-size="10">Best results</text>
81
+ <text x="700" y="320" text-anchor="middle" fill="#FFC107" font-size="9">~120ms</text>
82
+
83
+ <!-- Results merge -->
84
+ <line x1="200" y1="330" x2="450" y2="380" stroke="#2196F3" stroke-width="2"/>
85
+ <line x1="450" y1="330" x2="450" y2="380" stroke="#4CAF50" stroke-width="2"/>
86
+ <line x1="700" y1="330" x2="450" y2="380" stroke="#9C27B0" stroke-width="2"/>
87
+
88
+ <!-- Step 4: Ranked Results -->
89
+ <rect x="300" y="390" width="300" height="110" fill="rgba(255, 152, 0, 0.2)" stroke="#FF9800" stroke-width="2" rx="5"/>
90
+ <text x="450" y="415" text-anchor="middle" fill="#FF9800" font-size="14" font-weight="bold">4. Ranked Results</text>
91
+ <text x="320" y="440" fill="#B0B0B0" font-size="10">1. "PostgreSQL design" (0.92)</text>
92
+ <text x="320" y="460" fill="#B0B0B0" font-size="10">2. "Database schema" (0.89)</text>
93
+ <text x="320" y="480" fill="#B0B0B0" font-size="10">3. "Table relationships" (0.85)</text>
94
+
95
+ <!-- Arrow 4 to 5 -->
96
+ <line x1="450" y1="500" x2="450" y2="530" stroke="#FF9800" stroke-width="2" marker-end="url(#arrow-o)"/>
97
+
98
+ <!-- Step 5: Load to Working Memory -->
99
+ <rect x="300" y="540" width="300" height="110" fill="rgba(76, 175, 80, 0.2)" stroke="#4CAF50" stroke-width="2" rx="5"/>
100
+ <text x="450" y="565" text-anchor="middle" fill="#4CAF50" font-size="14" font-weight="bold">5. Load to Working Memory</text>
101
+ <text x="320" y="590" fill="#B0B0B0" font-size="10">• Add to in-memory cache</text>
102
+ <text x="320" y="610" fill="#B0B0B0" font-size="10">• Fast LLM access</text>
103
+ <text x="320" y="630" fill="#B0B0B0" font-size="10">• Return to user</text>
104
+
105
+ <!-- Key Features -->
106
+ <rect x="50" y="540" width="200" height="110" fill="rgba(33, 150, 243, 0.1)" stroke="#2196F3" stroke-width="1" rx="3"/>
107
+ <text x="150" y="560" text-anchor="middle" fill="#2196F3" font-size="11" font-weight="bold">Key Features:</text>
108
+ <text x="70" y="580" fill="#B0B0B0" font-size="9">✓ Temporal filtering</text>
109
+ <text x="70" y="600" fill="#B0B0B0" font-size="9">✓ Semantic search</text>
110
+ <text x="70" y="620" fill="#B0B0B0" font-size="9">✓ Keyword matching</text>
111
+ <text x="70" y="640" fill="#B0B0B0" font-size="9">✓ Importance ranking</text>
112
+
113
+ <!-- Performance -->
114
+ <rect x="650" y="540" width="200" height="110" fill="rgba(255, 193, 7, 0.1)" stroke="#FFC107" stroke-width="1" rx="3"/>
115
+ <text x="750" y="560" text-anchor="middle" fill="#FFC107" font-size="11" font-weight="bold">Performance:</text>
116
+ <text x="670" y="580" fill="#B0B0B0" font-size="9">Vector: ~80ms</text>
117
+ <text x="670" y="600" fill="#B0B0B0" font-size="9">Full-text: ~30ms</text>
118
+ <text x="670" y="620" fill="#B0B0B0" font-size="9">Hybrid: ~120ms</text>
119
+ <text x="670" y="640" fill="#B0B0B0" font-size="9">✓ Optimized indexes</text>
120
+
121
+ <defs>
122
+ <marker id="arrow-g" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto">
123
+ <polygon points="0 0, 10 3, 0 6" fill="#4CAF50"/>
124
+ </marker>
125
+ <marker id="arrow-g2" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto">
126
+ <polygon points="0 0, 10 3, 0 6" fill="#4CAF50"/>
127
+ </marker>
128
+ <marker id="arrow-b" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto">
129
+ <polygon points="0 0, 10 3, 0 6" fill="#2196F3"/>
130
+ </marker>
131
+ <marker id="arrow-b2" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto">
132
+ <polygon points="0 0, 10 3, 0 6" fill="#2196F3"/>
133
+ </marker>
134
+ <marker id="arrow-p" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto">
135
+ <polygon points="0 0, 10 3, 0 6" fill="#9C27B0"/>
136
+ </marker>
137
+ <marker id="arrow-o" markerWidth="10" markerHeight="10" refX="9" refY="3" orient="auto">
138
+ <polygon points="0 0, 10 3, 0 6" fill="#FF9800"/>
139
+ </marker>
140
+ </defs>
141
+ </svg>
142
+
143
+ ## Understanding Timeframes
144
+
145
+ HTM supports both natural language timeframes and explicit ranges.
146
+
147
+ ### Natural Language Timeframes
148
+
149
+ ```ruby
150
+ # Last 24 hours (default if unparseable)
151
+ htm.recall(timeframe: "today", topic: "...")
152
+
153
+ # Yesterday
154
+ htm.recall(timeframe: "yesterday", topic: "...")
155
+
156
+ # Last week
157
+ htm.recall(timeframe: "last week", topic: "...")
158
+
159
+ # Last N days
160
+ htm.recall(timeframe: "last 7 days", topic: "...")
161
+ htm.recall(timeframe: "last 30 days", topic: "...")
162
+
163
+ # This month
164
+ htm.recall(timeframe: "this month", topic: "...")
165
+
166
+ # Last month
167
+ htm.recall(timeframe: "last month", topic: "...")
168
+ ```
169
+
170
+ ### Explicit Time Ranges
171
+
172
+ For precise control, use Ruby time ranges:
173
+
174
+ ```ruby
175
+ # Specific date range
176
+ start_date = Time.new(2024, 1, 1)
177
+ end_date = Time.new(2024, 12, 31)
178
+ htm.recall(
179
+ timeframe: start_date..end_date,
180
+ topic: "annual report"
181
+ )
182
+
183
+ # Last 24 hours precisely
184
+ htm.recall(
185
+ timeframe: (Time.now - 24*3600)..Time.now,
186
+ topic: "errors"
187
+ )
188
+
189
+ # All time
190
+ htm.recall(
191
+ timeframe: Time.at(0)..Time.now,
192
+ topic: "architecture decisions"
193
+ )
194
+
195
+ # Relative to current time
196
+ three_days_ago = Time.now - (3 * 24 * 3600)
197
+ htm.recall(
198
+ timeframe: three_days_ago..Time.now,
199
+ topic: "bug fixes"
200
+ )
201
+ ```
202
+
203
+ !!! tip "Choosing Timeframes"
204
+ - Use **narrow timeframes** (days/weeks) for recent context
205
+ - Use **wide timeframes** (months/years) for historical facts
206
+ - Use **"all time"** for searching unchanging facts or decisions
207
+
208
+ ## Search Strategies
209
+
210
+ HTM provides three search strategies, each with different strengths.
211
+
212
+ ### Vector Search (Semantic)
213
+
214
+ Vector search uses embeddings to find semantically similar memories.
215
+
216
+ ```ruby
217
+ memories = htm.recall(
218
+ timeframe: "last month",
219
+ topic: "improving application performance",
220
+ strategy: :vector,
221
+ limit: 10
222
+ )
223
+ ```
224
+
225
+ **How it works**:
226
+
227
+ 1. Converts your topic to a vector embedding via Ollama
228
+ 2. Finds memories with similar embeddings using cosine similarity
229
+ 3. Returns results ordered by semantic similarity
230
+
231
+ **Best for**:
232
+
233
+ - Conceptual searches ("how to optimize queries")
234
+ - Related topics ("database" finds "PostgreSQL", "SQL")
235
+ - Fuzzy matching ("ML" finds "machine learning")
236
+ - Understanding user intent
237
+
238
+ **Example**:
239
+
240
+ ```ruby
241
+ # Will find memories about databases, even without the word "PostgreSQL"
242
+ memories = htm.recall(
243
+ timeframe: "last year",
244
+ topic: "data persistence strategies",
245
+ strategy: :vector
246
+ )
247
+
248
+ # Finds: "Use PostgreSQL", "Database indexing", "SQL optimization"
249
+ ```
250
+
251
+ !!! note "Similarity Scores"
252
+ Vector search returns a `similarity` score (0-1). Scores > 0.8 indicate high relevance, 0.6-0.8 moderate relevance, < 0.6 low relevance.
253
+
254
+ ### Full-text Search (Keywords)
255
+
256
+ Full-text search uses PostgreSQL's text search for exact keyword matching.
257
+
258
+ ```ruby
259
+ memories = htm.recall(
260
+ timeframe: "last week",
261
+ topic: "PostgreSQL indexing",
262
+ strategy: :fulltext,
263
+ limit: 10
264
+ )
265
+ ```
266
+
267
+ **How it works**:
268
+
269
+ 1. Tokenizes your query into keywords
270
+ 2. Uses PostgreSQL's `ts_vector` and `ts_query` for matching
271
+ 3. Returns results ranked by text relevance
272
+
273
+ **Best for**:
274
+
275
+ - Exact keyword matches ("PostgreSQL", "Redis")
276
+ - Technical terms ("JWT", "OAuth")
277
+ - Proper nouns ("Alice", "Project Phoenix")
278
+ - Acronyms ("API", "SQL", "REST")
279
+
280
+ **Example**:
281
+
282
+ ```ruby
283
+ # Will only find memories containing "JWT"
284
+ memories = htm.recall(
285
+ timeframe: "all time",
286
+ topic: "JWT authentication",
287
+ strategy: :fulltext
288
+ )
289
+
290
+ # Finds: "JWT token validation", "Implemented JWT auth"
291
+ # Misses: "Token-based authentication" (no keyword match)
292
+ ```
293
+
294
+ !!! note "Ranking Scores"
295
+ Full-text search returns a `rank` score. Higher values indicate better keyword matches.
296
+
297
+ ### Hybrid Search (Best of Both)
298
+
299
+ Hybrid search combines full-text and vector search for optimal results.
300
+
301
+ ```ruby
302
+ memories = htm.recall(
303
+ timeframe: "last month",
304
+ topic: "database performance issues",
305
+ strategy: :hybrid,
306
+ limit: 10
307
+ )
308
+ ```
309
+
310
+ **How it works**:
311
+
312
+ 1. First, runs full-text search to find keyword matches (prefilter)
313
+ 2. Then, ranks those results by vector similarity
314
+ 3. Combines precision of keywords with understanding of semantics
315
+
316
+ **Best for**:
317
+
318
+ - General-purpose searches (default recommendation)
319
+ - When you want both keyword matches and related concepts
320
+ - Balancing precision and recall
321
+ - Production applications
322
+
323
+ **Example**:
324
+
325
+ ```ruby
326
+ # Combines keyword matching with semantic understanding
327
+ memories = htm.recall(
328
+ timeframe: "last quarter",
329
+ topic: "scaling our PostgreSQL database",
330
+ strategy: :hybrid
331
+ )
332
+
333
+ # Prefilter: Finds all memories mentioning "PostgreSQL" or "database"
334
+ # Ranking: Orders by semantic similarity to "scaling" concepts
335
+ ```
336
+
337
+ !!! tip "When to Use Hybrid"
338
+ Hybrid is the recommended default strategy. It provides good results across different query types without needing to choose between vector and full-text.
339
+
340
+ ## Search Strategy Comparison
341
+
342
+ | Strategy | Speed | Accuracy | Best Use Case |
343
+ |----------|-------|----------|---------------|
344
+ | Vector | Medium | High for concepts | Understanding intent, related topics |
345
+ | Full-text | Fast | High for keywords | Exact terms, proper nouns |
346
+ | Hybrid | Medium | Highest overall | General purpose, best default |
347
+
348
+ ## Query Optimization Tips
349
+
350
+ ### 1. Be Specific
351
+
352
+ ```ruby
353
+ # Vague: Returns too many irrelevant results
354
+ htm.recall(timeframe: "last year", topic: "data")
355
+
356
+ # Specific: Returns targeted results
357
+ htm.recall(timeframe: "last year", topic: "PostgreSQL query optimization")
358
+ ```
359
+
360
+ ### 2. Use Appropriate Timeframes
361
+
362
+ ```ruby
363
+ # Too wide: Includes outdated information
364
+ htm.recall(timeframe: "last 5 years", topic: "current project status")
365
+
366
+ # Right size: Recent context
367
+ htm.recall(timeframe: "last week", topic: "current project status")
368
+ ```
369
+
370
+ ### 3. Adjust Limit Based on Need
371
+
372
+ ```ruby
373
+ # Few results: Quick overview
374
+ htm.recall(timeframe: "last month", topic: "errors", limit: 5)
375
+
376
+ # Many results: Comprehensive search
377
+ htm.recall(timeframe: "last year", topic: "architecture decisions", limit: 50)
378
+ ```
379
+
380
+ ### 4. Try Different Strategies
381
+
382
+ ```ruby
383
+ # Start with hybrid (best all-around)
384
+ results = htm.recall(topic: "authentication", strategy: :hybrid)
385
+
386
+ # If too many results, try full-text (more precise)
387
+ results = htm.recall(topic: "JWT authentication", strategy: :fulltext)
388
+
389
+ # If no results, try vector (more flexible)
390
+ results = htm.recall(topic: "user validation methods", strategy: :vector)
391
+ ```
392
+
393
+ ## Combining Search with Filters
394
+
395
+ While `recall` handles timeframes and topics, you can filter results further:
396
+
397
+ ```ruby
398
+ # Recall memories
399
+ memories = htm.recall(
400
+ timeframe: "last month",
401
+ topic: "database",
402
+ strategy: :hybrid,
403
+ limit: 50
404
+ )
405
+
406
+ # Filter by type
407
+ decisions = memories.select { |m| m['type'] == 'decision' }
408
+
409
+ # Filter by importance
410
+ critical = memories.select { |m| m['importance'].to_f >= 8.0 }
411
+
412
+ # Filter by robot
413
+ my_memories = memories.select { |m| m['robot_id'] == htm.robot_id }
414
+
415
+ # Filter by date
416
+ recent = memories.select do |m|
417
+ Time.parse(m['created_at']) > Time.now - 7*24*3600
418
+ end
419
+ ```
420
+
421
+ ## Advanced Query Patterns
422
+
423
+ ### Pattern 1: Multi-Topic Search
424
+
425
+ Search for multiple related topics:
426
+
427
+ ```ruby
428
+ def search_multiple_topics(timeframe, topics, strategy: :hybrid, limit: 10)
429
+ results = []
430
+
431
+ topics.each do |topic|
432
+ results.concat(
433
+ htm.recall(
434
+ timeframe: timeframe,
435
+ topic: topic,
436
+ strategy: strategy,
437
+ limit: limit
438
+ )
439
+ )
440
+ end
441
+
442
+ # Remove duplicates by key
443
+ results.uniq { |m| m['key'] }
444
+ end
445
+
446
+ # Usage
447
+ memories = search_multiple_topics(
448
+ "last month",
449
+ ["database optimization", "query performance", "indexing strategies"]
450
+ )
451
+ ```
452
+
453
+ ### Pattern 2: Iterative Refinement
454
+
455
+ Start broad, then narrow:
456
+
457
+ ```ruby
458
+ # First pass: Broad search
459
+ broad_results = htm.recall(
460
+ timeframe: "last year",
461
+ topic: "architecture",
462
+ strategy: :vector,
463
+ limit: 100
464
+ )
465
+
466
+ # Analyze results, refine query
467
+ relevant_terms = broad_results
468
+ .select { |m| m['similarity'].to_f > 0.7 }
469
+ .flat_map { |m| m['tags'] }
470
+ .uniq
471
+
472
+ # Second pass: Refined search
473
+ refined_results = htm.recall(
474
+ timeframe: "last year",
475
+ topic: "architecture #{relevant_terms.join(' ')}",
476
+ strategy: :hybrid,
477
+ limit: 20
478
+ )
479
+ ```
480
+
481
+ ### Pattern 3: Threshold Filtering
482
+
483
+ Only keep high-quality matches:
484
+
485
+ ```ruby
486
+ def recall_with_threshold(timeframe:, topic:, threshold: 0.7, strategy: :vector)
487
+ results = htm.recall(
488
+ timeframe: timeframe,
489
+ topic: topic,
490
+ strategy: strategy,
491
+ limit: 50 # Get more candidates
492
+ )
493
+
494
+ # Filter by similarity threshold
495
+ case strategy
496
+ when :vector, :hybrid
497
+ results.select { |m| m['similarity'].to_f >= threshold }
498
+ when :fulltext
499
+ # For fulltext, use rank threshold (adjust as needed)
500
+ results.select { |m| m['rank'].to_f >= threshold }
501
+ end
502
+ end
503
+
504
+ # Usage
505
+ high_quality = recall_with_threshold(
506
+ timeframe: "last month",
507
+ topic: "performance optimization",
508
+ threshold: 0.8
509
+ )
510
+ ```
511
+
512
+ ### Pattern 4: Time-Weighted Search
513
+
514
+ Weight results by recency:
515
+
516
+ ```ruby
517
+ def recall_time_weighted(timeframe:, topic:, recency_weight: 0.3)
518
+ memories = htm.recall(
519
+ timeframe: timeframe,
520
+ topic: topic,
521
+ strategy: :hybrid,
522
+ limit: 50
523
+ )
524
+
525
+ # Calculate time-weighted score
526
+ now = Time.now
527
+ memories.each do |m|
528
+ created = Time.parse(m['created_at'])
529
+ age_days = (now - created) / (24 * 3600)
530
+
531
+ # Decay factor: newer is better
532
+ recency_score = Math.exp(-age_days / 30.0) # 30-day half-life
533
+
534
+ # Combine similarity and recency
535
+ similarity = m['similarity'].to_f
536
+ m['weighted_score'] = (
537
+ similarity * (1 - recency_weight) +
538
+ recency_score * recency_weight
539
+ )
540
+ end
541
+
542
+ # Sort by weighted score
543
+ memories.sort_by { |m| -m['weighted_score'] }
544
+ end
545
+ ```
546
+
547
+ ### Pattern 5: Context-Aware Search
548
+
549
+ Include current context in search:
550
+
551
+ ```ruby
552
+ class ContextualRecall
553
+ def initialize(htm)
554
+ @htm = htm
555
+ @current_context = []
556
+ end
557
+
558
+ def add_context(key, value)
559
+ @current_context << { key: key, value: value }
560
+ end
561
+
562
+ def recall(timeframe:, topic:, strategy: :hybrid)
563
+ # Enhance topic with current context
564
+ context_terms = @current_context.map { |c| c[:value] }.join(" ")
565
+ enhanced_topic = "#{topic} #{context_terms}"
566
+
567
+ @htm.recall(
568
+ timeframe: timeframe,
569
+ topic: enhanced_topic,
570
+ strategy: strategy,
571
+ limit: 20
572
+ )
573
+ end
574
+ end
575
+
576
+ # Usage
577
+ recall = ContextualRecall.new(htm)
578
+ recall.add_context("project", "e-commerce platform")
579
+ recall.add_context("focus", "checkout flow")
580
+
581
+ # Search includes context automatically
582
+ results = recall.recall(
583
+ timeframe: "last month",
584
+ topic: "payment processing"
585
+ )
586
+ ```
587
+
588
+ ## Retrieving Specific Memories
589
+
590
+ For known keys, use `retrieve` instead of `recall`:
591
+
592
+ ```ruby
593
+ # Retrieve by exact key
594
+ memory = htm.retrieve("decision_database")
595
+
596
+ if memory
597
+ puts memory['value']
598
+ puts "Type: #{memory['type']}"
599
+ puts "Created: #{memory['created_at']}"
600
+ else
601
+ puts "Memory not found"
602
+ end
603
+ ```
604
+
605
+ !!! note
606
+ `retrieve` is faster than `recall` because it doesn't require embedding generation or similarity calculation.
607
+
608
+ ## Working with Search Results
609
+
610
+ ### Result Structure
611
+
612
+ Each memory returned by `recall` has these fields:
613
+
614
+ ```ruby
615
+ memory = {
616
+ 'id' => 123, # Database ID
617
+ 'key' => "decision_001", # Unique key
618
+ 'value' => "Decision text...", # Content
619
+ 'type' => "decision", # Memory type
620
+ 'category' => "architecture", # Category (if set)
621
+ 'importance' => 9.0, # Importance score
622
+ 'created_at' => "2024-01-15 10:30:00", # Timestamp
623
+ 'robot_id' => "uuid...", # Which robot added it
624
+ 'token_count' => 150, # Token count
625
+ 'similarity' => 0.85 # Similarity score (vector/hybrid)
626
+ # or 'rank' for fulltext
627
+ }
628
+ ```
629
+
630
+ ### Processing Results
631
+
632
+ ```ruby
633
+ memories = htm.recall(timeframe: "last month", topic: "errors")
634
+
635
+ # Sort by importance
636
+ by_importance = memories.sort_by { |m| -m['importance'].to_f }
637
+
638
+ # Group by type
639
+ by_type = memories.group_by { |m| m['type'] }
640
+
641
+ # Extract just the content
642
+ content = memories.map { |m| m['value'] }
643
+
644
+ # Create summary
645
+ summary = memories.map do |m|
646
+ "[#{m['type']}] #{m['value'][0..100]}... (#{m['importance']})"
647
+ end.join("\n\n")
648
+ ```
649
+
650
+ ## Common Use Cases
651
+
652
+ ### Use Case 1: Error Analysis
653
+
654
+ Find recent errors and their solutions:
655
+
656
+ ```ruby
657
+ # Find recent errors
658
+ errors = htm.recall(
659
+ timeframe: "last 7 days",
660
+ topic: "error exception failure",
661
+ strategy: :fulltext,
662
+ limit: 20
663
+ )
664
+
665
+ # Group by error type
666
+ error_types = errors
667
+ .map { |e| e['value'][/Error: (.+?)\\n/, 1] }
668
+ .compact
669
+ .tally
670
+
671
+ puts "Error frequency:"
672
+ error_types.sort_by { |_, count| -count }.each do |type, count|
673
+ puts " #{type}: #{count} occurrences"
674
+ end
675
+ ```
676
+
677
+ ### Use Case 2: Decision History
678
+
679
+ Track decision evolution:
680
+
681
+ ```ruby
682
+ # Get all decisions about a topic
683
+ decisions = htm.recall(
684
+ timeframe: Time.at(0)..Time.now, # All time
685
+ topic: "authentication",
686
+ strategy: :hybrid,
687
+ limit: 50
688
+ ).select { |m| m['type'] == 'decision' }
689
+
690
+ # Sort chronologically
691
+ timeline = decisions.sort_by { |d| d['created_at'] }
692
+
693
+ puts "Decision timeline:"
694
+ timeline.each do |decision|
695
+ puts "#{decision['created_at']}: #{decision['value'][0..100]}..."
696
+ end
697
+ ```
698
+
699
+ ### Use Case 3: Knowledge Aggregation
700
+
701
+ Gather all knowledge about a topic:
702
+
703
+ ```ruby
704
+ def gather_knowledge(topic)
705
+ # Gather different types of memories
706
+ facts = htm.recall(
707
+ timeframe: "all time",
708
+ topic: topic,
709
+ strategy: :hybrid
710
+ ).select { |m| m['type'] == 'fact' }
711
+
712
+ decisions = htm.recall(
713
+ timeframe: "all time",
714
+ topic: topic,
715
+ strategy: :hybrid
716
+ ).select { |m| m['type'] == 'decision' }
717
+
718
+ code = htm.recall(
719
+ timeframe: "all time",
720
+ topic: topic,
721
+ strategy: :hybrid
722
+ ).select { |m| m['type'] == 'code' }
723
+
724
+ {
725
+ facts: facts,
726
+ decisions: decisions,
727
+ code_examples: code
728
+ }
729
+ end
730
+
731
+ knowledge = gather_knowledge("PostgreSQL")
732
+ ```
733
+
734
+ ### Use Case 4: Conversation Context
735
+
736
+ Recall recent conversation:
737
+
738
+ ```ruby
739
+ def get_conversation_context(session_id, turns: 5)
740
+ # Get recent conversation turns
741
+ htm.recall(
742
+ timeframe: "last 24 hours",
743
+ topic: "session_#{session_id}",
744
+ strategy: :fulltext,
745
+ limit: turns * 2 # user + assistant messages
746
+ ).select { |m| m['type'] == 'context' }
747
+ .sort_by { |m| m['created_at'] }
748
+ .last(turns * 2)
749
+ end
750
+ ```
751
+
752
+ ## Performance Considerations
753
+
754
+ ### Search Speed
755
+
756
+ - **Full-text**: Fastest (~50-100ms)
757
+ - **Vector**: Medium (~100-300ms)
758
+ - **Hybrid**: Medium (~150-350ms)
759
+
760
+ Times vary based on database size and query complexity.
761
+
762
+ ### Optimizing Queries
763
+
764
+ ```ruby
765
+ # Slow: Wide timeframe + high limit
766
+ htm.recall(timeframe: "last 5 years", topic: "...", limit: 1000)
767
+
768
+ # Fast: Narrow timeframe + reasonable limit
769
+ htm.recall(timeframe: "last week", topic: "...", limit: 20)
770
+ ```
771
+
772
+ ### Caching Results
773
+
774
+ For repeated queries:
775
+
776
+ ```ruby
777
+ class CachedRecall
778
+ def initialize(htm, cache_ttl: 300)
779
+ @htm = htm
780
+ @cache = {}
781
+ @cache_ttl = cache_ttl
782
+ end
783
+
784
+ def recall(**args)
785
+ cache_key = args.hash
786
+
787
+ if cached = @cache[cache_key]
788
+ return cached[:results] if Time.now - cached[:time] < @cache_ttl
789
+ end
790
+
791
+ results = @htm.recall(**args)
792
+ @cache[cache_key] = { results: results, time: Time.now }
793
+ results
794
+ end
795
+ end
796
+ ```
797
+
798
+ ## Troubleshooting
799
+
800
+ ### No Results
801
+
802
+ ```ruby
803
+ results = htm.recall(timeframe: "last week", topic: "xyz")
804
+
805
+ if results.empty?
806
+ # Try wider timeframe
807
+ results = htm.recall(timeframe: "last month", topic: "xyz")
808
+
809
+ # Try different strategy
810
+ results = htm.recall(
811
+ timeframe: "last month",
812
+ topic: "xyz",
813
+ strategy: :vector # More flexible
814
+ )
815
+
816
+ # Try related terms
817
+ results = htm.recall(
818
+ timeframe: "last month",
819
+ topic: "xyz related similar",
820
+ strategy: :vector
821
+ )
822
+ end
823
+ ```
824
+
825
+ ### Low-Quality Results
826
+
827
+ ```ruby
828
+ # Filter by similarity threshold
829
+ good_results = results.select do |m|
830
+ m['similarity'].to_f > 0.7 # Only high-quality matches
831
+ end
832
+
833
+ # Or boost limit and take top results
834
+ htm.recall(timeframe: "...", topic: "...", limit: 100)
835
+ .sort_by { |m| -m['similarity'].to_f }
836
+ .first(10)
837
+ ```
838
+
839
+ ### Ollama Connection Issues
840
+
841
+ If vector search fails:
842
+
843
+ ```ruby
844
+ begin
845
+ results = htm.recall(topic: "...", strategy: :vector)
846
+ rescue => e
847
+ warn "Vector search failed: #{e.message}"
848
+ warn "Falling back to full-text search"
849
+ results = htm.recall(topic: "...", strategy: :fulltext)
850
+ end
851
+ ```
852
+
853
+ ## Next Steps
854
+
855
+ - [**Context Assembly**](context-assembly.md) - Use recalled memories with your LLM
856
+ - [**Search Strategies**](search-strategies.md) - Deep dive into search algorithms
857
+ - [**Working Memory**](working-memory.md) - Understand how recall populates working memory
858
+
859
+ ## Complete Example
860
+
861
+ ```ruby
862
+ require 'htm'
863
+
864
+ htm = HTM.new(robot_name: "Search Demo")
865
+
866
+ # Add test memories
867
+ htm.add_node(
868
+ "decision_db",
869
+ "Chose PostgreSQL for its reliability and ACID compliance",
870
+ type: :decision,
871
+ importance: 9.0,
872
+ tags: ["database", "postgresql", "architecture"]
873
+ )
874
+
875
+ htm.add_node(
876
+ "code_connection",
877
+ "conn = PG.connect(dbname: 'mydb')",
878
+ type: :code,
879
+ importance: 6.0,
880
+ tags: ["postgresql", "ruby", "connection"]
881
+ )
882
+
883
+ # Vector search: Semantic understanding
884
+ puts "=== Vector Search ==="
885
+ vector_results = htm.recall(
886
+ timeframe: "all time",
887
+ topic: "data persistence strategies",
888
+ strategy: :vector,
889
+ limit: 10
890
+ )
891
+
892
+ vector_results.each do |m|
893
+ puts "#{m['value'][0..80]}..."
894
+ puts " Similarity: #{m['similarity']}"
895
+ puts
896
+ end
897
+
898
+ # Full-text search: Exact keywords
899
+ puts "\n=== Full-text Search ==="
900
+ fulltext_results = htm.recall(
901
+ timeframe: "all time",
902
+ topic: "PostgreSQL",
903
+ strategy: :fulltext,
904
+ limit: 10
905
+ )
906
+
907
+ fulltext_results.each do |m|
908
+ puts "#{m['value'][0..80]}..."
909
+ puts " Rank: #{m['rank']}"
910
+ puts
911
+ end
912
+
913
+ # Hybrid search: Best of both
914
+ puts "\n=== Hybrid Search ==="
915
+ hybrid_results = htm.recall(
916
+ timeframe: "all time",
917
+ topic: "database connection setup",
918
+ strategy: :hybrid,
919
+ limit: 10
920
+ )
921
+
922
+ hybrid_results.each do |m|
923
+ puts "[#{m['type']}] #{m['value'][0..80]}..."
924
+ puts " Importance: #{m['importance']}, Similarity: #{m['similarity']}"
925
+ puts
926
+ end
927
+ ```