htm 0.0.14 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +33 -0
  3. data/README.md +269 -79
  4. data/db/migrate/00003_create_file_sources.rb +5 -0
  5. data/db/migrate/00004_create_nodes.rb +17 -0
  6. data/db/migrate/00005_create_tags.rb +7 -0
  7. data/db/migrate/00006_create_node_tags.rb +2 -0
  8. data/db/migrate/00007_create_robot_nodes.rb +7 -0
  9. data/db/schema.sql +41 -29
  10. data/docs/api/yard/HTM/Configuration.md +54 -0
  11. data/docs/api/yard/HTM/Database.md +13 -10
  12. data/docs/api/yard/HTM/EmbeddingService.md +5 -1
  13. data/docs/api/yard/HTM/LongTermMemory.md +18 -277
  14. data/docs/api/yard/HTM/PropositionError.md +18 -0
  15. data/docs/api/yard/HTM/PropositionService.md +66 -0
  16. data/docs/api/yard/HTM/QueryCache.md +88 -0
  17. data/docs/api/yard/HTM/RobotGroup.md +481 -0
  18. data/docs/api/yard/HTM/SqlBuilder.md +108 -0
  19. data/docs/api/yard/HTM/TagService.md +4 -0
  20. data/docs/api/yard/HTM/Telemetry/NullInstrument.md +13 -0
  21. data/docs/api/yard/HTM/Telemetry/NullMeter.md +15 -0
  22. data/docs/api/yard/HTM/Telemetry.md +109 -0
  23. data/docs/api/yard/HTM/WorkingMemoryChannel.md +176 -0
  24. data/docs/api/yard/HTM.md +11 -23
  25. data/docs/api/yard/index.csv +102 -25
  26. data/docs/api/yard-reference.md +8 -0
  27. data/docs/assets/images/multi-provider-failover.svg +51 -0
  28. data/docs/assets/images/robot-group-architecture.svg +65 -0
  29. data/docs/database/README.md +3 -3
  30. data/docs/database/public.file_sources.svg +29 -21
  31. data/docs/database/public.node_tags.md +2 -0
  32. data/docs/database/public.node_tags.svg +53 -41
  33. data/docs/database/public.nodes.md +2 -0
  34. data/docs/database/public.nodes.svg +52 -40
  35. data/docs/database/public.robot_nodes.md +2 -0
  36. data/docs/database/public.robot_nodes.svg +30 -22
  37. data/docs/database/public.robots.svg +16 -12
  38. data/docs/database/public.tags.md +3 -0
  39. data/docs/database/public.tags.svg +41 -33
  40. data/docs/database/schema.json +66 -0
  41. data/docs/database/schema.svg +60 -48
  42. data/docs/development/index.md +13 -0
  43. data/docs/development/rake-tasks.md +1068 -0
  44. data/docs/getting-started/quick-start.md +144 -155
  45. data/docs/guides/adding-memories.md +2 -3
  46. data/docs/guides/context-assembly.md +185 -184
  47. data/docs/guides/getting-started.md +154 -148
  48. data/docs/guides/index.md +7 -0
  49. data/docs/guides/long-term-memory.md +60 -92
  50. data/docs/guides/mcp-server.md +617 -0
  51. data/docs/guides/multi-robot.md +249 -345
  52. data/docs/guides/recalling-memories.md +153 -163
  53. data/docs/guides/robot-groups.md +604 -0
  54. data/docs/guides/search-strategies.md +61 -58
  55. data/docs/guides/working-memory.md +103 -136
  56. data/docs/index.md +30 -26
  57. data/examples/robot_groups/robot_worker.rb +1 -2
  58. data/examples/robot_groups/same_process.rb +1 -4
  59. data/lib/htm/robot_group.rb +721 -0
  60. data/lib/htm/version.rb +1 -1
  61. data/lib/htm/working_memory_channel.rb +250 -0
  62. data/lib/htm.rb +2 -0
  63. data/mkdocs.yml +2 -0
  64. metadata +18 -9
  65. data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +0 -12
  66. data/db/migrate/00010_add_soft_delete_to_associations.rb +0 -29
  67. data/db/migrate/00011_add_performance_indexes.rb +0 -21
  68. data/db/migrate/00012_add_tags_trigram_index.rb +0 -18
  69. data/db/migrate/00013_enable_lz4_compression.rb +0 -43
  70. data/examples/robot_groups/lib/robot_group.rb +0 -419
  71. data/examples/robot_groups/lib/working_memory_channel.rb +0 -140
@@ -110,11 +110,11 @@ Context assembly transforms working memory into LLM-ready context:
110
110
 
111
111
  ## Basic Usage
112
112
 
113
- The `create_context` method assembles context from working memory:
113
+ The `assemble_context` method on working memory creates a context string:
114
114
 
115
115
  ```ruby
116
116
  # Basic context assembly
117
- context = htm.create_context(
117
+ context = htm.working_memory.assemble_context(
118
118
  strategy: :balanced, # Assembly strategy
119
119
  max_tokens: nil # Optional token limit
120
120
  )
@@ -142,7 +142,7 @@ HTM provides three strategies for assembling context, each optimized for differe
142
142
  The `:recent` strategy prioritizes newest memories first.
143
143
 
144
144
  ```ruby
145
- context = htm.create_context(strategy: :recent)
145
+ context = htm.working_memory.assemble_context(strategy: :recent)
146
146
  ```
147
147
 
148
148
  **How it works:**
@@ -172,15 +172,13 @@ class ChatBot
172
172
  @turn += 1
173
173
 
174
174
  # Add user message
175
- @htm.add_node(
176
- "turn_#{@turn}_user",
175
+ @htm.remember(
177
176
  "User: #{user_message}",
178
- type: :context,
179
- importance: 6.0
177
+ metadata: { turn: @turn, role: "user" }
180
178
  )
181
179
 
182
180
  # Get recent context
183
- context = @htm.create_context(
181
+ context = @htm.working_memory.assemble_context(
184
182
  strategy: :recent,
185
183
  max_tokens: 10_000
186
184
  )
@@ -189,11 +187,9 @@ class ChatBot
189
187
  response = llm_generate(context, user_message)
190
188
 
191
189
  # Store assistant response
192
- @htm.add_node(
193
- "turn_#{@turn}_assistant",
190
+ @htm.remember(
194
191
  "Assistant: #{response}",
195
- type: :context,
196
- importance: 6.0
192
+ metadata: { turn: @turn, role: "assistant" }
197
193
  )
198
194
 
199
195
  response
@@ -208,17 +204,17 @@ class ChatBot
208
204
  end
209
205
  ```
210
206
 
211
- ### Important Strategy
207
+ ### Frequent Strategy
212
208
 
213
- The `:important` strategy prioritizes high-importance memories.
209
+ The `:frequent` strategy prioritizes frequently accessed memories.
214
210
 
215
211
  ```ruby
216
- context = htm.create_context(strategy: :important)
212
+ context = htm.working_memory.assemble_context(strategy: :frequent)
217
213
  ```
218
214
 
219
215
  **How it works:**
220
216
 
221
- 1. Sort memories by importance (highest first)
217
+ 1. Sort memories by access count (highest first)
222
218
  2. Add memories in order until token limit reached
223
219
  3. Return assembled context
224
220
 
@@ -233,44 +229,38 @@ context = htm.create_context(strategy: :important)
233
229
  **Example:**
234
230
 
235
231
  ```ruby
236
- # Knowledge base with priorities
232
+ # Knowledge base with priorities tracked via access patterns
237
233
  class KnowledgeBot
238
234
  def initialize
239
235
  @htm = HTM.new(robot_name: "Knowledge")
240
236
 
241
237
  # Add critical system constraints
242
- @htm.add_node(
243
- "constraint_001",
238
+ @htm.remember(
244
239
  "CRITICAL: Never expose API keys in responses",
245
- type: :fact,
246
- importance: 10.0
240
+ metadata: { priority: "critical", category: "constraint" }
247
241
  )
248
242
 
249
243
  # Add important user preferences
250
- @htm.add_node(
251
- "pref_001",
244
+ @htm.remember(
252
245
  "User prefers concise explanations",
253
- type: :preference,
254
- importance: 8.0
246
+ metadata: { priority: "high", category: "preference" }
255
247
  )
256
248
 
257
249
  # Add general knowledge
258
- @htm.add_node(
259
- "fact_001",
250
+ @htm.remember(
260
251
  "Python uses indentation for code blocks",
261
- type: :fact,
262
- importance: 5.0
252
+ metadata: { priority: "medium", category: "fact" }
263
253
  )
264
254
  end
265
255
 
266
256
  def answer_question(question)
267
- # Get most important context first
268
- context = @htm.create_context(
269
- strategy: :important,
257
+ # Get most frequently accessed context first
258
+ context = @htm.working_memory.assemble_context(
259
+ strategy: :frequent,
270
260
  max_tokens: 5_000
271
261
  )
272
262
 
273
- # Critical constraints and preferences are included first
263
+ # Frequently accessed constraints and preferences are included first
274
264
  generate_answer(context, question)
275
265
  end
276
266
 
@@ -278,22 +268,22 @@ class KnowledgeBot
278
268
 
279
269
  def generate_answer(context, question)
280
270
  # LLM integration
281
- "Answer based on important context"
271
+ "Answer based on frequently accessed context"
282
272
  end
283
273
  end
284
274
  ```
285
275
 
286
276
  ### Balanced Strategy (Recommended)
287
277
 
288
- The `:balanced` strategy combines importance and recency using a weighted formula.
278
+ The `:balanced` strategy combines access frequency and recency using a weighted formula.
289
279
 
290
280
  ```ruby
291
- context = htm.create_context(strategy: :balanced)
281
+ context = htm.working_memory.assemble_context(strategy: :balanced)
292
282
  ```
293
283
 
294
284
  **How it works:**
295
285
 
296
- 1. Calculate score: `importance × (1 / (1 + age_in_hours))`
286
+ 1. Calculate score: `log(1 + access_count) × recency_factor`
297
287
  2. Sort by score (highest first)
298
288
  3. Add memories until token limit reached
299
289
  4. Return assembled context
@@ -301,21 +291,21 @@ context = htm.create_context(strategy: :balanced)
301
291
  **Scoring examples:**
302
292
 
303
293
  ```ruby
304
- # Recent + Important: High score
305
- # Importance: 9.0, Age: 1 hour
306
- # Score: 9.0 × (1 / (1 + 1)) = 4.5 ✓ Included
294
+ # Recent + Frequently accessed: High score
295
+ # Access count: 10, Age: 1 hour
296
+ # Score: log(11) × (1 / (1 + 1/3600)) 2.4 ✓ Included
307
297
 
308
- # Old + Important: Medium score
309
- # Importance: 9.0, Age: 24 hours
310
- # Score: 9.0 × (1 / (1 + 24)) = 0.36 ≈ Maybe
298
+ # Old + Frequently accessed: Medium score
299
+ # Access count: 10, Age: 24 hours
300
+ # Score: log(11) × (1 / (1 + 24)) 0.10 ≈ Maybe
311
301
 
312
- # Recent + Unimportant: Low score
313
- # Importance: 2.0, Age: 1 hour
314
- # Score: 2.0 × (1 / (1 + 1)) = 1.0 ≈ Maybe
302
+ # Recent + Rarely accessed: Low score
303
+ # Access count: 1, Age: 1 hour
304
+ # Score: log(2) × (1 / (1 + 1/3600)) 0.69 ≈ Maybe
315
305
 
316
- # Old + Unimportant: Very low score
317
- # Importance: 2.0, Age: 24 hours
318
- # Score: 2.0 × (1 / (1 + 24)) = 0.08 ✗ Excluded
306
+ # Old + Rarely accessed: Very low score
307
+ # Access count: 1, Age: 24 hours
308
+ # Score: log(2) × (1 / (1 + 24)) 0.03 ✗ Excluded
319
309
  ```
320
310
 
321
311
  **Best for:**
@@ -339,15 +329,13 @@ class Assistant
339
329
 
340
330
  def process(user_input)
341
331
  # Add user input
342
- @htm.add_node(
343
- "input_#{Time.now.to_i}",
332
+ @htm.remember(
344
333
  user_input,
345
- type: :context,
346
- importance: 7.0
334
+ metadata: { role: "user", timestamp: Time.now.to_i }
347
335
  )
348
336
 
349
- # Get balanced context (recent + important)
350
- context = @htm.create_context(
337
+ # Get balanced context (frequent + recent)
338
+ context = @htm.working_memory.assemble_context(
351
339
  strategy: :balanced,
352
340
  max_tokens: 50_000
353
341
  )
@@ -387,23 +375,23 @@ Control context size with token limits:
387
375
 
388
376
  ```ruby
389
377
  # Use default (working memory size)
390
- context = htm.create_context(strategy: :balanced)
378
+ context = htm.working_memory.assemble_context(strategy: :balanced)
391
379
 
392
380
  # Custom limit
393
- context = htm.create_context(
381
+ context = htm.working_memory.assemble_context(
394
382
  strategy: :balanced,
395
383
  max_tokens: 50_000
396
384
  )
397
385
 
398
386
  # Small context for simple queries
399
- context = htm.create_context(
387
+ context = htm.working_memory.assemble_context(
400
388
  strategy: :recent,
401
389
  max_tokens: 5_000
402
390
  )
403
391
 
404
392
  # Large context for complex tasks
405
- context = htm.create_context(
406
- strategy: :important,
393
+ context = htm.working_memory.assemble_context(
394
+ strategy: :frequent,
407
395
  max_tokens: 200_000
408
396
  )
409
397
  ```
@@ -433,76 +421,72 @@ require 'benchmark'
433
421
 
434
422
  # Add 1000 test memories
435
423
  1000.times do |i|
436
- htm.add_node(
437
- "test_#{i}",
438
- "Memory #{i}",
439
- importance: rand(1.0..10.0)
440
- )
424
+ htm.remember("Memory #{i}")
441
425
  end
442
426
 
443
427
  # Benchmark strategies
444
428
  Benchmark.bm(15) do |x|
445
429
  x.report("Recent:") do
446
- 100.times { htm.create_context(strategy: :recent) }
430
+ 100.times { htm.working_memory.assemble_context(strategy: :recent) }
447
431
  end
448
432
 
449
- x.report("Important:") do
450
- 100.times { htm.create_context(strategy: :important) }
433
+ x.report("Frequent:") do
434
+ 100.times { htm.working_memory.assemble_context(strategy: :frequent) }
451
435
  end
452
436
 
453
437
  x.report("Balanced:") do
454
- 100.times { htm.create_context(strategy: :balanced) }
438
+ 100.times { htm.working_memory.assemble_context(strategy: :balanced) }
455
439
  end
456
440
  end
457
441
 
458
442
  # Typical results:
459
443
  # user system total real
460
444
  # Recent: 0.050000 0.000000 0.050000 ( 0.051234)
461
- # Important: 0.045000 0.000000 0.045000 ( 0.047891)
445
+ # Frequent: 0.045000 0.000000 0.045000 ( 0.047891)
462
446
  # Balanced: 0.080000 0.000000 0.080000 ( 0.082456)
463
447
  ```
464
448
 
465
449
  **Notes:**
466
450
 
467
451
  - `:recent` is fastest (simple sort)
468
- - `:important` is fast (simple sort)
452
+ - `:frequent` is fast (simple sort)
469
453
  - `:balanced` is slower (complex calculation)
470
454
  - All are typically < 100ms for normal working memory sizes
471
455
 
472
456
  ### Quality Comparison
473
457
 
474
458
  ```ruby
475
- # Test scenario: Mix of old important and recent unimportant data
459
+ # Test scenario: Mix of frequently accessed old data and recent rarely accessed data
476
460
 
477
461
  # Setup
478
462
  htm = HTM.new(robot_name: "Test")
479
463
 
480
- # Add old important data
481
- htm.add_node("old_critical", "Critical system constraint", importance: 10.0)
464
+ # Add frequently accessed data (simulate high access count)
465
+ htm.remember("Critical system constraint", metadata: { priority: "critical" })
482
466
  sleep 1 # Simulate age
483
467
 
484
- # Add recent unimportant data
468
+ # Add recent but rarely accessed data
485
469
  20.times do |i|
486
- htm.add_node("recent_#{i}", "Recent note #{i}", importance: 2.0)
470
+ htm.remember("Recent note #{i}", metadata: { priority: "low" })
487
471
  end
488
472
 
489
473
  # Compare strategies
490
474
  puts "=== Recent Strategy ==="
491
- context = htm.create_context(strategy: :recent, max_tokens: 1000)
475
+ context = htm.working_memory.assemble_context(strategy: :recent, max_tokens: 1000)
492
476
  puts context.include?("Critical system constraint") ? "✓ Has critical" : "✗ Missing critical"
493
477
 
494
- puts "\n=== Important Strategy ==="
495
- context = htm.create_context(strategy: :important, max_tokens: 1000)
478
+ puts "\n=== Frequent Strategy ==="
479
+ context = htm.working_memory.assemble_context(strategy: :frequent, max_tokens: 1000)
496
480
  puts context.include?("Critical system constraint") ? "✓ Has critical" : "✗ Missing critical"
497
481
 
498
482
  puts "\n=== Balanced Strategy ==="
499
- context = htm.create_context(strategy: :balanced, max_tokens: 1000)
483
+ context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: 1000)
500
484
  puts context.include?("Critical system constraint") ? "✓ Has critical" : "✗ Missing critical"
501
485
 
502
- # Results:
503
- # Recent: Missing critical (prioritized recent notes)
504
- # Important: Has critical (prioritized by importance)
505
- # Balanced: Has critical (balanced approach)
486
+ # Results depend on actual access patterns:
487
+ # Recent: May miss older frequently accessed data
488
+ # Frequent: Prioritizes frequently accessed items
489
+ # Balanced: Combines frequency and recency
506
490
  ```
507
491
 
508
492
  ## Advanced Techniques
@@ -514,13 +498,13 @@ Use multiple strategies for comprehensive context:
514
498
  ```ruby
515
499
  def multi_strategy_context(max_tokens_per_strategy: 10_000)
516
500
  # Get different perspectives
517
- recent = htm.create_context(
501
+ recent = htm.working_memory.assemble_context(
518
502
  strategy: :recent,
519
503
  max_tokens: max_tokens_per_strategy
520
504
  )
521
505
 
522
- important = htm.create_context(
523
- strategy: :important,
506
+ frequent = htm.working_memory.assemble_context(
507
+ strategy: :frequent,
524
508
  max_tokens: max_tokens_per_strategy
525
509
  )
526
510
 
@@ -529,8 +513,8 @@ def multi_strategy_context(max_tokens_per_strategy: 10_000)
529
513
  === Recent Context ===
530
514
  #{recent}
531
515
 
532
- === Important Context ===
533
- #{important}
516
+ === Frequently Accessed Context ===
517
+ #{frequent}
534
518
  CONTEXT
535
519
 
536
520
  combined
@@ -545,45 +529,44 @@ Choose strategy based on query type:
545
529
  def smart_context(query)
546
530
  strategy = if query.match?(/recent|latest|current/)
547
531
  :recent
548
- elsif query.match?(/important|critical|must/)
549
- :important
532
+ elsif query.match?(/important|critical|must|frequent/)
533
+ :frequent
550
534
  else
551
535
  :balanced
552
536
  end
553
537
 
554
- htm.create_context(strategy: strategy, max_tokens: 20_000)
538
+ htm.working_memory.assemble_context(strategy: strategy, max_tokens: 20_000)
555
539
  end
556
540
 
557
541
  # Usage
558
542
  context = smart_context("What are the recent changes?") # Uses :recent
559
- context = smart_context("What are critical constraints?") # Uses :important
543
+ context = smart_context("What are critical constraints?") # Uses :frequent
560
544
  context = smart_context("How do we handle auth?") # Uses :balanced
561
545
  ```
562
546
 
563
547
  ### 3. Filtered Context
564
548
 
565
- Include only specific types of memories:
549
+ Include only memories matching specific metadata:
566
550
 
567
551
  ```ruby
568
- def filtered_context(type:, strategy: :balanced)
569
- # This requires custom implementation
570
- # HTM doesn't expose working memory internals directly
571
-
572
- # Workaround: Recall specific types
552
+ def filtered_context(category:)
553
+ # Recall memories with specific metadata
573
554
  memories = htm.recall(
555
+ category,
574
556
  timeframe: "last 24 hours",
575
- topic: "type:#{type}", # Pseudo-filter
557
+ metadata: { category: category },
576
558
  strategy: :hybrid,
577
- limit: 50
578
- ).select { |m| m['type'] == type.to_s }
559
+ limit: 50,
560
+ raw: true
561
+ )
579
562
 
580
- # Manually assemble context
581
- memories.map { |m| m['value'] }.join("\n\n")
563
+ # Manually assemble context from results
564
+ memories.map { |m| m['content'] }.join("\n\n")
582
565
  end
583
566
 
584
567
  # Usage
585
- facts_only = filtered_context(type: :fact)
586
- decisions_only = filtered_context(type: :decision)
568
+ facts_only = filtered_context(category: "fact")
569
+ decisions_only = filtered_context(category: "decision")
587
570
  ```
588
571
 
589
572
  ### 4. Sectioned Context
@@ -592,27 +575,40 @@ Organize context into sections:
592
575
 
593
576
  ```ruby
594
577
  def sectioned_context
595
- # Get different types of context
596
- facts = htm.recall(timeframe: "all time", topic: "fact")
597
- .select { |m| m['type'] == 'fact' }
598
- .first(5)
578
+ # Get different types of context using metadata filtering
579
+ facts = htm.recall(
580
+ "facts",
581
+ timeframe: "all time",
582
+ metadata: { category: "fact" },
583
+ limit: 5,
584
+ raw: true
585
+ )
599
586
 
600
- decisions = htm.recall(timeframe: "all time", topic: "decision")
601
- .select { |m| m['type'] == 'decision' }
602
- .first(5)
587
+ decisions = htm.recall(
588
+ "decisions",
589
+ timeframe: "all time",
590
+ metadata: { category: "decision" },
591
+ limit: 5,
592
+ raw: true
593
+ )
603
594
 
604
- recent = htm.recall(timeframe: "last hour", topic: "", limit: 5)
595
+ recent = htm.recall(
596
+ "recent",
597
+ timeframe: "last hour",
598
+ limit: 5,
599
+ raw: true
600
+ )
605
601
 
606
602
  # Format as sections
607
603
  <<~CONTEXT
608
604
  === Core Facts ===
609
- #{facts.map { |f| "- #{f['value']}" }.join("\n")}
605
+ #{facts.map { |f| "- #{f['content']}" }.join("\n")}
610
606
 
611
607
  === Key Decisions ===
612
- #{decisions.map { |d| "- #{d['value']}" }.join("\n")}
608
+ #{decisions.map { |d| "- #{d['content']}" }.join("\n")}
613
609
 
614
610
  === Recent Activity ===
615
- #{recent.map { |r| "- #{r['value']}" }.join("\n")}
611
+ #{recent.map { |r| "- #{r['content']}" }.join("\n")}
616
612
  CONTEXT
617
613
  end
618
614
  ```
@@ -623,9 +619,8 @@ Ensure context fits LLM limits:
623
619
 
624
620
  ```ruby
625
621
  class TokenAwareContext
626
- def initialize(htm, embedding_service)
622
+ def initialize(htm)
627
623
  @htm = htm
628
- @embedding_service = embedding_service
629
624
  end
630
625
 
631
626
  def create(strategy:, llm_context_window:, reserve_for_prompt: 1000)
@@ -633,20 +628,20 @@ class TokenAwareContext
633
628
  available = llm_context_window - reserve_for_prompt
634
629
 
635
630
  # Get context
636
- context = @htm.create_context(
631
+ context = @htm.working_memory.assemble_context(
637
632
  strategy: strategy,
638
633
  max_tokens: available
639
634
  )
640
635
 
641
636
  # Verify token count
642
- actual_tokens = @embedding_service.count_tokens(context)
637
+ actual_tokens = HTM.configuration.count_tokens(context)
643
638
 
644
639
  if actual_tokens > available
645
640
  warn "Context exceeded limit! Truncating..."
646
641
  # Retry with smaller limit
647
- context = @htm.create_context(
642
+ context = @htm.working_memory.assemble_context(
648
643
  strategy: strategy,
649
- max_tokens: available * 0.9 # 90% to be safe
644
+ max_tokens: (available * 0.9).to_i # 90% to be safe
650
645
  )
651
646
  end
652
647
 
@@ -655,8 +650,7 @@ class TokenAwareContext
655
650
  end
656
651
 
657
652
  # Usage
658
- embedding_service = HTM::EmbeddingService.new
659
- context_builder = TokenAwareContext.new(htm, embedding_service)
653
+ context_builder = TokenAwareContext.new(htm)
660
654
 
661
655
  context = context_builder.create(
662
656
  strategy: :balanced,
@@ -671,7 +665,7 @@ context = context_builder.create(
671
665
 
672
666
  ```ruby
673
667
  def generate_with_context(user_query)
674
- context = htm.create_context(strategy: :balanced, max_tokens: 50_000)
668
+ context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: 50_000)
675
669
 
676
670
  system_prompt = <<~SYSTEM
677
671
  You are a helpful AI assistant with access to memory.
@@ -706,26 +700,22 @@ class ConversationManager
706
700
  def add_turn(user_msg, assistant_msg)
707
701
  timestamp = Time.now.to_i
708
702
 
709
- @htm.add_node(
710
- "#{@conversation_id}_#{timestamp}_user",
703
+ @htm.remember(
711
704
  user_msg,
712
- type: :context,
713
- importance: 6.0,
714
- tags: ["conversation", @conversation_id]
705
+ tags: ["conversation:#{@conversation_id}"],
706
+ metadata: { role: "user", timestamp: timestamp }
715
707
  )
716
708
 
717
- @htm.add_node(
718
- "#{@conversation_id}_#{timestamp}_assistant",
709
+ @htm.remember(
719
710
  assistant_msg,
720
- type: :context,
721
- importance: 6.0,
722
- tags: ["conversation", @conversation_id]
711
+ tags: ["conversation:#{@conversation_id}"],
712
+ metadata: { role: "assistant", timestamp: timestamp }
723
713
  )
724
714
  end
725
715
 
726
716
  def get_context_for_llm
727
717
  # Get recent conversation
728
- @htm.create_context(
718
+ @htm.working_memory.assemble_context(
729
719
  strategy: :recent,
730
720
  max_tokens: 10_000
731
721
  )
@@ -737,16 +727,16 @@ end
737
727
 
738
728
  ```ruby
739
729
  def rag_query(question)
740
- # 1. Retrieve relevant memories
730
+ # 1. Retrieve relevant memories (adds to working memory)
741
731
  relevant = htm.recall(
732
+ question,
742
733
  timeframe: "last month",
743
- topic: question,
744
734
  strategy: :hybrid,
745
735
  limit: 10
746
736
  )
747
737
 
748
738
  # 2. Create context from working memory (includes retrieved + existing)
749
- context = htm.create_context(
739
+ context = htm.working_memory.assemble_context(
750
740
  strategy: :balanced,
751
741
  max_tokens: 30_000
752
742
  )
@@ -788,7 +778,7 @@ class ContextCache
788
778
  end
789
779
 
790
780
  # Generate new context
791
- context = @htm.create_context(
781
+ context = @htm.working_memory.assemble_context(
792
782
  strategy: strategy,
793
783
  max_tokens: max_tokens
794
784
  )
@@ -817,17 +807,17 @@ context = cache.get_context(strategy: :balanced) # Cached for 30s
817
807
  ```ruby
818
808
  def progressive_context(start_tokens: 5_000, max_tokens: 50_000)
819
809
  # Start small
820
- context = htm.create_context(strategy: :balanced, max_tokens: start_tokens)
810
+ context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: start_tokens)
821
811
 
822
812
  # Check if more context needed (based on your logic)
823
813
  if needs_more_context?(context)
824
814
  # Expand gradually
825
- context = htm.create_context(strategy: :balanced, max_tokens: start_tokens * 2)
815
+ context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: start_tokens * 2)
826
816
  end
827
817
 
828
818
  if still_needs_more?(context)
829
819
  # Expand to max
830
- context = htm.create_context(strategy: :balanced, max_tokens: max_tokens)
820
+ context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: max_tokens)
831
821
  end
832
822
 
833
823
  context
@@ -848,33 +838,45 @@ end
848
838
 
849
839
  ```ruby
850
840
  def selective_context(query)
851
- # Determine what's relevant
841
+ # Determine what's relevant based on query
852
842
  include_facts = query.match?(/fact|truth|information/)
853
843
  include_decisions = query.match?(/decision|choice|why/)
854
844
  include_code = query.match?(/code|implement|example/)
855
845
 
856
- # Build custom context
846
+ # Build custom context using metadata filtering
857
847
  parts = []
858
848
 
859
849
  if include_facts
860
- facts = htm.recall(timeframe: "all time", topic: query)
861
- .select { |m| m['type'] == 'fact' }
862
- .first(5)
863
- parts << "Facts:\n" + facts.map { |f| "- #{f['value']}" }.join("\n")
850
+ facts = htm.recall(
851
+ query,
852
+ timeframe: "all time",
853
+ metadata: { category: "fact" },
854
+ limit: 5,
855
+ raw: true
856
+ )
857
+ parts << "Facts:\n" + facts.map { |f| "- #{f['content']}" }.join("\n")
864
858
  end
865
859
 
866
860
  if include_decisions
867
- decisions = htm.recall(timeframe: "all time", topic: query)
868
- .select { |m| m['type'] == 'decision' }
869
- .first(5)
870
- parts << "Decisions:\n" + decisions.map { |d| "- #{d['value']}" }.join("\n")
861
+ decisions = htm.recall(
862
+ query,
863
+ timeframe: "all time",
864
+ metadata: { category: "decision" },
865
+ limit: 5,
866
+ raw: true
867
+ )
868
+ parts << "Decisions:\n" + decisions.map { |d| "- #{d['content']}" }.join("\n")
871
869
  end
872
870
 
873
871
  if include_code
874
- code = htm.recall(timeframe: "all time", topic: query)
875
- .select { |m| m['type'] == 'code' }
876
- .first(3)
877
- parts << "Code Examples:\n" + code.map { |c| c['value'] }.join("\n\n")
872
+ code = htm.recall(
873
+ query,
874
+ timeframe: "all time",
875
+ metadata: { category: "code" },
876
+ limit: 3,
877
+ raw: true
878
+ )
879
+ parts << "Code Examples:\n" + code.map { |c| c['content'] }.join("\n\n")
878
880
  end
879
881
 
880
882
  parts.join("\n\n")
@@ -887,26 +889,26 @@ end
887
889
 
888
890
  ```ruby
889
891
  # Use :recent for conversations
890
- context = htm.create_context(strategy: :recent)
892
+ context = htm.working_memory.assemble_context(strategy: :recent)
891
893
 
892
- # Use :important for critical operations
893
- context = htm.create_context(strategy: :important)
894
+ # Use :frequent for critical operations
895
+ context = htm.working_memory.assemble_context(strategy: :frequent)
894
896
 
895
897
  # Use :balanced as default (recommended)
896
- context = htm.create_context(strategy: :balanced)
898
+ context = htm.working_memory.assemble_context(strategy: :balanced)
897
899
  ```
898
900
 
899
901
  ### 2. Set Appropriate Token Limits
900
902
 
901
903
  ```ruby
902
904
  # Don't exceed LLM context window
903
- context = htm.create_context(
905
+ context = htm.working_memory.assemble_context(
904
906
  strategy: :balanced,
905
907
  max_tokens: 100_000 # Leave room for prompt
906
908
  )
907
909
 
908
910
  # Smaller contexts are faster
909
- context = htm.create_context(
911
+ context = htm.working_memory.assemble_context(
910
912
  strategy: :recent,
911
913
  max_tokens: 5_000 # Quick queries
912
914
  )
@@ -916,13 +918,12 @@ context = htm.create_context(
916
918
 
917
919
  ```ruby
918
920
  def monitor_context
919
- context = htm.create_context(strategy: :balanced)
921
+ context = htm.working_memory.assemble_context(strategy: :balanced)
920
922
 
921
923
  puts "Context length: #{context.length} characters"
922
924
 
923
- # Count token estimate
924
- embedding_service = HTM::EmbeddingService.new
925
- tokens = embedding_service.count_tokens(context)
925
+ # Count tokens
926
+ tokens = HTM.configuration.count_tokens(context)
926
927
  puts "Estimated tokens: #{tokens}"
927
928
 
928
929
  # Check if too small or too large
@@ -935,15 +936,15 @@ end
935
936
 
936
937
  ```ruby
937
938
  def context_with_metadata
938
- context = htm.create_context(strategy: :balanced, max_tokens: 20_000)
939
+ context = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: 20_000)
939
940
 
940
941
  # Add metadata header
941
- stats = htm.memory_stats
942
+ wm = htm.working_memory
942
943
 
943
944
  <<~CONTEXT
944
945
  [Context assembled at #{Time.now}]
945
946
  [Strategy: balanced]
946
- [Working memory: #{stats[:working_memory][:node_count]} nodes]
947
+ [Working memory: #{wm.node_count} nodes]
947
948
  [Robot: #{htm.robot_name}]
948
949
 
949
950
  #{context}
@@ -962,27 +963,27 @@ htm = HTM.new(
962
963
  working_memory_size: 128_000
963
964
  )
964
965
 
965
- # Add various memories
966
- htm.add_node("fact_001", "User prefers Ruby", type: :fact, importance: 9.0)
967
- htm.add_node("decision_001", "Use PostgreSQL", type: :decision, importance: 8.0)
968
- htm.add_node("context_001", "Currently debugging auth", type: :context, importance: 7.0)
969
- htm.add_node("code_001", "def auth...", type: :code, importance: 6.0)
970
- htm.add_node("note_001", "Check logs later", type: :context, importance: 2.0)
966
+ # Add various memories with metadata for categorization
967
+ htm.remember("User prefers Ruby", metadata: { category: "fact", priority: "high" })
968
+ htm.remember("Use PostgreSQL for database", metadata: { category: "decision", priority: "high" })
969
+ htm.remember("Currently debugging auth module", metadata: { category: "context", priority: "medium" })
970
+ htm.remember("def authenticate(token)...", metadata: { category: "code", priority: "medium" })
971
+ htm.remember("Check logs later", metadata: { category: "note", priority: "low" })
971
972
 
972
973
  puts "=== Recent Strategy ==="
973
- recent = htm.create_context(strategy: :recent, max_tokens: 5_000)
974
+ recent = htm.working_memory.assemble_context(strategy: :recent, max_tokens: 5_000)
974
975
  puts recent
975
976
  puts "\n(Newest first)"
976
977
 
977
- puts "\n=== Important Strategy ==="
978
- important = htm.create_context(strategy: :important, max_tokens: 5_000)
979
- puts important
980
- puts "\n(Most important first)"
978
+ puts "\n=== Frequent Strategy ==="
979
+ frequent = htm.working_memory.assemble_context(strategy: :frequent, max_tokens: 5_000)
980
+ puts frequent
981
+ puts "\n(Most frequently accessed first)"
981
982
 
982
983
  puts "\n=== Balanced Strategy ==="
983
- balanced = htm.create_context(strategy: :balanced, max_tokens: 5_000)
984
+ balanced = htm.working_memory.assemble_context(strategy: :balanced, max_tokens: 5_000)
984
985
  puts balanced
985
- puts "\n(Recent + important)"
986
+ puts "\n(Balanced frequency + recency)"
986
987
 
987
988
  # Use with LLM
988
989
  def ask_llm(context, question)