htm 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.envrc +1 -0
- data/.tbls.yml +30 -0
- data/CHANGELOG.md +30 -0
- data/SETUP.md +132 -101
- data/db/migrate/20250125000001_add_content_hash_to_nodes.rb +14 -0
- data/db/migrate/20250125000002_create_robot_nodes.rb +35 -0
- data/db/migrate/20250125000003_remove_source_and_robot_id_from_nodes.rb +28 -0
- data/db/migrate/20250126000001_create_working_memories.rb +19 -0
- data/db/migrate/20250126000002_remove_unused_columns.rb +12 -0
- data/db/schema.sql +226 -43
- data/docs/api/database.md +20 -232
- data/docs/api/embedding-service.md +1 -7
- data/docs/api/htm.md +195 -449
- data/docs/api/index.md +1 -7
- data/docs/api/long-term-memory.md +342 -590
- data/docs/architecture/adrs/001-postgresql-timescaledb.md +1 -1
- data/docs/architecture/adrs/003-ollama-embeddings.md +1 -1
- data/docs/architecture/adrs/010-redis-working-memory-rejected.md +2 -27
- data/docs/architecture/adrs/index.md +2 -13
- data/docs/architecture/hive-mind.md +165 -166
- data/docs/architecture/index.md +2 -2
- data/docs/architecture/overview.md +5 -171
- data/docs/architecture/two-tier-memory.md +1 -35
- data/docs/assets/images/adr-010-current-architecture.svg +37 -0
- data/docs/assets/images/adr-010-proposed-architecture.svg +48 -0
- data/docs/assets/images/adr-dependency-tree.svg +93 -0
- data/docs/assets/images/class-hierarchy.svg +55 -0
- data/docs/assets/images/exception-hierarchy.svg +45 -0
- data/docs/assets/images/htm-architecture-overview.svg +83 -0
- data/docs/assets/images/htm-complete-memory-flow.svg +160 -0
- data/docs/assets/images/htm-context-assembly-flow.svg +148 -0
- data/docs/assets/images/htm-eviction-process.svg +141 -0
- data/docs/assets/images/htm-memory-addition-flow.svg +138 -0
- data/docs/assets/images/htm-memory-recall-flow.svg +152 -0
- data/docs/assets/images/htm-node-states.svg +123 -0
- data/docs/assets/images/project-structure.svg +78 -0
- data/docs/assets/images/test-directory-structure.svg +38 -0
- data/{dbdoc → docs/database}/README.md +5 -3
- data/{dbdoc → docs/database}/public.node_tags.md +4 -5
- data/docs/database/public.node_tags.svg +106 -0
- data/{dbdoc → docs/database}/public.nodes.md +3 -8
- data/docs/database/public.nodes.svg +152 -0
- data/docs/database/public.robot_nodes.md +44 -0
- data/docs/database/public.robot_nodes.svg +121 -0
- data/{dbdoc → docs/database}/public.robots.md +1 -2
- data/docs/database/public.robots.svg +106 -0
- data/docs/database/public.working_memories.md +40 -0
- data/docs/database/public.working_memories.svg +112 -0
- data/{dbdoc → docs/database}/schema.json +342 -110
- data/docs/database/schema.svg +223 -0
- data/docs/development/index.md +1 -29
- data/docs/development/schema.md +84 -324
- data/docs/development/testing.md +1 -9
- data/docs/getting-started/index.md +47 -0
- data/docs/{installation.md → getting-started/installation.md} +2 -2
- data/docs/{quick-start.md → getting-started/quick-start.md} +5 -5
- data/docs/guides/adding-memories.md +221 -655
- data/docs/guides/search-strategies.md +85 -51
- data/docs/images/htm-er-diagram.svg +156 -0
- data/docs/index.md +16 -31
- data/docs/multi_framework_support.md +4 -4
- data/examples/basic_usage.rb +18 -16
- data/examples/cli_app/htm_cli.rb +86 -8
- data/examples/custom_llm_configuration.rb +1 -2
- data/examples/example_app/app.rb +11 -14
- data/examples/sinatra_app/Gemfile +1 -0
- data/examples/sinatra_app/Gemfile.lock +166 -0
- data/examples/sinatra_app/app.rb +219 -24
- data/lib/htm/active_record_config.rb +10 -3
- data/lib/htm/configuration.rb +265 -78
- data/lib/htm/{sinatra.rb → integrations/sinatra.rb} +87 -12
- data/lib/htm/job_adapter.rb +10 -3
- data/lib/htm/long_term_memory.rb +220 -57
- data/lib/htm/models/node.rb +36 -7
- data/lib/htm/models/robot.rb +30 -4
- data/lib/htm/models/robot_node.rb +50 -0
- data/lib/htm/models/tag.rb +52 -0
- data/lib/htm/models/working_memory_entry.rb +88 -0
- data/lib/htm/tasks.rb +4 -0
- data/lib/htm/version.rb +1 -1
- data/lib/htm.rb +34 -13
- data/lib/tasks/htm.rake +32 -1
- data/lib/tasks/jobs.rake +7 -3
- data/lib/tasks/tags.rake +34 -0
- data/mkdocs.yml +56 -9
- metadata +61 -31
- data/dbdoc/public.node_tags.svg +0 -112
- data/dbdoc/public.nodes.svg +0 -118
- data/dbdoc/public.robots.svg +0 -90
- data/dbdoc/schema.svg +0 -154
- /data/{dbdoc → docs/database}/public.node_stats.md +0 -0
- /data/{dbdoc → docs/database}/public.node_stats.svg +0 -0
- /data/{dbdoc → docs/database}/public.nodes_tags.md +0 -0
- /data/{dbdoc → docs/database}/public.nodes_tags.svg +0 -0
- /data/{dbdoc → docs/database}/public.ontology_structure.md +0 -0
- /data/{dbdoc → docs/database}/public.ontology_structure.svg +0 -0
- /data/{dbdoc → docs/database}/public.operations_log.md +0 -0
- /data/{dbdoc → docs/database}/public.operations_log.svg +0 -0
- /data/{dbdoc → docs/database}/public.relationships.md +0 -0
- /data/{dbdoc → docs/database}/public.relationships.svg +0 -0
- /data/{dbdoc → docs/database}/public.robot_activity.md +0 -0
- /data/{dbdoc → docs/database}/public.robot_activity.svg +0 -0
- /data/{dbdoc → docs/database}/public.schema_migrations.md +0 -0
- /data/{dbdoc → docs/database}/public.schema_migrations.svg +0 -0
- /data/{dbdoc → docs/database}/public.tags.md +0 -0
- /data/{dbdoc → docs/database}/public.tags.svg +0 -0
- /data/{dbdoc → docs/database}/public.topic_relationships.md +0 -0
- /data/{dbdoc → docs/database}/public.topic_relationships.svg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Search Strategies Deep Dive
|
|
2
2
|
|
|
3
|
-
HTM provides three search strategies for retrieving memories: vector search, full-text search, and hybrid search. This guide explores each strategy in depth, when to use them, and how to optimize performance.
|
|
3
|
+
HTM provides three search strategies for retrieving memories: vector search, full-text search, and tag-enhanced hybrid search. This guide explores each strategy in depth, when to use them, and how to optimize performance.
|
|
4
4
|
|
|
5
5
|
## Overview
|
|
6
6
|
|
|
@@ -8,7 +8,7 @@ HTM provides three search strategies for retrieving memories: vector search, ful
|
|
|
8
8
|
|----------|--------|----------|----------|
|
|
9
9
|
| **Vector** | Semantic similarity via embeddings | Understanding meaning | Conceptual queries, related topics |
|
|
10
10
|
| **Full-text** | PostgreSQL text search | Exact keyword matching | Specific terms, proper nouns |
|
|
11
|
-
| **Hybrid** |
|
|
11
|
+
| **Hybrid** | Vector + fulltext + tag matching | Best overall accuracy | General purpose queries |
|
|
12
12
|
|
|
13
13
|
<svg viewBox="0 0 900 650" xmlns="http://www.w3.org/2000/svg" style="background: transparent;">
|
|
14
14
|
<!-- Title -->
|
|
@@ -123,14 +123,15 @@ User Query: "database optimization techniques"
|
|
|
123
123
|
|
|
124
124
|
```ruby
|
|
125
125
|
memories = htm.recall(
|
|
126
|
+
"improving application performance",
|
|
126
127
|
timeframe: "last month",
|
|
127
|
-
topic: "improving application performance",
|
|
128
128
|
strategy: :vector,
|
|
129
|
-
limit: 10
|
|
129
|
+
limit: 10,
|
|
130
|
+
raw: true # Get full node data with scores
|
|
130
131
|
)
|
|
131
132
|
|
|
132
133
|
memories.each do |m|
|
|
133
|
-
puts "#{m['
|
|
134
|
+
puts "#{m['content']}"
|
|
134
135
|
puts "Similarity: #{m['similarity']}" # 0.0 to 1.0
|
|
135
136
|
puts
|
|
136
137
|
end
|
|
@@ -496,45 +497,69 @@ result = conn.exec_params(
|
|
|
496
497
|
conn.close
|
|
497
498
|
```
|
|
498
499
|
|
|
499
|
-
## Hybrid Search (
|
|
500
|
+
## Hybrid Search (Tag-Enhanced)
|
|
500
501
|
|
|
501
|
-
Hybrid search combines full-text and
|
|
502
|
+
Hybrid search combines full-text, vector, and tag matching for optimal results. This is the recommended strategy for most use cases.
|
|
502
503
|
|
|
503
504
|
### How It Works
|
|
504
505
|
|
|
505
506
|
```
|
|
506
507
|
User Query: "PostgreSQL performance tuning"
|
|
507
508
|
↓
|
|
508
|
-
Step 1:
|
|
509
|
-
-
|
|
510
|
-
-
|
|
509
|
+
Step 1: Find Matching Tags
|
|
510
|
+
- Search tags for query terms (3+ chars)
|
|
511
|
+
- E.g., finds "database:postgresql", "performance:optimization"
|
|
511
512
|
↓
|
|
512
|
-
Step 2:
|
|
513
|
-
-
|
|
514
|
-
-
|
|
513
|
+
Step 2: Build Candidate Pool
|
|
514
|
+
- Full-text matches (keyword)
|
|
515
|
+
- Nodes with matching tags (categorical)
|
|
516
|
+
↓
|
|
517
|
+
Step 3: Score and Rank
|
|
518
|
+
- Vector similarity (semantic)
|
|
519
|
+
- Tag boost (categorical match)
|
|
520
|
+
- Combined score: (similarity × 0.7) + (tag_boost × 0.3)
|
|
515
521
|
↓
|
|
516
522
|
Final Results
|
|
517
|
-
- Keyword precision + Semantic understanding
|
|
523
|
+
- Keyword precision + Semantic understanding + Tag relevance
|
|
518
524
|
```
|
|
519
525
|
|
|
520
526
|
### Basic Usage
|
|
521
527
|
|
|
522
528
|
```ruby
|
|
523
529
|
memories = htm.recall(
|
|
530
|
+
"PostgreSQL performance optimization",
|
|
524
531
|
timeframe: "last month",
|
|
525
|
-
topic: "PostgreSQL performance optimization",
|
|
526
532
|
strategy: :hybrid,
|
|
527
|
-
limit: 10
|
|
533
|
+
limit: 10,
|
|
534
|
+
raw: true # Get full node data with scores
|
|
528
535
|
)
|
|
529
536
|
|
|
530
|
-
# Results have
|
|
537
|
+
# Results have keyword matches, semantic relevance, AND tag boosting
|
|
531
538
|
memories.each do |m|
|
|
532
|
-
puts "#{m['
|
|
533
|
-
puts "Similarity: #{m['similarity']}"
|
|
539
|
+
puts "#{m['content']}"
|
|
540
|
+
puts "Similarity: #{m['similarity']}" # Vector similarity (0-1)
|
|
541
|
+
puts "Tag Boost: #{m['tag_boost']}" # Tag match score (0-1)
|
|
542
|
+
puts "Combined: #{m['combined_score']}" # Weighted combination
|
|
534
543
|
puts
|
|
535
544
|
end
|
|
536
545
|
```
|
|
537
546
|
|
|
547
|
+
### Tag-Enhanced Scoring
|
|
548
|
+
|
|
549
|
+
The hybrid search automatically:
|
|
550
|
+
|
|
551
|
+
1. **Finds matching tags**: Searches tags for query term matches
|
|
552
|
+
2. **Includes tagged nodes**: Adds nodes with matching tags to candidate pool
|
|
553
|
+
3. **Calculates combined score**: `(similarity × 0.7) + (tag_boost × 0.3)`
|
|
554
|
+
|
|
555
|
+
```ruby
|
|
556
|
+
# Check which tags match a query
|
|
557
|
+
matching_tags = htm.long_term_memory.find_query_matching_tags("PostgreSQL database")
|
|
558
|
+
# => ["database:postgresql", "database:postgresql:extensions", "database:sql"]
|
|
559
|
+
|
|
560
|
+
# These tags boost relevance of associated nodes in hybrid search
|
|
561
|
+
```
|
|
562
|
+
|
|
538
563
|
### When Hybrid Search Excels
|
|
539
564
|
|
|
540
565
|
**1. General Purpose Queries**
|
|
@@ -542,14 +567,16 @@ end
|
|
|
542
567
|
```ruby
|
|
543
568
|
# Best for most use cases
|
|
544
569
|
memories = htm.recall(
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
strategy: :hybrid
|
|
570
|
+
"how to improve database query speed",
|
|
571
|
+
timeframe: "last year",
|
|
572
|
+
strategy: :hybrid,
|
|
573
|
+
raw: true
|
|
548
574
|
)
|
|
549
575
|
|
|
550
576
|
# Combines:
|
|
551
577
|
# - Keyword matches (database, query, speed)
|
|
552
578
|
# - Semantic understanding (optimization, performance)
|
|
579
|
+
# - Tag boost (nodes tagged with "database:*")
|
|
553
580
|
```
|
|
554
581
|
|
|
555
582
|
**2. Mixed Terminology**
|
|
@@ -557,14 +584,16 @@ memories = htm.recall(
|
|
|
557
584
|
```ruby
|
|
558
585
|
# Query with both specific and general terms
|
|
559
586
|
memories = htm.recall(
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
strategy: :hybrid
|
|
587
|
+
"JWT token authentication security best practices",
|
|
588
|
+
timeframe: "last year",
|
|
589
|
+
strategy: :hybrid,
|
|
590
|
+
raw: true
|
|
563
591
|
)
|
|
564
592
|
|
|
565
593
|
# Finds:
|
|
566
594
|
# - Exact "JWT" mentions (full-text)
|
|
567
595
|
# - Related security concepts (vector)
|
|
596
|
+
# - Nodes tagged "auth:jwt", "security:*" (tag boost)
|
|
568
597
|
```
|
|
569
598
|
|
|
570
599
|
**3. Production Applications**
|
|
@@ -572,12 +601,17 @@ memories = htm.recall(
|
|
|
572
601
|
```ruby
|
|
573
602
|
# Recommended default for production
|
|
574
603
|
class ProductionSearch
|
|
575
|
-
def
|
|
576
|
-
htm
|
|
577
|
-
|
|
578
|
-
|
|
604
|
+
def initialize(htm)
|
|
605
|
+
@htm = htm
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
def search(query, timeframe: "last 90 days")
|
|
609
|
+
@htm.recall(
|
|
610
|
+
query,
|
|
611
|
+
timeframe: timeframe,
|
|
579
612
|
strategy: :hybrid, # Best all-around
|
|
580
|
-
limit: 20
|
|
613
|
+
limit: 20,
|
|
614
|
+
raw: true
|
|
581
615
|
)
|
|
582
616
|
end
|
|
583
617
|
end
|
|
@@ -587,28 +621,18 @@ end
|
|
|
587
621
|
|
|
588
622
|
**Prefilter Limit**
|
|
589
623
|
|
|
590
|
-
The number of candidates from
|
|
624
|
+
The number of candidates considered from each source (fulltext and tags):
|
|
591
625
|
|
|
592
626
|
```ruby
|
|
593
|
-
# Internal parameter (not exposed in public API)
|
|
594
|
-
# Default: 100 candidates
|
|
595
|
-
|
|
596
627
|
# In LongTermMemory#search_hybrid:
|
|
597
|
-
# prefilter_limit: 100
|
|
598
|
-
```
|
|
599
|
-
|
|
600
|
-
For very large databases, you might want to adjust this:
|
|
601
|
-
|
|
602
|
-
```ruby
|
|
603
|
-
# Direct database query with custom prefilter
|
|
604
|
-
ltm = HTM::LongTermMemory.new(HTM::Database.default_config)
|
|
605
|
-
embedding_service = HTM::EmbeddingService.new
|
|
628
|
+
# prefilter_limit: 100 (default)
|
|
606
629
|
|
|
607
|
-
|
|
608
|
-
|
|
630
|
+
# Direct access with custom prefilter
|
|
631
|
+
results = htm.long_term_memory.search_hybrid(
|
|
632
|
+
timeframe: (Time.now - 365*24*3600)..Time.now,
|
|
609
633
|
query: "database optimization",
|
|
610
634
|
limit: 10,
|
|
611
|
-
embedding_service:
|
|
635
|
+
embedding_service: HTM::EmbeddingService.new,
|
|
612
636
|
prefilter_limit: 200 # More candidates
|
|
613
637
|
)
|
|
614
638
|
```
|
|
@@ -620,15 +644,15 @@ results = ltm.search_hybrid(
|
|
|
620
644
|
```ruby
|
|
621
645
|
# Good: Mix of specific keywords and concepts
|
|
622
646
|
htm.recall(
|
|
623
|
-
|
|
647
|
+
"PostgreSQL query optimization indexing performance",
|
|
624
648
|
strategy: :hybrid
|
|
625
649
|
)
|
|
626
650
|
|
|
627
651
|
# Suboptimal: Only keywords
|
|
628
|
-
htm.recall(
|
|
652
|
+
htm.recall("PostgreSQL SQL", strategy: :hybrid)
|
|
629
653
|
|
|
630
654
|
# Suboptimal: Only concepts
|
|
631
|
-
htm.recall(
|
|
655
|
+
htm.recall("making things faster", strategy: :hybrid)
|
|
632
656
|
```
|
|
633
657
|
|
|
634
658
|
**2. Use Appropriate Timeframes**
|
|
@@ -636,19 +660,29 @@ htm.recall(topic: "making things faster", strategy: :hybrid)
|
|
|
636
660
|
```ruby
|
|
637
661
|
# Narrow timeframe: Faster, more recent results
|
|
638
662
|
htm.recall(
|
|
663
|
+
"recent errors",
|
|
639
664
|
timeframe: "last week",
|
|
640
|
-
topic: "recent errors",
|
|
641
665
|
strategy: :hybrid
|
|
642
666
|
)
|
|
643
667
|
|
|
644
668
|
# Wide timeframe: Comprehensive, slower
|
|
645
669
|
htm.recall(
|
|
670
|
+
"architecture decisions",
|
|
646
671
|
timeframe: "last year",
|
|
647
|
-
topic: "architecture decisions",
|
|
648
672
|
strategy: :hybrid
|
|
649
673
|
)
|
|
650
674
|
```
|
|
651
675
|
|
|
676
|
+
**3. Check Tag Coverage**
|
|
677
|
+
|
|
678
|
+
```ruby
|
|
679
|
+
# See which tags exist for better query formulation
|
|
680
|
+
popular = htm.long_term_memory.popular_tags(limit: 20)
|
|
681
|
+
popular.each do |tag|
|
|
682
|
+
puts "#{tag[:name]}: #{tag[:usage_count]} nodes"
|
|
683
|
+
end
|
|
684
|
+
```
|
|
685
|
+
|
|
652
686
|
## Strategy Comparison
|
|
653
687
|
|
|
654
688
|
### Performance Benchmarks
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 900" style="background: transparent;">
|
|
2
|
+
<defs>
|
|
3
|
+
<style>
|
|
4
|
+
.table-box { fill: #1e1e1e; stroke: #4a9eff; stroke-width: 2; }
|
|
5
|
+
.table-header { fill: #2d5a8e; }
|
|
6
|
+
.text-header { fill: #ffffff; font-family: monospace; font-size: 14px; font-weight: bold; }
|
|
7
|
+
.text-field { fill: #d4d4d4; font-family: monospace; font-size: 11px; }
|
|
8
|
+
.text-type { fill: #8cb4e8; font-family: monospace; font-size: 10px; }
|
|
9
|
+
.relation-line { stroke: #4a9eff; stroke-width: 1.5; fill: none; }
|
|
10
|
+
.arrow { fill: #4a9eff; }
|
|
11
|
+
.join-table { fill: #1e3a1e; stroke: #4a9eff; stroke-width: 2; }
|
|
12
|
+
</style>
|
|
13
|
+
</defs>
|
|
14
|
+
|
|
15
|
+
<!-- Robots Table -->
|
|
16
|
+
<rect class="table-box" x="50" y="50" width="280" height="140" rx="5"/>
|
|
17
|
+
<rect class="table-header" x="50" y="50" width="280" height="35" rx="5"/>
|
|
18
|
+
<text class="text-header" x="190" y="73" text-anchor="middle">robots</text>
|
|
19
|
+
|
|
20
|
+
<text class="text-field" x="60" y="100">id</text>
|
|
21
|
+
<text class="text-type" x="320" y="100" text-anchor="end">BIGSERIAL PK</text>
|
|
22
|
+
|
|
23
|
+
<text class="text-field" x="60" y="120">name</text>
|
|
24
|
+
<text class="text-type" x="320" y="120" text-anchor="end">TEXT</text>
|
|
25
|
+
|
|
26
|
+
<text class="text-field" x="60" y="140">created_at</text>
|
|
27
|
+
<text class="text-type" x="320" y="140" text-anchor="end">TIMESTAMPTZ</text>
|
|
28
|
+
|
|
29
|
+
<text class="text-field" x="60" y="160">last_active</text>
|
|
30
|
+
<text class="text-type" x="320" y="160" text-anchor="end">TIMESTAMPTZ</text>
|
|
31
|
+
|
|
32
|
+
<text class="text-field" x="60" y="180">metadata</text>
|
|
33
|
+
<text class="text-type" x="320" y="180" text-anchor="end">JSONB</text>
|
|
34
|
+
|
|
35
|
+
<!-- robot_nodes Join Table -->
|
|
36
|
+
<rect class="join-table" x="400" y="50" width="320" height="180" rx="5"/>
|
|
37
|
+
<rect class="table-header" x="400" y="50" width="320" height="35" rx="5"/>
|
|
38
|
+
<text class="text-header" x="560" y="73" text-anchor="middle">robot_nodes</text>
|
|
39
|
+
|
|
40
|
+
<text class="text-field" x="410" y="100">id</text>
|
|
41
|
+
<text class="text-type" x="710" y="100" text-anchor="end">BIGSERIAL PK</text>
|
|
42
|
+
|
|
43
|
+
<text class="text-field" x="410" y="120">robot_id</text>
|
|
44
|
+
<text class="text-type" x="710" y="120" text-anchor="end">BIGINT FK</text>
|
|
45
|
+
|
|
46
|
+
<text class="text-field" x="410" y="140">node_id</text>
|
|
47
|
+
<text class="text-type" x="710" y="140" text-anchor="end">BIGINT FK</text>
|
|
48
|
+
|
|
49
|
+
<text class="text-field" x="410" y="160">first_remembered_at</text>
|
|
50
|
+
<text class="text-type" x="710" y="160" text-anchor="end">TIMESTAMPTZ</text>
|
|
51
|
+
|
|
52
|
+
<text class="text-field" x="410" y="180">last_remembered_at</text>
|
|
53
|
+
<text class="text-type" x="710" y="180" text-anchor="end">TIMESTAMPTZ</text>
|
|
54
|
+
|
|
55
|
+
<text class="text-field" x="410" y="200">remember_count</text>
|
|
56
|
+
<text class="text-type" x="710" y="200" text-anchor="end">INTEGER</text>
|
|
57
|
+
|
|
58
|
+
<text class="text-field" x="410" y="220">created_at, updated_at</text>
|
|
59
|
+
<text class="text-type" x="710" y="220" text-anchor="end">TIMESTAMPTZ</text>
|
|
60
|
+
|
|
61
|
+
<!-- Nodes Table -->
|
|
62
|
+
<rect class="table-box" x="50" y="280" width="280" height="280" rx="5"/>
|
|
63
|
+
<rect class="table-header" x="50" y="280" width="280" height="35" rx="5"/>
|
|
64
|
+
<text class="text-header" x="190" y="303" text-anchor="middle">nodes</text>
|
|
65
|
+
|
|
66
|
+
<text class="text-field" x="60" y="330">id</text>
|
|
67
|
+
<text class="text-type" x="320" y="330" text-anchor="end">BIGSERIAL PK</text>
|
|
68
|
+
|
|
69
|
+
<text class="text-field" x="60" y="350">content</text>
|
|
70
|
+
<text class="text-type" x="320" y="350" text-anchor="end">TEXT NOT NULL</text>
|
|
71
|
+
|
|
72
|
+
<text class="text-field" x="60" y="370">content_hash</text>
|
|
73
|
+
<text class="text-type" x="320" y="370" text-anchor="end">VARCHAR(64) UNIQUE</text>
|
|
74
|
+
|
|
75
|
+
<text class="text-field" x="60" y="390">access_count</text>
|
|
76
|
+
<text class="text-type" x="320" y="390" text-anchor="end">INTEGER</text>
|
|
77
|
+
|
|
78
|
+
<text class="text-field" x="60" y="410">created_at</text>
|
|
79
|
+
<text class="text-type" x="320" y="410" text-anchor="end">TIMESTAMPTZ</text>
|
|
80
|
+
|
|
81
|
+
<text class="text-field" x="60" y="430">updated_at</text>
|
|
82
|
+
<text class="text-type" x="320" y="430" text-anchor="end">TIMESTAMPTZ</text>
|
|
83
|
+
|
|
84
|
+
<text class="text-field" x="60" y="450">last_accessed</text>
|
|
85
|
+
<text class="text-type" x="320" y="450" text-anchor="end">TIMESTAMPTZ</text>
|
|
86
|
+
|
|
87
|
+
<text class="text-field" x="60" y="470">token_count</text>
|
|
88
|
+
<text class="text-type" x="320" y="470" text-anchor="end">INTEGER</text>
|
|
89
|
+
|
|
90
|
+
<text class="text-field" x="60" y="490">in_working_memory</text>
|
|
91
|
+
<text class="text-type" x="320" y="490" text-anchor="end">BOOLEAN</text>
|
|
92
|
+
|
|
93
|
+
<text class="text-field" x="60" y="510">embedding</text>
|
|
94
|
+
<text class="text-type" x="320" y="510" text-anchor="end">vector(2000)</text>
|
|
95
|
+
|
|
96
|
+
<text class="text-field" x="60" y="530">embedding_dimension</text>
|
|
97
|
+
<text class="text-type" x="320" y="530" text-anchor="end">INTEGER</text>
|
|
98
|
+
|
|
99
|
+
<!-- Tags Table -->
|
|
100
|
+
<rect class="table-box" x="850" y="280" width="280" height="120" rx="5"/>
|
|
101
|
+
<rect class="table-header" x="850" y="280" width="280" height="35" rx="5"/>
|
|
102
|
+
<text class="text-header" x="990" y="303" text-anchor="middle">tags</text>
|
|
103
|
+
|
|
104
|
+
<text class="text-field" x="860" y="330">id</text>
|
|
105
|
+
<text class="text-type" x="1120" y="330" text-anchor="end">BIGSERIAL PK</text>
|
|
106
|
+
|
|
107
|
+
<text class="text-field" x="860" y="350">name</text>
|
|
108
|
+
<text class="text-type" x="1120" y="350" text-anchor="end">TEXT UNIQUE</text>
|
|
109
|
+
|
|
110
|
+
<text class="text-field" x="860" y="370">created_at</text>
|
|
111
|
+
<text class="text-type" x="1120" y="370" text-anchor="end">TIMESTAMPTZ</text>
|
|
112
|
+
|
|
113
|
+
<!-- node_tags Join Table -->
|
|
114
|
+
<rect class="join-table" x="450" y="420" width="280" height="140" rx="5"/>
|
|
115
|
+
<rect class="table-header" x="450" y="420" width="280" height="35" rx="5"/>
|
|
116
|
+
<text class="text-header" x="590" y="443" text-anchor="middle">node_tags</text>
|
|
117
|
+
|
|
118
|
+
<text class="text-field" x="460" y="470">id</text>
|
|
119
|
+
<text class="text-type" x="720" y="470" text-anchor="end">BIGSERIAL PK</text>
|
|
120
|
+
|
|
121
|
+
<text class="text-field" x="460" y="490">node_id</text>
|
|
122
|
+
<text class="text-type" x="720" y="490" text-anchor="end">BIGINT FK</text>
|
|
123
|
+
|
|
124
|
+
<text class="text-field" x="460" y="510">tag_id</text>
|
|
125
|
+
<text class="text-type" x="720" y="510" text-anchor="end">BIGINT FK</text>
|
|
126
|
+
|
|
127
|
+
<text class="text-field" x="460" y="530">created_at</text>
|
|
128
|
+
<text class="text-type" x="720" y="530" text-anchor="end">TIMESTAMPTZ</text>
|
|
129
|
+
|
|
130
|
+
<!-- Relationships: robots -> robot_nodes -->
|
|
131
|
+
<path class="relation-line" d="M 330 120 L 400 120"/>
|
|
132
|
+
<polygon class="arrow" points="400,120 390,115 390,125"/>
|
|
133
|
+
|
|
134
|
+
<!-- Relationships: robot_nodes -> nodes -->
|
|
135
|
+
<path class="relation-line" d="M 560 230 L 560 280 L 330 280 L 330 330"/>
|
|
136
|
+
<polygon class="arrow" points="330,280 325,290 335,290"/>
|
|
137
|
+
|
|
138
|
+
<!-- Relationships: nodes -> node_tags -->
|
|
139
|
+
<path class="relation-line" d="M 330 490 L 450 490"/>
|
|
140
|
+
<polygon class="arrow" points="450,490 440,485 440,495"/>
|
|
141
|
+
|
|
142
|
+
<!-- Relationships: tags -> node_tags -->
|
|
143
|
+
<path class="relation-line" d="M 850 350 L 730 350 L 730 510"/>
|
|
144
|
+
<polygon class="arrow" points="730,510 725,500 735,500"/>
|
|
145
|
+
|
|
146
|
+
<!-- Legend -->
|
|
147
|
+
<text class="text-field" x="50" y="620" font-weight="bold">Legend:</text>
|
|
148
|
+
<text class="text-field" x="50" y="640">PK = Primary Key</text>
|
|
149
|
+
<text class="text-field" x="200" y="640">FK = Foreign Key</text>
|
|
150
|
+
<text class="text-field" x="350" y="640">Green box = Join table (many-to-many)</text>
|
|
151
|
+
|
|
152
|
+
<!-- Annotations -->
|
|
153
|
+
<text class="text-field" x="350" y="110" font-style="italic">N:M</text>
|
|
154
|
+
<text class="text-field" x="380" y="480" font-style="italic">N:M</text>
|
|
155
|
+
<text class="text-field" x="770" y="430" font-style="italic">N:M</text>
|
|
156
|
+
</svg>
|
data/docs/index.md
CHANGED
|
@@ -1,4 +1,15 @@
|
|
|
1
|
+
|
|
1
2
|
<div align="center">
|
|
3
|
+
<div style="background: linear-gradient(135deg, #ff6b6b 0%, #ee5a6f 100%); border: 4px solid #ff0000; border-radius: 12px; padding: 20px; margin: 20px auto; max-width: 800px; box-shadow: 0 8px 16px rgba(255, 0, 0, 0.3);">
|
|
4
|
+
<p style="color: #ffffff; font-size: 24px; font-weight: bold; margin: 0; text-shadow: 2px 2px 4px rgba(0,0,0,0.5);">
|
|
5
|
+
⚠️ WARNING ⚠️
|
|
6
|
+
</p>
|
|
7
|
+
<p style="color: #fff; font-size: 16px; margin: 10px 0 0 0; line-height: 1.6;">
|
|
8
|
+
This documentation is AI-generated and may contain <strong>hallucinations</strong>, <strong>inaccuracies</strong>, or <strong>outdated information</strong>.<br/>
|
|
9
|
+
Always verify critical details in the source code.
|
|
10
|
+
</p>
|
|
11
|
+
</div>
|
|
12
|
+
|
|
2
13
|
<img src="assets/images/htm_demo.gif" alt="Tree of Knowledge" width="800">
|
|
3
14
|
</div>
|
|
4
15
|
|
|
@@ -124,33 +135,7 @@ HTM is perfect for:
|
|
|
124
135
|
|
|
125
136
|
HTM consists of several key components working together:
|
|
126
137
|
|
|
127
|
-
|
|
128
|
-
┌─────────────────────────────────────────────────┐
|
|
129
|
-
│ Your LLM │
|
|
130
|
-
│ (Claude, GPT, etc.) │
|
|
131
|
-
└─────────────────┬───────────────────────────────┘
|
|
132
|
-
│
|
|
133
|
-
▼
|
|
134
|
-
┌─────────────────────────────────────────────────┐
|
|
135
|
-
│ HTM API │
|
|
136
|
-
│ add_node(), recall(), create_context(), etc. │
|
|
137
|
-
└──────┬──────────────────────────────────┬──────┘
|
|
138
|
-
│ │
|
|
139
|
-
▼ ▼
|
|
140
|
-
┌─────────────────┐ ┌──────────────────┐
|
|
141
|
-
│ Working Memory │ │ Long-term Memory │
|
|
142
|
-
│ (In-Memory) │◄────────────►│ (PostgreSQL) │
|
|
143
|
-
│ 128k tokens │ Eviction │ ∞ storage │
|
|
144
|
-
│ Fast access │ & Recall │ │
|
|
145
|
-
└─────────────────┘ └──────────────────┘
|
|
146
|
-
│
|
|
147
|
-
▼
|
|
148
|
-
┌──────────────────┐
|
|
149
|
-
│ Embedding Service│
|
|
150
|
-
│ (Ollama/RubyLLM)│
|
|
151
|
-
│ gpt-oss model │
|
|
152
|
-
└──────────────────┘
|
|
153
|
-
```
|
|
138
|
+

|
|
154
139
|
|
|
155
140
|
### Component Breakdown
|
|
156
141
|
|
|
@@ -177,8 +162,8 @@ Each type can have custom importance scores, tags, and relationships.
|
|
|
177
162
|
|
|
178
163
|
Ready to add intelligent memory to your LLM application? Follow these steps:
|
|
179
164
|
|
|
180
|
-
1. **[Installation](installation.md)**: Set up HTM, PostgreSQL, TimescaleDB, and Ollama
|
|
181
|
-
2. **[Quick Start](quick-start.md)**: Build your first HTM-powered application in 5 minutes
|
|
165
|
+
1. **[Installation](getting-started/installation.md)**: Set up HTM, PostgreSQL, TimescaleDB, and Ollama
|
|
166
|
+
2. **[Quick Start](getting-started/quick-start.md)**: Build your first HTM-powered application in 5 minutes
|
|
182
167
|
3. **[User Guide](guides/getting-started.md)**: Deep dive into all HTM features
|
|
183
168
|
4. **[API Reference](api/htm.md)**: Complete API documentation
|
|
184
169
|
|
|
@@ -208,7 +193,7 @@ Licensed under the MIT License.
|
|
|
208
193
|
|
|
209
194
|
**Next Steps:**
|
|
210
195
|
|
|
211
|
-
- [Install HTM](installation.md) and set up your environment
|
|
212
|
-
- Follow the [Quick Start Guide](quick-start.md) to build your first application
|
|
196
|
+
- [Install HTM](getting-started/installation.md) and set up your environment
|
|
197
|
+
- Follow the [Quick Start Guide](getting-started/quick-start.md) to build your first application
|
|
213
198
|
- Explore the [User Guide](guides/getting-started.md) for advanced features
|
|
214
199
|
- Check out the [API Reference](api/htm.md) for detailed documentation
|
|
@@ -38,7 +38,7 @@ puts "Found #{memories.length} memories"
|
|
|
38
38
|
```ruby
|
|
39
39
|
require 'sinatra'
|
|
40
40
|
require 'htm'
|
|
41
|
-
require 'htm/sinatra'
|
|
41
|
+
require 'htm/integrations/sinatra'
|
|
42
42
|
|
|
43
43
|
class MyApp < Sinatra::Base
|
|
44
44
|
# Automatically configures HTM with Sidekiq
|
|
@@ -51,7 +51,7 @@ class MyApp < Sinatra::Base
|
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
post '/remember' do
|
|
54
|
-
node_id = remember(params[:content]
|
|
54
|
+
node_id = remember(params[:content])
|
|
55
55
|
json status: 'ok', node_id: node_id
|
|
56
56
|
end
|
|
57
57
|
|
|
@@ -209,7 +209,7 @@ gem 'redis'
|
|
|
209
209
|
gem 'htm'
|
|
210
210
|
|
|
211
211
|
# app.rb
|
|
212
|
-
require 'htm/sinatra'
|
|
212
|
+
require 'htm/integrations/sinatra'
|
|
213
213
|
|
|
214
214
|
class MyApp < Sinatra::Base
|
|
215
215
|
register_htm # Auto-configures HTM
|
|
@@ -451,7 +451,7 @@ require 'htm'
|
|
|
451
451
|
# Threads used (not production-ready)
|
|
452
452
|
|
|
453
453
|
# After:
|
|
454
|
-
require 'htm/sinatra'
|
|
454
|
+
require 'htm/integrations/sinatra'
|
|
455
455
|
register_htm # Auto-configures Sidekiq
|
|
456
456
|
# Production-ready background jobs
|
|
457
457
|
```
|
data/examples/basic_usage.rb
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
# Basic usage example for HTM
|
|
5
5
|
#
|
|
6
6
|
# Prerequisites:
|
|
7
|
-
# 1.
|
|
8
|
-
# 2. Initialize database schema:
|
|
7
|
+
# 1. Set HTM_DBURL environment variable (see SETUP.md)
|
|
8
|
+
# 2. Initialize database schema: rake db_setup
|
|
9
9
|
# 3. Install dependencies: bundle install
|
|
10
10
|
|
|
11
11
|
require_relative '../lib/htm'
|
|
@@ -15,19 +15,21 @@ puts "=" * 60
|
|
|
15
15
|
|
|
16
16
|
# Check environment
|
|
17
17
|
unless ENV['HTM_DBURL']
|
|
18
|
-
puts "ERROR: HTM_DBURL not set. Please
|
|
18
|
+
puts "ERROR: HTM_DBURL not set. Please set it:"
|
|
19
|
+
puts " export HTM_DBURL=\"postgresql://postgres@localhost:5432/htm_development\""
|
|
20
|
+
puts "See SETUP.md for details."
|
|
19
21
|
exit 1
|
|
20
22
|
end
|
|
21
23
|
|
|
22
24
|
begin
|
|
23
|
-
# Configure HTM globally (uses
|
|
25
|
+
# Configure HTM globally (uses Ollama by default)
|
|
24
26
|
puts "\n1. Configuring HTM with Ollama provider..."
|
|
25
27
|
HTM.configure do |config|
|
|
26
28
|
config.embedding_provider = :ollama
|
|
27
|
-
config.embedding_model = 'nomic-embed-text'
|
|
29
|
+
config.embedding_model = 'nomic-embed-text:latest' # Ollama models need :tag suffix
|
|
28
30
|
config.embedding_dimensions = 768
|
|
29
31
|
config.tag_provider = :ollama
|
|
30
|
-
config.tag_model = '
|
|
32
|
+
config.tag_model = 'gemma3:latest' # Ollama models need :tag suffix
|
|
31
33
|
config.reset_to_defaults # Apply settings
|
|
32
34
|
end
|
|
33
35
|
puts "✓ HTM configured with Ollama provider"
|
|
@@ -47,20 +49,17 @@ begin
|
|
|
47
49
|
puts "\n3. Remembering information..."
|
|
48
50
|
|
|
49
51
|
node_id_1 = htm.remember(
|
|
50
|
-
"We decided to use PostgreSQL for HTM storage because it provides excellent time-series optimization and native vector search with pgvector."
|
|
51
|
-
source: "architect"
|
|
52
|
+
"We decided to use PostgreSQL for HTM storage because it provides excellent time-series optimization and native vector search with pgvector."
|
|
52
53
|
)
|
|
53
54
|
puts "✓ Remembered decision about database choice (node #{node_id_1})"
|
|
54
55
|
|
|
55
56
|
node_id_2 = htm.remember(
|
|
56
|
-
"We chose RAG (Retrieval-Augmented Generation) for memory recall, combining temporal filtering with semantic vector search."
|
|
57
|
-
source: "architect"
|
|
57
|
+
"We chose RAG (Retrieval-Augmented Generation) for memory recall, combining temporal filtering with semantic vector search."
|
|
58
58
|
)
|
|
59
59
|
puts "✓ Remembered decision about RAG approach (node #{node_id_2})"
|
|
60
60
|
|
|
61
61
|
node_id_3 = htm.remember(
|
|
62
|
-
"The user's name is Dewayne and they prefer using debug_me for debugging instead of puts."
|
|
63
|
-
source: "system"
|
|
62
|
+
"The user's name is Dewayne and they prefer using debug_me for debugging instead of puts."
|
|
64
63
|
)
|
|
65
64
|
puts "✓ Remembered fact about user preferences (node #{node_id_3})"
|
|
66
65
|
|
|
@@ -72,18 +71,21 @@ begin
|
|
|
72
71
|
memories = htm.recall(
|
|
73
72
|
"database",
|
|
74
73
|
timeframe: (Time.now - 3600)..Time.now, # Last hour
|
|
75
|
-
limit: 5
|
|
74
|
+
limit: 5,
|
|
75
|
+
raw: true # Return full node data (id, content, etc.)
|
|
76
76
|
)
|
|
77
77
|
puts "✓ Found #{memories.length} memories"
|
|
78
78
|
memories.each do |memory|
|
|
79
|
-
|
|
79
|
+
content = memory['content'] || memory[:content]
|
|
80
|
+
node_id = memory['id'] || memory[:id]
|
|
81
|
+
puts " - Node #{node_id}: #{content[0..60]}..."
|
|
80
82
|
end
|
|
81
83
|
|
|
82
84
|
puts "\n" + "=" * 60
|
|
83
85
|
puts "✓ Example completed successfully!"
|
|
84
86
|
puts "\nThe HTM API provides 3 core methods:"
|
|
85
|
-
puts " - htm.remember(content,
|
|
86
|
-
puts " - htm.recall(timeframe:,
|
|
87
|
+
puts " - htm.remember(content, tags: []) - Store information"
|
|
88
|
+
puts " - htm.recall(topic, timeframe:, ...) - Retrieve memories"
|
|
87
89
|
puts " - htm.forget(node_id, confirm:) - Delete a memory"
|
|
88
90
|
|
|
89
91
|
rescue => e
|