htm 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -0
- data/README.md +269 -79
- data/db/migrate/00003_create_file_sources.rb +5 -0
- data/db/migrate/00004_create_nodes.rb +17 -0
- data/db/migrate/00005_create_tags.rb +7 -0
- data/db/migrate/00006_create_node_tags.rb +2 -0
- data/db/migrate/00007_create_robot_nodes.rb +7 -0
- data/db/schema.sql +41 -29
- data/docs/api/yard/HTM/Configuration.md +54 -0
- data/docs/api/yard/HTM/Database.md +13 -10
- data/docs/api/yard/HTM/EmbeddingService.md +5 -1
- data/docs/api/yard/HTM/LongTermMemory.md +18 -277
- data/docs/api/yard/HTM/PropositionError.md +18 -0
- data/docs/api/yard/HTM/PropositionService.md +66 -0
- data/docs/api/yard/HTM/QueryCache.md +88 -0
- data/docs/api/yard/HTM/RobotGroup.md +481 -0
- data/docs/api/yard/HTM/SqlBuilder.md +108 -0
- data/docs/api/yard/HTM/TagService.md +4 -0
- data/docs/api/yard/HTM/Telemetry/NullInstrument.md +13 -0
- data/docs/api/yard/HTM/Telemetry/NullMeter.md +15 -0
- data/docs/api/yard/HTM/Telemetry.md +109 -0
- data/docs/api/yard/HTM/WorkingMemoryChannel.md +176 -0
- data/docs/api/yard/HTM.md +11 -23
- data/docs/api/yard/index.csv +102 -25
- data/docs/api/yard-reference.md +8 -0
- data/docs/assets/images/multi-provider-failover.svg +51 -0
- data/docs/assets/images/robot-group-architecture.svg +65 -0
- data/docs/database/README.md +3 -3
- data/docs/database/public.file_sources.svg +29 -21
- data/docs/database/public.node_tags.md +2 -0
- data/docs/database/public.node_tags.svg +53 -41
- data/docs/database/public.nodes.md +2 -0
- data/docs/database/public.nodes.svg +52 -40
- data/docs/database/public.robot_nodes.md +2 -0
- data/docs/database/public.robot_nodes.svg +30 -22
- data/docs/database/public.robots.svg +16 -12
- data/docs/database/public.tags.md +3 -0
- data/docs/database/public.tags.svg +41 -33
- data/docs/database/schema.json +66 -0
- data/docs/database/schema.svg +60 -48
- data/docs/development/index.md +13 -0
- data/docs/development/rake-tasks.md +1068 -0
- data/docs/getting-started/quick-start.md +144 -155
- data/docs/guides/adding-memories.md +2 -3
- data/docs/guides/context-assembly.md +185 -184
- data/docs/guides/getting-started.md +154 -148
- data/docs/guides/index.md +7 -0
- data/docs/guides/long-term-memory.md +60 -92
- data/docs/guides/mcp-server.md +617 -0
- data/docs/guides/multi-robot.md +249 -345
- data/docs/guides/recalling-memories.md +153 -163
- data/docs/guides/robot-groups.md +604 -0
- data/docs/guides/search-strategies.md +61 -58
- data/docs/guides/working-memory.md +103 -136
- data/docs/index.md +30 -26
- data/examples/robot_groups/robot_worker.rb +1 -2
- data/examples/robot_groups/same_process.rb +1 -4
- data/lib/htm/robot_group.rb +721 -0
- data/lib/htm/version.rb +1 -1
- data/lib/htm/working_memory_channel.rb +250 -0
- data/lib/htm.rb +2 -0
- data/mkdocs.yml +2 -0
- metadata +18 -9
- data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +0 -12
- data/db/migrate/00010_add_soft_delete_to_associations.rb +0 -29
- data/db/migrate/00011_add_performance_indexes.rb +0 -21
- data/db/migrate/00012_add_tags_trigram_index.rb +0 -18
- data/db/migrate/00013_enable_lz4_compression.rb +0 -43
- data/examples/robot_groups/lib/robot_group.rb +0 -419
- data/examples/robot_groups/lib/working_memory_channel.rb +0 -140
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9ae4db5f168c495f5afef19f740f16ee9b39136cd54403f54090921cb6952275
|
|
4
|
+
data.tar.gz: 454816f53e620a72efeae7bd8b6c0a30c459403b4f1b6a3d0ca7adf30f3cda7c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 52adbf798f4004961936f9112878d245c6872bf812611efa47bc9b7808ba39510d79d0902f109db8169e1cd0d5420338094edd7719f995f67c1b3def07a2a4d2
|
|
7
|
+
data.tar.gz: 7ac9e6a8913b9b836ddfdfdb909857ac62bdd2c5d6c6c904c2a257224fc80499a118afd3d63a3a14d0daed838e11c02b2f2149eddb8c0385ebb2d5be9ab258f8
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.0.15] - 2025-12-06
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **"Why Robots Instead of Agents?" README section** - Explains terminology choice
|
|
14
|
+
- Opens with Shakespeare quote from Romeo and Juliet
|
|
15
|
+
- Discusses semantic clarity, honest capabilities, avoiding hype cycle
|
|
16
|
+
- Critiques "agent" frameworks that are just prompt wrappers
|
|
17
|
+
- References rich heritage (Čapek's R.U.R., Asimov's Three Laws)
|
|
18
|
+
- Notes cultural personality of robots (R2-D2, Wall-E, Bender)
|
|
19
|
+
- **MCP Server documentation** - Comprehensive guide for AI assistant integration
|
|
20
|
+
- Added MCP Server section to README.md with tools/resources reference
|
|
21
|
+
- Created `docs/guides/mcp-server.md` with detailed configuration for Claude Desktop, Claude Code, and AIA
|
|
22
|
+
- Covers all 11 tools (SetRobot, Remember, Recall, Forget, Restore, ListTags, SearchTags, FindByTopic, Stats, GetRobot, GetWorkingMemory)
|
|
23
|
+
- Covers all 3 resources (statistics, tags/hierarchy, memories/recent)
|
|
24
|
+
- Usage examples, session management, and troubleshooting guide
|
|
25
|
+
- **HTM Rake Tasks Reference** - Complete documentation for all 44 rake tasks
|
|
26
|
+
- Added rake tasks summary table to README.md Development section
|
|
27
|
+
- Created `docs/development/rake-tasks.md` with detailed reference organized by namespace:
|
|
28
|
+
- Database Tasks (`htm:db:*`) - 16 tasks
|
|
29
|
+
- Documentation Tasks (`htm:doc:*`) - 9 tasks
|
|
30
|
+
- File Loading Tasks (`htm:files:*`) - 7 tasks
|
|
31
|
+
- Job Tasks (`htm:jobs:*`) - 7 tasks
|
|
32
|
+
- Tag Tasks (`htm:tags:*`) - 5 tasks
|
|
33
|
+
- Common workflows, environment variables, and troubleshooting
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
- **Documentation code examples** - Fixed incorrect API usage in docs/guides/*.md
|
|
37
|
+
- `add_node()` → `remember()`
|
|
38
|
+
- `topic:` named parameter → positional argument in `recall()`
|
|
39
|
+
- `retrieve(key)` → `long_term_memory.retrieve(node_id)`
|
|
40
|
+
- `create_context()` → `working_memory.assemble_context()`
|
|
41
|
+
- Removed invalid parameters: `type:`, `importance:`, `category:`, `related_to:`
|
|
42
|
+
|
|
10
43
|
## [0.0.14] - 2025-12-05
|
|
11
44
|
|
|
12
45
|
### Added
|
data/README.md
CHANGED
|
@@ -51,6 +51,12 @@
|
|
|
51
51
|
- Cross-robot context awareness
|
|
52
52
|
- Track which robot said what
|
|
53
53
|
|
|
54
|
+
- **Robot Groups**
|
|
55
|
+
- Coordinate multiple robots with shared working memory
|
|
56
|
+
- Real-time synchronization via PostgreSQL LISTEN/NOTIFY
|
|
57
|
+
- Active/passive roles for instant failover
|
|
58
|
+
- Dynamic scaling (add/remove robots at runtime)
|
|
59
|
+
|
|
54
60
|
- **LLM-Driven Tag Extraction**
|
|
55
61
|
- Automatic hierarchical tag extraction from content
|
|
56
62
|
- Tags in colon-delimited format (e.g., `database:postgresql:performance`)
|
|
@@ -312,6 +318,82 @@ rake htm:files:sync # Sync all files (reload changed)
|
|
|
312
318
|
rake htm:files:stats # Show file loading statistics
|
|
313
319
|
```
|
|
314
320
|
|
|
321
|
+
### Robot Groups
|
|
322
|
+
|
|
323
|
+
Robot Groups enable coordination of multiple robots with shared working memory and real-time synchronization. This is useful for high-availability setups, load balancing, and collaborative AI agents.
|
|
324
|
+
|
|
325
|
+
**Key Classes:**
|
|
326
|
+
- `HTM::RobotGroup` - Coordinates multiple robots with active/passive roles and shared working memory
|
|
327
|
+
- `HTM::WorkingMemoryChannel` - PostgreSQL LISTEN/NOTIFY pub/sub for real-time cross-process synchronization
|
|
328
|
+
|
|
329
|
+
**Basic Usage:**
|
|
330
|
+
|
|
331
|
+
```ruby
|
|
332
|
+
require 'htm'
|
|
333
|
+
|
|
334
|
+
HTM.configure do |config|
|
|
335
|
+
config.embedding_provider = :ollama
|
|
336
|
+
config.embedding_model = 'nomic-embed-text'
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
# Create a robot group with primary + standby
|
|
340
|
+
group = HTM::RobotGroup.new(
|
|
341
|
+
name: 'customer-support-ha',
|
|
342
|
+
active: ['support-primary'],
|
|
343
|
+
passive: ['support-standby'],
|
|
344
|
+
max_tokens: 8000
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
# Add memories to the shared working memory
|
|
348
|
+
group.remember("Customer #123 prefers email communication", originator: 'support-primary')
|
|
349
|
+
|
|
350
|
+
# All robots in the group can recall shared memories
|
|
351
|
+
memories = group.recall('customer', limit: 5, strategy: :fulltext)
|
|
352
|
+
|
|
353
|
+
# Simulate failover when primary fails
|
|
354
|
+
group.failover! # Promotes standby to active
|
|
355
|
+
|
|
356
|
+
# Dynamic scaling - add more robots at runtime
|
|
357
|
+
group.add_active('support-secondary')
|
|
358
|
+
group.sync_robot('support-secondary') # Sync existing memories
|
|
359
|
+
|
|
360
|
+
# Check group status
|
|
361
|
+
status = group.status
|
|
362
|
+
# => { name: "customer-support-ha", active: [...], passive: [...], in_sync: true, ... }
|
|
363
|
+
|
|
364
|
+
# Cleanup
|
|
365
|
+
group.shutdown
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
**Cross-Process Synchronization:**
|
|
369
|
+
|
|
370
|
+
For multi-process deployments, use `HTM::WorkingMemoryChannel` directly:
|
|
371
|
+
|
|
372
|
+
```ruby
|
|
373
|
+
# In each worker process
|
|
374
|
+
channel = HTM::WorkingMemoryChannel.new('my-group', HTM::Database.default_config)
|
|
375
|
+
|
|
376
|
+
# Listen for memory changes from other processes
|
|
377
|
+
channel.on_change do |event, node_id, origin_robot_id|
|
|
378
|
+
case event
|
|
379
|
+
when :added # Another robot added a memory
|
|
380
|
+
when :evicted # Another robot evicted a memory
|
|
381
|
+
when :cleared # Another robot cleared working memory
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
channel.start_listening
|
|
386
|
+
|
|
387
|
+
# Notify other processes when you add a memory
|
|
388
|
+
channel.notify(:added, node_id: node.id, robot_id: my_robot_id)
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
**Demo Applications:**
|
|
392
|
+
|
|
393
|
+
See `examples/robot_groups/` for complete examples:
|
|
394
|
+
- `same_process.rb` - Single-process demo with multiple robots
|
|
395
|
+
- `multi_process.rb` - Multi-process demo with real-time sync
|
|
396
|
+
|
|
315
397
|
### Automatic Tag Extraction
|
|
316
398
|
|
|
317
399
|
HTM automatically extracts hierarchical tags from content using LLM analysis. Tags are inferred from the content itself - you never specify them manually.
|
|
@@ -441,6 +523,75 @@ HTM uses a **null object pattern** for telemetry. When disabled or when the SDK
|
|
|
441
523
|
|
|
442
524
|
See [docs/telemetry.md](docs/telemetry.md) for detailed configuration and usage examples.
|
|
443
525
|
|
|
526
|
+
## MCP Server (Model Context Protocol)
|
|
527
|
+
|
|
528
|
+
HTM includes an MCP server that exposes memory capabilities to AI assistants like Claude Desktop, Claude Code, and AIA. This enables any MCP-compatible client to store, recall, and manage memories through a standardized protocol.
|
|
529
|
+
|
|
530
|
+
### Available Tools
|
|
531
|
+
|
|
532
|
+
| Tool | Description |
|
|
533
|
+
|------|-------------|
|
|
534
|
+
| `SetRobotTool` | Set the robot identity for the session (call first) |
|
|
535
|
+
| `GetRobotTool` | Get current robot information |
|
|
536
|
+
| `GetWorkingMemoryTool` | Get working memory contents for session restore |
|
|
537
|
+
| `RememberTool` | Store information with optional tags and metadata |
|
|
538
|
+
| `RecallTool` | Search memories using vector, fulltext, or hybrid strategies |
|
|
539
|
+
| `ForgetTool` | Soft-delete a memory (recoverable) |
|
|
540
|
+
| `RestoreTool` | Restore a soft-deleted memory |
|
|
541
|
+
| `ListTagsTool` | List tags with optional prefix filtering |
|
|
542
|
+
| `SearchTagsTool` | Fuzzy tag search with typo tolerance |
|
|
543
|
+
| `FindByTopicTool` | Find nodes by topic with optional fuzzy matching |
|
|
544
|
+
| `StatsTool` | Get memory usage statistics |
|
|
545
|
+
|
|
546
|
+
### Available Resources
|
|
547
|
+
|
|
548
|
+
| URI | Description |
|
|
549
|
+
|-----|-------------|
|
|
550
|
+
| `htm://statistics` | Memory statistics as JSON |
|
|
551
|
+
| `htm://tags/hierarchy` | Tag hierarchy as text tree |
|
|
552
|
+
| `htm://memories/recent` | Last 20 memories |
|
|
553
|
+
|
|
554
|
+
### Client Configuration
|
|
555
|
+
|
|
556
|
+
**Claude Desktop** (`~/.config/claude/claude_desktop_config.json`):
|
|
557
|
+
```json
|
|
558
|
+
{
|
|
559
|
+
"mcpServers": {
|
|
560
|
+
"htm-memory": {
|
|
561
|
+
"command": "htm_mcp.rb",
|
|
562
|
+
"env": {
|
|
563
|
+
"HTM_DBURL": "postgresql://user@localhost:5432/htm_development"
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
```
|
|
569
|
+
|
|
570
|
+
**Claude Code** (`~/.claude/claude_code_config.json`):
|
|
571
|
+
```json
|
|
572
|
+
{
|
|
573
|
+
"mcpServers": {
|
|
574
|
+
"htm-memory": {
|
|
575
|
+
"command": "htm_mcp.rb",
|
|
576
|
+
"env": {
|
|
577
|
+
"HTM_DBURL": "postgresql://user@localhost:5432/htm_development"
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
**AIA** (`~/.config/aia/config.yml`):
|
|
585
|
+
```yaml
|
|
586
|
+
mcp_servers:
|
|
587
|
+
htm-memory:
|
|
588
|
+
command: htm_mcp.rb
|
|
589
|
+
env:
|
|
590
|
+
HTM_DBURL: postgresql://user@localhost:5432/htm_development
|
|
591
|
+
```
|
|
592
|
+
|
|
593
|
+
See [docs/guides/mcp-server.md](docs/guides/mcp-server.md) for detailed configuration, usage examples, and troubleshooting.
|
|
594
|
+
|
|
444
595
|
## Configuration
|
|
445
596
|
|
|
446
597
|
HTM uses dependency injection for LLM access, allowing you to configure embedding generation, tag extraction, logging, and token counting.
|
|
@@ -1318,50 +1469,86 @@ rake test
|
|
|
1318
1469
|
ruby examples/basic_usage.rb
|
|
1319
1470
|
```
|
|
1320
1471
|
|
|
1321
|
-
### Database Management
|
|
1322
|
-
|
|
1323
|
-
HTM provides comprehensive rake tasks under the `htm:db` namespace for managing the database:
|
|
1324
|
-
|
|
1325
|
-
```bash
|
|
1326
|
-
# List all database tasks
|
|
1327
|
-
rake -T htm:db
|
|
1328
|
-
|
|
1329
|
-
# Set up database (create schema + run migrations)
|
|
1330
|
-
rake htm:db:setup
|
|
1331
|
-
|
|
1332
|
-
# Run pending migrations only
|
|
1333
|
-
rake htm:db:migrate
|
|
1334
|
-
|
|
1335
|
-
# Show migration status (which migrations are applied)
|
|
1336
|
-
rake htm:db:status
|
|
1337
|
-
|
|
1338
|
-
# Show database info (size, tables, extensions, row counts)
|
|
1339
|
-
rake htm:db:info
|
|
1340
|
-
|
|
1341
|
-
# Test database connection
|
|
1342
|
-
rake htm:db:test
|
|
1343
|
-
|
|
1344
|
-
# Open PostgreSQL console (interactive psql session)
|
|
1345
|
-
rake htm:db:console
|
|
1346
|
-
|
|
1347
|
-
# Seed database with sample data
|
|
1348
|
-
rake htm:db:seed
|
|
1349
|
-
|
|
1350
|
-
# Drop all HTM tables (WARNING: destructive!)
|
|
1351
|
-
rake htm:db:drop
|
|
1352
|
-
|
|
1353
|
-
# Drop and recreate database (WARNING: destructive!)
|
|
1354
|
-
rake htm:db:reset
|
|
1355
|
-
```
|
|
1356
|
-
|
|
1357
1472
|
**Important**: Make sure `direnv allow` has been run once in the project directory to load database environment variables from `.envrc`. Alternatively, you can manually export the environment variables:
|
|
1358
1473
|
|
|
1359
1474
|
```bash
|
|
1360
|
-
# Manually export HTM_DBURL
|
|
1361
1475
|
export HTM_DBURL="postgresql://user:password@host:port/dbname?sslmode=require"
|
|
1362
1476
|
```
|
|
1363
1477
|
|
|
1364
|
-
|
|
1478
|
+
### HTM Rake Tasks Reference
|
|
1479
|
+
|
|
1480
|
+
HTM provides comprehensive rake tasks for database management, documentation, file loading, job processing, and tag management. Use `rake -T htm` to see all available tasks.
|
|
1481
|
+
|
|
1482
|
+
#### Database Tasks (`htm:db:*`)
|
|
1483
|
+
|
|
1484
|
+
| Task | Description |
|
|
1485
|
+
|------|-------------|
|
|
1486
|
+
| `rake htm:db:setup` | Set up HTM database schema and run migrations (sets up from scratch) |
|
|
1487
|
+
| `rake htm:db:create` | Create database if it doesn't exist (respects RAILS_ENV) |
|
|
1488
|
+
| `rake htm:db:migrate` | Run pending database migrations |
|
|
1489
|
+
| `rake htm:db:status` | Show migration status |
|
|
1490
|
+
| `rake htm:db:verify` | Verify database connection (respects RAILS_ENV) |
|
|
1491
|
+
| `rake htm:db:info` | Show database info (size, tables, extensions) |
|
|
1492
|
+
| `rake htm:db:stats` | Show record counts for all HTM tables |
|
|
1493
|
+
| `rake htm:db:console` | Open PostgreSQL console (respects RAILS_ENV) |
|
|
1494
|
+
| `rake htm:db:seed` | Seed database with sample data |
|
|
1495
|
+
| `rake htm:db:schema:dump` | Dump current schema to db/schema.sql |
|
|
1496
|
+
| `rake htm:db:schema:load` | Load schema from db/schema.sql |
|
|
1497
|
+
| `rake htm:db:rebuild:embeddings` | Rebuild embeddings for all nodes |
|
|
1498
|
+
| `rake htm:db:rebuild:propositions` | Rebuild propositions for all non-proposition nodes |
|
|
1499
|
+
| `rake htm:db:tags:cleanup` | Soft delete orphaned tags and stale node_tags entries |
|
|
1500
|
+
| `rake htm:db:drop` | Drop all HTM tables (WARNING: destructive!) |
|
|
1501
|
+
| `rake htm:db:reset` | Drop and recreate database (WARNING: destructive!) |
|
|
1502
|
+
|
|
1503
|
+
#### Documentation Tasks (`htm:doc:*`)
|
|
1504
|
+
|
|
1505
|
+
| Task | Description |
|
|
1506
|
+
|------|-------------|
|
|
1507
|
+
| `rake htm:doc:all` | Generate DB docs, YARD API docs, build site, and serve |
|
|
1508
|
+
| `rake htm:doc:yard` | Build YARD API documentation (markdown format for MkDocs) |
|
|
1509
|
+
| `rake htm:doc:db` | Generate/update database documentation in docs/database/ |
|
|
1510
|
+
| `rake htm:doc:build` | Build documentation site with MkDocs |
|
|
1511
|
+
| `rake htm:doc:serve` | Serve documentation site locally with MkDocs |
|
|
1512
|
+
| `rake htm:doc:server[port]` | Start YARD documentation server (live reload) |
|
|
1513
|
+
| `rake htm:doc:fix_anchors` | Fix YARD anchor links for MkDocs compatibility |
|
|
1514
|
+
| `rake htm:doc:stats` | Show documentation coverage statistics |
|
|
1515
|
+
| `rake htm:doc:clean` | Clean generated documentation |
|
|
1516
|
+
|
|
1517
|
+
#### File Loading Tasks (`htm:files:*`)
|
|
1518
|
+
|
|
1519
|
+
| Task | Description |
|
|
1520
|
+
|------|-------------|
|
|
1521
|
+
| `rake htm:files:load[path]` | Load a markdown file into long-term memory |
|
|
1522
|
+
| `rake htm:files:load_dir[path,pattern]` | Load all markdown files from a directory |
|
|
1523
|
+
| `rake htm:files:list` | List all loaded file sources |
|
|
1524
|
+
| `rake htm:files:info[path]` | Show details for a loaded file |
|
|
1525
|
+
| `rake htm:files:sync` | Sync all loaded files (reload changed files) |
|
|
1526
|
+
| `rake htm:files:unload[path]` | Unload a file from memory |
|
|
1527
|
+
| `rake htm:files:stats` | Show file loading statistics |
|
|
1528
|
+
|
|
1529
|
+
#### Job Processing Tasks (`htm:jobs:*`)
|
|
1530
|
+
|
|
1531
|
+
| Task | Description |
|
|
1532
|
+
|------|-------------|
|
|
1533
|
+
| `rake htm:jobs:stats` | Show statistics for nodes and async job processing |
|
|
1534
|
+
| `rake htm:jobs:process_embeddings` | Process pending embedding jobs for nodes without embeddings |
|
|
1535
|
+
| `rake htm:jobs:process_tags` | Process pending tag extraction jobs for nodes without tags |
|
|
1536
|
+
| `rake htm:jobs:process_all` | Process all pending jobs (embeddings and tags) |
|
|
1537
|
+
| `rake htm:jobs:reprocess_embeddings` | Reprocess embeddings for all nodes (force regeneration) |
|
|
1538
|
+
| `rake htm:jobs:failed` | Show nodes that failed async processing |
|
|
1539
|
+
| `rake htm:jobs:clear_all` | Clear all embeddings and tags (for testing/development) |
|
|
1540
|
+
|
|
1541
|
+
#### Tag Management Tasks (`htm:tags:*`)
|
|
1542
|
+
|
|
1543
|
+
| Task | Description |
|
|
1544
|
+
|------|-------------|
|
|
1545
|
+
| `rake htm:tags:tree[prefix]` | Display tags as a hierarchical tree (text format) |
|
|
1546
|
+
| `rake htm:tags:mermaid[prefix]` | Export tags as Mermaid flowchart to tags.md |
|
|
1547
|
+
| `rake htm:tags:svg[prefix]` | Export tags as SVG visualization to tags.svg |
|
|
1548
|
+
| `rake htm:tags:export[prefix]` | Export tags in all formats (tags.txt, tags.md, tags.svg) |
|
|
1549
|
+
| `rake htm:tags:rebuild` | Rebuild all tags from node content |
|
|
1550
|
+
|
|
1551
|
+
### Common Workflows
|
|
1365
1552
|
|
|
1366
1553
|
```bash
|
|
1367
1554
|
# Initial setup (first time)
|
|
@@ -1374,57 +1561,35 @@ rake htm:db:migrate # Run pending migrations
|
|
|
1374
1561
|
# Check database state
|
|
1375
1562
|
rake htm:db:status # See migration status
|
|
1376
1563
|
rake htm:db:info # See database details
|
|
1564
|
+
rake htm:db:stats # See record counts
|
|
1377
1565
|
|
|
1378
|
-
#
|
|
1379
|
-
rake htm:db:reset # Drops and recreates everything
|
|
1380
|
-
|
|
1381
|
-
# Open database console for debugging
|
|
1382
|
-
rake htm:db:console # Opens psql
|
|
1383
|
-
```
|
|
1384
|
-
|
|
1385
|
-
### Async Job Management
|
|
1386
|
-
|
|
1387
|
-
HTM provides rake tasks under the `htm:jobs` namespace for managing async embedding and tag extraction jobs:
|
|
1388
|
-
|
|
1389
|
-
```bash
|
|
1390
|
-
# List all job management tasks
|
|
1391
|
-
rake -T htm:jobs
|
|
1392
|
-
|
|
1393
|
-
# Show statistics for async processing
|
|
1566
|
+
# Monitor async processing
|
|
1394
1567
|
rake htm:jobs:stats
|
|
1568
|
+
rake htm:jobs:failed # Show stuck jobs
|
|
1395
1569
|
|
|
1396
|
-
# Process pending jobs
|
|
1397
|
-
rake htm:jobs:
|
|
1398
|
-
rake htm:jobs:process_tags # Process nodes without tags
|
|
1399
|
-
rake htm:jobs:process_all # Process both
|
|
1400
|
-
|
|
1401
|
-
# Reprocess all nodes (force regeneration)
|
|
1402
|
-
rake htm:jobs:reprocess_embeddings # WARNING: Prompts for confirmation
|
|
1403
|
-
|
|
1404
|
-
# Debugging and maintenance
|
|
1405
|
-
rake htm:jobs:failed # Show stuck jobs (>1 hour old)
|
|
1406
|
-
rake htm:jobs:clear_all # Clear all embeddings/tags (testing only)
|
|
1407
|
-
```
|
|
1570
|
+
# Process pending jobs manually
|
|
1571
|
+
rake htm:jobs:process_all
|
|
1408
1572
|
|
|
1409
|
-
|
|
1573
|
+
# Generate documentation
|
|
1574
|
+
rake htm:doc:all # Full documentation build
|
|
1575
|
+
rake htm:doc:serve # Local preview
|
|
1410
1576
|
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
rake htm:
|
|
1577
|
+
# Load files into memory
|
|
1578
|
+
rake 'htm:files:load[docs/guide.md]'
|
|
1579
|
+
rake 'htm:files:load_dir[docs/,**/*.md]'
|
|
1414
1580
|
|
|
1415
|
-
#
|
|
1416
|
-
rake htm:
|
|
1581
|
+
# View tag hierarchy
|
|
1582
|
+
rake htm:tags:tree
|
|
1583
|
+
rake 'htm:tags:svg[database]' # Export filtered tags
|
|
1417
1584
|
|
|
1418
|
-
#
|
|
1419
|
-
rake htm:
|
|
1420
|
-
rake htm:jobs:process_embeddings # Retry failed embeddings
|
|
1585
|
+
# Reset database (development only!)
|
|
1586
|
+
rake htm:db:reset # Drops and recreates everything
|
|
1421
1587
|
|
|
1422
|
-
#
|
|
1423
|
-
rake htm:
|
|
1424
|
-
rake htm:jobs:clear_all # Clear everything and start fresh
|
|
1588
|
+
# Open database console for debugging
|
|
1589
|
+
rake htm:db:console # Opens psql
|
|
1425
1590
|
```
|
|
1426
1591
|
|
|
1427
|
-
See the [Async Job Processing](#async-job-processing) section for more details.
|
|
1592
|
+
See the [Async Job Processing](#async-job-processing) section for more details on background jobs.
|
|
1428
1593
|
|
|
1429
1594
|
## Testing
|
|
1430
1595
|
|
|
@@ -1488,6 +1653,31 @@ This separation allows you to provide any LLM implementation while HTM handles r
|
|
|
1488
1653
|
- [x] Phase 8: Production-ready gem
|
|
1489
1654
|
|
|
1490
1655
|
|
|
1656
|
+
## Why "Robots" Instead of "Agents"?
|
|
1657
|
+
|
|
1658
|
+
> "What's in a name? That which we call a rose
|
|
1659
|
+
> By any other name would smell as sweet."
|
|
1660
|
+
> — Shakespeare, *Romeo and Juliet*
|
|
1661
|
+
|
|
1662
|
+
Shakespeare argues names are arbitrary. In software, we respectfully disagree—names shape expectations and understanding.
|
|
1663
|
+
|
|
1664
|
+
HTM uses "robots" rather than the fashionable "agents" for several reasons:
|
|
1665
|
+
|
|
1666
|
+
- **Semantic clarity**: "Agent" is overloaded—user agents, software agents, real estate agents, secret agents. "Robot" is specific to automated workers.
|
|
1667
|
+
|
|
1668
|
+
- **Honest about capabilities**: "Agent" implies autonomy and genuine decision-making. These systems follow instructions—they don't have agency. "Robot" acknowledges what they actually are: tools.
|
|
1669
|
+
|
|
1670
|
+
- **Avoiding the hype cycle**: "AI Agent" and "Agentic AI" became buzzwords in 2023-2024, often meaning nothing more than "LLM with a prompt." We prefer terminology that will age well.
|
|
1671
|
+
|
|
1672
|
+
- **Many "agent" frameworks are prompt wrappers**: Look under the hood of popular agent libraries and you'll often find a single prompt in a for-loop. Calling that an "agent" sets false expectations.
|
|
1673
|
+
|
|
1674
|
+
- **Rich heritage**: "Robot" comes from Karel Čapek's 1920 play *R.U.R.* (from Czech *robota*, meaning labor). Isaac Asimov gave us the Three Laws. There's decades of thoughtful writing about robot ethics and behavior. "Agent" has no comparable tradition.
|
|
1675
|
+
|
|
1676
|
+
- **Personality**: Robots are endearing—R2-D2, Wall-E, Bender. They have cultural weight that "agent" lacks.
|
|
1677
|
+
|
|
1678
|
+
Honest terminology leads to clearer thinking. These are robots: tireless workers with perfect memory, executing instructions on our behalf. That's exactly what HTM helps them do better.
|
|
1679
|
+
|
|
1680
|
+
|
|
1491
1681
|
## Contributing
|
|
1492
1682
|
|
|
1493
1683
|
Bug reports and pull requests are welcome on GitHub at https://github.com/madbomber/htm.
|
|
@@ -16,5 +16,10 @@ class CreateFileSources < ActiveRecord::Migration[7.1]
|
|
|
16
16
|
add_index :file_sources, :file_path, unique: true, name: 'idx_file_sources_path_unique'
|
|
17
17
|
add_index :file_sources, :file_hash, name: 'idx_file_sources_hash'
|
|
18
18
|
add_index :file_sources, :last_synced_at, name: 'idx_file_sources_last_synced'
|
|
19
|
+
|
|
20
|
+
# LZ4 compression for better read performance on JSONB column
|
|
21
|
+
execute <<~SQL
|
|
22
|
+
ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
|
|
23
|
+
SQL
|
|
19
24
|
end
|
|
20
25
|
end
|
|
@@ -59,6 +59,23 @@ class CreateNodes < ActiveRecord::Migration[7.1]
|
|
|
59
59
|
CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
|
|
60
60
|
SQL
|
|
61
61
|
|
|
62
|
+
# Partial index for active (non-deleted) node queries
|
|
63
|
+
add_index :nodes, :id,
|
|
64
|
+
name: 'idx_nodes_active',
|
|
65
|
+
where: 'deleted_at IS NULL'
|
|
66
|
+
|
|
67
|
+
# Composite index for embedding-based searches on active nodes
|
|
68
|
+
execute <<-SQL
|
|
69
|
+
CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
|
|
70
|
+
WHERE deleted_at IS NULL AND embedding IS NOT NULL
|
|
71
|
+
SQL
|
|
72
|
+
|
|
73
|
+
# LZ4 compression for better read performance
|
|
74
|
+
execute <<-SQL
|
|
75
|
+
ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
|
|
76
|
+
ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
|
|
77
|
+
SQL
|
|
78
|
+
|
|
62
79
|
# Foreign key to file_sources table
|
|
63
80
|
add_foreign_key :nodes, :file_sources, column: :source_id, on_delete: :nullify
|
|
64
81
|
end
|
|
@@ -5,9 +5,16 @@ class CreateTags < ActiveRecord::Migration[7.1]
|
|
|
5
5
|
create_table :tags, comment: 'Unique tag names for categorization' do |t|
|
|
6
6
|
t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
|
|
7
7
|
t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
|
|
8
|
+
t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
|
|
8
9
|
end
|
|
9
10
|
|
|
10
11
|
add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
|
|
11
12
|
add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
|
|
13
|
+
add_index :tags, :deleted_at, name: 'idx_tags_deleted_at'
|
|
14
|
+
|
|
15
|
+
# GIN trigram index for fuzzy search (typo-tolerant queries)
|
|
16
|
+
execute <<~SQL
|
|
17
|
+
CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
|
|
18
|
+
SQL
|
|
12
19
|
end
|
|
13
20
|
end
|
|
@@ -6,11 +6,13 @@ class CreateNodeTags < ActiveRecord::Migration[7.1]
|
|
|
6
6
|
t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
|
|
7
7
|
t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
|
|
8
8
|
t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
|
|
9
|
+
t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
|
|
9
10
|
end
|
|
10
11
|
|
|
11
12
|
add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
|
|
12
13
|
add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
|
|
13
14
|
add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
|
|
15
|
+
add_index :node_tags, :deleted_at, name: 'idx_node_tags_deleted_at'
|
|
14
16
|
|
|
15
17
|
add_foreign_key :node_tags, :nodes, column: :node_id, on_delete: :cascade
|
|
16
18
|
add_foreign_key :node_tags, :tags, column: :tag_id, on_delete: :cascade
|
|
@@ -11,14 +11,21 @@ class CreateRobotNodes < ActiveRecord::Migration[7.1]
|
|
|
11
11
|
comment: 'When this robot last tried to remember this content'
|
|
12
12
|
t.integer :remember_count, default: 1, null: false,
|
|
13
13
|
comment: 'Number of times this robot has tried to remember this content'
|
|
14
|
+
t.boolean :working_memory, default: false, null: false,
|
|
15
|
+
comment: 'True if this node is currently in the robot working memory'
|
|
14
16
|
t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }
|
|
15
17
|
t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }
|
|
18
|
+
t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
|
|
16
19
|
end
|
|
17
20
|
|
|
18
21
|
add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: 'idx_robot_nodes_unique'
|
|
19
22
|
add_index :robot_nodes, :robot_id, name: 'idx_robot_nodes_robot_id'
|
|
20
23
|
add_index :robot_nodes, :node_id, name: 'idx_robot_nodes_node_id'
|
|
21
24
|
add_index :robot_nodes, :last_remembered_at, name: 'idx_robot_nodes_last_remembered_at'
|
|
25
|
+
add_index :robot_nodes, :deleted_at, name: 'idx_robot_nodes_deleted_at'
|
|
26
|
+
add_index :robot_nodes, [:robot_id, :working_memory],
|
|
27
|
+
where: 'working_memory = true',
|
|
28
|
+
name: 'idx_robot_nodes_working_memory'
|
|
22
29
|
|
|
23
30
|
add_foreign_key :robot_nodes, :robots, column: :robot_id, on_delete: :cascade
|
|
24
31
|
add_foreign_key :robot_nodes, :nodes, column: :node_id, on_delete: :cascade
|