htm 0.0.14 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +33 -0
  3. data/README.md +269 -79
  4. data/db/migrate/00003_create_file_sources.rb +5 -0
  5. data/db/migrate/00004_create_nodes.rb +17 -0
  6. data/db/migrate/00005_create_tags.rb +7 -0
  7. data/db/migrate/00006_create_node_tags.rb +2 -0
  8. data/db/migrate/00007_create_robot_nodes.rb +7 -0
  9. data/db/schema.sql +41 -29
  10. data/docs/api/yard/HTM/Configuration.md +54 -0
  11. data/docs/api/yard/HTM/Database.md +13 -10
  12. data/docs/api/yard/HTM/EmbeddingService.md +5 -1
  13. data/docs/api/yard/HTM/LongTermMemory.md +18 -277
  14. data/docs/api/yard/HTM/PropositionError.md +18 -0
  15. data/docs/api/yard/HTM/PropositionService.md +66 -0
  16. data/docs/api/yard/HTM/QueryCache.md +88 -0
  17. data/docs/api/yard/HTM/RobotGroup.md +481 -0
  18. data/docs/api/yard/HTM/SqlBuilder.md +108 -0
  19. data/docs/api/yard/HTM/TagService.md +4 -0
  20. data/docs/api/yard/HTM/Telemetry/NullInstrument.md +13 -0
  21. data/docs/api/yard/HTM/Telemetry/NullMeter.md +15 -0
  22. data/docs/api/yard/HTM/Telemetry.md +109 -0
  23. data/docs/api/yard/HTM/WorkingMemoryChannel.md +176 -0
  24. data/docs/api/yard/HTM.md +11 -23
  25. data/docs/api/yard/index.csv +102 -25
  26. data/docs/api/yard-reference.md +8 -0
  27. data/docs/assets/images/multi-provider-failover.svg +51 -0
  28. data/docs/assets/images/robot-group-architecture.svg +65 -0
  29. data/docs/database/README.md +3 -3
  30. data/docs/database/public.file_sources.svg +29 -21
  31. data/docs/database/public.node_tags.md +2 -0
  32. data/docs/database/public.node_tags.svg +53 -41
  33. data/docs/database/public.nodes.md +2 -0
  34. data/docs/database/public.nodes.svg +52 -40
  35. data/docs/database/public.robot_nodes.md +2 -0
  36. data/docs/database/public.robot_nodes.svg +30 -22
  37. data/docs/database/public.robots.svg +16 -12
  38. data/docs/database/public.tags.md +3 -0
  39. data/docs/database/public.tags.svg +41 -33
  40. data/docs/database/schema.json +66 -0
  41. data/docs/database/schema.svg +60 -48
  42. data/docs/development/index.md +13 -0
  43. data/docs/development/rake-tasks.md +1068 -0
  44. data/docs/getting-started/quick-start.md +144 -155
  45. data/docs/guides/adding-memories.md +2 -3
  46. data/docs/guides/context-assembly.md +185 -184
  47. data/docs/guides/getting-started.md +154 -148
  48. data/docs/guides/index.md +7 -0
  49. data/docs/guides/long-term-memory.md +60 -92
  50. data/docs/guides/mcp-server.md +617 -0
  51. data/docs/guides/multi-robot.md +249 -345
  52. data/docs/guides/recalling-memories.md +153 -163
  53. data/docs/guides/robot-groups.md +604 -0
  54. data/docs/guides/search-strategies.md +61 -58
  55. data/docs/guides/working-memory.md +103 -136
  56. data/docs/index.md +30 -26
  57. data/examples/robot_groups/robot_worker.rb +1 -2
  58. data/examples/robot_groups/same_process.rb +1 -4
  59. data/lib/htm/robot_group.rb +721 -0
  60. data/lib/htm/version.rb +1 -1
  61. data/lib/htm/working_memory_channel.rb +250 -0
  62. data/lib/htm.rb +2 -0
  63. data/mkdocs.yml +2 -0
  64. metadata +18 -9
  65. data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +0 -12
  66. data/db/migrate/00010_add_soft_delete_to_associations.rb +0 -29
  67. data/db/migrate/00011_add_performance_indexes.rb +0 -21
  68. data/db/migrate/00012_add_tags_trigram_index.rb +0 -18
  69. data/db/migrate/00013_enable_lz4_compression.rb +0 -43
  70. data/examples/robot_groups/lib/robot_group.rb +0 -419
  71. data/examples/robot_groups/lib/working_memory_channel.rb +0 -140
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5658951570d47a6988a3e6c5078ae90c83a8bbbaebd4bc18b8a70152d0648ab1
4
- data.tar.gz: e4216f5bbe7cbbcfb1b78058f16fb5c190a8902f4778874c858acf7401f0da14
3
+ metadata.gz: 9ae4db5f168c495f5afef19f740f16ee9b39136cd54403f54090921cb6952275
4
+ data.tar.gz: 454816f53e620a72efeae7bd8b6c0a30c459403b4f1b6a3d0ca7adf30f3cda7c
5
5
  SHA512:
6
- metadata.gz: 3d206a81c9120d7c2ffb8a8afb41056ee93a9b614a7abfabe1e14196e60570daaab048e37cc74601bd04afed0af5290616ff925a7610dc37e8816cb4d797956a
7
- data.tar.gz: d2977a3e77e2d7a0b6cd886d3edadbfe8c04b3af55e4280d8b70393de41ad019a76fe12e19c1bf141d3f6b604ac4a221caca5ea914c64985521a4ccbe2fbfce9
6
+ metadata.gz: 52adbf798f4004961936f9112878d245c6872bf812611efa47bc9b7808ba39510d79d0902f109db8169e1cd0d5420338094edd7719f995f67c1b3def07a2a4d2
7
+ data.tar.gz: 7ac9e6a8913b9b836ddfdfdb909857ac62bdd2c5d6c6c904c2a257224fc80499a118afd3d63a3a14d0daed838e11c02b2f2149eddb8c0385ebb2d5be9ab258f8
data/CHANGELOG.md CHANGED
@@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.0.15] - 2025-12-06
11
+
12
+ ### Added
13
+ - **"Why Robots Instead of Agents?" README section** - Explains terminology choice
14
+ - Opens with Shakespeare quote from Romeo and Juliet
15
+ - Discusses semantic clarity, honest capabilities, avoiding hype cycle
16
+ - Critiques "agent" frameworks that are just prompt wrappers
17
+ - References rich heritage (Čapek's R.U.R., Asimov's Three Laws)
18
+ - Notes cultural personality of robots (R2-D2, Wall-E, Bender)
19
+ - **MCP Server documentation** - Comprehensive guide for AI assistant integration
20
+ - Added MCP Server section to README.md with tools/resources reference
21
+ - Created `docs/guides/mcp-server.md` with detailed configuration for Claude Desktop, Claude Code, and AIA
22
+ - Covers all 11 tools (SetRobot, Remember, Recall, Forget, Restore, ListTags, SearchTags, FindByTopic, Stats, GetRobot, GetWorkingMemory)
23
+ - Covers all 3 resources (statistics, tags/hierarchy, memories/recent)
24
+ - Usage examples, session management, and troubleshooting guide
25
+ - **HTM Rake Tasks Reference** - Complete documentation for all 44 rake tasks
26
+ - Added rake tasks summary table to README.md Development section
27
+ - Created `docs/development/rake-tasks.md` with detailed reference organized by namespace:
28
+ - Database Tasks (`htm:db:*`) - 16 tasks
29
+ - Documentation Tasks (`htm:doc:*`) - 9 tasks
30
+ - File Loading Tasks (`htm:files:*`) - 7 tasks
31
+ - Job Tasks (`htm:jobs:*`) - 7 tasks
32
+ - Tag Tasks (`htm:tags:*`) - 5 tasks
33
+ - Common workflows, environment variables, and troubleshooting
34
+
35
+ ### Fixed
36
+ - **Documentation code examples** - Fixed incorrect API usage in docs/guides/*.md
37
+ - `add_node()` → `remember()`
38
+ - `topic:` named parameter → positional argument in `recall()`
39
+ - `retrieve(key)` → `long_term_memory.retrieve(node_id)`
40
+ - `create_context()` → `working_memory.assemble_context()`
41
+ - Removed invalid parameters: `type:`, `importance:`, `category:`, `related_to:`
42
+
10
43
  ## [0.0.14] - 2025-12-05
11
44
 
12
45
  ### Added
data/README.md CHANGED
@@ -51,6 +51,12 @@
51
51
  - Cross-robot context awareness
52
52
  - Track which robot said what
53
53
 
54
+ - **Robot Groups**
55
+ - Coordinate multiple robots with shared working memory
56
+ - Real-time synchronization via PostgreSQL LISTEN/NOTIFY
57
+ - Active/passive roles for instant failover
58
+ - Dynamic scaling (add/remove robots at runtime)
59
+
54
60
  - **LLM-Driven Tag Extraction**
55
61
  - Automatic hierarchical tag extraction from content
56
62
  - Tags in colon-delimited format (e.g., `database:postgresql:performance`)
@@ -312,6 +318,82 @@ rake htm:files:sync # Sync all files (reload changed)
312
318
  rake htm:files:stats # Show file loading statistics
313
319
  ```
314
320
 
321
+ ### Robot Groups
322
+
323
+ Robot Groups enable coordination of multiple robots with shared working memory and real-time synchronization. This is useful for high-availability setups, load balancing, and collaborative AI agents.
324
+
325
+ **Key Classes:**
326
+ - `HTM::RobotGroup` - Coordinates multiple robots with active/passive roles and shared working memory
327
+ - `HTM::WorkingMemoryChannel` - PostgreSQL LISTEN/NOTIFY pub/sub for real-time cross-process synchronization
328
+
329
+ **Basic Usage:**
330
+
331
+ ```ruby
332
+ require 'htm'
333
+
334
+ HTM.configure do |config|
335
+ config.embedding_provider = :ollama
336
+ config.embedding_model = 'nomic-embed-text'
337
+ end
338
+
339
+ # Create a robot group with primary + standby
340
+ group = HTM::RobotGroup.new(
341
+ name: 'customer-support-ha',
342
+ active: ['support-primary'],
343
+ passive: ['support-standby'],
344
+ max_tokens: 8000
345
+ )
346
+
347
+ # Add memories to the shared working memory
348
+ group.remember("Customer #123 prefers email communication", originator: 'support-primary')
349
+
350
+ # All robots in the group can recall shared memories
351
+ memories = group.recall('customer', limit: 5, strategy: :fulltext)
352
+
353
+ # Simulate failover when primary fails
354
+ group.failover! # Promotes standby to active
355
+
356
+ # Dynamic scaling - add more robots at runtime
357
+ group.add_active('support-secondary')
358
+ group.sync_robot('support-secondary') # Sync existing memories
359
+
360
+ # Check group status
361
+ status = group.status
362
+ # => { name: "customer-support-ha", active: [...], passive: [...], in_sync: true, ... }
363
+
364
+ # Cleanup
365
+ group.shutdown
366
+ ```
367
+
368
+ **Cross-Process Synchronization:**
369
+
370
+ For multi-process deployments, use `HTM::WorkingMemoryChannel` directly:
371
+
372
+ ```ruby
373
+ # In each worker process
374
+ channel = HTM::WorkingMemoryChannel.new('my-group', HTM::Database.default_config)
375
+
376
+ # Listen for memory changes from other processes
377
+ channel.on_change do |event, node_id, origin_robot_id|
378
+ case event
379
+ when :added # Another robot added a memory
380
+ when :evicted # Another robot evicted a memory
381
+ when :cleared # Another robot cleared working memory
382
+ end
383
+ end
384
+
385
+ channel.start_listening
386
+
387
+ # Notify other processes when you add a memory
388
+ channel.notify(:added, node_id: node.id, robot_id: my_robot_id)
389
+ ```
390
+
391
+ **Demo Applications:**
392
+
393
+ See `examples/robot_groups/` for complete examples:
394
+ - `same_process.rb` - Single-process demo with multiple robots
395
+ - `multi_process.rb` - Multi-process demo with real-time sync
396
+
315
397
  ### Automatic Tag Extraction
316
398
 
317
399
  HTM automatically extracts hierarchical tags from content using LLM analysis. Tags are inferred from the content itself - you never specify them manually.
@@ -441,6 +523,75 @@ HTM uses a **null object pattern** for telemetry. When disabled or when the SDK
441
523
 
442
524
  See [docs/telemetry.md](docs/telemetry.md) for detailed configuration and usage examples.
443
525
 
526
+ ## MCP Server (Model Context Protocol)
527
+
528
+ HTM includes an MCP server that exposes memory capabilities to AI assistants like Claude Desktop, Claude Code, and AIA. This enables any MCP-compatible client to store, recall, and manage memories through a standardized protocol.
529
+
530
+ ### Available Tools
531
+
532
+ | Tool | Description |
533
+ |------|-------------|
534
+ | `SetRobotTool` | Set the robot identity for the session (call first) |
535
+ | `GetRobotTool` | Get current robot information |
536
+ | `GetWorkingMemoryTool` | Get working memory contents for session restore |
537
+ | `RememberTool` | Store information with optional tags and metadata |
538
+ | `RecallTool` | Search memories using vector, fulltext, or hybrid strategies |
539
+ | `ForgetTool` | Soft-delete a memory (recoverable) |
540
+ | `RestoreTool` | Restore a soft-deleted memory |
541
+ | `ListTagsTool` | List tags with optional prefix filtering |
542
+ | `SearchTagsTool` | Fuzzy tag search with typo tolerance |
543
+ | `FindByTopicTool` | Find nodes by topic with optional fuzzy matching |
544
+ | `StatsTool` | Get memory usage statistics |
545
+
546
+ ### Available Resources
547
+
548
+ | URI | Description |
549
+ |-----|-------------|
550
+ | `htm://statistics` | Memory statistics as JSON |
551
+ | `htm://tags/hierarchy` | Tag hierarchy as text tree |
552
+ | `htm://memories/recent` | Last 20 memories |
553
+
554
+ ### Client Configuration
555
+
556
+ **Claude Desktop** (`~/.config/claude/claude_desktop_config.json`):
557
+ ```json
558
+ {
559
+ "mcpServers": {
560
+ "htm-memory": {
561
+ "command": "htm_mcp.rb",
562
+ "env": {
563
+ "HTM_DBURL": "postgresql://user@localhost:5432/htm_development"
564
+ }
565
+ }
566
+ }
567
+ }
568
+ ```
569
+
570
+ **Claude Code** (`~/.claude/claude_code_config.json`):
571
+ ```json
572
+ {
573
+ "mcpServers": {
574
+ "htm-memory": {
575
+ "command": "htm_mcp.rb",
576
+ "env": {
577
+ "HTM_DBURL": "postgresql://user@localhost:5432/htm_development"
578
+ }
579
+ }
580
+ }
581
+ }
582
+ ```
583
+
584
+ **AIA** (`~/.config/aia/config.yml`):
585
+ ```yaml
586
+ mcp_servers:
587
+ htm-memory:
588
+ command: htm_mcp.rb
589
+ env:
590
+ HTM_DBURL: postgresql://user@localhost:5432/htm_development
591
+ ```
592
+
593
+ See [docs/guides/mcp-server.md](docs/guides/mcp-server.md) for detailed configuration, usage examples, and troubleshooting.
594
+
444
595
  ## Configuration
445
596
 
446
597
  HTM uses dependency injection for LLM access, allowing you to configure embedding generation, tag extraction, logging, and token counting.
@@ -1318,50 +1469,86 @@ rake test
1318
1469
  ruby examples/basic_usage.rb
1319
1470
  ```
1320
1471
 
1321
- ### Database Management
1322
-
1323
- HTM provides comprehensive rake tasks under the `htm:db` namespace for managing the database:
1324
-
1325
- ```bash
1326
- # List all database tasks
1327
- rake -T htm:db
1328
-
1329
- # Set up database (create schema + run migrations)
1330
- rake htm:db:setup
1331
-
1332
- # Run pending migrations only
1333
- rake htm:db:migrate
1334
-
1335
- # Show migration status (which migrations are applied)
1336
- rake htm:db:status
1337
-
1338
- # Show database info (size, tables, extensions, row counts)
1339
- rake htm:db:info
1340
-
1341
- # Test database connection
1342
- rake htm:db:test
1343
-
1344
- # Open PostgreSQL console (interactive psql session)
1345
- rake htm:db:console
1346
-
1347
- # Seed database with sample data
1348
- rake htm:db:seed
1349
-
1350
- # Drop all HTM tables (WARNING: destructive!)
1351
- rake htm:db:drop
1352
-
1353
- # Drop and recreate database (WARNING: destructive!)
1354
- rake htm:db:reset
1355
- ```
1356
-
1357
1472
  **Important**: Make sure `direnv allow` has been run once in the project directory to load database environment variables from `.envrc`. Alternatively, you can manually export the environment variables:
1358
1473
 
1359
1474
  ```bash
1360
- # Manually export HTM_DBURL
1361
1475
  export HTM_DBURL="postgresql://user:password@host:port/dbname?sslmode=require"
1362
1476
  ```
1363
1477
 
1364
- **Common Workflows**:
1478
+ ### HTM Rake Tasks Reference
1479
+
1480
+ HTM provides comprehensive rake tasks for database management, documentation, file loading, job processing, and tag management. Use `rake -T htm` to see all available tasks.
1481
+
1482
+ #### Database Tasks (`htm:db:*`)
1483
+
1484
+ | Task | Description |
1485
+ |------|-------------|
1486
+ | `rake htm:db:setup` | Set up HTM database schema and run migrations (sets up from scratch) |
1487
+ | `rake htm:db:create` | Create database if it doesn't exist (respects RAILS_ENV) |
1488
+ | `rake htm:db:migrate` | Run pending database migrations |
1489
+ | `rake htm:db:status` | Show migration status |
1490
+ | `rake htm:db:verify` | Verify database connection (respects RAILS_ENV) |
1491
+ | `rake htm:db:info` | Show database info (size, tables, extensions) |
1492
+ | `rake htm:db:stats` | Show record counts for all HTM tables |
1493
+ | `rake htm:db:console` | Open PostgreSQL console (respects RAILS_ENV) |
1494
+ | `rake htm:db:seed` | Seed database with sample data |
1495
+ | `rake htm:db:schema:dump` | Dump current schema to db/schema.sql |
1496
+ | `rake htm:db:schema:load` | Load schema from db/schema.sql |
1497
+ | `rake htm:db:rebuild:embeddings` | Rebuild embeddings for all nodes |
1498
+ | `rake htm:db:rebuild:propositions` | Rebuild propositions for all non-proposition nodes |
1499
+ | `rake htm:db:tags:cleanup` | Soft delete orphaned tags and stale node_tags entries |
1500
+ | `rake htm:db:drop` | Drop all HTM tables (WARNING: destructive!) |
1501
+ | `rake htm:db:reset` | Drop and recreate database (WARNING: destructive!) |
1502
+
1503
+ #### Documentation Tasks (`htm:doc:*`)
1504
+
1505
+ | Task | Description |
1506
+ |------|-------------|
1507
+ | `rake htm:doc:all` | Generate DB docs, YARD API docs, build site, and serve |
1508
+ | `rake htm:doc:yard` | Build YARD API documentation (markdown format for MkDocs) |
1509
+ | `rake htm:doc:db` | Generate/update database documentation in docs/database/ |
1510
+ | `rake htm:doc:build` | Build documentation site with MkDocs |
1511
+ | `rake htm:doc:serve` | Serve documentation site locally with MkDocs |
1512
+ | `rake htm:doc:server[port]` | Start YARD documentation server (live reload) |
1513
+ | `rake htm:doc:fix_anchors` | Fix YARD anchor links for MkDocs compatibility |
1514
+ | `rake htm:doc:stats` | Show documentation coverage statistics |
1515
+ | `rake htm:doc:clean` | Clean generated documentation |
1516
+
1517
+ #### File Loading Tasks (`htm:files:*`)
1518
+
1519
+ | Task | Description |
1520
+ |------|-------------|
1521
+ | `rake htm:files:load[path]` | Load a markdown file into long-term memory |
1522
+ | `rake htm:files:load_dir[path,pattern]` | Load all markdown files from a directory |
1523
+ | `rake htm:files:list` | List all loaded file sources |
1524
+ | `rake htm:files:info[path]` | Show details for a loaded file |
1525
+ | `rake htm:files:sync` | Sync all loaded files (reload changed files) |
1526
+ | `rake htm:files:unload[path]` | Unload a file from memory |
1527
+ | `rake htm:files:stats` | Show file loading statistics |
1528
+
1529
+ #### Job Processing Tasks (`htm:jobs:*`)
1530
+
1531
+ | Task | Description |
1532
+ |------|-------------|
1533
+ | `rake htm:jobs:stats` | Show statistics for nodes and async job processing |
1534
+ | `rake htm:jobs:process_embeddings` | Process pending embedding jobs for nodes without embeddings |
1535
+ | `rake htm:jobs:process_tags` | Process pending tag extraction jobs for nodes without tags |
1536
+ | `rake htm:jobs:process_all` | Process all pending jobs (embeddings and tags) |
1537
+ | `rake htm:jobs:reprocess_embeddings` | Reprocess embeddings for all nodes (force regeneration) |
1538
+ | `rake htm:jobs:failed` | Show nodes that failed async processing |
1539
+ | `rake htm:jobs:clear_all` | Clear all embeddings and tags (for testing/development) |
1540
+
1541
+ #### Tag Management Tasks (`htm:tags:*`)
1542
+
1543
+ | Task | Description |
1544
+ |------|-------------|
1545
+ | `rake htm:tags:tree[prefix]` | Display tags as a hierarchical tree (text format) |
1546
+ | `rake htm:tags:mermaid[prefix]` | Export tags as Mermaid flowchart to tags.md |
1547
+ | `rake htm:tags:svg[prefix]` | Export tags as SVG visualization to tags.svg |
1548
+ | `rake htm:tags:export[prefix]` | Export tags in all formats (tags.txt, tags.md, tags.svg) |
1549
+ | `rake htm:tags:rebuild` | Rebuild all tags from node content |
1550
+
1551
+ ### Common Workflows
1365
1552
 
1366
1553
  ```bash
1367
1554
  # Initial setup (first time)
@@ -1374,57 +1561,35 @@ rake htm:db:migrate # Run pending migrations
1374
1561
  # Check database state
1375
1562
  rake htm:db:status # See migration status
1376
1563
  rake htm:db:info # See database details
1564
+ rake htm:db:stats # See record counts
1377
1565
 
1378
- # Reset database (development only!)
1379
- rake htm:db:reset # Drops and recreates everything
1380
-
1381
- # Open database console for debugging
1382
- rake htm:db:console # Opens psql
1383
- ```
1384
-
1385
- ### Async Job Management
1386
-
1387
- HTM provides rake tasks under the `htm:jobs` namespace for managing async embedding and tag extraction jobs:
1388
-
1389
- ```bash
1390
- # List all job management tasks
1391
- rake -T htm:jobs
1392
-
1393
- # Show statistics for async processing
1566
+ # Monitor async processing
1394
1567
  rake htm:jobs:stats
1568
+ rake htm:jobs:failed # Show stuck jobs
1395
1569
 
1396
- # Process pending jobs
1397
- rake htm:jobs:process_embeddings # Process nodes without embeddings
1398
- rake htm:jobs:process_tags # Process nodes without tags
1399
- rake htm:jobs:process_all # Process both
1400
-
1401
- # Reprocess all nodes (force regeneration)
1402
- rake htm:jobs:reprocess_embeddings # WARNING: Prompts for confirmation
1403
-
1404
- # Debugging and maintenance
1405
- rake htm:jobs:failed # Show stuck jobs (>1 hour old)
1406
- rake htm:jobs:clear_all # Clear all embeddings/tags (testing only)
1407
- ```
1570
+ # Process pending jobs manually
1571
+ rake htm:jobs:process_all
1408
1572
 
1409
- **Common Workflows**:
1573
+ # Generate documentation
1574
+ rake htm:doc:all # Full documentation build
1575
+ rake htm:doc:serve # Local preview
1410
1576
 
1411
- ```bash
1412
- # Monitor async processing
1413
- rake htm:jobs:stats
1577
+ # Load files into memory
1578
+ rake 'htm:files:load[docs/guide.md]'
1579
+ rake 'htm:files:load_dir[docs/,**/*.md]'
1414
1580
 
1415
- # Manually process pending jobs (if async job runner is not running)
1416
- rake htm:jobs:process_all
1581
+ # View tag hierarchy
1582
+ rake htm:tags:tree
1583
+ rake 'htm:tags:svg[database]' # Export filtered tags
1417
1584
 
1418
- # Debug stuck jobs
1419
- rake htm:jobs:failed
1420
- rake htm:jobs:process_embeddings # Retry failed embeddings
1585
+ # Reset database (development only!)
1586
+ rake htm:db:reset # Drops and recreates everything
1421
1587
 
1422
- # Development/testing: force regeneration
1423
- rake htm:jobs:reprocess_embeddings # Regenerate all embeddings
1424
- rake htm:jobs:clear_all # Clear everything and start fresh
1588
+ # Open database console for debugging
1589
+ rake htm:db:console # Opens psql
1425
1590
  ```
1426
1591
 
1427
- See the [Async Job Processing](#async-job-processing) section for more details.
1592
+ See the [Async Job Processing](#async-job-processing) section for more details on background jobs.
1428
1593
 
1429
1594
  ## Testing
1430
1595
 
@@ -1488,6 +1653,31 @@ This separation allows you to provide any LLM implementation while HTM handles r
1488
1653
  - [x] Phase 8: Production-ready gem
1489
1654
 
1490
1655
 
1656
+ ## Why "Robots" Instead of "Agents"?
1657
+
1658
+ > "What's in a name? That which we call a rose
1659
+ > By any other name would smell as sweet."
1660
+ > — Shakespeare, *Romeo and Juliet*
1661
+
1662
+ Shakespeare argues names are arbitrary. In software, we respectfully disagree—names shape expectations and understanding.
1663
+
1664
+ HTM uses "robots" rather than the fashionable "agents" for several reasons:
1665
+
1666
+ - **Semantic clarity**: "Agent" is overloaded—user agents, software agents, real estate agents, secret agents. "Robot" is specific to automated workers.
1667
+
1668
+ - **Honest about capabilities**: "Agent" implies autonomy and genuine decision-making. These systems follow instructions—they don't have agency. "Robot" acknowledges what they actually are: tools.
1669
+
1670
+ - **Avoiding the hype cycle**: "AI Agent" and "Agentic AI" became buzzwords in 2023-2024, often meaning nothing more than "LLM with a prompt." We prefer terminology that will age well.
1671
+
1672
+ - **Many "agent" frameworks are prompt wrappers**: Look under the hood of popular agent libraries and you'll often find a single prompt in a for-loop. Calling that an "agent" sets false expectations.
1673
+
1674
+ - **Rich heritage**: "Robot" comes from Karel Čapek's 1920 play *R.U.R.* (from Czech *robota*, meaning labor). Isaac Asimov gave us the Three Laws. There's decades of thoughtful writing about robot ethics and behavior. "Agent" has no comparable tradition.
1675
+
1676
+ - **Personality**: Robots are endearing—R2-D2, Wall-E, Bender. They have cultural weight that "agent" lacks.
1677
+
1678
+ Honest terminology leads to clearer thinking. These are robots: tireless workers with perfect memory, executing instructions on our behalf. That's exactly what HTM helps them do better.
1679
+
1680
+
1491
1681
  ## Contributing
1492
1682
 
1493
1683
  Bug reports and pull requests are welcome on GitHub at https://github.com/madbomber/htm.
@@ -16,5 +16,10 @@ class CreateFileSources < ActiveRecord::Migration[7.1]
16
16
  add_index :file_sources, :file_path, unique: true, name: 'idx_file_sources_path_unique'
17
17
  add_index :file_sources, :file_hash, name: 'idx_file_sources_hash'
18
18
  add_index :file_sources, :last_synced_at, name: 'idx_file_sources_last_synced'
19
+
20
+ # LZ4 compression for better read performance on JSONB column
21
+ execute <<~SQL
22
+ ALTER TABLE file_sources ALTER COLUMN frontmatter SET COMPRESSION lz4;
23
+ SQL
19
24
  end
20
25
  end
@@ -59,6 +59,23 @@ class CreateNodes < ActiveRecord::Migration[7.1]
59
59
  CHECK (embedding_dimension IS NULL OR (embedding_dimension > 0 AND embedding_dimension <= 2000))
60
60
  SQL
61
61
 
62
+ # Partial index for active (non-deleted) node queries
63
+ add_index :nodes, :id,
64
+ name: 'idx_nodes_active',
65
+ where: 'deleted_at IS NULL'
66
+
67
+ # Composite index for embedding-based searches on active nodes
68
+ execute <<-SQL
69
+ CREATE INDEX idx_nodes_active_with_embedding ON nodes (id)
70
+ WHERE deleted_at IS NULL AND embedding IS NOT NULL
71
+ SQL
72
+
73
+ # LZ4 compression for better read performance
74
+ execute <<-SQL
75
+ ALTER TABLE nodes ALTER COLUMN metadata SET COMPRESSION lz4;
76
+ ALTER TABLE nodes ALTER COLUMN content SET COMPRESSION lz4;
77
+ SQL
78
+
62
79
  # Foreign key to file_sources table
63
80
  add_foreign_key :nodes, :file_sources, column: :source_id, on_delete: :nullify
64
81
  end
@@ -5,9 +5,16 @@ class CreateTags < ActiveRecord::Migration[7.1]
5
5
  create_table :tags, comment: 'Unique tag names for categorization' do |t|
6
6
  t.text :name, null: false, comment: 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)'
7
7
  t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this tag was created'
8
+ t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
8
9
  end
9
10
 
10
11
  add_index :tags, :name, unique: true, name: 'idx_tags_name_unique'
11
12
  add_index :tags, :name, using: :btree, opclass: :text_pattern_ops, name: 'idx_tags_name_pattern'
13
+ add_index :tags, :deleted_at, name: 'idx_tags_deleted_at'
14
+
15
+ # GIN trigram index for fuzzy search (typo-tolerant queries)
16
+ execute <<~SQL
17
+ CREATE INDEX idx_tags_name_trgm ON tags USING gin(name gin_trgm_ops);
18
+ SQL
12
19
  end
13
20
  end
@@ -6,11 +6,13 @@ class CreateNodeTags < ActiveRecord::Migration[7.1]
6
6
  t.bigint :node_id, null: false, comment: 'ID of the node being tagged'
7
7
  t.bigint :tag_id, null: false, comment: 'ID of the tag being applied'
8
8
  t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }, comment: 'When this association was created'
9
+ t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
9
10
  end
10
11
 
11
12
  add_index :node_tags, [:node_id, :tag_id], unique: true, name: 'idx_node_tags_unique'
12
13
  add_index :node_tags, :node_id, name: 'idx_node_tags_node_id'
13
14
  add_index :node_tags, :tag_id, name: 'idx_node_tags_tag_id'
15
+ add_index :node_tags, :deleted_at, name: 'idx_node_tags_deleted_at'
14
16
 
15
17
  add_foreign_key :node_tags, :nodes, column: :node_id, on_delete: :cascade
16
18
  add_foreign_key :node_tags, :tags, column: :tag_id, on_delete: :cascade
@@ -11,14 +11,21 @@ class CreateRobotNodes < ActiveRecord::Migration[7.1]
11
11
  comment: 'When this robot last tried to remember this content'
12
12
  t.integer :remember_count, default: 1, null: false,
13
13
  comment: 'Number of times this robot has tried to remember this content'
14
+ t.boolean :working_memory, default: false, null: false,
15
+ comment: 'True if this node is currently in the robot working memory'
14
16
  t.timestamptz :created_at, default: -> { 'CURRENT_TIMESTAMP' }
15
17
  t.timestamptz :updated_at, default: -> { 'CURRENT_TIMESTAMP' }
18
+ t.timestamptz :deleted_at, comment: 'Soft delete timestamp'
16
19
  end
17
20
 
18
21
  add_index :robot_nodes, [:robot_id, :node_id], unique: true, name: 'idx_robot_nodes_unique'
19
22
  add_index :robot_nodes, :robot_id, name: 'idx_robot_nodes_robot_id'
20
23
  add_index :robot_nodes, :node_id, name: 'idx_robot_nodes_node_id'
21
24
  add_index :robot_nodes, :last_remembered_at, name: 'idx_robot_nodes_last_remembered_at'
25
+ add_index :robot_nodes, :deleted_at, name: 'idx_robot_nodes_deleted_at'
26
+ add_index :robot_nodes, [:robot_id, :working_memory],
27
+ where: 'working_memory = true',
28
+ name: 'idx_robot_nodes_working_memory'
22
29
 
23
30
  add_foreign_key :robot_nodes, :robots, column: :robot_id, on_delete: :cascade
24
31
  add_foreign_key :robot_nodes, :nodes, column: :node_id, on_delete: :cascade