htm 0.0.11 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/.dictate.toml +46 -0
  3. data/.envrc +2 -0
  4. data/CHANGELOG.md +85 -2
  5. data/README.md +348 -79
  6. data/Rakefile +14 -2
  7. data/bin/htm_mcp.rb +94 -0
  8. data/config/database.yml +20 -13
  9. data/db/migrate/00003_create_file_sources.rb +5 -0
  10. data/db/migrate/00004_create_nodes.rb +17 -0
  11. data/db/migrate/00005_create_tags.rb +7 -0
  12. data/db/migrate/00006_create_node_tags.rb +2 -0
  13. data/db/migrate/00007_create_robot_nodes.rb +7 -0
  14. data/db/schema.sql +69 -100
  15. data/docs/api/index.md +1 -1
  16. data/docs/api/yard/HTM/Configuration.md +54 -0
  17. data/docs/api/yard/HTM/Database.md +13 -10
  18. data/docs/api/yard/HTM/EmbeddingService.md +5 -1
  19. data/docs/api/yard/HTM/LongTermMemory.md +18 -277
  20. data/docs/api/yard/HTM/PropositionError.md +18 -0
  21. data/docs/api/yard/HTM/PropositionService.md +66 -0
  22. data/docs/api/yard/HTM/QueryCache.md +88 -0
  23. data/docs/api/yard/HTM/RobotGroup.md +481 -0
  24. data/docs/api/yard/HTM/SqlBuilder.md +108 -0
  25. data/docs/api/yard/HTM/TagService.md +4 -0
  26. data/docs/api/yard/HTM/Telemetry/NullInstrument.md +13 -0
  27. data/docs/api/yard/HTM/Telemetry/NullMeter.md +15 -0
  28. data/docs/api/yard/HTM/Telemetry.md +109 -0
  29. data/docs/api/yard/HTM/WorkingMemoryChannel.md +176 -0
  30. data/docs/api/yard/HTM.md +8 -22
  31. data/docs/api/yard/index.csv +102 -25
  32. data/docs/api/yard-reference.md +8 -0
  33. data/docs/architecture/index.md +1 -1
  34. data/docs/assets/images/multi-provider-failover.svg +51 -0
  35. data/docs/assets/images/robot-group-architecture.svg +65 -0
  36. data/docs/database/README.md +3 -3
  37. data/docs/database/public.file_sources.svg +29 -21
  38. data/docs/database/public.node_tags.md +2 -0
  39. data/docs/database/public.node_tags.svg +53 -41
  40. data/docs/database/public.nodes.md +2 -0
  41. data/docs/database/public.nodes.svg +52 -40
  42. data/docs/database/public.robot_nodes.md +2 -0
  43. data/docs/database/public.robot_nodes.svg +30 -22
  44. data/docs/database/public.robots.svg +16 -12
  45. data/docs/database/public.tags.md +3 -0
  46. data/docs/database/public.tags.svg +41 -33
  47. data/docs/database/schema.json +66 -0
  48. data/docs/database/schema.svg +60 -48
  49. data/docs/development/index.md +14 -1
  50. data/docs/development/rake-tasks.md +1068 -0
  51. data/docs/getting-started/index.md +1 -1
  52. data/docs/getting-started/quick-start.md +144 -155
  53. data/docs/guides/adding-memories.md +2 -3
  54. data/docs/guides/context-assembly.md +185 -184
  55. data/docs/guides/getting-started.md +154 -148
  56. data/docs/guides/index.md +8 -1
  57. data/docs/guides/long-term-memory.md +60 -92
  58. data/docs/guides/mcp-server.md +617 -0
  59. data/docs/guides/multi-robot.md +249 -345
  60. data/docs/guides/recalling-memories.md +153 -163
  61. data/docs/guides/robot-groups.md +604 -0
  62. data/docs/guides/search-strategies.md +61 -58
  63. data/docs/guides/working-memory.md +103 -136
  64. data/docs/images/telemetry-architecture.svg +153 -0
  65. data/docs/index.md +30 -26
  66. data/docs/telemetry.md +391 -0
  67. data/examples/README.md +46 -1
  68. data/examples/cli_app/README.md +1 -1
  69. data/examples/cli_app/htm_cli.rb +1 -1
  70. data/examples/robot_groups/robot_worker.rb +1 -2
  71. data/examples/robot_groups/same_process.rb +1 -4
  72. data/examples/sinatra_app/app.rb +1 -1
  73. data/examples/telemetry/README.md +147 -0
  74. data/examples/telemetry/SETUP_README.md +169 -0
  75. data/examples/telemetry/demo.rb +498 -0
  76. data/examples/telemetry/grafana/dashboards/htm-metrics.json +457 -0
  77. data/lib/htm/configuration.rb +261 -70
  78. data/lib/htm/database.rb +46 -22
  79. data/lib/htm/embedding_service.rb +24 -14
  80. data/lib/htm/errors.rb +15 -1
  81. data/lib/htm/jobs/generate_embedding_job.rb +19 -0
  82. data/lib/htm/jobs/generate_propositions_job.rb +103 -0
  83. data/lib/htm/jobs/generate_tags_job.rb +24 -0
  84. data/lib/htm/loaders/markdown_chunker.rb +79 -0
  85. data/lib/htm/loaders/markdown_loader.rb +41 -15
  86. data/lib/htm/long_term_memory/fulltext_search.rb +138 -0
  87. data/lib/htm/long_term_memory/hybrid_search.rb +324 -0
  88. data/lib/htm/long_term_memory/node_operations.rb +209 -0
  89. data/lib/htm/long_term_memory/relevance_scorer.rb +355 -0
  90. data/lib/htm/long_term_memory/robot_operations.rb +34 -0
  91. data/lib/htm/long_term_memory/tag_operations.rb +428 -0
  92. data/lib/htm/long_term_memory/vector_search.rb +109 -0
  93. data/lib/htm/long_term_memory.rb +51 -1153
  94. data/lib/htm/models/node.rb +35 -2
  95. data/lib/htm/models/node_tag.rb +31 -0
  96. data/lib/htm/models/robot_node.rb +31 -0
  97. data/lib/htm/models/tag.rb +44 -0
  98. data/lib/htm/proposition_service.rb +169 -0
  99. data/lib/htm/query_cache.rb +214 -0
  100. data/lib/htm/robot_group.rb +721 -0
  101. data/lib/htm/sql_builder.rb +178 -0
  102. data/lib/htm/tag_service.rb +16 -6
  103. data/lib/htm/tasks.rb +8 -2
  104. data/lib/htm/telemetry.rb +224 -0
  105. data/lib/htm/version.rb +1 -1
  106. data/lib/htm/working_memory_channel.rb +250 -0
  107. data/lib/htm.rb +66 -3
  108. data/lib/tasks/doc.rake +1 -1
  109. data/lib/tasks/htm.rake +259 -13
  110. data/mkdocs.yml +98 -96
  111. metadata +55 -20
  112. data/.aigcm_msg +0 -1
  113. data/.claude/settings.local.json +0 -95
  114. data/CLAUDE.md +0 -603
  115. data/db/migrate/00009_add_working_memory_to_robot_nodes.rb +0 -12
  116. data/examples/cli_app/temp.log +0 -93
  117. data/examples/robot_groups/lib/robot_group.rb +0 -419
  118. data/examples/robot_groups/lib/working_memory_channel.rb +0 -140
  119. data/lib/htm/loaders/paragraph_chunker.rb +0 -112
  120. data/notes/ARCHITECTURE_REVIEW.md +0 -1167
  121. data/notes/IMPLEMENTATION_SUMMARY.md +0 -606
  122. data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +0 -451
  123. data/notes/next_steps.md +0 -100
  124. data/notes/plan.md +0 -627
  125. data/notes/tag_ontology_enhancement_ideas.md +0 -222
  126. data/notes/timescaledb_removal_summary.md +0 -200
@@ -0,0 +1,457 @@
1
+ {
2
+ "annotations": {
3
+ "list": []
4
+ },
5
+ "editable": true,
6
+ "fiscalYearStartMonth": 0,
7
+ "graphTooltip": 0,
8
+ "id": null,
9
+ "links": [],
10
+ "liveNow": false,
11
+ "panels": [
12
+ {
13
+ "datasource": {
14
+ "type": "prometheus",
15
+ "uid": "prometheus"
16
+ },
17
+ "fieldConfig": {
18
+ "defaults": {
19
+ "color": {
20
+ "mode": "palette-classic"
21
+ },
22
+ "mappings": [],
23
+ "thresholds": {
24
+ "mode": "absolute",
25
+ "steps": [
26
+ { "color": "green", "value": null }
27
+ ]
28
+ },
29
+ "unit": "short"
30
+ },
31
+ "overrides": []
32
+ },
33
+ "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
34
+ "id": 1,
35
+ "options": {
36
+ "colorMode": "value",
37
+ "graphMode": "area",
38
+ "justifyMode": "auto",
39
+ "orientation": "auto",
40
+ "reduceOptions": {
41
+ "calcs": ["lastNotNull"],
42
+ "fields": "",
43
+ "values": false
44
+ },
45
+ "textMode": "auto"
46
+ },
47
+ "pluginVersion": "10.0.0",
48
+ "targets": [
49
+ {
50
+ "expr": "sum(htm_jobs_total{status=\"success\"})",
51
+ "legendFormat": "Successful Jobs",
52
+ "refId": "A"
53
+ }
54
+ ],
55
+ "title": "Total Successful Jobs",
56
+ "type": "stat"
57
+ },
58
+ {
59
+ "datasource": {
60
+ "type": "prometheus",
61
+ "uid": "prometheus"
62
+ },
63
+ "fieldConfig": {
64
+ "defaults": {
65
+ "color": {
66
+ "mode": "palette-classic"
67
+ },
68
+ "mappings": [],
69
+ "thresholds": {
70
+ "mode": "absolute",
71
+ "steps": [
72
+ { "color": "red", "value": null }
73
+ ]
74
+ },
75
+ "unit": "short"
76
+ },
77
+ "overrides": []
78
+ },
79
+ "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 },
80
+ "id": 2,
81
+ "options": {
82
+ "colorMode": "value",
83
+ "graphMode": "area",
84
+ "justifyMode": "auto",
85
+ "orientation": "auto",
86
+ "reduceOptions": {
87
+ "calcs": ["lastNotNull"],
88
+ "fields": "",
89
+ "values": false
90
+ },
91
+ "textMode": "auto"
92
+ },
93
+ "pluginVersion": "10.0.0",
94
+ "targets": [
95
+ {
96
+ "expr": "sum(htm_jobs_total{status=\"error\"})",
97
+ "legendFormat": "Failed Jobs",
98
+ "refId": "A"
99
+ }
100
+ ],
101
+ "title": "Total Failed Jobs",
102
+ "type": "stat"
103
+ },
104
+ {
105
+ "datasource": {
106
+ "type": "prometheus",
107
+ "uid": "prometheus"
108
+ },
109
+ "fieldConfig": {
110
+ "defaults": {
111
+ "color": {
112
+ "mode": "palette-classic"
113
+ },
114
+ "mappings": [],
115
+ "thresholds": {
116
+ "mode": "absolute",
117
+ "steps": [
118
+ { "color": "blue", "value": null }
119
+ ]
120
+ },
121
+ "unit": "percent"
122
+ },
123
+ "overrides": []
124
+ },
125
+ "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 },
126
+ "id": 3,
127
+ "options": {
128
+ "colorMode": "value",
129
+ "graphMode": "area",
130
+ "justifyMode": "auto",
131
+ "orientation": "auto",
132
+ "reduceOptions": {
133
+ "calcs": ["lastNotNull"],
134
+ "fields": "",
135
+ "values": false
136
+ },
137
+ "textMode": "auto"
138
+ },
139
+ "pluginVersion": "10.0.0",
140
+ "targets": [
141
+ {
142
+ "expr": "sum(htm_cache_operations_total{operation=\"hit\"}) / (sum(htm_cache_operations_total{operation=\"hit\"}) + sum(htm_cache_operations_total{operation=\"miss\"})) * 100",
143
+ "legendFormat": "Cache Hit Rate",
144
+ "refId": "A"
145
+ }
146
+ ],
147
+ "title": "Cache Hit Rate",
148
+ "type": "stat"
149
+ },
150
+ {
151
+ "datasource": {
152
+ "type": "prometheus",
153
+ "uid": "prometheus"
154
+ },
155
+ "fieldConfig": {
156
+ "defaults": {
157
+ "color": {
158
+ "mode": "palette-classic"
159
+ },
160
+ "custom": {
161
+ "axisCenteredZero": false,
162
+ "axisColorMode": "text",
163
+ "axisLabel": "",
164
+ "axisPlacement": "auto",
165
+ "barAlignment": 0,
166
+ "drawStyle": "line",
167
+ "fillOpacity": 10,
168
+ "gradientMode": "none",
169
+ "hideFrom": {
170
+ "legend": false,
171
+ "tooltip": false,
172
+ "viz": false
173
+ },
174
+ "lineInterpolation": "smooth",
175
+ "lineWidth": 2,
176
+ "pointSize": 5,
177
+ "scaleDistribution": { "type": "linear" },
178
+ "showPoints": "auto",
179
+ "spanNulls": false,
180
+ "stacking": { "group": "A", "mode": "none" },
181
+ "thresholdsStyle": { "mode": "off" }
182
+ },
183
+ "mappings": [],
184
+ "thresholds": {
185
+ "mode": "absolute",
186
+ "steps": [
187
+ { "color": "green", "value": null }
188
+ ]
189
+ },
190
+ "unit": "ms"
191
+ },
192
+ "overrides": []
193
+ },
194
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
195
+ "id": 4,
196
+ "options": {
197
+ "legend": {
198
+ "calcs": ["mean", "max"],
199
+ "displayMode": "table",
200
+ "placement": "bottom",
201
+ "showLegend": true
202
+ },
203
+ "tooltip": { "mode": "single", "sort": "none" }
204
+ },
205
+ "pluginVersion": "10.0.0",
206
+ "targets": [
207
+ {
208
+ "expr": "histogram_quantile(0.95, sum(rate(htm_embedding_latency_bucket[1m])) by (le))",
209
+ "legendFormat": "p95 Embedding",
210
+ "refId": "A"
211
+ },
212
+ {
213
+ "expr": "histogram_quantile(0.95, sum(rate(htm_tag_latency_bucket[1m])) by (le))",
214
+ "legendFormat": "p95 Tags",
215
+ "refId": "B"
216
+ }
217
+ ],
218
+ "title": "LLM Job Latency (p95)",
219
+ "type": "timeseries"
220
+ },
221
+ {
222
+ "datasource": {
223
+ "type": "prometheus",
224
+ "uid": "prometheus"
225
+ },
226
+ "fieldConfig": {
227
+ "defaults": {
228
+ "color": {
229
+ "mode": "palette-classic"
230
+ },
231
+ "custom": {
232
+ "axisCenteredZero": false,
233
+ "axisColorMode": "text",
234
+ "axisLabel": "",
235
+ "axisPlacement": "auto",
236
+ "barAlignment": 0,
237
+ "drawStyle": "line",
238
+ "fillOpacity": 10,
239
+ "gradientMode": "none",
240
+ "hideFrom": {
241
+ "legend": false,
242
+ "tooltip": false,
243
+ "viz": false
244
+ },
245
+ "lineInterpolation": "smooth",
246
+ "lineWidth": 2,
247
+ "pointSize": 5,
248
+ "scaleDistribution": { "type": "linear" },
249
+ "showPoints": "auto",
250
+ "spanNulls": false,
251
+ "stacking": { "group": "A", "mode": "none" },
252
+ "thresholdsStyle": { "mode": "off" }
253
+ },
254
+ "mappings": [],
255
+ "thresholds": {
256
+ "mode": "absolute",
257
+ "steps": [
258
+ { "color": "green", "value": null }
259
+ ]
260
+ },
261
+ "unit": "ms"
262
+ },
263
+ "overrides": []
264
+ },
265
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
266
+ "id": 5,
267
+ "options": {
268
+ "legend": {
269
+ "calcs": ["mean", "max"],
270
+ "displayMode": "table",
271
+ "placement": "bottom",
272
+ "showLegend": true
273
+ },
274
+ "tooltip": { "mode": "single", "sort": "none" }
275
+ },
276
+ "pluginVersion": "10.0.0",
277
+ "targets": [
278
+ {
279
+ "expr": "histogram_quantile(0.95, sum(rate(htm_search_latency_bucket{strategy=\"vector\"}[1m])) by (le))",
280
+ "legendFormat": "p95 Vector",
281
+ "refId": "A"
282
+ },
283
+ {
284
+ "expr": "histogram_quantile(0.95, sum(rate(htm_search_latency_bucket{strategy=\"fulltext\"}[1m])) by (le))",
285
+ "legendFormat": "p95 Fulltext",
286
+ "refId": "B"
287
+ },
288
+ {
289
+ "expr": "histogram_quantile(0.95, sum(rate(htm_search_latency_bucket{strategy=\"hybrid\"}[1m])) by (le))",
290
+ "legendFormat": "p95 Hybrid",
291
+ "refId": "C"
292
+ }
293
+ ],
294
+ "title": "Search Latency by Strategy (p95)",
295
+ "type": "timeseries"
296
+ },
297
+ {
298
+ "datasource": {
299
+ "type": "prometheus",
300
+ "uid": "prometheus"
301
+ },
302
+ "fieldConfig": {
303
+ "defaults": {
304
+ "color": {
305
+ "mode": "palette-classic"
306
+ },
307
+ "custom": {
308
+ "axisCenteredZero": false,
309
+ "axisColorMode": "text",
310
+ "axisLabel": "",
311
+ "axisPlacement": "auto",
312
+ "barAlignment": 0,
313
+ "drawStyle": "bars",
314
+ "fillOpacity": 80,
315
+ "gradientMode": "none",
316
+ "hideFrom": {
317
+ "legend": false,
318
+ "tooltip": false,
319
+ "viz": false
320
+ },
321
+ "lineInterpolation": "linear",
322
+ "lineWidth": 1,
323
+ "pointSize": 5,
324
+ "scaleDistribution": { "type": "linear" },
325
+ "showPoints": "never",
326
+ "spanNulls": false,
327
+ "stacking": { "group": "A", "mode": "normal" },
328
+ "thresholdsStyle": { "mode": "off" }
329
+ },
330
+ "mappings": [],
331
+ "thresholds": {
332
+ "mode": "absolute",
333
+ "steps": [
334
+ { "color": "green", "value": null }
335
+ ]
336
+ },
337
+ "unit": "short"
338
+ },
339
+ "overrides": []
340
+ },
341
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
342
+ "id": 6,
343
+ "options": {
344
+ "legend": {
345
+ "calcs": [],
346
+ "displayMode": "list",
347
+ "placement": "bottom",
348
+ "showLegend": true
349
+ },
350
+ "tooltip": { "mode": "single", "sort": "none" }
351
+ },
352
+ "pluginVersion": "10.0.0",
353
+ "targets": [
354
+ {
355
+ "expr": "sum(rate(htm_jobs_total{job=\"embedding\"}[1m])) by (status)",
356
+ "legendFormat": "Embedding - {{status}}",
357
+ "refId": "A"
358
+ },
359
+ {
360
+ "expr": "sum(rate(htm_jobs_total{job=\"tags\"}[1m])) by (status)",
361
+ "legendFormat": "Tags - {{status}}",
362
+ "refId": "B"
363
+ }
364
+ ],
365
+ "title": "Jobs per Minute by Type",
366
+ "type": "timeseries"
367
+ },
368
+ {
369
+ "datasource": {
370
+ "type": "prometheus",
371
+ "uid": "prometheus"
372
+ },
373
+ "fieldConfig": {
374
+ "defaults": {
375
+ "color": {
376
+ "mode": "palette-classic"
377
+ },
378
+ "custom": {
379
+ "axisCenteredZero": false,
380
+ "axisColorMode": "text",
381
+ "axisLabel": "",
382
+ "axisPlacement": "auto",
383
+ "barAlignment": 0,
384
+ "drawStyle": "bars",
385
+ "fillOpacity": 80,
386
+ "gradientMode": "none",
387
+ "hideFrom": {
388
+ "legend": false,
389
+ "tooltip": false,
390
+ "viz": false
391
+ },
392
+ "lineInterpolation": "linear",
393
+ "lineWidth": 1,
394
+ "pointSize": 5,
395
+ "scaleDistribution": { "type": "linear" },
396
+ "showPoints": "never",
397
+ "spanNulls": false,
398
+ "stacking": { "group": "A", "mode": "normal" },
399
+ "thresholdsStyle": { "mode": "off" }
400
+ },
401
+ "mappings": [],
402
+ "thresholds": {
403
+ "mode": "absolute",
404
+ "steps": [
405
+ { "color": "green", "value": null },
406
+ { "color": "red", "value": 80 }
407
+ ]
408
+ },
409
+ "unit": "short"
410
+ },
411
+ "overrides": [
412
+ {
413
+ "matcher": { "id": "byName", "options": "hit" },
414
+ "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
415
+ },
416
+ {
417
+ "matcher": { "id": "byName", "options": "miss" },
418
+ "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
419
+ }
420
+ ]
421
+ },
422
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
423
+ "id": 7,
424
+ "options": {
425
+ "legend": {
426
+ "calcs": [],
427
+ "displayMode": "list",
428
+ "placement": "bottom",
429
+ "showLegend": true
430
+ },
431
+ "tooltip": { "mode": "single", "sort": "none" }
432
+ },
433
+ "pluginVersion": "10.0.0",
434
+ "targets": [
435
+ {
436
+ "expr": "sum(rate(htm_cache_operations_total[1m])) by (operation)",
437
+ "legendFormat": "{{operation}}",
438
+ "refId": "A"
439
+ }
440
+ ],
441
+ "title": "Cache Operations per Minute",
442
+ "type": "timeseries"
443
+ }
444
+ ],
445
+ "refresh": "5s",
446
+ "schemaVersion": 38,
447
+ "style": "dark",
448
+ "tags": ["htm", "memory", "llm"],
449
+ "templating": { "list": [] },
450
+ "time": { "from": "now-15m", "to": "now" },
451
+ "timepicker": {},
452
+ "timezone": "",
453
+ "title": "HTM Metrics",
454
+ "uid": "htm-metrics",
455
+ "version": 1,
456
+ "weekStart": ""
457
+ }