htm 0.0.10 → 0.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/.dictate.toml +46 -0
  3. data/.envrc +2 -0
  4. data/CHANGELOG.md +86 -3
  5. data/README.md +86 -7
  6. data/Rakefile +14 -2
  7. data/bin/htm_mcp.rb +621 -0
  8. data/config/database.yml +20 -13
  9. data/db/migrate/00010_add_soft_delete_to_associations.rb +29 -0
  10. data/db/migrate/00011_add_performance_indexes.rb +21 -0
  11. data/db/migrate/00012_add_tags_trigram_index.rb +18 -0
  12. data/db/migrate/00013_enable_lz4_compression.rb +43 -0
  13. data/db/schema.sql +49 -92
  14. data/docs/api/index.md +1 -1
  15. data/docs/api/yard/HTM.md +2 -4
  16. data/docs/architecture/index.md +1 -1
  17. data/docs/development/index.md +1 -1
  18. data/docs/getting-started/index.md +1 -1
  19. data/docs/guides/index.md +1 -1
  20. data/docs/images/telemetry-architecture.svg +153 -0
  21. data/docs/telemetry.md +391 -0
  22. data/examples/README.md +171 -1
  23. data/examples/cli_app/README.md +1 -1
  24. data/examples/cli_app/htm_cli.rb +1 -1
  25. data/examples/mcp_client.rb +529 -0
  26. data/examples/sinatra_app/app.rb +1 -1
  27. data/examples/telemetry/README.md +147 -0
  28. data/examples/telemetry/SETUP_README.md +169 -0
  29. data/examples/telemetry/demo.rb +498 -0
  30. data/examples/telemetry/grafana/dashboards/htm-metrics.json +457 -0
  31. data/lib/htm/configuration.rb +261 -70
  32. data/lib/htm/database.rb +46 -22
  33. data/lib/htm/embedding_service.rb +24 -14
  34. data/lib/htm/errors.rb +15 -1
  35. data/lib/htm/jobs/generate_embedding_job.rb +19 -0
  36. data/lib/htm/jobs/generate_propositions_job.rb +103 -0
  37. data/lib/htm/jobs/generate_tags_job.rb +24 -0
  38. data/lib/htm/loaders/markdown_chunker.rb +79 -0
  39. data/lib/htm/loaders/markdown_loader.rb +41 -15
  40. data/lib/htm/long_term_memory/fulltext_search.rb +138 -0
  41. data/lib/htm/long_term_memory/hybrid_search.rb +324 -0
  42. data/lib/htm/long_term_memory/node_operations.rb +209 -0
  43. data/lib/htm/long_term_memory/relevance_scorer.rb +355 -0
  44. data/lib/htm/long_term_memory/robot_operations.rb +34 -0
  45. data/lib/htm/long_term_memory/tag_operations.rb +428 -0
  46. data/lib/htm/long_term_memory/vector_search.rb +109 -0
  47. data/lib/htm/long_term_memory.rb +51 -1153
  48. data/lib/htm/models/node.rb +35 -2
  49. data/lib/htm/models/node_tag.rb +31 -0
  50. data/lib/htm/models/robot_node.rb +31 -0
  51. data/lib/htm/models/tag.rb +44 -0
  52. data/lib/htm/proposition_service.rb +169 -0
  53. data/lib/htm/query_cache.rb +214 -0
  54. data/lib/htm/sql_builder.rb +178 -0
  55. data/lib/htm/tag_service.rb +16 -6
  56. data/lib/htm/tasks.rb +8 -2
  57. data/lib/htm/telemetry.rb +224 -0
  58. data/lib/htm/version.rb +1 -1
  59. data/lib/htm.rb +64 -3
  60. data/lib/tasks/doc.rake +1 -1
  61. data/lib/tasks/htm.rake +259 -13
  62. data/mkdocs.yml +96 -96
  63. metadata +75 -18
  64. data/.aigcm_msg +0 -1
  65. data/.claude/settings.local.json +0 -92
  66. data/CLAUDE.md +0 -603
  67. data/examples/cli_app/temp.log +0 -93
  68. data/lib/htm/loaders/paragraph_chunker.rb +0 -112
  69. data/notes/ARCHITECTURE_REVIEW.md +0 -1167
  70. data/notes/IMPLEMENTATION_SUMMARY.md +0 -606
  71. data/notes/MULTI_FRAMEWORK_IMPLEMENTATION.md +0 -451
  72. data/notes/next_steps.md +0 -100
  73. data/notes/plan.md +0 -627
  74. data/notes/tag_ontology_enhancement_ideas.md +0 -222
  75. data/notes/timescaledb_removal_summary.md +0 -200
@@ -0,0 +1,457 @@
1
+ {
2
+ "annotations": {
3
+ "list": []
4
+ },
5
+ "editable": true,
6
+ "fiscalYearStartMonth": 0,
7
+ "graphTooltip": 0,
8
+ "id": null,
9
+ "links": [],
10
+ "liveNow": false,
11
+ "panels": [
12
+ {
13
+ "datasource": {
14
+ "type": "prometheus",
15
+ "uid": "prometheus"
16
+ },
17
+ "fieldConfig": {
18
+ "defaults": {
19
+ "color": {
20
+ "mode": "palette-classic"
21
+ },
22
+ "mappings": [],
23
+ "thresholds": {
24
+ "mode": "absolute",
25
+ "steps": [
26
+ { "color": "green", "value": null }
27
+ ]
28
+ },
29
+ "unit": "short"
30
+ },
31
+ "overrides": []
32
+ },
33
+ "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
34
+ "id": 1,
35
+ "options": {
36
+ "colorMode": "value",
37
+ "graphMode": "area",
38
+ "justifyMode": "auto",
39
+ "orientation": "auto",
40
+ "reduceOptions": {
41
+ "calcs": ["lastNotNull"],
42
+ "fields": "",
43
+ "values": false
44
+ },
45
+ "textMode": "auto"
46
+ },
47
+ "pluginVersion": "10.0.0",
48
+ "targets": [
49
+ {
50
+ "expr": "sum(htm_jobs_total{status=\"success\"})",
51
+ "legendFormat": "Successful Jobs",
52
+ "refId": "A"
53
+ }
54
+ ],
55
+ "title": "Total Successful Jobs",
56
+ "type": "stat"
57
+ },
58
+ {
59
+ "datasource": {
60
+ "type": "prometheus",
61
+ "uid": "prometheus"
62
+ },
63
+ "fieldConfig": {
64
+ "defaults": {
65
+ "color": {
66
+ "mode": "palette-classic"
67
+ },
68
+ "mappings": [],
69
+ "thresholds": {
70
+ "mode": "absolute",
71
+ "steps": [
72
+ { "color": "red", "value": null }
73
+ ]
74
+ },
75
+ "unit": "short"
76
+ },
77
+ "overrides": []
78
+ },
79
+ "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 },
80
+ "id": 2,
81
+ "options": {
82
+ "colorMode": "value",
83
+ "graphMode": "area",
84
+ "justifyMode": "auto",
85
+ "orientation": "auto",
86
+ "reduceOptions": {
87
+ "calcs": ["lastNotNull"],
88
+ "fields": "",
89
+ "values": false
90
+ },
91
+ "textMode": "auto"
92
+ },
93
+ "pluginVersion": "10.0.0",
94
+ "targets": [
95
+ {
96
+ "expr": "sum(htm_jobs_total{status=\"error\"})",
97
+ "legendFormat": "Failed Jobs",
98
+ "refId": "A"
99
+ }
100
+ ],
101
+ "title": "Total Failed Jobs",
102
+ "type": "stat"
103
+ },
104
+ {
105
+ "datasource": {
106
+ "type": "prometheus",
107
+ "uid": "prometheus"
108
+ },
109
+ "fieldConfig": {
110
+ "defaults": {
111
+ "color": {
112
+ "mode": "palette-classic"
113
+ },
114
+ "mappings": [],
115
+ "thresholds": {
116
+ "mode": "absolute",
117
+ "steps": [
118
+ { "color": "blue", "value": null }
119
+ ]
120
+ },
121
+ "unit": "percent"
122
+ },
123
+ "overrides": []
124
+ },
125
+ "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 },
126
+ "id": 3,
127
+ "options": {
128
+ "colorMode": "value",
129
+ "graphMode": "area",
130
+ "justifyMode": "auto",
131
+ "orientation": "auto",
132
+ "reduceOptions": {
133
+ "calcs": ["lastNotNull"],
134
+ "fields": "",
135
+ "values": false
136
+ },
137
+ "textMode": "auto"
138
+ },
139
+ "pluginVersion": "10.0.0",
140
+ "targets": [
141
+ {
142
+ "expr": "sum(htm_cache_operations_total{operation=\"hit\"}) / (sum(htm_cache_operations_total{operation=\"hit\"}) + sum(htm_cache_operations_total{operation=\"miss\"})) * 100",
143
+ "legendFormat": "Cache Hit Rate",
144
+ "refId": "A"
145
+ }
146
+ ],
147
+ "title": "Cache Hit Rate",
148
+ "type": "stat"
149
+ },
150
+ {
151
+ "datasource": {
152
+ "type": "prometheus",
153
+ "uid": "prometheus"
154
+ },
155
+ "fieldConfig": {
156
+ "defaults": {
157
+ "color": {
158
+ "mode": "palette-classic"
159
+ },
160
+ "custom": {
161
+ "axisCenteredZero": false,
162
+ "axisColorMode": "text",
163
+ "axisLabel": "",
164
+ "axisPlacement": "auto",
165
+ "barAlignment": 0,
166
+ "drawStyle": "line",
167
+ "fillOpacity": 10,
168
+ "gradientMode": "none",
169
+ "hideFrom": {
170
+ "legend": false,
171
+ "tooltip": false,
172
+ "viz": false
173
+ },
174
+ "lineInterpolation": "smooth",
175
+ "lineWidth": 2,
176
+ "pointSize": 5,
177
+ "scaleDistribution": { "type": "linear" },
178
+ "showPoints": "auto",
179
+ "spanNulls": false,
180
+ "stacking": { "group": "A", "mode": "none" },
181
+ "thresholdsStyle": { "mode": "off" }
182
+ },
183
+ "mappings": [],
184
+ "thresholds": {
185
+ "mode": "absolute",
186
+ "steps": [
187
+ { "color": "green", "value": null }
188
+ ]
189
+ },
190
+ "unit": "ms"
191
+ },
192
+ "overrides": []
193
+ },
194
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
195
+ "id": 4,
196
+ "options": {
197
+ "legend": {
198
+ "calcs": ["mean", "max"],
199
+ "displayMode": "table",
200
+ "placement": "bottom",
201
+ "showLegend": true
202
+ },
203
+ "tooltip": { "mode": "single", "sort": "none" }
204
+ },
205
+ "pluginVersion": "10.0.0",
206
+ "targets": [
207
+ {
208
+ "expr": "histogram_quantile(0.95, sum(rate(htm_embedding_latency_bucket[1m])) by (le))",
209
+ "legendFormat": "p95 Embedding",
210
+ "refId": "A"
211
+ },
212
+ {
213
+ "expr": "histogram_quantile(0.95, sum(rate(htm_tag_latency_bucket[1m])) by (le))",
214
+ "legendFormat": "p95 Tags",
215
+ "refId": "B"
216
+ }
217
+ ],
218
+ "title": "LLM Job Latency (p95)",
219
+ "type": "timeseries"
220
+ },
221
+ {
222
+ "datasource": {
223
+ "type": "prometheus",
224
+ "uid": "prometheus"
225
+ },
226
+ "fieldConfig": {
227
+ "defaults": {
228
+ "color": {
229
+ "mode": "palette-classic"
230
+ },
231
+ "custom": {
232
+ "axisCenteredZero": false,
233
+ "axisColorMode": "text",
234
+ "axisLabel": "",
235
+ "axisPlacement": "auto",
236
+ "barAlignment": 0,
237
+ "drawStyle": "line",
238
+ "fillOpacity": 10,
239
+ "gradientMode": "none",
240
+ "hideFrom": {
241
+ "legend": false,
242
+ "tooltip": false,
243
+ "viz": false
244
+ },
245
+ "lineInterpolation": "smooth",
246
+ "lineWidth": 2,
247
+ "pointSize": 5,
248
+ "scaleDistribution": { "type": "linear" },
249
+ "showPoints": "auto",
250
+ "spanNulls": false,
251
+ "stacking": { "group": "A", "mode": "none" },
252
+ "thresholdsStyle": { "mode": "off" }
253
+ },
254
+ "mappings": [],
255
+ "thresholds": {
256
+ "mode": "absolute",
257
+ "steps": [
258
+ { "color": "green", "value": null }
259
+ ]
260
+ },
261
+ "unit": "ms"
262
+ },
263
+ "overrides": []
264
+ },
265
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
266
+ "id": 5,
267
+ "options": {
268
+ "legend": {
269
+ "calcs": ["mean", "max"],
270
+ "displayMode": "table",
271
+ "placement": "bottom",
272
+ "showLegend": true
273
+ },
274
+ "tooltip": { "mode": "single", "sort": "none" }
275
+ },
276
+ "pluginVersion": "10.0.0",
277
+ "targets": [
278
+ {
279
+ "expr": "histogram_quantile(0.95, sum(rate(htm_search_latency_bucket{strategy=\"vector\"}[1m])) by (le))",
280
+ "legendFormat": "p95 Vector",
281
+ "refId": "A"
282
+ },
283
+ {
284
+ "expr": "histogram_quantile(0.95, sum(rate(htm_search_latency_bucket{strategy=\"fulltext\"}[1m])) by (le))",
285
+ "legendFormat": "p95 Fulltext",
286
+ "refId": "B"
287
+ },
288
+ {
289
+ "expr": "histogram_quantile(0.95, sum(rate(htm_search_latency_bucket{strategy=\"hybrid\"}[1m])) by (le))",
290
+ "legendFormat": "p95 Hybrid",
291
+ "refId": "C"
292
+ }
293
+ ],
294
+ "title": "Search Latency by Strategy (p95)",
295
+ "type": "timeseries"
296
+ },
297
+ {
298
+ "datasource": {
299
+ "type": "prometheus",
300
+ "uid": "prometheus"
301
+ },
302
+ "fieldConfig": {
303
+ "defaults": {
304
+ "color": {
305
+ "mode": "palette-classic"
306
+ },
307
+ "custom": {
308
+ "axisCenteredZero": false,
309
+ "axisColorMode": "text",
310
+ "axisLabel": "",
311
+ "axisPlacement": "auto",
312
+ "barAlignment": 0,
313
+ "drawStyle": "bars",
314
+ "fillOpacity": 80,
315
+ "gradientMode": "none",
316
+ "hideFrom": {
317
+ "legend": false,
318
+ "tooltip": false,
319
+ "viz": false
320
+ },
321
+ "lineInterpolation": "linear",
322
+ "lineWidth": 1,
323
+ "pointSize": 5,
324
+ "scaleDistribution": { "type": "linear" },
325
+ "showPoints": "never",
326
+ "spanNulls": false,
327
+ "stacking": { "group": "A", "mode": "normal" },
328
+ "thresholdsStyle": { "mode": "off" }
329
+ },
330
+ "mappings": [],
331
+ "thresholds": {
332
+ "mode": "absolute",
333
+ "steps": [
334
+ { "color": "green", "value": null }
335
+ ]
336
+ },
337
+ "unit": "short"
338
+ },
339
+ "overrides": []
340
+ },
341
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
342
+ "id": 6,
343
+ "options": {
344
+ "legend": {
345
+ "calcs": [],
346
+ "displayMode": "list",
347
+ "placement": "bottom",
348
+ "showLegend": true
349
+ },
350
+ "tooltip": { "mode": "single", "sort": "none" }
351
+ },
352
+ "pluginVersion": "10.0.0",
353
+ "targets": [
354
+ {
355
+ "expr": "sum(rate(htm_jobs_total{job=\"embedding\"}[1m])) by (status)",
356
+ "legendFormat": "Embedding - {{status}}",
357
+ "refId": "A"
358
+ },
359
+ {
360
+ "expr": "sum(rate(htm_jobs_total{job=\"tags\"}[1m])) by (status)",
361
+ "legendFormat": "Tags - {{status}}",
362
+ "refId": "B"
363
+ }
364
+ ],
365
+ "title": "Jobs per Minute by Type",
366
+ "type": "timeseries"
367
+ },
368
+ {
369
+ "datasource": {
370
+ "type": "prometheus",
371
+ "uid": "prometheus"
372
+ },
373
+ "fieldConfig": {
374
+ "defaults": {
375
+ "color": {
376
+ "mode": "palette-classic"
377
+ },
378
+ "custom": {
379
+ "axisCenteredZero": false,
380
+ "axisColorMode": "text",
381
+ "axisLabel": "",
382
+ "axisPlacement": "auto",
383
+ "barAlignment": 0,
384
+ "drawStyle": "bars",
385
+ "fillOpacity": 80,
386
+ "gradientMode": "none",
387
+ "hideFrom": {
388
+ "legend": false,
389
+ "tooltip": false,
390
+ "viz": false
391
+ },
392
+ "lineInterpolation": "linear",
393
+ "lineWidth": 1,
394
+ "pointSize": 5,
395
+ "scaleDistribution": { "type": "linear" },
396
+ "showPoints": "never",
397
+ "spanNulls": false,
398
+ "stacking": { "group": "A", "mode": "normal" },
399
+ "thresholdsStyle": { "mode": "off" }
400
+ },
401
+ "mappings": [],
402
+ "thresholds": {
403
+ "mode": "absolute",
404
+ "steps": [
405
+ { "color": "green", "value": null },
406
+ { "color": "red", "value": 80 }
407
+ ]
408
+ },
409
+ "unit": "short"
410
+ },
411
+ "overrides": [
412
+ {
413
+ "matcher": { "id": "byName", "options": "hit" },
414
+ "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
415
+ },
416
+ {
417
+ "matcher": { "id": "byName", "options": "miss" },
418
+ "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
419
+ }
420
+ ]
421
+ },
422
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
423
+ "id": 7,
424
+ "options": {
425
+ "legend": {
426
+ "calcs": [],
427
+ "displayMode": "list",
428
+ "placement": "bottom",
429
+ "showLegend": true
430
+ },
431
+ "tooltip": { "mode": "single", "sort": "none" }
432
+ },
433
+ "pluginVersion": "10.0.0",
434
+ "targets": [
435
+ {
436
+ "expr": "sum(rate(htm_cache_operations_total[1m])) by (operation)",
437
+ "legendFormat": "{{operation}}",
438
+ "refId": "A"
439
+ }
440
+ ],
441
+ "title": "Cache Operations per Minute",
442
+ "type": "timeseries"
443
+ }
444
+ ],
445
+ "refresh": "5s",
446
+ "schemaVersion": 38,
447
+ "style": "dark",
448
+ "tags": ["htm", "memory", "llm"],
449
+ "templating": { "list": [] },
450
+ "time": { "from": "now-15m", "to": "now" },
451
+ "timepicker": {},
452
+ "timezone": "",
453
+ "title": "HTM Metrics",
454
+ "uid": "htm-metrics",
455
+ "version": 1,
456
+ "weekStart": ""
457
+ }