htm 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.envrc +1 -0
- data/.tbls.yml +30 -0
- data/CHANGELOG.md +30 -0
- data/SETUP.md +132 -101
- data/db/migrate/20250125000001_add_content_hash_to_nodes.rb +14 -0
- data/db/migrate/20250125000002_create_robot_nodes.rb +35 -0
- data/db/migrate/20250125000003_remove_source_and_robot_id_from_nodes.rb +28 -0
- data/db/migrate/20250126000001_create_working_memories.rb +19 -0
- data/db/migrate/20250126000002_remove_unused_columns.rb +12 -0
- data/db/schema.sql +226 -43
- data/docs/api/database.md +20 -232
- data/docs/api/embedding-service.md +1 -7
- data/docs/api/htm.md +195 -449
- data/docs/api/index.md +1 -7
- data/docs/api/long-term-memory.md +342 -590
- data/docs/architecture/adrs/001-postgresql-timescaledb.md +1 -1
- data/docs/architecture/adrs/003-ollama-embeddings.md +1 -1
- data/docs/architecture/adrs/010-redis-working-memory-rejected.md +2 -27
- data/docs/architecture/adrs/index.md +2 -13
- data/docs/architecture/hive-mind.md +165 -166
- data/docs/architecture/index.md +2 -2
- data/docs/architecture/overview.md +5 -171
- data/docs/architecture/two-tier-memory.md +1 -35
- data/docs/assets/images/adr-010-current-architecture.svg +37 -0
- data/docs/assets/images/adr-010-proposed-architecture.svg +48 -0
- data/docs/assets/images/adr-dependency-tree.svg +93 -0
- data/docs/assets/images/class-hierarchy.svg +55 -0
- data/docs/assets/images/exception-hierarchy.svg +45 -0
- data/docs/assets/images/htm-architecture-overview.svg +83 -0
- data/docs/assets/images/htm-complete-memory-flow.svg +160 -0
- data/docs/assets/images/htm-context-assembly-flow.svg +148 -0
- data/docs/assets/images/htm-eviction-process.svg +141 -0
- data/docs/assets/images/htm-memory-addition-flow.svg +138 -0
- data/docs/assets/images/htm-memory-recall-flow.svg +152 -0
- data/docs/assets/images/htm-node-states.svg +123 -0
- data/docs/assets/images/project-structure.svg +78 -0
- data/docs/assets/images/test-directory-structure.svg +38 -0
- data/{dbdoc → docs/database}/README.md +5 -3
- data/{dbdoc → docs/database}/public.node_tags.md +4 -5
- data/docs/database/public.node_tags.svg +106 -0
- data/{dbdoc → docs/database}/public.nodes.md +3 -8
- data/docs/database/public.nodes.svg +152 -0
- data/docs/database/public.robot_nodes.md +44 -0
- data/docs/database/public.robot_nodes.svg +121 -0
- data/{dbdoc → docs/database}/public.robots.md +1 -2
- data/docs/database/public.robots.svg +106 -0
- data/docs/database/public.working_memories.md +40 -0
- data/docs/database/public.working_memories.svg +112 -0
- data/{dbdoc → docs/database}/schema.json +342 -110
- data/docs/database/schema.svg +223 -0
- data/docs/development/index.md +1 -29
- data/docs/development/schema.md +84 -324
- data/docs/development/testing.md +1 -9
- data/docs/getting-started/index.md +47 -0
- data/docs/{installation.md → getting-started/installation.md} +2 -2
- data/docs/{quick-start.md → getting-started/quick-start.md} +5 -5
- data/docs/guides/adding-memories.md +221 -655
- data/docs/guides/search-strategies.md +85 -51
- data/docs/images/htm-er-diagram.svg +156 -0
- data/docs/index.md +16 -31
- data/docs/multi_framework_support.md +4 -4
- data/examples/basic_usage.rb +18 -16
- data/examples/cli_app/htm_cli.rb +86 -8
- data/examples/custom_llm_configuration.rb +1 -2
- data/examples/example_app/app.rb +11 -14
- data/examples/sinatra_app/Gemfile +1 -0
- data/examples/sinatra_app/Gemfile.lock +166 -0
- data/examples/sinatra_app/app.rb +219 -24
- data/lib/htm/active_record_config.rb +10 -3
- data/lib/htm/configuration.rb +265 -78
- data/lib/htm/{sinatra.rb → integrations/sinatra.rb} +87 -12
- data/lib/htm/job_adapter.rb +10 -3
- data/lib/htm/long_term_memory.rb +220 -57
- data/lib/htm/models/node.rb +36 -7
- data/lib/htm/models/robot.rb +30 -4
- data/lib/htm/models/robot_node.rb +50 -0
- data/lib/htm/models/tag.rb +52 -0
- data/lib/htm/models/working_memory_entry.rb +88 -0
- data/lib/htm/tasks.rb +4 -0
- data/lib/htm/version.rb +1 -1
- data/lib/htm.rb +34 -13
- data/lib/tasks/htm.rake +32 -1
- data/lib/tasks/jobs.rake +7 -3
- data/lib/tasks/tags.rake +34 -0
- data/mkdocs.yml +56 -9
- metadata +61 -31
- data/dbdoc/public.node_tags.svg +0 -112
- data/dbdoc/public.nodes.svg +0 -118
- data/dbdoc/public.robots.svg +0 -90
- data/dbdoc/schema.svg +0 -154
- /data/{dbdoc → docs/database}/public.node_stats.md +0 -0
- /data/{dbdoc → docs/database}/public.node_stats.svg +0 -0
- /data/{dbdoc → docs/database}/public.nodes_tags.md +0 -0
- /data/{dbdoc → docs/database}/public.nodes_tags.svg +0 -0
- /data/{dbdoc → docs/database}/public.ontology_structure.md +0 -0
- /data/{dbdoc → docs/database}/public.ontology_structure.svg +0 -0
- /data/{dbdoc → docs/database}/public.operations_log.md +0 -0
- /data/{dbdoc → docs/database}/public.operations_log.svg +0 -0
- /data/{dbdoc → docs/database}/public.relationships.md +0 -0
- /data/{dbdoc → docs/database}/public.relationships.svg +0 -0
- /data/{dbdoc → docs/database}/public.robot_activity.md +0 -0
- /data/{dbdoc → docs/database}/public.robot_activity.svg +0 -0
- /data/{dbdoc → docs/database}/public.schema_migrations.md +0 -0
- /data/{dbdoc → docs/database}/public.schema_migrations.svg +0 -0
- /data/{dbdoc → docs/database}/public.tags.md +0 -0
- /data/{dbdoc → docs/database}/public.tags.svg +0 -0
- /data/{dbdoc → docs/database}/public.topic_relationships.md +0 -0
- /data/{dbdoc → docs/database}/public.topic_relationships.svg +0 -0
data/db/schema.sql
CHANGED
|
@@ -82,16 +82,14 @@ ALTER SEQUENCE public.node_tags_id_seq OWNED BY public.node_tags.id;
|
|
|
82
82
|
CREATE TABLE public.nodes (
|
|
83
83
|
id bigint NOT NULL,
|
|
84
84
|
content text NOT NULL,
|
|
85
|
-
source text DEFAULT ''::text,
|
|
86
85
|
access_count integer DEFAULT 0 NOT NULL,
|
|
87
86
|
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
88
87
|
updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
89
88
|
last_accessed timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
90
89
|
token_count integer,
|
|
91
|
-
in_working_memory boolean DEFAULT false,
|
|
92
|
-
robot_id bigint NOT NULL,
|
|
93
90
|
embedding public.vector(2000),
|
|
94
91
|
embedding_dimension integer,
|
|
92
|
+
content_hash character varying(64),
|
|
95
93
|
CONSTRAINT check_embedding_dimension CHECK (((embedding_dimension IS NULL) OR ((embedding_dimension > 0) AND (embedding_dimension <= 2000))))
|
|
96
94
|
);
|
|
97
95
|
|
|
@@ -107,12 +105,6 @@ COMMENT ON TABLE public.nodes IS 'Core memory storage for conversation messages
|
|
|
107
105
|
|
|
108
106
|
COMMENT ON COLUMN public.nodes.content IS 'The conversation message/utterance content';
|
|
109
107
|
|
|
110
|
-
--
|
|
111
|
-
-- Name: COLUMN nodes.source; Type: COMMENT; Schema: public; Owner: -
|
|
112
|
-
--
|
|
113
|
-
|
|
114
|
-
COMMENT ON COLUMN public.nodes.source IS 'From where the content came (empty string if unknown)';
|
|
115
|
-
|
|
116
108
|
--
|
|
117
109
|
-- Name: COLUMN nodes.access_count; Type: COMMENT; Schema: public; Owner: -
|
|
118
110
|
--
|
|
@@ -143,18 +135,6 @@ COMMENT ON COLUMN public.nodes.last_accessed IS 'When this memory was last acces
|
|
|
143
135
|
|
|
144
136
|
COMMENT ON COLUMN public.nodes.token_count IS 'Number of tokens in the content (for context budget management)';
|
|
145
137
|
|
|
146
|
-
--
|
|
147
|
-
-- Name: COLUMN nodes.in_working_memory; Type: COMMENT; Schema: public; Owner: -
|
|
148
|
-
--
|
|
149
|
-
|
|
150
|
-
COMMENT ON COLUMN public.nodes.in_working_memory IS 'Whether this memory is currently in working memory';
|
|
151
|
-
|
|
152
|
-
--
|
|
153
|
-
-- Name: COLUMN nodes.robot_id; Type: COMMENT; Schema: public; Owner: -
|
|
154
|
-
--
|
|
155
|
-
|
|
156
|
-
COMMENT ON COLUMN public.nodes.robot_id IS 'ID of the robot that owns this memory';
|
|
157
|
-
|
|
158
138
|
--
|
|
159
139
|
-- Name: COLUMN nodes.embedding; Type: COMMENT; Schema: public; Owner: -
|
|
160
140
|
--
|
|
@@ -167,6 +147,12 @@ COMMENT ON COLUMN public.nodes.embedding IS 'Vector embedding (max 2000 dimensio
|
|
|
167
147
|
|
|
168
148
|
COMMENT ON COLUMN public.nodes.embedding_dimension IS 'Actual number of dimensions used in the embedding vector (max 2000)';
|
|
169
149
|
|
|
150
|
+
--
|
|
151
|
+
-- Name: COLUMN nodes.content_hash; Type: COMMENT; Schema: public; Owner: -
|
|
152
|
+
--
|
|
153
|
+
|
|
154
|
+
COMMENT ON COLUMN public.nodes.content_hash IS 'SHA-256 hash of content for deduplication';
|
|
155
|
+
|
|
170
156
|
--
|
|
171
157
|
-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
172
158
|
--
|
|
@@ -184,6 +170,74 @@ CREATE SEQUENCE public.nodes_id_seq
|
|
|
184
170
|
|
|
185
171
|
ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id;
|
|
186
172
|
|
|
173
|
+
--
|
|
174
|
+
-- Name: robot_nodes; Type: TABLE; Schema: public; Owner: -
|
|
175
|
+
--
|
|
176
|
+
|
|
177
|
+
CREATE TABLE public.robot_nodes (
|
|
178
|
+
id bigint NOT NULL,
|
|
179
|
+
robot_id bigint NOT NULL,
|
|
180
|
+
node_id bigint NOT NULL,
|
|
181
|
+
first_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
182
|
+
last_remembered_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
183
|
+
remember_count integer DEFAULT 1 NOT NULL,
|
|
184
|
+
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
185
|
+
updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
--
|
|
189
|
+
-- Name: TABLE robot_nodes; Type: COMMENT; Schema: public; Owner: -
|
|
190
|
+
--
|
|
191
|
+
|
|
192
|
+
COMMENT ON TABLE public.robot_nodes IS 'Join table connecting robots to nodes (many-to-many)';
|
|
193
|
+
|
|
194
|
+
--
|
|
195
|
+
-- Name: COLUMN robot_nodes.robot_id; Type: COMMENT; Schema: public; Owner: -
|
|
196
|
+
--
|
|
197
|
+
|
|
198
|
+
COMMENT ON COLUMN public.robot_nodes.robot_id IS 'ID of the robot that remembered this node';
|
|
199
|
+
|
|
200
|
+
--
|
|
201
|
+
-- Name: COLUMN robot_nodes.node_id; Type: COMMENT; Schema: public; Owner: -
|
|
202
|
+
--
|
|
203
|
+
|
|
204
|
+
COMMENT ON COLUMN public.robot_nodes.node_id IS 'ID of the node being remembered';
|
|
205
|
+
|
|
206
|
+
--
|
|
207
|
+
-- Name: COLUMN robot_nodes.first_remembered_at; Type: COMMENT; Schema: public; Owner: -
|
|
208
|
+
--
|
|
209
|
+
|
|
210
|
+
COMMENT ON COLUMN public.robot_nodes.first_remembered_at IS 'When this robot first remembered this content';
|
|
211
|
+
|
|
212
|
+
--
|
|
213
|
+
-- Name: COLUMN robot_nodes.last_remembered_at; Type: COMMENT; Schema: public; Owner: -
|
|
214
|
+
--
|
|
215
|
+
|
|
216
|
+
COMMENT ON COLUMN public.robot_nodes.last_remembered_at IS 'When this robot last tried to remember this content';
|
|
217
|
+
|
|
218
|
+
--
|
|
219
|
+
-- Name: COLUMN robot_nodes.remember_count; Type: COMMENT; Schema: public; Owner: -
|
|
220
|
+
--
|
|
221
|
+
|
|
222
|
+
COMMENT ON COLUMN public.robot_nodes.remember_count IS 'Number of times this robot has tried to remember this content';
|
|
223
|
+
|
|
224
|
+
--
|
|
225
|
+
-- Name: robot_nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
226
|
+
--
|
|
227
|
+
|
|
228
|
+
CREATE SEQUENCE public.robot_nodes_id_seq
|
|
229
|
+
START WITH 1
|
|
230
|
+
INCREMENT BY 1
|
|
231
|
+
NO MINVALUE
|
|
232
|
+
NO MAXVALUE
|
|
233
|
+
CACHE 1;
|
|
234
|
+
|
|
235
|
+
--
|
|
236
|
+
-- Name: robot_nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
|
|
237
|
+
--
|
|
238
|
+
|
|
239
|
+
ALTER SEQUENCE public.robot_nodes_id_seq OWNED BY public.robot_nodes.id;
|
|
240
|
+
|
|
187
241
|
--
|
|
188
242
|
-- Name: robots; Type: TABLE; Schema: public; Owner: -
|
|
189
243
|
--
|
|
@@ -192,8 +246,7 @@ CREATE TABLE public.robots (
|
|
|
192
246
|
id bigint NOT NULL,
|
|
193
247
|
name text,
|
|
194
248
|
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
195
|
-
last_active timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
196
|
-
metadata jsonb
|
|
249
|
+
last_active timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
|
197
250
|
);
|
|
198
251
|
|
|
199
252
|
--
|
|
@@ -220,12 +273,6 @@ COMMENT ON COLUMN public.robots.created_at IS 'When the robot was first register
|
|
|
220
273
|
|
|
221
274
|
COMMENT ON COLUMN public.robots.last_active IS 'Last time the robot accessed the system';
|
|
222
275
|
|
|
223
|
-
--
|
|
224
|
-
-- Name: COLUMN robots.metadata; Type: COMMENT; Schema: public; Owner: -
|
|
225
|
-
--
|
|
226
|
-
|
|
227
|
-
COMMENT ON COLUMN public.robots.metadata IS 'Robot-specific configuration and metadata';
|
|
228
|
-
|
|
229
276
|
--
|
|
230
277
|
-- Name: robots_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
231
278
|
--
|
|
@@ -296,6 +343,65 @@ CREATE SEQUENCE public.tags_id_seq
|
|
|
296
343
|
|
|
297
344
|
ALTER SEQUENCE public.tags_id_seq OWNED BY public.tags.id;
|
|
298
345
|
|
|
346
|
+
--
|
|
347
|
+
-- Name: working_memories; Type: TABLE; Schema: public; Owner: -
|
|
348
|
+
--
|
|
349
|
+
|
|
350
|
+
CREATE TABLE public.working_memories (
|
|
351
|
+
id bigint NOT NULL,
|
|
352
|
+
robot_id bigint NOT NULL,
|
|
353
|
+
node_id bigint NOT NULL,
|
|
354
|
+
added_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
|
355
|
+
token_count integer
|
|
356
|
+
);
|
|
357
|
+
|
|
358
|
+
--
|
|
359
|
+
-- Name: TABLE working_memories; Type: COMMENT; Schema: public; Owner: -
|
|
360
|
+
--
|
|
361
|
+
|
|
362
|
+
COMMENT ON TABLE public.working_memories IS 'Per-robot working memory state (optional persistence)';
|
|
363
|
+
|
|
364
|
+
--
|
|
365
|
+
-- Name: COLUMN working_memories.robot_id; Type: COMMENT; Schema: public; Owner: -
|
|
366
|
+
--
|
|
367
|
+
|
|
368
|
+
COMMENT ON COLUMN public.working_memories.robot_id IS 'Robot whose working memory this belongs to';
|
|
369
|
+
|
|
370
|
+
--
|
|
371
|
+
-- Name: COLUMN working_memories.node_id; Type: COMMENT; Schema: public; Owner: -
|
|
372
|
+
--
|
|
373
|
+
|
|
374
|
+
COMMENT ON COLUMN public.working_memories.node_id IS 'Node currently in working memory';
|
|
375
|
+
|
|
376
|
+
--
|
|
377
|
+
-- Name: COLUMN working_memories.added_at; Type: COMMENT; Schema: public; Owner: -
|
|
378
|
+
--
|
|
379
|
+
|
|
380
|
+
COMMENT ON COLUMN public.working_memories.added_at IS 'When node was added to working memory';
|
|
381
|
+
|
|
382
|
+
--
|
|
383
|
+
-- Name: COLUMN working_memories.token_count; Type: COMMENT; Schema: public; Owner: -
|
|
384
|
+
--
|
|
385
|
+
|
|
386
|
+
COMMENT ON COLUMN public.working_memories.token_count IS 'Cached token count for budget tracking';
|
|
387
|
+
|
|
388
|
+
--
|
|
389
|
+
-- Name: working_memories_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
|
390
|
+
--
|
|
391
|
+
|
|
392
|
+
CREATE SEQUENCE public.working_memories_id_seq
|
|
393
|
+
START WITH 1
|
|
394
|
+
INCREMENT BY 1
|
|
395
|
+
NO MINVALUE
|
|
396
|
+
NO MAXVALUE
|
|
397
|
+
CACHE 1;
|
|
398
|
+
|
|
399
|
+
--
|
|
400
|
+
-- Name: working_memories_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
|
|
401
|
+
--
|
|
402
|
+
|
|
403
|
+
ALTER SEQUENCE public.working_memories_id_seq OWNED BY public.working_memories.id;
|
|
404
|
+
|
|
299
405
|
--
|
|
300
406
|
-- Name: node_tags id; Type: DEFAULT; Schema: public; Owner: -
|
|
301
407
|
--
|
|
@@ -308,6 +414,12 @@ ALTER TABLE ONLY public.node_tags ALTER COLUMN id SET DEFAULT nextval('public.no
|
|
|
308
414
|
|
|
309
415
|
ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
|
|
310
416
|
|
|
417
|
+
--
|
|
418
|
+
-- Name: robot_nodes id; Type: DEFAULT; Schema: public; Owner: -
|
|
419
|
+
--
|
|
420
|
+
|
|
421
|
+
ALTER TABLE ONLY public.robot_nodes ALTER COLUMN id SET DEFAULT nextval('public.robot_nodes_id_seq'::regclass);
|
|
422
|
+
|
|
311
423
|
--
|
|
312
424
|
-- Name: robots id; Type: DEFAULT; Schema: public; Owner: -
|
|
313
425
|
--
|
|
@@ -320,6 +432,12 @@ ALTER TABLE ONLY public.robots ALTER COLUMN id SET DEFAULT nextval('public.robot
|
|
|
320
432
|
|
|
321
433
|
ALTER TABLE ONLY public.tags ALTER COLUMN id SET DEFAULT nextval('public.tags_id_seq'::regclass);
|
|
322
434
|
|
|
435
|
+
--
|
|
436
|
+
-- Name: working_memories id; Type: DEFAULT; Schema: public; Owner: -
|
|
437
|
+
--
|
|
438
|
+
|
|
439
|
+
ALTER TABLE ONLY public.working_memories ALTER COLUMN id SET DEFAULT nextval('public.working_memories_id_seq'::regclass);
|
|
440
|
+
|
|
323
441
|
--
|
|
324
442
|
-- Name: node_tags node_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
|
325
443
|
--
|
|
@@ -334,6 +452,13 @@ ALTER TABLE ONLY public.node_tags
|
|
|
334
452
|
ALTER TABLE ONLY public.nodes
|
|
335
453
|
ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
|
|
336
454
|
|
|
455
|
+
--
|
|
456
|
+
-- Name: robot_nodes robot_nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
|
457
|
+
--
|
|
458
|
+
|
|
459
|
+
ALTER TABLE ONLY public.robot_nodes
|
|
460
|
+
ADD CONSTRAINT robot_nodes_pkey PRIMARY KEY (id);
|
|
461
|
+
|
|
337
462
|
--
|
|
338
463
|
-- Name: robots robots_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
|
339
464
|
--
|
|
@@ -355,6 +480,13 @@ ALTER TABLE ONLY public.schema_migrations
|
|
|
355
480
|
ALTER TABLE ONLY public.tags
|
|
356
481
|
ADD CONSTRAINT tags_pkey PRIMARY KEY (id);
|
|
357
482
|
|
|
483
|
+
--
|
|
484
|
+
-- Name: working_memories working_memories_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
|
485
|
+
--
|
|
486
|
+
|
|
487
|
+
ALTER TABLE ONLY public.working_memories
|
|
488
|
+
ADD CONSTRAINT working_memories_pkey PRIMARY KEY (id);
|
|
489
|
+
|
|
358
490
|
--
|
|
359
491
|
-- Name: idx_node_tags_node_id; Type: INDEX; Schema: public; Owner: -
|
|
360
492
|
--
|
|
@@ -385,6 +517,12 @@ CREATE INDEX idx_nodes_access_count ON public.nodes USING btree (access_count);
|
|
|
385
517
|
|
|
386
518
|
CREATE INDEX idx_nodes_content_gin ON public.nodes USING gin (to_tsvector('english'::regconfig, content));
|
|
387
519
|
|
|
520
|
+
--
|
|
521
|
+
-- Name: idx_nodes_content_hash_unique; Type: INDEX; Schema: public; Owner: -
|
|
522
|
+
--
|
|
523
|
+
|
|
524
|
+
CREATE UNIQUE INDEX idx_nodes_content_hash_unique ON public.nodes USING btree (content_hash);
|
|
525
|
+
|
|
388
526
|
--
|
|
389
527
|
-- Name: idx_nodes_content_trgm; Type: INDEX; Schema: public; Owner: -
|
|
390
528
|
--
|
|
@@ -404,34 +542,40 @@ CREATE INDEX idx_nodes_created_at ON public.nodes USING btree (created_at);
|
|
|
404
542
|
CREATE INDEX idx_nodes_embedding ON public.nodes USING hnsw (embedding public.vector_cosine_ops) WITH (m='16', ef_construction='64');
|
|
405
543
|
|
|
406
544
|
--
|
|
407
|
-
-- Name:
|
|
545
|
+
-- Name: idx_nodes_last_accessed; Type: INDEX; Schema: public; Owner: -
|
|
408
546
|
--
|
|
409
547
|
|
|
410
|
-
CREATE INDEX
|
|
548
|
+
CREATE INDEX idx_nodes_last_accessed ON public.nodes USING btree (last_accessed);
|
|
411
549
|
|
|
412
550
|
--
|
|
413
|
-
-- Name:
|
|
551
|
+
-- Name: idx_nodes_updated_at; Type: INDEX; Schema: public; Owner: -
|
|
414
552
|
--
|
|
415
553
|
|
|
416
|
-
CREATE INDEX
|
|
554
|
+
CREATE INDEX idx_nodes_updated_at ON public.nodes USING btree (updated_at);
|
|
417
555
|
|
|
418
556
|
--
|
|
419
|
-
-- Name:
|
|
557
|
+
-- Name: idx_robot_nodes_last_remembered_at; Type: INDEX; Schema: public; Owner: -
|
|
420
558
|
--
|
|
421
559
|
|
|
422
|
-
CREATE INDEX
|
|
560
|
+
CREATE INDEX idx_robot_nodes_last_remembered_at ON public.robot_nodes USING btree (last_remembered_at);
|
|
423
561
|
|
|
424
562
|
--
|
|
425
|
-
-- Name:
|
|
563
|
+
-- Name: idx_robot_nodes_node_id; Type: INDEX; Schema: public; Owner: -
|
|
426
564
|
--
|
|
427
565
|
|
|
428
|
-
CREATE INDEX
|
|
566
|
+
CREATE INDEX idx_robot_nodes_node_id ON public.robot_nodes USING btree (node_id);
|
|
429
567
|
|
|
430
568
|
--
|
|
431
|
-
-- Name:
|
|
569
|
+
-- Name: idx_robot_nodes_robot_id; Type: INDEX; Schema: public; Owner: -
|
|
432
570
|
--
|
|
433
571
|
|
|
434
|
-
CREATE INDEX
|
|
572
|
+
CREATE INDEX idx_robot_nodes_robot_id ON public.robot_nodes USING btree (robot_id);
|
|
573
|
+
|
|
574
|
+
--
|
|
575
|
+
-- Name: idx_robot_nodes_unique; Type: INDEX; Schema: public; Owner: -
|
|
576
|
+
--
|
|
577
|
+
|
|
578
|
+
CREATE UNIQUE INDEX idx_robot_nodes_unique ON public.robot_nodes USING btree (robot_id, node_id);
|
|
435
579
|
|
|
436
580
|
--
|
|
437
581
|
-- Name: idx_tags_name_pattern; Type: INDEX; Schema: public; Owner: -
|
|
@@ -446,11 +590,43 @@ CREATE INDEX idx_tags_name_pattern ON public.tags USING btree (name text_pattern
|
|
|
446
590
|
CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
|
|
447
591
|
|
|
448
592
|
--
|
|
449
|
-
-- Name:
|
|
593
|
+
-- Name: idx_working_memories_node_id; Type: INDEX; Schema: public; Owner: -
|
|
450
594
|
--
|
|
451
595
|
|
|
452
|
-
|
|
453
|
-
|
|
596
|
+
CREATE INDEX idx_working_memories_node_id ON public.working_memories USING btree (node_id);
|
|
597
|
+
|
|
598
|
+
--
|
|
599
|
+
-- Name: idx_working_memories_robot_id; Type: INDEX; Schema: public; Owner: -
|
|
600
|
+
--
|
|
601
|
+
|
|
602
|
+
CREATE INDEX idx_working_memories_robot_id ON public.working_memories USING btree (robot_id);
|
|
603
|
+
|
|
604
|
+
--
|
|
605
|
+
-- Name: idx_working_memories_unique; Type: INDEX; Schema: public; Owner: -
|
|
606
|
+
--
|
|
607
|
+
|
|
608
|
+
CREATE UNIQUE INDEX idx_working_memories_unique ON public.working_memories USING btree (robot_id, node_id);
|
|
609
|
+
|
|
610
|
+
--
|
|
611
|
+
-- Name: working_memories fk_rails_2c1d8b383c; Type: FK CONSTRAINT; Schema: public; Owner: -
|
|
612
|
+
--
|
|
613
|
+
|
|
614
|
+
ALTER TABLE ONLY public.working_memories
|
|
615
|
+
ADD CONSTRAINT fk_rails_2c1d8b383c FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
|
|
616
|
+
|
|
617
|
+
--
|
|
618
|
+
-- Name: working_memories fk_rails_4b7c3eb07b; Type: FK CONSTRAINT; Schema: public; Owner: -
|
|
619
|
+
--
|
|
620
|
+
|
|
621
|
+
ALTER TABLE ONLY public.working_memories
|
|
622
|
+
ADD CONSTRAINT fk_rails_4b7c3eb07b FOREIGN KEY (robot_id) REFERENCES public.robots(id) ON DELETE CASCADE;
|
|
623
|
+
|
|
624
|
+
--
|
|
625
|
+
-- Name: robot_nodes fk_rails_9b003078a8; Type: FK CONSTRAINT; Schema: public; Owner: -
|
|
626
|
+
--
|
|
627
|
+
|
|
628
|
+
ALTER TABLE ONLY public.robot_nodes
|
|
629
|
+
ADD CONSTRAINT fk_rails_9b003078a8 FOREIGN KEY (robot_id) REFERENCES public.robots(id) ON DELETE CASCADE;
|
|
454
630
|
|
|
455
631
|
--
|
|
456
632
|
-- Name: node_tags fk_rails_b51cdcc57f; Type: FK CONSTRAINT; Schema: public; Owner: -
|
|
@@ -466,8 +642,15 @@ ALTER TABLE ONLY public.node_tags
|
|
|
466
642
|
ALTER TABLE ONLY public.node_tags
|
|
467
643
|
ADD CONSTRAINT fk_rails_ebc9aafd9f FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
|
|
468
644
|
|
|
645
|
+
--
|
|
646
|
+
-- Name: robot_nodes fk_rails_f2fc98d49e; Type: FK CONSTRAINT; Schema: public; Owner: -
|
|
647
|
+
--
|
|
648
|
+
|
|
649
|
+
ALTER TABLE ONLY public.robot_nodes
|
|
650
|
+
ADD CONSTRAINT fk_rails_f2fc98d49e FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
|
|
651
|
+
|
|
469
652
|
--
|
|
470
653
|
-- PostgreSQL database dump complete
|
|
471
654
|
--
|
|
472
655
|
|
|
473
|
-
\unrestrict
|
|
656
|
+
\unrestrict 6qynyffXXn5BTZM7u0DVZKV2Nc24dPezkY3OOwzriuYfchXNsoQuf114yBOqrIb
|
data/docs/api/database.md
CHANGED
|
@@ -264,241 +264,28 @@ htm = HTM.new(db_config: HTM::Database.default_config)
|
|
|
264
264
|
|
|
265
265
|
## Database Schema
|
|
266
266
|
|
|
267
|
-
|
|
267
|
+
For detailed database schema documentation, see:
|
|
268
268
|
|
|
269
|
-
|
|
269
|
+
- **[Database Schema Documentation](../development/schema.md)** - Query patterns, optimization tips, and best practices
|
|
270
|
+
- **[Database Tables Overview](../database/README.md)** - Auto-generated table reference with ER diagram
|
|
270
271
|
|
|
271
|
-
|
|
272
|
+
### Quick Reference
|
|
272
273
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
token_count INTEGER DEFAULT 0,
|
|
282
|
-
robot_id TEXT NOT NULL REFERENCES robots(id),
|
|
283
|
-
embedding vector(1536),
|
|
284
|
-
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
|
285
|
-
last_accessed TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
|
286
|
-
in_working_memory BOOLEAN DEFAULT TRUE,
|
|
287
|
-
evicted_at TIMESTAMPTZ
|
|
288
|
-
);
|
|
289
|
-
|
|
290
|
-
-- Indexes
|
|
291
|
-
CREATE UNIQUE INDEX idx_nodes_key ON nodes(key);
|
|
292
|
-
CREATE INDEX idx_nodes_created_at ON nodes(created_at DESC);
|
|
293
|
-
CREATE INDEX idx_nodes_robot_id ON nodes(robot_id);
|
|
294
|
-
CREATE INDEX idx_nodes_type ON nodes(type);
|
|
295
|
-
CREATE INDEX idx_nodes_embedding ON nodes USING ivfflat (embedding vector_cosine_ops);
|
|
296
|
-
CREATE INDEX idx_nodes_value_fts ON nodes USING GIN (to_tsvector('english', value));
|
|
297
|
-
```
|
|
298
|
-
|
|
299
|
-
#### `relationships`
|
|
300
|
-
|
|
301
|
-
Node relationship graph.
|
|
302
|
-
|
|
303
|
-
```sql
|
|
304
|
-
CREATE TABLE relationships (
|
|
305
|
-
id SERIAL PRIMARY KEY,
|
|
306
|
-
from_node_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
|
|
307
|
-
to_node_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
|
|
308
|
-
relationship_type TEXT,
|
|
309
|
-
strength REAL DEFAULT 1.0,
|
|
310
|
-
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
|
311
|
-
UNIQUE(from_node_id, to_node_id, relationship_type)
|
|
312
|
-
);
|
|
313
|
-
|
|
314
|
-
-- Indexes
|
|
315
|
-
CREATE INDEX idx_relationships_from ON relationships(from_node_id);
|
|
316
|
-
CREATE INDEX idx_relationships_to ON relationships(to_node_id);
|
|
317
|
-
```
|
|
318
|
-
|
|
319
|
-
#### `tags`
|
|
320
|
-
|
|
321
|
-
Flexible tagging system.
|
|
322
|
-
|
|
323
|
-
```sql
|
|
324
|
-
CREATE TABLE tags (
|
|
325
|
-
id SERIAL PRIMARY KEY,
|
|
326
|
-
node_id INTEGER NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
|
|
327
|
-
tag TEXT NOT NULL,
|
|
328
|
-
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
|
329
|
-
UNIQUE(node_id, tag)
|
|
330
|
-
);
|
|
331
|
-
|
|
332
|
-
-- Indexes
|
|
333
|
-
CREATE INDEX idx_tags_node_id ON tags(node_id);
|
|
334
|
-
CREATE INDEX idx_tags_tag ON tags(tag);
|
|
335
|
-
```
|
|
336
|
-
|
|
337
|
-
#### `robots`
|
|
338
|
-
|
|
339
|
-
Robot registry for multi-robot tracking.
|
|
340
|
-
|
|
341
|
-
```sql
|
|
342
|
-
CREATE TABLE robots (
|
|
343
|
-
id TEXT PRIMARY KEY,
|
|
344
|
-
name TEXT NOT NULL,
|
|
345
|
-
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
|
346
|
-
last_active TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
|
347
|
-
);
|
|
348
|
-
```
|
|
349
|
-
|
|
350
|
-
#### `operations_log`
|
|
351
|
-
|
|
352
|
-
Audit log for all operations (hypertable partitioned by `timestamp`).
|
|
353
|
-
|
|
354
|
-
```sql
|
|
355
|
-
CREATE TABLE operations_log (
|
|
356
|
-
id SERIAL,
|
|
357
|
-
operation TEXT NOT NULL,
|
|
358
|
-
node_id INTEGER REFERENCES nodes(id) ON DELETE SET NULL,
|
|
359
|
-
robot_id TEXT NOT NULL REFERENCES robots(id),
|
|
360
|
-
timestamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
|
361
|
-
details JSONB
|
|
362
|
-
);
|
|
363
|
-
|
|
364
|
-
-- Indexes
|
|
365
|
-
CREATE INDEX idx_operations_log_timestamp ON operations_log(timestamp DESC);
|
|
366
|
-
CREATE INDEX idx_operations_log_robot_id ON operations_log(robot_id);
|
|
367
|
-
CREATE INDEX idx_operations_log_operation ON operations_log(operation);
|
|
368
|
-
```
|
|
369
|
-
|
|
370
|
-
### Views
|
|
371
|
-
|
|
372
|
-
#### `node_stats`
|
|
373
|
-
|
|
374
|
-
Aggregate statistics by node type.
|
|
375
|
-
|
|
376
|
-
```sql
|
|
377
|
-
CREATE VIEW node_stats AS
|
|
378
|
-
SELECT
|
|
379
|
-
type,
|
|
380
|
-
COUNT(*) as count,
|
|
381
|
-
AVG(importance) as avg_importance,
|
|
382
|
-
MIN(created_at) as oldest,
|
|
383
|
-
MAX(created_at) as newest
|
|
384
|
-
FROM nodes
|
|
385
|
-
GROUP BY type;
|
|
386
|
-
```
|
|
387
|
-
|
|
388
|
-
#### `robot_activity`
|
|
389
|
-
|
|
390
|
-
Robot activity summary.
|
|
391
|
-
|
|
392
|
-
```sql
|
|
393
|
-
CREATE VIEW robot_activity AS
|
|
394
|
-
SELECT
|
|
395
|
-
id,
|
|
396
|
-
name,
|
|
397
|
-
last_active,
|
|
398
|
-
(SELECT COUNT(*) FROM nodes WHERE robot_id = robots.id) as node_count
|
|
399
|
-
FROM robots
|
|
400
|
-
ORDER BY last_active DESC;
|
|
401
|
-
```
|
|
402
|
-
|
|
403
|
-
---
|
|
404
|
-
|
|
405
|
-
## TimescaleDB Hypertables
|
|
406
|
-
|
|
407
|
-
### `nodes` Hypertable
|
|
408
|
-
|
|
409
|
-
```sql
|
|
410
|
-
SELECT create_hypertable('nodes', 'created_at',
|
|
411
|
-
if_not_exists => TRUE,
|
|
412
|
-
migrate_data => TRUE
|
|
413
|
-
);
|
|
414
|
-
```
|
|
415
|
-
|
|
416
|
-
**Partitioning**: By `created_at` timestamp
|
|
417
|
-
|
|
418
|
-
**Chunk Interval**: 7 days (default)
|
|
419
|
-
|
|
420
|
-
**Compression**:
|
|
421
|
-
|
|
422
|
-
```sql
|
|
423
|
-
ALTER TABLE nodes SET (
|
|
424
|
-
timescaledb.compress,
|
|
425
|
-
timescaledb.compress_segmentby = 'robot_id,type'
|
|
426
|
-
);
|
|
427
|
-
|
|
428
|
-
SELECT add_compression_policy('nodes', INTERVAL '30 days',
|
|
429
|
-
if_not_exists => TRUE
|
|
430
|
-
);
|
|
431
|
-
```
|
|
432
|
-
|
|
433
|
-
- Automatically compresses chunks older than 30 days
|
|
434
|
-
- Segments by `robot_id` and `type` for efficient queries
|
|
435
|
-
- Reduces storage by ~90% for old data
|
|
436
|
-
|
|
437
|
-
### `operations_log` Hypertable
|
|
438
|
-
|
|
439
|
-
```sql
|
|
440
|
-
SELECT create_hypertable('operations_log', 'timestamp',
|
|
441
|
-
if_not_exists => TRUE,
|
|
442
|
-
migrate_data => TRUE
|
|
443
|
-
);
|
|
444
|
-
```
|
|
445
|
-
|
|
446
|
-
**Partitioning**: By `timestamp`
|
|
447
|
-
|
|
448
|
-
**Chunk Interval**: 1 day (default)
|
|
449
|
-
|
|
450
|
-
**Benefits**:
|
|
451
|
-
|
|
452
|
-
- Fast time-range queries
|
|
453
|
-
- Automatic data retention policies (can be added)
|
|
454
|
-
- Optimized for append-only workload
|
|
455
|
-
|
|
456
|
-
---
|
|
457
|
-
|
|
458
|
-
## Required PostgreSQL Extensions
|
|
459
|
-
|
|
460
|
-
### TimescaleDB
|
|
461
|
-
|
|
462
|
-
Time-series database extension.
|
|
463
|
-
|
|
464
|
-
```sql
|
|
465
|
-
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
|
466
|
-
```
|
|
467
|
-
|
|
468
|
-
**Features Used**:
|
|
469
|
-
|
|
470
|
-
- Hypertables for time-series optimization
|
|
471
|
-
- Automatic chunking and partitioning
|
|
472
|
-
- Compression policies
|
|
473
|
-
- Continuous aggregates (planned)
|
|
474
|
-
|
|
475
|
-
### pgvector
|
|
476
|
-
|
|
477
|
-
Vector similarity search.
|
|
478
|
-
|
|
479
|
-
```sql
|
|
480
|
-
CREATE EXTENSION IF NOT EXISTS vector;
|
|
481
|
-
```
|
|
482
|
-
|
|
483
|
-
**Features Used**:
|
|
484
|
-
|
|
485
|
-
- `vector(1536)` data type for embeddings
|
|
486
|
-
- Cosine similarity operator `<=>`
|
|
487
|
-
- IVFFlat index for approximate nearest neighbor
|
|
488
|
-
|
|
489
|
-
### pg_trgm
|
|
490
|
-
|
|
491
|
-
Trigram-based text search.
|
|
492
|
-
|
|
493
|
-
```sql
|
|
494
|
-
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
|
495
|
-
```
|
|
274
|
+
| Table | Purpose |
|
|
275
|
+
|-------|---------|
|
|
276
|
+
| [robots](../database/public.robots.md) | Robot registry for multi-robot tracking |
|
|
277
|
+
| [nodes](../database/public.nodes.md) | Primary memory storage with vector embeddings |
|
|
278
|
+
| [tags](../database/public.tags.md) | Hierarchical tag names for categorization |
|
|
279
|
+
| [robot_nodes](../database/public.robot_nodes.md) | Robot-to-node associations (hive mind) |
|
|
280
|
+
| [node_tags](../database/public.node_tags.md) | Node-to-tag associations |
|
|
281
|
+
| [working_memories](../database/public.working_memories.md) | Per-robot working memory state |
|
|
496
282
|
|
|
497
|
-
|
|
283
|
+
### Required Extensions
|
|
498
284
|
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
285
|
+
| Extension | Purpose |
|
|
286
|
+
|-----------|---------|
|
|
287
|
+
| `pgvector` | Vector similarity search with HNSW indexes |
|
|
288
|
+
| `pg_trgm` | Trigram-based fuzzy text matching |
|
|
502
289
|
|
|
503
290
|
---
|
|
504
291
|
|
|
@@ -801,6 +588,7 @@ end
|
|
|
801
588
|
|
|
802
589
|
- [HTM API](htm.md) - Main class that uses Database config
|
|
803
590
|
- [LongTermMemory API](long-term-memory.md) - Uses database for storage
|
|
804
|
-
- [Database Schema](../development/schema.md) -
|
|
805
|
-
- [
|
|
591
|
+
- [Database Schema](../development/schema.md) - Query patterns, optimization tips, and best practices
|
|
592
|
+
- [Database Tables](../database/README.md) - Auto-generated table reference with ER diagram
|
|
806
593
|
- [pgvector Documentation](https://github.com/pgvector/pgvector) - Vector search
|
|
594
|
+
- [pg_trgm Documentation](https://www.postgresql.org/docs/current/pgtrgm.html) - Trigram fuzzy matching
|
|
@@ -353,13 +353,7 @@ system("ollama pull nomic-embed-text")
|
|
|
353
353
|
|
|
354
354
|
### Exception Types
|
|
355
355
|
|
|
356
|
-
|
|
357
|
-
HTM::EmbeddingError
|
|
358
|
-
├─ "Ollama connection failed"
|
|
359
|
-
├─ "OpenAI API error: ..."
|
|
360
|
-
├─ "Invalid model: ..."
|
|
361
|
-
└─ "Empty text provided"
|
|
362
|
-
```
|
|
356
|
+

|
|
363
357
|
|
|
364
358
|
---
|
|
365
359
|
|