htm 0.0.1

data/db/schema.sql

@@ -0,0 +1,473 @@
+-- HTM Database Schema
+-- Auto-generated from database using pg_dump
+-- DO NOT EDIT THIS FILE MANUALLY
+-- Run 'rake htm:db:schema:dump' to regenerate
+
+--
+-- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
+
+--
+-- Name: EXTENSION pg_trgm; Type: COMMENT; Schema: -; Owner: -
+--
+
+--
+-- Name: vector; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public;
+
+--
+-- Name: EXTENSION vector; Type: COMMENT; Schema: -; Owner: -
+--
+
+--
+-- Name: node_tags; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.node_tags (
+    id bigint NOT NULL,
+    node_id bigint NOT NULL,
+    tag_id bigint NOT NULL,
+    created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
+);
+
+--
+-- Name: TABLE node_tags; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON TABLE public.node_tags IS 'Join table connecting nodes to tags (many-to-many)';
+
+--
+-- Name: COLUMN node_tags.node_id; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.node_tags.node_id IS 'ID of the node being tagged';
+
+--
+-- Name: COLUMN node_tags.tag_id; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.node_tags.tag_id IS 'ID of the tag being applied';
+
+--
+-- Name: COLUMN node_tags.created_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.node_tags.created_at IS 'When this association was created';
+
+--
+-- Name: node_tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.node_tags_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+--
+-- Name: node_tags_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.node_tags_id_seq OWNED BY public.node_tags.id;
+
+--
+-- Name: nodes; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.nodes (
+    id bigint NOT NULL,
+    content text NOT NULL,
+    source text DEFAULT ''::text,
+    access_count integer DEFAULT 0 NOT NULL,
+    created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    last_accessed timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    token_count integer,
+    in_working_memory boolean DEFAULT false,
+    robot_id bigint NOT NULL,
+    embedding public.vector(2000),
+    embedding_dimension integer,
+    CONSTRAINT check_embedding_dimension CHECK (((embedding_dimension IS NULL) OR ((embedding_dimension > 0) AND (embedding_dimension <= 2000))))
+);
+
+--
+-- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON TABLE public.nodes IS 'Core memory storage for conversation messages and context';
+
+--
+-- Name: COLUMN nodes.content; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.content IS 'The conversation message/utterance content';
+
+--
+-- Name: COLUMN nodes.source; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.source IS 'From where the content came (empty string if unknown)';
+
+--
+-- Name: COLUMN nodes.access_count; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.access_count IS 'Number of times this node has been accessed/retrieved';
+
+--
+-- Name: COLUMN nodes.created_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.created_at IS 'When this memory was created';
+
+--
+-- Name: COLUMN nodes.updated_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.updated_at IS 'When this memory was last modified';
+
+--
+-- Name: COLUMN nodes.last_accessed; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.last_accessed IS 'When this memory was last accessed';
+
+--
+-- Name: COLUMN nodes.token_count; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.token_count IS 'Number of tokens in the content (for context budget management)';
+
+--
+-- Name: COLUMN nodes.in_working_memory; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.in_working_memory IS 'Whether this memory is currently in working memory';
+
+--
+-- Name: COLUMN nodes.robot_id; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.robot_id IS 'ID of the robot that owns this memory';
+
+--
+-- Name: COLUMN nodes.embedding; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.embedding IS 'Vector embedding (max 2000 dimensions) for semantic search';
+
+--
+-- Name: COLUMN nodes.embedding_dimension; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.nodes.embedding_dimension IS 'Actual number of dimensions used in the embedding vector (max 2000)';
+
+--
+-- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.nodes_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+--
+-- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id;
+
+--
+-- Name: robots; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.robots (
+    id bigint NOT NULL,
+    name text,
+    created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    last_active timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+    metadata jsonb
+);
+
+--
+-- Name: TABLE robots; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON TABLE public.robots IS 'Registry of all LLM robots using the HTM system';
+
+--
+-- Name: COLUMN robots.name; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.robots.name IS 'Human-readable name for the robot';
+
+--
+-- Name: COLUMN robots.created_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.robots.created_at IS 'When the robot was first registered';
+
+--
+-- Name: COLUMN robots.last_active; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.robots.last_active IS 'Last time the robot accessed the system';
+
+--
+-- Name: COLUMN robots.metadata; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.robots.metadata IS 'Robot-specific configuration and metadata';
+
+--
+-- Name: robots_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.robots_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+--
+-- Name: robots_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.robots_id_seq OWNED BY public.robots.id;
+
+--
+-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.schema_migrations (
+    version character varying NOT NULL
+);
+
+--
+-- Name: tags; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.tags (
+    id bigint NOT NULL,
+    name text NOT NULL,
+    created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP
+);
+
+--
+-- Name: TABLE tags; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON TABLE public.tags IS 'Unique tag names for categorization';
+
+--
+-- Name: COLUMN tags.name; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.tags.name IS 'Hierarchical tag in format: root:level1:level2 (e.g., database:postgresql:timescaledb)';
+
+--
+-- Name: COLUMN tags.created_at; Type: COMMENT; Schema: public; Owner: -
+--
+
+COMMENT ON COLUMN public.tags.created_at IS 'When this tag was created';
+
+--
+-- Name: tags_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.tags_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+--
+-- Name: tags_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.tags_id_seq OWNED BY public.tags.id;
+
+--
+-- Name: node_tags id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.node_tags ALTER COLUMN id SET DEFAULT nextval('public.node_tags_id_seq'::regclass);
+
+--
+-- Name: nodes id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
+
+--
+-- Name: robots id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.robots ALTER COLUMN id SET DEFAULT nextval('public.robots_id_seq'::regclass);
+
+--
+-- Name: tags id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tags ALTER COLUMN id SET DEFAULT nextval('public.tags_id_seq'::regclass);
+
+--
+-- Name: node_tags node_tags_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.node_tags
+    ADD CONSTRAINT node_tags_pkey PRIMARY KEY (id);
+
+--
+-- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.nodes
+    ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
+
+--
+-- Name: robots robots_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.robots
+    ADD CONSTRAINT robots_pkey PRIMARY KEY (id);
+
+--
+-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.schema_migrations
+    ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version);
+
+--
+-- Name: tags tags_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.tags
+    ADD CONSTRAINT tags_pkey PRIMARY KEY (id);
+
+--
+-- Name: idx_node_tags_node_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_node_tags_node_id ON public.node_tags USING btree (node_id);
+
+--
+-- Name: idx_node_tags_tag_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_node_tags_tag_id ON public.node_tags USING btree (tag_id);
+
+--
+-- Name: idx_node_tags_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX idx_node_tags_unique ON public.node_tags USING btree (node_id, tag_id);
+
+--
+-- Name: idx_nodes_access_count; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_access_count ON public.nodes USING btree (access_count);
+
+--
+-- Name: idx_nodes_content_gin; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_content_gin ON public.nodes USING gin (to_tsvector('english'::regconfig, content));
+
+--
+-- Name: idx_nodes_content_trgm; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_content_trgm ON public.nodes USING gin (content public.gin_trgm_ops);
+
+--
+-- Name: idx_nodes_created_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_created_at ON public.nodes USING btree (created_at);
+
+--
+-- Name: idx_nodes_embedding; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_embedding ON public.nodes USING hnsw (embedding public.vector_cosine_ops) WITH (m='16', ef_construction='64');
+
+--
+-- Name: idx_nodes_in_working_memory; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_in_working_memory ON public.nodes USING btree (in_working_memory);
+
+--
+-- Name: idx_nodes_last_accessed; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_last_accessed ON public.nodes USING btree (last_accessed);
+
+--
+-- Name: idx_nodes_robot_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_robot_id ON public.nodes USING btree (robot_id);
+
+--
+-- Name: idx_nodes_source; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_source ON public.nodes USING btree (source);
+
+--
+-- Name: idx_nodes_updated_at; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_nodes_updated_at ON public.nodes USING btree (updated_at);
+
+--
+-- Name: idx_tags_name_pattern; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_tags_name_pattern ON public.tags USING btree (name text_pattern_ops);
+
+--
+-- Name: idx_tags_name_unique; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX idx_tags_name_unique ON public.tags USING btree (name);
+
+--
+-- Name: nodes fk_rails_60162e9d3a; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.nodes
+    ADD CONSTRAINT fk_rails_60162e9d3a FOREIGN KEY (robot_id) REFERENCES public.robots(id) ON DELETE CASCADE;
+
+--
+-- Name: node_tags fk_rails_b51cdcc57f; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.node_tags
+    ADD CONSTRAINT fk_rails_b51cdcc57f FOREIGN KEY (tag_id) REFERENCES public.tags(id) ON DELETE CASCADE;
+
+--
+-- Name: node_tags fk_rails_ebc9aafd9f; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.node_tags
+    ADD CONSTRAINT fk_rails_ebc9aafd9f FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
+
+--
+-- PostgreSQL database dump complete
+--
+
+\unrestrict f5a75Zsnuw7NUeDmu1kxeQ3pRMbaORhrsWHJyDdXV4wbRfzQweTumJBXu85kf1z

data/db/seed_data/README.md

@@ -0,0 +1,100 @@
+# HTM Seed Data
+
+This directory contains markdown files used to seed the HTM database with sample data.
+
+## Configuration
+
+The seeding process uses environment variables for configuration. All settings have sensible defaults:
+
+### LLM Provider Settings
+
+- `HTM_EMBEDDING_PROVIDER` - Embedding provider (default: `ollama`)
+- `HTM_EMBEDDING_MODEL` - Embedding model (default: `nomic-embed-text`)
+- `HTM_EMBEDDING_DIMENSIONS` - Embedding dimensions (default: `768`)
+- `HTM_TAG_PROVIDER` - Tag extraction provider (default: `ollama`)
+- `HTM_TAG_MODEL` - Tag extraction model (default: `gemma3`)
+- `OLLAMA_URL` - Ollama server URL (default: `http://localhost:11434`)
+
+### Timeout Settings
+
+- `HTM_EMBEDDING_TIMEOUT` - Embedding generation timeout in seconds (default: `120`)
+- `HTM_TAG_TIMEOUT` - Tag generation timeout in seconds (default: `180`)
+- `HTM_CONNECTION_TIMEOUT` - LLM connection timeout in seconds (default: `30`)
+
+### Database Settings
+
+- `HTM_DBURL` - Full PostgreSQL connection URL (required)
+- Or individual settings: `HTM_DBHOST`, `HTM_DBPORT`, `HTM_DBNAME`, `HTM_DBUSER`, `HTM_DBPASS`
+
+### Other Settings
+
+- `HTM_ROBOT_NAME` - Name for the seeding robot (default: `"Seed Robot"`)
+
+## Format
+
+All `.md` files in this directory will be automatically processed by `db/seeds.rb`.
+
+Each markdown file should follow this structure:
+
+```markdown
+# Title (optional, will be ignored)
+
+## Section Name 1
+Paragraph of content for this section. This entire paragraph will be stored
+as a single memory node in HTM.
+
+## Section Name 2
+Another paragraph of content. Each ## header denotes a new section that will
+become a separate memory node.
+
+## Section Name 3
+Content for the third section...
+```
+
+## Processing
+
+The seeding script (`db/seeds.rb`):
+
+1. Reads all `*.md` files from this directory
+2. Parses each file looking for `## Header` sections
+3. Extracts the paragraph(s) following each header
+4. Creates an HTM memory node for each section
+5. Uses the filename (without `.md`) as the `source` field
+6. Automatically generates embeddings and hierarchical tags for each node
+
+## Current Seed Data
+
+- **states.md**: Interesting facts about all 50 US states
+- **presidents.md**: Interesting facts about all 45 US presidents
+
+## Adding New Seed Data
+
+To add new seed data:
+
+1. Create a new `.md` file in this directory
+2. Follow the format above with `## Header` sections
+3. Run `rake htm:db:seed` to populate the database
+
+The filename (without extension) will be used as the source identifier for all nodes
+created from that file.
+
+## Example
+
+Given a file `countries.md`:
+
+```markdown
+# World Countries
+
+## France
+France is known for the Eiffel Tower...
+
+## Japan
+Japan is an island nation...
+```
+
+Running `rake htm:db:seed` will create:
+- 2 memory nodes
+- Both with `source: "countries"`
+- Each with embeddings (768-dimensional vectors)
+- Each with hierarchical tags extracted by LLM
+- Stored in the `nodes` table with full-text and vector search capabilities