RubyGems - htm - Versions diffs - 0.0.1 - Mend

htm 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (155) hide show

checksums.yaml +7 -0
data/.architecture/decisions/adrs/001-use-postgresql-timescaledb-storage.md +227 -0
data/.architecture/decisions/adrs/002-two-tier-memory-architecture.md +322 -0
data/.architecture/decisions/adrs/003-ollama-default-embedding-provider.md +339 -0
data/.architecture/decisions/adrs/004-multi-robot-shared-memory-hive-mind.md +374 -0
data/.architecture/decisions/adrs/005-rag-based-retrieval-with-hybrid-search.md +443 -0
data/.architecture/decisions/adrs/006-context-assembly-strategies.md +444 -0
data/.architecture/decisions/adrs/007-working-memory-eviction-strategy.md +461 -0
data/.architecture/decisions/adrs/008-robot-identification-system.md +550 -0
data/.architecture/decisions/adrs/009-never-forget-explicit-deletion-only.md +570 -0
data/.architecture/decisions/adrs/010-redis-working-memory-rejected.md +323 -0
data/.architecture/decisions/adrs/011-database-side-embedding-generation-with-pgai.md +585 -0
data/.architecture/decisions/adrs/012-llm-driven-ontology-topic-extraction.md +583 -0
data/.architecture/decisions/adrs/013-activerecord-orm-and-many-to-many-tagging.md +299 -0
data/.architecture/decisions/adrs/014-client-side-embedding-generation-workflow.md +569 -0
data/.architecture/decisions/adrs/015-hierarchical-tag-ontology-and-llm-extraction.md +701 -0
data/.architecture/decisions/adrs/016-async-embedding-and-tag-generation.md +694 -0
data/.architecture/members.yml +144 -0
data/.architecture/reviews/2025-10-29-llm-configuration-and-async-processing-review.md +1137 -0
data/.architecture/reviews/initial-system-analysis.md +330 -0
data/.envrc +32 -0
data/.irbrc +145 -0
data/CHANGELOG.md +150 -0
data/COMMITS.md +196 -0
data/LICENSE +21 -0
data/README.md +1347 -0
data/Rakefile +51 -0
data/SETUP.md +268 -0
data/config/database.yml +67 -0
data/db/migrate/20250101000001_enable_extensions.rb +14 -0
data/db/migrate/20250101000002_create_robots.rb +14 -0
data/db/migrate/20250101000003_create_nodes.rb +42 -0
data/db/migrate/20250101000005_create_tags.rb +38 -0
data/db/migrate/20250101000007_add_node_vector_indexes.rb +30 -0
data/db/schema.sql +473 -0
data/db/seed_data/README.md +100 -0
data/db/seed_data/presidents.md +136 -0
data/db/seed_data/states.md +151 -0
data/db/seeds.rb +208 -0
data/dbdoc/README.md +173 -0
data/dbdoc/public.node_stats.md +48 -0
data/dbdoc/public.node_stats.svg +41 -0
data/dbdoc/public.node_tags.md +40 -0
data/dbdoc/public.node_tags.svg +112 -0
data/dbdoc/public.nodes.md +54 -0
data/dbdoc/public.nodes.svg +118 -0
data/dbdoc/public.nodes_tags.md +39 -0
data/dbdoc/public.nodes_tags.svg +112 -0
data/dbdoc/public.ontology_structure.md +48 -0
data/dbdoc/public.ontology_structure.svg +38 -0
data/dbdoc/public.operations_log.md +42 -0
data/dbdoc/public.operations_log.svg +130 -0
data/dbdoc/public.relationships.md +39 -0
data/dbdoc/public.relationships.svg +41 -0
data/dbdoc/public.robot_activity.md +46 -0
data/dbdoc/public.robot_activity.svg +35 -0
data/dbdoc/public.robots.md +35 -0
data/dbdoc/public.robots.svg +90 -0
data/dbdoc/public.schema_migrations.md +29 -0
data/dbdoc/public.schema_migrations.svg +26 -0
data/dbdoc/public.tags.md +35 -0
data/dbdoc/public.tags.svg +60 -0
data/dbdoc/public.topic_relationships.md +45 -0
data/dbdoc/public.topic_relationships.svg +32 -0
data/dbdoc/schema.json +1437 -0
data/dbdoc/schema.svg +154 -0
data/docs/api/database.md +806 -0
data/docs/api/embedding-service.md +532 -0
data/docs/api/htm.md +797 -0
data/docs/api/index.md +259 -0
data/docs/api/long-term-memory.md +1096 -0
data/docs/api/working-memory.md +665 -0
data/docs/architecture/adrs/001-postgresql-timescaledb.md +314 -0
data/docs/architecture/adrs/002-two-tier-memory.md +411 -0
data/docs/architecture/adrs/003-ollama-embeddings.md +421 -0
data/docs/architecture/adrs/004-hive-mind.md +437 -0
data/docs/architecture/adrs/005-rag-retrieval.md +531 -0
data/docs/architecture/adrs/006-context-assembly.md +496 -0
data/docs/architecture/adrs/007-eviction-strategy.md +645 -0
data/docs/architecture/adrs/008-robot-identification.md +625 -0
data/docs/architecture/adrs/009-never-forget.md +648 -0
data/docs/architecture/adrs/010-redis-working-memory-rejected.md +323 -0
data/docs/architecture/adrs/011-pgai-integration.md +494 -0
data/docs/architecture/adrs/index.md +215 -0
data/docs/architecture/hive-mind.md +736 -0
data/docs/architecture/index.md +351 -0
data/docs/architecture/overview.md +538 -0
data/docs/architecture/two-tier-memory.md +873 -0
data/docs/assets/css/custom.css +83 -0
data/docs/assets/images/htm-core-components.svg +63 -0
data/docs/assets/images/htm-database-schema.svg +93 -0
data/docs/assets/images/htm-hive-mind-architecture.svg +125 -0
data/docs/assets/images/htm-importance-scoring-framework.svg +83 -0
data/docs/assets/images/htm-layered-architecture.svg +71 -0
data/docs/assets/images/htm-long-term-memory-architecture.svg +115 -0
data/docs/assets/images/htm-working-memory-architecture.svg +120 -0
data/docs/assets/images/htm.jpg +0 -0
data/docs/assets/images/htm_demo.gif +0 -0
data/docs/assets/js/mathjax.js +18 -0
data/docs/assets/videos/htm_video.mp4 +0 -0
data/docs/database_rake_tasks.md +322 -0
data/docs/development/contributing.md +787 -0
data/docs/development/index.md +336 -0
data/docs/development/schema.md +596 -0
data/docs/development/setup.md +719 -0
data/docs/development/testing.md +819 -0
data/docs/guides/adding-memories.md +824 -0
data/docs/guides/context-assembly.md +1009 -0
data/docs/guides/getting-started.md +577 -0
data/docs/guides/index.md +118 -0
data/docs/guides/long-term-memory.md +941 -0
data/docs/guides/multi-robot.md +866 -0
data/docs/guides/recalling-memories.md +927 -0
data/docs/guides/search-strategies.md +953 -0
data/docs/guides/working-memory.md +717 -0
data/docs/index.md +214 -0
data/docs/installation.md +477 -0
data/docs/multi_framework_support.md +519 -0
data/docs/quick-start.md +655 -0
data/docs/setup_local_database.md +302 -0
data/docs/using_rake_tasks_in_your_app.md +383 -0
data/examples/basic_usage.rb +93 -0
data/examples/cli_app/README.md +317 -0
data/examples/cli_app/htm_cli.rb +270 -0
data/examples/custom_llm_configuration.rb +183 -0
data/examples/example_app/Rakefile +71 -0
data/examples/example_app/app.rb +206 -0
data/examples/sinatra_app/Gemfile +21 -0
data/examples/sinatra_app/app.rb +335 -0
data/lib/htm/active_record_config.rb +113 -0
data/lib/htm/configuration.rb +342 -0
data/lib/htm/database.rb +594 -0
data/lib/htm/embedding_service.rb +115 -0
data/lib/htm/errors.rb +34 -0
data/lib/htm/job_adapter.rb +154 -0
data/lib/htm/jobs/generate_embedding_job.rb +65 -0
data/lib/htm/jobs/generate_tags_job.rb +82 -0
data/lib/htm/long_term_memory.rb +965 -0
data/lib/htm/models/node.rb +109 -0
data/lib/htm/models/node_tag.rb +33 -0
data/lib/htm/models/robot.rb +52 -0
data/lib/htm/models/tag.rb +76 -0
data/lib/htm/railtie.rb +76 -0
data/lib/htm/sinatra.rb +157 -0
data/lib/htm/tag_service.rb +135 -0
data/lib/htm/tasks.rb +38 -0
data/lib/htm/version.rb +5 -0
data/lib/htm/working_memory.rb +182 -0
data/lib/htm.rb +400 -0
data/lib/tasks/db.rake +19 -0
data/lib/tasks/htm.rake +147 -0
data/lib/tasks/jobs.rake +312 -0
data/mkdocs.yml +190 -0
data/scripts/install_local_database.sh +309 -0
metadata +341 -0

data/examples/sinatra_app/app.rb ADDED Viewed

@@ -0,0 +1,335 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+# HTM Sinatra Application Example
+#
+# Demonstrates using HTM in a Sinatra web application with:
+# - Sidekiq for background job processing
+# - Session-based robot identification
+# - RESTful API endpoints
+# - Thread-safe concurrent request handling
+#
+# Usage:
+#   bundle install
+#   bundle exec ruby app.rb
+#
+# Environment:
+#   HTM_DBURL - PostgreSQL connection URL (required)
+#   REDIS_URL - Redis connection URL (for Sidekiq, default: redis://localhost:6379/0)
+#   OLLAMA_URL - Ollama server URL (default: http://localhost:11434)
+#
+require 'sinatra'
+require 'sinatra/json'
+require 'sidekiq'
+require_relative '../../lib/htm'
+require_relative '../../lib/htm/sinatra'
+# Sidekiq configuration
+Sidekiq.configure_server do |config|
+  config.redis = { url: ENV.fetch('REDIS_URL', 'redis://localhost:6379/0') }
+end
+Sidekiq.configure_client do |config|
+  config.redis = { url: ENV.fetch('REDIS_URL', 'redis://localhost:6379/0') }
+end
+# Sinatra Application
+class HTMApp < Sinatra::Base
+  # Register HTM with automatic configuration
+  register_htm
+  # Enable sessions for robot identification
+  enable :sessions
+  set :session_secret, ENV.fetch('SESSION_SECRET', 'change_me_in_production')
+  # Initialize HTM for each request
+  before do
+    # Use session ID as robot identifier
+    robot_name = session[:robot_id] ||= SecureRandom.uuid[0..7]
+    init_htm(robot_name: "web_user_#{robot_name}")
+    # Set content type for API responses
+    content_type :json if request.path.start_with?('/api/')
+  end
+  # Home page
+  get '/' do
+    erb :index
+  end
+  # API: Remember information
+  post '/api/remember' do
+    content = params[:content]
+    unless content && !content.empty?
+      halt 400, json(error: 'Content is required')
+    end
+    node_id = remember(content, source: 'web_user')
+    json(
+      status: 'ok',
+      node_id: node_id,
+      message: 'Memory stored. Embedding and tags are being generated in background.'
+    )
+  end
+  # API: Recall memories
+  get '/api/recall' do
+    topic = params[:topic]
+    limit = (params[:limit] || 10).to_i
+    strategy = (params[:strategy] || 'hybrid').to_sym
+    unless topic && !topic.empty?
+      halt 400, json(error: 'Topic is required')
+    end
+    unless [:vector, :fulltext, :hybrid].include?(strategy)
+      halt 400, json(error: 'Invalid strategy. Use: vector, fulltext, or hybrid')
+    end
+    memories = recall(topic, limit: limit, strategy: strategy, raw: true)
+    json(
+      status: 'ok',
+      count: memories.length,
+      memories: memories.map { |m| format_memory(m) }
+    )
+  end
+  # API: Get memory statistics
+  get '/api/stats' do
+    total_nodes = HTM::Models::Node.count
+    nodes_with_embeddings = HTM::Models::Node.where.not(embedding: nil).count
+    nodes_with_tags = HTM::Models::Node.joins(:tags).distinct.count
+    total_tags = HTM::Models::Tag.count
+    robot_nodes = HTM::Models::Node.where(robot_id: htm.robot_id).count
+    json(
+      status: 'ok',
+      stats: {
+        total_nodes: total_nodes,
+        nodes_with_embeddings: nodes_with_embeddings,
+        nodes_with_tags: nodes_with_tags,
+        total_tags: total_tags,
+        current_robot: {
+          id: htm.robot_id,
+          name: htm.robot_name,
+          nodes: robot_nodes
+        }
+      }
+    )
+  end
+  # API: Health check
+  get '/api/health' do
+    json(
+      status: 'ok',
+      job_backend: HTM.configuration.job_backend,
+      database: HTM::ActiveRecordConfig.connected?,
+      timestamp: Time.now.iso8601
+    )
+  end
+  private
+  def format_memory(memory)
+    {
+      id: memory['id'],
+      content: memory['content'],
+      source: memory['source'],
+      created_at: memory['created_at'],
+      token_count: memory['token_count']
+    }
+  end
+  # Run the app
+  run! if app_file == $0
+end
+__END__
+@@index
+<!DOCTYPE html>
+<html>
+<head>
+  <title>HTM Sinatra Example</title>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <style>
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+      max-width: 800px;
+      margin: 40px auto;
+      padding: 0 20px;
+      line-height: 1.6;
+    }
+    h1 { color: #333; }
+    .section {
+      background: #f5f5f5;
+      padding: 20px;
+      margin: 20px 0;
+      border-radius: 8px;
+    }
+    input[type="text"], textarea {
+      width: 100%;
+      padding: 10px;
+      margin: 10px 0;
+      border: 1px solid #ddd;
+      border-radius: 4px;
+      box-sizing: border-box;
+    }
+    button {
+      background: #007bff;
+      color: white;
+      padding: 10px 20px;
+      border: none;
+      border-radius: 4px;
+      cursor: pointer;
+    }
+    button:hover { background: #0056b3; }
+    .result {
+      background: white;
+      padding: 15px;
+      margin: 10px 0;
+      border-radius: 4px;
+      border-left: 4px solid #007bff;
+    }
+    .error {
+      border-left-color: #dc3545;
+      background: #fff5f5;
+    }
+  </style>
+</head>
+<body>
+  <h1>HTM Sinatra Example</h1>
+  <p>Hierarchical Temporary Memory with Sidekiq background jobs</p>
+  <div class="section">
+    <h2>Remember Information</h2>
+    <textarea id="rememberContent" rows="4" placeholder="Enter information to remember..."></textarea>
+    <button onclick="remember()">Remember</button>
+    <div id="rememberResult"></div>
+  </div>
+  <div class="section">
+    <h2>Recall Memories</h2>
+    <input type="text" id="recallTopic" placeholder="Enter topic to search...">
+    <select id="recallStrategy">
+      <option value="hybrid">Hybrid (Vector + Fulltext)</option>
+      <option value="vector">Vector Only</option>
+      <option value="fulltext">Fulltext Only</option>
+    </select>
+    <button onclick="recall()">Recall</button>
+    <div id="recallResult"></div>
+  </div>
+  <div class="section">
+    <h2>Memory Statistics</h2>
+    <button onclick="getStats()">Refresh Stats</button>
+    <div id="statsResult"></div>
+  </div>
+  <script>
+    async function remember() {
+      const content = document.getElementById('rememberContent').value;
+      const resultDiv = document.getElementById('rememberResult');
+      if (!content) {
+        resultDiv.innerHTML = '<div class="result error">Please enter some content</div>';
+        return;
+      }
+      try {
+        const response = await fetch('/api/remember', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+          body: `content=${encodeURIComponent(content)}`
+        });
+        const data = await response.json();
+        if (response.ok) {
+          resultDiv.innerHTML = `<div class="result">
+            ✓ ${data.message}<br>
+            Node ID: ${data.node_id}
+          </div>`;
+          document.getElementById('rememberContent').value = '';
+        } else {
+          resultDiv.innerHTML = `<div class="result error">✗ ${data.error}</div>`;
+        }
+      } catch (error) {
+        resultDiv.innerHTML = `<div class="result error">✗ ${error.message}</div>`;
+      }
+    }
+    async function recall() {
+      const topic = document.getElementById('recallTopic').value;
+      const strategy = document.getElementById('recallStrategy').value;
+      const resultDiv = document.getElementById('recallResult');
+      if (!topic) {
+        resultDiv.innerHTML = '<div class="result error">Please enter a topic</div>';
+        return;
+      }
+      try {
+        const response = await fetch(`/api/recall?topic=${encodeURIComponent(topic)}&strategy=${strategy}&limit=10`);
+        const data = await response.json();
+        if (response.ok) {
+          if (data.count === 0) {
+            resultDiv.innerHTML = '<div class="result">No memories found</div>';
+          } else {
+            const memoriesHtml = data.memories.map(m => `
+              <div class="result">
+                <strong>Node ${m.id}</strong> (${m.source})<br>
+                ${m.content}<br>
+                <small>${new Date(m.created_at).toLocaleString()} • ${m.token_count} tokens</small>
+              </div>
+            `).join('');
+            resultDiv.innerHTML = `<p>Found ${data.count} memories:</p>${memoriesHtml}`;
+          }
+        } else {
+          resultDiv.innerHTML = `<div class="result error">✗ ${data.error}</div>`;
+        }
+      } catch (error) {
+        resultDiv.innerHTML = `<div class="result error">✗ ${error.message}</div>`;
+      }
+    }
+    async function getStats() {
+      const resultDiv = document.getElementById('statsResult');
+      try {
+        const response = await fetch('/api/stats');
+        const data = await response.json();
+        if (response.ok) {
+          const stats = data.stats;
+          resultDiv.innerHTML = `
+            <div class="result">
+              <strong>Global Statistics:</strong><br>
+              Total Nodes: ${stats.total_nodes}<br>
+              With Embeddings: ${stats.nodes_with_embeddings}<br>
+              With Tags: ${stats.nodes_with_tags}<br>
+              Total Tags: ${stats.total_tags}<br><br>
+              <strong>Your Session (${stats.current_robot.name}):</strong><br>
+              Nodes: ${stats.current_robot.nodes}
+            </div>
+          `;
+        } else {
+          resultDiv.innerHTML = `<div class="result error">✗ ${data.error}</div>`;
+        }
+      } catch (error) {
+        resultDiv.innerHTML = `<div class="result error">✗ ${error.message}</div>`;
+      }
+    }
+    // Load stats on page load
+    window.onload = getStats;
+  </script>
+</body>
+</html>

data/lib/htm/active_record_config.rb ADDED Viewed

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+require 'active_record'
+require 'pg'
+require 'neighbor'
+require 'erb'
+require 'yaml'
+class HTM
+  # ActiveRecord database configuration and model loading
+  class ActiveRecordConfig
+    class << self
+      # Establish database connection from config/database.yml
+      def establish_connection!
+        config = load_database_config
+        ActiveRecord::Base.establish_connection(config)
+        # Set search path
+        ActiveRecord::Base.connection.execute("SET search_path TO public")
+        # Load models after connection is established
+        require_models
+        true
+      end
+      # Load and parse database configuration from YAML with ERB
+      def load_database_config
+        config_path = File.expand_path('../../config/database.yml', __dir__)
+        unless File.exist?(config_path)
+          raise "Database configuration file not found at #{config_path}"
+        end
+        # Read and parse ERB
+        erb_content = ERB.new(File.read(config_path)).result
+        db_config = YAML.safe_load(erb_content, aliases: true)
+        # Determine environment
+        env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+        # Get configuration for current environment
+        config = db_config[env]
+        unless config
+          raise "No database configuration found for environment: #{env}"
+        end
+        # Convert string keys to symbols for ActiveRecord
+        config.transform_keys(&:to_sym)
+      end
+      # Check if connection is established and active
+      def connected?
+        ActiveRecord::Base.connected? &&
+          ActiveRecord::Base.connection.active?
+      rescue StandardError
+        false
+      end
+      # Close all database connections
+      def disconnect!
+        ActiveRecord::Base.connection_pool.disconnect!
+      end
+      # Verify required extensions are available
+      def verify_extensions!
+        conn = ActiveRecord::Base.connection
+        required_extensions = {
+          'vector' => 'pgvector extension',
+          'pg_trgm' => 'PostgreSQL trigram extension'
+        }
+        missing = []
+        required_extensions.each do |ext, name|
+          result = conn.select_value(
+            "SELECT COUNT(*) FROM pg_extension WHERE extname = '#{ext}'"
+          )
+          missing << name if result.to_i.zero?
+        end
+        if missing.any?
+          raise "Missing required PostgreSQL extensions: #{missing.join(', ')}"
+        end
+        true
+      end
+      # Get connection pool statistics
+      def connection_stats
+        pool = ActiveRecord::Base.connection_pool
+        {
+          size: pool.size,
+          connections: pool.connections.size,
+          in_use: pool.connections.count(&:in_use?),
+          available: pool.connections.count { |c| !c.in_use? }
+        }
+      end
+      private
+      # Require all model files
+      def require_models
+        require_relative 'models/robot'
+        require_relative 'models/node'
+        require_relative 'models/tag'
+        require_relative 'models/node_tag'
+      end
+    end
+  end
+end