activematrix 0.0.7 → 0.0.9

data/lib/active_matrix/client_pool.rb

@@ -1,26 +1,36 @@
 # frozen_string_literal: true
 
 require 'singleton'
-require 'concurrent'
+require 'async'
+require 'async/semaphore'
+require 'async/condition'
 
 module ActiveMatrix
-  # Manages a pool of Matrix client connections for efficiency
+  # Manages Matrix client connections per homeserver with rate limiting.
+  #
+  # NOTE: Despite the name, this is not a traditional connection pool.
+  # Each agent gets a dedicated long-lived client. The "pool" provides:
+  # - Semaphore-based rate limiting on client creation per homeserver
+  # - Tracking of active clients for health monitoring
+  #
+  # Clients are NOT returned to the pool after use - they remain active
+  # for the agent's lifetime. The semaphore prevents too many agents
+  # from connecting to a single homeserver simultaneously.
+  #
   class ClientPool
     include Singleton
     include ActiveMatrix::Logging
 
     def initialize
-      @pools = Concurrent::Hash.new
+      @pools = {}
       @config = ActiveMatrix.config
       @mutex = Mutex.new
     end
 
     # Get or create a client for a homeserver
     def get_client(homeserver, **)
-      @mutex.synchronize do
-        pool = get_or_create_pool(homeserver)
-        pool.checkout(**)
-      end
+      pool = @mutex.synchronize { get_or_create_pool(homeserver) }
+      pool.checkout(**)
     end
 
     # Return a client to the pool
@@ -78,32 +88,37 @@ module ActiveMatrix
         @available = []
         @in_use = {}
         @mutex = Mutex.new
-        @condition = ConditionVariable.new
+        @semaphore = Async::Semaphore.new(max_size)
       end
 
       def checkout(**)
-        @mutex.synchronize do
-          # Try to find an available client
-          client = find_available_client
+        # Acquire semaphore temporarily to rate-limit client creation
+        @semaphore.acquire
 
-          # Create new client if needed and pool not full
-          client = create_client(**) if client.nil? && @in_use.size < @max_size
+        client = @mutex.synchronize do
+          # Try to find an available client
+          existing = find_available_client
 
-          # Wait for a client if pool is full
-          while client.nil?
-            @condition.wait(@mutex, 1)
-            client = find_available_client
+          if existing
+            @available.delete(existing)
+            existing
+          else
+            create_client(**)
           end
+        end
 
-          # Mark as in use
-          @available.delete(client)
+        # Track as in use
+        @mutex.synchronize do
           @in_use[client.object_id] = {
             client: client,
             checked_out_at: Time.current
           }
-
-          client
         end
+
+        client
+      ensure
+        # Release immediately - semaphore only rate-limits creation, not usage
+        @semaphore.release
       end
 
       def checkin(client)
@@ -115,12 +130,8 @@ module ActiveMatrix
           if client_valid?(client)
             @available << client
           else
-            # Client is no longer valid, don't return to pool
             logger.debug "Discarding invalid client for #{@homeserver}"
           end
-
-          # Signal waiting threads
-          @condition.signal
         end
       end
 
@@ -140,7 +151,7 @@ module ActiveMatrix
         @mutex.synchronize do
           # Stop all clients
           (@available + @in_use.values.map { |e| e[:client] }).each do |client|
-            client.stop_listener_thread if client.listening?
+            client.stop_listener if client.listening?
             client.logout if client.logged_in?
           rescue StandardError => e
             logger.error "Error cleaning up client: #{e.message}"
@@ -158,7 +169,7 @@ module ActiveMatrix
         @available.select! { |client| client_valid?(client) }
 
         # Return first available
-        @available.first
+        @available.shift
       end
 
       def create_client(**)

data/lib/active_matrix/daemon/probe_server.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'async'
+require 'async/http'
+require 'json'
+
+module ActiveMatrix
+  class Daemon
+    # Lightweight HTTP health probe server using Async::HTTP
+    #
+    # Endpoints:
+    # - GET /health - Returns 200 if healthy, 503 if shutting down
+    # - GET /status - Returns detailed JSON status
+    # - GET /metrics - Prometheus-compatible metrics
+    #
+    class ProbeServer
+      attr_reader :host, :port, :daemon
+
+      def initialize(host:, port:, daemon:)
+        @host = host
+        @port = port
+        @daemon = daemon
+        @thread = nil
+      end
+
+      def start
+        @thread = Thread.new do
+          Sync do
+            endpoint = Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
+
+            @server = Async::HTTP::Server.for(endpoint) do |request|
+              handle_request(request)
+            end
+
+            logger.info "Probe server listening on #{host}:#{port}"
+            @server.run
+          end
+        rescue StandardError => e
+          logger.error "Probe server error: #{e.message}"
+        end
+      end
+
+      def stop
+        @thread&.kill
+        @thread&.join(1)
+      end
+
+      private
+
+      def handle_request(request)
+        path = request.path
+
+        case path
+        when '/health'
+          health_response
+        when '/status'
+          status_response
+        when '/metrics'
+          metrics_response
+        else
+          not_found_response
+        end
+      end
+
+      def health_response
+        status = daemon.status
+        code = status[:status] == 'ok' ? 200 : 503
+
+        ::Protocol::HTTP::Response[code, { 'content-type' => 'text/plain' }, [status[:status]]]
+      end
+
+      def status_response
+        body = JSON.pretty_generate(daemon.status)
+
+        ::Protocol::HTTP::Response[200, { 'content-type' => 'application/json' }, [body]]
+      end
+
+      def metrics_response
+        status = daemon.status
+
+        lines = [
+          '# HELP activematrix_up Is the daemon running',
+          '# TYPE activematrix_up gauge',
+          "activematrix_up #{status[:status] == 'ok' ? 1 : 0}",
+          '',
+          '# HELP activematrix_uptime_seconds Daemon uptime in seconds',
+          '# TYPE activematrix_uptime_seconds counter',
+          "activematrix_uptime_seconds #{status[:uptime]}",
+          '',
+          '# HELP activematrix_workers Number of worker processes',
+          '# TYPE activematrix_workers gauge',
+          "activematrix_workers #{status[:workers]}",
+          '',
+          '# HELP activematrix_agents_total Total number of agents',
+          '# TYPE activematrix_agents_total gauge',
+          "activematrix_agents_total #{status.dig(:agents, :total) || 0}",
+          '',
+          '# HELP activematrix_agents Agent count by state',
+          '# TYPE activematrix_agents gauge',
+          "activematrix_agents{state=\"online\"} #{status.dig(:agents, :online) || 0}",
+          "activematrix_agents{state=\"connecting\"} #{status.dig(:agents, :connecting) || 0}",
+          "activematrix_agents{state=\"error\"} #{status.dig(:agents, :error) || 0}",
+          "activematrix_agents{state=\"offline\"} #{status.dig(:agents, :offline) || 0}"
+        ]
+
+        ::Protocol::HTTP::Response[200, { 'content-type' => 'text/plain; version=0.0.4' }, [lines.join("\n")]]
+      end
+
+      def not_found_response
+        ::Protocol::HTTP::Response[404, { 'content-type' => 'text/plain' }, ['Not Found']]
+      end
+
+      def logger
+        ActiveMatrix.logger
+      end
+    end
+  end
+end

data/lib/active_matrix/daemon/signal_handler.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module ActiveMatrix
+  class Daemon
+    # Handles Unix signals for the daemon coordinator
+    #
+    # Signals:
+    # - TERM/INT: Graceful shutdown
+    # - HUP: Reload configuration and restart agents
+    # - USR1: Log rotation (reopen log files)
+    # - USR2: Dump debug information
+    #
+    class SignalHandler
+      SIGNALS = %w[TERM INT HUP USR1 USR2].freeze
+
+      attr_reader :daemon
+
+      def initialize(daemon)
+        @daemon = daemon
+        @self_pipe_reader, @self_pipe_writer = IO.pipe
+        @old_handlers = {}
+      end
+
+      def install
+        SIGNALS.each do |signal|
+          install_handler(signal)
+        end
+
+        # Start signal processing thread
+        start_processor
+      end
+
+      def uninstall
+        SIGNALS.each do |signal|
+          restore_handler(signal)
+        end
+
+        @self_pipe_writer.close
+        @self_pipe_reader.close
+      end
+
+      private
+
+      def install_handler(signal)
+        @old_handlers[signal] = Signal.trap(signal) do
+          # Write signal to pipe (non-blocking, safe in signal handler)
+          @self_pipe_writer.write_nonblock("#{signal}\n")
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK
+          # Pipe full, signal will be coalesced
+        end
+      rescue ArgumentError
+        # Signal not supported on this platform
+        logger.debug "Signal #{signal} not supported"
+      end
+
+      def restore_handler(signal)
+        return unless @old_handlers.key?(signal)
+
+        Signal.trap(signal, @old_handlers[signal])
+      end
+
+      def start_processor
+        Thread.new do
+          Thread.current.name = 'activematrix-signal-processor'
+
+          loop do
+            # rubocop:disable Lint/IncompatibleIoSelectWithFiberScheduler
+            ready = IO.select([@self_pipe_reader], nil, nil, 1)
+            # rubocop:enable Lint/IncompatibleIoSelectWithFiberScheduler
+            next unless ready
+
+            signal = @self_pipe_reader.gets&.strip
+            next unless signal
+
+            handle_signal(signal)
+          rescue IOError
+            break
+          end
+        end
+      end
+
+      def handle_signal(signal)
+        logger.info "Received signal: #{signal}"
+
+        case signal
+        when 'TERM', 'INT'
+          handle_shutdown
+        when 'HUP'
+          handle_reload
+        when 'USR1'
+          handle_log_rotation
+        when 'USR2'
+          handle_debug_dump
+        end
+      end
+
+      def handle_shutdown
+        logger.info 'Initiating graceful shutdown...'
+        daemon.shutdown
+      end
+
+      def handle_reload
+        logger.info 'Reloading agent configuration...'
+        # TODO: Implement reload
+        # 1. Query for new/removed agents
+        # 2. Send HUP to workers for them to reload
+        daemon.worker_pids.each do |pid|
+          Process.kill('HUP', pid)
+        rescue Errno::ESRCH
+          # Worker already dead
+        end
+      end
+
+      def handle_log_rotation
+        logger.info 'Rotating log files...'
+
+        # Reopen stdout/stderr if they're files
+        if $stdout.respond_to?(:path) && $stdout.path
+          $stdout.reopen($stdout.path, 'a')
+          $stdout.sync = true
+        end
+
+        if $stderr.respond_to?(:path) && $stderr.path
+          $stderr.reopen($stderr.path, 'a')
+          $stderr.sync = true
+        end
+
+        # Signal workers to rotate their logs too
+        daemon.worker_pids.each do |pid|
+          Process.kill('USR1', pid)
+        rescue Errno::ESRCH
+          # Worker already dead
+        end
+      end
+
+      def handle_debug_dump
+        logger.info 'Dumping debug information...'
+
+        # Dump current state
+        status = daemon.status
+        logger.info "Status: #{status.inspect}"
+
+        # Dump thread backtraces
+        Thread.list.each do |thread|
+          logger.info "Thread: #{thread.name || thread.object_id}"
+          logger.info thread.backtrace&.join("\n") || '(no backtrace)'
+          logger.info '---'
+        end
+      end
+
+      def logger
+        ActiveMatrix.logger
+      end
+    end
+  end
+end

data/lib/active_matrix/daemon/worker.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module ActiveMatrix
+  class Daemon
+    # Worker process that runs a subset of agents
+    #
+    # Each worker is a forked child process that:
+    # - Initializes its own AgentManager
+    # - Runs only assigned agents (by ID)
+    # - Handles signals for graceful shutdown
+    #
+    class Worker
+      attr_reader :index, :agent_ids
+
+      def initialize(index:, agent_ids:)
+        @index = index
+        @agent_ids = agent_ids
+        @running = false
+      end
+
+      def run
+        @running = true
+
+        set_process_name
+        install_signal_handlers
+        reconnect_database
+
+        logger.info "Worker #{index} starting with agents: #{agent_ids.join(', ')}"
+
+        run_agents
+      rescue StandardError => e
+        logger.error "Worker #{index} crashed: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        raise
+      ensure
+        logger.info "Worker #{index} exiting"
+      end
+
+      private
+
+      def set_process_name
+        Process.setproctitle("activematrix[#{index}]: #{agent_ids.size} agents")
+      end
+
+      def install_signal_handlers
+        Signal.trap('TERM') do
+          @running = false
+          AgentManager.instance.stop_all
+        end
+
+        Signal.trap('INT') do
+          @running = false
+          AgentManager.instance.stop_all
+        end
+
+        Signal.trap('HUP') do
+          # Reload - restart agents with new config
+          logger.info "Worker #{index} received HUP, reloading..."
+          # For now, just log. Full reload would require:
+          # 1. Stop all agents
+          # 2. Re-query DB for agent list
+          # 3. Start new agents
+        end
+
+        Signal.trap('USR1') do
+          # Log rotation
+          if $stdout.respond_to?(:path) && $stdout.path
+            $stdout.reopen($stdout.path, 'a')
+            $stdout.sync = true
+          end
+          if $stderr.respond_to?(:path) && $stderr.path
+            $stderr.reopen($stderr.path, 'a')
+            $stderr.sync = true
+          end
+        end
+      end
+
+      def reconnect_database
+        # After fork, we need fresh database connections
+        ActiveRecord::Base.connection_handler.clear_active_connections!
+        ActiveRecord::Base.establish_connection
+      end
+
+      def run_agents
+        manager = AgentManager.instance
+
+        # Install signal handlers for the manager
+        manager.install_signal_handlers!
+
+        # Load only our assigned agents
+        agents = ActiveMatrix::Agent.where(id: agent_ids)
+
+        if agents.empty?
+          logger.warn "Worker #{index} has no agents to run"
+          return
+        end
+
+        # Start the manager with only our agents
+        Sync do
+          manager.start_agents(agents)
+        end
+      end
+
+      def logger
+        ActiveMatrix.logger
+      end
+    end
+  end
+end