activematrix 0.0.5 → 0.0.8

data/lib/active_matrix/daemon/signal_handler.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module ActiveMatrix
+  class Daemon
+    # Handles Unix signals for the daemon coordinator
+    #
+    # Signals:
+    # - TERM/INT: Graceful shutdown
+    # - HUP: Reload configuration and restart agents
+    # - USR1: Log rotation (reopen log files)
+    # - USR2: Dump debug information
+    #
+    class SignalHandler
+      SIGNALS = %w[TERM INT HUP USR1 USR2].freeze
+
+      attr_reader :daemon
+
+      def initialize(daemon)
+        @daemon = daemon
+        @self_pipe_reader, @self_pipe_writer = IO.pipe
+        @old_handlers = {}
+      end
+
+      def install
+        SIGNALS.each do |signal|
+          install_handler(signal)
+        end
+
+        # Start signal processing thread
+        start_processor
+      end
+
+      def uninstall
+        SIGNALS.each do |signal|
+          restore_handler(signal)
+        end
+
+        @self_pipe_writer.close
+        @self_pipe_reader.close
+      end
+
+      private
+
+      def install_handler(signal)
+        @old_handlers[signal] = Signal.trap(signal) do
+          # Write signal to pipe (non-blocking, safe in signal handler)
+          @self_pipe_writer.write_nonblock("#{signal}\n")
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK
+          # Pipe full, signal will be coalesced
+        end
+      rescue ArgumentError
+        # Signal not supported on this platform
+        logger.debug "Signal #{signal} not supported"
+      end
+
+      def restore_handler(signal)
+        return unless @old_handlers.key?(signal)
+
+        Signal.trap(signal, @old_handlers[signal])
+      end
+
+      def start_processor
+        Thread.new do
+          Thread.current.name = 'activematrix-signal-processor'
+
+          loop do
+            # rubocop:disable Lint/IncompatibleIoSelectWithFiberScheduler
+            ready = IO.select([@self_pipe_reader], nil, nil, 1)
+            # rubocop:enable Lint/IncompatibleIoSelectWithFiberScheduler
+            next unless ready
+
+            signal = @self_pipe_reader.gets&.strip
+            next unless signal
+
+            handle_signal(signal)
+          rescue IOError
+            break
+          end
+        end
+      end
+
+      def handle_signal(signal)
+        logger.info "Received signal: #{signal}"
+
+        case signal
+        when 'TERM', 'INT'
+          handle_shutdown
+        when 'HUP'
+          handle_reload
+        when 'USR1'
+          handle_log_rotation
+        when 'USR2'
+          handle_debug_dump
+        end
+      end
+
+      def handle_shutdown
+        logger.info 'Initiating graceful shutdown...'
+        daemon.shutdown
+      end
+
+      def handle_reload
+        logger.info 'Reloading agent configuration...'
+        # TODO: Implement reload
+        # 1. Query for new/removed agents
+        # 2. Send HUP to workers for them to reload
+        daemon.worker_pids.each do |pid|
+          Process.kill('HUP', pid)
+        rescue Errno::ESRCH
+          # Worker already dead
+        end
+      end
+
+      def handle_log_rotation
+        logger.info 'Rotating log files...'
+
+        # Reopen stdout/stderr if they're files
+        if $stdout.respond_to?(:path) && $stdout.path
+          $stdout.reopen($stdout.path, 'a')
+          $stdout.sync = true
+        end
+
+        if $stderr.respond_to?(:path) && $stderr.path
+          $stderr.reopen($stderr.path, 'a')
+          $stderr.sync = true
+        end
+
+        # Signal workers to rotate their logs too
+        daemon.worker_pids.each do |pid|
+          Process.kill('USR1', pid)
+        rescue Errno::ESRCH
+          # Worker already dead
+        end
+      end
+
+      def handle_debug_dump
+        logger.info 'Dumping debug information...'
+
+        # Dump current state
+        status = daemon.status
+        logger.info "Status: #{status.inspect}"
+
+        # Dump thread backtraces
+        Thread.list.each do |thread|
+          logger.info "Thread: #{thread.name || thread.object_id}"
+          logger.info thread.backtrace&.join("\n") || '(no backtrace)'
+          logger.info '---'
+        end
+      end
+
+      def logger
+        ActiveMatrix.logger
+      end
+    end
+  end
+end

data/lib/active_matrix/daemon/worker.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module ActiveMatrix
+  class Daemon
+    # Worker process that runs a subset of agents
+    #
+    # Each worker is a forked child process that:
+    # - Initializes its own AgentManager
+    # - Runs only assigned agents (by ID)
+    # - Handles signals for graceful shutdown
+    #
+    class Worker
+      attr_reader :index, :agent_ids
+
+      def initialize(index:, agent_ids:)
+        @index = index
+        @agent_ids = agent_ids
+        @running = false
+      end
+
+      def run
+        @running = true
+
+        set_process_name
+        install_signal_handlers
+        reconnect_database
+
+        logger.info "Worker #{index} starting with agents: #{agent_ids.join(', ')}"
+
+        run_agents
+      rescue StandardError => e
+        logger.error "Worker #{index} crashed: #{e.message}"
+        logger.error e.backtrace.join("\n")
+        raise
+      ensure
+        logger.info "Worker #{index} exiting"
+      end
+
+      private
+
+      def set_process_name
+        Process.setproctitle("activematrix[#{index}]: #{agent_ids.size} agents")
+      end
+
+      def install_signal_handlers
+        Signal.trap('TERM') do
+          @running = false
+          AgentManager.instance.stop_all
+        end
+
+        Signal.trap('INT') do
+          @running = false
+          AgentManager.instance.stop_all
+        end
+
+        Signal.trap('HUP') do
+          # Reload - restart agents with new config
+          logger.info "Worker #{index} received HUP, reloading..."
+          # For now, just log. Full reload would require:
+          # 1. Stop all agents
+          # 2. Re-query DB for agent list
+          # 3. Start new agents
+        end
+
+        Signal.trap('USR1') do
+          # Log rotation
+          if $stdout.respond_to?(:path) && $stdout.path
+            $stdout.reopen($stdout.path, 'a')
+            $stdout.sync = true
+          end
+          if $stderr.respond_to?(:path) && $stderr.path
+            $stderr.reopen($stderr.path, 'a')
+            $stderr.sync = true
+          end
+        end
+      end
+
+      def reconnect_database
+        # After fork, we need fresh database connections
+        ActiveRecord::Base.connection_handler.clear_active_connections!
+        ActiveRecord::Base.establish_connection
+      end
+
+      def run_agents
+        manager = AgentManager.instance
+
+        # Install signal handlers for the manager
+        manager.install_signal_handlers!
+
+        # Load only our assigned agents
+        agents = ActiveMatrix::Agent.where(id: agent_ids)
+
+        if agents.empty?
+          logger.warn "Worker #{index} has no agents to run"
+          return
+        end
+
+        # Start the manager with only our agents
+        Sync do
+          manager.start_agents(agents)
+        end
+      end
+
+      def logger
+        ActiveMatrix.logger
+      end
+    end
+  end
+end

data/lib/active_matrix/daemon.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require_relative 'daemon/signal_handler'
+require_relative 'daemon/probe_server'
+require_relative 'daemon/worker'
+require_relative 'telemetry'
+
+module ActiveMatrix
+  # Main daemon coordinator for managing Matrix bot agents
+  #
+  # Responsibilities:
+  # - Fork and manage worker processes
+  # - Distribute agents across workers
+  # - Handle signals (TERM, INT, HUP, USR1, USR2)
+  # - Run HTTP health probe server
+  # - Monitor worker health and restart on crash
+  #
+  class Daemon
+    attr_reader :workers_count, :probe_port, :probe_host, :agent_names, :start_time
+
+    def initialize(workers: 1, probe_port: 3042, probe_host: '127.0.0.1', agent_names: nil)
+      @workers_count = workers
+      @probe_port = probe_port
+      @probe_host = probe_host
+      @agent_names = agent_names
+      @workers = {} # { pid => { index:, agent_ids: } }
+      @running = false
+      @start_time = nil
+    end
+
+    def worker_pids
+      @workers.keys
+    end
+
+    def run
+      @start_time = Time.zone.now
+      @running = true
+
+      logger.info "Starting ActiveMatrix daemon (workers: #{workers_count}, probe: #{probe_host}:#{probe_port})"
+
+      # Initialize OpenTelemetry if available
+      logger.info 'OpenTelemetry tracing enabled' if Telemetry.configure!
+
+      install_signal_handlers
+      start_probe_server
+      start_workers
+
+      monitor_loop
+    ensure
+      shutdown
+    end
+
+    def shutdown
+      return unless @running
+
+      @running = false
+      logger.info 'Shutting down ActiveMatrix daemon...'
+
+      stop_probe_server
+      stop_workers
+
+      Telemetry.shutdown
+
+      logger.info 'ActiveMatrix daemon stopped'
+    end
+
+    def status
+      {
+        status: @running ? 'ok' : 'stopping',
+        uptime: @start_time ? (Time.zone.now - @start_time).to_i : 0,
+        workers: @workers.size,
+        agents: aggregate_agent_status
+      }
+    end
+
+    private
+
+    def logger
+      ActiveMatrix.logger
+    end
+
+    def install_signal_handlers
+      @signal_handler = SignalHandler.new(self)
+      @signal_handler.install
+    end
+
+    def start_probe_server
+      @probe_server = ProbeServer.new(
+        host: probe_host,
+        port: probe_port,
+        daemon: self
+      )
+      @probe_server.start
+    end
+
+    def stop_probe_server
+      @probe_server&.stop
+    end
+
+    def start_workers
+      agents = load_agents
+      agent_groups = distribute_agents(agents, workers_count)
+
+      agent_groups.each_with_index do |agent_ids, index|
+        next if agent_ids.empty?
+
+        pid = fork_worker(index, agent_ids)
+        @workers[pid] = { index: index, agent_ids: agent_ids }
+        logger.info "Started worker #{index} (PID: #{pid}) with #{agent_ids.size} agents"
+      end
+    end
+
+    def fork_worker(index, agent_ids)
+      fork do
+        # Reset signal handlers in child
+        Signal.trap('TERM') { exit } # rubocop:disable Rails/Exit
+        Signal.trap('INT') { exit } # rubocop:disable Rails/Exit
+
+        # Close parent's probe server socket
+        @probe_server&.stop
+
+        # Run worker
+        worker = Worker.new(index: index, agent_ids: agent_ids)
+        worker.run
+      end
+    end
+
+    def stop_workers
+      timeout = ActiveMatrix.config.shutdown_timeout || 30
+
+      # Send TERM to all workers
+      @workers.each_key do |pid|
+        Process.kill('TERM', pid)
+      rescue Errno::ESRCH
+        # Already dead
+      end
+
+      # Wait for graceful shutdown
+      deadline = Time.zone.now + timeout
+      while @workers.any? && Time.zone.now < deadline
+        @workers.reject! do |pid, _|
+          Process.waitpid(pid, Process::WNOHANG)
+        rescue Errno::ECHILD
+          true
+        end
+        sleep 0.5 if @workers.any?
+      end
+
+      # Force kill remaining
+      @workers.each_key do |pid|
+        logger.warn "Force killing worker #{pid}"
+        Process.kill('KILL', pid)
+        Process.waitpid(pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Already dead
+      end
+
+      @workers.clear
+    end
+
+    def monitor_loop
+      while @running
+        # Reap dead children
+        reap_workers
+
+        # Restart crashed workers
+        restart_crashed_workers if @running
+
+        sleep 1
+      end
+    end
+
+    def reap_workers
+      @crashed_workers = []
+
+      loop do
+        pid = Process.waitpid(-1, Process::WNOHANG)
+        break unless pid
+
+        if @workers.key?(pid)
+          worker_info = @workers.delete(pid)
+          logger.warn "Worker #{worker_info[:index]} (PID: #{pid}) exited"
+          @crashed_workers << worker_info
+        end
+      rescue Errno::ECHILD
+        break
+      end
+    end
+
+    def restart_crashed_workers
+      return if @crashed_workers.blank?
+
+      @crashed_workers.each do |worker_info|
+        break unless @running
+
+        # Restart with same agent assignment
+        pid = fork_worker(worker_info[:index], worker_info[:agent_ids])
+        @workers[pid] = worker_info.merge(pid: pid)
+        logger.info "Restarted worker #{worker_info[:index]} (PID: #{pid}) with #{worker_info[:agent_ids].size} agents"
+      end
+
+      @crashed_workers.clear
+    end
+
+    def load_agents
+      scope = ActiveMatrix::Agent.where.not(state: :offline)
+      scope = scope.where(name: agent_names) if agent_names.present?
+      scope.pluck(:id)
+    end
+
+    def distribute_agents(agent_ids, num_workers)
+      return [agent_ids] if num_workers <= 1
+
+      # Round-robin distribution
+      groups = Array.new(num_workers) { [] }
+      agent_ids.each_with_index do |id, index|
+        groups[index % num_workers] << id
+      end
+      groups
+    end
+
+    def aggregate_agent_status
+      # In multi-process mode, we'd need IPC to get real status
+      # For now, query the database
+      agents = ActiveMatrix::Agent.all
+
+      {
+        total: agents.count,
+        online: agents.where(state: %i[online_idle online_busy]).count,
+        connecting: agents.where(state: :connecting).count,
+        error: agents.where(state: :error).count,
+        offline: agents.where(state: :offline).count
+      }
+    end
+  end
+end

data/lib/active_matrix/engine.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'rails/engine'
+
+module ActiveMatrix
+  class Engine < Rails::Engine
+    engine_name 'activematrix'
+
+    initializer 'activematrix.configure_logger' do
+      ActiveMatrix.logger = Rails.logger
+    end
+
+    initializer 'activematrix.initialize_metrics', after: 'activematrix.configure_logger' do
+      # Eagerly initialize Metrics singleton to subscribe to notifications
+      ActiveMatrix::Metrics.instance
+    end
+  end
+end

data/lib/active_matrix/errors.rb

@@ -54,7 +54,7 @@ module ActiveMatrix
     # An error raised when errors occur in the connection layer
     class MatrixConnectionError < MatrixError
       def self.class_by_code(code)
-        return MatrixTimeoutError if code == 504
+        return MatrixTimeoutError if code.to_i == 504
 
         MatrixConnectionError
       end