activematrix 0.0.7 → 0.0.8

data/lib/active_matrix/daemon.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require_relative 'daemon/signal_handler'
+require_relative 'daemon/probe_server'
+require_relative 'daemon/worker'
+require_relative 'telemetry'
+
+module ActiveMatrix
+  # Main daemon coordinator for managing Matrix bot agents
+  #
+  # Responsibilities:
+  # - Fork and manage worker processes
+  # - Distribute agents across workers
+  # - Handle signals (TERM, INT, HUP, USR1, USR2)
+  # - Run HTTP health probe server
+  # - Monitor worker health and restart on crash
+  #
+  class Daemon
+    attr_reader :workers_count, :probe_port, :probe_host, :agent_names, :start_time
+
+    def initialize(workers: 1, probe_port: 3042, probe_host: '127.0.0.1', agent_names: nil)
+      @workers_count = workers
+      @probe_port = probe_port
+      @probe_host = probe_host
+      @agent_names = agent_names
+      @workers = {} # { pid => { index:, agent_ids: } }
+      @running = false
+      @start_time = nil
+    end
+
+    def worker_pids
+      @workers.keys
+    end
+
+    def run
+      @start_time = Time.zone.now
+      @running = true
+
+      logger.info "Starting ActiveMatrix daemon (workers: #{workers_count}, probe: #{probe_host}:#{probe_port})"
+
+      # Initialize OpenTelemetry if available
+      logger.info 'OpenTelemetry tracing enabled' if Telemetry.configure!
+
+      install_signal_handlers
+      start_probe_server
+      start_workers
+
+      monitor_loop
+    ensure
+      shutdown
+    end
+
+    def shutdown
+      return unless @running
+
+      @running = false
+      logger.info 'Shutting down ActiveMatrix daemon...'
+
+      stop_probe_server
+      stop_workers
+
+      Telemetry.shutdown
+
+      logger.info 'ActiveMatrix daemon stopped'
+    end
+
+    def status
+      {
+        status: @running ? 'ok' : 'stopping',
+        uptime: @start_time ? (Time.zone.now - @start_time).to_i : 0,
+        workers: @workers.size,
+        agents: aggregate_agent_status
+      }
+    end
+
+    private
+
+    def logger
+      ActiveMatrix.logger
+    end
+
+    def install_signal_handlers
+      @signal_handler = SignalHandler.new(self)
+      @signal_handler.install
+    end
+
+    def start_probe_server
+      @probe_server = ProbeServer.new(
+        host: probe_host,
+        port: probe_port,
+        daemon: self
+      )
+      @probe_server.start
+    end
+
+    def stop_probe_server
+      @probe_server&.stop
+    end
+
+    def start_workers
+      agents = load_agents
+      agent_groups = distribute_agents(agents, workers_count)
+
+      agent_groups.each_with_index do |agent_ids, index|
+        next if agent_ids.empty?
+
+        pid = fork_worker(index, agent_ids)
+        @workers[pid] = { index: index, agent_ids: agent_ids }
+        logger.info "Started worker #{index} (PID: #{pid}) with #{agent_ids.size} agents"
+      end
+    end
+
+    def fork_worker(index, agent_ids)
+      fork do
+        # Reset signal handlers in child
+        Signal.trap('TERM') { exit } # rubocop:disable Rails/Exit
+        Signal.trap('INT') { exit } # rubocop:disable Rails/Exit
+
+        # Close parent's probe server socket
+        @probe_server&.stop
+
+        # Run worker
+        worker = Worker.new(index: index, agent_ids: agent_ids)
+        worker.run
+      end
+    end
+
+    def stop_workers
+      timeout = ActiveMatrix.config.shutdown_timeout || 30
+
+      # Send TERM to all workers
+      @workers.each_key do |pid|
+        Process.kill('TERM', pid)
+      rescue Errno::ESRCH
+        # Already dead
+      end
+
+      # Wait for graceful shutdown
+      deadline = Time.zone.now + timeout
+      while @workers.any? && Time.zone.now < deadline
+        @workers.reject! do |pid, _|
+          Process.waitpid(pid, Process::WNOHANG)
+        rescue Errno::ECHILD
+          true
+        end
+        sleep 0.5 if @workers.any?
+      end
+
+      # Force kill remaining
+      @workers.each_key do |pid|
+        logger.warn "Force killing worker #{pid}"
+        Process.kill('KILL', pid)
+        Process.waitpid(pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Already dead
+      end
+
+      @workers.clear
+    end
+
+    def monitor_loop
+      while @running
+        # Reap dead children
+        reap_workers
+
+        # Restart crashed workers
+        restart_crashed_workers if @running
+
+        sleep 1
+      end
+    end
+
+    def reap_workers
+      @crashed_workers = []
+
+      loop do
+        pid = Process.waitpid(-1, Process::WNOHANG)
+        break unless pid
+
+        if @workers.key?(pid)
+          worker_info = @workers.delete(pid)
+          logger.warn "Worker #{worker_info[:index]} (PID: #{pid}) exited"
+          @crashed_workers << worker_info
+        end
+      rescue Errno::ECHILD
+        break
+      end
+    end
+
+    def restart_crashed_workers
+      return if @crashed_workers.blank?
+
+      @crashed_workers.each do |worker_info|
+        break unless @running
+
+        # Restart with same agent assignment
+        pid = fork_worker(worker_info[:index], worker_info[:agent_ids])
+        @workers[pid] = worker_info.merge(pid: pid)
+        logger.info "Restarted worker #{worker_info[:index]} (PID: #{pid}) with #{worker_info[:agent_ids].size} agents"
+      end
+
+      @crashed_workers.clear
+    end
+
+    def load_agents
+      scope = ActiveMatrix::Agent.where.not(state: :offline)
+      scope = scope.where(name: agent_names) if agent_names.present?
+      scope.pluck(:id)
+    end
+
+    def distribute_agents(agent_ids, num_workers)
+      return [agent_ids] if num_workers <= 1
+
+      # Round-robin distribution
+      groups = Array.new(num_workers) { [] }
+      agent_ids.each_with_index do |id, index|
+        groups[index % num_workers] << id
+      end
+      groups
+    end
+
+    def aggregate_agent_status
+      # In multi-process mode, we'd need IPC to get real status
+      # For now, query the database
+      agents = ActiveMatrix::Agent.all
+
+      {
+        total: agents.count,
+        online: agents.where(state: %i[online_idle online_busy]).count,
+        connecting: agents.where(state: :connecting).count,
+        error: agents.where(state: :error).count,
+        offline: agents.where(state: :offline).count
+      }
+    end
+  end
+end

data/lib/active_matrix/engine.rb

@@ -7,8 +7,12 @@ module ActiveMatrix
     engine_name 'activematrix'
 
     initializer 'activematrix.configure_logger' do
-      # Configure logger
       ActiveMatrix.logger = Rails.logger
     end
+
+    initializer 'activematrix.initialize_metrics', after: 'activematrix.configure_logger' do
+      # Eagerly initialize Metrics singleton to subscribe to notifications
+      ActiveMatrix::Metrics.instance
+    end
   end
 end

data/lib/active_matrix/errors.rb

@@ -54,7 +54,7 @@ module ActiveMatrix
     # An error raised when errors occur in the connection layer
     class MatrixConnectionError < MatrixError
       def self.class_by_code(code)
-        return MatrixTimeoutError if code == 504
+        return MatrixTimeoutError if code.to_i == 504
 
         MatrixConnectionError
       end

data/lib/active_matrix/event_router.rb

@@ -1,19 +1,21 @@
 # frozen_string_literal: true
 
 require 'singleton'
-require 'concurrent'
+require 'async'
+require 'async/queue'
 
 module ActiveMatrix
-  # Routes Matrix events to appropriate agents
+  # Routes Matrix events to appropriate agents using async fibers
   class EventRouter
     include Singleton
     include ActiveMatrix::Logging
 
     def initialize
-      @routes = Concurrent::Array.new
-      @event_queue = Queue.new
+      @routes = []
+      @mutex = Mutex.new
+      @event_queue = nil
       @processing = false
-      @worker_thread = nil
+      @worker_task = nil
     end
 
     # Register an event route
@@ -28,8 +30,10 @@ module ActiveMatrix
         handler: block
       }
 
-      @routes << route
-      @routes.sort_by! { |r| -r[:priority] } # Higher priority first
+      @mutex.synchronize do
+        @routes << route
+        @routes.sort_by! { |r| -r[:priority] } # Higher priority first
+      end
 
       logger.debug "Registered route: #{route.except(:handler).inspect}"
       route[:id]
@@ -37,29 +41,35 @@ module ActiveMatrix
 
     # Unregister a route
     def unregister_route(route_id)
-      @routes.delete_if { |route| route[:id] == route_id }
+      @mutex.synchronize do
+        @routes.delete_if { |route| route[:id] == route_id }
+      end
     end
 
     # Clear all routes for an agent
     def clear_agent_routes(agent_id)
-      @routes.delete_if { |route| route[:agent_id] == agent_id }
+      @mutex.synchronize do
+        @routes.delete_if { |route| route[:agent_id] == agent_id }
+      end
     end
 
     # Route an event to appropriate agents
     def route_event(event)
-      return unless @processing
+      return unless @processing && @event_queue
 
       # Queue the event for processing
-      @event_queue << event
+      @event_queue.enqueue(event)
     end
 
-    # Start the event router
+    # Start the event router (call from within async context)
     def start
       return if @processing
 
       @processing = true
-      @worker_thread = Thread.new do
-        Thread.current.name = 'event-router'
+      @event_queue = Async::Queue.new
+
+      @worker_task = Async(transient: true) do |task|
+        task.annotate 'event-router'
         process_events
       end
 
@@ -69,20 +79,22 @@ module ActiveMatrix
     # Stop the event router
     def stop
       @processing = false
-      @worker_thread&.kill
-      @event_queue.clear
+      @worker_task&.stop
+      @event_queue = nil
 
       logger.info 'Event router stopped'
     end
 
     # Check if router is running
     def running?
-      @processing && @worker_thread&.alive?
+      @processing && @worker_task&.alive?
     end
 
     # Get routes for debugging
     def routes_summary
-      @routes.map { |r| r.except(:handler) }
+      @mutex.synchronize do
+        @routes.map { |r| r.except(:handler) }
+      end
     end
 
     # Broadcast an event to all agents
@@ -99,9 +111,7 @@ module ActiveMatrix
     def process_events
       while @processing
         begin
-          # Wait for event with timeout
-          event = nil
-          Timeout.timeout(1) { event = @event_queue.pop }
+          event = @event_queue.dequeue
 
           next unless event
 
@@ -113,36 +123,40 @@ module ActiveMatrix
             next
           end
 
-          # Process routes in priority order
+          # Process routes in priority order (each in its own fiber)
           matching_routes.each do |route|
-            process_route(route, event)
+            Async do
+              process_route(route, event)
+            end
           end
-        rescue Timeout::Error
-          # Normal timeout, continue loop
+        rescue Async::Stop
+          break
         rescue StandardError => e
           logger.error "Event router error: #{e.message}"
-          logger.error e.backtrace.join("\n")
+          logger.error e.backtrace.first(10).join("\n")
         end
       end
     end
 
     def find_matching_routes(event)
-      @routes.select do |route|
-        # Check room match
-        next false if route[:room_id] && route[:room_id] != event[:room_id]
+      @mutex.synchronize do
+        @routes.select do |route|
+          # Check room match
+          next false if route[:room_id] && route[:room_id] != event[:room_id]
 
-        # Check event type match
-        next false if route[:event_type] && route[:event_type] != event[:type]
+          # Check event type match
+          next false if route[:event_type] && route[:event_type] != event[:type]
 
-        # Check user match
-        next false if route[:user_id] && route[:user_id] != event[:sender]
+          # Check user match
+          next false if route[:user_id] && route[:user_id] != event[:sender]
 
-        # Check if agent is running
-        registry = AgentRegistry.instance
-        agent_entry = registry.get(route[:agent_id])
-        next false unless agent_entry
+          # Check if agent is running
+          registry = AgentRegistry.instance
+          agent_entry = registry.get(route[:agent_id])
+          next false unless agent_entry
 
-        true
+          true
+        end
       end
     end
 
@@ -154,18 +168,16 @@ module ActiveMatrix
 
       bot = agent_entry[:instance]
 
-      begin
-        if route[:handler]
-          # Custom handler
-          route[:handler].call(bot, event)
-        elsif bot.respond_to?(:_handle_event)
-          # Default handling
-          bot._handle_event(event)
-        end
-      rescue StandardError => e
-        logger.error "Error processing route for agent #{agent_entry[:record].name}: #{e.message}"
-        logger.error e.backtrace.first(5).join("\n")
+      if route[:handler]
+        # Custom handler
+        route[:handler].call(bot, event)
+      elsif bot.respond_to?(:_handle_event)
+        # Default handling
+        bot._handle_event(event)
       end
+    rescue StandardError => e
+      logger.error "Error processing route for agent #{agent_entry[:record].name}: #{e.message}"
+      logger.error e.backtrace.first(5).join("\n")
     end
   end
 

data/lib/active_matrix/events.rb

@@ -3,6 +3,7 @@
 module ActiveMatrix
   class EventHandlerArray < Hash
     include ActiveMatrix::Logging
+
     attr_accessor :reraise_exceptions
 
     def initialize(*)

data/lib/active_matrix/instrumentation.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+require 'active_support/notifications'
+require 'timeout'
+require 'socket'
+require 'json'
+
+module ActiveMatrix
+  # Instrumentation module for Matrix bot operations
+  # Provides ActiveSupport::Notifications events and structured logging
+  #
+  # @example Include in a class
+  #   class MyService
+  #     include ActiveMatrix::Instrumentation
+  #
+  #     def perform
+  #       instrument_operation(:my_operation, room_id: '!abc:matrix.org') do
+  #         # ... operation code
+  #       end
+  #     end
+  #   end
+  #
+  module Instrumentation
+    extend ActiveSupport::Concern
+
+    private
+
+    # Instrument a Matrix bot operation with timing and result tracking
+    #
+    # @param operation [Symbol, String] Operation name (e.g., :send_message, :sync)
+    # @param metadata [Hash] Additional context to include in the event
+    # @yield Block to execute and instrument
+    # @return [Object] Result of the block
+    # @raise [StandardError] Re-raises any exception after logging
+    def instrument_operation(operation, **metadata)
+      event_data = metadata.merge(
+        operation: operation,
+        agent_id: respond_to?(:agent_id) ? agent_id : nil,
+        component: self.class.name&.demodulize || 'Unknown'
+      )
+
+      ActiveSupport::Notifications.instrument(
+        "activematrix.#{operation}",
+        event_data
+      ) do |payload|
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        begin
+          result = yield
+
+          payload[:status] = 'success'
+          payload[:duration_ms] = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(2)
+          payload[:result] = summarize_result(result)
+
+          log_operation_result(operation, 'SUCCESS', payload)
+
+          result
+        rescue StandardError => e
+          payload[:status] = 'error'
+          payload[:duration_ms] = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(2)
+          payload[:error_class] = e.class.name
+          payload[:error_message] = e.message
+          payload[:error_category] = classify_error(e)
+
+          log_operation_result(operation, 'ERROR', payload)
+
+          raise
+        end
+      end
+    end
+
+    # Classify errors for better monitoring and alerting
+    #
+    # @param error [StandardError] The error to classify
+    # @return [String] Error category
+    def classify_error(error)
+      case error
+      when Timeout::Error
+        'timeout'
+      when SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH
+        'network'
+      when JSON::ParserError
+        'parse'
+      when OpenSSL::SSL::SSLError
+        'ssl'
+      else
+        case error.class.name
+        when /ActiveMatrix::Errors::MatrixConnectionError/
+          'matrix_connection'
+        when /ActiveMatrix::Errors::MatrixRequestError/
+          'matrix_request'
+        when /ActiveMatrix::Errors::MatrixNotAuthorizedError/
+          'matrix_auth'
+        when /ActiveMatrix::Errors::MatrixForbiddenError/
+          'matrix_forbidden'
+        when /ActiveMatrix::Errors::MatrixNotFoundError/
+          'matrix_not_found'
+        when /PG::/
+          'database'
+        else
+          'application'
+        end
+      end
+    end
+
+    # Log operation result with structured data
+    #
+    # @param operation [Symbol, String] Operation name
+    # @param status [String] 'SUCCESS' or 'ERROR'
+    # @param data [Hash] Event payload data
+    def log_operation_result(operation, status, data)
+      component = data[:component] || 'Unknown'
+      agent_id = data[:agent_id] || 'unknown'
+
+      message = "#{operation} - #{status}"
+      message += " (#{data[:duration_ms]}ms)" if data[:duration_ms]
+      message = "[#{component}][agent:#{agent_id}] #{message}"
+
+      if status == 'ERROR'
+        ActiveMatrix.logger.error("#{message}: #{data[:error_class]} - #{data[:error_message]}")
+      else
+        ActiveMatrix.logger.debug(message)
+      end
+    end
+
+    # Summarize result for logging without exposing sensitive data
+    #
+    # @param result [Object] The result to summarize
+    # @return [String] Human-readable summary
+    def summarize_result(result)
+      case result
+      when String
+        result.length > 100 ? "#{result[0...97]}..." : result
+      when Numeric, true, false
+        result.to_s
+      when nil
+        'nil'
+      when Hash
+        "Hash(#{result.keys.size} keys)"
+      when Array
+        "Array(#{result.size} items)"
+      else
+        result.class.name
+      end
+    end
+  end
+end

data/lib/active_matrix/memory/agent_memory.rb

@@ -14,19 +14,15 @@ module ActiveMatrix
       # Get a value from agent memory
       def get(key)
         fetch_with_cache(key) do
-          return nil unless defined?(::AgentMemory)
-
-          memory = @agent.agent_memories.active.find_by(key: key)
+          memory = @agent.agent_stores.active.find_by(key: key)
           memory&.value
         end
       end
 
       # Set a value in agent memory
       def set(key, value, expires_in: nil)
-        return false unless defined?(::AgentMemory)
-
         write_through(key, value, expires_in: expires_in) do
-          memory = @agent.agent_memories.find_or_initialize_by(key: key)
+          memory = @agent.agent_stores.find_or_initialize_by(key: key)
           memory.value = value
           memory.expires_at = expires_in.present? ? Time.current + expires_in : nil
           memory.save!
@@ -35,43 +31,33 @@ module ActiveMatrix
 
       # Check if a key exists
       def exists?(key)
-        return false unless defined?(::AgentMemory)
-
         if @cache_enabled && Rails.cache.exist?(cache_key(key))
           true
         else
-          @agent.agent_memories.active.exists?(key: key)
+          @agent.agent_stores.active.exists?(key: key)
         end
       end
 
       # Delete a key
       def delete(key)
-        return false unless defined?(::AgentMemory)
-
         delete_through(key) do
-          @agent.agent_memories.where(key: key).destroy_all.any?
+          @agent.agent_stores.where(key: key).destroy_all.any?
         end
       end
 
       # Get all keys
       def keys
-        return [] unless defined?(::AgentMemory)
-
-        @agent.agent_memories.active.pluck(:key)
+        AsyncQuery.async_pluck(@agent.agent_stores.active, :key)
       end
 
       # Get all memory as hash
       def all
-        return {} unless defined?(::AgentMemory)
-
-        @agent.agent_memories.active.pluck(:key, :value).to_h
+        AsyncQuery.async_pluck(@agent.agent_stores.active, :key, :value).to_h
       end
 
       # Clear all agent memory
       def clear!
-        return false unless defined?(::AgentMemory)
-
-        @agent.agent_memories.destroy_all
+        @agent.agent_stores.destroy_all
 
         # Clear cache entries
         keys.each { |key| Rails.cache.delete(cache_key(key)) } if @cache_enabled