decision_agent 0.1.2 → 0.1.3

data/lib/decision_agent/monitoring/dashboard_server.rb

@@ -0,0 +1,340 @@
+require "sinatra/base"
+require "json"
+
+# Faye/WebSocket is optional for real-time features
+begin
+  require "faye/websocket"
+  WEBSOCKET_AVAILABLE = true
+rescue LoadError
+  WEBSOCKET_AVAILABLE = false
+  warn "Warning: faye-websocket gem not found. Real-time dashboard features will be disabled."
+  warn "Install with: gem install faye-websocket"
+end
+
+module DecisionAgent
+  module Monitoring
+    # Real-time monitoring dashboard server
+    class DashboardServer < Sinatra::Base
+      set :public_folder, File.expand_path("dashboard/public", __dir__)
+      set :views, File.expand_path("dashboard/views", __dir__)
+      set :bind, "0.0.0.0"
+      set :port, 4568
+      set :server, :puma
+
+      # Enable CORS
+      before do
+        headers["Access-Control-Allow-Origin"] = "*"
+        headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
+        headers["Access-Control-Allow-Headers"] = "Content-Type"
+      end
+
+      options "*" do
+        200
+      end
+
+      # Class-level configuration
+      class << self
+        attr_accessor :metrics_collector, :prometheus_exporter, :alert_manager
+        attr_reader :websocket_clients
+
+        def configure_monitoring(metrics_collector:, prometheus_exporter:, alert_manager:)
+          @metrics_collector = metrics_collector
+          @prometheus_exporter = prometheus_exporter
+          @alert_manager = alert_manager
+          @websocket_clients = []
+
+          setup_real_time_updates
+        end
+
+        def setup_real_time_updates
+          # Register observer for real-time metric updates
+          @metrics_collector.add_observer do |event_type, metric|
+            broadcast_to_clients({
+                                   type: "metric_update",
+                                   event: event_type,
+                                   data: metric,
+                                   timestamp: Time.now.utc.iso8601
+                                 })
+          end
+
+          # Register alert handler
+          @alert_manager.add_handler do |alert|
+            broadcast_to_clients({
+                                   type: "alert",
+                                   data: alert,
+                                   timestamp: Time.now.utc.iso8601
+                                 })
+          end
+        end
+
+        def broadcast_to_clients(message)
+          return unless WEBSOCKET_AVAILABLE
+
+          json_message = message.to_json
+          @websocket_clients.each do |client|
+            client.send(json_message) if client.ready_state == Faye::WebSocket::API::OPEN
+          rescue StandardError => e
+            warn "WebSocket send failed: #{e.message}"
+          end
+        end
+
+        def add_websocket_client(ws)
+          @websocket_clients << ws
+        end
+
+        def remove_websocket_client(ws)
+          @websocket_clients.delete(ws)
+        end
+      end
+
+      # Main dashboard page
+      get "/" do
+        send_file File.join(settings.public_folder, "index.html")
+      end
+
+      # WebSocket endpoint for real-time updates
+      get "/ws" do
+        halt 503, { error: "WebSocket support not available. Install faye-websocket gem." }.to_json unless WEBSOCKET_AVAILABLE
+
+        if Faye::WebSocket.websocket?(request.env)
+          ws = Faye::WebSocket.new(request.env)
+
+          ws.on :open do |_event|
+            self.class.add_websocket_client(ws)
+
+            # Send initial state
+            ws.send({
+              type: "connected",
+              message: "Connected to DecisionAgent monitoring",
+              timestamp: Time.now.utc.iso8601
+            }.to_json)
+          end
+
+          ws.on :message do |event|
+            # Handle client messages
+            handle_websocket_message(ws, event.data)
+          end
+
+          ws.on :close do |_event|
+            self.class.remove_websocket_client(ws)
+          end
+
+          ws.rack_response
+        else
+          status 426
+          { error: "WebSocket connection required" }.to_json
+        end
+      end
+
+      # API: Get current statistics
+      get "/api/stats" do
+        content_type :json
+
+        time_range = params[:time_range]&.to_i
+        stats = self.class.metrics_collector.statistics(time_range: time_range)
+
+        stats.to_json
+      end
+
+      # API: Get time series data
+      get "/api/timeseries/:metric_type" do
+        content_type :json
+
+        metric_type = params[:metric_type].to_sym
+        bucket_size = (params[:bucket_size] || 60).to_i
+        time_range = (params[:time_range] || 3600).to_i
+
+        data = self.class.metrics_collector.time_series(
+          metric_type: metric_type,
+          bucket_size: bucket_size,
+          time_range: time_range
+        )
+
+        data.to_json
+      end
+
+      # API: Prometheus metrics endpoint
+      get "/metrics" do
+        content_type PrometheusExporter::CONTENT_TYPE
+        self.class.prometheus_exporter.export
+      end
+
+      # API: Get Prometheus metrics in JSON format
+      get "/api/metrics" do
+        content_type :json
+        self.class.prometheus_exporter.metrics_hash.to_json
+      end
+
+      # API: Register custom KPI
+      post "/api/kpi" do
+        content_type :json
+
+        begin
+          data = JSON.parse(request.body.read, symbolize_names: true)
+
+          self.class.prometheus_exporter.register_kpi(
+            name: data[:name],
+            value: data[:value],
+            labels: data[:labels] || {},
+            help: data[:help]
+          )
+
+          { success: true, message: "KPI registered" }.to_json
+        rescue StandardError => e
+          status 400
+          { error: e.message }.to_json
+        end
+      end
+
+      # API: Get active alerts
+      get "/api/alerts" do
+        content_type :json
+        self.class.alert_manager.active_alerts.to_json
+      end
+
+      # API: Get all alerts
+      get "/api/alerts/all" do
+        content_type :json
+        limit = (params[:limit] || 100).to_i
+        self.class.alert_manager.all_alerts(limit: limit).to_json
+      end
+
+      # API: Create alert rule
+      post "/api/alerts/rules" do
+        content_type :json
+
+        begin
+          data = JSON.parse(request.body.read, symbolize_names: true)
+
+          # Parse condition
+          condition = parse_alert_condition(data[:condition], data[:condition_type])
+
+          rule = self.class.alert_manager.add_rule(
+            name: data[:name],
+            condition: condition,
+            severity: (data[:severity] || :warning).to_sym,
+            threshold: data[:threshold],
+            message: data[:message],
+            cooldown: data[:cooldown] || 300
+          )
+
+          status 201
+          rule.to_json
+        rescue StandardError => e
+          status 400
+          { error: e.message }.to_json
+        end
+      end
+
+      # API: Toggle alert rule
+      put "/api/alerts/rules/:rule_id/toggle" do
+        content_type :json
+
+        begin
+          data = JSON.parse(request.body.read, symbolize_names: true)
+          enabled = data[:enabled] || false
+
+          self.class.alert_manager.toggle_rule(params[:rule_id], enabled)
+
+          { success: true, message: "Rule #{enabled ? 'enabled' : 'disabled'}" }.to_json
+        rescue StandardError => e
+          status 400
+          { error: e.message }.to_json
+        end
+      end
+
+      # API: Acknowledge alert
+      post "/api/alerts/:alert_id/acknowledge" do
+        content_type :json
+
+        begin
+          data = JSON.parse(request.body.read, symbolize_names: true)
+          acknowledged_by = data[:acknowledged_by] || "user"
+
+          self.class.alert_manager.acknowledge_alert(params[:alert_id], acknowledged_by: acknowledged_by)
+
+          { success: true, message: "Alert acknowledged" }.to_json
+        rescue StandardError => e
+          status 400
+          { error: e.message }.to_json
+        end
+      end
+
+      # API: Resolve alert
+      post "/api/alerts/:alert_id/resolve" do
+        content_type :json
+
+        begin
+          data = JSON.parse(request.body.read, symbolize_names: true)
+          resolved_by = data[:resolved_by] || "user"
+
+          self.class.alert_manager.resolve_alert(params[:alert_id], resolved_by: resolved_by)
+
+          { success: true, message: "Alert resolved" }.to_json
+        rescue StandardError => e
+          status 400
+          { error: e.message }.to_json
+        end
+      end
+
+      # Health check
+      get "/health" do
+        content_type :json
+        {
+          status: "ok",
+          version: DecisionAgent::VERSION,
+          websocket_clients: self.class.websocket_clients.size,
+          metrics_count: self.class.metrics_collector.metrics_count
+        }.to_json
+      end
+
+      # Class method to start the server
+      def self.start!(metrics_collector:, prometheus_exporter:, alert_manager:, port: 4568, host: "0.0.0.0")
+        configure_monitoring(
+          metrics_collector: metrics_collector,
+          prometheus_exporter: prometheus_exporter,
+          alert_manager: alert_manager
+        )
+
+        set :port, port
+        set :bind, host
+        run!
+      end
+
+      private
+
+      def handle_websocket_message(ws, data)
+        message = JSON.parse(data, symbolize_names: true)
+
+        case message[:action]
+        when "subscribe"
+          # Send current stats
+          stats = self.class.metrics_collector.statistics
+          ws.send({ type: "stats", data: stats }.to_json)
+        when "get_alerts"
+          alerts = self.class.alert_manager.active_alerts
+          ws.send({ type: "alerts", data: alerts }.to_json)
+        end
+      rescue StandardError => e
+        ws.send({ type: "error", message: e.message }.to_json)
+      end
+
+      def parse_alert_condition(condition_data, condition_type)
+        case condition_type
+        when "high_error_rate"
+          AlertManager.high_error_rate(threshold: condition_data[:threshold] || 0.1)
+        when "low_confidence"
+          AlertManager.low_confidence(threshold: condition_data[:threshold] || 0.5)
+        when "high_latency"
+          AlertManager.high_latency(threshold_ms: condition_data[:threshold_ms] || 1000)
+        when "error_spike"
+          AlertManager.error_spike(threshold: condition_data[:threshold] || 10)
+        when "custom"
+          condition_data
+        else
+          raise "Unknown condition type: #{condition_type}"
+        end
+      end
+    end
+  end
+end

data/lib/decision_agent/monitoring/metrics_collector.rb

@@ -0,0 +1,278 @@
+require "monitor"
+require "time"
+
+module DecisionAgent
+  module Monitoring
+    # Thread-safe metrics collector for decision analytics
+    class MetricsCollector
+      include MonitorMixin
+
+      attr_reader :metrics, :window_size
+
+      def initialize(window_size: 3600)
+        super()
+        @window_size = window_size # Default: 1 hour window
+        @metrics = {
+          decisions: [],
+          evaluations: [],
+          performance: [],
+          errors: []
+        }
+        @observers = []
+        freeze_config
+      end
+
+      # Record a decision for analytics
+      def record_decision(decision, context, duration_ms: nil)
+        synchronize do
+          metric = {
+            timestamp: Time.now.utc,
+            decision: decision.decision,
+            confidence: decision.confidence,
+            evaluations_count: decision.evaluations.size,
+            context_size: context.to_h.size,
+            duration_ms: duration_ms,
+            evaluator_names: decision.evaluations.map(&:evaluator_name).uniq
+          }
+
+          @metrics[:decisions] << metric
+          cleanup_old_metrics!
+          notify_observers(:decision, metric)
+          metric
+        end
+      end
+
+      # Record individual evaluation metrics
+      def record_evaluation(evaluation)
+        synchronize do
+          metric = {
+            timestamp: Time.now.utc,
+            decision: evaluation.decision,
+            weight: evaluation.weight,
+            evaluator_name: evaluation.evaluator_name
+          }
+
+          @metrics[:evaluations] << metric
+          cleanup_old_metrics!
+          notify_observers(:evaluation, metric)
+          metric
+        end
+      end
+
+      # Record performance metrics
+      def record_performance(operation:, duration_ms:, success: true, metadata: {})
+        synchronize do
+          metric = {
+            timestamp: Time.now.utc,
+            operation: operation,
+            duration_ms: duration_ms,
+            success: success,
+            metadata: metadata
+          }
+
+          @metrics[:performance] << metric
+          cleanup_old_metrics!
+          notify_observers(:performance, metric)
+          metric
+        end
+      end
+
+      # Record error
+      def record_error(error, context: {})
+        synchronize do
+          metric = {
+            timestamp: Time.now.utc,
+            error_class: error.class.name,
+            error_message: error.message,
+            context: context
+          }
+
+          @metrics[:errors] << metric
+          cleanup_old_metrics!
+          notify_observers(:error, metric)
+          metric
+        end
+      end
+
+      # Get aggregated statistics
+      def statistics(time_range: nil)
+        synchronize do
+          range_start = time_range ? Time.now.utc - time_range : nil
+
+          decisions = filter_by_time(@metrics[:decisions], range_start)
+          evaluations = filter_by_time(@metrics[:evaluations], range_start)
+          performance = filter_by_time(@metrics[:performance], range_start)
+          errors = filter_by_time(@metrics[:errors], range_start)
+
+          {
+            summary: {
+              total_decisions: decisions.size,
+              total_evaluations: evaluations.size,
+              total_errors: errors.size,
+              time_range: range_start ? "Last #{time_range}s" : "All time"
+            },
+            decisions: compute_decision_stats(decisions),
+            evaluations: compute_evaluation_stats(evaluations),
+            performance: compute_performance_stats(performance),
+            errors: compute_error_stats(errors),
+            timestamp: Time.now.utc
+          }
+        end
+      end
+
+      # Get time-series data for graphing
+      def time_series(metric_type:, bucket_size: 60, time_range: 3600)
+        synchronize do
+          data = @metrics[metric_type] || []
+          range_start = Time.now.utc - time_range
+
+          buckets = {}
+          data.each do |metric|
+            next if metric[:timestamp] < range_start
+
+            bucket_key = (metric[:timestamp].to_i / bucket_size) * bucket_size
+            buckets[bucket_key] ||= []
+            buckets[bucket_key] << metric
+          end
+
+          buckets.sort.map do |timestamp, metrics|
+            {
+              timestamp: Time.at(timestamp).utc,
+              count: metrics.size,
+              metrics: metrics
+            }
+          end
+        end
+      end
+
+      # Register observer for real-time updates
+      def add_observer(&block)
+        synchronize do
+          @observers << block
+        end
+      end
+
+      # Clear all metrics
+      def clear!
+        synchronize do
+          @metrics.each_value(&:clear)
+        end
+      end
+
+      # Get current metrics count
+      def metrics_count
+        synchronize do
+          @metrics.transform_values(&:size)
+        end
+      end
+
+      private
+
+      def freeze_config
+        @window_size.freeze
+      end
+
+      def cleanup_old_metrics!
+        cutoff_time = Time.now.utc - @window_size
+
+        @metrics.each_value do |data|
+          data.delete_if { |m| m[:timestamp] < cutoff_time }
+        end
+      end
+
+      def filter_by_time(data, start_time)
+        return data unless start_time
+
+        data.select { |m| m[:timestamp] >= start_time }
+      end
+
+      def compute_decision_stats(decisions)
+        return {} if decisions.empty?
+
+        confidences = decisions.map { |d| d[:confidence] }
+        durations = decisions.map { |d| d[:duration_ms] }.compact
+
+        decision_distribution = decisions.group_by { |d| d[:decision] }
+                                         .transform_values(&:size)
+
+        {
+          total: decisions.size,
+          avg_confidence: (confidences.sum / confidences.size.to_f).round(4),
+          min_confidence: confidences.min.round(4),
+          max_confidence: confidences.max.round(4),
+          decision_distribution: decision_distribution,
+          avg_duration_ms: durations.empty? ? nil : (durations.sum / durations.size.to_f).round(2),
+          evaluators_used: decisions.flat_map { |d| d[:evaluator_names] }.uniq
+        }
+      end
+
+      def compute_evaluation_stats(evaluations)
+        return {} if evaluations.empty?
+
+        weights = evaluations.map { |e| e[:weight] }
+        evaluator_distribution = evaluations.group_by { |e| e[:evaluator_name] }
+                                            .transform_values(&:size)
+
+        {
+          total: evaluations.size,
+          avg_weight: (weights.sum / weights.size.to_f).round(4),
+          evaluator_distribution: evaluator_distribution,
+          decision_distribution: evaluations.group_by { |e| e[:decision] }
+                                            .transform_values(&:size)
+        }
+      end
+
+      def compute_performance_stats(performance)
+        return {} if performance.empty?
+
+        durations = performance.map { |p| p[:duration_ms] }
+        successes = performance.count { |p| p[:success] }
+
+        {
+          total_operations: performance.size,
+          successful: successes,
+          failed: performance.size - successes,
+          success_rate: (successes / performance.size.to_f).round(4),
+          avg_duration_ms: (durations.sum / durations.size.to_f).round(2),
+          min_duration_ms: durations.min.round(2),
+          max_duration_ms: durations.max.round(2),
+          p95_duration_ms: percentile(durations, 0.95).round(2),
+          p99_duration_ms: percentile(durations, 0.99).round(2)
+        }
+      end
+
+      def compute_error_stats(errors)
+        return {} if errors.empty?
+
+        {
+          total: errors.size,
+          by_type: errors.group_by { |e| e[:error_class] }.transform_values(&:size),
+          recent_errors: errors.last(10).map do |e|
+            {
+              timestamp: e[:timestamp],
+              error: e[:error_class],
+              message: e[:error_message]
+            }
+          end
+        }
+      end
+
+      def percentile(array, percentile)
+        return 0 if array.empty?
+
+        sorted = array.sort
+        index = (percentile * sorted.length).ceil - 1
+        sorted[[index, 0].max]
+      end
+
+      def notify_observers(event_type, metric)
+        @observers.each do |observer|
+          observer.call(event_type, metric)
+        rescue StandardError => e
+          # Silently fail observer notifications to prevent disruption
+          warn "Observer notification failed: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/decision_agent/monitoring/monitored_agent.rb

@@ -0,0 +1,71 @@
+module DecisionAgent
+  module Monitoring
+    # Wrapper around Agent that automatically records metrics
+    class MonitoredAgent
+      attr_reader :agent, :metrics_collector
+
+      def initialize(agent:, metrics_collector:)
+        @agent = agent
+        @metrics_collector = metrics_collector
+      end
+
+      # Make a decision and automatically record metrics
+      def decide(context:, feedback: {})
+        ctx = context.is_a?(Context) ? context : Context.new(context)
+
+        start_time = Time.now
+
+        begin
+          result = @agent.decide(context: ctx, feedback: feedback)
+          duration_ms = (Time.now - start_time) * 1000
+
+          # Record decision metrics
+          @metrics_collector.record_decision(result, ctx, duration_ms: duration_ms)
+
+          # Record each evaluation
+          result.evaluations.each do |evaluation|
+            @metrics_collector.record_evaluation(evaluation)
+          end
+
+          # Record successful performance
+          @metrics_collector.record_performance(
+            operation: "decide",
+            duration_ms: duration_ms,
+            success: true,
+            metadata: {
+              evaluators_count: result.evaluations.size,
+              decision: result.decision,
+              confidence: result.confidence
+            }
+          )
+
+          result
+        rescue StandardError => e
+          duration_ms = (Time.now - start_time) * 1000
+
+          # Record error
+          @metrics_collector.record_error(e, context: ctx.to_h)
+
+          # Record failed performance
+          @metrics_collector.record_performance(
+            operation: "decide",
+            duration_ms: duration_ms,
+            success: false,
+            metadata: { error_class: e.class.name }
+          )
+
+          raise
+        end
+      end
+
+      # Delegate other methods to the wrapped agent
+      def method_missing(method, ...)
+        @agent.send(method, ...)
+      end
+
+      def respond_to_missing?(method, include_private = false)
+        @agent.respond_to?(method, include_private) || super
+      end
+    end
+  end
+end