natswork-server 0.0.1

data/lib/natswork/error_tracker.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'concurrent'
+require 'socket'
+
+module NatsWork
+  class ErrorTracker
+    attr_reader :errors, :error_rates
+
+    def initialize(options = {})
+      @errors = Concurrent::Hash.new
+      @error_counts = Concurrent::Hash.new(0)
+      @error_rates = Concurrent::Hash.new { |h, k| h[k] = RateTracker.new }
+      @max_errors = options.fetch(:max_errors, 1000)
+      @retention_period = options.fetch(:retention_hours, 24) * 3600
+      @notifiers = []
+
+      setup_cleanup_timer
+    end
+
+    def track_error(error, context = {})
+      error_info = extract_error_info(error, context)
+      error_key = generate_error_key(error_info)
+
+      # Store error details
+      @errors[error_key] = error_info.merge(
+        first_seen: @errors[error_key]&.dig(:first_seen) || Time.now,
+        last_seen: Time.now,
+        count: @error_counts[error_key] + 1
+      )
+
+      @error_counts[error_key] += 1
+      @error_rates[error_key].increment
+
+      # Notify error handlers
+      notify_error(error_info)
+
+      # Cleanup old errors if we're over the limit
+      cleanup_old_errors if @errors.size > @max_errors
+
+      error_info
+    end
+
+    def get_error(error_key)
+      @errors[error_key]
+    end
+
+    def list_errors(limit = 50, filter = {})
+      filtered_errors = @errors.select do |_key, error|
+        matches_filter?(error, filter)
+      end
+
+      sorted_errors = filtered_errors.sort_by { |_key, error| -error[:last_seen].to_f }
+      sorted_errors.first(limit).to_h
+    end
+
+    def error_stats(error_key = nil)
+      if error_key
+        error = @errors[error_key]
+        return nil unless error
+
+        {
+          key: error_key,
+          count: error[:count],
+          first_seen: error[:first_seen],
+          last_seen: error[:last_seen],
+          rate_1h: @error_rates[error_key].rate(3600),
+          rate_24h: @error_rates[error_key].rate(86_400)
+        }
+      else
+        {
+          total_errors: @errors.size,
+          total_occurrences: @error_counts.values.sum,
+          most_frequent: most_frequent_errors(5),
+          recent_errors: recent_errors(10)
+        }
+      end
+    end
+
+    def add_notifier(&block)
+      @notifiers << block if block_given?
+    end
+
+    def clear_errors!
+      @errors.clear
+      @error_counts.clear
+      @error_rates.clear
+    end
+
+    def categorize_errors
+      categories = {
+        timeout: [],
+        connection: [],
+        permission: [],
+        validation: [],
+        system: [],
+        unknown: []
+      }
+
+      @errors.each do |key, error|
+        category = determine_category(error)
+        categories[category] << { key: key, error: error }
+      end
+
+      categories
+    end
+
+    private
+
+    def extract_error_info(error, context)
+      {
+        type: error.class.name,
+        message: error.message,
+        backtrace: error.backtrace&.first(20),
+        fingerprint: generate_fingerprint(error),
+        context: sanitize_context(context),
+        timestamp: Time.now,
+        hostname: Socket.gethostname,
+        pid: Process.pid,
+        thread_id: Thread.current.object_id
+      }
+    end
+
+    def generate_error_key(error_info)
+      "#{error_info[:type]}:#{error_info[:fingerprint]}"
+    end
+
+    def generate_fingerprint(error)
+      # Create a fingerprint based on error type, message, and location
+      location = error.backtrace&.first&.split(':')&.first(2)&.join(':') || 'unknown'
+      fingerprint_data = "#{error.class.name}:#{error.message}:#{location}"
+      Digest::SHA256.hexdigest(fingerprint_data)[0, 16]
+    end
+
+    def sanitize_context(context)
+      # Remove sensitive data from context
+      sanitized = context.dup
+
+      sensitive_keys = %w[password token secret key credential auth]
+      sensitive_keys.each do |sensitive_key|
+        sanitized.keys.each do |key|
+          sanitized[key] = '[FILTERED]' if key.to_s.downcase.include?(sensitive_key)
+        end
+      end
+
+      sanitized
+    end
+
+    def notify_error(error_info)
+      @notifiers.each do |notifier|
+        notifier.call(error_info)
+      rescue StandardError => e
+        # Don't let notifier errors break error tracking
+        warn "Error notifier failed: #{e.message}"
+      end
+    end
+
+    def matches_filter?(error, filter)
+      return true if filter.empty?
+
+      filter.all? do |key, value|
+        case key.to_sym
+        when :type
+          error[:type] == value
+        when :since
+          error[:last_seen] >= value
+        when :context
+          value.all? { |k, v| error[:context][k] == v }
+        else
+          true
+        end
+      end
+    end
+
+    def most_frequent_errors(limit)
+      @error_counts.sort_by { |_key, count| -count }
+                   .first(limit)
+                   .map { |key, count| { key: key, count: count, error: @errors[key] } }
+    end
+
+    def recent_errors(limit)
+      @errors.values
+             .sort_by { |error| -error[:last_seen].to_f }
+             .first(limit)
+    end
+
+    def cleanup_old_errors
+      cutoff_time = Time.now - @retention_period
+
+      @errors.delete_if { |_key, error| error[:last_seen] < cutoff_time }
+      @error_counts.delete_if { |key, _count| !@errors.key?(key) }
+      @error_rates.delete_if { |key, _rate| !@errors.key?(key) }
+    end
+
+    def setup_cleanup_timer
+      Thread.new do
+        loop do
+          sleep 3600 # Run cleanup every hour
+          cleanup_old_errors
+        end
+      end
+    end
+
+    def determine_category(error)
+      message = error[:message].downcase
+      type = error[:type].downcase
+
+      return :timeout if message.include?('timeout') || type.include?('timeout')
+      return :connection if message.include?('connection') || message.include?('network')
+      return :permission if message.include?('permission') || message.include?('unauthorized')
+      return :validation if message.include?('invalid') || message.include?('validation')
+      return :system if type.include?('system') || type.include?('runtime')
+
+      :unknown
+    end
+
+    class << self
+      def global
+        @global ||= new
+      end
+
+      def track(error, context = {})
+        global.track_error(error, context)
+      end
+
+      def stats
+        global.error_stats
+      end
+
+      def errors(limit = 50, filter = {})
+        global.list_errors(limit, filter)
+      end
+    end
+  end
+
+  # Rate tracker for monitoring error frequencies
+  class RateTracker
+    def initialize(window_size = 3600)
+      @timestamps = Concurrent::Array.new
+      @window_size = window_size
+      @mutex = Mutex.new
+    end
+
+    def increment
+      @mutex.synchronize do
+        @timestamps << Time.now.to_f
+        cleanup_old_timestamps
+      end
+    end
+
+    def rate(period_seconds = @window_size)
+      @mutex.synchronize do
+        cleanup_old_timestamps
+        cutoff_time = Time.now.to_f - period_seconds
+        recent_count = @timestamps.count { |ts| ts >= cutoff_time }
+        recent_count.to_f / period_seconds
+      end
+    end
+
+    def count(period_seconds = @window_size)
+      @mutex.synchronize do
+        cleanup_old_timestamps
+        cutoff_time = Time.now.to_f - period_seconds
+        @timestamps.count { |ts| ts >= cutoff_time }
+      end
+    end
+
+    private
+
+    def cleanup_old_timestamps
+      cutoff_time = Time.now.to_f - @window_size
+      @timestamps.delete_if { |ts| ts < cutoff_time }
+    end
+  end
+
+  # Built-in error notifiers
+  module ErrorNotifiers
+    class LogNotifier
+      def initialize(logger = nil)
+        @logger = logger || NatsWork::Logger.global
+      end
+
+      def call(error_info)
+        @logger.error('Error tracked',
+                      error_key: "#{error_info[:type]}:#{error_info[:fingerprint]}",
+                      error_type: error_info[:type],
+                      error_message: error_info[:message],
+                      context: error_info[:context])
+      end
+    end
+
+    class MetricsNotifier
+      def initialize(metrics = nil)
+        @metrics = metrics || NatsWork::Metrics.global
+      end
+
+      def call(error_info)
+        @metrics.increment('errors.tracked', 1,
+                           error_type: error_info[:type],
+                           hostname: error_info[:hostname])
+      end
+    end
+
+    class WebhookNotifier
+      def initialize(webhook_url, options = {})
+        @webhook_url = webhook_url
+        @http_timeout = options.fetch(:timeout, 5)
+        @retry_count = options.fetch(:retries, 2)
+      end
+
+      def call(error_info)
+        payload = {
+          error: {
+            type: error_info[:type],
+            message: error_info[:message],
+            fingerprint: error_info[:fingerprint],
+            timestamp: error_info[:timestamp].iso8601,
+            context: error_info[:context]
+          }
+        }
+
+        send_webhook(payload)
+      end
+
+      private
+
+      def send_webhook(payload)
+        # This would integrate with an HTTP client
+        # For now, just log the webhook call
+        require_relative 'logger'
+        NatsWork::Logger.debug('Webhook notification',
+                               url: @webhook_url,
+                               payload_size: payload.to_s.length)
+      end
+    end
+  end
+end

data/lib/natswork/health_check.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module NatsWork
+  class HealthCheck
+    attr_reader :name, :status, :last_checked, :message, :metadata
+
+    STATUSES = %i[healthy degraded unhealthy unknown].freeze
+
+    def initialize(name, &check_block)
+      @name = name.to_s
+      @check_block = check_block
+      @status = :unknown
+      @message = ''
+      @metadata = {}
+      @last_checked = nil
+      @mutex = Mutex.new
+    end
+
+    def check!
+      @mutex.synchronize do
+        @last_checked = Time.now
+
+        begin
+          if @check_block
+            result = @check_block.call
+
+            if result.is_a?(Hash)
+              @status = result.fetch(:status, :healthy)
+              @message = result.fetch(:message, '')
+              @metadata = result.fetch(:metadata, {})
+            elsif result
+              @status = :healthy
+              @message = 'Check passed'
+            else
+              @status = :unhealthy
+              @message = 'Check failed'
+            end
+          else
+            @status = :unknown
+            @message = 'No check defined'
+          end
+        rescue StandardError => e
+          @status = :unhealthy
+          @message = "Check error: #{e.message}"
+          @metadata = { error: e.class.name, backtrace: e.backtrace&.first(5) }
+        end
+
+        @status
+      end
+    end
+
+    def healthy?
+      @status == :healthy
+    end
+
+    def unhealthy?
+      @status == :unhealthy
+    end
+
+    def degraded?
+      @status == :degraded
+    end
+
+    def to_h
+      {
+        name: @name,
+        status: @status,
+        message: @message,
+        last_checked: @last_checked,
+        metadata: @metadata
+      }
+    end
+  end
+
+  class HealthChecker
+    def initialize
+      @checks = Concurrent::Hash.new
+      @thresholds = {
+        memory_mb: 1000,
+        cpu_percent: 90.0,
+        queue_depth: 1000,
+        error_rate: 0.05
+      }
+    end
+
+    def add_check(name, &block)
+      @checks[name.to_s] = HealthCheck.new(name, &block)
+    end
+
+    def remove_check(name)
+      @checks.delete(name.to_s)
+    end
+
+    def check_all!
+      results = {}
+
+      @checks.each do |name, check|
+        results[name] = check.check!
+      end
+
+      results
+    end
+
+    def check(name)
+      check = @checks[name.to_s]
+      check&.check!
+    end
+
+    def status(name = nil)
+      if name
+        @checks[name.to_s]&.status || :unknown
+      else
+        overall_status
+      end
+    end
+
+    def healthy?(name = nil)
+      if name
+        @checks[name.to_s]&.healthy? || false
+      else
+        @checks.values.all?(&:healthy?)
+      end
+    end
+
+    def report
+      {
+        status: overall_status,
+        timestamp: Time.now.iso8601,
+        checks: @checks.transform_values(&:to_h)
+      }
+    end
+
+    def set_threshold(metric, value)
+      @thresholds[metric.to_sym] = value
+    end
+
+    private
+
+    def overall_status
+      statuses = @checks.values.map(&:status)
+
+      return :unhealthy if statuses.include?(:unhealthy)
+      return :degraded if statuses.include?(:degraded)
+      return :healthy if statuses.all? { |s| s == :healthy }
+
+      :unknown
+    end
+
+    class << self
+      def global
+        @global ||= new.tap { |checker| setup_default_checks(checker) }
+      end
+
+      private
+
+      def setup_default_checks(checker)
+        # NATS connection health
+        checker.add_check('nats_connection') do
+          # This will be implemented when we have connection reference
+          { status: :healthy, message: 'NATS connection active' }
+        end
+
+        # Memory usage
+        checker.add_check('memory_usage') do
+          if defined?(GC.stat)
+            memory_mb = GC.stat(:heap_allocated_pages) * GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] / (1024 * 1024)
+            threshold = checker.instance_variable_get(:@thresholds)[:memory_mb]
+
+            if memory_mb > threshold
+              { status: :unhealthy, message: "Memory usage #{memory_mb}MB exceeds threshold #{threshold}MB" }
+            elsif memory_mb > threshold * 0.8
+              { status: :degraded, message: "Memory usage #{memory_mb}MB approaching threshold" }
+            else
+              { status: :healthy, message: "Memory usage #{memory_mb}MB within limits" }
+            end
+          else
+            { status: :unknown, message: 'Memory stats not available' }
+          end
+        end
+
+        # Process health
+        checker.add_check('process') do
+          {
+            status: :healthy,
+            message: 'Process running normally',
+            metadata: {
+              pid: Process.pid,
+              uptime: Time.now - $PROGRAM_START_TIME
+            }
+          }
+        end
+      end
+    end
+  end
+
+  # Worker-specific health checks
+  module WorkerHealth
+    def self.setup_worker_checks(worker, health_checker = HealthChecker.global)
+      # Worker status check
+      health_checker.add_check('worker_status') do
+        if worker.running?
+          if worker.paused?
+            { status: :degraded, message: 'Worker is paused' }
+          elsif worker.stopping?
+            { status: :degraded, message: 'Worker is stopping' }
+          else
+            { status: :healthy, message: 'Worker running normally' }
+          end
+        else
+          { status: :unhealthy, message: 'Worker is stopped' }
+        end
+      end
+
+      # Active jobs check
+      health_checker.add_check('active_jobs') do
+        stats = worker.stats
+        active = stats[:active_jobs]
+        concurrency = stats[:concurrency]
+
+        if active >= concurrency
+          { status: :degraded, message: "All #{concurrency} job slots occupied" }
+        elsif active > concurrency * 0.8
+          { status: :degraded, message: "#{active}/#{concurrency} job slots occupied" }
+        else
+          { status: :healthy, message: "#{active}/#{concurrency} job slots occupied" }
+        end
+      end
+
+      # Job processing rate check
+      health_checker.add_check('job_processing') do
+        stats = worker.stats
+        processed = stats[:jobs_processed]
+        failed = stats[:jobs_failed]
+
+        if processed.zero?
+          { status: :unknown, message: 'No jobs processed yet' }
+        else
+          error_rate = failed.to_f / processed
+          if error_rate > 0.5
+            { status: :unhealthy, message: "High error rate: #{(error_rate * 100).round(1)}%" }
+          elsif error_rate > 0.1
+            { status: :degraded, message: "Elevated error rate: #{(error_rate * 100).round(1)}%" }
+          else
+            { status: :healthy, message: "Error rate: #{(error_rate * 100).round(1)}%" }
+          end
+        end
+      end
+    end
+  end
+end