hyperprobe-agent 1.2.27.pre.3-java

data/lib/hyperprobe/agent.rb

@@ -0,0 +1,537 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'uri'
+require 'thread'
+require 'monitor'
+require_relative 'version'
+require_relative 'core/quota'
+require_relative 'core/safety'
+require_relative 'core/serializer'
+require_relative 'core/evaluator'
+require_relative 'core/trace_extractor'
+require_relative 'core/monitoring_engine'
+require_relative 'core/broker'
+
+module HyperProbe
+  UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.freeze
+
+  class Agent
+    attr_reader :options, :agent_id, :owner_pid, :is_shutdown, :active_probes, :global_config, :quota_manager, :safety_monitor, :broker_client, :engine
+
+    def initialize(options = {})
+      @options = (options || {}).dup
+      @owner_pid = Process.pid
+      @agent_id = SecureRandom.uuid
+      @is_shutdown = false
+      @is_agent_disabled = false
+      @mutex = Monitor.new # Reentrant mutex in Ruby
+      @stop_event = @mutex.new_cond
+      @sync_mutex = Mutex.new
+      @flush_mutex = Mutex.new
+      @shutdown_mutex = Mutex.new
+      @probe_generation = 0
+      @pending_probe_update = nil
+      @finished_probes = {}
+      @telemetry_queue = Queue.new
+      @active_probes = {}
+      @local_hits = {}
+
+      @log = Core::Logger.get_logger('hyperprobe:agent')
+      @log_broker = Core::Logger.get_logger('hyperprobe:broker')
+      @log_safety = Core::Logger.get_logger('hyperprobe:safety')
+      @log_stats = Core::Logger.get_logger('hyperprobe:stats')
+
+      parse_configuration
+
+      if @global_config[:disable_safe_evaluation]
+        @log.warn 'Safe evaluation is DISABLED. Probe expressions can execute arbitrary Ruby code, mutate application state, or block execution.'
+      end
+
+      @quota_manager = Core::QuotaManager.new(@hits_per_sec, @bandwidth_kb_per_sec * 1024)
+      @safety_monitor = Core::SafetyMonitor.new(
+        method(:handle_health_change),
+        max_lag_ms: @max_lag_ms,
+        pause_budget_ms: @pause_budget_ms,
+        is_ephemeral_lambda: @is_ephemeral_lambda
+      )
+
+      @broker_client = Core::BrokerClient.new(
+        broker_url: @broker_url,
+        service_id: @service_id,
+        environment: @environment,
+        commit_sha: @commit_sha,
+        agent_id: @agent_id,
+        agent_version: VERSION,
+        rpc_timeout_sec: @rpc_timeout_sec,
+        enable_keep_alive: @enable_keep_alive
+      )
+
+      @cooldown_timer = nil
+      @cooldown_until = nil
+
+      @engine = Core::MonitoringEngine.new(
+        @quota_manager,
+        @safety_monitor,
+        method(:handle_capture),
+        @options[:set_trace_id]
+      )
+      @engine.set_global_config(@global_config)
+
+      @sync_thread = nil
+      @flush_thread = nil
+      @stats_thread = nil
+      @apply_thread = Thread.new { apply_loop }
+      @apply_thread.name = 'hyperprobe-apply'
+
+      if @is_ephemeral_lambda
+        @log.debug 'Running in Ephemeral AWS Lambda Mode.'
+      else
+        @safety_monitor.start
+        start_background_loops
+      end
+
+      @log.info "Agent started for #{@service_id} in #{@environment} (v#{VERSION})"
+    rescue SignalException, SystemExit
+      shutdown
+      raise
+    rescue Exception # Agent-owned startup failures must not escape into the host.
+      shutdown
+    end
+
+    def agent_disabled?
+      @is_agent_disabled
+    end
+
+    def telemetry_queue_length
+      @telemetry_queue&.size || 0
+    end
+
+    def force_sync(timeout_sec = nil)
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+
+      sync_with_broker(timeout_sec)
+    end
+
+    def force_flush(timeout_sec = nil)
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+
+      flush_telemetry(timeout_sec)
+    end
+
+    def shutdown
+      return after_fork if @owner_pid && @owner_pid != Process.pid
+      @shutdown_mutex ||= Mutex.new
+      return unless @shutdown_mutex.try_lock
+
+      shutdown_locked = true
+      return if @is_shutdown
+
+      @is_shutdown = true
+      @is_agent_disabled = true
+      threads = [@sync_thread, @flush_thread, @stats_thread, @apply_thread, @cooldown_timer].compact
+      threads.each { |thread| thread.kill unless thread == Thread.current }
+
+      # Cleanup independently: a broken or stuck resource cannot prevent the rest.
+      cleanup = [[@engine, :close], [@safety_monitor, :stop], [@broker_client, :shutdown]].each_with_object([]) do |(resource, method), workers|
+        next unless resource
+
+        workers << Thread.new do
+          resource.public_send(method)
+        rescue Exception
+          # Only agent-owned cleanup runs in this thread.
+        end
+      end
+      cleanup << Thread.new do
+        @mutex.synchronize do
+          until @telemetry_queue.empty?
+            settle_reservation(@telemetry_queue.pop(true)[:reservation], :release)
+          end
+        end
+      rescue Exception
+        # Reservations belong only to the stopped agent.
+      end
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.5
+      (threads + cleanup).each do |thread|
+        next if thread == Thread.current
+
+        remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        thread.join(remaining) if remaining.positive?
+      rescue SignalException, SystemExit
+        raise
+      rescue Exception
+        # Continue closing the other resources, including from a signal trap.
+      end
+      cleanup.each do |thread|
+        thread.kill if thread.alive?
+        thread.join(0.01)
+      rescue SignalException, SystemExit
+        raise
+      rescue Exception
+        # Never wait indefinitely for a broken cleanup thread.
+      end
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      @is_shutdown = true
+      @is_agent_disabled = true
+    ensure
+      @shutdown_mutex.unlock if shutdown_locked
+    end
+
+    def after_fork
+      return if @owner_pid == Process.pid
+
+      # Never call, close, or replace an inherited native gRPC client here.
+      @is_shutdown = true
+      @is_agent_disabled = true
+      @engine&.after_fork
+      false
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      false
+    end
+
+    private
+
+    def parse_configuration
+      @service_id = get_opt(:service_id, :serviceId) || ENV['HYPERPROBE_SERVICE_ID']
+      @environment = get_opt(:environment) || ENV['HYPERPROBE_ENVIRONMENT']
+      @broker_url = get_opt(:broker_url, :brokerUrl) || ENV['HYPERPROBE_BROKER_URL']
+      @commit_sha = get_opt(:commit_sha, :commitSha) || ENV['GIT_COMMIT'] || ENV['HYPERPROBE_COMMIT_SHA']
+
+      raw_sync = get_opt(:sync_interval_ms, :syncIntervalMs) || parse_env_int('HYPERPROBE_SYNC_INTERVAL_MS', 60_000)
+      @sync_interval_sec = [raw_sync.to_f / 1000.0, 0.1].max
+
+      raw_flush = get_opt(:flush_interval_ms, :flushIntervalMs) || parse_env_int('HYPERPROBE_FLUSH_INTERVAL_MS', 1000)
+      @flush_interval_sec = [raw_flush.to_f / 1000.0, 0.05].max
+      @max_queue_size = get_opt(:max_queue_size, :maxQueueSize) || parse_env_int('HYPERPROBE_MAX_QUEUE_SIZE', 100)
+      @cooldown_sec = get_opt(:cooldown_sec, :cooldownSec) || parse_env_int('HYPERPROBE_COOLDOWN_SEC', 10)
+
+      @hits_per_sec = get_opt(:hits_per_sec, :hitsPerSec) || parse_env_int('HYPERPROBE_HITS_PER_SEC', 10)
+      @bandwidth_kb_per_sec = get_opt(:bandwidth_kb_per_sec, :bandwidthKbPerSec) || parse_env_int('HYPERPROBE_BANDWIDTH_KB_PER_SEC', 1024)
+      @max_lag_ms = (get_opt(:max_lag_ms, :maxLagMs) || parse_env_int('HYPERPROBE_MAX_LAG_MS', 50)).to_f
+      @pause_budget_ms = (get_opt(:pause_budget_ms, :pauseBudgetMs) || parse_env_int('HYPERPROBE_PAUSE_BUDGET_MS', 15)).to_f
+      @rpc_timeout_sec = (get_opt(:rpc_timeout_sec, :rpcTimeoutSec) || parse_env_int('HYPERPROBE_RPC_TIMEOUT_SEC', 10)).to_f
+
+      is_aws_lambda = !ENV['AWS_LAMBDA_FUNCTION_NAME'].nil?
+      is_local_emulator = ENV['IS_OFFLINE'] == 'true' || ENV['AWS_SAM_LOCAL'] == 'true'
+      lambda_opt = get_opt(:is_lambda, :isLambda)
+      @is_ephemeral_lambda = (lambda_opt.nil? ? is_aws_lambda : lambda_opt) && !is_local_emulator
+      enable_keep_alive_opt = get_opt(:enable_keep_alive, :enableKeepAlive)
+      @enable_keep_alive = enable_keep_alive_opt.nil? ? !@is_ephemeral_lambda : enable_keep_alive_opt
+
+      redact_keys_raw = get_opt(:redact_keys, :redactKeys) || (ENV['HYPERPROBE_REDACT_KEYS'] ? ENV['HYPERPROBE_REDACT_KEYS'].split(',') : %w[password secret token authorization cookie key signature])
+      redact_values_raw = get_opt(:redact_values, :redactValues) || (ENV['HYPERPROBE_REDACT_VALUES'] ? ENV['HYPERPROBE_REDACT_VALUES'].split(',') : [])
+
+      @global_config = {
+        redact_keys: Array(redact_keys_raw).map(&:to_s).map(&:strip).reject(&:empty?),
+        redact_values: Array(redact_values_raw).map(&:to_s).map(&:strip).reject(&:empty?),
+        max_object_depth: get_opt(:max_object_depth, :maxObjectDepth) || parse_env_int('HYPERPROBE_MAX_OBJECT_DEPTH', 3),
+        max_array_length: get_opt(:max_array_length, :maxArrayLength) || parse_env_int('HYPERPROBE_MAX_ARRAY_LENGTH', 3),
+        stack_frame_depth: get_opt(:stack_frame_depth, :stackFrameDepth) || parse_env_int('HYPERPROBE_STACK_FRAME_DEPTH', 3),
+        max_object_properties: get_opt(:max_object_properties, :maxObjectProperties) || parse_env_int('HYPERPROBE_MAX_OBJECT_PROPERTIES', 50),
+        max_string_length: get_opt(:max_string_length, :maxStringLength) || parse_env_int('HYPERPROBE_MAX_STRING_LENGTH', 1024),
+        capture_closures: get_opt(:capture_closures, :captureClosures) == true,
+        disable_safe_evaluation: !Core::Evaluator.safe_evaluation_enabled?(get_opt(:disable_safe_evaluation, :disableSafeEvaluation))
+      }
+    end
+
+    def get_opt(*keys)
+      keys.each do |k|
+        return @options[k] if @options.key?(k)
+        return @options[k.to_s] if @options.key?(k.to_s)
+      end
+      nil
+    end
+
+    def parse_env_int(key, default_val)
+      val = ENV[key]
+      return default_val if val.nil? || val.strip.empty?
+
+      val.to_i rescue default_val
+    end
+
+    def start_background_loops
+      @sync_thread = Thread.new { sync_loop }
+      @sync_thread.name = 'hyperprobe-sync' if @sync_thread.respond_to?(:name=)
+
+      @flush_thread = Thread.new { flush_loop }
+      @flush_thread.name = 'hyperprobe-flush' if @flush_thread.respond_to?(:name=)
+
+      @stats_thread = Thread.new { stats_loop }
+      @stats_thread.name = 'hyperprobe-stats' if @stats_thread.respond_to?(:name=)
+    end
+
+    def sync_loop
+      until @is_shutdown
+        begin
+          sync_with_broker
+          sleep @sync_interval_sec
+        rescue Exception
+          # Silently handle transient sync error
+          break
+        end
+      end
+    end
+
+    def sync_with_broker(timeout_sec = nil)
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+      return false unless @sync_mutex.try_lock
+
+      sync_locked = true
+
+      @log_broker.debug "sync_with_broker started #{timeout_sec ? "with timeout #{timeout_sec}" : ''}"
+      response = @broker_client.get_probes(timeout_sec)
+      return unless response
+
+      @log_broker.debug "sync_with_broker: got #{response.probes.length} probes from broker"
+
+      server_probes = response.probes.to_a
+      server_probe_ids = server_probes.map(&:id)
+      now_ms = (Time.now.to_f * 1000).to_i
+      config = nil
+      snapshot = @mutex.synchronize do
+        return false if @is_shutdown
+
+        if response.global_config
+          gc = response.global_config
+          @global_config[:redact_keys] = gc.redact_keys.to_a unless gc.redact_keys.empty?
+          @global_config[:redact_values] = gc.redact_values.to_a unless gc.redact_values.empty?
+          %i[max_object_depth max_array_length stack_frame_depth max_object_properties max_string_length capture_closures].each do |field|
+            @global_config[field] = gc.public_send(field) if gc.public_send("has_#{field}?")
+          end
+          config = @global_config.dup
+        end
+        # Clean stale local probes
+        @active_probes.delete_if { |pid, _| !server_probe_ids.include?(pid) }
+        @local_hits.delete_if { |pid, _| !server_probe_ids.include?(pid) }
+        @finished_probes.delete_if { |pid, _| !server_probe_ids.include?(pid) }
+
+        server_probes.each do |probe|
+          hits = @local_hits[probe.id] || 0
+          is_expired = probe.expiry_time.positive? && probe.expiry_time <= now_ms
+
+          if hits < probe.hit_limit && !is_expired && !@finished_probes[probe.id]
+            @active_probes[probe.id] = probe
+          else
+            @active_probes.delete(probe.id)
+          end
+        end
+        probe_snapshot_locked
+      end
+
+      @engine.set_global_config(config) if config
+      @engine.set_probes(snapshot[0], generation: snapshot[1])
+      true
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      false
+    ensure
+      @sync_mutex.unlock if sync_locked
+    end
+
+    def flush_loop
+      until @is_shutdown
+        begin
+          sleep @flush_interval_sec
+          break if @is_shutdown
+
+          flush_telemetry
+        rescue Exception
+          # Silently handle flush loop error
+        end
+      end
+    end
+
+    def flush_telemetry(timeout_sec = nil)
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+      return false unless @flush_mutex.try_lock
+
+      flush_locked = true
+
+      batch = []
+      @mutex.synchronize do
+        batch << @telemetry_queue.pop(true) until @telemetry_queue.empty?
+      end
+
+      return if batch.empty?
+
+      begin
+        events = batch.map { |item| item[:event] }
+        finished_probe_ids = @broker_client.report_telemetry(events, timeout_sec)
+        delivered = true
+
+        # Commit bandwidth reservations
+        batch.each do |item|
+          settle_reservation(item[:reservation], :commit)
+        end
+
+        # Handle globally finished probes
+        if finished_probe_ids && !finished_probe_ids.empty?
+          snapshot = @mutex.synchronize do
+            return if @is_shutdown
+
+            finished_probe_ids.each do |pid|
+              @finished_probes[pid] = true
+              @active_probes.delete(pid)
+            end
+            probe_snapshot_locked
+          end
+          @engine.set_probes(snapshot[0], generation: snapshot[1])
+        end
+      rescue SignalException, SystemExit
+        raise
+      rescue Exception
+        return false if delivered
+
+        # On flush failure, re-queue events up to max capacity
+        batch.each do |item|
+          retained = @mutex.synchronize do
+            if !@is_shutdown && @telemetry_queue.size < @max_queue_size
+              @telemetry_queue.push(item)
+              true
+            end
+          end
+          settle_reservation(item[:reservation], :release) unless retained
+        end
+      end
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      false
+    ensure
+      if @is_shutdown && batch && !delivered
+        batch.each { |item| settle_reservation(item[:reservation], :release) }
+      end
+      @flush_mutex.unlock if flush_locked
+    end
+
+    def stats_loop
+      until @is_shutdown
+        begin
+          sleep 5.0
+          break if @is_shutdown
+
+          stats = @engine.get_stats
+          if stats[:hits].positive? || stats[:skips].positive?
+            @log_stats.info "Probes Hit: #{stats[:hits]}, Probes Skipped: #{stats[:skips]}"
+          end
+        rescue Exception
+          # Ignore stats error
+        end
+      end
+    end
+
+    def handle_capture(event)
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+      return unless @mutex.try_enter
+
+      locked = true
+      return if @is_shutdown || @is_agent_disabled
+
+      probe_id = event[:probe_id]
+      probe = @active_probes[probe_id]
+      return unless probe
+
+      raw_limit = probe.respond_to?(:hit_limit) ? probe.hit_limit : probe[:hit_limit]
+      hit_limit = raw_limit.to_i
+      hits = @local_hits[probe_id] || 0
+      return if hit_limit.positive? && hits >= hit_limit
+
+      # Attempted captures consume the safety fuse even when delivery is dropped.
+      @local_hits[probe_id] = hits + 1
+      if hit_limit.positive? && hits + 1 >= hit_limit
+        @active_probes.delete(probe_id)
+        @pending_probe_update = probe_snapshot_locked
+        @stop_event.broadcast
+      end
+
+      return if @telemetry_queue.size >= @max_queue_size
+
+      event_size = @broker_client.estimate_event_size(event)
+      reservation = @quota_manager.reserve_bandwidth(event_size)
+      return unless reservation
+
+      @telemetry_queue.push(event: event, reservation: reservation)
+      admitted = true
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      # This callback is agent code running on an application thread.
+      nil
+    ensure
+      @mutex.exit if locked
+      settle_reservation(reservation, :release) if reservation && !admitted
+    end
+
+    def settle_reservation(reservation, action)
+      reservation&.public_send(action)
+    rescue Exception
+      nil
+    end
+
+    def probe_snapshot_locked
+      @probe_generation += 1
+      [@active_probes.values, @probe_generation]
+    end
+
+    def apply_loop
+      loop do
+        snapshot = @mutex.synchronize do
+          @stop_event.wait until @is_shutdown || @pending_probe_update
+          break if @is_shutdown
+
+          update = @pending_probe_update
+          @pending_probe_update = nil
+          update
+        end
+        break unless snapshot
+
+        @engine.set_probes(snapshot[0], generation: snapshot[1])
+      end
+    rescue Exception
+      shutdown
+    end
+
+    def handle_health_change(health, reason = nil)
+      return if @owner_pid != Process.pid || @is_shutdown
+
+      case health
+      when Core::AgentHealth::RED
+        @log_safety.error "Safety Shield Triggered: RED Status. #{reason || ''}"
+        @engine.suspend
+
+        @cooldown_timer&.exit if @cooldown_timer&.alive?
+        cooldown_duration = @cooldown_sec
+        @cooldown_until = Process.clock_gettime(Process::CLOCK_MONOTONIC) + cooldown_duration
+        @cooldown_timer = Thread.new do
+          @log_safety.info "Cooldown period started (#{cooldown_duration}s)..."
+          sleep cooldown_duration
+          @safety_monitor.with_health(Core::AgentHealth::GREEN) do
+            if !@is_shutdown && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @cooldown_until
+              @log_safety.info "Cooldown period ended. Resuming instrumentation..."
+              @engine.resume
+            end
+          end
+        rescue Exception
+          # Agent-owned cooldown work must never abort the application.
+        end
+        @cooldown_timer.kill if @is_shutdown
+      when Core::AgentHealth::YELLOW
+        @log_safety.warn "Safety Warning: YELLOW Status (Moderate Overhead). #{reason || ''}"
+      when Core::AgentHealth::GREEN
+        @log_safety.info "Status back to GREEN. #{reason || ''}"
+        # A timer may be alive briefly after its final health check. Use the
+        # deadline, not thread liveness, so a concurrent GREEN is never lost.
+        if @engine.is_suspended && (!@cooldown_until || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @cooldown_until)
+          @engine.resume
+        end
+      end
+    rescue Exception
+      @is_agent_disabled = true
+    end
+  end
+end