hyperprobe-agent 1.2.27.pre.1 → 1.2.27.pre.3

data/lib/hyperprobe/agent.rb

@@ -3,6 +3,7 @@
 require 'securerandom'
 require 'uri'
 require 'thread'
+require 'monitor'
 require_relative 'version'
 require_relative 'core/quota'
 require_relative 'core/safety'
@@ -24,8 +25,17 @@ module HyperProbe
       @agent_id = SecureRandom.uuid
       @is_shutdown = false
       @is_agent_disabled = false
-      @stop_event = ConditionVariable.new
       @mutex = Monitor.new # Reentrant mutex in Ruby
+      @stop_event = @mutex.new_cond
+      @sync_mutex = Mutex.new
+      @flush_mutex = Mutex.new
+      @shutdown_mutex = Mutex.new
+      @probe_generation = 0
+      @pending_probe_update = nil
+      @finished_probes = {}
+      @telemetry_queue = Queue.new
+      @active_probes = {}
+      @local_hits = {}
 
       @log = Core::Logger.get_logger('hyperprobe:agent')
       @log_broker = Core::Logger.get_logger('hyperprobe:broker')
@@ -34,6 +44,10 @@ module HyperProbe
 
       parse_configuration
 
+      if @global_config[:disable_safe_evaluation]
+        @log.warn 'Safe evaluation is DISABLED. Probe expressions can execute arbitrary Ruby code, mutate application state, or block execution.'
+      end
+
       @quota_manager = Core::QuotaManager.new(@hits_per_sec, @bandwidth_kb_per_sec * 1024)
       @safety_monitor = Core::SafetyMonitor.new(
         method(:handle_health_change),
@@ -53,10 +67,8 @@ module HyperProbe
         enable_keep_alive: @enable_keep_alive
       )
 
-      @telemetry_queue = Queue.new
-      @active_probes = {}
-      @local_hits = {}
       @cooldown_timer = nil
+      @cooldown_until = nil
 
       @engine = Core::MonitoringEngine.new(
         @quota_manager,
@@ -69,16 +81,22 @@ module HyperProbe
       @sync_thread = nil
       @flush_thread = nil
       @stats_thread = nil
+      @apply_thread = Thread.new { apply_loop }
+      @apply_thread.name = 'hyperprobe-apply'
 
       if @is_ephemeral_lambda
         @log.debug 'Running in Ephemeral AWS Lambda Mode.'
       else
         @safety_monitor.start
         start_background_loops
-        sync_with_broker
       end
 
       @log.info "Agent started for #{@service_id} in #{@environment} (v#{VERSION})"
+    rescue SignalException, SystemExit
+      shutdown
+      raise
+    rescue Exception # Agent-owned startup failures must not escape into the host.
+      shutdown
     end
 
     def agent_disabled?
@@ -86,65 +104,93 @@ module HyperProbe
     end
 
     def telemetry_queue_length
-      @telemetry_queue.size
+      @telemetry_queue&.size || 0
     end
 
     def force_sync(timeout_sec = nil)
-      return if @is_shutdown || @is_agent_disabled
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
 
       sync_with_broker(timeout_sec)
     end
 
     def force_flush(timeout_sec = nil)
-      return if @is_shutdown || @is_agent_disabled
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
 
       flush_telemetry(timeout_sec)
     end
 
     def shutdown
-      @mutex.synchronize do
-        return if @is_shutdown
+      return after_fork if @owner_pid && @owner_pid != Process.pid
+      @shutdown_mutex ||= Mutex.new
+      return unless @shutdown_mutex.try_lock
 
-        @is_shutdown = true
-      end
+      shutdown_locked = true
+      return if @is_shutdown
 
-      @safety_monitor&.stop
-      @engine&.close
-      @cooldown_timer&.exit if @cooldown_timer&.alive?
-      @broker_client&.shutdown
+      @is_shutdown = true
+      @is_agent_disabled = true
+      threads = [@sync_thread, @flush_thread, @stats_thread, @apply_thread, @cooldown_timer].compact
+      threads.each { |thread| thread.kill unless thread == Thread.current }
 
-      [@sync_thread, @flush_thread, @stats_thread].compact.each do |t|
-        t.kill if t.alive? && t != Thread.current
-        t.join(1.0) rescue nil
+      # Cleanup independently: a broken or stuck resource cannot prevent the rest.
+      cleanup = [[@engine, :close], [@safety_monitor, :stop], [@broker_client, :shutdown]].each_with_object([]) do |(resource, method), workers|
+        next unless resource
+
+        workers << Thread.new do
+          resource.public_send(method)
+        rescue Exception
+          # Only agent-owned cleanup runs in this thread.
+        end
+      end
+      cleanup << Thread.new do
+        @mutex.synchronize do
+          until @telemetry_queue.empty?
+            settle_reservation(@telemetry_queue.pop(true)[:reservation], :release)
+          end
+        end
+      rescue Exception
+        # Reservations belong only to the stopped agent.
+      end
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.5
+      (threads + cleanup).each do |thread|
+        next if thread == Thread.current
+
+        remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        thread.join(remaining) if remaining.positive?
+      rescue SignalException, SystemExit
+        raise
+      rescue Exception
+        # Continue closing the other resources, including from a signal trap.
       end
+      cleanup.each do |thread|
+        thread.kill if thread.alive?
+        thread.join(0.01)
+      rescue SignalException, SystemExit
+        raise
+      rescue Exception
+        # Never wait indefinitely for a broken cleanup thread.
+      end
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      @is_shutdown = true
+      @is_agent_disabled = true
+    ensure
+      @shutdown_mutex.unlock if shutdown_locked
     end
 
     def after_fork
-      return if defined?(JRUBY_VERSION)
-
-      @mutex.synchronize do
-        return if @is_shutdown
-
-        @owner_pid = Process.pid
-        @agent_id = SecureRandom.uuid
-
-        # Reset queues and client for new process
-        @telemetry_queue = Queue.new
-        @broker_client = Core::BrokerClient.new(
-          broker_url: @broker_url,
-          service_id: @service_id,
-          environment: @environment,
-          commit_sha: @commit_sha,
-          agent_id: @agent_id,
-          agent_version: VERSION,
-          rpc_timeout_sec: @rpc_timeout_sec,
-          enable_keep_alive: @enable_keep_alive
-        )
+      return if @owner_pid == Process.pid
 
-        @safety_monitor.start
-        start_background_loops
-        sync_with_broker
-      end
+      # Never call, close, or replace an inherited native gRPC client here.
+      @is_shutdown = true
+      @is_agent_disabled = true
+      @engine&.after_fork
+      false
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      false
     end
 
     private
@@ -155,8 +201,11 @@ module HyperProbe
       @broker_url = get_opt(:broker_url, :brokerUrl) || ENV['HYPERPROBE_BROKER_URL']
       @commit_sha = get_opt(:commit_sha, :commitSha) || ENV['GIT_COMMIT'] || ENV['HYPERPROBE_COMMIT_SHA']
 
-      @sync_interval_sec = ((get_opt(:sync_interval_ms, :syncIntervalMs) || parse_env_int('HYPERPROBE_SYNC_INTERVAL_MS', 60_000)).to_f / 1000.0)
-      @flush_interval_sec = ((get_opt(:flush_interval_ms, :flushIntervalMs) || parse_env_int('HYPERPROBE_FLUSH_INTERVAL_MS', 1000)).to_f / 1000.0)
+      raw_sync = get_opt(:sync_interval_ms, :syncIntervalMs) || parse_env_int('HYPERPROBE_SYNC_INTERVAL_MS', 60_000)
+      @sync_interval_sec = [raw_sync.to_f / 1000.0, 0.1].max
+
+      raw_flush = get_opt(:flush_interval_ms, :flushIntervalMs) || parse_env_int('HYPERPROBE_FLUSH_INTERVAL_MS', 1000)
+      @flush_interval_sec = [raw_flush.to_f / 1000.0, 0.05].max
       @max_queue_size = get_opt(:max_queue_size, :maxQueueSize) || parse_env_int('HYPERPROBE_MAX_QUEUE_SIZE', 100)
       @cooldown_sec = get_opt(:cooldown_sec, :cooldownSec) || parse_env_int('HYPERPROBE_COOLDOWN_SEC', 10)
 
@@ -168,7 +217,8 @@ module HyperProbe
 
       is_aws_lambda = !ENV['AWS_LAMBDA_FUNCTION_NAME'].nil?
       is_local_emulator = ENV['IS_OFFLINE'] == 'true' || ENV['AWS_SAM_LOCAL'] == 'true'
-      @is_ephemeral_lambda = (get_opt(:is_lambda, :isLambda) || is_aws_lambda) && !is_local_emulator
+      lambda_opt = get_opt(:is_lambda, :isLambda)
+      @is_ephemeral_lambda = (lambda_opt.nil? ? is_aws_lambda : lambda_opt) && !is_local_emulator
       enable_keep_alive_opt = get_opt(:enable_keep_alive, :enableKeepAlive)
       @enable_keep_alive = enable_keep_alive_opt.nil? ? !@is_ephemeral_lambda : enable_keep_alive_opt
 
@@ -182,7 +232,9 @@ module HyperProbe
         max_array_length: get_opt(:max_array_length, :maxArrayLength) || parse_env_int('HYPERPROBE_MAX_ARRAY_LENGTH', 3),
         stack_frame_depth: get_opt(:stack_frame_depth, :stackFrameDepth) || parse_env_int('HYPERPROBE_STACK_FRAME_DEPTH', 3),
         max_object_properties: get_opt(:max_object_properties, :maxObjectProperties) || parse_env_int('HYPERPROBE_MAX_OBJECT_PROPERTIES', 50),
-        max_string_length: get_opt(:max_string_length, :maxStringLength) || parse_env_int('HYPERPROBE_MAX_STRING_LENGTH', 1024)
+        max_string_length: get_opt(:max_string_length, :maxStringLength) || parse_env_int('HYPERPROBE_MAX_STRING_LENGTH', 1024),
+        capture_closures: get_opt(:capture_closures, :captureClosures) == true,
+        disable_safe_evaluation: !Core::Evaluator.safe_evaluation_enabled?(get_opt(:disable_safe_evaluation, :disableSafeEvaluation))
       }
     end
 
@@ -214,19 +266,21 @@ module HyperProbe
 
     def sync_loop
       until @is_shutdown
-        sleep @sync_interval_sec
-        break if @is_shutdown
-
         begin
           sync_with_broker
-        rescue StandardError => e
+          sleep @sync_interval_sec
+        rescue Exception
           # Silently handle transient sync error
+          break
         end
       end
     end
 
     def sync_with_broker(timeout_sec = nil)
-      return if @is_shutdown
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+      return false unless @sync_mutex.try_lock
+
+      sync_locked = true
 
       @log_broker.debug "sync_with_broker started #{timeout_sec ? "with timeout #{timeout_sec}" : ''}"
       response = @broker_client.get_probes(timeout_sec)
@@ -234,181 +288,216 @@ module HyperProbe
 
       @log_broker.debug "sync_with_broker: got #{response.probes.length} probes from broker"
 
-      # Dynamic global config updates from broker
-      if response.global_config
-        gc = response.global_config
-        @mutex.synchronize do
-          @global_config[:redact_keys] = gc.redact_keys.to_a unless gc.redact_keys.empty?
-          @global_config[:redact_values] = gc.redact_values.to_a unless gc.redact_values.empty?
-          @global_config[:max_object_depth] = gc.max_object_depth if gc.max_object_depth.positive?
-          @global_config[:max_array_length] = gc.max_array_length if gc.max_array_length.positive?
-          @global_config[:stack_frame_depth] = gc.stack_frame_depth if gc.stack_frame_depth.positive?
-          @global_config[:max_object_properties] = gc.max_object_properties if gc.max_object_properties.positive?
-          @global_config[:max_string_length] = gc.max_string_length if gc.max_string_length.positive?
-        end
-        @engine.set_global_config(@global_config)
-      end
-
       server_probes = response.probes.to_a
       server_probe_ids = server_probes.map(&:id)
-
-      to_apply = []
       now_ms = (Time.now.to_f * 1000).to_i
+      config = nil
+      snapshot = @mutex.synchronize do
+        return false if @is_shutdown
 
-      @mutex.synchronize do
+        if response.global_config
+          gc = response.global_config
+          @global_config[:redact_keys] = gc.redact_keys.to_a unless gc.redact_keys.empty?
+          @global_config[:redact_values] = gc.redact_values.to_a unless gc.redact_values.empty?
+          %i[max_object_depth max_array_length stack_frame_depth max_object_properties max_string_length capture_closures].each do |field|
+            @global_config[field] = gc.public_send(field) if gc.public_send("has_#{field}?")
+          end
+          config = @global_config.dup
+        end
         # Clean stale local probes
         @active_probes.delete_if { |pid, _| !server_probe_ids.include?(pid) }
         @local_hits.delete_if { |pid, _| !server_probe_ids.include?(pid) }
+        @finished_probes.delete_if { |pid, _| !server_probe_ids.include?(pid) }
 
         server_probes.each do |probe|
           hits = @local_hits[probe.id] || 0
           is_expired = probe.expiry_time.positive? && probe.expiry_time <= now_ms
 
-          if hits < probe.hit_limit && !is_expired
-            to_apply << probe
+          if hits < probe.hit_limit && !is_expired && !@finished_probes[probe.id]
             @active_probes[probe.id] = probe
           else
             @active_probes.delete(probe.id)
           end
         end
+        probe_snapshot_locked
       end
 
-      @log_broker.debug "Found #{to_apply.length} active probes to apply locally"
-      @engine.set_probes(to_apply)
+      @engine.set_global_config(config) if config
+      @engine.set_probes(snapshot[0], generation: snapshot[1])
       true
-    rescue StandardError => e
-      @log_broker.error("Failed to sync with broker: #{e.message}", e)
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
       false
+    ensure
+      @sync_mutex.unlock if sync_locked
     end
 
     def flush_loop
       until @is_shutdown
-        sleep @flush_interval_sec
-        break if @is_shutdown
-
         begin
+          sleep @flush_interval_sec
+          break if @is_shutdown
+
           flush_telemetry
-        rescue StandardError => e
+        rescue Exception
           # Silently handle flush loop error
         end
       end
     end
 
     def flush_telemetry(timeout_sec = nil)
-      return if @is_shutdown || @telemetry_queue.empty?
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+      return false unless @flush_mutex.try_lock
+
+      flush_locked = true
 
       batch = []
-      until @telemetry_queue.empty?
-        begin
-          batch << @telemetry_queue.pop(true)
-        rescue ThreadError
-          break
-        end
+      @mutex.synchronize do
+        batch << @telemetry_queue.pop(true) until @telemetry_queue.empty?
       end
 
       return if batch.empty?
 
-      @log_broker.info "Flushing #{batch.length} telemetry events to broker..."
-
       begin
         events = batch.map { |item| item[:event] }
         finished_probe_ids = @broker_client.report_telemetry(events, timeout_sec)
-
-        @log_broker.info "Successfully flushed #{batch.length} telemetry events to broker"
+        delivered = true
 
         # Commit bandwidth reservations
         batch.each do |item|
-          item[:reservation]&.commit
+          settle_reservation(item[:reservation], :commit)
         end
 
         # Handle globally finished probes
         if finished_probe_ids && !finished_probe_ids.empty?
-          changed = false
-          @mutex.synchronize do
+          snapshot = @mutex.synchronize do
+            return if @is_shutdown
+
             finished_probe_ids.each do |pid|
-              if @active_probes.key?(pid)
-                @active_probes.delete(pid)
-                changed = true
-              end
+              @finished_probes[pid] = true
+              @active_probes.delete(pid)
             end
+            probe_snapshot_locked
           end
-          @engine.set_probes(@active_probes.values) if changed
+          @engine.set_probes(snapshot[0], generation: snapshot[1])
         end
-      rescue StandardError => e
-        @log_broker.error("Failed to flush telemetry: #{e.message}", e)
+      rescue SignalException, SystemExit
+        raise
+      rescue Exception
+        return false if delivered
+
         # On flush failure, re-queue events up to max capacity
         batch.each do |item|
-          item[:reservation]&.release
-          if @telemetry_queue.size < @max_queue_size
-            @telemetry_queue.push(item)
+          retained = @mutex.synchronize do
+            if !@is_shutdown && @telemetry_queue.size < @max_queue_size
+              @telemetry_queue.push(item)
+              true
+            end
           end
+          settle_reservation(item[:reservation], :release) unless retained
         end
       end
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      false
+    ensure
+      if @is_shutdown && batch && !delivered
+        batch.each { |item| settle_reservation(item[:reservation], :release) }
+      end
+      @flush_mutex.unlock if flush_locked
     end
 
     def stats_loop
       until @is_shutdown
-        sleep 5.0
-        break if @is_shutdown
-
         begin
+          sleep 5.0
+          break if @is_shutdown
+
           stats = @engine.get_stats
           if stats[:hits].positive? || stats[:skips].positive?
             @log_stats.info "Probes Hit: #{stats[:hits]}, Probes Skipped: #{stats[:skips]}"
           end
-        rescue StandardError
+        rescue Exception
           # Ignore stats error
         end
       end
     end
 
     def handle_capture(event)
+      return if @owner_pid != Process.pid || @is_shutdown || @is_agent_disabled
+      return unless @mutex.try_enter
+
+      locked = true
       return if @is_shutdown || @is_agent_disabled
 
       probe_id = event[:probe_id]
-      @log.debug "Captured event for probe #{probe_id}"
-      probe = nil
-      hits = 0
-
-      @mutex.synchronize do
-        probe = @active_probes[probe_id]
-        return unless probe
-
-        hits = (@local_hits[probe_id] || 0) + 1
-        @local_hits[probe_id] = hits
+      probe = @active_probes[probe_id]
+      return unless probe
+
+      raw_limit = probe.respond_to?(:hit_limit) ? probe.hit_limit : probe[:hit_limit]
+      hit_limit = raw_limit.to_i
+      hits = @local_hits[probe_id] || 0
+      return if hit_limit.positive? && hits >= hit_limit
+
+      # Attempted captures consume the safety fuse even when delivery is dropped.
+      @local_hits[probe_id] = hits + 1
+      if hit_limit.positive? && hits + 1 >= hit_limit
+        @active_probes.delete(probe_id)
+        @pending_probe_update = probe_snapshot_locked
+        @stop_event.broadcast
       end
 
-      # Reserve bandwidth
+      return if @telemetry_queue.size >= @max_queue_size
+
       event_size = @broker_client.estimate_event_size(event)
       reservation = @quota_manager.reserve_bandwidth(event_size)
-      unless reservation
-        @log.debug "Bandwidth quota exceeded; dropping event for probe #{probe_id}"
-        return
-      end
+      return unless reservation
+
+      @telemetry_queue.push(event: event, reservation: reservation)
+      admitted = true
+    rescue SignalException, SystemExit
+      raise
+    rescue Exception
+      # This callback is agent code running on an application thread.
+      nil
+    ensure
+      @mutex.exit if locked
+      settle_reservation(reservation, :release) if reservation && !admitted
+    end
 
-      # Admission to telemetry queue
-      item = { event: event, reservation: reservation }
-      if @telemetry_queue.size < @max_queue_size
-        @telemetry_queue.push(item)
-      else
-        @log.debug "Telemetry queue full. Dropping event for probe #{probe_id}"
-        reservation.release
-      end
+    def settle_reservation(reservation, action)
+      reservation&.public_send(action)
+    rescue Exception
+      nil
+    end
 
-      # Enforce hitLimit fuse
-      hit_limit = probe.respond_to?(:hit_limit) ? probe.hit_limit : probe[:hit_limit]
-      if hits >= hit_limit
-        remaining_probes = nil
-        @mutex.synchronize do
-          @active_probes.delete(probe_id)
-          remaining_probes = @active_probes.values
+    def probe_snapshot_locked
+      @probe_generation += 1
+      [@active_probes.values, @probe_generation]
+    end
+
+    def apply_loop
+      loop do
+        snapshot = @mutex.synchronize do
+          @stop_event.wait until @is_shutdown || @pending_probe_update
+          break if @is_shutdown
+
+          update = @pending_probe_update
+          @pending_probe_update = nil
+          update
         end
-        @engine.set_probes(remaining_probes)
+        break unless snapshot
+
+        @engine.set_probes(snapshot[0], generation: snapshot[1])
       end
+    rescue Exception
+      shutdown
     end
 
     def handle_health_change(health, reason = nil)
-      return if @is_shutdown
+      return if @owner_pid != Process.pid || @is_shutdown
 
       case health
       when Core::AgentHealth::RED
@@ -417,22 +506,32 @@ module HyperProbe
 
         @cooldown_timer&.exit if @cooldown_timer&.alive?
         cooldown_duration = @cooldown_sec
+        @cooldown_until = Process.clock_gettime(Process::CLOCK_MONOTONIC) + cooldown_duration
         @cooldown_timer = Thread.new do
           @log_safety.info "Cooldown period started (#{cooldown_duration}s)..."
           sleep cooldown_duration
-          if @safety_monitor.get_health == Core::AgentHealth::GREEN
-            @log_safety.info "Cooldown period ended. Resuming instrumentation..."
-            @engine.resume
+          @safety_monitor.with_health(Core::AgentHealth::GREEN) do
+            if !@is_shutdown && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @cooldown_until
+              @log_safety.info "Cooldown period ended. Resuming instrumentation..."
+              @engine.resume
+            end
           end
+        rescue Exception
+          # Agent-owned cooldown work must never abort the application.
         end
+        @cooldown_timer.kill if @is_shutdown
       when Core::AgentHealth::YELLOW
         @log_safety.warn "Safety Warning: YELLOW Status (Moderate Overhead). #{reason || ''}"
       when Core::AgentHealth::GREEN
         @log_safety.info "Status back to GREEN. #{reason || ''}"
-        if @engine.is_suspended && (!@cooldown_timer || !@cooldown_timer.alive?)
+        # A timer may be alive briefly after its final health check. Use the
+        # deadline, not thread liveness, so a concurrent GREEN is never lost.
+        if @engine.is_suspended && (!@cooldown_until || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @cooldown_until)
           @engine.resume
         end
       end
+    rescue Exception
+      @is_agent_disabled = true
     end
   end
 end