fiber_audit 0.2.1 → 0.3.1

data/lib/fiber_audit/runtime/lifecycle.rb

@@ -14,6 +14,8 @@ require_relative 'scheduler_observer'
 require_relative 'session'
 require_relative 'watchdog'
 require_relative 'watchdog_policy'
+require_relative 'operation_liveness_monitor'
+require_relative 'operation_liveness_policy'
 
 module FiberAudit
   module Runtime
@@ -21,6 +23,7 @@ module FiberAudit
     class Lifecycle
       attr_reader :settings, :owner_pid, :output_path, :recorder,
                   :watchdog, :active_operations, :watchdog_policy,
+                  :operation_liveness_monitor, :operation_liveness_policy,
                   :redactor, :probe_registry, :execution_context_store,
                   :rails_integration
 
@@ -31,6 +34,7 @@ module FiberAudit
       def initialize(
         settings:,
         watchdog_policy: nil,
+        operation_liveness_policy: nil,
         probes_enabled: false,
         clock: Clock.new,
         session_id_source: SecureRandom.method(:uuid),
@@ -39,11 +43,12 @@ module FiberAudit
         random: Sampler::RANDOM_SOURCE
       )
         validate_dependencies!(
-          settings, watchdog_policy, probes_enabled, clock,
+          settings, watchdog_policy, operation_liveness_policy, probes_enabled, clock,
           session_id_source, pid_source, writer_factory, random
         )
         @settings = settings
         @watchdog_policy = watchdog_policy
+        @operation_liveness_policy = operation_liveness_policy
         @probes_enabled = probes_enabled
         @clock = clock
         @session_id_source = session_id_source
@@ -52,15 +57,7 @@ module FiberAudit
         @random = random
         @owner_pid = current_pid
         @state = :starting
-        @output_path = nil
-        @recorder = nil
-        @watchdog = nil
-        @scheduler_observer = nil
-        @active_operations = nil
-        @redactor = nil
-        @probe_registry = nil
-        @execution_context_store = nil
-        @rails_integration = nil
+        reset_process_components!
         start_process_session!
       rescue StandardError => e
         startup_failure!(e)
@@ -84,15 +81,7 @@ module FiberAudit
 
         abandon_inherited_runtime!
         @owner_pid = pid
-        @output_path = nil
-        @recorder = nil
-        @watchdog = nil
-        @scheduler_observer = nil
-        @active_operations = nil
-        @redactor = nil
-        @probe_registry = nil
-        @execution_context_store = nil
-        @rails_integration = nil
+        reset_process_components!
         @state = :starting
         # Reset fiber-local context after fork
         ExecutionContext.reset!
@@ -122,13 +111,18 @@ module FiberAudit
 
       private
 
-      def validate_dependencies!(candidate_settings, watchdog, probes, candidate_clock, session_ids, pids, writers, random)
+      def validate_dependencies!(candidate_settings, watchdog, liveness, probes, candidate_clock,
+                                 session_ids, pids, writers, random)
         unless candidate_settings.is_a?(Environment::Settings)
           raise RuntimeContractError, 'settings must be FiberAudit::Runtime::Environment::Settings'
         end
         unless watchdog.nil? || watchdog.is_a?(WatchdogPolicy)
           raise RuntimeContractError, 'watchdog_policy must be a FiberAudit::Runtime::WatchdogPolicy or nil'
         end
+        unless liveness.nil? || liveness.is_a?(OperationLivenessPolicy)
+          raise RuntimeContractError,
+                'operation_liveness_policy must be a FiberAudit::Runtime::OperationLivenessPolicy or nil'
+        end
         raise RuntimeContractError, 'probes_enabled must be a Boolean' unless [true, false].include?(probes)
         raise RuntimeContractError, 'clock must be a FiberAudit::Runtime::Clock' unless candidate_clock.is_a?(Clock)
 
@@ -142,6 +136,19 @@ module FiberAudit
         end
       end
 
+      def reset_process_components!
+        @output_path = nil
+        @recorder = nil
+        @watchdog = nil
+        @operation_liveness_monitor = nil
+        @scheduler_observer = nil
+        @active_operations = nil
+        @redactor = nil
+        @probe_registry = nil
+        @execution_context_store = nil
+        @rails_integration = nil
+      end
+
       def start_process_session!
         session_id = @session_id_source.call
         started_at = @clock.wall_time
@@ -175,6 +182,7 @@ module FiberAudit
         @redactor = Redactor.new(root: settings.project_root, policy: settings.policy)
         @execution_context_store = ExecutionContext if @probes_enabled
         setup_watchdog! if watchdog_policy
+        setup_operation_liveness! if @probes_enabled && operation_liveness_policy
         setup_rails_integration! if @probes_enabled
         setup_probes! if @probes_enabled
       rescue StandardError => e
@@ -202,6 +210,13 @@ module FiberAudit
         ensure
           @rails_integration = nil
         end
+        begin
+          @operation_liveness_monitor&.stop
+        rescue StandardError
+          nil
+        ensure
+          @operation_liveness_monitor = nil
+        end
         begin
           if @scheduler_observer
             SchedulerObserver.deactivate(@scheduler_observer)
@@ -215,6 +230,15 @@ module FiberAudit
         end
       end
 
+      def setup_operation_liveness!
+        @operation_liveness_monitor = OperationLivenessMonitor.new(
+          policy: operation_liveness_policy,
+          recorder: recorder,
+          active_operations: active_operations,
+          clock: @clock
+        )
+      end
+
       def setup_rails_integration!
         @rails_integration = RailsIntegration.activate(
           context_store: @execution_context_store,
@@ -272,44 +296,61 @@ module FiberAudit
       def abandon_inherited_runtime!
         recorder&.writer&.close
       ensure
-        @recorder = nil
-        @watchdog = nil
-        @scheduler_observer = nil
-        @active_operations = nil
-        @redactor = nil
-        @probe_registry = nil
-        @execution_context_store = nil
-        @rails_integration = nil
+        reset_process_components!
       end
 
       def stop_runtime_observers
         error = nil
-        begin
-          RailsIntegration.deactivate(@rails_integration) if @rails_integration
-        rescue StandardError => e
-          error ||= e
-        ensure
-          @rails_integration = nil
-        end
-        begin
-          Probes::Registry.deactivate(probe_registry) if probe_registry
-        rescue StandardError => e
-          error ||= e
-        ensure
-          @probe_registry = nil
-        end
-        begin
-          SchedulerObserver.deactivate(@scheduler_observer) if @scheduler_observer
-          watchdog&.stop
-        rescue StandardError => e
-          error ||= e
-        ensure
-          @scheduler_observer = nil
-        end
+        component_error = stop_rails_integration
+        error ||= component_error
+        component_error = stop_probe_registry
+        error ||= component_error
+        component_error = stop_operation_liveness_monitor
+        error ||= component_error
+        component_error = stop_watchdog
+        error ||= component_error
         recorder&.internal_error! if error
         error
       end
 
+      def stop_rails_integration
+        RailsIntegration.deactivate(@rails_integration) if @rails_integration
+        nil
+      rescue StandardError => e
+        e
+      ensure
+        @rails_integration = nil
+      end
+
+      def stop_probe_registry
+        Probes::Registry.deactivate(probe_registry) if probe_registry
+        nil
+      rescue StandardError => e
+        e
+      ensure
+        @probe_registry = nil
+      end
+
+      def stop_operation_liveness_monitor
+        operation_liveness_monitor&.stop
+        nil
+      rescue StandardError => e
+        e
+      ensure
+        @operation_liveness_monitor = nil
+      end
+
+      def stop_watchdog
+        SchedulerObserver.deactivate(@scheduler_observer) if @scheduler_observer
+        watchdog&.stop
+        nil
+      rescue StandardError => e
+        e
+      ensure
+        @scheduler_observer = nil
+        @watchdog = nil
+      end
+
       def startup_failure!(error)
         raise error unless defined?(@settings) && settings.policy.fail_open?
 

data/lib/fiber_audit/runtime/operation_liveness_monitor.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+require_relative 'active_operations'
+require_relative 'clock'
+require_relative 'event'
+require_relative 'operation_liveness_policy'
+require_relative 'recorder'
+require_relative 'scheduler_evidence_classifier'
+
+module FiberAudit
+  module Runtime
+    class OperationLivenessMonitor
+      SOURCE = :operation_liveness_monitor
+      STOP_TIMEOUT_SECONDS = 1
+      MAX_START_EVENTS_PER_POLL = 10
+      Tracked = Data.define(:long_active_sequence, :entry)
+
+      attr_reader :policy, :recorder, :active_operations, :state
+
+      def initialize(policy:, recorder:, active_operations:, clock: Clock.new,
+                     thread_factory: ->(&block) { Thread.new(&block) })
+        validate_dependencies!(policy, recorder, active_operations, clock, thread_factory)
+        @policy = policy
+        @recorder = recorder
+        @active_operations = active_operations
+        @clock = clock
+        @thread_factory = thread_factory
+        @poll_mutex = Mutex.new
+        @wait_mutex = Mutex.new
+        @condition = ConditionVariable.new
+        @tracked = {}
+        @long_active_sequence = 0
+        @monitor_thread = nil
+        @state = policy.enabled? ? :starting : :disabled
+        @stopping = false
+        @stopped = false
+        activate!
+      end
+
+      def enabled? = policy.enabled?
+      def fail_open? = recorder.session.policy.fail_open?
+
+      def poll(now_ns: nil)
+        return state unless enabled? && state == :active
+
+        observed_now = now_ns.nil? ? @clock.monotonic_ns : Validation.integer(now_ns, 'operation liveness poll time')
+        @poll_mutex.synchronize { poll_operations(observed_now) }
+        state
+      rescue StandardError => e
+        handle_failure(e)
+        raise e unless fail_open?
+
+        :unsupported
+      end
+
+      def stop
+        return self if @stopped
+
+        @wait_mutex.synchronize do
+          @stopping = true
+          @condition.broadcast
+        end
+        stop_monitor_thread
+        now_ns = safe_monotonic_ns
+        @poll_mutex.synchronize { close_tracked(now_ns, operation_finished: false) }
+        @stopped = true
+        self
+      rescue StandardError => e
+        handle_failure(e)
+        @stopped = true
+        raise e unless fail_open?
+
+        self
+      end
+
+      private
+
+      def activate!
+        if policy.enabled?
+          start_monitor_thread
+          @state = :active
+          emit_state(:operation_liveness_active)
+        else
+          emit_state(:operation_liveness_disabled)
+        end
+      rescue StandardError => e
+        handle_failure(e)
+        raise e unless fail_open?
+      end
+
+      def validate_dependencies!(candidate_policy, candidate_recorder, operations, candidate_clock, threads)
+        unless candidate_policy.is_a?(OperationLivenessPolicy)
+          raise RuntimeContractError, 'policy must be a FiberAudit::Runtime::OperationLivenessPolicy'
+        end
+        unless candidate_recorder.is_a?(Recorder)
+          raise RuntimeContractError,
+                'recorder must be a FiberAudit::Runtime::Recorder'
+        end
+        unless operations.is_a?(ActiveOperations)
+          raise RuntimeContractError, 'active_operations must be FiberAudit::Runtime::ActiveOperations'
+        end
+        raise RuntimeContractError, 'clock must be a FiberAudit::Runtime::Clock' unless candidate_clock.is_a?(Clock)
+        raise RuntimeContractError, 'thread_factory must respond to call' unless threads.respond_to?(:call)
+      end
+
+      def poll_operations(now_ns)
+        snapshot = active_operations.snapshot_with_metadata
+        current = snapshot.entries.to_h { |entry| [entry.sequence, entry] }
+        complete_absent(current, now_ns)
+        start_overdue(snapshot, current, now_ns)
+      end
+
+      def complete_absent(current, now_ns)
+        @tracked.keys.reject { |sequence| current.key?(sequence) }.each do |sequence|
+          emit_completion(@tracked.delete(sequence), now_ns, operation_finished: true)
+        end
+      end
+
+      def start_overdue(snapshot, current, now_ns)
+        overdue = current.values.reject do |entry|
+          @tracked.key?(entry.sequence) || !long_active?(entry, now_ns)
+        end
+        bounded = overdue.first(MAX_START_EVENTS_PER_POLL)
+        batch_truncated = overdue.size > bounded.size
+        bounded.each do |entry|
+          @long_active_sequence += 1
+          tracked = Tracked.new(long_active_sequence: @long_active_sequence, entry: entry)
+          @tracked[entry.sequence] = tracked
+          emit_start(tracked, now_ns, snapshot, batch_truncated, overdue.size)
+        end
+      end
+
+      def long_active?(entry, now_ns)
+        raise RuntimeSafetyError, 'operation liveness clock moved backwards' if now_ns < entry.started_monotonic_ns
+
+        policy.long_active?(age_ns: now_ns - entry.started_monotonic_ns)
+      end
+
+      def classifier_measurements(entry)
+        SchedulerEvidenceClassifier.measurements(
+          operation: entry.operation,
+          scheduler_snapshot: entry.scheduler_snapshot
+        )
+      end
+
+      def emit_start(tracked, now_ns, snapshot, batch_truncated, candidate_count)
+        entry = tracked.entry
+        age_ns = now_ns - entry.started_monotonic_ns
+        measurements = {
+          long_active_sequence: tracked.long_active_sequence,
+          operation_sequence: entry.sequence,
+          operation_started_monotonic_ns: entry.started_monotonic_ns,
+          observed_age_ns: age_ns,
+          long_active_threshold_ns: policy.long_active_threshold_ns,
+          poll_interval_ns: policy.poll_interval_ns,
+          active_operation_total_count: snapshot.total_count,
+          active_operation_snapshot_truncated: snapshot.truncated?,
+          long_active_batch_truncated: batch_truncated,
+          long_active_candidate_count: candidate_count
+        }
+        measurements.merge!(entry.scheduler_snapshot.to_measurements) if entry.scheduler_snapshot
+        measurements.merge!(classifier_measurements(entry))
+        emit_event(kind: :operation_long_active_started, entry: entry, monotonic_ns: now_ns,
+                   duration_ns: age_ns, measurements: measurements)
+      end
+
+      def emit_completion(tracked, now_ns, operation_finished:)
+        entry = tracked.entry
+        duration_ns = [now_ns - entry.started_monotonic_ns, 0].max
+        measurements = {
+          long_active_sequence: tracked.long_active_sequence,
+          operation_sequence: entry.sequence,
+          operation_started_monotonic_ns: entry.started_monotonic_ns,
+          long_active_threshold_ns: policy.long_active_threshold_ns,
+          operation_finished: operation_finished
+        }
+        measurements.merge!(entry.scheduler_snapshot.to_measurements) if entry.scheduler_snapshot
+        measurements.merge!(classifier_measurements(entry))
+        emit_event(kind: :operation_long_active_completed, entry: entry, monotonic_ns: now_ns,
+                   duration_ns: duration_ns, measurements: measurements)
+      end
+
+      def close_tracked(now_ns, operation_finished:)
+        tracked = @tracked.values
+        @tracked.clear
+        tracked.each { |entry| emit_completion(entry, now_ns, operation_finished: operation_finished) }
+      end
+
+      def emit_state(kind)
+        emit_event(kind: kind, entry: nil, monotonic_ns: @clock.monotonic_ns,
+                   measurements: { poll_interval_ns: policy.poll_interval_ns,
+                                   long_active_threshold_ns: policy.long_active_threshold_ns })
+      end
+
+      def emit_event(kind:, entry:, monotonic_ns:, duration_ns: nil, measurements: {})
+        recorder.record_control do
+          Event.new(
+            kind: kind,
+            source: SOURCE,
+            occurred_at: @clock.wall_time,
+            monotonic_ns: monotonic_ns,
+            duration_ns: duration_ns,
+            operation: entry&.operation,
+            location: entry&.location,
+            execution_context: entry&.execution_context || :unknown,
+            thread_id: entry&.thread_id,
+            fiber_id: entry&.fiber_id,
+            measurements: measurements
+          )
+        end
+      end
+
+      def start_monitor_thread
+        thread = @thread_factory.call { monitor_loop }
+        raise RuntimeContractError, 'thread_factory must return a Thread' unless thread.is_a?(Thread)
+
+        @monitor_thread = thread
+        thread.report_on_exception = false
+        thread.abort_on_exception = !fail_open?
+        thread.name = 'fiber-audit-operation-liveness' if thread.respond_to?(:name=)
+      end
+
+      def monitor_loop
+        loop do
+          should_stop = @wait_mutex.synchronize do
+            @condition.wait(@wait_mutex, policy.poll_interval_ms.fdiv(1_000)) unless @stopping
+            @stopping
+          end
+          break if should_stop
+
+          poll
+        end
+      rescue StandardError => e
+        handle_failure(e)
+        raise e unless fail_open?
+      end
+
+      def stop_monitor_thread
+        thread = @monitor_thread
+        return unless thread && thread != Thread.current
+        return if thread.join(STOP_TIMEOUT_SECONDS)
+
+        account_internal_error
+        thread.kill
+        thread.join
+      ensure
+        @monitor_thread = nil
+      end
+
+      def handle_failure(error)
+        first_failure = @state != :unsupported
+        @state = :unsupported
+        @wait_mutex&.synchronize do
+          @stopping = true
+          @condition&.broadcast
+        end
+        account_internal_error
+        begin
+          emit_state(:operation_liveness_unsupported) if first_failure
+        rescue StandardError
+          nil
+        end
+        error
+      end
+
+      def account_internal_error
+        recorder.internal_error!
+      rescue StandardError
+        nil
+      end
+
+      def safe_monotonic_ns
+        @clock.monotonic_ns
+      rescue StandardError => e
+        account_internal_error
+        raise e unless fail_open?
+
+        recorder.session.started_monotonic_ns
+      end
+    end
+  end
+end

data/lib/fiber_audit/runtime/operation_liveness_policy.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative 'validation'
+
+module FiberAudit
+  module Runtime
+    OperationLivenessPolicy = Data.define(:enabled, :poll_interval_ms, :long_active_threshold_ms) do
+      def initialize(**values)
+        unknown = values.keys - OperationLivenessPolicy::DEFAULTS.keys
+        raise RuntimeContractError, "unknown operation liveness policy field: #{unknown.first}" unless unknown.empty?
+
+        fields = OperationLivenessPolicy::DEFAULTS.merge(values)
+        raise RuntimeContractError, 'enabled must be a Boolean' unless [true, false].include?(fields[:enabled])
+
+        limits = OperationLivenessPolicy::LIMITS.to_h do |name, range|
+          value = fields.fetch(name)
+          unless value.is_a?(Integer) && range.cover?(value)
+            raise RuntimeContractError, "#{name} must be an Integer in #{range}"
+          end
+
+          [name, value]
+        end
+        super(enabled: fields[:enabled], **limits)
+      end
+
+      def enabled? = enabled
+      def poll_interval_ns = poll_interval_ms * OperationLivenessPolicy::NANOSECONDS_PER_MILLISECOND
+      def long_active_threshold_ns = long_active_threshold_ms * OperationLivenessPolicy::NANOSECONDS_PER_MILLISECOND
+
+      def long_active?(age_ns:)
+        Validation.integer(age_ns, 'active operation age') > long_active_threshold_ns
+      end
+    end
+
+    OperationLivenessPolicy.const_set(:NANOSECONDS_PER_MILLISECOND, 1_000_000)
+    OperationLivenessPolicy.const_set(:DEFAULTS, {
+      enabled: true,
+      poll_interval_ms: 100,
+      long_active_threshold_ms: 1_000
+    }.freeze)
+    OperationLivenessPolicy.const_set(:LIMITS, {
+      poll_interval_ms: 1..60_000,
+      long_active_threshold_ms: 1..86_400_000
+    }.freeze)
+    OperationLivenessPolicy.const_set(:DISABLED, OperationLivenessPolicy.new(enabled: false))
+  end
+end

data/lib/fiber_audit/runtime/probes/base.rb

@@ -7,6 +7,8 @@ require_relative '../execution_context'
 require_relative '../rails_integration'
 require_relative '../recorder'
 require_relative '../redactor'
+require_relative '../scheduler_evidence_classifier'
+require_relative '../scheduler_snapshot'
 
 module FiberAudit
   module Runtime
@@ -27,7 +29,8 @@ module FiberAudit
           :thread_id,
           :fiber_id,
           :measurements,
-          :execution_context
+          :execution_context,
+          :scheduler_snapshot
         )
 
         attr_reader :recorder, :clock, :redactor, :active_operations, :owner_pid, :execution_context_store
@@ -125,13 +128,15 @@ module FiberAudit
             thread = Thread.current
             fiber = Fiber.current
             captured_context = capture_execution_context
+            captured_scheduler_snapshot = capture_scheduler_snapshot
             handle = active_operations.register(
               operation: canonical_operation,
               monotonic_ns: started_ns,
               location: location,
               execution_context: captured_context,
               thread: thread,
-              fiber: fiber
+              fiber: fiber,
+              scheduler_snapshot: captured_scheduler_snapshot
             )
             Observation.new(
               operation: canonical_operation,
@@ -141,7 +146,8 @@ module FiberAudit
               thread_id: thread.object_id,
               fiber_id: fiber.object_id,
               measurements: normalized_measurements,
-              execution_context: captured_context
+              execution_context: captured_context,
+              scheduler_snapshot: captured_scheduler_snapshot
             )
           end
           # rubocop:enable Metrics/BlockLength
@@ -155,6 +161,10 @@ module FiberAudit
           Context::UNKNOWN
         end
 
+        def capture_scheduler_snapshot
+          SchedulerSnapshotCapture.capture
+        end
+
         def emit_start_observation(observation)
           emit_observation(:operation_started, observation, monotonic_ns: observation.started_monotonic_ns)
         rescue StandardError => e
@@ -207,11 +217,12 @@ module FiberAudit
             values.merge!(generated)
           end
           values[:operation_sequence] = observation.handle&.sequence
-          values
+          enrich_scheduler_evidence(values, observation)
         end
 
         def emit_observation(kind, observation, monotonic_ns:, duration_ns: nil, measurements: nil)
           values = measurements || observation.measurements.merge(operation_sequence: observation.handle&.sequence)
+          values = enrich_scheduler_evidence(values.dup, observation)
           recorder.record do
             Event.new(
               kind: kind,
@@ -229,6 +240,18 @@ module FiberAudit
           end
         end
 
+        def enrich_scheduler_evidence(values, observation)
+          snapshot = observation.scheduler_snapshot
+          values.merge!(snapshot.to_measurements) if snapshot
+          values.merge!(
+            SchedulerEvidenceClassifier.measurements(
+              operation: observation.operation,
+              scheduler_snapshot: snapshot
+            )
+          )
+          values
+        end
+
         def normalize_measurements(value)
           raise RuntimeContractError, 'probe measurements must be a Hash' unless value.is_a?(Hash)