fiber_audit 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.fiber-audit.example.yml +7 -8
- data/ARCHITECTURE.md +92 -670
- data/CHANGELOG.md +29 -0
- data/README.md +145 -92
- data/lib/fiber_audit/cli.rb +3 -1
- data/lib/fiber_audit/configuration.rb +31 -6
- data/lib/fiber_audit/operation_semantics.rb +172 -0
- data/lib/fiber_audit/runtime/active_operations.rb +25 -9
- data/lib/fiber_audit/runtime/boot.rb +4 -0
- data/lib/fiber_audit/runtime/environment.rb +58 -0
- data/lib/fiber_audit/runtime/lifecycle.rb +91 -50
- data/lib/fiber_audit/runtime/operation_liveness_monitor.rb +282 -0
- data/lib/fiber_audit/runtime/operation_liveness_policy.rb +47 -0
- data/lib/fiber_audit/runtime/probes/base.rb +12 -10
- data/lib/fiber_audit/runtime/scheduler_evidence_classifier.rb +60 -0
- data/lib/fiber_audit/runtime/scheduler_snapshot.rb +6 -24
- data/lib/fiber_audit/runtime/watchdog.rb +17 -9
- data/lib/fiber_audit/runtime.rb +4 -0
- data/lib/fiber_audit/static/rules/blocking_subprocess.rb +3 -26
- data/lib/fiber_audit/static/rules/direct_socket.rb +52 -16
- data/lib/fiber_audit/static/rules/thread_current_state.rb +12 -20
- data/lib/fiber_audit/version.rb +1 -1
- metadata +5 -1
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'active_operations'
|
|
4
|
+
require_relative 'clock'
|
|
5
|
+
require_relative 'event'
|
|
6
|
+
require_relative 'operation_liveness_policy'
|
|
7
|
+
require_relative 'recorder'
|
|
8
|
+
require_relative 'scheduler_evidence_classifier'
|
|
9
|
+
|
|
10
|
+
module FiberAudit
|
|
11
|
+
module Runtime
|
|
12
|
+
class OperationLivenessMonitor
|
|
13
|
+
SOURCE = :operation_liveness_monitor
|
|
14
|
+
STOP_TIMEOUT_SECONDS = 1
|
|
15
|
+
MAX_START_EVENTS_PER_POLL = 10
|
|
16
|
+
Tracked = Data.define(:long_active_sequence, :entry)
|
|
17
|
+
|
|
18
|
+
attr_reader :policy, :recorder, :active_operations, :state
|
|
19
|
+
|
|
20
|
+
def initialize(policy:, recorder:, active_operations:, clock: Clock.new,
|
|
21
|
+
thread_factory: ->(&block) { Thread.new(&block) })
|
|
22
|
+
validate_dependencies!(policy, recorder, active_operations, clock, thread_factory)
|
|
23
|
+
@policy = policy
|
|
24
|
+
@recorder = recorder
|
|
25
|
+
@active_operations = active_operations
|
|
26
|
+
@clock = clock
|
|
27
|
+
@thread_factory = thread_factory
|
|
28
|
+
@poll_mutex = Mutex.new
|
|
29
|
+
@wait_mutex = Mutex.new
|
|
30
|
+
@condition = ConditionVariable.new
|
|
31
|
+
@tracked = {}
|
|
32
|
+
@long_active_sequence = 0
|
|
33
|
+
@monitor_thread = nil
|
|
34
|
+
@state = policy.enabled? ? :starting : :disabled
|
|
35
|
+
@stopping = false
|
|
36
|
+
@stopped = false
|
|
37
|
+
activate!
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def enabled? = policy.enabled?
|
|
41
|
+
def fail_open? = recorder.session.policy.fail_open?
|
|
42
|
+
|
|
43
|
+
def poll(now_ns: nil)
|
|
44
|
+
return state unless enabled? && state == :active
|
|
45
|
+
|
|
46
|
+
observed_now = now_ns.nil? ? @clock.monotonic_ns : Validation.integer(now_ns, 'operation liveness poll time')
|
|
47
|
+
@poll_mutex.synchronize { poll_operations(observed_now) }
|
|
48
|
+
state
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
handle_failure(e)
|
|
51
|
+
raise e unless fail_open?
|
|
52
|
+
|
|
53
|
+
:unsupported
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def stop
|
|
57
|
+
return self if @stopped
|
|
58
|
+
|
|
59
|
+
@wait_mutex.synchronize do
|
|
60
|
+
@stopping = true
|
|
61
|
+
@condition.broadcast
|
|
62
|
+
end
|
|
63
|
+
stop_monitor_thread
|
|
64
|
+
now_ns = safe_monotonic_ns
|
|
65
|
+
@poll_mutex.synchronize { close_tracked(now_ns, operation_finished: false) }
|
|
66
|
+
@stopped = true
|
|
67
|
+
self
|
|
68
|
+
rescue StandardError => e
|
|
69
|
+
handle_failure(e)
|
|
70
|
+
@stopped = true
|
|
71
|
+
raise e unless fail_open?
|
|
72
|
+
|
|
73
|
+
self
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def activate!
|
|
79
|
+
if policy.enabled?
|
|
80
|
+
start_monitor_thread
|
|
81
|
+
@state = :active
|
|
82
|
+
emit_state(:operation_liveness_active)
|
|
83
|
+
else
|
|
84
|
+
emit_state(:operation_liveness_disabled)
|
|
85
|
+
end
|
|
86
|
+
rescue StandardError => e
|
|
87
|
+
handle_failure(e)
|
|
88
|
+
raise e unless fail_open?
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def validate_dependencies!(candidate_policy, candidate_recorder, operations, candidate_clock, threads)
|
|
92
|
+
unless candidate_policy.is_a?(OperationLivenessPolicy)
|
|
93
|
+
raise RuntimeContractError, 'policy must be a FiberAudit::Runtime::OperationLivenessPolicy'
|
|
94
|
+
end
|
|
95
|
+
unless candidate_recorder.is_a?(Recorder)
|
|
96
|
+
raise RuntimeContractError,
|
|
97
|
+
'recorder must be a FiberAudit::Runtime::Recorder'
|
|
98
|
+
end
|
|
99
|
+
unless operations.is_a?(ActiveOperations)
|
|
100
|
+
raise RuntimeContractError, 'active_operations must be FiberAudit::Runtime::ActiveOperations'
|
|
101
|
+
end
|
|
102
|
+
raise RuntimeContractError, 'clock must be a FiberAudit::Runtime::Clock' unless candidate_clock.is_a?(Clock)
|
|
103
|
+
raise RuntimeContractError, 'thread_factory must respond to call' unless threads.respond_to?(:call)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def poll_operations(now_ns)
|
|
107
|
+
snapshot = active_operations.snapshot_with_metadata
|
|
108
|
+
current = snapshot.entries.to_h { |entry| [entry.sequence, entry] }
|
|
109
|
+
complete_absent(current, now_ns)
|
|
110
|
+
start_overdue(snapshot, current, now_ns)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def complete_absent(current, now_ns)
|
|
114
|
+
@tracked.keys.reject { |sequence| current.key?(sequence) }.each do |sequence|
|
|
115
|
+
emit_completion(@tracked.delete(sequence), now_ns, operation_finished: true)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def start_overdue(snapshot, current, now_ns)
|
|
120
|
+
overdue = current.values.reject do |entry|
|
|
121
|
+
@tracked.key?(entry.sequence) || !long_active?(entry, now_ns)
|
|
122
|
+
end
|
|
123
|
+
bounded = overdue.first(MAX_START_EVENTS_PER_POLL)
|
|
124
|
+
batch_truncated = overdue.size > bounded.size
|
|
125
|
+
bounded.each do |entry|
|
|
126
|
+
@long_active_sequence += 1
|
|
127
|
+
tracked = Tracked.new(long_active_sequence: @long_active_sequence, entry: entry)
|
|
128
|
+
@tracked[entry.sequence] = tracked
|
|
129
|
+
emit_start(tracked, now_ns, snapshot, batch_truncated, overdue.size)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def long_active?(entry, now_ns)
|
|
134
|
+
raise RuntimeSafetyError, 'operation liveness clock moved backwards' if now_ns < entry.started_monotonic_ns
|
|
135
|
+
|
|
136
|
+
policy.long_active?(age_ns: now_ns - entry.started_monotonic_ns)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def classifier_measurements(entry)
|
|
140
|
+
SchedulerEvidenceClassifier.measurements(
|
|
141
|
+
operation: entry.operation,
|
|
142
|
+
scheduler_snapshot: entry.scheduler_snapshot
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def emit_start(tracked, now_ns, snapshot, batch_truncated, candidate_count)
|
|
147
|
+
entry = tracked.entry
|
|
148
|
+
age_ns = now_ns - entry.started_monotonic_ns
|
|
149
|
+
measurements = {
|
|
150
|
+
long_active_sequence: tracked.long_active_sequence,
|
|
151
|
+
operation_sequence: entry.sequence,
|
|
152
|
+
operation_started_monotonic_ns: entry.started_monotonic_ns,
|
|
153
|
+
observed_age_ns: age_ns,
|
|
154
|
+
long_active_threshold_ns: policy.long_active_threshold_ns,
|
|
155
|
+
poll_interval_ns: policy.poll_interval_ns,
|
|
156
|
+
active_operation_total_count: snapshot.total_count,
|
|
157
|
+
active_operation_snapshot_truncated: snapshot.truncated?,
|
|
158
|
+
long_active_batch_truncated: batch_truncated,
|
|
159
|
+
long_active_candidate_count: candidate_count
|
|
160
|
+
}
|
|
161
|
+
measurements.merge!(entry.scheduler_snapshot.to_measurements) if entry.scheduler_snapshot
|
|
162
|
+
measurements.merge!(classifier_measurements(entry))
|
|
163
|
+
emit_event(kind: :operation_long_active_started, entry: entry, monotonic_ns: now_ns,
|
|
164
|
+
duration_ns: age_ns, measurements: measurements)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def emit_completion(tracked, now_ns, operation_finished:)
|
|
168
|
+
entry = tracked.entry
|
|
169
|
+
duration_ns = [now_ns - entry.started_monotonic_ns, 0].max
|
|
170
|
+
measurements = {
|
|
171
|
+
long_active_sequence: tracked.long_active_sequence,
|
|
172
|
+
operation_sequence: entry.sequence,
|
|
173
|
+
operation_started_monotonic_ns: entry.started_monotonic_ns,
|
|
174
|
+
long_active_threshold_ns: policy.long_active_threshold_ns,
|
|
175
|
+
operation_finished: operation_finished
|
|
176
|
+
}
|
|
177
|
+
measurements.merge!(entry.scheduler_snapshot.to_measurements) if entry.scheduler_snapshot
|
|
178
|
+
measurements.merge!(classifier_measurements(entry))
|
|
179
|
+
emit_event(kind: :operation_long_active_completed, entry: entry, monotonic_ns: now_ns,
|
|
180
|
+
duration_ns: duration_ns, measurements: measurements)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def close_tracked(now_ns, operation_finished:)
|
|
184
|
+
tracked = @tracked.values
|
|
185
|
+
@tracked.clear
|
|
186
|
+
tracked.each { |entry| emit_completion(entry, now_ns, operation_finished: operation_finished) }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def emit_state(kind)
|
|
190
|
+
emit_event(kind: kind, entry: nil, monotonic_ns: @clock.monotonic_ns,
|
|
191
|
+
measurements: { poll_interval_ns: policy.poll_interval_ns,
|
|
192
|
+
long_active_threshold_ns: policy.long_active_threshold_ns })
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def emit_event(kind:, entry:, monotonic_ns:, duration_ns: nil, measurements: {})
|
|
196
|
+
recorder.record_control do
|
|
197
|
+
Event.new(
|
|
198
|
+
kind: kind,
|
|
199
|
+
source: SOURCE,
|
|
200
|
+
occurred_at: @clock.wall_time,
|
|
201
|
+
monotonic_ns: monotonic_ns,
|
|
202
|
+
duration_ns: duration_ns,
|
|
203
|
+
operation: entry&.operation,
|
|
204
|
+
location: entry&.location,
|
|
205
|
+
execution_context: entry&.execution_context || :unknown,
|
|
206
|
+
thread_id: entry&.thread_id,
|
|
207
|
+
fiber_id: entry&.fiber_id,
|
|
208
|
+
measurements: measurements
|
|
209
|
+
)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def start_monitor_thread
|
|
214
|
+
thread = @thread_factory.call { monitor_loop }
|
|
215
|
+
raise RuntimeContractError, 'thread_factory must return a Thread' unless thread.is_a?(Thread)
|
|
216
|
+
|
|
217
|
+
@monitor_thread = thread
|
|
218
|
+
thread.report_on_exception = false
|
|
219
|
+
thread.abort_on_exception = !fail_open?
|
|
220
|
+
thread.name = 'fiber-audit-operation-liveness' if thread.respond_to?(:name=)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def monitor_loop
|
|
224
|
+
loop do
|
|
225
|
+
should_stop = @wait_mutex.synchronize do
|
|
226
|
+
@condition.wait(@wait_mutex, policy.poll_interval_ms.fdiv(1_000)) unless @stopping
|
|
227
|
+
@stopping
|
|
228
|
+
end
|
|
229
|
+
break if should_stop
|
|
230
|
+
|
|
231
|
+
poll
|
|
232
|
+
end
|
|
233
|
+
rescue StandardError => e
|
|
234
|
+
handle_failure(e)
|
|
235
|
+
raise e unless fail_open?
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def stop_monitor_thread
|
|
239
|
+
thread = @monitor_thread
|
|
240
|
+
return unless thread && thread != Thread.current
|
|
241
|
+
return if thread.join(STOP_TIMEOUT_SECONDS)
|
|
242
|
+
|
|
243
|
+
account_internal_error
|
|
244
|
+
thread.kill
|
|
245
|
+
thread.join
|
|
246
|
+
ensure
|
|
247
|
+
@monitor_thread = nil
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def handle_failure(error)
|
|
251
|
+
first_failure = @state != :unsupported
|
|
252
|
+
@state = :unsupported
|
|
253
|
+
@wait_mutex&.synchronize do
|
|
254
|
+
@stopping = true
|
|
255
|
+
@condition&.broadcast
|
|
256
|
+
end
|
|
257
|
+
account_internal_error
|
|
258
|
+
begin
|
|
259
|
+
emit_state(:operation_liveness_unsupported) if first_failure
|
|
260
|
+
rescue StandardError
|
|
261
|
+
nil
|
|
262
|
+
end
|
|
263
|
+
error
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def account_internal_error
|
|
267
|
+
recorder.internal_error!
|
|
268
|
+
rescue StandardError
|
|
269
|
+
nil
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def safe_monotonic_ns
|
|
273
|
+
@clock.monotonic_ns
|
|
274
|
+
rescue StandardError => e
|
|
275
|
+
account_internal_error
|
|
276
|
+
raise e unless fail_open?
|
|
277
|
+
|
|
278
|
+
recorder.session.started_monotonic_ns
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'validation'
|
|
4
|
+
|
|
5
|
+
module FiberAudit
|
|
6
|
+
module Runtime
|
|
7
|
+
OperationLivenessPolicy = Data.define(:enabled, :poll_interval_ms, :long_active_threshold_ms) do
|
|
8
|
+
def initialize(**values)
|
|
9
|
+
unknown = values.keys - OperationLivenessPolicy::DEFAULTS.keys
|
|
10
|
+
raise RuntimeContractError, "unknown operation liveness policy field: #{unknown.first}" unless unknown.empty?
|
|
11
|
+
|
|
12
|
+
fields = OperationLivenessPolicy::DEFAULTS.merge(values)
|
|
13
|
+
raise RuntimeContractError, 'enabled must be a Boolean' unless [true, false].include?(fields[:enabled])
|
|
14
|
+
|
|
15
|
+
limits = OperationLivenessPolicy::LIMITS.to_h do |name, range|
|
|
16
|
+
value = fields.fetch(name)
|
|
17
|
+
unless value.is_a?(Integer) && range.cover?(value)
|
|
18
|
+
raise RuntimeContractError, "#{name} must be an Integer in #{range}"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
[name, value]
|
|
22
|
+
end
|
|
23
|
+
super(enabled: fields[:enabled], **limits)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def enabled? = enabled
|
|
27
|
+
def poll_interval_ns = poll_interval_ms * OperationLivenessPolicy::NANOSECONDS_PER_MILLISECOND
|
|
28
|
+
def long_active_threshold_ns = long_active_threshold_ms * OperationLivenessPolicy::NANOSECONDS_PER_MILLISECOND
|
|
29
|
+
|
|
30
|
+
def long_active?(age_ns:)
|
|
31
|
+
Validation.integer(age_ns, 'active operation age') > long_active_threshold_ns
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
OperationLivenessPolicy.const_set(:NANOSECONDS_PER_MILLISECOND, 1_000_000)
|
|
36
|
+
OperationLivenessPolicy.const_set(:DEFAULTS, {
|
|
37
|
+
enabled: true,
|
|
38
|
+
poll_interval_ms: 100,
|
|
39
|
+
long_active_threshold_ms: 1_000
|
|
40
|
+
}.freeze)
|
|
41
|
+
OperationLivenessPolicy.const_set(:LIMITS, {
|
|
42
|
+
poll_interval_ms: 1..60_000,
|
|
43
|
+
long_active_threshold_ms: 1..86_400_000
|
|
44
|
+
}.freeze)
|
|
45
|
+
OperationLivenessPolicy.const_set(:DISABLED, OperationLivenessPolicy.new(enabled: false))
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -7,6 +7,7 @@ require_relative '../execution_context'
|
|
|
7
7
|
require_relative '../rails_integration'
|
|
8
8
|
require_relative '../recorder'
|
|
9
9
|
require_relative '../redactor'
|
|
10
|
+
require_relative '../scheduler_evidence_classifier'
|
|
10
11
|
require_relative '../scheduler_snapshot'
|
|
11
12
|
|
|
12
13
|
module FiberAudit
|
|
@@ -216,14 +217,12 @@ module FiberAudit
|
|
|
216
217
|
values.merge!(generated)
|
|
217
218
|
end
|
|
218
219
|
values[:operation_sequence] = observation.handle&.sequence
|
|
219
|
-
|
|
220
|
-
values.merge!(observation.scheduler_snapshot.to_measurements) if observation.scheduler_snapshot
|
|
221
|
-
values
|
|
220
|
+
enrich_scheduler_evidence(values, observation)
|
|
222
221
|
end
|
|
223
222
|
|
|
224
223
|
def emit_observation(kind, observation, monotonic_ns:, duration_ns: nil, measurements: nil)
|
|
225
224
|
values = measurements || observation.measurements.merge(operation_sequence: observation.handle&.sequence)
|
|
226
|
-
values =
|
|
225
|
+
values = enrich_scheduler_evidence(values.dup, observation)
|
|
227
226
|
recorder.record do
|
|
228
227
|
Event.new(
|
|
229
228
|
kind: kind,
|
|
@@ -241,12 +240,15 @@ module FiberAudit
|
|
|
241
240
|
end
|
|
242
241
|
end
|
|
243
242
|
|
|
244
|
-
def
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
243
|
+
def enrich_scheduler_evidence(values, observation)
|
|
244
|
+
snapshot = observation.scheduler_snapshot
|
|
245
|
+
values.merge!(snapshot.to_measurements) if snapshot
|
|
246
|
+
values.merge!(
|
|
247
|
+
SchedulerEvidenceClassifier.measurements(
|
|
248
|
+
operation: observation.operation,
|
|
249
|
+
scheduler_snapshot: snapshot
|
|
250
|
+
)
|
|
251
|
+
)
|
|
250
252
|
values
|
|
251
253
|
end
|
|
252
254
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../operation_semantics'
|
|
4
|
+
require_relative 'scheduler_snapshot'
|
|
5
|
+
|
|
6
|
+
module FiberAudit
|
|
7
|
+
module Runtime
|
|
8
|
+
module SchedulerEvidenceClassifier
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def measurements(operation:, scheduler_snapshot:)
|
|
12
|
+
unless scheduler_snapshot.nil? || scheduler_snapshot.is_a?(SchedulerSnapshot)
|
|
13
|
+
raise RuntimeContractError, 'scheduler_snapshot must be a SchedulerSnapshot or nil'
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
profile = OperationSemantics.resolve_runtime_operation(operation)
|
|
17
|
+
capability_supported = capability_supported(profile.scheduler_capability, scheduler_snapshot)
|
|
18
|
+
{
|
|
19
|
+
operation_wait_possible: profile.wait_possible,
|
|
20
|
+
operation_inventory_only: profile.inventory_only,
|
|
21
|
+
operation_scheduler_capability_required: profile.known? ? profile.scheduler_capability_required? : nil,
|
|
22
|
+
operation_scheduler_capability_supported: capability_supported,
|
|
23
|
+
operation_scheduler_cooperation_available: cooperation_available(
|
|
24
|
+
profile,
|
|
25
|
+
scheduler_snapshot,
|
|
26
|
+
capability_supported
|
|
27
|
+
)
|
|
28
|
+
}.freeze
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def capability_supported(capability, snapshot)
|
|
32
|
+
return nil if capability.nil? || snapshot.nil?
|
|
33
|
+
|
|
34
|
+
case capability
|
|
35
|
+
when :block, :kernel_sleep
|
|
36
|
+
snapshot.scheduler_present
|
|
37
|
+
when :io_select
|
|
38
|
+
snapshot.scheduler_io_select_supported
|
|
39
|
+
when :process_wait
|
|
40
|
+
snapshot.scheduler_process_wait_supported
|
|
41
|
+
when :address_resolve
|
|
42
|
+
snapshot.scheduler_address_resolve_supported
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
private_class_method :capability_supported
|
|
46
|
+
|
|
47
|
+
def cooperation_available(profile, snapshot, capability_supported)
|
|
48
|
+
return nil unless profile.wait_possible == true
|
|
49
|
+
return nil if snapshot.nil? || snapshot.scheduler_present.nil?
|
|
50
|
+
return false unless snapshot.scheduler_present
|
|
51
|
+
return nil if snapshot.fiber_blocking.nil?
|
|
52
|
+
return false if snapshot.fiber_blocking
|
|
53
|
+
return nil unless profile.scheduler_capability_required?
|
|
54
|
+
|
|
55
|
+
capability_supported
|
|
56
|
+
end
|
|
57
|
+
private_class_method :cooperation_available
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -19,8 +19,8 @@ module FiberAudit
|
|
|
19
19
|
scheduler_address_resolve_supported: nil
|
|
20
20
|
)
|
|
21
21
|
super(
|
|
22
|
-
scheduler_present:
|
|
23
|
-
fiber_blocking:
|
|
22
|
+
scheduler_present: normalize_optional_boolean(scheduler_present, 'scheduler_present'),
|
|
23
|
+
fiber_blocking: normalize_optional_boolean(fiber_blocking, 'fiber_blocking'),
|
|
24
24
|
scheduler_io_select_supported: normalize_optional_boolean(
|
|
25
25
|
scheduler_io_select_supported,
|
|
26
26
|
'scheduler_io_select_supported'
|
|
@@ -48,12 +48,6 @@ module FiberAudit
|
|
|
48
48
|
|
|
49
49
|
private
|
|
50
50
|
|
|
51
|
-
def normalize_boolean(value, field)
|
|
52
|
-
raise RuntimeContractError, "#{field} must be a Boolean" unless [true, false].include?(value)
|
|
53
|
-
|
|
54
|
-
value
|
|
55
|
-
end
|
|
56
|
-
|
|
57
51
|
def normalize_optional_boolean(value, field)
|
|
58
52
|
return value if value.nil?
|
|
59
53
|
raise RuntimeContractError, "#{field} must be a Boolean or nil" unless [true, false].include?(value)
|
|
@@ -71,17 +65,8 @@ module FiberAudit
|
|
|
71
65
|
scheduler = Fiber.scheduler
|
|
72
66
|
scheduler_present = !scheduler.nil?
|
|
73
67
|
|
|
74
|
-
|
|
75
|
-
fiber_blocking =
|
|
76
|
-
current_fiber = Fiber.current
|
|
77
|
-
if current_fiber.respond_to?(:blocking?)
|
|
78
|
-
!current_fiber.blocking?.nil?
|
|
79
|
-
else
|
|
80
|
-
false
|
|
81
|
-
end
|
|
82
|
-
rescue StandardError
|
|
83
|
-
false
|
|
84
|
-
end
|
|
68
|
+
current_fiber = Fiber.current
|
|
69
|
+
fiber_blocking = current_fiber.respond_to?(:blocking?) ? current_fiber.blocking? : nil
|
|
85
70
|
|
|
86
71
|
# Query scheduler capabilities if present
|
|
87
72
|
io_select_supported = nil
|
|
@@ -102,11 +87,8 @@ module FiberAudit
|
|
|
102
87
|
scheduler_address_resolve_supported: address_resolve_supported
|
|
103
88
|
)
|
|
104
89
|
rescue StandardError
|
|
105
|
-
# Fail open
|
|
106
|
-
SchedulerSnapshot.new(
|
|
107
|
-
scheduler_present: false,
|
|
108
|
-
fiber_blocking: false
|
|
109
|
-
)
|
|
90
|
+
# Fail open without inventing scheduler or Fiber state.
|
|
91
|
+
SchedulerSnapshot.new(scheduler_present: nil, fiber_blocking: nil)
|
|
110
92
|
end
|
|
111
93
|
end
|
|
112
94
|
end
|
|
@@ -6,6 +6,7 @@ require_relative 'event'
|
|
|
6
6
|
require_relative 'heartbeat'
|
|
7
7
|
require_relative 'recorder'
|
|
8
8
|
require_relative 'redactor'
|
|
9
|
+
require_relative 'scheduler_evidence_classifier'
|
|
9
10
|
require_relative 'watchdog_policy'
|
|
10
11
|
|
|
11
12
|
module FiberAudit
|
|
@@ -260,6 +261,7 @@ module FiberAudit
|
|
|
260
261
|
start_stall(channel, snapshot, age_ns, now_ns) if policy.stalled?(age_ns: age_ns)
|
|
261
262
|
end
|
|
262
263
|
|
|
264
|
+
# rubocop:disable Metrics/AbcSize
|
|
263
265
|
def start_stall(channel, snapshot, age_ns, now_ns)
|
|
264
266
|
@stall_sequence += 1
|
|
265
267
|
channel.stall = Stall.new(
|
|
@@ -267,7 +269,8 @@ module FiberAudit
|
|
|
267
269
|
progress_sequence: snapshot.sequence,
|
|
268
270
|
began_monotonic_ns: snapshot.last_progress_ns
|
|
269
271
|
)
|
|
270
|
-
|
|
272
|
+
operation_snapshot = active_operations.snapshot_with_metadata(thread_id: snapshot.thread_id)
|
|
273
|
+
operations = operation_snapshot.entries
|
|
271
274
|
frames = safe_frames(channel.thread)
|
|
272
275
|
measurements = {
|
|
273
276
|
stall_sequence: @stall_sequence,
|
|
@@ -276,7 +279,8 @@ module FiberAudit
|
|
|
276
279
|
stall_threshold_ns: policy.stall_threshold_ns,
|
|
277
280
|
heartbeat_interval_ns: policy.heartbeat_interval_ns,
|
|
278
281
|
frame_count: frames.size,
|
|
279
|
-
active_operation_count:
|
|
282
|
+
active_operation_count: operation_snapshot.total_count,
|
|
283
|
+
active_operation_snapshot_truncated: operation_snapshot.truncated?,
|
|
280
284
|
active_operation_first_sequence: operations.first&.sequence,
|
|
281
285
|
active_operation_last_sequence: operations.last&.sequence
|
|
282
286
|
}
|
|
@@ -297,17 +301,16 @@ module FiberAudit
|
|
|
297
301
|
measurements: { stall_sequence: @stall_sequence, frame_index: index }
|
|
298
302
|
)
|
|
299
303
|
end
|
|
300
|
-
emit_stall_operation_overlap_events(operations,
|
|
304
|
+
emit_stall_operation_overlap_events(operations, operation_snapshot.total_count, now_ns)
|
|
301
305
|
end
|
|
306
|
+
# rubocop:enable Metrics/AbcSize
|
|
302
307
|
|
|
303
|
-
def emit_stall_operation_overlap_events(operations,
|
|
308
|
+
def emit_stall_operation_overlap_events(operations, total_count, now_ns)
|
|
304
309
|
return if operations.empty?
|
|
305
310
|
|
|
306
|
-
truncated = operations.size > MAX_OVERLAP_EVENTS
|
|
307
311
|
bounded_operations = operations.first(MAX_OVERLAP_EVENTS)
|
|
308
|
-
|
|
312
|
+
truncated = total_count > bounded_operations.size
|
|
309
313
|
bounded_operations.each do |entry|
|
|
310
|
-
overlap_measurements = build_overlap_measurements(entry, truncated, operations.size)
|
|
311
314
|
emit_event(
|
|
312
315
|
kind: :scheduler_stall_operation_overlap,
|
|
313
316
|
monotonic_ns: now_ns,
|
|
@@ -316,7 +319,7 @@ module FiberAudit
|
|
|
316
319
|
execution_context: entry.execution_context,
|
|
317
320
|
thread_id: entry.thread_id,
|
|
318
321
|
fiber_id: entry.fiber_id,
|
|
319
|
-
measurements:
|
|
322
|
+
measurements: build_overlap_measurements(entry, truncated, total_count)
|
|
320
323
|
)
|
|
321
324
|
end
|
|
322
325
|
rescue StandardError => e
|
|
@@ -334,7 +337,12 @@ module FiberAudit
|
|
|
334
337
|
}
|
|
335
338
|
|
|
336
339
|
measurements.merge!(entry.scheduler_snapshot.to_measurements) if entry.scheduler_snapshot
|
|
337
|
-
|
|
340
|
+
measurements.merge!(
|
|
341
|
+
SchedulerEvidenceClassifier.measurements(
|
|
342
|
+
operation: entry.operation,
|
|
343
|
+
scheduler_snapshot: entry.scheduler_snapshot
|
|
344
|
+
)
|
|
345
|
+
)
|
|
338
346
|
measurements
|
|
339
347
|
end
|
|
340
348
|
|
data/lib/fiber_audit/runtime.rb
CHANGED
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
require_relative 'version'
|
|
4
4
|
require_relative 'errors'
|
|
5
5
|
require_relative 'operation_vocabulary'
|
|
6
|
+
require_relative 'operation_semantics'
|
|
6
7
|
require_relative 'execution_context'
|
|
7
8
|
require_relative 'runtime/validation'
|
|
8
9
|
require_relative 'runtime/policy'
|
|
9
10
|
require_relative 'runtime/watchdog_policy'
|
|
11
|
+
require_relative 'runtime/operation_liveness_policy'
|
|
10
12
|
require_relative 'runtime/location'
|
|
11
13
|
require_relative 'runtime/event'
|
|
12
14
|
require_relative 'runtime/session'
|
|
@@ -23,7 +25,9 @@ require_relative 'runtime/environment'
|
|
|
23
25
|
require_relative 'runtime/active_operations'
|
|
24
26
|
require_relative 'runtime/heartbeat'
|
|
25
27
|
require_relative 'runtime/scheduler_snapshot'
|
|
28
|
+
require_relative 'runtime/scheduler_evidence_classifier'
|
|
26
29
|
require_relative 'runtime/watchdog'
|
|
30
|
+
require_relative 'runtime/operation_liveness_monitor'
|
|
27
31
|
require_relative 'runtime/scheduler_observer'
|
|
28
32
|
require_relative 'runtime/probes/base'
|
|
29
33
|
require_relative 'runtime/probes/subprocess'
|
|
@@ -4,6 +4,7 @@ require_relative 'base'
|
|
|
4
4
|
require_relative '../../findings/evidence'
|
|
5
5
|
require_relative '../../correlation/fingerprint'
|
|
6
6
|
require_relative '../../findings/finding'
|
|
7
|
+
require_relative '../../operation_semantics'
|
|
7
8
|
require_relative '../../operation_vocabulary'
|
|
8
9
|
|
|
9
10
|
module FiberAudit
|
|
@@ -32,31 +33,7 @@ module FiberAudit
|
|
|
32
33
|
TARGETS = OperationVocabulary::FA1001_TARGETS
|
|
33
34
|
BARE_KERNEL_METHODS = OperationVocabulary::FA1001_KERNEL_METHODS
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
OPERATION_CATEGORY = {
|
|
37
|
-
# Creation (info) - spawning a new process
|
|
38
|
-
'Kernel.spawn' => :creation,
|
|
39
|
-
'Process.spawn' => :creation,
|
|
40
|
-
# Replacement (info) - replacing current process via exec
|
|
41
|
-
'Kernel.exec' => :replacement,
|
|
42
|
-
'Process.exec' => :replacement,
|
|
43
|
-
# Waiting (medium) - blocking waits for subprocess completion
|
|
44
|
-
'Kernel.system' => :waiting,
|
|
45
|
-
'Process.wait' => :waiting,
|
|
46
|
-
'Process.wait2' => :waiting,
|
|
47
|
-
'Process.waitpid' => :waiting,
|
|
48
|
-
'Process.waitpid2' => :waiting,
|
|
49
|
-
'Process.waitall' => :waiting,
|
|
50
|
-
'Process::Status.wait' => :waiting,
|
|
51
|
-
'Open3.capture2' => :waiting,
|
|
52
|
-
'Open3.capture2e' => :waiting,
|
|
53
|
-
'Open3.capture3' => :waiting,
|
|
54
|
-
'Open3.pipeline' => :waiting,
|
|
55
|
-
# Detach (info) - detaching subprocess without waiting
|
|
56
|
-
'Process.detach' => :detach,
|
|
57
|
-
# Stream (medium) - subprocess pipe/stream lifecycle
|
|
58
|
-
'IO.popen' => :stream
|
|
59
|
-
}.freeze
|
|
36
|
+
OPERATION_CATEGORY = OperationSemantics::FA1001_CATEGORIES
|
|
60
37
|
|
|
61
38
|
# Per-category metadata
|
|
62
39
|
CATEGORY_METADATA = {
|
|
@@ -133,7 +110,7 @@ module FiberAudit
|
|
|
133
110
|
def build_finding(site, match)
|
|
134
111
|
operation = "#{match[:constant]}.#{match[:method]}"
|
|
135
112
|
context = site.execution_context || :unknown
|
|
136
|
-
category =
|
|
113
|
+
category = OperationSemantics.resolve(operation).category
|
|
137
114
|
metadata = CATEGORY_METADATA.fetch(category)
|
|
138
115
|
base_severity = metadata[:severity]
|
|
139
116
|
sev = advisory_severity(base_severity)
|