fiber_audit 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.fiber-audit.example.yml +7 -8
- data/ARCHITECTURE.md +92 -670
- data/CHANGELOG.md +29 -0
- data/README.md +145 -92
- data/lib/fiber_audit/cli.rb +3 -1
- data/lib/fiber_audit/configuration.rb +31 -6
- data/lib/fiber_audit/operation_semantics.rb +172 -0
- data/lib/fiber_audit/runtime/active_operations.rb +25 -9
- data/lib/fiber_audit/runtime/boot.rb +4 -0
- data/lib/fiber_audit/runtime/environment.rb +58 -0
- data/lib/fiber_audit/runtime/lifecycle.rb +91 -50
- data/lib/fiber_audit/runtime/operation_liveness_monitor.rb +282 -0
- data/lib/fiber_audit/runtime/operation_liveness_policy.rb +47 -0
- data/lib/fiber_audit/runtime/probes/base.rb +12 -10
- data/lib/fiber_audit/runtime/scheduler_evidence_classifier.rb +60 -0
- data/lib/fiber_audit/runtime/scheduler_snapshot.rb +6 -24
- data/lib/fiber_audit/runtime/watchdog.rb +17 -9
- data/lib/fiber_audit/runtime.rb +4 -0
- data/lib/fiber_audit/static/rules/blocking_subprocess.rb +3 -26
- data/lib/fiber_audit/static/rules/direct_socket.rb +52 -16
- data/lib/fiber_audit/static/rules/thread_current_state.rb +12 -20
- data/lib/fiber_audit/version.rb +1 -1
- metadata +5 -1
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'errors'
|
|
4
|
+
|
|
5
|
+
module FiberAudit
|
|
6
|
+
# rubocop:disable Metrics/ModuleLength
|
|
7
|
+
module OperationSemantics
|
|
8
|
+
CAPABILITIES = %i[block kernel_sleep io_select process_wait address_resolve].freeze
|
|
9
|
+
CATEGORIES = %i[
|
|
10
|
+
creation replacement waiting detach stream thread_wait synchronization
|
|
11
|
+
nonblocking_try thread_state io_select socket_allocation socket_resolve_connect
|
|
12
|
+
socket_local_connect socket_constructor_unknown http_io unknown
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
Profile = Data.define(:category, :wait_possible, :inventory_only, :scheduler_capability) do
|
|
16
|
+
def initialize(category:, wait_possible:, inventory_only:, scheduler_capability: nil)
|
|
17
|
+
normalized_category = category.to_sym if category.respond_to?(:to_sym)
|
|
18
|
+
unless CATEGORIES.include?(normalized_category)
|
|
19
|
+
raise RuntimeContractError, "unknown operation semantic category: #{category.inspect}"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
[[:wait_possible, wait_possible], [:inventory_only, inventory_only]].each do |name, value|
|
|
23
|
+
next if value.nil? || [true, false].include?(value)
|
|
24
|
+
|
|
25
|
+
raise RuntimeContractError, "#{name} must be a Boolean or nil"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
capability = scheduler_capability&.to_sym
|
|
29
|
+
unless capability.nil? || CAPABILITIES.include?(capability)
|
|
30
|
+
raise RuntimeContractError, "unknown scheduler capability: #{scheduler_capability.inspect}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
super(category: normalized_category, wait_possible: wait_possible,
|
|
34
|
+
inventory_only: inventory_only, scheduler_capability: capability)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def known? = category != :unknown
|
|
38
|
+
def scheduler_capability_required? = !scheduler_capability.nil?
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
UNKNOWN = Profile.new(category: :unknown, wait_possible: nil, inventory_only: nil).freeze
|
|
42
|
+
SOCKET_SUBCLASS = Profile.new(
|
|
43
|
+
category: :socket_constructor_unknown,
|
|
44
|
+
wait_possible: true,
|
|
45
|
+
inventory_only: false
|
|
46
|
+
).freeze
|
|
47
|
+
|
|
48
|
+
TABLE = {
|
|
49
|
+
'Kernel.spawn' => Profile.new(category: :creation, wait_possible: false, inventory_only: true),
|
|
50
|
+
'Process.spawn' => Profile.new(category: :creation, wait_possible: false, inventory_only: true),
|
|
51
|
+
'Kernel.exec' => Profile.new(category: :replacement, wait_possible: false, inventory_only: true),
|
|
52
|
+
'Process.exec' => Profile.new(category: :replacement, wait_possible: false, inventory_only: true),
|
|
53
|
+
'Kernel.system' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
54
|
+
scheduler_capability: :process_wait),
|
|
55
|
+
'Process.wait' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
56
|
+
scheduler_capability: :process_wait),
|
|
57
|
+
'Process.wait2' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
58
|
+
scheduler_capability: :process_wait),
|
|
59
|
+
'Process.waitpid' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
60
|
+
scheduler_capability: :process_wait),
|
|
61
|
+
'Process.waitpid2' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
62
|
+
scheduler_capability: :process_wait),
|
|
63
|
+
'Process.waitall' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
64
|
+
scheduler_capability: :process_wait),
|
|
65
|
+
'Process::Status.wait' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
66
|
+
scheduler_capability: :process_wait),
|
|
67
|
+
'Open3.capture2' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
68
|
+
scheduler_capability: :process_wait),
|
|
69
|
+
'Open3.capture2e' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
70
|
+
scheduler_capability: :process_wait),
|
|
71
|
+
'Open3.capture3' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
72
|
+
scheduler_capability: :process_wait),
|
|
73
|
+
'Open3.pipeline' => Profile.new(category: :waiting, wait_possible: true, inventory_only: false,
|
|
74
|
+
scheduler_capability: :process_wait),
|
|
75
|
+
'Process.detach' => Profile.new(category: :detach, wait_possible: false, inventory_only: true),
|
|
76
|
+
'IO.popen' => Profile.new(category: :stream, wait_possible: true, inventory_only: false),
|
|
77
|
+
'Thread.join' => Profile.new(category: :thread_wait, wait_possible: true, inventory_only: false,
|
|
78
|
+
scheduler_capability: :block),
|
|
79
|
+
'Thread.value' => Profile.new(category: :thread_wait, wait_possible: true, inventory_only: false,
|
|
80
|
+
scheduler_capability: :block),
|
|
81
|
+
'Mutex#lock' => Profile.new(category: :synchronization, wait_possible: true, inventory_only: false,
|
|
82
|
+
scheduler_capability: :block),
|
|
83
|
+
'Mutex#synchronize' => Profile.new(category: :synchronization, wait_possible: true, inventory_only: false,
|
|
84
|
+
scheduler_capability: :block),
|
|
85
|
+
'Mutex#try_lock' => Profile.new(category: :nonblocking_try, wait_possible: false, inventory_only: false),
|
|
86
|
+
'ConditionVariable#wait' => Profile.new(category: :synchronization, wait_possible: true,
|
|
87
|
+
inventory_only: false, scheduler_capability: :kernel_sleep),
|
|
88
|
+
'Monitor#synchronize' => Profile.new(category: :synchronization, wait_possible: true,
|
|
89
|
+
inventory_only: false, scheduler_capability: :block),
|
|
90
|
+
'MonitorMixin#synchronize' => Profile.new(category: :synchronization, wait_possible: true,
|
|
91
|
+
inventory_only: false, scheduler_capability: :block),
|
|
92
|
+
'Thread.thread_variable_get' => Profile.new(category: :thread_state, wait_possible: false,
|
|
93
|
+
inventory_only: true),
|
|
94
|
+
'Thread.thread_variable_set' => Profile.new(category: :thread_state, wait_possible: false,
|
|
95
|
+
inventory_only: true),
|
|
96
|
+
'IO.select' => Profile.new(category: :io_select, wait_possible: true, inventory_only: false,
|
|
97
|
+
scheduler_capability: :io_select),
|
|
98
|
+
'Kernel.select' => Profile.new(category: :io_select, wait_possible: true, inventory_only: false,
|
|
99
|
+
scheduler_capability: :io_select),
|
|
100
|
+
'Socket.new' => Profile.new(category: :socket_allocation, wait_possible: false, inventory_only: true),
|
|
101
|
+
'IPSocket.new' => Profile.new(category: :socket_allocation, wait_possible: false, inventory_only: true),
|
|
102
|
+
'UDPSocket.new' => Profile.new(category: :socket_allocation, wait_possible: false, inventory_only: true),
|
|
103
|
+
'TCPSocket.new' => Profile.new(category: :socket_resolve_connect, wait_possible: true, inventory_only: false,
|
|
104
|
+
scheduler_capability: :address_resolve),
|
|
105
|
+
'TCPServer.new' => Profile.new(category: :socket_resolve_connect, wait_possible: true, inventory_only: false,
|
|
106
|
+
scheduler_capability: :address_resolve),
|
|
107
|
+
'UNIXSocket.new' => Profile.new(category: :socket_local_connect, wait_possible: true, inventory_only: false),
|
|
108
|
+
'UNIXServer.new' => Profile.new(category: :socket_allocation, wait_possible: false, inventory_only: true),
|
|
109
|
+
'Net::HTTP.get' => Profile.new(category: :http_io, wait_possible: true, inventory_only: false,
|
|
110
|
+
scheduler_capability: :address_resolve),
|
|
111
|
+
'Net::HTTP.get_response' => Profile.new(category: :http_io, wait_possible: true, inventory_only: false,
|
|
112
|
+
scheduler_capability: :address_resolve),
|
|
113
|
+
'Net::HTTP.start' => Profile.new(category: :http_io, wait_possible: true, inventory_only: false,
|
|
114
|
+
scheduler_capability: :address_resolve),
|
|
115
|
+
'Net::HTTP.request' => Profile.new(category: :http_io, wait_possible: true, inventory_only: false,
|
|
116
|
+
scheduler_capability: :address_resolve),
|
|
117
|
+
'URI.open' => Profile.new(category: :http_io, wait_possible: true, inventory_only: false,
|
|
118
|
+
scheduler_capability: :address_resolve),
|
|
119
|
+
'OpenURI.open_uri' => Profile.new(category: :http_io, wait_possible: true, inventory_only: false,
|
|
120
|
+
scheduler_capability: :address_resolve)
|
|
121
|
+
}.freeze
|
|
122
|
+
|
|
123
|
+
FA1001_CATEGORIES = TABLE.slice(
|
|
124
|
+
'Kernel.spawn', 'Process.spawn', 'Kernel.exec', 'Process.exec', 'Kernel.system',
|
|
125
|
+
'Process.wait', 'Process.wait2', 'Process.waitpid', 'Process.waitpid2',
|
|
126
|
+
'Process.waitall', 'Process::Status.wait', 'Open3.capture2', 'Open3.capture2e',
|
|
127
|
+
'Open3.capture3', 'Open3.pipeline', 'Process.detach', 'IO.popen'
|
|
128
|
+
).transform_values(&:category).freeze
|
|
129
|
+
|
|
130
|
+
module_function
|
|
131
|
+
|
|
132
|
+
def resolve(operation)
|
|
133
|
+
TABLE.fetch(normalize_operation(operation), UNKNOWN)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def resolve_socket_constructor(operation)
|
|
137
|
+
resolve_with_socket_fallback(operation)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Runtime callers are the controlled targeted-probe registry; the only
|
|
141
|
+
# canonical dynamic `<Class>.new` operations it emits are proven IPSocket
|
|
142
|
+
# subclasses from Probes::Socket.
|
|
143
|
+
def resolve_runtime_operation(operation)
|
|
144
|
+
resolve_with_socket_fallback(operation)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def known?(operation) = resolve(operation).known?
|
|
148
|
+
|
|
149
|
+
def normalize_operation(value)
|
|
150
|
+
unless value.is_a?(String) && !value.empty? && value.valid_encoding? && value.bytesize <= 240
|
|
151
|
+
raise RuntimeContractError, 'operation must be a non-empty UTF-8 String of at most 240 bytes'
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
value
|
|
155
|
+
end
|
|
156
|
+
private_class_method :normalize_operation
|
|
157
|
+
|
|
158
|
+
def resolve_with_socket_fallback(operation)
|
|
159
|
+
normalized = normalize_operation(operation)
|
|
160
|
+
TABLE.fetch(normalized) do
|
|
161
|
+
socket_constructor_operation?(normalized) ? SOCKET_SUBCLASS : UNKNOWN
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
private_class_method :resolve_with_socket_fallback
|
|
165
|
+
|
|
166
|
+
def socket_constructor_operation?(operation)
|
|
167
|
+
operation.match?(/\A[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*\.new\z/)
|
|
168
|
+
end
|
|
169
|
+
private_class_method :socket_constructor_operation?
|
|
170
|
+
end
|
|
171
|
+
# rubocop:enable Metrics/ModuleLength
|
|
172
|
+
end
|
|
@@ -7,7 +7,7 @@ require_relative 'validation'
|
|
|
7
7
|
|
|
8
8
|
module FiberAudit
|
|
9
9
|
module Runtime
|
|
10
|
-
# Bounded process-local registry shared by
|
|
10
|
+
# Bounded process-local registry shared by runtime observers.
|
|
11
11
|
class ActiveOperations
|
|
12
12
|
MAX_ENTRIES = 10_000
|
|
13
13
|
MAX_SNAPSHOT = 100
|
|
@@ -23,6 +23,20 @@ module FiberAudit
|
|
|
23
23
|
:started_monotonic_ns,
|
|
24
24
|
:scheduler_snapshot
|
|
25
25
|
)
|
|
26
|
+
Snapshot = Data.define(:entries, :total_count) do
|
|
27
|
+
def initialize(entries:, total_count:)
|
|
28
|
+
unless entries.is_a?(Array) && entries.all?(Entry)
|
|
29
|
+
raise RuntimeContractError, 'snapshot entries must be ActiveOperations::Entry values'
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
super(
|
|
33
|
+
entries: entries.dup.freeze,
|
|
34
|
+
total_count: Validation.integer(total_count, 'active operation total count')
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def truncated? = total_count > entries.size
|
|
39
|
+
end
|
|
26
40
|
|
|
27
41
|
def initialize(pid_source: Process.method(:pid), capacity: MAX_ENTRIES, snapshot_limit: MAX_SNAPSHOT)
|
|
28
42
|
validate_source!(pid_source)
|
|
@@ -77,13 +91,16 @@ module FiberAudit
|
|
|
77
91
|
@mutex.synchronize { @entries.delete(handle) }
|
|
78
92
|
end
|
|
79
93
|
|
|
80
|
-
def snapshot(thread_id: nil)
|
|
94
|
+
def snapshot(thread_id: nil) = snapshot_with_metadata(thread_id: thread_id).entries
|
|
95
|
+
|
|
96
|
+
def snapshot_with_metadata(thread_id: nil)
|
|
81
97
|
ensure_current_process!
|
|
82
98
|
normalized_thread_id = Validation.integer(thread_id, 'thread_id', allow_nil: true)
|
|
83
99
|
@mutex.synchronize do
|
|
84
100
|
entries = @entries.values
|
|
85
101
|
entries = entries.select { |entry| entry.thread_id == normalized_thread_id } if normalized_thread_id
|
|
86
|
-
entries.sort_by(&:sequence)
|
|
102
|
+
ordered = entries.sort_by(&:sequence)
|
|
103
|
+
Snapshot.new(entries: ordered.first(@snapshot_limit), total_count: ordered.size)
|
|
87
104
|
end
|
|
88
105
|
end
|
|
89
106
|
|
|
@@ -107,8 +124,6 @@ module FiberAudit
|
|
|
107
124
|
raise RuntimeContractError, 'thread and fiber identities are invalid'
|
|
108
125
|
end
|
|
109
126
|
|
|
110
|
-
normalized_snapshot = normalize_scheduler_snapshot(scheduler_snapshot)
|
|
111
|
-
|
|
112
127
|
{
|
|
113
128
|
thread_id: Validation.integer(thread.object_id, 'thread_id'),
|
|
114
129
|
fiber_id: Validation.integer(fiber.object_id, 'fiber_id'),
|
|
@@ -116,13 +131,12 @@ module FiberAudit
|
|
|
116
131
|
location: location,
|
|
117
132
|
execution_context: normalized_context,
|
|
118
133
|
started_monotonic_ns: Validation.integer(monotonic_ns, 'monotonic_ns'),
|
|
119
|
-
scheduler_snapshot:
|
|
134
|
+
scheduler_snapshot: normalize_scheduler_snapshot(scheduler_snapshot)
|
|
120
135
|
}
|
|
121
136
|
end
|
|
122
137
|
|
|
123
138
|
def normalize_scheduler_snapshot(value)
|
|
124
|
-
return value if value.nil?
|
|
125
|
-
return value if value.is_a?(SchedulerSnapshot)
|
|
139
|
+
return value if value.nil? || value.is_a?(SchedulerSnapshot)
|
|
126
140
|
|
|
127
141
|
raise RuntimeContractError, 'scheduler_snapshot must be a FiberAudit::Runtime::SchedulerSnapshot or nil'
|
|
128
142
|
end
|
|
@@ -147,7 +161,9 @@ module FiberAudit
|
|
|
147
161
|
end
|
|
148
162
|
|
|
149
163
|
def validate_source!(source)
|
|
150
|
-
|
|
164
|
+
return if source.respond_to?(:call)
|
|
165
|
+
|
|
166
|
+
raise RuntimeContractError, 'pid_source must respond to call'
|
|
151
167
|
end
|
|
152
168
|
|
|
153
169
|
def validate_limit!(value, name, maximum)
|
|
@@ -25,9 +25,13 @@ module FiberAudit
|
|
|
25
25
|
watchdog_policy = if environment.key?(Environment::WATCHDOG_SETTINGS_KEY)
|
|
26
26
|
Environment.load_watchdog_policy(environment)
|
|
27
27
|
end
|
|
28
|
+
operation_liveness_policy = if environment.key?(Environment::OPERATION_LIVENESS_SETTINGS_KEY)
|
|
29
|
+
Environment.load_operation_liveness_policy(environment)
|
|
30
|
+
end
|
|
28
31
|
@lifecycle = Lifecycle.start(
|
|
29
32
|
settings: settings,
|
|
30
33
|
watchdog_policy: watchdog_policy,
|
|
34
|
+
operation_liveness_policy: operation_liveness_policy,
|
|
31
35
|
probes_enabled: Environment.probes_enabled?(environment),
|
|
32
36
|
clock: clock,
|
|
33
37
|
session_id_source: session_id_source,
|
|
@@ -7,6 +7,7 @@ require 'securerandom'
|
|
|
7
7
|
require_relative 'policy'
|
|
8
8
|
require_relative 'validation'
|
|
9
9
|
require_relative 'watchdog_policy'
|
|
10
|
+
require_relative 'operation_liveness_policy'
|
|
10
11
|
|
|
11
12
|
module FiberAudit
|
|
12
13
|
module Runtime
|
|
@@ -17,10 +18,12 @@ module FiberAudit
|
|
|
17
18
|
SETTINGS_KEY = 'FIBER_AUDIT_RUNTIME_SETTINGS'
|
|
18
19
|
FAILURE_MODE_KEY = 'FIBER_AUDIT_RUNTIME_FAILURE_MODE'
|
|
19
20
|
WATCHDOG_SETTINGS_KEY = 'FIBER_AUDIT_RUNTIME_WATCHDOG_SETTINGS'
|
|
21
|
+
OPERATION_LIVENESS_SETTINGS_KEY = 'FIBER_AUDIT_RUNTIME_OPERATION_LIVENESS_SETTINGS'
|
|
20
22
|
PROBES_KEY = 'FIBER_AUDIT_RUNTIME_PROBES'
|
|
21
23
|
BOOT_REQUIRE = '-rfiber_audit/runtime/boot'
|
|
22
24
|
MAX_SETTINGS_BYTES = 16_384
|
|
23
25
|
MAX_WATCHDOG_SETTINGS_BYTES = 1_024
|
|
26
|
+
MAX_OPERATION_LIVENESS_SETTINGS_BYTES = 1_024
|
|
24
27
|
SETTINGS_KEYS = %w[protocol_version launch_id project_root output_directory policy].freeze
|
|
25
28
|
POLICY_KEYS = %w[
|
|
26
29
|
redaction sampling_rate max_events_per_second max_events_per_session
|
|
@@ -29,6 +32,9 @@ module FiberAudit
|
|
|
29
32
|
WATCHDOG_KEYS = %w[
|
|
30
33
|
protocol_version enabled heartbeat_interval_ms stall_threshold_ms max_frames
|
|
31
34
|
].freeze
|
|
35
|
+
OPERATION_LIVENESS_KEYS = %w[
|
|
36
|
+
protocol_version enabled poll_interval_ms long_active_threshold_ms
|
|
37
|
+
].freeze
|
|
32
38
|
|
|
33
39
|
Settings = Data.define(:protocol_version, :launch_id, :project_root, :output_directory, :policy) do
|
|
34
40
|
def initialize(protocol_version:, launch_id:, project_root:, output_directory:, policy:)
|
|
@@ -146,6 +152,45 @@ module FiberAudit
|
|
|
146
152
|
raise RuntimeContractError, "invalid runtime watchdog activation settings: #{e.message}"
|
|
147
153
|
end
|
|
148
154
|
|
|
155
|
+
def dump_operation_liveness_policy(policy)
|
|
156
|
+
require_operation_liveness_policy!(policy)
|
|
157
|
+
payload = {
|
|
158
|
+
'protocol_version' => PROTOCOL_VERSION,
|
|
159
|
+
'enabled' => policy.enabled,
|
|
160
|
+
'poll_interval_ms' => policy.poll_interval_ms,
|
|
161
|
+
'long_active_threshold_ms' => policy.long_active_threshold_ms
|
|
162
|
+
}
|
|
163
|
+
encoded = JSON.generate(payload)
|
|
164
|
+
if encoded.bytesize > MAX_OPERATION_LIVENESS_SETTINGS_BYTES
|
|
165
|
+
raise RuntimeSafetyError, 'runtime operation-liveness activation settings are too large'
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
encoded.freeze
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def load_operation_liveness_policy(environment = ENV)
|
|
172
|
+
value = environment[OPERATION_LIVENESS_SETTINGS_KEY]
|
|
173
|
+
return OperationLivenessPolicy::DISABLED if value.nil?
|
|
174
|
+
unless value.is_a?(String) && value.valid_encoding? &&
|
|
175
|
+
value.bytesize <= MAX_OPERATION_LIVENESS_SETTINGS_BYTES
|
|
176
|
+
raise RuntimeContractError, 'runtime operation-liveness activation settings are invalid'
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
payload = JSON.parse(value)
|
|
180
|
+
require_exact_keys!(payload, OPERATION_LIVENESS_KEYS, 'runtime operation-liveness activation settings')
|
|
181
|
+
unless payload.fetch('protocol_version') == PROTOCOL_VERSION
|
|
182
|
+
raise RuntimeContractError, "runtime operation-liveness activation protocol must be #{PROTOCOL_VERSION}"
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
OperationLivenessPolicy.new(
|
|
186
|
+
enabled: payload.fetch('enabled'),
|
|
187
|
+
poll_interval_ms: payload.fetch('poll_interval_ms'),
|
|
188
|
+
long_active_threshold_ms: payload.fetch('long_active_threshold_ms')
|
|
189
|
+
)
|
|
190
|
+
rescue JSON::ParserError, ArgumentError => e
|
|
191
|
+
raise RuntimeContractError, "invalid runtime operation-liveness activation settings: #{e.message}"
|
|
192
|
+
end
|
|
193
|
+
|
|
149
194
|
def activated?(environment = ENV)
|
|
150
195
|
marker = environment[ACTIVATION_KEY]
|
|
151
196
|
return false if marker.nil?
|
|
@@ -173,12 +218,14 @@ module FiberAudit
|
|
|
173
218
|
def child_environment(
|
|
174
219
|
settings:,
|
|
175
220
|
watchdog_policy: nil,
|
|
221
|
+
operation_liveness_policy: nil,
|
|
176
222
|
probes_enabled: false,
|
|
177
223
|
base_environment: ENV,
|
|
178
224
|
library_path: default_library_path
|
|
179
225
|
)
|
|
180
226
|
require_settings!(settings)
|
|
181
227
|
require_watchdog_policy!(watchdog_policy) if watchdog_policy
|
|
228
|
+
require_operation_liveness_policy!(operation_liveness_policy) if operation_liveness_policy
|
|
182
229
|
raise RuntimeContractError, 'probes_enabled must be a Boolean' unless [true, false].include?(probes_enabled)
|
|
183
230
|
raise RuntimeContractError, 'base_environment must be a Hash-like object' unless base_environment.respond_to?(:[])
|
|
184
231
|
|
|
@@ -190,6 +237,9 @@ module FiberAudit
|
|
|
190
237
|
'RUBYLIB' => prepend_token(base_environment['RUBYLIB'], library_path, separator: File::PATH_SEPARATOR)
|
|
191
238
|
}
|
|
192
239
|
environment[WATCHDOG_SETTINGS_KEY] = dump_watchdog_policy(watchdog_policy) if watchdog_policy
|
|
240
|
+
if operation_liveness_policy
|
|
241
|
+
environment[OPERATION_LIVENESS_SETTINGS_KEY] = dump_operation_liveness_policy(operation_liveness_policy)
|
|
242
|
+
end
|
|
193
243
|
environment[PROBES_KEY] = '1' if probes_enabled
|
|
194
244
|
environment.transform_values(&:freeze).freeze
|
|
195
245
|
end
|
|
@@ -255,6 +305,14 @@ module FiberAudit
|
|
|
255
305
|
end
|
|
256
306
|
private_class_method :require_watchdog_policy!
|
|
257
307
|
|
|
308
|
+
def require_operation_liveness_policy!(value)
|
|
309
|
+
return if value.is_a?(OperationLivenessPolicy)
|
|
310
|
+
|
|
311
|
+
raise RuntimeContractError,
|
|
312
|
+
'operation_liveness_policy must be FiberAudit::Runtime::OperationLivenessPolicy'
|
|
313
|
+
end
|
|
314
|
+
private_class_method :require_operation_liveness_policy!
|
|
315
|
+
|
|
258
316
|
def require_exact_keys!(value, expected, path)
|
|
259
317
|
raise RuntimeContractError, "#{path} must be an object" unless value.is_a?(Hash)
|
|
260
318
|
|
|
@@ -14,6 +14,8 @@ require_relative 'scheduler_observer'
|
|
|
14
14
|
require_relative 'session'
|
|
15
15
|
require_relative 'watchdog'
|
|
16
16
|
require_relative 'watchdog_policy'
|
|
17
|
+
require_relative 'operation_liveness_monitor'
|
|
18
|
+
require_relative 'operation_liveness_policy'
|
|
17
19
|
|
|
18
20
|
module FiberAudit
|
|
19
21
|
module Runtime
|
|
@@ -21,6 +23,7 @@ module FiberAudit
|
|
|
21
23
|
class Lifecycle
|
|
22
24
|
attr_reader :settings, :owner_pid, :output_path, :recorder,
|
|
23
25
|
:watchdog, :active_operations, :watchdog_policy,
|
|
26
|
+
:operation_liveness_monitor, :operation_liveness_policy,
|
|
24
27
|
:redactor, :probe_registry, :execution_context_store,
|
|
25
28
|
:rails_integration
|
|
26
29
|
|
|
@@ -31,6 +34,7 @@ module FiberAudit
|
|
|
31
34
|
def initialize(
|
|
32
35
|
settings:,
|
|
33
36
|
watchdog_policy: nil,
|
|
37
|
+
operation_liveness_policy: nil,
|
|
34
38
|
probes_enabled: false,
|
|
35
39
|
clock: Clock.new,
|
|
36
40
|
session_id_source: SecureRandom.method(:uuid),
|
|
@@ -39,11 +43,12 @@ module FiberAudit
|
|
|
39
43
|
random: Sampler::RANDOM_SOURCE
|
|
40
44
|
)
|
|
41
45
|
validate_dependencies!(
|
|
42
|
-
settings, watchdog_policy, probes_enabled, clock,
|
|
46
|
+
settings, watchdog_policy, operation_liveness_policy, probes_enabled, clock,
|
|
43
47
|
session_id_source, pid_source, writer_factory, random
|
|
44
48
|
)
|
|
45
49
|
@settings = settings
|
|
46
50
|
@watchdog_policy = watchdog_policy
|
|
51
|
+
@operation_liveness_policy = operation_liveness_policy
|
|
47
52
|
@probes_enabled = probes_enabled
|
|
48
53
|
@clock = clock
|
|
49
54
|
@session_id_source = session_id_source
|
|
@@ -52,15 +57,7 @@ module FiberAudit
|
|
|
52
57
|
@random = random
|
|
53
58
|
@owner_pid = current_pid
|
|
54
59
|
@state = :starting
|
|
55
|
-
|
|
56
|
-
@recorder = nil
|
|
57
|
-
@watchdog = nil
|
|
58
|
-
@scheduler_observer = nil
|
|
59
|
-
@active_operations = nil
|
|
60
|
-
@redactor = nil
|
|
61
|
-
@probe_registry = nil
|
|
62
|
-
@execution_context_store = nil
|
|
63
|
-
@rails_integration = nil
|
|
60
|
+
reset_process_components!
|
|
64
61
|
start_process_session!
|
|
65
62
|
rescue StandardError => e
|
|
66
63
|
startup_failure!(e)
|
|
@@ -84,15 +81,7 @@ module FiberAudit
|
|
|
84
81
|
|
|
85
82
|
abandon_inherited_runtime!
|
|
86
83
|
@owner_pid = pid
|
|
87
|
-
|
|
88
|
-
@recorder = nil
|
|
89
|
-
@watchdog = nil
|
|
90
|
-
@scheduler_observer = nil
|
|
91
|
-
@active_operations = nil
|
|
92
|
-
@redactor = nil
|
|
93
|
-
@probe_registry = nil
|
|
94
|
-
@execution_context_store = nil
|
|
95
|
-
@rails_integration = nil
|
|
84
|
+
reset_process_components!
|
|
96
85
|
@state = :starting
|
|
97
86
|
# Reset fiber-local context after fork
|
|
98
87
|
ExecutionContext.reset!
|
|
@@ -122,13 +111,18 @@ module FiberAudit
|
|
|
122
111
|
|
|
123
112
|
private
|
|
124
113
|
|
|
125
|
-
def validate_dependencies!(candidate_settings, watchdog, probes, candidate_clock,
|
|
114
|
+
def validate_dependencies!(candidate_settings, watchdog, liveness, probes, candidate_clock,
|
|
115
|
+
session_ids, pids, writers, random)
|
|
126
116
|
unless candidate_settings.is_a?(Environment::Settings)
|
|
127
117
|
raise RuntimeContractError, 'settings must be FiberAudit::Runtime::Environment::Settings'
|
|
128
118
|
end
|
|
129
119
|
unless watchdog.nil? || watchdog.is_a?(WatchdogPolicy)
|
|
130
120
|
raise RuntimeContractError, 'watchdog_policy must be a FiberAudit::Runtime::WatchdogPolicy or nil'
|
|
131
121
|
end
|
|
122
|
+
unless liveness.nil? || liveness.is_a?(OperationLivenessPolicy)
|
|
123
|
+
raise RuntimeContractError,
|
|
124
|
+
'operation_liveness_policy must be a FiberAudit::Runtime::OperationLivenessPolicy or nil'
|
|
125
|
+
end
|
|
132
126
|
raise RuntimeContractError, 'probes_enabled must be a Boolean' unless [true, false].include?(probes)
|
|
133
127
|
raise RuntimeContractError, 'clock must be a FiberAudit::Runtime::Clock' unless candidate_clock.is_a?(Clock)
|
|
134
128
|
|
|
@@ -142,6 +136,19 @@ module FiberAudit
|
|
|
142
136
|
end
|
|
143
137
|
end
|
|
144
138
|
|
|
139
|
+
def reset_process_components!
|
|
140
|
+
@output_path = nil
|
|
141
|
+
@recorder = nil
|
|
142
|
+
@watchdog = nil
|
|
143
|
+
@operation_liveness_monitor = nil
|
|
144
|
+
@scheduler_observer = nil
|
|
145
|
+
@active_operations = nil
|
|
146
|
+
@redactor = nil
|
|
147
|
+
@probe_registry = nil
|
|
148
|
+
@execution_context_store = nil
|
|
149
|
+
@rails_integration = nil
|
|
150
|
+
end
|
|
151
|
+
|
|
145
152
|
def start_process_session!
|
|
146
153
|
session_id = @session_id_source.call
|
|
147
154
|
started_at = @clock.wall_time
|
|
@@ -175,6 +182,7 @@ module FiberAudit
|
|
|
175
182
|
@redactor = Redactor.new(root: settings.project_root, policy: settings.policy)
|
|
176
183
|
@execution_context_store = ExecutionContext if @probes_enabled
|
|
177
184
|
setup_watchdog! if watchdog_policy
|
|
185
|
+
setup_operation_liveness! if @probes_enabled && operation_liveness_policy
|
|
178
186
|
setup_rails_integration! if @probes_enabled
|
|
179
187
|
setup_probes! if @probes_enabled
|
|
180
188
|
rescue StandardError => e
|
|
@@ -202,6 +210,13 @@ module FiberAudit
|
|
|
202
210
|
ensure
|
|
203
211
|
@rails_integration = nil
|
|
204
212
|
end
|
|
213
|
+
begin
|
|
214
|
+
@operation_liveness_monitor&.stop
|
|
215
|
+
rescue StandardError
|
|
216
|
+
nil
|
|
217
|
+
ensure
|
|
218
|
+
@operation_liveness_monitor = nil
|
|
219
|
+
end
|
|
205
220
|
begin
|
|
206
221
|
if @scheduler_observer
|
|
207
222
|
SchedulerObserver.deactivate(@scheduler_observer)
|
|
@@ -215,6 +230,15 @@ module FiberAudit
|
|
|
215
230
|
end
|
|
216
231
|
end
|
|
217
232
|
|
|
233
|
+
def setup_operation_liveness!
|
|
234
|
+
@operation_liveness_monitor = OperationLivenessMonitor.new(
|
|
235
|
+
policy: operation_liveness_policy,
|
|
236
|
+
recorder: recorder,
|
|
237
|
+
active_operations: active_operations,
|
|
238
|
+
clock: @clock
|
|
239
|
+
)
|
|
240
|
+
end
|
|
241
|
+
|
|
218
242
|
def setup_rails_integration!
|
|
219
243
|
@rails_integration = RailsIntegration.activate(
|
|
220
244
|
context_store: @execution_context_store,
|
|
@@ -272,44 +296,61 @@ module FiberAudit
|
|
|
272
296
|
def abandon_inherited_runtime!
|
|
273
297
|
recorder&.writer&.close
|
|
274
298
|
ensure
|
|
275
|
-
|
|
276
|
-
@watchdog = nil
|
|
277
|
-
@scheduler_observer = nil
|
|
278
|
-
@active_operations = nil
|
|
279
|
-
@redactor = nil
|
|
280
|
-
@probe_registry = nil
|
|
281
|
-
@execution_context_store = nil
|
|
282
|
-
@rails_integration = nil
|
|
299
|
+
reset_process_components!
|
|
283
300
|
end
|
|
284
301
|
|
|
285
302
|
def stop_runtime_observers
|
|
286
303
|
error = nil
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
Probes::Registry.deactivate(probe_registry) if probe_registry
|
|
296
|
-
rescue StandardError => e
|
|
297
|
-
error ||= e
|
|
298
|
-
ensure
|
|
299
|
-
@probe_registry = nil
|
|
300
|
-
end
|
|
301
|
-
begin
|
|
302
|
-
SchedulerObserver.deactivate(@scheduler_observer) if @scheduler_observer
|
|
303
|
-
watchdog&.stop
|
|
304
|
-
rescue StandardError => e
|
|
305
|
-
error ||= e
|
|
306
|
-
ensure
|
|
307
|
-
@scheduler_observer = nil
|
|
308
|
-
end
|
|
304
|
+
component_error = stop_rails_integration
|
|
305
|
+
error ||= component_error
|
|
306
|
+
component_error = stop_probe_registry
|
|
307
|
+
error ||= component_error
|
|
308
|
+
component_error = stop_operation_liveness_monitor
|
|
309
|
+
error ||= component_error
|
|
310
|
+
component_error = stop_watchdog
|
|
311
|
+
error ||= component_error
|
|
309
312
|
recorder&.internal_error! if error
|
|
310
313
|
error
|
|
311
314
|
end
|
|
312
315
|
|
|
316
|
+
def stop_rails_integration
|
|
317
|
+
RailsIntegration.deactivate(@rails_integration) if @rails_integration
|
|
318
|
+
nil
|
|
319
|
+
rescue StandardError => e
|
|
320
|
+
e
|
|
321
|
+
ensure
|
|
322
|
+
@rails_integration = nil
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def stop_probe_registry
|
|
326
|
+
Probes::Registry.deactivate(probe_registry) if probe_registry
|
|
327
|
+
nil
|
|
328
|
+
rescue StandardError => e
|
|
329
|
+
e
|
|
330
|
+
ensure
|
|
331
|
+
@probe_registry = nil
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
def stop_operation_liveness_monitor
|
|
335
|
+
operation_liveness_monitor&.stop
|
|
336
|
+
nil
|
|
337
|
+
rescue StandardError => e
|
|
338
|
+
e
|
|
339
|
+
ensure
|
|
340
|
+
@operation_liveness_monitor = nil
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def stop_watchdog
|
|
344
|
+
SchedulerObserver.deactivate(@scheduler_observer) if @scheduler_observer
|
|
345
|
+
watchdog&.stop
|
|
346
|
+
nil
|
|
347
|
+
rescue StandardError => e
|
|
348
|
+
e
|
|
349
|
+
ensure
|
|
350
|
+
@scheduler_observer = nil
|
|
351
|
+
@watchdog = nil
|
|
352
|
+
end
|
|
353
|
+
|
|
313
354
|
def startup_failure!(error)
|
|
314
355
|
raise error unless defined?(@settings) && settings.policy.fail_open?
|
|
315
356
|
|