async-service-supervisor 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/bake/async/service/supervisor.rb +86 -0
- data/context/getting-started.md +10 -26
- data/context/index.yaml +4 -0
- data/context/memory-diagnostics.md +249 -0
- data/context/memory-monitor.md +12 -43
- data/context/migration.md +1 -1
- data/lib/async/service/supervisor/version.rb +1 -1
- data/lib/async/service/supervisor/worker_controller.rb +37 -3
- data/readme.md +2 -0
- data.tar.gz.sig +0 -0
- metadata +2 -1
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 537a7bbe49f8304f907d4b258931dd3de900c591d781dd305b0daa8c285d4432
|
|
4
|
+
data.tar.gz: 2a1da9aa30958fd2628312aa053aaf2f6d61215689761c6bf8247b9e09510b4b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 86c98af76a9bd87f0c31bdf590765ac3e851d259604f1a62e0a5df134943fce4bfccf67ddaa3870c4c202f34eae9eab0a71cbcc72879ec7191f868147c3ad8ba
|
|
7
|
+
data.tar.gz: 7daac497ebd34ff0cd408bb34284979da4bb727a70e754be0b3c30e35ceaabefb60545523b9d47be9741551df2b54893d3535c44e612c9e7139f3b90298f726b
|
checksums.yaml.gz.sig
CHANGED
|
Binary file
|
|
@@ -32,12 +32,98 @@ def status
|
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
# List the connection IDs of all registered workers.
|
|
36
|
+
def workers
|
|
37
|
+
client do |connection|
|
|
38
|
+
supervisor = connection[:supervisor]
|
|
39
|
+
supervisor.keys
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Dump the object space of a worker to a file on the worker's filesystem.
|
|
44
|
+
#
|
|
45
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
46
|
+
# @parameter path [String] The file path where the worker should write the dump.
|
|
47
|
+
# @parameter shapes [Boolean] Whether to include Ruby shape-tree records.
|
|
48
|
+
def memory_dump(connection_id:, path:, shapes: true)
|
|
49
|
+
with_worker(connection_id) do |worker|
|
|
50
|
+
worker.memory_dump(path: path, shapes: shapes)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Start recording object allocation metadata in a worker.
|
|
55
|
+
#
|
|
56
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
57
|
+
def allocation_trace_start(connection_id:)
|
|
58
|
+
with_worker(connection_id) do |worker|
|
|
59
|
+
worker.allocation_trace_start
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Stop recording object allocations, dump the traced heap, and clear the metadata.
|
|
64
|
+
#
|
|
65
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
66
|
+
# @parameter path [String] The file path where the worker should write the dump.
|
|
67
|
+
# @parameter shapes [Boolean] Whether to include Ruby shape-tree records.
|
|
68
|
+
def allocation_trace_stop(connection_id:, path:, shapes: true)
|
|
69
|
+
with_worker(connection_id) do |worker|
|
|
70
|
+
worker.allocation_trace_stop(path: path, shapes: shapes)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Dump the fiber scheduler hierarchy of a worker.
|
|
75
|
+
#
|
|
76
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
77
|
+
# @parameter path [String | Nil] An optional file path on the worker's filesystem.
|
|
78
|
+
# @parameter log [String | Nil] An optional message to log with the dump.
|
|
79
|
+
def scheduler_dump(connection_id:, path: nil, log: nil)
|
|
80
|
+
with_worker(connection_id) do |worker|
|
|
81
|
+
worker.scheduler_dump(path: path, log: log)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Dump information about all threads in a worker.
|
|
86
|
+
#
|
|
87
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
88
|
+
# @parameter path [String | Nil] An optional file path on the worker's filesystem.
|
|
89
|
+
def thread_dump(connection_id:, path: nil)
|
|
90
|
+
with_worker(connection_id) do |worker|
|
|
91
|
+
worker.thread_dump(path: path)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Start garbage collection profiling in a worker.
|
|
96
|
+
#
|
|
97
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
98
|
+
def garbage_profile_start(connection_id:)
|
|
99
|
+
with_worker(connection_id) do |worker|
|
|
100
|
+
worker.garbage_profile_start
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Stop garbage collection profiling in a worker and return or save the results.
|
|
105
|
+
#
|
|
106
|
+
# @parameter connection_id [Integer] The connection ID of the worker to target.
|
|
107
|
+
# @parameter path [String | Nil] An optional file path on the worker's filesystem.
|
|
108
|
+
def garbage_profile_stop(connection_id:, path: nil)
|
|
109
|
+
with_worker(connection_id) do |worker|
|
|
110
|
+
worker.garbage_profile_stop(path: path)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
35
114
|
private
|
|
36
115
|
|
|
37
116
|
def endpoint
|
|
38
117
|
Async::Service::Supervisor.endpoint
|
|
39
118
|
end
|
|
40
119
|
|
|
120
|
+
def with_worker(connection_id)
|
|
121
|
+
client do |connection|
|
|
122
|
+
supervisor = connection[:supervisor]
|
|
123
|
+
yield supervisor[connection_id]
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
41
127
|
def client(&block)
|
|
42
128
|
Sync do
|
|
43
129
|
Async::Service::Supervisor::Client.new(endpoint: self.endpoint).connect(&block)
|
data/context/getting-started.md
CHANGED
|
@@ -149,44 +149,28 @@ end
|
|
|
149
149
|
The supervisor can collect various diagnostics from workers on demand:
|
|
150
150
|
|
|
151
151
|
- **Memory dumps**: Full heap dumps for memory analysis via `ObjectSpace.dump_all`.
|
|
152
|
-
- **Memory samples**: Lightweight sampling to identify memory leaks.
|
|
153
152
|
- **Thread dumps**: Stack traces of all threads.
|
|
154
|
-
- **Scheduler dumps**: Async fiber hierarchy
|
|
155
|
-
- **Garbage collection profiles**: GC performance data
|
|
153
|
+
- **Scheduler dumps**: Async fiber hierarchy.
|
|
154
|
+
- **Garbage collection profiles**: GC performance data.
|
|
156
155
|
|
|
157
|
-
These can be triggered programmatically or
|
|
156
|
+
These can be triggered programmatically or with Bake tasks.
|
|
158
157
|
|
|
159
158
|
#### Memory Leak Diagnosis
|
|
160
159
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
**Using the bake task:**
|
|
160
|
+
Start by listing the workers registered with the supervisor:
|
|
164
161
|
|
|
165
162
|
```bash
|
|
166
|
-
|
|
167
|
-
$ bake async:container:supervisor:memory_sample duration=30
|
|
163
|
+
$ bake async:service:supervisor:workers
|
|
168
164
|
```
|
|
169
165
|
|
|
170
|
-
|
|
166
|
+
Then capture heap dumps from the same worker before and after the suspected growth period:
|
|
171
167
|
|
|
172
|
-
```
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
puts result[:data]
|
|
168
|
+
```bash
|
|
169
|
+
$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-before.json
|
|
170
|
+
$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-after.json
|
|
176
171
|
```
|
|
177
172
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
The JSON report includes:
|
|
181
|
-
- `total_allocated`: Total allocated memory and count
|
|
182
|
-
- `total_retained`: Total retained memory and count
|
|
183
|
-
- `by_gem`: Breakdown by gem/library
|
|
184
|
-
- `by_file`: Breakdown by source file
|
|
185
|
-
- `by_location`: Breakdown by specific file:line locations
|
|
186
|
-
- `by_class`: Breakdown by object class
|
|
187
|
-
- `strings`: String allocation analysis
|
|
188
|
-
|
|
189
|
-
This is much more efficient than `do: :memory_dump` which uses `ObjectSpace.dump_all` and can be slow and blocking on large heaps. The JSON format also makes it easy to integrate with monitoring and analysis tools.
|
|
173
|
+
Heap dumps are heavyweight and may contain sensitive application data. See the [Memory Diagnostics](../memory-diagnostics/index) guide for the complete capture, comparison, and GC profiling workflow.
|
|
190
174
|
|
|
191
175
|
## Advanced Usage
|
|
192
176
|
|
data/context/index.yaml
CHANGED
|
@@ -19,6 +19,10 @@ files:
|
|
|
19
19
|
title: Memory Monitor
|
|
20
20
|
description: This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::MemoryMonitor</code>
|
|
21
21
|
to detect and restart workers that exceed memory limits or develop memory leaks.
|
|
22
|
+
- path: memory-diagnostics.md
|
|
23
|
+
title: Memory Diagnostics
|
|
24
|
+
description: This guide explains how to capture Ruby heap dumps and garbage collection
|
|
25
|
+
profiles from supervised workers, then use them to investigate memory growth.
|
|
22
26
|
- path: process-monitor.md
|
|
23
27
|
title: Process Monitor
|
|
24
28
|
description: This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::ProcessMonitor</code>
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
# Memory Diagnostics
|
|
2
|
+
|
|
3
|
+
This guide explains how to capture Ruby heap dumps and garbage collection profiles from supervised workers, then use them to investigate memory growth.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
Use the diagnostic Bake tasks when a worker's memory grows unexpectedly and you need to understand what it is retaining. A typical investigation combines:
|
|
8
|
+
|
|
9
|
+
- {ruby Async::Service::Supervisor::MemoryMonitor} or process metrics to identify sustained growth.
|
|
10
|
+
- Heap dumps to compare the objects present before and after representative load.
|
|
11
|
+
- Garbage collection profiles to measure time spent collecting objects.
|
|
12
|
+
- Scheduler and thread dumps to correlate memory growth with stuck or unusually long-running work.
|
|
13
|
+
|
|
14
|
+
These operations run inside the selected worker. They are intended for diagnosis rather than continuous monitoring.
|
|
15
|
+
|
|
16
|
+
## Finding a Worker
|
|
17
|
+
|
|
18
|
+
Run Bake from the service root where `supervisor.ipc` is located, then list the registered workers:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
$ bake async:service:supervisor:workers
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The result contains supervisor-assigned connection IDs. These IDs are not process IDs and may change whenever a worker or supervisor restarts. List the workers again if an operation reports that the connection no longer exists.
|
|
25
|
+
|
|
26
|
+
When workers are interchangeable, choose one ID and use that same worker throughout an investigation. If a worker restarts between snapshots, its replacement has a new heap and a new connection ID, so the snapshots are not directly comparable.
|
|
27
|
+
|
|
28
|
+
## Capturing a Heap Dump
|
|
29
|
+
|
|
30
|
+
Choose a path that is writable from the worker and has enough free space:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
$ bake async:service:supervisor:memory_dump \
|
|
34
|
+
connection_id=1 \
|
|
35
|
+
path=/var/tmp/worker-1-before.json
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The returned path identifies the file written by the worker:
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
{:path=>"/var/tmp/worker-1-before.json"}
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The path is resolved in the worker's filesystem namespace, not the shell running Bake. For a containerized or remote worker, retrieve the file from that worker or write it to a shared volume.
|
|
45
|
+
|
|
46
|
+
Ruby heap dumps use newline-delimited JSON: each line describes an object, root, or heap record. They are not a single JSON array.
|
|
47
|
+
|
|
48
|
+
### Operational Safety
|
|
49
|
+
|
|
50
|
+
A full heap dump calls `ObjectSpace.dump_all`. Before running it in production, consider the following:
|
|
51
|
+
|
|
52
|
+
- Dumping a large heap is slow and blocks the selected worker while Ruby walks its object space.
|
|
53
|
+
- The output can be much larger than the worker's resident memory. Check available disk space first.
|
|
54
|
+
- Heap records may contain application strings and other sensitive data. Store and transfer dumps accordingly.
|
|
55
|
+
- Avoid capturing every worker at once. Start with one representative worker during a low-risk period.
|
|
56
|
+
- The dump includes objects present at that instant, including garbage that Ruby has not collected yet. For retained-growth comparisons, capture snapshots shortly after comparable GC activity when possible.
|
|
57
|
+
|
|
58
|
+
## Analyzing Heap Dumps
|
|
59
|
+
|
|
60
|
+
Different tools answer different questions about a heap dump:
|
|
61
|
+
|
|
62
|
+
| Tool | Best for | Snapshots |
|
|
63
|
+
| --- | --- | --- |
|
|
64
|
+
| [`heap-profiler`](https://github.com/Shopify/heap-profiler) | Aggregate memory and object counts by class, gem, file, and location | One |
|
|
65
|
+
| [`sheap`](https://github.com/jhawthorn/sheap) | Finding objects retained across snapshots and tracing paths back to roots | Two or three |
|
|
66
|
+
| [Reap](https://github.com/oxidize-rb/reap) | Finding objects that dominate and retain large portions of a heap | One |
|
|
67
|
+
|
|
68
|
+
Install and run these tools on a trusted analysis system after retrieving the dumps from the worker. The dumps can contain sensitive application data.
|
|
69
|
+
|
|
70
|
+
### Aggregate Reports with `heap-profiler`
|
|
71
|
+
|
|
72
|
+
Capture a heap dump and pass it directly to `heap-profiler`:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
$ bake async:service:supervisor:memory_dump \
|
|
76
|
+
connection_id=1 \
|
|
77
|
+
path=/var/tmp/worker-1.json
|
|
78
|
+
|
|
79
|
+
$ gem install heap-profiler
|
|
80
|
+
$ heap-profiler /var/tmp/worker-1.json
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Use `--max` to show more entries when the default report is too short:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
$ heap-profiler --max=100 /var/tmp/worker-1.json
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The report is a useful overview of the largest classes, strings, and allocation locations in one snapshot. It does not calculate a delta between arbitrary Bake snapshots. Capture reports at comparable points in the workload if you want to compare their aggregate counts manually.
|
|
90
|
+
|
|
91
|
+
### Retention Diffs with `sheap`
|
|
92
|
+
|
|
93
|
+
Capture a baseline and two later snapshots from the same worker. The third snapshot distinguishes objects that remain retained from temporary allocations present only in the second dump:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-before.json
|
|
97
|
+
|
|
98
|
+
# Run representative traffic or wait through the suspected growth period.
|
|
99
|
+
$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-after.json
|
|
100
|
+
|
|
101
|
+
# Run another comparable workload and GC cycle.
|
|
102
|
+
$ bake async:service:supervisor:memory_dump connection_id=1 path=/var/tmp/worker-1-later.json
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Install `sheap` and open an interactive two-snapshot diff:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
$ gem install sheap
|
|
109
|
+
$ sheap /var/tmp/worker-1-before.json /var/tmp/worker-1-after.json
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The command opens IRB with `$before`, `$after`, and `$diff` available. For example:
|
|
113
|
+
|
|
114
|
+
```ruby
|
|
115
|
+
# Count newly retained objects by Ruby heap type:
|
|
116
|
+
$diff.retained.map(&:type_str).tally.sort_by(&:last).last(20)
|
|
117
|
+
|
|
118
|
+
# Inspect a large retained collection and find its path from a heap root:
|
|
119
|
+
large_array = $diff.retained.arrays.max_by(&:length)
|
|
120
|
+
$after.find_path(large_array)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Use the library API for a three-snapshot diff:
|
|
124
|
+
|
|
125
|
+
```ruby
|
|
126
|
+
three_way = Sheap::Diff.new(
|
|
127
|
+
"/var/tmp/worker-1-before.json",
|
|
128
|
+
"/var/tmp/worker-1-after.json",
|
|
129
|
+
"/var/tmp/worker-1-later.json"
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
three_way.retained.map(&:type_str).tally.sort_by(&:last).last(20)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
`sheap` identifies objects by heap address and type. Heap compaction can move objects, while freed addresses can be reused, so disable automatic compaction during the investigation and prefer a three-snapshot diff. Always compare dumps from the same live worker.
|
|
136
|
+
|
|
137
|
+
Look for:
|
|
138
|
+
|
|
139
|
+
- Collection classes whose object count or shallow memory continually grows.
|
|
140
|
+
- Repeated strings or payloads that should have expired.
|
|
141
|
+
- Paths from roots through registries, caches, queues, or other long-lived state.
|
|
142
|
+
- Objects that remain in the three-snapshot diff after comparable GC activity.
|
|
143
|
+
|
|
144
|
+
### Dominator Analysis with Reap
|
|
145
|
+
|
|
146
|
+
Reap builds a dominator tree from a single heap's reference graph. An object dominates another object when every path from a heap root to the second object passes through the first. This makes Reap useful for finding a small cache, queue, thread, or registry that keeps a much larger object graph alive.
|
|
147
|
+
|
|
148
|
+
Reap does not currently accept Ruby's address-less `SHAPE` records. Disable them when capturing a dump for Reap:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
$ bake async:service:supervisor:memory_dump \
|
|
152
|
+
connection_id=1 \
|
|
153
|
+
path=/var/tmp/worker-1-reap.json \
|
|
154
|
+
shapes=false
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Install Reap with Cargo, then print the largest dominators and optionally generate an inverted flame graph:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
$ cargo install reap
|
|
161
|
+
$ reap /var/tmp/worker-1-reap.json \
|
|
162
|
+
--count 20 \
|
|
163
|
+
--flamegraph /var/tmp/worker-1-retained.svg
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The report separates an object's shallow size from the total memory it dominates. Use `--root ADDRESS` to repeat the analysis for a suspicious subtree, or `--dot FILE` to write its dominator graph. Dumps captured with `shapes=false` remain compatible with `heap-profiler` and `sheap`.
|
|
167
|
+
|
|
168
|
+
Reap does not compare snapshots or use allocation generations. Combine it with a `sheap` diff when you need both evidence of continued retention and the aggregate size of the retained graph.
|
|
169
|
+
|
|
170
|
+
## Recording Allocation Locations
|
|
171
|
+
|
|
172
|
+
By default, a heap dump may not contain allocation source locations. Start tracing on the selected worker before the workload you want to investigate:
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
$ bake async:service:supervisor:allocation_trace_start connection_id=1
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Exercise the workload, then stop tracing and write the heap dump:
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
$ bake async:service:supervisor:allocation_trace_stop \
|
|
182
|
+
connection_id=1 \
|
|
183
|
+
path=/var/tmp/worker-1-traced.json
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
The stop operation disables tracing before dumping the heap, then clears the allocation metadata after the dump is complete. Start and stop must target the same live connection ID. Use `shapes=false` with the stop task when the dump will be analyzed by Reap.
|
|
187
|
+
|
|
188
|
+
Allocation tracing is process-global and adds runtime and memory overhead, so enable it selectively and measure its impact before using it in production. The start task refuses to run if tracing is already active, avoiding interference with another profiler. While tracing is enabled, heap records can include `file`, `line`, `method`, and allocation generation fields. `heap-profiler` automatically includes breakdowns by gem, file, and location when this information is present.
|
|
189
|
+
|
|
190
|
+
High allocation counts identify hot allocation sites. Compare the `heap-profiler` reports and focus on locations whose objects remain present and continue accumulating, rather than sites that merely allocate many short-lived objects.
|
|
191
|
+
|
|
192
|
+
## Profiling Garbage Collection
|
|
193
|
+
|
|
194
|
+
Start the Ruby GC profiler on the same worker you are investigating:
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
$ bake async:service:supervisor:garbage_profile_start connection_id=1
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Exercise the workload, then stop profiling and write the report:
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
$ bake async:service:supervisor:garbage_profile_stop \
|
|
204
|
+
connection_id=1 \
|
|
205
|
+
path=/var/tmp/worker-1-gc.txt
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
The start and stop commands must target the same live connection ID. The report is written in the worker's filesystem namespace.
|
|
209
|
+
|
|
210
|
+
GC profiling helps distinguish several patterns:
|
|
211
|
+
|
|
212
|
+
- Growing heap size and increasing GC time can indicate retained objects or a cache without a bound.
|
|
213
|
+
- High GC time without retained heap growth usually indicates allocation churn rather than a leak.
|
|
214
|
+
- Heap growth with little GC activity may mean the workload has not yet forced collection; compare snapshots after similar GC activity.
|
|
215
|
+
|
|
216
|
+
GC profiling records collection timing and heap statistics. It does not identify which objects retain memory, so use it alongside heap dumps.
|
|
217
|
+
|
|
218
|
+
## Capturing Scheduler and Thread State
|
|
219
|
+
|
|
220
|
+
Memory growth can be caused by queued work, blocked requests, or fibers retaining large request graphs. Capture scheduler and thread state from the same worker:
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
$ bake async:service:supervisor:scheduler_dump \
|
|
224
|
+
connection_id=1 \
|
|
225
|
+
path=/var/tmp/worker-1-scheduler.txt
|
|
226
|
+
|
|
227
|
+
$ bake async:service:supervisor:thread_dump \
|
|
228
|
+
connection_id=1 \
|
|
229
|
+
path=/var/tmp/worker-1-threads.txt
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Without `path`, scheduler and thread dumps are returned to Bake instead of being written by the worker. A path is usually more convenient for large diagnostic output.
|
|
233
|
+
|
|
234
|
+
Look for fibers or threads that remain in every snapshot, queues that never drain, and repeated backtraces corresponding to the workload that causes memory growth.
|
|
235
|
+
|
|
236
|
+
## Suggested Investigation Workflow
|
|
237
|
+
|
|
238
|
+
1. Confirm sustained growth with the `MemoryMonitor`, `ProcessMonitor`, or external process metrics.
|
|
239
|
+
2. List worker connection IDs and select one representative worker.
|
|
240
|
+
3. Enable allocation tracing before the workload if its overhead is acceptable.
|
|
241
|
+
4. Capture a baseline heap dump after normal warm-up, using `shapes=false` if you plan to use Reap.
|
|
242
|
+
5. Start GC profiling and run representative traffic or wait through the suspected leak interval.
|
|
243
|
+
6. Capture scheduler and thread dumps if work appears stuck or backlogged.
|
|
244
|
+
7. Stop GC profiling and capture the second heap dump from the same worker.
|
|
245
|
+
8. Repeat the workload and capture a third dump for a stronger `sheap` retention signal.
|
|
246
|
+
9. Use `heap-profiler` for aggregate counts, `sheap` for retained objects and root paths, and Reap for dominator sizes.
|
|
247
|
+
10. Repeat the experiment if necessary to distinguish sustained retention from normal cache warm-up.
|
|
248
|
+
|
|
249
|
+
Use the {ruby Async::Service::Supervisor::MemoryMonitor} to protect production from unbounded growth, but complete diagnostics before its configured limit restarts the worker. A restarted worker receives a new connection ID and loses the heap state you were observing.
|
data/context/memory-monitor.md
CHANGED
|
@@ -11,7 +11,7 @@ Use the `MemoryMonitor` when you need:
|
|
|
11
11
|
- **Memory leak protection**: Automatically restart workers that continuously accumulate memory.
|
|
12
12
|
- **Resource limits**: Enforce maximum memory usage per worker.
|
|
13
13
|
- **System stability**: Prevent runaway processes from exhausting system memory.
|
|
14
|
-
- **Leak diagnosis**:
|
|
14
|
+
- **Leak diagnosis**: Identify workers that should be investigated with heap diagnostics.
|
|
15
15
|
|
|
16
16
|
The monitor uses the `memory-leak` gem to track process memory usage over time, detecting abnormal growth patterns that indicate leaks.
|
|
17
17
|
|
|
@@ -39,9 +39,8 @@ end
|
|
|
39
39
|
|
|
40
40
|
When a worker exceeds the limit:
|
|
41
41
|
1. The monitor logs the leak detection.
|
|
42
|
-
2.
|
|
43
|
-
3.
|
|
44
|
-
4. The container automatically spawns a replacement worker.
|
|
42
|
+
2. Sends `SIGINT` to gracefully shut down the worker.
|
|
43
|
+
3. The container automatically spawns a replacement worker.
|
|
45
44
|
|
|
46
45
|
## Configuration Options
|
|
47
46
|
|
|
@@ -79,51 +78,21 @@ Async::Service::Supervisor::MemoryMonitor.new(
|
|
|
79
78
|
)
|
|
80
79
|
```
|
|
81
80
|
|
|
82
|
-
### `memory_sample`
|
|
83
|
-
|
|
84
|
-
Options for capturing memory samples when a leak is detected. If `nil`, memory sampling is disabled.
|
|
85
|
-
|
|
86
|
-
Default: `{duration: 30, timeout: 120}`
|
|
87
|
-
|
|
88
|
-
```ruby
|
|
89
|
-
# Customize memory sampling:
|
|
90
|
-
Async::Service::Supervisor::MemoryMonitor.new(
|
|
91
|
-
memory_sample: {
|
|
92
|
-
duration: 60, # Sample for 60 seconds
|
|
93
|
-
timeout: 180 # Timeout after 180 seconds
|
|
94
|
-
}
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
# Disable memory sampling:
|
|
98
|
-
Async::Service::Supervisor::MemoryMonitor.new(
|
|
99
|
-
memory_sample: nil
|
|
100
|
-
)
|
|
101
|
-
```
|
|
102
|
-
|
|
103
81
|
## Memory Leak Detection
|
|
104
82
|
|
|
105
83
|
When a memory leak is detected, the monitor will:
|
|
106
84
|
|
|
107
85
|
1. Log the leak detection with process details.
|
|
108
|
-
2.
|
|
109
|
-
3.
|
|
110
|
-
4. The container will automatically restart the worker process.
|
|
111
|
-
|
|
112
|
-
### Memory Sampling
|
|
86
|
+
2. Send a `SIGINT` signal to gracefully restart the worker.
|
|
87
|
+
3. The container will automatically restart the worker process.
|
|
113
88
|
|
|
114
|
-
|
|
89
|
+
### Heap Diagnostics
|
|
115
90
|
|
|
116
|
-
|
|
117
|
-
- Forces a garbage collection.
|
|
118
|
-
- Returns a JSON report showing retained objects.
|
|
91
|
+
The monitor does not automatically capture heap data before restarting a worker. When investigating growth, use the worker diagnostic Bake tasks to:
|
|
119
92
|
|
|
120
|
-
|
|
121
|
-
- `
|
|
122
|
-
-
|
|
123
|
-
-
|
|
124
|
-
- `by_file`: Breakdown by source file.
|
|
125
|
-
- `by_location`: Breakdown by specific file:line locations.
|
|
126
|
-
- `by_class`: Breakdown by object class.
|
|
127
|
-
- `strings`: String allocation analysis.
|
|
93
|
+
- List live worker connection IDs.
|
|
94
|
+
- Capture full `ObjectSpace` heap dumps before and after representative load.
|
|
95
|
+
- Record garbage collection profiles.
|
|
96
|
+
- Capture scheduler and thread state.
|
|
128
97
|
|
|
129
|
-
|
|
98
|
+
See the [Memory Diagnostics](../memory-diagnostics/index) guide for a safe capture and comparison workflow. Complete the diagnostic capture before the configured limit restarts the worker.
|
data/context/migration.md
CHANGED
|
@@ -223,7 +223,7 @@ $ bake async:service:supervisor:reload
|
|
|
223
223
|
$ bake async:service:supervisor:status
|
|
224
224
|
```
|
|
225
225
|
|
|
226
|
-
**Note:** The `memory_sample`
|
|
226
|
+
**Note:** The `memory_sample` Bake task has been removed in `async-service-supervisor`. Use the `MemoryMonitor` to detect sustained growth and the worker diagnostic Bake tasks to capture heap dumps and GC profiles. See the [Memory Diagnostics](../memory-diagnostics/index) guide for the recommended workflow.
|
|
227
227
|
|
|
228
228
|
### 8. Update Programmatic Client Usage
|
|
229
229
|
|
|
@@ -57,14 +57,49 @@ module Async
|
|
|
57
57
|
# This is a heavyweight operation that dumps all objects in the heap.
|
|
58
58
|
#
|
|
59
59
|
# @parameter path [String] Optional file path to save the dump.
|
|
60
|
-
|
|
60
|
+
# @parameter shapes [Boolean] Whether to include Ruby shape-tree records.
|
|
61
|
+
def memory_dump(path: nil, shapes: true)
|
|
61
62
|
require "objspace"
|
|
62
63
|
|
|
63
64
|
dump(path: path, buffer: false) do |file|
|
|
64
|
-
ObjectSpace.dump_all(output: file)
|
|
65
|
+
ObjectSpace.dump_all(output: file, shapes: shapes)
|
|
65
66
|
end
|
|
66
67
|
end
|
|
67
68
|
|
|
69
|
+
# Start recording object allocation metadata.
|
|
70
|
+
#
|
|
71
|
+
# Allocation tracing is process-global and can add significant overhead.
|
|
72
|
+
def allocation_trace_start
|
|
73
|
+
require "objspace"
|
|
74
|
+
|
|
75
|
+
raise "Object allocation tracing was already started by this controller!" if @allocation_trace_active
|
|
76
|
+
raise "Object allocation tracing is already active!" if ObjectSpace.allocation_sourcefile(Object.new)
|
|
77
|
+
|
|
78
|
+
ObjectSpace.trace_object_allocations_start
|
|
79
|
+
@allocation_trace_active = true
|
|
80
|
+
|
|
81
|
+
return {started: true}
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Stop recording allocations, dump the traced heap, and clear the metadata.
|
|
85
|
+
#
|
|
86
|
+
# @parameter path [String] File path where the heap dump should be written.
|
|
87
|
+
# @parameter shapes [Boolean] Whether to include Ruby shape-tree records.
|
|
88
|
+
def allocation_trace_stop(path:, shapes: true)
|
|
89
|
+
require "objspace"
|
|
90
|
+
stopped = false
|
|
91
|
+
|
|
92
|
+
raise "Object allocation tracing was not started by this controller!" unless @allocation_trace_active
|
|
93
|
+
|
|
94
|
+
ObjectSpace.trace_object_allocations_stop
|
|
95
|
+
stopped = true
|
|
96
|
+
@allocation_trace_active = false
|
|
97
|
+
|
|
98
|
+
memory_dump(path: path, shapes: shapes)
|
|
99
|
+
ensure
|
|
100
|
+
ObjectSpace.trace_object_allocations_clear if stopped
|
|
101
|
+
end
|
|
102
|
+
|
|
68
103
|
# Dump information about all running threads.
|
|
69
104
|
#
|
|
70
105
|
# Includes thread inspection and backtraces for debugging.
|
|
@@ -118,4 +153,3 @@ module Async
|
|
|
118
153
|
end
|
|
119
154
|
end
|
|
120
155
|
end
|
|
121
|
-
|
data/readme.md
CHANGED
|
@@ -20,6 +20,8 @@ Please see the [project documentation](https://socketry.github.io/async-service-
|
|
|
20
20
|
|
|
21
21
|
- [Memory Monitor](https://socketry.github.io/async-service-supervisor/guides/memory-monitor/index) - This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::MemoryMonitor</code> to detect and restart workers that exceed memory limits or develop memory leaks.
|
|
22
22
|
|
|
23
|
+
- [Memory Diagnostics](https://socketry.github.io/async-service-supervisor/guides/memory-diagnostics/index) - This guide explains how to capture Ruby heap dumps and garbage collection profiles from supervised workers, then use them to investigate memory growth.
|
|
24
|
+
|
|
23
25
|
- [Process Monitor](https://socketry.github.io/async-service-supervisor/guides/process-monitor/index) - This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::ProcessMonitor</code> to log CPU and memory metrics for your worker processes.
|
|
24
26
|
|
|
25
27
|
- [Utilization Monitor](https://socketry.github.io/async-service-supervisor/guides/utilization-monitor/index) - This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::UtilizationMonitor</code> to collect and aggregate application-level utilization metrics from your worker processes.
|
data.tar.gz.sig
CHANGED
|
Binary file
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: async-service-supervisor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.19.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Samuel Williams
|
|
@@ -157,6 +157,7 @@ files:
|
|
|
157
157
|
- bake/async/service/supervisor.rb
|
|
158
158
|
- context/getting-started.md
|
|
159
159
|
- context/index.yaml
|
|
160
|
+
- context/memory-diagnostics.md
|
|
160
161
|
- context/memory-monitor.md
|
|
161
162
|
- context/migration.md
|
|
162
163
|
- context/process-monitor.md
|
metadata.gz.sig
CHANGED
|
Binary file
|