async-service-supervisor 0.10.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e5058893b1b7ea249bcd4e273f169fb9a4649523380d545c08a281f20d6f56c0
+  data.tar.gz: d17a7bc6d6443ffd962118ae45d1dbfa6cf7a890d5b91ca1d74c9f050d45177b
+SHA512:
+  metadata.gz: 344766f9eb24ca6fae65844a8e5654fb197b829efe89363b0bdff1e9bfd322fcc1e0f1b121ef1ae2579d03174b14226c0a2f76a5de3a7e3ceff4ff6f48416366
+  data.tar.gz: 2efa90fef449b5a0b52ea738a4f078a32c9df70d41a2d42dcf3e7e0d02b1f67c6290bca1b5c073ef838e482b56abd97d9d2e1fe1019e383c62659946df52cee1

data/bake/async/service/supervisor.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2025, by Samuel Williams.
+
+def initialize(...)
+	super
+	
+	require "async/service/supervisor"
+end
+
+# Restart the container, typically causing it to exit (the parent process should then restart it).
+def restart
+	client do |connection|
+		supervisor = connection[:supervisor]
+		supervisor.restart
+	end
+end
+
+# Reload the services gracefully, allowing them to reconfigure without dropping connections.
+def reload
+	client do |connection|
+		supervisor = connection[:supervisor]
+		supervisor.restart(signal: :HUP)
+	end
+end
+
+def status
+	client do |connection|
+		supervisor = connection[:supervisor]
+		supervisor.status
+	end
+end
+
+private
+
+def endpoint
+	Async::Service::Supervisor.endpoint
+end
+
+def client(&block)
+	Sync do
+		Async::Service::Supervisor::Client.new(endpoint: self.endpoint).connect(&block)
+	end
+end

data/context/getting-started.md

@@ -0,0 +1,205 @@
+# Getting Started
+
+This guide explains how to get started with `async-service-supervisor` to supervise and monitor worker processes in your Ruby applications.
+
+## Installation
+
+Add the gem to your project:
+
+```bash
+$ bundle add async-service-supervisor
+```
+
+## Core Concepts
+
+`async-service-supervisor` provides a robust process supervision system built on top of {ruby Async::Service::Generic}. The key components are:
+
+- {ruby Async::Service::Supervisor::Environment}: An environment mixin that sets up a supervisor service in your application.
+- {ruby Async::Service::Supervisor::Supervised}: An environment mixin that enables workers to connect to and be supervised by the supervisor.
+- {ruby Async::Service::Supervisor::Server}: The server that handles communication with workers and performs monitoring.
+- {ruby Async::Service::Supervisor::Worker}: A client that connects workers to the supervisor for health monitoring and diagnostics.
+
+### Process Architecture
+
+The supervisor operates as a multi-process architecture with three layers:
+
+```mermaid
+graph TD
+    Controller[Async::Container::Controller<br/>Root Process]
+    
+    Controller -->|spawns & manages| Supervisor[Supervisor Process<br/>async-service-supervisor]
+    Controller -->|spawns & manages| Worker1[Worker Process 1]
+    Controller -->|spawns & manages| Worker2[Worker Process 2]
+    Controller -->|spawns & manages| WorkerN[Worker Process N...]
+    
+    Worker1 -.->|connects via IPC| Supervisor
+    Worker2 -.->|connects via IPC| Supervisor
+    WorkerN -.->|connects via IPC| Supervisor
+```
+
+**Important:** The supervisor process is itself just another process managed by the root controller. If the supervisor crashes, the controller will restart it, and all worker processes will automatically reconnect to the new supervisor. This design ensures high availability and fault tolerance.
+
+## Usage
+
+To use the supervisor, you need to define two services: one for the supervisor itself and one for your workers that will be supervised.
+
+### Basic Example
+
+Create a service configuration file (e.g., `service.rb`):
+
+```ruby
+#!/usr/bin/env async-service
+# frozen_string_literal: true
+
+require "async/service/supervisor"
+
+class MyWorkerService < Async::Service::Generic
+	def setup(container)
+		super
+		
+		container.run(name: self.class.name, count: 4, restart: true) do |instance|
+			Async do
+				# Get the environment evaluator:
+				evaluator = self.environment.evaluator
+				
+				# Prepare the instance (connects to supervisor if available):
+				evaluator.prepare!(instance)
+				
+				# Mark the worker as ready:
+				instance.ready!
+				
+				# Your worker logic here:
+				loop do
+					# Do work...
+					sleep 1
+					
+					# Periodically update readiness:
+					instance.ready!
+				end
+			end
+		end
+	end
+end
+
+# Define the worker service:
+service "worker" do
+	service_class MyWorkerService
+	
+	# Enable supervision for this service:
+	include Async::Service::Supervisor::Supervised
+end
+
+# Define the supervisor service:
+service "supervisor" do
+	include Async::Service::Supervisor::Environment
+end
+```
+
+### Running the Service
+
+Make the service executable and run it:
+
+```bash
+$ chmod +x service.rb
+$ ./service.rb
+```
+
+This will start:
+- A supervisor process listening on a Unix socket
+- Four worker processes that connect to the supervisor
+
+### Adding Health Monitors
+
+You can add monitors to observe worker health and automatically respond to issues. Monitors are useful for:
+
+- **Memory leak detection**: Automatically restart workers consuming excessive memory.
+- **Performance monitoring**: Track CPU and memory usage trends.
+- **Capacity planning**: Understand resource requirements.
+
+For example, to add monitoring:
+
+```ruby
+service "supervisor" do
+	include Async::Service::Supervisor::Environment
+	
+	monitors do
+		[
+			# Log process metrics for observability:
+			Async::Service::Supervisor::ProcessMonitor.new(
+				interval: 60
+			),
+			
+			# Restart workers exceeding memory limits:
+			Async::Service::Supervisor::MemoryMonitor.new(
+				interval: 10,
+				maximum_size_limit: 1024 * 1024 * 500  # 500MB limit per process
+			)
+		]
+	end
+end
+```
+
+See the {ruby Async::Service::Supervisor::MemoryMonitor Memory Monitor} and {ruby Async::Service::Supervisor::ProcessMonitor Process Monitor} guides for detailed configuration options and best practices.
+
+### Collecting Diagnostics
+
+The supervisor can collect various diagnostics from workers on demand:
+
+- **Memory dumps**: Full heap dumps for memory analysis via `ObjectSpace.dump_all`.
+- **Memory samples**: Lightweight sampling to identify memory leaks.
+- **Thread dumps**: Stack traces of all threads.
+- **Scheduler dumps**: Async fiber hierarchy
+- **Garbage collection profiles**: GC performance data
+
+These can be triggered programmatically or via command-line tools (when available).
+
+#### Memory Leak Diagnosis
+
+To identify memory leaks, you can use the memory sampling feature which is much lighter weight than a full memory dump. It tracks allocations over a time period and focuses on retained objects.
+
+**Using the bake task:**
+
+```bash
+# Sample for 30 seconds and print report to console
+$ bake async:container:supervisor:memory_sample duration=30
+```
+
+**Programmatically:**
+
+```ruby
+# Assuming you have a connection to a worker:
+result = connection.call(do: :memory_sample, duration: 30)
+puts result[:data]
+```
+
+This will sample memory allocations for the specified duration, then force a garbage collection and return a JSON report showing what objects were allocated during that period and retained after GC. Late-lifecycle allocations that are retained are likely memory leaks.
+
+The JSON report includes:
+- `total_allocated`: Total allocated memory and count
+- `total_retained`: Total retained memory and count  
+- `by_gem`: Breakdown by gem/library
+- `by_file`: Breakdown by source file
+- `by_location`: Breakdown by specific file:line locations
+- `by_class`: Breakdown by object class
+- `strings`: String allocation analysis
+
+This is much more efficient than `do: :memory_dump` which uses `ObjectSpace.dump_all` and can be slow and blocking on large heaps. The JSON format also makes it easy to integrate with monitoring and analysis tools.
+
+## Advanced Usage
+
+### Custom Monitors
+
+You can create custom monitors by implementing the monitor interface. A monitor should:
+
+1. Accept connections and periodically check worker health
+2. Take action (like restarting workers) when unhealthy conditions are detected
+
+### Fault Tolerance
+
+The supervisor architecture is designed for fault tolerance:
+
+- **Supervisor crashes**: When the supervisor process crashes, the root controller automatically restarts it. Workers detect the disconnection and reconnect to the new supervisor.
+- **Worker crashes**: The container automatically restarts crashed workers based on the `restart: true` configuration.
+- **Communication failures**: Workers gracefully handle supervisor unavailability and will attempt to reconnect.
+
+This design ensures your application remains operational even when individual processes fail.

data/context/index.yaml

@@ -0,0 +1,20 @@
+# Automatically generated context index for Utopia::Project guides.
+# Do not edit then files in this directory directly, instead edit the guides and then run `bake utopia:project:agent:context:update`.
+---
+description: A supervisor for managing multiple container processes.
+metadata:
+  documentation_uri: https://socketry.github.io/async-service-supervisor/
+  source_code_uri: https://github.com/socketry/async-service-supervisor.git
+files:
+- path: getting-started.md
+  title: Getting Started
+  description: This guide explains how to get started with `async-service-supervisor`
+    to supervise and monitor worker processes in your Ruby applications.
+- path: memory-monitor.md
+  title: Memory Monitor
+  description: This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::MemoryMonitor</code>
+    to detect and restart workers that exceed memory limits or develop memory leaks.
+- path: process-monitor.md
+  title: Process Monitor
+  description: This guide explains how to use the <code class="language-ruby">Async::Service::Supervisor::ProcessMonitor</code>
+    to log CPU and memory metrics for your worker processes.

data/context/memory-monitor.md

@@ -0,0 +1,129 @@
+# Memory Monitor
+
+This guide explains how to use the {ruby Async::Service::Supervisor::MemoryMonitor} to detect and restart workers that exceed memory limits or develop memory leaks.
+
+## Overview
+
+Long-running worker processes often accumulate memory over time, either through legitimate growth or memory leaks. Without intervention, workers can consume all available system memory, causing performance degradation or system crashes. The `MemoryMonitor` solves this by automatically detecting and restarting problematic workers before they impact system stability.
+
+Use the `MemoryMonitor` when you need:
+
+- **Memory leak protection**: Automatically restart workers that continuously accumulate memory.
+- **Resource limits**: Enforce maximum memory usage per worker.
+- **System stability**: Prevent runaway processes from exhausting system memory.
+- **Leak diagnosis**: Capture memory samples when leaks are detected for debugging.
+
+The monitor uses the `memory-leak` gem to track process memory usage over time, detecting abnormal growth patterns that indicate leaks.
+
+## Usage
+
+Add a memory monitor to your supervisor service to automatically restart workers that exceed 500MB:
+
+```ruby
+service "supervisor" do
+	include Async::Service::Supervisor::Environment
+	
+	monitors do
+		[
+			Async::Service::Supervisor::MemoryMonitor.new(
+				# Check worker memory every 10 seconds:
+				interval: 10,
+				
+				# Restart workers exceeding 500MB:
+				maximum_size_limit: 1024 * 1024 * 500
+			)
+		]
+	end
+end
+```
+
+When a worker exceeds the limit:
+1. The monitor logs the leak detection.
+2. Optionally captures a memory sample for debugging.
+3. Sends `SIGINT` to gracefully shut down the worker.
+4. The container automatically spawns a replacement worker.
+
+## Configuration Options
+
+The `MemoryMonitor` accepts the following options:
+
+### `interval`
+
+The interval (in seconds) at which to check for memory leaks. Default: `10` seconds.
+
+```ruby
+Async::Service::Supervisor::MemoryMonitor.new(interval: 30)
+```
+
+### `maximum_size_limit`
+
+The maximum memory size (in bytes) per process. When a process exceeds this limit, it will be restarted.
+
+```ruby
+# 500MB limit
+Async::Service::Supervisor::MemoryMonitor.new(maximum_size_limit: 1024 * 1024 * 500)
+
+# 1GB limit
+Async::Service::Supervisor::MemoryMonitor.new(maximum_size_limit: 1024 * 1024 * 1024)
+```
+
+### `total_size_limit`
+
+The total size limit (in bytes) for all monitored processes combined. If not specified, only per-process limits are enforced.
+
+```ruby
+# Total limit of 2GB across all workers
+Async::Service::Supervisor::MemoryMonitor.new(
+	maximum_size_limit: 1024 * 1024 * 500,  # 500MB per process
+	total_size_limit: 1024 * 1024 * 1024 * 2  # 2GB total
+)
+```
+
+### `memory_sample`
+
+Options for capturing memory samples when a leak is detected. If `nil`, memory sampling is disabled.
+
+Default: `{duration: 30, timeout: 120}`
+
+```ruby
+# Customize memory sampling:
+Async::Service::Supervisor::MemoryMonitor.new(
+	memory_sample: {
+		duration: 60,  # Sample for 60 seconds
+		timeout: 180   # Timeout after 180 seconds
+	}
+)
+
+# Disable memory sampling:
+Async::Service::Supervisor::MemoryMonitor.new(
+	memory_sample: nil
+)
+```
+
+## Memory Leak Detection
+
+When a memory leak is detected, the monitor will:
+
+1. Log the leak detection with process details.
+2. If `memory_sample` is configured, capture a memory sample from the worker.
+3. Send a `SIGINT` signal to gracefully restart the worker.
+4. The container will automatically restart the worker process.
+
+### Memory Sampling
+
+When a memory leak is detected and `memory_sample` is configured, the monitor requests a lightweight memory sample from the worker. This sample:
+
+- Tracks allocations during the sampling period.
+- Forces a garbage collection.
+- Returns a JSON report showing retained objects.
+
+The report includes:
+- `total_allocated`: Total allocated memory and object count.
+- `total_retained`: Total retained memory and count after GC.
+- `by_gem`: Breakdown by gem/library.
+- `by_file`: Breakdown by source file.
+- `by_location`: Breakdown by specific file:line locations.
+- `by_class`: Breakdown by object class.
+- `strings`: String allocation analysis.
+
+This is much more efficient than a full heap dump using `ObjectSpace.dump_all`.

data/context/process-monitor.md

@@ -0,0 +1,91 @@
+# Process Monitor
+
+This guide explains how to use the {ruby Async::Service::Supervisor::ProcessMonitor} to log CPU and memory metrics for your worker processes.
+
+## Overview
+
+Understanding how your workers consume resources over time is essential for performance optimization, capacity planning, and debugging. Without visibility into CPU and memory usage, you can't identify bottlenecks, plan infrastructure scaling, or diagnose production issues effectively.
+
+The `ProcessMonitor` provides this observability by periodically capturing and logging comprehensive metrics for your entire application process tree.
+
+Use the `ProcessMonitor` when you need:
+
+- **Performance analysis**: Identify which workers consume the most CPU or memory.
+- **Capacity planning**: Determine optimal worker counts and memory requirements.
+- **Trend monitoring**: Track resource usage patterns over time.
+- **Debugging assistance**: Correlate resource usage with application behavior.
+- **Cost optimization**: Right-size infrastructure based on actual usage.
+
+Unlike the {ruby Async::Service::Supervisor::MemoryMonitor}, which takes action when limits are exceeded, the `ProcessMonitor` is purely observational - it logs metrics without interfering with worker processes.
+
+## Usage
+
+Add a process monitor to log resource usage every minute:
+
+```ruby
+service "supervisor" do
+	include Async::Service::Supervisor::Environment
+	
+	monitors do
+		[
+			# Log CPU and memory metrics for all processes:
+			Async::Service::Supervisor::ProcessMonitor.new(
+				interval: 60  # Capture metrics every minute
+			)
+		]
+	end
+end
+```
+
+This allows you to easily search and filter by specific fields:
+- `general.process_id = 12347` - Find metrics for a specific process.
+- `general.command = "worker-1"` - Find all metrics for worker processes.
+- `general.processor_utilization > 50` - Find high CPU usage processes.
+- `general.resident_size > 500000` - Find processes using more than 500MB.
+
+## Configuration Options
+
+### `interval`
+
+The interval (in seconds) at which to capture and log process metrics. Default: `60` seconds.
+
+```ruby
+# Log every 30 seconds
+Async::Service::Supervisor::ProcessMonitor.new(interval: 30)
+
+# Log every 5 minutes
+Async::Service::Supervisor::ProcessMonitor.new(interval: 300)
+```
+
+## Captured Metrics
+
+The `ProcessMonitor` captures the following metrics for each process:
+
+### Core Metrics
+
+- **process_id**: Unique identifier for the process.
+- **parent_process_id**: The parent process that spawned this one.
+- **process_group_id**: Process group identifier.
+- **command**: The command name.
+- **processor_utilization**: CPU usage percentage.
+- **resident_size**: Physical memory used (KB).
+- **total_size**: Total memory space including shared memory (KB).
+- **processor_time**: Total CPU time used (seconds).
+- **elapsed_time**: How long the process has been running (seconds).
+
+### Detailed Memory Metrics
+
+When available (OS-dependent), additional memory details are captured:
+
+- **map_count**: Number of memory mappings (stacks, libraries, etc.).
+- **proportional_size**: Memory usage accounting for shared memory (KB).
+- **shared_clean_size**: Unmodified shared memory (KB).
+- **shared_dirty_size**: Modified shared memory (KB).
+- **private_clean_size**: Unmodified private memory (KB).
+- **private_dirty_size**: Modified private memory (KB).
+- **referenced_size**: Active page-cache (KB).
+- **anonymous_size**: Memory not backed by files (KB)
+- **swap_size**: Memory swapped to disk (KB).
+- **proportional_swap_size**: Proportional swap usage (KB).
+- **major_faults**: The number of page faults requiring I/O.
+- **minor_faults**: The number of page faults that don't require I/O (e.g. CoW).

data/lib/async/service/supervisor/client.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2025, by Samuel Williams.
+
+require "async/bus/client"
+
+module Async
+	module Service
+		module Supervisor
+			# A client provides a mechanism to connect to a supervisor server in order to execute operations.
+			class Client < Async::Bus::Client
+				# Initialize a new client.
+				#
+				# @parameter endpoint [IO::Endpoint] The supervisor endpoint to connect to.
+				def initialize(endpoint: Supervisor.endpoint, **options)
+					super(endpoint, **options)
+				end
+			end
+		end
+	end
+end

data/lib/async/service/supervisor/endpoint.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2025, by Samuel Williams.
+
+require "io/endpoint/unix_endpoint"
+
+module Async
+	module Service
+		module Supervisor
+			# Get the supervisor IPC endpoint.
+			#
+			# @parameter path [String] The path for the Unix socket (default: "supervisor.ipc").
+			# @returns [IO::Endpoint] The Unix socket endpoint.
+			def self.endpoint(path = "supervisor.ipc")
+				::IO::Endpoint.unix(path)
+			end
+		end
+	end
+end
+

data/lib/async/service/supervisor/environment.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2025, by Samuel Williams.
+
+require "async/service/environment"
+require "async/service/managed/environment"
+
+require_relative "service"
+
+module Async
+	module Service
+		module Supervisor
+			# An environment mixin for supervisor services.
+			#
+			# Provides configuration and setup for supervisor processes that monitor workers.
+			module Environment
+				include Async::Service::Managed::Environment
+				
+				# The service class to use for the supervisor.
+				# @returns [Class]
+				def service_class
+					Supervisor::Service
+				end
+				
+				# The name of the supervisor
+				# @returns [String]
+				def name
+					"supervisor"
+				end
+				
+				# The IPC path to use for communication with the supervisor.
+				# @returns [String]
+				def ipc_path
+					::File.expand_path("supervisor.ipc", root)
+				end
+				
+				# The endpoint the supervisor will bind to.
+				# @returns [::IO::Endpoint::Generic]
+				def endpoint
+					::IO::Endpoint.unix(ipc_path)
+				end
+				
+				# Number of supervisor instances (always 1).
+				# @returns [Integer]
+				def count
+					1
+				end
+				
+				# Options to use when creating the container.
+				# Merges with Managed::Environment defaults.
+				def container_options
+					super.merge(restart: true, count: self.count)
+				end
+				
+				# Get the list of monitors to run in the supervisor.
+				#
+				# Override this method to provide custom monitors.
+				#
+				# @returns [Array] The list of monitor instances.
+				def monitors
+					[]
+				end
+				
+				# Create the supervisor server instance.
+				#
+				# @returns [Server] The supervisor server.
+				def make_server(endpoint)
+					Server.new(endpoint: endpoint, monitors: self.monitors)
+				end
+			end
+		end
+	end
+end
+

data/lib/async/service/supervisor/loop.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Async
+	module Service
+		module Supervisor
+			# A helper for running loops at aligned intervals.
+			module Loop
+				# A robust loop that executes a block at aligned intervals.
+				#
+				# The alignment is modulo the current clock in seconds.
+				#
+				# If an error occurs during the execution of the block, it is logged and the loop continues.
+				#
+				# @parameter interval [Integer] The interval in seconds between executions of the block.
+				def self.run(interval: 60, &block)
+					while true
+						# Compute the wait time to the next interval:
+						wait = interval - (Time.now.to_f % interval)
+						if wait.positive?
+							# Sleep until the next interval boundary:
+							sleep(wait)
+						end
+						
+						begin
+							yield
+						rescue => error
+							Console.error(self, "Loop error:", error)
+						end
+					end
+				end
+			end
+			
+			private_constant :Loop
+		end
+	end
+end
+