@opengroundplan/core 0.1.1

package/templates/layers/health-checks/files/internal/health/health_test.go

@@ -0,0 +1,102 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func testConfig() Config {
+	return Config{
+		LivePath:       "/health/live",
+		ReadyPath:      "/health/ready",
+		StartupPath:    "/health/startup",
+		ProbeTimeout:   50 * time.Millisecond,
+		ReadyCache:     0,
+		ServiceName:    "test",
+		ServiceVersion: "0.0.0",
+	}
+}
+
+func TestHealthyWithoutProbes(t *testing.T) {
+	reg := NewRegistry(testConfig())
+
+	if got := reg.Run(context.Background(), Readiness).Status; got != StatusHealthy {
+		t.Fatalf("status = %q, want %q", got, StatusHealthy)
+	}
+}
+
+func TestCriticalFailureIsUnhealthy(t *testing.T) {
+	reg := NewRegistry(testConfig())
+	reg.Register(CallbackProbe("db", func(context.Context) error {
+		return errors.New("connection refused")
+	}))
+
+	if got := reg.Run(context.Background(), Readiness).Status; got != StatusUnhealthy {
+		t.Fatalf("status = %q, want %q", got, StatusUnhealthy)
+	}
+}
+
+func TestNonCriticalFailureOnlyDegrades(t *testing.T) {
+	reg := NewRegistry(testConfig())
+	probe := CallbackProbe("search", func(context.Context) error { return errors.New("down") })
+	probe.Critical = false
+	reg.Register(probe)
+
+	if got := reg.Run(context.Background(), Readiness).Status; got != StatusDegraded {
+		t.Fatalf("status = %q, want %q", got, StatusDegraded)
+	}
+}
+
+func TestLivenessIgnoresReadinessProbes(t *testing.T) {
+	reg := NewRegistry(testConfig())
+	reg.Register(CallbackProbe("queue", func(context.Context) error { return errors.New("down") }))
+	reg.Register(ProcessProbe())
+
+	if got := reg.Run(context.Background(), Liveness).Status; got != StatusHealthy {
+		t.Fatalf("status = %q, want %q", got, StatusHealthy)
+	}
+}
+
+func TestSlowProbeTimesOut(t *testing.T) {
+	reg := NewRegistry(testConfig())
+	reg.Register(CallbackProbe("slow", func(ctx context.Context) error {
+		<-ctx.Done()
+		return ctx.Err()
+	}))
+
+	report := reg.Run(context.Background(), Readiness)
+	if report.Status != StatusUnhealthy {
+		t.Fatalf("status = %q, want %q", report.Status, StatusUnhealthy)
+	}
+}
+
+func TestReadinessEndpointFailsClosed(t *testing.T) {
+	reg := NewRegistry(testConfig())
+	reg.Register(CallbackProbe("db", func(context.Context) error { return errors.New("down") }))
+
+	rec := httptest.NewRecorder()
+	reg.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/health/ready", nil))
+
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("code = %d, want %d", rec.Code, http.StatusServiceUnavailable)
+	}
+}
+
+func TestDetailIsHiddenByDefault(t *testing.T) {
+	reg := NewRegistry(testConfig())
+	reg.Register(CallbackProbe("db", func(context.Context) error {
+		return errors.New("host=db-primary port=5432")
+	}))
+
+	rec := httptest.NewRecorder()
+	reg.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/health/ready", nil))
+
+	if body := rec.Body.String(); strings.Contains(body, "db-primary") {
+		t.Fatalf("probe detail leaked into the response: %s", body)
+	}
+}

package/templates/layers/health-checks/files/internal/health/http.go

@@ -0,0 +1,56 @@
+package health
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+)
+
+// Handler serves the three probe paths from the resolved config. Mount it on any mux:
+//
+//	mux.Handle("/", reg.Handler())
+//
+// or register the paths individually with reg.HandlerFor(health.Readiness).
+func (r *Registry) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.Handle("GET "+r.cfg.LivePath, r.HandlerFor(Liveness))
+	mux.Handle("GET "+r.cfg.ReadyPath, r.HandlerFor(Readiness))
+	mux.Handle("GET "+r.cfg.StartupPath, r.HandlerFor(Startup))
+	return mux
+}
+
+// HandlerFor serves one probe kind, so an existing router can mount the paths itself.
+func (r *Registry) HandlerFor(kind Kind) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		report := r.Run(req.Context(), kind)
+
+		status := http.StatusOK
+		// Degraded still serves traffic, so only a hard failure may fail the probe.
+		if report.Status == StatusUnhealthy {
+			status = http.StatusServiceUnavailable
+		}
+
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		// A cached probe answer is a stale probe answer.
+		w.Header().Set("Cache-Control", "no-store")
+		w.WriteHeader(status)
+
+		if err := json.NewEncoder(w).Encode(r.body(report)); err != nil {
+			slog.ErrorContext(req.Context(), "encode health report", "err", err)
+		}
+	})
+}
+
+// body hides probe detail unless it is explicitly exposed: names and statuses are safe to
+// publish, messages and payloads can leak topology.
+func (r *Registry) body(report Report) Report {
+	if r.cfg.ExposeDetails {
+		return report
+	}
+	redacted := report
+	redacted.Checks = make([]Entry, len(report.Checks))
+	for i, check := range report.Checks {
+		redacted.Checks[i] = Entry{Name: check.Name, Status: check.Status, Critical: check.Critical}
+	}
+	return redacted
+}

package/templates/layers/health-checks/files/internal/health/probes.go

@@ -0,0 +1,88 @@
+package health
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+)
+
+// HTTPProbe checks a dependency over HTTP. The URL is supplied by the caller from its own
+// config layer, never inlined here.
+func HTTPProbe(name, url string, kinds ...Kind) Probe {
+	client := &http.Client{}
+	return Probe{
+		Name:     name,
+		Kinds:    withDefault(kinds),
+		Critical: true,
+		Check: func(ctx context.Context) Result {
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+			if err != nil {
+				return Result{Status: StatusUnhealthy, Detail: err.Error()}
+			}
+			resp, err := client.Do(req)
+			if err != nil {
+				return Result{Status: StatusUnhealthy, Detail: err.Error()}
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode >= http.StatusBadRequest {
+				return Result{Status: StatusUnhealthy, Detail: fmt.Sprintf("HTTP %d", resp.StatusCode)}
+			}
+			return Result{Status: StatusHealthy}
+		},
+	}
+}
+
+// TCPProbe is for dependencies with no HTTP surface — a broker, a cache, a database socket.
+func TCPProbe(name, address string, kinds ...Kind) Probe {
+	var dialer net.Dialer
+	return Probe{
+		Name:     name,
+		Kinds:    withDefault(kinds),
+		Critical: true,
+		Check: func(ctx context.Context) Result {
+			conn, err := dialer.DialContext(ctx, "tcp", address)
+			if err != nil {
+				return Result{Status: StatusUnhealthy, Detail: err.Error()}
+			}
+			_ = conn.Close()
+			return Result{Status: StatusHealthy}
+		},
+	}
+}
+
+// CallbackProbe wraps any "does this still work?" call — a SELECT 1, a PING, a queue read.
+func CallbackProbe(name string, run func(ctx context.Context) error, kinds ...Kind) Probe {
+	return Probe{
+		Name:     name,
+		Kinds:    withDefault(kinds),
+		Critical: true,
+		Check: func(ctx context.Context) Result {
+			if err := run(ctx); err != nil {
+				return Result{Status: StatusUnhealthy, Detail: err.Error()}
+			}
+			return Result{Status: StatusHealthy}
+		},
+	}
+}
+
+// ProcessProbe answers liveness without touching the network: the process still schedules work.
+func ProcessProbe() Probe {
+	return Probe{
+		Name: "process",
+		// Also covers startup, so the endpoint answers before the app registers a real boot gate.
+		Kinds:    []Kind{Liveness, Startup},
+		Critical: true,
+		Check: func(context.Context) Result {
+			return Result{Status: StatusHealthy}
+		},
+	}
+}
+
+func withDefault(kinds []Kind) []Kind {
+	if len(kinds) == 0 {
+		return []Kind{Readiness}
+	}
+	return kinds
+}

package/templates/layers/health-checks/files/internal/health/runner.go

@@ -0,0 +1,100 @@
+package health
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// Run executes every probe for the kind concurrently under the configured timeout budget.
+func (r *Registry) Run(ctx context.Context, kind Kind) Report {
+	if kind == Readiness {
+		if cached, ok := r.readCache(); ok {
+			return cached
+		}
+	}
+
+	started := time.Now()
+	probes := r.probesFor(kind)
+	entries := make([]Entry, len(probes))
+
+	var wg sync.WaitGroup
+	for i, p := range probes {
+		wg.Add(1)
+		go func(i int, p Probe) {
+			defer wg.Done()
+			entries[i] = r.runProbe(ctx, p)
+		}(i, p)
+	}
+	wg.Wait()
+
+	report := Report{
+		Status:        aggregate(entries),
+		Service:       r.cfg.ServiceName,
+		Version:       r.cfg.ServiceVersion,
+		UptimeSeconds: int64(time.Since(r.startedAt).Seconds()),
+		CheckedAt:     time.Now().UTC().Format(time.RFC3339),
+		DurationMS:    time.Since(started).Milliseconds(),
+		Checks:        entries,
+	}
+
+	if kind == Readiness && r.cfg.ReadyCache > 0 {
+		r.writeCache(report)
+	}
+	return report
+}
+
+func (r *Registry) runProbe(ctx context.Context, p Probe) Entry {
+	ctx, cancel := context.WithTimeout(ctx, r.cfg.ProbeTimeout)
+	defer cancel()
+
+	started := time.Now()
+	done := make(chan Result, 1)
+	// Buffered and abandoned on timeout: a wedged probe must not block the endpoint.
+	go func() { done <- p.Check(ctx) }()
+
+	entry := Entry{Name: p.Name, Critical: p.Critical}
+	select {
+	case result := <-done:
+		entry.Status = result.Status
+		entry.Detail = result.Detail
+		entry.Data = result.Data
+	case <-ctx.Done():
+		entry.Status = StatusUnhealthy
+		entry.Detail = "timeout"
+	}
+	entry.DurationMS = time.Since(started).Milliseconds()
+	return entry
+}
+
+func (r *Registry) readCache() (Report, bool) {
+	r.cacheMu.Lock()
+	defer r.cacheMu.Unlock()
+	if r.cached != nil && time.Now().Before(r.cacheUntil) {
+		return *r.cached, true
+	}
+	return Report{}, false
+}
+
+func (r *Registry) writeCache(report Report) {
+	r.cacheMu.Lock()
+	defer r.cacheMu.Unlock()
+	r.cached = &report
+	r.cacheUntil = time.Now().Add(r.cfg.ReadyCache)
+}
+
+func aggregate(entries []Entry) Status {
+	status := StatusHealthy
+	for _, e := range entries {
+		if e.Status == StatusUnhealthy {
+			// A non-critical outage costs a feature, not the whole instance.
+			if e.Critical {
+				return StatusUnhealthy
+			}
+			status = StatusDegraded
+		} else if e.Status == StatusDegraded {
+			status = StatusDegraded
+		}
+	}
+	return status
+}

package/templates/layers/health-checks/files/package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "vitest": "^4.1.10"
+  }
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/DependencyProbeOptions.cs

@@ -0,0 +1,26 @@
+namespace {{ solutionName | dot }}.Health;
+
+public enum DependencyProbeKind
+{
+    Http,
+    Tcp,
+}
+
+// One configured dependency. Addresses live in configuration, never in a call site.
+public sealed class DependencyProbeOptions
+{
+    public string Name { get; set; } = string.Empty;
+
+    public DependencyProbeKind Kind { get; set; } = DependencyProbeKind.Http;
+
+    public Uri? Url { get; set; }
+
+    public string? Host { get; set; }
+
+    public int Port { get; set; }
+
+    // False reports Degraded instead of Unhealthy, so the instance keeps serving traffic.
+    public bool Critical { get; set; } = true;
+
+    public string Tag { get; set; } = HealthTags.Ready;
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/HealthEndpointExtensions.cs

@@ -0,0 +1,34 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+
+namespace {{ solutionName | dot }}.Health;
+
+public static class HealthEndpointExtensions
+{
+    // Maps the three probe paths from configuration, so an orchestrator convention is one setting away.
+    public static IEndpointRouteBuilder MapAppHealthChecks(this IEndpointRouteBuilder endpoints)
+    {
+        ArgumentNullException.ThrowIfNull(endpoints);
+
+        var options = endpoints.ServiceProvider.GetRequiredService<IOptions<HealthOptions>>().Value;
+        MapProbe(endpoints, options.LivePath, HealthTags.Live);
+        MapProbe(endpoints, options.ReadyPath, HealthTags.Ready);
+        MapProbe(endpoints, options.StartupPath, HealthTags.Startup);
+        return endpoints;
+    }
+
+    private static void MapProbe(IEndpointRouteBuilder endpoints, string path, string tag) =>
+        endpoints
+            .MapGet(path, (
+                HttpContext http,
+                HealthCheckService checks,
+                HealthProbeResponder responder,
+                CancellationToken cancellationToken) =>
+                responder.RespondAsync(http, checks, tag, cancellationToken))
+            // Orchestrators probe these; clients do not call them.
+            .ExcludeFromDescription();
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/HealthOptions.cs

@@ -0,0 +1,25 @@
+namespace {{ solutionName | dot }}.Health;
+
+// The one typed default table for the health surface. Bound from the "Health" configuration
+// section, so every value is overridable by appsettings, environment, or user secrets.
+public sealed class HealthOptions
+{
+    public const string SectionName = "Health";
+
+    public string LivePath { get; set; } = "{{ healthLivePath }}";
+
+    public string ReadyPath { get; set; } = "{{ healthReadyPath }}";
+
+    public string StartupPath { get; set; } = "{{ healthStartupPath }}";
+
+    // Must stay below the orchestrator's own probe timeout or the probe is cut off first.
+    public int ProbeTimeoutMs { get; set; } = {{ healthProbeTimeoutMs }};
+
+    // Probes fire every few seconds from every replica; without this they stampede a dependency.
+    public int ReadyCacheMs { get; set; } = {{ healthReadyCacheMs }};
+
+    // Probe endpoints are usually unauthenticated, so failure detail is opt-in.
+    public bool ExposeDetails { get; set; }
+
+    public IList<DependencyProbeOptions> Dependencies { get; } = [];
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/HealthProbeResponder.cs

@@ -0,0 +1,121 @@
+using System.Text.Json;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+
+namespace {{ solutionName | dot }}.Health;
+
+// Runs the checks for one tag and writes the response. Registered as a singleton so the
+// readiness answer can be shared across concurrent probes instead of re-running the checks.
+public sealed class HealthProbeResponder(IOptions<HealthOptions> options)
+{
+    private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
+
+    private readonly HealthOptions _options = options.Value;
+    private readonly object _gate = new();
+
+    private DateTimeOffset _readyCachedUntil = DateTimeOffset.MinValue;
+    private byte[]? _readyPayload;
+    private int _readyStatusCode;
+
+    public async Task RespondAsync(
+        HttpContext http,
+        HealthCheckService checks,
+        string tag,
+        CancellationToken cancellationToken)
+    {
+        // A cached probe answer is a stale probe answer, as far as the client is concerned.
+        http.Response.Headers.CacheControl = "no-store";
+
+        if (tag == HealthTags.Ready && TryReadCache(out var cached, out var cachedCode))
+        {
+            await WriteAsync(http, cachedCode, cached, cancellationToken);
+            return;
+        }
+
+        var report = await checks.CheckHealthAsync(
+            registration => registration.Tags.Contains(tag),
+            cancellationToken);
+
+        // Degraded still serves traffic, so only a hard failure may fail the probe.
+        var code = report.Status == HealthStatus.Unhealthy
+            ? StatusCodes.Status503ServiceUnavailable
+            : StatusCodes.Status200OK;
+        var body = JsonSerializer.SerializeToUtf8Bytes(Describe(report), SerializerOptions);
+
+        if (tag == HealthTags.Ready && _options.ReadyCacheMs > 0)
+        {
+            WriteCache(body, code);
+        }
+
+        await WriteAsync(http, code, body, cancellationToken);
+    }
+
+    // Drop the cached answer so a draining instance stops advertising itself.
+    public void InvalidateReadyCache()
+    {
+        lock (_gate)
+        {
+            _readyCachedUntil = DateTimeOffset.MinValue;
+            _readyPayload = null;
+        }
+    }
+
+    private object Describe(HealthReport report)
+    {
+        // Names and statuses are safe to publish; descriptions can leak topology.
+        var entries = report.Entries.Select(entry => new
+        {
+            name = entry.Key,
+            status = entry.Value.Status.ToString(),
+            durationMs = _options.ExposeDetails ? (long)entry.Value.Duration.TotalMilliseconds : 0L,
+            detail = _options.ExposeDetails ? entry.Value.Description : null,
+        });
+
+        return new
+        {
+            status = report.Status.ToString(),
+            durationMs = (long)report.TotalDuration.TotalMilliseconds,
+            checkedAt = DateTimeOffset.UtcNow,
+            checks = entries,
+        };
+    }
+
+    private bool TryReadCache(out byte[] payload, out int statusCode)
+    {
+        lock (_gate)
+        {
+            if (_readyPayload is not null && DateTimeOffset.UtcNow < _readyCachedUntil)
+            {
+                payload = _readyPayload;
+                statusCode = _readyStatusCode;
+                return true;
+            }
+        }
+
+        payload = [];
+        statusCode = 0;
+        return false;
+    }
+
+    private void WriteCache(byte[] payload, int statusCode)
+    {
+        lock (_gate)
+        {
+            _readyPayload = payload;
+            _readyStatusCode = statusCode;
+            _readyCachedUntil = DateTimeOffset.UtcNow.AddMilliseconds(_options.ReadyCacheMs);
+        }
+    }
+
+    private static async Task WriteAsync(
+        HttpContext http,
+        int statusCode,
+        byte[] payload,
+        CancellationToken cancellationToken)
+    {
+        http.Response.StatusCode = statusCode;
+        http.Response.ContentType = "application/json; charset=utf-8";
+        await http.Response.Body.WriteAsync(payload, cancellationToken);
+    }
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/HealthServiceCollectionExtensions.cs

@@ -0,0 +1,51 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace {{ solutionName | dot }}.Health;
+
+public static class HealthServiceCollectionExtensions
+{
+    // Registers one health check per configured dependency. Nothing is named in code:
+    // the Health:Dependencies section decides what this service considers itself ready for.
+    public static IServiceCollection AddAppHealthChecks(
+        this IServiceCollection services,
+        IConfiguration configuration)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+        ArgumentNullException.ThrowIfNull(configuration);
+
+        var section = configuration.GetSection(HealthOptions.SectionName);
+        services.Configure<HealthOptions>(section);
+        services.AddHttpClient();
+        services.AddSingleton<HealthProbeResponder>();
+
+        // Checks are registered up front, so the binding is read once here rather than per request.
+        var options = new HealthOptions();
+        section.Bind(options);
+        var timeout = TimeSpan.FromMilliseconds(options.ProbeTimeoutMs);
+
+        var builder = services.AddHealthChecks();
+        // Liveness must not touch the network: it only reports that the process still schedules work.
+        builder.AddCheck("process", () => HealthCheckResult.Healthy(), [HealthTags.Live, HealthTags.Startup]);
+
+        foreach (var dependency in options.Dependencies)
+        {
+            builder.Add(new HealthCheckRegistration(
+                dependency.Name,
+                provider => Create(provider, dependency),
+                dependency.Critical ? HealthStatus.Unhealthy : HealthStatus.Degraded,
+                [dependency.Tag],
+                timeout));
+        }
+
+        return services;
+    }
+
+    private static IHealthCheck Create(IServiceProvider provider, DependencyProbeOptions dependency) =>
+        dependency.Kind == DependencyProbeKind.Tcp
+            ? new TcpDependencyHealthCheck(dependency)
+            : new HttpDependencyHealthCheck(
+                provider.GetRequiredService<IHttpClientFactory>(),
+                dependency);
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/HealthTags.cs

@@ -0,0 +1,10 @@
+namespace {{ solutionName | dot }}.Health;
+
+// Liveness answers "restart me?", readiness "send me traffic?", startup "still booting?".
+// Collapsing them into one endpoint is what turns a slow dependency into a restart loop.
+public static class HealthTags
+{
+    public const string Live = "live";
+    public const string Ready = "ready";
+    public const string Startup = "startup";
+}

package/templates/layers/health-checks/files/src/__solutionName.dot__.Health/HttpDependencyHealthCheck.cs

@@ -0,0 +1,44 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace {{ solutionName | dot }}.Health;
+
+// Probes a dependency over HTTP. The URL arrives from configuration through the options object.
+public sealed class HttpDependencyHealthCheck(IHttpClientFactory factory, DependencyProbeOptions options)
+    : IHealthCheck
+{
+    private readonly HealthStatus _failureStatus =
+        options.Critical ? HealthStatus.Unhealthy : HealthStatus.Degraded;
+
+    public async Task<HealthCheckResult> CheckHealthAsync(
+        HealthCheckContext context,
+        CancellationToken cancellationToken = default)
+    {
+        if (options.Url is null)
+        {
+            return new HealthCheckResult(_failureStatus, $"{options.Name}: no Url configured");
+        }
+
+        using var client = factory.CreateClient(nameof(HttpDependencyHealthCheck));
+        try
+        {
+            using var request = new HttpRequestMessage(HttpMethod.Get, options.Url);
+            // Headers are enough to judge reachability; the body may be large.
+            using var response = await client.SendAsync(
+                request,
+                HttpCompletionOption.ResponseHeadersRead,
+                cancellationToken);
+
+            return response.IsSuccessStatusCode
+                ? HealthCheckResult.Healthy()
+                : new HealthCheckResult(_failureStatus, $"HTTP {(int)response.StatusCode}");
+        }
+        catch (HttpRequestException error)
+        {
+            return new HealthCheckResult(_failureStatus, error.Message, error);
+        }
+        catch (TaskCanceledException error) when (!cancellationToken.IsCancellationRequested)
+        {
+            return new HealthCheckResult(_failureStatus, "timeout", error);
+        }
+    }
+}