@senad-d/observme 0.1.2 → 0.1.4

package/examples/README.md

@@ -11,6 +11,7 @@ The npm package ships configuration examples and an extension-integration exampl
 - Local self-signed TLS and IPv4 preference are enabled for that profile.
 - Prompt, response, thinking, tool, Bash, and path capture remain disabled.
 - Redaction remains enabled and unsafe capture remains disabled.
+- Metrics export every 15 seconds and renew a 60-second active-agent lease; keep producer and Prometheus clocks synchronized within 5 seconds.
 
 To use it in a trusted project:
 
@@ -27,13 +28,17 @@ Read [`../docs/configuration.md`](../docs/configuration.md) for the quick guide
 
 - traces are exported to Tempo;
 - logs are exported to Loki;
-- metrics are remote-written to Mimir;
+- metrics are exposed on a Prometheus scrape endpoint;
+- the five-minute `metric_expiration` applies to all exported metric types only as stale-series/cardinality cleanup, not active-agent liveness;
+- lease-aware PromQL joins the positive lifecycle claim and lease on generated `observme_instance_id`; raw `observme_active_agents` sums are diagnostics, not production live totals;
 - high-cardinality workflow, agent, session, and spawn attributes are removed from metrics;
 - accidental content attributes are removed from logs as defense in depth.
 
 The example uses Collector components commonly provided by the Collector Contrib distribution. Verify that your deployed distribution includes every receiver, processor, and exporter, then replace backend endpoints and security settings for your environment.
 
-Read [`../docs/reference/05-otel-pipeline-and-collector.md`](../docs/reference/05-otel-pipeline-and-collector.md) for pipeline design and [`../docs/reference/11-deployment-runbooks.md`](../docs/reference/11-deployment-runbooks.md) for deployment and incident checks.
+Read [`../docs/reference/05-otel-pipeline-and-collector.md`](../docs/reference/05-otel-pipeline-and-collector.md) for pipeline design, [`../docs/reference/09-dashboards-alerts-slos.md`](../docs/reference/09-dashboards-alerts-slos.md#131-canonical-active-agent-promql) for canonical queries and raw-query migration, and [`../docs/reference/11-deployment-runbooks.md`](../docs/reference/11-deployment-runbooks.md) for deployment and incident checks.
+
+For GitHub Actions, prefer graceful Pi/sidecar shutdown and an `if: always()` cleanup step, but do not depend on either after force cancellation. Lease expiry provides correctness without a Collector restart. GitHub-hosted runner clocks meet the supported expectation; self-hosted runners must use reliable time synchronization.
 
 ## `integrations/subagent-runner.ts`
 

package/examples/collector.yaml

@@ -1,4 +1,4 @@
-# Production-oriented Collector reference for Tempo, Loki, and Mimir.
+# Production-oriented Collector reference for Tempo, Loki, and Prometheus.
 # Verify that your Collector distribution contains every configured component,
 # replace backend endpoints/security for your deployment, and never add raw
 # content or high-cardinality execution identifiers to metric labels.
@@ -97,14 +97,14 @@ exporters:
     retry_on_failure:
       enabled: true
 
-  prometheusremotewrite/mimir:
-    endpoint: http://mimir:9009/api/v1/push
+  prometheus:
+    endpoint: 0.0.0.0:8889
+    # Exporter-wide stale-series/cardinality cleanup for gauges, counters, and
+    # histograms. Five minutes exceeds ObservMe's default one-minute active-agent
+    # lease; lease-aware PromQL, not expiration, determines liveness.
+    metric_expiration: 5m
     resource_to_telemetry_conversion:
       enabled: true
-    sending_queue:
-      enabled: true
-    retry_on_failure:
-      enabled: true
 
   debug:
     verbosity: basic
@@ -120,7 +120,7 @@ service:
     metrics:
       receivers: [otlp]
       processors: [memory_limiter, resource/observme, resource/drop_high_cardinality_metric_attrs, batch]
-      exporters: [prometheusremotewrite/mimir]
+      exporters: [prometheus]
 
     logs:
       receivers: [otlp]

package/examples/observme.yaml

@@ -61,6 +61,7 @@ observme:
     enabled: true
     exportIntervalMillis: 15000
     exportTimeoutMillis: 3000
+    activeAgentLeaseDurationMillis: 60000
 
   logs:
     enabled: true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@senad-d/observme",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "type": "module",
   "description": "ObservMe: a Pi extension that instruments Pi agent sessions and exports OpenTelemetry traces, metrics, and logs to an external observability stack (OTel Collector, Grafana Tempo/Loki/Prometheus).",
   "license": "MIT",
@@ -57,7 +57,9 @@
     "lint:fix": "eslint . --fix",
     "format:check": "node scripts/check-format.mjs",
     "test": "node --test test/*.test.mjs test/*.test.ts",
+    "test:active-agent-lease:contracts": "node --test test/active-agent-lease.test.ts test/alerts.test.mjs test/cardinality.test.ts test/config-defaults.test.mjs test/config-loader.test.mjs test/config-validation.test.mjs test/dashboards.test.mjs test/metrics.test.ts test/pi-handlers.test.mjs test/semconv-metrics.test.mjs",
     "test:integration:collector": "node --test test/integration/collector-debug.test.mjs",
+    "test:integration:active-agent-lease": "node --test test/integration/active-agent-lease.test.mjs",
     "test:integration:grafana-stack": "node --test test/integration/grafana-stack.test.mjs",
     "test:coverage": "node scripts/test-coverage.mjs",
     "smoke:discover": "node scripts/smoke-observability.mjs",

package/src/config/bootstrap-project-config.ts

@@ -101,6 +101,7 @@ export const PROJECT_OBSERVME_YAML_TEMPLATE = `observme:
     enabled: true
     exportIntervalMillis: 15000
     exportTimeoutMillis: 3000
+    activeAgentLeaseDurationMillis: 60000
 
   logs:
     enabled: true

package/src/config/defaults.ts

@@ -60,6 +60,7 @@ export const defaultObservMeConfig = {
     enabled: true,
     exportIntervalMillis: 15000,
     exportTimeoutMillis: 3000,
+    activeAgentLeaseDurationMillis: 60000,
   },
   logs: {
     enabled: true,

package/src/config/load-config.ts

@@ -147,6 +147,11 @@ export function envToConfig(env: NodeJS.ProcessEnv = process.env): DeepPartial<O
   setString(config, ["otlp", "signalEndpoints", "metrics"], env.OBSERVME_OTLP_METRICS_ENDPOINT);
   setString(config, ["otlp", "signalEndpoints", "logs"], env.OBSERVME_OTLP_LOGS_ENDPOINT);
   setNumber(config, ["otlp", "timeoutMs"], env.OBSERVME_OTLP_TIMEOUT_MS);
+  setNumberForValidation(
+    config,
+    ["metrics", "activeAgentLeaseDurationMillis"],
+    env.OBSERVME_ACTIVE_AGENT_LEASE_DURATION_MS,
+  );
   setAuthorizationHeader(config, env.OBSERVME_OTLP_TOKEN);
   setNumber(config, ["workflow", "maxDepthWarning"], env.OBSERVME_WORKFLOW_MAX_DEPTH_WARNING);
   setNumber(config, ["workflow", "maxFanoutWarning"], env.OBSERVME_WORKFLOW_MAX_FANOUT_WARNING);
@@ -592,6 +597,12 @@ function setNumber(target: DeepPartial<ObservMeConfig>, path: string[], value: s
   setPathValue(target as Record<string, unknown>, path, parsed);
 }
 
+function setNumberForValidation(target: DeepPartial<ObservMeConfig>, path: string[], value: string | undefined) {
+  if (value === undefined || value === "") return;
+  const parsed = Number(value);
+  setPathValue(target as Record<string, unknown>, path, Number.isFinite(parsed) ? parsed : value);
+}
+
 function setBoolean(target: DeepPartial<ObservMeConfig>, path: string[], value: string | undefined) {
   const parsed = parseBooleanEnv(value);
   if (parsed === undefined) return;

package/src/config/schema.ts

@@ -8,6 +8,10 @@ export type ObservMeEnvironment = (typeof observMeEnvironments)[number];
 export type OtlpProtocol = "http/protobuf";
 export type PrivacyPathMode = (typeof privacyPathModes)[number];
 
+export const ACTIVE_AGENT_LEASE_DURATION_MILLIS_MINIMUM = 10_000;
+export const ACTIVE_AGENT_LEASE_DURATION_MILLIS_MAXIMUM = 300_000;
+export const ACTIVE_AGENT_LEASE_EXPORT_SAFETY_MARGIN_MILLIS = 5_000;
+
 export interface OtlpTlsConfig {
   enabled: boolean;
   insecureSkipVerify: boolean;
@@ -71,6 +75,7 @@ export interface MetricsConfig {
   enabled: boolean;
   exportIntervalMillis: number;
   exportTimeoutMillis: number;
+  activeAgentLeaseDurationMillis: number;
   labels?: string[];
 }
 
@@ -288,6 +293,10 @@ export const observMeConfigSchema = Type.Object(
         enabled: Type.Boolean(),
         exportIntervalMillis: positiveIntegerSchema,
         exportTimeoutMillis: positiveIntegerSchema,
+        activeAgentLeaseDurationMillis: Type.Integer({
+          minimum: ACTIVE_AGENT_LEASE_DURATION_MILLIS_MINIMUM,
+          maximum: ACTIVE_AGENT_LEASE_DURATION_MILLIS_MAXIMUM,
+        }),
         labels: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
       },
       { additionalProperties: false },

package/src/config/validate.ts

@@ -1,6 +1,9 @@
 import { Compile } from "typebox/compile";
 import { defaultObservMeConfig } from "./defaults.ts";
-import { observMeConfigSchema } from "./schema.ts";
+import {
+  ACTIVE_AGENT_LEASE_EXPORT_SAFETY_MARGIN_MILLIS,
+  observMeConfigSchema,
+} from "./schema.ts";
 import type { ObservMeConfig } from "./schema.ts";
 import { validateCustomRedactionPatterns } from "../privacy/redact.ts";
 
@@ -58,6 +61,7 @@ const knownConfigValidationIssueCodes = new Set([
   "insecure_production_transport",
   "invalid_signal_endpoint_path",
   "high_cardinality_metric_label",
+  "active_agent_lease_too_short_for_export_interval",
   "custom_redaction_pattern_limit",
   "custom_redaction_pattern_too_long",
   "custom_redaction_pattern_unsupported_construct",
@@ -82,6 +86,7 @@ export function validateObservMeConfig(
     ...validateTransportSecurity(config),
     ...validateSignalEndpoints(config),
     ...validateMetricLabels(config),
+    ...validateActiveAgentLeaseDuration(config),
     ...validateCustomRedactionPatternConfig(config),
     ...validateProjectTrust(options),
     ...validateLineageEnvironment(config, options.env ?? process.env),
@@ -307,6 +312,20 @@ function validateMetricLabels(config: ObservMeConfig): ValidationIssue[] {
   }));
 }
 
+function validateActiveAgentLeaseDuration(config: ObservMeConfig): ValidationIssue[] {
+  const requiredDuration =
+    (2 * config.metrics.exportIntervalMillis) + ACTIVE_AGENT_LEASE_EXPORT_SAFETY_MARGIN_MILLIS;
+  if (config.metrics.activeAgentLeaseDurationMillis >= requiredDuration) return [];
+
+  return [
+    {
+      code: "active_agent_lease_too_short_for_export_interval",
+      message:
+        "metrics.activeAgentLeaseDurationMillis must be at least twice metrics.exportIntervalMillis plus the clock-skew safety margin.",
+    },
+  ];
+}
+
 function validateCustomRedactionPatternConfig(config: ObservMeConfig): ValidationIssue[] {
   return validateCustomRedactionPatterns(config.privacy.customRedactionPatterns).map(issue => ({
     code: issue.code,

package/src/integration.ts

@@ -10,6 +10,10 @@ export type ObservMeChildStatus = "starting" | "active" | "completed" | "failed"
 export type ObservMeJoinStatus = "completed" | "failed" | "cancelled" | "timeout" | "unknown" | "waiting";
 export type ObservMeIntegrationFailureReason =
   | "session_unavailable"
+  | "invalid_request"
+  | "spawn_already_exists"
+  | "wait_already_exists"
+  | "join_already_exists"
   | "spawn_not_found"
   | "wait_not_found"
   | "join_not_found"
@@ -118,17 +122,55 @@ interface IntegrationResponseHolder {
 }
 
 export function requestObservMeIntegration(host: ObservMeIntegrationHost): ObservMeIntegrationApi | undefined {
+  const events = resolveIntegrationEventBus(host);
+  if (!events) return undefined;
+
   const holder: IntegrationResponseHolder = {};
   const request: ObservMeIntegrationRequest = {
     supportedVersions: [OBSERVME_INTEGRATION_VERSION],
     respond: receiveObservMeIntegration.bind(undefined, holder),
   };
 
-  host.events.emit(OBSERVME_INTEGRATION_CHANNEL, request);
+  try {
+    events.emit(OBSERVME_INTEGRATION_CHANNEL, request);
+  } catch {
+    return undefined;
+  }
   return holder.api;
 }
 
-function receiveObservMeIntegration(holder: IntegrationResponseHolder, api: ObservMeIntegrationApi): void {
-  if (api.version !== OBSERVME_INTEGRATION_VERSION || holder.api) return;
-  holder.api = api;
+function resolveIntegrationEventBus(host: unknown): ObservMeIntegrationEventBus | undefined {
+  if (!host || typeof host !== "object") return undefined;
+  try {
+    const events = (host as Partial<ObservMeIntegrationHost>).events;
+    return events && typeof events.emit === "function" ? events : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function receiveObservMeIntegration(holder: IntegrationResponseHolder, value: unknown): void {
+  if (holder.api || !isObservMeIntegrationApi(value)) return;
+  holder.api = value;
+}
+
+function isObservMeIntegrationApi(value: unknown): value is ObservMeIntegrationApi {
+  if (!value || typeof value !== "object") return false;
+
+  try {
+    const api = value as Partial<ObservMeIntegrationApi>;
+    return (
+      api.version === OBSERVME_INTEGRATION_VERSION &&
+      typeof api.getContext === "function" &&
+      typeof api.startSubagent === "function" &&
+      typeof api.completeSubagent === "function" &&
+      typeof api.failSubagent === "function" &&
+      typeof api.startWait === "function" &&
+      typeof api.endWait === "function" &&
+      typeof api.startJoin === "function" &&
+      typeof api.endJoin === "function"
+    );
+  } catch {
+    return false;
+  }
 }

package/src/pi/active-agent-lease.ts

@@ -0,0 +1,101 @@
+import type { Attributes, ObservableCallback, ObservableGauge, ObservableResult } from "@opentelemetry/api";
+
+export interface ActiveAgentLeaseController {
+  readonly active: boolean;
+  readonly disposed: boolean;
+  activate: () => void;
+  deactivate: () => void;
+  dispose: () => void;
+}
+
+export interface CreateActiveAgentLeaseOptions {
+  readonly instrument: ObservableGauge;
+  readonly leaseDurationMillis: number;
+  readonly attributes?: Attributes;
+  readonly wallClockNow?: () => number;
+  readonly enabled?: boolean;
+}
+
+interface ActiveAgentLeaseState {
+  readonly leaseDurationMillis: number;
+  readonly attributes: Attributes;
+  readonly wallClockNow: () => number;
+  active: boolean;
+  disposed: boolean;
+}
+
+export function createActiveAgentLease(
+  options: CreateActiveAgentLeaseOptions,
+): ActiveAgentLeaseController | undefined {
+  if (options.enabled === false) return undefined;
+  return new SessionActiveAgentLease(options);
+}
+
+export function computeActiveAgentLeaseExpiryUnixSeconds(
+  wallClockUnixMillis: number,
+  leaseDurationMillis: number,
+): number | undefined {
+  if (!Number.isFinite(wallClockUnixMillis) || !Number.isFinite(leaseDurationMillis)) return undefined;
+
+  const expiryMillis = wallClockUnixMillis + leaseDurationMillis;
+  if (!Number.isFinite(expiryMillis)) return undefined;
+
+  const expiryUnixSeconds = expiryMillis / 1000;
+  return Number.isFinite(expiryUnixSeconds) ? expiryUnixSeconds : undefined;
+}
+
+function observeActiveAgentLease(state: ActiveAgentLeaseState, result: ObservableResult): void {
+  if (!state.active || state.disposed) return;
+
+  const expiryUnixSeconds = computeActiveAgentLeaseExpiryUnixSeconds(
+    state.wallClockNow(),
+    state.leaseDurationMillis,
+  );
+  if (expiryUnixSeconds === undefined) return;
+
+  result.observe(expiryUnixSeconds, state.attributes);
+}
+
+class SessionActiveAgentLease implements ActiveAgentLeaseController {
+  readonly #instrument: ObservableGauge;
+  readonly #state: ActiveAgentLeaseState;
+  readonly #callback: ObservableCallback;
+
+  constructor(options: CreateActiveAgentLeaseOptions) {
+    this.#instrument = options.instrument;
+    this.#state = {
+      leaseDurationMillis: options.leaseDurationMillis,
+      attributes: { ...options.attributes },
+      wallClockNow: options.wallClockNow ?? Date.now,
+      active: false,
+      disposed: false,
+    };
+    this.#callback = observeActiveAgentLease.bind(undefined, this.#state);
+    this.#instrument.addCallback(this.#callback);
+  }
+
+  get active(): boolean {
+    return this.#state.active;
+  }
+
+  get disposed(): boolean {
+    return this.#state.disposed;
+  }
+
+  activate(): void {
+    if (this.#state.disposed) return;
+    this.#state.active = true;
+  }
+
+  deactivate(): void {
+    this.#state.active = false;
+  }
+
+  dispose(): void {
+    if (this.#state.disposed) return;
+
+    this.#state.active = false;
+    this.#state.disposed = true;
+    this.#instrument.removeCallback(this.#callback);
+  }
+}

package/src/pi/event-handlers/lifecycle.ts

@@ -183,45 +183,54 @@ async function handleSessionStart(
       options.requireCompleteParentEnvelope ?? (options.trustedParentContext === true && recoveryCorrelation === undefined),
     failOpenInvalidPropagation: true,
   });
-  const session = await startTelemetryFn({ config, lineage, now: options.now });
-  session.now = options.now ?? session.now ?? monotonicNowMs;
-  updateObsStatusRuntimeState({ config: session.config, configDiagnostics: loadedConfig.diagnostics });
-  clearObsStatusExportError();
-  state.session = session;
-  const attributes = buildSessionAttributes(event, ctx, session.config, lineage, recovery);
-  const labels = metricLabels(session.config, lineage);
-
-  session.sessionAttributes = attributes;
-  const traceParent = resolveSessionTraceParent(lineage);
-  session.sessionSpan = startActiveRootSpan(session, SPAN_NAMES.PI_SESSION, attributes, "session", traceParent);
-  emitConfigRejectionDiagnostic(session, loadedConfig.diagnostics, ctx);
-  recordSessionTracePropagationFailure(session, traceParent);
-  startObsSessionRuntimeState({
-    sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
-    traceId: readSpanTraceId(session.sessionSpan),
-    traceUrlTemplate: session.config.query.links.traceUrlTemplate,
-  });
-  startObsAgentsRuntimeState({
+  const session = await startTelemetryFn({
+    config,
     lineage,
-    agentTree: session.agentTree,
-    sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
-    traceId: readSpanTraceId(session.sessionSpan),
+    now: options.now,
+    wallClockNow: options.wallClockNow,
   });
-  session.workflowStartedAtMs = session.now();
-  session.metrics.sessionsStarted.add(1, labels);
-  session.metrics.activeAgents.add(1, labels);
-  session.activeAgentRecorded = true;
-  session.sessionSpan.addEvent(LOG_EVENT_NAMES.SESSION_STARTED, attributes);
-  emitLifecycleLog(session.logger, LOG_EVENT_NAMES.SESSION_STARTED, attributes);
-  if (recovery.resumed && session.config.replayOnStart) emitStartupReplayTelemetry(session, attributes);
+  session.now = options.now ?? session.now ?? monotonicNowMs;
+  state.session = session;
 
-  if (isRootWorkflow(lineage)) {
-    session.metrics.workflowsStarted.add(1, labels);
-    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.WORKFLOW_STARTED, attributes);
+  try {
+    updateObsStatusRuntimeState({ config: session.config, configDiagnostics: loadedConfig.diagnostics });
+    clearObsStatusExportError();
+    const attributes = buildSessionAttributes(event, ctx, session.config, lineage, recovery);
+    const labels = metricLabels(session.config, lineage);
+
+    session.sessionAttributes = attributes;
+    const traceParent = resolveSessionTraceParent(lineage);
+    session.sessionSpan = startActiveRootSpan(session, SPAN_NAMES.PI_SESSION, attributes, "session", traceParent);
+    emitConfigRejectionDiagnostic(session, loadedConfig.diagnostics, ctx);
+    recordSessionTracePropagationFailure(session, traceParent);
+    startObsSessionRuntimeState({
+      sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
+      traceId: readSpanTraceId(session.sessionSpan),
+      traceUrlTemplate: session.config.query.links.traceUrlTemplate,
+    });
+    startObsAgentsRuntimeState({
+      lineage,
+      agentTree: session.agentTree,
+      sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
+      traceId: readSpanTraceId(session.sessionSpan),
+    });
+    session.workflowStartedAtMs = session.now();
+    session.metrics.sessionsStarted.add(1, labels);
+    session.sessionSpan.addEvent(LOG_EVENT_NAMES.SESSION_STARTED, attributes);
+    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.SESSION_STARTED, attributes);
+    if (recovery.resumed && session.config.replayOnStart) emitStartupReplayTelemetry(session, attributes);
+
+    if (isRootWorkflow(lineage)) {
+      session.metrics.workflowsStarted.add(1, labels);
+      emitLifecycleLog(session.logger, LOG_EVENT_NAMES.WORKFLOW_STARTED, attributes);
+    }
+
+    await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, EXTENSION_STATUS_VALUE);
+    activateSessionActiveAgent(session, labels);
+  } catch (error) {
+    await cleanUpFailedSessionStart(session, ctx, state);
+    throw error;
   }
-
-  await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, EXTENSION_STATUS_VALUE);
-  state.session = session;
 }
 
 function createSessionShutdownHandler(state: HandlerSessionState): Handler {
@@ -429,6 +438,26 @@ function recordDuplicateSessionStartShutdownError(session: ObservMeTelemetrySess
   );
 }
 
+async function cleanUpFailedSessionStart(
+  session: ObservMeTelemetrySession,
+  ctx: ObservMeHandlerContext,
+  state: HandlerSessionState,
+): Promise<void> {
+  const labels = metricLabels(session.config, session.lineage);
+
+  try {
+    deactivateSessionActiveAgent(session, labels);
+    endAllActiveSpans(session);
+    endActiveSpan(session, session.sessionSpan);
+    await clearExtensionStatus(ctx);
+    await recordControllerOperationResult(session, "flush");
+    await recordControllerOperationResult(session, "shutdown");
+  } finally {
+    disposeSessionActiveAgentLease(session);
+    clearTelemetrySessionRuntimeState(session, state);
+  }
+}
+
 async function shutDownTelemetrySession(
   session: ObservMeTelemetrySession,
   event: unknown,
@@ -439,19 +468,64 @@ async function shutDownTelemetrySession(
   const shutdownAttributes = buildShutdownAttributes(event, session);
   const failed = workflowFailed(event);
 
-  session.metrics.sessionsShutdown.add(1, labels);
-  if (session.activeAgentRecorded) session.metrics.activeAgents.add(-1, labels);
-  recordWorkflowShutdownTelemetry(session, shutdownAttributes, failed, labels);
-  endAllActiveSpans(session);
-  session.sessionSpan?.addEvent(LOG_EVENT_NAMES.SESSION_SHUTDOWN, shutdownAttributes);
-  if (failed) session.sessionSpan?.setStatus({ code: SpanStatusCode.ERROR });
-  endActiveSpan(session, session.sessionSpan);
-  await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, undefined);
-  await recordControllerOperationResult(session, "flush");
-  await recordControllerOperationResult(session, "shutdown");
+  try {
+    deactivateSessionActiveAgent(session, labels);
+    session.metrics.sessionsShutdown.add(1, labels);
+    recordWorkflowShutdownTelemetry(session, shutdownAttributes, failed, labels);
+    endAllActiveSpans(session);
+    session.sessionSpan?.addEvent(LOG_EVENT_NAMES.SESSION_SHUTDOWN, shutdownAttributes);
+    if (failed) session.sessionSpan?.setStatus({ code: SpanStatusCode.ERROR });
+    endActiveSpan(session, session.sessionSpan);
+    await clearExtensionStatus(ctx);
+    await recordControllerOperationResult(session, "flush");
+    await recordControllerOperationResult(session, "shutdown");
+  } finally {
+    disposeSessionActiveAgentLease(session);
+    clearTelemetrySessionRuntimeState(session, state);
+  }
+}
+
+function activateSessionActiveAgent(
+  session: ObservMeTelemetrySession,
+  labels: Record<string, string>,
+): void {
+  if (session.activeAgentRecorded) return;
+
+  session.metrics.activeAgents.add(1, labels);
+  session.activeAgentRecorded = true;
+  session.activeAgentLease?.activate();
+}
+
+function deactivateSessionActiveAgent(
+  session: ObservMeTelemetrySession,
+  labels: Record<string, string>,
+): void {
+  session.activeAgentLease?.deactivate();
+  if (!session.activeAgentRecorded) return;
+
+  session.activeAgentRecorded = false;
+  session.metrics.activeAgents.add(-1, labels);
+}
+
+function disposeSessionActiveAgentLease(session: ObservMeTelemetrySession): void {
+  session.activeAgentLease?.dispose();
+}
+
+function clearTelemetrySessionRuntimeState(
+  session: ObservMeTelemetrySession,
+  state: HandlerSessionState,
+): void {
   clearObsSessionRuntimeState();
   clearObsAgentsRuntimeState();
-  state.session = undefined;
+  if (state.session === session) state.session = undefined;
+}
+
+async function clearExtensionStatus(ctx: ObservMeHandlerContext): Promise<void> {
+  try {
+    await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, undefined);
+  } catch {
+    return;
+  }
 }
 
 async function recordControllerOperationResult(

package/src/pi/handler-runtime.ts

@@ -11,6 +11,7 @@ import { inheritTenantSaltEnvironment } from "../privacy/hash.ts";
 import { LOG_ATTRIBUTES, RESOURCE_ATTRIBUTES } from "../semconv/attributes.ts";
 import {
   LOG_EVENT_NAMES,
+  OBSERVME_AGENT_LEASE_METRIC_OPTIONS,
   OBSERVME_COUNTER_METRIC_NAMES,
   OBSERVME_GAUGE_METRIC_NAMES,
   OBSERVME_HISTOGRAM_METRIC_NAMES,
@@ -18,6 +19,7 @@ import {
   OFFICIAL_GENAI_METRIC_NAMES,
 } from "../semconv/metrics.ts";
 import { BoundedMap, type BoundedMapEviction } from "../util/bounded-map.ts";
+import { createActiveAgentLease } from "./active-agent-lease.ts";
 import type { AgentLineageContext } from "./agent-lineage.ts";
 import { buildResourceLineageAttributes } from "./agent-lineage.ts";
 import type { AgentTreeNode } from "./agent-tree-tracker.ts";
@@ -30,6 +32,7 @@ import {
   evictToolCallState,
   evictWaitJoinState,
   isRecord,
+  metricLabels,
   normalizeMetricValue,
   readBoolean,
   readString,
@@ -180,6 +183,16 @@ export async function startSessionTelemetry(options: StartSessionTelemetryOption
   const metrics = createObservMeMetrics(metricSdk.meter);
   const sessionReference: HandlerSessionState = {};
   const getTelemetryDropTarget = resolveSessionTelemetryDropTarget.bind(undefined, sessionReference, metrics);
+  const spans = createSpanRegistry(config, metrics, getTelemetryDropTarget);
+  const agentTree = createAgentTreeTracker(config, options.lineage, metrics, getTelemetryDropTarget);
+  const turnSequences = createTurnSequenceRegistry(config, metrics, getTelemetryDropTarget);
+  const activeAgentLease = createActiveAgentLease({
+    instrument: metrics.agentLeaseExpiresUnixTimeSeconds,
+    leaseDurationMillis: config.metrics.activeAgentLeaseDurationMillis,
+    attributes: metricLabels(config, options.lineage),
+    wallClockNow: options.wallClockNow,
+    enabled: config.metrics.enabled,
+  });
   const session: ObservMeTelemetrySession = {
     config,
     lineage: options.lineage,
@@ -188,14 +201,15 @@ export async function startSessionTelemetry(options: StartSessionTelemetryOption
     meter: metricSdk.meter,
     logger: logSdk.logger,
     metrics,
-    spans: createSpanRegistry(config, metrics, getTelemetryDropTarget),
-    agentTree: createAgentTreeTracker(config, options.lineage, metrics, getTelemetryDropTarget),
+    spans,
+    activeAgentLease,
+    agentTree,
     now: options.now ?? monotonicNowMs,
     activeAgentRecorded: false,
     agentRunSequence: 0,
     llmRequestSequence: 0,
     toolCallSequence: 0,
-    turnSequences: createTurnSequenceRegistry(config, metrics, getTelemetryDropTarget),
+    turnSequences,
   };
 
   sessionReference.session = session;
@@ -252,6 +266,10 @@ export function createObservMeMetrics(meter: TelemetryMeter): ObservMeMetrics {
     eventsObserved: meter.createCounter(OBSERVME_COUNTER_METRIC_NAMES.EVENTS_OBSERVED_TOTAL),
     activeSpans: meter.createUpDownCounter(OBSERVME_GAUGE_METRIC_NAMES.ACTIVE_SPANS),
     activeAgents: meter.createUpDownCounter(OBSERVME_GAUGE_METRIC_NAMES.ACTIVE_AGENTS),
+    agentLeaseExpiresUnixTimeSeconds: meter.createObservableGauge(
+      OBSERVME_GAUGE_METRIC_NAMES.AGENT_LEASE_EXPIRES_UNIXTIME_SECONDS,
+      OBSERVME_AGENT_LEASE_METRIC_OPTIONS,
+    ),
     workflowDurationMs: createHistogram(meter, OBSERVME_HISTOGRAM_METRIC_NAMES.WORKFLOW_DURATION_MS),
     agentRunDurationMs: createHistogram(meter, OBSERVME_HISTOGRAM_METRIC_NAMES.AGENT_RUN_DURATION_MS),
     agentLifetimeDurationMs: createHistogram(meter, OBSERVME_HISTOGRAM_METRIC_NAMES.AGENT_LIFETIME_DURATION_MS),