@senad-d/observme 0.1.3 → 0.1.4

package/examples/collector.yaml

@@ -1,4 +1,4 @@
-# Production-oriented Collector reference for Tempo, Loki, and Mimir.
+# Production-oriented Collector reference for Tempo, Loki, and Prometheus.
 # Verify that your Collector distribution contains every configured component,
 # replace backend endpoints/security for your deployment, and never add raw
 # content or high-cardinality execution identifiers to metric labels.
@@ -97,14 +97,14 @@ exporters:
     retry_on_failure:
       enabled: true
 
-  prometheusremotewrite/mimir:
-    endpoint: http://mimir:9009/api/v1/push
+  prometheus:
+    endpoint: 0.0.0.0:8889
+    # Exporter-wide stale-series/cardinality cleanup for gauges, counters, and
+    # histograms. Five minutes exceeds ObservMe's default one-minute active-agent
+    # lease; lease-aware PromQL, not expiration, determines liveness.
+    metric_expiration: 5m
     resource_to_telemetry_conversion:
       enabled: true
-    sending_queue:
-      enabled: true
-    retry_on_failure:
-      enabled: true
 
   debug:
     verbosity: basic
@@ -120,7 +120,7 @@ service:
     metrics:
       receivers: [otlp]
       processors: [memory_limiter, resource/observme, resource/drop_high_cardinality_metric_attrs, batch]
-      exporters: [prometheusremotewrite/mimir]
+      exporters: [prometheus]
 
     logs:
       receivers: [otlp]

package/examples/observme.yaml

@@ -61,6 +61,7 @@ observme:
     enabled: true
     exportIntervalMillis: 15000
     exportTimeoutMillis: 3000
+    activeAgentLeaseDurationMillis: 60000
 
   logs:
     enabled: true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@senad-d/observme",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "type": "module",
   "description": "ObservMe: a Pi extension that instruments Pi agent sessions and exports OpenTelemetry traces, metrics, and logs to an external observability stack (OTel Collector, Grafana Tempo/Loki/Prometheus).",
   "license": "MIT",
@@ -57,7 +57,9 @@
     "lint:fix": "eslint . --fix",
     "format:check": "node scripts/check-format.mjs",
     "test": "node --test test/*.test.mjs test/*.test.ts",
+    "test:active-agent-lease:contracts": "node --test test/active-agent-lease.test.ts test/alerts.test.mjs test/cardinality.test.ts test/config-defaults.test.mjs test/config-loader.test.mjs test/config-validation.test.mjs test/dashboards.test.mjs test/metrics.test.ts test/pi-handlers.test.mjs test/semconv-metrics.test.mjs",
     "test:integration:collector": "node --test test/integration/collector-debug.test.mjs",
+    "test:integration:active-agent-lease": "node --test test/integration/active-agent-lease.test.mjs",
     "test:integration:grafana-stack": "node --test test/integration/grafana-stack.test.mjs",
     "test:coverage": "node scripts/test-coverage.mjs",
     "smoke:discover": "node scripts/smoke-observability.mjs",

package/src/config/bootstrap-project-config.ts

@@ -101,6 +101,7 @@ export const PROJECT_OBSERVME_YAML_TEMPLATE = `observme:
     enabled: true
     exportIntervalMillis: 15000
     exportTimeoutMillis: 3000
+    activeAgentLeaseDurationMillis: 60000
 
   logs:
     enabled: true

package/src/config/defaults.ts

@@ -60,6 +60,7 @@ export const defaultObservMeConfig = {
     enabled: true,
     exportIntervalMillis: 15000,
     exportTimeoutMillis: 3000,
+    activeAgentLeaseDurationMillis: 60000,
   },
   logs: {
     enabled: true,

package/src/config/load-config.ts

@@ -147,6 +147,11 @@ export function envToConfig(env: NodeJS.ProcessEnv = process.env): DeepPartial<O
   setString(config, ["otlp", "signalEndpoints", "metrics"], env.OBSERVME_OTLP_METRICS_ENDPOINT);
   setString(config, ["otlp", "signalEndpoints", "logs"], env.OBSERVME_OTLP_LOGS_ENDPOINT);
   setNumber(config, ["otlp", "timeoutMs"], env.OBSERVME_OTLP_TIMEOUT_MS);
+  setNumberForValidation(
+    config,
+    ["metrics", "activeAgentLeaseDurationMillis"],
+    env.OBSERVME_ACTIVE_AGENT_LEASE_DURATION_MS,
+  );
   setAuthorizationHeader(config, env.OBSERVME_OTLP_TOKEN);
   setNumber(config, ["workflow", "maxDepthWarning"], env.OBSERVME_WORKFLOW_MAX_DEPTH_WARNING);
   setNumber(config, ["workflow", "maxFanoutWarning"], env.OBSERVME_WORKFLOW_MAX_FANOUT_WARNING);
@@ -592,6 +597,12 @@ function setNumber(target: DeepPartial<ObservMeConfig>, path: string[], value: s
   setPathValue(target as Record<string, unknown>, path, parsed);
 }
 
+function setNumberForValidation(target: DeepPartial<ObservMeConfig>, path: string[], value: string | undefined) {
+  if (value === undefined || value === "") return;
+  const parsed = Number(value);
+  setPathValue(target as Record<string, unknown>, path, Number.isFinite(parsed) ? parsed : value);
+}
+
 function setBoolean(target: DeepPartial<ObservMeConfig>, path: string[], value: string | undefined) {
   const parsed = parseBooleanEnv(value);
   if (parsed === undefined) return;

package/src/config/schema.ts

@@ -8,6 +8,10 @@ export type ObservMeEnvironment = (typeof observMeEnvironments)[number];
 export type OtlpProtocol = "http/protobuf";
 export type PrivacyPathMode = (typeof privacyPathModes)[number];
 
+export const ACTIVE_AGENT_LEASE_DURATION_MILLIS_MINIMUM = 10_000;
+export const ACTIVE_AGENT_LEASE_DURATION_MILLIS_MAXIMUM = 300_000;
+export const ACTIVE_AGENT_LEASE_EXPORT_SAFETY_MARGIN_MILLIS = 5_000;
+
 export interface OtlpTlsConfig {
   enabled: boolean;
   insecureSkipVerify: boolean;
@@ -71,6 +75,7 @@ export interface MetricsConfig {
   enabled: boolean;
   exportIntervalMillis: number;
   exportTimeoutMillis: number;
+  activeAgentLeaseDurationMillis: number;
   labels?: string[];
 }
 
@@ -288,6 +293,10 @@ export const observMeConfigSchema = Type.Object(
         enabled: Type.Boolean(),
         exportIntervalMillis: positiveIntegerSchema,
         exportTimeoutMillis: positiveIntegerSchema,
+        activeAgentLeaseDurationMillis: Type.Integer({
+          minimum: ACTIVE_AGENT_LEASE_DURATION_MILLIS_MINIMUM,
+          maximum: ACTIVE_AGENT_LEASE_DURATION_MILLIS_MAXIMUM,
+        }),
         labels: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
       },
       { additionalProperties: false },

package/src/config/validate.ts

@@ -1,6 +1,9 @@
 import { Compile } from "typebox/compile";
 import { defaultObservMeConfig } from "./defaults.ts";
-import { observMeConfigSchema } from "./schema.ts";
+import {
+  ACTIVE_AGENT_LEASE_EXPORT_SAFETY_MARGIN_MILLIS,
+  observMeConfigSchema,
+} from "./schema.ts";
 import type { ObservMeConfig } from "./schema.ts";
 import { validateCustomRedactionPatterns } from "../privacy/redact.ts";
 
@@ -58,6 +61,7 @@ const knownConfigValidationIssueCodes = new Set([
   "insecure_production_transport",
   "invalid_signal_endpoint_path",
   "high_cardinality_metric_label",
+  "active_agent_lease_too_short_for_export_interval",
   "custom_redaction_pattern_limit",
   "custom_redaction_pattern_too_long",
   "custom_redaction_pattern_unsupported_construct",
@@ -82,6 +86,7 @@ export function validateObservMeConfig(
     ...validateTransportSecurity(config),
     ...validateSignalEndpoints(config),
     ...validateMetricLabels(config),
+    ...validateActiveAgentLeaseDuration(config),
     ...validateCustomRedactionPatternConfig(config),
     ...validateProjectTrust(options),
     ...validateLineageEnvironment(config, options.env ?? process.env),
@@ -307,6 +312,20 @@ function validateMetricLabels(config: ObservMeConfig): ValidationIssue[] {
   }));
 }
 
+function validateActiveAgentLeaseDuration(config: ObservMeConfig): ValidationIssue[] {
+  const requiredDuration =
+    (2 * config.metrics.exportIntervalMillis) + ACTIVE_AGENT_LEASE_EXPORT_SAFETY_MARGIN_MILLIS;
+  if (config.metrics.activeAgentLeaseDurationMillis >= requiredDuration) return [];
+
+  return [
+    {
+      code: "active_agent_lease_too_short_for_export_interval",
+      message:
+        "metrics.activeAgentLeaseDurationMillis must be at least twice metrics.exportIntervalMillis plus the clock-skew safety margin.",
+    },
+  ];
+}
+
 function validateCustomRedactionPatternConfig(config: ObservMeConfig): ValidationIssue[] {
   return validateCustomRedactionPatterns(config.privacy.customRedactionPatterns).map(issue => ({
     code: issue.code,

package/src/pi/active-agent-lease.ts

@@ -0,0 +1,101 @@
+import type { Attributes, ObservableCallback, ObservableGauge, ObservableResult } from "@opentelemetry/api";
+
+export interface ActiveAgentLeaseController {
+  readonly active: boolean;
+  readonly disposed: boolean;
+  activate: () => void;
+  deactivate: () => void;
+  dispose: () => void;
+}
+
+export interface CreateActiveAgentLeaseOptions {
+  readonly instrument: ObservableGauge;
+  readonly leaseDurationMillis: number;
+  readonly attributes?: Attributes;
+  readonly wallClockNow?: () => number;
+  readonly enabled?: boolean;
+}
+
+interface ActiveAgentLeaseState {
+  readonly leaseDurationMillis: number;
+  readonly attributes: Attributes;
+  readonly wallClockNow: () => number;
+  active: boolean;
+  disposed: boolean;
+}
+
+export function createActiveAgentLease(
+  options: CreateActiveAgentLeaseOptions,
+): ActiveAgentLeaseController | undefined {
+  if (options.enabled === false) return undefined;
+  return new SessionActiveAgentLease(options);
+}
+
+export function computeActiveAgentLeaseExpiryUnixSeconds(
+  wallClockUnixMillis: number,
+  leaseDurationMillis: number,
+): number | undefined {
+  if (!Number.isFinite(wallClockUnixMillis) || !Number.isFinite(leaseDurationMillis)) return undefined;
+
+  const expiryMillis = wallClockUnixMillis + leaseDurationMillis;
+  if (!Number.isFinite(expiryMillis)) return undefined;
+
+  const expiryUnixSeconds = expiryMillis / 1000;
+  return Number.isFinite(expiryUnixSeconds) ? expiryUnixSeconds : undefined;
+}
+
+function observeActiveAgentLease(state: ActiveAgentLeaseState, result: ObservableResult): void {
+  if (!state.active || state.disposed) return;
+
+  const expiryUnixSeconds = computeActiveAgentLeaseExpiryUnixSeconds(
+    state.wallClockNow(),
+    state.leaseDurationMillis,
+  );
+  if (expiryUnixSeconds === undefined) return;
+
+  result.observe(expiryUnixSeconds, state.attributes);
+}
+
+class SessionActiveAgentLease implements ActiveAgentLeaseController {
+  readonly #instrument: ObservableGauge;
+  readonly #state: ActiveAgentLeaseState;
+  readonly #callback: ObservableCallback;
+
+  constructor(options: CreateActiveAgentLeaseOptions) {
+    this.#instrument = options.instrument;
+    this.#state = {
+      leaseDurationMillis: options.leaseDurationMillis,
+      attributes: { ...options.attributes },
+      wallClockNow: options.wallClockNow ?? Date.now,
+      active: false,
+      disposed: false,
+    };
+    this.#callback = observeActiveAgentLease.bind(undefined, this.#state);
+    this.#instrument.addCallback(this.#callback);
+  }
+
+  get active(): boolean {
+    return this.#state.active;
+  }
+
+  get disposed(): boolean {
+    return this.#state.disposed;
+  }
+
+  activate(): void {
+    if (this.#state.disposed) return;
+    this.#state.active = true;
+  }
+
+  deactivate(): void {
+    this.#state.active = false;
+  }
+
+  dispose(): void {
+    if (this.#state.disposed) return;
+
+    this.#state.active = false;
+    this.#state.disposed = true;
+    this.#instrument.removeCallback(this.#callback);
+  }
+}

package/src/pi/event-handlers/lifecycle.ts

@@ -183,45 +183,54 @@ async function handleSessionStart(
       options.requireCompleteParentEnvelope ?? (options.trustedParentContext === true && recoveryCorrelation === undefined),
     failOpenInvalidPropagation: true,
   });
-  const session = await startTelemetryFn({ config, lineage, now: options.now });
-  session.now = options.now ?? session.now ?? monotonicNowMs;
-  updateObsStatusRuntimeState({ config: session.config, configDiagnostics: loadedConfig.diagnostics });
-  clearObsStatusExportError();
-  state.session = session;
-  const attributes = buildSessionAttributes(event, ctx, session.config, lineage, recovery);
-  const labels = metricLabels(session.config, lineage);
-
-  session.sessionAttributes = attributes;
-  const traceParent = resolveSessionTraceParent(lineage);
-  session.sessionSpan = startActiveRootSpan(session, SPAN_NAMES.PI_SESSION, attributes, "session", traceParent);
-  emitConfigRejectionDiagnostic(session, loadedConfig.diagnostics, ctx);
-  recordSessionTracePropagationFailure(session, traceParent);
-  startObsSessionRuntimeState({
-    sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
-    traceId: readSpanTraceId(session.sessionSpan),
-    traceUrlTemplate: session.config.query.links.traceUrlTemplate,
-  });
-  startObsAgentsRuntimeState({
+  const session = await startTelemetryFn({
+    config,
     lineage,
-    agentTree: session.agentTree,
-    sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
-    traceId: readSpanTraceId(session.sessionSpan),
+    now: options.now,
+    wallClockNow: options.wallClockNow,
   });
-  session.workflowStartedAtMs = session.now();
-  session.metrics.sessionsStarted.add(1, labels);
-  session.metrics.activeAgents.add(1, labels);
-  session.activeAgentRecorded = true;
-  session.sessionSpan.addEvent(LOG_EVENT_NAMES.SESSION_STARTED, attributes);
-  emitLifecycleLog(session.logger, LOG_EVENT_NAMES.SESSION_STARTED, attributes);
-  if (recovery.resumed && session.config.replayOnStart) emitStartupReplayTelemetry(session, attributes);
+  session.now = options.now ?? session.now ?? monotonicNowMs;
+  state.session = session;
 
-  if (isRootWorkflow(lineage)) {
-    session.metrics.workflowsStarted.add(1, labels);
-    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.WORKFLOW_STARTED, attributes);
+  try {
+    updateObsStatusRuntimeState({ config: session.config, configDiagnostics: loadedConfig.diagnostics });
+    clearObsStatusExportError();
+    const attributes = buildSessionAttributes(event, ctx, session.config, lineage, recovery);
+    const labels = metricLabels(session.config, lineage);
+
+    session.sessionAttributes = attributes;
+    const traceParent = resolveSessionTraceParent(lineage);
+    session.sessionSpan = startActiveRootSpan(session, SPAN_NAMES.PI_SESSION, attributes, "session", traceParent);
+    emitConfigRejectionDiagnostic(session, loadedConfig.diagnostics, ctx);
+    recordSessionTracePropagationFailure(session, traceParent);
+    startObsSessionRuntimeState({
+      sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
+      traceId: readSpanTraceId(session.sessionSpan),
+      traceUrlTemplate: session.config.query.links.traceUrlTemplate,
+    });
+    startObsAgentsRuntimeState({
+      lineage,
+      agentTree: session.agentTree,
+      sessionId: readString(attributes, SESSION_ATTRIBUTES.PI_SESSION_ID),
+      traceId: readSpanTraceId(session.sessionSpan),
+    });
+    session.workflowStartedAtMs = session.now();
+    session.metrics.sessionsStarted.add(1, labels);
+    session.sessionSpan.addEvent(LOG_EVENT_NAMES.SESSION_STARTED, attributes);
+    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.SESSION_STARTED, attributes);
+    if (recovery.resumed && session.config.replayOnStart) emitStartupReplayTelemetry(session, attributes);
+
+    if (isRootWorkflow(lineage)) {
+      session.metrics.workflowsStarted.add(1, labels);
+      emitLifecycleLog(session.logger, LOG_EVENT_NAMES.WORKFLOW_STARTED, attributes);
+    }
+
+    await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, EXTENSION_STATUS_VALUE);
+    activateSessionActiveAgent(session, labels);
+  } catch (error) {
+    await cleanUpFailedSessionStart(session, ctx, state);
+    throw error;
   }
-
-  await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, EXTENSION_STATUS_VALUE);
-  state.session = session;
 }
 
 function createSessionShutdownHandler(state: HandlerSessionState): Handler {
@@ -429,6 +438,26 @@ function recordDuplicateSessionStartShutdownError(session: ObservMeTelemetrySess
   );
 }
 
+async function cleanUpFailedSessionStart(
+  session: ObservMeTelemetrySession,
+  ctx: ObservMeHandlerContext,
+  state: HandlerSessionState,
+): Promise<void> {
+  const labels = metricLabels(session.config, session.lineage);
+
+  try {
+    deactivateSessionActiveAgent(session, labels);
+    endAllActiveSpans(session);
+    endActiveSpan(session, session.sessionSpan);
+    await clearExtensionStatus(ctx);
+    await recordControllerOperationResult(session, "flush");
+    await recordControllerOperationResult(session, "shutdown");
+  } finally {
+    disposeSessionActiveAgentLease(session);
+    clearTelemetrySessionRuntimeState(session, state);
+  }
+}
+
 async function shutDownTelemetrySession(
   session: ObservMeTelemetrySession,
   event: unknown,
@@ -439,19 +468,64 @@ async function shutDownTelemetrySession(
   const shutdownAttributes = buildShutdownAttributes(event, session);
   const failed = workflowFailed(event);
 
-  session.metrics.sessionsShutdown.add(1, labels);
-  if (session.activeAgentRecorded) session.metrics.activeAgents.add(-1, labels);
-  recordWorkflowShutdownTelemetry(session, shutdownAttributes, failed, labels);
-  endAllActiveSpans(session);
-  session.sessionSpan?.addEvent(LOG_EVENT_NAMES.SESSION_SHUTDOWN, shutdownAttributes);
-  if (failed) session.sessionSpan?.setStatus({ code: SpanStatusCode.ERROR });
-  endActiveSpan(session, session.sessionSpan);
-  await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, undefined);
-  await recordControllerOperationResult(session, "flush");
-  await recordControllerOperationResult(session, "shutdown");
+  try {
+    deactivateSessionActiveAgent(session, labels);
+    session.metrics.sessionsShutdown.add(1, labels);
+    recordWorkflowShutdownTelemetry(session, shutdownAttributes, failed, labels);
+    endAllActiveSpans(session);
+    session.sessionSpan?.addEvent(LOG_EVENT_NAMES.SESSION_SHUTDOWN, shutdownAttributes);
+    if (failed) session.sessionSpan?.setStatus({ code: SpanStatusCode.ERROR });
+    endActiveSpan(session, session.sessionSpan);
+    await clearExtensionStatus(ctx);
+    await recordControllerOperationResult(session, "flush");
+    await recordControllerOperationResult(session, "shutdown");
+  } finally {
+    disposeSessionActiveAgentLease(session);
+    clearTelemetrySessionRuntimeState(session, state);
+  }
+}
+
+function activateSessionActiveAgent(
+  session: ObservMeTelemetrySession,
+  labels: Record<string, string>,
+): void {
+  if (session.activeAgentRecorded) return;
+
+  session.metrics.activeAgents.add(1, labels);
+  session.activeAgentRecorded = true;
+  session.activeAgentLease?.activate();
+}
+
+function deactivateSessionActiveAgent(
+  session: ObservMeTelemetrySession,
+  labels: Record<string, string>,
+): void {
+  session.activeAgentLease?.deactivate();
+  if (!session.activeAgentRecorded) return;
+
+  session.activeAgentRecorded = false;
+  session.metrics.activeAgents.add(-1, labels);
+}
+
+function disposeSessionActiveAgentLease(session: ObservMeTelemetrySession): void {
+  session.activeAgentLease?.dispose();
+}
+
+function clearTelemetrySessionRuntimeState(
+  session: ObservMeTelemetrySession,
+  state: HandlerSessionState,
+): void {
   clearObsSessionRuntimeState();
   clearObsAgentsRuntimeState();
-  state.session = undefined;
+  if (state.session === session) state.session = undefined;
+}
+
+async function clearExtensionStatus(ctx: ObservMeHandlerContext): Promise<void> {
+  try {
+    await ctx.ui?.setStatus?.(EXTENSION_STATUS_KEY, undefined);
+  } catch {
+    return;
+  }
 }
 
 async function recordControllerOperationResult(

package/src/pi/handler-runtime.ts

@@ -11,6 +11,7 @@ import { inheritTenantSaltEnvironment } from "../privacy/hash.ts";
 import { LOG_ATTRIBUTES, RESOURCE_ATTRIBUTES } from "../semconv/attributes.ts";
 import {
   LOG_EVENT_NAMES,
+  OBSERVME_AGENT_LEASE_METRIC_OPTIONS,
   OBSERVME_COUNTER_METRIC_NAMES,
   OBSERVME_GAUGE_METRIC_NAMES,
   OBSERVME_HISTOGRAM_METRIC_NAMES,
@@ -18,6 +19,7 @@ import {
   OFFICIAL_GENAI_METRIC_NAMES,
 } from "../semconv/metrics.ts";
 import { BoundedMap, type BoundedMapEviction } from "../util/bounded-map.ts";
+import { createActiveAgentLease } from "./active-agent-lease.ts";
 import type { AgentLineageContext } from "./agent-lineage.ts";
 import { buildResourceLineageAttributes } from "./agent-lineage.ts";
 import type { AgentTreeNode } from "./agent-tree-tracker.ts";
@@ -30,6 +32,7 @@ import {
   evictToolCallState,
   evictWaitJoinState,
   isRecord,
+  metricLabels,
   normalizeMetricValue,
   readBoolean,
   readString,
@@ -180,6 +183,16 @@ export async function startSessionTelemetry(options: StartSessionTelemetryOption
   const metrics = createObservMeMetrics(metricSdk.meter);
   const sessionReference: HandlerSessionState = {};
   const getTelemetryDropTarget = resolveSessionTelemetryDropTarget.bind(undefined, sessionReference, metrics);
+  const spans = createSpanRegistry(config, metrics, getTelemetryDropTarget);
+  const agentTree = createAgentTreeTracker(config, options.lineage, metrics, getTelemetryDropTarget);
+  const turnSequences = createTurnSequenceRegistry(config, metrics, getTelemetryDropTarget);
+  const activeAgentLease = createActiveAgentLease({
+    instrument: metrics.agentLeaseExpiresUnixTimeSeconds,
+    leaseDurationMillis: config.metrics.activeAgentLeaseDurationMillis,
+    attributes: metricLabels(config, options.lineage),
+    wallClockNow: options.wallClockNow,
+    enabled: config.metrics.enabled,
+  });
   const session: ObservMeTelemetrySession = {
     config,
     lineage: options.lineage,
@@ -188,14 +201,15 @@ export async function startSessionTelemetry(options: StartSessionTelemetryOption
     meter: metricSdk.meter,
     logger: logSdk.logger,
     metrics,
-    spans: createSpanRegistry(config, metrics, getTelemetryDropTarget),
-    agentTree: createAgentTreeTracker(config, options.lineage, metrics, getTelemetryDropTarget),
+    spans,
+    activeAgentLease,
+    agentTree,
     now: options.now ?? monotonicNowMs,
     activeAgentRecorded: false,
     agentRunSequence: 0,
     llmRequestSequence: 0,
     toolCallSequence: 0,
-    turnSequences: createTurnSequenceRegistry(config, metrics, getTelemetryDropTarget),
+    turnSequences,
   };
 
   sessionReference.session = session;
@@ -252,6 +266,10 @@ export function createObservMeMetrics(meter: TelemetryMeter): ObservMeMetrics {
     eventsObserved: meter.createCounter(OBSERVME_COUNTER_METRIC_NAMES.EVENTS_OBSERVED_TOTAL),
     activeSpans: meter.createUpDownCounter(OBSERVME_GAUGE_METRIC_NAMES.ACTIVE_SPANS),
     activeAgents: meter.createUpDownCounter(OBSERVME_GAUGE_METRIC_NAMES.ACTIVE_AGENTS),
+    agentLeaseExpiresUnixTimeSeconds: meter.createObservableGauge(
+      OBSERVME_GAUGE_METRIC_NAMES.AGENT_LEASE_EXPIRES_UNIXTIME_SECONDS,
+      OBSERVME_AGENT_LEASE_METRIC_OPTIONS,
+    ),
     workflowDurationMs: createHistogram(meter, OBSERVME_HISTOGRAM_METRIC_NAMES.WORKFLOW_DURATION_MS),
     agentRunDurationMs: createHistogram(meter, OBSERVME_HISTOGRAM_METRIC_NAMES.AGENT_RUN_DURATION_MS),
     agentLifetimeDurationMs: createHistogram(meter, OBSERVME_HISTOGRAM_METRIC_NAMES.AGENT_LIFETIME_DURATION_MS),

package/src/pi/handler-types.ts

@@ -1,4 +1,4 @@
-import type { Counter, Histogram, Meter, Span, Tracer, UpDownCounter } from "@opentelemetry/api";
+import type { Counter, Histogram, Meter, ObservableGauge, Span, Tracer, UpDownCounter } from "@opentelemetry/api";
 import type { Logger } from "@opentelemetry/api-logs";
 import type { EnsureProjectConfig as BootstrapEnsureProjectConfig } from "../config/bootstrap-project-config.ts";
 import type {
@@ -13,6 +13,7 @@ import type { ObservMeOtelSdkController } from "../otel/sdk.ts";
 import type { ObservMeTraceSdk } from "../otel/traces.ts";
 import type { BoundedMap } from "../util/bounded-map.ts";
 import type { AgentLineageContext } from "./agent-lineage.ts";
+import type { ActiveAgentLeaseController } from "./active-agent-lease.ts";
 import type { AgentTreeTracker } from "./agent-tree-tracker.ts";
 import type {
   AgentWaitJoinState,
@@ -22,7 +23,10 @@ import type {
 
 export type AttributePrimitive = boolean | number | string | string[];
 export type AttributeMap = Record<string, AttributePrimitive>;
-export type TelemetryMeter = Pick<Meter, "createCounter" | "createHistogram" | "createUpDownCounter">;
+export type TelemetryMeter = Pick<
+  Meter,
+  "createCounter" | "createHistogram" | "createObservableGauge" | "createUpDownCounter"
+>;
 export type TelemetryTracer = Pick<Tracer, "startSpan">;
 export type TelemetryLogger = Pick<Logger, "emit">;
 export type Handler = (event: unknown, ctx: ObservMeHandlerContext) => Promise<void> | void;
@@ -88,6 +92,7 @@ export interface RegisterHandlersOptions {
   readonly startTelemetry?: StartSessionTelemetry;
   readonly env?: NodeJS.ProcessEnv;
   readonly now?: () => number;
+  readonly wallClockNow?: () => number;
   readonly configDirName?: string;
   readonly trustedParentContext?: boolean;
   readonly requireCompleteParentEnvelope?: boolean;
@@ -100,6 +105,7 @@ export interface StartSessionTelemetryOptions {
   readonly config: ObservMeConfig;
   readonly lineage: AgentLineageContext;
   readonly now?: () => number;
+  readonly wallClockNow?: () => number;
 }
 
 export interface TurnSequenceRegistry {
@@ -119,6 +125,7 @@ export interface ObservMeTelemetrySession {
   readonly logger: TelemetryLogger;
   readonly metrics: ObservMeMetrics;
   readonly spans: SpanRegistry;
+  readonly activeAgentLease?: ActiveAgentLeaseController;
   agentTree: AgentTreeTracker;
   sessionSpan?: Span;
   sessionAttributes?: AttributeMap;
@@ -179,6 +186,7 @@ export interface ObservMeMetrics {
   readonly eventsObserved: Counter;
   readonly activeSpans: UpDownCounter;
   readonly activeAgents: UpDownCounter;
+  readonly agentLeaseExpiresUnixTimeSeconds: ObservableGauge;
   readonly workflowDurationMs: Histogram;
   readonly agentRunDurationMs: Histogram;
   readonly agentLifetimeDurationMs: Histogram;

package/src/semconv/metrics.ts

@@ -66,6 +66,12 @@ export const OBSERVME_HISTOGRAM_METRIC_NAMES = {
 export const OBSERVME_GAUGE_METRIC_NAMES = {
   ACTIVE_SPANS: "observme_active_spans",
   ACTIVE_AGENTS: "observme_active_agents",
+  AGENT_LEASE_EXPIRES_UNIXTIME_SECONDS: "observme_agent_lease_expires_unixtime_seconds",
+} as const;
+
+export const OBSERVME_AGENT_LEASE_METRIC_OPTIONS = {
+  unit: "s",
+  description: "Absolute Unix timestamp in seconds when the active Pi agent lease expires.",
 } as const;
 
 export const OFFICIAL_GENAI_METRIC_NAMES = {