@senad-d/observme 0.1.3 → 0.1.5

package/src/otel/sdk.ts

@@ -1,9 +1,24 @@
 import type { ObservMeConfig } from "../config/schema.ts";
+import { readDiagnosticMessage, sanitizeDiagnosticText } from "../diagnostics/sanitize.ts";
 import type { AgentLineageContext } from "../pi/agent-lineage.ts";
-import type { BoundedOtelOperationResult, OtelShutdownLogSink, ShutdownableOtelSdk } from "./shutdown.ts";
+import type {
+  BoundedOtelOperationResult,
+  OtelOperationSettlement,
+  OtelShutdownLogSink,
+  ShutdownableOtelSdk,
+} from "./shutdown.ts";
 import { flushOtelSdk, shutdownOtelSdk } from "./shutdown.ts";
 
-export type OtelSdkLifecycleState = "idle" | "starting" | "started" | "shutting_down" | "shutdown";
+// `failed` is terminal for startup; shutdown cleanup can remain owned for observation or retry.
+export type OtelSdkLifecycleState =
+  | "idle"
+  | "starting"
+  | "started"
+  | "failed"
+  | "shutting_down"
+  | "shutdown_pending"
+  | "shutdown_failed"
+  | "shutdown";
 
 export interface StartOtelSdkFactoryOptions {
   readonly config: ObservMeConfig;
@@ -35,6 +50,10 @@ export class ObservMeOtelSdkController {
   readonly #sdkFactory: SessionScopedOtelSdkFactory;
   readonly #logger?: OtelShutdownLogSink;
   #sdk?: SessionScopedOtelSdk;
+  #startPromise?: Promise<SessionScopedOtelSdk>;
+  #shutdownPromise?: Promise<BoundedOtelOperationResult>;
+  #pendingShutdown?: Promise<OtelOperationSettlement>;
+  #startupCleanupResult?: BoundedOtelOperationResult;
   #state: OtelSdkLifecycleState = "idle";
 
   constructor(options: ObservMeOtelSdkControllerOptions) {
@@ -62,13 +81,42 @@ export class ObservMeOtelSdkController {
 
   async start(): Promise<SessionScopedOtelSdk> {
     if (this.#state === "started" && this.#sdk) return this.#sdk;
+    if (this.#state === "starting" && this.#startPromise) return this.#startPromise;
+    if (this.#state === "failed") throw new Error("ObservMe OTEL SDK controller cannot be restarted after failed startup.");
     if (this.#state === "shutdown") throw new Error("ObservMe OTEL SDK controller cannot be restarted after shutdown.");
+    if (this.#state !== "idle") {
+      throw new Error(`ObservMe OTEL SDK controller cannot start while cleanup state is ${this.#state}.`);
+    }
 
     this.#state = "starting";
-    this.#sdk = this.#sdkFactory({ config: this.#config, agent: this.#agent });
-    await this.#sdk.start?.();
-    this.#state = "started";
-    return this.#sdk;
+    this.#startPromise = this.startOnce();
+    return this.#startPromise;
+  }
+
+  private async startOnce(): Promise<SessionScopedOtelSdk> {
+    try {
+      this.#sdk = this.#sdkFactory({ config: this.#config, agent: this.#agent });
+      await this.#sdk.start?.();
+      this.#state = "started";
+      return this.#sdk;
+    } catch (error) {
+      if (error instanceof ObservMeOtelStartupError) {
+        this.#startupCleanupResult = error.cleanup ?? completedShutdown();
+      } else {
+        this.#startupCleanupResult = sanitizeStartupCleanupResult(
+          await shutdownOtelSdk(
+            this.#sdk,
+            this.#config.shutdown.flushTimeoutMs,
+            this.#logger,
+          ),
+        );
+      }
+      this.#sdk = undefined;
+      this.#state = "failed";
+      throw toOtelStartupError(error, this.#startupCleanupResult);
+    } finally {
+      this.#startPromise = undefined;
+    }
   }
 
   async flush(timeoutMs: number = this.#config.shutdown.flushTimeoutMs): Promise<BoundedOtelOperationResult> {
@@ -76,12 +124,86 @@ export class ObservMeOtelSdkController {
   }
 
   async shutdown(timeoutMs: number = this.#config.shutdown.flushTimeoutMs): Promise<BoundedOtelOperationResult> {
-    if (this.#state === "shutdown") return { operation: "shutdown", completed: true, timedOut: false };
+    if (this.#state === "shutdown") return completedShutdown();
+    if (this.#state === "failed") return this.#startupCleanupResult ?? completedShutdown();
+    if (this.#state === "shutting_down" && this.#shutdownPromise) return this.#shutdownPromise;
+    if (this.#state === "shutdown_pending" && this.#pendingShutdown) {
+      return pendingShutdown(this.#pendingShutdown);
+    }
 
     this.#state = "shutting_down";
-    const result = await shutdownOtelSdk(this.#sdk, timeoutMs, this.#logger);
+    this.#shutdownPromise = this.shutdownOnce(timeoutMs);
+    return this.#shutdownPromise;
+  }
+
+  private async shutdownOnce(timeoutMs: number): Promise<BoundedOtelOperationResult> {
+    try {
+      const result = await shutdownOtelSdk(this.#sdk, timeoutMs, this.#logger);
+      this.applyShutdownResult(result);
+      return result;
+    } finally {
+      this.#shutdownPromise = undefined;
+    }
+  }
+
+  private applyShutdownResult(result: BoundedOtelOperationResult): void {
+    if (result.timedOut && result.settlement) {
+      this.#pendingShutdown = result.settlement;
+      this.#state = "shutdown_pending";
+      void result.settlement.then(this.observePendingShutdown.bind(this, result.settlement));
+      return;
+    }
+
+    if (result.error || !result.completed) {
+      this.#state = "shutdown_failed";
+      return;
+    }
+
+    this.releaseShutdownOwnership();
+  }
+
+  private observePendingShutdown(
+    pending: Promise<OtelOperationSettlement>,
+    settlement: OtelOperationSettlement,
+  ): void {
+    if (this.#pendingShutdown !== pending) return;
+
+    this.#pendingShutdown = undefined;
+    if (settlement.error || !settlement.completed) {
+      this.#state = "shutdown_failed";
+      return;
+    }
+
+    this.releaseShutdownOwnership();
+  }
+
+  private releaseShutdownOwnership(): void {
+    this.#sdk = undefined;
     this.#state = "shutdown";
-    return result;
+  }
+}
+
+export function toOtelStartupError(
+  error: unknown,
+  cleanup?: BoundedOtelOperationResult,
+): Error {
+  if (error instanceof ObservMeOtelStartupError) return error;
+
+  const detail = sanitizeDiagnosticText(readDiagnosticMessage(error));
+  const safeCleanup = cleanup ? sanitizeStartupCleanupResult(cleanup) : undefined;
+  return new ObservMeOtelStartupError(
+    `ObservMe OTEL startup failed: ${detail}. ${startupCleanupGuidance(safeCleanup)}`,
+    safeCleanup,
+  );
+}
+
+export class ObservMeOtelStartupError extends Error {
+  readonly cleanup?: BoundedOtelOperationResult;
+
+  constructor(message: string, cleanup?: BoundedOtelOperationResult) {
+    super(message);
+    this.name = "ObservMeOtelStartupError";
+    this.cleanup = cleanup;
   }
 }
 
@@ -99,6 +221,30 @@ export function createNoopSessionScopedOtelSdk(): SessionScopedOtelSdk {
   return new NoopSessionScopedOtelSdk();
 }
 
+function startupCleanupGuidance(cleanup: BoundedOtelOperationResult | undefined): string {
+  if (!cleanup) return "Check OTLP settings and Collector availability before retrying.";
+  if (cleanup.timedOut) return "Cleanup exceeded its timeout; restart Pi before retrying.";
+  if (cleanup.error) return "Cleanup also failed; restart Pi before retrying.";
+  return "Started providers were cleaned up; check OTLP settings and Collector availability before retrying.";
+}
+
+function completedShutdown(): BoundedOtelOperationResult {
+  return { operation: "shutdown", completed: true, timedOut: false };
+}
+
+function pendingShutdown(settlement: Promise<OtelOperationSettlement>): BoundedOtelOperationResult {
+  return { operation: "shutdown", completed: false, timedOut: true, settlement };
+}
+
+function sanitizeStartupCleanupResult(result: BoundedOtelOperationResult): BoundedOtelOperationResult {
+  if (!result.error) return result;
+
+  return {
+    ...result,
+    error: new Error(sanitizeDiagnosticText(readDiagnosticMessage(result.error))),
+  };
+}
+
 class NoopSessionScopedOtelSdk implements SessionScopedOtelSdk {
   async start(): Promise<void> {
     await Promise.resolve();

package/src/otel/shutdown.ts

@@ -9,11 +9,19 @@ export interface ShutdownableOtelSdk extends FlushableOtelSdk {
   shutdown?: () => Promise<void> | void;
 }
 
+export interface OtelOperationSettlement {
+  readonly operation: "flush" | "shutdown";
+  readonly completed: boolean;
+  readonly timedOut: false;
+  readonly error?: unknown;
+}
+
 export interface BoundedOtelOperationResult {
   readonly operation: "flush" | "shutdown";
   readonly completed: boolean;
   readonly timedOut: boolean;
   readonly error?: unknown;
+  readonly settlement?: Promise<OtelOperationSettlement>;
 }
 
 export interface OtelShutdownLogSink {
@@ -54,10 +62,10 @@ export async function runBoundedOtelOperation(
   const timeoutResult = timeoutOperation(operation, normalizedTimeoutMs, timeoutController.signal);
 
   try {
-    const completed = completeOperation(operation, action);
-    return await Promise.race([completed, timeoutResult]);
-  } catch (error) {
-    return { operation, completed: false, timedOut: false, error };
+    const settlement = settleOperation(operation, action);
+    const result = await Promise.race([settlement, timeoutResult]);
+    if (!result.timedOut) return result;
+    return { ...result, settlement };
   } finally {
     timeoutController.abort();
   }
@@ -68,12 +76,16 @@ function normalizeTimeoutMs(timeoutMs: number): number {
   return timeoutMs;
 }
 
-async function completeOperation(
+async function settleOperation(
   operation: "flush" | "shutdown",
   action: () => Promise<void> | void,
-): Promise<BoundedOtelOperationResult> {
-  await action();
-  return completedOperation(operation);
+): Promise<OtelOperationSettlement> {
+  try {
+    await action();
+    return completedOperation(operation);
+  } catch (error) {
+    return { operation, completed: false, timedOut: false, error };
+  }
 }
 
 async function timeoutOperation(
@@ -84,17 +96,33 @@ async function timeoutOperation(
   return delay(timeoutMs, { operation, completed: false, timedOut: true } as const, { signal });
 }
 
-function completedOperation(operation: "flush" | "shutdown"): BoundedOtelOperationResult {
+function completedOperation(operation: "flush" | "shutdown"): OtelOperationSettlement {
   return { operation, completed: true, timedOut: false };
 }
 
 function logBoundedOperationIssue(result: BoundedOtelOperationResult, logger: OtelShutdownLogSink | undefined): void {
   if (result.timedOut) {
-    logger?.warn?.(`ObservMe OTEL ${result.operation} exceeded timeout and was abandoned.`);
+    warnOtelOperation(logger, `ObservMe OTEL ${result.operation} exceeded timeout and remains pending.`);
+    void result.settlement?.then(logLateOperationIssue.bind(undefined, logger));
     return;
   }
 
-  if (result.error) logger?.warn?.(`ObservMe OTEL ${result.operation} failed: ${formatError(result.error)}`);
+  logLateOperationIssue(logger, result);
+}
+
+function logLateOperationIssue(
+  logger: OtelShutdownLogSink | undefined,
+  result: Pick<BoundedOtelOperationResult, "operation" | "error">,
+): void {
+  if (result.error) warnOtelOperation(logger, `ObservMe OTEL ${result.operation} failed: ${formatError(result.error)}`);
+}
+
+function warnOtelOperation(logger: OtelShutdownLogSink | undefined, message: string): void {
+  try {
+    logger?.warn?.(message);
+  } catch {
+    return;
+  }
 }
 
 function formatError(error: unknown): string {

package/src/otel/traces.ts

@@ -3,15 +3,24 @@ import type { Resource } from "@opentelemetry/resources";
 import { resourceFromAttributes } from "@opentelemetry/resources";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
 import type { Sampler, SpanExporter, SpanProcessor } from "@opentelemetry/sdk-trace-base";
-import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
+import {
+  AlwaysOffSampler,
+  BasicTracerProvider,
+  BatchSpanProcessor,
+  ParentBasedSampler,
+  TraceIdRatioBasedSampler,
+} from "@opentelemetry/sdk-trace-base";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
 import type { ObservMeConfig, TraceBatchConfig } from "../config/schema.ts";
 import { appendOtlpSignalPath } from "./otlp-endpoint.ts";
+import { buildOtlpHttpAgentOptions, type OtlpHttpAgentOptions } from "./otlp-http-options.ts";
 import type { StartOtelSdkFactoryOptions } from "./sdk.ts";
 
 export const OBSERVME_TRACER_NAME = "@senad-d/observme";
 export const OTLP_TRACE_SIGNAL_PATH = "/v1/traces";
 
+const noopTracer = new BasicTracerProvider({ sampler: new AlwaysOffSampler() }).getTracer(OBSERVME_TRACER_NAME);
+
 export const DOCUMENTED_TRACE_BATCH_DEFAULTS = {
   maxQueueSize: 2048,
   maxExportBatchSize: 512,
@@ -25,6 +34,7 @@ export interface OtlpTraceExporterOptions {
   readonly url: string;
   readonly headers: Record<string, string>;
   readonly timeoutMillis: number;
+  readonly httpAgentOptions: OtlpHttpAgentOptions;
 }
 
 export interface TraceProviderOptions {
@@ -35,7 +45,6 @@ export interface TraceProviderOptions {
 }
 
 export interface TraceProviderLike {
-  register?: () => void;
   getTracer: (name: string) => Tracer;
   forceFlush?: () => Promise<void> | void;
   shutdown?: () => Promise<void> | void;
@@ -48,7 +57,6 @@ export type TraceProviderFactory = (options: TraceProviderOptions) => TraceProvi
 export interface ObservMeTraceSdkOptions {
   readonly config: ObservMeConfig;
   readonly tracerName?: string;
-  readonly registerGlobal?: boolean;
   readonly exporterFactory?: TraceExporterFactory;
   readonly spanProcessorFactory?: TraceSpanProcessorFactory;
   readonly tracerProviderFactory?: TraceProviderFactory;
@@ -63,18 +71,18 @@ export interface TraceExporterWiring {
 export class ObservMeTraceSdk {
   readonly #config: ObservMeConfig;
   readonly #tracerName: string;
-  readonly #registerGlobal: boolean;
   readonly #exporterFactory: TraceExporterFactory;
   readonly #spanProcessorFactory: TraceSpanProcessorFactory;
   readonly #tracerProviderFactory: TraceProviderFactory;
+  #exporter?: SpanExporter;
+  #processor?: SpanProcessor;
   #provider?: TraceProviderLike;
-  #tracer?: Tracer;
+  #tracer: Tracer = noopTracer;
   #state: TracePipelineState = "idle";
 
   constructor(options: ObservMeTraceSdkOptions) {
     this.#config = options.config;
     this.#tracerName = options.tracerName ?? OBSERVME_TRACER_NAME;
-    this.#registerGlobal = options.registerGlobal ?? true;
     this.#exporterFactory = options.exporterFactory ?? createOtlpTraceExporter;
     this.#spanProcessorFactory = options.spanProcessorFactory ?? createBatchSpanProcessor;
     this.#tracerProviderFactory = options.tracerProviderFactory ?? createNodeTraceProvider;
@@ -84,28 +92,27 @@ export class ObservMeTraceSdk {
     return this.#state;
   }
 
-  get tracer(): Tracer | undefined {
+  get tracer(): Tracer {
     return this.#tracer;
   }
 
   start(): void {
     if (this.#state === "started" || this.#state === "disabled") return;
-    if (!this.#config.traces.enabled) {
+    if (!this.#config.enabled || !this.#config.traces.enabled) {
       this.#state = "disabled";
       return;
     }
 
     const wiring = buildTraceExporterWiring(this.#config);
-    const exporter = this.#exporterFactory(wiring.exporter);
-    const processor = this.#spanProcessorFactory(exporter, wiring.batch);
+    this.#exporter = this.#exporterFactory(wiring.exporter);
+    this.#processor = this.#spanProcessorFactory(this.#exporter, wiring.batch);
     this.#provider = this.#tracerProviderFactory({
       resource: createTraceResource(this.#config),
       sampler: createTraceSampler(this.#config),
-      spanProcessors: [processor],
+      spanProcessors: [this.#processor],
       forceFlushTimeoutMillis: this.#config.shutdown.flushTimeoutMs,
     });
 
-    if (this.#registerGlobal) this.#provider.register?.();
     this.#tracer = this.#provider.getTracer(this.#tracerName);
     this.#state = "started";
   }
@@ -115,9 +122,19 @@ export class ObservMeTraceSdk {
   }
 
   async shutdown(): Promise<void> {
-    await this.#provider?.shutdown?.();
-    this.#state = "shutdown";
-    this.#tracer = undefined;
+    if (this.#state === "shutdown") return;
+
+    try {
+      if (this.#provider?.shutdown) await this.#provider.shutdown();
+      else if (this.#processor) await this.#processor.shutdown();
+      else await this.#exporter?.shutdown();
+    } finally {
+      this.#provider = undefined;
+      this.#processor = undefined;
+      this.#exporter = undefined;
+      this.#state = "shutdown";
+      this.#tracer = noopTracer;
+    }
   }
 }
 
@@ -127,7 +144,7 @@ export function createTraceSessionScopedOtelSdk(options: StartOtelSdkFactoryOpti
 
 export function buildTraceExporterWiring(config: ObservMeConfig): TraceExporterWiring {
   return {
-    enabled: config.traces.enabled,
+    enabled: config.enabled && config.traces.enabled,
     exporter: buildOtlpTraceExporterOptions(config),
     batch: { ...config.traces.batch },
   };
@@ -138,6 +155,7 @@ export function buildOtlpTraceExporterOptions(config: ObservMeConfig): OtlpTrace
     url: resolveTraceEndpoint(config),
     headers: { ...config.otlp.headers },
     timeoutMillis: config.otlp.timeoutMs,
+    httpAgentOptions: buildOtlpHttpAgentOptions(config),
   };
 }
 

package/src/pi/active-agent-lease.ts

@@ -0,0 +1,101 @@
+import type { Attributes, ObservableCallback, ObservableGauge, ObservableResult } from "@opentelemetry/api";
+
+export interface ActiveAgentLeaseController {
+  readonly active: boolean;
+  readonly disposed: boolean;
+  activate: () => void;
+  deactivate: () => void;
+  dispose: () => void;
+}
+
+export interface CreateActiveAgentLeaseOptions {
+  readonly instrument: ObservableGauge;
+  readonly leaseDurationMillis: number;
+  readonly attributes?: Attributes;
+  readonly wallClockNow?: () => number;
+  readonly enabled?: boolean;
+}
+
+interface ActiveAgentLeaseState {
+  readonly leaseDurationMillis: number;
+  readonly attributes: Attributes;
+  readonly wallClockNow: () => number;
+  active: boolean;
+  disposed: boolean;
+}
+
+export function createActiveAgentLease(
+  options: CreateActiveAgentLeaseOptions,
+): ActiveAgentLeaseController | undefined {
+  if (options.enabled === false) return undefined;
+  return new SessionActiveAgentLease(options);
+}
+
+export function computeActiveAgentLeaseExpiryUnixSeconds(
+  wallClockUnixMillis: number,
+  leaseDurationMillis: number,
+): number | undefined {
+  if (!Number.isFinite(wallClockUnixMillis) || !Number.isFinite(leaseDurationMillis)) return undefined;
+
+  const expiryMillis = wallClockUnixMillis + leaseDurationMillis;
+  if (!Number.isFinite(expiryMillis)) return undefined;
+
+  const expiryUnixSeconds = expiryMillis / 1000;
+  return Number.isFinite(expiryUnixSeconds) ? expiryUnixSeconds : undefined;
+}
+
+function observeActiveAgentLease(state: ActiveAgentLeaseState, result: ObservableResult): void {
+  if (!state.active || state.disposed) return;
+
+  const expiryUnixSeconds = computeActiveAgentLeaseExpiryUnixSeconds(
+    state.wallClockNow(),
+    state.leaseDurationMillis,
+  );
+  if (expiryUnixSeconds === undefined) return;
+
+  result.observe(expiryUnixSeconds, state.attributes);
+}
+
+class SessionActiveAgentLease implements ActiveAgentLeaseController {
+  readonly #instrument: ObservableGauge;
+  readonly #state: ActiveAgentLeaseState;
+  readonly #callback: ObservableCallback;
+
+  constructor(options: CreateActiveAgentLeaseOptions) {
+    this.#instrument = options.instrument;
+    this.#state = {
+      leaseDurationMillis: options.leaseDurationMillis,
+      attributes: { ...options.attributes },
+      wallClockNow: options.wallClockNow ?? Date.now,
+      active: false,
+      disposed: false,
+    };
+    this.#callback = observeActiveAgentLease.bind(undefined, this.#state);
+    this.#instrument.addCallback(this.#callback);
+  }
+
+  get active(): boolean {
+    return this.#state.active;
+  }
+
+  get disposed(): boolean {
+    return this.#state.disposed;
+  }
+
+  activate(): void {
+    if (this.#state.disposed) return;
+    this.#state.active = true;
+  }
+
+  deactivate(): void {
+    this.#state.active = false;
+  }
+
+  dispose(): void {
+    if (this.#state.disposed) return;
+
+    this.#state.active = false;
+    this.#state.disposed = true;
+    this.#instrument.removeCallback(this.#callback);
+  }
+}

package/src/pi/agent-tree-tracker.ts

@@ -48,6 +48,7 @@ interface MutableAgentTreeNode {
 }
 
 const activeChildStatuses = new Set<AgentChildStatus>(["starting", "active"]);
+const terminalChildStatuses = new Set<AgentChildStatus>(["completed", "failed", "cancelled", "orphaned"]);
 const agentStatusOrder: AgentChildStatus[] = ["starting", "active", "completed", "failed", "cancelled", "orphaned"];
 
 export class AgentTreeTracker {
@@ -68,6 +69,11 @@ export class AgentTreeTracker {
   }
 
   registerAgent(lineage: AgentLineageContext, status: AgentChildStatus = "active"): AgentTreeNode {
+    const existingNode = this.#nodes.get(lineage.agentId);
+    if (existingNode && !isAgentStatusTransitionAllowed(existingNode.status, status)) {
+      return snapshotNode(existingNode, this.#nodes);
+    }
+
     const node = createMutableNode(lineage, status, this.isOrphan(lineage));
     this.retainParentForInsertion(node.parentAgentId);
     this.#nodes.set(lineage.agentId, node);
@@ -77,7 +83,7 @@ export class AgentTreeTracker {
 
   updateStatus(agentId: string, status: AgentChildStatus): AgentTreeNode | undefined {
     const node = this.#nodes.get(agentId);
-    if (!node) return undefined;
+    if (!node || !isAgentStatusTransitionAllowed(node.status, status)) return undefined;
 
     node.status = status;
     if (status === "orphaned") node.orphaned = true;
@@ -177,6 +183,13 @@ export class AgentTreeTracker {
   }
 }
 
+export function isAgentStatusTransitionAllowed(current: AgentChildStatus, next: AgentChildStatus): boolean {
+  if (current === next) return true;
+  if (terminalChildStatuses.has(current)) return false;
+  if (current === "starting") return true;
+  return next !== "starting";
+}
+
 export function assertNoHighCardinalityMetricLabels(labels: Record<string, string>): void {
   const forbiddenKey = Object.keys(labels).find(isHighCardinalityLineageKey);
   if (forbiddenKey) throw new Error(`High-cardinality lineage value must not be used as a metric label: ${forbiddenKey}`);