@omercnet/paseo-omp 0.3.0-next.108.1 → 0.3.0-next.111.1

package/server/protocol-violation-diagnostics.ts

@@ -0,0 +1,169 @@
+import {
+  OMP_PROTOCOL_VIOLATION_CATEGORIES,
+  OMP_PROTOCOL_VIOLATION_REASONS,
+  type OmpProtocolViolationCategory,
+  type OmpProtocolViolationDiagnostic,
+  type OmpProtocolViolationReason,
+} from "./provider/omp-rpc";
+
+export interface OmpProtocolViolationSummary {
+  category: OmpProtocolViolationCategory;
+  occurrenceCount: number;
+  batchCount: number;
+  maxOccurrenceCount: number;
+  firstAt: string | null;
+  lastAt: string | null;
+  reasonCounts: Record<OmpProtocolViolationReason, number>;
+  latestReason: OmpProtocolViolationDiagnostic["reason"] | null;
+  latestPhase: OmpProtocolViolationDiagnostic["phase"] | null;
+  latestEventType: OmpProtocolViolationDiagnostic["eventType"] | null;
+  latestFrameType: OmpProtocolViolationDiagnostic["frameType"] | null;
+  latestField: OmpProtocolViolationDiagnostic["field"] | null;
+  latestExpected: OmpProtocolViolationDiagnostic["expected"] | null;
+  latestActualType: OmpProtocolViolationDiagnostic["actualType"] | null;
+  maxByteSize: number | null;
+  latestLimitBytes: number | null;
+}
+
+type MutableSummary = OmpProtocolViolationSummary;
+
+const FRAME_TYPES: Record<NonNullable<OmpProtocolViolationDiagnostic["frameType"]>, true> = {
+  ready: true,
+  response: true,
+  rpc_chunk: true,
+  rpc_frame_error: true,
+  notice: true,
+};
+
+function boundedCount(value: unknown): number {
+  if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) return 0;
+  return Math.min(value, Number.MAX_SAFE_INTEGER);
+}
+
+function boundedAdd(left: number, right: number): number {
+  return Math.min(Number.MAX_SAFE_INTEGER, left + right);
+}
+
+function safeTimestamp(now: () => Date): string | null {
+  try {
+    const value = now();
+    return Number.isFinite(value.getTime()) ? value.toISOString() : null;
+  } catch {
+    return null;
+  }
+}
+
+function logProtocolViolation(message: string, diagnostic: OmpProtocolViolationDiagnostic): void {
+  const fields = [
+    `category: ${diagnostic.category}`,
+    `reason: ${diagnostic.reason}`,
+    `occurrenceCount: ${diagnostic.occurrenceCount}`,
+    `phase: ${diagnostic.phase}`,
+    ...(diagnostic.eventType ? [`eventType: ${diagnostic.eventType}`] : []),
+    ...(diagnostic.frameType ? [`frameType: ${diagnostic.frameType}`] : []),
+    ...(diagnostic.field ? [`field: ${diagnostic.field}`] : []),
+    ...(diagnostic.expected ? [`expected: ${diagnostic.expected}`] : []),
+    ...(diagnostic.actualType ? [`actualType: ${diagnostic.actualType}`] : []),
+    ...(diagnostic.maxByteSize ? [`maxByteSize: ${diagnostic.maxByteSize}`] : []),
+    ...(diagnostic.limitBytes ? [`limitBytes: ${diagnostic.limitBytes}`] : []),
+  ];
+  console.error(`${message} { ${fields.join(", ")} }`);
+}
+
+/** Fixed-category, saturating in-process aggregation. A new instance is created on each reload. */
+export class OmpProtocolViolationCollector {
+  private readonly summaries = Object.fromEntries(
+    OMP_PROTOCOL_VIOLATION_CATEGORIES.map((category) => [
+      category,
+      {
+        category,
+        occurrenceCount: 0,
+        batchCount: 0,
+        maxOccurrenceCount: 0,
+        firstAt: null,
+        lastAt: null,
+        reasonCounts: Object.fromEntries(
+          OMP_PROTOCOL_VIOLATION_REASONS.map((reason) => [reason, 0]),
+        ),
+        latestReason: null,
+        latestPhase: null,
+        latestEventType: null,
+        latestFrameType: null,
+        latestField: null,
+        latestExpected: null,
+        latestActualType: null,
+        maxByteSize: null,
+        latestLimitBytes: null,
+      },
+    ]),
+  ) as Record<OmpProtocolViolationCategory, MutableSummary>;
+
+  constructor(
+    private readonly now: () => Date = () => new Date(),
+    private readonly log: (
+      message: string,
+      diagnostic: OmpProtocolViolationDiagnostic,
+    ) => void | PromiseLike<void> = logProtocolViolation,
+  ) {}
+
+  report = (diagnostic: OmpProtocolViolationDiagnostic): void => {
+    try {
+      const summary = this.summaries[diagnostic.category];
+      if (!summary) return;
+      const occurrenceCount = boundedCount(diagnostic.occurrenceCount);
+      const timestamp = safeTimestamp(this.now);
+      summary.occurrenceCount = boundedAdd(summary.occurrenceCount, occurrenceCount);
+      summary.batchCount = boundedAdd(summary.batchCount, 1);
+      summary.maxOccurrenceCount = Math.max(summary.maxOccurrenceCount, occurrenceCount);
+      if (timestamp) {
+        summary.firstAt ??= timestamp;
+        summary.lastAt = timestamp;
+      }
+      summary.reasonCounts[diagnostic.reason] = boundedAdd(
+        summary.reasonCounts[diagnostic.reason],
+        occurrenceCount,
+      );
+      summary.latestReason = diagnostic.reason;
+      summary.latestPhase = diagnostic.phase;
+      summary.latestEventType = diagnostic.eventType ?? null;
+      summary.latestFrameType =
+        diagnostic.frameType && FRAME_TYPES[diagnostic.frameType] ? diagnostic.frameType : null;
+      summary.latestField = diagnostic.field ?? null;
+      summary.latestExpected = diagnostic.expected ?? null;
+      summary.latestActualType = diagnostic.actualType ?? null;
+      const byteSize = boundedCount(diagnostic.maxByteSize);
+      if (byteSize > 0) summary.maxByteSize = Math.max(summary.maxByteSize ?? 0, byteSize);
+      const limitBytes = boundedCount(diagnostic.limitBytes);
+      summary.latestLimitBytes = limitBytes || null;
+
+      const safeDiagnostic: OmpProtocolViolationDiagnostic = {
+        category: diagnostic.category,
+        reason: diagnostic.reason,
+        phase: diagnostic.phase,
+        occurrenceCount,
+        ...(summary.latestEventType ? { eventType: summary.latestEventType } : {}),
+        ...(summary.latestFrameType ? { frameType: summary.latestFrameType } : {}),
+        ...(summary.latestField ? { field: summary.latestField } : {}),
+        ...(summary.latestExpected ? { expected: summary.latestExpected } : {}),
+        ...(summary.latestActualType ? { actualType: summary.latestActualType } : {}),
+        ...(byteSize > 0 ? { maxByteSize: byteSize } : {}),
+        ...(limitBytes > 0 ? { limitBytes } : {}),
+      };
+      try {
+        const logging = this.log("OMP protocol violation", safeDiagnostic);
+        if (logging) void Promise.resolve(logging).catch(() => undefined);
+      } catch {
+        // Diagnostics must never affect transport or provider flow.
+      }
+    } catch {
+      // Diagnostics must never affect transport or provider flow.
+    }
+  };
+
+  snapshot(): OmpProtocolViolationSummary[] {
+    return OMP_PROTOCOL_VIOLATION_CATEGORIES.map((category) => ({
+      ...this.summaries[category],
+      reasonCounts: { ...this.summaries[category].reasonCounts },
+    }));
+  }
+}

package/server/provider/connection.ts

@@ -9,6 +9,10 @@ import {
   requireProviderCapabilities,
 } from "@getpaseo/plugin/server/provider";
 import type { OmpBrowserAuthorizationRegistry } from "../mcp-browser";
+import type {
+  OmpOperationalFailure,
+  OmpOperationalFailureReporter,
+} from "../operational-failure-diagnostics";
 import { discoverOmpCatalog } from "./catalog";
 import { normalizeOmpCatalogOptions } from "./config-normalization";
 import type { OmpMcpConnector } from "./host-tools";
@@ -683,6 +687,7 @@ export function createOmpConnection(
   browserAuthorizationRegistry?: OmpBrowserAuthorizationRegistry,
   reportDiagnostic: (diagnostic: OmpConnectionDiagnostic) => void = (diagnostic) =>
     console.error("OMP provider failure", diagnostic),
+  reportOperationalFailure: OmpOperationalFailureReporter = () => {},
 ): ProviderConnection {
   const errorDetails = (error: unknown, fallback: string): { message: string } => {
     if (isOmpPublicError(error)) return { message: error.message };
@@ -696,6 +701,13 @@ export function createOmpConnection(
     }
     return { message: `${fallback} (diagnostic ${diagnosticId})` };
   };
+  const recordOperationalFailure = (failure: OmpOperationalFailure) => {
+    try {
+      reportOperationalFailure(failure);
+    } catch {
+      // Diagnostics must never affect provider requests or cleanup.
+    }
+  };
   const safeCapabilities = [...new Set(capabilities)].filter(
     (capability) =>
       SUPPORTED_CAPABILITIES[capability] &&
@@ -802,6 +814,9 @@ export function createOmpConnection(
                 },
           });
         } catch (error) {
+          if (!closing) {
+            recordOperationalFailure({ category: "session-open", stage: "catalog" });
+          }
           if (isOmpCleanupFailure(error)) catalogCleanup = error.cleanup;
           if (!closing) requestFailure(input.requestId, error, "OMP catalog discovery failed");
         }
@@ -925,6 +940,7 @@ export function createOmpConnection(
             environment,
             mcpConnector,
             mcpInitializationTimeoutMs,
+            reportOperationalFailure,
           );
           const discoveredNativeSessionId = session.persistenceSessionId;
           if (discoveredNativeSessionId) nativeSessionId = discoveredNativeSessionId;
@@ -977,6 +993,15 @@ export function createOmpConnection(
             nativeReservations.release(nativeSessionId, token);
           }
         } catch (error) {
+          const openingCancelled =
+            closing || controller.signal.aborted || opening.get(input.sessionId)?.token !== token;
+          if (!openingCancelled) {
+            recordOperationalFailure(
+              input.history === "replay" || nativeSessionId
+                ? { category: "replay-recovery", stage: "persisted-replay" }
+                : { category: "session-open", stage: "startup" },
+            );
+          }
           deleteSession(input.sessionId, token);
           let cleanupError: unknown;
           if (session) {

package/server/provider/host-tools.ts

@@ -3,6 +3,10 @@ import type {
   ProviderMcpServerConfig,
   ProviderSessionConfig,
 } from "@getpaseo/plugin/server/provider";
+import type {
+  OmpOperationalFailure,
+  OmpOperationalFailureReporter,
+} from "../operational-failure-diagnostics";
 import { isValidImagePayload } from "./image";
 import {
   type ConnectedMcpClient,
@@ -26,6 +30,11 @@ import {
   utf8Bytes,
 } from "./security";
 
+type HostToolFailureStage = Extract<
+  OmpOperationalFailure,
+  { category: "tool-projector"; stage: `host-tool-${string}` }
+>["stage"];
+
 const INTERNAL_PASEO_MCP_PATH = "/mcp/agents";
 const RESERVED_PASEO_NAMESPACE = "paseo";
 const MAX_MCP_SERVERS = 32;
@@ -79,6 +88,7 @@ export interface OmpHostToolsOpenOptions {
   initializationTimeoutMs?: number;
   callTimeoutMs?: number;
   callScheduler?: OmpHostToolScheduler;
+  reportOperationalFailure?: OmpOperationalFailureReporter;
 }
 
 type ClassifiedServer = {
@@ -496,6 +506,7 @@ export class OmpHostToolsBridge {
     private readonly targets: ReadonlyMap<string, ToolTarget>,
     private readonly callTimeoutMs: number,
     private readonly callScheduler: OmpHostToolScheduler,
+    private readonly reportOperationalFailure: OmpOperationalFailureReporter,
   ) {
     this.labels = new Map(definitions.map(({ name, label }) => [name, label ?? name]));
   }
@@ -612,6 +623,7 @@ export class OmpHostToolsBridge {
         targets,
         callTimeoutMs,
         callScheduler,
+        options.reportOperationalFailure ?? (() => {}),
       );
     } catch (error) {
       const cleanupTasks = [
@@ -740,7 +752,11 @@ export class OmpHostToolsBridge {
     if (!runtime) return true;
     const target = this.targets.get(event.toolName);
     if (!target) {
-      this.sendTerminal(runtime, errorResult(event.id, "Unknown OMP host tool"));
+      this.sendOperationalError(
+        "host-tool-unknown",
+        runtime,
+        errorResult(event.id, "Unknown OMP host tool"),
+      );
       return true;
     }
     const retainedBytes = boundedJsonBytes(
@@ -756,7 +772,11 @@ export class OmpHostToolsBridge {
       retainedBytes === Number.POSITIVE_INFINITY ||
       this.pendingBytes + retainedBytes > MAX_PENDING_HOST_TOOL_BYTES
     ) {
-      this.sendTerminal(runtime, errorResult(event.id, "OMP host tool bridge is at capacity"));
+      this.sendOperationalError(
+        "host-tool-capacity",
+        runtime,
+        errorResult(event.id, "OMP host tool bridge is at capacity"),
+      );
       return true;
     }
     const pending: PendingCall = {
@@ -794,23 +814,30 @@ export class OmpHostToolsBridge {
       })
       .then((result) => {
         if (!this.isCurrent(event.id, pending)) return;
-        let terminal: OmpHostToolResult;
         try {
           const normalized = normalizeResult(result);
-          terminal = {
+          const terminal: OmpHostToolResult = {
             type: "host_tool_result",
             id: event.id,
             result: normalized,
             ...(normalized.isError !== undefined ? { isError: normalized.isError } : {}),
           };
+          if (normalized.isError) {
+            this.sendOperationalError("host-tool-call", runtime, terminal, pending);
+          } else this.sendTerminal(runtime, terminal, pending);
         } catch {
-          terminal = errorResult(event.id, "MCP host tool execution failed");
+          this.sendOperationalError(
+            "host-tool-normalization",
+            runtime,
+            errorResult(event.id, "MCP host tool execution failed"),
+            pending,
+          );
         }
-        this.sendTerminal(runtime, terminal, pending);
       })
       .catch(() => {
         if (!this.isCurrent(event.id, pending)) return;
-        this.sendTerminal(
+        this.sendOperationalError(
+          "host-tool-call",
           runtime,
           errorResult(event.id, "MCP host tool execution failed"),
           pending,
@@ -909,22 +936,44 @@ export class OmpHostToolsBridge {
     return fits(bounded) ? bounded : errorResult(result.id, OMP_HOST_TOOL_FRAME_LIMIT_ERROR);
   }
 
-  private sendTerminal(
+  private recordOperationalFailure(failure: OmpOperationalFailure): void {
+    try {
+      this.reportOperationalFailure(failure);
+    } catch {
+      // Diagnostics must never affect host tool execution or result delivery.
+    }
+  }
+
+  private sendOperationalError(
+    stage: HostToolFailureStage,
     runtime: OmpRuntimeSession,
     result: OmpHostToolResult,
     pending?: PendingCall,
   ): void {
+    if (this.sendTerminal(runtime, result, pending)) {
+      this.recordOperationalFailure({ category: "tool-projector", stage });
+    }
+  }
+
+  private sendTerminal(
+    runtime: OmpRuntimeSession,
+    result: OmpHostToolResult,
+    pending?: PendingCall,
+  ): boolean {
     const bounded = this.boundedTerminal(runtime, result);
-    if (pending && !this.isCurrent(bounded.id, pending)) return;
+    if (pending && !this.isCurrent(bounded.id, pending)) return false;
     try {
       runtime.sendHostToolResult(bounded);
+      return true;
     } catch (error) {
       this.failRuntime(runtime, error);
+      return false;
     }
   }
 
   private failRuntime(runtime: OmpRuntimeSession, error: unknown): void {
     if (this.runtime !== runtime) return;
+    this.recordOperationalFailure({ category: "tool-projector", stage: "host-tool-delivery" });
     const failure =
       error instanceof Error ? error : new Error("OMP host tool result delivery failed");
     this.detach();
@@ -945,7 +994,11 @@ export class OmpHostToolsBridge {
     if (!this.isCurrent(id, pending)) return;
     this.releasePending(id, pending);
     pending.controller.abort(new Error("OMP MCP host tool call timed out"));
-    this.sendTerminal(pending.runtime, errorResult(id, "OMP MCP host tool call timed out"));
+    this.sendOperationalError(
+      "host-tool-timeout",
+      pending.runtime,
+      errorResult(id, "OMP MCP host tool call timed out"),
+    );
   }
 
   private releasePending(id: string, pending: PendingCall): void {