@opengeni/runtime 0.6.0 → 0.7.0

package/src/sandbox/selfhosted/control-rpc.ts

@@ -20,12 +20,7 @@
 // hardening, retries) behind this SAME `ControlRpc` interface — design for that,
 // do not duplicate.
 
-import {
-  AgentError,
-  ControlRequest,
-  ControlResponse,
-  ErrorCode,
-} from "@opengeni/agent-proto";
+import { AgentError, ControlRequest, ControlResponse, ErrorCode } from "@opengeni/agent-proto";
 import type { CapabilityUnavailableReason } from "@opengeni/contracts";
 
 // Re-export the contracts reason union locally so callers of the mapping below
@@ -93,6 +88,15 @@ export class SelfhostedControlError extends Error {
   readonly draining: boolean;
   readonly agentOffline: boolean;
   readonly osNotFound: boolean;
+  /** The reply exceeded the transport's max payload (a distinguishing flag so the
+   *  fault renderer can name the size wall + recovery instead of a generic OS error). */
+  readonly payloadTooLarge: boolean;
+  /** The transport synthesized this fault PRE-SEND — it KNOWS the request never
+   *  reached the machine (no connection, or no responder on the subject), so the op
+   *  provably did not execute. Only such a fault is safe to re-issue for ANY op kind
+   *  (a post-send timeout is ambiguous — at-least-once — and must not be re-sent
+   *  unless the op is read-only). Never set for an agent-returned error. */
+  readonly neverSent: boolean;
   readonly detail: Record<string, string>;
 
   constructor(input: {
@@ -104,6 +108,8 @@ export class SelfhostedControlError extends Error {
     draining?: boolean;
     agentOffline?: boolean;
     osNotFound?: boolean;
+    payloadTooLarge?: boolean;
+    neverSent?: boolean;
     detail?: Record<string, string>;
   }) {
     super(input.message);
@@ -114,6 +120,8 @@ export class SelfhostedControlError extends Error {
     this.draining = input.draining ?? false;
     this.agentOffline = input.agentOffline ?? false;
     this.osNotFound = input.osNotFound ?? false;
+    this.payloadTooLarge = input.payloadTooLarge ?? false;
+    this.neverSent = input.neverSent ?? false;
     this.detail = input.detail ?? {};
   }
 }
@@ -133,6 +141,9 @@ export class SelfhostedControlError extends Error {
  *                             real "file does not exist"), no machine-liveness
  *                             reason. (This is the ONLY NotFound; it is NOT the
  *                             box-gone NotFound that licenses a cold restore.)
+ *   - PAYLOAD_TOO_LARGE     → the reply exceeded the transport's max payload;
+ *                             non-retryable, with actionable copy (the sizes + a
+ *                             redirect-to-file workaround).
  *   - OS / UNSUPPORTED / STREAM / PROTOCOL / UNSPECIFIED → op-level error, no
  *                             reason, non-retryable.
  */
@@ -141,12 +152,17 @@ export function agentErrorToControlError(err: AgentError): SelfhostedControlErro
   const detail = err.detail ?? {};
   switch (err.code) {
     case ErrorCode.ERROR_CODE_AGENT_OFFLINE:
+      // AGENT_OFFLINE is ONLY ever synthesized by our transport (the agent never
+      // sends it). The synthesis marks `neverSent` on the pre-send paths (no NATS
+      // connection, or no responder on the subject) — the op provably never reached
+      // the machine — via the reserved detail key `NEVER_SENT_DETAIL_KEY`.
       return new SelfhostedControlError({
         message: message || "the enrolled agent is offline",
         code: err.code,
         reason: "agent_offline",
         retryable: false,
         agentOffline: true,
+        neverSent: detail[NEVER_SENT_DETAIL_KEY] === "1",
         detail,
       });
     case ErrorCode.ERROR_CODE_TIMEOUT:
@@ -168,14 +184,33 @@ export function agentErrorToControlError(err: AgentError): SelfhostedControlErro
         detail,
       });
     case ErrorCode.ERROR_CODE_DRAINING:
+      // A pre-admission backpressure rejection (the machine's bounded host-work
+      // pool is full, or it is shutting down): the op NEVER started, so it is safe
+      // to retry (SelfhostedSession.call retries it a bounded number of times).
+      // Replace the agent's internal "N in flight" phrasing with human-language,
+      // actionable copy; call() appends the retry count when it finally surfaces.
       return new SelfhostedControlError({
-        message: message || "the agent is draining and cannot accept new work",
+        message: drainingMessage(0),
         code: err.code,
         reason: null,
         retryable: true,
         draining: true,
         detail,
       });
+    case ErrorCode.ERROR_CODE_PAYLOAD_TOO_LARGE:
+      // The op's reply exceeded the control transport's negotiated max payload and
+      // could not be published (a huge file read, an unbounded exec dump, a full-res
+      // screenshot). Not retryable — the same op reproduces the same oversized reply.
+      // Surface the actual sizes + a concrete workaround (write to a file, read in
+      // ranges) instead of the raw byte-count message.
+      return new SelfhostedControlError({
+        message: payloadTooLargeMessage(detail),
+        code: err.code,
+        reason: null,
+        retryable: false,
+        payloadTooLarge: true,
+        detail,
+      });
     case ErrorCode.ERROR_CODE_FENCED:
       return new SelfhostedControlError({
         message: message || "a stale op was fenced by the epoch guard; re-resolve and retry",
@@ -211,14 +246,97 @@ export function agentErrorToControlError(err: AgentError): SelfhostedControlErro
   }
 }
 
+// ── Actionable, human-language error copy ─────────────────────────────────────
+// These build the messages the model / API caller sees for the failure modes that
+// have a concrete workaround. They live here (next to the mapping) so the phrasing
+// is single-sourced and unit-testable, and deliberately avoid the wire's internal
+// vocabulary ("in flight", "negotiated max payload", raw byte counts alone).
+
+/**
+ * PAYLOAD_TOO_LARGE copy: state the actual sizes (from the agent's `detail` map —
+ * `encoded_bytes` / `max_payload`) and tell the caller how to get the data anyway:
+ * redirect the output to a file and read it back in ranges/chunks.
+ */
+export function payloadTooLargeMessage(detail: Record<string, string>): string {
+  const encoded = detail.encoded_bytes;
+  const max = detail.max_payload;
+  const sizes =
+    encoded && max
+      ? `The result was ${encoded} bytes, over the machine link's ${max}-byte per-message limit. `
+      : "The result was larger than the machine link can deliver in a single message. ";
+  return (
+    `${sizes}Redirect the command's output to a file (for example ` +
+    "`<command> > /tmp/out.log 2>&1`) and then read that file back in ranges or chunks " +
+    "instead of returning it all at once."
+  );
+}
+
+/**
+ * DRAINING copy: the machine is at its concurrent-work capacity. `retries` is the
+ * number of times `SelfhostedSession.call` already re-tried before giving up (0 at
+ * the mapping layer, the final count when it surfaces after exhausting retries).
+ */
+export function drainingMessage(retries: number): string {
+  const retried =
+    retries > 0 ? ` It was retried ${retries} time${retries === 1 ? "" : "s"} first.` : "";
+  return (
+    "The machine is at its concurrent-work capacity and could not accept this command." +
+    retried +
+    " Try again shortly, or reduce the number of commands you run in parallel."
+  );
+}
+
+/** Rebuild a DRAINING error with the final retry count folded into its message —
+ *  used by `SelfhostedSession.call` when the bounded DRAINING retries are exhausted. */
+export function drainingExhaustedError(
+  base: SelfhostedControlError,
+  retries: number,
+): SelfhostedControlError {
+  return new SelfhostedControlError({
+    message: drainingMessage(retries),
+    code: base.code,
+    reason: base.reason,
+    retryable: base.retryable,
+    draining: base.draining,
+    // Carry the retry count in `detail` so the fault renderer can state it.
+    detail: { ...base.detail, retries: String(retries) },
+  });
+}
+
+/**
+ * exec-deadline copy: the command was terminated at the exec time limit. Advise
+ * running long jobs in the background and polling their output rather than blocking
+ * the turn on one long command. Surfaced on a timed-out exec result's stderr.
+ */
+export function execDeadlineHint(seconds: number): string {
+  return (
+    `The command was terminated at the ${seconds}-second execution limit. ` +
+    "Run long jobs in the background (for example " +
+    "`nohup <command> > /tmp/job.log 2>&1 &`, or start them in a terminal session) " +
+    "and poll the output file instead of blocking on a single command."
+  );
+}
+
+/** The reserved `detail` key the transport sets to "1" when it KNOWS a synthesized
+ *  AGENT_OFFLINE fault occurred pre-send (no connection / no responder) — so the op
+ *  provably never reached the machine. `agentErrorToControlError` reads it into
+ *  `SelfhostedControlError.neverSent`. Reserved (never sent by the agent). */
+export const NEVER_SENT_DETAIL_KEY = "_never_sent";
+
 /** Build a synthesized AGENT_OFFLINE `AgentError` — the control plane uses this
- *  when no agent responds on the subject at all. */
-export function offlineAgentError(message = "no agent responded (offline)"): AgentError {
+ *  when no agent responds on the subject at all. `neverSent` marks that the request
+ *  provably never reached the machine (no connection / no responder), which makes it
+ *  safe to re-issue for ANY op kind; leave it false for an AMBIGUOUS transport error
+ *  (a connection torn mid-request may have delivered the op). */
+export function offlineAgentError(
+  message = "no agent responded (offline)",
+  neverSent = false,
+): AgentError {
   return {
     code: ErrorCode.ERROR_CODE_AGENT_OFFLINE,
     message,
     retryable: false,
-    detail: {},
+    detail: neverSent ? { [NEVER_SENT_DETAIL_KEY]: "1" } : {},
   };
 }
 
@@ -288,24 +406,33 @@ function isRequestTimeoutError(err: unknown): boolean {
  */
 export class NatsControlRpc implements ControlRpc {
   private readonly connect: () => Promise<NatsRequestConnection | null>;
-  private connection: NatsRequestConnection | null | undefined;
+  private connection: NatsRequestConnection | undefined;
+  private connecting: Promise<NatsRequestConnection | null> | undefined;
 
   constructor(connect: () => Promise<NatsRequestConnection | null>) {
     this.connect = connect;
   }
 
   private async resolveConnection(): Promise<NatsRequestConnection | null> {
-    if (this.connection === undefined) {
-      try {
-        this.connection = await this.connect();
-      } catch {
-        // A connect failure is an OFFLINE condition, not a crash. Cache null so a
-        // later request retries the factory (set undefined to allow retry).
-        this.connection = null;
-        return null;
-      }
+    if (this.connection) {
+      return this.connection;
     }
-    return this.connection ?? null;
+    // Share one in-flight dial across concurrent callers, but cache only a real
+    // connection. A transient null/throw must be retried by the next request;
+    // pinning null here would make a recovered NATS bus look offline until the
+    // API/worker process restarted.
+    this.connecting ??= this.connect()
+      .then((connection) => {
+        if (connection) {
+          this.connection = connection;
+        }
+        return connection;
+      })
+      .catch(() => null)
+      .finally(() => {
+        this.connecting = undefined;
+      });
+    return this.connecting;
   }
 
   async request(
@@ -316,7 +443,8 @@ export class NatsControlRpc implements ControlRpc {
     const conn = await this.resolveConnection();
     if (!conn) {
       // No NATS configured / not reachable → the agent is unaddressable → offline.
-      return offlineControlResponse(req.requestId);
+      // The request was never published (no connection) → NEVER SENT: safe to re-issue.
+      return offlineControlResponse(req.requestId, true);
     }
     const payload = ControlRequest.encode(req).finish();
     try {
@@ -325,11 +453,13 @@ export class NatsControlRpc implements ControlRpc {
     } catch (err) {
       // Re-allow a future request to re-dial if the cached conn was torn down.
       if (isNoRespondersError(err)) {
-        // No subscriber on the subject at all → the machine is offline.
-        return offlineControlResponse(req.requestId);
+        // No subscriber on the subject at all → the request reached no responder and
+        // was NOT delivered → NEVER SENT (provably not executed): safe to re-issue.
+        return offlineControlResponse(req.requestId, true);
       }
       if (isRequestTimeoutError(err)) {
-        // A responder may exist but the request timed out → a transient blip.
+        // A responder may exist but the request timed out → a transient blip. AMBIGUOUS
+        // (the op may have run and only the reply was lost) — never marked never-sent.
         return timeoutControlResponse(req.requestId);
       }
       // Any other transport error → treat as offline (never a NotFound). The op
@@ -340,9 +470,11 @@ export class NatsControlRpc implements ControlRpc {
   }
 }
 
-/** A `ControlResponse` carrying a synthesized AGENT_OFFLINE error. */
-export function offlineControlResponse(requestId: string): ControlResponse {
-  return { requestId, error: offlineAgentError(), result: undefined };
+/** A `ControlResponse` carrying a synthesized AGENT_OFFLINE error. `neverSent` marks
+ *  a pre-send synthesis (no connection / no responder) — the op provably never
+ *  reached the machine, so it is safe to re-issue for any op kind. */
+export function offlineControlResponse(requestId: string, neverSent = false): ControlResponse {
+  return { requestId, error: offlineAgentError(undefined, neverSent), result: undefined };
 }
 
 /** A `ControlResponse` carrying a synthesized TIMEOUT error (→ reconnecting). */

package/src/sandbox/selfhosted/fault-rendering.ts

@@ -0,0 +1,226 @@
+// The in-band fault renderer for Connected Machine (selfhosted) control faults.
+//
+// The failure-visibility doctrine: every fault the model sees at the point of
+// impact must carry FOUR fields — what happened, which layer, what was preserved,
+// what to try — with a retry verdict that is actually correct for the fault class.
+//
+// Why this exists: a thrown `SelfhostedControlError` reaches the model already
+// wrapped by the SDK's generic tool-error function as
+// "An error occurred while running the tool. Please try again. Error: …". That
+// "Please try again" is actively wrong for a machine that is offline, a consent
+// that has not been granted, or a reply that was too large — the model retries a
+// fault that a retry cannot fix, and must parse prose to recover which layer
+// failed. The typed `SelfhostedControlError` flags carry the truth; this renders
+// them into the four structural fields so the capability tools can surface a
+// legible fault as their result instead of the misleading wrapper.
+
+import { errorCodeToJSON, ErrorCode } from "@opengeni/agent-proto";
+import { SelfhostedControlError } from "./control-rpc";
+
+/**
+ * A stable, low-cardinality fault-class string for a `SelfhostedControlError` — the
+ * out-of-band twin of the in-band renderer's class taxonomy (single-sourced here).
+ * Used for the `machine.op.*` event class + the metrics label + the infra filter.
+ */
+export function selfhostedFaultClass(error: SelfhostedControlError): string {
+  if (error.agentOffline || error.code === ErrorCode.ERROR_CODE_AGENT_OFFLINE) return "offline";
+  if (error.draining) return "draining";
+  if (error.payloadTooLarge || error.code === ErrorCode.ERROR_CODE_PAYLOAD_TOO_LARGE)
+    return "payload_too_large";
+  if (error.reason === "agent_reconnecting") return "reconnecting";
+  if (error.reason === "consent_required") return "consent";
+  if (error.osNotFound || error.code === ErrorCode.ERROR_CODE_NOT_FOUND) return "not_found";
+  if (error.fenced || error.code === ErrorCode.ERROR_CODE_FENCED) return "fenced";
+  switch (error.code) {
+    case ErrorCode.ERROR_CODE_OS:
+      return "os";
+    case ErrorCode.ERROR_CODE_STREAM:
+      return "stream";
+    case ErrorCode.ERROR_CODE_PROTOCOL:
+      return "protocol";
+    default:
+      return "unspecified";
+  }
+}
+
+/**
+ * The fault classes that count as an INFRASTRUCTURE fault (a `machine.op.failed`
+ * event). A semantic miss the model asked about — a missing path (`not_found`), a
+ * consent gate (`consent`), a nonzero exit (never a fault at all) — is an OUTCOME,
+ * not an infra fault, and is DELIBERATELY excluded; `fenced` is a routing/session
+ * condition, not a machine fault. (Doctrine two-planes / the team ruling.)
+ */
+export const SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES: ReadonlySet<string> = new Set([
+  "offline",
+  "draining",
+  "payload_too_large",
+  "reconnecting",
+  "os",
+  "stream",
+  "protocol",
+]);
+
+/** The four mandatory field labels (exported so tests can assert their presence). */
+export const FAULT_FIELD_WHAT_HAPPENED = "What happened:";
+export const FAULT_FIELD_WHICH_LAYER = "Which layer:";
+export const FAULT_FIELD_WHAT_PRESERVED = "What was preserved:";
+export const FAULT_FIELD_WHAT_TO_TRY = "What to try:";
+
+interface FaultFields {
+  headline: string;
+  happened: string;
+  layer: string;
+  preserved: string;
+  tryNext: string;
+}
+
+/** Assemble the four-field block. The typed wire code is folded into "what
+ *  happened" so the model has both the plain language and the precise code. */
+function assemble(error: SelfhostedControlError, f: FaultFields): string {
+  const code = errorCodeToJSON(error.code);
+  return [
+    `[connected machine] ${f.headline}`,
+    `${FAULT_FIELD_WHAT_HAPPENED} ${f.happened} (control code ${code}).`,
+    `${FAULT_FIELD_WHICH_LAYER} ${f.layer}`,
+    `${FAULT_FIELD_WHAT_PRESERVED} ${f.preserved}`,
+    `${FAULT_FIELD_WHAT_TO_TRY} ${f.tryNext}`,
+  ].join("\n");
+}
+
+/**
+ * Render a `SelfhostedControlError` into the doctrine's four in-band fields with a
+ * correct retry verdict. The fault CLASS is read from the typed flags (never the
+ * message prose), so each class gets the right "which layer" and "what to try":
+ *
+ *  - PAYLOAD_TOO_LARGE → the command ran but its reply was too big to return; the
+ *    fix is to bound the output, not to retry.
+ *  - DRAINING → pre-admission backpressure; nothing ran; safe to retry shortly.
+ *  - CONSENT_REQUIRED → a machine-side consent gate; retrying cannot help.
+ *  - AGENT_OFFLINE → the machine link is down. If the transport KNOWS the request
+ *    never left (`neverSent`), nothing ran; otherwise the effect is ambiguous and
+ *    the model must check before re-running. Either way "try again" is wrong until
+ *    the machine reconnects.
+ *  - agent_reconnecting (TIMEOUT) → a transient link blip AFTER send: ambiguous, so
+ *    a state-changing command must be checked before re-running (at-least-once).
+ *  - NOT_FOUND → a command/filesystem miss, not a link fault.
+ *  - FENCED → the active machine session changed (a swap); re-run.
+ *  - default (OS/UNSUPPORTED/STREAM/PROTOCOL) → the command/op failed on the machine.
+ */
+export function renderSelfhostedFault(error: SelfhostedControlError): string {
+  const detail = error.detail;
+
+  if (error.payloadTooLarge || error.code === ErrorCode.ERROR_CODE_PAYLOAD_TOO_LARGE) {
+    // The STREAMING retention-ceiling overflow (op-stream OP_OVERFLOW) is a
+    // different truth than the legacy reply-size wall: the machine STOPPED the
+    // command at the ceiling (it did not run to completion), and none of its
+    // output was returned. Same correct next action, honest distinct fields.
+    if (detail.failure_code === "OP_OVERFLOW") {
+      const retained = detail.retained_bytes
+        ? `after retaining ${detail.retained_bytes} bytes of output`
+        : "when its output exceeded what the machine can retain for delivery";
+      return assemble(error, {
+        headline: "the command produced more output than the machine link can deliver",
+        happened: `the machine stopped the command ${retained} — its output-retention ceiling was reached`,
+        layer:
+          "the machine link's output-retention ceiling — the command was terminated at the ceiling, not by its own logic",
+        preserved:
+          "no output was returned, and the command did NOT run to completion — any side effects up to the stop point may have applied.",
+        tryNext:
+          "re-run with the output bounded: redirect to a file (`> /tmp/out.log 2>&1`) then read it " +
+          "back in ranges/chunks, or slice it with `head -c` / `tail -c` / `head` / `tail`.",
+      });
+    }
+    const sizes =
+      detail.encoded_bytes && detail.max_payload
+        ? `its ${detail.encoded_bytes}-byte reply exceeded the machine link's ${detail.max_payload}-byte per-message limit`
+        : "its reply was larger than the machine link can deliver in one message";
+    return assemble(error, {
+      headline: "the command's output was too large to return",
+      happened: `the command ran on the machine, but ${sizes}`,
+      layer: "the machine link's per-message size limit — the command itself executed",
+      preserved: "nothing was returned; the oversized reply was dropped whole.",
+      tryNext:
+        "re-run with the output bounded: redirect to a file (`> /tmp/out.log 2>&1`) then read it " +
+        "back in ranges/chunks, or slice it with `head -c` / `tail -c` / `head` / `tail`.",
+    });
+  }
+
+  if (error.draining) {
+    const retried = detail.retries ? ` after ${detail.retries} retries` : "";
+    return assemble(error, {
+      headline: "the machine is at its concurrent-work capacity",
+      happened: `the machine rejected this command at its admission gate${retried} because its work pool is full`,
+      layer: "the machine's host-work admission — the machine is online, just saturated",
+      preserved: "nothing ran; the command was rejected before it started.",
+      tryNext: "try again shortly, or reduce the number of commands you run in parallel.",
+    });
+  }
+
+  if (error.reason === "consent_required") {
+    return assemble(error, {
+      headline: "the operation needs consent that has not been granted",
+      happened: "the machine rejected the operation because the required consent is not granted",
+      layer: "the machine's consent gate — not your command",
+      preserved: "nothing ran; the operation was rejected.",
+      tryNext:
+        "the consent must be granted on the machine itself; retrying will keep failing until it is.",
+    });
+  }
+
+  if (error.agentOffline || error.code === ErrorCode.ERROR_CODE_AGENT_OFFLINE) {
+    const preserved = error.neverSent
+      ? "nothing ran; the command never reached the machine."
+      : "unknown — the link dropped and the command may or may not have run.";
+    const tryNext = error.neverSent
+      ? "the machine appears offline; check that it is powered on and connected. Retrying will not help until it reconnects."
+      : "check the machine, and check whether the command already took effect before re-running it.";
+    return assemble(error, {
+      headline: "the machine is unreachable",
+      happened: "the enrolled machine did not respond; it appears offline",
+      layer: "the machine link — the machine itself, not your command",
+      preserved,
+      tryNext,
+    });
+  }
+
+  if (error.reason === "agent_reconnecting") {
+    return assemble(error, {
+      headline: "the machine link blipped",
+      happened: "the machine did not respond in time; the link may be reconnecting",
+      layer: "the machine link (transport) — not your command",
+      preserved: "unknown — the command may or may not have run (its reply was lost).",
+      tryNext:
+        "wait a moment and try again; if this was a state-changing command, check whether it " +
+        "already took effect before re-running it.",
+    });
+  }
+
+  if (error.osNotFound || error.code === ErrorCode.ERROR_CODE_NOT_FOUND) {
+    return assemble(error, {
+      headline: "the path or reference was not found on the machine",
+      happened: `the machine reported: ${error.message}`,
+      layer: "the command / filesystem on the machine — not the machine link",
+      preserved: "not applicable; the operation did not apply.",
+      tryNext: "check the path or reference exists on the machine, then re-run.",
+    });
+  }
+
+  if (error.fenced || error.code === ErrorCode.ERROR_CODE_FENCED) {
+    return assemble(error, {
+      headline: "the active machine session changed",
+      happened: "the machine session was swapped underneath the command",
+      layer: "the session-routing layer — not your command",
+      preserved: "nothing ran on this session.",
+      tryNext: "re-run the command; it will target the machine's current session.",
+    });
+  }
+
+  // OS / UNSUPPORTED / STREAM / PROTOCOL / UNSPECIFIED — a command/op-level failure.
+  return assemble(error, {
+    headline: "the command failed on the machine",
+    happened: `the machine reported: ${error.message}`,
+    layer: "the command / operating system on the machine",
+    preserved: "not applicable; the command did not complete successfully.",
+    tryNext: "read the message above and adjust the command; a blind retry is unlikely to help.",
+  });
+}

package/src/sandbox/selfhosted/op-observer.ts

@@ -0,0 +1,54 @@
+// The transport-agnostic per-op observation seam for the Connected Machine
+// (selfhosted) control path — the out-of-band audience of the failure-visibility
+// doctrine (op outcomes + healed faults land in metrics + `machine.*` events so
+// problems are learnable over time).
+//
+// It is DELIBERATELY transport-agnostic (op-shaped, not request/reply-shaped): the
+// op-engine's future op-stream client (ENGINE-INTEGRATION.md step 6) emits through
+// this SAME interface, so the metrics + `machine.*` events keep flowing unchanged
+// across the transport swap. The runtime leaf only INVOKES the observer; the
+// worker/api adapt it to the concrete sinks (a metrics hook, a durable session
+// event), so the leaf stays agent-loop-free and db-free.
+
+import type { ErrorCode } from "@opengeni/agent-proto";
+import type { SelfhostedUnavailableReason } from "./control-rpc";
+
+/**
+ * One completed control-op observation. `outcome` is the terminal result AFTER any
+ * in-call retries: `ok` (the op succeeded) or `failed` (a terminal fault surfaced).
+ * `healed` marks a success that only landed after ≥1 retry — the doctrine's
+ * "healed and invisible in-band, always out-of-band" breadcrumb (drives
+ * `machine.op.recovered`). Every field is derivable at the `SelfhostedSession.call`
+ * exit points with no extra round-trip.
+ */
+export interface SelfhostedOpObservation {
+  /** The control-op kind (`ControlRequest["op"].$case`): exec / fsRead / ping / … */
+  op: string;
+  outcome: "ok" | "failed";
+  /** True when `outcome==="ok"` only after ≥1 in-call retry (draining/timeout/never-sent). */
+  healed: boolean;
+  /** Total in-call retries before this outcome (summed across the retry classes). */
+  retries: number;
+  /** Wall time from the first attempt to the terminal outcome, in ms. */
+  durationMs: number;
+  /** The typed wire code — present on `failed`. */
+  code?: ErrorCode;
+  /** The negotiated capability reason (agent_offline / agent_reconnecting / …), if any. */
+  reason?: SelfhostedUnavailableReason | null;
+  /** Whether a failed fault provably never reached the machine (a pre-send synthesis). */
+  neverSent?: boolean;
+  /** The enrolled machine (agent) id the op addressed — for machine-scoped fan-out. */
+  machineId?: string;
+  /** A stable fault-class string (`selfhostedFaultClass`): the fault class on a
+   *  `failed` op, or the class of retries that a `healed` op recovered from. Absent
+   *  on a clean success. Drives the `machine.op.*` event mapping + its infra filter. */
+  faultClass?: string;
+  /** The reply size in bytes when known — set on a PAYLOAD_TOO_LARGE fault (from the
+   *  agent's `encoded_bytes` detail). The op-stream client fills the success-reply
+   *  size once it owns the framed transport; today's request/reply does not surface it. */
+  replyBytes?: number;
+}
+
+/** A fire-and-forget observer of completed control ops. It MUST NOT throw (the
+ *  session guards the call) and MUST NOT block the op — it is a telemetry tap. */
+export type SelfhostedOpObserver = (observation: SelfhostedOpObservation) => void;