machine-bridge-mcp 2.0.0 → 3.0.0-beta.102

package/src/local/computer-use-recovery.mjs

@@ -0,0 +1,82 @@
+export function buildRetryGuidance({
+  dispatchStatus,
+  effectStatus,
+  expectationRequested,
+  postObservation,
+  continuation,
+}) {
+  const postSnapshotId = postObservation?.snapshot_id || null;
+  const mappedPostTargetRef = continuation?.surface === "application"
+    && continuation?.target_identity_continues === true
+    && typeof continuation?.mapped_post_target_ref === "string"
+    && continuation.mapped_post_target_ref
+    ? continuation.mapped_post_target_ref
+    : "";
+  const base = {
+    same_action_retry_allowed: false,
+    post_snapshot_id: postSnapshotId,
+    ...(mappedPostTargetRef ? {
+      mapped_post_target_ref: mappedPostTargetRef,
+      mapped_ref_for_replanning_only: true,
+    } : {}),
+  };
+
+  if (effectStatus === "confirmed") {
+    return {
+      ...base,
+      disposition: "do_not_retry",
+      reason: "effect_confirmed",
+      next_step: nextObservationStep(postObservation, continuation),
+      message: "The requested effect was confirmed. Do not replay the same action.",
+    };
+  }
+
+  if (dispatchStatus === "completed" && expectationRequested !== true) {
+    return {
+      ...base,
+      disposition: "do_not_retry",
+      reason: "dispatch_completed_without_effect_contract",
+      next_step: nextObservationStep(postObservation, continuation),
+      message: "Dispatch completed without an explicit effect contract. Do not replay the same action automatically; continue from observed state.",
+    };
+  }
+
+  if (!postObservation) {
+    return {
+      ...base,
+      disposition: "reobserve_before_retry",
+      reason: dispatchStatus === "unknown" ? "dispatch_unknown_and_post_state_unavailable" : "post_state_unavailable",
+      next_step: "computer_observe",
+      message: "Post-state is unavailable. Re-observe before deciding whether any retry is appropriate.",
+    };
+  }
+
+  if (continuation?.reobserve_recommended === true) {
+    return {
+      ...base,
+      disposition: "reobserve_before_retry",
+      reason: continuation.reason || "post_state_requires_fresh_observation",
+      next_step: "computer_observe",
+      message: "The post-state cannot be safely continued from the previous ref namespace. Re-observe before deciding whether any retry is appropriate.",
+    };
+  }
+
+  return {
+    ...base,
+    disposition: "use_post_snapshot",
+    reason: dispatchStatus === "unknown"
+      ? "dispatch_unknown_inspect_post_state_before_any_retry"
+      : effectStatus === "not_observed"
+        ? "requested_effect_not_observed_use_post_state_to_replan"
+        : "effect_unconfirmed_use_post_state_before_any_retry",
+    next_step: "continue_from_post_snapshot",
+    message: dispatchStatus === "unknown"
+      ? "Dispatch outcome is ambiguous. Inspect and continue from the post snapshot; do not replay the same action automatically."
+      : "The requested effect was not confirmed. Use the post snapshot to replan before considering any retry.",
+  };
+}
+
+function nextObservationStep(postObservation, continuation) {
+  if (!postObservation || continuation?.reobserve_recommended === true) return "computer_observe";
+  return "continue_from_post_snapshot";
+}

package/src/local/computer-use-result-budget.mjs

@@ -0,0 +1,145 @@
+import { buildContinuation, observationDiff, projectPostObservation } from "./computer-use-observation.mjs";
+import { buildRetryGuidance } from "./computer-use-recovery.mjs";
+import { normalizeToolResult } from "./tool-result-boundary.mjs";
+
+const WORST_CASE_SNAPSHOT_ID = `cu_${"A".repeat(80)}`;
+export const RESULT_BUDGET_OMISSION_REASON = "tool_result_budget";
+
+export function observationResult(observation, imageContent) {
+  if (!imageContent.length) return observation;
+  return {
+    $mcp: {
+      content: [{ type: "text", text: JSON.stringify(observation) }, ...imageContent],
+      structuredContent: observation,
+    },
+  };
+}
+
+export function assertObservationResultFits(observation, imageContent) {
+  const projected = { ...observation, snapshot_id: WORST_CASE_SNAPSHOT_ID };
+  normalizeToolResult(observationResult(projected, imageContent));
+}
+
+export function isResultLimitExceeded(error) {
+  return error?.code === "limit_exceeded";
+}
+
+export function omitBrowserScreenshotForResultBudget(observation) {
+  if (!observation?.capture || !observation?.capabilities) return;
+  observation.capture.screenshot = false;
+  observation.capture.screenshot_source = "none";
+  observation.capture.screenshot_sha256 = "";
+  observation.capture.screenshot_fallback_reason = "browser screenshot omitted because the MCP result budget would be exceeded";
+  observation.capture.screenshot_omitted_reason = RESULT_BUDGET_OMISSION_REASON;
+  observation.capabilities.cdp_surface_screenshot = false;
+  observation.capabilities.snapshot_bound_visual_points = false;
+}
+
+export function omitApplicationScreenshotForResultBudget(observation) {
+  if (!observation?.capture || !observation?.capabilities) return;
+  observation.capture.screenshot = false;
+  observation.capture.screenshot_source = "none";
+  observation.capture.screenshot_sha256 = "";
+  observation.capture.screenshot_error = "application screenshot omitted because the MCP result budget would be exceeded";
+  observation.capture.screenshot_omitted_reason = RESULT_BUDGET_OMISSION_REASON;
+  observation.capture.coherence = "window_screenshot_then_accessibility_image_omitted_for_result_budget";
+  observation.capabilities.snapshot_bound_visual_points = false;
+  observation.capabilities.snapshot_bound_semantic_points = false;
+}
+
+function actionResult(result, imageContent) {
+  if (!imageContent.length) return result;
+  return {
+    $mcp: {
+      content: [{ type: "text", text: JSON.stringify(result) }, ...imageContent],
+      structuredContent: result,
+    },
+  };
+}
+
+export function fitActionResultToBudget({
+  result, imageContent, beforeObservation, beforePrivateState, postCapture, target,
+  effectStatus, expectationRequested, postObservationDetail,
+}) {
+  try {
+    const value = actionResult(result, imageContent);
+    normalizeToolResult(value);
+    return value;
+  } catch (error) {
+    if (!isResultLimitExceeded(error)) throw error;
+  }
+
+  let candidate = result;
+  if (postCapture && imageContent.length) {
+    if (postCapture.observation?.surface === "browser") omitBrowserScreenshotForResultBudget(postCapture.observation);
+    else if (postCapture.observation?.surface === "application") omitApplicationScreenshotForResultBudget(postCapture.observation);
+    postCapture.imageContent = [];
+    const observedDiff = observationDiff(beforeObservation, postCapture.observation, beforePrivateState, postCapture.privateState);
+    const continuation = buildContinuation(
+      beforeObservation, postCapture.observation, target, observedDiff, beforePrivateState, postCapture.privateState,
+    );
+    const retryGuidance = buildRetryGuidance({
+      dispatchStatus: result.dispatch_status, effectStatus, expectationRequested,
+      postObservation: postCapture.observation, continuation,
+    });
+    candidate = {
+      ...result,
+      observed_diff: observedDiff,
+      post_observation: projectPostObservation(postCapture.observation, postObservationDetail),
+      continuation,
+      retry_guidance: retryGuidance,
+      recovery: retryGuidance.message,
+      post_screenshot_included: false,
+      post_screenshot_omitted_reason: RESULT_BUDGET_OMISSION_REASON,
+      result_budget_compacted: true,
+    };
+    try {
+      normalizeToolResult(candidate);
+      return candidate;
+    } catch (error) {
+      if (!isResultLimitExceeded(error)) throw error;
+    }
+  }
+
+  const compact = compactActionSettlement(candidate);
+  normalizeToolResult(compact);
+  return compact;
+}
+
+function compactActionSettlement(result) {
+  const verification = result?.verification && typeof result.verification === "object" && !Array.isArray(result.verification)
+    ? { requested: result.verification.requested === true, matched: result.verification.matched === true, reason: boundedText(result.verification.reason, 200) }
+    : null;
+  const postSnapshotId = typeof result?.post_snapshot_id === "string" ? result.post_snapshot_id : null;
+  const retryGuidance = result?.retry_guidance && typeof result.retry_guidance === "object" && !Array.isArray(result.retry_guidance)
+    ? {
+        same_action_retry_allowed: false,
+        disposition: boundedText(result.retry_guidance.disposition, 80),
+        reason: boundedText(result.retry_guidance.reason, 200),
+        next_step: boundedText(result.retry_guidance.next_step, 80),
+        post_snapshot_id: postSnapshotId,
+        message: boundedText(result.retry_guidance.message, 500),
+      }
+    : null;
+  return {
+    surface: result.surface, action: result.action, snapshot_id: result.snapshot_id,
+    dispatch_status: result.dispatch_status, effect_status: result.effect_status, verification,
+    post_snapshot_id: postSnapshotId, post_observation_detail: result.post_observation_detail,
+    post_screenshot_policy: result.post_screenshot_policy, post_screenshot_included: false,
+    ...(result.post_screenshot_omitted_reason === RESULT_BUDGET_OMISSION_REASON ? { post_screenshot_omitted_reason: RESULT_BUDGET_OMISSION_REASON } : {}), post_observation: null,
+    continuation: result.continuation && typeof result.continuation === "object" && !Array.isArray(result.continuation)
+      ? result.continuation
+      : postSnapshotId
+        ? { available: true, snapshot_id: postSnapshotId, reobserve_recommended: true, reason: "post_observation_omitted_from_result_budget" }
+        : { available: false, snapshot_id: null, reobserve_recommended: true, reason: "post_observation_unavailable" },
+    retry_guidance: retryGuidance,
+    ...(result.dispatch_error ? { dispatch_error: boundedText(result.dispatch_error, 1000) } : {}),
+    ...(result.post_observation_error ? { post_observation_error: boundedText(result.post_observation_error, 1000) } : {}),
+    recovery: retryGuidance?.message || "Use the returned post snapshot when available; otherwise observe again before any retry.",
+    result_budget_compacted: true,
+  };
+}
+
+function boundedText(value, maximum) {
+  return typeof value === "string" ? value.slice(0, maximum) : "";
+}

package/src/local/computer-use-snapshot-store.mjs

@@ -0,0 +1,67 @@
+// @ts-check
+import { performance } from "node:perf_hooks";
+import { BridgeError } from "./errors.mjs";
+const SNAPSHOT_TTL_MS = 10 * 60 * 1000;
+const MAX_SNAPSHOTS = 64;
+/** @typedef {{ createdAt: number, observation: Record<string, any>, privateState: any }} SnapshotItem */
+export class ComputerUseSnapshotStore {
+  /** @param {{ now?: () => number, createId: () => string }} options */
+  constructor({ now = () => performance.now(), createId }) {
+    if (typeof createId !== "function") throw new TypeError("computer snapshot store requires an id factory");
+    this.now = now;
+    this.createId = createId;
+    /** @type {Map<string, SnapshotItem>} */
+    this.items = new Map();
+    /** @type {number | null} */
+    this.lastNow = null;
+  }
+  /** @param {Record<string, any>} observation @param {any} [privateState] */
+  add(observation, privateState = null) {
+    this.prune();
+    while (this.items.size >= MAX_SNAPSHOTS) {
+      const oldest = this.items.keys().next().value;
+      if (oldest === undefined) break;
+      this.items.delete(oldest);
+    }
+    let id = this.createId();
+    while (this.items.has(id)) id = this.createId();
+    this.items.set(id, { createdAt: this.readNow(), observation, privateState });
+    return id;
+  }
+  /** @param {string} id */
+  get(id) {
+    this.prune();
+    const item = this.items.get(id);
+    if (!item) throw missingSnapshot("computer snapshot is missing or expired; observe again before acting");
+    this.items.delete(id);
+    this.items.set(id, item);
+    return item;
+  }
+  /** @param {string} id @param {SnapshotItem} expected */
+  claim(id, expected) {
+    this.prune();
+    const item = this.items.get(id);
+    if (!item || item !== expected) throw missingSnapshot("computer snapshot is missing, expired, or already consumed; observe again before acting");
+    this.items.delete(id);
+    return item;
+  }
+  /** @param {string} id */
+  discard(id) {
+    if (id) this.items.delete(id);
+  }
+  prune() {
+    const cutoff = this.readNow() - SNAPSHOT_TTL_MS;
+    for (const [id, item] of this.items) if (item.createdAt < cutoff) this.items.delete(id);
+  }
+  /** @returns {number} */
+  readNow() {
+    const value = Number(this.now());
+    if (!Number.isFinite(value)) throw new TypeError("computer snapshot monotonic clock returned a non-finite value");
+    if (this.lastNow === null || value > this.lastNow) this.lastNow = value;
+    return /** @type {number} */ (this.lastNow);
+  }
+}
+/** @param {string} message */
+function missingSnapshot(message) {
+  return new BridgeError("conflict", message, { details: { reason: "snapshot_missing_or_expired" } });
+}