@intentcomputing/koko-agent-sdk 1.0.1

package/lib/module/client/AgentSession.js

@@ -0,0 +1,854 @@
+"use strict";
+
+import { Platform } from 'react-native';
+import { runPreflightConfirm } from "../confirmation/gate.js";
+import { viewTreeOutline } from "../privacy/redact.js";
+import { sensitiveScreenKind, SensitiveScreenError } from "../privacy/sensitiveScreen.js";
+import { screenModelText, screenSnapshotLine, extractAffordances } from "../privacy/screenModel.js";
+import { HostActionsStore, filterVisibleHostActions, hostActionsModelText } from "../host/HostActions.js";
+import { AppUserGateStore, detectAppUserGate, gateMessageValue, userGateModelText } from "../privacy/userGate.js";
+import { GatewayClient, GatewayError } from "./GatewayClient.js";
+import { UsageBus } from "../commercial/UsageBus.js";
+import { AccountStore } from "../commercial/AccountStore.js";
+import { ExecutionGuard } from "../commercial/ExecutionGuard.js";
+import { AgentDebug } from "../commercial/AgentDebug.js";
+import { scrollPosition } from "../native/scrollViewport.js";
+import { TaskStore } from "../task/TaskStore.js";
+import { isLooping } from "./loopDetect.js";
+import { ExplorationGraph, exploreHintsModelText, goalExploreQuery } from "../explore/ExplorationGraph.js";
+import { RunCheckpoint } from "./RunCheckpoint.js";
+import { ActionOutcomeUnknownError, ActionValidationError } from "./ActionOutcomeUnknownError.js";
+import { OperationReplayGuard } from "./OperationReplayGuard.js";
+import { recentSearchAction } from "./recentSearch.js";
+import { webResultCategory } from "./webResultCategory.js";
+import { wrongHostWalletChoice } from "./walletChoiceGuard.js";
+import { currentLocale, gatewayLocale, runtimeText, message, resolveText } from "../i18n/index.js";
+import { buildOnDeviceAssist } from "../assist/OnDeviceAssist.js";
+import { NativeBridge } from "../native/NativeBridge.js";
+import { webVisualChanged, webVisualLabels } from "../assist/webVisualChange.js";
+import { ocrHitInFrames, visibleWebViewFrames } from "../assist/ocrTarget.js";
+
+/** Soft ETA-style start; extend while progressing. Absolute safety wall only. */
+const SOFT_MAX_STEPS = 40;
+const HARD_MAX_STEPS = 200;
+function affordanceLabels(tree) {
+  if (!tree) return [];
+  return extractAffordances(tree, 40).map(a => a.label);
+}
+/**
+ * Cursor-like session with:
+ * - Preflight risk confirm (before loop)
+ * - Unattended execution (stop only)
+ * - Pause when App shows password/PIN/OTP until user finishes
+ */
+export class AgentSession {
+  history = [];
+  stopped = false;
+  running = false;
+  operationReplayGuard = new OperationReplayGuard();
+  constructor(agent, opts) {
+    this.agent = agent;
+    this.gateway = new GatewayClient({
+      token: opts.token,
+      baseUrl: opts.baseUrl
+    });
+    this.opts = opts;
+  }
+  stop() {
+    this.stopped = true;
+    this.agent.cancelPendingActions();
+  }
+  uiLocale() {
+    return currentLocale(this.opts.locale ?? this.agent.getConfig().locale);
+  }
+  text(key, vars) {
+    return runtimeText(key, vars, this.opts.locale ?? this.agent.getConfig().locale);
+  }
+  message(key, vars) {
+    return message(key, vars, this.opts.locale ?? this.agent.getConfig().locale);
+  }
+  assertRunning() {
+    if (this.stopped) throw new Error('KokoAgent: task cancelled');
+  }
+  async run(goal) {
+    if (this.running) throw new Error(this.text('runtime.alreadyRunning'));
+    const acquired = this.agent.acquireSession(this);
+    this.running = true;
+    try {
+      return await this.runInternal(goal);
+    } finally {
+      this.running = false;
+      if (acquired) this.agent.releaseSession(this);
+    }
+  }
+  async runInternal(goal) {
+    // A stop issued while runGoal persists its checkpoint must remain effective.
+    this.operationReplayGuard.reset();
+    this.history = [];
+    this.sessionId = undefined;
+    // Mark running immediately so host freezeOnBlur stays off across sheet→run gap.
+    this.agent.startTask(goal);
+    const cfg = this.agent.getConfig();
+    AgentDebug.runStart(cfg, goal);
+    // Only after startTask: appendNode with no active task invents a task of
+    // its own that nothing ever finishes.
+    if (this.opts.resumedFromReload) {
+      this.agent.log('info', this.message('resume.note'));
+    }
+    // A resumed run starts blind: the action that reloaded the App probably
+    // already achieved the goal, and labels learned before the reload may be
+    // gone (a locale change renames every one of them). Without this the
+    // planner starts over, fails to re-find its entry point and reports a
+    // false failure for work that in fact succeeded. Planner-only — the task
+    // title and every goal heuristic keep the user's original wording.
+    const plannerGoal = () => this.opts.resumedFromReload ? `${goal}\n\n${this.text('resume.verifyFirst')}` : goal;
+    if (this.stopped) {
+      this.agent.finishTask('cancelled', this.message('runtime.cancelled'));
+      return {
+        success: false,
+        summary: this.text('runtime.cancelled'),
+        steps: 0,
+        sessionId: ''
+      };
+    }
+    const preflightOk = await runPreflightConfirm(goal, cfg.confirmation, cfg.onConfirmSensitive);
+    if (!preflightOk || this.stopped) {
+      const summary = this.stopped ? this.text('runtime.cancelled') : this.text('runtime.confirmationCancelled');
+      this.agent.finishTask('cancelled', summary);
+      return {
+        success: false,
+        summary,
+        steps: 0,
+        sessionId: ''
+      };
+    }
+
+    // Soft budget extends while making progress; only HARD + isLooping stop a run.
+    let stepBudget = SOFT_MAX_STEPS;
+    let userReply;
+    let awaitingObservationVerdict = false;
+    this.agent.activeGoal = goal;
+    try {
+      // Session + permit before first dump (production soft gate).
+      this.assertRunning();
+      const began = await this.gateway.beginSession();
+      this.assertRunning();
+      this.sessionId = began.session_id;
+      ExecutionGuard.setPermit(began.execution_permit, began.session_id);
+      const initialGate = await this.waitForAppUserGateIfNeeded();
+      if (initialGate === 'stopped' || initialGate === 'timeout') {
+        const summary = initialGate === 'stopped' ? this.text('runtime.cancelled') : this.text('runtime.gateTimeout');
+        this.agent.finishTask(initialGate === 'stopped' ? 'cancelled' : 'failed', summary);
+        return {
+          success: false,
+          summary,
+          steps: 0,
+          sessionId: this.sessionId || ''
+        };
+      }
+      await ExplorationGraph.ensureLoaded();
+      this.assertRunning();
+      const exploreQuery = goalExploreQuery(goal);
+
+      // Learned paths are suggestions in the screen model. Every action still
+      // goes through the planner, current-screen guards and result observation.
+
+      for (let step = 0; step < HARD_MAX_STEPS; step++) {
+        // Age of the checkpoint gates resume, so it has to track liveness
+        // rather than when the run began.
+        void RunCheckpoint.touch().catch(() => {});
+        if (this.stopped) {
+          this.agent.finishTask('cancelled', this.message('runtime.cancelled'));
+          return {
+            success: false,
+            summary: this.text('runtime.cancelled'),
+            steps: this.history.length,
+            sessionId: this.sessionId || ''
+          };
+        }
+
+        // Still progressing → extend soft budget (KokoAgent-style).
+        if (step >= stepBudget && stepBudget < HARD_MAX_STEPS && !isLooping(this.history)) {
+          stepBudget = Math.min(HARD_MAX_STEPS, stepBudget + 8);
+          this.agent.log('info', this.message('runtime.progressBudget', {
+            steps: stepBudget
+          }));
+        }
+
+        // App-owned auth: wait for user, do not open Koko Q&A.
+        const gateWait = await this.waitForAppUserGateIfNeeded();
+        if (gateWait === 'stopped') {
+          this.agent.finishTask('cancelled', this.message('runtime.cancelled'));
+          return {
+            success: false,
+            summary: this.text('runtime.cancelled'),
+            steps: this.history.length,
+            sessionId: this.sessionId || ''
+          };
+        }
+        if (gateWait === 'timeout') {
+          this.agent.finishTask('failed', this.message('runtime.gateTimeout'));
+          return {
+            success: false,
+            summary: this.text('runtime.gateTimeout'),
+            steps: this.history.length,
+            sessionId: this.sessionId || ''
+          };
+        }
+        const tree = await this.safeDump();
+        this.assertRunning();
+        // The screen can change between the gate poll and this dump. Do not
+        // run OCR or construct a cloud request for a newly presented gate.
+        if (detectAppUserGate(tree)) {
+          step--;
+          continue;
+        }
+        this.agent.setPlanningTree(tree);
+        const outline = tree ? viewTreeOutline(tree) : undefined;
+        const gate = detectAppUserGate(tree);
+        const visibleHosts = filterVisibleHostActions(HostActionsStore.getActions(), tree);
+        const exploreHints = tree ? ExplorationGraph.hintLabelsTowardGoal(affordanceLabels(tree), exploreQuery.targetTokens, 6) : [];
+        // On-device assist (JS intent/rank + iOS Vision/Core ML OCR) enriches
+        // the planner model; cloud LLM still owns action planning.
+        const assist = await buildOnDeviceAssist(goal, tree);
+        this.assertRunning();
+        let hostWalletAliases = cfg.webWalletAliases || [];
+        if (assist.walletChoices?.length && hostWalletAliases.length && tree) {
+          const frame = visibleWebViewFrames(tree)[0];
+          if (frame) {
+            try {
+              const flags = await NativeBridge.inspectWebWalletProvider({
+                x: frame.x + frame.width / 2,
+                y: frame.y + frame.height / 2
+              });
+              if (flags && (!flags.present || !flags.hasRequest)) hostWalletAliases = [];else if (flags && !flags.isMetaMask) hostWalletAliases = hostWalletAliases.filter(alias => alias !== 'MetaMask');
+            } catch {
+              // A failed provider inspection is not evidence of an injected wallet.
+              hostWalletAliases = [];
+            }
+            this.assertRunning();
+          }
+        }
+        const hostModel = [hostActionsModelText(visibleHosts), exploreHintsModelText(exploreHints), userGateModelText(gate), assist.modelText].filter(Boolean).join('\n');
+        const model = tree ? screenModelText(tree, 40, hostModel || undefined) : hostModel || undefined;
+        if (assist.modelText && AgentDebug.enabled(cfg)) {
+          // eslint-disable-next-line no-console
+          console.log(`[koko] on-device assist: intent=${assist.intent} backend=${assist.backend} ranked=${assist.ranked.length} ocr=${assist.ocrTokens.length}`);
+        }
+        // Debug: what raw texts did native dump actually return?
+        if (tree && AgentDebug.enabled(cfg)) {
+          const texts = [];
+          const walk = n => {
+            const t = (n.text || n.accessibilityLabel || '').trim();
+            if (t && t.length <= 40) texts.push(t);
+            if (n.testID) texts.push(`#${n.testID}`);
+            for (const c of n.children) walk(c);
+          };
+          walk(tree);
+          const uniq = [...new Set(texts)].slice(0, 40);
+          // eslint-disable-next-line no-console
+          console.log(`[koko] dump texts (${uniq.length}): ${uniq.join(' · ').slice(0, 500)}`);
+        }
+        AgentDebug.screenCapabilities(cfg, tree);
+        const planningLocale = gatewayLocale(this.uiLocale());
+        let next;
+        try {
+          next = await this.nextWithRetry({
+            session_id: this.sessionId,
+            goal: this.operationReplayGuard.hasHandoff ? `${plannerGoal()}\n\n${this.text('gate.verifyResultFirst')}` : plannerGoal(),
+            ui_tree: tree || undefined,
+            ui_outline: outline,
+            ui_screen_model: model,
+            local_ocr_goal_labels: assist.goalOcrLabels,
+            local_ocr_wallet_choices: assist.walletChoices,
+            local_web_wallet_aliases: hostWalletAliases,
+            history: this.history,
+            locale: planningLocale,
+            user_reply: userReply,
+            platform: Platform.OS === 'ios' ? 'ios' : Platform.OS === 'android' ? 'android' : 'unknown'
+          });
+        } catch (error) {
+          if (error instanceof SensitiveScreenError) {
+            this.operationReplayGuard.markHandoff();
+            this.history = [];
+            step--;
+            continue;
+          }
+          throw error;
+        }
+        this.assertRunning();
+        userReply = undefined;
+        this.sessionId = next.session_id;
+        ExecutionGuard.setPermit(next.execution_permit, next.session_id);
+        // A response planned in the previous language must not put stale
+        // narration/questions/results back into the live progress UI.
+        if (planningLocale !== gatewayLocale(this.uiLocale())) {
+          this.pushHistory({
+            tool: 'dump',
+            args: {}
+          }, this.text('runtime.localeChanged'), false);
+          step--;
+          continue;
+        }
+        UsageBus.emit({
+          type: 'agent_step',
+          sessionId: next.session_id,
+          step: next.step,
+          tool: next.action.tool,
+          at: Date.now()
+        });
+        AgentDebug.appAction(cfg, next.step, next.action.tool, actionDetail(next.action));
+
+        // After repeated read-only observations, the gateway may only return a
+        // final evidence-based verdict. Never turn this review into another act.
+        if (awaitingObservationVerdict && next.action.tool !== 'done') {
+          this.agent.finishTask('failed', this.message('runtime.loopStopped'));
+          return {
+            success: false,
+            summary: this.text('runtime.loopStopped'),
+            steps: this.history.length,
+            sessionId: this.sessionId || ''
+          };
+        }
+        let recentAction = recentSearchAction(next.action, tree, goal) || (!awaitingObservationVerdict ? webResultCategory(next.action, tree, goal, this.history) : null);
+        let hostWalletChoice = false;
+        if (!recentAction && !awaitingObservationVerdict && next.action.tool === 'dump' && tree && assist.walletChoices?.length) {
+          const aliases = hostWalletAliases.filter(alias => assist.walletChoices.includes(alias));
+          const alias = aliases.length === 1 ? aliases[0] : undefined;
+          const attempted = alias && this.history.some(item => item.tool === 'tap' && String(item.args?.label || '').toLocaleLowerCase().includes(alias.toLocaleLowerCase()));
+          if (alias && !attempted) {
+            const target = await this.agent.find({
+              match: [{
+                text: alias,
+                exact: true
+              }]
+            }, tree);
+            this.assertRunning();
+            if (target && ocrHitInFrames({
+              text: alias,
+              confidence: 1,
+              frame: target.frame
+            }, visibleWebViewFrames(tree))) {
+              recentAction = {
+                tool: 'tap',
+                args: {
+                  nodeId: target.id,
+                  label: alias
+                }
+              };
+              hostWalletChoice = true;
+            }
+          }
+        }
+        if (recentAction) {
+          this.agent.log('info', gatewayLocale(this.uiLocale()) === 'zh' ? hostWalletChoice ? '选择宿主钱包入口。' : next.action.tool === 'type' ? '使用匹配的最近搜索。' : '选择相关应用分类。' : hostWalletChoice ? 'Selecting the host wallet.' : next.action.tool === 'type' ? 'Using the matching recent search.' : 'Selecting the relevant app category.');
+        } else if (next.narration) {
+          this.opts.onNarration?.(next.narration);
+          this.agent.log('info', next.narration);
+        }
+        const action = recentAction || next.action;
+        if (wrongHostWalletChoice(goal, action, assist.walletChoices || [], hostWalletAliases, action.tool === 'tap' ? this.agent.observedOcrLabel(action.args.nodeId) : undefined)) {
+          this.pushHistory(action, 'Host wallet does not match this option; select the configured host wallet or try another browser mode.', false);
+          if (isLooping(this.history)) {
+            this.agent.finishTask('failed', this.message('runtime.loopStopped'));
+            return {
+              success: false,
+              summary: this.text('runtime.loopStopped'),
+              steps: this.history.length,
+              sessionId: this.sessionId || ''
+            };
+          }
+          continue;
+        }
+        if (action.tool === 'done') {
+          this.pushHistory(action, action.args.summary, action.args.success);
+          this.agent.finishTask(action.args.success ? 'success' : 'failed', action.args.summary);
+          AgentDebug.runEnd(cfg, action.args.summary, action.args.success);
+          return {
+            success: action.args.success,
+            summary: action.args.summary,
+            steps: this.history.length,
+            sessionId: this.sessionId
+          };
+        }
+        if (action.tool === 'ask_user') {
+          this.agent.log('info', this.message('runtime.waitForApp', {
+            request: action.args.question
+          }));
+          this.opts.onNarration?.(this.text('runtime.waitForApp', {
+            request: action.args.question
+          }));
+          const w = await this.waitForAppUserGateIfNeeded({
+            forceMessage: action.args.question,
+            minWaitMs: 1500
+          });
+          this.pushHistory(action, w === 'cleared' ? this.text('runtime.userContinued') : w === 'timeout' ? this.text('runtime.userTimeout') : this.text('runtime.cancelled'), w === 'cleared');
+          if (w === 'stopped' || w === 'timeout') {
+            const summary = w === 'stopped' ? this.text('runtime.cancelled') : this.text('runtime.userTimeout');
+            this.agent.finishTask(w === 'stopped' ? 'cancelled' : 'failed', summary);
+            return {
+              success: false,
+              summary,
+              steps: this.history.length,
+              sessionId: this.sessionId
+            };
+          }
+          if (isLooping(this.history)) {
+            this.agent.finishTask('failed', this.message('runtime.loopStopped'));
+            return {
+              success: false,
+              summary: this.text('runtime.loopStopped'),
+              steps: this.history.length,
+              sessionId: this.sessionId
+            };
+          }
+          userReply = this.text('runtime.userContinued');
+          continue;
+        }
+        const beforeSnap = tree ? screenSnapshotLine(tree) : '';
+        const webTap = action.tool === 'tap' && Boolean(action.args.nodeId?.startsWith('ocr:'));
+        const beforeWeb = webTap ? await this.webVisualLabels(tree) : null;
+        this.assertRunning();
+        const beforeLabels = affordanceLabels(tree);
+        if (this.operationReplayGuard.blocks(action, tree)) {
+          const summary = this.text('gate.replayStopped');
+          this.agent.log('info', summary);
+          this.agent.finishTask('failed', summary);
+          return {
+            success: false,
+            summary,
+            steps: this.history.length,
+            sessionId: this.sessionId || ''
+          };
+        }
+        const exec = await this.executeWithRetry(action);
+        this.assertRunning();
+        if (exec.ok) this.operationReplayGuard.record(action, tree);
+        // Let host animations / navigation commit before noop detection.
+        if (action.tool === 'tap' || action.tool === 'swipe' || action.tool === 'type') {
+          await sleep(280);
+        }
+        let after = await this.safeDump();
+        this.assertRunning();
+        let snap = after ? screenSnapshotLine(after) : '';
+        // Dump raced navigation / host async work — give UI one more beat.
+        if (action.tool === 'tap' && exec.ok && beforeSnap && snap && beforeSnap === snap) {
+          await sleep(550);
+          after = await this.safeDump();
+          this.assertRunning();
+          snap = after ? screenSnapshotLine(after) : '';
+        }
+        // Window subviews stack in the order they were added, so a host sheet
+        // that animates in after the action lands above our chrome and buries
+        // the progress pill and its stop button. Re-front once the screen has
+        // settled, which is the first moment that sheet actually exists.
+        TaskStore.liftChrome();
+        // Grow on-device exploration graph from successful taps.
+        if (action.tool === 'tap' && exec.ok && !sensitiveScreenKind(after)) {
+          const tapLabel = String(action.args.label || this.agent.lastTapViaHost?.label || '').trim();
+          if (tapLabel) {
+            ExplorationGraph.recordTransition(beforeLabels, tapLabel, affordanceLabels(after));
+          }
+        }
+        let ok = exec.ok;
+        if (action.tool === 'swipe' && exec.ok && tree && after && !sensitiveScreenKind(after)) {
+          const beforePosition = scrollPosition(tree, action.args.from);
+          const afterPosition = scrollPosition(after, action.args.from);
+          if (beforePosition && afterPosition) {
+            snap += ` scroll_position=${afterPosition}`;
+            if (beforePosition === afterPosition) {
+              ok = false;
+              exec.result = this.text('runtime.scrollNoop');
+            } else {
+              exec.result = this.text('runtime.scrollChanged');
+            }
+          }
+        }
+        let resultWithObs = snap ? `${exec.result} || ${snap}` : exec.result;
+        const changedWeb = webTap && exec.ok ? webVisualChanged(beforeWeb, await this.webVisualLabels(after)) : false;
+        this.assertRunning();
+        if (changedWeb) resultWithObs = `${exec.result} (web content changed; verify the new state) || ${snap}`;
+        // `lastTapViaHost` is only set once the agent has confirmed the host
+        // action moved the screen, so it annotates the trace but no longer
+        // exempts the step from noop detection.
+        const viaHost = Boolean(this.agent.lastTapViaHost);
+        if (viaHost && action.tool === 'tap' && exec.ok) {
+          resultWithObs = snap ? `${exec.result} (host run) || ${snap}` : `${exec.result} (host run)`;
+        }
+        if (
+        // Tap claimed success but UI fingerprint unchanged → treat as noop so
+        // loop detection / planner can escape instead of empty dumping.
+        // Skip when both sides are empty — that is dump failure, not a dead tap.
+        action.tool === 'tap' && exec.ok && !changedWeb && beforeSnap && snap && beforeSnap === snap && snap !== 'screen: (empty)') {
+          ok = false;
+          resultWithObs = `tap_noop (screen unchanged) || ${snap}`;
+        }
+        if (assist.walletChoices?.length) {
+          // Only allowlisted public names are kept; no raw OCR or account data.
+          resultWithObs += ` || visible wallet options: ${assist.walletChoices.join(', ')}`;
+        }
+        this.pushHistory(action, resultWithObs, ok);
+        if (isLooping(this.history)) {
+          const lastThree = this.history.slice(-3);
+          if (!awaitingObservationVerdict && lastThree.length === 3 && lastThree.every(h => h.tool === 'dump')) {
+            // The last observation may already prove the goal. Ask for one
+            // read-only verdict instead of treating a stall as evidence of failure.
+            awaitingObservationVerdict = true;
+            continue;
+          }
+          this.agent.finishTask('failed', this.message('runtime.loopStopped'));
+          AgentDebug.runEnd(cfg, this.text('runtime.loopStopped'), false);
+          return {
+            success: false,
+            summary: this.text('runtime.loopStopped'),
+            steps: this.history.length,
+            sessionId: this.sessionId || ''
+          };
+        }
+        await this.waitForAppUserGateIfNeeded();
+      }
+      this.agent.finishTask('failed', this.message('runtime.stepLimit'));
+      return {
+        success: false,
+        summary: this.text('runtime.stepLimit'),
+        steps: this.history.length,
+        sessionId: this.sessionId || ''
+      };
+    } catch (e) {
+      if (this.stopped) {
+        this.agent.finishTask('cancelled', this.message('runtime.cancelled'));
+        return {
+          success: false,
+          summary: this.text('runtime.cancelled'),
+          steps: this.history.length,
+          sessionId: this.sessionId || ''
+        };
+      }
+      const diagnostic = e instanceof Error ? e.message : String(e);
+      const copy = e instanceof ActionOutcomeUnknownError || e instanceof ActionValidationError ? e.localizedMessage : this.message('runtime.operationFailed');
+      const failureMessage = resolveText(copy);
+      if (e instanceof GatewayError && e.code === 'quota_exceeded') {
+        UsageBus.emit({
+          type: 'quota_exceeded',
+          message: failureMessage,
+          quota: e.quota,
+          at: Date.now()
+        });
+        if (this.opts.token) {
+          void AccountStore.refresh({
+            token: this.opts.token,
+            baseUrl: this.opts.baseUrl
+          });
+        }
+      }
+      AgentDebug.runIssue(cfg, diagnostic);
+      this.agent.log('error', copy);
+      this.agent.finishTask('failed', copy);
+      return {
+        success: false,
+        summary: failureMessage,
+        steps: this.history.length,
+        sessionId: this.sessionId || ''
+      };
+    } finally {
+      this.agent.activeGoal = undefined;
+      ExecutionGuard.clear();
+    }
+  }
+  pushHistory(action, result, ok) {
+    this.history.push({
+      tool: action.tool,
+      args: 'args' in action ? action.args : undefined,
+      result,
+      ok
+    });
+  }
+
+  /** Retry transient upstream / network failures so a completed UI goal is not
+   * marked failed just because the next planning round could not reach the LLM. */
+  async nextWithRetry(body) {
+    const maxAttempts = 3;
+    let last;
+    for (let i = 0; i < maxAttempts; i++) {
+      this.assertRunning();
+      try {
+        const result = await this.gateway.next(body);
+        this.assertRunning();
+        return result;
+      } catch (e) {
+        this.assertRunning();
+        last = e;
+        const msg = e instanceof Error ? e.message : String(e);
+        const status = e instanceof GatewayError ? e.status : 0;
+        const retryable = status === 502 || status === 503 || status === 504 || /fetch failed|network request failed|ECONNRESET|ETIMEDOUT|upstream_failed|socket/i.test(msg);
+        if (!retryable || i === maxAttempts - 1) throw e;
+        AgentDebug.runIssue(this.agent.getConfig(), this.text('runtime.gatewayRetry', {
+          attempt: i + 1,
+          total: maxAttempts - 1
+        }));
+        await sleep(500 * (i + 1));
+      }
+    }
+    throw last;
+  }
+  async safeDump() {
+    try {
+      let tree = await this.agent.dump({
+        silent: true
+      });
+      if (sensitiveScreenKind(tree)) return tree;
+      // Loading / transition settle: empty screens often need a beat.
+      for (let i = 0; i < 3; i++) {
+        const items = tree ? extractAffordances(tree, 8) : [];
+        if (items.length > 0) break;
+        await sleep(350);
+        if (this.stopped) break;
+        tree = await this.agent.dump({
+          silent: true
+        });
+      }
+      return tree;
+    } catch {
+      return null;
+    }
+  }
+  async webVisualLabels(tree) {
+    if (!tree || sensitiveScreenKind(tree) || !NativeBridge.isAvailable()) return null;
+    try {
+      const ocr = await NativeBridge.recognizeScreenText({
+        maxTokens: 64,
+        hideOverlays: true
+      });
+      return webVisualLabels(tree, ocr.texts || []);
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * When the App presents credentials or recovery material, wait locally.
+   * Koko does not collect secrets — user types in the App UI.
+   */
+  async waitForAppUserGateIfNeeded(opts) {
+    const tree = await this.safeDump();
+    const gate = tree ? detectAppUserGate(tree) : detectAppUserGate(null);
+    if (!gate && !opts?.forceMessage) return 'none';
+    if (sensitiveScreenKind(tree)) {
+      this.operationReplayGuard.markHandoff();
+      this.history = [];
+    }
+    const msg = gate ? gateMessageValue(gate, this.opts.locale ?? this.agent.getConfig().locale) : opts?.forceMessage ? this.message('runtime.waitForApp', {
+      request: opts.forceMessage
+    }) : this.message('runtime.inputInApp');
+    this.agent.log('info', msg);
+    this.opts.onNarration?.(resolveText(msg));
+    TaskStore.setUserGateWaiting(msg);
+    const waitNode = TaskStore.appendNode({
+      kind: 'confirm',
+      title: msg,
+      status: 'running'
+    });
+    const timeout = this.opts.userGateTimeoutMs ?? 300_000;
+    const pollMs = 700;
+    const started = Date.now();
+    const minWait = opts?.minWaitMs ?? 0;
+    let checkedResumeAt = started;
+    let fingerprint = gate ? `${gate.kind}:${gate.nodeId}:${gate.label}` : `force:${opts?.forceMessage ?? ''}`;
+    try {
+      while (Date.now() - started < timeout) {
+        if (this.stopped) return 'stopped';
+        await sleep(pollMs);
+        if (this.stopped) return 'stopped';
+
+        // A status check is not permission to bypass a still-private page.
+        if (AppUserGateStore.wasManuallyResumedSince(checkedResumeAt)) {
+          checkedResumeAt = Date.now() + 1;
+          const current = await this.safeDump();
+          if (current && !sensitiveScreenKind(current) && !AppUserGateStore.getHostGate()) {
+            TaskStore.updateNode(waitNode.id, {
+              status: 'done',
+              title: this.message('runtime.gateConfirmed')
+            });
+            return 'cleared';
+          }
+          const feedback = this.message(current ? 'gate.stillPrivate' : 'gate.unreadable');
+          TaskStore.setUserGateWaiting(feedback);
+          TaskStore.updateNode(waitNode.id, {
+            title: feedback
+          });
+          continue;
+        }
+        const now = await this.safeDump();
+        if (!now) continue;
+        const g2 = detectAppUserGate(now);
+        if (!g2) {
+          if (Date.now() - started >= minWait) {
+            TaskStore.updateNode(waitNode.id, {
+              status: 'done',
+              title: this.message('runtime.gateCompleted')
+            });
+            return 'cleared';
+          }
+          continue;
+        }
+        const fp2 = `${g2.kind}:${g2.nodeId}:${g2.label}`;
+        if (fp2 !== fingerprint) {
+          fingerprint = fp2;
+          const nextMsg = gateMessageValue(g2, this.opts.locale ?? this.agent.getConfig().locale);
+          this.opts.onNarration?.(resolveText(nextMsg));
+          TaskStore.setUserGateWaiting(nextMsg);
+          TaskStore.updateNode(waitNode.id, {
+            title: nextMsg
+          });
+        }
+      }
+      TaskStore.updateNode(waitNode.id, {
+        status: 'failed',
+        title: this.message('runtime.gateInputTimeout')
+      });
+      return 'timeout';
+    } finally {
+      TaskStore.setUserGateWaiting(null);
+    }
+  }
+  async executeWithRetry(action) {
+    // Only observations are safe to repeat after an exception. A rejected
+    // native promise or timed-out host callback can follow a committed action.
+    const readOnly = action.tool === 'dump' || action.tool === 'find';
+    const configured = this.opts.localRetries ?? 3;
+    const retries = readOnly && Number.isFinite(configured) ? Math.max(0, Math.min(3, Math.floor(configured))) : 0;
+    let last = '';
+    for (let attempt = 0; attempt <= retries; attempt++) {
+      this.assertRunning();
+      try {
+        const result = await this.executeOnce(action);
+        this.assertRunning();
+        return {
+          ok: true,
+          result
+        };
+      } catch (e) {
+        this.assertRunning();
+        last = e instanceof Error ? e.message : String(e);
+        if (e instanceof ActionValidationError || e instanceof SensitiveScreenError) {
+          // The action was rejected before dispatch; let the planner observe
+          // and choose a different target without replaying a real touch.
+          return {
+            ok: false,
+            result: `pre_dispatch: ${last}`
+          };
+        }
+        if (!readOnly) throw e instanceof ActionOutcomeUnknownError ? e : new ActionOutcomeUnknownError(last);
+        if (attempt < retries) await sleep(200);
+      }
+    }
+    return {
+      ok: false,
+      result: last || 'failed'
+    };
+  }
+  async executeOnce(action) {
+    switch (action.tool) {
+      case 'dump':
+        {
+          const tree = await this.agent.dump();
+          return `dumped || ${screenSnapshotLine(tree)}`;
+        }
+      case 'find':
+        {
+          const selector = {
+            match: action.args.match,
+            index: action.args.index
+          };
+          // A model lookup observes the current page; it must not secretly turn
+          // into a scroll search before the planner considers visible navigation.
+          const currentTree = await this.safeDump();
+          if (!currentTree) throw new Error(this.text('runtime.unreadable'));
+          const node = await this.agent.find(selector, currentTree);
+          if (!node) return 'null';
+          return JSON.stringify({
+            id: node.id,
+            text: node.text,
+            accessibilityLabel: node.accessibilityLabel,
+            clickable: node.clickable,
+            selected: node.selected,
+            enabled: node.enabled,
+            visible: node.visible,
+            secure: node.secure,
+            frame: node.frame
+          });
+        }
+      case 'tap':
+        {
+          await this.agent.tap({
+            selector: action.args.selector,
+            nodeId: action.args.nodeId,
+            point: action.args.point,
+            label: action.args.label,
+            options: {
+              revealIfNeeded: false
+            }
+          });
+          return `tapped ${action.args.label || action.args.nodeId || 'target'}`;
+        }
+      case 'swipe':
+        {
+          await this.agent.swipe(action.args.from, action.args.to);
+          return 'swiped';
+        }
+      case 'pinch':
+        {
+          await this.agent.pinch(action.args.center, action.args.scale);
+          return `pinched scale=${action.args.scale}`;
+        }
+      case 'type':
+        {
+          // Never type into secure fields — wait for user instead.
+          const root = await this.safeDump();
+          this.assertRunning();
+          if (root && detectAppUserGate(root)) {
+            await this.waitForAppUserGateIfNeeded();
+            return 'skipped type: waited for user secure input';
+          }
+          await this.agent.type(action.args.text, {
+            selector: action.args.selector,
+            nodeId: action.args.nodeId,
+            options: {
+              clear: action.args.clear
+            },
+            label: action.args.label
+          });
+          return `typed ${action.args.text.slice(0, 40)}`;
+        }
+      default:
+        return 'noop';
+    }
+  }
+}
+function actionDetail(action) {
+  switch (action.tool) {
+    case 'tap':
+      return action.args.label;
+    case 'type':
+      return action.args.label || action.args.text?.slice(0, 24);
+    case 'swipe':
+      return action.args.label;
+    case 'pinch':
+      return action.args.label || `×${action.args.scale}`;
+    case 'find':
+      {
+        const first = action.args.match?.[0];
+        return first?.text;
+      }
+    case 'ask_user':
+      return action.args.question?.slice(0, 60);
+    case 'done':
+      return action.args.summary?.slice(0, 80);
+    default:
+      return undefined;
+  }
+}
+function sleep(ms) {
+  return new Promise(r => setTimeout(r, ms));
+}
+//# sourceMappingURL=AgentSession.js.map