@agent-native/core 0.93.0 → 0.94.0

package/corpus/core/src/integrations/remote-run-events-store.ts

@@ -5,7 +5,11 @@ import {
   retryOnDdlRace,
 } from "../db/client.js";
 import { ensureTableExists, ensureIndexExists } from "../db/ddl-guard.js";
-import type { RemoteRunEvent } from "./remote-types.js";
+import { serializeBoundedRemoteJson } from "./remote-json-safety.js";
+import type { RemoteLiveViewEvent, RemoteRunEvent } from "./remote-types.js";
+
+const MAX_EVENT_JSON_BYTES = 256_000;
+const MAX_EVENT_BATCH_JSON_BYTES = 1_000_000;
 
 let _initPromise: Promise<void> | undefined;
 
@@ -79,28 +83,56 @@ export async function insertRemoteRunEvents(input: {
   events: Array<{ seq: number; event: unknown }>;
 }): Promise<{ inserted: number }> {
   await ensureTable();
+  if (input.events.length === 0) return { inserted: 0 };
+  if (input.events.length > 1_000) {
+    throw new Error("A remote run event batch cannot exceed 1,000 events");
+  }
   const client = getDbExec();
   const now = Date.now();
-  let inserted = 0;
-
-  for (const event of input.events) {
-    const result = await client.execute({
-      sql: `INSERT INTO integration_remote_run_events
-              (device_id, remote_run_id, seq, event_json, created_at)
-            VALUES (?, ?, ?, ?, ?)
-            ON CONFLICT(device_id, remote_run_id, seq) DO NOTHING`,
-      args: [
-        input.deviceId,
-        input.remoteRunId,
-        event.seq,
-        JSON.stringify(event.event ?? null),
-        now,
-      ],
+  const values = input.events.map(() => "(?, ?, ?, ?, ?)").join(", ");
+  let batchBytes = 0;
+  const args = input.events.flatMap((event) => {
+    const value = isLiveViewEvent(event.event)
+      ? normalizeLiveViewEvent(event.event)
+      : (event.event ?? null);
+    const eventJson = serializeBoundedRemoteJson(value, {
+      label: "Remote run event",
+      maxBytes: MAX_EVENT_JSON_BYTES,
     });
-    inserted += result.rowsAffected ?? (result as any).rowCount ?? 0;
-  }
+    batchBytes += new TextEncoder().encode(eventJson).byteLength;
+    if (batchBytes > MAX_EVENT_BATCH_JSON_BYTES) {
+      throw new Error(
+        `Remote run event batch exceeds ${MAX_EVENT_BATCH_JSON_BYTES} JSON bytes`,
+      );
+    }
+    return [input.deviceId, input.remoteRunId, event.seq, eventJson, now];
+  });
+  const result = await client.execute({
+    sql: `INSERT INTO integration_remote_run_events
+            (device_id, remote_run_id, seq, event_json, created_at)
+          VALUES ${values}
+          ON CONFLICT(device_id, remote_run_id, seq) DO NOTHING`,
+    args,
+  });
+  return {
+    inserted:
+      result.rowsAffected ??
+      Number((result as { rowCount?: unknown }).rowCount ?? 0),
+  };
+}
 
-  return { inserted };
+export async function insertRemoteLiveViewEvents(input: {
+  deviceId: string;
+  remoteRunId: string;
+  events: Array<{ seq: number; event: RemoteLiveViewEvent }>;
+}): Promise<{ inserted: number }> {
+  return insertRemoteRunEvents({
+    ...input,
+    events: input.events.map(({ seq, event }) => ({
+      seq,
+      event: normalizeLiveViewEvent(event),
+    })),
+  });
 }
 
 export async function listRemoteRunEvents(input: {
@@ -135,3 +167,56 @@ function parseJson(value: unknown, fallback: unknown): unknown {
     return fallback;
   }
 }
+
+function normalizeLiveViewEvent(
+  event: RemoteLiveViewEvent,
+): RemoteLiveViewEvent {
+  if (!event || event.type !== "computer.live-view") {
+    throw new Error("Invalid computer live-view event");
+  }
+  const frameHandle = sanitizeString(event.frameHandle, 240);
+  if (/^data:/i.test(frameHandle) || looksLikeLargeBase64(frameHandle)) {
+    throw new Error(
+      "Live-view frames must use ephemeral handles, not image data",
+    );
+  }
+  if (!Number.isSafeInteger(event.capturedAt) || event.capturedAt < 0) {
+    throw new Error("Live-view capturedAt must be an epoch timestamp");
+  }
+  return {
+    type: "computer.live-view",
+    frameHandle,
+    capturedAt: event.capturedAt,
+    width: boundedDimension(event.width),
+    height: boundedDimension(event.height),
+    targetLabel:
+      event.targetLabel == null ? null : sanitizeString(event.targetLabel, 240),
+  };
+}
+
+function isLiveViewEvent(value: unknown): value is RemoteLiveViewEvent {
+  return (
+    Boolean(value) &&
+    typeof value === "object" &&
+    (value as { type?: unknown }).type === "computer.live-view"
+  );
+}
+
+function boundedDimension(value: number | null | undefined): number | null {
+  if (value == null) return null;
+  if (!Number.isInteger(value) || value < 1 || value > 100_000) {
+    throw new Error("Live-view dimensions must be positive integers");
+  }
+  return value;
+}
+
+function sanitizeString(value: unknown, max: number): string {
+  if (typeof value !== "string" || !value.trim() || value.length > max) {
+    throw new Error(`Expected a non-empty string of at most ${max} chars`);
+  }
+  return value.trim();
+}
+
+function looksLikeLargeBase64(value: string): boolean {
+  return value.length > 512 && /^[A-Za-z0-9+/]+={0,2}$/.test(value);
+}

package/corpus/core/src/integrations/remote-types.ts

@@ -6,7 +6,60 @@ export type RemoteCommandKind =
   | "approve"
   | "deny"
   | "stop"
-  | "status";
+  | "status"
+  | "computer-operation";
+
+export type ComputerOperationClass =
+  | "browser.observe"
+  | "browser.control"
+  | "desktop.observe"
+  | "desktop.control";
+
+export type ComputerApprovalScope = "once" | "run" | "task";
+
+export interface ComputerCommandAction {
+  type: string;
+  target?: Record<string, unknown> | null;
+  input?: unknown;
+}
+
+export interface ComputerCommandEnvelope {
+  version: 1;
+  taskId: string;
+  runId: string;
+  sequence: number;
+  idempotencyKey: string;
+  operationClass: ComputerOperationClass;
+  action: ComputerCommandAction;
+  approval: {
+    id?: string | null;
+    scope: ComputerApprovalScope;
+    actionHash: string;
+  };
+  issuedAt: number;
+  leaseExpiresAt: number;
+}
+
+export interface RemoteComputerCapabilities {
+  browser?: {
+    observe: boolean;
+    control: boolean;
+    provider?: string | null;
+    version?: string | null;
+  };
+  desktop?: {
+    observe: boolean;
+    control: boolean;
+    accessibility?: boolean;
+    screenCapture?: boolean;
+    provider?: string | null;
+    version?: string | null;
+  };
+}
+
+export interface RemoteDeviceMetadata extends Record<string, unknown> {
+  computerCapabilities?: RemoteComputerCapabilities;
+}
 
 export type RemoteCommandStatus =
   | "pending"
@@ -25,7 +78,7 @@ export interface RemoteDevice {
   platform: string | null;
   appVersion: string | null;
   hostName: string | null;
-  metadata: Record<string, unknown> | null;
+  metadata: RemoteDeviceMetadata | null;
   deviceTokenHash: string;
   lastSeenAt: number | null;
   status: RemoteDeviceStatus;
@@ -42,7 +95,7 @@ export interface PublicRemoteDevice {
   platform: string | null;
   appVersion: string | null;
   hostName: string | null;
-  metadata: Record<string, unknown> | null;
+  metadata: RemoteDeviceMetadata | null;
   lastSeenAt: number | null;
   status: RemoteDeviceStatus;
   revokedAt: number | null;
@@ -61,6 +114,7 @@ export interface RemoteCommand {
   result: unknown;
   platform: string | null;
   externalThreadId: string | null;
+  computerOperation?: ComputerCommandEnvelope | null;
   attempts: number;
   nextCheckAt: number;
   claimedAt: number | null;
@@ -78,6 +132,15 @@ export interface RemoteRunEvent {
   createdAt: number;
 }
 
+export interface RemoteLiveViewEvent {
+  type: "computer.live-view";
+  frameHandle: string;
+  capturedAt: number;
+  width?: number | null;
+  height?: number | null;
+  targetLabel?: string | null;
+}
+
 export type RemotePushRegistrationStatus = "active" | "inactive";
 
 export interface RemotePushRegistration {

package/corpus/core/src/integrations/webhook-handler.ts

@@ -10,6 +10,7 @@ import {
   isLlmCredentialError,
 } from "../agent/engine/credential-errors.js";
 import {
+  getConfiguredEngineNameForRequest,
   getStoredModelForEngine,
   normalizeModelForEngine,
   resolveEngine,
@@ -45,6 +46,10 @@ import {
 } from "../server/credential-provider.js";
 import { runWithRequestContext } from "../server/request-context.js";
 import { A2A_CONTINUATION_QUEUED_MARKER } from "./a2a-continuation-marker.js";
+import {
+  clearIntegrationAwaitingInput,
+  setIntegrationAwaitingInput,
+} from "./awaiting-input-store.js";
 import { loadIntegrationMemoryPrompt } from "./integration-memory.js";
 import { signInternalToken } from "./internal-token.js";
 import {
@@ -158,6 +163,18 @@ function explicitEngineName(
   return undefined;
 }
 
+async function resolveIntegrationEngineOption(
+  engineOption: WebhookHandlerOptions["engine"],
+  appId?: string,
+): Promise<WebhookHandlerOptions["engine"]> {
+  // A custom engine instance/config is an intentional per-plugin override and
+  // must remain authoritative. A string option is the normal integration
+  // plugin default; org/user Agent settings should override that default just
+  // as they do in web chat.
+  if (engineOption && typeof engineOption === "object") return engineOption;
+  return (await getConfiguredEngineNameForRequest({ appId })) ?? engineOption;
+}
+
 function collectToolResultSummaries(
   completedRun: ActiveRun,
 ): A2AToolResultSummary[] {
@@ -801,13 +818,17 @@ async function processIncomingMessage(
               : undefined,
           },
           async () => {
-            const effectiveApiKey = await resolveIntegrationApiKey(
+            const effectiveEngineOption = await resolveIntegrationEngineOption(
               engineOption,
+              options.appId,
+            );
+            const effectiveApiKey = await resolveIntegrationApiKey(
+              effectiveEngineOption,
               ownerEmail,
               apiKey,
             );
             const engine = await resolveEngine({
-              engineOption,
+              engineOption: effectiveEngineOption,
               apiKey: effectiveApiKey,
               model,
               appId: options.appId,
@@ -850,12 +871,23 @@ async function processIncomingMessage(
         );
       },
       async (completedRun: ActiveRun) => {
+        let keepSlackInputWindow = false;
         try {
           const queuedA2AContinuation = hasQueuedA2AContinuation(completedRun);
+          const slackInputRequest =
+            incoming.platform === "slack"
+              ? extractSlackInputRequest(completedRun)
+              : null;
           let responseText = collectFinalResponseTextFromAgentEvents(
             completedRun.events.map((runEvent) => runEvent.event),
             { fallbackToPreToolText: !queuedA2AContinuation },
           );
+          // `ask-question` is a native web-chat interaction. When an
+          // integration run invokes it successfully, project the same
+          // validated question into Slack text and open a tightly-bound reply
+          // window for the originating user instead of leaving a web-only
+          // card with no way to answer in the channel.
+          if (slackInputRequest) responseText = slackInputRequest.text;
           if (!queuedA2AContinuation && !responseText.trim()) {
             const recoverableA2AArtifactText =
               extractRecoverableA2AArtifactToolResult(completedRun);
@@ -930,18 +962,50 @@ async function processIncomingMessage(
             const outgoing = adapter.formatAgentResponse(responseText, {
               threadDeepLinkUrl,
             });
+            let delivered = false;
             if (progress) {
               try {
                 await progress.complete(outgoing);
+                delivered = true;
               } catch {
                 await adapter.sendResponse(outgoing, incoming, {
                   placeholderRef: opts.placeholderRef,
                 });
+                delivered = true;
               }
             } else {
               await adapter.sendResponse(outgoing, incoming, {
                 placeholderRef: opts.placeholderRef,
               });
+              delivered = true;
+            }
+            if (slackInputRequest && delivered && incoming.senderId) {
+              await setIntegrationAwaitingInput({
+                platform: "slack",
+                externalThreadId: incoming.externalThreadId,
+                requesterId: incoming.senderId,
+              });
+              keepSlackInputWindow = true;
+            }
+          } else if (progress) {
+            // The downstream agent owns the eventual reply, but this parent
+            // integration run owns the native progress stream it opened. End
+            // that stream now so Slack does not leave an eternal task card;
+            // the continuation processor will post the final result into the
+            // same thread when the downstream task completes.
+            const deferred = adapter.formatAgentResponse(
+              "The delegated agent is still working. I’ll post its final result in this thread automatically.",
+            );
+            try {
+              await progress.complete(deferred);
+            } catch {
+              // A failed complete must still terminate a provider-native
+              // stream when the adapter offers a failure lifecycle. Do not
+              // duplicate the deferred text as a regular reply: a later
+              // continuation delivery is authoritative.
+              await progress.fail?.(
+                "The delegated agent is still working. I’ll post its final result in this thread automatically.",
+              );
             }
           }
 
@@ -983,6 +1047,15 @@ async function processIncomingMessage(
             if (!progress?.fail) await adapter.sendResponse(fallback, incoming);
           } catch {}
         } finally {
+          // Any terminal path (including a failed run or an unrelated new
+          // mention) invalidates an older clarification window. The only
+          // exception is the just-delivered, verified `ask-question` flow.
+          if (incoming.platform === "slack" && !keepSlackInputWindow) {
+            await clearIntegrationAwaitingInput(
+              "slack",
+              incoming.externalThreadId,
+            ).catch(() => {});
+          }
           if (!budgetsSettled) {
             await releaseApplicableIntegrationBudgets(
               budgetReservations.reservations,
@@ -991,6 +1064,13 @@ async function processIncomingMessage(
           resolve();
         }
       },
+      // Integration workers are ordinary self-dispatched serverless requests,
+      // not a Netlify background-function route. Without the hosted soft
+      // timeout, a wedged model connection can outlive the worker and leave
+      // Slack's native stream in "working" forever when the host kills the
+      // process. Checkpoint at the foreground-safe boundary so onComplete can
+      // always close the provider progress surface before the function wall.
+      { useHostedSoftTimeoutDefault: true },
     );
   });
 }
@@ -1242,6 +1322,85 @@ function hasQueuedA2AContinuation(completedRun: ActiveRun): boolean {
   });
 }
 
+function extractSlackInputRequest(
+  completedRun: ActiveRun,
+): { text: string } | null {
+  const events = completedRun.events.map((runEvent) => runEvent.event);
+  const didRequestInput = events.some(
+    (event) =>
+      event.type === "tool_done" &&
+      event.tool === "ask-question" &&
+      String(event.result ?? "").startsWith(
+        "Asked the user a clarifying question and rendered it in the chat.",
+      ),
+  );
+  if (!didRequestInput) return null;
+
+  for (let index = events.length - 1; index >= 0; index--) {
+    const event = events[index];
+    if (event.type !== "tool_start" || event.tool !== "ask-question") {
+      continue;
+    }
+    const input = event.input as Record<string, unknown> | undefined;
+    const question =
+      typeof input?.question === "string" ? input.question.trim() : "";
+    if (!question) return null;
+
+    let rawOptions: unknown;
+    try {
+      rawOptions = JSON.parse(String(input?.options ?? "[]"));
+    } catch {
+      return null;
+    }
+    if (!Array.isArray(rawOptions) || rawOptions.length === 0) return null;
+    const options = rawOptions
+      .slice(0, 4)
+      .map((option) => {
+        const value = option as Record<string, unknown> | null;
+        const label =
+          typeof value?.label === "string"
+            ? value.label.trim()
+            : typeof value?.value === "string"
+              ? value.value.trim()
+              : "";
+        if (!label) return null;
+        const description =
+          typeof value?.description === "string"
+            ? value.description.trim()
+            : "";
+        return {
+          label: label.slice(0, 200),
+          description: description.slice(0, 400),
+        };
+      })
+      .filter(
+        (option): option is { label: string; description: string } =>
+          option !== null,
+      );
+    if (!options.length) return null;
+
+    const header =
+      typeof input?.header === "string" ? input.header.trim().slice(0, 80) : "";
+    const allowFreeText = String(input?.allowFreeText ?? "true") !== "false";
+    return {
+      text: [
+        header ? `*${header}*` : null,
+        question.slice(0, 1_500),
+        "",
+        ...options.map(
+          (option, optionIndex) =>
+            `${optionIndex + 1}. ${option.label}${option.description ? ` — ${option.description}` : ""}`,
+        ),
+        "",
+        `Reply in this thread with your choice${allowFreeText ? " or a short answer" : ""}.`,
+      ]
+        .filter((line): line is string => line !== null)
+        .join("\n"),
+    };
+  }
+  return null;
+}
+
 function extractRecoverableA2AArtifactToolResult(
   completedRun: ActiveRun,
 ): string | null {

package/corpus/core/src/localization/default-messages.ts

@@ -211,6 +211,34 @@ const messages = {
     cliRequiresDevModeDescription:
       "Run this app locally with pnpm dev or use Builder.io to access the CLI terminal.",
     toggleAgent: "Toggle agent",
+    voiceMode: {
+      entryButtonLabel: "Use microphone",
+      promptTitle: "Talk to your app",
+      promptDescription:
+        "Voice mode keeps listening while the agent navigates and takes actions.",
+      start: "Start voice mode",
+      keepDictating: "Keep dictating",
+      showChat: "Show chat",
+      hideChat: "Hide chat",
+      end: "End voice mode",
+      status: {
+        connecting: "Connecting",
+        listening: "Listening",
+        speaking: "Speaking",
+        working: "Working",
+        error: "Voice mode needs attention",
+        ending: "Ending voice mode",
+      },
+      errors: {
+        unsupported:
+          "This browser does not support realtime voice conversations.",
+        responseFailed: "OpenAI could not complete the voice response.",
+        sessionFailed: "The realtime voice session encountered an error.",
+        channelDisconnected: "The realtime voice control channel disconnected.",
+        connectionFailed: "The realtime voice connection failed.",
+        offerFailed: "The browser did not create an audio offer.",
+      },
+    },
   },
   share: {
     titleWithResource: 'Share "{{title}}"',

package/corpus/core/src/mcp-client/config.ts

@@ -71,6 +71,8 @@ export interface McpConfig {
   source?: string;
 }
 
+const DESKTOP_COMPUTER_SERVER_ID = "agent-native-desktop-computer";
+
 function isNode(): boolean {
   return (
     typeof process !== "undefined" &&
@@ -148,8 +150,52 @@ export function loadMcpConfig(startDir?: string): McpConfig | null {
     }
   }
 
-  if (fileConfig) return fileConfig;
-  return envConfig;
+  return mergeDesktopChildComputerConfig(fileConfig ?? envConfig);
+}
+
+/**
+ * The desktop process supplies this configuration only to a child it spawned
+ * for a specific Agent run. Raw MCP config cannot opt into this trust path:
+ * the explicit child gate, loopback URL, and strong per-run bearer are all
+ * required. Existing user servers always win on key collisions.
+ */
+function mergeDesktopChildComputerConfig(
+  base: McpConfig | null,
+): McpConfig | null {
+  if (process.env.AGENT_NATIVE_DESKTOP_CHILD !== "1") return base;
+  const url = process.env.AGENT_NATIVE_DESKTOP_COMPUTER_MCP_URL?.trim();
+  const token = process.env.AGENT_NATIVE_DESKTOP_COMPUTER_MCP_TOKEN?.trim();
+  if (!url || !token || !/^[A-Za-z0-9_-]{32,}$/.test(token)) return base;
+  try {
+    const parsed = new URL(url);
+    if (
+      parsed.protocol !== "http:" ||
+      parsed.hostname !== "127.0.0.1" ||
+      parsed.pathname !== "/mcp" ||
+      parsed.username ||
+      parsed.password
+    ) {
+      return base;
+    }
+  } catch {
+    return base;
+  }
+  const servers = { ...(base?.servers ?? {}) };
+  let serverId = DESKTOP_COMPUTER_SERVER_ID;
+  for (let suffix = 2; servers[serverId]; suffix += 1) {
+    serverId = `${DESKTOP_COMPUTER_SERVER_ID}-${suffix}`;
+  }
+  servers[serverId] = {
+    type: "http",
+    url,
+    headers: { Authorization: `Bearer ${token}` },
+    description:
+      "Authenticated computer control for this Agent Native desktop task",
+  };
+  return {
+    servers,
+    source: base?.source ? `${base.source}+desktop-child` : "desktop-child",
+  };
 }
 
 function readEnvConfig(): McpConfig | null {

package/corpus/core/src/mcp-client/index.ts

@@ -81,6 +81,15 @@ export { fetchHubServers } from "./hub-client.js";
 
 export { isMcpToolAllowedForRequest } from "./visibility.js";
 import { isMcpToolAllowedForRequest } from "./visibility.js";
+export {
+  classifyMcpToolCall,
+  evaluateMcpToolCallPolicy,
+  type McpToolCallClassification,
+  type McpToolEffect,
+  type McpToolFamily,
+  type McpToolInvocationPolicy,
+  type McpToolPolicyDecision,
+} from "./tool-policy.js";
 export {
   configureScreenMemory,
   queryScreenMemoryContext,
@@ -120,13 +129,22 @@ import {
   type McpActionResult,
 } from "./app-result.js";
 import type { McpClientManager, McpTool } from "./manager.js";
+import {
+  evaluateMcpToolCallPolicy,
+  type McpToolInvocationPolicy,
+} from "./tool-policy.js";
+
+export interface McpActionEntryOptions {
+  invocationPolicy?: McpToolInvocationPolicy;
+}
 
 export function mcpToolsToActionEntries(
   manager: McpClientManager,
+  options: McpActionEntryOptions = {},
 ): Record<string, ActionEntry> {
   const entries: Record<string, ActionEntry> = {};
   for (const tool of manager.getTools().filter(isVisibleToModel)) {
-    entries[tool.name] = mcpToolToActionEntry(manager, tool);
+    entries[tool.name] = mcpToolToActionEntry(manager, tool, options);
   }
   return entries;
 }
@@ -159,6 +177,7 @@ export function syncMcpActionEntries(
 function mcpToolToActionEntry(
   manager: McpClientManager,
   tool: McpTool,
+  options: McpActionEntryOptions = {},
 ): ActionEntry {
   return {
     tool: {
@@ -167,7 +186,7 @@ function mcpToolToActionEntry(
     },
     http: false,
     ...(tool.annotations?.readOnlyHint === true ? { readOnly: true } : {}),
-    run: async (args: Record<string, string>) => {
+    run: async (args: Record<string, unknown>) => {
       // Defense-in-depth: even if a cross-scope MCP tool somehow makes it
       // into the LLM's visible tool list, reject invocation here so we never
       // execute a user's credentials on behalf of another user.
@@ -178,6 +197,20 @@ function mcpToolToActionEntry(
           `Error: MCP tool ${tool.name} is not available in the current request scope.`,
         );
       }
+      if (options.invocationPolicy) {
+        const decision = evaluateMcpToolCallPolicy(
+          options.invocationPolicy,
+          tool,
+          args,
+        );
+        if (!decision.allowed) {
+          return buildMcpErrorActionResult(
+            tool,
+            args,
+            `Error: MCP tool ${tool.name} is unavailable in read-only mode: ${decision.reason}.`,
+          );
+        }
+      }
       try {
         const result = await manager.callTool(tool.name, args);
         return await buildMcpActionResult(manager, tool, args, result);