@agent-native/core 0.97.0 → 0.98.1

package/corpus/core/src/server/realtime-voice.ts

@@ -18,6 +18,8 @@ import {
   sanitizeToolErrorText,
   sanitizeToolErrorValue,
 } from "../agent/tool-error-redaction.js";
+import { TOOL_SEARCH_ACTION_NAME } from "../agent/tool-search.js";
+import { parseAcceptLanguage } from "../localization/server.js";
 import { getSession } from "./auth.js";
 import {
   getBuilderGatewayBaseUrl,
@@ -34,6 +36,13 @@ export const REALTIME_VOICE_TOOL_PATH = "/_agent-native/realtime-voice/tool";
 export const REALTIME_VOICE_MAX_SDP_BYTES = 64 * 1024;
 export const REALTIME_VOICE_MAX_TOOL_BODY_BYTES = 64 * 1024;
 export const REALTIME_VOICE_MAX_TOOL_OUTPUT_CHARS = 16_000;
+export const REALTIME_VOICE_MAX_TOOLS = 32;
+export const REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES = 32_000;
+export const REALTIME_VOICE_MAX_SESSION_BYTES = 64_000;
+export const REALTIME_VOICE_TOOL_GRANT_TTL_MS = 10 * 60 * 1_000;
+export const REALTIME_VOICE_MAX_TOOL_GRANT_SESSIONS = 256;
+export const REALTIME_VOICE_CAPABILITY_HEADER =
+  "X-Agent-Native-Realtime-Capability";
 
 const OPENAI_REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
 const DEFAULT_MODEL = "gpt-realtime-2.1";
@@ -42,12 +51,42 @@ const DEFAULT_INSTRUCTIONS =
   "You are the live voice interface for this Agent Native app. Speak naturally, briefly, and conversationally. Use the available function tools when the user asks you to navigate or take an action. Never claim an action succeeded until its tool result confirms success. If a tool requires approval, explain that the user must approve it in chat.";
 const MAX_INSTRUCTIONS_CHARS = 16_000;
 const MAX_TOOL_DESCRIPTION_CHARS = 2_000;
-const MAX_TOOL_SCHEMA_CHARS = 64 * 1024;
 const MAX_APPROVAL_KEY_CHARS = 1_024;
 const REALTIME_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/;
 const CALL_ID = /^[A-Za-z0-9_-]{1,256}$/;
 const SESSION_ID = /^[A-Za-z0-9_.:-]{1,256}$/;
 const BROWSER_TAB_ID = /^[A-Za-z0-9_-]{1,96}$/;
+const ISO_639_1_LANGUAGE = /^[A-Za-z]{2}$/;
+const REALTIME_VOICE_BUILT_INS = new Set([
+  "alloy",
+  "ash",
+  "ballad",
+  "coral",
+  "echo",
+  "sage",
+  "shimmer",
+  "verse",
+  "marin",
+  "cedar",
+]);
+const REALTIME_VOICE_REASONING_EFFORT = {
+  instant: "minimal",
+  balanced: "low",
+  deep: "medium",
+} as const;
+
+/**
+ * Realtime sessions have a deliberately bounded tool manifest. Keep the
+ * context/navigation tools ahead of large template registries so voice can
+ * always see and operate the same UI navigation surface as text chat.
+ */
+const REALTIME_VOICE_PRIORITY_TOOLS = [
+  "navigate",
+  "set-url-path",
+  "set-search-params",
+  "view-screen",
+  TOOL_SEARCH_ACTION_NAME,
+] as const;
 
 export interface RealtimeVoiceRequestContext {
   event: H3Event;
@@ -103,6 +142,17 @@ interface RealtimeFunctionTool {
   parameters: Record<string, unknown>;
 }
 
+interface RealtimeToolCapability {
+  userEmail: string;
+  orgId?: string;
+  browserTabId?: string;
+  expiresAt: number;
+  initialNames: Set<string>;
+  names: Set<string>;
+}
+
+type RealtimeToolCapabilityStore = Map<string, RealtimeToolCapability>;
+
 interface AuthenticatedVoiceContext extends RealtimeVoiceRequestContext {
   timezone?: string;
 }
@@ -124,6 +174,50 @@ function configuredIdentifier(
     : fallback;
 }
 
+export function resolveRealtimeVoiceTranscriptionLanguage(
+  acceptLanguage: string | null | undefined,
+): string {
+  for (const locale of parseAcceptLanguage(acceptLanguage)) {
+    const primaryLanguage = locale.split("-")[0];
+    if (ISO_639_1_LANGUAGE.test(primaryLanguage)) {
+      return primaryLanguage.toLowerCase();
+    }
+  }
+  return "en";
+}
+
+export function resolveRealtimeVoiceLanguagePreference(
+  value: string | null | undefined,
+  acceptLanguage: string | null | undefined,
+): string {
+  const normalized = value?.trim().toLowerCase();
+  return normalized && ISO_639_1_LANGUAGE.test(normalized)
+    ? normalized
+    : resolveRealtimeVoiceTranscriptionLanguage(acceptLanguage);
+}
+
+export function resolveRealtimeVoiceReasoningEffort(
+  value: string | null | undefined,
+): "minimal" | "low" | "medium" {
+  const normalized = value?.trim().toLowerCase();
+  return normalized &&
+    Object.hasOwn(REALTIME_VOICE_REASONING_EFFORT, normalized)
+    ? REALTIME_VOICE_REASONING_EFFORT[
+        normalized as keyof typeof REALTIME_VOICE_REASONING_EFFORT
+      ]
+    : "low";
+}
+
+export function resolveRealtimeVoicePreference(
+  value: string | null | undefined,
+  fallback: string,
+): string {
+  const normalized = value?.trim().toLowerCase();
+  return normalized && REALTIME_VOICE_BUILT_INS.has(normalized)
+    ? normalized
+    : fallback;
+}
+
 function truncate(value: string, maxChars: number): string {
   if (value.length <= maxChars) return value;
   return `${value.slice(0, maxChars)}\n\n...[truncated]`;
@@ -181,7 +275,13 @@ function normalizeToolSchema(
 ): Record<string, unknown> | null {
   try {
     const serialized = JSON.stringify(inputSchema);
-    if (!serialized || serialized.length > MAX_TOOL_SCHEMA_CHARS) return null;
+    if (
+      !serialized ||
+      Buffer.byteLength(serialized, "utf8") >
+        REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES
+    ) {
+      return null;
+    }
     const parsed = JSON.parse(serialized) as unknown;
     if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
       return null;
@@ -210,7 +310,162 @@ function buildRealtimeTools(
       parameters,
     });
   }
-  return tools;
+  const priority = new Map<string, number>(
+    REALTIME_VOICE_PRIORITY_TOOLS.map((name, index) => [name, index]),
+  );
+  return tools
+    .map((tool, index) => ({ tool, index }))
+    .sort((left, right) => {
+      const leftPriority = priority.get(left.tool.name);
+      const rightPriority = priority.get(right.tool.name);
+      if (leftPriority !== undefined || rightPriority !== undefined) {
+        if (leftPriority === undefined) return 1;
+        if (rightPriority === undefined) return -1;
+        return leftPriority - rightPriority;
+      }
+      return left.index - right.index;
+    })
+    .map(({ tool }) => tool);
+}
+
+function packRealtimeTools(
+  session: Record<string, unknown>,
+  eligibleTools: RealtimeFunctionTool[],
+): RealtimeFunctionTool[] {
+  const packed: RealtimeFunctionTool[] = [];
+  for (const tool of eligibleTools.slice(0, REALTIME_VOICE_MAX_TOOLS)) {
+    const candidate = [...packed, tool];
+    const candidateSession = {
+      ...session,
+      tools: candidate,
+      tool_choice: "auto",
+    };
+    if (
+      Buffer.byteLength(JSON.stringify(candidateSession), "utf8") <=
+      REALTIME_VOICE_MAX_SESSION_BYTES
+    ) {
+      packed.push(tool);
+    }
+  }
+  return packed;
+}
+
+function mintRealtimeToolCapability(): string {
+  return globalThis.crypto.randomUUID().replaceAll("-", "");
+}
+
+function cleanRealtimeToolCapabilities(
+  capabilities: RealtimeToolCapabilityStore,
+  now = Date.now(),
+): void {
+  for (const [key, capability] of capabilities) {
+    if (capability.expiresAt <= now) capabilities.delete(key);
+  }
+  while (capabilities.size > REALTIME_VOICE_MAX_TOOL_GRANT_SESSIONS) {
+    let oldestKey: string | undefined;
+    let oldestExpiry = Number.POSITIVE_INFINITY;
+    for (const [key, capability] of capabilities) {
+      if (capability.expiresAt < oldestExpiry) {
+        oldestKey = key;
+        oldestExpiry = capability.expiresAt;
+      }
+    }
+    if (!oldestKey) break;
+    capabilities.delete(oldestKey);
+  }
+}
+
+function registerRealtimeToolCapability(
+  capabilities: RealtimeToolCapabilityStore,
+  auth: AuthenticatedVoiceContext,
+  initialNames: Iterable<string>,
+): string {
+  cleanRealtimeToolCapabilities(capabilities);
+  const token = mintRealtimeToolCapability();
+  capabilities.set(token, {
+    userEmail: auth.userEmail.trim().toLowerCase(),
+    ...(auth.orgId ? { orgId: auth.orgId } : {}),
+    ...(auth.browserTabId ? { browserTabId: auth.browserTabId } : {}),
+    expiresAt: Date.now() + REALTIME_VOICE_TOOL_GRANT_TTL_MS,
+    initialNames: new Set(initialNames),
+    names: new Set(),
+  });
+  cleanRealtimeToolCapabilities(capabilities);
+  return token;
+}
+
+function resolveRealtimeToolCapability(
+  capabilities: RealtimeToolCapabilityStore,
+  token: string | undefined,
+  auth: AuthenticatedVoiceContext,
+): RealtimeToolCapability | null {
+  cleanRealtimeToolCapabilities(capabilities);
+  if (!token) return null;
+  const capability = capabilities.get(token);
+  if (!capability) return null;
+  if (
+    capability.userEmail !== auth.userEmail.trim().toLowerCase() ||
+    capability.orgId !== auth.orgId ||
+    capability.browserTabId !== auth.browserTabId
+  ) {
+    return null;
+  }
+  capability.expiresAt = Date.now() + REALTIME_VOICE_TOOL_GRANT_TTL_MS;
+  return capability;
+}
+
+function parseSuccessfulToolSearchNames(output: string): string[] {
+  try {
+    const parsed = JSON.parse(output) as { results?: unknown };
+    if (!Array.isArray(parsed.results)) return [];
+    const names: string[] = [];
+    for (const result of parsed.results) {
+      if (!isRecord(result) || typeof result.name !== "string") continue;
+      if (!REALTIME_TOOL_NAME.test(result.name)) continue;
+      if (!names.includes(result.name)) names.push(result.name);
+      if (names.length >= REALTIME_VOICE_MAX_TOOLS) break;
+    }
+    return names;
+  } catch {
+    return [];
+  }
+}
+
+function grantDiscoveredRealtimeTools(input: {
+  request: NonNullable<ReturnType<typeof parseToolRequest>>;
+  result: RealtimeVoiceToolExecutionResult;
+  toolsByName: ReadonlyMap<string, RealtimeFunctionTool>;
+  initialAllowedNames: ReadonlySet<string>;
+  capability: RealtimeToolCapability;
+}): RealtimeFunctionTool[] {
+  const query = input.request.args.query;
+  if (
+    input.request.name !== TOOL_SEARCH_ACTION_NAME ||
+    input.result.status !== "completed" ||
+    typeof query !== "string" ||
+    !query.trim()
+  ) {
+    return [];
+  }
+
+  const candidates = parseSuccessfulToolSearchNames(input.result.output)
+    .filter((name) => !input.initialAllowedNames.has(name))
+    .map((name) => input.toolsByName.get(name))
+    .filter((tool): tool is RealtimeFunctionTool => Boolean(tool));
+  const boundedTools = packRealtimeTools({}, candidates);
+  const expandedTools: RealtimeFunctionTool[] = [];
+  for (const tool of boundedTools) {
+    if (
+      !input.capability.names.has(tool.name) &&
+      input.capability.names.size >= REALTIME_VOICE_MAX_TOOLS
+    ) {
+      continue;
+    }
+    input.capability.names.add(tool.name);
+    expandedTools.push(tool);
+  }
+  input.capability.expiresAt = Date.now() + REALTIME_VOICE_TOOL_GRANT_TTL_MS;
+  return expandedTools;
 }
 
 function declaredBodyBytes(event: H3Event): number | undefined {
@@ -298,6 +553,7 @@ function invalidMethod(event: H3Event): { error: string } {
 
 function createSessionHandler(
   tools: RealtimeFunctionTool[],
+  capabilities: RealtimeToolCapabilityStore,
   options: MountRealtimeVoiceRoutesOptions,
 ) {
   return defineEventHandler(async (event: H3Event) => {
@@ -364,14 +620,30 @@ function createSessionHandler(
         }
 
         const instructions = await buildInstructions(auth, options);
-        const session = {
+        const transcriptionLanguage = resolveRealtimeVoiceLanguagePreference(
+          readSafeHeader(event, "x-agent-native-realtime-language"),
+          readSafeHeader(event, "accept-language"),
+        );
+        const reasoningEffort = resolveRealtimeVoiceReasoningEffort(
+          readSafeHeader(event, "x-agent-native-realtime-intelligence"),
+        );
+        const voice = resolveRealtimeVoicePreference(
+          readSafeHeader(event, "x-agent-native-realtime-voice"),
+          configuredIdentifier(options.voice, DEFAULT_VOICE),
+        );
+        const sessionBase = {
           type: "realtime",
           model: configuredIdentifier(options.model, DEFAULT_MODEL),
           instructions,
+          parallel_tool_calls: false,
+          reasoning: { effort: reasoningEffort },
           output_modalities: ["audio"],
           audio: {
             input: {
-              transcription: { model: "gpt-4o-mini-transcribe" },
+              transcription: {
+                model: "gpt-4o-mini-transcribe",
+                language: transcriptionLanguage,
+              },
               turn_detection: {
                 type: "semantic_vad",
                 create_response: true,
@@ -380,10 +652,14 @@ function createSessionHandler(
               },
             },
             output: {
-              voice: configuredIdentifier(options.voice, DEFAULT_VOICE),
+              voice,
             },
           },
-          tools,
+        };
+        const packedTools = packRealtimeTools(sessionBase, tools);
+        const session = {
+          ...sessionBase,
+          tools: packedTools,
           tool_choice: "auto",
         };
 
@@ -458,6 +734,15 @@ function createSessionHandler(
 
         setResponseStatus(event, upstream.status);
         setResponseHeader(event, "Content-Type", "application/sdp");
+        setResponseHeader(
+          event,
+          REALTIME_VOICE_CAPABILITY_HEADER,
+          registerRealtimeToolCapability(
+            capabilities,
+            auth,
+            packedTools.map((tool) => tool.name),
+          ),
+        );
         return answerSdp;
       },
     );
@@ -529,7 +814,8 @@ function normalizeExecutionResult(
 }
 
 function createToolHandler(
-  allowedToolNames: ReadonlySet<string>,
+  toolsByName: ReadonlyMap<string, RealtimeFunctionTool>,
+  capabilities: RealtimeToolCapabilityStore,
   options: MountRealtimeVoiceRoutesOptions,
 ) {
   return defineEventHandler(async (event: H3Event) => {
@@ -545,6 +831,15 @@ function createToolHandler(
       setResponseStatus(event, 401);
       return { error: "Authentication required" };
     }
+    const capability = resolveRealtimeToolCapability(
+      capabilities,
+      readSafeHeader(event, REALTIME_VOICE_CAPABILITY_HEADER),
+      auth,
+    );
+    if (!capability) {
+      setResponseStatus(event, 403);
+      return { error: "Invalid or expired realtime voice capability" };
+    }
 
     const contentType = readSafeHeader(event, "content-type")?.toLowerCase();
     if (!contentType?.includes("application/json")) {
@@ -575,12 +870,22 @@ function createToolHandler(
       setResponseStatus(event, 400);
       return { error: "Invalid realtime tool request" };
     }
-    if (!allowedToolNames.has(request.name)) {
+    if (
+      request.browserTabId !== undefined &&
+      request.browserTabId !== capability.browserTabId
+    ) {
+      setResponseStatus(event, 403);
+      return { error: "Realtime voice browser tab mismatch" };
+    }
+    if (
+      !capability.initialNames.has(request.name) &&
+      !capability.names.has(request.name)
+    ) {
       setResponseStatus(event, 404);
       return { error: "Unknown realtime voice tool" };
     }
 
-    const browserTabId = request.browserTabId ?? auth.browserTabId;
+    const browserTabId = capability.browserTabId;
     const threadId = request.sessionId
       ? `realtime:${request.sessionId}`
       : `realtime:${request.callId}`;
@@ -602,9 +907,21 @@ function createToolHandler(
               userEmail: auth.userEmail,
               orgId: auth.orgId,
               ...request,
+              ...(browserTabId ? { browserTabId } : {}),
             }),
           );
-          return { callId: request.callId, ...result };
+          const expandedTools = grantDiscoveredRealtimeTools({
+            request,
+            result,
+            toolsByName,
+            initialAllowedNames: capability.initialNames,
+            capability,
+          });
+          return {
+            callId: request.callId,
+            ...result,
+            ...(expandedTools.length > 0 ? { expandedTools } : {}),
+          };
         } catch (error) {
           setResponseStatus(event, 500);
           return {
@@ -632,12 +949,16 @@ export function mountRealtimeVoiceRoutes(
   }
 
   const tools = buildRealtimeTools(actions);
-  const allowedToolNames = new Set(tools.map((tool) => tool.name));
+  const toolsByName = new Map(tools.map((tool) => [tool.name, tool]));
+  const capabilities: RealtimeToolCapabilityStore = new Map();
   const app = getH3App(nitroApp);
-  app.use(REALTIME_VOICE_SESSION_PATH, createSessionHandler(tools, options));
+  app.use(
+    REALTIME_VOICE_SESSION_PATH,
+    createSessionHandler(tools, capabilities, options),
+  );
   app.use(
     REALTIME_VOICE_TOOL_PATH,
-    createToolHandler(allowedToolNames, options),
+    createToolHandler(toolsByName, capabilities, options),
   );
   return {
     sessionPath: REALTIME_VOICE_SESSION_PATH,

package/corpus/core/src/server/request-origin.ts

@@ -12,6 +12,12 @@ function isLoopbackHost(host: string): boolean {
  * back to `Sec-Fetch-Site` so Safari's fetch-spec behavior still works.
  */
 export function isSameOriginRequest(event: H3Event): boolean {
+  // Fetch metadata is browser-controlled and describes the relationship
+  // before a reverse proxy rewrites Host (dev-lazy maps :8080 to :8088).
+  // Reject an explicit cross-site signal before consulting forgeable headers.
+  const fetchSite = getRequestHeader(event, "sec-fetch-site");
+  if (fetchSite) return fetchSite === "same-origin" || fetchSite === "none";
+
   const host = getRequestHeader(event, "host");
   const origin = getRequestHeader(event, "origin");
   if (origin && host) {
@@ -55,8 +61,6 @@ export function isSameOriginRequest(event: H3Event): boolean {
       return false;
     }
   }
-  const fetchSite = getRequestHeader(event, "sec-fetch-site");
-  if (fetchSite) return fetchSite === "same-origin" || fetchSite === "none";
   // No Origin and no Sec-Fetch-Site: likely a non-browser client (curl,
   // server-side) — safe to allow, CSRF requires a browser with ambient cookies.
   return true;

package/corpus/core/src/server/ssr-handler.ts

@@ -98,6 +98,22 @@ function requestWithPathname(
   return new Request(url, init);
 }
 
+function requestForAnonymousSsr(request: Request): Request {
+  const headers = new Headers(request.headers);
+  headers.delete("cookie");
+  headers.delete("authorization");
+  const init: RequestInit & { duplex?: "half" } = {
+    method: request.method,
+    headers,
+    signal: request.signal,
+  };
+  if (request.body && !["GET", "HEAD"].includes(request.method.toUpperCase())) {
+    init.body = request.body;
+    init.duplex = "half";
+  }
+  return new Request(request.url, init);
+}
+
 function prefixMountedPath(path: string, basePath: string): string {
   if (!basePath || !path.startsWith("/") || path.startsWith("//")) return path;
   if (path === basePath || path.startsWith(`${basePath}/`)) return path;
@@ -106,7 +122,7 @@ function prefixMountedPath(path: string, basePath: string): string {
 
 function prefixMountedHtml(html: string, basePath: string): string {
   if (!basePath) return html;
-  return html
+  const prefixedHtml = html
     .replace(
       /\b(href|src|action|formaction|poster)=(["'])(\/(?!\/)[^"']*)\2/g,
       (_match, attr: string, quote: string, path: string) =>
@@ -116,6 +132,20 @@ function prefixMountedHtml(html: string, basePath: string): string {
       const q = quote || "";
       return `url(${q}${prefixMountedPath(path, basePath)}${q})`;
     });
+
+  // React Router serializes the server-side basename into its hydration
+  // context. The request above is deliberately rendered mount-relative, so
+  // that value is normally "/" even though the browser URL is mounted at a
+  // workspace prefix such as "/analytics". If the client hydrates that
+  // context unchanged, the mounted pathname no longer matches the route tree:
+  // index redirects can stall and child pages can fall through to the 404.
+  // Keep the serialized router state consistent with the URLs we just
+  // prefixed. Template entry clients also set this defensively for older
+  // responses, but the initial hydration state must be correct at the source.
+  return prefixedHtml.replace(
+    /(window\.__reactRouterContext\s*=\s*\{\s*"basename"\s*:\s*)"(?:\\.|[^"\\])*"/,
+    `$1${JSON.stringify(basePath)}`,
+  );
 }
 
 function injectHeadScript(html: string, script: string | null): string {
@@ -219,6 +249,27 @@ function applyDefaultSsrCacheHeader(
 ) {
   if (!isSsrHtmlOrDataResponse(headers, status, pathname)) return;
 
+  // A public shell must never set a viewer cookie or vary by credentials.
+  // Preserve harmless content-negotiation dimensions such as Accept-Encoding.
+  headers.delete("set-cookie");
+  const vary = headers.get("vary");
+  if (vary) {
+    const publicVary = vary
+      .split(",")
+      .map((value) => value.trim())
+      .filter((value) => {
+        const normalized = value.toLowerCase();
+        return (
+          normalized &&
+          normalized !== "*" &&
+          normalized !== "cookie" &&
+          normalized !== "authorization"
+        );
+      });
+    if (publicVary.length > 0) headers.set("vary", publicVary.join(", "));
+    else headers.delete("vary");
+  }
+
   // Netlify Functions/proxies are not cached by default. Set all three cache
   // headers: Cache-Control for browsers, CDN-Cache-Control for generic CDNs,
   // and Netlify-CDN-Cache-Control (with durable) so Netlify's shared cache
@@ -347,7 +398,9 @@ export function createH3SSRHandler(getBuild: () => Promise<unknown> | unknown) {
       return new Response(null, { status: 404 });
     }
     try {
-      const request = requestWithPathname(event.req as Request, p, basePath);
+      const request = requestForAnonymousSsr(
+        requestWithPathname(event.req as Request, p, basePath),
+      );
       // SSR renders an IMPERSONAL public shell — we deliberately do NOT read the
       // request's session/cookies here, and pin an explicitly anonymous request
       // context. That keeps the SSR HTML/.data identical for every visitor so it