@agent-native/core 0.79.27 → 0.80.1

package/corpus/core/src/server/builder-design-systems.ts

@@ -0,0 +1,308 @@
+import { FeatureNotConfiguredError } from "./credential-provider.js";
+import {
+  getBuilderProxyOrigin,
+  resolveBuilderCredentials,
+} from "./credential-provider.js";
+
+const DEFAULT_TIMEOUT_MS = 120_000;
+
+export interface BuilderDesignSystemIndexFile {
+  name: string;
+  data: Uint8Array;
+  mimeType?: string;
+}
+
+export interface BuilderDesignSystemIndexOptions {
+  projectName?: string;
+  description?: string;
+  githubRepoUrl?: string;
+  connectedProjectId?: string;
+  files?: BuilderDesignSystemIndexFile[];
+  selection?: Record<string, string[]>;
+  devToolsVersion?: string;
+}
+
+export interface BuilderDesignSystemIndexResult {
+  ok: true;
+  source: "builder";
+  projectId: string;
+  jobId: string;
+  designSystemId: string;
+  suggestedTitle: string | null;
+  builderUrl: string;
+  status: "in-progress";
+}
+
+interface BuilderDesignSystemCredentials {
+  privateKey: string;
+  publicKey: string;
+  userId: string | null;
+}
+
+interface UploadStartResponse {
+  uploads?: Array<{ idx: number; uploadUrl: string; uploadToken: string }>;
+}
+
+interface GenerateResponse {
+  projectId?: string;
+  jobId?: string;
+  designSystemId?: string;
+}
+
+function trimTrailingSlash(value: string): string {
+  return value.replace(/\/+$/, "");
+}
+
+export function getBuilderDesignSystemsBaseUrl(): string {
+  return (
+    process.env.BUILDER_DESIGN_SYSTEMS_BASE_URL ||
+    `${trimTrailingSlash(getBuilderProxyOrigin())}/design-systems/v1`
+  );
+}
+
+function getBuilderAppHost(): string {
+  return (
+    process.env.BUILDER_APP_HOST ||
+    process.env.BUILDER_PUBLIC_APP_HOST ||
+    "https://builder.io"
+  );
+}
+
+function makeBuilderDesignSystemUrl(
+  path: string,
+  credentials: BuilderDesignSystemCredentials,
+): URL {
+  const base = `${trimTrailingSlash(getBuilderDesignSystemsBaseUrl())}/`;
+  const url = new URL(path.replace(/^\/+/, ""), base);
+  url.searchParams.set("apiKey", credentials.publicKey);
+  if (credentials.userId) url.searchParams.set("userId", credentials.userId);
+  return url;
+}
+
+function makeBuilderHeaders(
+  credentials: BuilderDesignSystemCredentials,
+): Record<string, string> {
+  return {
+    Authorization: `Bearer ${credentials.privateKey}`,
+    "x-builder-api-key": credentials.publicKey,
+    ...(credentials.userId ? { "x-builder-user-id": credentials.userId } : {}),
+  };
+}
+
+async function resolveBuilderDesignSystemCredentials(): Promise<BuilderDesignSystemCredentials> {
+  const credentials = await resolveBuilderCredentials();
+  if (!credentials.privateKey || !credentials.publicKey) {
+    throw new FeatureNotConfiguredError({
+      requiredCredential: "BUILDER_PRIVATE_KEY",
+      message:
+        "Connect Builder.io before indexing a design system from Figma or code.",
+      builderConnectUrl: "/_agent-native/builder/connect",
+    });
+  }
+  return {
+    privateKey: credentials.privateKey,
+    publicKey: credentials.publicKey,
+    userId: credentials.userId ?? null,
+  };
+}
+
+function mimeTypeForFile(file: BuilderDesignSystemIndexFile): string {
+  if (file.mimeType?.trim()) return file.mimeType.trim();
+  const lower = file.name.toLowerCase();
+  if (lower.endsWith(".fig")) return "application/octet-stream";
+  if (lower.endsWith(".pdf")) return "application/pdf";
+  if (lower.endsWith(".md") || lower.endsWith(".markdown"))
+    return "text/markdown";
+  if (lower.endsWith(".json")) return "application/json";
+  if (lower.endsWith(".css")) return "text/css";
+  return "text/plain";
+}
+
+function makeBody(bytes: Uint8Array, mimeType: string): BodyInit {
+  return typeof Blob !== "undefined"
+    ? new Blob([bytes as unknown as BlobPart], { type: mimeType })
+    : (bytes as unknown as BodyInit);
+}
+
+async function fetchWithTimeout(
+  url: string | URL,
+  init: RequestInit,
+  timeoutMs = DEFAULT_TIMEOUT_MS,
+): Promise<Response> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fetch(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function parseErrorBody(response: Response): Promise<string> {
+  const text = await response.text().catch(() => "");
+  if (!text) return response.statusText || `HTTP ${response.status}`;
+  try {
+    const json = JSON.parse(text) as { error?: unknown };
+    if (typeof json.error === "string") return json.error;
+    if (json.error && typeof json.error === "object") {
+      return JSON.stringify(json.error).slice(0, 500);
+    }
+  } catch {}
+  return text.replace(/\s+/g, " ").trim().slice(0, 500);
+}
+
+async function assertOk(response: Response, label: string): Promise<void> {
+  if (response.ok) return;
+  throw new Error(
+    `${label} (${response.status}): ${await parseErrorBody(response)}`,
+  );
+}
+
+async function uploadToResumableUrl(
+  slot: { uploadUrl: string },
+  file: BuilderDesignSystemIndexFile,
+): Promise<void> {
+  const mimeType = mimeTypeForFile(file);
+  const bytes = file.data;
+  const start = await fetchWithTimeout(slot.uploadUrl, {
+    method: "POST",
+    headers: {
+      "x-goog-resumable": "start",
+      "x-goog-content-length-range": `0,${bytes.byteLength}`,
+      "Content-Type": mimeType,
+    },
+  });
+  await assertOk(start, "Builder design-system upload session failed");
+  const sessionUrl = start.headers.get("Location");
+  if (!sessionUrl) {
+    throw new Error("Builder design-system upload session returned no URL.");
+  }
+
+  const response = await fetchWithTimeout(sessionUrl, {
+    method: "PUT",
+    headers: {
+      "Content-Range": `bytes 0-${bytes.byteLength - 1}/${bytes.byteLength}`,
+      "Content-Type": mimeType,
+    },
+    body: makeBody(bytes, mimeType),
+  });
+  await assertOk(response, "Builder design-system file upload failed");
+}
+
+function nonEmptyFiles(
+  files: BuilderDesignSystemIndexFile[] | undefined,
+): BuilderDesignSystemIndexFile[] {
+  return (files ?? []).filter((file) => file.data.byteLength > 0);
+}
+
+export function builderDesignSystemUrl(designSystemId?: string | null): string {
+  const host = trimTrailingSlash(getBuilderAppHost());
+  return designSystemId
+    ? `${host}/app/design-system-intelligence/${encodeURIComponent(
+        designSystemId,
+      )}`
+    : `${host}/app/design-system-intelligence`;
+}
+
+export async function startBuilderDesignSystemIndex(
+  options: BuilderDesignSystemIndexOptions,
+): Promise<BuilderDesignSystemIndexResult> {
+  const files = nonEmptyFiles(options.files);
+  const description = options.description?.trim();
+  if (description) {
+    files.unshift({
+      name: "additional-context.txt",
+      data: new TextEncoder().encode(description),
+      mimeType: "text/plain",
+    });
+  }
+  if (
+    files.length === 0 &&
+    !options.githubRepoUrl &&
+    !options.connectedProjectId
+  ) {
+    throw new Error(
+      "Provide at least one .fig/code/text file or a GitHub repository URL to index with Builder.",
+    );
+  }
+
+  const credentials = await resolveBuilderDesignSystemCredentials();
+  let uploadTokens: string[] = [];
+  if (files.length > 0) {
+    const uploadStart = await fetchWithTimeout(
+      makeBuilderDesignSystemUrl("upload/start", credentials),
+      {
+        method: "POST",
+        headers: {
+          ...makeBuilderHeaders(credentials),
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          attachments: files.map((file) => ({
+            name: file.name,
+            mimetype: mimeTypeForFile(file),
+            declaredSize: file.data.byteLength,
+          })),
+        }),
+      },
+    );
+    await assertOk(uploadStart, "Builder design-system upload start failed");
+    const uploadJson = (await uploadStart.json()) as UploadStartResponse;
+    const slots = [...(uploadJson.uploads ?? [])].sort((a, b) => a.idx - b.idx);
+    if (slots.length !== files.length) {
+      throw new Error("Builder did not return upload slots for all files.");
+    }
+    for (let i = 0; i < slots.length; i++) {
+      if (slots[i].idx !== i) {
+        throw new Error(`Builder upload slot mismatch: expected ${i}.`);
+      }
+      await uploadToResumableUrl(slots[i], files[i]);
+    }
+    uploadTokens = slots.map((slot) => slot.uploadToken);
+  }
+
+  const generate = await fetchWithTimeout(
+    makeBuilderDesignSystemUrl("generate", credentials),
+    {
+      method: "POST",
+      headers: {
+        ...makeBuilderHeaders(credentials),
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        uploads: uploadTokens,
+        ...(options.projectName?.trim()
+          ? { projectName: options.projectName.trim() }
+          : {}),
+        ...(options.githubRepoUrl?.trim()
+          ? { githubRepoUrl: options.githubRepoUrl.trim() }
+          : {}),
+        ...(options.connectedProjectId?.trim()
+          ? { connectedProjectId: options.connectedProjectId.trim() }
+          : {}),
+        ...(options.selection ? { selection: options.selection } : {}),
+        ...(options.devToolsVersion?.trim()
+          ? { devToolsVersion: options.devToolsVersion.trim() }
+          : {}),
+      }),
+    },
+  );
+  await assertOk(generate, "Builder design-system indexing failed");
+  const generated = (await generate.json()) as GenerateResponse;
+  if (!generated.projectId || !generated.jobId || !generated.designSystemId) {
+    throw new Error(
+      "Builder design-system indexing returned an incomplete response.",
+    );
+  }
+
+  return {
+    ok: true,
+    source: "builder",
+    projectId: generated.projectId,
+    jobId: generated.jobId,
+    designSystemId: generated.designSystemId,
+    suggestedTitle: options.projectName?.trim() || null,
+    builderUrl: builderDesignSystemUrl(generated.designSystemId),
+    status: "in-progress",
+  };
+}

package/corpus/core/src/server/core-routes-plugin.ts

@@ -383,6 +383,31 @@ function parseBuilderCallbackBoolean(
 // inlines them in the DOM. Mirrors SAFE_DATA_IMAGE in sanitize-html.ts.
 export const AVATAR_RASTER_MIME = /^data:image\/(png|jpe?g|gif|webp);/i;
 
+export function resolveAvatarEmailParam(
+  pathname: string,
+  appBasePath = "",
+): string {
+  const base = appBasePath.replace(/\/+$/, "");
+  const avatarPaths = Array.from(
+    new Set([`${base}/_agent-native/avatar/`, "/_agent-native/avatar/"]),
+  );
+
+  for (const avatarPath of avatarPaths) {
+    const avatarIndex = pathname.indexOf(avatarPath);
+    if (avatarIndex >= 0) {
+      return pathname
+        .slice(avatarIndex + avatarPath.length)
+        .replace(/^\/+/, "")
+        .split("/")[0];
+    }
+  }
+
+  const firstSegment = pathname.replace(/^\/+/, "").split("/")[0] ?? "";
+  if (!firstSegment || firstSegment === "_agent-native") return "";
+  if (base && firstSegment === base.replace(/^\/+/, "")) return "";
+  return firstSegment;
+}
+
 async function detectUsageEngineName(
   event: H3Event,
   userEmail: string | undefined,
@@ -3010,9 +3035,10 @@ export function createCoreRoutesPlugin(
         `${P}/avatar`,
         defineEventHandler(async (event: H3Event) => {
           const method = getMethod(event);
-          const emailParam = (event.url?.pathname || "")
-            .replace(/^\/+/, "")
-            .split("/")[0];
+          const emailParam = resolveAvatarEmailParam(
+            event.url?.pathname || "",
+            getConfiguredAppBasePath(),
+          );
 
           if (method === "GET") {
             if (!emailParam) {

package/corpus/core/src/server/index.ts

@@ -386,6 +386,14 @@ export {
   deleteBuilderCredentials,
   resolveSecret,
 } from "./credential-provider.js";
+export {
+  builderDesignSystemUrl,
+  getBuilderDesignSystemsBaseUrl,
+  startBuilderDesignSystemIndex,
+  type BuilderDesignSystemIndexFile,
+  type BuilderDesignSystemIndexOptions,
+  type BuilderDesignSystemIndexResult,
+} from "./builder-design-systems.js";
 export {
   getBuilderBranchProjectId,
   isBuilderBranchingEnabled,

package/corpus/core/src/server/transcribe-voice.ts

@@ -30,6 +30,13 @@ import { createBuilderEngine } from "../agent/engine/builder-engine.js";
 import { appStateGet } from "../application-state/store.js";
 import { getOrgContext } from "../org/context.js";
 import { transcribeWithBuilder } from "../transcription/builder-transcription.js";
+import {
+  applyVoiceContextReplacements,
+  buildVoiceGuidanceBlock,
+  parseVoiceContextPack,
+  voiceContextTermsOnly,
+  type VoiceContextPack,
+} from "../voice/index.js";
 import { getSession } from "./auth.js";
 import {
   resolveHasBuilderPrivateKey,
@@ -146,6 +153,20 @@ export function createTranscribeVoiceHandler() {
           Buffer.from(instructionsPart.data).toString("utf8"),
         )
       : undefined;
+    const voiceContextPart = parts?.find(
+      (p) => p.name === "voiceContext" || p.name === "contextPack",
+    );
+    const voiceContext = voiceContextPart?.data
+      ? parseVoiceContextPack(
+          Buffer.from(voiceContextPart.data).toString("utf8"),
+        )
+      : undefined;
+    const voiceGuidance = buildVoiceGuidanceBlock({
+      instructions,
+      contextPack: voiceContextTermsOnly(voiceContext),
+    });
+    const applyVoiceContext = (value: string) =>
+      applyVoiceContextReplacements(value, voiceContext).trim();
 
     // Resolve provider preference. Per-request "provider" form field takes
     // precedence (the desktop client sends it on every dictation press),
@@ -220,7 +241,7 @@ export function createTranscribeVoiceHandler() {
     // does not want audio uploaded to any external provider. The client
     // shouldn't hit this endpoint when "browser" is selected; this is a
     // defense-in-depth refusal.
-    if (providerPref === "browser") {
+    if (providerPref === "browser" && !transcriptText) {
       setResponseStatus(event, 400);
       return {
         error:
@@ -238,7 +259,8 @@ export function createTranscribeVoiceHandler() {
       return await cleanupTranscriptText({
         event,
         text: transcriptText,
-        instructions,
+        instructions: voiceGuidance,
+        contextPack: voiceContext,
         providerPref,
         hasBuilderPrivateKey,
         withRequestContext,
@@ -282,9 +304,9 @@ export function createTranscribeVoiceHandler() {
           mimeType: mime,
           apiKey: geminiKey,
           language: language || undefined,
-          instructions,
+          instructions: voiceGuidance,
         });
-        const trimmed = text.trim();
+        const trimmed = applyVoiceContext(text);
         if (!trimmed) {
           setResponseStatus(event, 502);
           return { error: "Gemini returned an empty transcript." };
@@ -318,9 +340,9 @@ export function createTranscribeVoiceHandler() {
               ? BUILDER_GEMINI_TRANSCRIPTION_MODEL
               : undefined,
           language: language || undefined,
-          instructions,
+          instructions: voiceGuidance,
         });
-        return { text: (result.text ?? "").trim() };
+        return { text: applyVoiceContext(result.text ?? "") };
       } catch (err) {
         const message = (err as Error)?.message ?? String(err);
         if (message.includes("credits exhausted")) {
@@ -352,7 +374,8 @@ export function createTranscribeVoiceHandler() {
         audioBytes,
         mime,
         language,
-        instructions,
+        instructions: voiceGuidance,
+        contextPack: voiceContext,
       });
     }
 
@@ -368,9 +391,9 @@ export function createTranscribeVoiceHandler() {
           mimeType: mime,
           model: BUILDER_GEMINI_TRANSCRIPTION_MODEL,
           language: language || undefined,
-          instructions,
+          instructions: voiceGuidance,
         });
-        return { text: (result.text ?? "").trim() };
+        return { text: applyVoiceContext(result.text ?? "") };
       } catch (err) {
         const message = (err as Error)?.message ?? String(err);
         // Surface 402 (credits exhausted) as a 402 so the client can show
@@ -395,9 +418,9 @@ export function createTranscribeVoiceHandler() {
             mimeType: mime,
             apiKey: geminiKey,
             language: language || undefined,
-            instructions,
+            instructions: voiceGuidance,
           });
-          const trimmed = text.trim();
+          const trimmed = applyVoiceContext(text);
           if (trimmed) {
             console.log(`[transcribe-voice] Gemini → ${trimmed.length} chars`);
             return { text: trimmed };
@@ -465,7 +488,8 @@ export function createTranscribeVoiceHandler() {
       audioBytes,
       mime,
       language,
-      instructions,
+      instructions: voiceGuidance,
+      contextPack: voiceContext,
     });
   });
 }
@@ -484,6 +508,7 @@ async function callWhisperCompat({
   mime,
   language,
   instructions,
+  contextPack,
 }: {
   event: H3Event;
   provider: {
@@ -496,6 +521,7 @@ async function callWhisperCompat({
   mime: string;
   language?: string;
   instructions?: string;
+  contextPack?: VoiceContextPack;
 }): Promise<{ text: string } | { error: string }> {
   const ext = pickExtension(mime);
   const filename = `composer-voice.${ext}`;
@@ -531,7 +557,12 @@ async function callWhisperCompat({
       };
     }
     const data = (await res.json()) as { text?: string };
-    return { text: (data.text ?? "").trim() };
+    return {
+      text: applyVoiceContextReplacements(
+        (data.text ?? "").trim(),
+        contextPack,
+      ).trim(),
+    };
   } catch (err) {
     setResponseStatus(event, 502);
     return {
@@ -549,6 +580,7 @@ async function cleanupTranscriptText({
   event,
   text,
   instructions,
+  contextPack,
   providerPref,
   hasBuilderPrivateKey,
   withRequestContext,
@@ -557,6 +589,7 @@ async function cleanupTranscriptText({
   event: H3Event;
   text: string;
   instructions?: string;
+  contextPack?: VoiceContextPack;
   providerPref?: string;
   hasBuilderPrivateKey: () => Promise<boolean>;
   withRequestContext: <T>(fn: () => Promise<T>) => Promise<T>;
@@ -564,9 +597,11 @@ async function cleanupTranscriptText({
 }): Promise<{ text: string } | { error: string }> {
   const original = text.trim();
   if (!original) return { text: "" };
+  const finalizeText = (value: string) =>
+    applyVoiceContextReplacements(value || original, contextPack).trim();
 
   if (providerPref === "browser") {
-    return { text: original };
+    return { text: finalizeText(original) };
   }
 
   if (providerPref === "builder" || providerPref === "builder-gemini") {
@@ -581,7 +616,7 @@ async function cleanupTranscriptText({
       const cleaned = await withRequestContext(() =>
         cleanupWithBuilder({ text: original, instructions }),
       );
-      return { text: cleaned || original };
+      return { text: finalizeText(cleaned || original) };
     } catch (err) {
       setResponseStatus(event, 502);
       return {
@@ -605,7 +640,7 @@ async function cleanupTranscriptText({
         apiKey: geminiKey,
         instructions,
       });
-      return { text: cleaned || original };
+      return { text: finalizeText(cleaned || original) };
     } catch (err) {
       setResponseStatus(event, 502);
       return {
@@ -631,7 +666,7 @@ async function cleanupTranscriptText({
         apiKey,
         instructions,
       });
-      return { text: cleaned || original };
+      return { text: finalizeText(cleaned || original) };
     } catch (err) {
       setResponseStatus(event, 502);
       return {
@@ -645,7 +680,7 @@ async function cleanupTranscriptText({
       const cleaned = await withRequestContext(() =>
         cleanupWithBuilder({ text: original, instructions }),
       );
-      if (cleaned) return { text: cleaned };
+      if (cleaned) return { text: finalizeText(cleaned) };
     } catch {
       // Fall through to BYOK providers, then raw text.
     }
@@ -659,7 +694,7 @@ async function cleanupTranscriptText({
         apiKey: geminiKey,
         instructions,
       });
-      if (cleaned) return { text: cleaned };
+      if (cleaned) return { text: finalizeText(cleaned) };
     } catch {
       // Fall through.
     }
@@ -674,7 +709,7 @@ async function cleanupTranscriptText({
         apiKey: groqKey,
         instructions,
       });
-      if (cleaned) return { text: cleaned };
+      if (cleaned) return { text: finalizeText(cleaned) };
     } catch {
       // Fall through.
     }
@@ -689,13 +724,13 @@ async function cleanupTranscriptText({
         apiKey: openaiKey,
         instructions,
       });
-      if (cleaned) return { text: cleaned };
+      if (cleaned) return { text: finalizeText(cleaned) };
     } catch {
       // Fall through.
     }
   }
 
-  return { text: original };
+  return { text: finalizeText(original) };
 }
 
 async function cleanupWithBuilder({

package/corpus/core/src/shared/mcp-embed-headers.ts

@@ -87,6 +87,27 @@ export function isAgentNativeFirstPartyAppOrigin(
   }
 }
 
+export function isBuilderIoEmbedOrigin(
+  origin: string | null | undefined,
+): boolean {
+  if (!origin) return false;
+  try {
+    const url = new URL(origin);
+    const hostname = url.hostname.toLowerCase();
+    return (
+      url.protocol === "https:" &&
+      !url.username &&
+      !url.password &&
+      (hostname === "builder.io" ||
+        hostname.endsWith(".builder.io") ||
+        hostname === "builder.my" ||
+        hostname.endsWith(".builder.my"))
+    );
+  } catch {
+    return false;
+  }
+}
+
 export function isMcpEmbedCorsOrigin(
   origin: string | null | undefined,
 ): boolean {
@@ -96,7 +117,8 @@ export function isMcpEmbedCorsOrigin(
     isClaudeMcpContentOrigin(origin) ||
     isChatGptMcpSandboxOrigin(origin) ||
     isMcpProductHostOrigin(origin) ||
-    isAgentNativeFirstPartyAppOrigin(origin)
+    isAgentNativeFirstPartyAppOrigin(origin) ||
+    isBuilderIoEmbedOrigin(origin)
   );
 }