@agent-native/toolkit 0.0.0-beta-20260819143124

package/src/composer/VoiceButton.spec.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  isRealtimeVoiceSetupRequired,
+  normalizeVoiceInputPreference,
+} from "./VoiceButton.js";
+
+describe("normalizeVoiceInputPreference", () => {
+  it("accepts only persisted realtime and dictation choices", () => {
+    expect(normalizeVoiceInputPreference({ mode: "realtime" })).toBe(
+      "realtime",
+    );
+    expect(normalizeVoiceInputPreference({ mode: "dictation" })).toBe(
+      "dictation",
+    );
+    expect(normalizeVoiceInputPreference({ mode: "voice" })).toBeNull();
+    expect(normalizeVoiceInputPreference(null)).toBeNull();
+  });
+});
+
+describe("isRealtimeVoiceSetupRequired", () => {
+  it("waits for the voice-specific provider status before prompting setup", () => {
+    expect(isRealtimeVoiceSetupRequired(null, false)).toBe(false);
+  });
+
+  it("does not wait on the redundant Builder status after voice status resolves", () => {
+    expect(
+      isRealtimeVoiceSetupRequired({ builder: false, openai: false }, null),
+    ).toBe(true);
+  });
+
+  it("accepts either managed Builder voice or an OpenAI key", () => {
+    expect(
+      isRealtimeVoiceSetupRequired({ builder: true, openai: false }, false),
+    ).toBe(false);
+    expect(
+      isRealtimeVoiceSetupRequired({ builder: false, openai: true }, false),
+    ).toBe(false);
+    expect(
+      isRealtimeVoiceSetupRequired({ builder: false, openai: false }, true),
+    ).toBe(false);
+  });
+
+  it("prompts setup only when neither realtime provider is configured", () => {
+    expect(
+      isRealtimeVoiceSetupRequired({ builder: false, openai: false }, false),
+    ).toBe(true);
+  });
+});

package/src/composer/VoiceButton.tsx

@@ -0,0 +1,329 @@
+/**
+ * Voice dictation button + recording overlay for the agent composer.
+ *
+ * UX mirrors Lovable: click-to-toggle record, a live amplitude bar and
+ * MM:SS timer replace the editor area while recording, and a cancel X
+ * discards without transcribing. The mic is always visible alongside the
+ * send button (Cursor replaces send with mic; their users complain — we
+ * don't copy that).
+ */
+
+import {
+  IconMicrophone,
+  IconPlayerStopFilled,
+  IconLoader2,
+  IconX,
+} from "@tabler/icons-react";
+import { useCallback, useEffect, useState } from "react";
+
+import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip.js";
+import { RealtimeVoiceModeEntry } from "./RealtimeVoiceMode.js";
+import type { RealtimeVoiceInputMode } from "./RealtimeVoiceMode.js";
+import { useComposerRuntimeAdapters } from "./runtime-adapters.js";
+import {
+  useRealtimeVoiceModeCopy,
+  useRealtimeVoiceModeOptional,
+} from "./useRealtimeVoiceMode.js";
+import type { VoiceDictationApi } from "./useVoiceDictation.js";
+
+const VOICE_INPUT_PREFERENCE_KEY = "voice-input-preference";
+
+export function normalizeVoiceInputPreference(
+  value: unknown,
+): RealtimeVoiceInputMode | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const mode = (value as { mode?: unknown }).mode;
+  return mode === "realtime" || mode === "dictation" ? mode : null;
+}
+
+function openOpenAiKeySettings(): void {
+  window.location.hash = "#secrets:OPENAI_API_KEY";
+  window.dispatchEvent(new Event("agent-panel:open"));
+  window.dispatchEvent(
+    new CustomEvent("agent-panel:open-settings", {
+      detail: { section: "secrets" },
+    }),
+  );
+}
+
+export interface VoiceButtonProps {
+  voice: VoiceDictationApi;
+  isMac: boolean;
+  disabled?: boolean;
+}
+
+export function isRealtimeVoiceSetupRequired(
+  status: { builder?: boolean; openai?: boolean } | null,
+  builderConfigured: boolean | null,
+): boolean {
+  return (
+    status !== null &&
+    !status.builder &&
+    !status.openai &&
+    builderConfigured !== true
+  );
+}
+
+export function VoiceButton({ voice, isMac, disabled }: VoiceButtonProps) {
+  const adapters = useComposerRuntimeAdapters();
+  const t = adapters.translate!;
+  const { state, start, stop, supported } = voice;
+  const realtimeVoice = useRealtimeVoiceModeOptional();
+  const realtimeCopy = useRealtimeVoiceModeCopy();
+  const voiceProviders = adapters.voice!.useProviderStatus!();
+  const builderConnect = adapters.builder!.useConnectFlow!({
+    trackingSource: "realtime_voice",
+    trackingFlow: "voice_transcription",
+    onConnected: () => voiceProviders.refresh(),
+  });
+  const [inputPreference, setInputPreference] = useState<{
+    ready: boolean;
+    mode: RealtimeVoiceInputMode | null;
+  }>({ ready: false, mode: null });
+
+  useEffect(() => {
+    let cancelled = false;
+    void Promise.resolve(
+      adapters.voice!.readAppState!(VOICE_INPUT_PREFERENCE_KEY),
+    )
+      .then((value) => {
+        if (!cancelled) {
+          setInputPreference({
+            ready: true,
+            mode: normalizeVoiceInputPreference(value),
+          });
+        }
+      })
+      .catch(() => {
+        if (!cancelled) setInputPreference({ ready: true, mode: null });
+      });
+    return () => {
+      cancelled = true;
+    };
+  }, [adapters]);
+
+  const rememberInputPreference = useCallback(
+    (mode: RealtimeVoiceInputMode) => {
+      setInputPreference({ ready: true, mode });
+      void Promise.resolve(
+        adapters.voice!.setAppState!(VOICE_INPUT_PREFERENCE_KEY, { mode }),
+      ).catch(() => undefined);
+    },
+    [adapters],
+  );
+
+  if (!supported) return null;
+
+  const recording = state === "recording" || state === "starting";
+  const transcribing = state === "transcribing";
+
+  if (realtimeVoice?.active && !recording && !transcribing) return null;
+
+  if (realtimeVoice && !recording && !transcribing) {
+    return (
+      <RealtimeVoiceModeEntry
+        copy={realtimeCopy}
+        disabled={disabled}
+        providerStatusPending={voiceProviders.status === null}
+        setupRequired={isRealtimeVoiceSetupRequired(
+          voiceProviders.status,
+          builderConnect.statusResolved ? builderConnect.configured : null,
+        )}
+        openAiConfigured={voiceProviders.status?.openai === true}
+        connectingBuilder={builderConnect.connecting}
+        onConnectBuilder={builderConnect.start}
+        onUseOpenAiKey={() => {
+          if (voiceProviders.status?.openai) void realtimeVoice.start();
+          else openOpenAiKeySettings();
+        }}
+        onStartVoiceMode={() => void realtimeVoice.start()}
+        onKeepDictating={() => void start()}
+        preferredMode={inputPreference.ready ? inputPreference.mode : null}
+        onRememberPreference={rememberInputPreference}
+      />
+    );
+  }
+
+  const label = recording
+    ? t("agentChat.voice.dictation.stopRecording", {
+        defaultValue: "Stop recording",
+      })
+    : transcribing
+      ? t("agentChat.voice.dictation.transcribing", {
+          defaultValue: "Transcribing…",
+        })
+      : t("agentChat.voice.dictation.start", {
+          shortcut: isMac ? "⌘⇧M" : "Ctrl+Shift+M",
+          defaultValue: `Dictate (${isMac ? "⌘⇧M" : "Ctrl+Shift+M"})`,
+        });
+
+  const onClick = () => {
+    if (recording) stop();
+    else if (!transcribing) void start();
+  };
+
+  return (
+    <Tooltip>
+      <TooltipTrigger asChild>
+        <button
+          type="button"
+          onClick={onClick}
+          disabled={disabled || transcribing}
+          aria-label={label}
+          aria-pressed={recording}
+          className={`shrink-0 flex h-7 w-7 items-center justify-center rounded-md transition-colors duration-150 disabled:opacity-30 disabled:cursor-not-allowed ${
+            recording
+              ? "text-[#00B5FF] bg-[#00B5FF]/10 hover:bg-[#00B5FF]/20"
+              : "text-muted-foreground hover:text-foreground hover:bg-accent/50"
+          }`}
+        >
+          {transcribing ? (
+            <IconLoader2 className="h-4 w-4 animate-spin" />
+          ) : recording ? (
+            <IconPlayerStopFilled className="h-3.5 w-3.5" />
+          ) : (
+            <IconMicrophone className="h-4 w-4" />
+          )}
+        </button>
+      </TooltipTrigger>
+      <TooltipContent>{label}</TooltipContent>
+    </Tooltip>
+  );
+}
+
+export interface VoiceRecordingOverlayProps {
+  voice: VoiceDictationApi;
+}
+
+export function VoiceRecordingOverlay({ voice }: VoiceRecordingOverlayProps) {
+  const { state, amplitude, durationMs, errorMessage, cancel } = voice;
+  const { dismissError, start } = voice;
+  const t = useComposerRuntimeAdapters().translate!;
+
+  if (state === "error" && errorMessage) {
+    return (
+      <div
+        role="alert"
+        className="mx-2 mt-1 flex items-start gap-2 rounded-md border border-red-500/40 bg-red-500/10 px-2 py-1.5 text-[11px] text-red-500"
+      >
+        <span className="flex-1 min-w-0">{errorMessage}</span>
+        <Tooltip>
+          <TooltipTrigger asChild>
+            <button
+              type="button"
+              onClick={() => {
+                dismissError();
+                void start();
+              }}
+              className="shrink-0 cursor-pointer rounded px-1.5 py-0.5 text-[11px] font-medium text-red-500 hover:bg-red-500/20"
+              aria-label={t("agentChat.common.retry", {
+                defaultValue: "Try again",
+              })}
+            >
+              {t("agentChat.common.retry", { defaultValue: "Try again" })}
+            </button>
+          </TooltipTrigger>
+          <TooltipContent>
+            {t("agentChat.common.retry", { defaultValue: "Try again" })}
+          </TooltipContent>
+        </Tooltip>
+        <Tooltip>
+          <TooltipTrigger asChild>
+            <button
+              type="button"
+              onClick={dismissError}
+              className="shrink-0 flex h-4 w-4 cursor-pointer items-center justify-center rounded text-red-500 hover:bg-red-500/20"
+              aria-label={t("agentChat.common.dismiss", {
+                defaultValue: "Dismiss",
+              })}
+            >
+              <IconX className="h-3 w-3" />
+            </button>
+          </TooltipTrigger>
+          <TooltipContent>
+            {t("agentChat.common.dismiss", { defaultValue: "Dismiss" })}
+          </TooltipContent>
+        </Tooltip>
+      </div>
+    );
+  }
+
+  if (state !== "recording" && state !== "starting" && state !== "transcribing")
+    return null;
+
+  return (
+    <div
+      className="flex items-center gap-2 mx-2 mt-2 mb-1 h-[2rem] rounded-md border border-[#00B5FF]/40 bg-[#00B5FF]/10 px-2"
+      aria-live="polite"
+    >
+      <Tooltip>
+        <TooltipTrigger asChild>
+          <button
+            type="button"
+            onClick={cancel}
+            className="shrink-0 flex h-5 w-5 items-center justify-center rounded text-muted-foreground hover:text-foreground hover:bg-accent/40"
+            aria-label={t("agentChat.voice.dictation.cancelRecording", {
+              defaultValue: "Cancel recording",
+            })}
+          >
+            <IconX className="h-3 w-3" />
+          </button>
+        </TooltipTrigger>
+        <TooltipContent>
+          {t("agentChat.voice.dictation.cancel", {
+            defaultValue: "Cancel (Esc)",
+          })}
+        </TooltipContent>
+      </Tooltip>
+
+      <div className="flex-1 flex items-center gap-[2px] min-w-0 h-4">
+        {state === "transcribing" ? (
+          <span className="text-[11px] text-muted-foreground">
+            {t("agentChat.voice.dictation.transcribing", {
+              defaultValue: "Transcribing…",
+            })}
+          </span>
+        ) : (
+          <AmplitudeBars amplitude={amplitude} />
+        )}
+      </div>
+
+      <span className="shrink-0 text-[11px] font-medium tabular-nums text-muted-foreground">
+        {state === "transcribing" ? (
+          <IconLoader2 className="h-3 w-3 animate-spin" />
+        ) : (
+          formatDuration(durationMs)
+        )}
+      </span>
+    </div>
+  );
+}
+
+const BAR_COUNT = 24;
+
+function AmplitudeBars({ amplitude }: { amplitude: number }) {
+  // Render a symmetric meter — the middle bars peak first so the visual
+  // matches what voice input looks like in Lovable / iOS dictation.
+  const bars = [];
+  for (let i = 0; i < BAR_COUNT; i++) {
+    const centerDistance =
+      Math.abs(i - (BAR_COUNT - 1) / 2) / ((BAR_COUNT - 1) / 2);
+    const heightPct =
+      Math.max(0.1, amplitude * (1 - centerDistance * 0.6)) * 100;
+    bars.push(
+      <span
+        key={i}
+        className="flex-1 rounded-full bg-[#00B5FF]"
+        style={{ height: `${heightPct}%`, minHeight: 2 }}
+      />,
+    );
+  }
+  return <>{bars}</>;
+}
+
+function formatDuration(ms: number): string {
+  const total = Math.floor(ms / 1000);
+  const m = Math.floor(total / 60);
+  const s = total % 60;
+  return `${m}:${s.toString().padStart(2, "0")}`;
+}

package/src/composer/asset-picker-url.spec.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  isExternalAssetPickerUrl,
+  standaloneAssetPickerUrl,
+} from "./asset-picker-url.js";
+
+describe("asset picker auth handoff", () => {
+  it("treats the hosted Assets picker as external to the host app", () => {
+    expect(
+      isExternalAssetPickerUrl(
+        "https://assets.agent-native.com/picker",
+        "https://clips.agent-native.com",
+      ),
+    ).toBe(true);
+  });
+
+  it("keeps same-origin pickers eligible for inline rendering", () => {
+    expect(
+      isExternalAssetPickerUrl("/picker", "https://clips.agent-native.com"),
+    ).toBe(false);
+  });
+
+  it("removes iframe flags from the top-level fallback URL", () => {
+    expect(
+      standaloneAssetPickerUrl(
+        "https://assets.agent-native.com/picker?embedded=1&__an_embed_token=fake-token",
+      ),
+    ).toBe("https://assets.agent-native.com/picker?mediaType=image");
+  });
+
+  it("adds a nonce-bound exact-origin callback to the top-level URL", () => {
+    expect(
+      standaloneAssetPickerUrl(
+        "https://assets.agent-native.com/picker?embedded=1",
+        "https://clips.agent-native.com",
+        {
+          handoffId: "handoff-123",
+          returnOrigin: "https://clips.agent-native.com/chat",
+        },
+      ),
+    ).toBe(
+      "https://assets.agent-native.com/picker?mediaType=image&__an_asset_picker_handoff=handoff-123&__an_asset_picker_return_origin=https%3A%2F%2Fclips.agent-native.com",
+    );
+  });
+
+  it("omits an invalid callback target", () => {
+    expect(
+      standaloneAssetPickerUrl(
+        "https://assets.agent-native.com/picker",
+        "https://clips.agent-native.com",
+        { handoffId: "handoff-123", returnOrigin: "javascript:alert(1)" },
+      ),
+    ).toBe("https://assets.agent-native.com/picker?mediaType=image");
+  });
+});

package/src/composer/asset-picker-url.ts

@@ -0,0 +1,80 @@
+const FALLBACK_BASE_URL = "http://agent-native.invalid";
+
+export const ASSET_PICKER_HANDOFF_PARAM = "__an_asset_picker_handoff";
+export const ASSET_PICKER_RETURN_ORIGIN_PARAM =
+  "__an_asset_picker_return_origin";
+
+export interface StandaloneAssetPickerOptions {
+  handoffId?: string;
+  returnOrigin?: string;
+}
+
+export function createAssetPickerHandoffId(): string {
+  if (typeof crypto !== "undefined" && crypto.randomUUID) {
+    return crypto.randomUUID();
+  }
+  if (typeof crypto !== "undefined" && crypto.getRandomValues) {
+    const bytes = crypto.getRandomValues(new Uint8Array(16));
+    return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
+      "",
+    );
+  }
+  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
+}
+
+/**
+ * Cross-origin Assets pages cannot inherit the host app's authenticated
+ * session inside an iframe. Treat those URLs as link-out targets so provider
+ * sign-in runs in a normal top-level browser context.
+ */
+export function isExternalAssetPickerUrl(
+  value: string,
+  currentOrigin: string,
+): boolean {
+  try {
+    return new URL(value, currentOrigin).origin !== currentOrigin;
+  } catch {
+    // A malformed configured URL should fail closed instead of being loaded in
+    // an iframe with an unknown auth boundary.
+    return true;
+  }
+}
+
+/** Build a top-level picker URL without iframe-only auth flags. */
+export function standaloneAssetPickerUrl(
+  value: string,
+  baseUrl = FALLBACK_BASE_URL,
+  options: StandaloneAssetPickerOptions = {},
+): string {
+  try {
+    const parsed = new URL(value, baseUrl);
+    parsed.searchParams.delete("embedded");
+    parsed.searchParams.delete("__an_embed_token");
+    parsed.searchParams.set("mediaType", "image");
+    let returnOrigin: string | null = null;
+    if (options.returnOrigin) {
+      try {
+        const parsedOrigin = new URL(options.returnOrigin);
+        if (
+          parsedOrigin.protocol === "http:" ||
+          parsedOrigin.protocol === "https:"
+        ) {
+          returnOrigin = parsedOrigin.origin;
+        }
+      } catch {
+        // Omit an invalid return target rather than emitting an unverified
+        // cross-origin callback URL.
+      }
+    }
+    if (options.handoffId && returnOrigin) {
+      parsed.searchParams.set(ASSET_PICKER_HANDOFF_PARAM, options.handoffId);
+      parsed.searchParams.set(ASSET_PICKER_RETURN_ORIGIN_PARAM, returnOrigin);
+    } else {
+      parsed.searchParams.delete(ASSET_PICKER_HANDOFF_PARAM);
+      parsed.searchParams.delete(ASSET_PICKER_RETURN_ORIGIN_PARAM);
+    }
+    return parsed.toString();
+  } catch {
+    return value;
+  }
+}

package/src/composer/attachment-accept.spec.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  CHAT_DOCUMENT_ATTACHMENT_ACCEPT,
+  PROMPT_DOCUMENT_ATTACHMENT_ACCEPT,
+} from "./attachment-accept.js";
+
+describe("attachment accept lists", () => {
+  it("keeps SVGs out of standalone prompt composer document uploads", () => {
+    const accept = PROMPT_DOCUMENT_ATTACHMENT_ACCEPT.split(",");
+
+    expect(accept).not.toContain("image/svg+xml");
+    expect(accept).not.toContain(".svg");
+  });
+
+  it("allows SVGs in the main chat document upload path", () => {
+    const accept = CHAT_DOCUMENT_ATTACHMENT_ACCEPT.split(",");
+
+    expect(accept).toContain("image/svg+xml");
+    expect(accept).toContain(".svg");
+  });
+
+  it("allows Excel workbooks in both composer paths", () => {
+    for (const accept of [
+      PROMPT_DOCUMENT_ATTACHMENT_ACCEPT,
+      CHAT_DOCUMENT_ATTACHMENT_ACCEPT,
+    ]) {
+      expect(accept.split(",")).toContain(".xlsx");
+      expect(accept.split(",")).toContain(".xls");
+    }
+  });
+});

package/src/composer/attachment-accept.ts

@@ -0,0 +1,75 @@
+import { SimpleTextAttachmentAdapter } from "@assistant-ui/react";
+
+const BASE_DOCUMENT_ATTACHMENT_ACCEPT = [
+  "application/pdf",
+  "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  "application/vnd.ms-excel",
+  ".pdf",
+  ".pptx",
+  ".docx",
+  ".xlsx",
+  ".xls",
+];
+
+export const PROMPT_DOCUMENT_ATTACHMENT_ACCEPT =
+  BASE_DOCUMENT_ATTACHMENT_ACCEPT.join(",");
+
+export const CHAT_DOCUMENT_ATTACHMENT_ACCEPT = [
+  ...BASE_DOCUMENT_ATTACHMENT_ACCEPT,
+  "image/svg+xml",
+  ".svg",
+].join(",");
+
+export const IMAGE_ATTACHMENT_ACCEPT = [
+  "image/jpeg",
+  "image/jpg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+  "image/heic",
+  "image/heif",
+  "image/avif",
+  "image/bmp",
+  "image/tiff",
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".webp",
+  ".heic",
+  ".heif",
+  ".avif",
+  ".bmp",
+  ".tif",
+  ".tiff",
+].join(",");
+
+export const TEXT_ATTACHMENT_ACCEPT = [
+  "text/plain",
+  "text/html",
+  "text/markdown",
+  "text/csv",
+  "text/xml",
+  "text/json",
+  "text/css",
+  "text/yaml",
+  "application/json",
+  "application/x-yaml",
+  ".txt",
+  ".md",
+  ".markdown",
+  ".csv",
+  ".json",
+  ".html",
+  ".htm",
+  ".css",
+  ".xml",
+  ".yaml",
+  ".yml",
+].join(",");
+
+export class TextAttachmentAdapter extends SimpleTextAttachmentAdapter {
+  public accept = TEXT_ATTACHMENT_ACCEPT;
+}

package/src/composer/draft-key.spec.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+
+import { getComposerDraftKey } from "./draft-key.js";
+
+describe("getComposerDraftKey", () => {
+  it("uses the legacy key when no scope is available", () => {
+    expect(getComposerDraftKey()).toBe("an-composer-draft");
+    expect(getComposerDraftKey("   ")).toBe("an-composer-draft");
+  });
+
+  it("scopes drafts by thread or tab id", () => {
+    expect(getComposerDraftKey("thread-qa")).toBe(
+      "an-composer-draft:thread-qa",
+    );
+    expect(getComposerDraftKey("tab with spaces")).toBe(
+      "an-composer-draft:tab%20with%20spaces",
+    );
+  });
+});

package/src/composer/draft-key.ts

@@ -0,0 +1,7 @@
+const BASE_COMPOSER_DRAFT_KEY = "an-composer-draft";
+
+export function getComposerDraftKey(scope?: string | null): string {
+  const trimmed = scope?.trim();
+  if (!trimmed) return BASE_COMPOSER_DRAFT_KEY;
+  return `${BASE_COMPOSER_DRAFT_KEY}:${encodeURIComponent(trimmed)}`;
+}