@agent-native/core 0.111.4 → 0.113.0

package/corpus/toolkit/src/composer/pasted-text.ts

@@ -0,0 +1,160 @@
+// Page-sized pastes turn into a `Pasted text` attachment chip instead of being
+// dumped into the editor. Short paragraphs and everyday lists should stay
+// inline so the composer still feels like a normal text field.
+const PASTED_TEXT_MIN_CHARS = 3200;
+const PASTED_TEXT_MIN_LINES = 24;
+
+const PASTED_TEXT_FILENAME_PREFIX = "pasted-text-";
+
+// A copied HTML document/source is recognizable from its markup: a closing tag
+// (`</div>`), a doctype, or a common structural/element tag. We key off the
+// *content* the user actually pasted, not the clipboard's `text/html` flavor —
+// editors (VS Code) and rich-text apps (Google Docs) populate `text/html` with
+// syntax-highlight spans or formatting wrappers even when the real content is
+// plain code/prose, so trusting `text/html` blindly would mangle those pastes.
+const HTML_SOURCE_SIGNAL =
+  /<!doctype\s+html|<html[\s>]|<\/[a-z][a-z0-9-]*\s*>|<(?:body|head|div|span|section|main|header|footer|nav|article|aside|ul|ol|li|table|thead|tbody|tr|td|th|h[1-6]|p|a|img|button|input|textarea|select|form|label|script|style|link|meta|svg|canvas|template)\b/i;
+
+// A real HTML *document* announces itself with a doctype / html / head / body.
+// When one of these is present we keep the HTML classification even if an inline
+// <script> contains JS keywords — it's a genuine page.
+const HTML_DOCUMENT_SIGNAL =
+  /<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>]/i;
+
+// JSX/TSX source contains the same `</div>`/`<span>` tags as an HTML document,
+// so the HTML tag signal alone misfiles a pasted React/TS component (even a bare
+// function component) as an `.html` artifact — the agent then mishandles it as a
+// hostable document instead of source. These markers appear in JS/TS/JSX source
+// but not in a plain HTML *fragment*: `className=` (JSX uses it; HTML uses
+// `class=`), ES module import/export, arrow functions, TS type/React annotations,
+// React hooks, and the basic JS declaration/return keywords that make up a
+// component body.
+const CODE_SOURCE_SIGNAL =
+  /\bclassName=|\bimport\b|\bexport\b|=>|:\s*React\.|\buse[A-Z]\w*\(|\b(?:function|const|let|var|return|class|interface|type|enum)\b/;
+
+function looksLikeHtml(value: string): boolean {
+  if (!value || !HTML_SOURCE_SIGNAL.test(value)) return false;
+  // A self-announcing HTML document stays HTML even if it embeds a <script>.
+  if (HTML_DOCUMENT_SIGNAL.test(value)) return true;
+  // Otherwise it's a bare fragment — if it carries JS/TS/JSX code signals,
+  // treat it as source code, not an HTML attachment.
+  if (CODE_SOURCE_SIGNAL.test(value)) return false;
+  return true;
+}
+
+/** The clipboard flavors we care about for a paste. */
+export interface ClipboardPaste {
+  /** `text/plain` flavor — used for size heuristics and as the default body. */
+  text: string;
+  /** `text/html` flavor when the source provided one. */
+  html?: string;
+}
+
+/** Read the relevant clipboard flavors from a paste/drop DataTransfer. */
+export function readClipboardPaste(
+  data: { getData(type: string): string } | null | undefined,
+): ClipboardPaste {
+  const text = data?.getData("text/plain") ?? "";
+  const html = data?.getData("text/html") ?? "";
+  return { text, html: html.trim() ? html : undefined };
+}
+
+interface SelectedPasteBody {
+  body: string;
+  ext: "html" | "txt";
+  type: "text/html" | "text/plain";
+}
+
+// Decide what to actually store for a paste. Preserving HTML markup means a
+// pasted HTML document behaves exactly like uploading that .html file: the agent
+// recognizes it as a hostable artifact and reads it verbatim via
+// `contentFromAttachment` instead of retyping it inline (which cuts off
+// mid-stream on large files and triggers a continuation loop / "spin").
+function selectPasteBody(paste: ClipboardPaste): SelectedPasteBody {
+  const plain = paste.text ?? "";
+  const html = paste.html ?? "";
+
+  // 1) The pasted text is itself HTML source (copied from an editor, a file, or
+  //    view-source). The plain text *is* the markup, so keep it as .html.
+  if (plain.trim() && looksLikeHtml(plain)) {
+    return { body: plain, ext: "html", type: "text/html" };
+  }
+
+  // 2) No usable plain text, but a real HTML flavor exists (some apps only
+  //    expose text/html). Fall back to the markup so nothing is dropped.
+  if (!plain.trim() && looksLikeHtml(html)) {
+    return { body: html, ext: "html", type: "text/html" };
+  }
+
+  // 3) Default: keep the clean plain text. Avoids the syntax-highlight /
+  //    rich-text noise that lives in text/html when the plain text is already
+  //    the real content (code from an editor, prose from a doc).
+  return { body: plain, ext: "txt", type: "text/plain" };
+}
+
+export function shouldConvertPasteToAttachment(text: string): boolean {
+  if (!text) return false;
+  if (text.length >= PASTED_TEXT_MIN_CHARS) return true;
+  let lines = 1;
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) === 10) {
+      lines++;
+      if (lines >= PASTED_TEXT_MIN_LINES) return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Whether a clipboard paste is large enough to become a `Pasted text`
+ * attachment chip. Mirrors `shouldConvertPasteToAttachment` but evaluates the
+ * representation we'd actually store, so an HTML-only paste (empty text/plain)
+ * still converts on the strength of its markup.
+ */
+export function shouldConvertClipboardToAttachment(
+  paste: ClipboardPaste,
+): boolean {
+  return shouldConvertPasteToAttachment(selectPasteBody(paste).body);
+}
+
+function pastedAttachmentName(ext: "html" | "txt"): string {
+  return `${PASTED_TEXT_FILENAME_PREFIX}${Date.now()}-${Math.random()
+    .toString(36)
+    .slice(2, 8)}.${ext}`;
+}
+
+/**
+ * Build the attachment File for a page-sized paste, preserving HTML markup when
+ * the pasted content is an HTML document so it travels the same rail as an
+ * uploaded .html file.
+ */
+export function createPastedAttachmentFile(paste: ClipboardPaste): File {
+  const { body, ext, type } = selectPasteBody(paste);
+  return new File([body], pastedAttachmentName(ext), { type });
+}
+
+/** Back-compat helper for callers that only have plain text. */
+export function createPastedTextFile(text: string): File {
+  return createPastedAttachmentFile({ text });
+}
+
+export function isPastedTextAttachmentName(name: string | undefined): boolean {
+  return !!name && name.startsWith(PASTED_TEXT_FILENAME_PREFIX);
+}
+
+// Strips the `<attachment name=...>\n` / `\n</attachment>` envelope that
+// SimpleTextAttachmentAdapter wraps the file body in when sending. Returns the
+// raw body for previewing.
+export function unwrapAttachmentEnvelope(text: string): string {
+  const match = text.match(/^<attachment\b[^>]*>\n([\s\S]*)\n<\/attachment>$/);
+  return match ? match[1] : text;
+}
+
+export function countLines(text: string): number {
+  if (!text) return 0;
+  let lines = 1;
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) === 10) lines++;
+  }
+  return lines;
+}

package/corpus/toolkit/src/composer/prompt-attachments.ts

@@ -0,0 +1,93 @@
+export interface AgentPromptAttachment {
+  name: string;
+  type?: string;
+  size?: number;
+  text?: string;
+  dataUrl?: string;
+}
+
+export function escapePromptAttachmentAttribute(value: string): string {
+  return value.replace(/&/g, "&").replace(/"/g, """);
+}
+
+export function formatPromptWithAttachments(
+  prompt: string,
+  attachments: readonly AgentPromptAttachment[],
+): string {
+  if (attachments.length === 0) return prompt;
+  const attachmentText = attachments
+    .map((attachment) => {
+      const size = attachment.size ? ` size="${attachment.size}"` : "";
+      const type = attachment.type
+        ? ` type="${escapePromptAttachmentAttribute(attachment.type)}"`
+        : "";
+      if (attachment.dataUrl) {
+        return `<attached-image name="${escapePromptAttachmentAttribute(attachment.name)}"${type}${size}>\n${attachment.dataUrl}\n</attached-image>`;
+      }
+      const body =
+        attachment.text?.trim() ||
+        "Selected in the UI. If this file is needed, inspect it from the workspace or ask for a readable copy.";
+      return `<attached-file name="${escapePromptAttachmentAttribute(attachment.name)}"${type}${size}>\n${body}\n</attached-file>`;
+    })
+    .join("\n\n");
+  return `${prompt.trimEnd()}\n\nAttached context:\n${attachmentText}`;
+}
+
+export const AGENT_PROMPT_MAX_INLINE_TEXT_CHARS = 60_000;
+export const AGENT_PROMPT_MAX_INLINE_IMAGE_BYTES = 2 * 1024 * 1024;
+
+export interface ReadAgentPromptAttachmentOptions {
+  maxInlineTextChars?: number;
+  maxInlineImageBytes?: number;
+}
+
+export async function readAgentPromptAttachment(
+  file: File,
+  options: ReadAgentPromptAttachmentOptions = {},
+): Promise<AgentPromptAttachment> {
+  const maxInlineTextChars =
+    options.maxInlineTextChars ?? AGENT_PROMPT_MAX_INLINE_TEXT_CHARS;
+  const maxInlineImageBytes =
+    options.maxInlineImageBytes ?? AGENT_PROMPT_MAX_INLINE_IMAGE_BYTES;
+  const attachment: AgentPromptAttachment = {
+    name: file.name,
+    type: file.type || undefined,
+    size: file.size,
+  };
+
+  if (isInlineableAgentPromptFile(file) && file.size <= maxInlineTextChars) {
+    try {
+      attachment.text = await file.text();
+    } catch {
+      // Keep the filename-only attachment if the browser cannot read it.
+    }
+  } else if (
+    file.type.startsWith("image/") &&
+    file.size <= maxInlineImageBytes
+  ) {
+    try {
+      attachment.dataUrl = await readFileAsDataUrl(file);
+    } catch {
+      // Keep the filename-only attachment if the browser cannot read it.
+    }
+  }
+
+  return attachment;
+}
+
+export function isInlineableAgentPromptFile(file: File): boolean {
+  if (file.type.startsWith("text/")) return true;
+  return /\.(cjs|css|csv|html|js|json|jsx|md|mdx|mjs|sql|tsx?|txt|xml|yaml|yml)$/i.test(
+    file.name,
+  );
+}
+
+function readFileAsDataUrl(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(String(reader.result ?? ""));
+    reader.onerror = () =>
+      reject(reader.error ?? new Error("Could not read file"));
+    reader.readAsDataURL(file);
+  });
+}

package/corpus/toolkit/src/composer/realtime-voice-audio-level.ts

@@ -0,0 +1,64 @@
+export interface RealtimeVoiceAudioLevels {
+  input: number;
+  output: number;
+}
+
+export interface RealtimeVoiceAudioLevelStore {
+  getSnapshot: () => RealtimeVoiceAudioLevels;
+  subscribe: (listener: () => void) => () => void;
+  set: (levels: RealtimeVoiceAudioLevels) => void;
+  reset: () => void;
+}
+
+const SILENT_LEVELS: RealtimeVoiceAudioLevels = { input: 0, output: 0 };
+
+function clampLevel(value: number): number {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
+export function normalizeRealtimeVoiceRms(samples: Uint8Array): number {
+  if (samples.length === 0) return 0;
+  let sumSquares = 0;
+  for (const sample of samples) {
+    const normalized = (sample - 128) / 128;
+    sumSquares += normalized * normalized;
+  }
+  const rms = Math.sqrt(sumSquares / samples.length);
+  return clampLevel(Math.max(0, rms - 0.012) * 3.2);
+}
+
+export function smoothRealtimeVoiceLevel(
+  previous: number,
+  next: number,
+): number {
+  const target = clampLevel(next);
+  const factor = target > previous ? 0.55 : 0.2;
+  return clampLevel(previous + (target - previous) * factor);
+}
+
+export function createRealtimeVoiceAudioLevelStore(): RealtimeVoiceAudioLevelStore {
+  let snapshot = SILENT_LEVELS;
+  const listeners = new Set<() => void>();
+
+  const set = (levels: RealtimeVoiceAudioLevels) => {
+    const next = {
+      input: clampLevel(levels.input),
+      output: clampLevel(levels.output),
+    };
+    if (next.input === snapshot.input && next.output === snapshot.output)
+      return;
+    snapshot = next;
+    for (const listener of listeners) listener();
+  };
+
+  return {
+    getSnapshot: () => snapshot,
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    set,
+    reset: () => set(SILENT_LEVELS),
+  };
+}

package/corpus/toolkit/src/composer/realtime-voice-transcript.ts

@@ -0,0 +1,134 @@
+export type RealtimeVoiceTranscriptRole = "user" | "assistant";
+
+export interface RealtimeVoiceTranscriptMessage {
+  id: string;
+  threadId: string;
+  role: RealtimeVoiceTranscriptRole;
+  text: string;
+  createdAt: string;
+}
+
+export interface RealtimeVoiceTranscriptSink {
+  threadId: string;
+  active: boolean;
+  append: (message: RealtimeVoiceTranscriptMessage) => boolean;
+}
+
+export interface RealtimeVoiceTranscriptRegistry {
+  activeThreadId: () => string | undefined;
+  publish: (message: RealtimeVoiceTranscriptMessage) => void;
+  register: (sink: RealtimeVoiceTranscriptSink) => () => void;
+  pendingCount: () => number;
+}
+
+export function appendRealtimeVoiceTranscriptToRepository(
+  value: unknown,
+  transcript: RealtimeVoiceTranscriptMessage,
+): { appended: boolean; repository: Record<string, unknown> } {
+  const repository =
+    value && typeof value === "object" && !Array.isArray(value)
+      ? (value as Record<string, unknown>)
+      : {};
+  const messages = Array.isArray(repository.messages)
+    ? [...repository.messages]
+    : [];
+  const duplicate = messages.some((entry) => {
+    if (!entry || typeof entry !== "object") return false;
+    const record = entry as Record<string, unknown>;
+    const message =
+      record.message && typeof record.message === "object"
+        ? (record.message as Record<string, unknown>)
+        : record;
+    return message.id === transcript.id;
+  });
+  if (duplicate) return { appended: false, repository };
+
+  const parentId =
+    typeof repository.headId === "string" ? repository.headId : null;
+  const createdAt = new Date(transcript.createdAt);
+  const message = {
+    id: transcript.id,
+    role: transcript.role,
+    content: [{ type: "text", text: transcript.text }],
+    createdAt: Number.isNaN(createdAt.getTime()) ? new Date() : createdAt,
+    metadata: {
+      custom: {
+        source: "realtime-voice",
+      },
+    },
+    ...(transcript.role === "assistant"
+      ? { status: { type: "complete", reason: "stop" } }
+      : {}),
+  };
+
+  return {
+    appended: true,
+    repository: {
+      ...repository,
+      messages: [...messages, { message, parentId }],
+      headId: transcript.id,
+    },
+  };
+}
+
+export function createRealtimeVoiceTranscriptRegistry(): RealtimeVoiceTranscriptRegistry {
+  const sinks = new Map<
+    symbol,
+    RealtimeVoiceTranscriptSink & { order: number }
+  >();
+  const pending = new Map<string, RealtimeVoiceTranscriptMessage>();
+  const delivered = new Set<string>();
+  let order = 0;
+
+  const flush = () => {
+    for (const [id, message] of pending) {
+      let consumed = false;
+      for (const sink of sinks.values()) {
+        if (sink.threadId !== message.threadId) continue;
+        try {
+          consumed = sink.append(message) || consumed;
+        } catch {
+          // Keep the completed transcript queued until the chat can import it.
+        }
+      }
+      if (!consumed) continue;
+      pending.delete(id);
+      delivered.add(id);
+      if (delivered.size > 1_000) {
+        const oldest = delivered.values().next().value;
+        if (typeof oldest === "string") delivered.delete(oldest);
+      }
+    }
+  };
+
+  return {
+    activeThreadId() {
+      let active: (RealtimeVoiceTranscriptSink & { order: number }) | undefined;
+      for (const sink of sinks.values()) {
+        if (!sink.active || (active && sink.order <= active.order)) continue;
+        active = sink;
+      }
+      return active?.threadId;
+    },
+    publish(message) {
+      const id = message.id.trim();
+      const threadId = message.threadId.trim();
+      const text = message.text.trim();
+      if (!id || !threadId || !text || delivered.has(id) || pending.has(id)) {
+        return;
+      }
+      pending.set(id, { ...message, id, threadId, text });
+      flush();
+    },
+    register(sink) {
+      const token = Symbol("realtime-voice-transcript-sink");
+      sinks.set(token, { ...sink, order: ++order });
+      flush();
+      return () => sinks.delete(token);
+    },
+    pendingCount: () => pending.size,
+  };
+}
+
+export const realtimeVoiceTranscriptRegistry =
+  createRealtimeVoiceTranscriptRegistry();