@agent-native/toolkit 0.6.0 → 0.8.0

package/src/composer/realtime-voice-audio-level.ts

@@ -0,0 +1,64 @@
+export interface RealtimeVoiceAudioLevels {
+  input: number;
+  output: number;
+}
+
+export interface RealtimeVoiceAudioLevelStore {
+  getSnapshot: () => RealtimeVoiceAudioLevels;
+  subscribe: (listener: () => void) => () => void;
+  set: (levels: RealtimeVoiceAudioLevels) => void;
+  reset: () => void;
+}
+
+const SILENT_LEVELS: RealtimeVoiceAudioLevels = { input: 0, output: 0 };
+
+function clampLevel(value: number): number {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
+export function normalizeRealtimeVoiceRms(samples: Uint8Array): number {
+  if (samples.length === 0) return 0;
+  let sumSquares = 0;
+  for (const sample of samples) {
+    const normalized = (sample - 128) / 128;
+    sumSquares += normalized * normalized;
+  }
+  const rms = Math.sqrt(sumSquares / samples.length);
+  return clampLevel(Math.max(0, rms - 0.012) * 3.2);
+}
+
+export function smoothRealtimeVoiceLevel(
+  previous: number,
+  next: number,
+): number {
+  const target = clampLevel(next);
+  const factor = target > previous ? 0.55 : 0.2;
+  return clampLevel(previous + (target - previous) * factor);
+}
+
+export function createRealtimeVoiceAudioLevelStore(): RealtimeVoiceAudioLevelStore {
+  let snapshot = SILENT_LEVELS;
+  const listeners = new Set<() => void>();
+
+  const set = (levels: RealtimeVoiceAudioLevels) => {
+    const next = {
+      input: clampLevel(levels.input),
+      output: clampLevel(levels.output),
+    };
+    if (next.input === snapshot.input && next.output === snapshot.output)
+      return;
+    snapshot = next;
+    for (const listener of listeners) listener();
+  };
+
+  return {
+    getSnapshot: () => snapshot,
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    set,
+    reset: () => set(SILENT_LEVELS),
+  };
+}

package/src/composer/realtime-voice-transcript.spec.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  appendRealtimeVoiceTranscriptToRepository,
+  createRealtimeVoiceTranscriptRegistry,
+  type RealtimeVoiceTranscriptMessage,
+} from "./realtime-voice-transcript.js";
+
+function transcript(
+  overrides: Partial<RealtimeVoiceTranscriptMessage> = {},
+): RealtimeVoiceTranscriptMessage {
+  return {
+    id: "voice-1",
+    threadId: "thread-1",
+    role: "user",
+    text: "Open sources",
+    createdAt: "2026-07-10T20:00:00.000Z",
+    ...overrides,
+  };
+}
+
+describe("realtime voice transcript registry", () => {
+  it("captures the latest active thread and buffers until its sink mounts", () => {
+    const registry = createRealtimeVoiceTranscriptRegistry();
+    registry.register({
+      threadId: "thread-old",
+      active: true,
+      append: vi.fn(() => true),
+    });
+    registry.register({
+      threadId: "thread-1",
+      active: true,
+      // Simulate the chat surface being selected but not yet ready to apply
+      // the completed transcript. The registry must retain it for the sink
+      // that mounts once the surface is ready.
+      append: vi.fn(() => false),
+    });
+    expect(registry.activeThreadId()).toBe("thread-1");
+
+    registry.publish(transcript());
+    expect(registry.pendingCount()).toBe(1);
+    const append = vi.fn(() => true);
+    registry.register({ threadId: "thread-1", active: false, append });
+    expect(append).toHaveBeenCalledOnce();
+    expect(registry.pendingCount()).toBe(0);
+  });
+
+  it("deduplicates completed provider transcript events", () => {
+    const registry = createRealtimeVoiceTranscriptRegistry();
+    const append = vi.fn(() => true);
+    registry.register({ threadId: "thread-1", active: true, append });
+    registry.publish(transcript());
+    registry.publish(transcript());
+    expect(append).toHaveBeenCalledOnce();
+  });
+});
+
+describe("appendRealtimeVoiceTranscriptToRepository", () => {
+  it("links completed user and assistant messages without starting a run", () => {
+    const first = appendRealtimeVoiceTranscriptToRepository({}, transcript());
+    const second = appendRealtimeVoiceTranscriptToRepository(
+      first.repository,
+      transcript({
+        id: "voice-2",
+        role: "assistant",
+        text: "Opening Sources.",
+      }),
+    );
+    expect(second.repository.headId).toBe("voice-2");
+    expect(second.repository.messages).toMatchObject([
+      {
+        parentId: null,
+        message: {
+          id: "voice-1",
+          role: "user",
+          metadata: { custom: { source: "realtime-voice" } },
+        },
+      },
+      {
+        parentId: "voice-1",
+        message: {
+          id: "voice-2",
+          role: "assistant",
+          status: { type: "complete", reason: "stop" },
+        },
+      },
+    ]);
+    expect(
+      appendRealtimeVoiceTranscriptToRepository(
+        second.repository,
+        transcript({ id: "voice-2" }),
+      ).appended,
+    ).toBe(false);
+  });
+});

package/src/composer/realtime-voice-transcript.ts

@@ -0,0 +1,134 @@
+export type RealtimeVoiceTranscriptRole = "user" | "assistant";
+
+export interface RealtimeVoiceTranscriptMessage {
+  id: string;
+  threadId: string;
+  role: RealtimeVoiceTranscriptRole;
+  text: string;
+  createdAt: string;
+}
+
+export interface RealtimeVoiceTranscriptSink {
+  threadId: string;
+  active: boolean;
+  append: (message: RealtimeVoiceTranscriptMessage) => boolean;
+}
+
+export interface RealtimeVoiceTranscriptRegistry {
+  activeThreadId: () => string | undefined;
+  publish: (message: RealtimeVoiceTranscriptMessage) => void;
+  register: (sink: RealtimeVoiceTranscriptSink) => () => void;
+  pendingCount: () => number;
+}
+
+export function appendRealtimeVoiceTranscriptToRepository(
+  value: unknown,
+  transcript: RealtimeVoiceTranscriptMessage,
+): { appended: boolean; repository: Record<string, unknown> } {
+  const repository =
+    value && typeof value === "object" && !Array.isArray(value)
+      ? (value as Record<string, unknown>)
+      : {};
+  const messages = Array.isArray(repository.messages)
+    ? [...repository.messages]
+    : [];
+  const duplicate = messages.some((entry) => {
+    if (!entry || typeof entry !== "object") return false;
+    const record = entry as Record<string, unknown>;
+    const message =
+      record.message && typeof record.message === "object"
+        ? (record.message as Record<string, unknown>)
+        : record;
+    return message.id === transcript.id;
+  });
+  if (duplicate) return { appended: false, repository };
+
+  const parentId =
+    typeof repository.headId === "string" ? repository.headId : null;
+  const createdAt = new Date(transcript.createdAt);
+  const message = {
+    id: transcript.id,
+    role: transcript.role,
+    content: [{ type: "text", text: transcript.text }],
+    createdAt: Number.isNaN(createdAt.getTime()) ? new Date() : createdAt,
+    metadata: {
+      custom: {
+        source: "realtime-voice",
+      },
+    },
+    ...(transcript.role === "assistant"
+      ? { status: { type: "complete", reason: "stop" } }
+      : {}),
+  };
+
+  return {
+    appended: true,
+    repository: {
+      ...repository,
+      messages: [...messages, { message, parentId }],
+      headId: transcript.id,
+    },
+  };
+}
+
+export function createRealtimeVoiceTranscriptRegistry(): RealtimeVoiceTranscriptRegistry {
+  const sinks = new Map<
+    symbol,
+    RealtimeVoiceTranscriptSink & { order: number }
+  >();
+  const pending = new Map<string, RealtimeVoiceTranscriptMessage>();
+  const delivered = new Set<string>();
+  let order = 0;
+
+  const flush = () => {
+    for (const [id, message] of pending) {
+      let consumed = false;
+      for (const sink of sinks.values()) {
+        if (sink.threadId !== message.threadId) continue;
+        try {
+          consumed = sink.append(message) || consumed;
+        } catch {
+          // Keep the completed transcript queued until the chat can import it.
+        }
+      }
+      if (!consumed) continue;
+      pending.delete(id);
+      delivered.add(id);
+      if (delivered.size > 1_000) {
+        const oldest = delivered.values().next().value;
+        if (typeof oldest === "string") delivered.delete(oldest);
+      }
+    }
+  };
+
+  return {
+    activeThreadId() {
+      let active: (RealtimeVoiceTranscriptSink & { order: number }) | undefined;
+      for (const sink of sinks.values()) {
+        if (!sink.active || (active && sink.order <= active.order)) continue;
+        active = sink;
+      }
+      return active?.threadId;
+    },
+    publish(message) {
+      const id = message.id.trim();
+      const threadId = message.threadId.trim();
+      const text = message.text.trim();
+      if (!id || !threadId || !text || delivered.has(id) || pending.has(id)) {
+        return;
+      }
+      pending.set(id, { ...message, id, threadId, text });
+      flush();
+    },
+    register(sink) {
+      const token = Symbol("realtime-voice-transcript-sink");
+      sinks.set(token, { ...sink, order: ++order });
+      flush();
+      return () => sinks.delete(token);
+    },
+    pendingCount: () => pending.size,
+  };
+}
+
+export const realtimeVoiceTranscriptRegistry =
+  createRealtimeVoiceTranscriptRegistry();

package/src/composer/runtime-adapters.tsx

@@ -0,0 +1,343 @@
+import {
+  createContext,
+  useContext,
+  type ComponentType,
+  type ReactNode,
+} from "react";
+
+export type ComposerTranslate = (
+  key: string,
+  options?: Record<string, unknown>,
+) => string;
+
+export type ReasoningEffort =
+  | "auto"
+  | "none"
+  | "minimal"
+  | "low"
+  | "medium"
+  | "high"
+  | "xhigh"
+  | "max";
+
+export interface EngineModelGroup {
+  engine: string;
+  label: string;
+  models: string[];
+  configured: boolean;
+}
+
+export interface ComposerModelState {
+  selectedModel: string;
+  selectedEngine: string;
+  selectedEffort: ReasoningEffort;
+  availableModels: EngineModelGroup[];
+  isLoading: boolean;
+  onModelChange: (model: string, engine: string) => void;
+  onEffortChange: (effort: ReasoningEffort) => void;
+}
+
+export interface ComposerBuilderConnectFlow {
+  hasFetchedStatus: boolean;
+  configured: boolean;
+  envManaged: boolean;
+  connecting: boolean;
+  statusResolved: boolean;
+  start: () => void;
+}
+
+export interface AgentChatContextItem {
+  key: string;
+  title: string;
+  context: string;
+}
+
+export interface ComposerAgentChatMessage {
+  message: string;
+  context?: string;
+  mode?: "plan" | "act";
+  submit?: boolean;
+}
+
+export interface ComposerAgentChatContextSetOptions extends AgentChatContextItem {
+  openSidebar?: boolean;
+  focus?: boolean;
+}
+
+export interface ComposerAgentChatOpenThreadRequest {
+  threadId: string;
+  onlyIfActiveThreadId?: string;
+}
+
+export interface ComposerBuilderConnectFlowOptions {
+  enabled?: boolean;
+  popupUrl?: string;
+  trackingSource?: string;
+  trackingFlow?: string;
+  onConnected?: (state: { orgName: string | null }) => void | Promise<void>;
+}
+
+export interface AgentComposerReference {
+  label: string;
+  icon?: string;
+  source?: string;
+  refType: string;
+  refId?: string | null;
+  refPath?: string | null;
+  slotKey?: string;
+  slotLabel?: string;
+  metadata?: Record<string, unknown>;
+  clearsSlots?: string[];
+  relatedReferences?: AgentComposerReference[];
+}
+export interface AgentComposerReferenceInsertPayload extends AgentComposerReference {
+  insertMessageId: string;
+}
+
+export interface VoiceContextPack {
+  surface?: string;
+  mode?: string;
+  snippets?: Array<{ label: string; value: string }>;
+  terms?: Array<{ term: string; replacement?: string }>;
+}
+
+export interface ComposerRuntimeAdapters {
+  resolvePath?: (path: string) => string;
+  translate?: ComposerTranslate;
+  models?: {
+    useChatModels?: (options: { enabled: boolean }) => ComposerModelState;
+    useAgentEngineConfigured?: (enabled: boolean) => {
+      missing: boolean;
+      state: string;
+    };
+    fetchAgentEngineConfiguredState?: (
+      enabled: boolean,
+      options: { timeoutMs: number },
+    ) => Promise<"missing" | "configured" | string>;
+    BuilderSetupCard?: ComponentType<any>;
+    BuilderSetupContent?: ComponentType<any>;
+    reasoning?: {
+      defaultEffort?: ReasoningEffort;
+      getOptionsForModel?: (model?: string) => ReasoningEffort[];
+      label?: (effort: ReasoningEffort) => string;
+      resolve?: (model: string, effort?: ReasoningEffort) => ReasoningEffort;
+    };
+  };
+  agentChat?: {
+    sendToAgentChat?: (input: ComposerAgentChatMessage) => void;
+    setContextItem?: (input: ComposerAgentChatContextSetOptions) => void;
+    requestThreadOpen?: (input: ComposerAgentChatOpenThreadRequest) => void;
+    formatContextItems?: (
+      items: readonly AgentChatContextItem[] | undefined,
+    ) => string;
+    normalizeReference?: (reference: unknown) => AgentComposerReference | null;
+    StaleIndexBoundary?: ComponentType<any>;
+  };
+  builder?: {
+    useConnectFlow?: (
+      options: ComposerBuilderConnectFlowOptions,
+    ) => ComposerBuilderConnectFlow;
+    tryDelegateBuildRequest?: (text: string) => boolean;
+    isTrustedFrameMessage?: (event: MessageEvent) => boolean;
+    isTrustedBuilderMessage?: (event: MessageEvent) => boolean;
+  };
+  resources?: {
+    useOrg?: () => {
+      data?: { orgId?: string | null; role?: string | null } | null;
+    };
+    isMcpIntegrationAvailable?: () => boolean;
+    useCreateMcpServer?: () => {
+      mutateAsync: (input: any) => Promise<unknown>;
+    };
+    McpIntegrationDialog?: ComponentType<any>;
+  };
+  voice?: {
+    useProviderStatus?: () => {
+      status: { builder?: boolean; openai?: boolean } | null;
+      refresh: () => void;
+    };
+    getBrowserTabId?: () => string;
+    readAppState?: <T>(
+      key: string,
+    ) => T | null | undefined | Promise<T | null | undefined>;
+    setAppState?: (key: string, value: unknown) => void | Promise<unknown>;
+    subscribeSidebarState?: (
+      listener: (detail: { open?: boolean } | undefined) => void,
+    ) => () => void;
+    applyContextReplacements?: (
+      text: string,
+      context: VoiceContextPack | undefined,
+    ) => string;
+  };
+}
+
+const identityPath = (path: string) => path;
+const fallbackTranslate: ComposerTranslate = (key, options) =>
+  typeof options?.defaultValue === "string" ? options.defaultValue : key;
+const fallbackModels = {
+  useChatModels: () => ({
+    selectedModel: "auto",
+    selectedEngine: "auto",
+    selectedEffort: "medium" as ReasoningEffort,
+    availableModels: [],
+    isLoading: false,
+    onModelChange: () => {},
+    onEffortChange: () => {},
+  }),
+  useAgentEngineConfigured: () => ({ missing: false, state: "configured" }),
+  fetchAgentEngineConfiguredState: async () => "configured",
+};
+const FragmentBoundary: ComponentType<{ children?: ReactNode }> = ({
+  children,
+}) => <>{children}</>;
+const fallbackBuilderFlow = {
+  hasFetchedStatus: false,
+  configured: false,
+  envManaged: false,
+  connecting: false,
+  statusResolved: false,
+  start: () => {},
+};
+
+const fallbackAdapters: Required<Pick<ComposerRuntimeAdapters, "resolvePath">> &
+  ComposerRuntimeAdapters = {
+  resolvePath: identityPath,
+  translate: fallbackTranslate,
+  models: fallbackModels,
+  agentChat: {
+    sendToAgentChat: () => {},
+    setContextItem: () => {},
+    requestThreadOpen: () => {},
+    formatContextItems: () => "",
+    normalizeReference: (reference) =>
+      reference &&
+      typeof reference === "object" &&
+      typeof (reference as AgentComposerReference).label === "string" &&
+      typeof (reference as AgentComposerReference).refType === "string"
+        ? (reference as AgentComposerReference)
+        : null,
+    StaleIndexBoundary: FragmentBoundary,
+  },
+  builder: {
+    useConnectFlow: () => fallbackBuilderFlow,
+    tryDelegateBuildRequest: () => false,
+    isTrustedFrameMessage: () => false,
+    isTrustedBuilderMessage: () => false,
+  },
+  resources: {
+    useOrg: () => ({ data: null }),
+    isMcpIntegrationAvailable: () => false,
+    useCreateMcpServer: () => ({ mutateAsync: async () => undefined }),
+  },
+  voice: {
+    useProviderStatus: () => ({ status: null, refresh: () => {} }),
+    getBrowserTabId: () => "",
+    readAppState: () => undefined,
+    setAppState: () => {},
+    subscribeSidebarState: () => () => {},
+    applyContextReplacements: applyVoiceContextReplacements,
+  },
+};
+
+const ComposerRuntimeAdaptersContext =
+  createContext<ComposerRuntimeAdapters>(fallbackAdapters);
+
+export function ComposerRuntimeAdaptersProvider({
+  adapters,
+  children,
+}: {
+  adapters: ComposerRuntimeAdapters;
+  children: ReactNode;
+}) {
+  return (
+    <ComposerRuntimeAdaptersContext.Provider
+      value={{
+        ...fallbackAdapters,
+        ...adapters,
+        models: { ...fallbackModels, ...adapters.models },
+        agentChat: { ...fallbackAdapters.agentChat, ...adapters.agentChat },
+        builder: { ...fallbackAdapters.builder, ...adapters.builder },
+        resources: { ...fallbackAdapters.resources, ...adapters.resources },
+        voice: { ...fallbackAdapters.voice, ...adapters.voice },
+      }}
+    >
+      {children}
+    </ComposerRuntimeAdaptersContext.Provider>
+  );
+}
+
+export function useComposerRuntimeAdapters() {
+  return useContext(ComposerRuntimeAdaptersContext);
+}
+
+export const DEFAULT_REASONING_EFFORT: ReasoningEffort = "medium";
+export function getReasoningEffortOptionsForModel(
+  _model?: string,
+): ReasoningEffort[] {
+  return ["low", "medium", "high"];
+}
+export function reasoningEffortLabel(effort: ReasoningEffort): string {
+  return effort === "xhigh"
+    ? "Extra high"
+    : effort[0].toUpperCase() + effort.slice(1);
+}
+export function resolveReasoningEffortSelection(
+  _model: string,
+  effort?: ReasoningEffort,
+): ReasoningEffort {
+  return effort ?? DEFAULT_REASONING_EFFORT;
+}
+
+export const AGENT_CHAT_INSERT_REFERENCE_EVENT =
+  "agent-native:insert-composer-reference";
+export const AGENT_CHAT_INSERT_REFERENCE_MESSAGE_TYPE =
+  "agent-native:insert-composer-reference";
+
+export function formatPromptContextItems(
+  items: AgentChatContextItem[] | undefined,
+): string {
+  return (
+    items
+      ?.map((item) => item.context ?? item.title ?? "")
+      .filter(Boolean)
+      .join("\n\n") ?? ""
+  );
+}
+
+export function applyVoiceContextReplacements(
+  text: string,
+  context: VoiceContextPack | undefined,
+): string {
+  const isAlphaNumeric = (value: string | undefined) =>
+    value != null && /[\p{L}\p{N}_]/u.test(value);
+  const isSafeBoundary = (
+    value: string,
+    index: number,
+    direction: "before" | "after",
+  ) => {
+    const char = value[index];
+    if (char == null) return true;
+    if (isAlphaNumeric(char) || "@/\\:".includes(char)) return false;
+    if (".-+".includes(char)) {
+      const neighbor =
+        direction === "before" ? value[index - 1] : value[index + 1];
+      return !isAlphaNumeric(neighbor);
+    }
+    return true;
+  };
+  let next = text;
+  for (const term of context?.terms ?? []) {
+    const source = term.term.trim();
+    const replacement = term.replacement?.trim();
+    if (!replacement || source.length < 2 || replacement === source) continue;
+    const escaped = source.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    next = next.replace(new RegExp(escaped, "giu"), (match, offset: number) => {
+      const end = offset + match.length;
+      return isSafeBoundary(next, offset - 1, "before") &&
+        isSafeBoundary(next, end, "after")
+        ? replacement
+        : match;
+    });
+  }
+  return next;
+}

package/src/composer/types.ts

@@ -0,0 +1,66 @@
+export interface FileResult {
+  path: string;
+  name: string;
+  source: "codebase" | "resource";
+  type: "file" | "folder";
+}
+
+export interface SkillResult {
+  name: string;
+  description: string;
+  path: string;
+  source: "codebase" | "resource";
+}
+
+export interface MentionItem {
+  id: string;
+  label: string;
+  description?: string;
+  icon?: string;
+  source: string;
+  refType: string;
+  refPath?: string;
+  refId?: string;
+  section?: string;
+  slotKey?: string;
+  slotLabel?: string;
+  metadata?: Record<string, unknown>;
+  clearsSlots?: string[];
+  relatedReferences?: MentionReferenceInsert[];
+}
+
+export interface Reference {
+  type: "file" | "skill" | "mention" | "agent" | "custom-agent";
+  path: string;
+  name: string;
+  source: string;
+  refType?: string;
+  refId?: string;
+  slotKey?: string;
+  slotLabel?: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface MentionReferenceInsert {
+  label: string;
+  icon?: string;
+  source?: string;
+  refType: string;
+  refId?: string | null;
+  refPath?: string | null;
+  slotKey?: string;
+  slotLabel?: string;
+  metadata?: Record<string, unknown>;
+  clearsSlots?: string[];
+  relatedReferences?: MentionReferenceInsert[];
+}
+
+export interface SlashCommand {
+  name: string;
+  description: string;
+  icon?: string;
+}
+
+export type ComposerMode = "skill" | "job" | "automation" | "extension";
+
+export type AgentComposerLayoutVariant = "default" | "compact" | "hero";