@runtypelabs/persona 4.25.0 → 4.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/dist/animations/glyph-cycle.d.cts +1 -1
  2. package/dist/animations/glyph-cycle.d.ts +1 -1
  3. package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
  4. package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
  5. package/dist/animations/wipe.d.cts +1 -1
  6. package/dist/animations/wipe.d.ts +1 -1
  7. package/dist/animations-extra.d.cts +14 -0
  8. package/dist/animations-extra.d.ts +14 -0
  9. package/dist/approval-ui-QAKWNFNM.js +1 -0
  10. package/dist/approval-ui.cjs +2 -2
  11. package/dist/approval-ui.d.cts +180 -8
  12. package/dist/approval-ui.d.ts +180 -8
  13. package/dist/approval-ui.js +2 -2
  14. package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
  15. package/dist/artifacts-ui.cjs +4 -4
  16. package/dist/artifacts-ui.d.cts +180 -8
  17. package/dist/artifacts-ui.d.ts +180 -8
  18. package/dist/artifacts-ui.js +5 -5
  19. package/dist/chunk-2ANTRKTV.js +1 -0
  20. package/dist/chunk-4HPTPUC6.js +1 -0
  21. package/dist/chunk-6V3MNEKB.js +1 -0
  22. package/dist/chunk-HGUCGM3K.js +9 -0
  23. package/dist/chunk-IO5VVUKP.js +3 -0
  24. package/dist/chunk-J4J3Y2P2.js +1 -0
  25. package/dist/chunk-K6HKIP5K.js +1 -0
  26. package/dist/chunk-K7NA7G7J.js +1 -0
  27. package/dist/chunk-MXUTS3LB.js +1 -0
  28. package/dist/chunk-PDPVORUF.js +1 -0
  29. package/dist/chunk-QPPJUSZH.js +1 -0
  30. package/dist/chunk-QRKGQPU7.js +1 -0
  31. package/dist/chunk-S6PBWE75.js +1 -0
  32. package/dist/chunk-T7QM7YMF.js +1 -0
  33. package/dist/chunk-TR2H475E.js +1 -0
  34. package/dist/chunk-UPO4GUFC.js +1 -0
  35. package/dist/chunk-ZC66NAIR.js +1 -0
  36. package/dist/client-history-YPEG45Q4.js +1 -0
  37. package/dist/client-history.js +1 -0
  38. package/dist/client-stream-42FOI3EC.js +4 -0
  39. package/dist/client-stream.js +5 -0
  40. package/dist/codegen.cjs +1 -1
  41. package/dist/codegen.js +1 -1
  42. package/dist/context-mentions-E7HJYWO6.js +161 -0
  43. package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
  44. package/dist/context-mentions.cjs +7 -7
  45. package/dist/context-mentions.d.cts +180 -8
  46. package/dist/context-mentions.d.ts +180 -8
  47. package/dist/context-mentions.js +8 -8
  48. package/dist/event-stream-view-FOIJHFQO.js +1 -0
  49. package/dist/event-stream-view.cjs +1 -1
  50. package/dist/event-stream-view.d.cts +183 -31
  51. package/dist/event-stream-view.d.ts +183 -31
  52. package/dist/event-stream-view.js +1 -1
  53. package/dist/forms-ui.d.cts +221 -667
  54. package/dist/forms-ui.d.ts +221 -667
  55. package/dist/history-shell-QDQ44PSE.js +3 -0
  56. package/dist/history-shell.js +3 -0
  57. package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
  58. package/dist/history-view.cjs +2 -2
  59. package/dist/history-view.d.cts +35 -1
  60. package/dist/history-view.d.ts +35 -1
  61. package/dist/history-view.js +2 -2
  62. package/dist/index.cjs +80 -74
  63. package/dist/index.cjs.map +1 -1
  64. package/dist/index.d.cts +227 -738
  65. package/dist/index.d.ts +227 -738
  66. package/dist/index.global.js +3 -215
  67. package/dist/index.global.js.map +1 -1
  68. package/dist/index.js +81 -75
  69. package/dist/index.js.map +1 -1
  70. package/dist/install.global.js +1 -1
  71. package/dist/install.global.js.map +1 -1
  72. package/dist/launcher.global.js +3 -67
  73. package/dist/launcher.global.js.map +1 -1
  74. package/dist/markdown-parsers.js +22 -22
  75. package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
  76. package/dist/runtype-tts.js +1 -1
  77. package/dist/session-actions-ISSKSI66.js +1 -0
  78. package/dist/session-actions.js +1 -0
  79. package/dist/session-reconnect.d.cts +180 -8
  80. package/dist/session-reconnect.d.ts +180 -8
  81. package/dist/smart-dom-reader.cjs +13 -13
  82. package/dist/smart-dom-reader.d.cts +180 -8
  83. package/dist/smart-dom-reader.d.ts +180 -8
  84. package/dist/smart-dom-reader.js +11 -11
  85. package/dist/theme-editor-preview.cjs +116 -98
  86. package/dist/theme-editor-preview.d.cts +182 -8
  87. package/dist/theme-editor-preview.d.ts +182 -8
  88. package/dist/theme-editor-preview.js +49 -54
  89. package/dist/theme-editor.cjs +2 -2
  90. package/dist/theme-editor.d.cts +181 -23
  91. package/dist/theme-editor.d.ts +181 -23
  92. package/dist/theme-editor.js +2 -2
  93. package/dist/theme-reference.d.cts +14 -0
  94. package/dist/theme-reference.d.ts +14 -0
  95. package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
  96. package/dist/ui-extras.js +72 -0
  97. package/dist/voice-runtime-UOQOFAFY.js +17 -0
  98. package/dist/voice-runtime.cjs +17 -1
  99. package/dist/voice-runtime.d.cts +11650 -232
  100. package/dist/voice-runtime.d.ts +11650 -232
  101. package/dist/voice-runtime.js +17 -1
  102. package/dist/voice-worklet-player.cjs +19 -4
  103. package/dist/voice-worklet-player.d.cts +9 -10
  104. package/dist/voice-worklet-player.d.ts +9 -10
  105. package/dist/voice-worklet-player.js +19 -4
  106. package/dist/webmcp-polyfill.js +1 -1
  107. package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
  108. package/dist/webmcp-runtime.cjs +2 -2
  109. package/dist/webmcp-runtime.d.cts +1 -33
  110. package/dist/webmcp-runtime.d.ts +1 -33
  111. package/dist/webmcp-runtime.js +2 -2
  112. package/dist/widget.css +1 -1
  113. package/package.json +11 -5
  114. package/src/approval-paused-tool-settle.test.ts +221 -0
  115. package/src/client-approve.test.ts +311 -0
  116. package/src/client-history-eager.ts +9 -0
  117. package/src/client-history-loader.ts +23 -0
  118. package/src/client-history.ts +579 -0
  119. package/src/client-resume.ts +177 -0
  120. package/src/client-stream-eager.ts +12 -0
  121. package/src/client-stream-loader.ts +25 -0
  122. package/src/client-stream.ts +1880 -0
  123. package/src/client.test.ts +31 -1
  124. package/src/client.ts +293 -2607
  125. package/src/components/approval-actions.test.ts +27 -0
  126. package/src/components/approval-actions.ts +27 -3
  127. package/src/components/message-bubble.test.ts +19 -0
  128. package/src/components/message-bubble.ts +2 -0
  129. package/src/components/registry.ts +6 -6
  130. package/src/components/tool-bubble.test.ts +9 -0
  131. package/src/components/tool-bubble.ts +1 -0
  132. package/src/history-shell-bundle.test.ts +54 -0
  133. package/src/history-shell-eager.ts +12 -0
  134. package/src/history-shell-loader.ts +27 -0
  135. package/src/history-shell.ts +2362 -0
  136. package/src/history-view-loader.ts +2 -0
  137. package/src/history-view.ts +3 -0
  138. package/src/index-core.ts +6 -0
  139. package/src/index-global.ts +90 -248
  140. package/src/index.ts +13 -0
  141. package/src/install.ts +6 -4
  142. package/src/internal/runtype-history-provider.test.ts +1 -0
  143. package/src/internal/runtype-history-provider.ts +28 -13
  144. package/src/plugins/registry.ts +8 -8
  145. package/src/session-actions-chunk.test.ts +55 -0
  146. package/src/session-actions-eager.ts +11 -0
  147. package/src/session-actions-loader.ts +37 -0
  148. package/src/session-actions.ts +909 -0
  149. package/src/session-history.test.ts +3 -1
  150. package/src/session.ts +852 -1536
  151. package/src/session.voice.test.ts +1170 -5
  152. package/src/styles/widget.css +58 -0
  153. package/src/theme-editor/state.ts +91 -91
  154. package/src/types.ts +198 -5
  155. package/src/ui-ask-user-sheet.ts +382 -0
  156. package/src/ui-extras-eager.ts +10 -0
  157. package/src/ui-extras-entry.ts +9 -0
  158. package/src/ui-extras-loader.ts +26 -0
  159. package/src/ui.artifact-rail-resize.test.ts +2 -1
  160. package/src/ui.ask-user-chunk.test.ts +93 -0
  161. package/src/ui.history-chunk.test.ts +2 -1
  162. package/src/ui.history-config-provider.test.ts +2 -1
  163. package/src/ui.history-confirm-fallback.test.ts +142 -0
  164. package/src/ui.history-controller.test.ts +2 -1
  165. package/src/ui.history-lifecycle.test.ts +2 -1
  166. package/src/ui.history-render-hooks.test.ts +2 -1
  167. package/src/ui.history-shell.test.ts +2 -1
  168. package/src/ui.rail-overlay.test.ts +2 -1
  169. package/src/ui.rail-resize.test.ts +2 -1
  170. package/src/ui.rail-trigger-pending.test.ts +2 -1
  171. package/src/ui.ts +340 -2770
  172. package/src/utils/attachment-manager.ts +67 -67
  173. package/src/utils/constants.ts +3 -0
  174. package/src/utils/context-mention-controller.ts +323 -323
  175. package/src/utils/context-mention-manager.ts +40 -40
  176. package/src/utils/context-mention-orchestrator.ts +28 -0
  177. package/src/utils/event-stream-buffer.ts +50 -50
  178. package/src/utils/event-stream-store.ts +43 -43
  179. package/src/utils/throughput-tracker.ts +34 -34
  180. package/src/utils/virtual-scroller.ts +67 -67
  181. package/src/utils/visitor-store.test.ts +50 -0
  182. package/src/utils/visitor-store.ts +23 -5
  183. package/src/voice/__fixtures__/voice-wire/README.md +1 -0
  184. package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
  185. package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
  186. package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
  187. package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
  188. package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
  189. package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
  190. package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
  191. package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
  192. package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
  193. package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
  194. package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
  195. package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
  196. package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
  197. package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
  198. package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
  199. package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
  200. package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
  201. package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
  202. package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
  203. package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
  204. package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
  205. package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
  206. package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
  207. package/src/voice/audio-playback-manager.test.ts +57 -0
  208. package/src/voice/audio-playback-manager.ts +120 -98
  209. package/src/voice/browser-voice-provider.ts +37 -37
  210. package/src/voice/fallback-speech-engine.ts +22 -16
  211. package/src/voice/keyed-voice-transcript.ts +314 -0
  212. package/src/voice/mic-state-styles.ts +182 -0
  213. package/src/voice/read-aloud-controller.ts +46 -44
  214. package/src/voice/runtype-speech-engine.ts +35 -33
  215. package/src/voice/runtype-voice-provider.ts +670 -189
  216. package/src/voice/session-voice-wiring.ts +266 -0
  217. package/src/voice/voice-delegation.test.ts +71 -0
  218. package/src/voice/voice-delegation.ts +364 -0
  219. package/src/voice/voice-turn-tracker.ts +6 -6
  220. package/src/voice/voice.test.ts +981 -8
  221. package/src/voice/worklet-playback-engine.test.ts +81 -0
  222. package/src/voice/worklet-playback-engine.ts +21 -3
  223. package/src/voice-runtime-loader.ts +4 -1
  224. package/src/voice-runtime.ts +7 -0
  225. package/src/webmcp-runtime-entry.ts +41 -39
  226. package/dist/approval-ui-3NMYNXBD.js +0 -1
  227. package/dist/chunk-5EIIHQLQ.js +0 -1
  228. package/dist/chunk-7CO4T5KB.js +0 -1
  229. package/dist/chunk-EFJBUH5F.js +0 -1
  230. package/dist/chunk-R2TMIMDI.js +0 -3
  231. package/dist/chunk-UTVLOTWB.js +0 -9
  232. package/dist/context-mentions-45MODWOY.js +0 -161
  233. package/dist/event-stream-view-GL3ADRY2.js +0 -1
  234. package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
  235. package/dist/voice-runtime-7DA7MJH4.js +0 -1
  236. package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
package/src/types.ts CHANGED
@@ -4766,6 +4766,45 @@ export type AgentWidgetVoiceRecognitionConfig = {
4766
4766
  * @default 30000
4767
4767
  */
4768
4768
  attachIdleMs?: number;
4769
+ /**
4770
+ * Full-duplex (speech-to-speech, e.g. GPT-Live) calls: run the turns the
4771
+ * voice model hands off to the agent through this widget's normal chat
4772
+ * pipeline. The spoken request becomes the user message, and the answer
4773
+ * renders as a regular assistant message (streaming, tool and approval
4774
+ * UI, WebMCP page tools, persistence) that the voice model then reads
4775
+ * aloud. Only takes effect when the server supports it (it confirms in
4776
+ * `session_config`); otherwise the agent turn runs on the server and its
4777
+ * spoken reply is transcribed, as before. Set `false` to always keep the
4778
+ * agent turn on the server.
4779
+ * @default true
4780
+ */
4781
+ clientDelegation?: boolean;
4782
+ /**
4783
+ * Full-duplex calls: extra context for the voice model (for example the
4784
+ * page the visitor is on), sent after the chat history as of call start.
4785
+ * The frame goes out once, when the visitor finishes their first
4786
+ * utterance (or at the first agent hand-off). A function is
4787
+ * called at call start and may be async. This text is capped at 4000 characters and the whole frame
4788
+ * (history included) at 8000. Sent only to servers that accept call
4789
+ * context (they announce it in `session_config`).
4790
+ */
4791
+ callContext?: string | (() => string | Promise<string>);
4792
+ /**
4793
+ * Full-duplex calls with client delegation: how long a tool approval
4794
+ * that a spoken request raised may wait before it is declined as
4795
+ * expired (and the voice model says so). Server-side approval gates
4796
+ * keep their own timeout too. Hanging up never declines it: the
4797
+ * approval card in the chat stays usable. Capped at 540000 (9 minutes),
4798
+ * under the server's 10-minute delegation deadline.
4799
+ * @default 300000
4800
+ */
4801
+ approvalTimeoutMs?: number;
4802
+ /**
4803
+ * Speech-to-speech calls: the one-line notice shown in the composer
4804
+ * status area when the call starts. `false` hides it.
4805
+ * @default "You're talking to an AI assistant. Voice is processed by OpenAI."
4806
+ */
4807
+ disclosureText?: string | false;
4769
4808
  /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
4770
4809
  pauseDuration?: number;
4771
4810
  /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
@@ -5024,6 +5063,13 @@ export interface VoicePlaybackEngine {
5024
5063
  onFinished(callback: () => void): void;
5025
5064
  /** Release all audio resources. */
5026
5065
  destroy(): Promise<void> | void;
5066
+ /**
5067
+ * Optional. Speech-to-speech calls stream reply audio with no end-of-stream
5068
+ * marker, so an engine that holds audio below a prebuffer waterline must, while
5069
+ * continuous, release that held tail on its own (e.g. after a short input gap)
5070
+ * instead of waiting for {@link markStreamEnd}. Engines without a prebuffer can omit it.
5071
+ */
5072
+ setContinuousMode?(enabled: boolean): void;
5027
5073
  }
5028
5074
 
5029
5075
  /**
@@ -5077,6 +5123,14 @@ export type VoiceConfig = {
5077
5123
  prewarmMode?: 'request' | 'attach';
5078
5124
  /** See `voiceRecognition.provider.runtype.attachIdleMs`. @default 30000 */
5079
5125
  attachIdleMs?: number;
5126
+ /** See `voiceRecognition.provider.runtype.clientDelegation`. @default true */
5127
+ clientDelegation?: boolean;
5128
+ /** See `voiceRecognition.provider.runtype.callContext`. */
5129
+ callContext?: string | (() => string | Promise<string>);
5130
+ /** See `voiceRecognition.provider.runtype.approvalTimeoutMs`. @default 300000 (capped at 540000) */
5131
+ approvalTimeoutMs?: number;
5132
+ /** See `voiceRecognition.provider.runtype.disclosureText`. */
5133
+ disclosureText?: string | false;
5080
5134
  /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
5081
5135
  pauseDuration?: number;
5082
5136
  /** @deprecated No-op on the realtime path: the server's STT owns turn-taking. */
@@ -5114,13 +5168,24 @@ export interface VoiceProvider {
5114
5168
  * Register a callback for incremental transcript updates during a voice turn.
5115
5169
  * `isFinal=false` is a live interim update (user partials, or assistant deltas
5116
5170
  * on providers that stream them); `isFinal=true` finalizes that role's text.
5117
- * On the realtime `runtype` path, interim updates fire for the `user` only and
5118
- * the `assistant` arrives as a single final. Providers with overlapping turns
5119
- * must include the same turnId on user and assistant finals; without IDs,
5120
- * providers must discard cancelled output before emitting the next user final.
5171
+ * On the classic realtime `runtype` path, interim updates fire for the `user`
5172
+ * only and the `assistant` arrives as a single final; without a turnId the
5173
+ * session assumes strictly alternating turns, and providers must discard
5174
+ * cancelled output before emitting the next user final.
5175
+ *
5176
+ * When `metadata.turnId` is present (full-duplex / speech-to-speech, e.g.
5177
+ * GPT-Live), the session reconciles by `(turnId, role)`: each pair owns one
5178
+ * bubble that later updates replace in place, turns may overlap or arrive in
5179
+ * any order, and no empty assistant placeholder is injected. User and
5180
+ * assistant may share a turnId (a late user bubble then renders above its
5181
+ * reply) or use distinct ones. `metadata.startMs` (call-relative ms when the
5182
+ * utterance began) orders a new bubble among the call's other keyed bubbles;
5183
+ * without it bubbles order by first arrival. After a stop, keyed providers
5184
+ * must drop the cancelled reply's output until the server acknowledges the
5185
+ * cancel: the session renders every reply it receives.
5121
5186
  */
5122
5187
  onTranscript?(
5123
- callback: (role: 'user' | 'assistant', text: string, isFinal: boolean, metadata?: { turnId?: string }) => void,
5188
+ callback: (role: 'user' | 'assistant', text: string, isFinal: boolean, metadata?: VoiceTranscriptMetadata) => void,
5124
5189
  ): void;
5125
5190
 
5126
5191
  /** Register a callback for per-turn latency metrics (realtime path). */
@@ -5156,6 +5221,117 @@ export interface VoiceProvider {
5156
5221
  * repeatedly; throttle as needed.
5157
5222
  */
5158
5223
  prewarm?(): void;
5224
+
5225
+ /**
5226
+ * Called by the session once after construction with the chat-side surface
5227
+ * a full-duplex provider uses to hand agent turns to the chat pipeline
5228
+ * ({@link VoiceSessionBridge}). Providers without client delegation omit it.
5229
+ */
5230
+ setSessionBridge?(bridge: VoiceSessionBridge): void;
5231
+ /** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
5232
+ getDisclosure?(): string | null;
5233
+ }
5234
+
5235
+ /** Metadata on a {@link VoiceProvider.onTranscript} update. */
5236
+ export type VoiceTranscriptMetadata = {
5237
+ /** Groups updates that belong to one utterance bubble (full duplex). */
5238
+ turnId?: string;
5239
+ /** Call-relative ms when the utterance began; orders late bubbles. */
5240
+ startMs?: number;
5241
+ /** Call-relative ms of the latest audio this update covers. */
5242
+ endMs?: number;
5243
+ /**
5244
+ * The bubble only captions speech (see `AgentWidgetMessage.voiceCaption`).
5245
+ * The chat pipeline owns the conversation, as with client delegation.
5246
+ */
5247
+ caption?: boolean;
5248
+ };
5249
+
5250
+ /** An agent turn a full-duplex voice model handed to the widget. */
5251
+ export type VoiceDelegationRequest = {
5252
+ /** The voice model's delegation id (`delegation_started.delegationId`). */
5253
+ delegationId: string;
5254
+ /** What the visitor asked, as the voice model heard it. */
5255
+ userText: string;
5256
+ /**
5257
+ * The user transcript utterances `userText` joins, oldest first (the last
5258
+ * is the one it came from). May be empty.
5259
+ */
5260
+ userUtteranceIds: string[];
5261
+ /** The voice model's view of the conversation it delegated from. */
5262
+ messages: Array<{ role: string; content: unknown }>;
5263
+ };
5264
+
5265
+ /**
5266
+ * How a delegated turn ended, or that it waits on an approval (contract
5267
+ * Amendment 5). `pending_approval` is the only non-terminal status. Open:
5268
+ * a receiver treats an unknown terminal status as `failed`.
5269
+ * - `completed`: the work ran.
5270
+ * - `denied`: the visitor said no (Deny tap, or a spoken decline).
5271
+ * - `timeout`: the approval lapsed (`approvalTimeoutMs`, or the server's own timeout).
5272
+ * - `cancelled`: replaced by a newer request, or abandoned by the system.
5273
+ * - `failed`: an error.
5274
+ */
5275
+ export type VoiceDelegationStatus =
5276
+ | 'completed'
5277
+ | 'pending_approval'
5278
+ | 'denied'
5279
+ | 'timeout'
5280
+ | 'cancelled'
5281
+ | 'failed'
5282
+ | (string & {});
5283
+
5284
+ /** The terminal result of a turn that parked on approvals. */
5285
+ export type VoiceDelegationFollowUp = {
5286
+ status: VoiceDelegationStatus;
5287
+ /** What to read aloud (Markdown allowed). May be empty: nothing more to say. */
5288
+ text: string;
5289
+ };
5290
+
5291
+ /** The delegated turn's answer, sent back for the voice model to read aloud. */
5292
+ export type VoiceDelegationResult = {
5293
+ /** `pending_approval` when `followUp` is set; otherwise terminal. */
5294
+ status: VoiceDelegationStatus;
5295
+ /** The assistant text (Markdown allowed). */
5296
+ text: string;
5297
+ /**
5298
+ * Set when the turn parked on approvals (`text` then asks for the decision).
5299
+ * Resolves with the terminal result once the visitor decides (or the
5300
+ * approval is replaced, declined by voice, or unanswered for
5301
+ * `approvalTimeoutMs`) and the resumed turn finishes; `null` only when
5302
+ * `signal` aborts first (hang-up, or the server cancelled the delegation).
5303
+ */
5304
+ followUp?: (options: {
5305
+ signal: AbortSignal;
5306
+ approvalTimeoutMs?: number;
5307
+ /** `false`: run only the approval bookkeeping (expiry, supersede); nothing will be read back. @default true */
5308
+ readBack?: boolean;
5309
+ /** The resumed turn stopped on another approval: ask for it too (`text` is its approval script). */
5310
+ onUpdate?: (text: string) => void;
5311
+ }) => Promise<VoiceDelegationFollowUp | null>;
5312
+ };
5313
+
5314
+ /**
5315
+ * Chat-side surface the session hands a full-duplex provider: the visible
5316
+ * history (for the call-start context) and a way to run a delegated turn
5317
+ * through the normal chat pipeline.
5318
+ */
5319
+ export interface VoiceSessionBridge {
5320
+ /** Visible user/assistant messages, oldest first. */
5321
+ getHistory(): Array<{ role: 'user' | 'assistant'; content: string }>;
5322
+ /**
5323
+ * Submit the turn as the visitor's message (reusing its transcript bubble)
5324
+ * and resolve with the assistant's final answer once the turn settles.
5325
+ * Waits for any chat turn already in flight first. Never rejects.
5326
+ */
5327
+ runDelegatedTurn(request: VoiceDelegationRequest): Promise<VoiceDelegationResult>;
5328
+ /**
5329
+ * The server dropped this delegation's result (cancelled it, or refused a
5330
+ * late result): its chat answer is no longer spoken, so browser TTS may read it.
5331
+ * `expired`: the server's deadline passed, so its pending approvals are
5332
+ * declined too, as the approval TTL would.
5333
+ */
5334
+ dropDelegation?(delegationId: string, expired?: boolean): void;
5159
5335
  }
5160
5336
 
5161
5337
  /**
@@ -8038,6 +8214,11 @@ export type AgentWidgetToolCall = {
8038
8214
  startedAt?: number;
8039
8215
  completedAt?: number;
8040
8216
  durationMs?: number;
8217
+ /**
8218
+ * The call paused for an approval that was granted, and the agent re-ran it
8219
+ * as a new tool call: that later call carries the result, this one has none.
8220
+ */
8221
+ superseded?: boolean;
8041
8222
  };
8042
8223
 
8043
8224
  /**
@@ -8060,6 +8241,8 @@ export type AgentWidgetApproval = {
8060
8241
  */
8061
8242
  reason?: string;
8062
8243
  parameters?: unknown;
8244
+ /** The paused tool call this approval gates (`approval_start.toolCallId`). */
8245
+ toolCallId?: string;
8063
8246
  resolvedAt?: number;
8064
8247
  };
8065
8248
 
@@ -8162,6 +8345,13 @@ export type AgentWidgetMessage = {
8162
8345
  * }
8163
8346
  */
8164
8347
  voiceProcessing?: boolean;
8348
+ /**
8349
+ * Display-only caption of full-duplex voice speech, such as the voice
8350
+ * model's own words, or something the visitor said that wasn't submitted
8351
+ * as a chat turn. It renders in the transcript but is never sent to the
8352
+ * model as conversation.
8353
+ */
8354
+ voiceCaption?: boolean;
8165
8355
  /**
8166
8356
  * Raw structured payload for this message (e.g., JSON action response).
8167
8357
  * Populated automatically when structured parsers run.
@@ -8291,6 +8481,9 @@ export type InjectMessageOptions = {
8291
8481
  */
8292
8482
  voiceProcessing?: boolean;
8293
8483
 
8484
+ /** See {@link AgentWidgetMessage.voiceCaption}. */
8485
+ voiceCaption?: boolean;
8486
+
8294
8487
  /**
8295
8488
  * Raw structured payload (typically a JSON string) representing the
8296
8489
  * full directive that produced this message: e.g. `{ "text": "...",
@@ -0,0 +1,382 @@
1
+ /**
2
+ * Event handlers for the `ask_user_question` answer sheet that mounts in the
3
+ * composer overlay: pill pick (single), multi-select toggle + submit, free-text
4
+ * expansion + submit, grouped-question paging, skip, and dismissal. A selection
5
+ * becomes a regular user message (or a LOCAL-tool resume) so the agent
6
+ * continues on the next turn.
7
+ *
8
+ * Lives in the lazy `ui-extras` chunk: `ui.ts` warms it when a sheet mounts
9
+ * and routes overlay events here once loaded.
10
+ */
11
+ import type { AgentWidgetConfig } from "./types";
12
+ import type { AgentWidgetSession } from "./session";
13
+ import {
14
+ buildStructuredAnswers,
15
+ getCurrentIndex,
16
+ getQuestionCount,
17
+ getSelectedLabels,
18
+ isGroupedSheet,
19
+ navigateToPage,
20
+ readAnswersFromSheet,
21
+ removeAskUserQuestionSheet,
22
+ setCurrentAnswer,
23
+ } from "./components/ask-user-question-bubble";
24
+
25
+ export type AskUserSheetContext = {
26
+ mount: HTMLElement;
27
+ overlay: HTMLElement;
28
+ sessionRef: { current: AgentWidgetSession | null };
29
+ /** Read live: `controller.update()` replaces the config. */
30
+ config: () => AgentWidgetConfig;
31
+ };
32
+
33
+ export type AskUserSheetHandlers = {
34
+ click: (event: MouseEvent) => void;
35
+ keydown: (event: KeyboardEvent) => void;
36
+ };
37
+
38
+ export const createAskUserSheetHandlers = (ctx: AskUserSheetContext): AskUserSheetHandlers => {
39
+ const { mount, overlay: askUserOverlay, sessionRef } = ctx;
40
+
41
+ const submitAskUserAnswer = (
42
+ sheet: HTMLElement,
43
+ text: string,
44
+ meta: {
45
+ source: "pick" | "multi" | "free-text" | "submit-all";
46
+ values?: string[];
47
+ structured?: Record<string, string | string[]>;
48
+ }
49
+ ): void => {
50
+ const trimmed = text.trim();
51
+ if (!trimmed || !sessionRef.current) return;
52
+ const toolCallId = sheet.getAttribute("data-tool-call-id") ?? "";
53
+ const isFreeText = meta.source === "free-text";
54
+
55
+ // Dispatch before removing the sheet so listeners can still query DOM state.
56
+ mount.dispatchEvent(
57
+ new CustomEvent("persona:askUserQuestion:answered", {
58
+ detail: {
59
+ toolUseId: toolCallId,
60
+ answer: trimmed,
61
+ answers: meta.structured,
62
+ values: meta.values ?? (meta.source === "multi" ? trimmed.split(", ") : [trimmed]),
63
+ isFreeText,
64
+ source: meta.source,
65
+ },
66
+ bubbles: true,
67
+ composed: true,
68
+ })
69
+ );
70
+
71
+ removeAskUserQuestionSheet(askUserOverlay, toolCallId);
72
+
73
+ // Branch: LOCAL-tool pause (await) resumes via /resume with structured
74
+ // toolOutputs; legacy path sends as a plain user message.
75
+ const sourceMessage = sessionRef.current
76
+ .getMessages()
77
+ .find((m) => m.toolCall?.id === toolCallId);
78
+ if (sourceMessage?.agentMetadata?.awaitingLocalTool) {
79
+ sessionRef.current.resolveAskUserQuestion(sourceMessage, meta.structured ?? trimmed);
80
+ } else {
81
+ sessionRef.current.sendMessage(trimmed);
82
+ }
83
+ };
84
+
85
+ /**
86
+ * Persist in-progress grouped-question answers + page index back to the
87
+ * source message so a refresh restores the user's spot.
88
+ */
89
+ const persistGroupedProgress = (sheet: HTMLElement): void => {
90
+ const session = sessionRef.current;
91
+ if (!session) return;
92
+ const toolCallId = sheet.getAttribute("data-tool-call-id") ?? "";
93
+ const sourceMessage = session.getMessages().find((m) => m.toolCall?.id === toolCallId);
94
+ if (!sourceMessage) return;
95
+ session.persistAskUserQuestionProgress(sourceMessage, {
96
+ answers: buildStructuredAnswers(sheet, sourceMessage),
97
+ currentIndex: getCurrentIndex(sheet),
98
+ });
99
+ };
100
+
101
+ /**
102
+ * Build a one-line summary string for the legacy `answer` field on the
103
+ * answered event when submit-all fires from a grouped sheet.
104
+ */
105
+ const stringifyStructured = (answers: Record<string, string | string[]>): string => {
106
+ return Object.entries(answers)
107
+ .map(([q, v]) => `${q}: ${Array.isArray(v) ? v.join(", ") : v}`)
108
+ .join(" | ");
109
+ };
110
+
111
+ /**
112
+ * If `groupedAutoAdvance` is enabled (default) and we're not on the final
113
+ * page, advance one step. The final page never auto-submits: users always
114
+ * confirm with an explicit Submit-all click so they can review.
115
+ */
116
+ const maybeAutoAdvance = (sheet: HTMLElement): void => {
117
+ if (ctx.config().features?.askUserQuestion?.groupedAutoAdvance === false) return;
118
+ const idx = getCurrentIndex(sheet);
119
+ const count = getQuestionCount(sheet);
120
+ if (idx >= count - 1) return;
121
+ const sourceMessage = sessionRef.current
122
+ ?.getMessages()
123
+ .find((m) => m.toolCall?.id === sheet.getAttribute("data-tool-call-id"));
124
+ if (!sourceMessage) return;
125
+ navigateToPage(sheet, sourceMessage, ctx.config(), idx + 1);
126
+ persistGroupedProgress(sheet);
127
+ };
128
+
129
+ const click = (event: MouseEvent): void => {
130
+ const target = event.target as HTMLElement;
131
+ const trigger = target.closest<HTMLElement>("[data-ask-user-action]");
132
+ if (!trigger) return;
133
+ const sheet = trigger.closest<HTMLElement>("[data-persona-ask-sheet-for]");
134
+ if (!sheet) return;
135
+
136
+ const action = trigger.getAttribute("data-ask-user-action");
137
+ event.preventDefault();
138
+ event.stopPropagation();
139
+
140
+ if (action === "dismiss") {
141
+ const toolCallId = sheet.getAttribute("data-tool-call-id") ?? "";
142
+ mount.dispatchEvent(
143
+ new CustomEvent("persona:askUserQuestion:dismissed", {
144
+ detail: { toolUseId: toolCallId },
145
+ bubbles: true,
146
+ composed: true,
147
+ })
148
+ );
149
+ removeAskUserQuestionSheet(askUserOverlay, toolCallId);
150
+
151
+ // Best-effort: if this sheet corresponds to a LOCAL-awaiting tool,
152
+ // unblock the paused execution with a sentinel answer so the server
153
+ // doesn't sit in waiting_for_local forever. Fire-and-forget: errors
154
+ // are surfaced to the onError callback. Flip the answered flag first
155
+ // so a racing render pass doesn't re-mount the sheet mid-dismissal.
156
+ const sourceMessage = sessionRef.current
157
+ ?.getMessages()
158
+ .find((m) => m.toolCall?.id === toolCallId);
159
+ if (sourceMessage?.agentMetadata?.awaitingLocalTool) {
160
+ sessionRef.current?.markAskUserQuestionResolved(sourceMessage);
161
+ sessionRef.current?.resolveAskUserQuestion(sourceMessage, "(dismissed)");
162
+ }
163
+ return;
164
+ }
165
+
166
+ if (action === "pick") {
167
+ const label = trigger.getAttribute("data-option-label");
168
+ if (!label) return;
169
+ const multiSelect = sheet.getAttribute("data-multi-select") === "true";
170
+ const grouped = isGroupedSheet(sheet);
171
+
172
+ if (grouped && multiSelect) {
173
+ const stored = readAnswersFromSheet(sheet)[getCurrentIndex(sheet)];
174
+ const set = new Set<string>(Array.isArray(stored) ? stored : []);
175
+ if (set.has(label)) set.delete(label);
176
+ else set.add(label);
177
+ setCurrentAnswer(sheet, Array.from(set));
178
+ persistGroupedProgress(sheet);
179
+ return;
180
+ }
181
+
182
+ if (grouped) {
183
+ setCurrentAnswer(sheet, label);
184
+ persistGroupedProgress(sheet);
185
+ maybeAutoAdvance(sheet);
186
+ return;
187
+ }
188
+
189
+ // 1-question modes: preserve original UX.
190
+ if (multiSelect) {
191
+ const pressed = trigger.getAttribute("aria-pressed") === "true";
192
+ trigger.setAttribute("aria-pressed", pressed ? "false" : "true");
193
+ trigger.classList.toggle("persona-ask-pill-selected", !pressed);
194
+ const submitBtn = sheet.querySelector<HTMLButtonElement>(
195
+ '[data-ask-user-action="submit-multi"]'
196
+ );
197
+ if (submitBtn) {
198
+ submitBtn.disabled = getSelectedLabels(sheet).length === 0;
199
+ }
200
+ return;
201
+ }
202
+ submitAskUserAnswer(sheet, label, { source: "pick", values: [label] });
203
+ return;
204
+ }
205
+
206
+ if (action === "submit-multi") {
207
+ const labels = getSelectedLabels(sheet);
208
+ if (labels.length === 0) return;
209
+ submitAskUserAnswer(sheet, labels.join(", "), {
210
+ source: "multi",
211
+ values: labels,
212
+ });
213
+ return;
214
+ }
215
+
216
+ if (action === "open-free-text") {
217
+ const row = sheet.querySelector<HTMLElement>('[data-ask-free-text-row="true"]');
218
+ if (row) {
219
+ row.classList.remove("persona-hidden");
220
+ const input = row.querySelector<HTMLInputElement>('[data-ask-free-text-input="true"]');
221
+ input?.focus();
222
+ }
223
+ return;
224
+ }
225
+
226
+ if (action === "focus-free-text") {
227
+ // Rows-layout Other row: input lives inside the row container itself.
228
+ // Native click on the input already focuses it; this branch handles
229
+ // clicks on the badge or row chrome AND digit-shortcut activations.
230
+ const input = sheet.querySelector<HTMLInputElement>('[data-ask-free-text-input="true"]');
231
+ input?.focus();
232
+ return;
233
+ }
234
+
235
+ if (action === "submit-free-text") {
236
+ const input = sheet.querySelector<HTMLInputElement>('[data-ask-free-text-input="true"]');
237
+ const text = input?.value ?? "";
238
+ if (!text.trim()) return;
239
+ if (isGroupedSheet(sheet)) {
240
+ setCurrentAnswer(sheet, text.trim());
241
+ persistGroupedProgress(sheet);
242
+ maybeAutoAdvance(sheet);
243
+ return;
244
+ }
245
+ submitAskUserAnswer(sheet, text, { source: "free-text" });
246
+ return;
247
+ }
248
+
249
+ if (action === "next" || action === "back") {
250
+ if (!sessionRef.current) return;
251
+ const toolCallId = sheet.getAttribute("data-tool-call-id") ?? "";
252
+ const sourceMessage = sessionRef.current
253
+ .getMessages()
254
+ .find((m) => m.toolCall?.id === toolCallId);
255
+ if (!sourceMessage) return;
256
+ // Flush any unsubmitted free-text input as the current answer.
257
+ const freeInput = sheet.querySelector<HTMLInputElement>('[data-ask-free-text-input="true"]');
258
+ const pending = freeInput?.value?.trim() ?? "";
259
+ if (pending) {
260
+ const stored = readAnswersFromSheet(sheet)[getCurrentIndex(sheet)];
261
+ if (typeof stored !== "string" || stored !== pending) {
262
+ setCurrentAnswer(sheet, pending);
263
+ }
264
+ }
265
+ const direction = action === "next" ? 1 : -1;
266
+ const nextIdx = getCurrentIndex(sheet) + direction;
267
+ navigateToPage(sheet, sourceMessage, ctx.config(), nextIdx);
268
+ persistGroupedProgress(sheet);
269
+ return;
270
+ }
271
+
272
+ if (action === "submit-all") {
273
+ if (!sessionRef.current) return;
274
+ const toolCallId = sheet.getAttribute("data-tool-call-id") ?? "";
275
+ const sourceMessage = sessionRef.current
276
+ .getMessages()
277
+ .find((m) => m.toolCall?.id === toolCallId);
278
+ if (!sourceMessage) return;
279
+ // Flush any pending free-text on the final page first.
280
+ const freeInput = sheet.querySelector<HTMLInputElement>('[data-ask-free-text-input="true"]');
281
+ const pending = freeInput?.value?.trim() ?? "";
282
+ if (pending) setCurrentAnswer(sheet, pending);
283
+
284
+ const structured = buildStructuredAnswers(sheet, sourceMessage);
285
+ // Persist final answers to message metadata BEFORE resolving so the
286
+ // answered-state review card (which reads `agentMetadata
287
+ // .askUserQuestionAnswers`) shows the user's actual picks instead of
288
+ // "(skipped)" placeholders. Without this, any answer set only via the
289
+ // pending-flush above (or via paths that bypassed the per-pick persist
290
+ // hook) would be missing from the transcript review even though it
291
+ // landed in the structured payload sent to the agent.
292
+ sessionRef.current.persistAskUserQuestionProgress(sourceMessage, {
293
+ answers: structured,
294
+ currentIndex: getCurrentIndex(sheet),
295
+ });
296
+ const summary = stringifyStructured(structured);
297
+ submitAskUserAnswer(sheet, summary || "(submitted)", {
298
+ source: "submit-all",
299
+ structured,
300
+ });
301
+ return;
302
+ }
303
+
304
+ if (action === "skip") {
305
+ if (!sessionRef.current) return;
306
+ const toolCallId = sheet.getAttribute("data-tool-call-id") ?? "";
307
+ const sourceMessage = sessionRef.current
308
+ .getMessages()
309
+ .find((m) => m.toolCall?.id === toolCallId);
310
+ if (!sourceMessage) return;
311
+
312
+ const grouped = isGroupedSheet(sheet);
313
+ const idx = getCurrentIndex(sheet);
314
+ const count = getQuestionCount(sheet);
315
+ const isFinal = idx >= count - 1;
316
+
317
+ // Single-question payloads behave like dismiss.
318
+ if (!grouped) {
319
+ mount.dispatchEvent(
320
+ new CustomEvent("persona:askUserQuestion:dismissed", {
321
+ detail: { toolUseId: toolCallId },
322
+ bubbles: true,
323
+ composed: true,
324
+ })
325
+ );
326
+ removeAskUserQuestionSheet(askUserOverlay, toolCallId);
327
+ if (sourceMessage.agentMetadata?.awaitingLocalTool) {
328
+ sessionRef.current.markAskUserQuestionResolved(sourceMessage);
329
+ sessionRef.current.resolveAskUserQuestion(sourceMessage, "(dismissed)");
330
+ }
331
+ return;
332
+ }
333
+
334
+ // Drop the current question's answer (if any) so it's absent from the
335
+ // resolved Record. setCurrentAnswer with an empty string deletes the
336
+ // index from the in-memory map.
337
+ setCurrentAnswer(sheet, "");
338
+ // Also clear any unsubmitted free-text on this page.
339
+ const freeInput = sheet.querySelector<HTMLInputElement>('[data-ask-free-text-input="true"]');
340
+ if (freeInput) freeInput.value = "";
341
+
342
+ if (isFinal) {
343
+ // Submit with whatever has been recorded so far.
344
+ const structured = buildStructuredAnswers(sheet, sourceMessage);
345
+ const summary = stringifyStructured(structured);
346
+ submitAskUserAnswer(sheet, summary || "(skipped)", {
347
+ source: "submit-all",
348
+ structured,
349
+ });
350
+ return;
351
+ }
352
+
353
+ // Intermediate page: advance one step without recording.
354
+ navigateToPage(sheet, sourceMessage, ctx.config(), idx + 1);
355
+ persistGroupedProgress(sheet);
356
+ return;
357
+ }
358
+ };
359
+
360
+ // Enter on the free-text input → submit. Stays on the overlay because the
361
+ // event target IS the input, which lives inside the overlay subtree.
362
+ const keydown = (event: KeyboardEvent): void => {
363
+ if (event.key !== "Enter") return;
364
+ const target = event.target as HTMLElement;
365
+ const input = target as HTMLInputElement;
366
+ if (!input.matches?.('[data-ask-free-text-input="true"]')) return;
367
+ const sheet = input.closest<HTMLElement>("[data-persona-ask-sheet-for]");
368
+ if (!sheet) return;
369
+ event.preventDefault();
370
+ const text = input.value;
371
+ if (!text.trim()) return;
372
+ if (isGroupedSheet(sheet)) {
373
+ setCurrentAnswer(sheet, text.trim());
374
+ persistGroupedProgress(sheet);
375
+ maybeAutoAdvance(sheet);
376
+ return;
377
+ }
378
+ submitAskUserAnswer(sheet, text, { source: "free-text" });
379
+ };
380
+
381
+ return { click, keydown };
382
+ };
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Eager `ui-extras` registration for the bundled (ESM / CJS) builds and tests,
3
+ * so the features behind it respond synchronously. Like
4
+ * `markdown-parsers-eager.ts`, this module must NOT be reachable from
5
+ * `index-global.ts`: the IIFE/CDN build fetches the `ui-extras.js` chunk.
6
+ */
7
+ import * as uiExtras from "./ui-extras-entry";
8
+ import { provideUiExtras } from "./ui-extras-loader";
9
+
10
+ provideUiExtras(uiExtras);
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Entry for the lazy `ui-extras` chunk: `ui.ts` glue for opt-in or
3
+ * interaction-only features, kept out of the IIFE/CDN core bundle. See
4
+ * `ui-extras-loader.ts`.
5
+ */
6
+ export { createAskUserSheetHandlers } from "./ui-ask-user-sheet";
7
+ // Context-mention orchestrator (opt-in `contextMentions.enabled`). Callers pass
8
+ // core's loaders / icon registry via `deps` so this chunk's copies stay unused.
9
+ export { createContextMentionOrchestrator } from "./utils/context-mention-orchestrator";