@memberjunction/ng-conversations 6.2.0-edge.5 → 6.2.0-edge.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/dist/lib/components/attachment/image-viewer.component.d.ts +3 -1
  2. package/dist/lib/components/attachment/image-viewer.component.d.ts.map +1 -1
  3. package/dist/lib/components/attachment/image-viewer.component.js +5 -1
  4. package/dist/lib/components/attachment/image-viewer.component.js.map +1 -1
  5. package/dist/lib/components/collection/collection-view.component.d.ts.map +1 -1
  6. package/dist/lib/components/collection/collection-view.component.js +1 -0
  7. package/dist/lib/components/collection/collection-view.component.js.map +1 -1
  8. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts +188 -9
  9. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts.map +1 -1
  10. package/dist/lib/components/conversation/conversation-chat-area.component.js +1333 -464
  11. package/dist/lib/components/conversation/conversation-chat-area.component.js.map +1 -1
  12. package/dist/lib/components/conversation/conversation-mode-picker.component.js +2 -2
  13. package/dist/lib/components/conversation/conversation-mode-picker.component.js.map +1 -1
  14. package/dist/lib/components/conversation/conversation-remote-browser-controller.d.ts +109 -0
  15. package/dist/lib/components/conversation/conversation-remote-browser-controller.d.ts.map +1 -0
  16. package/dist/lib/components/conversation/conversation-remote-browser-controller.js +298 -0
  17. package/dist/lib/components/conversation/conversation-remote-browser-controller.js.map +1 -0
  18. package/dist/lib/components/message/agent-handoff-card.component.d.ts +102 -0
  19. package/dist/lib/components/message/agent-handoff-card.component.d.ts.map +1 -0
  20. package/dist/lib/components/message/agent-handoff-card.component.js +439 -0
  21. package/dist/lib/components/message/agent-handoff-card.component.js.map +1 -0
  22. package/dist/lib/components/message/message-input.component.d.ts +126 -59
  23. package/dist/lib/components/message/message-input.component.d.ts.map +1 -1
  24. package/dist/lib/components/message/message-input.component.js +559 -554
  25. package/dist/lib/components/message/message-input.component.js.map +1 -1
  26. package/dist/lib/components/message/message-item.component.d.ts +34 -2
  27. package/dist/lib/components/message/message-item.component.d.ts.map +1 -1
  28. package/dist/lib/components/message/message-item.component.js +678 -520
  29. package/dist/lib/components/message/message-item.component.js.map +1 -1
  30. package/dist/lib/components/message/message-list.component.d.ts +13 -1
  31. package/dist/lib/components/message/message-list.component.d.ts.map +1 -1
  32. package/dist/lib/components/message/message-list.component.js +92 -39
  33. package/dist/lib/components/message/message-list.component.js.map +1 -1
  34. package/dist/lib/components/overlay/chat-overlay.component.d.ts +19 -0
  35. package/dist/lib/components/overlay/chat-overlay.component.d.ts.map +1 -1
  36. package/dist/lib/components/overlay/chat-overlay.component.js +125 -13
  37. package/dist/lib/components/overlay/chat-overlay.component.js.map +1 -1
  38. package/dist/lib/components/project/project-form-modal.component.js +2 -2
  39. package/dist/lib/components/project/project-form-modal.component.js.map +1 -1
  40. package/dist/lib/components/realtime/interactive-component/run-view-component-artifact-source.js +1 -1
  41. package/dist/lib/components/realtime/interactive-component/run-view-component-artifact-source.js.map +1 -1
  42. package/dist/lib/components/realtime/media/media-channel-state.d.ts +5 -0
  43. package/dist/lib/components/realtime/media/media-channel-state.d.ts.map +1 -1
  44. package/dist/lib/components/realtime/media/media-channel-state.js +27 -1
  45. package/dist/lib/components/realtime/media/media-channel-state.js.map +1 -1
  46. package/dist/lib/components/realtime/media/media-channel.d.ts.map +1 -1
  47. package/dist/lib/components/realtime/media/media-channel.js +7 -2
  48. package/dist/lib/components/realtime/media/media-channel.js.map +1 -1
  49. package/dist/lib/components/realtime/media/realtime-media-surface.component.d.ts +19 -2
  50. package/dist/lib/components/realtime/media/realtime-media-surface.component.d.ts.map +1 -1
  51. package/dist/lib/components/realtime/media/realtime-media-surface.component.js +59 -10
  52. package/dist/lib/components/realtime/media/realtime-media-surface.component.js.map +1 -1
  53. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts +22 -6
  54. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts.map +1 -1
  55. package/dist/lib/components/realtime/realtime-agent-banner.component.js +160 -98
  56. package/dist/lib/components/realtime/realtime-agent-banner.component.js.map +1 -1
  57. package/dist/lib/components/realtime/realtime-call-failure-notice.d.ts +13 -0
  58. package/dist/lib/components/realtime/realtime-call-failure-notice.d.ts.map +1 -0
  59. package/dist/lib/components/realtime/realtime-call-failure-notice.js +21 -0
  60. package/dist/lib/components/realtime/realtime-call-failure-notice.js.map +1 -0
  61. package/dist/lib/components/realtime/realtime-composer.component.d.ts +26 -3
  62. package/dist/lib/components/realtime/realtime-composer.component.d.ts.map +1 -1
  63. package/dist/lib/components/realtime/realtime-composer.component.js +226 -120
  64. package/dist/lib/components/realtime/realtime-composer.component.js.map +1 -1
  65. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts +73 -16
  66. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts.map +1 -1
  67. package/dist/lib/components/realtime/realtime-session-overlay.component.js +219 -108
  68. package/dist/lib/components/realtime/realtime-session-overlay.component.js.map +1 -1
  69. package/dist/lib/components/realtime/realtime-ui-config.d.ts +9 -12
  70. package/dist/lib/components/realtime/realtime-ui-config.d.ts.map +1 -1
  71. package/dist/lib/components/realtime/realtime-ui-config.js +38 -18
  72. package/dist/lib/components/realtime/realtime-ui-config.js.map +1 -1
  73. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map +1 -1
  74. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js +34 -5
  75. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js.map +1 -1
  76. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts +8 -2
  77. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts.map +1 -1
  78. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js +51 -10
  79. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js.map +1 -1
  80. package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.d.ts +46 -0
  81. package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.d.ts.map +1 -0
  82. package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.js +64 -0
  83. package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.js.map +1 -0
  84. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts +131 -9
  85. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts.map +1 -1
  86. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js +347 -38
  87. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js.map +1 -1
  88. package/dist/lib/conversations.module.d.ts +2 -1
  89. package/dist/lib/conversations.module.d.ts.map +1 -1
  90. package/dist/lib/conversations.module.js +9 -4
  91. package/dist/lib/conversations.module.js.map +1 -1
  92. package/dist/lib/events/chat-events.d.ts +5 -4
  93. package/dist/lib/events/chat-events.d.ts.map +1 -1
  94. package/dist/lib/events/chat-events.js +2 -2
  95. package/dist/lib/events/chat-events.js.map +1 -1
  96. package/dist/lib/models/agent-turn.model.d.ts +11 -7
  97. package/dist/lib/models/agent-turn.model.d.ts.map +1 -1
  98. package/dist/lib/models/agent-turn.model.js.map +1 -1
  99. package/dist/lib/services/active-tasks.service.d.ts +5 -0
  100. package/dist/lib/services/active-tasks.service.d.ts.map +1 -1
  101. package/dist/lib/services/active-tasks.service.js +16 -0
  102. package/dist/lib/services/active-tasks.service.js.map +1 -1
  103. package/dist/lib/services/agent-state.service.d.ts.map +1 -1
  104. package/dist/lib/services/agent-state.service.js +7 -1
  105. package/dist/lib/services/agent-state.service.js.map +1 -1
  106. package/dist/lib/services/artifact-permission.service.d.ts.map +1 -1
  107. package/dist/lib/services/artifact-permission.service.js +2 -0
  108. package/dist/lib/services/artifact-permission.service.js.map +1 -1
  109. package/dist/lib/services/artifact-state.service.d.ts.map +1 -1
  110. package/dist/lib/services/artifact-state.service.js +1 -0
  111. package/dist/lib/services/artifact-state.service.js.map +1 -1
  112. package/dist/lib/services/collection-permission.service.d.ts.map +1 -1
  113. package/dist/lib/services/collection-permission.service.js +1 -0
  114. package/dist/lib/services/collection-permission.service.js.map +1 -1
  115. package/dist/lib/services/conversation-agent.service.d.ts +38 -16
  116. package/dist/lib/services/conversation-agent.service.d.ts.map +1 -1
  117. package/dist/lib/services/conversation-agent.service.js +57 -71
  118. package/dist/lib/services/conversation-agent.service.js.map +1 -1
  119. package/dist/lib/services/conversation-detail-window.store.d.ts +5 -0
  120. package/dist/lib/services/conversation-detail-window.store.d.ts.map +1 -1
  121. package/dist/lib/services/conversation-detail-window.store.js +52 -5
  122. package/dist/lib/services/conversation-detail-window.store.js.map +1 -1
  123. package/dist/lib/services/realtime-session-review.service.d.ts +2 -1
  124. package/dist/lib/services/realtime-session-review.service.d.ts.map +1 -1
  125. package/dist/lib/services/realtime-session-review.service.js.map +1 -1
  126. package/dist/lib/services/realtime-session.service.d.ts.map +1 -1
  127. package/dist/lib/services/realtime-session.service.js +2 -1
  128. package/dist/lib/services/realtime-session.service.js.map +1 -1
  129. package/dist/lib/services/search.service.d.ts.map +1 -1
  130. package/dist/lib/services/search.service.js +1 -0
  131. package/dist/lib/services/search.service.js.map +1 -1
  132. package/dist/lib/utils/agent-failure-display.d.ts +14 -0
  133. package/dist/lib/utils/agent-failure-display.d.ts.map +1 -0
  134. package/dist/lib/utils/agent-failure-display.js +26 -0
  135. package/dist/lib/utils/agent-failure-display.js.map +1 -0
  136. package/dist/lib/utils/agent-handoff-map.d.ts +52 -0
  137. package/dist/lib/utils/agent-handoff-map.d.ts.map +1 -0
  138. package/dist/lib/utils/agent-handoff-map.js +270 -0
  139. package/dist/lib/utils/agent-handoff-map.js.map +1 -0
  140. package/dist/lib/utils/agent-turn-routing.d.ts +62 -7
  141. package/dist/lib/utils/agent-turn-routing.d.ts.map +1 -1
  142. package/dist/lib/utils/agent-turn-routing.js +147 -6
  143. package/dist/lib/utils/agent-turn-routing.js.map +1 -1
  144. package/dist/lib/utils/decision-routing.d.ts +17 -9
  145. package/dist/lib/utils/decision-routing.d.ts.map +1 -1
  146. package/dist/lib/utils/decision-routing.js +23 -14
  147. package/dist/lib/utils/decision-routing.js.map +1 -1
  148. package/dist/lib/utils/message-list-order.d.ts +13 -0
  149. package/dist/lib/utils/message-list-order.d.ts.map +1 -0
  150. package/dist/lib/utils/message-list-order.js +31 -0
  151. package/dist/lib/utils/message-list-order.js.map +1 -0
  152. package/dist/lib/utils/pin-state.d.ts +11 -0
  153. package/dist/lib/utils/pin-state.d.ts.map +1 -0
  154. package/dist/lib/utils/pin-state.js +9 -0
  155. package/dist/lib/utils/pin-state.js.map +1 -0
  156. package/dist/lib/utils/thread-replies.d.ts +34 -0
  157. package/dist/lib/utils/thread-replies.d.ts.map +1 -0
  158. package/dist/lib/utils/thread-replies.js +49 -0
  159. package/dist/lib/utils/thread-replies.js.map +1 -0
  160. package/dist/public-api.d.ts +6 -0
  161. package/dist/public-api.d.ts.map +1 -1
  162. package/dist/public-api.js +6 -0
  163. package/dist/public-api.js.map +1 -1
  164. package/package.json +36 -36
@@ -4,10 +4,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
4
4
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
5
  return c > 3 && r && Object.defineProperty(target, key, r), r;
6
6
  };
7
- import { RegisterClass } from '@memberjunction/global';
8
- import { REALTIME_CHANNEL_CONTRACT_VERSION } from '@memberjunction/ai-core-plus';
7
+ import { CompareExposure, REALTIME_CHANNEL_CONTRACT_VERSION } from '@memberjunction/ai-core-plus';
9
8
  import { BaseRealtimeChannelClient, BuildToolBackedVerbs } from '@memberjunction/realtime-runtime';
10
- import { ApplyWhiteboardAgentTool, BuildWhiteboardExportSvg, RealtimeWhiteboardHostComponent, WHITEBOARD_TOOL_DEFINITIONS, WHITEBOARD_TOOL_PREFIX, WhiteboardState } from '@memberjunction/ng-whiteboard';
9
+ import { IsPlainObject, RegisterClass } from '@memberjunction/global';
10
+ import { ApplyWhiteboardAgentTool, BuildWhiteboardExportSvg, IsWhiteboardWidgetToolAllowed, IsWhiteboardWidgetToolName, MinWhiteboardWidgetAccess, RealtimeWhiteboardHostComponent, WHITEBOARD_TOOL_DEFINITIONS, WHITEBOARD_TOOL_NAMES, WHITEBOARD_TOOL_PREFIX, WHITEBOARD_WIDGET_TOOL_NAMES, WhiteboardState, SameWhiteboardWidgetActions } from '@memberjunction/ng-whiteboard';
11
+ import { DEFAULT_WHITEBOARD_CHANNEL_CONFIG, ParseWhiteboardChannelConfig } from './whiteboard-channel-config';
11
12
  /**
12
13
  * Whether a whiteboard tool result reports success.
13
14
  *
@@ -104,6 +105,103 @@ export async function RasterizeSvgToJpegBase64(svg, width = 1280, height = 720)
104
105
  * collapse to one trailing-edge note instead of flooding the model's context).
105
106
  */
106
107
  export const WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS = 4000;
108
+ /** Every widget tool's result carries the widget's contents (outline, state), so it requires `'state'` exposure. */
109
+ const WIDGET_TOOLS_RETURN_STATE = Object.fromEntries(WHITEBOARD_WIDGET_TOOL_NAMES.map(name => [name, 'state']));
110
+ /**
111
+ * Per-widget throttle window for "widget offers actions" perception notes (same leading +
112
+ * trailing, latest-wins shape as the ambient interaction notes).
113
+ */
114
+ export const WHITEBOARD_ACTIONS_NOTE_THROTTLE_MS = 4000;
115
+ /**
116
+ * Minimum spacing (ms) between the save requests a widget-state change triggers. The
117
+ * session runtime debounces saves 3 s and RE-ARMS on every request, so a chatty widget
118
+ * requesting on every keystroke would never save until teardown; spacing requests further
119
+ * apart than the debounce guarantees each one lands.
120
+ */
121
+ export const WHITEBOARD_WIDGET_STATE_SAVE_THROTTLE_MS = 5000;
122
+ /** The JSON key the widget states ride under, beside the board's own fields, in the session save payload. */
123
+ export const WHITEBOARD_WIDGET_STATES_SAVE_KEY = 'widgetStates';
124
+ /** A JSON tool result for a widget command the channel refused before it reached the board. */
125
+ function widgetToolFailure(errorCode, error) {
126
+ const result = { success: false, error, errorCode };
127
+ return JSON.stringify(result);
128
+ }
129
+ /** Parses tool arguments into a JSON object, or null when they are not one. */
130
+ function parseToolArgs(argsJson) {
131
+ try {
132
+ const parsed = JSON.parse(argsJson);
133
+ // JSON.parse output is JSON by construction, so narrowing a plain object to JSONObject is not a type lie.
134
+ return IsPlainObject(parsed) ? parsed : null;
135
+ }
136
+ catch {
137
+ return null;
138
+ }
139
+ }
140
+ /** Whether a widget event came from the agent's own operate command (absent Author = the user). */
141
+ function isAgentAuthored(event) {
142
+ return event.Author === 'agent';
143
+ }
144
+ /** Reads the `widgetStates` sibling key of a saved payload; anything malformed reads as none. */
145
+ function readSavedWidgetStates(stateJson) {
146
+ const saved = parseToolArgs(stateJson);
147
+ const raw = saved ? saved[WHITEBOARD_WIDGET_STATES_SAVE_KEY] : undefined;
148
+ const out = {};
149
+ if (IsPlainObject(raw)) {
150
+ for (const [id, value] of Object.entries(raw)) {
151
+ if (typeof value === 'string') {
152
+ out[id] = value;
153
+ }
154
+ }
155
+ }
156
+ return out;
157
+ }
158
+ /**
159
+ * Per-key leading + trailing throttle for background notes: outside an open window a value
160
+ * is sent immediately and opens a window; within the window the LATEST value is stashed and
161
+ * a trailing-edge timer sends it when the window closes (one note per key per window).
162
+ */
163
+ class KeyedNoteThrottle {
164
+ windowMs;
165
+ send;
166
+ entries = new Map();
167
+ constructor(windowMs, send) {
168
+ this.windowMs = windowMs;
169
+ this.send = send;
170
+ }
171
+ /** Offers one value for `key`; it is sent now or at the end of the key's open window. */
172
+ Push(key, value) {
173
+ const now = Date.now();
174
+ const entry = this.entries.get(key);
175
+ if (!entry || (entry.Timer === null && now - entry.LastSentAt >= this.windowMs)) {
176
+ this.send(value);
177
+ this.entries.set(key, { LastSentAt: now, Timer: null, Pending: null });
178
+ return;
179
+ }
180
+ entry.Pending = value; // latest wins within the window
181
+ if (entry.Timer === null) {
182
+ const wait = Math.max(0, entry.LastSentAt + this.windowMs - now);
183
+ entry.Timer = setTimeout(() => this.flush(entry), wait);
184
+ }
185
+ }
186
+ /** Cancels every pending trailing note and resets all windows. */
187
+ Clear() {
188
+ for (const entry of this.entries.values()) {
189
+ if (entry.Timer !== null) {
190
+ clearTimeout(entry.Timer);
191
+ }
192
+ }
193
+ this.entries.clear();
194
+ }
195
+ flush(entry) {
196
+ entry.Timer = null;
197
+ const pending = entry.Pending;
198
+ entry.Pending = null;
199
+ if (pending !== null) {
200
+ entry.LastSentAt = Date.now();
201
+ this.send(pending);
202
+ }
203
+ }
204
+ }
107
205
  /**
108
206
  * The LIVE WHITEBOARD as a pluggable interactive channel — the canonical
109
207
  * {@link BaseRealtimeChannelClient} implementation, resolved from the `MJ: AI Agent
@@ -144,7 +242,21 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
144
242
  /** Board-mutation subscription driving the debounced state-of-record save. */
145
243
  stateChangedSub = null;
146
244
  /** Per-widget ambient-interaction note throttles (ItemID → window state). */
147
- interactionThrottles = new Map();
245
+ interactionThrottles = new KeyedNoteThrottle(WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS, (interaction) => this.sendInteractionNote(interaction));
246
+ /** Per-widget "offers actions" note throttles (ItemID → window state). */
247
+ actionsNoteThrottles = new KeyedNoteThrottle(WHITEBOARD_ACTIONS_NOTE_THROTTLE_MS, (actions) => this.sendActionsNote(actions));
248
+ /** Per-widget action list of the last "offers actions" note SENT (ItemID → actions), for dedupe. */
249
+ lastActionsNoted = new Map();
250
+ /** The parsed `ChannelConfig` (defaults until {@link OnInitialize}). */
251
+ config = { ...DEFAULT_WHITEBOARD_CHANNEL_CONFIG };
252
+ /** Widget-state subscription driving the throttled save. */
253
+ widgetStateSub = null;
254
+ /** Trailing timer of the widget-state save throttle, or null when idle. */
255
+ widgetStateSaveTimer = null;
256
+ /** When the widget-state path last requested a save (epoch ms). */
257
+ lastWidgetStateSaveAt = Number.NEGATIVE_INFINITY;
258
+ /** True while {@link RestoreState} runs: saves are held so none carries a half-restored board. */
259
+ restoring = false;
148
260
  get ChannelName() {
149
261
  return 'Whiteboard';
150
262
  }
@@ -194,8 +306,46 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
194
306
  get SurfaceShareable() {
195
307
  return true;
196
308
  }
309
+ /**
310
+ * The board's tools. The widget tools are declared only as far as {@link MaxWidgetAccess}
311
+ * allows (`'inspect-only'` declares Inspect only, `'off'` none). Read before the channel is
312
+ * initialized, so it depends on the class alone, never on the session's config.
313
+ */
197
314
  GetToolDefinitions() {
198
- return WHITEBOARD_TOOL_DEFINITIONS;
315
+ const max = this.MaxWidgetAccess;
316
+ if (max === 'full') {
317
+ return WHITEBOARD_TOOL_DEFINITIONS;
318
+ }
319
+ return WHITEBOARD_TOOL_DEFINITIONS.filter(tool => !IsWhiteboardWidgetToolName(tool.Name) || IsWhiteboardWidgetToolAllowed(max, tool.Name));
320
+ }
321
+ // ────────────────────────────────────────────── widget access policy (extension points)
322
+ /**
323
+ * The most widget access this channel class ever offers — which widget tools it DECLARES to the
324
+ * model (tools are collected before the channel is initialized, so this cannot depend on the
325
+ * session). Default `'full'`. A host app lowers it by subclassing the channel and registering the
326
+ * subclass under `'RealtimeWhiteboardChannel'` (or via `HostChannels[].Create`).
327
+ */
328
+ get MaxWidgetAccess() {
329
+ return 'full';
330
+ }
331
+ /**
332
+ * The widget access in force for ONE call, evaluated per call (so it may change mid-session,
333
+ * e.g. `'off'` while an assessment is open). Default: the lower of {@link MaxWidgetAccess} and
334
+ * the session's `widgetAccess` config. A call it does not allow returns `access_denied`.
335
+ */
336
+ ResolveWidgetAccess() {
337
+ return MinWhiteboardWidgetAccess(this.MaxWidgetAccess, this.config.WidgetAccess);
338
+ }
339
+ /**
340
+ * Last-moment veto of one widget command that access already allows: return the refusal text the
341
+ * model should see (the call returns `vetoed` with it) or `null` to let it through. Default allows all.
342
+ */
343
+ OnBeforeWidgetCommand(_command) {
344
+ return null;
345
+ }
346
+ /** Whether widget runtime state rides in this session's saves (config `persistWidgetState`). */
347
+ get PersistWidgetState() {
348
+ return this.config.PersistWidgetState;
199
349
  }
200
350
  GetSurfaceComponent() {
201
351
  return RealtimeWhiteboardHostComponent;
@@ -234,7 +384,8 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
234
384
  Schema: { type: 'array', items: { type: 'object' } }
235
385
  }
236
386
  ],
237
- Verbs: BuildToolBackedVerbs(this.GetToolDefinitions(), this.ToolNamePrefix),
387
+ // Each widget tool returns the widget's contents, so the runtime refuses it below 'state' exposure.
388
+ Verbs: BuildToolBackedVerbs(this.GetToolDefinitions(), this.ToolNamePrefix, 'agent', WIDGET_TOOLS_RETURN_STATE),
238
389
  Events: [
239
390
  { Name: 'state_changed', Description: 'The board changed (by you or the user).' },
240
391
  { Name: 'frame_pushed', Description: 'A picture of the board was sent to you.' }
@@ -251,6 +402,7 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
251
402
  * the structured note would duplicate the deltas.
252
403
  */
253
404
  OnInitialize() {
405
+ this.applyChannelConfig();
254
406
  this.stateChangedSub?.unsubscribe();
255
407
  this.EnableVisualPerception(this, {
256
408
  ConfirmationNote: '[whiteboard] visual confirmation of your action (background — do NOT narrate or announce your own change; continue naturally)',
@@ -259,7 +411,9 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
259
411
  StartBridgePoller: true
260
412
  });
261
413
  this.stateChangedSub = this.State.Changed$.subscribe((change) => {
262
- this.Context?.RequestSave(this.State.ToJSON());
414
+ if (!this.restoring) {
415
+ this.Context?.RequestSave(this.BuildSavePayload());
416
+ }
263
417
  this.RecordChange({ Author: change.Author, Perceive: false });
264
418
  // Only user edits (and scene replacements like undo) drive the settle-debounce pipeline.
265
419
  // Agent edits are confirmed with a single frame in ApplyAgentTool.
@@ -267,6 +421,80 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
267
421
  void this.NotifyVisualChange();
268
422
  }
269
423
  });
424
+ // Widget state is runtime data (no Changed$): save it on its own, throttled cadence.
425
+ this.widgetStateSub?.unsubscribe();
426
+ this.widgetStateSub = this.State.WidgetStateChanged$.subscribe(() => this.scheduleWidgetStateSave());
427
+ }
428
+ /** Parses `ChannelConfig`; an invalid member falls back to its default with a console warning. */
429
+ applyChannelConfig() {
430
+ const parsed = ParseWhiteboardChannelConfig(this.Context?.ChannelConfig);
431
+ this.config = parsed.Config;
432
+ for (const problem of parsed.Problems) {
433
+ console.warn(`[RealtimeWhiteboardChannel] channel config: ${problem}`);
434
+ }
435
+ }
436
+ /**
437
+ * Tells the model, once the session is live, which widget tools this session's access leaves
438
+ * unavailable (notes sent before the session is live are dropped, so not from OnInitialize).
439
+ */
440
+ OnSessionStarted() {
441
+ super.OnSessionStarted();
442
+ const limit = this.describeWidgetAccessLimit(this.ResolveWidgetAccess());
443
+ if (limit) {
444
+ this.Context?.SendContextNote(`[whiteboard] ${limit}`);
445
+ }
446
+ }
447
+ /** One sentence naming the widget tools `access` leaves unavailable, or null at full access. */
448
+ describeWidgetAccessLimit(access) {
449
+ if (access === 'full') {
450
+ return null;
451
+ }
452
+ const unavailable = WHITEBOARD_WIDGET_TOOL_NAMES.filter(name => !IsWhiteboardWidgetToolAllowed(access, name));
453
+ const reason = access === 'off'
454
+ ? 'widget tools are turned off for this session'
455
+ : `widgets are inspect-only for this session — read them with ${WHITEBOARD_TOOL_NAMES.InspectWidget}, but do not try to change them`;
456
+ return `${unavailable.join(', ')} ${unavailable.length === 1 ? 'is' : 'are'} unavailable: ${reason}.`;
457
+ }
458
+ /**
459
+ * The session save payload: the board's own JSON plus, when `persistWidgetState` is on and any
460
+ * widget holds state, a sibling `widgetStates` key. Old readers ignore the unknown key; LiveKit
461
+ * sync and Save-to-artifact keep plain {@link WhiteboardState.ToJSON}.
462
+ */
463
+ BuildSavePayload() {
464
+ const boardJson = this.State.ToJSON();
465
+ if (!this.config.PersistWidgetState) {
466
+ return boardJson;
467
+ }
468
+ const widgetStates = this.State.WidgetStatesToJSON();
469
+ if (Object.keys(widgetStates).length === 0) {
470
+ return boardJson;
471
+ }
472
+ const payload = parseToolArgs(boardJson) ?? {};
473
+ payload[WHITEBOARD_WIDGET_STATES_SAVE_KEY] = widgetStates;
474
+ return JSON.stringify(payload);
475
+ }
476
+ /**
477
+ * Requests a save for a widget-state change at most once per
478
+ * {@link WHITEBOARD_WIDGET_STATE_SAVE_THROTTLE_MS} (leading edge, then one trailing request with
479
+ * the latest state). Nothing is requested when `persistWidgetState` is off.
480
+ */
481
+ scheduleWidgetStateSave() {
482
+ if (this.restoring || !this.config.PersistWidgetState || this.widgetStateSaveTimer !== null) {
483
+ return;
484
+ }
485
+ const wait = this.lastWidgetStateSaveAt + WHITEBOARD_WIDGET_STATE_SAVE_THROTTLE_MS - Date.now();
486
+ if (wait <= 0) {
487
+ this.requestWidgetStateSave();
488
+ return;
489
+ }
490
+ this.widgetStateSaveTimer = setTimeout(() => {
491
+ this.widgetStateSaveTimer = null;
492
+ this.requestWidgetStateSave();
493
+ }, wait);
494
+ }
495
+ requestWidgetStateSave() {
496
+ this.lastWidgetStateSaveAt = Date.now();
497
+ this.Context?.RequestSave(this.BuildSavePayload());
270
498
  }
271
499
  /**
272
500
  * Wires the dynamically-created board host: inputs (shared state engine + agent name)
@@ -279,6 +507,7 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
279
507
  this.host = instance;
280
508
  instance.State = this.State;
281
509
  instance.AgentName = this.Context?.AgentName ?? 'Agent';
510
+ instance.AgentWidgetAccess = this.ResolveWidgetAccess();
282
511
  this.surfaceSubs.push(
283
512
  // The board's coalesced scene delta — the perception feed the agent "sees".
284
513
  instance.SceneDelta.subscribe((deltaJson) => {
@@ -298,6 +527,11 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
298
527
  // event ran first). Surface it to the agent so it can react to quiz answers /
299
528
  // micro-form input it asked for.
300
529
  instance.WidgetSubmitted.subscribe((submit) => {
530
+ // The agent's own operate command submitted it: the tool result already carries
531
+ // `submitted`, so no note and no spoken reaction (it would answer itself).
532
+ if (isAgentAuthored(submit)) {
533
+ return;
534
+ }
301
535
  // Durable awareness first (the note persists in the model's context)…
302
536
  this.Context?.SendContextNote(`[whiteboard] the user submitted input in widget "${submit.Title || submit.ItemID}": ${submit.DataJson}`);
303
537
  // …then make the model REACT — a submission is explicit user input the user is
@@ -313,6 +547,10 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
313
547
  instance.WidgetInteraction.subscribe((interaction) => {
314
548
  this.onWidgetInteraction(interaction);
315
549
  }),
550
+ // A widget registered or changed its named actions — a throttled perception note.
551
+ instance.WidgetActionsChanged.subscribe((actions) => {
552
+ this.actionsNoteThrottles.Push(actions.ItemID, actions);
553
+ }),
316
554
  // The board's Focus toggle — ask the shell to collapse/restore the main call column.
317
555
  instance.FocusModeChange.subscribe((focused) => {
318
556
  this.Context?.SetFocusMode(focused);
@@ -330,29 +568,14 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
330
568
  * context note: outside an open window the note goes out immediately and opens a
331
569
  * {@link WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS} window; within the window the LATEST
332
570
  * event is stashed and a trailing-edge timer sends it when the window closes (one note
333
- * per widget per window, latest summary wins).
571
+ * per widget per window, latest summary wins). The agent's own interactions (its operate
572
+ * steps) are not echoed back to it.
334
573
  */
335
574
  onWidgetInteraction(interaction) {
336
- const now = Date.now();
337
- const entry = this.interactionThrottles.get(interaction.ItemID);
338
- if (!entry || (entry.Timer === null && now - entry.LastSentAt >= WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS)) {
339
- this.sendInteractionNote(interaction);
340
- this.interactionThrottles.set(interaction.ItemID, { LastSentAt: now, Timer: null, Pending: null });
575
+ if (isAgentAuthored(interaction)) {
341
576
  return;
342
577
  }
343
- entry.Pending = interaction; // latest wins within the window
344
- if (entry.Timer === null) {
345
- const wait = Math.max(0, entry.LastSentAt + WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS - now);
346
- entry.Timer = setTimeout(() => {
347
- entry.Timer = null;
348
- const pending = entry.Pending;
349
- entry.Pending = null;
350
- if (pending) {
351
- entry.LastSentAt = Date.now();
352
- this.sendInteractionNote(pending);
353
- }
354
- }, wait);
355
- }
578
+ this.interactionThrottles.Push(interaction.ItemID, interaction);
356
579
  }
357
580
  /** The ambient-note framing: background etiquette rides in the note itself (like scene deltas). */
358
581
  sendInteractionNote(interaction) {
@@ -360,14 +583,31 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
360
583
  '(background — do NOT respond unless it is significant or you are asked; ' +
361
584
  `explicit submissions arrive separately): ${interaction.Summary}`);
362
585
  }
363
- /** Cancels all pending ambient-note timers and resets the per-widget throttle windows. */
364
- clearInteractionThrottles() {
365
- for (const entry of this.interactionThrottles.values()) {
366
- if (entry.Timer !== null) {
367
- clearTimeout(entry.Timer);
368
- }
586
+ /**
587
+ * "[whiteboard] widget … offers actions: …" — sent only when the channel's exposure lets the
588
+ * model see board contents (`'state'` or above) and this session may call widget actions,
589
+ * and only when the list (names + descriptions) differs from the last note sent for that
590
+ * widget: a remount, a `hello` answer or the agent's own inspect re-reports the same list.
591
+ */
592
+ sendActionsNote(event) {
593
+ if (event.Actions.length === 0 || CompareExposure(this.Exposure, 'state') < 0) {
594
+ return;
369
595
  }
370
- this.interactionThrottles.clear();
596
+ if (!IsWhiteboardWidgetToolAllowed(this.ResolveWidgetAccess(), WHITEBOARD_TOOL_NAMES.CallWidget)) {
597
+ return;
598
+ }
599
+ const lastSent = this.lastActionsNoted.get(event.ItemID);
600
+ if (lastSent && SameWhiteboardWidgetActions(lastSent, event.Actions)) {
601
+ return;
602
+ }
603
+ this.lastActionsNoted.set(event.ItemID, event.Actions.map(action => ({ ...action })));
604
+ const names = event.Actions.map(action => action.Name).join(', ');
605
+ this.Context?.SendContextNote(`[whiteboard] widget "${event.Title || event.ItemID}" (${event.ItemID}) offers actions: ${names}`);
606
+ }
607
+ /** Cancels all pending ambient / actions note timers and resets the per-widget throttle windows. */
608
+ clearInteractionThrottles() {
609
+ this.interactionThrottles.Clear();
610
+ this.actionsNoteThrottles.Clear();
371
611
  }
372
612
  /**
373
613
  * Persists the current board as a `MJ: Artifacts` snapshot via the host context
@@ -387,11 +627,23 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
387
627
  }
388
628
  }
389
629
  /**
390
- * Executes one `Whiteboard_*` tool call LOCALLY. Prefers the live bound host (board
391
- * mutation + UI garnish); falls back to the pure engine function when no surface is
392
- * bound so the channel keeps working before the board is first shown.
630
+ * Executes one `Whiteboard_*` tool call LOCALLY. Board tools prefer the live bound host
631
+ * (board mutation + UI garnish) and fall back to the pure engine function when no surface
632
+ * is bound, so the channel keeps working before the board is first shown. Widget tools
633
+ * (inspect / operate / call) need the live board and resolve asynchronously; see
634
+ * {@link applyWidgetTool}.
635
+ *
636
+ * Return contract: **board tools return a plain `string` synchronously** (bound or not);
637
+ * **the three widget tools** ({@link IsWhiteboardWidgetToolName}) **return a
638
+ * `Promise<string>`** (policy refusals may also come back as a plain string). The declared
639
+ * type is therefore `string | Promise<string>`. An override that post-processes results
640
+ * should route `IsWhiteboardWidgetToolName(toolName)` calls straight to `super` and, for the
641
+ * rest, narrow the board result with `typeof result === 'string'`.
393
642
  */
394
643
  ApplyAgentTool(toolName, argsJson) {
644
+ if (IsWhiteboardWidgetToolName(toolName)) {
645
+ return this.applyWidgetTool(toolName, argsJson);
646
+ }
395
647
  let result;
396
648
  if (this.host) {
397
649
  result = this.host.ApplyAgentTool(toolName, argsJson);
@@ -410,7 +662,34 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
410
662
  }
411
663
  return result;
412
664
  }
413
- /** The board's serialized state of record (persisted under {@link ChannelName}). */
665
+ /**
666
+ * One widget command: access policy (`access_denied`, with the reason), then the
667
+ * {@link OnBeforeWidgetCommand} veto (`vetoed`), then the live board (`surface_unavailable`
668
+ * when none is bound). No confirmation frame: a widget's picture is a placeholder in frames.
669
+ */
670
+ applyWidgetTool(toolName, argsJson) {
671
+ const access = this.ResolveWidgetAccess();
672
+ if (!IsWhiteboardWidgetToolAllowed(access, toolName)) {
673
+ return widgetToolFailure('access_denied', this.describeWidgetAccessLimit(access) ?? `${toolName} is not available.`);
674
+ }
675
+ const args = parseToolArgs(argsJson);
676
+ const itemId = args && typeof args['itemId'] === 'string' ? args['itemId'] : null;
677
+ const refusal = this.OnBeforeWidgetCommand({ ToolName: toolName, ItemID: itemId, Args: args, ArgsJson: argsJson, Access: access });
678
+ if (refusal !== null) {
679
+ return widgetToolFailure('vetoed', refusal);
680
+ }
681
+ const host = this.host;
682
+ if (!host) {
683
+ return widgetToolFailure('surface_unavailable', 'The whiteboard isn\'t open, so its widgets can\'t be reached; ask the user to open the Whiteboard panel, then try again.');
684
+ }
685
+ host.AgentWidgetAccess = access; // per-call policy may have changed since BindSurface
686
+ return host.ApplyAgentToolAsync(toolName, argsJson).catch((error) => widgetToolFailure('surface_unavailable', `The whiteboard could not run ${toolName}: ${error?.message ?? String(error)}`));
687
+ }
688
+ /**
689
+ * The board's serialized state of record — the board JSON only. It also backs the channel's
690
+ * `GetState()` snapshot, so widget states (learner answers) stay out of it; they ride only in
691
+ * the session save payload ({@link BuildSavePayload}).
692
+ */
414
693
  SerializeState() {
415
694
  return this.State.ToJSON();
416
695
  }
@@ -419,9 +698,31 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
419
698
  * the {@link State} instance and its subscriptions are preserved). Returns `true` on
420
699
  * success; malformed / incompatible JSON returns `false` and the board stays fresh
421
700
  * (never throws — {@link WhiteboardState.LoadFromJSON} is tolerant by contract).
701
+ *
702
+ * Saves: none are requested while the board and its widget states are being loaded (so no
703
+ * save can pair the restored board with stale in-memory widget states); after a successful
704
+ * restore, exactly one save is requested carrying both. A failed restore requests nothing.
422
705
  */
423
706
  RestoreState(stateJson) {
424
- return this.State.LoadFromJSON(stateJson);
707
+ // A trailing widget-state save armed before the restore would fire after it as a second save.
708
+ if (this.widgetStateSaveTimer !== null) {
709
+ clearTimeout(this.widgetStateSaveTimer);
710
+ this.widgetStateSaveTimer = null;
711
+ }
712
+ this.restoring = true;
713
+ try {
714
+ if (!this.State.LoadFromJSON(stateJson)) {
715
+ return false;
716
+ }
717
+ // A legacy payload (no `widgetStates` key) simply restores no widget state. With
718
+ // `persistWidgetState` off nothing is restored either: state lives in memory only.
719
+ this.State.LoadWidgetStates(this.config.PersistWidgetState ? readSavedWidgetStates(stateJson) : {});
720
+ }
721
+ finally {
722
+ this.restoring = false;
723
+ }
724
+ this.Context?.RequestSave(this.BuildSavePayload());
725
+ return true;
425
726
  }
426
727
  /** Keeps the board's "Move to stage" / "Back to tab" button true to where the host placed it. */
427
728
  OnSurfacePlacementChange(placement) {
@@ -440,6 +741,14 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
440
741
  Dispose() {
441
742
  this.stateChangedSub?.unsubscribe();
442
743
  this.stateChangedSub = null;
744
+ this.widgetStateSub?.unsubscribe();
745
+ this.widgetStateSub = null;
746
+ if (this.widgetStateSaveTimer !== null) {
747
+ // Best effort: hand the latest widget state over before the context is released.
748
+ clearTimeout(this.widgetStateSaveTimer);
749
+ this.widgetStateSaveTimer = null;
750
+ this.requestWidgetStateSave();
751
+ }
443
752
  super.Dispose(); // releases the surface binding, the visual pump + bridge, and the context
444
753
  }
445
754
  /** Unsubscribes surface outputs, cancels pending ambient notes and drops the host reference. */