@memberjunction/ng-conversations 6.2.0-edge.6 → 6.2.0-edge.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/components/collection/collection-view.component.d.ts.map +1 -1
- package/dist/lib/components/collection/collection-view.component.js +1 -0
- package/dist/lib/components/collection/collection-view.component.js.map +1 -1
- package/dist/lib/components/conversation/conversation-chat-area.component.d.ts +174 -2
- package/dist/lib/components/conversation/conversation-chat-area.component.d.ts.map +1 -1
- package/dist/lib/components/conversation/conversation-chat-area.component.js +1266 -443
- package/dist/lib/components/conversation/conversation-chat-area.component.js.map +1 -1
- package/dist/lib/components/conversation/conversation-mode-picker.component.js +2 -2
- package/dist/lib/components/conversation/conversation-mode-picker.component.js.map +1 -1
- package/dist/lib/components/conversation/conversation-remote-browser-controller.d.ts +109 -0
- package/dist/lib/components/conversation/conversation-remote-browser-controller.d.ts.map +1 -0
- package/dist/lib/components/conversation/conversation-remote-browser-controller.js +298 -0
- package/dist/lib/components/conversation/conversation-remote-browser-controller.js.map +1 -0
- package/dist/lib/components/message/agent-handoff-card.component.d.ts +102 -0
- package/dist/lib/components/message/agent-handoff-card.component.d.ts.map +1 -0
- package/dist/lib/components/message/agent-handoff-card.component.js +439 -0
- package/dist/lib/components/message/agent-handoff-card.component.js.map +1 -0
- package/dist/lib/components/message/message-input.component.d.ts +80 -53
- package/dist/lib/components/message/message-input.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-input.component.js +381 -277
- package/dist/lib/components/message/message-input.component.js.map +1 -1
- package/dist/lib/components/message/message-item.component.d.ts +18 -1
- package/dist/lib/components/message/message-item.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-item.component.js +548 -426
- package/dist/lib/components/message/message-item.component.js.map +1 -1
- package/dist/lib/components/message/message-list.component.d.ts +8 -1
- package/dist/lib/components/message/message-list.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-list.component.js +40 -2
- package/dist/lib/components/message/message-list.component.js.map +1 -1
- package/dist/lib/components/realtime/interactive-component/run-view-component-artifact-source.js +1 -1
- package/dist/lib/components/realtime/interactive-component/run-view-component-artifact-source.js.map +1 -1
- package/dist/lib/components/realtime/media/media-channel-state.d.ts +5 -0
- package/dist/lib/components/realtime/media/media-channel-state.d.ts.map +1 -1
- package/dist/lib/components/realtime/media/media-channel-state.js +27 -1
- package/dist/lib/components/realtime/media/media-channel-state.js.map +1 -1
- package/dist/lib/components/realtime/media/media-channel.d.ts.map +1 -1
- package/dist/lib/components/realtime/media/media-channel.js +7 -2
- package/dist/lib/components/realtime/media/media-channel.js.map +1 -1
- package/dist/lib/components/realtime/media/realtime-media-surface.component.d.ts +19 -2
- package/dist/lib/components/realtime/media/realtime-media-surface.component.d.ts.map +1 -1
- package/dist/lib/components/realtime/media/realtime-media-surface.component.js +59 -10
- package/dist/lib/components/realtime/media/realtime-media-surface.component.js.map +1 -1
- package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts +17 -6
- package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts.map +1 -1
- package/dist/lib/components/realtime/realtime-agent-banner.component.js +61 -27
- package/dist/lib/components/realtime/realtime-agent-banner.component.js.map +1 -1
- package/dist/lib/components/realtime/realtime-composer.component.d.ts +26 -3
- package/dist/lib/components/realtime/realtime-composer.component.d.ts.map +1 -1
- package/dist/lib/components/realtime/realtime-composer.component.js +196 -102
- package/dist/lib/components/realtime/realtime-composer.component.js.map +1 -1
- package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts +53 -15
- package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts.map +1 -1
- package/dist/lib/components/realtime/realtime-session-overlay.component.js +125 -51
- package/dist/lib/components/realtime/realtime-session-overlay.component.js.map +1 -1
- package/dist/lib/components/realtime/realtime-ui-config.d.ts +9 -12
- package/dist/lib/components/realtime/realtime-ui-config.d.ts.map +1 -1
- package/dist/lib/components/realtime/realtime-ui-config.js +38 -18
- package/dist/lib/components/realtime/realtime-ui-config.js.map +1 -1
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map +1 -1
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js +27 -4
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js.map +1 -1
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts +8 -2
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts.map +1 -1
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js +51 -10
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js.map +1 -1
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.d.ts +46 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.d.ts.map +1 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.js +64 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel-config.js.map +1 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts +131 -9
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts.map +1 -1
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js +347 -38
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js.map +1 -1
- package/dist/lib/conversations.module.d.ts +2 -1
- package/dist/lib/conversations.module.d.ts.map +1 -1
- package/dist/lib/conversations.module.js +9 -4
- package/dist/lib/conversations.module.js.map +1 -1
- package/dist/lib/events/chat-events.d.ts +5 -4
- package/dist/lib/events/chat-events.d.ts.map +1 -1
- package/dist/lib/events/chat-events.js +2 -2
- package/dist/lib/events/chat-events.js.map +1 -1
- package/dist/lib/models/agent-turn.model.d.ts +11 -7
- package/dist/lib/models/agent-turn.model.d.ts.map +1 -1
- package/dist/lib/models/agent-turn.model.js.map +1 -1
- package/dist/lib/services/active-tasks.service.d.ts +5 -0
- package/dist/lib/services/active-tasks.service.d.ts.map +1 -1
- package/dist/lib/services/active-tasks.service.js +16 -0
- package/dist/lib/services/active-tasks.service.js.map +1 -1
- package/dist/lib/services/agent-state.service.d.ts.map +1 -1
- package/dist/lib/services/agent-state.service.js +7 -1
- package/dist/lib/services/agent-state.service.js.map +1 -1
- package/dist/lib/services/artifact-permission.service.d.ts.map +1 -1
- package/dist/lib/services/artifact-permission.service.js +2 -0
- package/dist/lib/services/artifact-permission.service.js.map +1 -1
- package/dist/lib/services/artifact-state.service.d.ts.map +1 -1
- package/dist/lib/services/artifact-state.service.js +1 -0
- package/dist/lib/services/artifact-state.service.js.map +1 -1
- package/dist/lib/services/collection-permission.service.d.ts.map +1 -1
- package/dist/lib/services/collection-permission.service.js +1 -0
- package/dist/lib/services/collection-permission.service.js.map +1 -1
- package/dist/lib/services/conversation-agent.service.d.ts +1 -4
- package/dist/lib/services/conversation-agent.service.d.ts.map +1 -1
- package/dist/lib/services/conversation-agent.service.js +7 -45
- package/dist/lib/services/conversation-agent.service.js.map +1 -1
- package/dist/lib/services/realtime-session-review.service.d.ts +2 -1
- package/dist/lib/services/realtime-session-review.service.d.ts.map +1 -1
- package/dist/lib/services/realtime-session-review.service.js.map +1 -1
- package/dist/lib/services/search.service.d.ts.map +1 -1
- package/dist/lib/services/search.service.js +1 -0
- package/dist/lib/services/search.service.js.map +1 -1
- package/dist/lib/utils/agent-handoff-map.d.ts +52 -0
- package/dist/lib/utils/agent-handoff-map.d.ts.map +1 -0
- package/dist/lib/utils/agent-handoff-map.js +270 -0
- package/dist/lib/utils/agent-handoff-map.js.map +1 -0
- package/dist/lib/utils/agent-turn-routing.d.ts +62 -7
- package/dist/lib/utils/agent-turn-routing.d.ts.map +1 -1
- package/dist/lib/utils/agent-turn-routing.js +147 -6
- package/dist/lib/utils/agent-turn-routing.js.map +1 -1
- package/dist/lib/utils/decision-routing.d.ts +17 -9
- package/dist/lib/utils/decision-routing.d.ts.map +1 -1
- package/dist/lib/utils/decision-routing.js +23 -14
- package/dist/lib/utils/decision-routing.js.map +1 -1
- package/dist/lib/utils/thread-replies.d.ts +34 -0
- package/dist/lib/utils/thread-replies.d.ts.map +1 -0
- package/dist/lib/utils/thread-replies.js +49 -0
- package/dist/lib/utils/thread-replies.js.map +1 -0
- package/dist/public-api.d.ts +5 -0
- package/dist/public-api.d.ts.map +1 -1
- package/dist/public-api.js +5 -0
- package/dist/public-api.js.map +1 -1
- package/package.json +36 -36
|
@@ -4,10 +4,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
4
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
6
|
};
|
|
7
|
-
import {
|
|
8
|
-
import { REALTIME_CHANNEL_CONTRACT_VERSION } from '@memberjunction/ai-core-plus';
|
|
7
|
+
import { CompareExposure, REALTIME_CHANNEL_CONTRACT_VERSION } from '@memberjunction/ai-core-plus';
|
|
9
8
|
import { BaseRealtimeChannelClient, BuildToolBackedVerbs } from '@memberjunction/realtime-runtime';
|
|
10
|
-
import {
|
|
9
|
+
import { IsPlainObject, RegisterClass } from '@memberjunction/global';
|
|
10
|
+
import { ApplyWhiteboardAgentTool, BuildWhiteboardExportSvg, IsWhiteboardWidgetToolAllowed, IsWhiteboardWidgetToolName, MinWhiteboardWidgetAccess, RealtimeWhiteboardHostComponent, WHITEBOARD_TOOL_DEFINITIONS, WHITEBOARD_TOOL_NAMES, WHITEBOARD_TOOL_PREFIX, WHITEBOARD_WIDGET_TOOL_NAMES, WhiteboardState, SameWhiteboardWidgetActions } from '@memberjunction/ng-whiteboard';
|
|
11
|
+
import { DEFAULT_WHITEBOARD_CHANNEL_CONFIG, ParseWhiteboardChannelConfig } from './whiteboard-channel-config';
|
|
11
12
|
/**
|
|
12
13
|
* Whether a whiteboard tool result reports success.
|
|
13
14
|
*
|
|
@@ -104,6 +105,103 @@ export async function RasterizeSvgToJpegBase64(svg, width = 1280, height = 720)
|
|
|
104
105
|
* collapse to one trailing-edge note instead of flooding the model's context).
|
|
105
106
|
*/
|
|
106
107
|
export const WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS = 4000;
|
|
108
|
+
/** Every widget tool's result carries the widget's contents (outline, state), so it requires `'state'` exposure. */
|
|
109
|
+
const WIDGET_TOOLS_RETURN_STATE = Object.fromEntries(WHITEBOARD_WIDGET_TOOL_NAMES.map(name => [name, 'state']));
|
|
110
|
+
/**
|
|
111
|
+
* Per-widget throttle window for "widget offers actions" perception notes (same leading +
|
|
112
|
+
* trailing, latest-wins shape as the ambient interaction notes).
|
|
113
|
+
*/
|
|
114
|
+
export const WHITEBOARD_ACTIONS_NOTE_THROTTLE_MS = 4000;
|
|
115
|
+
/**
|
|
116
|
+
* Minimum spacing (ms) between the save requests a widget-state change triggers. The
|
|
117
|
+
* session runtime debounces saves 3 s and RE-ARMS on every request, so a chatty widget
|
|
118
|
+
* requesting on every keystroke would never save until teardown; spacing requests further
|
|
119
|
+
* apart than the debounce guarantees each one lands.
|
|
120
|
+
*/
|
|
121
|
+
export const WHITEBOARD_WIDGET_STATE_SAVE_THROTTLE_MS = 5000;
|
|
122
|
+
/** The JSON key the widget states ride under, beside the board's own fields, in the session save payload. */
|
|
123
|
+
export const WHITEBOARD_WIDGET_STATES_SAVE_KEY = 'widgetStates';
|
|
124
|
+
/** A JSON tool result for a widget command the channel refused before it reached the board. */
|
|
125
|
+
function widgetToolFailure(errorCode, error) {
|
|
126
|
+
const result = { success: false, error, errorCode };
|
|
127
|
+
return JSON.stringify(result);
|
|
128
|
+
}
|
|
129
|
+
/** Parses tool arguments into a JSON object, or null when they are not one. */
|
|
130
|
+
function parseToolArgs(argsJson) {
|
|
131
|
+
try {
|
|
132
|
+
const parsed = JSON.parse(argsJson);
|
|
133
|
+
// JSON.parse output is JSON by construction, so narrowing a plain object to JSONObject is not a type lie.
|
|
134
|
+
return IsPlainObject(parsed) ? parsed : null;
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/** Whether a widget event came from the agent's own operate command (absent Author = the user). */
|
|
141
|
+
function isAgentAuthored(event) {
|
|
142
|
+
return event.Author === 'agent';
|
|
143
|
+
}
|
|
144
|
+
/** Reads the `widgetStates` sibling key of a saved payload; anything malformed reads as none. */
|
|
145
|
+
function readSavedWidgetStates(stateJson) {
|
|
146
|
+
const saved = parseToolArgs(stateJson);
|
|
147
|
+
const raw = saved ? saved[WHITEBOARD_WIDGET_STATES_SAVE_KEY] : undefined;
|
|
148
|
+
const out = {};
|
|
149
|
+
if (IsPlainObject(raw)) {
|
|
150
|
+
for (const [id, value] of Object.entries(raw)) {
|
|
151
|
+
if (typeof value === 'string') {
|
|
152
|
+
out[id] = value;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Per-key leading + trailing throttle for background notes: outside an open window a value
|
|
160
|
+
* is sent immediately and opens a window; within the window the LATEST value is stashed and
|
|
161
|
+
* a trailing-edge timer sends it when the window closes (one note per key per window).
|
|
162
|
+
*/
|
|
163
|
+
class KeyedNoteThrottle {
|
|
164
|
+
windowMs;
|
|
165
|
+
send;
|
|
166
|
+
entries = new Map();
|
|
167
|
+
constructor(windowMs, send) {
|
|
168
|
+
this.windowMs = windowMs;
|
|
169
|
+
this.send = send;
|
|
170
|
+
}
|
|
171
|
+
/** Offers one value for `key`; it is sent now or at the end of the key's open window. */
|
|
172
|
+
Push(key, value) {
|
|
173
|
+
const now = Date.now();
|
|
174
|
+
const entry = this.entries.get(key);
|
|
175
|
+
if (!entry || (entry.Timer === null && now - entry.LastSentAt >= this.windowMs)) {
|
|
176
|
+
this.send(value);
|
|
177
|
+
this.entries.set(key, { LastSentAt: now, Timer: null, Pending: null });
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
entry.Pending = value; // latest wins within the window
|
|
181
|
+
if (entry.Timer === null) {
|
|
182
|
+
const wait = Math.max(0, entry.LastSentAt + this.windowMs - now);
|
|
183
|
+
entry.Timer = setTimeout(() => this.flush(entry), wait);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
/** Cancels every pending trailing note and resets all windows. */
|
|
187
|
+
Clear() {
|
|
188
|
+
for (const entry of this.entries.values()) {
|
|
189
|
+
if (entry.Timer !== null) {
|
|
190
|
+
clearTimeout(entry.Timer);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
this.entries.clear();
|
|
194
|
+
}
|
|
195
|
+
flush(entry) {
|
|
196
|
+
entry.Timer = null;
|
|
197
|
+
const pending = entry.Pending;
|
|
198
|
+
entry.Pending = null;
|
|
199
|
+
if (pending !== null) {
|
|
200
|
+
entry.LastSentAt = Date.now();
|
|
201
|
+
this.send(pending);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
107
205
|
/**
|
|
108
206
|
* The LIVE WHITEBOARD as a pluggable interactive channel — the canonical
|
|
109
207
|
* {@link BaseRealtimeChannelClient} implementation, resolved from the `MJ: AI Agent
|
|
@@ -144,7 +242,21 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
144
242
|
/** Board-mutation subscription driving the debounced state-of-record save. */
|
|
145
243
|
stateChangedSub = null;
|
|
146
244
|
/** Per-widget ambient-interaction note throttles (ItemID → window state). */
|
|
147
|
-
interactionThrottles = new
|
|
245
|
+
interactionThrottles = new KeyedNoteThrottle(WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS, (interaction) => this.sendInteractionNote(interaction));
|
|
246
|
+
/** Per-widget "offers actions" note throttles (ItemID → window state). */
|
|
247
|
+
actionsNoteThrottles = new KeyedNoteThrottle(WHITEBOARD_ACTIONS_NOTE_THROTTLE_MS, (actions) => this.sendActionsNote(actions));
|
|
248
|
+
/** Per-widget action list of the last "offers actions" note SENT (ItemID → actions), for dedupe. */
|
|
249
|
+
lastActionsNoted = new Map();
|
|
250
|
+
/** The parsed `ChannelConfig` (defaults until {@link OnInitialize}). */
|
|
251
|
+
config = { ...DEFAULT_WHITEBOARD_CHANNEL_CONFIG };
|
|
252
|
+
/** Widget-state subscription driving the throttled save. */
|
|
253
|
+
widgetStateSub = null;
|
|
254
|
+
/** Trailing timer of the widget-state save throttle, or null when idle. */
|
|
255
|
+
widgetStateSaveTimer = null;
|
|
256
|
+
/** When the widget-state path last requested a save (epoch ms). */
|
|
257
|
+
lastWidgetStateSaveAt = Number.NEGATIVE_INFINITY;
|
|
258
|
+
/** True while {@link RestoreState} runs: saves are held so none carries a half-restored board. */
|
|
259
|
+
restoring = false;
|
|
148
260
|
get ChannelName() {
|
|
149
261
|
return 'Whiteboard';
|
|
150
262
|
}
|
|
@@ -194,8 +306,46 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
194
306
|
get SurfaceShareable() {
|
|
195
307
|
return true;
|
|
196
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* The board's tools. The widget tools are declared only as far as {@link MaxWidgetAccess}
|
|
311
|
+
* allows (`'inspect-only'` declares Inspect only, `'off'` none). Read before the channel is
|
|
312
|
+
* initialized, so it depends on the class alone, never on the session's config.
|
|
313
|
+
*/
|
|
197
314
|
GetToolDefinitions() {
|
|
198
|
-
|
|
315
|
+
const max = this.MaxWidgetAccess;
|
|
316
|
+
if (max === 'full') {
|
|
317
|
+
return WHITEBOARD_TOOL_DEFINITIONS;
|
|
318
|
+
}
|
|
319
|
+
return WHITEBOARD_TOOL_DEFINITIONS.filter(tool => !IsWhiteboardWidgetToolName(tool.Name) || IsWhiteboardWidgetToolAllowed(max, tool.Name));
|
|
320
|
+
}
|
|
321
|
+
// ────────────────────────────────────────────── widget access policy (extension points)
|
|
322
|
+
/**
|
|
323
|
+
* The most widget access this channel class ever offers — which widget tools it DECLARES to the
|
|
324
|
+
* model (tools are collected before the channel is initialized, so this cannot depend on the
|
|
325
|
+
* session). Default `'full'`. A host app lowers it by subclassing the channel and registering the
|
|
326
|
+
* subclass under `'RealtimeWhiteboardChannel'` (or via `HostChannels[].Create`).
|
|
327
|
+
*/
|
|
328
|
+
get MaxWidgetAccess() {
|
|
329
|
+
return 'full';
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* The widget access in force for ONE call, evaluated per call (so it may change mid-session,
|
|
333
|
+
* e.g. `'off'` while an assessment is open). Default: the lower of {@link MaxWidgetAccess} and
|
|
334
|
+
* the session's `widgetAccess` config. A call it does not allow returns `access_denied`.
|
|
335
|
+
*/
|
|
336
|
+
ResolveWidgetAccess() {
|
|
337
|
+
return MinWhiteboardWidgetAccess(this.MaxWidgetAccess, this.config.WidgetAccess);
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Last-moment veto of one widget command that access already allows: return the refusal text the
|
|
341
|
+
* model should see (the call returns `vetoed` with it) or `null` to let it through. Default allows all.
|
|
342
|
+
*/
|
|
343
|
+
OnBeforeWidgetCommand(_command) {
|
|
344
|
+
return null;
|
|
345
|
+
}
|
|
346
|
+
/** Whether widget runtime state rides in this session's saves (config `persistWidgetState`). */
|
|
347
|
+
get PersistWidgetState() {
|
|
348
|
+
return this.config.PersistWidgetState;
|
|
199
349
|
}
|
|
200
350
|
GetSurfaceComponent() {
|
|
201
351
|
return RealtimeWhiteboardHostComponent;
|
|
@@ -234,7 +384,8 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
234
384
|
Schema: { type: 'array', items: { type: 'object' } }
|
|
235
385
|
}
|
|
236
386
|
],
|
|
237
|
-
|
|
387
|
+
// Each widget tool returns the widget's contents, so the runtime refuses it below 'state' exposure.
|
|
388
|
+
Verbs: BuildToolBackedVerbs(this.GetToolDefinitions(), this.ToolNamePrefix, 'agent', WIDGET_TOOLS_RETURN_STATE),
|
|
238
389
|
Events: [
|
|
239
390
|
{ Name: 'state_changed', Description: 'The board changed (by you or the user).' },
|
|
240
391
|
{ Name: 'frame_pushed', Description: 'A picture of the board was sent to you.' }
|
|
@@ -251,6 +402,7 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
251
402
|
* the structured note would duplicate the deltas.
|
|
252
403
|
*/
|
|
253
404
|
OnInitialize() {
|
|
405
|
+
this.applyChannelConfig();
|
|
254
406
|
this.stateChangedSub?.unsubscribe();
|
|
255
407
|
this.EnableVisualPerception(this, {
|
|
256
408
|
ConfirmationNote: '[whiteboard] visual confirmation of your action (background — do NOT narrate or announce your own change; continue naturally)',
|
|
@@ -259,7 +411,9 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
259
411
|
StartBridgePoller: true
|
|
260
412
|
});
|
|
261
413
|
this.stateChangedSub = this.State.Changed$.subscribe((change) => {
|
|
262
|
-
|
|
414
|
+
if (!this.restoring) {
|
|
415
|
+
this.Context?.RequestSave(this.BuildSavePayload());
|
|
416
|
+
}
|
|
263
417
|
this.RecordChange({ Author: change.Author, Perceive: false });
|
|
264
418
|
// Only user edits (and scene replacements like undo) drive the settle-debounce pipeline.
|
|
265
419
|
// Agent edits are confirmed with a single frame in ApplyAgentTool.
|
|
@@ -267,6 +421,80 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
267
421
|
void this.NotifyVisualChange();
|
|
268
422
|
}
|
|
269
423
|
});
|
|
424
|
+
// Widget state is runtime data (no Changed$): save it on its own, throttled cadence.
|
|
425
|
+
this.widgetStateSub?.unsubscribe();
|
|
426
|
+
this.widgetStateSub = this.State.WidgetStateChanged$.subscribe(() => this.scheduleWidgetStateSave());
|
|
427
|
+
}
|
|
428
|
+
/** Parses `ChannelConfig`; an invalid member falls back to its default with a console warning. */
|
|
429
|
+
applyChannelConfig() {
|
|
430
|
+
const parsed = ParseWhiteboardChannelConfig(this.Context?.ChannelConfig);
|
|
431
|
+
this.config = parsed.Config;
|
|
432
|
+
for (const problem of parsed.Problems) {
|
|
433
|
+
console.warn(`[RealtimeWhiteboardChannel] channel config: ${problem}`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Tells the model, once the session is live, which widget tools this session's access leaves
|
|
438
|
+
* unavailable (notes sent before the session is live are dropped, so not from OnInitialize).
|
|
439
|
+
*/
|
|
440
|
+
OnSessionStarted() {
|
|
441
|
+
super.OnSessionStarted();
|
|
442
|
+
const limit = this.describeWidgetAccessLimit(this.ResolveWidgetAccess());
|
|
443
|
+
if (limit) {
|
|
444
|
+
this.Context?.SendContextNote(`[whiteboard] ${limit}`);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
/** One sentence naming the widget tools `access` leaves unavailable, or null at full access. */
|
|
448
|
+
describeWidgetAccessLimit(access) {
|
|
449
|
+
if (access === 'full') {
|
|
450
|
+
return null;
|
|
451
|
+
}
|
|
452
|
+
const unavailable = WHITEBOARD_WIDGET_TOOL_NAMES.filter(name => !IsWhiteboardWidgetToolAllowed(access, name));
|
|
453
|
+
const reason = access === 'off'
|
|
454
|
+
? 'widget tools are turned off for this session'
|
|
455
|
+
: `widgets are inspect-only for this session — read them with ${WHITEBOARD_TOOL_NAMES.InspectWidget}, but do not try to change them`;
|
|
456
|
+
return `${unavailable.join(', ')} ${unavailable.length === 1 ? 'is' : 'are'} unavailable: ${reason}.`;
|
|
457
|
+
}
|
|
458
|
+
/**
|
|
459
|
+
* The session save payload: the board's own JSON plus, when `persistWidgetState` is on and any
|
|
460
|
+
* widget holds state, a sibling `widgetStates` key. Old readers ignore the unknown key; LiveKit
|
|
461
|
+
* sync and Save-to-artifact keep plain {@link WhiteboardState.ToJSON}.
|
|
462
|
+
*/
|
|
463
|
+
BuildSavePayload() {
|
|
464
|
+
const boardJson = this.State.ToJSON();
|
|
465
|
+
if (!this.config.PersistWidgetState) {
|
|
466
|
+
return boardJson;
|
|
467
|
+
}
|
|
468
|
+
const widgetStates = this.State.WidgetStatesToJSON();
|
|
469
|
+
if (Object.keys(widgetStates).length === 0) {
|
|
470
|
+
return boardJson;
|
|
471
|
+
}
|
|
472
|
+
const payload = parseToolArgs(boardJson) ?? {};
|
|
473
|
+
payload[WHITEBOARD_WIDGET_STATES_SAVE_KEY] = widgetStates;
|
|
474
|
+
return JSON.stringify(payload);
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* Requests a save for a widget-state change at most once per
|
|
478
|
+
* {@link WHITEBOARD_WIDGET_STATE_SAVE_THROTTLE_MS} (leading edge, then one trailing request with
|
|
479
|
+
* the latest state). Nothing is requested when `persistWidgetState` is off.
|
|
480
|
+
*/
|
|
481
|
+
scheduleWidgetStateSave() {
|
|
482
|
+
if (this.restoring || !this.config.PersistWidgetState || this.widgetStateSaveTimer !== null) {
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
const wait = this.lastWidgetStateSaveAt + WHITEBOARD_WIDGET_STATE_SAVE_THROTTLE_MS - Date.now();
|
|
486
|
+
if (wait <= 0) {
|
|
487
|
+
this.requestWidgetStateSave();
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
this.widgetStateSaveTimer = setTimeout(() => {
|
|
491
|
+
this.widgetStateSaveTimer = null;
|
|
492
|
+
this.requestWidgetStateSave();
|
|
493
|
+
}, wait);
|
|
494
|
+
}
|
|
495
|
+
requestWidgetStateSave() {
|
|
496
|
+
this.lastWidgetStateSaveAt = Date.now();
|
|
497
|
+
this.Context?.RequestSave(this.BuildSavePayload());
|
|
270
498
|
}
|
|
271
499
|
/**
|
|
272
500
|
* Wires the dynamically-created board host: inputs (shared state engine + agent name)
|
|
@@ -279,6 +507,7 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
279
507
|
this.host = instance;
|
|
280
508
|
instance.State = this.State;
|
|
281
509
|
instance.AgentName = this.Context?.AgentName ?? 'Agent';
|
|
510
|
+
instance.AgentWidgetAccess = this.ResolveWidgetAccess();
|
|
282
511
|
this.surfaceSubs.push(
|
|
283
512
|
// The board's coalesced scene delta — the perception feed the agent "sees".
|
|
284
513
|
instance.SceneDelta.subscribe((deltaJson) => {
|
|
@@ -298,6 +527,11 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
298
527
|
// event ran first). Surface it to the agent so it can react to quiz answers /
|
|
299
528
|
// micro-form input it asked for.
|
|
300
529
|
instance.WidgetSubmitted.subscribe((submit) => {
|
|
530
|
+
// The agent's own operate command submitted it: the tool result already carries
|
|
531
|
+
// `submitted`, so no note and no spoken reaction (it would answer itself).
|
|
532
|
+
if (isAgentAuthored(submit)) {
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
301
535
|
// Durable awareness first (the note persists in the model's context)…
|
|
302
536
|
this.Context?.SendContextNote(`[whiteboard] the user submitted input in widget "${submit.Title || submit.ItemID}": ${submit.DataJson}`);
|
|
303
537
|
// …then make the model REACT — a submission is explicit user input the user is
|
|
@@ -313,6 +547,10 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
313
547
|
instance.WidgetInteraction.subscribe((interaction) => {
|
|
314
548
|
this.onWidgetInteraction(interaction);
|
|
315
549
|
}),
|
|
550
|
+
// A widget registered or changed its named actions — a throttled perception note.
|
|
551
|
+
instance.WidgetActionsChanged.subscribe((actions) => {
|
|
552
|
+
this.actionsNoteThrottles.Push(actions.ItemID, actions);
|
|
553
|
+
}),
|
|
316
554
|
// The board's Focus toggle — ask the shell to collapse/restore the main call column.
|
|
317
555
|
instance.FocusModeChange.subscribe((focused) => {
|
|
318
556
|
this.Context?.SetFocusMode(focused);
|
|
@@ -330,29 +568,14 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
330
568
|
* context note: outside an open window the note goes out immediately and opens a
|
|
331
569
|
* {@link WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS} window; within the window the LATEST
|
|
332
570
|
* event is stashed and a trailing-edge timer sends it when the window closes (one note
|
|
333
|
-
* per widget per window, latest summary wins).
|
|
571
|
+
* per widget per window, latest summary wins). The agent's own interactions (its operate
|
|
572
|
+
* steps) are not echoed back to it.
|
|
334
573
|
*/
|
|
335
574
|
onWidgetInteraction(interaction) {
|
|
336
|
-
|
|
337
|
-
const entry = this.interactionThrottles.get(interaction.ItemID);
|
|
338
|
-
if (!entry || (entry.Timer === null && now - entry.LastSentAt >= WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS)) {
|
|
339
|
-
this.sendInteractionNote(interaction);
|
|
340
|
-
this.interactionThrottles.set(interaction.ItemID, { LastSentAt: now, Timer: null, Pending: null });
|
|
575
|
+
if (isAgentAuthored(interaction)) {
|
|
341
576
|
return;
|
|
342
577
|
}
|
|
343
|
-
|
|
344
|
-
if (entry.Timer === null) {
|
|
345
|
-
const wait = Math.max(0, entry.LastSentAt + WHITEBOARD_INTERACTION_NOTE_THROTTLE_MS - now);
|
|
346
|
-
entry.Timer = setTimeout(() => {
|
|
347
|
-
entry.Timer = null;
|
|
348
|
-
const pending = entry.Pending;
|
|
349
|
-
entry.Pending = null;
|
|
350
|
-
if (pending) {
|
|
351
|
-
entry.LastSentAt = Date.now();
|
|
352
|
-
this.sendInteractionNote(pending);
|
|
353
|
-
}
|
|
354
|
-
}, wait);
|
|
355
|
-
}
|
|
578
|
+
this.interactionThrottles.Push(interaction.ItemID, interaction);
|
|
356
579
|
}
|
|
357
580
|
/** The ambient-note framing: background etiquette rides in the note itself (like scene deltas). */
|
|
358
581
|
sendInteractionNote(interaction) {
|
|
@@ -360,14 +583,31 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
360
583
|
'(background — do NOT respond unless it is significant or you are asked; ' +
|
|
361
584
|
`explicit submissions arrive separately): ${interaction.Summary}`);
|
|
362
585
|
}
|
|
363
|
-
/**
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
586
|
+
/**
|
|
587
|
+
* "[whiteboard] widget … offers actions: …" — sent only when the channel's exposure lets the
|
|
588
|
+
* model see board contents (`'state'` or above) and this session may call widget actions,
|
|
589
|
+
* and only when the list (names + descriptions) differs from the last note sent for that
|
|
590
|
+
* widget: a remount, a `hello` answer or the agent's own inspect re-reports the same list.
|
|
591
|
+
*/
|
|
592
|
+
sendActionsNote(event) {
|
|
593
|
+
if (event.Actions.length === 0 || CompareExposure(this.Exposure, 'state') < 0) {
|
|
594
|
+
return;
|
|
369
595
|
}
|
|
370
|
-
this.
|
|
596
|
+
if (!IsWhiteboardWidgetToolAllowed(this.ResolveWidgetAccess(), WHITEBOARD_TOOL_NAMES.CallWidget)) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const lastSent = this.lastActionsNoted.get(event.ItemID);
|
|
600
|
+
if (lastSent && SameWhiteboardWidgetActions(lastSent, event.Actions)) {
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
this.lastActionsNoted.set(event.ItemID, event.Actions.map(action => ({ ...action })));
|
|
604
|
+
const names = event.Actions.map(action => action.Name).join(', ');
|
|
605
|
+
this.Context?.SendContextNote(`[whiteboard] widget "${event.Title || event.ItemID}" (${event.ItemID}) offers actions: ${names}`);
|
|
606
|
+
}
|
|
607
|
+
/** Cancels all pending ambient / actions note timers and resets the per-widget throttle windows. */
|
|
608
|
+
clearInteractionThrottles() {
|
|
609
|
+
this.interactionThrottles.Clear();
|
|
610
|
+
this.actionsNoteThrottles.Clear();
|
|
371
611
|
}
|
|
372
612
|
/**
|
|
373
613
|
* Persists the current board as a `MJ: Artifacts` snapshot via the host context
|
|
@@ -387,11 +627,23 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
387
627
|
}
|
|
388
628
|
}
|
|
389
629
|
/**
|
|
390
|
-
* Executes one `Whiteboard_*` tool call LOCALLY.
|
|
391
|
-
* mutation + UI garnish)
|
|
392
|
-
* bound so the channel keeps working before the board is first shown.
|
|
630
|
+
* Executes one `Whiteboard_*` tool call LOCALLY. Board tools prefer the live bound host
|
|
631
|
+
* (board mutation + UI garnish) and fall back to the pure engine function when no surface
|
|
632
|
+
* is bound, so the channel keeps working before the board is first shown. Widget tools
|
|
633
|
+
* (inspect / operate / call) need the live board and resolve asynchronously; see
|
|
634
|
+
* {@link applyWidgetTool}.
|
|
635
|
+
*
|
|
636
|
+
* Return contract: **board tools return a plain `string` synchronously** (bound or not);
|
|
637
|
+
* **the three widget tools** ({@link IsWhiteboardWidgetToolName}) **return a
|
|
638
|
+
* `Promise<string>`** (policy refusals may also come back as a plain string). The declared
|
|
639
|
+
* type is therefore `string | Promise<string>`. An override that post-processes results
|
|
640
|
+
* should route `IsWhiteboardWidgetToolName(toolName)` calls straight to `super` and, for the
|
|
641
|
+
* rest, narrow the board result with `typeof result === 'string'`.
|
|
393
642
|
*/
|
|
394
643
|
ApplyAgentTool(toolName, argsJson) {
|
|
644
|
+
if (IsWhiteboardWidgetToolName(toolName)) {
|
|
645
|
+
return this.applyWidgetTool(toolName, argsJson);
|
|
646
|
+
}
|
|
395
647
|
let result;
|
|
396
648
|
if (this.host) {
|
|
397
649
|
result = this.host.ApplyAgentTool(toolName, argsJson);
|
|
@@ -410,7 +662,34 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
410
662
|
}
|
|
411
663
|
return result;
|
|
412
664
|
}
|
|
413
|
-
/**
|
|
665
|
+
/**
|
|
666
|
+
* One widget command: access policy (`access_denied`, with the reason), then the
|
|
667
|
+
* {@link OnBeforeWidgetCommand} veto (`vetoed`), then the live board (`surface_unavailable`
|
|
668
|
+
* when none is bound). No confirmation frame: a widget's picture is a placeholder in frames.
|
|
669
|
+
*/
|
|
670
|
+
applyWidgetTool(toolName, argsJson) {
|
|
671
|
+
const access = this.ResolveWidgetAccess();
|
|
672
|
+
if (!IsWhiteboardWidgetToolAllowed(access, toolName)) {
|
|
673
|
+
return widgetToolFailure('access_denied', this.describeWidgetAccessLimit(access) ?? `${toolName} is not available.`);
|
|
674
|
+
}
|
|
675
|
+
const args = parseToolArgs(argsJson);
|
|
676
|
+
const itemId = args && typeof args['itemId'] === 'string' ? args['itemId'] : null;
|
|
677
|
+
const refusal = this.OnBeforeWidgetCommand({ ToolName: toolName, ItemID: itemId, Args: args, ArgsJson: argsJson, Access: access });
|
|
678
|
+
if (refusal !== null) {
|
|
679
|
+
return widgetToolFailure('vetoed', refusal);
|
|
680
|
+
}
|
|
681
|
+
const host = this.host;
|
|
682
|
+
if (!host) {
|
|
683
|
+
return widgetToolFailure('surface_unavailable', 'The whiteboard isn\'t open, so its widgets can\'t be reached; ask the user to open the Whiteboard panel, then try again.');
|
|
684
|
+
}
|
|
685
|
+
host.AgentWidgetAccess = access; // per-call policy may have changed since BindSurface
|
|
686
|
+
return host.ApplyAgentToolAsync(toolName, argsJson).catch((error) => widgetToolFailure('surface_unavailable', `The whiteboard could not run ${toolName}: ${error?.message ?? String(error)}`));
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* The board's serialized state of record — the board JSON only. It also backs the channel's
|
|
690
|
+
* `GetState()` snapshot, so widget states (learner answers) stay out of it; they ride only in
|
|
691
|
+
* the session save payload ({@link BuildSavePayload}).
|
|
692
|
+
*/
|
|
414
693
|
SerializeState() {
|
|
415
694
|
return this.State.ToJSON();
|
|
416
695
|
}
|
|
@@ -419,9 +698,31 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
419
698
|
* the {@link State} instance and its subscriptions are preserved). Returns `true` on
|
|
420
699
|
* success; malformed / incompatible JSON returns `false` and the board stays fresh
|
|
421
700
|
* (never throws — {@link WhiteboardState.LoadFromJSON} is tolerant by contract).
|
|
701
|
+
*
|
|
702
|
+
* Saves: none are requested while the board and its widget states are being loaded (so no
|
|
703
|
+
* save can pair the restored board with stale in-memory widget states); after a successful
|
|
704
|
+
* restore, exactly one save is requested carrying both. A failed restore requests nothing.
|
|
422
705
|
*/
|
|
423
706
|
RestoreState(stateJson) {
|
|
424
|
-
|
|
707
|
+
// A trailing widget-state save armed before the restore would fire after it as a second save.
|
|
708
|
+
if (this.widgetStateSaveTimer !== null) {
|
|
709
|
+
clearTimeout(this.widgetStateSaveTimer);
|
|
710
|
+
this.widgetStateSaveTimer = null;
|
|
711
|
+
}
|
|
712
|
+
this.restoring = true;
|
|
713
|
+
try {
|
|
714
|
+
if (!this.State.LoadFromJSON(stateJson)) {
|
|
715
|
+
return false;
|
|
716
|
+
}
|
|
717
|
+
// A legacy payload (no `widgetStates` key) simply restores no widget state. With
|
|
718
|
+
// `persistWidgetState` off nothing is restored either: state lives in memory only.
|
|
719
|
+
this.State.LoadWidgetStates(this.config.PersistWidgetState ? readSavedWidgetStates(stateJson) : {});
|
|
720
|
+
}
|
|
721
|
+
finally {
|
|
722
|
+
this.restoring = false;
|
|
723
|
+
}
|
|
724
|
+
this.Context?.RequestSave(this.BuildSavePayload());
|
|
725
|
+
return true;
|
|
425
726
|
}
|
|
426
727
|
/** Keeps the board's "Move to stage" / "Back to tab" button true to where the host placed it. */
|
|
427
728
|
OnSurfacePlacementChange(placement) {
|
|
@@ -440,6 +741,14 @@ let RealtimeWhiteboardChannel = class RealtimeWhiteboardChannel extends BaseReal
|
|
|
440
741
|
Dispose() {
|
|
441
742
|
this.stateChangedSub?.unsubscribe();
|
|
442
743
|
this.stateChangedSub = null;
|
|
744
|
+
this.widgetStateSub?.unsubscribe();
|
|
745
|
+
this.widgetStateSub = null;
|
|
746
|
+
if (this.widgetStateSaveTimer !== null) {
|
|
747
|
+
// Best effort: hand the latest widget state over before the context is released.
|
|
748
|
+
clearTimeout(this.widgetStateSaveTimer);
|
|
749
|
+
this.widgetStateSaveTimer = null;
|
|
750
|
+
this.requestWidgetStateSave();
|
|
751
|
+
}
|
|
443
752
|
super.Dispose(); // releases the surface binding, the visual pump + bridge, and the context
|
|
444
753
|
}
|
|
445
754
|
/** Unsubscribes surface outputs, cancels pending ambient notes and drops the host reference. */
|