clay-server 2.26.0 → 2.27.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,631 @@
1
+ var path = require("path");
2
+ var fs = require("fs");
3
+
4
+ /**
5
+ * Attach user-message handler and remaining small handlers
6
+ * (sticky notes, terminals, context sources, browser extension,
7
+ * scheduled tasks gate, loop delegation, schedule_message,
8
+ * and the main "message" dispatch) to a project context.
9
+ *
10
+ * ctx fields:
11
+ * cwd, slug, isMate, osUsers,
12
+ * sm, sdk, nm, tm,
13
+ * send, sendTo, sendToSession, sendToSessionOthers,
14
+ * clients, opts,
15
+ * usersModule, matesModule,
16
+ * getSessionForWs, getLinuxUserForSession, getOsUserInfoForWs,
17
+ * hydrateImageRefs, saveImageFile, imagesDir,
18
+ * onProcessingChanged, onSessionDone,
19
+ * _loop - { handleLoopMessage: fn(ws, msg) }
20
+ * browserState - { _browserTabList, _extensionWs, pendingExtensionRequests } (mutable refs)
21
+ * sendExtensionCommandAny, requestTabContext,
22
+ * startFileWatch, stopFileWatch,
23
+ * scheduleMessage, cancelScheduledMessage,
24
+ * loadContextSources, saveContextSources,
25
+ * digestDmTurn, gateMemory,
26
+ * getSDK - lazy ESM loader returning Promise<sdk>
27
+ */
28
+ function attachUserMessage(ctx) {
29
+ var cwd = ctx.cwd;
30
+ var slug = ctx.slug;
31
+ var isMate = ctx.isMate;
32
+ var osUsers = ctx.osUsers;
33
+
34
+ var sm = ctx.sm;
35
+ var sdk = ctx.sdk;
36
+ var nm = ctx.nm;
37
+ var tm = ctx.tm;
38
+
39
+ var send = ctx.send;
40
+ var sendTo = ctx.sendTo;
41
+ var sendToSession = ctx.sendToSession;
42
+ var sendToSessionOthers = ctx.sendToSessionOthers;
43
+
44
+ var clients = ctx.clients;
45
+ var opts = ctx.opts;
46
+
47
+ var usersModule = ctx.usersModule;
48
+ var matesModule = ctx.matesModule;
49
+
50
+ var getSessionForWs = ctx.getSessionForWs;
51
+ var getLinuxUserForSession = ctx.getLinuxUserForSession;
52
+ var getOsUserInfoForWs = ctx.getOsUserInfoForWs;
53
+
54
+ var hydrateImageRefs = ctx.hydrateImageRefs;
55
+ var saveImageFile = ctx.saveImageFile;
56
+ var imagesDir = ctx.imagesDir;
57
+
58
+ var onProcessingChanged = ctx.onProcessingChanged;
59
+
60
+ var _loop = ctx._loop;
61
+ var browserState = ctx.browserState;
62
+
63
+ var sendExtensionCommandAny = ctx.sendExtensionCommandAny;
64
+ var requestTabContext = ctx.requestTabContext;
65
+
66
+ var scheduleMessage = ctx.scheduleMessage;
67
+ var cancelScheduledMessage = ctx.cancelScheduledMessage;
68
+
69
+ var loadContextSources = ctx.loadContextSources;
70
+ var saveContextSources = ctx.saveContextSources;
71
+
72
+ var getSDK = ctx.getSDK;
73
+
74
+ // --------------- Sticky notes ---------------
75
+
76
+ function syncNotesKnowledge() {
77
+ if (!isMate) return;
78
+ try {
79
+ var knDir = path.join(cwd, "knowledge");
80
+ var knFile = path.join(knDir, "sticky-notes.md");
81
+ var text = nm.getActiveNotesText();
82
+ if (text) {
83
+ fs.mkdirSync(knDir, { recursive: true });
84
+ fs.writeFileSync(knFile, text);
85
+ } else {
86
+ try { fs.unlinkSync(knFile); } catch (e) {}
87
+ }
88
+ } catch (e) {
89
+ console.error("[project] Failed to sync sticky-notes.md:", e.message);
90
+ }
91
+ }
92
+
93
+ // --------------- Main handler ---------------
94
+
95
+ function handleUserMessage(ws, msg) {
96
+ // --- Sticky notes ---
97
+ if (msg.type === "note_create") {
98
+ var note = nm.create(msg);
99
+ if (note) {
100
+ send({ type: "note_created", note: note });
101
+ syncNotesKnowledge();
102
+ }
103
+ return true;
104
+ }
105
+
106
+ if (msg.type === "note_update") {
107
+ if (!msg.id) return true;
108
+ var updated = nm.update(msg.id, msg);
109
+ if (updated) {
110
+ send({ type: "note_updated", note: updated });
111
+ if (msg.text !== undefined || msg.hidden !== undefined) syncNotesKnowledge();
112
+ }
113
+ return true;
114
+ }
115
+
116
+ if (msg.type === "note_delete") {
117
+ if (!msg.id) return true;
118
+ if (nm.remove(msg.id)) {
119
+ send({ type: "note_deleted", id: msg.id });
120
+ syncNotesKnowledge();
121
+ }
122
+ return true;
123
+ }
124
+
125
+ if (msg.type === "note_list_request") {
126
+ sendTo(ws, { type: "notes_list", notes: nm.list() });
127
+ return true;
128
+ }
129
+
130
+ if (msg.type === "note_bring_front") {
131
+ if (!msg.id) return true;
132
+ var front = nm.bringToFront(msg.id);
133
+ if (front) send({ type: "note_updated", note: front });
134
+ return true;
135
+ }
136
+
137
+ // --- Web terminal ---
138
+ if (msg.type === "term_create") {
139
+ if (ws._clayUser) {
140
+ var termPerms = usersModule.getEffectivePermissions(ws._clayUser, osUsers);
141
+ if (!termPerms.terminal) {
142
+ sendTo(ws, { type: "term_error", error: "Terminal access is not permitted" });
143
+ return true;
144
+ }
145
+ }
146
+ var t = tm.create(msg.cols || 80, msg.rows || 24, getOsUserInfoForWs(ws), ws);
147
+ if (!t) {
148
+ sendTo(ws, { type: "term_error", error: "Cannot create terminal (node-pty not available or limit reached)" });
149
+ return true;
150
+ }
151
+ tm.attach(t.id, ws);
152
+ send({ type: "term_list", terminals: tm.list() });
153
+ sendTo(ws, { type: "term_created", id: t.id });
154
+ return true;
155
+ }
156
+
157
+ if (msg.type === "term_attach") {
158
+ if (msg.id) tm.attach(msg.id, ws);
159
+ return true;
160
+ }
161
+
162
+ if (msg.type === "term_detach") {
163
+ if (msg.id) tm.detach(msg.id, ws);
164
+ return true;
165
+ }
166
+
167
+ if (msg.type === "term_input") {
168
+ if (msg.id) tm.write(msg.id, msg.data);
169
+ return true;
170
+ }
171
+
172
+ if (msg.type === "term_resize") {
173
+ if (msg.id && msg.cols > 0 && msg.rows > 0) {
174
+ tm.resize(msg.id, msg.cols, msg.rows, ws);
175
+ }
176
+ return true;
177
+ }
178
+
179
+ if (msg.type === "term_close") {
180
+ if (msg.id) {
181
+ tm.close(msg.id);
182
+ send({ type: "term_list", terminals: tm.list() });
183
+ // Remove closed terminal from context sources
184
+ var saved = loadContextSources(slug);
185
+ var termKey = "term:" + msg.id;
186
+ var filtered = saved.filter(function(id) { return id !== termKey; });
187
+ if (filtered.length !== saved.length) {
188
+ saveContextSources(slug, filtered);
189
+ send({ type: "context_sources_state", active: filtered });
190
+ }
191
+ }
192
+ return true;
193
+ }
194
+
195
+ if (msg.type === "term_rename") {
196
+ if (msg.id && msg.title) {
197
+ tm.rename(msg.id, msg.title);
198
+ send({ type: "term_list", terminals: tm.list() });
199
+ }
200
+ return true;
201
+ }
202
+
203
+ // --- Context Sources ---
204
+ if (msg.type === "context_sources_save") {
205
+ var activeIds = msg.active || [];
206
+ saveContextSources(slug, activeIds);
207
+ return true;
208
+ }
209
+
210
+ // --- Browser Extension ---
211
+ if (msg.type === "browser_tab_list") {
212
+ browserState._extensionWs = ws; // Track which client has the extension
213
+ var tabs = msg.tabs || [];
214
+ browserState._browserTabList = {};
215
+ for (var bti = 0; bti < tabs.length; bti++) {
216
+ browserState._browserTabList[tabs[bti].id] = tabs[bti];
217
+ }
218
+ return true;
219
+ }
220
+
221
+ if (msg.type === "extension_result") {
222
+ var pending = browserState.pendingExtensionRequests[msg.requestId];
223
+ if (pending) {
224
+ clearTimeout(pending.timer);
225
+ pending.resolve(msg.result);
226
+ delete browserState.pendingExtensionRequests[msg.requestId];
227
+ }
228
+ return true;
229
+ }
230
+
231
+ // --- Scheduled tasks permission gate ---
232
+ if (msg.type === "loop_start" || msg.type === "loop_stop" || msg.type === "loop_registry_files" ||
233
+ msg.type === "loop_registry_list" || msg.type === "loop_registry_update" || msg.type === "loop_registry_rename" ||
234
+ msg.type === "loop_registry_remove" || msg.type === "loop_registry_convert" || msg.type === "loop_registry_toggle" ||
235
+ msg.type === "loop_registry_rerun" || msg.type === "schedule_create" || msg.type === "schedule_move") {
236
+ if (ws._clayUser) {
237
+ var schPerms = usersModule.getEffectivePermissions(ws._clayUser, osUsers);
238
+ if (!schPerms.scheduledTasks) {
239
+ sendTo(ws, { type: "error", text: "Scheduled tasks access is not permitted" });
240
+ return true;
241
+ }
242
+ }
243
+ }
244
+
245
+ // --- Loop message delegation (project-loop.js) ---
246
+ if (_loop.handleLoopMessage(ws, msg)) return true;
247
+
248
+ // --- Schedule message for after rate limit resets ---
249
+ if (msg.type === "schedule_message") {
250
+ var schedSession = getSessionForWs(ws);
251
+ if (!schedSession || !msg.text || !msg.resetsAt) return true;
252
+ scheduleMessage(schedSession, msg.text, msg.resetsAt);
253
+ return true;
254
+ }
255
+
256
+ if (msg.type === "cancel_scheduled_message") {
257
+ var cancelSession = getSessionForWs(ws);
258
+ if (!cancelSession) return true;
259
+ cancelScheduledMessage(cancelSession);
260
+ return true;
261
+ }
262
+
263
+ if (msg.type === "send_scheduled_now") {
264
+ var nowSession = getSessionForWs(ws);
265
+ if (!nowSession || !nowSession.scheduledMessage) return true;
266
+ var schedText = nowSession.scheduledMessage.text;
267
+ clearTimeout(nowSession.scheduledMessage.timer);
268
+ nowSession.scheduledMessage = null;
269
+ console.log("[project] Scheduled message sent immediately for session " + nowSession.localId);
270
+ sm.sendAndRecord(nowSession, { type: "scheduled_message_sent" });
271
+ var userMsg = { type: "user_message", text: schedText };
272
+ nowSession.history.push(userMsg);
273
+ sm.appendToSessionFile(nowSession, userMsg);
274
+ sendToSession(nowSession.localId, userMsg);
275
+ nowSession.isProcessing = true;
276
+ onProcessingChanged();
277
+ sendToSession(nowSession.localId, { type: "status", status: "processing" });
278
+ sdk.startQuery(nowSession, schedText, null, getLinuxUserForSession(nowSession));
279
+ sm.broadcastSessionList();
280
+ return true;
281
+ }
282
+
283
+ if (msg.type !== "message") return false;
284
+ if (!msg.text && (!msg.images || msg.images.length === 0) && (!msg.pastes || msg.pastes.length === 0)) return true;
285
+
286
+ var session = getSessionForWs(ws);
287
+ if (!session) return true;
288
+
289
+ // Backfill ownerId for legacy sessions restored without one (multi-user only)
290
+ if (!session.ownerId && ws._clayUser && usersModule.isMultiUser()) {
291
+ session.ownerId = ws._clayUser.id;
292
+ sm.saveSessionFile(session);
293
+ }
294
+
295
+ // Keep any pending scheduled message alive when user sends a regular message
296
+
297
+ var userMsg2 = { type: "user_message", text: msg.text || "" };
298
+ // Attach sender info for multi-user attribution (backward-compatible: old clients ignore these)
299
+ if (ws._clayUser) {
300
+ userMsg2.from = ws._clayUser.id;
301
+ userMsg2.fromName = ws._clayUser.displayName || ws._clayUser.username || "";
302
+ }
303
+ var savedImagePaths = [];
304
+ if (msg.images && msg.images.length > 0) {
305
+ userMsg2.imageCount = msg.images.length;
306
+ // Save images as files, store URL references in history
307
+ var imageRefs = [];
308
+ for (var imgIdx = 0; imgIdx < msg.images.length; imgIdx++) {
309
+ var img = msg.images[imgIdx];
310
+ var savedName = saveImageFile(img.mediaType, img.data, getLinuxUserForSession(session));
311
+ if (savedName) {
312
+ imageRefs.push({ mediaType: img.mediaType, file: savedName });
313
+ savedImagePaths.push(path.join(imagesDir, savedName));
314
+ }
315
+ }
316
+ if (imageRefs.length > 0) {
317
+ userMsg2.imageRefs = imageRefs;
318
+ }
319
+ }
320
+ if (msg.pastes && msg.pastes.length > 0) {
321
+ userMsg2.pastes = msg.pastes;
322
+ }
323
+ session.history.push(userMsg2);
324
+ sm.appendToSessionFile(session, userMsg2);
325
+ sendToSessionOthers(ws, session.localId, hydrateImageRefs(userMsg2));
326
+
327
+ if (!session.title) {
328
+ session.title = (msg.text || "Image").substring(0, 50);
329
+ sm.saveSessionFile(session);
330
+ sm.broadcastSessionList();
331
+ // Sync auto-title to SDK
332
+ if (session.cliSessionId) {
333
+ getSDK().then(function(sdkMod) {
334
+ sdkMod.renameSession(session.cliSessionId, session.title, { dir: cwd }).catch(function(e) {
335
+ console.error("[project] SDK renameSession failed:", e.message);
336
+ });
337
+ }).catch(function() {});
338
+ }
339
+ }
340
+
341
+ var fullText = msg.text || "";
342
+ // Prepend saved image paths so Claude can copy/save them
343
+ if (savedImagePaths.length > 0) {
344
+ var imgPathLines = savedImagePaths.map(function (p) { return "[Uploaded image: " + p + "]"; }).join("\n");
345
+ fullText = imgPathLines + (fullText ? "\n" + fullText : "");
346
+ }
347
+ if (msg.pastes && msg.pastes.length > 0) {
348
+ for (var pi = 0; pi < msg.pastes.length; pi++) {
349
+ if (fullText) fullText += "\n\n";
350
+ fullText += msg.pastes[pi];
351
+ }
352
+ }
353
+
354
+ // Inject pending @mention context so the current agent sees the exchange
355
+ if (session.pendingMentionContexts && session.pendingMentionContexts.length > 0) {
356
+ var mentionPrefix = session.pendingMentionContexts.join("\n\n");
357
+ session.pendingMentionContexts = [];
358
+ fullText = mentionPrefix + "\n\n" + fullText;
359
+ }
360
+
361
+ // Inject active terminal context sources (delta only: send new output since last message)
362
+ var TERM_CONTEXT_MAX = 8192; // 8KB max per terminal per message
363
+ var TERM_HEAD_SIZE = 2048; // keep first 2KB for error context
364
+ var TERM_TAIL_SIZE = 6144; // keep last 6KB for recent state
365
+ var ctxSources = loadContextSources(slug);
366
+ if (ctxSources.length > 0) {
367
+ if (!session._termContextCursors) session._termContextCursors = {};
368
+ var termContextParts = [];
369
+ for (var ci = 0; ci < ctxSources.length; ci++) {
370
+ var srcId = ctxSources[ci];
371
+ if (srcId.startsWith("term:")) {
372
+ var termId = parseInt(srcId.split(":")[1], 10);
373
+ var sb = tm.getScrollback(termId);
374
+ if (sb) {
375
+ var lastCursor;
376
+ if (termId in session._termContextCursors) {
377
+ lastCursor = session._termContextCursors[termId];
378
+ // Terminal was recycled (closed and reopened with same ID) -- reset cursor
379
+ if (lastCursor > sb.totalBytesWritten) lastCursor = 0;
380
+ } else {
381
+ // First time seeing this terminal -- include last 8KB (what user can see now)
382
+ lastCursor = Math.max(0, sb.totalBytesWritten - TERM_CONTEXT_MAX);
383
+ }
384
+ var newBytes = sb.totalBytesWritten - lastCursor;
385
+ session._termContextCursors[termId] = sb.totalBytesWritten;
386
+ if (newBytes <= 0) continue;
387
+ // Build timestamped delta from chunks
388
+ var deltaChunks = [];
389
+ var bytePos = sb.bufferStart;
390
+ for (var chunkIdx = 0; chunkIdx < sb.chunks.length; chunkIdx++) {
391
+ var chunk = sb.chunks[chunkIdx];
392
+ var chunkEnd = bytePos + chunk.data.length;
393
+ if (chunkEnd > lastCursor) {
394
+ // This chunk has new content
395
+ var chunkData = chunk.data;
396
+ if (bytePos < lastCursor) {
397
+ // Partial chunk: only the part after lastCursor
398
+ chunkData = chunkData.slice(lastCursor - bytePos);
399
+ }
400
+ deltaChunks.push({ ts: chunk.ts, data: chunkData });
401
+ }
402
+ bytePos = chunkEnd;
403
+ }
404
+ if (deltaChunks.length === 0) continue;
405
+ // Format with timestamps: group by second to avoid excessive timestamps
406
+ var lines = [];
407
+ var lastTimeSec = 0;
408
+ for (var di = 0; di < deltaChunks.length; di++) {
409
+ var dc = deltaChunks[di];
410
+ var cleaned = dc.data.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
411
+ if (!cleaned) continue;
412
+ var timeSec = Math.floor(dc.ts / 1000);
413
+ if (timeSec !== lastTimeSec) {
414
+ var d = new Date(dc.ts);
415
+ var timeStr = d.toTimeString().slice(0, 8); // HH:MM:SS
416
+ lines.push("[" + timeStr + "] " + cleaned);
417
+ lastTimeSec = timeSec;
418
+ } else {
419
+ lines.push(cleaned);
420
+ }
421
+ }
422
+ var delta = lines.join("").trim();
423
+ if (!delta) continue;
424
+ var termInfo = tm.list().find(function(t) { return t.id === termId; });
425
+ var termTitle = termInfo ? termInfo.title : "Terminal " + termId;
426
+ var header;
427
+ if (delta.length > TERM_CONTEXT_MAX) {
428
+ var head = delta.slice(0, TERM_HEAD_SIZE);
429
+ var tail = delta.slice(-TERM_TAIL_SIZE);
430
+ var omittedBytes = delta.length - TERM_HEAD_SIZE - TERM_TAIL_SIZE;
431
+ var omittedLines = delta.slice(TERM_HEAD_SIZE, delta.length - TERM_TAIL_SIZE).split("\n").length;
432
+ delta = head + "\n\n... (" + omittedLines + " lines / " + Math.round(omittedBytes / 1024) + "KB omitted) ...\n\n" + tail;
433
+ header = "[New terminal output from " + termTitle + " (large output, head+tail shown)]";
434
+ } else {
435
+ header = "[New terminal output from " + termTitle + "]";
436
+ }
437
+ termContextParts.push(header + "\n```\n" + delta + "\n```");
438
+ }
439
+ }
440
+ }
441
+ if (termContextParts.length > 0) {
442
+ fullText = termContextParts.join("\n\n") + "\n\n" + fullText;
443
+ }
444
+ }
445
+
446
+ // Collect browser tab context (async: requires round-trip to client extension)
447
+ var _browserTabList = browserState._browserTabList;
448
+ var tabSources = ctxSources.filter(function(id) {
449
+ if (!id.startsWith("tab:")) return false;
450
+ // Only include tabs that currently exist in the browser
451
+ var tid = parseInt(id.split(":")[1], 10);
452
+ return !!_browserTabList[tid];
453
+ });
454
+
455
+ function dispatchToSdk(finalText) {
456
+ if (!session.isProcessing) {
457
+ session.isProcessing = true;
458
+ onProcessingChanged();
459
+ session.sentToolResults = {};
460
+ sendToSession(session.localId, { type: "status", status: "processing" });
461
+ if (!session.queryInstance && (!session.worker || session.messageQueue !== "worker")) {
462
+ // No active query (or worker idle between queries): start a new query
463
+ session._queryStartTs = Date.now();
464
+ console.log("[PERF] project.js: startQuery called, localId=" + session.localId + " t=0ms");
465
+ sdk.startQuery(session, finalText, msg.images, getLinuxUserForSession(session));
466
+ } else {
467
+ sdk.pushMessage(session, finalText, msg.images);
468
+ }
469
+ } else {
470
+ sdk.pushMessage(session, finalText, msg.images);
471
+ }
472
+ sm.broadcastSessionList();
473
+ }
474
+
475
+ if (tabSources.length > 0) {
476
+ // Request tab context from all active browser tab sources
477
+ var tabPromises = tabSources.map(function(srcId) {
478
+ var tabId = parseInt(srcId.split(":")[1], 10);
479
+ return requestTabContext(ws, tabId);
480
+ });
481
+ Promise.all(tabPromises).then(function(results) {
482
+ var tabContextParts = [];
483
+ var screenshotImages = [];
484
+
485
+ for (var ti = 0; ti < results.length; ti++) {
486
+ if (!results[ti]) continue;
487
+ var tabId2 = parseInt(tabSources[ti].split(":")[1], 10);
488
+ var tabInfo = _browserTabList[tabId2];
489
+ var tabLabel = tabInfo ? (tabInfo.title || tabInfo.url || "Tab " + tabId2) : "Tab " + tabId2;
490
+ var r = results[ti];
491
+ var parts = [];
492
+
493
+ // Console logs
494
+ if (r.console && r.console.logs) {
495
+ try {
496
+ var logs = typeof r.console.logs === "string" ? JSON.parse(r.console.logs) : r.console.logs;
497
+ if (logs && logs.length > 0) {
498
+ var logLines = [];
499
+ var logSlice = logs.slice(-50);
500
+ for (var li = 0; li < logSlice.length; li++) {
501
+ var entry = logSlice[li];
502
+ var ts = entry.ts ? new Date(entry.ts).toTimeString().slice(0, 8) : "";
503
+ var lvl = (entry.level || "log").toUpperCase();
504
+ logLines.push("[" + ts + " " + lvl + "] " + (entry.text || ""));
505
+ }
506
+ parts.push("Console:\n" + logLines.join("\n"));
507
+ }
508
+ } catch (e) {
509
+ // ignore parse errors
510
+ }
511
+ }
512
+
513
+ // Network requests
514
+ if (r.network && r.network.network) {
515
+ try {
516
+ var netLog = typeof r.network.network === "string" ? JSON.parse(r.network.network) : r.network.network;
517
+ if (netLog && netLog.length > 0) {
518
+ var netLines = [];
519
+ var netSlice = netLog.slice(-30);
520
+ for (var ni = 0; ni < netSlice.length; ni++) {
521
+ var req = netSlice[ni];
522
+ var line = (req.method || "GET") + " " + (req.url || "") + " " + (req.status || 0) + " " + (req.duration || 0) + "ms";
523
+ if (req.error) line += " [" + req.error + "]";
524
+ netLines.push(line);
525
+ }
526
+ parts.push("Network (last " + netSlice.length + " requests):\n" + netLines.join("\n"));
527
+ }
528
+ } catch (e) {
529
+ // ignore parse errors
530
+ }
531
+ }
532
+
533
+ // Page text (from tab_page_text command)
534
+ if (r.pageText && (r.pageText.text || r.pageText.value)) {
535
+ var pageContent = r.pageText.text || r.pageText.value;
536
+ if (pageContent.length > 0) {
537
+ if (pageContent.length > 32768) {
538
+ pageContent = pageContent.substring(0, 32768) + "\n... (truncated)";
539
+ }
540
+ parts.push("Page text:\n" + pageContent);
541
+ }
542
+ }
543
+
544
+ // Screenshot -- save to disk and add to images for SDK
545
+ if (r.screenshot && r.screenshot.image) {
546
+ try {
547
+ var screenshotData = r.screenshot.image;
548
+ var screenshotName = saveImageFile("image/png", screenshotData, getLinuxUserForSession(session));
549
+ if (screenshotName) {
550
+ var screenshotPath = path.join(imagesDir, screenshotName);
551
+ // Add to images array for SDK multimodal
552
+ screenshotImages.push({
553
+ mediaType: "image/png",
554
+ data: screenshotData,
555
+ file: screenshotName,
556
+ tabTitle: tabLabel,
557
+ tabUrl: tabInfo ? tabInfo.url : "",
558
+ tabFavIconUrl: tabInfo ? tabInfo.favIconUrl : ""
559
+ });
560
+ parts.push("[Screenshot saved: " + screenshotPath + "]");
561
+ }
562
+ } catch (e) {
563
+ // ignore screenshot save errors
564
+ }
565
+ }
566
+
567
+ if (r.console && r.console.error) {
568
+ parts.push("(Console error: " + r.console.error + ")");
569
+ }
570
+ if (r.network && r.network.error) {
571
+ parts.push("(Network error: " + r.network.error + ")");
572
+ }
573
+
574
+ if (parts.length > 0) {
575
+ tabContextParts.push("[Browser tab: " + tabLabel + "]\n" + parts.join("\n\n"));
576
+ }
577
+ }
578
+
579
+ if (tabContextParts.length > 0) {
580
+ fullText = "[The following browser tab data is automatically attached as context sources. Do NOT call browser_read_page, browser_console, browser_network, or browser_screenshot for these tabs -- the data is already here.]\n\n" +
581
+ tabContextParts.join("\n\n---\n\n") + "\n\n" + fullText;
582
+ }
583
+
584
+ // If screenshots were captured, send context preview cards and add to SDK images
585
+ if (screenshotImages.length > 0) {
586
+ if (!msg.images) msg.images = [];
587
+ for (var si = 0; si < screenshotImages.length; si++) {
588
+ var ss = screenshotImages[si];
589
+ // Save context_preview to history so it restores on session load
590
+ var previewEntry = {
591
+ type: "context_preview",
592
+ tab: {
593
+ title: ss.tabTitle || "",
594
+ url: ss.tabUrl || "",
595
+ favIconUrl: ss.tabFavIconUrl || "",
596
+ screenshotFile: ss.file
597
+ }
598
+ };
599
+ session.history.push(previewEntry);
600
+ // Send context card to all clients
601
+ sendToSession(session.localId, {
602
+ type: "context_preview",
603
+ tab: {
604
+ title: ss.tabTitle || "",
605
+ url: ss.tabUrl || "",
606
+ favIconUrl: ss.tabFavIconUrl || "",
607
+ screenshotUrl: "/p/" + slug + "/images/" + ss.file
608
+ }
609
+ });
610
+ // Add to SDK images for multimodal
611
+ msg.images.push({ mediaType: ss.mediaType, data: ss.data });
612
+ }
613
+ sm.saveSessionFile(session);
614
+ }
615
+
616
+ dispatchToSdk(fullText);
617
+ });
618
+ } else {
619
+ dispatchToSdk(fullText);
620
+ }
621
+
622
+ return true;
623
+ }
624
+
625
+ return {
626
+ handleUserMessage: handleUserMessage,
627
+ syncNotesKnowledge: syncNotesKnowledge,
628
+ };
629
+ }
630
+
631
+ module.exports = { attachUserMessage: attachUserMessage };