@jupyterlite/ai 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/chat-model.ts CHANGED
@@ -36,7 +36,19 @@ import type { UserContent, ImagePart, FilePart, ModelMessage } from 'ai';
36
36
 
37
37
  import { AI_AVATAR } from './icons';
38
38
 
39
- import type { IAgentManager, IAISettingsModel, ITokenUsage } from './tokens';
39
+ import type {
40
+ IAgentManager,
41
+ IAIChatModel,
42
+ IAISettingsModel,
43
+ IProviderRegistry,
44
+ ITokenUsage
45
+ } from './tokens';
46
+
47
+ import {
48
+ modelSupportsAudio,
49
+ modelSupportsImages,
50
+ modelSupportsPdf
51
+ } from './providers/model-info';
40
52
 
41
53
  /**
42
54
  * Tool call status types.
@@ -69,10 +81,6 @@ interface IToolExecutionContext {
69
81
  * The tool input (formatted).
70
82
  */
71
83
  input: string;
72
- /**
73
- * Optional approval ID if awaiting approval.
74
- */
75
- approvalId?: string;
76
84
  /**
77
85
  * Current status.
78
86
  */
@@ -91,7 +99,7 @@ interface IToolExecutionContext {
91
99
  * AI Chat Model implementation that provides chat functionality tool integration,
92
100
  * and MCP server support.
93
101
  */
94
- export class AIChatModel extends AbstractChatModel {
102
+ export class AIChatModel extends AbstractChatModel implements IAIChatModel {
95
103
  /**
96
104
  * Constructs a new AIChatModel instance.
97
105
  * @param options Configuration options for the chat model
@@ -109,6 +117,7 @@ export class AIChatModel extends AbstractChatModel {
109
117
  this._user = options.user;
110
118
  this._agentManager = options.agentManager;
111
119
  this._contentsManager = options.contentsManager;
120
+ this._providerRegistry = options.providerRegistry;
112
121
 
113
122
  // Listen for agent events
114
123
  this._agentManager.agentEvent.connect(this._onAgentEvent, this);
@@ -116,6 +125,13 @@ export class AIChatModel extends AbstractChatModel {
116
125
  // Listen for settings changes to update chat behavior
117
126
  this._settingsModel.stateChanged.connect(this._onSettingsChanged, this);
118
127
 
128
+ // Rebuild history when the model changes
129
+ this._agentManager.activeProviderChanged.connect(
130
+ this._onModelChanged,
131
+ this
132
+ );
133
+ this._settingsModel.stateChanged.connect(this._onModelChanged, this);
134
+
119
135
  this._autosaveDebouncer = new Debouncer(this.save, 3000);
120
136
  }
121
137
 
@@ -139,7 +155,7 @@ export class AIChatModel extends AbstractChatModel {
139
155
  /**
140
156
  * A signal emitting when the chat name has changed.
141
157
  */
142
- get nameChanged(): ISignal<AIChatModel, string> {
158
+ get nameChanged(): ISignal<IAIChatModel, string> {
143
159
  return this._nameChanged;
144
160
  }
145
161
 
@@ -160,7 +176,7 @@ export class AIChatModel extends AbstractChatModel {
160
176
  /**
161
177
  * A signal emitting when the chat title has changed.
162
178
  */
163
- get titleChanged(): ISignal<AIChatModel, string | null> {
179
+ get titleChanged(): ISignal<IAIChatModel, string | null> {
164
180
  return this._titleChanged;
165
181
  }
166
182
 
@@ -171,6 +187,9 @@ export class AIChatModel extends AbstractChatModel {
171
187
  return this._autosave;
172
188
  }
173
189
  set autosave(value: boolean) {
190
+ if (value === this._autosave) {
191
+ return;
192
+ }
174
193
  this._autosave = value;
175
194
  this._autosaveChanged.emit(value);
176
195
  if (value) {
@@ -182,7 +201,6 @@ export class AIChatModel extends AbstractChatModel {
182
201
  this._autosaveDebouncer.invoke,
183
202
  this._autosaveDebouncer
184
203
  );
185
- this._autosaveDebouncer.invoke();
186
204
  } else {
187
205
  this.messagesUpdated.disconnect(
188
206
  this._autosaveDebouncer.invoke,
@@ -193,12 +211,13 @@ export class AIChatModel extends AbstractChatModel {
193
211
  this._autosaveDebouncer
194
212
  );
195
213
  }
214
+ this._autosaveDebouncer.invoke();
196
215
  }
197
216
 
198
217
  /**
199
218
  * A signal emitting when the autosave flag changed.
200
219
  */
201
- get autosaveChanged(): ISignal<AIChatModel, boolean> {
220
+ get autosaveChanged(): ISignal<IAIChatModel, boolean> {
202
221
  return this._autosaveChanged;
203
222
  }
204
223
 
@@ -217,7 +236,7 @@ export class AIChatModel extends AbstractChatModel {
217
236
  }
218
237
 
219
238
  /**
220
- * Get the agent manager associated to the model.
239
+ * The agent manager used in the model.
221
240
  */
222
241
  get agentManager(): IAgentManager {
223
242
  return this._agentManager;
@@ -234,6 +253,7 @@ export class AIChatModel extends AbstractChatModel {
234
253
  * Dispose of the model.
235
254
  */
236
255
  dispose(): void {
256
+ this.stopStreaming();
237
257
  this.messagesUpdated.disconnect(
238
258
  this._autosaveDebouncer.invoke,
239
259
  this._autosaveDebouncer
@@ -253,7 +273,7 @@ export class AIChatModel extends AbstractChatModel {
253
273
  stopStreaming: () => this.stopStreaming(),
254
274
  clearMessages: () => this.clearMessages(),
255
275
  agentManager: this._agentManager,
256
- addSystemMessage: (body: string) => this.addSystemMessage(body)
276
+ addSystemMessage: (body: string) => this._addSystemMessage(body)
257
277
  };
258
278
  }
259
279
 
@@ -268,15 +288,31 @@ export class AIChatModel extends AbstractChatModel {
268
288
  * Clears all messages from the chat and resets conversation state.
269
289
  */
270
290
  clearMessages = async (): Promise<void> => {
291
+ this.stopStreaming();
292
+ this._messageQueue = [];
293
+ this._isBusy = false;
294
+ this._queueMessageId = null;
295
+ this._currentStreamingMessage = null;
271
296
  this.messagesDeleted(0, this.messages.length);
297
+ this.title = null;
272
298
  this._toolContexts.clear();
273
299
  await this._agentManager.clearHistory();
274
300
  };
275
301
 
302
+ /**
303
+ * Overrides messageAdded to ensure queued messages stay at the bottom.
304
+ */
305
+ override messageAdded(message: IMessageContent): void {
306
+ super.messageAdded(message);
307
+ if (this._queueMessageId && message.id !== this._queueMessageId) {
308
+ this._updateQueueUI();
309
+ }
310
+ }
311
+
276
312
  /**
277
313
  * Adds a non-user message to the chat (used by chat commands).
278
314
  */
279
- addSystemMessage(body: string): void {
315
+ private _addSystemMessage(body: string): void {
280
316
  const message: IMessageContent = {
281
317
  body,
282
318
  sender: this._getAIUser(),
@@ -309,10 +345,10 @@ export class AIChatModel extends AbstractChatModel {
309
345
  raw_time: false,
310
346
  attachments: [...this.input.attachments]
311
347
  };
312
- this.messageAdded(userMessage);
313
348
 
314
349
  // Check if we have valid configuration
315
350
  if (!this._agentManager.hasValidConfig()) {
351
+ this.messageAdded(userMessage);
316
352
  const errorMessage: IMessageContent = {
317
353
  body: 'Please configure your AI settings first. Open the AI Settings to set your API key and model.',
318
354
  sender: this._getAIUser(),
@@ -325,35 +361,71 @@ export class AIChatModel extends AbstractChatModel {
325
361
  return;
326
362
  }
327
363
 
328
- try {
329
- // Process attachments and add their content to the message
330
- let enhancedMessage: UserContent = message.body;
331
- if (this.input.attachments.length > 0) {
332
- const { textContents, binaryParts } = await Private.processAttachments(
333
- this.input.attachments,
334
- this.input.documentManager
335
- );
336
- this.input.clearAttachments();
364
+ if (this._isBusy) {
365
+ this._messageQueue.push({
366
+ id: UUID.uuid4(),
367
+ body: message.body,
368
+ _originalMsg: userMessage
369
+ });
370
+ this.input.clearAttachments();
371
+ this._updateQueueUI();
372
+ return;
373
+ }
337
374
 
338
- let textPart = message.body;
339
- if (textContents.length > 0) {
340
- textPart +=
341
- '\n\n--- Attached Files ---\n' + textContents.join('\n\n');
342
- }
375
+ this._isBusy = true;
376
+ this.messageAdded(userMessage);
377
+ this.input.clearAttachments();
343
378
 
344
- if (binaryParts.length > 0) {
345
- enhancedMessage = [{ type: 'text', text: textPart }, ...binaryParts];
346
- } else {
347
- enhancedMessage = textPart;
348
- }
349
- }
379
+ await this._processMessage(userMessage);
380
+ }
350
381
 
382
+ /**
383
+ * Internal method to process attachments and send the message to the agent.
384
+ */
385
+ private async _processMessage(userMessage: IMessageContent): Promise<void> {
386
+ try {
351
387
  this.updateWriters([{ user: this._getAIUser() }]);
352
388
 
389
+ let enhancedMessage: UserContent = userMessage.body;
390
+ if (userMessage.attachments && userMessage.attachments.length > 0) {
391
+ const providerConfig = this._settingsModel.getProvider(
392
+ this._agentManager.activeProvider
393
+ );
394
+ const supportsImages = modelSupportsImages(
395
+ providerConfig,
396
+ this._providerRegistry
397
+ );
398
+ const supportsPdf = modelSupportsPdf(
399
+ providerConfig,
400
+ this._providerRegistry
401
+ );
402
+ const supportsAudio = modelSupportsAudio(
403
+ providerConfig,
404
+ this._providerRegistry
405
+ );
406
+
407
+ enhancedMessage = await Private.processAttachments(
408
+ userMessage.attachments,
409
+ this.input.documentManager,
410
+ userMessage.body,
411
+ supportsImages,
412
+ supportsPdf,
413
+ supportsAudio
414
+ );
415
+ }
416
+
353
417
  await this._agentManager.generateResponse(enhancedMessage);
354
418
  } catch (error) {
355
419
  const errorMessage: IMessageContent = {
356
- body: `Error generating AI response: ${(error as Error).message}`,
420
+ body: '',
421
+ mime_model: {
422
+ data: {
423
+ 'application/vnd.jupyter.chat.components': 'error'
424
+ },
425
+ metadata: {
426
+ errorMessage: `Error generating AI response: ${(error as Error).message}`
427
+ }
428
+ },
357
429
  sender: this._getAIUser(),
358
430
  id: UUID.uuid4(),
359
431
  time: Date.now() / 1000,
@@ -362,8 +434,102 @@ export class AIChatModel extends AbstractChatModel {
362
434
  };
363
435
  this.messageAdded(errorMessage);
364
436
  } finally {
437
+ this._drainQueue();
438
+
439
+ if (
440
+ this._settingsModel.config.autoTitle &&
441
+ (this.messages.length <= 5 || this.title === null)
442
+ ) {
443
+ try {
444
+ this.title = await this.requestTitle();
445
+ } catch (e) {
446
+ console.warn('Error while generating a title\n', e);
447
+ }
448
+ }
449
+ }
450
+ }
451
+
452
+ /**
453
+ * Removes the message-queue chat component.
454
+ */
455
+ private _removeQueueUI(): void {
456
+ if (this._queueMessageId) {
457
+ const existingMsg = this.messages.find(
458
+ msg => msg.id === this._queueMessageId
459
+ );
460
+ if (existingMsg) {
461
+ const idx = this.messages.indexOf(existingMsg);
462
+ if (idx !== -1) {
463
+ this.messagesDeleted(idx, 1);
464
+ }
465
+ }
466
+ this._queueMessageId = null;
467
+ }
468
+ }
469
+
470
+ /**
471
+ * Creates or updates the message-queue chat component.
472
+ */
473
+ private _updateQueueUI(): void {
474
+ this._removeQueueUI();
475
+
476
+ if (this._messageQueue.length === 0) {
477
+ return;
478
+ }
479
+
480
+ const queueBody = {
481
+ data: {
482
+ 'application/vnd.jupyter.chat.components': 'message-queue'
483
+ },
484
+ metadata: {
485
+ messages: this._messageQueue.map(m => ({
486
+ id: m.id,
487
+ body: m.body,
488
+ attachments: m._originalMsg.attachments
489
+ })),
490
+ targetId: this.name
491
+ }
492
+ } as IMimeModelBody;
493
+
494
+ this._queueMessageId = UUID.uuid4();
495
+ const queueMessage: IMessageContent = {
496
+ body: '',
497
+ mime_model: queueBody,
498
+ sender: { username: 'system', display_name: '' },
499
+ id: this._queueMessageId,
500
+ time: Date.now() / 1000,
501
+ type: 'msg',
502
+ raw_time: false
503
+ };
504
+ this.messageAdded(queueMessage);
505
+ }
506
+
507
+ /**
508
+ * Processes the next message in the queue, or marks the agent as idle.
509
+ */
510
+ private async _drainQueue(): Promise<void> {
511
+ if (this._messageQueue.length === 0) {
512
+ this._isBusy = false;
365
513
  this.updateWriters([]);
514
+ this._removeQueueUI();
515
+ return;
366
516
  }
517
+
518
+ // Dequeue and push to chat
519
+ const next = this._messageQueue.shift()!;
520
+ next._originalMsg.time = Date.now() / 1000;
521
+ this.messageAdded(next._originalMsg);
522
+
523
+ await this._processMessage(next._originalMsg);
524
+ }
525
+
526
+ /**
527
+ * Removes a queued message by its ID.
528
+ * @param messageId The ID of the queued message to remove
529
+ */
530
+ removeQueuedMessage(messageId: string): void {
531
+ this._messageQueue = this._messageQueue.filter(msg => msg.id !== messageId);
532
+ this._updateQueueUI();
367
533
  }
368
534
 
369
535
  /**
@@ -452,7 +618,7 @@ export class AIChatModel extends AbstractChatModel {
452
618
  });
453
619
  await this.clearMessages();
454
620
  this.messagesInserted(0, messages);
455
- this._agentManager.setHistory(messages);
621
+ await this._rebuildHistory();
456
622
  this.autosave = content.metadata?.autosave ?? false;
457
623
  this.title = content.metadata?.title ?? null;
458
624
  return true;
@@ -473,7 +639,7 @@ export class AIChatModel extends AbstractChatModel {
473
639
  {
474
640
  role: 'system',
475
641
  content:
476
- "Generate a concise title (no more than 10 words) for the following conversation. Do not use formatting. Focus on the user's main intent."
642
+ "Generate a concise title (no more than 10 words) for the following conversation. Do not use formatting, quotes, or punctuation. Focus on the subject matter and specific content the user is working on, not on the actions taken (e.g. prefer 'Pandas DataFrame filtering' over 'Opening a notebook'). The title should be a noun phrase describing the topic."
477
643
  },
478
644
  {
479
645
  role: 'user',
@@ -494,6 +660,13 @@ export class AIChatModel extends AbstractChatModel {
494
660
  const attachmentsList: IAttachment[] = []; // Actual attachments
495
661
 
496
662
  this.messages.forEach(message => {
663
+ if (
664
+ message.content?.mime_model?.data?.[
665
+ 'application/vnd.jupyter.chat.components'
666
+ ] === 'message-queue'
667
+ ) {
668
+ return;
669
+ }
497
670
  let attachmentIndexes: string[] = [];
498
671
  if (message.attachments) {
499
672
  attachmentIndexes = message.attachments.map(attachment => {
@@ -559,6 +732,75 @@ export class AIChatModel extends AbstractChatModel {
559
732
  // Agent manager handles agent recreation automatically via its own settings listener
560
733
  }
561
734
 
735
+ /**
736
+ * Rebuild history when the active model changes.
737
+ */
738
+ private _onModelChanged(): void {
739
+ const providerConfig = this._settingsModel.getProvider(
740
+ this._agentManager.activeProvider
741
+ );
742
+ const modelKey = providerConfig
743
+ ? `${providerConfig.provider}:${providerConfig.model}`
744
+ : undefined;
745
+ if (modelKey && modelKey !== this._currentModelKey) {
746
+ this._currentModelKey = modelKey;
747
+ this._rebuildHistory().catch(e =>
748
+ console.warn('Failed to rebuild history on model change:', e)
749
+ );
750
+ }
751
+ }
752
+
753
+ /**
754
+ * Rebuilds the agent history from the current messages.
755
+ * For vision-capable models, re-reads binary attachments from disk.
756
+ * For text-only models, uses message text only.
757
+ */
758
+ private async _rebuildHistory(): Promise<void> {
759
+ const providerConfig = this._settingsModel.getProvider(
760
+ this._agentManager.activeProvider
761
+ );
762
+ const supportsImages = modelSupportsImages(
763
+ providerConfig,
764
+ this._providerRegistry
765
+ );
766
+ const supportsPdf = modelSupportsPdf(
767
+ providerConfig,
768
+ this._providerRegistry
769
+ );
770
+ const supportsAudio = modelSupportsAudio(
771
+ providerConfig,
772
+ this._providerRegistry
773
+ );
774
+
775
+ const modelMessages: ModelMessage[] = [];
776
+ for (const msg of this.messages) {
777
+ const isAI = msg.sender.username === 'ai-assistant';
778
+ if (!isAI && msg.attachments?.length) {
779
+ const enhancedContent = await Private.processAttachments(
780
+ msg.attachments,
781
+ this.input.documentManager,
782
+ msg.body,
783
+ supportsImages,
784
+ supportsPdf,
785
+ supportsAudio
786
+ );
787
+
788
+ modelMessages.push({
789
+ role: 'user',
790
+ content: enhancedContent
791
+ } as ModelMessage);
792
+ } else if (msg.body) {
793
+ modelMessages.push({
794
+ role: isAI ? 'assistant' : 'user',
795
+ content: msg.body
796
+ } as ModelMessage);
797
+ }
798
+ // Skip messages with empty body like tool calls
799
+ }
800
+
801
+ this._agentManager.setHistory(modelMessages);
802
+ }
803
+
562
804
  /**
563
805
  * Handles events emitted by the agent manager.
564
806
  * @param event The event data containing type and payload
@@ -759,13 +1001,18 @@ export class AIChatModel extends AbstractChatModel {
759
1001
  body: '',
760
1002
  mime_model: {
761
1003
  data: {
762
- 'application/vnd.jupyter.chat.components': 'tool-call'
1004
+ 'application/vnd.jupyter.chat.components': 'grouped-tool-calls'
763
1005
  },
764
1006
  metadata: {
765
- toolName: context.toolName,
766
- input: context.input,
767
- status: context.status,
768
- summary: context.summary
1007
+ toolCalls: [
1008
+ {
1009
+ toolCallId: context.toolCallId,
1010
+ title: `${context.toolName}${context.summary ? ' : ' + context.summary : ''}`,
1011
+ kind: context.toolName,
1012
+ status: 'in_progress',
1013
+ rawInput: context.input
1014
+ }
1015
+ ]
769
1016
  }
770
1017
  },
771
1018
  sender: this._getAIUser(),
@@ -837,7 +1084,15 @@ export class AIChatModel extends AbstractChatModel {
837
1084
  */
838
1085
  private _handleErrorEvent(event: IAgentManager.IAgentEvent<'error'>): void {
839
1086
  this.messageAdded({
840
- body: `Error generating response: ${event.data.error.message}`,
1087
+ body: '',
1088
+ mime_model: {
1089
+ data: {
1090
+ 'application/vnd.jupyter.chat.components': 'error'
1091
+ },
1092
+ metadata: {
1093
+ errorMessage: `Error generating response: ${event.data.error.message}`
1094
+ }
1095
+ },
841
1096
  sender: this._getAIUser(),
842
1097
  id: UUID.uuid4(),
843
1098
  time: Date.now() / 1000,
@@ -856,7 +1111,6 @@ export class AIChatModel extends AbstractChatModel {
856
1111
  if (!context) {
857
1112
  return;
858
1113
  }
859
- context.approvalId = event.data.approvalId;
860
1114
  context.input = JSON.stringify(event.data.args, null, 2);
861
1115
  this._updateToolCallUI(event.data.toolCallId, 'awaiting_approval');
862
1116
  }
@@ -867,15 +1121,13 @@ export class AIChatModel extends AbstractChatModel {
867
1121
  private _handleToolApprovalResolved(
868
1122
  event: IAgentManager.IAgentEvent<'tool_approval_resolved'>
869
1123
  ): void {
870
- const context = Array.from(this._toolContexts.values()).find(
871
- ctx => ctx.approvalId === event.data.approvalId
872
- );
1124
+ const context = this._toolContexts.get(event.data.toolCallId);
873
1125
  if (!context) {
874
1126
  return;
875
1127
  }
876
1128
 
877
1129
  const status = event.data.approved ? 'approved' : 'rejected';
878
- this._updateToolCallUI(context.toolCallId, status);
1130
+ this._updateToolCallUI(event.data.toolCallId, status);
879
1131
 
880
1132
  if (!event.data.approved) {
881
1133
  this._toolContexts.delete(context.toolCallId);
@@ -906,37 +1158,84 @@ export class AIChatModel extends AbstractChatModel {
906
1158
  existingMessage.update({
907
1159
  mime_model: {
908
1160
  data: {
909
- 'application/vnd.jupyter.chat.components': 'tool-call'
1161
+ 'application/vnd.jupyter.chat.components': 'grouped-tool-calls'
910
1162
  },
911
1163
  metadata: {
912
- toolName: context.toolName,
913
- input: context.input,
914
- status: context.status,
915
- summary: context.summary,
916
- output,
917
- targetId: this.name,
918
- approvalId: context.approvalId
1164
+ toolCalls: [
1165
+ {
1166
+ toolCallId: context.toolCallId,
1167
+ title: `${context.toolName}${context.summary ? ' : ' + context.summary : ''}`,
1168
+ kind: context.toolName,
1169
+ status: context.status,
1170
+ rawInput: context.input,
1171
+ rawOutput: output,
1172
+ sessionId: this.name,
1173
+ permissionStatus:
1174
+ status === 'awaiting_approval' ? 'pending' : 'resolved',
1175
+ ...(status === 'awaiting_approval' && {
1176
+ permissionOptions: [
1177
+ { optionId: 'approve', name: 'Approve', kind: 'allow_once' },
1178
+ { optionId: 'reject', name: 'Reject', kind: 'reject_once' }
1179
+ ]
1180
+ })
1181
+ }
1182
+ ]
919
1183
  }
920
1184
  }
921
1185
  });
922
1186
  }
923
1187
 
1188
+ /**
1189
+ * The current message queue
1190
+ */
1191
+ get messageQueue(): Private.IQueuedItem[] {
1192
+ return this._messageQueue;
1193
+ }
1194
+ set messageQueue(value: Private.IQueuedItem[]) {
1195
+ this._messageQueue = value;
1196
+ this._updateQueueUI();
1197
+ if (this._messageQueue.length > 0 && !this._isBusy) {
1198
+ this._drainQueue();
1199
+ }
1200
+ }
1201
+
1202
+ /**
1203
+ * Whether the chat is busy
1204
+ */
1205
+ get isBusy(): boolean {
1206
+ return this._isBusy;
1207
+ }
1208
+ set isBusy(value: boolean) {
1209
+ this._isBusy = value;
1210
+ }
1211
+
924
1212
  // Private fields
925
1213
  private _settingsModel: IAISettingsModel;
926
1214
  private _user: IUser;
927
1215
  private _toolContexts: Map<string, IToolExecutionContext> = new Map();
928
1216
  private _agentManager: IAgentManager;
1217
+ private _providerRegistry?: IProviderRegistry;
1218
+ private _currentModelKey: string | undefined;
929
1219
  private _currentStreamingMessage: IMessage | null = null;
930
- private _nameChanged = new Signal<AIChatModel, string>(this);
1220
+ private _nameChanged = new Signal<IAIChatModel, string>(this);
931
1221
  private _contentsManager?: Contents.IManager;
932
1222
  private _autosave: boolean = false;
933
- private _autosaveChanged = new Signal<AIChatModel, boolean>(this);
1223
+ private _autosaveChanged = new Signal<IAIChatModel, boolean>(this);
934
1224
  private _autosaveDebouncer: Debouncer;
1225
+ private _messageQueue: Private.IQueuedItem[] = [];
1226
+ private _isBusy: boolean = false;
1227
+ private _queueMessageId: string | null = null;
935
1228
  private _title: string | null = null;
936
- private _titleChanged = new Signal<AIChatModel, string | null>(this);
1229
+ private _titleChanged = new Signal<IAIChatModel, string | null>(this);
937
1230
  }
938
1231
 
939
1232
  namespace Private {
1233
+ export interface IQueuedItem {
1234
+ id: string;
1235
+ body: string;
1236
+ _originalMsg: IMessageContent;
1237
+ }
1238
+
940
1239
  type IDisplayOutput =
941
1240
  | nbformat.IDisplayData
942
1241
  | nbformat.IDisplayUpdate
@@ -1043,23 +1342,29 @@ namespace Private {
1043
1342
  }
1044
1343
 
1045
1344
  /**
1046
- * Processes file attachments and returns text contents and binary parts separately.
1345
+ * Processes file attachments and returns the message content with the attachments.
1047
1346
  * @param attachments Array of file attachments to process
1048
1347
  * @param documentManager Optional document manager for file operations
1049
- * @returns Text contents and binary parts
1348
+ * @param body The message body
1349
+ * @param supportsImages Whether the model supports images
1350
+ * @param supportsPdf Whether the model supports pdfs
1351
+ * @param supportsAudio Whether the model supports audio
1352
+ * @returns Enhanced message content
1050
1353
  */
1051
1354
  export async function processAttachments(
1052
1355
  attachments: IAttachment[],
1053
- documentManager: IDocumentManager | null | undefined
1054
- ): Promise<{
1055
- textContents: string[];
1056
- binaryParts: Array<ImagePart | FilePart>;
1057
- }> {
1356
+ documentManager: IDocumentManager | null | undefined,
1357
+ body: string,
1358
+ supportsImages: boolean,
1359
+ supportsPdf: boolean,
1360
+ supportsAudio: boolean
1361
+ ): Promise<UserContent> {
1058
1362
  const textContents: string[] = [];
1059
- const binaryParts: Array<ImagePart | FilePart> = [];
1363
+ const includedParts: Array<ImagePart | FilePart> = [];
1364
+ const omittedNames: string[] = [];
1060
1365
 
1061
1366
  if (!documentManager) {
1062
- return { textContents, binaryParts };
1367
+ return body;
1063
1368
  }
1064
1369
 
1065
1370
  for (const attachment of attachments) {
@@ -1093,29 +1398,54 @@ namespace Private {
1093
1398
  }
1094
1399
 
1095
1400
  if (mimetype?.startsWith('image/')) {
1096
- const data = await readBinaryAttachment(
1097
- attachment,
1098
- documentManager
1099
- );
1100
- if (data) {
1101
- binaryParts.push({
1102
- type: 'image',
1103
- image: data,
1104
- mediaType: mimetype
1105
- });
1401
+ if (supportsImages) {
1402
+ const data = await readBinaryAttachment(
1403
+ attachment,
1404
+ documentManager
1405
+ );
1406
+ if (data) {
1407
+ includedParts.push({
1408
+ type: 'image',
1409
+ image: data,
1410
+ mediaType: mimetype
1411
+ });
1412
+ }
1413
+ } else {
1414
+ omittedNames.push(PathExt.basename(attachment.value));
1106
1415
  }
1107
1416
  } else if (mimetype === 'application/pdf') {
1108
- const data = await readBinaryAttachment(
1109
- attachment,
1110
- documentManager
1111
- );
1112
- if (data) {
1113
- binaryParts.push({
1114
- type: 'file',
1115
- data,
1116
- mediaType: mimetype,
1117
- filename: PathExt.basename(attachment.value)
1118
- });
1417
+ if (supportsPdf) {
1418
+ const data = await readBinaryAttachment(
1419
+ attachment,
1420
+ documentManager
1421
+ );
1422
+ if (data) {
1423
+ includedParts.push({
1424
+ type: 'file',
1425
+ data,
1426
+ mediaType: mimetype,
1427
+ filename: PathExt.basename(attachment.value)
1428
+ });
1429
+ }
1430
+ } else {
1431
+ omittedNames.push(PathExt.basename(attachment.value));
1432
+ }
1433
+ } else if (mimetype?.startsWith('audio/')) {
1434
+ if (supportsAudio) {
1435
+ const data = await readBinaryAttachment(
1436
+ attachment,
1437
+ documentManager
1438
+ );
1439
+ if (data) {
1440
+ includedParts.push({
1441
+ type: 'file',
1442
+ data,
1443
+ mediaType: mimetype,
1444
+ filename: PathExt.basename(attachment.value)
1445
+ });
1446
+ }
1447
+ } else {
1448
+ omittedNames.push(PathExt.basename(attachment.value));
1119
1449
  }
1120
1450
  } else {
1121
1451
  const fileContent = await readFileAttachment(
@@ -1142,7 +1472,18 @@ namespace Private {
1142
1472
  }
1143
1473
  }
1144
1474
 
1145
- return { textContents, binaryParts };
1475
+ let textPart = body;
1476
+ if (textContents.length > 0) {
1477
+ textPart += '\n\n--- Attached Files ---\n' + textContents.join('\n\n');
1478
+ }
1479
+
1480
+ if (omittedNames.length > 0) {
1481
+ textPart += `\n[Attachments omitted (not supported by this model): ${omittedNames.join(', ')}.]`;
1482
+ }
1483
+
1484
+ return includedParts.length > 0
1485
+ ? [{ type: 'text', text: textPart }, ...includedParts]
1486
+ : textPart;
1146
1487
  }
1147
1488
 
1148
1489
  /**
@@ -1462,6 +1803,10 @@ export namespace AIChatModel {
1462
1803
  * The contents manager.
1463
1804
  */
1464
1805
  contentsManager?: Contents.IManager;
1806
+ /**
1807
+ * Optional provider registry for model capability lookups.
1808
+ */
1809
+ providerRegistry?: IProviderRegistry;
1465
1810
  /**
1466
1811
  * Whether to restore or not the message (default to true)
1467
1812
  */