@wabot-dev/framework 0.9.24 → 0.9.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,6 @@ import '../../../feature/chat-bot/ChatBot.js';
7
7
  import '../../../feature/chat-bot/ChatOperator.js';
8
8
  import '../../../feature/chat-bot/UnionChatAdapter.js';
9
9
  import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
10
- import '../../../feature/chat-bot/ImageDescriber.js';
11
10
  import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
12
11
  import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
13
12
  import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
@@ -15,7 +14,6 @@ import 'uuid';
15
14
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
16
15
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
17
16
  import { safeJsonParse } from '../../../feature/chat-bot/safeJsonParse.js';
18
- import { unconsumedMediaStartIndex } from '../../../feature/chat-bot/unconsumedMediaStartIndex.js';
19
17
  import { Anthropic } from '@anthropic-ai/sdk';
20
18
 
21
19
  const ANTHROPIC_SUPPORTED_IMAGE_MIME_TYPES = [
@@ -62,11 +60,10 @@ let AnthropicChatAdapter = class AnthropicChatAdapter {
62
60
  }
63
61
  mapChatItems(chatItems) {
64
62
  const messages = [];
65
- const mediaStart = unconsumedMediaStartIndex(chatItems);
66
- chatItems.forEach((chatItem, index) => {
63
+ chatItems.forEach((chatItem) => {
67
64
  switch (chatItem.type) {
68
65
  case 'humanMessage':
69
- messages.push(this.mapHumanMessage(chatItem.humanMessage, index >= mediaStart));
66
+ messages.push(this.mapHumanMessage(chatItem.humanMessage));
70
67
  break;
71
68
  case 'botMessage':
72
69
  messages.push(this.mapBotMessage(chatItem.botMessage));
@@ -78,7 +75,7 @@ let AnthropicChatAdapter = class AnthropicChatAdapter {
78
75
  });
79
76
  return messages;
80
77
  }
81
- mapHumanMessage(item, includeMedia) {
78
+ mapHumanMessage(item) {
82
79
  if (isChatMessageEmpty(item)) {
83
80
  throw new Error('User message content is empty');
84
81
  }
@@ -90,14 +87,14 @@ let AnthropicChatAdapter = class AnthropicChatAdapter {
90
87
  supportedDocumentMimeTypes: ANTHROPIC_SUPPORTED_DOCUMENT_MIME_TYPES,
91
88
  }),
92
89
  });
93
- if (includeMedia && item.images) {
90
+ if (item.images) {
94
91
  for (const image of item.images) {
95
92
  if (!ANTHROPIC_SUPPORTED_IMAGE_MIME_TYPES.includes(image.mimeType))
96
93
  continue;
97
94
  blocks.push({ type: 'image', source: this.toAnthropicImageSource(image) });
98
95
  }
99
96
  }
100
- if (includeMedia && item.documents) {
97
+ if (item.documents) {
101
98
  for (const doc of item.documents) {
102
99
  if (!ANTHROPIC_SUPPORTED_DOCUMENT_MIME_TYPES.includes(doc.mimeType))
103
100
  continue;
@@ -6,7 +6,6 @@ import '../../../feature/chat-bot/ChatBot.js';
6
6
  import '../../../feature/chat-bot/ChatOperator.js';
7
7
  import '../../../feature/chat-bot/UnionChatAdapter.js';
8
8
  import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
9
- import '../../../feature/chat-bot/ImageDescriber.js';
10
9
  import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
11
10
  import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
12
11
  import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
@@ -8,7 +8,6 @@ import '../../../feature/chat-bot/ChatBot.js';
8
8
  import '../../../feature/chat-bot/ChatOperator.js';
9
9
  import '../../../feature/chat-bot/UnionChatAdapter.js';
10
10
  import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
11
- import '../../../feature/chat-bot/ImageDescriber.js';
12
11
  import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
13
12
  import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
14
13
  import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
@@ -16,7 +15,6 @@ import 'uuid';
16
15
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
17
16
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
18
17
  import { safeJsonParse } from '../../../feature/chat-bot/safeJsonParse.js';
19
- import { unconsumedMediaStartIndex } from '../../../feature/chat-bot/unconsumedMediaStartIndex.js';
20
18
  import { GoogleGenAI } from '@google/genai';
21
19
 
22
20
  const GOOGLE_SUPPORTED_IMAGE_MIME_TYPES = [
@@ -84,11 +82,10 @@ let GoogleChatAdapter = class GoogleChatAdapter {
84
82
  }
85
83
  async mapChatItems(chatItems) {
86
84
  const contents = [];
87
- const mediaStart = unconsumedMediaStartIndex(chatItems);
88
- for (const [index, chatItem] of chatItems.entries()) {
85
+ for (const chatItem of chatItems) {
89
86
  switch (chatItem.type) {
90
87
  case 'humanMessage':
91
- contents.push(await this.mapHumanMessage(chatItem.humanMessage, index >= mediaStart));
88
+ contents.push(await this.mapHumanMessage(chatItem.humanMessage));
92
89
  break;
93
90
  case 'botMessage':
94
91
  contents.push(this.mapBotMessage(chatItem.botMessage));
@@ -100,7 +97,7 @@ let GoogleChatAdapter = class GoogleChatAdapter {
100
97
  }
101
98
  return contents;
102
99
  }
103
- async mapHumanMessage(item, includeMedia) {
100
+ async mapHumanMessage(item) {
104
101
  if (isChatMessageEmpty(item)) {
105
102
  throw new Error('User message content is empty');
106
103
  }
@@ -112,14 +109,14 @@ let GoogleChatAdapter = class GoogleChatAdapter {
112
109
  }),
113
110
  });
114
111
  const filesToSend = [];
115
- if (includeMedia && item.images) {
112
+ if (item.images) {
116
113
  for (const image of item.images) {
117
114
  if (!GOOGLE_SUPPORTED_IMAGE_MIME_TYPES.includes(image.mimeType))
118
115
  continue;
119
116
  filesToSend.push(image);
120
117
  }
121
118
  }
122
- if (includeMedia && item.documents) {
119
+ if (item.documents) {
123
120
  for (const doc of item.documents) {
124
121
  if (!GOOGLE_SUPPORTED_DOCUMENT_MIME_TYPES.includes(doc.mimeType))
125
122
  continue;
@@ -6,7 +6,6 @@ import '../../../feature/chat-bot/ChatBot.js';
6
6
  import { ChatItem } from '../../../feature/chat-bot/ChatItem.js';
7
7
  import '../../../feature/chat-bot/ChatOperator.js';
8
8
  import '../../../feature/chat-bot/UnionChatAdapter.js';
9
- import '../../../feature/chat-bot/ImageDescriber.js';
10
9
  import '../../../core/injection/index.js';
11
10
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
12
11
  import 'uuid';
@@ -10,7 +10,6 @@ import '../../../feature/chat-bot/ChatAdapterRegistry.js';
10
10
  import '../../../feature/chat-bot/ChatBot.js';
11
11
  import { ChatOperator } from '../../../feature/chat-bot/ChatOperator.js';
12
12
  import '../../../feature/chat-bot/UnionChatAdapter.js';
13
- import '../../../feature/chat-bot/ImageDescriber.js';
14
13
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
15
14
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
16
15
  import '../../../core/error/setupErrorHandlers.js';
@@ -4,7 +4,6 @@ import '../../../feature/chat-bot/ChatBot.js';
4
4
  import '../../../feature/chat-bot/ChatOperator.js';
5
5
  import '../../../feature/chat-bot/UnionChatAdapter.js';
6
6
  import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
7
- import '../../../feature/chat-bot/ImageDescriber.js';
8
7
  import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
9
8
  import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
10
9
  import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
@@ -13,7 +12,6 @@ import 'uuid';
13
12
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
14
13
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
15
14
  import '../../../core/error/setupErrorHandlers.js';
16
- import { unconsumedMediaStartIndex } from '../../../feature/chat-bot/unconsumedMediaStartIndex.js';
17
15
  import { Logger } from '../../../core/logger/Logger.js';
18
16
  import { OpenAI } from 'openai';
19
17
 
@@ -53,11 +51,10 @@ let OpenaiChatAdapter = class OpenaiChatAdapter {
53
51
  }
54
52
  mapChatItems(chatItems) {
55
53
  const openIaInput = [];
56
- const mediaStart = unconsumedMediaStartIndex(chatItems);
57
- chatItems.forEach((chatItem, index) => {
54
+ chatItems.forEach((chatItem) => {
58
55
  switch (chatItem.type) {
59
56
  case 'humanMessage':
60
- openIaInput.push(this.mapConectionMessage(chatItem.humanMessage, index >= mediaStart));
57
+ openIaInput.push(this.mapConectionMessage(chatItem.humanMessage));
61
58
  break;
62
59
  case 'botMessage':
63
60
  openIaInput.push(this.mapBotMessage(chatItem.botMessage));
@@ -69,7 +66,7 @@ let OpenaiChatAdapter = class OpenaiChatAdapter {
69
66
  });
70
67
  return openIaInput;
71
68
  }
72
- mapConectionMessage(item, includeMedia) {
69
+ mapConectionMessage(item) {
73
70
  if (isChatMessageEmpty(item)) {
74
71
  throw new Error('User message content is empty');
75
72
  }
@@ -81,7 +78,7 @@ let OpenaiChatAdapter = class OpenaiChatAdapter {
81
78
  supportedDocumentMimeTypes: OPENAI_SUPPORTED_DOCUMENT_MIME_TYPES,
82
79
  }),
83
80
  });
84
- if (includeMedia && item.images) {
81
+ if (item.images) {
85
82
  for (const image of item.images) {
86
83
  if (!OPENAI_SUPPORTED_IMAGE_MIME_TYPES.includes(image.mimeType))
87
84
  continue;
@@ -92,7 +89,7 @@ let OpenaiChatAdapter = class OpenaiChatAdapter {
92
89
  });
93
90
  }
94
91
  }
95
- if (includeMedia && item.documents) {
92
+ if (item.documents) {
96
93
  for (const doc of item.documents) {
97
94
  if (!OPENAI_SUPPORTED_DOCUMENT_MIME_TYPES.includes(doc.mimeType))
98
95
  continue;
@@ -122,7 +119,7 @@ let OpenaiChatAdapter = class OpenaiChatAdapter {
122
119
  type: 'function_call',
123
120
  call_id: item.id,
124
121
  name: item.name,
125
- arguments: JSON.stringify(item.arguments),
122
+ arguments: item.arguments || '{}',
126
123
  },
127
124
  {
128
125
  type: 'function_call_output',
@@ -7,7 +7,6 @@ import '../../../feature/chat-bot/ChatBot.js';
7
7
  import '../../../feature/chat-bot/ChatOperator.js';
8
8
  import '../../../feature/chat-bot/UnionChatAdapter.js';
9
9
  import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
10
- import '../../../feature/chat-bot/ImageDescriber.js';
11
10
  import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
12
11
  import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
13
12
  import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
@@ -15,7 +14,6 @@ import 'uuid';
15
14
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
16
15
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
17
16
  import '../../../core/error/setupErrorHandlers.js';
18
- import { unconsumedMediaStartIndex } from '../../../feature/chat-bot/unconsumedMediaStartIndex.js';
19
17
  import { OpenRouter } from '@openrouter/sdk';
20
18
 
21
19
  const OPENROUTER_SUPPORTED_IMAGE_MIME_TYPES = [
@@ -68,11 +66,10 @@ let OpenRouterChatAdapter = class OpenRouterChatAdapter {
68
66
  }
69
67
  mapChatItems(chatItems) {
70
68
  const messages = [];
71
- const mediaStart = unconsumedMediaStartIndex(chatItems);
72
- chatItems.forEach((chatItem, index) => {
69
+ chatItems.forEach((chatItem) => {
73
70
  switch (chatItem.type) {
74
71
  case 'humanMessage':
75
- messages.push(this.mapHumanMessage(chatItem.humanMessage, index >= mediaStart));
72
+ messages.push(this.mapHumanMessage(chatItem.humanMessage));
76
73
  break;
77
74
  case 'botMessage':
78
75
  messages.push(this.mapBotMessage(chatItem.botMessage));
@@ -84,34 +81,38 @@ let OpenRouterChatAdapter = class OpenRouterChatAdapter {
84
81
  });
85
82
  return messages;
86
83
  }
87
- mapHumanMessage(item, includeMedia) {
84
+ mapHumanMessage(item) {
88
85
  if (isChatMessageEmpty(item)) {
89
86
  throw new Error('User message content is empty');
90
87
  }
91
88
  const contentParts = [];
92
- contentParts.push(extractChatMessageText(item, {
93
- supportedImageMimeTypes: OPENROUTER_SUPPORTED_IMAGE_MIME_TYPES,
94
- supportedDocumentMimeTypes: OPENROUTER_SUPPORTED_DOCUMENT_MIME_TYPES,
95
- }));
96
- if (includeMedia && item.images) {
89
+ contentParts.push({
90
+ type: 'text',
91
+ text: extractChatMessageText(item, {
92
+ supportedImageMimeTypes: OPENROUTER_SUPPORTED_IMAGE_MIME_TYPES,
93
+ supportedDocumentMimeTypes: OPENROUTER_SUPPORTED_DOCUMENT_MIME_TYPES,
94
+ }),
95
+ });
96
+ if (item.images) {
97
97
  for (const image of item.images) {
98
98
  if (!OPENROUTER_SUPPORTED_IMAGE_MIME_TYPES.includes(image.mimeType))
99
99
  continue;
100
100
  const imageUrl = image.publicUrl ?? image.base64Url;
101
101
  if (imageUrl)
102
- contentParts.push(imageUrl);
102
+ contentParts.push({ type: 'image_url', imageUrl: { url: imageUrl } });
103
103
  }
104
104
  }
105
- if (includeMedia && item.documents) {
105
+ if (item.documents) {
106
106
  for (const doc of item.documents) {
107
107
  if (!OPENROUTER_SUPPORTED_DOCUMENT_MIME_TYPES.includes(doc.mimeType))
108
108
  continue;
109
109
  const docUrl = doc.publicUrl ?? doc.base64Url;
110
- if (docUrl)
111
- contentParts.push(docUrl);
110
+ if (docUrl) {
111
+ contentParts.push({ type: 'file', file: { fileData: docUrl, filename: doc.name } });
112
+ }
112
113
  }
113
114
  }
114
- return { role: 'user', content: contentParts.join('\n') };
115
+ return { role: 'user', content: contentParts };
115
116
  }
116
117
  mapBotMessage(item) {
117
118
  if (!item.text) {
@@ -13,7 +13,6 @@ import '../../../feature/chat-bot/ChatBot.js';
13
13
  import { ChatItem } from '../../../feature/chat-bot/ChatItem.js';
14
14
  import '../../../feature/chat-bot/ChatOperator.js';
15
15
  import '../../../feature/chat-bot/UnionChatAdapter.js';
16
- import '../../../feature/chat-bot/ImageDescriber.js';
17
16
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
18
17
  import 'uuid';
19
18
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
@@ -16,7 +16,6 @@ import '../../../feature/chat-bot/ChatAdapterRegistry.js';
16
16
  import '../../../feature/chat-bot/ChatBot.js';
17
17
  import { ChatOperator } from '../../../feature/chat-bot/ChatOperator.js';
18
18
  import '../../../feature/chat-bot/UnionChatAdapter.js';
19
- import '../../../feature/chat-bot/ImageDescriber.js';
20
19
  import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
21
20
  import 'uuid';
22
21
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
@@ -8,7 +8,6 @@ import '../../../feature/chat-bot/ChatAdapterRegistry.js';
8
8
  import '../../../feature/chat-bot/ChatBot.js';
9
9
  import '../../../feature/chat-bot/ChatOperator.js';
10
10
  import '../../../feature/chat-bot/UnionChatAdapter.js';
11
- import '../../../feature/chat-bot/ImageDescriber.js';
12
11
  import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
13
12
  import 'uuid';
14
13
  import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
@@ -39,7 +39,8 @@ let CmdChannelServer = class CmdChannelServer {
39
39
  this.server = net.createServer((socket) => this.handleConnection(socket));
40
40
  this.server.on('error', (err) => logger.error('cmd channel server error', err));
41
41
  this.server.listen(socketPath, () => {
42
- logger.info(`cmd channel server listening at ${socketPath}`);
42
+ const displayPath = path.relative(process.cwd(), socketPath) || socketPath;
43
+ logger.info(`cmd channel server listening at ${displayPath}`);
43
44
  });
44
45
  process.once('exit', () => this.shutdown());
45
46
  process.once('SIGINT', () => {
@@ -5,8 +5,7 @@ import { MindsetOperator } from '../mindset/MindsetOperator.js';
5
5
  import { ChatAdapter } from './ChatAdapter.js';
6
6
  import { ChatItem } from './ChatItem.js';
7
7
  import { ChatMemory } from './ChatMemory.js';
8
- import { ImageDescriber } from './ImageDescriber.js';
9
- import { pendingMediaStartIndex } from './pendingMediaStartIndex.js';
8
+ import { stripAnsweredMedia } from './stripAnsweredMedia.js';
10
9
  import { Logger } from '../../core/logger/Logger.js';
11
10
 
12
11
  const MAX_CONSECUTIVE_INVALID_ARGS = 2;
@@ -20,16 +19,13 @@ let ChatBot = class ChatBot {
20
19
  memory;
21
20
  adapter;
22
21
  mindset;
23
- imageDescriber;
24
22
  logger = new Logger('wabot:chat-bot');
25
- constructor(memory, adapter, mindset, imageDescriber) {
23
+ constructor(memory, adapter, mindset) {
26
24
  this.memory = memory;
27
25
  this.adapter = adapter;
28
26
  this.mindset = mindset;
29
- this.imageDescriber = imageDescriber;
30
27
  }
31
28
  async sendMessage(message, callback) {
32
- await this.describeImages(message);
33
29
  const newChatItem = new ChatItem({
34
30
  type: 'humanMessage',
35
31
  humanMessage: message,
@@ -37,20 +33,6 @@ let ChatBot = class ChatBot {
37
33
  await this.memory.create(newChatItem);
38
34
  await this.processLoop(callback, 0);
39
35
  }
40
- // Caption incoming images once, before persisting, so later turns can recall
41
- // them from the stored description instead of re-sending the binary. The
42
- // mindset's context/skills/limits focus the description on what matters here.
43
- async describeImages(message) {
44
- if (!message.images?.some((image) => !image.description))
45
- return;
46
- const [models, context, skills, limits] = await Promise.all([
47
- this.mindset.resolveModels('visionLlm'),
48
- this.mindset.context(),
49
- this.mindset.skills(),
50
- this.mindset.limits(),
51
- ]);
52
- await this.imageDescriber.describeMessageImages(message, models, { context, skills, limits });
53
- }
54
36
  async processLoop(callback, invalidArgsCount) {
55
37
  const prevItems = await this.memory.findLastItems(16);
56
38
  if (prevItems.length === 0) {
@@ -64,12 +46,12 @@ let ChatBot = class ChatBot {
64
46
  const tools = this.mindset.tools();
65
47
  const identity = await this.mindset.identity();
66
48
  const prevItemsData = prevItems.map((x) => x.getData());
67
- // Only media from the pending exchange is actually sent to the model; images
68
- // in already-answered messages are not, so they must not force a vision model.
69
- const mediaStart = pendingMediaStartIndex(prevItemsData);
70
- const needsVision = prevItemsData.some((data, index) => index >= mediaStart &&
71
- data.type === 'humanMessage' &&
72
- (data.humanMessage.images?.length ?? 0) > 0);
49
+ // The bot not the provider adapters decides which media reaches the
50
+ // model: keep binaries only for the pending exchange and drop already-
51
+ // answered ones. The leftover media also determines whether this call needs
52
+ // a vision model.
53
+ const sentItems = stripAnsweredMedia(prevItemsData);
54
+ const needsVision = sentItems.some((data) => data.type === 'humanMessage' && (data.humanMessage.images?.length ?? 0) > 0);
73
55
  const kind = needsVision ? 'visionLlm' : 'llm';
74
56
  const candidates = await this.mindset.resolveModels(kind);
75
57
  if (candidates.length === 0) {
@@ -79,7 +61,7 @@ let ChatBot = class ChatBot {
79
61
  models: candidates,
80
62
  systemPrompt,
81
63
  tools,
82
- prevItems: prevItemsData,
64
+ prevItems: sentItems,
83
65
  });
84
66
  for (const newItemData of newItemsData) {
85
67
  if (newItemData.type === 'functionCall') {
@@ -114,8 +96,7 @@ ChatBot = __decorate([
114
96
  injectable(),
115
97
  __metadata("design:paramtypes", [ChatMemory,
116
98
  ChatAdapter,
117
- MindsetOperator,
118
- ImageDescriber])
99
+ MindsetOperator])
119
100
  ], ChatBot);
120
101
 
121
102
  export { ChatBot };
@@ -9,7 +9,6 @@ function extractChatMessageText(message, options = {}) {
9
9
  id: x.id,
10
10
  name: x.name,
11
11
  mimeType: x.mimeType,
12
- description: x.description,
13
12
  notAnalyzed: isAnalyzed(x.mimeType, options.supportedImageMimeTypes) ? undefined : true,
14
13
  })),
15
14
  documents: message.documents?.map((x) => ({
@@ -8,6 +8,14 @@
8
8
  * re-sending the binary would make the model analyze the same files again on
9
9
  * every turn (and re-upload them, wasting tokens).
10
10
  *
11
+ * The boundary is the last bot message, not the last model output, so media
12
+ * stays attached for every call of a tool loop within the pending exchange. The
13
+ * provider APIs are stateless — each call resends the whole history — so the
14
+ * bytes must travel on every call up to the bot's reply for the model to keep
15
+ * seeing the file while it works through tool calls (prompt caching keeps the
16
+ * re-send cheap). Do not stop at function calls, or the model goes blind to the
17
+ * image right when it composes its post-tool answer.
18
+ *
11
19
  * Returns 0 when the bot has not replied yet, so the whole pending exchange
12
20
  * keeps its media.
13
21
  */
@@ -0,0 +1,17 @@
1
+ import { pendingMediaStartIndex } from './pendingMediaStartIndex.js';
2
+
3
+ function stripAnsweredMedia(items) {
4
+ const start = pendingMediaStartIndex(items);
5
+ return items.map((item, index) => {
6
+ if (index >= start || item.type !== 'humanMessage')
7
+ return item;
8
+ if (!item.humanMessage.images && !item.humanMessage.documents)
9
+ return item;
10
+ const humanMessage = { ...item.humanMessage };
11
+ delete humanMessage.images;
12
+ delete humanMessage.documents;
13
+ return { type: 'humanMessage', humanMessage };
14
+ });
15
+ }
16
+
17
+ export { stripAnsweredMedia };
@@ -5,7 +5,6 @@ import '../chat-bot/ChatBot.js';
5
5
  import '../chat-bot/ChatOperator.js';
6
6
  import { ChatRepository } from '../chat-bot/ChatRepository.js';
7
7
  import '../chat-bot/UnionChatAdapter.js';
8
- import '../chat-bot/ImageDescriber.js';
9
8
  import { injectable } from '../../core/injection/index.js';
10
9
  import '../chat-bot/metadata/ChatAdapterMetadataStore.js';
11
10
  import 'uuid';
@@ -8,7 +8,6 @@ import { ChatMemory } from '../chat-bot/ChatMemory.js';
8
8
  import '../chat-bot/ChatOperator.js';
9
9
  import { ChatRepository } from '../chat-bot/ChatRepository.js';
10
10
  import '../chat-bot/UnionChatAdapter.js';
11
- import '../chat-bot/ImageDescriber.js';
12
11
  import '../chat-bot/metadata/ChatAdapterMetadataStore.js';
13
12
  import 'uuid';
14
13
  import { ChatBotMetadataStore } from '../chat-bot/metadata/ChatBotMetadataStore.js';
@@ -7,7 +7,6 @@ import '../chat-bot/ChatBot.js';
7
7
  import '../chat-bot/ChatOperator.js';
8
8
  import { ChatRepository } from '../chat-bot/ChatRepository.js';
9
9
  import '../chat-bot/UnionChatAdapter.js';
10
- import '../chat-bot/ImageDescriber.js';
11
10
  import '../chat-bot/metadata/ChatAdapterMetadataStore.js';
12
11
  import 'uuid';
13
12
  import '../chat-bot/metadata/ChatBotMetadataStore.js';
@@ -953,8 +953,6 @@ interface IChatMessagesPublicFile {
953
953
  base64Url?: undefined;
954
954
  mimeType: string;
955
955
  id: string;
956
- /** Text description of the file content, used to recall it without re-analyzing the binary. */
957
- description?: string;
958
956
  }
959
957
  interface IChatMessagesPrivateFile {
960
958
  name?: string;
@@ -962,8 +960,6 @@ interface IChatMessagesPrivateFile {
962
960
  base64Url: string;
963
961
  mimeType: string;
964
962
  id: string;
965
- /** Text description of the file content, used to recall it without re-analyzing the binary. */
966
- description?: string;
967
963
  }
968
964
  type IChatMessageFile = IChatMessagesPrivateFile | IChatMessagesPublicFile;
969
965
 
@@ -1071,43 +1067,13 @@ interface IChatBot {
1071
1067
  sendMessage(message: IChatMessage, callback: (message: IChatMessage) => Promise<void>): void | Promise<void>;
1072
1068
  }
1073
1069
 
1074
- /** Slice of the mindset used to focus the description on what the assistant cares about. */
1075
- interface IImageDescribeContext {
1076
- context?: string;
1077
- skills?: string;
1078
- limits?: string;
1079
- }
1080
- /**
1081
- * Generates a text description for images so the conversation can recall their
1082
- * content on later turns without re-sending (and re-analyzing) the binary.
1083
- *
1084
- * The description is produced once, on arrival, with a single vision-model call
1085
- * and stored on the image itself; from then on it travels with the message text
1086
- * via {@link extractChatMessageText}.
1087
- */
1088
- declare class ImageDescriber {
1089
- private adapter;
1090
- private logger;
1091
- constructor(adapter: ChatAdapter);
1092
- /**
1093
- * Fills `description` for every image on the message that does not have one
1094
- * yet, mutating the images in place. Safe to call with no images or no models:
1095
- * it simply does nothing. Failures are logged and leave the image undescribed
1096
- * rather than aborting the message.
1097
- */
1098
- describeMessageImages(message: IChatMessage, models: IMindsetModelRef[], describeContext?: IImageDescribeContext): Promise<void>;
1099
- describe(image: IChatMessageImage, models: IMindsetModelRef[], describeContext?: IImageDescribeContext): Promise<string | undefined>;
1100
- }
1101
-
1102
1070
  declare class ChatBot implements IChatBot {
1103
1071
  private memory;
1104
1072
  private adapter;
1105
1073
  private mindset;
1106
- private imageDescriber;
1107
1074
  private logger;
1108
- constructor(memory: ChatMemory, adapter: ChatAdapter, mindset: MindsetOperator, imageDescriber: ImageDescriber);
1075
+ constructor(memory: ChatMemory, adapter: ChatAdapter, mindset: MindsetOperator);
1109
1076
  sendMessage(message: IChatMessage, callback: (message: IChatMessage) => Promise<void>): Promise<void>;
1110
- private describeImages;
1111
1077
  protected processLoop(callback: (message: IChatMessage) => Promise<void>, invalidArgsCount: number): Promise<void>;
1112
1078
  }
1113
1079
 
@@ -1207,6 +1173,14 @@ declare class ChatBotMetadataStore {
1207
1173
  * re-sending the binary would make the model analyze the same files again on
1208
1174
  * every turn (and re-upload them, wasting tokens).
1209
1175
  *
1176
+ * The boundary is the last bot message, not the last model output, so media
1177
+ * stays attached for every call of a tool loop within the pending exchange. The
1178
+ * provider APIs are stateless — each call resends the whole history — so the
1179
+ * bytes must travel on every call up to the bot's reply for the model to keep
1180
+ * seeing the file while it works through tool calls (prompt caching keeps the
1181
+ * re-send cheap). Do not stop at function calls, or the model goes blind to the
1182
+ * image right when it composes its post-tool answer.
1183
+ *
1210
1184
  * Returns 0 when the bot has not replied yet, so the whole pending exchange
1211
1185
  * keeps its media.
1212
1186
  */
@@ -1216,24 +1190,7 @@ declare function runChatAdapters(adapters: IConstructor<IChatAdapter>[]): void;
1216
1190
 
1217
1191
  declare function safeJsonParse<T = unknown>(json: string | undefined | null, context?: string): T;
1218
1192
 
1219
- /**
1220
- * Index of the first item whose media (images/documents) the model has not been
1221
- * shown yet — i.e. right after the last model output, be it a bot message or a
1222
- * function call.
1223
- *
1224
- * A file's binary should only be sent on the model's first exposure to it. Once
1225
- * the model has produced anything in response to a human message (a function
1226
- * call during a tool loop, or a reply), it has already analyzed that message's
1227
- * files; re-sending the bytes on the next call would analyze the same file
1228
- * again. This is the within-turn counterpart of {@link pendingMediaStartIndex}:
1229
- * that one keeps a turn's media "active" for vision-model selection across the
1230
- * whole tool loop, while this one stops re-uploading the bytes after the first
1231
- * call.
1232
- *
1233
- * Returns 0 when the model has not produced any output yet, so a brand-new
1234
- * message keeps its media.
1235
- */
1236
- declare function unconsumedMediaStartIndex(items: IChatItem[]): number;
1193
+ declare function stripAnsweredMedia(items: IChatItem[]): IChatItem[];
1237
1194
 
1238
1195
  interface IProjectRunnerConfig {
1239
1196
  directories?: string[];
@@ -2765,4 +2722,4 @@ declare function HtmlModule(options: IHtmlModuleOptions): {
2765
2722
  new (): {};
2766
2723
  };
2767
2724
 
2768
- export { AnthropicChatAdapter, ApiKey, ApiKeyGuardMiddleware, ApiKeyHandshakeGuardMiddleware, ApiKeyRepository, Async, AsyncMetadataStore, Auth, Chat, ChatAdapter, ChatAdapterMetadataStore, ChatAdapterRegistry, ChatBot, ChatBotMetadataStore, ChatItem, ChatMemory, ChatOperator, ChatRepository, ChatResolver, type ClientMap, CmdChannel, CmdChannelConfig, CmdChannelServer, type CmdClientMessage, type CmdServerMessage, type ConfigReference, type ConfigReferenceType, ConfigResolver, Container, ControllerMetadataStore, CronJob, CronJobRepository, CrudRepository, CustomError, DeepSeekChatAdapter, DescriptionMetadataStore, EXPRESS_REQ, EXPRESS_RES, Entity, Env, type ErrorSeverity, ExpressProvider, GoogleChatAdapter, type GoogleChatAdapterV2Options, HtmlModule, HttpServerProvider, type IApiKeyData, type IApiKeyRepository, type IArrayValidationError, type IArrayValidationResult, type IBotMessageItem, type IBuiltQuery, type IChannelMessage, type IChannelMetadata, type IChatAdapter, type IChatAdapterDecoratorConfig, type IChatAdapterMetadata, type IChatAdapterNextItemsReq, type IChatAdapterNextItemsRes, type IChatAssociation, type IChatBot, type IChatBotMetadata, type IChatChannel, type IChatConnection, type IChatControllerMetadata, type IChatData, type IChatItem, type IChatItemData, type IChatItemType, type IChatMemory, type IChatMessage, type IChatMessageDocument, type IChatMessageFile, type IChatMessageImage, type IChatMessagesPrivateFile, type IChatMessagesPublicFile, type IChatRepository, type IChatType, type ICmdChannelEntry, type ICmdChannelHandlers, type ICmdChannelMessage, type ICmdImage, type ICmdIncomingMessage, type ICmdReceivedMessage, type ICommandConfig, type ICommandHandler, type ICommandHandlerConfig, type IConstructor, type ICronConfig, type ICronHandler, type ICronJobData, type ICronJobRepository, type ICrudRepository, type ICustomErrorData, type IDedupConfig, type IDescriptionMetadata, type IEndPointConfig, type IEndPointMetadata, type IEntityData, type IEnvType, type IErrorHandlersConfig, type IErrorMonitor, type IErrorMonitorContext, type IExtractChatMessageTextOptions, type IFunctionCall, type IFunctionCallItem, type IGenerateApiKeyReq, type IGenerateApiKeyRes, type IHandshakeMiddleware, type IHandshakeMiddlewareMetadata, type IHtmlModuleOptions, type IHumanMessageItem, type IImageDescribeContext, type IJobData, type IJobOptions, type IJobRepository, type IJwtRefreshTokenData, type IJwtRefreshTokenRepository, type IKapsoChannelConfig, type IKapsoChannelMessage, type IKapsoChannelMessageListener, type IKapsoChatMessage, type IKapsoConversation, type IKapsoEvent, type IKapsoIncomingMessage, type IKapsoMessageReceivedEvent, type IKapsoReceivedMessage, type IKapsoUnknownEvent, type ILanguageModelUsage, type ILockKey, type ILocker, type ILockerKey, type IMemoryRepositoryAdapterOptions, type IMessageContext, type IMiddleware, type IMiddlewareMetadata, type IMindset, type IMindsetConfig, type IMindsetIdentity, type IMindsetLlm, type IMindsetMetadata, type IMindsetModelKind, type IMindsetModelRef, type IMindsetModels, type IMindsetModuleConfig, type IMindsetModuleMetadata, type IMindsetParameterSchema, type IMindsetTool, type IMindsetToolParameter, type IModelValidationError, type IModelValidationResult, type IModelValidatorsInfo, type IMoneyData, type IPersistentData, type IPgRepositoryConfig, type IProjectRunnerConfig, type IPropertyValidatorInfo, type IQueryAst, type IQueryCondition, type IQueryMethodMetadata, type IQueryOrderBy, type IReceivedMessage, type IRemoteApiKeyFetcher, type IRepositoryAdapter, type IRepositoryConfig, type IRepositoryRuntime, type IRestControllerConfig, type IRestControllerMetadata, type IScanProjectFilesOptions, type IScheduleAt, type IScheduleDelay, type ISendWhatsAppMessageReq, type ISendWhatsAppTemplateReq, type ISocketChannelConfig, type ISocketChannelMessage, type ISocketChannelReceivedMessage, type ISocketControllerConfig, type ISocketControllerMetadata, type ISocketEventConfig, type ISocketEventMetadata, type ISocketReceivedMessage, type IStorableData, type ITelegramChannelConfig, type ITelegramChannelMessage, type ITelegramReceivedMessage, type ITransactionAdapter, type IValidateArrayOptions, type IValidateArrayOptionsWithItemsValidators, type IValidateInputShape, type IValidateIsInOptions, type IValidateIsRecordOptions, type IValidateMaxOptions, type IValidateMinOptions, type IValidationError, type IValidationResult, type IValidator, type IValidatorMetadata, type IWasenderChannelConfig, type IWasenderChannelMessageListener, type IWasenderDeviceListMetadata, type IWasenderEvent, type IWasenderMessageContent, type IWasenderMessageContextInfo, type IWasenderMessageKey, type IWasenderMessageReceivedData, type IWasenderMessageReceivedEvent, type IWasenderQrUpdatedEvent, type IWasenderReceivedMessage, type IWhatsAppCloudContact, type IWhatsAppCloudMessage, type IWhatsAppCloudMessageMetadata, type IWhatsAppCloudTemplate, type IWhatsAppCloudTemplateComponent, type IWhatsAppCloudTemplateResponse, type IWhatsAppCloudWebhookPayload, type IWhatsAppSender, type IWhatsAppTemplateData, type IWhatsAppTemplateParameter, type IchatControllerConfig, ImageDescriber, InMemoryChatMemory, InMemoryChatRepository, InMemoryCronJobRepository, InMemoryJobRepository, InMemoryLockKey, InMemoryLocker, Job, JobRepository, JobRunner, Jwt, JwtAccessAndRefreshTokenDto, JwtConfig, JwtGuardMiddleware, JwtHandshakeGuardMiddleware, JwtRefreshToken, JwtRefreshTokenRepository, JwtSigner, JwtTokenDto, KapsoChannel, KapsoChannelConfig, KapsoReceiver, KapsoSender, KapsoWebhookController, Lifecycle, Locker, Logger, MEMORY_ADAPTER_ID, Mapper, MemoryRepositoryAdapter, MemoryRepositoryExtension, Mindset, MindsetMetadataStore, MindsetOperator, Money, MoneyDto, OpenRouterChatAdapter, OpenaiChatAdapter, PG_ADAPTER_ID, Password, type PasswordHashOptions, Persistent, PgApiKeyRepository, PgChatMemory, PgChatRepository, PgCronJobRepository, PgCrudRepository, PgJobRepository, PgJsonRepositoryAdapter, PgJwtRefreshTokenRepository, PgLockKey, PgLocker, PgRepositoryBase, PgRepositoryBase as PgRepositoryExtension, PgTransactionAdapter, ProjectRunner, type QueryConnector, type QueryOperator, type QueryPrefix, Random, RemoteApiKeyRepository, RepositoryAdapterRegistry, RepositoryMetadataStore, type ResolvedConfig, RestControllerMetadataStore, RestRequest, SocketChannel, SocketChannelConfig, SocketChannelMessageFile, SocketChannelReceivedMessage, SocketControllerMetadataStore, SocketServerConfig, SocketServerProvider, Storable, TelegramChannel, TelegramChannelConfig, TransactionMetadataStore, UnionChatAdapter, ValidationMetadataStore, WabotChatAdapter, WasenderChannel, WasenderChannelConfig, WasenderReceiver, WasenderSender, WasenderWebhookController, WhatsAppApiSender, WhatsAppReceiverByCloudApi, WhatsAppSender, apiKeyGuard, apiKeyHandshakeGuard, bool, boolArr, buildQuerySql, chatAdapter, chatBot, chatController, chatItemTypeOptions, cmd, cmdChannelName, cmdChannelSocketPath, command, commandHandler, computeDedupKey, container, cronHandler, description, errorToPlainObject, evaluateQueryAst, extractChatMessageText, extractNumberFromWasenderMessageKey, getClientMap, getPgClient, handshakeMiddlewares, inject, injectable, isArray, isBoolean, isChatMessageEmpty, isDate, isIn, isModel, isNotEmpty, isNumber, isOptional, isPresent, isRecord, isRetryableError, isString, jwtGuard, jwtHandshakeGuard, kapso, kapsoChannelName, markdownToTelegramHtml, max, memExtension, middleware, min, mindset, mindsetModule, modelInfo, num, numArr, obj, onDelete, onGet, onPost, onPut, onSocketEvent, parseQueryMethodName, pendingMediaStartIndex, pgExtension, pgStorage, query, queryExtension, readJsonFromFile, repository, resolveConfigReferences, restController, run, runChatAdapters, runChatControllers, runCmdClient, runCommandHandlers, runCronHandlers, runRestControllers, runSocketControllers, safeJsonParse, scanProjectFiles, scoped, setupErrorHandlers, singleton, socket, socketChannelName, socketController, stopCommandHandlers, stopCronHandlers, str, strArr, telegram, telegramChannelName, transaction, unconsumedMediaStartIndex, validateAndTransform, validateArray, validateIsBoolean, validateIsDate, validateIsIn, validateIsNotEmpty, validateIsNumber, validateIsPresent, validateIsRecord, validateIsString, validateMax, validateMin, validateModel, wasender, wasenderChannelName, withPgClient, withPgTransaction, writeJsonToFile };
2725
+ export { AnthropicChatAdapter, ApiKey, ApiKeyGuardMiddleware, ApiKeyHandshakeGuardMiddleware, ApiKeyRepository, Async, AsyncMetadataStore, Auth, Chat, ChatAdapter, ChatAdapterMetadataStore, ChatAdapterRegistry, ChatBot, ChatBotMetadataStore, ChatItem, ChatMemory, ChatOperator, ChatRepository, ChatResolver, type ClientMap, CmdChannel, CmdChannelConfig, CmdChannelServer, type CmdClientMessage, type CmdServerMessage, type ConfigReference, type ConfigReferenceType, ConfigResolver, Container, ControllerMetadataStore, CronJob, CronJobRepository, CrudRepository, CustomError, DeepSeekChatAdapter, DescriptionMetadataStore, EXPRESS_REQ, EXPRESS_RES, Entity, Env, type ErrorSeverity, ExpressProvider, GoogleChatAdapter, type GoogleChatAdapterV2Options, HtmlModule, HttpServerProvider, type IApiKeyData, type IApiKeyRepository, type IArrayValidationError, type IArrayValidationResult, type IBotMessageItem, type IBuiltQuery, type IChannelMessage, type IChannelMetadata, type IChatAdapter, type IChatAdapterDecoratorConfig, type IChatAdapterMetadata, type IChatAdapterNextItemsReq, type IChatAdapterNextItemsRes, type IChatAssociation, type IChatBot, type IChatBotMetadata, type IChatChannel, type IChatConnection, type IChatControllerMetadata, type IChatData, type IChatItem, type IChatItemData, type IChatItemType, type IChatMemory, type IChatMessage, type IChatMessageDocument, type IChatMessageFile, type IChatMessageImage, type IChatMessagesPrivateFile, type IChatMessagesPublicFile, type IChatRepository, type IChatType, type ICmdChannelEntry, type ICmdChannelHandlers, type ICmdChannelMessage, type ICmdImage, type ICmdIncomingMessage, type ICmdReceivedMessage, type ICommandConfig, type ICommandHandler, type ICommandHandlerConfig, type IConstructor, type ICronConfig, type ICronHandler, type ICronJobData, type ICronJobRepository, type ICrudRepository, type ICustomErrorData, type IDedupConfig, type IDescriptionMetadata, type IEndPointConfig, type IEndPointMetadata, type IEntityData, type IEnvType, type IErrorHandlersConfig, type IErrorMonitor, type IErrorMonitorContext, type IExtractChatMessageTextOptions, type IFunctionCall, type IFunctionCallItem, type IGenerateApiKeyReq, type IGenerateApiKeyRes, type IHandshakeMiddleware, type IHandshakeMiddlewareMetadata, type IHtmlModuleOptions, type IHumanMessageItem, type IJobData, type IJobOptions, type IJobRepository, type IJwtRefreshTokenData, type IJwtRefreshTokenRepository, type IKapsoChannelConfig, type IKapsoChannelMessage, type IKapsoChannelMessageListener, type IKapsoChatMessage, type IKapsoConversation, type IKapsoEvent, type IKapsoIncomingMessage, type IKapsoMessageReceivedEvent, type IKapsoReceivedMessage, type IKapsoUnknownEvent, type ILanguageModelUsage, type ILockKey, type ILocker, type ILockerKey, type IMemoryRepositoryAdapterOptions, type IMessageContext, type IMiddleware, type IMiddlewareMetadata, type IMindset, type IMindsetConfig, type IMindsetIdentity, type IMindsetLlm, type IMindsetMetadata, type IMindsetModelKind, type IMindsetModelRef, type IMindsetModels, type IMindsetModuleConfig, type IMindsetModuleMetadata, type IMindsetParameterSchema, type IMindsetTool, type IMindsetToolParameter, type IModelValidationError, type IModelValidationResult, type IModelValidatorsInfo, type IMoneyData, type IPersistentData, type IPgRepositoryConfig, type IProjectRunnerConfig, type IPropertyValidatorInfo, type IQueryAst, type IQueryCondition, type IQueryMethodMetadata, type IQueryOrderBy, type IReceivedMessage, type IRemoteApiKeyFetcher, type IRepositoryAdapter, type IRepositoryConfig, type IRepositoryRuntime, type IRestControllerConfig, type IRestControllerMetadata, type IScanProjectFilesOptions, type IScheduleAt, type IScheduleDelay, type ISendWhatsAppMessageReq, type ISendWhatsAppTemplateReq, type ISocketChannelConfig, type ISocketChannelMessage, type ISocketChannelReceivedMessage, type ISocketControllerConfig, type ISocketControllerMetadata, type ISocketEventConfig, type ISocketEventMetadata, type ISocketReceivedMessage, type IStorableData, type ITelegramChannelConfig, type ITelegramChannelMessage, type ITelegramReceivedMessage, type ITransactionAdapter, type IValidateArrayOptions, type IValidateArrayOptionsWithItemsValidators, type IValidateInputShape, type IValidateIsInOptions, type IValidateIsRecordOptions, type IValidateMaxOptions, type IValidateMinOptions, type IValidationError, type IValidationResult, type IValidator, type IValidatorMetadata, type IWasenderChannelConfig, type IWasenderChannelMessageListener, type IWasenderDeviceListMetadata, type IWasenderEvent, type IWasenderMessageContent, type IWasenderMessageContextInfo, type IWasenderMessageKey, type IWasenderMessageReceivedData, type IWasenderMessageReceivedEvent, type IWasenderQrUpdatedEvent, type IWasenderReceivedMessage, type IWhatsAppCloudContact, type IWhatsAppCloudMessage, type IWhatsAppCloudMessageMetadata, type IWhatsAppCloudTemplate, type IWhatsAppCloudTemplateComponent, type IWhatsAppCloudTemplateResponse, type IWhatsAppCloudWebhookPayload, type IWhatsAppSender, type IWhatsAppTemplateData, type IWhatsAppTemplateParameter, type IchatControllerConfig, InMemoryChatMemory, InMemoryChatRepository, InMemoryCronJobRepository, InMemoryJobRepository, InMemoryLockKey, InMemoryLocker, Job, JobRepository, JobRunner, Jwt, JwtAccessAndRefreshTokenDto, JwtConfig, JwtGuardMiddleware, JwtHandshakeGuardMiddleware, JwtRefreshToken, JwtRefreshTokenRepository, JwtSigner, JwtTokenDto, KapsoChannel, KapsoChannelConfig, KapsoReceiver, KapsoSender, KapsoWebhookController, Lifecycle, Locker, Logger, MEMORY_ADAPTER_ID, Mapper, MemoryRepositoryAdapter, MemoryRepositoryExtension, Mindset, MindsetMetadataStore, MindsetOperator, Money, MoneyDto, OpenRouterChatAdapter, OpenaiChatAdapter, PG_ADAPTER_ID, Password, type PasswordHashOptions, Persistent, PgApiKeyRepository, PgChatMemory, PgChatRepository, PgCronJobRepository, PgCrudRepository, PgJobRepository, PgJsonRepositoryAdapter, PgJwtRefreshTokenRepository, PgLockKey, PgLocker, PgRepositoryBase, PgRepositoryBase as PgRepositoryExtension, PgTransactionAdapter, ProjectRunner, type QueryConnector, type QueryOperator, type QueryPrefix, Random, RemoteApiKeyRepository, RepositoryAdapterRegistry, RepositoryMetadataStore, type ResolvedConfig, RestControllerMetadataStore, RestRequest, SocketChannel, SocketChannelConfig, SocketChannelMessageFile, SocketChannelReceivedMessage, SocketControllerMetadataStore, SocketServerConfig, SocketServerProvider, Storable, TelegramChannel, TelegramChannelConfig, TransactionMetadataStore, UnionChatAdapter, ValidationMetadataStore, WabotChatAdapter, WasenderChannel, WasenderChannelConfig, WasenderReceiver, WasenderSender, WasenderWebhookController, WhatsAppApiSender, WhatsAppReceiverByCloudApi, WhatsAppSender, apiKeyGuard, apiKeyHandshakeGuard, bool, boolArr, buildQuerySql, chatAdapter, chatBot, chatController, chatItemTypeOptions, cmd, cmdChannelName, cmdChannelSocketPath, command, commandHandler, computeDedupKey, container, cronHandler, description, errorToPlainObject, evaluateQueryAst, extractChatMessageText, extractNumberFromWasenderMessageKey, getClientMap, getPgClient, handshakeMiddlewares, inject, injectable, isArray, isBoolean, isChatMessageEmpty, isDate, isIn, isModel, isNotEmpty, isNumber, isOptional, isPresent, isRecord, isRetryableError, isString, jwtGuard, jwtHandshakeGuard, kapso, kapsoChannelName, markdownToTelegramHtml, max, memExtension, middleware, min, mindset, mindsetModule, modelInfo, num, numArr, obj, onDelete, onGet, onPost, onPut, onSocketEvent, parseQueryMethodName, pendingMediaStartIndex, pgExtension, pgStorage, query, queryExtension, readJsonFromFile, repository, resolveConfigReferences, restController, run, runChatAdapters, runChatControllers, runCmdClient, runCommandHandlers, runCronHandlers, runRestControllers, runSocketControllers, safeJsonParse, scanProjectFiles, scoped, setupErrorHandlers, singleton, socket, socketChannelName, socketController, stopCommandHandlers, stopCronHandlers, str, strArr, stripAnsweredMedia, telegram, telegramChannelName, transaction, validateAndTransform, validateArray, validateIsBoolean, validateIsDate, validateIsIn, validateIsNotEmpty, validateIsNumber, validateIsPresent, validateIsRecord, validateIsString, validateMax, validateMin, validateModel, wasender, wasenderChannelName, withPgClient, withPgTransaction, writeJsonToFile };
package/dist/src/index.js CHANGED
@@ -71,7 +71,6 @@ export { ChatRepository } from './feature/chat-bot/ChatRepository.js';
71
71
  export { chatItemTypeOptions } from './feature/chat-bot/IChatItem.js';
72
72
  export { UnionChatAdapter } from './feature/chat-bot/UnionChatAdapter.js';
73
73
  export { extractChatMessageText } from './feature/chat-bot/extractChatMessageText.js';
74
- export { ImageDescriber } from './feature/chat-bot/ImageDescriber.js';
75
74
  export { isChatMessageEmpty } from './feature/chat-bot/isChatMessageEmpty.js';
76
75
  export { isRetryableError } from './feature/chat-bot/isRetryableError.js';
77
76
  export { chatAdapter } from './feature/chat-bot/metadata/@chatAdapter.js';
@@ -81,7 +80,7 @@ export { ChatBotMetadataStore } from './feature/chat-bot/metadata/ChatBotMetadat
81
80
  export { pendingMediaStartIndex } from './feature/chat-bot/pendingMediaStartIndex.js';
82
81
  export { runChatAdapters } from './feature/chat-bot/runChatAdapters.js';
83
82
  export { safeJsonParse } from './feature/chat-bot/safeJsonParse.js';
84
- export { unconsumedMediaStartIndex } from './feature/chat-bot/unconsumedMediaStartIndex.js';
83
+ export { stripAnsweredMedia } from './feature/chat-bot/stripAnsweredMedia.js';
85
84
  export { chatController } from './feature/chat-controller/metadata/controller/@chatController.js';
86
85
  export { ControllerMetadataStore } from './feature/chat-controller/metadata/ControllerMetadataStore.js';
87
86
  export { ChatResolver } from './feature/chat-controller/ChatResolver.js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wabot-dev/framework",
3
- "version": "0.9.24",
3
+ "version": "0.9.26",
4
4
  "description": "Framework for IA Chat Bots",
5
5
  "type": "module",
6
6
  "main": "dist/src/index.js",
@@ -1,95 +0,0 @@
1
- import { __decorate, __metadata } from 'tslib';
2
- import { injectable } from '../../core/injection/index.js';
3
- import { Logger } from '../../core/logger/Logger.js';
4
- import { ChatAdapter } from './ChatAdapter.js';
5
-
6
- const DESCRIBE_INSTRUCTIONS = `
7
- You describe images so an assistant can reference them later in a conversation
8
- without seeing the image again.
9
-
10
- Write a single, factual, self-contained paragraph covering the visible content:
11
- objects, people, scene, layout, colors, and any readable text. Be thorough but
12
- concise. Do not add opinions, greetings, or markdown — output only the
13
- description.
14
- `.trim();
15
- function buildSystemPrompt(describeContext) {
16
- const sections = [DESCRIBE_INSTRUCTIONS];
17
- const mindsetSections = [
18
- ['Assistant context', describeContext.context],
19
- ['Assistant skills', describeContext.skills],
20
- ['Assistant limits', describeContext.limits],
21
- ].filter(([, value]) => value && value.trim());
22
- if (mindsetSections.length > 0) {
23
- sections.push('The assistant you describe images for is outlined below. Use it only to ' +
24
- 'decide which details deserve emphasis; still describe anything else that ' +
25
- 'is clearly visible.');
26
- for (const [title, value] of mindsetSections) {
27
- sections.push(`## ${title}\n${value.trim()}`);
28
- }
29
- }
30
- return sections.join('\n\n');
31
- }
32
- /**
33
- * Generates a text description for images so the conversation can recall their
34
- * content on later turns without re-sending (and re-analyzing) the binary.
35
- *
36
- * The description is produced once, on arrival, with a single vision-model call
37
- * and stored on the image itself; from then on it travels with the message text
38
- * via {@link extractChatMessageText}.
39
- */
40
- let ImageDescriber = class ImageDescriber {
41
- adapter;
42
- logger = new Logger('wabot:image-describer');
43
- constructor(adapter) {
44
- this.adapter = adapter;
45
- }
46
- /**
47
- * Fills `description` for every image on the message that does not have one
48
- * yet, mutating the images in place. Safe to call with no images or no models:
49
- * it simply does nothing. Failures are logged and leave the image undescribed
50
- * rather than aborting the message.
51
- */
52
- async describeMessageImages(message, models, describeContext = {}) {
53
- const pending = message.images?.filter((image) => !image.description);
54
- if (!pending || pending.length === 0 || models.length === 0)
55
- return;
56
- await Promise.all(pending.map(async (image) => {
57
- const description = await this.describe(image, models, describeContext);
58
- if (description)
59
- image.description = description;
60
- }));
61
- }
62
- async describe(image, models, describeContext = {}) {
63
- if (models.length === 0)
64
- return undefined;
65
- try {
66
- const { nextItems } = await this.adapter.nextItems({
67
- models,
68
- systemPrompt: buildSystemPrompt(describeContext),
69
- tools: [],
70
- prevItems: [
71
- {
72
- type: 'humanMessage',
73
- humanMessage: { text: 'Describe this image.', images: [image] },
74
- },
75
- ],
76
- });
77
- const parts = [];
78
- for (const item of nextItems) {
79
- if (item.type === 'botMessage' && item.botMessage.text)
80
- parts.push(item.botMessage.text);
81
- }
82
- return parts.join('\n').trim() || undefined;
83
- }
84
- catch (err) {
85
- this.logger.warn(`failed to describe image '${image.id}'`, err instanceof Error ? { message: err.message } : { err });
86
- return undefined;
87
- }
88
- }
89
- };
90
- ImageDescriber = __decorate([
91
- injectable(),
92
- __metadata("design:paramtypes", [ChatAdapter])
93
- ], ImageDescriber);
94
-
95
- export { ImageDescriber };
@@ -1,27 +0,0 @@
1
- /**
2
- * Index of the first item whose media (images/documents) the model has not been
3
- * shown yet — i.e. right after the last model output, be it a bot message or a
4
- * function call.
5
- *
6
- * A file's binary should only be sent on the model's first exposure to it. Once
7
- * the model has produced anything in response to a human message (a function
8
- * call during a tool loop, or a reply), it has already analyzed that message's
9
- * files; re-sending the bytes on the next call would analyze the same file
10
- * again. This is the within-turn counterpart of {@link pendingMediaStartIndex}:
11
- * that one keeps a turn's media "active" for vision-model selection across the
12
- * whole tool loop, while this one stops re-uploading the bytes after the first
13
- * call.
14
- *
15
- * Returns 0 when the model has not produced any output yet, so a brand-new
16
- * message keeps its media.
17
- */
18
- function unconsumedMediaStartIndex(items) {
19
- for (let i = items.length - 1; i >= 0; i--) {
20
- const type = items[i].type;
21
- if (type === 'botMessage' || type === 'functionCall')
22
- return i + 1;
23
- }
24
- return 0;
25
- }
26
-
27
- export { unconsumedMediaStartIndex };