@wabot-dev/framework 0.9.24 → 0.9.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/addon/chat-bot/anthropic/AnthropicChatAdapter.js +0 -1
- package/dist/src/addon/chat-bot/deepseek/DeepSeekChatAdapter.js +0 -1
- package/dist/src/addon/chat-bot/google/GoogleChatAdapter.js +0 -1
- package/dist/src/addon/chat-bot/in-memory/InMemoryChatMemory.js +0 -1
- package/dist/src/addon/chat-bot/in-memory/InMemoryChatRepository.js +0 -1
- package/dist/src/addon/chat-bot/openia/OpenaiChatAdapter.js +0 -1
- package/dist/src/addon/chat-bot/openrouter/OpenRouterChatAdapter.js +0 -1
- package/dist/src/addon/chat-bot/pg/PgChatMemory.js +0 -1
- package/dist/src/addon/chat-bot/pg/PgChatRepository.js +0 -1
- package/dist/src/addon/chat-bot/wabot/WabotChatAdapter.js +0 -1
- package/dist/src/feature/chat-bot/ChatBot.js +2 -21
- package/dist/src/feature/chat-bot/extractChatMessageText.js +0 -1
- package/dist/src/feature/chat-controller/ChatResolver.js +0 -1
- package/dist/src/feature/chat-controller/runChatControllers.js +0 -1
- package/dist/src/feature/project-runner/ProjectRunner.js +0 -1
- package/dist/src/index.d.ts +2 -36
- package/dist/src/index.js +0 -1
- package/package.json +1 -1
- package/dist/src/feature/chat-bot/ImageDescriber.js +0 -95
|
@@ -7,7 +7,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
7
7
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
8
8
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
9
9
|
import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
|
|
10
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
11
10
|
import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
|
|
12
11
|
import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
|
|
13
12
|
import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
|
|
@@ -6,7 +6,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
6
6
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
7
7
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
8
8
|
import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
|
|
9
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
10
9
|
import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
|
|
11
10
|
import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
|
|
12
11
|
import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
|
|
@@ -8,7 +8,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
8
8
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
9
9
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
10
10
|
import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
|
|
11
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
12
11
|
import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
|
|
13
12
|
import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
|
|
14
13
|
import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
|
|
@@ -6,7 +6,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
6
6
|
import { ChatItem } from '../../../feature/chat-bot/ChatItem.js';
|
|
7
7
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
8
8
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
9
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
10
9
|
import '../../../core/injection/index.js';
|
|
11
10
|
import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
12
11
|
import 'uuid';
|
|
@@ -10,7 +10,6 @@ import '../../../feature/chat-bot/ChatAdapterRegistry.js';
|
|
|
10
10
|
import '../../../feature/chat-bot/ChatBot.js';
|
|
11
11
|
import { ChatOperator } from '../../../feature/chat-bot/ChatOperator.js';
|
|
12
12
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
13
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
14
13
|
import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
15
14
|
import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
|
|
16
15
|
import '../../../core/error/setupErrorHandlers.js';
|
|
@@ -4,7 +4,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
4
4
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
5
5
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
6
6
|
import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
|
|
7
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
8
7
|
import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
|
|
9
8
|
import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
|
|
10
9
|
import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
|
|
@@ -7,7 +7,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
7
7
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
8
8
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
9
9
|
import { extractChatMessageText } from '../../../feature/chat-bot/extractChatMessageText.js';
|
|
10
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
11
10
|
import { isChatMessageEmpty } from '../../../feature/chat-bot/isChatMessageEmpty.js';
|
|
12
11
|
import { isRetryableError } from '../../../feature/chat-bot/isRetryableError.js';
|
|
13
12
|
import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
|
|
@@ -13,7 +13,6 @@ import '../../../feature/chat-bot/ChatBot.js';
|
|
|
13
13
|
import { ChatItem } from '../../../feature/chat-bot/ChatItem.js';
|
|
14
14
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
15
15
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
16
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
17
16
|
import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
18
17
|
import 'uuid';
|
|
19
18
|
import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
|
|
@@ -16,7 +16,6 @@ import '../../../feature/chat-bot/ChatAdapterRegistry.js';
|
|
|
16
16
|
import '../../../feature/chat-bot/ChatBot.js';
|
|
17
17
|
import { ChatOperator } from '../../../feature/chat-bot/ChatOperator.js';
|
|
18
18
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
19
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
20
19
|
import '../../../feature/chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
21
20
|
import 'uuid';
|
|
22
21
|
import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
|
|
@@ -8,7 +8,6 @@ import '../../../feature/chat-bot/ChatAdapterRegistry.js';
|
|
|
8
8
|
import '../../../feature/chat-bot/ChatBot.js';
|
|
9
9
|
import '../../../feature/chat-bot/ChatOperator.js';
|
|
10
10
|
import '../../../feature/chat-bot/UnionChatAdapter.js';
|
|
11
|
-
import '../../../feature/chat-bot/ImageDescriber.js';
|
|
12
11
|
import { chatAdapter } from '../../../feature/chat-bot/metadata/@chatAdapter.js';
|
|
13
12
|
import 'uuid';
|
|
14
13
|
import '../../../feature/chat-bot/metadata/ChatBotMetadataStore.js';
|
|
@@ -5,7 +5,6 @@ import { MindsetOperator } from '../mindset/MindsetOperator.js';
|
|
|
5
5
|
import { ChatAdapter } from './ChatAdapter.js';
|
|
6
6
|
import { ChatItem } from './ChatItem.js';
|
|
7
7
|
import { ChatMemory } from './ChatMemory.js';
|
|
8
|
-
import { ImageDescriber } from './ImageDescriber.js';
|
|
9
8
|
import { pendingMediaStartIndex } from './pendingMediaStartIndex.js';
|
|
10
9
|
import { Logger } from '../../core/logger/Logger.js';
|
|
11
10
|
|
|
@@ -20,16 +19,13 @@ let ChatBot = class ChatBot {
|
|
|
20
19
|
memory;
|
|
21
20
|
adapter;
|
|
22
21
|
mindset;
|
|
23
|
-
imageDescriber;
|
|
24
22
|
logger = new Logger('wabot:chat-bot');
|
|
25
|
-
constructor(memory, adapter, mindset
|
|
23
|
+
constructor(memory, adapter, mindset) {
|
|
26
24
|
this.memory = memory;
|
|
27
25
|
this.adapter = adapter;
|
|
28
26
|
this.mindset = mindset;
|
|
29
|
-
this.imageDescriber = imageDescriber;
|
|
30
27
|
}
|
|
31
28
|
async sendMessage(message, callback) {
|
|
32
|
-
await this.describeImages(message);
|
|
33
29
|
const newChatItem = new ChatItem({
|
|
34
30
|
type: 'humanMessage',
|
|
35
31
|
humanMessage: message,
|
|
@@ -37,20 +33,6 @@ let ChatBot = class ChatBot {
|
|
|
37
33
|
await this.memory.create(newChatItem);
|
|
38
34
|
await this.processLoop(callback, 0);
|
|
39
35
|
}
|
|
40
|
-
// Caption incoming images once, before persisting, so later turns can recall
|
|
41
|
-
// them from the stored description instead of re-sending the binary. The
|
|
42
|
-
// mindset's context/skills/limits focus the description on what matters here.
|
|
43
|
-
async describeImages(message) {
|
|
44
|
-
if (!message.images?.some((image) => !image.description))
|
|
45
|
-
return;
|
|
46
|
-
const [models, context, skills, limits] = await Promise.all([
|
|
47
|
-
this.mindset.resolveModels('visionLlm'),
|
|
48
|
-
this.mindset.context(),
|
|
49
|
-
this.mindset.skills(),
|
|
50
|
-
this.mindset.limits(),
|
|
51
|
-
]);
|
|
52
|
-
await this.imageDescriber.describeMessageImages(message, models, { context, skills, limits });
|
|
53
|
-
}
|
|
54
36
|
async processLoop(callback, invalidArgsCount) {
|
|
55
37
|
const prevItems = await this.memory.findLastItems(16);
|
|
56
38
|
if (prevItems.length === 0) {
|
|
@@ -114,8 +96,7 @@ ChatBot = __decorate([
|
|
|
114
96
|
injectable(),
|
|
115
97
|
__metadata("design:paramtypes", [ChatMemory,
|
|
116
98
|
ChatAdapter,
|
|
117
|
-
MindsetOperator
|
|
118
|
-
ImageDescriber])
|
|
99
|
+
MindsetOperator])
|
|
119
100
|
], ChatBot);
|
|
120
101
|
|
|
121
102
|
export { ChatBot };
|
|
@@ -9,7 +9,6 @@ function extractChatMessageText(message, options = {}) {
|
|
|
9
9
|
id: x.id,
|
|
10
10
|
name: x.name,
|
|
11
11
|
mimeType: x.mimeType,
|
|
12
|
-
description: x.description,
|
|
13
12
|
notAnalyzed: isAnalyzed(x.mimeType, options.supportedImageMimeTypes) ? undefined : true,
|
|
14
13
|
})),
|
|
15
14
|
documents: message.documents?.map((x) => ({
|
|
@@ -5,7 +5,6 @@ import '../chat-bot/ChatBot.js';
|
|
|
5
5
|
import '../chat-bot/ChatOperator.js';
|
|
6
6
|
import { ChatRepository } from '../chat-bot/ChatRepository.js';
|
|
7
7
|
import '../chat-bot/UnionChatAdapter.js';
|
|
8
|
-
import '../chat-bot/ImageDescriber.js';
|
|
9
8
|
import { injectable } from '../../core/injection/index.js';
|
|
10
9
|
import '../chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
11
10
|
import 'uuid';
|
|
@@ -8,7 +8,6 @@ import { ChatMemory } from '../chat-bot/ChatMemory.js';
|
|
|
8
8
|
import '../chat-bot/ChatOperator.js';
|
|
9
9
|
import { ChatRepository } from '../chat-bot/ChatRepository.js';
|
|
10
10
|
import '../chat-bot/UnionChatAdapter.js';
|
|
11
|
-
import '../chat-bot/ImageDescriber.js';
|
|
12
11
|
import '../chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
13
12
|
import 'uuid';
|
|
14
13
|
import { ChatBotMetadataStore } from '../chat-bot/metadata/ChatBotMetadataStore.js';
|
|
@@ -7,7 +7,6 @@ import '../chat-bot/ChatBot.js';
|
|
|
7
7
|
import '../chat-bot/ChatOperator.js';
|
|
8
8
|
import { ChatRepository } from '../chat-bot/ChatRepository.js';
|
|
9
9
|
import '../chat-bot/UnionChatAdapter.js';
|
|
10
|
-
import '../chat-bot/ImageDescriber.js';
|
|
11
10
|
import '../chat-bot/metadata/ChatAdapterMetadataStore.js';
|
|
12
11
|
import 'uuid';
|
|
13
12
|
import '../chat-bot/metadata/ChatBotMetadataStore.js';
|
package/dist/src/index.d.ts
CHANGED
|
@@ -953,8 +953,6 @@ interface IChatMessagesPublicFile {
|
|
|
953
953
|
base64Url?: undefined;
|
|
954
954
|
mimeType: string;
|
|
955
955
|
id: string;
|
|
956
|
-
/** Text description of the file content, used to recall it without re-analyzing the binary. */
|
|
957
|
-
description?: string;
|
|
958
956
|
}
|
|
959
957
|
interface IChatMessagesPrivateFile {
|
|
960
958
|
name?: string;
|
|
@@ -962,8 +960,6 @@ interface IChatMessagesPrivateFile {
|
|
|
962
960
|
base64Url: string;
|
|
963
961
|
mimeType: string;
|
|
964
962
|
id: string;
|
|
965
|
-
/** Text description of the file content, used to recall it without re-analyzing the binary. */
|
|
966
|
-
description?: string;
|
|
967
963
|
}
|
|
968
964
|
type IChatMessageFile = IChatMessagesPrivateFile | IChatMessagesPublicFile;
|
|
969
965
|
|
|
@@ -1071,43 +1067,13 @@ interface IChatBot {
|
|
|
1071
1067
|
sendMessage(message: IChatMessage, callback: (message: IChatMessage) => Promise<void>): void | Promise<void>;
|
|
1072
1068
|
}
|
|
1073
1069
|
|
|
1074
|
-
/** Slice of the mindset used to focus the description on what the assistant cares about. */
|
|
1075
|
-
interface IImageDescribeContext {
|
|
1076
|
-
context?: string;
|
|
1077
|
-
skills?: string;
|
|
1078
|
-
limits?: string;
|
|
1079
|
-
}
|
|
1080
|
-
/**
|
|
1081
|
-
* Generates a text description for images so the conversation can recall their
|
|
1082
|
-
* content on later turns without re-sending (and re-analyzing) the binary.
|
|
1083
|
-
*
|
|
1084
|
-
* The description is produced once, on arrival, with a single vision-model call
|
|
1085
|
-
* and stored on the image itself; from then on it travels with the message text
|
|
1086
|
-
* via {@link extractChatMessageText}.
|
|
1087
|
-
*/
|
|
1088
|
-
declare class ImageDescriber {
|
|
1089
|
-
private adapter;
|
|
1090
|
-
private logger;
|
|
1091
|
-
constructor(adapter: ChatAdapter);
|
|
1092
|
-
/**
|
|
1093
|
-
* Fills `description` for every image on the message that does not have one
|
|
1094
|
-
* yet, mutating the images in place. Safe to call with no images or no models:
|
|
1095
|
-
* it simply does nothing. Failures are logged and leave the image undescribed
|
|
1096
|
-
* rather than aborting the message.
|
|
1097
|
-
*/
|
|
1098
|
-
describeMessageImages(message: IChatMessage, models: IMindsetModelRef[], describeContext?: IImageDescribeContext): Promise<void>;
|
|
1099
|
-
describe(image: IChatMessageImage, models: IMindsetModelRef[], describeContext?: IImageDescribeContext): Promise<string | undefined>;
|
|
1100
|
-
}
|
|
1101
|
-
|
|
1102
1070
|
declare class ChatBot implements IChatBot {
|
|
1103
1071
|
private memory;
|
|
1104
1072
|
private adapter;
|
|
1105
1073
|
private mindset;
|
|
1106
|
-
private imageDescriber;
|
|
1107
1074
|
private logger;
|
|
1108
|
-
constructor(memory: ChatMemory, adapter: ChatAdapter, mindset: MindsetOperator
|
|
1075
|
+
constructor(memory: ChatMemory, adapter: ChatAdapter, mindset: MindsetOperator);
|
|
1109
1076
|
sendMessage(message: IChatMessage, callback: (message: IChatMessage) => Promise<void>): Promise<void>;
|
|
1110
|
-
private describeImages;
|
|
1111
1077
|
protected processLoop(callback: (message: IChatMessage) => Promise<void>, invalidArgsCount: number): Promise<void>;
|
|
1112
1078
|
}
|
|
1113
1079
|
|
|
@@ -2765,4 +2731,4 @@ declare function HtmlModule(options: IHtmlModuleOptions): {
|
|
|
2765
2731
|
new (): {};
|
|
2766
2732
|
};
|
|
2767
2733
|
|
|
2768
|
-
export { AnthropicChatAdapter, ApiKey, ApiKeyGuardMiddleware, ApiKeyHandshakeGuardMiddleware, ApiKeyRepository, Async, AsyncMetadataStore, Auth, Chat, ChatAdapter, ChatAdapterMetadataStore, ChatAdapterRegistry, ChatBot, ChatBotMetadataStore, ChatItem, ChatMemory, ChatOperator, ChatRepository, ChatResolver, type ClientMap, CmdChannel, CmdChannelConfig, CmdChannelServer, type CmdClientMessage, type CmdServerMessage, type ConfigReference, type ConfigReferenceType, ConfigResolver, Container, ControllerMetadataStore, CronJob, CronJobRepository, CrudRepository, CustomError, DeepSeekChatAdapter, DescriptionMetadataStore, EXPRESS_REQ, EXPRESS_RES, Entity, Env, type ErrorSeverity, ExpressProvider, GoogleChatAdapter, type GoogleChatAdapterV2Options, HtmlModule, HttpServerProvider, type IApiKeyData, type IApiKeyRepository, type IArrayValidationError, type IArrayValidationResult, type IBotMessageItem, type IBuiltQuery, type IChannelMessage, type IChannelMetadata, type IChatAdapter, type IChatAdapterDecoratorConfig, type IChatAdapterMetadata, type IChatAdapterNextItemsReq, type IChatAdapterNextItemsRes, type IChatAssociation, type IChatBot, type IChatBotMetadata, type IChatChannel, type IChatConnection, type IChatControllerMetadata, type IChatData, type IChatItem, type IChatItemData, type IChatItemType, type IChatMemory, type IChatMessage, type IChatMessageDocument, type IChatMessageFile, type IChatMessageImage, type IChatMessagesPrivateFile, type IChatMessagesPublicFile, type IChatRepository, type IChatType, type ICmdChannelEntry, type ICmdChannelHandlers, type ICmdChannelMessage, type ICmdImage, type ICmdIncomingMessage, type ICmdReceivedMessage, type ICommandConfig, type ICommandHandler, type ICommandHandlerConfig, type IConstructor, type ICronConfig, type ICronHandler, type ICronJobData, type ICronJobRepository, type ICrudRepository, type ICustomErrorData, type IDedupConfig, type IDescriptionMetadata, type IEndPointConfig, type IEndPointMetadata, type IEntityData, type IEnvType, type IErrorHandlersConfig, type IErrorMonitor, type IErrorMonitorContext, type IExtractChatMessageTextOptions, type IFunctionCall, type IFunctionCallItem, type IGenerateApiKeyReq, type IGenerateApiKeyRes, type IHandshakeMiddleware, type IHandshakeMiddlewareMetadata, type IHtmlModuleOptions, type IHumanMessageItem, type
|
|
2734
|
+
export { AnthropicChatAdapter, ApiKey, ApiKeyGuardMiddleware, ApiKeyHandshakeGuardMiddleware, ApiKeyRepository, Async, AsyncMetadataStore, Auth, Chat, ChatAdapter, ChatAdapterMetadataStore, ChatAdapterRegistry, ChatBot, ChatBotMetadataStore, ChatItem, ChatMemory, ChatOperator, ChatRepository, ChatResolver, type ClientMap, CmdChannel, CmdChannelConfig, CmdChannelServer, type CmdClientMessage, type CmdServerMessage, type ConfigReference, type ConfigReferenceType, ConfigResolver, Container, ControllerMetadataStore, CronJob, CronJobRepository, CrudRepository, CustomError, DeepSeekChatAdapter, DescriptionMetadataStore, EXPRESS_REQ, EXPRESS_RES, Entity, Env, type ErrorSeverity, ExpressProvider, GoogleChatAdapter, type GoogleChatAdapterV2Options, HtmlModule, HttpServerProvider, type IApiKeyData, type IApiKeyRepository, type IArrayValidationError, type IArrayValidationResult, type IBotMessageItem, type IBuiltQuery, type IChannelMessage, type IChannelMetadata, type IChatAdapter, type IChatAdapterDecoratorConfig, type IChatAdapterMetadata, type IChatAdapterNextItemsReq, type IChatAdapterNextItemsRes, type IChatAssociation, type IChatBot, type IChatBotMetadata, type IChatChannel, type IChatConnection, type IChatControllerMetadata, type IChatData, type IChatItem, type IChatItemData, type IChatItemType, type IChatMemory, type IChatMessage, type IChatMessageDocument, type IChatMessageFile, type IChatMessageImage, type IChatMessagesPrivateFile, type IChatMessagesPublicFile, type IChatRepository, type IChatType, type ICmdChannelEntry, type ICmdChannelHandlers, type ICmdChannelMessage, type ICmdImage, type ICmdIncomingMessage, type ICmdReceivedMessage, type ICommandConfig, type ICommandHandler, type ICommandHandlerConfig, type IConstructor, type ICronConfig, type ICronHandler, type ICronJobData, type ICronJobRepository, type ICrudRepository, type ICustomErrorData, type IDedupConfig, type IDescriptionMetadata, type IEndPointConfig, type IEndPointMetadata, type IEntityData, type IEnvType, type IErrorHandlersConfig, type IErrorMonitor, type IErrorMonitorContext, type IExtractChatMessageTextOptions, type IFunctionCall, type IFunctionCallItem, type IGenerateApiKeyReq, type IGenerateApiKeyRes, type IHandshakeMiddleware, type IHandshakeMiddlewareMetadata, type IHtmlModuleOptions, type IHumanMessageItem, type IJobData, type IJobOptions, type IJobRepository, type IJwtRefreshTokenData, type IJwtRefreshTokenRepository, type IKapsoChannelConfig, type IKapsoChannelMessage, type IKapsoChannelMessageListener, type IKapsoChatMessage, type IKapsoConversation, type IKapsoEvent, type IKapsoIncomingMessage, type IKapsoMessageReceivedEvent, type IKapsoReceivedMessage, type IKapsoUnknownEvent, type ILanguageModelUsage, type ILockKey, type ILocker, type ILockerKey, type IMemoryRepositoryAdapterOptions, type IMessageContext, type IMiddleware, type IMiddlewareMetadata, type IMindset, type IMindsetConfig, type IMindsetIdentity, type IMindsetLlm, type IMindsetMetadata, type IMindsetModelKind, type IMindsetModelRef, type IMindsetModels, type IMindsetModuleConfig, type IMindsetModuleMetadata, type IMindsetParameterSchema, type IMindsetTool, type IMindsetToolParameter, type IModelValidationError, type IModelValidationResult, type IModelValidatorsInfo, type IMoneyData, type IPersistentData, type IPgRepositoryConfig, type IProjectRunnerConfig, type IPropertyValidatorInfo, type IQueryAst, type IQueryCondition, type IQueryMethodMetadata, type IQueryOrderBy, type IReceivedMessage, type IRemoteApiKeyFetcher, type IRepositoryAdapter, type IRepositoryConfig, type IRepositoryRuntime, type IRestControllerConfig, type IRestControllerMetadata, type IScanProjectFilesOptions, type IScheduleAt, type IScheduleDelay, type ISendWhatsAppMessageReq, type ISendWhatsAppTemplateReq, type ISocketChannelConfig, type ISocketChannelMessage, type ISocketChannelReceivedMessage, type ISocketControllerConfig, type ISocketControllerMetadata, type ISocketEventConfig, type ISocketEventMetadata, type ISocketReceivedMessage, type IStorableData, type ITelegramChannelConfig, type ITelegramChannelMessage, type ITelegramReceivedMessage, type ITransactionAdapter, type IValidateArrayOptions, type IValidateArrayOptionsWithItemsValidators, type IValidateInputShape, type IValidateIsInOptions, type IValidateIsRecordOptions, type IValidateMaxOptions, type IValidateMinOptions, type IValidationError, type IValidationResult, type IValidator, type IValidatorMetadata, type IWasenderChannelConfig, type IWasenderChannelMessageListener, type IWasenderDeviceListMetadata, type IWasenderEvent, type IWasenderMessageContent, type IWasenderMessageContextInfo, type IWasenderMessageKey, type IWasenderMessageReceivedData, type IWasenderMessageReceivedEvent, type IWasenderQrUpdatedEvent, type IWasenderReceivedMessage, type IWhatsAppCloudContact, type IWhatsAppCloudMessage, type IWhatsAppCloudMessageMetadata, type IWhatsAppCloudTemplate, type IWhatsAppCloudTemplateComponent, type IWhatsAppCloudTemplateResponse, type IWhatsAppCloudWebhookPayload, type IWhatsAppSender, type IWhatsAppTemplateData, type IWhatsAppTemplateParameter, type IchatControllerConfig, InMemoryChatMemory, InMemoryChatRepository, InMemoryCronJobRepository, InMemoryJobRepository, InMemoryLockKey, InMemoryLocker, Job, JobRepository, JobRunner, Jwt, JwtAccessAndRefreshTokenDto, JwtConfig, JwtGuardMiddleware, JwtHandshakeGuardMiddleware, JwtRefreshToken, JwtRefreshTokenRepository, JwtSigner, JwtTokenDto, KapsoChannel, KapsoChannelConfig, KapsoReceiver, KapsoSender, KapsoWebhookController, Lifecycle, Locker, Logger, MEMORY_ADAPTER_ID, Mapper, MemoryRepositoryAdapter, MemoryRepositoryExtension, Mindset, MindsetMetadataStore, MindsetOperator, Money, MoneyDto, OpenRouterChatAdapter, OpenaiChatAdapter, PG_ADAPTER_ID, Password, type PasswordHashOptions, Persistent, PgApiKeyRepository, PgChatMemory, PgChatRepository, PgCronJobRepository, PgCrudRepository, PgJobRepository, PgJsonRepositoryAdapter, PgJwtRefreshTokenRepository, PgLockKey, PgLocker, PgRepositoryBase, PgRepositoryBase as PgRepositoryExtension, PgTransactionAdapter, ProjectRunner, type QueryConnector, type QueryOperator, type QueryPrefix, Random, RemoteApiKeyRepository, RepositoryAdapterRegistry, RepositoryMetadataStore, type ResolvedConfig, RestControllerMetadataStore, RestRequest, SocketChannel, SocketChannelConfig, SocketChannelMessageFile, SocketChannelReceivedMessage, SocketControllerMetadataStore, SocketServerConfig, SocketServerProvider, Storable, TelegramChannel, TelegramChannelConfig, TransactionMetadataStore, UnionChatAdapter, ValidationMetadataStore, WabotChatAdapter, WasenderChannel, WasenderChannelConfig, WasenderReceiver, WasenderSender, WasenderWebhookController, WhatsAppApiSender, WhatsAppReceiverByCloudApi, WhatsAppSender, apiKeyGuard, apiKeyHandshakeGuard, bool, boolArr, buildQuerySql, chatAdapter, chatBot, chatController, chatItemTypeOptions, cmd, cmdChannelName, cmdChannelSocketPath, command, commandHandler, computeDedupKey, container, cronHandler, description, errorToPlainObject, evaluateQueryAst, extractChatMessageText, extractNumberFromWasenderMessageKey, getClientMap, getPgClient, handshakeMiddlewares, inject, injectable, isArray, isBoolean, isChatMessageEmpty, isDate, isIn, isModel, isNotEmpty, isNumber, isOptional, isPresent, isRecord, isRetryableError, isString, jwtGuard, jwtHandshakeGuard, kapso, kapsoChannelName, markdownToTelegramHtml, max, memExtension, middleware, min, mindset, mindsetModule, modelInfo, num, numArr, obj, onDelete, onGet, onPost, onPut, onSocketEvent, parseQueryMethodName, pendingMediaStartIndex, pgExtension, pgStorage, query, queryExtension, readJsonFromFile, repository, resolveConfigReferences, restController, run, runChatAdapters, runChatControllers, runCmdClient, runCommandHandlers, runCronHandlers, runRestControllers, runSocketControllers, safeJsonParse, scanProjectFiles, scoped, setupErrorHandlers, singleton, socket, socketChannelName, socketController, stopCommandHandlers, stopCronHandlers, str, strArr, telegram, telegramChannelName, transaction, unconsumedMediaStartIndex, validateAndTransform, validateArray, validateIsBoolean, validateIsDate, validateIsIn, validateIsNotEmpty, validateIsNumber, validateIsPresent, validateIsRecord, validateIsString, validateMax, validateMin, validateModel, wasender, wasenderChannelName, withPgClient, withPgTransaction, writeJsonToFile };
|
package/dist/src/index.js
CHANGED
|
@@ -71,7 +71,6 @@ export { ChatRepository } from './feature/chat-bot/ChatRepository.js';
|
|
|
71
71
|
export { chatItemTypeOptions } from './feature/chat-bot/IChatItem.js';
|
|
72
72
|
export { UnionChatAdapter } from './feature/chat-bot/UnionChatAdapter.js';
|
|
73
73
|
export { extractChatMessageText } from './feature/chat-bot/extractChatMessageText.js';
|
|
74
|
-
export { ImageDescriber } from './feature/chat-bot/ImageDescriber.js';
|
|
75
74
|
export { isChatMessageEmpty } from './feature/chat-bot/isChatMessageEmpty.js';
|
|
76
75
|
export { isRetryableError } from './feature/chat-bot/isRetryableError.js';
|
|
77
76
|
export { chatAdapter } from './feature/chat-bot/metadata/@chatAdapter.js';
|
package/package.json
CHANGED
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import { __decorate, __metadata } from 'tslib';
|
|
2
|
-
import { injectable } from '../../core/injection/index.js';
|
|
3
|
-
import { Logger } from '../../core/logger/Logger.js';
|
|
4
|
-
import { ChatAdapter } from './ChatAdapter.js';
|
|
5
|
-
|
|
6
|
-
const DESCRIBE_INSTRUCTIONS = `
|
|
7
|
-
You describe images so an assistant can reference them later in a conversation
|
|
8
|
-
without seeing the image again.
|
|
9
|
-
|
|
10
|
-
Write a single, factual, self-contained paragraph covering the visible content:
|
|
11
|
-
objects, people, scene, layout, colors, and any readable text. Be thorough but
|
|
12
|
-
concise. Do not add opinions, greetings, or markdown — output only the
|
|
13
|
-
description.
|
|
14
|
-
`.trim();
|
|
15
|
-
function buildSystemPrompt(describeContext) {
|
|
16
|
-
const sections = [DESCRIBE_INSTRUCTIONS];
|
|
17
|
-
const mindsetSections = [
|
|
18
|
-
['Assistant context', describeContext.context],
|
|
19
|
-
['Assistant skills', describeContext.skills],
|
|
20
|
-
['Assistant limits', describeContext.limits],
|
|
21
|
-
].filter(([, value]) => value && value.trim());
|
|
22
|
-
if (mindsetSections.length > 0) {
|
|
23
|
-
sections.push('The assistant you describe images for is outlined below. Use it only to ' +
|
|
24
|
-
'decide which details deserve emphasis; still describe anything else that ' +
|
|
25
|
-
'is clearly visible.');
|
|
26
|
-
for (const [title, value] of mindsetSections) {
|
|
27
|
-
sections.push(`## ${title}\n${value.trim()}`);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
return sections.join('\n\n');
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Generates a text description for images so the conversation can recall their
|
|
34
|
-
* content on later turns without re-sending (and re-analyzing) the binary.
|
|
35
|
-
*
|
|
36
|
-
* The description is produced once, on arrival, with a single vision-model call
|
|
37
|
-
* and stored on the image itself; from then on it travels with the message text
|
|
38
|
-
* via {@link extractChatMessageText}.
|
|
39
|
-
*/
|
|
40
|
-
let ImageDescriber = class ImageDescriber {
|
|
41
|
-
adapter;
|
|
42
|
-
logger = new Logger('wabot:image-describer');
|
|
43
|
-
constructor(adapter) {
|
|
44
|
-
this.adapter = adapter;
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Fills `description` for every image on the message that does not have one
|
|
48
|
-
* yet, mutating the images in place. Safe to call with no images or no models:
|
|
49
|
-
* it simply does nothing. Failures are logged and leave the image undescribed
|
|
50
|
-
* rather than aborting the message.
|
|
51
|
-
*/
|
|
52
|
-
async describeMessageImages(message, models, describeContext = {}) {
|
|
53
|
-
const pending = message.images?.filter((image) => !image.description);
|
|
54
|
-
if (!pending || pending.length === 0 || models.length === 0)
|
|
55
|
-
return;
|
|
56
|
-
await Promise.all(pending.map(async (image) => {
|
|
57
|
-
const description = await this.describe(image, models, describeContext);
|
|
58
|
-
if (description)
|
|
59
|
-
image.description = description;
|
|
60
|
-
}));
|
|
61
|
-
}
|
|
62
|
-
async describe(image, models, describeContext = {}) {
|
|
63
|
-
if (models.length === 0)
|
|
64
|
-
return undefined;
|
|
65
|
-
try {
|
|
66
|
-
const { nextItems } = await this.adapter.nextItems({
|
|
67
|
-
models,
|
|
68
|
-
systemPrompt: buildSystemPrompt(describeContext),
|
|
69
|
-
tools: [],
|
|
70
|
-
prevItems: [
|
|
71
|
-
{
|
|
72
|
-
type: 'humanMessage',
|
|
73
|
-
humanMessage: { text: 'Describe this image.', images: [image] },
|
|
74
|
-
},
|
|
75
|
-
],
|
|
76
|
-
});
|
|
77
|
-
const parts = [];
|
|
78
|
-
for (const item of nextItems) {
|
|
79
|
-
if (item.type === 'botMessage' && item.botMessage.text)
|
|
80
|
-
parts.push(item.botMessage.text);
|
|
81
|
-
}
|
|
82
|
-
return parts.join('\n').trim() || undefined;
|
|
83
|
-
}
|
|
84
|
-
catch (err) {
|
|
85
|
-
this.logger.warn(`failed to describe image '${image.id}'`, err instanceof Error ? { message: err.message } : { err });
|
|
86
|
-
return undefined;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
};
|
|
90
|
-
ImageDescriber = __decorate([
|
|
91
|
-
injectable(),
|
|
92
|
-
__metadata("design:paramtypes", [ChatAdapter])
|
|
93
|
-
], ImageDescriber);
|
|
94
|
-
|
|
95
|
-
export { ImageDescriber };
|