@loonylabs/llm-middleware 2.21.0 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/middleware/services/llm/llm.service.d.ts +3 -2
- package/dist/middleware/services/llm/llm.service.d.ts.map +1 -1
- package/dist/middleware/services/llm/llm.service.js.map +1 -1
- package/dist/middleware/services/llm/providers/anthropic-provider.d.ts +2 -1
- package/dist/middleware/services/llm/providers/anthropic-provider.d.ts.map +1 -1
- package/dist/middleware/services/llm/providers/anthropic-provider.js +23 -5
- package/dist/middleware/services/llm/providers/anthropic-provider.js.map +1 -1
- package/dist/middleware/services/llm/providers/base-llm-provider.d.ts +3 -2
- package/dist/middleware/services/llm/providers/base-llm-provider.d.ts.map +1 -1
- package/dist/middleware/services/llm/providers/base-llm-provider.js.map +1 -1
- package/dist/middleware/services/llm/providers/gemini/gemini-base.provider.d.ts +3 -2
- package/dist/middleware/services/llm/providers/gemini/gemini-base.provider.d.ts.map +1 -1
- package/dist/middleware/services/llm/providers/gemini/gemini-base.provider.js +21 -6
- package/dist/middleware/services/llm/providers/gemini/gemini-base.provider.js.map +1 -1
- package/dist/middleware/services/llm/providers/ollama-provider.d.ts +4 -3
- package/dist/middleware/services/llm/providers/ollama-provider.d.ts.map +1 -1
- package/dist/middleware/services/llm/providers/ollama-provider.js +8 -4
- package/dist/middleware/services/llm/providers/ollama-provider.js.map +1 -1
- package/dist/middleware/services/llm/providers/requesty-provider.d.ts +2 -1
- package/dist/middleware/services/llm/providers/requesty-provider.d.ts.map +1 -1
- package/dist/middleware/services/llm/providers/requesty-provider.js +23 -5
- package/dist/middleware/services/llm/providers/requesty-provider.js.map +1 -1
- package/dist/middleware/services/llm/types/anthropic.types.d.ts +21 -2
- package/dist/middleware/services/llm/types/anthropic.types.d.ts.map +1 -1
- package/dist/middleware/services/llm/types/gemini.types.d.ts +9 -1
- package/dist/middleware/services/llm/types/gemini.types.d.ts.map +1 -1
- package/dist/middleware/services/llm/types/index.d.ts +1 -0
- package/dist/middleware/services/llm/types/index.d.ts.map +1 -1
- package/dist/middleware/services/llm/types/index.js +2 -0
- package/dist/middleware/services/llm/types/index.js.map +1 -1
- package/dist/middleware/services/llm/types/multimodal.types.d.ts +47 -0
- package/dist/middleware/services/llm/types/multimodal.types.d.ts.map +1 -0
- package/dist/middleware/services/llm/types/multimodal.types.js +14 -0
- package/dist/middleware/services/llm/types/multimodal.types.js.map +1 -0
- package/dist/middleware/services/llm/types/requesty.types.d.ts +14 -1
- package/dist/middleware/services/llm/types/requesty.types.d.ts.map +1 -1
- package/dist/middleware/services/llm/utils/index.d.ts +1 -0
- package/dist/middleware/services/llm/utils/index.d.ts.map +1 -1
- package/dist/middleware/services/llm/utils/index.js +1 -0
- package/dist/middleware/services/llm/utils/index.js.map +1 -1
- package/dist/middleware/services/llm/utils/multimodal.utils.d.ts +56 -0
- package/dist/middleware/services/llm/utils/multimodal.utils.d.ts.map +1 -0
- package/dist/middleware/services/llm/utils/multimodal.utils.js +119 -0
- package/dist/middleware/services/llm/utils/multimodal.utils.js.map +1 -0
- package/dist/middleware/usecases/base/base-ai.usecase.d.ts +15 -0
- package/dist/middleware/usecases/base/base-ai.usecase.d.ts.map +1 -1
- package/dist/middleware/usecases/base/base-ai.usecase.js +36 -9
- package/dist/middleware/usecases/base/base-ai.usecase.js.map +1 -1
- package/package.json +2 -1
|
@@ -13,6 +13,7 @@ const types_1 = require("../types");
|
|
|
13
13
|
const debug_llm_utils_1 = require("../utils/debug-llm.utils");
|
|
14
14
|
const data_flow_logger_1 = require("../../data-flow-logger");
|
|
15
15
|
const retry_utils_1 = require("../utils/retry.utils");
|
|
16
|
+
const multimodal_utils_1 = require("../utils/multimodal.utils");
|
|
16
17
|
/**
|
|
17
18
|
* Check if the model is a Google/Gemini model.
|
|
18
19
|
* Google Gemini doesn't support 'none' - it needs 'min' for minimal reasoning.
|
|
@@ -85,12 +86,28 @@ class RequestyProvider extends base_llm_provider_1.BaseLLMProvider {
|
|
|
85
86
|
headers['HTTP-Referer'] = httpReferer;
|
|
86
87
|
if (xTitle)
|
|
87
88
|
headers['X-Title'] = xTitle;
|
|
89
|
+
// Build user message content: multimodal (with images) or plain string
|
|
90
|
+
const userMessageContent = (0, multimodal_utils_1.hasImages)(userPrompt)
|
|
91
|
+
? (0, multimodal_utils_1.normalizeContent)(userPrompt).map(part => {
|
|
92
|
+
if (part.type === 'text') {
|
|
93
|
+
return { type: 'text', text: part.text };
|
|
94
|
+
}
|
|
95
|
+
// OpenAI format: image_url with data URI
|
|
96
|
+
return {
|
|
97
|
+
type: 'image_url',
|
|
98
|
+
image_url: {
|
|
99
|
+
url: `data:${part.mimeType};base64,${part.data}`,
|
|
100
|
+
...(part.detail && { detail: part.detail })
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
})
|
|
104
|
+
: (typeof userPrompt === 'string' ? userPrompt : (0, multimodal_utils_1.normalizeContent)(userPrompt).map(p => p.text).join('\n'));
|
|
88
105
|
// Build request payload (OpenAI format)
|
|
89
106
|
const requestPayload = {
|
|
90
107
|
model: model,
|
|
91
108
|
messages: [
|
|
92
109
|
{ role: 'system', content: systemMessage },
|
|
93
|
-
{ role: 'user', content:
|
|
110
|
+
{ role: 'user', content: userMessageContent }
|
|
94
111
|
],
|
|
95
112
|
max_tokens: maxTokens,
|
|
96
113
|
temperature: temperature,
|
|
@@ -98,14 +115,15 @@ class RequestyProvider extends base_llm_provider_1.BaseLLMProvider {
|
|
|
98
115
|
// Note: For Gemini, 'none' is mapped to 'min' since Gemini doesn't support 'none'
|
|
99
116
|
...(reasoningEffort && { reasoning_effort: mapReasoningEffort(reasoningEffort, model) })
|
|
100
117
|
};
|
|
101
|
-
// Prepare debug info
|
|
118
|
+
// Prepare debug info (use debug string to avoid base64 blobs in logs)
|
|
119
|
+
const userMessageDebug = (0, multimodal_utils_1.contentToDebugString)(userPrompt);
|
|
102
120
|
const debugInfo = {
|
|
103
121
|
timestamp: new Date(),
|
|
104
122
|
provider: this.providerName,
|
|
105
123
|
model: model,
|
|
106
124
|
baseUrl: this.BASE_URL,
|
|
107
125
|
systemMessage: systemMessage,
|
|
108
|
-
userMessage:
|
|
126
|
+
userMessage: userMessageDebug,
|
|
109
127
|
requestData: requestPayload,
|
|
110
128
|
useCase: debugContext,
|
|
111
129
|
sessionId: sessionId,
|
|
@@ -126,7 +144,7 @@ class RequestyProvider extends base_llm_provider_1.BaseLLMProvider {
|
|
|
126
144
|
const requestId = this.dataFlowLogger.startRequest(debugContext || 'requesty-direct', contextForLogger);
|
|
127
145
|
this.dataFlowLogger.logLLMRequest({
|
|
128
146
|
stage: debugContext || 'requesty-direct',
|
|
129
|
-
prompt:
|
|
147
|
+
prompt: userMessageDebug,
|
|
130
148
|
systemMessage: systemMessage,
|
|
131
149
|
modelName: model,
|
|
132
150
|
temperature: temperature,
|
|
@@ -139,7 +157,7 @@ class RequestyProvider extends base_llm_provider_1.BaseLLMProvider {
|
|
|
139
157
|
metadata: {
|
|
140
158
|
url: `${this.BASE_URL}/chat/completions`,
|
|
141
159
|
model: model,
|
|
142
|
-
promptLength: userPrompt
|
|
160
|
+
promptLength: (0, multimodal_utils_1.contentLength)(userPrompt),
|
|
143
161
|
maxTokens: maxTokens
|
|
144
162
|
}
|
|
145
163
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"requesty-provider.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/providers/requesty-provider.ts"],"names":[],"mappings":";;;;;;AAAA,iCAAiC;AACjC,kDAA0B;AAC1B,+BAAoC;AACpC,uEAA6D;AAC7D,2DAAsD;AACtD,oCAAuF;
|
|
1
|
+
{"version":3,"file":"requesty-provider.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/providers/requesty-provider.ts"],"names":[],"mappings":";;;;;;AAAA,iCAAiC;AACjC,kDAA0B;AAC1B,+BAAoC;AACpC,uEAA6D;AAC7D,2DAAsD;AACtD,oCAAuF;AASvF,8DAAqE;AACrE,6DAA+D;AAC/D,sDAAwD;AACxD,gEAA6G;AAE7G;;;GAGG;AACH,SAAS,aAAa,CAAC,KAAa;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACvC,OAAO,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC7B,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC;QAChC,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC;QAChC,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,MAAuB,EAAE,KAAa;IAChE,wDAAwD;IACxD,IAAI,MAAM,KAAK,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,MAAM;YACT,OAAO,MAAM,CAAC,CAAE,yBAAyB;QAC3C,KAAK,KAAK;YACR,OAAO,KAAK,CAAC;QACf,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC;QAClB,KAAK,MAAM;YACT,OAAO,MAAM,CAAC;QAChB;YACE,OAAO,QAAQ,CAAC,CAAE,eAAe;IACrC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAa,gBAAiB,SAAQ,mCAAe;IAInD;QACE,KAAK,CAAC,mBAAW,CAAC,QAAQ,CAAC,CAAC;QAHb,aAAQ,GAAG,kCAAkC,CAAC;QAI7D,IAAI,CAAC,cAAc,GAAG,wCAAqB,CAAC,WAAW,EAAE,CAAC;IAC5D,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,qBAAqB,CAChC,UAA6B,EAC7B,aAAqB,EACrB,UAAkC,EAAE;QAEpC,MAAM,EACJ,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,EACxC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,eAAe,EACrD,WAAW,GAAG,GAAG,EACjB,SAAS,GAAG,IAAI,EAChB,WAAW,EACX,MAAM,EACN,YAAY,EACZ,SAAS,GAAG,IAAA,SAAM,GAAE,EACpB,aAAa,EACb,UAAU,EACV,QAAQ,EACR,eAAe,EAChB,GAAG,OAAO,CAAC;QAEZ,mBAAmB;QACnB,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iDAAiD;gBACjD,6EAA6E,CAC9E,CAAC;QACJ,CAAC;QAED,iBAAiB;QACjB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,2CAA2C;gBAC3C,uEAAuE,CACxE,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,8EAA8E;QAC9E,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;YAClC,QAAQ,EAAE,kBAAkB;YAC5B,eAAe,EAAE,UAAU,SAAS,EAAE;SACvC,CAAC;QAEF,IAAI,WAAW;YAAE,OAAO,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC;QACvD,IAAI,MAAM;YAAE,OAAO,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC;QAExC,uEAAuE;QACvE,MAAM,kBAAkB,GAAG,IAAA,4BAAS,EAAC,UAAU,CAAC;YAC9C,CAAC,CAAC,IAAA,mCAAgB,EAAC,UAAU,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBACtC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,yCAAyC;gBACzC,OAAO;oBACL,IAAI,EAAE,WAAoB;oBAC1B,SAAS,EAAE;wBACT,GAAG,EAAE,QAAQ,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,IAAI,EAAE;wBAChD,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;qBAC5C;iBACF,CAAC;YACJ,CAAC,CAAC;YACJ,CAAC,CAAC,CAAC,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAA,mCAAgB,EAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAAoC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAEjJ,wCAAwC;QACxC,MAAM,cAAc,GAAuB;YACzC,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE;gBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE;aAC9C;YACD,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,WAAW;YACxB,iFAAiF;YACjF,kFAAkF;YAClF,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,eAAe,EAAE,KAAK,CAAC,EAAE,CAAC;SACzF,CAAC;QAEF,sEAAsE;QACtE,MAAM,gBAAgB,GAAG,IAAA,uCAAoB,EAAC,UAAU,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAiB;YAC9B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,aAAa,EAAE,aAAa;YAC5B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,OAAO,EAAE,YAAY;YACrB,SAAS,EAAE,SAAS;YACpB,aAAa,EAAE,aAAa;YAC5B,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,QAAQ;YAClB,gDAAgD;YAChD,WAAW;YACX,eAAe;SAChB,CAAC;QAEF,cAAc;QACd,MAAM,6BAAW,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAExC,MAAM,gBAAgB,GAAG;YACvB,gBAAgB,EAAE,aAAa;YAC/B,WAAW,EAAE,UAAU;YACvB,YAAY;SACb,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,CAChD,YAAY,IAAI,iBAAiB,EACjC,gBAAgB,CACjB,CAAC;QAEF,IAAI,CAAC,cAAc,CAAC,aAAa,CAC/B;YACE,KAAK,EAAE,YAAY,IAAI,iBAAiB;YACxC,MAAM,EAAE,gBAAgB;YACxB,aAAa,EAAE,aAAa;YAC5B,SAAS,EAAE,KAAK;YAChB,WAAW,EAAE,WAAW;YACxB,WAAW,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE;SAChE,EACD,gBAAgB,EAChB,SAAS,CACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,sBAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;gBAC7C,OAAO,EAAE,kBAAkB;gBAC3B,QAAQ,EAAE;oBACR,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,mBAAmB;oBACxC,KAAK,EAAE,KAAK;oBACZ,YAAY,EAAE,IAAA,gCAAa,EAAC,UAAU,CAAC;oBACvC,SAAS,EAAE,SAAS;iBACrB;aACF,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAA,8BAAgB,EACrC,GAAG,EAAE,CAAC,eAAK,CAAC,IAAI,CACd,GAAG,IAAI,CAAC,QAAQ,mBAAmB,EACnC,cAAc,EACd;gBACE,OAAO;gBACP,OAAO,EAAE,MAAM,CAAC,qBAAqB;aACtC,CACF,EACD,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,OAAO,CAAC,KAAK,CACd,CAAC;YAEF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;YAEtD,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACxC,MAAM,WAAW,GAAwB,QAAQ,CAAC,IAAI,CAAC;gBAEvD,4BAA4B;gBAC5B,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;gBAEpE,wBAAwB;gBACxB,MAAM,UAAU,GAAe;oBAC7B,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,aAAa;oBAC5C,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,iBAAiB;oBACjD,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,YAAY;oBAC3C,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;iBACjF,CAAC;gBAEF,qBAAqB;gBACrB,MAAM,kBAAkB,GAAqB;oBAC3C,OAAO,EAAE;wBACP,OAAO,EAAE,YAAY;qBACtB;oBACD,SAAS,EAAE,SAAS;oBACpB,QAAQ,EAAE;wBACR,QAAQ,EAAE,IAAI,CAAC,YAAY;wBAC3B,KAAK,EAAE,WAAW,CAAC,KAAK;wBACxB,UAAU,EAAE,UAAU,CAAC,WAAW;wBAClC,cAAc,EAAE,eAAe;qBAChC;oBACD,KAAK,EAAE,UAAU;oBACjB,EAAE,EAAE,WAAW,CAAC,EAAE;oBAClB,aAAa,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,aAAa,IAAI,SAAS;iBAClE,CAAC;gBAEF,oBAAoB;gBACpB,SAAS,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzC,SAAS,CAAC,QAAQ,GAAG,YAAY,CAAC;gBAClC,SAAS,CAAC,eAAe,GAAG,WAAW,CAAC;gBAExC,eAAe;gBACf,MAAM,6BAAW,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAEzC,IAAI,CAAC,cAAc,CAAC,cAAc,CAChC,YAAY,IAAI,iBAAiB,EACjC;oBACE,WAAW,EAAE,YAAY;oBACzB,cAAc,EAAE,eAAe;iBAChC,EACD,gBAAgB,EAChB,SAAS,CACV,CAAC;gBAEF,OAAO,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,QAAQ,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;gBACnE,sBAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;oBACzC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;oBAC9B,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,QAAQ,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE;iBAC/B,CAAC,CAAC;gBAEH,IAAI,CAAC,cAAc,CAAC,cAAc,CAChC,YAAY,IAAI,iBAAiB,EACjC;oBACE,WAAW,EAAE,EAAE;oBACf,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB;oBAC7C,KAAK;iBACN,EACD,gBAAgB,EAChB,SAAS,CACV,CAAC;gBAEF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,YAAY,GAAG,eAAe,CAAC;YACnC,IAAI,YAAY,GAAwB,EAAE,CAAC;YAE3C,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC3B,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC;YAC/B,CAAC;YAED,sBAAsB;YACtB,IACE,KAAK;gBACL,OAAO,KAAK,KAAK,QAAQ;gBACzB,cAAc,IAAI,KAAK;gBACvB,KAAK,CAAC,YAAY,KAAK,IAAI,EAC3B,CAAC;gBACD,MAAM,UAAU,GAAG,KAAY,CAAC;gBAEhC,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;oBACxB,YAAY,GAAG;wBACb,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM;wBACtC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;wBAC1C,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI;qBAC/B,CAAC;oBAEF,oCAAoC;oBACpC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBACvC,sBAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE;4BACrD,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;4BAC9B,KAAK,EAAE,iBAAiB;4BACxB,QAAQ,EAAE;gCACR,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM;gCACtC,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO;6BAClD;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBAC9C,sBAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;4BAClC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;4BAC9B,KAAK,EAAE,mBAAmB;4BAC1B,QAAQ,EAAE;gCACR,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM;gCACtC,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC;6BACvD;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBAC9C,sBAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE;4BAC1C,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;4BAC9B,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,IAAI,iBAAiB;4BACpE,QAAQ,EAAE;gCACR,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI;gCAC3C,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK;6BACzC;yBACF,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,sBAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE;gBACnC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC9B,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR,GAAG,YAAY;oBACf,YAAY,EAAE,KAAK;oBACnB,SAAS,EAAE,SAAS;iBACrB;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,cAAc,CAAC,cAAc,CAChC,YAAY,IAAI,iBAAiB,EACjC;gBACE,WAAW,EAAE,EAAE;gBACf,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB;gBAC7C,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC;aAChE,EACD,gBAAgB,EAChB,SAAS,CACV,CAAC;YAEF,SAAS,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;YACzC,SAAS,CAAC,KAAK,GAAG;gBAChB,OAAO,EAAE,YAAY;gBACrB,OAAO,EAAE,YAAY;aACtB,CAAC;YAEF,MAAM,6BAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAEtC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF;AArUD,4CAqUC;AAM4B,2CAAe;AAJ5C,4BAA4B;AACf,QAAA,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;AAI1B,0BAJhB,wBAAgB,CAIe"}
|
|
@@ -31,11 +31,30 @@ export interface AnthropicRequestOptions extends CommonLLMOptions {
|
|
|
31
31
|
system?: string;
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
|
-
* Anthropic
|
|
34
|
+
* Anthropic image source (base64-encoded)
|
|
35
|
+
*/
|
|
36
|
+
export interface AnthropicImageSource {
|
|
37
|
+
type: 'base64';
|
|
38
|
+
media_type: string;
|
|
39
|
+
data: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Anthropic content part for multimodal messages
|
|
43
|
+
*/
|
|
44
|
+
export type AnthropicContentPart = {
|
|
45
|
+
type: 'text';
|
|
46
|
+
text: string;
|
|
47
|
+
} | {
|
|
48
|
+
type: 'image';
|
|
49
|
+
source: AnthropicImageSource;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Anthropic message format.
|
|
53
|
+
* Content can be a plain string (text-only) or an array of content parts (multimodal).
|
|
35
54
|
*/
|
|
36
55
|
export interface AnthropicMessage {
|
|
37
56
|
role: 'user' | 'assistant';
|
|
38
|
-
content: string;
|
|
57
|
+
content: string | AnthropicContentPart[];
|
|
39
58
|
}
|
|
40
59
|
/**
|
|
41
60
|
* Anthropic API request payload
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/anthropic.types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAmB,MAAM,gBAAgB,CAAC;AAEtF;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,2DAA2D;IAC3D,IAAI,EAAE,SAAS,CAAC;IAChB,6EAA6E;IAC7E,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,uBAAwB,SAAQ,gBAAgB;IAG/D,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,sDAAsD;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAE1B,qCAAqC;IACrC,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"anthropic.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/anthropic.types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAmB,MAAM,gBAAgB,CAAC;AAEtF;;;;GAIG;AACH,MAAM,WAAW,uBAAuB;IACtC,2DAA2D;IAC3D,IAAI,EAAE,SAAS,CAAC;IAChB,6EAA6E;IAC7E,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,uBAAwB,SAAQ,gBAAgB;IAG/D,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,sDAAsD;IACtD,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAE1B,qCAAqC;IACrC,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,QAAQ,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAC5B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAA;CAAE,CAAC;AAEpD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,oBAAoB,EAAE,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,uBAAuB,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,yDAAyD;IACzD,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,GAAG,YAAY,GAAG,eAAe,GAAG,IAAI,CAAC;IAChE,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,iBAAiB;IAE1D,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB"}
|
|
@@ -60,7 +60,15 @@ export interface GeminiRequestOptions extends CommonLLMOptions {
|
|
|
60
60
|
* - thoughtSignature - Present on the actual content part (non-thinking)
|
|
61
61
|
*/
|
|
62
62
|
export interface GeminiPart {
|
|
63
|
-
|
|
63
|
+
/** Text content. Optional when inlineData is provided. */
|
|
64
|
+
text?: string;
|
|
65
|
+
/** Inline binary data for images/media. Mutually exclusive with text. */
|
|
66
|
+
inlineData?: {
|
|
67
|
+
/** MIME type of the data (e.g., 'image/jpeg', 'image/png') */
|
|
68
|
+
mimeType: string;
|
|
69
|
+
/** Base64-encoded data */
|
|
70
|
+
data: string;
|
|
71
|
+
};
|
|
64
72
|
/** If true, this part contains thinking/reasoning content (not the actual response) */
|
|
65
73
|
thought?: boolean;
|
|
66
74
|
/** Signature present on content parts when thinking is enabled (indicates non-thinking content) */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/gemini.types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAmB,MAAM,gBAAgB,CAAC;AAEtF;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG,4BAA4B,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvG;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;IAEpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAG5D,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IAEzB,kEAAkE;IAClE,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,yEAAyE;IACzE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/gemini.types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAmB,MAAM,gBAAgB,CAAC;AAEtF;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG,4BAA4B,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvG;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;IAEpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAG5D,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IAEzB,kEAAkE;IAClE,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,yEAAyE;IACzE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,UAAU,CAAC,EAAE;QACX,8DAA8D;QAC9D,QAAQ,EAAE,MAAM,CAAC;QACjB,0BAA0B;QAC1B,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,uFAAuF;IACvF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,mGAAmG;IACnG,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,yEAAyE;IACzE,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,iBAAiB,CAAC,EAAE;QAClB,KAAK,EAAE,UAAU,EAAE,CAAC;KACrB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,KAAK,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,EAAE,MAAM,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,2EAA2E;IAC3E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,+FAA+F;IAC/F,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,cAAc,CAAC,EAAE;QACf,aAAa,CAAC,EAAE,KAAK,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;KACJ,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,iBAAiB;IAEvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,KAAK,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;CACJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/index.ts"],"names":[],"mappings":"AACA,cAAc,gBAAgB,CAAC;AAG/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/index.ts"],"names":[],"mappings":"AACA,cAAc,gBAAgB,CAAC;AAG/B,cAAc,oBAAoB,CAAC;AAGnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,mBAAmB,CAAC"}
|
|
@@ -16,6 +16,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
// Common types
|
|
18
18
|
__exportStar(require("./common.types"), exports);
|
|
19
|
+
// Multimodal types (vision/image input)
|
|
20
|
+
__exportStar(require("./multimodal.types"), exports);
|
|
19
21
|
// Provider-specific types
|
|
20
22
|
__exportStar(require("./ollama.types"), exports);
|
|
21
23
|
__exportStar(require("./anthropic.types"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,eAAe;AACf,iDAA+B;AAE/B,0BAA0B;AAC1B,iDAA+B;AAC/B,oDAAkC;AAClC,iDAA+B;AAC/B,mDAAiC;AACjC,oDAAkC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,eAAe;AACf,iDAA+B;AAE/B,wCAAwC;AACxC,qDAAmC;AAEnC,0BAA0B;AAC1B,iDAA+B;AAC/B,oDAAkC;AAClC,iDAA+B;AAC/B,mDAAiC;AACjC,oDAAkC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multimodal content types for vision/image input support.
|
|
3
|
+
* Enables sending images alongside text to LLM providers.
|
|
4
|
+
*
|
|
5
|
+
* Design decisions:
|
|
6
|
+
* - Base64 only (no URLs) — Gemini doesn't support direct URLs, base64 is universal
|
|
7
|
+
* - type: 'image' (not 'image_url') — follows Vercel AI SDK / Anthropic convention
|
|
8
|
+
* - Explicit mimeType — required by Gemini and Anthropic, safer than inference
|
|
9
|
+
*
|
|
10
|
+
* @since 2.22.0
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Text content part for multimodal messages.
|
|
14
|
+
*/
|
|
15
|
+
export interface TextContentPart {
|
|
16
|
+
type: 'text';
|
|
17
|
+
text: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Image content part for multimodal messages.
|
|
21
|
+
* Uses base64-encoded image data (without data-URI prefix).
|
|
22
|
+
*/
|
|
23
|
+
export interface ImageContentPart {
|
|
24
|
+
type: 'image';
|
|
25
|
+
/** Base64-encoded image data (without data: URI prefix) */
|
|
26
|
+
data: string;
|
|
27
|
+
/** MIME type of the image */
|
|
28
|
+
mimeType: 'image/jpeg' | 'image/png' | 'image/webp' | 'image/gif';
|
|
29
|
+
/**
|
|
30
|
+
* Optional resolution/detail hint.
|
|
31
|
+
* - OpenAI: maps to `detail` parameter (low/high/auto)
|
|
32
|
+
* - Gemini: could map to media_resolution (not currently used)
|
|
33
|
+
* - Anthropic: ignored (no equivalent)
|
|
34
|
+
*/
|
|
35
|
+
detail?: 'low' | 'high' | 'auto';
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Union type for all content part types.
|
|
39
|
+
* Extensible for future content types (audio, video, etc.).
|
|
40
|
+
*/
|
|
41
|
+
export type ContentPart = TextContentPart | ImageContentPart;
|
|
42
|
+
/**
|
|
43
|
+
* Multimodal content: either a plain string (backward-compatible) or an array of content parts.
|
|
44
|
+
* When a string is provided, it is internally normalized to [{ type: 'text', text: string }].
|
|
45
|
+
*/
|
|
46
|
+
export type MultimodalContent = string | ContentPart[];
|
|
47
|
+
//# sourceMappingURL=multimodal.types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multimodal.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/multimodal.types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,OAAO,CAAC;IACd,2DAA2D;IAC3D,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,QAAQ,EAAE,YAAY,GAAG,WAAW,GAAG,YAAY,GAAG,WAAW,CAAC;IAClE;;;;;OAKG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,eAAe,GAAG,gBAAgB,CAAC;AAE7D;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,WAAW,EAAE,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Multimodal content types for vision/image input support.
|
|
4
|
+
* Enables sending images alongside text to LLM providers.
|
|
5
|
+
*
|
|
6
|
+
* Design decisions:
|
|
7
|
+
* - Base64 only (no URLs) — Gemini doesn't support direct URLs, base64 is universal
|
|
8
|
+
* - type: 'image' (not 'image_url') — follows Vercel AI SDK / Anthropic convention
|
|
9
|
+
* - Explicit mimeType — required by Gemini and Anthropic, safer than inference
|
|
10
|
+
*
|
|
11
|
+
* @since 2.22.0
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
//# sourceMappingURL=multimodal.types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multimodal.types.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/multimodal.types.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG"}
|
|
@@ -16,6 +16,19 @@ export interface RequestyRequestOptions extends CommonLLMOptions {
|
|
|
16
16
|
* extended values ('min', 'max', 'none').
|
|
17
17
|
*/
|
|
18
18
|
export type RequestyReasoningEffort = 'none' | 'min' | 'low' | 'medium' | 'high' | 'max';
|
|
19
|
+
/**
|
|
20
|
+
* OpenAI-compatible content part for multimodal messages
|
|
21
|
+
*/
|
|
22
|
+
export type RequestyContentPart = {
|
|
23
|
+
type: 'text';
|
|
24
|
+
text: string;
|
|
25
|
+
} | {
|
|
26
|
+
type: 'image_url';
|
|
27
|
+
image_url: {
|
|
28
|
+
url: string;
|
|
29
|
+
detail?: 'low' | 'high' | 'auto';
|
|
30
|
+
};
|
|
31
|
+
};
|
|
19
32
|
/**
|
|
20
33
|
* OpenAI-compatible request format for Requesty API
|
|
21
34
|
*/
|
|
@@ -23,7 +36,7 @@ export interface RequestyAPIRequest {
|
|
|
23
36
|
model: string;
|
|
24
37
|
messages: Array<{
|
|
25
38
|
role: 'system' | 'user' | 'assistant';
|
|
26
|
-
content: string;
|
|
39
|
+
content: string | RequestyContentPart[];
|
|
27
40
|
}>;
|
|
28
41
|
temperature?: number;
|
|
29
42
|
max_tokens?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"requesty.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/requesty.types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAmB,MAAM,gBAAgB,CAAC;AAEtF;;;GAGG;AACH,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAC;AAEzF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACtC,OAAO,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"requesty.types.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/types/requesty.types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAmB,MAAM,gBAAgB,CAAC;AAEtF;;;GAGG;AACH,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAC;AAEzF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAC3B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAExF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACtC,OAAO,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAAC;KACzC,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,uBAAuB,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,iBAAiB,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE;YACP,IAAI,EAAE,WAAW,CAAC;YAClB,OAAO,EAAE,MAAM,CAAC;SACjB,CAAC;QACF,aAAa,EAAE,MAAM,GAAG,QAAQ,GAAG,gBAAgB,GAAG,IAAI,CAAC;KAC5D,CAAC,CAAC;IACH,KAAK,EAAE;QACL,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,iBAAiB;IACzD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC"}
|
|
@@ -17,4 +17,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
17
17
|
__exportStar(require("./debug-llm.utils"), exports);
|
|
18
18
|
__exportStar(require("./text-analysis.utils"), exports);
|
|
19
19
|
__exportStar(require("./retry.utils"), exports);
|
|
20
|
+
__exportStar(require("./multimodal.utils"), exports);
|
|
20
21
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAAkC;AAClC,wDAAsC;AACtC,gDAA8B"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAAkC;AAClC,wDAAsC;AACtC,gDAA8B;AAC9B,qDAAmC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Utility functions for multimodal content handling.
|
|
3
|
+
* Provides normalization, debug formatting, and inspection helpers.
|
|
4
|
+
*
|
|
5
|
+
* @since 2.22.0
|
|
6
|
+
*/
|
|
7
|
+
import { MultimodalContent, ContentPart } from '../types/multimodal.types';
|
|
8
|
+
/**
|
|
9
|
+
* Normalizes MultimodalContent to ContentPart[].
|
|
10
|
+
* Converts a plain string to a single TextContentPart array.
|
|
11
|
+
*
|
|
12
|
+
* @param content - String or ContentPart array
|
|
13
|
+
* @returns Normalized ContentPart array
|
|
14
|
+
*/
|
|
15
|
+
export declare function normalizeContent(content: MultimodalContent): ContentPart[];
|
|
16
|
+
/**
|
|
17
|
+
* Extracts the combined text content from MultimodalContent.
|
|
18
|
+
* Only includes text parts, ignores images.
|
|
19
|
+
*
|
|
20
|
+
* @param content - String or ContentPart array
|
|
21
|
+
* @returns Combined text from all text parts
|
|
22
|
+
*/
|
|
23
|
+
export declare function extractTextContent(content: MultimodalContent): string;
|
|
24
|
+
/**
|
|
25
|
+
* Creates a debug-safe string representation of MultimodalContent.
|
|
26
|
+
* Images are replaced with placeholder strings showing type and size.
|
|
27
|
+
* This prevents base64 blobs from appearing in logs.
|
|
28
|
+
*
|
|
29
|
+
* @param content - String or ContentPart array
|
|
30
|
+
* @returns Debug-safe string representation
|
|
31
|
+
*/
|
|
32
|
+
export declare function contentToDebugString(content: MultimodalContent): string;
|
|
33
|
+
/**
|
|
34
|
+
* Returns the effective "length" of MultimodalContent for metrics/logging.
|
|
35
|
+
* For strings, returns string length. For content parts, returns
|
|
36
|
+
* the debug string length (images counted as placeholder length).
|
|
37
|
+
*
|
|
38
|
+
* @param content - String or ContentPart array
|
|
39
|
+
* @returns Character length suitable for metrics
|
|
40
|
+
*/
|
|
41
|
+
export declare function contentLength(content: MultimodalContent): number;
|
|
42
|
+
/**
|
|
43
|
+
* Checks whether MultimodalContent contains any image parts.
|
|
44
|
+
*
|
|
45
|
+
* @param content - String or ContentPart array
|
|
46
|
+
* @returns true if content contains at least one image part
|
|
47
|
+
*/
|
|
48
|
+
export declare function hasImages(content: MultimodalContent): boolean;
|
|
49
|
+
/**
|
|
50
|
+
* Counts the number of image parts in MultimodalContent.
|
|
51
|
+
*
|
|
52
|
+
* @param content - String or ContentPart array
|
|
53
|
+
* @returns Number of image parts
|
|
54
|
+
*/
|
|
55
|
+
export declare function countImages(content: MultimodalContent): number;
|
|
56
|
+
//# sourceMappingURL=multimodal.utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multimodal.utils.d.ts","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/utils/multimodal.utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAoB,MAAM,2BAA2B,CAAC;AAE7F;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,WAAW,EAAE,CAK1E;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAQrE;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAgBvE;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAKhE;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAK7D;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAK9D"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Utility functions for multimodal content handling.
|
|
4
|
+
* Provides normalization, debug formatting, and inspection helpers.
|
|
5
|
+
*
|
|
6
|
+
* @since 2.22.0
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.normalizeContent = normalizeContent;
|
|
10
|
+
exports.extractTextContent = extractTextContent;
|
|
11
|
+
exports.contentToDebugString = contentToDebugString;
|
|
12
|
+
exports.contentLength = contentLength;
|
|
13
|
+
exports.hasImages = hasImages;
|
|
14
|
+
exports.countImages = countImages;
|
|
15
|
+
/**
|
|
16
|
+
* Normalizes MultimodalContent to ContentPart[].
|
|
17
|
+
* Converts a plain string to a single TextContentPart array.
|
|
18
|
+
*
|
|
19
|
+
* @param content - String or ContentPart array
|
|
20
|
+
* @returns Normalized ContentPart array
|
|
21
|
+
*/
|
|
22
|
+
function normalizeContent(content) {
|
|
23
|
+
if (typeof content === 'string') {
|
|
24
|
+
return [{ type: 'text', text: content }];
|
|
25
|
+
}
|
|
26
|
+
return content;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Extracts the combined text content from MultimodalContent.
|
|
30
|
+
* Only includes text parts, ignores images.
|
|
31
|
+
*
|
|
32
|
+
* @param content - String or ContentPart array
|
|
33
|
+
* @returns Combined text from all text parts
|
|
34
|
+
*/
|
|
35
|
+
function extractTextContent(content) {
|
|
36
|
+
if (typeof content === 'string') {
|
|
37
|
+
return content;
|
|
38
|
+
}
|
|
39
|
+
return content
|
|
40
|
+
.filter((part) => part.type === 'text')
|
|
41
|
+
.map(part => part.text)
|
|
42
|
+
.join('\n');
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Creates a debug-safe string representation of MultimodalContent.
|
|
46
|
+
* Images are replaced with placeholder strings showing type and size.
|
|
47
|
+
* This prevents base64 blobs from appearing in logs.
|
|
48
|
+
*
|
|
49
|
+
* @param content - String or ContentPart array
|
|
50
|
+
* @returns Debug-safe string representation
|
|
51
|
+
*/
|
|
52
|
+
function contentToDebugString(content) {
|
|
53
|
+
if (typeof content === 'string') {
|
|
54
|
+
return content;
|
|
55
|
+
}
|
|
56
|
+
return content
|
|
57
|
+
.map(part => {
|
|
58
|
+
if (part.type === 'text') {
|
|
59
|
+
return part.text;
|
|
60
|
+
}
|
|
61
|
+
// Calculate approximate size of base64 data
|
|
62
|
+
const sizeBytes = Math.ceil(part.data.length * 3 / 4);
|
|
63
|
+
const sizeStr = formatByteSize(sizeBytes);
|
|
64
|
+
return `[IMAGE: ${part.mimeType}, ${sizeStr}]`;
|
|
65
|
+
})
|
|
66
|
+
.join('\n');
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Returns the effective "length" of MultimodalContent for metrics/logging.
|
|
70
|
+
* For strings, returns string length. For content parts, returns
|
|
71
|
+
* the debug string length (images counted as placeholder length).
|
|
72
|
+
*
|
|
73
|
+
* @param content - String or ContentPart array
|
|
74
|
+
* @returns Character length suitable for metrics
|
|
75
|
+
*/
|
|
76
|
+
function contentLength(content) {
|
|
77
|
+
if (typeof content === 'string') {
|
|
78
|
+
return content.length;
|
|
79
|
+
}
|
|
80
|
+
return contentToDebugString(content).length;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Checks whether MultimodalContent contains any image parts.
|
|
84
|
+
*
|
|
85
|
+
* @param content - String or ContentPart array
|
|
86
|
+
* @returns true if content contains at least one image part
|
|
87
|
+
*/
|
|
88
|
+
function hasImages(content) {
|
|
89
|
+
if (typeof content === 'string') {
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
return content.some(part => part.type === 'image');
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Counts the number of image parts in MultimodalContent.
|
|
96
|
+
*
|
|
97
|
+
* @param content - String or ContentPart array
|
|
98
|
+
* @returns Number of image parts
|
|
99
|
+
*/
|
|
100
|
+
function countImages(content) {
|
|
101
|
+
if (typeof content === 'string') {
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
return content.filter(part => part.type === 'image').length;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Formats a byte size into a human-readable string.
|
|
108
|
+
* @internal
|
|
109
|
+
*/
|
|
110
|
+
function formatByteSize(bytes) {
|
|
111
|
+
if (bytes < 1024) {
|
|
112
|
+
return `${bytes}B`;
|
|
113
|
+
}
|
|
114
|
+
if (bytes < 1024 * 1024) {
|
|
115
|
+
return `${(bytes / 1024).toFixed(1)}KB`;
|
|
116
|
+
}
|
|
117
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=multimodal.utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multimodal.utils.js","sourceRoot":"","sources":["../../../../../src/middleware/services/llm/utils/multimodal.utils.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAWH,4CAKC;AASD,gDAQC;AAUD,oDAgBC;AAUD,sCAKC;AAQD,8BAKC;AAQD,kCAKC;AAhGD;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,OAA0B;IACzD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,kBAAkB,CAAC,OAA0B;IAC3D,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,IAAI,EAA0C,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;SAC9E,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SACtB,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,oBAAoB,CAAC,OAA0B;IAC7D,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,OAAO;SACX,GAAG,CAAC,IAAI,CAAC,EAAE;QACV,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,4CAA4C;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QAC1C,OAAO,WAAW,IAAI,CAAC,QAAQ,KAAK,OAAO,GAAG,CAAC;IACjD,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,aAAa,CAAC,OAA0B;IACtD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AAC9C,CAAC;AAED;;;;;GAKG;AACH,SAAgB,SAAS,CAAC,OAA0B;IAClD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AACrD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,OAA0B;IACpD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;QACjB,OAAO,GAAG,KAAK,GAAG,CAAC;IACrB,CAAC;IACD,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,CAAC"}
|
|
@@ -3,6 +3,7 @@ import { ModelConfigKey, ValidatedLLMModelConfig } from '../../shared/config/mod
|
|
|
3
3
|
import { ResponseProcessingOptions } from '../../services/response-processor/types';
|
|
4
4
|
import { BaseAIRequest, BaseAIResult } from '../../shared/types/base-request.types';
|
|
5
5
|
import { ModelParameterOverrides } from '../../services/model-parameter-manager/model-parameter-manager.service';
|
|
6
|
+
import { MultimodalContent } from '../../services/llm/types/multimodal.types';
|
|
6
7
|
/**
|
|
7
8
|
* Base abstract class for all AI use cases
|
|
8
9
|
* Provides common functionality and enforces configuration for each use case
|
|
@@ -86,6 +87,20 @@ export declare abstract class BaseAIUseCase<TPrompt = string, TRequest extends B
|
|
|
86
87
|
* This enforces the message pattern: each use case should have a corresponding message file
|
|
87
88
|
*/
|
|
88
89
|
protected abstract getUserTemplate(): (formattedPrompt: string) => string;
|
|
90
|
+
/**
|
|
91
|
+
* Optional override for multimodal content.
|
|
92
|
+
* When this method returns non-null, the result is sent directly to the provider,
|
|
93
|
+
* bypassing the text-only path (formatUserMessage + getUserTemplate).
|
|
94
|
+
*
|
|
95
|
+
* Override in child classes that need to send images alongside text.
|
|
96
|
+
*
|
|
97
|
+
* @param prompt - The raw prompt data
|
|
98
|
+
* @param request - The full request object
|
|
99
|
+
* @returns MultimodalContent or null (default: null = text-only path)
|
|
100
|
+
*
|
|
101
|
+
* @since 2.22.0
|
|
102
|
+
*/
|
|
103
|
+
protected formatMultimodalContent(prompt: TPrompt, request: TRequest): MultimodalContent | null;
|
|
89
104
|
/**
|
|
90
105
|
* Override model parameters for this specific use case
|
|
91
106
|
* Override this method in child classes to customize parameters
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-ai.usecase.d.ts","sourceRoot":"","sources":["../../../../src/middleware/usecases/base/base-ai.usecase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAkB,cAAc,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAE5G,OAAO,EAAE,yBAAyB,EAAE,MAAM,yCAAyC,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,uCAAuC,CAAC;AAEpF,OAAO,EAAgC,uBAAuB,EAAE,MAAM,wEAAwE,CAAC;
|
|
1
|
+
{"version":3,"file":"base-ai.usecase.d.ts","sourceRoot":"","sources":["../../../../src/middleware/usecases/base/base-ai.usecase.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAkB,cAAc,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAE5G,OAAO,EAAE,yBAAyB,EAAE,MAAM,yCAAyC,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,uCAAuC,CAAC;AAEpF,OAAO,EAAgC,uBAAuB,EAAE,MAAM,wEAAwE,CAAC;AAE/I,OAAO,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AAG9E;;;;;;GAMG;AACH,8BAAsB,aAAa,CACjC,OAAO,GAAG,MAAM,EAChB,QAAQ,SAAS,aAAa,CAAC,OAAO,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,EAChE,OAAO,SAAS,YAAY,GAAG,YAAY;IAE3C;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAElD;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAC,CAAW;IAEnC;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,MAAM;IAItD;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,wBAAwB,EAAE,cAAc,CAAY;IAE9E;;;OAGG;IACH,SAAS,KAAK,cAAc,IAAI,cAAc,CAE7C;IAED;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB;IAI9E;;;OAGG;IACH,SAAS,KAAK,WAAW,IAAI,uBAAuB,CAEnD;IAED;;;OAGG;IACH,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM;IAShD;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,eAAe,EAAE,MAAM,KAAK,MAAM;IAEzE;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,uBAAuB,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,GAAG,iBAAiB,GAAG,IAAI;IAI/F;;;;OAIG;IACH,SAAS,CAAC,qBAAqB,IAAI,uBAAuB;IAI1D;;;;OAIG;IACH,SAAS,CAAC,WAAW,IAAI,WAAW;IAIpC;;;;OAIG;IACU,OAAO,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IA0KzD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,4BAA4B,IAAI,yBAAyB;IAKnE;;;;;OAKG;cACa,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAKrG;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAC7B,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO;CACX"}
|