@vellumai/assistant 0.11.2-staging.2 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openapi.yaml +5 -2
- package/package.json +1 -1
- package/src/__tests__/compactor-truncation-pairing.test.ts +368 -0
- package/src/__tests__/history-repair.test.ts +39 -0
- package/src/agent/history-repair/history-repair.ts +8 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +1 -1
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -1
- package/src/config/bundled-skills/media-processing/__tests__/media-analysis-default-model.test.ts +177 -0
- package/src/config/bundled-skills/media-processing/services/gemini-map.ts +4 -3
- package/src/config/bundled-skills/media-processing/services/gemini-video.ts +2 -1
- package/src/config/bundled-skills/media-processing/services/media-analysis-model.ts +12 -0
- package/src/context/compactor.ts +41 -4
- package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts +478 -0
- package/src/providers/openai/chat-completions-provider.ts +46 -30
- package/src/providers/openai/orphaned-tool-result.ts +79 -0
- package/src/providers/openai/responses-provider.ts +50 -24
- package/src/runtime/routes/secret-routes.ts +8 -2
package/openapi.yaml
CHANGED
|
@@ -27286,10 +27286,13 @@ paths:
|
|
|
27286
27286
|
type: string
|
|
27287
27287
|
name:
|
|
27288
27288
|
type: string
|
|
27289
|
+
error:
|
|
27290
|
+
description:
|
|
27291
|
+
Why the secret was not stored (e.g. provider-side API key validation failed). Present only when success is
|
|
27292
|
+
false.
|
|
27293
|
+
type: string
|
|
27289
27294
|
required:
|
|
27290
27295
|
- success
|
|
27291
|
-
- type
|
|
27292
|
-
- name
|
|
27293
27296
|
additionalProperties: false
|
|
27294
27297
|
/v1/secrets/read:
|
|
27295
27298
|
post:
|
package/package.json
CHANGED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The compaction summary call's front truncation must never split a
|
|
3
|
+
* tool_use/tool_result pair, and the outbound request must be repaired
|
|
4
|
+
* before the provider call.
|
|
5
|
+
*
|
|
6
|
+
* The token-budget drop loop advances one message at a time, so without a
|
|
7
|
+
* boundary check its cut can drop an assistant `tool_use` while keeping its
|
|
8
|
+
* user `tool_result` in the retained portion. Providers that validate
|
|
9
|
+
* pairing (OpenAI Responses: "No tool call found for function call output
|
|
10
|
+
* with call_id ...") reject such a request, and the retry ladder reproduces
|
|
11
|
+
* the same rejection every pass. The cut must advance to a pair-safe user
|
|
12
|
+
* boundary, and `buildCompactionRequest` runs the deterministic history
|
|
13
|
+
* repair so even a malformed input history reaches the provider valid.
|
|
14
|
+
*/
|
|
15
|
+
import { describe, expect, mock, test } from "bun:test";
|
|
16
|
+
|
|
17
|
+
mock.module("../persistence/conversation-crud.js", () => ({
|
|
18
|
+
setConversationProcessingStartedAt: () => {},
|
|
19
|
+
isConversationProcessing: () => false,
|
|
20
|
+
getMessages: () => [],
|
|
21
|
+
}));
|
|
22
|
+
|
|
23
|
+
mock.module("../persistence/attachments-store.js", () => ({
|
|
24
|
+
getAttachmentMetadataForMessage: () => [],
|
|
25
|
+
getAttachmentContent: () => null,
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
mock.module("../persistence/llm-request-log-store.js", () => ({
|
|
29
|
+
recordRequestLog: () => {},
|
|
30
|
+
}));
|
|
31
|
+
|
|
32
|
+
import { runAssistantDrivenCompaction } from "../context/compactor.js";
|
|
33
|
+
import { estimatePromptTokens } from "../context/token-estimator.js";
|
|
34
|
+
import type { ContentBlock, Message, Provider } from "../providers/types.js";
|
|
35
|
+
|
|
36
|
+
const SUMMARY = "Earlier turns summarized in the assistant's own voice.";
|
|
37
|
+
|
|
38
|
+
function turnTimestamp(turn: number): string {
|
|
39
|
+
const hour = String(10 + Math.floor(turn / 60)).padStart(2, "0");
|
|
40
|
+
const minute = String(turn % 60).padStart(2, "0");
|
|
41
|
+
return `2026-05-21 (Thursday) ${hour}:${minute}:00 -05:00 (America/Chicago)`;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function userTurn(turn: number, body: string): Message {
|
|
45
|
+
return {
|
|
46
|
+
role: "user",
|
|
47
|
+
content: [
|
|
48
|
+
{
|
|
49
|
+
type: "text",
|
|
50
|
+
text: `<turn_context>\ncurrent_time: ${turnTimestamp(
|
|
51
|
+
turn,
|
|
52
|
+
)}\n</turn_context>\n[U${turn}] ${body}`,
|
|
53
|
+
},
|
|
54
|
+
],
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function compactionResponse(tailTurn: number, preview: string): string {
|
|
59
|
+
return `<compaction_result>
|
|
60
|
+
<summary>
|
|
61
|
+
${SUMMARY}
|
|
62
|
+
</summary>
|
|
63
|
+
<key_state>
|
|
64
|
+
- Nothing critical pending.
|
|
65
|
+
</key_state>
|
|
66
|
+
<tail_start timestamp="${turnTimestamp(tailTurn)}" preview="${preview}" />
|
|
67
|
+
</compaction_result>`;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Fake provider enforcing the same pairing contract the OpenAI Responses
|
|
72
|
+
* serialization is subject to: a `tool_result` may only reference a
|
|
73
|
+
* `tool_use` emitted earlier in the same request (a `function_call_output`
|
|
74
|
+
* with no preceding matching `function_call` is a 400). Rejects with the
|
|
75
|
+
* provider's live error phrasing so a regression reproduces the real
|
|
76
|
+
* failure mode.
|
|
77
|
+
*/
|
|
78
|
+
function makeValidatingProvider(response: string): {
|
|
79
|
+
provider: Provider;
|
|
80
|
+
lastRequest: () => Message[] | null;
|
|
81
|
+
} {
|
|
82
|
+
let captured: Message[] | null = null;
|
|
83
|
+
const provider: Provider = {
|
|
84
|
+
name: "mock-provider",
|
|
85
|
+
sendMessage: async (messages: Message[]) => {
|
|
86
|
+
const emittedToolUseIds = new Set<string>();
|
|
87
|
+
for (const msg of messages) {
|
|
88
|
+
for (const block of msg.content) {
|
|
89
|
+
if (msg.role === "assistant" && block.type === "tool_use") {
|
|
90
|
+
emittedToolUseIds.add(block.id);
|
|
91
|
+
}
|
|
92
|
+
if (block.type === "tool_result") {
|
|
93
|
+
// guard:allow-tool-result-only: the fake validates client-side pairing only
|
|
94
|
+
const toolUseId = (block as { tool_use_id: string }).tool_use_id;
|
|
95
|
+
if (!emittedToolUseIds.has(toolUseId)) {
|
|
96
|
+
throw new Error(
|
|
97
|
+
`No tool call found for function call output with call_id ${toolUseId}.`,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
captured = messages;
|
|
104
|
+
return {
|
|
105
|
+
content: [{ type: "text", text: response }],
|
|
106
|
+
model: "mock-model",
|
|
107
|
+
usage: { inputTokens: 100, outputTokens: 50 },
|
|
108
|
+
stopReason: "end_turn",
|
|
109
|
+
};
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
return { provider, lastRequest: () => captured };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function estimate(messages: Message[]): number {
|
|
116
|
+
return estimatePromptTokens(messages, "system", {
|
|
117
|
+
providerName: "mock-provider",
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Mirror of the compactor's pair-safe cut predicate for fixture guards. */
|
|
122
|
+
function isCleanUserBoundary(message: Message | undefined): boolean {
|
|
123
|
+
return (
|
|
124
|
+
message != null &&
|
|
125
|
+
message.role === "user" &&
|
|
126
|
+
// guard:allow-tool-result-only: mirrors the compactor's boundary predicate
|
|
127
|
+
!message.content.some((block) => block.type === "tool_result")
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Replay the token-budget drop loop without the pairing advance, to locate
|
|
133
|
+
* where the raw budget cut lands for a fixture.
|
|
134
|
+
*/
|
|
135
|
+
function naiveDropCount(messages: Message[], budgetTokens: number): number {
|
|
136
|
+
let dropCount = 0;
|
|
137
|
+
let estimated = estimate(messages);
|
|
138
|
+
while (estimated > budgetTokens && dropCount < messages.length - 1) {
|
|
139
|
+
dropCount++;
|
|
140
|
+
estimated = estimate(messages.slice(dropCount));
|
|
141
|
+
}
|
|
142
|
+
return dropCount;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function textOfMessage(message: Message | undefined): string {
|
|
146
|
+
return (message?.content ?? [])
|
|
147
|
+
.map((block) => ("text" in block ? (block.text as string) : ""))
|
|
148
|
+
.join("\n");
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function requestText(messages: Message[]): string {
|
|
152
|
+
return messages.map(textOfMessage).join("\n");
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
describe("compaction summary call: pair-safe front truncation", () => {
|
|
156
|
+
// Rounds of [user text, assistant tool_use (heavy), user tool_result,
|
|
157
|
+
// assistant text]: the heavy tool_use messages dominate the estimate, so
|
|
158
|
+
// the raw budget loop settles right after dropping one of them, landing
|
|
159
|
+
// the cut on its orphaned tool_result (or the assistant text behind it),
|
|
160
|
+
// never on a clean user boundary.
|
|
161
|
+
function buildToolHeavyHistory(rounds: number, idPrefix: string): Message[] {
|
|
162
|
+
const messages: Message[] = [];
|
|
163
|
+
for (let i = 0; i < rounds; i++) {
|
|
164
|
+
messages.push(userTurn(i, "please inspect the next data batch"));
|
|
165
|
+
messages.push({
|
|
166
|
+
role: "assistant",
|
|
167
|
+
content: [
|
|
168
|
+
{
|
|
169
|
+
type: "tool_use",
|
|
170
|
+
id: `${idPrefix}_${i}`,
|
|
171
|
+
name: "inspect_batch",
|
|
172
|
+
input: { payload: `batch ${i} `.repeat(600) },
|
|
173
|
+
},
|
|
174
|
+
],
|
|
175
|
+
});
|
|
176
|
+
messages.push({
|
|
177
|
+
role: "user",
|
|
178
|
+
content: [
|
|
179
|
+
{
|
|
180
|
+
type: "tool_result",
|
|
181
|
+
tool_use_id: `${idPrefix}_${i}`,
|
|
182
|
+
content: `inspection ${i} complete. `.repeat(8),
|
|
183
|
+
},
|
|
184
|
+
],
|
|
185
|
+
});
|
|
186
|
+
messages.push({
|
|
187
|
+
role: "assistant",
|
|
188
|
+
content: [
|
|
189
|
+
{ type: "text", text: `[A${i}] batch ${i} looks consistent.` },
|
|
190
|
+
],
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return messages;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** tool_result blocks in `messages` with no tool_use anywhere in `messages`. */
|
|
197
|
+
function orphanedResultIds(messages: Message[]): string[] {
|
|
198
|
+
const toolUseIds = new Set<string>();
|
|
199
|
+
for (const msg of messages) {
|
|
200
|
+
for (const block of msg.content) {
|
|
201
|
+
if (msg.role === "assistant" && block.type === "tool_use") {
|
|
202
|
+
toolUseIds.add(block.id);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
const orphans: string[] = [];
|
|
207
|
+
for (const msg of messages) {
|
|
208
|
+
for (const block of msg.content) {
|
|
209
|
+
if (block.type === "tool_result") {
|
|
210
|
+
// guard:allow-tool-result-only: counting client-side orphans
|
|
211
|
+
const toolUseId = (block as { tool_use_id: string }).tool_use_id;
|
|
212
|
+
if (!toolUseIds.has(toolUseId)) {
|
|
213
|
+
orphans.push(toolUseId);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return orphans;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Pairing must be id-format-agnostic, so the fixture runs across both
|
|
222
|
+
// persisted tool id shapes: Anthropic-style `toolu_` ids and OpenAI
|
|
223
|
+
// Responses-native `call_` ids.
|
|
224
|
+
for (const idPrefix of ["toolu", "call"] as const) {
|
|
225
|
+
test(`advances the cut to a pair-safe boundary and succeeds against a pairing-validating provider (${idPrefix}_ ids)`, async () => {
|
|
226
|
+
const rounds = 30;
|
|
227
|
+
const messages = buildToolHeavyHistory(rounds, idPrefix);
|
|
228
|
+
|
|
229
|
+
const maxInputTokens = 20_000;
|
|
230
|
+
// Mirrors compactor.compactionPrefixBudget: window minus the
|
|
231
|
+
// instruction reserve (800) and the 15% output reserve.
|
|
232
|
+
const prefixBudget =
|
|
233
|
+
maxInputTokens - 800 - Math.floor(maxInputTokens * 0.15);
|
|
234
|
+
expect(estimate(messages)).toBeGreaterThan(prefixBudget);
|
|
235
|
+
|
|
236
|
+
// Fixture guard: without the pairing advance, the budget cut lands on
|
|
237
|
+
// an unsafe index and the retained tail carries a tool_result whose
|
|
238
|
+
// tool_use sits in the dropped prefix (the exact shape the provider
|
|
239
|
+
// rejects). If token-estimation changes ever make this land safely,
|
|
240
|
+
// the fixture must be re-tuned or this test proves nothing.
|
|
241
|
+
const rawCut = naiveDropCount(messages, prefixBudget);
|
|
242
|
+
expect(rawCut).toBeGreaterThan(0);
|
|
243
|
+
expect(isCleanUserBoundary(messages[rawCut])).toBe(false);
|
|
244
|
+
expect(orphanedResultIds(messages.slice(rawCut)).length).toBeGreaterThan(
|
|
245
|
+
0,
|
|
246
|
+
);
|
|
247
|
+
|
|
248
|
+
const { provider, lastRequest } = makeValidatingProvider(
|
|
249
|
+
compactionResponse(rounds - 2, "please inspect the next"),
|
|
250
|
+
);
|
|
251
|
+
|
|
252
|
+
const result = await runAssistantDrivenCompaction({
|
|
253
|
+
conversationId: "conv-test",
|
|
254
|
+
messages,
|
|
255
|
+
provider,
|
|
256
|
+
systemPrompt: "system",
|
|
257
|
+
compaction: { enabled: true, autoThreshold: 0.7 },
|
|
258
|
+
maxInputTokens,
|
|
259
|
+
force: true,
|
|
260
|
+
previousEstimatedInputTokens: 90_000,
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
// The pairing-validating provider accepted the request and the pass
|
|
264
|
+
// applied.
|
|
265
|
+
expect(result.compacted).toBe(true);
|
|
266
|
+
|
|
267
|
+
const sent = lastRequest();
|
|
268
|
+
expect(sent).not.toBeNull();
|
|
269
|
+
const sentMessages = sent ?? [];
|
|
270
|
+
|
|
271
|
+
// Every tool_result in the outbound request is preceded by its
|
|
272
|
+
// tool_use.
|
|
273
|
+
expect(orphanedResultIds(sentMessages)).toEqual([]);
|
|
274
|
+
|
|
275
|
+
// The cut itself was pair-safe: nothing needed the repair pass's
|
|
276
|
+
// orphan downgrade, and the retained pairs survive intact.
|
|
277
|
+
const sentText = requestText(sentMessages);
|
|
278
|
+
expect(sentText).not.toContain("[orphaned");
|
|
279
|
+
|
|
280
|
+
// The request still fits the window and announces the truncation.
|
|
281
|
+
expect(estimate(sentMessages)).toBeLessThan(maxInputTokens);
|
|
282
|
+
expect(sentText).toContain("summary covers only the visible portion");
|
|
283
|
+
|
|
284
|
+
// Recent valid content is retained verbatim: the last round's user
|
|
285
|
+
// turn, tool pair, and assistant reply all reach the provider.
|
|
286
|
+
const lastRound = rounds - 1;
|
|
287
|
+
expect(sentText).toContain(`[U${lastRound}] please inspect`);
|
|
288
|
+
expect(sentText).toContain(`[A${lastRound}] batch ${lastRound}`);
|
|
289
|
+
const sentToolUseIds = new Set<string>();
|
|
290
|
+
for (const msg of sentMessages) {
|
|
291
|
+
for (const block of msg.content) {
|
|
292
|
+
if (msg.role === "assistant" && block.type === "tool_use") {
|
|
293
|
+
sentToolUseIds.add(block.id);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
expect(sentToolUseIds.has(`${idPrefix}_${lastRound}`)).toBe(true);
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
test("repairs a malformed below-budget history before the provider call", async () => {
|
|
302
|
+
// Orphan tool_result (its tool_use exists nowhere in the history) and a
|
|
303
|
+
// consecutive same-role user run, below the truncation budget so the
|
|
304
|
+
// request-build repair is the only transform in play.
|
|
305
|
+
const messages: Message[] = [
|
|
306
|
+
userTurn(0, "start the job"),
|
|
307
|
+
{
|
|
308
|
+
role: "user",
|
|
309
|
+
content: [
|
|
310
|
+
{
|
|
311
|
+
type: "tool_result",
|
|
312
|
+
tool_use_id: "toolu_missing",
|
|
313
|
+
content: "stale result payload",
|
|
314
|
+
},
|
|
315
|
+
],
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
role: "assistant",
|
|
319
|
+
content: [{ type: "text", text: "[A0] acknowledged." }],
|
|
320
|
+
},
|
|
321
|
+
userTurn(1, "carry on with the job"),
|
|
322
|
+
{
|
|
323
|
+
role: "assistant",
|
|
324
|
+
content: [{ type: "text", text: "[A1] done." }],
|
|
325
|
+
},
|
|
326
|
+
];
|
|
327
|
+
|
|
328
|
+
const { provider, lastRequest } = makeValidatingProvider(
|
|
329
|
+
compactionResponse(1, "carry on with the job"),
|
|
330
|
+
);
|
|
331
|
+
|
|
332
|
+
const result = await runAssistantDrivenCompaction({
|
|
333
|
+
conversationId: "conv-test",
|
|
334
|
+
messages,
|
|
335
|
+
provider,
|
|
336
|
+
systemPrompt: "system",
|
|
337
|
+
compaction: { enabled: true, autoThreshold: 0.7 },
|
|
338
|
+
maxInputTokens: 200_000,
|
|
339
|
+
force: true,
|
|
340
|
+
previousEstimatedInputTokens: 90_000,
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
expect(result.compacted).toBe(true);
|
|
344
|
+
|
|
345
|
+
const sentMessages = lastRequest() ?? [];
|
|
346
|
+
// The orphan was downgraded to text: no tool_result blocks reach the
|
|
347
|
+
// provider, and the payload is preserved in the degraded form.
|
|
348
|
+
const hasToolResult = sentMessages.some((msg) =>
|
|
349
|
+
// guard:allow-tool-result-only: asserting the repair removed them
|
|
350
|
+
msg.content.some((block: ContentBlock) => block.type === "tool_result"),
|
|
351
|
+
);
|
|
352
|
+
expect(hasToolResult).toBe(false);
|
|
353
|
+
const sentText = requestText(sentMessages);
|
|
354
|
+
expect(sentText).toContain("stale result payload");
|
|
355
|
+
expect(sentText).toContain("orphaned tool_result");
|
|
356
|
+
|
|
357
|
+
// Same-role runs were merged: roles strictly alternate in the history
|
|
358
|
+
// portion of the request (everything before the trailing instruction).
|
|
359
|
+
const historyPortion = sentMessages.slice(0, -1);
|
|
360
|
+
for (let i = 1; i < historyPortion.length; i++) {
|
|
361
|
+
expect(historyPortion[i].role).not.toBe(historyPortion[i - 1].role);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// Valid user text survives the repair verbatim.
|
|
365
|
+
expect(sentText).toContain("[U0] start the job");
|
|
366
|
+
expect(sentText).toContain("[U1] carry on with the job");
|
|
367
|
+
});
|
|
368
|
+
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
|
|
3
3
|
import { deepRepairHistory } from "../agent/history-repair/history-repair.js";
|
|
4
|
+
import { isRepairableOrderingError } from "../agent/history-repair/history-repair.js";
|
|
4
5
|
import { repairHistory } from "../agent/history-repair/history-repair.js";
|
|
5
6
|
import type { Message } from "../providers/types.js";
|
|
6
7
|
|
|
@@ -1110,3 +1111,41 @@ describe("deepRepairHistory", () => {
|
|
|
1110
1111
|
expect(stats.orphanToolResultsDowngraded).toBe(0);
|
|
1111
1112
|
});
|
|
1112
1113
|
});
|
|
1114
|
+
|
|
1115
|
+
describe("isRepairableOrderingError", () => {
|
|
1116
|
+
test("matches Anthropic tool ordering rejections", () => {
|
|
1117
|
+
expect(
|
|
1118
|
+
isRepairableOrderingError(
|
|
1119
|
+
"Requests which include `tool_use` blocks must have a corresponding `tool_result` block in the next message.",
|
|
1120
|
+
),
|
|
1121
|
+
).toBe(true);
|
|
1122
|
+
expect(
|
|
1123
|
+
isRepairableOrderingError(
|
|
1124
|
+
"messages.5: tool_result block references tool_use_id toolu_abc which was not found",
|
|
1125
|
+
),
|
|
1126
|
+
).toBe(true);
|
|
1127
|
+
});
|
|
1128
|
+
|
|
1129
|
+
test("matches the OpenAI Responses orphan function_call_output rejection", () => {
|
|
1130
|
+
expect(
|
|
1131
|
+
isRepairableOrderingError(
|
|
1132
|
+
"No tool call found for function call output with call_id call_abc123.",
|
|
1133
|
+
),
|
|
1134
|
+
).toBe(true);
|
|
1135
|
+
});
|
|
1136
|
+
|
|
1137
|
+
test("matches the OpenAI Chat Completions orphan tool_call_id rejection", () => {
|
|
1138
|
+
expect(
|
|
1139
|
+
isRepairableOrderingError(
|
|
1140
|
+
"Invalid parameter: 'tool_call_id' of 'call_abc123' not found in 'tool_calls' of previous message.",
|
|
1141
|
+
),
|
|
1142
|
+
).toBe(true);
|
|
1143
|
+
});
|
|
1144
|
+
|
|
1145
|
+
test("does not match unrelated provider errors", () => {
|
|
1146
|
+
expect(isRepairableOrderingError("Rate limit exceeded")).toBe(false);
|
|
1147
|
+
expect(
|
|
1148
|
+
isRepairableOrderingError("Request too large for model context window"),
|
|
1149
|
+
).toBe(false);
|
|
1150
|
+
});
|
|
1151
|
+
});
|
|
@@ -399,6 +399,14 @@ export const ORDERING_ERROR_PATTERNS: readonly RegExp[] = [
|
|
|
399
399
|
/tool_use_id.*without.*tool_result/i,
|
|
400
400
|
/tool_result.*tool_use_id.*not found/i,
|
|
401
401
|
/messages.*invalid.*order/i,
|
|
402
|
+
// OpenAI Responses API: a function_call_output whose call_id has no
|
|
403
|
+
// matching function_call earlier in the request ("No tool call found for
|
|
404
|
+
// function call output with call_id ...").
|
|
405
|
+
/no tool call found for function call output/i,
|
|
406
|
+
// OpenAI Chat Completions API: a tool message whose tool_call_id is not in
|
|
407
|
+
// a preceding assistant message's tool_calls ("Invalid parameter:
|
|
408
|
+
// 'tool_call_id' of '...' not found in 'tool_calls' of previous message").
|
|
409
|
+
/tool_call_id.*not found/i,
|
|
402
410
|
];
|
|
403
411
|
|
|
404
412
|
/**
|
|
@@ -72,7 +72,7 @@ Parameters:
|
|
|
72
72
|
- `output_schema` (required) - JSON Schema for structured output.
|
|
73
73
|
- `mode` - Analysis mode: `'keyframes'` (default) or `'direct_video'`.
|
|
74
74
|
- `context` - Additional context to include in the prompt.
|
|
75
|
-
- `model` - Gemini model to use (
|
|
75
|
+
- `model` - Gemini model to use (defaults to the Gemini provider's recommended vision model).
|
|
76
76
|
- `concurrency` - Maximum concurrent API requests (default: 10, keyframes mode only).
|
|
77
77
|
- `max_retries` - Retry attempts per segment on failure (default: 3).
|
|
78
78
|
|
|
@@ -130,7 +130,7 @@
|
|
|
130
130
|
},
|
|
131
131
|
"model": {
|
|
132
132
|
"type": "string",
|
|
133
|
-
"description": "Gemini model to use.
|
|
133
|
+
"description": "Gemini model to use. Defaults to the Gemini provider's recommended vision model."
|
|
134
134
|
},
|
|
135
135
|
"concurrency": {
|
|
136
136
|
"type": "number",
|
package/src/config/bundled-skills/media-processing/__tests__/media-analysis-default-model.test.ts
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { mkdtemp, writeFile } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
// Mock @google/genai module: must be before importing the services
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
let capturedModels: string[] = [];
|
|
11
|
+
|
|
12
|
+
mock.module("@google/genai", () => ({
|
|
13
|
+
GoogleGenAI: class MockGoogleGenAI {
|
|
14
|
+
constructor(_opts: Record<string, unknown>) {}
|
|
15
|
+
models = {
|
|
16
|
+
generateContent: async (params: { model: string }) => {
|
|
17
|
+
capturedModels.push(params.model);
|
|
18
|
+
return {
|
|
19
|
+
text: '{"frames":[]}',
|
|
20
|
+
usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5 },
|
|
21
|
+
modelVersion: params.model,
|
|
22
|
+
};
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
files = {
|
|
26
|
+
upload: async (_params: Record<string, unknown>) => ({
|
|
27
|
+
name: "files/test-upload",
|
|
28
|
+
uri: "https://example.com/files/test-upload",
|
|
29
|
+
mimeType: "video/mp4",
|
|
30
|
+
}),
|
|
31
|
+
get: async (_params: Record<string, unknown>) => ({ state: "ACTIVE" }),
|
|
32
|
+
delete: async (_params: Record<string, unknown>) => ({}),
|
|
33
|
+
};
|
|
34
|
+
},
|
|
35
|
+
ApiError: class MockApiError extends Error {
|
|
36
|
+
status: number;
|
|
37
|
+
constructor(status: number, message: string) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.status = status;
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
}));
|
|
43
|
+
|
|
44
|
+
// Import after mocking
|
|
45
|
+
import { resolveModelIntent } from "../../../../providers/model-intents.js";
|
|
46
|
+
import { mapSegments } from "../services/gemini-map.js";
|
|
47
|
+
import { analyzeVideoDirectly } from "../services/gemini-video.js";
|
|
48
|
+
import { resolveMediaAnalysisModel } from "../services/media-analysis-model.js";
|
|
49
|
+
import type { Segment } from "../services/preprocess.js";
|
|
50
|
+
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// Helpers
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
const EXPECTED_DEFAULT = resolveModelIntent("gemini", "vision-optimized");
|
|
56
|
+
|
|
57
|
+
const OUTPUT_SCHEMA = { type: "object", properties: {} };
|
|
58
|
+
|
|
59
|
+
async function createPipelineDir(): Promise<{
|
|
60
|
+
pipelineDir: string;
|
|
61
|
+
segments: Segment[];
|
|
62
|
+
}> {
|
|
63
|
+
const pipelineDir = await mkdtemp(join(tmpdir(), "media-model-test-"));
|
|
64
|
+
const framePath = join(pipelineDir, "frame-000.jpg");
|
|
65
|
+
await writeFile(framePath, Buffer.from("fake-jpeg-bytes"));
|
|
66
|
+
const segments: Segment[] = [
|
|
67
|
+
{
|
|
68
|
+
id: "seg-000",
|
|
69
|
+
startSeconds: 0,
|
|
70
|
+
endSeconds: 10,
|
|
71
|
+
framePaths: [framePath],
|
|
72
|
+
frameTimestamps: [0],
|
|
73
|
+
},
|
|
74
|
+
];
|
|
75
|
+
return { pipelineDir, segments };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function mapOptions(model?: string) {
|
|
79
|
+
return {
|
|
80
|
+
apiKey: "test-key",
|
|
81
|
+
systemPrompt: "Describe the frames.",
|
|
82
|
+
outputSchema: OUTPUT_SCHEMA,
|
|
83
|
+
model,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
beforeEach(() => {
|
|
88
|
+
capturedModels = [];
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
// Automatic no-override path (the background media-processing job passes no
|
|
93
|
+
// model, so the services' default is what runs in production)
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
describe("media analysis default model (no-override path)", () => {
|
|
97
|
+
test("resolveMediaAnalysisModel defaults to the vision-optimized catalog intent", () => {
|
|
98
|
+
expect(resolveMediaAnalysisModel(undefined)).toBe(EXPECTED_DEFAULT);
|
|
99
|
+
// Regression (LUM-3038): the default must not be the retired pinned model.
|
|
100
|
+
expect(resolveMediaAnalysisModel(undefined)).not.toBe("gemini-2.5-flash");
|
|
101
|
+
expect(resolveMediaAnalysisModel("custom-model")).toBe("custom-model");
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("keyframe mapping without a model override executes the resolved default", async () => {
|
|
105
|
+
const { pipelineDir, segments } = await createPipelineDir();
|
|
106
|
+
|
|
107
|
+
const output = await mapSegments(
|
|
108
|
+
"asset-1",
|
|
109
|
+
pipelineDir,
|
|
110
|
+
segments,
|
|
111
|
+
mapOptions(),
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
expect(capturedModels).toEqual([EXPECTED_DEFAULT]);
|
|
115
|
+
expect(output.model).toBe(EXPECTED_DEFAULT);
|
|
116
|
+
expect(output.successCount).toBe(1);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("cache identity agrees with the execution default", async () => {
|
|
120
|
+
const { pipelineDir, segments } = await createPipelineDir();
|
|
121
|
+
|
|
122
|
+
await mapSegments("asset-1", pipelineDir, segments, mapOptions());
|
|
123
|
+
expect(capturedModels).toHaveLength(1);
|
|
124
|
+
|
|
125
|
+
// A rerun that pins the same model the default resolved to must hit the
|
|
126
|
+
// per-segment cache: the config hash and the executed model agree.
|
|
127
|
+
const rerun = await mapSegments(
|
|
128
|
+
"asset-1",
|
|
129
|
+
pipelineDir,
|
|
130
|
+
segments,
|
|
131
|
+
mapOptions(EXPECTED_DEFAULT),
|
|
132
|
+
);
|
|
133
|
+
expect(capturedModels).toHaveLength(1);
|
|
134
|
+
expect(rerun.successCount).toBe(1);
|
|
135
|
+
|
|
136
|
+
// A different model must miss the cache (hash covers the model).
|
|
137
|
+
await mapSegments(
|
|
138
|
+
"asset-1",
|
|
139
|
+
pipelineDir,
|
|
140
|
+
segments,
|
|
141
|
+
mapOptions("custom-model"),
|
|
142
|
+
);
|
|
143
|
+
expect(capturedModels).toEqual([EXPECTED_DEFAULT, "custom-model"]);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("direct video analysis without a model override executes the resolved default", async () => {
|
|
147
|
+
const { pipelineDir } = await createPipelineDir();
|
|
148
|
+
const videoPath = join(pipelineDir, "video.mp4");
|
|
149
|
+
await writeFile(videoPath, Buffer.from("fake-mp4-bytes"));
|
|
150
|
+
|
|
151
|
+
const output = await analyzeVideoDirectly(
|
|
152
|
+
"asset-1",
|
|
153
|
+
pipelineDir,
|
|
154
|
+
mapOptions(),
|
|
155
|
+
videoPath,
|
|
156
|
+
12,
|
|
157
|
+
"video/mp4",
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
expect(capturedModels).toEqual([EXPECTED_DEFAULT]);
|
|
161
|
+
expect(output.model).toBe(EXPECTED_DEFAULT);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("an explicit model override wins over the default", async () => {
|
|
165
|
+
const { pipelineDir, segments } = await createPipelineDir();
|
|
166
|
+
|
|
167
|
+
const output = await mapSegments(
|
|
168
|
+
"asset-1",
|
|
169
|
+
pipelineDir,
|
|
170
|
+
segments,
|
|
171
|
+
mapOptions("gemini-explicit-override"),
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
expect(capturedModels).toEqual(["gemini-explicit-override"]);
|
|
175
|
+
expect(output.model).toBe("gemini-explicit-override");
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -15,6 +15,7 @@ import { ApiError, GoogleGenAI } from "@google/genai";
|
|
|
15
15
|
import { computeRetryDelay, sleep } from "../../../../util/retry.js";
|
|
16
16
|
import { ConcurrencyPool } from "./concurrency-pool.js";
|
|
17
17
|
import { type CostSummary, CostTracker } from "./cost-tracker.js";
|
|
18
|
+
import { resolveMediaAnalysisModel } from "./media-analysis-model.js";
|
|
18
19
|
import type { Segment } from "./preprocess.js";
|
|
19
20
|
|
|
20
21
|
// ---------------------------------------------------------------------------
|
|
@@ -92,7 +93,7 @@ function computeConfigHash(options: GeminiMapOptions): string {
|
|
|
92
93
|
const payload = JSON.stringify({
|
|
93
94
|
systemPrompt: options.systemPrompt,
|
|
94
95
|
outputSchema: options.outputSchema,
|
|
95
|
-
model: options.model
|
|
96
|
+
model: resolveMediaAnalysisModel(options.model),
|
|
96
97
|
context: options.context,
|
|
97
98
|
});
|
|
98
99
|
return createHash("sha256").update(payload).digest("hex").slice(0, 8);
|
|
@@ -137,7 +138,7 @@ async function processSegment(
|
|
|
137
138
|
|
|
138
139
|
parts.push({ text: promptText });
|
|
139
140
|
|
|
140
|
-
const model = options.model
|
|
141
|
+
const model = resolveMediaAnalysisModel(options.model);
|
|
141
142
|
|
|
142
143
|
const response = await client.models.generateContent({
|
|
143
144
|
model,
|
|
@@ -284,7 +285,7 @@ export async function mapSegments(
|
|
|
284
285
|
options: GeminiMapOptions,
|
|
285
286
|
onProgress?: (msg: string) => void,
|
|
286
287
|
): Promise<MapOutput> {
|
|
287
|
-
const model = options.model
|
|
288
|
+
const model = resolveMediaAnalysisModel(options.model);
|
|
288
289
|
const concurrency = options.concurrency ?? 10;
|
|
289
290
|
const maxRetries = options.maxRetries ?? 3;
|
|
290
291
|
|