@terreno/ai 57.7.0 → 57.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/models/gptHistory.js +9 -1
- package/dist/routes/gpt.js +15 -14
- package/dist/service/aiService.js +4 -0
- package/package.json +4 -4
- package/src/models/gptHistory.test.ts +26 -0
- package/src/models/gptHistory.ts +9 -1
- package/src/routes/gpt.ts +15 -18
- package/src/routes/routes.test.ts +58 -0
- package/src/service/aiService.test.ts +18 -0
- package/src/service/aiService.ts +3 -0
|
@@ -28,7 +28,15 @@ const gptHistoryPromptSchema = new mongoose_1.default.Schema({
|
|
|
28
28
|
type: String,
|
|
29
29
|
},
|
|
30
30
|
result: { description: "Result returned from a tool call", type: mongoose_1.default.Schema.Types.Mixed },
|
|
31
|
-
text: {
|
|
31
|
+
text: {
|
|
32
|
+
default: "",
|
|
33
|
+
description: "Text content of the prompt or response",
|
|
34
|
+
// Image-only responses carry their payload in content, not text
|
|
35
|
+
required: function () {
|
|
36
|
+
return !this.content?.length;
|
|
37
|
+
},
|
|
38
|
+
type: String,
|
|
39
|
+
},
|
|
32
40
|
toolCallId: {
|
|
33
41
|
description: "Identifier linking a tool result to its originating call",
|
|
34
42
|
type: String,
|
package/dist/routes/gpt.js
CHANGED
|
@@ -260,6 +260,15 @@ const addGptRoutes = (router, options) => {
|
|
|
260
260
|
sseStarted = true;
|
|
261
261
|
let fullResponse = "";
|
|
262
262
|
const generatedImages = [];
|
|
263
|
+
// Stream file parts and result.files report the same images; emit each data URL once.
|
|
264
|
+
const sendGeneratedImage = (file) => {
|
|
265
|
+
const url = `data:${file.mediaType};base64,${file.base64}`;
|
|
266
|
+
if (generatedImages.some((img) => img.url === url)) {
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
generatedImages.push({ mimeType: file.mediaType, url });
|
|
270
|
+
res.write(`data: ${JSON.stringify({ image: { mimeType: file.mediaType, url } })}\n\n`);
|
|
271
|
+
};
|
|
263
272
|
const startTime = luxon_1.DateTime.now().toMillis();
|
|
264
273
|
try {
|
|
265
274
|
api_1.logger.debug("Starting streamText", { model: modelId, supportsTools });
|
|
@@ -336,11 +345,7 @@ const addGptRoutes = (router, options) => {
|
|
|
336
345
|
}
|
|
337
346
|
if (part.type === "file") {
|
|
338
347
|
if (isGeneratedImageFile(part.file)) {
|
|
339
|
-
|
|
340
|
-
generatedImages.push({ mimeType: part.file.mediaType, url: dataUrl });
|
|
341
|
-
res.write(`data: ${JSON.stringify({
|
|
342
|
-
image: { mimeType: part.file.mediaType, url: dataUrl },
|
|
343
|
-
})}\n\n`);
|
|
348
|
+
sendGeneratedImage(part.file);
|
|
344
349
|
api_1.logger.debug("Sent inline image from stream");
|
|
345
350
|
}
|
|
346
351
|
continue;
|
|
@@ -413,20 +418,16 @@ const addGptRoutes = (router, options) => {
|
|
|
413
418
|
}
|
|
414
419
|
}
|
|
415
420
|
api_1.logger.debug("Stream completed", { fullResponseLength: fullResponse.length, partCount });
|
|
416
|
-
//
|
|
421
|
+
// Catch generated images the stream did not report as file parts
|
|
417
422
|
try {
|
|
418
423
|
const files = await result.files;
|
|
419
424
|
if (files && files.length > 0) {
|
|
420
425
|
for (const file of files) {
|
|
421
|
-
if (file
|
|
422
|
-
|
|
423
|
-
generatedImages.push({ mimeType: file.mediaType, url: dataUrl });
|
|
424
|
-
res.write(`data: ${JSON.stringify({
|
|
425
|
-
image: { mimeType: file.mediaType, url: dataUrl },
|
|
426
|
-
})}\n\n`);
|
|
426
|
+
if (isGeneratedImageFile(file)) {
|
|
427
|
+
sendGeneratedImage(file);
|
|
427
428
|
}
|
|
428
429
|
}
|
|
429
|
-
api_1.logger.debug("Sent generated images", { count:
|
|
430
|
+
api_1.logger.debug("Sent generated images", { count: generatedImages.length });
|
|
430
431
|
}
|
|
431
432
|
}
|
|
432
433
|
catch (fileErr) {
|
|
@@ -443,7 +444,7 @@ const addGptRoutes = (router, options) => {
|
|
|
443
444
|
}));
|
|
444
445
|
const assistantPrompt = {
|
|
445
446
|
model: aiService.modelId,
|
|
446
|
-
text: fullResponse
|
|
447
|
+
text: fullResponse,
|
|
447
448
|
type: "assistant",
|
|
448
449
|
...(contentParts.length > 0 ? { content: contentParts } : {}),
|
|
449
450
|
};
|
|
@@ -423,6 +423,10 @@ class AIService {
|
|
|
423
423
|
}
|
|
424
424
|
messages.push({ content: parts, role: "user" });
|
|
425
425
|
}
|
|
426
|
+
else if (!prompt.text && prompt.content?.length) {
|
|
427
|
+
// Providers reject empty assistant turns; stand in for an image-only response
|
|
428
|
+
messages.push({ content: "[Generated image]", role });
|
|
429
|
+
}
|
|
426
430
|
else {
|
|
427
431
|
messages.push({ content: prompt.text, role });
|
|
428
432
|
}
|
package/package.json
CHANGED
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
"description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
|
|
19
19
|
"devDependencies": {
|
|
20
20
|
"@biomejs/biome": "^2.3.6",
|
|
21
|
-
"@terreno/api": "57.
|
|
22
|
-
"@terreno/test": "57.
|
|
21
|
+
"@terreno/api": "57.8.0",
|
|
22
|
+
"@terreno/test": "57.8.0",
|
|
23
23
|
"@types/bun": "^1.4.0",
|
|
24
24
|
"@types/express": "^5.0.6",
|
|
25
25
|
"@types/luxon": "^3.7.1",
|
|
@@ -69,7 +69,7 @@
|
|
|
69
69
|
"name": "@terreno/ai",
|
|
70
70
|
"peerDependencies": {
|
|
71
71
|
"@ai-sdk/google-vertex": "^4.0.142",
|
|
72
|
-
"@terreno/api": "57.
|
|
72
|
+
"@terreno/api": "57.8.0",
|
|
73
73
|
"mongoose": "^8.0.0 || ^9.0.0"
|
|
74
74
|
},
|
|
75
75
|
"peerDependenciesMeta": {
|
|
@@ -97,5 +97,5 @@
|
|
|
97
97
|
"test:coverage": "bun run ../scripts/check-coverage.ts"
|
|
98
98
|
},
|
|
99
99
|
"types": "dist/index.d.ts",
|
|
100
|
-
"version": "57.
|
|
100
|
+
"version": "57.8.0"
|
|
101
101
|
}
|
|
@@ -45,6 +45,32 @@ describe("GptHistory Model", () => {
|
|
|
45
45
|
expect(history.prompts[1].model).toBe("gpt-4");
|
|
46
46
|
});
|
|
47
47
|
|
|
48
|
+
it("should require prompt text when the prompt has no content parts", async () => {
|
|
49
|
+
const userId = new mongoose.Types.ObjectId();
|
|
50
|
+
await expect(
|
|
51
|
+
GptHistory.create({prompts: [{text: "", type: "assistant"}], userId})
|
|
52
|
+
).rejects.toThrow();
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("should allow empty prompt text on an image-only response", async () => {
|
|
56
|
+
const userId = new mongoose.Types.ObjectId();
|
|
57
|
+
const history = await GptHistory.create({
|
|
58
|
+
prompts: [
|
|
59
|
+
{
|
|
60
|
+
content: [
|
|
61
|
+
{mimeType: "image/png", type: "image", url: "data:image/png;base64,aGVsbG8="},
|
|
62
|
+
],
|
|
63
|
+
text: "",
|
|
64
|
+
type: "assistant",
|
|
65
|
+
},
|
|
66
|
+
],
|
|
67
|
+
userId,
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
expect(history.prompts[0].text).toBe("");
|
|
71
|
+
expect(history.prompts[0].content).toHaveLength(1);
|
|
72
|
+
});
|
|
73
|
+
|
|
48
74
|
it("should require userId", async () => {
|
|
49
75
|
await expect(GptHistory.create({prompts: []})).rejects.toThrow();
|
|
50
76
|
});
|
package/src/models/gptHistory.ts
CHANGED
|
@@ -30,7 +30,15 @@ const gptHistoryPromptSchema = new mongoose.Schema(
|
|
|
30
30
|
type: String,
|
|
31
31
|
},
|
|
32
32
|
result: {description: "Result returned from a tool call", type: mongoose.Schema.Types.Mixed},
|
|
33
|
-
text: {
|
|
33
|
+
text: {
|
|
34
|
+
default: "",
|
|
35
|
+
description: "Text content of the prompt or response",
|
|
36
|
+
// Image-only responses carry their payload in content, not text
|
|
37
|
+
required: function (this: {content?: unknown[]}): boolean {
|
|
38
|
+
return !this.content?.length;
|
|
39
|
+
},
|
|
40
|
+
type: String,
|
|
41
|
+
},
|
|
34
42
|
toolCallId: {
|
|
35
43
|
description: "Identifier linking a tool result to its originating call",
|
|
36
44
|
type: String,
|
package/src/routes/gpt.ts
CHANGED
|
@@ -316,6 +316,15 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
316
316
|
|
|
317
317
|
let fullResponse = "";
|
|
318
318
|
const generatedImages: Array<{mimeType: string; url: string}> = [];
|
|
319
|
+
// Stream file parts and result.files report the same images; emit each data URL once.
|
|
320
|
+
const sendGeneratedImage = (file: GeneratedImageFile): void => {
|
|
321
|
+
const url = `data:${file.mediaType};base64,${file.base64}`;
|
|
322
|
+
if (generatedImages.some((img) => img.url === url)) {
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
generatedImages.push({mimeType: file.mediaType, url});
|
|
326
|
+
res.write(`data: ${JSON.stringify({image: {mimeType: file.mediaType, url}})}\n\n`);
|
|
327
|
+
};
|
|
319
328
|
const startTime = DateTime.now().toMillis();
|
|
320
329
|
try {
|
|
321
330
|
logger.debug("Starting streamText", {model: modelId, supportsTools});
|
|
@@ -400,13 +409,7 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
400
409
|
}
|
|
401
410
|
if (part.type === "file") {
|
|
402
411
|
if (isGeneratedImageFile(part.file)) {
|
|
403
|
-
|
|
404
|
-
generatedImages.push({mimeType: part.file.mediaType, url: dataUrl});
|
|
405
|
-
res.write(
|
|
406
|
-
`data: ${JSON.stringify({
|
|
407
|
-
image: {mimeType: part.file.mediaType, url: dataUrl},
|
|
408
|
-
})}\n\n`
|
|
409
|
-
);
|
|
412
|
+
sendGeneratedImage(part.file);
|
|
410
413
|
logger.debug("Sent inline image from stream");
|
|
411
414
|
}
|
|
412
415
|
continue;
|
|
@@ -489,22 +492,16 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
489
492
|
|
|
490
493
|
logger.debug("Stream completed", {fullResponseLength: fullResponse.length, partCount});
|
|
491
494
|
|
|
492
|
-
//
|
|
495
|
+
// Catch generated images the stream did not report as file parts
|
|
493
496
|
try {
|
|
494
497
|
const files = await result.files;
|
|
495
498
|
if (files && files.length > 0) {
|
|
496
499
|
for (const file of files) {
|
|
497
|
-
if (file
|
|
498
|
-
|
|
499
|
-
generatedImages.push({mimeType: file.mediaType, url: dataUrl});
|
|
500
|
-
res.write(
|
|
501
|
-
`data: ${JSON.stringify({
|
|
502
|
-
image: {mimeType: file.mediaType, url: dataUrl},
|
|
503
|
-
})}\n\n`
|
|
504
|
-
);
|
|
500
|
+
if (isGeneratedImageFile(file)) {
|
|
501
|
+
sendGeneratedImage(file);
|
|
505
502
|
}
|
|
506
503
|
}
|
|
507
|
-
logger.debug("Sent generated images", {count:
|
|
504
|
+
logger.debug("Sent generated images", {count: generatedImages.length});
|
|
508
505
|
}
|
|
509
506
|
} catch (fileErr) {
|
|
510
507
|
logger.debug("No files in response", {
|
|
@@ -521,7 +518,7 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
|
|
|
521
518
|
}));
|
|
522
519
|
const assistantPrompt: GptHistoryPrompt = {
|
|
523
520
|
model: aiService.modelId,
|
|
524
|
-
text: fullResponse
|
|
521
|
+
text: fullResponse,
|
|
525
522
|
type: "assistant",
|
|
526
523
|
...(contentParts.length > 0 ? {content: contentParts} : {}),
|
|
527
524
|
};
|
|
@@ -101,6 +101,36 @@ const createImageModel = () => ({
|
|
|
101
101
|
supportedUrls: {},
|
|
102
102
|
});
|
|
103
103
|
|
|
104
|
+
// Mock model that emits one generated image and no text
|
|
105
|
+
const createImageOnlyModel = () => ({
|
|
106
|
+
doGenerate: mock(async () => ({
|
|
107
|
+
content: [],
|
|
108
|
+
finishReason: "stop" as const,
|
|
109
|
+
usage: {inputTokens: 2, outputTokens: 2},
|
|
110
|
+
})),
|
|
111
|
+
doStream: mock(async () => ({
|
|
112
|
+
stream: new ReadableStream({
|
|
113
|
+
start(controller) {
|
|
114
|
+
controller.enqueue({
|
|
115
|
+
data: "aGVsbG8=",
|
|
116
|
+
mediaType: "image/png",
|
|
117
|
+
type: "file" as const,
|
|
118
|
+
});
|
|
119
|
+
controller.enqueue({
|
|
120
|
+
finishReason: "stop" as const,
|
|
121
|
+
type: "finish" as const,
|
|
122
|
+
usage: {inputTokens: 2, outputTokens: 2},
|
|
123
|
+
});
|
|
124
|
+
controller.close();
|
|
125
|
+
},
|
|
126
|
+
}),
|
|
127
|
+
})),
|
|
128
|
+
modelId: "gemini-3-pro-image",
|
|
129
|
+
provider: "mock-provider",
|
|
130
|
+
specificationVersion: "v2" as const,
|
|
131
|
+
supportedUrls: {},
|
|
132
|
+
});
|
|
133
|
+
|
|
104
134
|
// Mock model that emits text without a finish event (no finish-step → post-loop flush)
|
|
105
135
|
const createNoFinishModel = () => ({
|
|
106
136
|
doGenerate: mock(async () => ({
|
|
@@ -482,6 +512,34 @@ describe("AI Routes", () => {
|
|
|
482
512
|
);
|
|
483
513
|
});
|
|
484
514
|
|
|
515
|
+
it("emits and saves a generated image once, with no placeholder text", async () => {
|
|
516
|
+
const imageService = new AIService({
|
|
517
|
+
model: createImageOnlyModel() as unknown as LanguageModel,
|
|
518
|
+
});
|
|
519
|
+
const imgApp = new TerrenoApp({
|
|
520
|
+
configureApp: (router, options) => {
|
|
521
|
+
addGptRoutes(router, {aiService: imageService, openApiOptions: options});
|
|
522
|
+
},
|
|
523
|
+
skipListen: true,
|
|
524
|
+
userModel: UserModel,
|
|
525
|
+
}).build();
|
|
526
|
+
const agent = await authAsUser(imgApp, "notAdmin");
|
|
527
|
+
const res = await agent
|
|
528
|
+
.post("/gpt/prompt")
|
|
529
|
+
.send({prompt: "Make an image"})
|
|
530
|
+
.buffer(true)
|
|
531
|
+
.parse(sseCollect);
|
|
532
|
+
expect(res.status).toBe(200);
|
|
533
|
+
const body = (res as SseResponse).body;
|
|
534
|
+
expect(body.match(/"image":\{/g)?.length).toBe(1);
|
|
535
|
+
|
|
536
|
+
const history = await GptHistory.findExactlyOne({"prompts.text": "Make an image"});
|
|
537
|
+
const assistant = history.prompts.find((p) => p.type === "assistant");
|
|
538
|
+
expect(assistant?.text).toBe("");
|
|
539
|
+
expect(assistant?.content).toHaveLength(1);
|
|
540
|
+
expect(assistant?.content?.[0]?.url).toBe("data:image/png;base64,aGVsbG8=");
|
|
541
|
+
});
|
|
542
|
+
|
|
485
543
|
it("writes an SSE error event when the model stream throws", async () => {
|
|
486
544
|
const errService = new AIService({
|
|
487
545
|
model: createErrorModel() as unknown as LanguageModel,
|
|
@@ -524,6 +524,24 @@ describe("AIService", () => {
|
|
|
524
524
|
expect(messages[2]).toEqual({content: "How are you?", role: "user"});
|
|
525
525
|
});
|
|
526
526
|
|
|
527
|
+
it("should give image-only assistant prompts stand-in text", () => {
|
|
528
|
+
const model = createMockModel();
|
|
529
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
530
|
+
|
|
531
|
+
const messages = service.buildMessages([
|
|
532
|
+
{text: "Make an image", type: "user"},
|
|
533
|
+
{
|
|
534
|
+
content: [
|
|
535
|
+
{mimeType: "image/png", type: "image" as const, url: "data:image/png;base64,aGVsbG8="},
|
|
536
|
+
],
|
|
537
|
+
text: "",
|
|
538
|
+
type: "assistant",
|
|
539
|
+
},
|
|
540
|
+
]);
|
|
541
|
+
|
|
542
|
+
expect(messages[1]).toEqual({content: "[Generated image]", role: "assistant"});
|
|
543
|
+
});
|
|
544
|
+
|
|
527
545
|
it("should convert multi-modal user prompts with image content", () => {
|
|
528
546
|
const model = createMockModel();
|
|
529
547
|
const service = new AIService({model: model as unknown as LanguageModel});
|
package/src/service/aiService.ts
CHANGED
|
@@ -548,6 +548,9 @@ export class AIService {
|
|
|
548
548
|
}
|
|
549
549
|
}
|
|
550
550
|
messages.push({content: parts, role: "user"} as ModelMessage);
|
|
551
|
+
} else if (!prompt.text && prompt.content?.length) {
|
|
552
|
+
// Providers reject empty assistant turns; stand in for an image-only response
|
|
553
|
+
messages.push({content: "[Generated image]", role});
|
|
551
554
|
} else {
|
|
552
555
|
messages.push({content: prompt.text, role});
|
|
553
556
|
}
|