@terreno/ai 57.8.0 → 57.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/routes/gpt.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import {randomUUID} from "node:crypto";
1
2
  import {
2
3
  APIError,
3
4
  asyncHandler,
@@ -16,6 +17,8 @@ import {AIRequest} from "../models/aiRequest";
16
17
  import {GptHistory} from "../models/gptHistory";
17
18
  import {Project} from "../models/project";
18
19
  import {AIService} from "../service/aiService";
20
+ import type {FileStorageService} from "../service/fileStorage";
21
+ import {assertFileUploadsEnabled} from "../service/fileUploadsGate";
19
22
  import {TITLE_GENERATION_PROMPT} from "../service/prompts";
20
23
  import type {
21
24
  GptHistoryDocument,
@@ -24,6 +27,174 @@ import type {
24
27
  MessageContentPart,
25
28
  } from "../types";
26
29
 
30
+ const DEFAULT_STREAM_PERSIST_INTERVAL_MS = 1000;
31
+ const DEFAULT_STREAM_RESUME_POLL_INTERVAL_MS = 500;
32
+ const DEFAULT_STREAM_STALE_AFTER_MS = 60_000;
33
+ // Bump `updated` even without new text so long tool calls do not look stale to resumers
34
+ const STREAM_HEARTBEAT_MS = 10_000;
35
+
36
+ const DURABLE_URL_PATTERN = /^(https?:|data:)/i;
37
+ const DATA_URL_PATTERN = /^data:([^;,]*)((?:;[^;,]*)*?)(;base64)?,(.*)$/s;
38
+
39
+ interface PromptAttachment {
40
+ filename?: string;
41
+ mimeType?: string;
42
+ type?: string;
43
+ url?: string;
44
+ }
45
+
46
+ interface AttachmentContentParts {
47
+ /** Parts sent to the model for this request (original URLs, including data URLs). */
48
+ modelParts: MessageContentPart[];
49
+ /** Parts persisted to history (durable storage references when storage is configured). */
50
+ storedParts: MessageContentPart[];
51
+ }
52
+
53
+ const sseEvent = (payload: Record<string, unknown>): string =>
54
+ `data: ${JSON.stringify(payload)}\n\n`;
55
+
56
+ const sleep = (ms: number): Promise<void> =>
57
+ new Promise((resolve) => {
58
+ setTimeout(resolve, ms);
59
+ });
60
+
61
+ /**
62
+ * Reject attachment URLs that only exist on the client (blob:, file:, content:, ph:, ...).
63
+ * They cannot be read by the model provider or reloaded from history later.
64
+ */
65
+ const validateAttachments = (attachments: unknown): PromptAttachment[] => {
66
+ if (attachments === undefined || attachments === null) {
67
+ return [];
68
+ }
69
+ if (!Array.isArray(attachments)) {
70
+ throw new APIError({status: 400, title: "attachments must be an array"});
71
+ }
72
+ for (const attachment of attachments as PromptAttachment[]) {
73
+ if (attachment?.type !== "image" && attachment?.type !== "file") {
74
+ continue;
75
+ }
76
+ if (typeof attachment.url !== "string" || !DURABLE_URL_PATTERN.test(attachment.url)) {
77
+ const scheme =
78
+ typeof attachment.url === "string" ? (attachment.url.split(":")[0] ?? "") : "missing";
79
+ throw new APIError({
80
+ detail:
81
+ `Attachment "${attachment.filename ?? attachment.type}" uses a "${scheme}" URL. ` +
82
+ "Upload the file first (POST /files/upload) or send it as a data: URL.",
83
+ status: 400,
84
+ title: "Attachment URL must be an http(s) or data: URL",
85
+ });
86
+ }
87
+ }
88
+ return attachments as PromptAttachment[];
89
+ };
90
+
91
+ const decodeDataUrl = (url: string): {buffer: Buffer; mimeType?: string} | undefined => {
92
+ const match = DATA_URL_PATTERN.exec(url);
93
+ if (!match) {
94
+ return undefined;
95
+ }
96
+ const [, mimeType, , base64Flag, payload] = match;
97
+ const buffer = base64Flag
98
+ ? Buffer.from(payload, "base64")
99
+ : Buffer.from(decodeURIComponent(payload), "utf8");
100
+ return {buffer, mimeType: mimeType || undefined};
101
+ };
102
+
103
+ /** Build model and history content parts, uploading data: attachments when storage is configured. */
104
+ const buildAttachmentParts = async ({
105
+ attachments,
106
+ fileStorageService,
107
+ userId,
108
+ }: {
109
+ attachments: PromptAttachment[];
110
+ fileStorageService?: FileStorageService;
111
+ userId?: mongoose.Types.ObjectId;
112
+ }): Promise<AttachmentContentParts> => {
113
+ const modelParts: MessageContentPart[] = [];
114
+ const storedParts: MessageContentPart[] = [];
115
+ for (const attachment of attachments) {
116
+ if (attachment.type !== "image" && attachment.type !== "file") {
117
+ continue;
118
+ }
119
+ const url = attachment.url as string;
120
+ const modelPart: MessageContentPart =
121
+ attachment.type === "image"
122
+ ? {mimeType: attachment.mimeType, type: "image", url}
123
+ : {
124
+ filename: attachment.filename,
125
+ mimeType: attachment.mimeType as string,
126
+ type: "file",
127
+ url,
128
+ };
129
+ modelParts.push(modelPart);
130
+
131
+ const decoded = url.startsWith("data:") ? decodeDataUrl(url) : undefined;
132
+ if (!fileStorageService || !userId || !decoded) {
133
+ storedParts.push(modelPart);
134
+ continue;
135
+ }
136
+ const mimeType = attachment.mimeType ?? decoded.mimeType ?? "application/octet-stream";
137
+ const extension = mimeType.split("/")[1]?.split("+")[0] ?? "bin";
138
+ try {
139
+ const uploaded = await fileStorageService.upload({
140
+ buffer: decoded.buffer,
141
+ filename: attachment.filename ?? `attachment-${DateTime.now().toMillis()}.${extension}`,
142
+ mimeType,
143
+ userId,
144
+ });
145
+ storedParts.push({...modelPart, gcsKey: uploaded.gcsKey, url: uploaded.url});
146
+ } catch (error) {
147
+ throw new APIError({
148
+ cause: error,
149
+ detail: error instanceof Error ? error.message : String(error),
150
+ status: 502,
151
+ title: "Attachment upload failed",
152
+ });
153
+ }
154
+ }
155
+ return {modelParts, storedParts};
156
+ };
157
+
158
+ const toPlainPrompt = (prompt: GptHistoryPrompt): GptHistoryPrompt => {
159
+ const maybeSubdoc = prompt as GptHistoryPrompt & {toObject?: () => GptHistoryPrompt};
160
+ return typeof maybeSubdoc.toObject === "function" ? maybeSubdoc.toObject() : prompt;
161
+ };
162
+
163
+ /** Swap stored attachment references for short-lived signed URLs the model provider can fetch. */
164
+ const resolveStoredAttachmentUrls = async (
165
+ prompts: GptHistoryPrompt[],
166
+ fileStorageService?: FileStorageService
167
+ ): Promise<GptHistoryPrompt[]> => {
168
+ if (!fileStorageService) {
169
+ return prompts;
170
+ }
171
+ return Promise.all(
172
+ prompts.map(async (prompt) => {
173
+ if (!prompt.content?.some((part) => part.type !== "text" && part.gcsKey)) {
174
+ return prompt;
175
+ }
176
+ const plain = toPlainPrompt(prompt);
177
+ const content = await Promise.all(
178
+ (plain.content ?? []).map(async (part) => {
179
+ if (part.type === "text" || !part.gcsKey) {
180
+ return part;
181
+ }
182
+ try {
183
+ return {...part, url: await fileStorageService.getSignedUrl(part.gcsKey)};
184
+ } catch (error) {
185
+ logger.warn("Could not sign stored attachment URL", {
186
+ error: error instanceof Error ? error.message : String(error),
187
+ gcsKey: part.gcsKey,
188
+ });
189
+ return part;
190
+ }
191
+ })
192
+ );
193
+ return {...plain, content};
194
+ })
195
+ );
196
+ };
197
+
27
198
  interface GeneratedImageFile {
28
199
  base64: string;
29
200
  mediaType: string;
@@ -111,7 +282,18 @@ const generateTitle = async (
111
282
  };
112
283
 
113
284
  export const addGptRoutes = (router: express.Router, options: GptRouteOptions): void => {
114
- const {mcpService, tools: routeTools, createRequestTools, toolChoice, maxSteps} = options;
285
+ const {
286
+ mcpService,
287
+ tools: routeTools,
288
+ createRequestTools,
289
+ toolChoice,
290
+ maxSteps,
291
+ fileStorageService,
292
+ } = options;
293
+ const persistIntervalMs = options.streamPersistIntervalMs ?? DEFAULT_STREAM_PERSIST_INTERVAL_MS;
294
+ const resumePollIntervalMs =
295
+ options.streamResumePollIntervalMs ?? DEFAULT_STREAM_RESUME_POLL_INTERVAL_MS;
296
+ const staleAfterMs = options.streamStaleAfterMs ?? DEFAULT_STREAM_STALE_AFTER_MS;
115
297
 
116
298
  router.post(
117
299
  "/gpt/prompt",
@@ -160,6 +342,10 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
160
342
  if (!prompt || typeof prompt !== "string") {
161
343
  throw new APIError({status: 400, title: "prompt is required"});
162
344
  }
345
+ if (Array.isArray(attachments) && attachments.length > 0) {
346
+ await assertFileUploadsEnabled(req, options.fileUploadsEnabled);
347
+ }
348
+ const validAttachments = validateAttachments(attachments);
163
349
 
164
350
  // Resolve AI service (per-request key takes priority, then configured service)
165
351
  const hasPerRequestKey = !!req.headers["x-ai-api-key"];
@@ -244,33 +430,20 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
244
430
  }
245
431
 
246
432
  // Build content parts from attachments
247
- const contentParts: MessageContentPart[] = [{text: prompt, type: "text"}];
248
- if (attachments && Array.isArray(attachments)) {
249
- logger.debug("Processing attachments", {
250
- count: attachments.length,
251
- types: attachments.map((a: {mimeType?: string; type?: string; url?: string}) => ({
252
- mimeType: a.mimeType,
253
- type: a.type,
254
- urlLength: a.url?.length ?? 0,
255
- })),
256
- });
257
- for (const attachment of attachments) {
258
- if (attachment.type === "image") {
259
- contentParts.push({
260
- mimeType: attachment.mimeType,
261
- type: "image",
262
- url: attachment.url,
263
- });
264
- } else if (attachment.type === "file") {
265
- contentParts.push({
266
- filename: attachment.filename,
267
- mimeType: attachment.mimeType,
268
- type: "file",
269
- url: attachment.url,
270
- });
271
- }
272
- }
273
- }
433
+ logger.debug("Processing attachments", {
434
+ count: validAttachments.length,
435
+ types: validAttachments.map((a) => ({
436
+ mimeType: a.mimeType,
437
+ type: a.type,
438
+ urlLength: a.url?.length ?? 0,
439
+ })),
440
+ });
441
+ const {modelParts, storedParts} = await buildAttachmentParts({
442
+ attachments: validAttachments,
443
+ fileStorageService,
444
+ userId,
445
+ });
446
+ const contentParts: MessageContentPart[] = [{text: prompt, type: "text"}, ...storedParts];
274
447
 
275
448
  // Add user prompt to history
276
449
  const hasAttachments = contentParts.length > 1;
@@ -286,10 +459,18 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
286
459
  attachmentCount: contentParts.length - 1,
287
460
  historyLength: history.prompts.length,
288
461
  });
289
- const messages = aiService.buildMessages(history.prompts);
462
+ // Earlier turns reference stored attachments; this turn sends the original URLs
463
+ const earlierPrompts = await resolveStoredAttachmentUrls(
464
+ history.prompts.slice(0, -1),
465
+ fileStorageService
466
+ );
467
+ const modelUserPrompt: GptHistoryPrompt = hasAttachments
468
+ ? {...userPrompt, content: [{text: prompt, type: "text"}, ...modelParts]}
469
+ : userPrompt;
470
+ const messages = aiService.buildMessages([...earlierPrompts, modelUserPrompt]);
290
471
  logger.debug("Messages built", {messageCount: messages.length});
291
472
 
292
- // Some models (e.g. gemini-2.5-flash-image) don't support tool calling
473
+ // Some models (e.g. gemini-3-pro-image) don't support tool calling
293
474
  const modelId = aiService.modelId;
294
475
  const supportsTools = !modelId?.includes("image");
295
476
 
@@ -308,13 +489,74 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
308
489
  }
309
490
  }
310
491
 
492
+ // Persist the user turn and a streaming placeholder so a reload can resume the reply
493
+ const streamId = randomUUID();
494
+ history.prompts.push({
495
+ model: aiService.modelId,
496
+ status: "streaming",
497
+ streamId,
498
+ text: "",
499
+ type: "assistant",
500
+ });
501
+ await history.save();
502
+ const savedHistory = history;
503
+ const placeholderIndex = history.prompts.length - 1;
504
+
311
505
  // Stream response via SSE
312
506
  res.setHeader("Content-Type", "text/event-stream");
313
507
  res.setHeader("Cache-Control", "no-cache");
314
508
  res.setHeader("Connection", "keep-alive");
315
509
  sseStarted = true;
510
+ res.write(sseEvent({historyId: history._id.toString(), started: true, streamId}));
316
511
 
317
512
  let fullResponse = "";
513
+ // Buffer text per step so we can discard reasoning text when a tool call follows
514
+ let stepTextBuffer = "";
515
+ let stepHasToolCall = false;
516
+ const currentPartialText = (): string =>
517
+ fullResponse + (stepHasToolCall ? "" : stepTextBuffer);
518
+
519
+ let lastPersistedText = "";
520
+ let lastPersistedAt = DateTime.now();
521
+ let persistQueue: Promise<void> = Promise.resolve();
522
+ const persistTimer = setInterval(() => {
523
+ const text = currentPartialText();
524
+ const isHeartbeatDue =
525
+ DateTime.now().diff(lastPersistedAt).toMillis() >= STREAM_HEARTBEAT_MS;
526
+ if (text === lastPersistedText && !isHeartbeatDue) {
527
+ return;
528
+ }
529
+ lastPersistedText = text;
530
+ lastPersistedAt = DateTime.now();
531
+ persistQueue = persistQueue
532
+ .then(async () => {
533
+ await GptHistory.updateOne(
534
+ {_id: savedHistory._id, [`prompts.${placeholderIndex}.streamId`]: streamId},
535
+ {
536
+ $set: {
537
+ [`prompts.${placeholderIndex}.text`]: text,
538
+ updated: DateTime.now().toJSDate(),
539
+ },
540
+ }
541
+ );
542
+ })
543
+ .catch((persistErr) => {
544
+ logger.warn("Failed to persist partial GPT response", {
545
+ error: persistErr instanceof Error ? persistErr.message : String(persistErr),
546
+ });
547
+ });
548
+ }, persistIntervalMs);
549
+ const stopPersisting = async (): Promise<void> => {
550
+ clearInterval(persistTimer);
551
+ await persistQueue;
552
+ };
553
+ const removePlaceholder = (): void => {
554
+ const index = savedHistory.prompts.findIndex((p) => p.streamId === streamId);
555
+ if (index >= 0) {
556
+ savedHistory.prompts.splice(index, 1);
557
+ }
558
+ };
559
+
318
560
  const generatedImages: Array<{mimeType: string; url: string}> = [];
319
561
  // Stream file parts and result.files report the same images; emit each data URL once.
320
562
  const sendGeneratedImage = (file: GeneratedImageFile): void => {
@@ -355,9 +597,6 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
355
597
  });
356
598
 
357
599
  let partCount = 0;
358
- // Buffer text per step so we can discard reasoning text when a tool call follows
359
- let stepTextBuffer = "";
360
- let stepHasToolCall = false;
361
600
 
362
601
  for await (const part of result.fullStream as AsyncIterable<{
363
602
  type: string;
@@ -509,7 +748,9 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
509
748
  });
510
749
  }
511
750
 
512
- // Save assistant response to history
751
+ // Replace the streaming placeholder with the final reply, after any tool turns
752
+ await stopPersisting();
753
+ removePlaceholder();
513
754
  if (fullResponse || generatedImages.length > 0) {
514
755
  const contentParts: MessageContentPart[] = generatedImages.map((img) => ({
515
756
  mimeType: img.mimeType,
@@ -518,12 +759,15 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
518
759
  }));
519
760
  const assistantPrompt: GptHistoryPrompt = {
520
761
  model: aiService.modelId,
762
+ status: "complete",
763
+ streamId,
521
764
  text: fullResponse,
522
765
  type: "assistant",
523
766
  ...(contentParts.length > 0 ? {content: contentParts} : {}),
524
767
  };
525
768
  history.prompts.push(assistantPrompt);
526
769
  }
770
+ history.markModified("prompts");
527
771
  await history.save();
528
772
 
529
773
  try {
@@ -574,6 +818,27 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
574
818
  error: error instanceof Error ? error.message : String(error),
575
819
  });
576
820
 
821
+ // Keep any partial reply so the conversation shows what streamed before the failure
822
+ try {
823
+ await stopPersisting();
824
+ const partialText = currentPartialText();
825
+ const placeholder = savedHistory.prompts.find((p) => p.streamId === streamId);
826
+ if (placeholder?.status === "streaming") {
827
+ if (partialText) {
828
+ placeholder.text = partialText;
829
+ placeholder.status = "error";
830
+ } else {
831
+ removePlaceholder();
832
+ }
833
+ savedHistory.markModified("prompts");
834
+ await savedHistory.save();
835
+ }
836
+ } catch (saveErr) {
837
+ logger.warn("Failed to save interrupted GPT response", {
838
+ error: saveErr instanceof Error ? saveErr.message : String(saveErr),
839
+ });
840
+ }
841
+
577
842
  try {
578
843
  await AIRequest.logRequest({
579
844
  aiModel: modelId ?? "unknown",
@@ -617,6 +882,128 @@ export const addGptRoutes = (router: express.Router, options: GptRouteOptions):
617
882
  }
618
883
  );
619
884
 
885
+ // Re-attach to a reply that /gpt/prompt is still streaming (e.g. after a reload or remount).
886
+ // Polls the persisted partial output, so it works across server instances.
887
+ router.get(
888
+ "/gpt/histories/:id/stream",
889
+ [
890
+ authenticateMiddleware(),
891
+ createOpenApiBuilder(options.openApiOptions ?? {})
892
+ .withTags(["gpt"])
893
+ .withSummary("Resume an in-flight GPT reply as SSE")
894
+ .withPathParameter("id", {type: "string"})
895
+ .withQueryParameter(
896
+ "streamId",
897
+ {type: "string"},
898
+ {description: "Reply to follow. Defaults to the latest streaming reply."}
899
+ )
900
+ .withQueryParameter(
901
+ "offset",
902
+ {type: "number"},
903
+ {description: "Characters of the reply the client already shows. Defaults to 0."}
904
+ )
905
+ .withResponse(200, {data: {type: "string"}})
906
+ .build(),
907
+ ],
908
+ asyncHandler(async (req: express.Request, res: express.Response) => {
909
+ const {id} = req.params;
910
+ const userId = (req.user as {_id?: mongoose.Types.ObjectId} | undefined)?._id;
911
+ const requestedStreamId =
912
+ typeof req.query.streamId === "string" ? req.query.streamId : undefined;
913
+ const parsedOffset = Number(req.query.offset ?? 0);
914
+ const offset = Number.isFinite(parsedOffset) && parsedOffset > 0 ? parsedOffset : 0;
915
+
916
+ const history = await GptHistory.findById(id);
917
+ if (!history) {
918
+ throw new APIError({status: 404, title: "History not found"});
919
+ }
920
+ if (history.userId.toString() !== userId?.toString()) {
921
+ throw new APIError({status: 403, title: "Not authorized to access this history"});
922
+ }
923
+
924
+ const reply = requestedStreamId
925
+ ? history.prompts.find((p) => p.streamId === requestedStreamId)
926
+ : history.prompts.findLast((p) => p.status === "streaming");
927
+ const streamId = reply?.streamId;
928
+
929
+ res.setHeader("Content-Type", "text/event-stream");
930
+ res.setHeader("Cache-Control", "no-cache");
931
+ res.setHeader("Connection", "keep-alive");
932
+ res.write(sseEvent({historyId: id, resumed: true, ...(streamId ? {streamId} : {})}));
933
+
934
+ let isClosed = false;
935
+ req.on("close", () => {
936
+ isClosed = true;
937
+ });
938
+
939
+ let isInitialPoll = true;
940
+ let sentText: string | undefined;
941
+ let sentLength = offset;
942
+ try {
943
+ while (!isClosed) {
944
+ const current = streamId ? await GptHistory.findById(id) : history;
945
+ if (!current) {
946
+ res.write(sseEvent({error: "History not found"}));
947
+ break;
948
+ }
949
+ const currentReply = streamId
950
+ ? current.prompts.find((p) => p.streamId === streamId)
951
+ : undefined;
952
+ const text = currentReply?.text ?? "";
953
+
954
+ if (isInitialPoll && offset > 0) {
955
+ // The client can have loaded a stale placeholder before reconnecting.
956
+ res.write(sseEvent({replace: true, text}));
957
+ } else if (sentText !== undefined && !text.startsWith(sentText)) {
958
+ // Text from a step that turned into a tool call was discarded; resend the reply
959
+ res.write(sseEvent({replace: true, text}));
960
+ } else if (text.length > sentLength) {
961
+ res.write(sseEvent({text: text.slice(sentLength)}));
962
+ }
963
+ isInitialPoll = false;
964
+ sentText = text;
965
+ sentLength = text.length;
966
+
967
+ if (currentReply?.status === "streaming") {
968
+ const sinceUpdateMs = DateTime.now()
969
+ .diff(DateTime.fromJSDate(current.updated))
970
+ .toMillis();
971
+ if (sinceUpdateMs > staleAfterMs) {
972
+ await GptHistory.updateOne(
973
+ {_id: current._id, prompts: {$elemMatch: {status: "streaming", streamId}}},
974
+ {$set: {"prompts.$.status": "error"}}
975
+ );
976
+ res.write(sseEvent({error: "The reply was interrupted before it finished"}));
977
+ res.write(sseEvent({done: true, historyId: id}));
978
+ break;
979
+ }
980
+ await sleep(resumePollIntervalMs);
981
+ continue;
982
+ }
983
+
984
+ for (const part of currentReply?.content ?? []) {
985
+ if (part.type === "image") {
986
+ res.write(sseEvent({image: {mimeType: part.mimeType, url: part.url}}));
987
+ }
988
+ }
989
+ if (currentReply?.status === "error") {
990
+ res.write(sseEvent({error: "The reply was interrupted before it finished"}));
991
+ }
992
+ res.write(
993
+ sseEvent({done: true, historyId: id, ...(current.title ? {title: current.title} : {})})
994
+ );
995
+ break;
996
+ }
997
+ } catch (error) {
998
+ logger.error("Error resuming GPT stream", {
999
+ error: error instanceof Error ? error.message : String(error),
1000
+ });
1001
+ res.write(sseEvent({error: error instanceof Error ? error.message : "Unknown error"}));
1002
+ }
1003
+ res.end();
1004
+ })
1005
+ );
1006
+
620
1007
  router.patch(
621
1008
  "/gpt/histories/:id/rating",
622
1009
  [