@oar-sh/oar 0.9.23 → 0.9.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/.github/extensions/web-relay/skills/reasoning-stream.mjs +3 -1
  2. package/CHANGELOG.md +307 -0
  3. package/README.md +12 -11
  4. package/bin/oar.js +54 -4
  5. package/package.json +11 -7
  6. package/server/agent-sdk-loader.mjs +116 -0
  7. package/server/agent-sdks/claude/entry.mjs +4 -0
  8. package/server/agent-sdks/claude/package-lock.json +1438 -0
  9. package/server/agent-sdks/claude/package.json +13 -0
  10. package/server/agent-sdks/cursor/entry-sqlite.mjs +2 -0
  11. package/server/agent-sdks/cursor/entry.mjs +3 -0
  12. package/server/agent-sdks/cursor/package-lock.json +171 -0
  13. package/server/agent-sdks/cursor/package.json +11 -0
  14. package/server/agent-sdks/koffi/entry.cjs +2 -0
  15. package/server/agent-sdks/koffi/package-lock.json +368 -0
  16. package/server/agent-sdks/koffi/package.json +10 -0
  17. package/server/claude-worker/claude-preview-tool.mjs +5 -2
  18. package/server/claude-worker/claude-remote-relay-tool.mjs +20 -2
  19. package/server/claude-worker/claude-sdk-adapter.mjs +4 -1
  20. package/server/claude-worker/sdk-message-normalizer.mjs +29 -1
  21. package/server/copilot-worker/copilot-sdk-session-process.mjs +13 -8
  22. package/server/cursor-worker/cursor-sdk-adapter.mjs +5 -3
  23. package/server/cursor-worker/sdk-message-normalizer.mjs +13 -0
  24. package/server/db-schema.mjs +19 -0
  25. package/server/grok-worker/sdk-message-normalizer.mjs +3 -0
  26. package/server/migrations/0039-media-index.mjs +167 -0
  27. package/server/migrations/0040-image-session-defaults.mjs +93 -0
  28. package/server/public/app/agent-sdk-ui.js +349 -0
  29. package/server/public/app/api-client.js +149 -8
  30. package/server/public/app/ask-user-view.js +2 -1
  31. package/server/public/app/attachments-view.js +66 -9
  32. package/server/public/app/bootstrap.js +289 -94
  33. package/server/public/app/chat-actions-menu-state.mjs +1 -0
  34. package/server/public/app/context-media-tab.js +143 -0
  35. package/server/public/app/conversation-view.js +90 -5
  36. package/server/public/app/image-composer-ui.js +518 -0
  37. package/server/public/app/image-turn-placeholder.mjs +233 -0
  38. package/server/public/app/journal-view.js +12 -2
  39. package/server/public/app/media-gallery-model.mjs +202 -0
  40. package/server/public/app/media-gallery-ui.js +524 -0
  41. package/server/public/app/media-view.js +79 -0
  42. package/server/public/app/model-refresh-ui.mjs +413 -0
  43. package/server/public/app/no-conversation-state.mjs +5 -2
  44. package/server/public/app/openai-image-composer.mjs +492 -0
  45. package/server/public/app/openai-image-cost-ui.js +426 -0
  46. package/server/public/app/openai-image-spend.mjs +443 -0
  47. package/server/public/app/plan-usage-view.mjs +38 -6
  48. package/server/public/app/provider-readiness.mjs +67 -4
  49. package/server/public/app/relay-error-ctas.mjs +5 -0
  50. package/server/public/app/session-history-ui.js +2 -0
  51. package/server/public/app/settings-modal.js +171 -0
  52. package/server/public/app/socket-handlers.js +47 -0
  53. package/server/public/app/tab-strip-fade.mjs +1 -1
  54. package/server/public/app/viewer-gallery.mjs +23 -1
  55. package/server/public/index.html +478 -9
  56. package/server/relay-tools.md +1 -0
  57. package/server/repositories/image-conversation-repository.mjs +69 -0
  58. package/server/repositories/media-repository.mjs +305 -0
  59. package/server/routes/agent-sdk-routes.mjs +73 -0
  60. package/server/routes/media-routes.mjs +361 -0
  61. package/server/routes/messages-routes.mjs +1467 -343
  62. package/server/routes/sessions-routes.mjs +596 -123
  63. package/server/server-runtime.mjs +414 -81
  64. package/server/services/about-service.mjs +26 -6
  65. package/server/services/agent-sdk-launch-gate.mjs +124 -0
  66. package/server/services/agent-sdk-paths.mjs +307 -0
  67. package/server/services/agent-sdk-store.mjs +1211 -0
  68. package/server/services/claude-api-settings-service.mjs +54 -17
  69. package/server/services/cli-install-service.mjs +3 -2
  70. package/server/services/copilot-model-discovery-service.mjs +7 -1
  71. package/server/services/database-file.mjs +23 -5
  72. package/server/services/deepseek-settings-service.mjs +58 -20
  73. package/server/services/earlier-database-service.mjs +4 -2
  74. package/server/services/generated-image-cleanup-service.mjs +2 -1
  75. package/server/services/generated-image-files.mjs +110 -0
  76. package/server/services/image-input-paths.mjs +389 -0
  77. package/server/services/image-operation-service.mjs +6 -2
  78. package/server/services/media-extract.mjs +653 -0
  79. package/server/services/media-index-service.mjs +221 -0
  80. package/server/services/model-refresh-scheduler.mjs +153 -0
  81. package/server/services/model-variant-catalog-service.mjs +141 -9
  82. package/server/services/oar-cli-helpers.mjs +72 -9
  83. package/server/services/oar-state-migration-service.mjs +2 -12
  84. package/server/services/oar-state-root.mjs +17 -0
  85. package/server/services/openai-image-preview-turns.mjs +330 -0
  86. package/server/services/openai-image-spend-service.mjs +516 -0
  87. package/server/services/openai-image-stream.mjs +170 -0
  88. package/server/services/plan-usage-openai-image.mjs +148 -0
  89. package/server/services/provider-model-refresh.mjs +326 -0
  90. package/server/services/provider-model-state.mjs +395 -0
  91. package/server/services/relay-singleton-guard.mjs +72 -10
  92. package/server/services/remote-relay-caller-context.mjs +6 -0
  93. package/server/services/remote-relay-dispatcher.mjs +386 -22
  94. package/server/services/remote-relay-registry-service.mjs +3 -1
  95. package/server/services/session-worker-launch-service.mjs +13 -0
  96. package/server/services/session-worker-supervisor-service.mjs +19 -1
  97. package/server/services/session-worker-websocket-service.mjs +15 -0
  98. package/server/services/token-usage-service.mjs +15 -0
  99. package/server/services/update-install-service.mjs +2 -2
  100. package/server/services/worker-launch-note-service.mjs +14 -2
  101. package/server/services/worker-recycle-service.mjs +145 -11
  102. package/shared/image-header.mjs +90 -0
  103. package/shared/model-refresh-schedule.mjs +120 -0
  104. package/shared/openai-image-options.mjs +299 -0
  105. package/shared/openai-image-presets.mjs +175 -0
  106. package/shared/openai-image-previews.mjs +40 -0
  107. package/shared/openai-image-prices.mjs +291 -0
  108. package/shared/openai-image-spend.mjs +90 -0
  109. package/shared/remote-relay-contract.mjs +126 -4
  110. package/shared/worker-runtime/direct-openai-image.mjs +98 -48
@@ -6,6 +6,7 @@ import { randomUUID } from 'crypto';
6
6
  import { stripRelayPromptContext } from '../services/relay-prompt-sanitizer.mjs';
7
7
  import { resolveUploadMimeType } from '../services/mime-sniffer.mjs';
8
8
  import { fileVersionToken, serveFileWithRangeSupport, withFileVersion } from '../services/file-serving.mjs';
9
+ import { registerMediaRoutes } from './media-routes.mjs';
9
10
  import { validateEntryName } from '../services/explorer-entry-name.mjs';
10
11
  import { countEntries, createDirectory, removeEntry, renameEntry } from '../services/explorer-mutations.mjs';
11
12
  import {
@@ -64,6 +65,47 @@ import {
64
65
  unknownProviderModelText,
65
66
  } from '../services/provider-model-selection.mjs';
66
67
  import { stripRemotePromptHeader } from '../../shared/remote-relay-contract.mjs';
68
+ import {
69
+ OPENAI_IMAGE_LIMITS,
70
+ OPENAI_IMAGE_MIME_BY_FORMAT,
71
+ OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
72
+ checkOpenAIImageSize,
73
+ formatOpenAIImageSummary,
74
+ isOpenAIImageModelId,
75
+ mergeOpenAIImageOptions,
76
+ openAIImageModelFamily,
77
+ openAIImageModelNeedsOpenAIText,
78
+ validateOpenAIImageOptions,
79
+ } from '../../shared/openai-image-options.mjs';
80
+ import { OPENAI_IMAGE_PARTIAL_IMAGES, OPENAI_IMAGE_PARTIALS_DIR } from '../../shared/openai-image-previews.mjs';
81
+ import {
82
+ SSE_EVENT_TOO_LARGE_CODE,
83
+ interpretOpenAIImageStreamEvent,
84
+ isEventStreamResponse,
85
+ readSseEvents,
86
+ } from '../services/openai-image-stream.mjs';
87
+ import { createOpenAIImagePreviewTurns } from '../services/openai-image-preview-turns.mjs';
88
+ import {
89
+ OPENAI_IMAGE_CAP_WARN_RATIO,
90
+ OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
91
+ OPENAI_IMAGE_USAGE_PATH,
92
+ formatOpenAIImageUsd,
93
+ } from '../../shared/openai-image-spend.mjs';
94
+ import { estimateOpenAIImageCost } from '../../shared/openai-image-prices.mjs';
95
+ import {
96
+ OPENAI_IMAGE_PRESETS_SETTING_KEY,
97
+ findOpenAIImagePreset,
98
+ parseOwnOpenAIImagePresets,
99
+ splitOpenAIImagePreset,
100
+ } from '../../shared/openai-image-presets.mjs';
101
+ import {
102
+ createOpenAIImageSpendService,
103
+ openAIImageCapRefusalText,
104
+ openAIImageTryCharge,
105
+ openAIImageTryRunId,
106
+ } from '../services/openai-image-spend-service.mjs';
107
+ import { IMAGE_INPUT_REFUSED_CODE, resolveImageInputRoots, resolveImageInputs } from '../services/image-input-paths.mjs';
108
+ import { IMAGE_HEADER_BYTES, readImageHeader } from '../../shared/image-header.mjs';
67
109
  import { stripModelTierSuffix } from '../../shared/claude-cloud/repo-url.mjs';
68
110
  import {
69
111
  createSubagentTimelineService,
@@ -206,35 +248,33 @@ function isAutoModel(model) {
206
248
  return String(model || '').trim().toLowerCase() === AUTO_MODEL_SENTINEL;
207
249
  }
208
250
 
209
- function isOpenAIImageModelId(model = '') {
210
- const normalized = String(model || '').trim().toLowerCase().replace(/^openai\//, '');
211
- return normalized.startsWith('gpt-image-') || normalized.startsWith('dall-e-');
212
- }
213
-
214
- const OPENAI_IMAGE_QUALITY_VALUES = new Set(['auto', 'low', 'medium', 'high']);
215
- const OPENAI_IMAGE_SIZE_VALUES = new Set([
216
- 'auto',
217
- '256x256',
218
- '512x512',
219
- '1024x1024',
220
- '1536x1024',
221
- '1024x1536',
222
- '1792x1024',
223
- '1024x1792',
224
- ]);
225
-
251
+ // A quality or size any image model family takes (shared/openai-image-options.mjs),
252
+ // or ''. The model's own family is checked where the request is built.
226
253
  function normalizeOpenAIImageQuality(value = '') {
227
- const normalized = String(value || '').trim().toLowerCase();
228
- return OPENAI_IMAGE_QUALITY_VALUES.has(normalized) ? normalized : '';
254
+ const checked = validateOpenAIImageOptions({ quality: value });
255
+ return checked.ok ? (checked.values.quality || '') : '';
229
256
  }
230
257
 
231
258
  function normalizeOpenAIImageSize(value = '') {
232
259
  const normalized = String(value || '').trim().toLowerCase();
233
- if (OPENAI_IMAGE_SIZE_VALUES.has(normalized)) return normalized;
260
+ // The composer's size select is the context-tier select: its own values
261
+ // mean the API's default size.
234
262
  if (normalized === 'default' || normalized === 'long_context') return 'auto';
235
- return '';
263
+ if (!normalized) return '';
264
+ return checkOpenAIImageSize(normalized).size || '';
236
265
  }
237
266
 
267
+ // How long one OpenAI image request may take, from sending it to the last
268
+ // byte of the reply; config.json `openaiImageGenerateTimeoutMs` and
269
+ // `openaiImageEditTimeoutMs` override them. Above 300 s undici's own header
270
+ // timeout ends a request first.
271
+ export const OPENAI_IMAGE_GENERATE_TIMEOUT_MS = 180_000;
272
+ export const OPENAI_IMAGE_EDIT_TIMEOUT_MS = 300_000;
273
+ // The pause before the one retry of a request that may pass the second time.
274
+ const OPENAI_IMAGE_RETRY_DELAY_MS = 2_000;
275
+ // OpenAI takes up to 16 input images on an edit.
276
+ const OPENAI_IMAGE_EDIT_INPUTS_MAX = 16;
277
+
238
278
  function normalizeSessionWorkerId(value) {
239
279
  const text = String(value || '').trim();
240
280
  return text || null;
@@ -1445,6 +1485,7 @@ export function buildDequeuedRelayMessage({
1445
1485
  try {
1446
1486
  messageKind = String(stmts.getMessageByConversation?.get?.(msg.id, msg.conversation_id)?.kind || '').trim();
1447
1487
  } catch {}
1488
+ const imageOptions = parseQueueImageOptions(msg.image_options_json);
1448
1489
  return {
1449
1490
  id: msg.id,
1450
1491
  conversationId: msg.conversation_id,
@@ -1477,10 +1518,23 @@ export function buildDequeuedRelayMessage({
1477
1518
  ownerAssignedAt: msg.owner_assigned_at || null,
1478
1519
  ownerLeaseExpiresAt: msg.owner_lease_expires_at || null,
1479
1520
  imageOperationId: String(msg.image_operation_id || '').trim() || null,
1521
+ // An image turn's options as the send settled them; a turn without an
1522
+ // operation sends them with its direct request.
1523
+ ...(imageOptions ? { imageOptions } : {}),
1480
1524
  ...(isRelayAutomatedMessageKind(messageKind) ? { messageKind } : {}),
1481
1525
  };
1482
1526
  }
1483
1527
 
1528
+ function parseQueueImageOptions(value) {
1529
+ if (!value) return null;
1530
+ try {
1531
+ const parsed = JSON.parse(String(value));
1532
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
1533
+ } catch {
1534
+ return null;
1535
+ }
1536
+ }
1537
+
1484
1538
  function buildAttachmentPromptContext(attachments) {
1485
1539
  if (!Array.isArray(attachments) || !attachments.length) return '';
1486
1540
  const lines = [
@@ -1526,14 +1580,61 @@ function resolveGeneratedImageExtension(mimeType) {
1526
1580
  return GENERATED_IMAGE_EXTENSIONS[normalized] || 'png';
1527
1581
  }
1528
1582
 
1583
+ // Base64 payloads are checked with a character loop, never with a regular
1584
+ // expression over the whole string. V8 compiles a regexp without
1585
+ // optimisation once the process has generated more than 1 MB of regexp code
1586
+ // and holds more than 16 MB of executable memory (TooMuchRegExpCode in
1587
+ // regexp.cc), which a relay reaches after some uptime. An unoptimised
1588
+ // `[...]+` then pushes one backtrack frame per character and overflows the
1589
+ // 64 MiB regexp stack at 4,194,304 characters, a PNG of about 3 MB, with
1590
+ // "Maximum call stack size exceeded": every gpt-image reply above that size
1591
+ // was dropped after the provider had billed it, and the row was retried four
1592
+ // more times (2026-11-07). A fresh process optimises the same regexp and
1593
+ // never shows it.
1594
+ // An empty string is not base64 here: every caller checks for a missing
1595
+ // payload first and answers it with its own message.
1596
+ export function isBase64Text(value) {
1597
+ const text = String(value || '');
1598
+ if (!text) return false;
1599
+ for (let i = 0; i < text.length; i += 1) {
1600
+ const c = text.charCodeAt(i);
1601
+ if (
1602
+ (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122)
1603
+ || c === 43 || c === 47 || c === 61 // + / =
1604
+ || c === 32 || c === 9 || c === 10 || c === 13 || c === 11 || c === 12 // ASCII whitespace
1605
+ ) continue;
1606
+ return false;
1607
+ }
1608
+ return true;
1609
+ }
1610
+
1611
+ /**
1612
+ * `data:<mime>;base64,<payload>` taken apart without a regexp over the
1613
+ * payload: `{ mimeType, payload }`, or null when the string is not that
1614
+ * shape (no comma, no `;base64` right before it, a parameter in the type,
1615
+ * an empty payload).
1616
+ */
1617
+ export function splitBase64DataUrl(dataUrl) {
1618
+ const text = String(dataUrl || '');
1619
+ if (!/^data:/i.test(text)) return null;
1620
+ const comma = text.indexOf(',');
1621
+ if (comma < 0) return null;
1622
+ const header = text.slice('data:'.length, comma);
1623
+ const payload = text.slice(comma + 1);
1624
+ if (!/;base64$/i.test(header) || !payload) return null;
1625
+ const mimeType = header.slice(0, -';base64'.length);
1626
+ if (!mimeType || mimeType.includes(';') || mimeType.includes(',')) return null;
1627
+ return { mimeType, payload };
1628
+ }
1629
+
1529
1630
  function parseGeneratedImageData(raw) {
1530
1631
  const dataUrl = String(raw?.dataUrl || raw?.data_url || '').trim();
1531
1632
  if (dataUrl.startsWith('data:')) {
1532
- const match = dataUrl.match(/^data:([^;,]+);base64,(.+)$/i);
1533
- if (!match) throw new Error('Invalid generated image data URL');
1633
+ const parts = splitBase64DataUrl(dataUrl);
1634
+ if (!parts) throw new Error('Invalid generated image data URL');
1534
1635
  return {
1535
- mimeType: String(raw?.mimeType || raw?.mime_type || match[1] || '').trim().toLowerCase(),
1536
- dataBase64: String(match[2] || '').trim(),
1636
+ mimeType: String(raw?.mimeType || raw?.mime_type || parts.mimeType || '').trim().toLowerCase(),
1637
+ dataBase64: parts.payload.trim(),
1537
1638
  };
1538
1639
  }
1539
1640
  return {
@@ -1552,7 +1653,7 @@ export function normalizeGeneratedImageResponses(rawGeneratedImages, { maxImages
1552
1653
  const mimeType = parsed.mimeType || 'image/png';
1553
1654
  if (!mimeType.startsWith('image/')) throw new Error('Generated image type must be image/*');
1554
1655
  if (!parsed.dataBase64) throw new Error('Generated image payload is missing base64 data');
1555
- if (!/^[a-z0-9+/=\s]+$/i.test(parsed.dataBase64)) {
1656
+ if (!isBase64Text(parsed.dataBase64)) {
1556
1657
  throw new Error('Generated image payload is not valid base64');
1557
1658
  }
1558
1659
  const buffer = Buffer.from(parsed.dataBase64, 'base64');
@@ -1578,55 +1679,41 @@ export function shouldAcceptAssistantResponsePayload({ text = '', terminalFailur
1578
1679
  function parseOpenAIImageBase64Value(value) {
1579
1680
  const raw = String(value || '').trim();
1580
1681
  if (!raw) return '';
1581
- if (!/^[a-z0-9+/=\s]+$/i.test(raw)) {
1682
+ if (!isBase64Text(raw)) {
1582
1683
  throw new Error('OpenAI image payload contains invalid base64 data');
1583
1684
  }
1584
1685
  return Buffer.from(raw, 'base64').toString('base64');
1585
1686
  }
1586
1687
 
1587
- function normalizeOpenAIImageOptionString(value, fieldName) {
1588
- if (value === undefined || value === null) return undefined;
1589
- const normalized = String(value).trim();
1590
- if (!normalized) return undefined;
1591
- if (!/^[a-z0-9._-]{2,32}$/i.test(normalized)) {
1592
- throw new Error(`Invalid OpenAI image option "${fieldName}"`);
1593
- }
1594
- return normalized;
1595
- }
1596
-
1597
- export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 10 } = {}) {
1688
+ /**
1689
+ * The body of one image request (the generate route's, or the one the
1690
+ * execute route builds from an operation), checked against the model's
1691
+ * family. `n` is the count; size and quality "auto" pass and are left out
1692
+ * for a dall-e model when the request is built. A prompt that came from an
1693
+ * agent on another relay loses its "[Remote prompt from …]" header: OpenAI
1694
+ * would draw it.
1695
+ */
1696
+ export function normalizeOpenAIImageGenerateRequestBody(rawBody) {
1598
1697
  const body = rawBody && typeof rawBody === 'object' ? rawBody : {};
1599
1698
  const messageId = String(body.messageId || '').trim();
1600
1699
  const conversationId = String(body.conversationId || '').trim();
1601
1700
  const model = String(body.model || '').trim();
1602
- const prompt = String(body.prompt || '').trim();
1701
+ const prompt = stripRemotePromptHeader(String(body.prompt || '')).trim();
1603
1702
  if (!messageId) throw new Error('Missing messageId');
1604
1703
  if (!conversationId) throw new Error('Missing conversationId');
1605
1704
  if (!model) throw new Error('Missing model');
1606
1705
  if (!prompt) throw new Error('Missing prompt');
1607
1706
 
1608
- const hasN = Object.prototype.hasOwnProperty.call(body, 'n');
1609
- let n = undefined;
1610
- if (hasN) {
1611
- const parsed = Number(body.n);
1612
- const maxN = Math.max(1, Math.trunc(Number(maxImages) || 10));
1613
- if (!Number.isInteger(parsed) || parsed < 1 || parsed > maxN) {
1614
- throw new Error(`Invalid n value (must be an integer between 1 and ${maxN})`);
1615
- }
1616
- n = parsed;
1617
- }
1618
-
1619
- const sizeRaw = body.size;
1620
- let size = undefined;
1621
- if (sizeRaw !== undefined && sizeRaw !== null && String(sizeRaw).trim()) {
1622
- const normalizedSize = String(sizeRaw).trim().toLowerCase();
1623
- if (normalizedSize !== 'auto' && !/^\d{2,5}x\d{2,5}$/i.test(normalizedSize)) {
1624
- throw new Error('Invalid size value (expected "<width>x<height>" or "auto")');
1625
- }
1626
- size = normalizedSize;
1627
- }
1707
+ const legacySize = String(body.size ?? '').trim().toLowerCase();
1708
+ const checked = validateOpenAIImageOptions({
1709
+ size: legacySize === 'default' || legacySize === 'long_context' ? 'auto' : body.size,
1710
+ quality: body.quality,
1711
+ count: body.n,
1712
+ output_format: body.output_format,
1713
+ background: body.background,
1714
+ }, { model, field: 'image' });
1715
+ if (!checked.ok) throw new Error(checked.error);
1628
1716
 
1629
- const quality = normalizeOpenAIImageOptionString(body.quality, 'quality');
1630
1717
  const attachments = body.attachments;
1631
1718
  if (attachments !== undefined && !Array.isArray(attachments)) {
1632
1719
  throw new Error('attachments must be an array when provided');
@@ -1637,13 +1724,288 @@ export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 1
1637
1724
  conversationId,
1638
1725
  model,
1639
1726
  prompt,
1640
- n,
1641
- size,
1642
- quality,
1727
+ n: checked.values.count,
1728
+ size: checked.values.size,
1729
+ quality: checked.values.quality,
1730
+ output_format: checked.values.output_format,
1731
+ background: checked.values.background,
1643
1732
  attachments: Array.isArray(attachments) ? attachments : [],
1644
1733
  };
1645
1734
  }
1646
1735
 
1736
+ /**
1737
+ * The fields OpenAI gets for one image request. dall-e models take no
1738
+ * output_format or background, and their "auto" size or quality is not sent
1739
+ * (the API's own default); they answer in b64_json only when asked to.
1740
+ */
1741
+ export function buildOpenAIImageRequestFields(requestBody = {}) {
1742
+ const family = openAIImageModelFamily(requestBody.model);
1743
+ const isGptImage = family === 'gpt-image';
1744
+ const fields = { model: requestBody.model, prompt: requestBody.prompt };
1745
+ if (!isGptImage) fields.response_format = 'b64_json';
1746
+ if (requestBody.n !== undefined) fields.n = requestBody.n;
1747
+ for (const key of ['size', 'quality']) {
1748
+ const value = requestBody[key];
1749
+ if (value && (isGptImage || value !== 'auto')) fields[key] = value;
1750
+ }
1751
+ if (isGptImage) {
1752
+ if (requestBody.output_format) fields.output_format = requestBody.output_format;
1753
+ if (requestBody.background) fields.background = requestBody.background;
1754
+ }
1755
+ return fields;
1756
+ }
1757
+
1758
+ const OPENAI_IMAGE_USAGE_DETAIL_KEYS = Object.freeze(['input_tokens_details', 'output_tokens_details']);
1759
+
1760
+ /**
1761
+ * The `usage` block of an OpenAI image reply, numbers only (top level and
1762
+ * the two token-detail blocks), or null when there is none.
1763
+ */
1764
+ export function normalizeOpenAIImageUsage(raw) {
1765
+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
1766
+ const numbersOf = (source) => {
1767
+ const out = {};
1768
+ for (const [key, value] of Object.entries(source || {})) {
1769
+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) out[key] = value;
1770
+ }
1771
+ return out;
1772
+ };
1773
+ const usage = numbersOf(raw);
1774
+ for (const key of OPENAI_IMAGE_USAGE_DETAIL_KEYS) {
1775
+ const detail = raw[key];
1776
+ if (detail && typeof detail === 'object' && !Array.isArray(detail)) {
1777
+ const numbers = numbersOf(detail);
1778
+ if (Object.keys(numbers).length) usage[key] = numbers;
1779
+ }
1780
+ }
1781
+ return Object.keys(usage).length ? usage : null;
1782
+ }
1783
+
1784
+ /**
1785
+ * Two image usages added up, field by field and detail by detail (a stream
1786
+ * of a count > 1 request reports one per completed image); null + null is
1787
+ * null.
1788
+ */
1789
+ export function addOpenAIImageUsage(a, b) {
1790
+ if (!a) return b || null;
1791
+ if (!b) return a;
1792
+ const sum = { ...a };
1793
+ for (const [key, value] of Object.entries(b)) {
1794
+ if (typeof value === 'number') sum[key] = (typeof sum[key] === 'number' ? sum[key] : 0) + value;
1795
+ else if (value && typeof value === 'object') sum[key] = addOpenAIImageUsage(sum[key] && typeof sum[key] === 'object' ? sum[key] : null, value);
1796
+ }
1797
+ return sum;
1798
+ }
1799
+
1800
+ // The base64 characters that decode to the first IMAGE_HEADER_BYTES bytes.
1801
+ const IMAGE_HEADER_BASE64_CHARS = Math.ceil(IMAGE_HEADER_BYTES / 3) * 4;
1802
+
1803
+ /** readImageHeader of a canonical base64 payload, decoding only its start. */
1804
+ function readBase64ImageHeader(base64) {
1805
+ const text = String(base64 || '');
1806
+ if (!text) return null;
1807
+ return readImageHeader(Buffer.from(text.slice(0, IMAGE_HEADER_BASE64_CHARS), 'base64'));
1808
+ }
1809
+
1810
+ // What one OpenAI image request came to, as `kind`:
1811
+ // - 'unsent': it never reached OpenAI (no connection, no name, no TLS);
1812
+ // nothing was billed.
1813
+ // - 'timeout', 'broken', 'server': no answer in time, the connection broke
1814
+ // off after the request may have gone out, or a 5xx; it may have been
1815
+ // billed.
1816
+ // - 'stream': a streamed reply that broke off, or ended without its image,
1817
+ // after OpenAI had started on it; it may have been billed.
1818
+ // - 'refused': a 4xx (429 included); nothing came of it, a retry is refused
1819
+ // the same way.
1820
+ // - 'unprocessed': a 2xx the relay could not read; billed.
1821
+ // The first five are tried once more; the last two never.
1822
+ const OPENAI_IMAGE_RETRY_KINDS = new Set(['unsent', 'timeout', 'broken', 'server', 'stream']);
1823
+ const OPENAI_IMAGE_AMBIGUOUS_KINDS = new Set(['timeout', 'broken', 'server', 'stream']);
1824
+ const OPENAI_IMAGE_UNSENT_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT']);
1825
+ const OPENAI_IMAGE_TIMEOUT_ERROR_CODES = new Set(['UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']);
1826
+
1827
+ export const OPENAI_IMAGE_INTERRUPTED_NOTE = 'The relay was interrupted while OpenAI was working on the image; the request may have been billed; retry the message.';
1828
+ export const OPENAI_IMAGE_REPEATED_NOTE = 'This image request was already sent to OpenAI once and was not sent again; the first request may have been billed; retry the message.';
1829
+ // The guidance under every image failure note: the note itself says what to do.
1830
+ const OPENAI_IMAGE_FAILURE_GUIDANCE = 'If this keeps failing, include the error code when you report it.';
1831
+
1832
+ /** The `code` of a thrown fetch error and of every cause under it. */
1833
+ function fetchErrorCodes(error) {
1834
+ const codes = [];
1835
+ const seen = new Set();
1836
+ const visit = (entry, depth) => {
1837
+ if (!entry || typeof entry !== 'object' || seen.has(entry) || depth > 4) return;
1838
+ seen.add(entry);
1839
+ if (entry.code) codes.push(String(entry.code));
1840
+ visit(entry.cause, depth + 1);
1841
+ if (Array.isArray(entry.errors)) for (const inner of entry.errors) visit(inner, depth + 1);
1842
+ };
1843
+ visit(error, 0);
1844
+ return codes;
1845
+ }
1846
+
1847
+ function isTlsFailureCode(code) {
1848
+ return code.startsWith('ERR_TLS_') || code.startsWith('ERR_SSL_') || code.includes('CERT') || code === 'EPROTO';
1849
+ }
1850
+
1851
+ /** 'unsent', 'timeout' or 'broken' for an error fetch threw (see OPENAI_IMAGE_RETRY_KINDS). */
1852
+ export function classifyOpenAIImageFetchError(error) {
1853
+ if (error?.name === 'TimeoutError' || error?.cause?.name === 'TimeoutError') return 'timeout';
1854
+ const codes = fetchErrorCodes(error);
1855
+ if (codes.some((code) => OPENAI_IMAGE_TIMEOUT_ERROR_CODES.has(code))) return 'timeout';
1856
+ if (codes.length && codes.every((code) => OPENAI_IMAGE_UNSENT_ERROR_CODES.has(code) || isTlsFailureCode(code))) return 'unsent';
1857
+ return 'broken';
1858
+ }
1859
+
1860
+ function describeFetchError(error) {
1861
+ const codes = [...new Set(fetchErrorCodes(error))];
1862
+ const message = String(error?.message || error || 'request error').trim();
1863
+ return codes.length ? `${message} (${codes.join(', ')})` : message;
1864
+ }
1865
+
1866
+ function formatNoteDuration(ms) {
1867
+ const value = Math.max(0, Number(ms) || 0);
1868
+ return value >= 1000 ? `${Math.round(value / 1000)} s` : `${Math.round(value)} ms`;
1869
+ }
1870
+
1871
+ function asSentence(text) {
1872
+ const value = String(text || '').trim();
1873
+ return !value || /[.!?]$/.test(value) ? value : `${value}.`;
1874
+ }
1875
+
1876
+ /**
1877
+ * The failure of an image request after its tries: the note the chat shows
1878
+ * (`error`), the attempt's code, `outcome` 'uncertain' when it may have been
1879
+ * billed (else 'terminal'), and the HTTP status the generate route answers.
1880
+ * A request that never reached OpenAI on its last try but may have been
1881
+ * billed on the first is told as the first.
1882
+ */
1883
+ export function openAIImageFailureResult(tries, { timeoutMs = 0, resent = false } = {}) {
1884
+ const list = Array.isArray(tries) && tries.length ? tries : [{ kind: 'broken' }];
1885
+ const last = list[list.length - 1];
1886
+ const ambiguous = list.find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry.kind)) || null;
1887
+ const deciding = last.kind === 'unsent' && ambiguous ? ambiguous : last;
1888
+ // A resend without stream fields is the second request of a turn whose
1889
+ // first was refused for them before any work.
1890
+ const triedText = resent ? 'sent twice, once without previews' : (list.length > 1 ? 'tried twice' : 'tried once');
1891
+ const maybeBilled = 'the request may have been billed; retry the message.';
1892
+ // A refusal or an unreadable reply after a first try that may have been
1893
+ // billed: the note says so too.
1894
+ const earlierBilled = ambiguous && !OPENAI_IMAGE_AMBIGUOUS_KINDS.has(deciding.kind) && deciding.kind !== 'unsent'
1895
+ ? ' The first try may have been billed as well.'
1896
+ : '';
1897
+ const base = {
1898
+ ok: false,
1899
+ tries: list.length,
1900
+ sent: list.some((entry) => entry.kind !== 'unsent'),
1901
+ ...(deciding.detail ? { detail: String(deciding.detail).slice(0, 500) } : {}),
1902
+ upstreamStatus: Number.isFinite(Number(deciding.status)) && Number(deciding.status) > 0 ? Number(deciding.status) : null,
1903
+ };
1904
+ switch (deciding.kind) {
1905
+ case 'unsent':
1906
+ return { ...base, status: 502, outcome: 'terminal', code: 'IMAGE_PROVIDER_UNREACHABLE', error: 'OpenAI could not be reached; nothing was billed; retry the message.' };
1907
+ case 'timeout': {
1908
+ const waited = deciding.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
1909
+ return { ...base, status: 504, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `OpenAI did not answer ${waited} (${triedText}); ${maybeBilled}` };
1910
+ }
1911
+ case 'server':
1912
+ return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_UNAVAILABLE', error: `OpenAI answered ${deciding.status || 'with a server error'} (${triedText}); ${maybeBilled}` };
1913
+ case 'stream':
1914
+ return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_STREAM_BROKEN', error: `OpenAI's image stream broke off before the image was complete (${triedText}); ${maybeBilled}` };
1915
+ case 'refused': {
1916
+ const message = String(deciding.message || '').trim().slice(0, 240);
1917
+ return {
1918
+ ...base,
1919
+ status: deciding.status >= 400 && deciding.status < 500 ? deciding.status : 502,
1920
+ outcome: 'terminal',
1921
+ code: 'IMAGE_PROVIDER_REJECTED',
1922
+ error: `${asSentence(`OpenAI refused the image request (${deciding.status})${message ? `: ${message}` : ''}`)}${earlierBilled}`,
1923
+ };
1924
+ }
1925
+ case 'unprocessed':
1926
+ return {
1927
+ ...base,
1928
+ status: 502,
1929
+ outcome: 'terminal',
1930
+ code: 'IMAGE_REPLY_UNPROCESSED',
1931
+ relayProcessing: true,
1932
+ error: `The relay could not process the image OpenAI returned (${deciding.detail || 'unknown error'}). The generation was billed; the message was not retried.${earlierBilled}`,
1933
+ };
1934
+ default:
1935
+ return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `The connection to OpenAI broke off (${triedText}); ${maybeBilled}` };
1936
+ }
1937
+ }
1938
+
1939
+ /**
1940
+ * The note on an image request that came through on its second try after a
1941
+ * first that may have reached OpenAI (OPENAI_IMAGE_AMBIGUOUS_KINDS): both may
1942
+ * have been billed. Empty when the earlier tries never went out.
1943
+ */
1944
+ export function openAIImageRetriedNote(tries, { timeoutMs = 0 } = {}) {
1945
+ const first = (Array.isArray(tries) ? tries : []).find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
1946
+ if (!first) return '';
1947
+ const tail = 'it was sent once more and the first try may have been billed as well.';
1948
+ switch (first.kind) {
1949
+ case 'timeout': {
1950
+ const waited = first.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
1951
+ return `OpenAI did not answer the first try ${waited}; ${tail}`;
1952
+ }
1953
+ case 'server':
1954
+ return `OpenAI answered the first try with ${first.status || 'a server error'}; ${tail}`;
1955
+ case 'stream':
1956
+ return `OpenAI's image stream broke off on the first try; ${tail}`;
1957
+ default:
1958
+ return `The connection to OpenAI broke off on the first try; ${tail}`;
1959
+ }
1960
+ }
1961
+
1962
+ /**
1963
+ * The failure of an image request whose next try the daily cap stopped
1964
+ * before it was sent (`refusal`: the spend service's reserveTry). Nothing of
1965
+ * that try reached OpenAI; an earlier try may have been billed, and the note
1966
+ * says so then.
1967
+ */
1968
+ export function openAIImageCapRefusalResult(tries, refusal = {}) {
1969
+ const list = Array.isArray(tries) ? tries : [];
1970
+ const earlierBilled = list.some((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
1971
+ return {
1972
+ ok: false,
1973
+ tries: list.length,
1974
+ sent: list.some((entry) => entry?.kind !== 'unsent'),
1975
+ status: 409,
1976
+ outcome: earlierBilled ? 'uncertain' : 'terminal',
1977
+ code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
1978
+ error: openAIImageCapRefusalText({ tryNumber: earlierBilled ? 2 : 1, refusal }),
1979
+ upstreamStatus: null,
1980
+ };
1981
+ }
1982
+
1983
+ // The file system the partial preview route serves through: its open
1984
+ // follows no link.
1985
+ const PARTIAL_PREVIEW_FS = Object.freeze({
1986
+ ...fs,
1987
+ openSync: (file) => fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)),
1988
+ });
1989
+
1990
+ // The longest image note /api/response takes into the reply text.
1991
+ const OPENAI_IMAGE_NOTE_MAX = 500;
1992
+
1993
+ /** A request refused before anything was sent to OpenAI. */
1994
+ function openAIImageRequestRefusal(status, code, error) {
1995
+ return { ok: false, tries: 0, sent: false, status, outcome: 'terminal', code, error, upstreamStatus: null };
1996
+ }
1997
+
1998
+ /** The `terminalError` /api/response gets for a failed image request. */
1999
+ function openAIImageTerminalError(result) {
2000
+ return {
2001
+ code: result.code,
2002
+ message: String(result.error || 'Image operation failed').slice(0, 500),
2003
+ ...(result.detail ? { detail: String(result.detail).slice(0, 500) } : {}),
2004
+ guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
2005
+ retryable: false,
2006
+ };
2007
+ }
2008
+
1647
2009
  function isOpenAIImageAttachment(raw) {
1648
2010
  if (!raw || typeof raw !== 'object') return false;
1649
2011
  const type = String(raw.type || '').trim().toLowerCase();
@@ -1652,7 +2014,26 @@ function isOpenAIImageAttachment(raw) {
1652
2014
  return dataUrl.startsWith('data:image/');
1653
2015
  }
1654
2016
 
1655
- export function resolveOpenAIImageEditAttachment(rawAttachments, {
2017
+ export function resolveOpenAIImageEditAttachment(rawAttachments, options = {}) {
2018
+ return resolveOpenAIImageEditAttachments(rawAttachments, { ...options, maxImages: 1 })[0] || null;
2019
+ }
2020
+
2021
+ /**
2022
+ * Every image attachment of an edit, in the order given (the first is the
2023
+ * base image), at most `maxImages`; the others are ignored.
2024
+ */
2025
+ export function resolveOpenAIImageEditAttachments(rawAttachments, {
2026
+ maxImages = OPENAI_IMAGE_EDIT_INPUTS_MAX,
2027
+ ...options
2028
+ } = {}) {
2029
+ const attachments = Array.isArray(rawAttachments) ? rawAttachments : [];
2030
+ return attachments
2031
+ .filter((entry) => isOpenAIImageAttachment(entry))
2032
+ .slice(0, Math.max(1, Math.trunc(Number(maxImages) || 1)))
2033
+ .map((entry) => resolveOneOpenAIImageEditAttachment(entry, options));
2034
+ }
2035
+
2036
+ function resolveOneOpenAIImageEditAttachment(image, {
1656
2037
  maxImageBytes = 20 * 1024 * 1024,
1657
2038
  allowedRootPath = '',
1658
2039
  // Folders of earlier versions that uploads are still read from: only a
@@ -1660,22 +2041,18 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1660
2041
  allowedBlobRootPaths = [],
1661
2042
  allowedGeneratedImagesRootPath = '',
1662
2043
  } = {}) {
1663
- const attachments = Array.isArray(rawAttachments) ? rawAttachments : [];
1664
- const firstImage = attachments.find((entry) => isOpenAIImageAttachment(entry));
1665
- if (!firstImage) return null;
1666
-
1667
- const dataUrl = String(firstImage.dataUrl || '').trim();
1668
- let mimeType = String(firstImage.type || '').trim().toLowerCase();
2044
+ const dataUrl = String(image.dataUrl || '').trim();
2045
+ let mimeType = String(image.type || '').trim().toLowerCase();
1669
2046
  let bytes = null;
1670
2047
 
1671
2048
  if (dataUrl.startsWith('data:')) {
1672
- const match = dataUrl.match(/^data:([^;,]+);base64,(.+)$/i);
1673
- if (!match) throw new Error('Invalid image attachment data URL');
1674
- mimeType = mimeType || String(match[1] || '').trim().toLowerCase();
1675
- const payload = parseOpenAIImageBase64Value(match[2]);
2049
+ const parts = splitBase64DataUrl(dataUrl);
2050
+ if (!parts) throw new Error('Invalid image attachment data URL');
2051
+ mimeType = mimeType || String(parts.mimeType || '').trim().toLowerCase();
2052
+ const payload = parseOpenAIImageBase64Value(parts.payload);
1676
2053
  bytes = Buffer.from(payload, 'base64');
1677
2054
  } else {
1678
- const filePath = String(firstImage.path || '').trim();
2055
+ const filePath = String(image.path || '').trim();
1679
2056
  if (!filePath) throw new Error('Image attachment is missing file path or data URL');
1680
2057
  const allowedRoot = String(allowedRootPath || '').trim();
1681
2058
  const blobRoots = (Array.isArray(allowedBlobRootPaths) ? allowedBlobRootPaths : [])
@@ -1709,6 +2086,8 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1709
2086
  bytes = fs.readFileSync(filePath);
1710
2087
  }
1711
2088
 
2089
+ // What the bytes are wins over what the attachment says they are.
2090
+ mimeType = (Buffer.isBuffer(bytes) && readImageHeader(bytes)?.mime) || mimeType;
1712
2091
  if (!mimeType.startsWith('image/')) {
1713
2092
  throw new Error('Image attachment must use image/* MIME type');
1714
2093
  }
@@ -1722,13 +2101,20 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1722
2101
  return {
1723
2102
  bytes,
1724
2103
  mimeType,
1725
- name: String(firstImage.name || 'image').trim().slice(0, 120) || 'image',
2104
+ name: String(image.name || 'image').trim().slice(0, 120) || 'image',
1726
2105
  };
1727
2106
  }
1728
2107
 
1729
- export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8 } = {}) {
2108
+ /**
2109
+ * The images of an OpenAI image reply. Their type comes from the format
2110
+ * asked for (`outputFormat`), else the one the reply names, else PNG; the
2111
+ * image's own header has the last word.
2112
+ */
2113
+ export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8, outputFormat = '' } = {}) {
1730
2114
  const rows = Array.isArray(payload?.data) ? payload.data : [];
1731
2115
  const maxCount = Math.max(1, Math.trunc(Number(maxImages) || 8));
2116
+ const format = String(outputFormat || payload?.output_format || '').trim().toLowerCase();
2117
+ const declaredMime = OPENAI_IMAGE_MIME_BY_FORMAT[format] || OPENAI_IMAGE_MIME_BY_FORMAT.png;
1732
2118
  const generatedImages = [];
1733
2119
  for (const [index, row] of rows.entries()) {
1734
2120
  if (index >= maxCount) break;
@@ -1738,7 +2124,7 @@ export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8
1738
2124
  const revisedPrompt = String(row.revised_prompt || row.revisedPrompt || '').trim();
1739
2125
  generatedImages.push({
1740
2126
  data,
1741
- mimeType: 'image/png',
2127
+ mimeType: readBase64ImageHeader(data)?.mime || declaredMime,
1742
2128
  name: `generated-image-${index + 1}`,
1743
2129
  ...(revisedPrompt ? { revisedPrompt } : {}),
1744
2130
  });
@@ -1806,7 +2192,12 @@ function createGeneratedImageAttachment({
1806
2192
  resolveSessionStateRoot,
1807
2193
  }) {
1808
2194
  const imageId = `img-${String(imageIndex + 1).padStart(2, '0')}`;
1809
- const extension = resolveGeneratedImageExtension(image.mimeType);
2195
+ const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
2196
+ // The type and pixel size the image's own header gives; the type the
2197
+ // sender named only when the header is not one the reader knows.
2198
+ const header = readImageHeader(bytes);
2199
+ const mimeType = header?.mime || image.mimeType;
2200
+ const extension = resolveGeneratedImageExtension(mimeType);
1810
2201
  const location = resolveGeneratedImageStoragePath({
1811
2202
  resolveSessionStateRoot,
1812
2203
  sdkSessionId,
@@ -1817,7 +2208,6 @@ function createGeneratedImageAttachment({
1817
2208
  });
1818
2209
  fs.mkdirSync(location.absoluteDir, { recursive: true });
1819
2210
  const tmpPath = path.join(location.absoluteDir, `${location.fileName}.${process.pid}.${Date.now()}.tmp`);
1820
- const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
1821
2211
  fs.writeFileSync(tmpPath, bytes);
1822
2212
  try {
1823
2213
  fs.renameSync(tmpPath, location.absolutePath);
@@ -1829,8 +2219,9 @@ function createGeneratedImageAttachment({
1829
2219
  }
1830
2220
  return {
1831
2221
  name: `${String(image.name || 'generated-image').replace(/\.[a-z0-9]+$/i, '') || 'generated-image'}-${imageIndex + 1}.${extension}`,
1832
- type: image.mimeType,
2222
+ type: mimeType,
1833
2223
  size: bytes.length,
2224
+ ...(header ? { width: header.width, height: header.height } : {}),
1834
2225
  contentUrl: `/api/generated-image/${encodeURIComponent(conversationId)}/${encodeURIComponent(messageId)}/${encodeURIComponent(imageId)}/content`,
1835
2226
  generatedImage: {
1836
2227
  imageId,
@@ -1982,6 +2373,9 @@ export function registerMessagesRoutes(app, deps) {
1982
2373
  MAX_UPLOAD_ATTACHMENTS,
1983
2374
  uploadsDir,
1984
2375
  uploadReadDirs = [uploadsDir],
2376
+ // What a send's `imagePaths` never reads, beside the upload dirs: the
2377
+ // relay's data dir (`dirs`), its database and config files (`files`).
2378
+ imageInputDenyPaths = { dirs: [], files: [] },
1985
2379
  MAX_REPO_TREE_NODES,
1986
2380
  MAX_REQUEUE_RETRIES,
1987
2381
  MAX_IMAGE_DATA_URL_LENGTH,
@@ -2085,6 +2479,11 @@ export function registerMessagesRoutes(app, deps) {
2085
2479
  // (services/request-boundary.mjs). The runtime passes the check built
2086
2480
  // with the configured marker headers.
2087
2481
  isTrulyLocalRequest = createLocalRequestCheck().isTrulyLocal,
2482
+ // The pause before the one retry of an OpenAI image request.
2483
+ openAIImageRetryDelayMs = OPENAI_IMAGE_RETRY_DELAY_MS,
2484
+ // How long the partial previews of an image request that ended wait for
2485
+ // the turn's end (services/openai-image-preview-turns.mjs).
2486
+ openAIImagePreviewLeftoverMs,
2088
2487
  featureFlags,
2089
2488
  sessionWorkerRegistry,
2090
2489
  sessionWorkerSupervisor,
@@ -2107,6 +2506,10 @@ export function registerMessagesRoutes(app, deps) {
2107
2506
  claudeApiUsageService = null,
2108
2507
  // Tokens per turn for every provider (token-usage-service.mjs).
2109
2508
  tokenUsageService = null,
2509
+ // OpenAI image spend and its daily cap (openai-image-spend-service.mjs):
2510
+ // the runtime shares one instance with the settings routes. Without it
2511
+ // (tests) the routes make their own over tokenUsageService.
2512
+ openAIImageSpend = null,
2110
2513
  // The delivery fence's runtime half for GET /api/pending, as on the
2111
2514
  // socket: the Claude API claim check (null: no refusal) and what a fenced
2112
2515
  // row does (the swap, or fail closed).
@@ -2149,6 +2552,24 @@ export function registerMessagesRoutes(app, deps) {
2149
2552
  deepseekClaudeConfigDir = '',
2150
2553
  } = deps;
2151
2554
 
2555
+ // An app_settings value as text ('' when unset or unreadable).
2556
+ const readAppSettingText = (key) => {
2557
+ try {
2558
+ return String(stmts?.getAppSetting?.get?.(key)?.value ?? '').trim();
2559
+ } catch {
2560
+ return '';
2561
+ }
2562
+ };
2563
+ // OpenAI image spend: the cap check and reservation before each try, its
2564
+ // booking after, the send-time refusal and the spend state.
2565
+ const imageSpend = openAIImageSpend || createOpenAIImageSpendService({
2566
+ tokenUsageService,
2567
+ readSetting: readAppSettingText,
2568
+ db,
2569
+ emit: (event, payload) => io?.emit?.(event, payload),
2570
+ logger: console,
2571
+ });
2572
+
2152
2573
  // ─── Sub-agent runs: header, timeline, live events ─────────────────────────
2153
2574
  const subagentTimeline = createSubagentTimelineService({ db, stmts });
2154
2575
 
@@ -4012,6 +4433,19 @@ export function registerMessagesRoutes(app, deps) {
4012
4433
  });
4013
4434
  });
4014
4435
 
4436
+ // The media gallery (🧠 Media tab, global Media view) and its index.
4437
+ registerMediaRoutes(app, {
4438
+ auth,
4439
+ db,
4440
+ uploadPathForSha,
4441
+ fetchBrowsableDrives,
4442
+ resolveGeneratedImagePath: ({ sessionId, relativePath }) => resolveGeneratedImageReadPath({
4443
+ resolveSessionStateRoot,
4444
+ sessionId,
4445
+ relativePath,
4446
+ }),
4447
+ });
4448
+
4015
4449
  app.get('/api/files/{*splat}', auth, (req, res) => {
4016
4450
  const requestedPath = splatPath(req).trim();
4017
4451
  const rootOverride = resolveScopedWorkspaceRootPath(req);
@@ -4824,6 +5258,8 @@ export function registerMessagesRoutes(app, deps) {
4824
5258
  provider,
4825
5259
  attachments: rawAttachments,
4826
5260
  imageTarget: rawImageTarget,
5261
+ imagePaths: rawImagePaths,
5262
+ image: rawImageOptions,
4827
5263
  } = req.body;
4828
5264
  // A prompt from another relay's agent (body `origin` or the
4829
5265
  // X-OAR-Remote-Origin header) is refused while the inbound switch is off.
@@ -4860,7 +5296,7 @@ export function registerMessagesRoutes(app, deps) {
4860
5296
  Array.isArray(rawAttachments) ? rawAttachments.filter((item) => item && typeof item === 'object').length : (rawAttachments ? 1 : 0),
4861
5297
  );
4862
5298
  const referenceResolution = collectReferenceAttachmentsFromText(trimmedText);
4863
- const attachments = mergeMessageAttachments(normalizedAttachments, referenceResolution.attachments);
5299
+ let attachments = mergeMessageAttachments(normalizedAttachments, referenceResolution.attachments);
4864
5300
  const imageTarget = rawImageTarget && typeof rawImageTarget === 'object'
4865
5301
  ? {
4866
5302
  messageId: String(rawImageTarget.messageId || '').trim(),
@@ -4875,6 +5311,17 @@ export function registerMessagesRoutes(app, deps) {
4875
5311
  code: 'IMAGE_CONTINUITY_DISABLED',
4876
5312
  });
4877
5313
  }
5314
+ // Images named by path on this relay and the turn's own image options
5315
+ // (relay_tool send and create_session). Both go to an OpenAI image
5316
+ // session only; the paths are read before anything is stored, and one
5317
+ // that may not be read refuses the whole send.
5318
+ const refuseImageInput = (error) => res.status(400).json({ code: IMAGE_INPUT_REFUSED_CODE, error });
5319
+ const imagePathsGiven = rawImagePaths !== undefined && rawImagePaths !== null;
5320
+ const imageOptionsGiven = rawImageOptions !== undefined && rawImageOptions !== null;
5321
+ if (imagePathsGiven && (!Array.isArray(rawImagePaths) || rawImagePaths.some((entry) => typeof entry !== 'string'))) {
5322
+ return refuseImageInput('imagePaths must be a list of absolute file paths on this relay');
5323
+ }
5324
+ const imagePathList = imagePathsGiven ? rawImagePaths.filter((entry) => entry !== '') : [];
4878
5325
 
4879
5326
  if (trimmedText.toLowerCase() === '/compact') {
4880
5327
  if (attachments.length) return res.status(400).json({ error: 'Compact command does not accept attachments' });
@@ -5396,6 +5843,16 @@ export function registerMessagesRoutes(app, deps) {
5396
5843
  latestUserMessageModel: stmts.getLatestUserMessageModel?.get?.(conversationId)?.model || '',
5397
5844
  })
5398
5845
  : null;
5846
+ // Copilot lists image ids its runtime cannot serve: a Copilot chat on one
5847
+ // is refused with a pointer to an OpenAI Image chat, whoever sends.
5848
+ const copilotModelId = usesGitHubCatalog ? String(inheritedModel ? inheritedModel.model : (model || '')).trim() : '';
5849
+ if (copilotModelId && isOpenAIImageModelId(copilotModelId)) {
5850
+ return res.status(400).json({
5851
+ error: `${openAIImageModelNeedsOpenAIText(copilotModelId)} Nothing was sent.`,
5852
+ code: OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
5853
+ ...(conversationId ? { conversationId } : {}),
5854
+ });
5855
+ }
5399
5856
  const githubModelResolution = usesGitHubCatalog
5400
5857
  ? resolveRequestedModel(inheritedModel ? inheritedModel.model : model)
5401
5858
  : null;
@@ -5499,6 +5956,17 @@ export function registerMessagesRoutes(app, deps) {
5499
5956
  code: 'IMAGE_TARGET_REQUIRES_IMAGE_MODEL',
5500
5957
  });
5501
5958
  }
5959
+ if ((imagePathsGiven || imageOptionsGiven) && !openAIImageModel) {
5960
+ return refuseImageInput(`Images and image options go to an OpenAI image session; this session runs ${requestedModel || 'no image model'}`);
5961
+ }
5962
+ // An image turn from another relay's agent takes its images from
5963
+ // `imagePaths` alone: an @file: token resolves against this relay's
5964
+ // workspace root, not the session's folder, and carries no stored copy.
5965
+ let ignoredReferenceAttachments = 0;
5966
+ if (remoteRequest.remote && openAIImageModel && referenceResolution.attachments.length) {
5967
+ ignoredReferenceAttachments = referenceResolution.attachments.length;
5968
+ attachments = mergeMessageAttachments(normalizedAttachments, []);
5969
+ }
5502
5970
  // A send that names no effort runs with the conversation's own, clamped
5503
5971
  // to the nearest level the model has. Null when there is nothing to keep
5504
5972
  // or it neither matches nor clamps (an ultracode on a model without it):
@@ -5621,11 +6089,66 @@ export function registerMessagesRoutes(app, deps) {
5621
6089
  // is already 1M.
5622
6090
  normalizedContextTier = 'default';
5623
6091
  }
6092
+ // An image turn's options: the session's defaults (update_session
6093
+ // `image`), then the preset the turn names (`image.preset`), then the
6094
+ // turn's own. The owner's composer names quality and size in the effort
6095
+ // and context-tier fields, which are the turn's own; an agent's turn
6096
+ // names them in `image`, and its effort counts below the defaults. Only
6097
+ // what an agent sent is refused here: the composer's choice goes on as
6098
+ // before, checked against the model when it runs. The options are
6099
+ // resolved here, at queue time: a preset changed later does not change a
6100
+ // queued turn.
6101
+ let imageOptions = null;
6102
+ if (openAIImageModel) {
6103
+ const { preset: presetName, own: ownImageInput } = splitOpenAIImagePreset(rawImageOptions);
6104
+ const preset = findOpenAIImagePreset(presetName, parseOwnOpenAIImagePresets(readAppSettingText(OPENAI_IMAGE_PRESETS_SETTING_KEY)));
6105
+ if (!preset.ok) return refuseImageInput(`${asSentence(preset.error)} Nothing was sent.`);
6106
+ const ownImageOptions = validateOpenAIImageOptions(ownImageInput, { model: requestedModel });
6107
+ if (!ownImageOptions.ok) return refuseImageInput(ownImageOptions.error);
6108
+ const keptQuality = !explicitReasoningEffort && requestedReasoningEffort && !['auto', 'none'].includes(requestedReasoningEffort)
6109
+ ? { quality: requestedReasoningEffort }
6110
+ : {};
6111
+ const fieldOptions = {
6112
+ ...(explicitReasoningEffort ? { quality: requestedReasoningEffort } : {}),
6113
+ ...(String(contextTier || '').trim() && !['default', 'long_context'].includes(requestedContextTier) ? { size: normalizedContextTier } : {}),
6114
+ };
6115
+ let sessionDefaults = {};
6116
+ if (!shouldCreateConversation) {
6117
+ try {
6118
+ sessionDefaults = stmts.readConversationImageDefaults?.(conversationId) || {};
6119
+ } catch {}
6120
+ }
6121
+ const layers = remoteRequest.remote
6122
+ ? [keptQuality, fieldOptions, sessionDefaults, preset.values, ownImageOptions.values]
6123
+ : [keptQuality, sessionDefaults, preset.values, fieldOptions, ownImageOptions.values];
6124
+ const merged = layers.reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
6125
+ const checked = validateOpenAIImageOptions(merged, { model: requestedModel });
6126
+ if (checked.ok) {
6127
+ imageOptions = mergeOpenAIImageOptions(checked.values, null);
6128
+ } else if (imagePathsGiven || imageOptionsGiven) {
6129
+ return refuseImageInput(checked.error);
6130
+ } else {
6131
+ imageOptions = [keptQuality, fieldOptions].reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
6132
+ }
6133
+ // Today's image spend has reached the daily cap: refused for every
6134
+ // sender before anything is stored (the owner's override lifts it).
6135
+ let capRefusal = null;
6136
+ try {
6137
+ capRefusal = imageSpend.sendRefusal();
6138
+ } catch (error) {
6139
+ capRefusal = { code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE, error: `The relay could not check the daily cap of OpenAI images (${error?.message || error}). Nothing was sent.` };
6140
+ }
6141
+ if (capRefusal) {
6142
+ return res.status(409).json({ error: capRefusal.error, code: capRefusal.code, ...(conversationId ? { conversationId } : {}) });
6143
+ }
6144
+ }
5624
6145
  if (!requestedRelayMode) return res.status(400).json({ error: 'Unsupported relay mode' });
5625
6146
  const conversationWorkspaceState = (!shouldCreateConversation && typeof resolveConversationWorkspaceState === 'function')
5626
6147
  ? resolveConversationWorkspaceState({ conversationId })
5627
6148
  : null;
5628
- let workspaceRootUpdate = attachments.length === 0
6149
+ // Images by path count as attachments: a "cd …" in their prompt moves
6150
+ // no folder, and they are checked against the folder the session has now.
6151
+ let workspaceRootUpdate = attachments.length === 0 && imagePathList.length === 0
5629
6152
  ? maybeApplyWorkspaceRootFromMessage(
5630
6153
  trimmedText,
5631
6154
  conversationWorkspaceState?.currentWorkspaceRootPath || null,
@@ -5695,6 +6218,56 @@ export function registerMessagesRoutes(app, deps) {
5695
6218
  }
5696
6219
  }
5697
6220
 
6221
+ // The images by path: every one checked and read before anything is
6222
+ // stored, then kept in the upload store, so each becomes an attachment
6223
+ // with its bytes' hash and sniffed type (and an asset of the operation).
6224
+ // They come first: the first image is the base of an edit.
6225
+ let imageInputs = [];
6226
+ let imageInputsDropped = 0;
6227
+ if (imagePathsGiven) {
6228
+ if (shouldCreateConversation) {
6229
+ return refuseImageInput('imagePaths need an existing session: create it first, then send the images to it');
6230
+ }
6231
+ let sessionStateRoot = '';
6232
+ try {
6233
+ sessionStateRoot = String(resolveSessionStateRoot?.() || '');
6234
+ } catch {}
6235
+ const roots = resolveImageInputRoots({
6236
+ cwd: conversationWorkspaceState?.currentWorkspaceRootPath || '',
6237
+ sessionStateRoot,
6238
+ deniedDirs: [...(imageInputDenyPaths?.dirs || []), uploadsDir, ...(uploadReadDirs || [])],
6239
+ deniedFiles: imageInputDenyPaths?.files || [],
6240
+ });
6241
+ const resolvedImages = resolveImageInputs(imagePathList, {
6242
+ roots,
6243
+ isLiveConversation: (id) => {
6244
+ const row = stmts.getConvAnyStatus?.get?.(id) || null;
6245
+ return !!row && String(row.status || '').trim() !== 'deleted';
6246
+ },
6247
+ });
6248
+ if (!resolvedImages.ok) return refuseImageInput(resolvedImages.error);
6249
+ const resolvedHashes = new Set(resolvedImages.images.map((image) => image.sha256));
6250
+ const otherAttachments = attachments.filter((attachment) => !resolvedHashes.has(String(attachment?.sha256 || '').trim().toLowerCase()));
6251
+ const total = resolvedImages.images.length + otherAttachments.length;
6252
+ if (total > OPENAI_IMAGE_LIMITS.inputImagesMax) {
6253
+ return refuseImageInput(`An image turn takes at most ${OPENAI_IMAGE_LIMITS.inputImagesMax} images; this one would have ${total} with the message's other attachments`);
6254
+ }
6255
+ try {
6256
+ imageInputs = resolvedImages.images.map((image) => ({
6257
+ ...persistUploadBuffer(image.bytes, { name: image.name, type: image.mime }),
6258
+ // The type the bytes are, not the one an earlier upload of the
6259
+ // same bytes was stored with.
6260
+ type: image.mime,
6261
+ width: image.width,
6262
+ height: image.height,
6263
+ }));
6264
+ } catch (error) {
6265
+ return res.status(500).json({ code: 'IMAGE_INPUT_STORE_FAILED', error: `The images could not be stored on this relay: ${error?.message || error}` });
6266
+ }
6267
+ imageInputsDropped = imagePathList.length - imageInputs.length;
6268
+ attachments = [...imageInputs, ...otherAttachments];
6269
+ }
6270
+
5698
6271
  const convId = shouldCreateConversation ? uuidv4() : conversationId;
5699
6272
  if (shouldCreateConversation) {
5700
6273
  // An agent's prompt opens with the sending relay's header line, which
@@ -5931,6 +6504,7 @@ export function registerMessagesRoutes(app, deps) {
5931
6504
  null,
5932
6505
  null,
5933
6506
  );
6507
+ if (imageOptions) stmts.setQueueImageOptions?.run?.(JSON.stringify(imageOptions), msgId);
5934
6508
  if (imageContinuityEnabled && openAIImageModel) {
5935
6509
  if (!imageOperationService?.createEnqueuedOperation) {
5936
6510
  throw new Error('Image operation service is unavailable');
@@ -5942,11 +6516,7 @@ export function registerMessagesRoutes(app, deps) {
5942
6516
  prompt: trimmedText,
5943
6517
  selectedImageModel: requestedModel,
5944
6518
  executionMode: 'direct_images',
5945
- parameters: {
5946
- quality: requestedReasoningEffort,
5947
- size: normalizedContextTier,
5948
- count: 1,
5949
- },
6519
+ parameters: imageOptions || { count: 1 },
5950
6520
  attachments,
5951
6521
  imageTarget,
5952
6522
  createdAt: now,
@@ -6111,8 +6681,17 @@ export function registerMessagesRoutes(app, deps) {
6111
6681
  workspaceRootWarning: workspaceRootUpdate.error || null,
6112
6682
  workspaceRootChanged: !!workspaceRootUpdate.changed,
6113
6683
  ...workspaceRootPayload(),
6114
- referenceAttachmentCount: referenceResolution.attachments.length,
6684
+ referenceAttachmentCount: referenceResolution.attachments.length - ignoredReferenceAttachments,
6115
6685
  skippedReferenceAttachments: referenceResolution.skipped,
6686
+ ...(ignoredReferenceAttachments ? { ignoredReferenceAttachments } : {}),
6687
+ // How many images from imagePaths the turn has, and how many named
6688
+ // twice (the same file, or the same bytes) were kept once.
6689
+ ...(imagePathsGiven ? { imageInputs: imageInputs.length } : {}),
6690
+ ...(imageInputsDropped > 0 ? {
6691
+ imageInputsDropped,
6692
+ imageInputsNote: `${imageInputsDropped} of the ${imagePathList.length} image paths named an image already attached; it was attached once.`,
6693
+ } : {}),
6694
+ ...(imageOptions ? { imageOptions } : {}),
6116
6695
  // Attachment references that no longer resolve (a deleted upload) are
6117
6696
  // dropped silently by normalizeAttachments; the sender is told how many.
6118
6697
  droppedAttachmentCount: Math.max(0, requestedAttachmentCount - normalizedAttachments.length),
@@ -7605,8 +8184,516 @@ export function registerMessagesRoutes(app, deps) {
7605
8184
  return res.json({ ok: true, ...result });
7606
8185
  });
7607
8186
 
8187
+ // Image operations whose OpenAI request runs in this process, and the
8188
+ // direct image requests (no operation) this process sent, by queue
8189
+ // message: a second run of either would be billed again.
8190
+ const executingImageOperations = new Set();
8191
+ const dispatchedImageMessages = new Map();
8192
+ const DISPATCHED_IMAGE_MESSAGES_MAX = 500;
8193
+ const openAIImageTimeoutFromConfig = (key, fallback) => (Number(config?.[key]) > 0 ? Number(config[key]) : fallback);
8194
+ const openAIImageGenerateTimeoutMs = openAIImageTimeoutFromConfig('openaiImageGenerateTimeoutMs', OPENAI_IMAGE_GENERATE_TIMEOUT_MS);
8195
+ const openAIImageEditTimeoutMs = openAIImageTimeoutFromConfig('openaiImageEditTimeoutMs', OPENAI_IMAGE_EDIT_TIMEOUT_MS);
8196
+ // The partial previews of the running image turns: files beside the
8197
+ // turn's final images, events to the owner's clients (every socket is the
8198
+ // owner's: share views read over HTTP and have none).
8199
+ const imagePreviews = createOpenAIImagePreviewTurns({
8200
+ resolveSessionStateRoot,
8201
+ sessionIdFor: (conversationId) => String(stmts.getConvAnyStatus?.get?.(conversationId)?.sdk_session_id || '').trim() || conversationId,
8202
+ emit: (event, payload) => io?.emit?.(event, payload),
8203
+ ...(openAIImagePreviewLeftoverMs !== undefined ? { leftoverMs: openAIImagePreviewLeftoverMs } : {}),
8204
+ });
8205
+
8206
+ // A partial preview of a running image turn, for the owner's clients
8207
+ // only: only a file the turn wrote and still has is served (no path is
8208
+ // built from the request), never through a share link, and gone with the
8209
+ // turn.
8210
+ app.get(`/api/generated-image/:conversationId/:messageId/${OPENAI_IMAGE_PARTIALS_DIR}/:fileName`, auth, (req, res) => {
8211
+ const file = imagePreviews.partialFile({
8212
+ conversationId: String(req.params.conversationId || '').trim(),
8213
+ messageId: String(req.params.messageId || '').trim(),
8214
+ fileName: String(req.params.fileName || '').trim(),
8215
+ });
8216
+ // lstat: a link put in the turn's partials folder is not followed.
8217
+ let stat = null;
8218
+ try {
8219
+ stat = file ? fs.lstatSync(file.path) : null;
8220
+ } catch {}
8221
+ if (!stat?.isFile()) return res.status(404).json({ error: 'Preview not found' });
8222
+ return serveFileWithRangeSupport(req, res, file.path, { size: stat.size, contentType: file.mime }, {
8223
+ safeName: path.basename(file.path).replace(/"/g, ''),
8224
+ // And the open itself refuses a link swapped in after the lstat (POSIX;
8225
+ // Windows has no O_NOFOLLOW, the lstat is the guard there).
8226
+ fsImpl: PARTIAL_PREVIEW_FS,
8227
+ });
8228
+ });
8229
+
8230
+ function noteDispatchedImageMessage(messageId) {
8231
+ dispatchedImageMessages.delete(messageId);
8232
+ dispatchedImageMessages.set(messageId, { at: Date.now(), running: true });
8233
+ while (dispatchedImageMessages.size > DISPATCHED_IMAGE_MESSAGES_MAX) {
8234
+ dispatchedImageMessages.delete(dispatchedImageMessages.keys().next().value);
8235
+ }
8236
+ return dispatchedImageMessages.get(messageId);
8237
+ }
8238
+
8239
+ /**
8240
+ * One try of an OpenAI image request: `{ ok: true, generatedImages, usage }`
8241
+ * or `{ kind, status?, message?, detail? }` (see OPENAI_IMAGE_RETRY_KINDS).
8242
+ * The timeout covers the reply's body as well, a streamed one whole.
8243
+ * `fields` asking for a stream get one read event by event (a reply that is
8244
+ * no event stream is read as JSON); `onPartial` sees each partial image.
8245
+ */
8246
+ async function sendOpenAIImageRequestOnce({ url, headers, fields, editImages, family, timeoutMs, outputFormat, model, onPartial = null }) {
8247
+ // The images asked for: a stream's events past them are dropped.
8248
+ const expectedImages = Math.max(1, Math.trunc(Number(fields.n) || 1));
8249
+ let response;
8250
+ try {
8251
+ let body;
8252
+ if (editImages.length) {
8253
+ body = new FormData();
8254
+ for (const [key, value] of Object.entries(fields)) body.set(key, String(value));
8255
+ // gpt-image takes every image, the first being the base; dall-e-2
8256
+ // takes one.
8257
+ const images = family === 'gpt-image' ? editImages : editImages.slice(0, 1);
8258
+ for (const image of images) {
8259
+ body.append(
8260
+ family === 'gpt-image' ? 'image[]' : 'image',
8261
+ new Blob([image.bytes], { type: image.mimeType }),
8262
+ image.name,
8263
+ );
8264
+ }
8265
+ } else {
8266
+ body = JSON.stringify(fields);
8267
+ }
8268
+ response = await fetch(url, {
8269
+ method: 'POST',
8270
+ headers: editImages.length ? headers : { ...headers, 'Content-Type': 'application/json' },
8271
+ body,
8272
+ signal: AbortSignal.timeout(timeoutMs),
8273
+ });
8274
+ } catch (error) {
8275
+ const kind = classifyOpenAIImageFetchError(error);
8276
+ return { kind, ownTimeout: error?.name === 'TimeoutError', detail: describeFetchError(error) };
8277
+ }
8278
+
8279
+ const status = Number(response.status);
8280
+ if (response.ok && fields.stream === true && isEventStreamResponse(response)) {
8281
+ return readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages });
8282
+ }
8283
+ let rawText = '';
8284
+ try {
8285
+ rawText = String(await response.text()).trim();
8286
+ } catch (error) {
8287
+ // A 2xx whose body broke off is billed; an error status keeps its class.
8288
+ if (response.ok) return { kind: 'unprocessed', status, detail: `the reply broke off: ${describeFetchError(error)}` };
8289
+ return status >= 500 ? { kind: 'server', status } : { kind: 'refused', status };
8290
+ }
8291
+ let payload = {};
8292
+ if (rawText) {
8293
+ try {
8294
+ payload = JSON.parse(rawText);
8295
+ } catch {
8296
+ // After a 2xx the generation is billed; a retry pays again. An error
8297
+ // status with a body that is not JSON stays a provider failure.
8298
+ if (response.ok) return { kind: 'unprocessed', status, detail: 'OpenAI image API returned invalid JSON' };
8299
+ payload = null;
8300
+ }
8301
+ }
8302
+ if (!response.ok) {
8303
+ const message = String(payload?.error?.message || payload?.message || (payload ? '' : rawText)).trim().slice(0, 240);
8304
+ if (!Number.isFinite(status) || status >= 500) return { kind: 'server', status, message };
8305
+ return { kind: 'refused', status, message };
8306
+ }
8307
+ try {
8308
+ const generatedImages = normalizeOpenAIImageApiGeneratedImages(payload, {
8309
+ maxImages: MAX_UPLOAD_ATTACHMENTS,
8310
+ outputFormat,
8311
+ });
8312
+ return { ok: true, generatedImages, usage: normalizeOpenAIImageUsage(payload?.usage) };
8313
+ } catch (error) {
8314
+ // The usage the reply carried is what was billed.
8315
+ const usage = normalizeOpenAIImageUsage(payload?.usage);
8316
+ // The provider answered 2xx, so the generation is billed; what failed
8317
+ // is the relay's handling of the reply. Logged with the stack: a retry
8318
+ // pays again and fails the same way.
8319
+ console.error(
8320
+ `[${ts()}] IMAGE reply of ${model} not processed: ${error?.message || error} (body ${rawText.length} chars)\n${error?.stack || ''}`,
8321
+ );
8322
+ return { kind: 'unprocessed', status, detail: error?.message || 'OpenAI image API returned no images', ...(usage ? { usage } : {}) };
8323
+ }
8324
+ }
8325
+
8326
+ /**
8327
+ * The rest of a streamed reply (after a 2xx with an event stream): each
8328
+ * partial goes to `onPartial`, each completed event's image is kept in
8329
+ * order (a count > 1 turn may get several), and the completed events'
8330
+ * usages add up to the request's. A stream that breaks off, ends without an image
8331
+ * or reports a server error after OpenAI started may have been billed:
8332
+ * 'stream', 'timeout' or 'server', tried once more. An error event that
8333
+ * names a refusal is 'refused'; an image the relay cannot read, or an event
8334
+ * past the reader's cap, 'unprocessed'.
8335
+ *
8336
+ * Once every image asked for (`expectedImages`) is complete, a stream that
8337
+ * breaks off or times out before its end still answers them: they are
8338
+ * billed, and a retry would buy them again. Partials past
8339
+ * OPENAI_IMAGE_PARTIAL_IMAGES per image or past the images asked for, and
8340
+ * completed events past those images, are dropped unread.
8341
+ *
8342
+ * Every result carries `partials` (received) and, once a completed event
8343
+ * was read, `usage`: a try that fails after it books that usage
8344
+ * (openAIImageTryCharge).
8345
+ */
8346
+ async function readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages = 1 }) {
8347
+ const completed = [];
8348
+ let usage = null;
8349
+ let partials = 0;
8350
+ // What every result carries for the booking.
8351
+ const counted = (result) => ({ ...result, partials, ...(usage ? { usage } : {}) });
8352
+ const answer = () => {
8353
+ try {
8354
+ const generatedImages = normalizeOpenAIImageApiGeneratedImages({
8355
+ data: completed.map((payload) => ({ b64_json: payload.b64_json })),
8356
+ output_format: completed[0].output_format,
8357
+ }, { maxImages: MAX_UPLOAD_ATTACHMENTS, outputFormat });
8358
+ return counted({ ok: true, generatedImages });
8359
+ } catch (error) {
8360
+ console.error(`[${ts()}] IMAGE streamed reply of ${model} not processed: ${error?.message || error}\n${error?.stack || ''}`);
8361
+ return counted({ kind: 'unprocessed', status, detail: error?.message || 'OpenAI image stream returned no images' });
8362
+ }
8363
+ };
8364
+ try {
8365
+ for await (const sse of readSseEvents(response.body)) {
8366
+ const event = interpretOpenAIImageStreamEvent(sse);
8367
+ if (event.type === 'partial') {
8368
+ partials += 1;
8369
+ const partialIndex = Number.isInteger(event.payload.partial_image_index) ? event.payload.partial_image_index : partials - 1;
8370
+ const shown = completed.length < expectedImages && partialIndex >= 0 && partialIndex < OPENAI_IMAGE_PARTIAL_IMAGES;
8371
+ if (onPartial && shown) {
8372
+ try {
8373
+ await onPartial({
8374
+ imageIndex: completed.length,
8375
+ partialIndex,
8376
+ b64: event.payload.b64_json,
8377
+ outputFormat: event.payload.output_format || outputFormat,
8378
+ });
8379
+ } catch (error) {
8380
+ console.warn(`[${ts()}] IMAGE partial of ${model} not shown: ${error?.message || error}`);
8381
+ }
8382
+ }
8383
+ } else if (event.type === 'completed') {
8384
+ if (completed.length >= expectedImages) continue;
8385
+ completed.push(event.payload);
8386
+ usage = addOpenAIImageUsage(usage, normalizeOpenAIImageUsage(event.payload.usage));
8387
+ } else if (event.type === 'invalid' && sse.event.endsWith('.completed')) {
8388
+ return counted({ kind: 'unprocessed', status, detail: 'OpenAI image stream sent an unreadable image' });
8389
+ } else if (event.type === 'error') {
8390
+ const { message, type, code } = event.error;
8391
+ const named = `${type} ${code}`.toLowerCase();
8392
+ const detail = `error event in the stream${code ? ` (${code})` : ''} after ${partials} partial image(s)`;
8393
+ if (['server', 'internal', 'overloaded', 'timeout', 'unavailable'].some((word) => named.includes(word)) || !named.trim()) {
8394
+ return counted({ kind: 'server', message, detail });
8395
+ }
8396
+ return counted({ kind: 'refused', status: 400, message, detail, streamed: true });
8397
+ }
8398
+ }
8399
+ } catch (error) {
8400
+ if (error?.code === SSE_EVENT_TOO_LARGE_CODE) return counted({ kind: 'unprocessed', status, detail: error.message });
8401
+ if (completed.length >= expectedImages) {
8402
+ console.warn(`[${ts()}] IMAGE ${model} stream broke off after its ${completed.length} image(s): kept (${describeFetchError(error)})`);
8403
+ return answer();
8404
+ }
8405
+ const kind = classifyOpenAIImageFetchError(error);
8406
+ return counted({
8407
+ kind: kind === 'timeout' ? 'timeout' : 'stream',
8408
+ ownTimeout: error?.name === 'TimeoutError',
8409
+ detail: `the stream broke off after ${partials} partial image(s): ${describeFetchError(error)}`,
8410
+ });
8411
+ }
8412
+ if (!completed.length) return counted({ kind: 'stream', detail: `the stream ended without an image after ${partials} partial image(s)` });
8413
+ return answer();
8414
+ }
8415
+
8416
+ /**
8417
+ * The OpenAI image request of one turn, for the execute route and the
8418
+ * generate route alike: the request with every option and every input
8419
+ * image, a timeout per try (generate / edit), and one retry of a try that
8420
+ * may pass the second time, with the same Idempotency-Key (OpenAI may
8421
+ * ignore it; the note says the request may have been billed). Never more
8422
+ * than two requests: nothing else may send this turn again.
8423
+ *
8424
+ * `requestBody` is normalizeOpenAIImageGenerateRequestBody's. Resolves
8425
+ * `{ ok: true, generatedImages, model, endpoint, usage, tries, note? }`
8426
+ * (`note`: openAIImageRetriedNote's) or openAIImageFailureResult's failure.
8427
+ *
8428
+ * A gpt-image request is streamed with OPENAI_IMAGE_PARTIAL_IMAGES partials
8429
+ * per image (dall-e models do not stream). With `preview` ({ conversationId,
8430
+ * messageId }: the queue message the turn answers) the owner's clients see
8431
+ * each try start and each partial; a retry drops the first try's, and the
8432
+ * turn's end (/api/response) the rest.
8433
+ *
8434
+ * Each try is checked against the daily cap and reserves its estimate
8435
+ * right before it is sent (the resend without stream fields keeps the
8436
+ * reservation); the cap refuses a try with OPENAI_IMAGE_DAILY_CAP_REACHED
8437
+ * and sends nothing. When the try ends, its cost is booked under the queue
8438
+ * row (`scopeId`: the image attempt, or the claim, that runs it; run_id
8439
+ * image:<scopeId>:try<n>) and its reservation dropped, in one step.
8440
+ */
8441
+ async function runOpenAIImageRequest(requestBody, { idempotencyKey = '', preview = null, scopeId = '' } = {}) {
8442
+ const settings = getOpenAIProviderSettings();
8443
+ if (settings?.configured !== true || !String(settings?.apiKey || '').trim()) {
8444
+ return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI BYOK is not configured');
8445
+ }
8446
+ const baseUrl = String(settings.baseUrl || '').trim().replace(/\/+$/, '');
8447
+ if (!baseUrl) return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI base URL is not configured');
8448
+
8449
+ let editImages = [];
8450
+ try {
8451
+ editImages = resolveOpenAIImageEditAttachments(requestBody.attachments, {
8452
+ maxImageBytes: MAX_UPLOAD_BYTES,
8453
+ allowedRootPath: uploadsDir,
8454
+ allowedBlobRootPaths: uploadReadDirs.slice(1),
8455
+ allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
8456
+ });
8457
+ } catch (error) {
8458
+ return openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', error?.message || 'Invalid image attachment');
8459
+ }
8460
+
8461
+ const family = openAIImageModelFamily(requestBody.model);
8462
+ const fields = buildOpenAIImageRequestFields(requestBody);
8463
+ const endpoint = editImages.length ? '/images/edits' : '/images/generations';
8464
+ const timeoutMs = editImages.length ? openAIImageEditTimeoutMs : openAIImageGenerateTimeoutMs;
8465
+ const headers = {
8466
+ Authorization: `Bearer ${settings.apiKey}`,
8467
+ ...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
8468
+ };
8469
+ let streaming = family === 'gpt-image' && OPENAI_IMAGE_PARTIAL_IMAGES > 0;
8470
+ const previewTurn = preview
8471
+ ? imagePreviews.begin({
8472
+ conversationId: preview.conversationId,
8473
+ messageId: preview.messageId,
8474
+ size: requestBody.size,
8475
+ quality: requestBody.quality,
8476
+ count: requestBody.n,
8477
+ model: requestBody.model,
8478
+ })
8479
+ : null;
8480
+ const tries = [];
8481
+ // HTTP requests of this turn, the resend without stream fields included:
8482
+ // never more than two.
8483
+ let calls = 0;
8484
+ let resent = false;
8485
+ let result = null;
8486
+ // What a try of this request costs, and where it books.
8487
+ const spendRequest = {
8488
+ model: requestBody.model,
8489
+ size: requestBody.size,
8490
+ quality: requestBody.quality,
8491
+ count: requestBody.n,
8492
+ inputImages: editImages.length,
8493
+ };
8494
+ const bookingScope = String(scopeId || '').trim() || `claim_${randomUUID()}`;
8495
+ // The try in flight's reservation: held across the resend, released
8496
+ // (with its booking, or without one) when the try ends.
8497
+ let reservation = null;
8498
+ previewTurn?.started(1);
8499
+ try {
8500
+ for (;;) {
8501
+ if (!reservation) {
8502
+ const tryNumber = tries.length + 1;
8503
+ const reserved = imageSpend.reserveTry({ estimateUsd: estimateOpenAIImageCost(spendRequest).costUsd });
8504
+ if (!reserved.ok) {
8505
+ console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tryNumber} stopped by the daily cap`);
8506
+ result = openAIImageCapRefusalResult(tries, reserved);
8507
+ break;
8508
+ }
8509
+ reservation = reserved.reservation;
8510
+ }
8511
+ calls += 1;
8512
+ const tried = await sendOpenAIImageRequestOnce({
8513
+ url: `${baseUrl}${endpoint}`,
8514
+ // The resend's body differs from the first request's: its own key.
8515
+ headers: resent && idempotencyKey ? { ...headers, 'Idempotency-Key': `${idempotencyKey}-nostream` } : headers,
8516
+ fields: streaming ? { ...fields, stream: true, partial_images: OPENAI_IMAGE_PARTIAL_IMAGES } : fields,
8517
+ editImages,
8518
+ family,
8519
+ timeoutMs,
8520
+ outputFormat: requestBody.output_format,
8521
+ model: requestBody.model,
8522
+ onPartial: previewTurn ? (partial) => previewTurn.partial(partial) : null,
8523
+ });
8524
+ // The resend without stream fields (below) is the same try: it keeps
8525
+ // the reservation and books once.
8526
+ const resend = streaming && calls < 2 && tried.kind === 'refused' && Number(tried.status) === 400 && !tried.streamed
8527
+ && /\bstream|partial_image/i.test(String(tried.message || '').slice(0, 240));
8528
+ if (!resend) {
8529
+ imageSpend.settleTry(reservation, {
8530
+ conversationId: requestBody.conversationId,
8531
+ turnId: requestBody.messageId,
8532
+ runId: openAIImageTryRunId(bookingScope, tries.length + 1),
8533
+ model: requestBody.model,
8534
+ charge: openAIImageTryCharge({ tried, request: spendRequest }),
8535
+ });
8536
+ reservation = null;
8537
+ }
8538
+ if (tried.ok) {
8539
+ const note = openAIImageRetriedNote(tries, { timeoutMs });
8540
+ result = {
8541
+ ok: true,
8542
+ generatedImages: tried.generatedImages,
8543
+ model: requestBody.model,
8544
+ endpoint,
8545
+ usage: tried.usage,
8546
+ tries: tries.length + 1,
8547
+ ...(note ? { note } : {}),
8548
+ };
8549
+ break;
8550
+ }
8551
+ // An endpoint that takes no stream (an OpenAI-compatible one, or a
8552
+ // model OpenAI does not stream) refuses the fields with a 400 before
8553
+ // any work: the same try again without them, nothing billed. It
8554
+ // counts as one of the two requests, so a resend that fails is final.
8555
+ // Any other refusal (a 429 that names streaming, say) is no reason.
8556
+ if (resend) {
8557
+ console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} refused streaming; sent again without it`);
8558
+ streaming = false;
8559
+ resent = true;
8560
+ continue;
8561
+ }
8562
+ tries.push(tried);
8563
+ console.warn(
8564
+ `[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tries.length} ${tried.kind}`
8565
+ + `${tried.status ? ` status=${tried.status}` : ''}${tried.detail ? ` ${String(tried.detail).slice(0, 200)}` : ''}`,
8566
+ );
8567
+ if (calls >= 2 || !OPENAI_IMAGE_RETRY_KINDS.has(tried.kind)) break;
8568
+ await delay(openAIImageRetryDelayMs);
8569
+ previewTurn?.retry();
8570
+ previewTurn?.started(tries.length + 1);
8571
+ }
8572
+ } finally {
8573
+ // A throw or an abort: the try books nothing it did not book above.
8574
+ imageSpend.releaseTry(reservation);
8575
+ previewTurn?.ended({ ok: Boolean(result?.ok) });
8576
+ }
8577
+ return result || openAIImageFailureResult(tries, { timeoutMs, resent });
8578
+ }
8579
+
8580
+ /**
8581
+ * The reply line of an image turn: what came back, and what this turn's
8582
+ * tries were booked at ("~USD 0.21", marked "(estimate)" when any part is
8583
+ * one, "(price unknown, estimated)" for the stand-in); from 80 % of the
8584
+ * daily cap on, today's spend against it.
8585
+ */
8586
+ function openAIImageReplyLine({ images, model, turnId }) {
8587
+ let cost = null;
8588
+ let state = null;
8589
+ try {
8590
+ cost = imageSpend.turnCost(turnId);
8591
+ state = imageSpend.buildState();
8592
+ } catch (error) {
8593
+ console.warn(`[${ts()}] IMAGE the cost of ${String(turnId || '').slice(0, 8)} could not be read: ${error?.message || error}`);
8594
+ }
8595
+ let line = formatOpenAIImageSummary({ images, model, costUsd: cost ? cost.costUsd : null });
8596
+ if (cost?.unknownPrice) line += ' (price unknown, estimated)';
8597
+ else if (cost?.estimated) line += ' (estimate)';
8598
+ // The spend state's own rule (booked and in flight, `warning`), as the
8599
+ // banner reads it.
8600
+ if (state?.capUsd && state.warning) {
8601
+ const todayUsd = (Number(state.spentUsd) || 0) + (Number(state.reservedUsd) || 0);
8602
+ line += ` · today ${formatOpenAIImageUsd(todayUsd)} of ${state.capUsd.toFixed(2)} (${Math.round((todayUsd / state.capUsd) * 100)} %)`;
8603
+ }
8604
+ return line;
8605
+ }
8606
+
8607
+ // Today's OpenAI image spend against the daily cap (the Check Usage card,
8608
+ // the banner, a paired relay's check before create_session); with
8609
+ // `?conversationId=` also that chat's.
8610
+ app.get(OPENAI_IMAGE_USAGE_PATH, auth, (req, res) => {
8611
+ const conversationId = String(req.query?.conversationId || '').trim().slice(0, 200) || null;
8612
+ try {
8613
+ return res.json(imageSpend.buildState({ conversationId }));
8614
+ } catch (error) {
8615
+ return res.status(500).json({ error: `Image spend could not be read: ${error?.message || error}` });
8616
+ }
8617
+ });
8618
+
8619
+ /**
8620
+ * An image attempt the recovery found cut off while its request ran
8621
+ * ('started', now 'uncertain'): it may have been billed, so its estimate
8622
+ * is booked as image:<attemptId>:recovered, unless a try of that attempt
8623
+ * was booked already.
8624
+ */
8625
+ function bookRecoveredImageAttempt(operation, attempt) {
8626
+ if (!operation || !attempt?.id) return;
8627
+ let parameters = {};
8628
+ try {
8629
+ parameters = JSON.parse(operation.parameters_json || '{}') || {};
8630
+ } catch {}
8631
+ let inputImages = operation.parent_node_id ? 1 : 0;
8632
+ if (!operation.parent_node_id) {
8633
+ try {
8634
+ inputImages = (stmts.listAssets?.all?.(operation.id) || []).length;
8635
+ } catch {}
8636
+ }
8637
+ try {
8638
+ imageSpend.bookRecovered({
8639
+ conversationId: operation.conversation_id,
8640
+ turnId: operation.queue_message_id,
8641
+ attemptId: attempt.id,
8642
+ model: operation.selected_image_model,
8643
+ startedAt: attempt.started_at,
8644
+ request: { size: parameters.size, quality: parameters.quality, count: parameters.count ?? 1, inputImages },
8645
+ });
8646
+ } catch (error) {
8647
+ console.warn(`[${ts()}] IMAGE_OP the estimate of attempt=${attempt.id} was not booked: ${error?.message || error}`);
8648
+ }
8649
+ }
8650
+
8651
+ /**
8652
+ * Why a queue row may not go back to pending (/api/requeue): `{ running:
8653
+ * true }` while its image request runs in this process, `{ failure }`
8654
+ * (a terminalError) once its request went to OpenAI, or null for a row
8655
+ * whose request was never sent (requeueing it bills nothing).
8656
+ */
8657
+ function imageTurnRequeueStop(queueRow) {
8658
+ const operationId = String(queueRow?.image_operation_id || '').trim();
8659
+ if (operationId) {
8660
+ if (executingImageOperations.has(operationId)) return { running: true };
8661
+ const attempt = stmts.getLatestAttempt?.get?.(operationId) || null;
8662
+ if (!attempt) return null;
8663
+ if (attempt.status === 'started') {
8664
+ stmts.finishAttempt.run({
8665
+ id: attempt.id,
8666
+ status: 'uncertain',
8667
+ providerRequestId: null,
8668
+ providerConversationId: null,
8669
+ providerResponseId: null,
8670
+ httpStatus: null,
8671
+ errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
8672
+ errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
8673
+ completedAt: new Date().toISOString(),
8674
+ });
8675
+ bookRecoveredImageAttempt(stmts.getOperation?.get?.(operationId), attempt);
8676
+ return { failure: { code: 'IMAGE_ATTEMPT_INTERRUPTED', message: OPENAI_IMAGE_INTERRUPTED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
8677
+ }
8678
+ return {
8679
+ failure: {
8680
+ code: String(attempt.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED',
8681
+ message: String(attempt.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE,
8682
+ guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
8683
+ },
8684
+ };
8685
+ }
8686
+ const dispatched = dispatchedImageMessages.get(String(queueRow?.id || ''));
8687
+ if (!dispatched) return null;
8688
+ if (dispatched.running) return { running: true };
8689
+ return { failure: { code: 'IMAGE_ATTEMPT_REPEATED', message: OPENAI_IMAGE_REPEATED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
8690
+ }
8691
+
7608
8692
  // Called by the relay's own worker on this machine; nothing that came in
7609
8693
  // through a proxy or the tunnel may start a provider image request.
8694
+ // The request runs in this process (runOpenAIImageRequest): its timeout
8695
+ // and its one retry are the only ones. A failure ends the row with a note,
8696
+ // never with a requeue: a requeued image turn was billed again.
7610
8697
  app.post('/api/image-operations/:operationId/execute', auth, async (req, res) => {
7611
8698
  if (!isTrulyLocalRequest(req)) {
7612
8699
  return res.status(403).json({ error: 'Image operation execution is reachable from this machine only' });
@@ -7626,6 +8713,11 @@ export function registerMessagesRoutes(app, deps) {
7626
8713
  if (!queueRow || queueRow.image_operation_id !== operationId) {
7627
8714
  return res.status(409).json({ error: 'Image operation queue linkage is unavailable' });
7628
8715
  }
8716
+ // Delivered again while its request still runs here (a worker that
8717
+ // died and came back): that run finishes the row.
8718
+ if (executingImageOperations.has(operationId)) {
8719
+ return res.status(202).json({ ok: true, outcome: 'executing', operationId });
8720
+ }
7629
8721
 
7630
8722
  const now = new Date().toISOString();
7631
8723
  const selfHeaders = {
@@ -7633,6 +8725,23 @@ export function registerMessagesRoutes(app, deps) {
7633
8725
  'Content-Type': 'application/json',
7634
8726
  };
7635
8727
  const selfBaseUrl = `http://127.0.0.1:${config.port}`;
8728
+ const publishFailure = async (terminalError) => {
8729
+ // `terminalError` is the field /api/response reads; under any other
8730
+ // name the failure was refused as an empty reply and the row stayed
8731
+ // processing until the relay recovered and re-ran it.
8732
+ const response = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
8733
+ method: 'POST',
8734
+ headers: selfHeaders,
8735
+ body: JSON.stringify({
8736
+ messageId: operation.queue_message_id,
8737
+ conversationId: operation.conversation_id,
8738
+ terminalError,
8739
+ }),
8740
+ }).catch(() => null);
8741
+ if (!response?.ok) {
8742
+ console.warn(`[${ts()}] IMAGE_OP failed to publish the failure of operation=${operationId}`);
8743
+ }
8744
+ };
7636
8745
  const acquireAttempt = db.transaction(() => {
7637
8746
  const current = stmts.getLatestAttempt.get(operationId);
7638
8747
  if (current?.status === 'started') {
@@ -7650,15 +8759,19 @@ export function registerMessagesRoutes(app, deps) {
7650
8759
  providerResponseId: null,
7651
8760
  httpStatus: null,
7652
8761
  errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
7653
- errorMessage: 'Image execution was interrupted after provider dispatch may have begun',
8762
+ errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
7654
8763
  completedAt: now,
7655
8764
  });
7656
8765
  return { acquired: false, attempt: current, stale: true };
7657
8766
  }
8767
+ // An attempt that ended: its note was published, or publishing it
8768
+ // failed and the row came back. Either way the request is not sent
8769
+ // again.
8770
+ if (current) return { acquired: false, attempt: current, finished: true };
7658
8771
  const attempt = {
7659
8772
  id: `img_attempt_${uuidv4()}`,
7660
8773
  operationId,
7661
- attemptNumber: Math.max(1, Number(current?.attempt_number || 0) + 1),
8774
+ attemptNumber: 1,
7662
8775
  provider: operation.provider,
7663
8776
  capabilitySnapshotJson: JSON.stringify({
7664
8777
  executionMode: operation.execution_mode,
@@ -7671,312 +8784,250 @@ export function registerMessagesRoutes(app, deps) {
7671
8784
  });
7672
8785
  const acquired = acquireAttempt();
7673
8786
  if (acquired.stale) {
7674
- await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
7675
- method: 'POST',
7676
- headers: selfHeaders,
7677
- body: JSON.stringify({
7678
- messageId: operation.queue_message_id,
7679
- conversationId: operation.conversation_id,
7680
- // `terminalError` is the field /api/response reads; under any other
7681
- // name the failure was refused as an empty reply and the row stayed
7682
- // processing until the relay recovered and re-ran it.
7683
- terminalError: {
7684
- code: 'IMAGE_ATTEMPT_INTERRUPTED',
7685
- message: 'Image generation may have completed before the relay was interrupted. It was not retried to avoid duplicate images.',
7686
- retryable: false,
7687
- },
7688
- }),
7689
- }).catch(() => null);
8787
+ bookRecoveredImageAttempt(operation, acquired.attempt);
8788
+ await publishFailure({
8789
+ code: 'IMAGE_ATTEMPT_INTERRUPTED',
8790
+ message: OPENAI_IMAGE_INTERRUPTED_NOTE,
8791
+ guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
8792
+ retryable: false,
8793
+ });
7690
8794
  return res.status(409).json({
7691
8795
  ok: false,
7692
8796
  outcome: 'uncertain',
7693
8797
  operationId,
7694
- error: 'The previous image attempt ended in an uncertain state and was not retried',
8798
+ code: 'IMAGE_ATTEMPT_INTERRUPTED',
8799
+ error: OPENAI_IMAGE_INTERRUPTED_NOTE,
8800
+ });
8801
+ }
8802
+ if (acquired.finished) {
8803
+ const earlier = acquired.attempt;
8804
+ const message = String(earlier.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE;
8805
+ const code = String(earlier.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED';
8806
+ await publishFailure({ code, message, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE, retryable: false });
8807
+ return res.status(409).json({
8808
+ ok: false,
8809
+ outcome: ['failed_terminal', 'failed_retryable'].includes(earlier.status) ? 'terminal' : 'uncertain',
8810
+ operationId,
8811
+ code,
8812
+ error: message,
7695
8813
  });
7696
8814
  }
7697
8815
  if (!acquired.acquired) {
7698
8816
  return res.status(202).json({ ok: true, outcome: 'executing', operationId });
7699
8817
  }
7700
8818
 
7701
- const requestAttachments = [];
7702
- if (operation.parent_node_id) {
7703
- const parentNode = stmts.getNode.get(operation.parent_node_id);
7704
- const parentMessage = parentNode
7705
- ? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
7706
- : null;
7707
- const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
7708
- String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
7709
- ));
7710
- const generatedImage = parentAttachment?.generatedImage;
7711
- const parentPath = resolveGeneratedImageReadPath({
7712
- resolveSessionStateRoot,
7713
- sessionId: generatedImage?.sessionId,
7714
- relativePath: generatedImage?.relativePath,
7715
- });
7716
- if (!parentPath) {
8819
+ executingImageOperations.add(operationId);
8820
+ try {
8821
+ const finishAttempt = (result) => {
7717
8822
  stmts.finishAttempt.run({
7718
8823
  id: acquired.attempt.id,
7719
- status: 'failed_terminal',
8824
+ // A request that never reached OpenAI (unreachable, or stopped by
8825
+ // the daily cap before it was sent) closes as retryable: nothing
8826
+ // was billed, and the recovery books nothing for it.
8827
+ status: result.outcome === 'uncertain'
8828
+ ? 'uncertain'
8829
+ : ((result.code === 'IMAGE_PROVIDER_UNREACHABLE' || (result.code === OPENAI_IMAGE_DAILY_CAP_REACHED_CODE && !result.sent))
8830
+ ? 'failed_retryable'
8831
+ : 'failed_terminal'),
7720
8832
  providerRequestId: null,
7721
8833
  providerConversationId: null,
7722
8834
  providerResponseId: null,
7723
- httpStatus: 400,
7724
- errorCode: 'IMAGE_PARENT_UNAVAILABLE',
7725
- errorMessage: 'The parent generated image is unavailable',
8835
+ httpStatus: result.upstreamStatus || null,
8836
+ errorCode: result.code,
8837
+ errorMessage: String(result.error || '').slice(0, 500),
7726
8838
  completedAt: new Date().toISOString(),
7727
8839
  });
8840
+ };
8841
+ const fail = async (result) => {
8842
+ finishAttempt(result);
8843
+ await publishFailure(openAIImageTerminalError(result));
7728
8844
  return res.status(409).json({
7729
8845
  ok: false,
7730
- outcome: 'terminal',
7731
- code: 'IMAGE_PARENT_UNAVAILABLE',
7732
- error: 'The parent generated image is unavailable',
8846
+ outcome: result.outcome,
8847
+ operationId,
8848
+ code: result.code,
8849
+ error: result.error,
8850
+ ...(result.usage ? { usage: result.usage } : {}),
7733
8851
  });
7734
- }
7735
- requestAttachments.push({
7736
- name: parentAttachment.name,
7737
- type: parentAttachment.type,
7738
- path: parentPath,
7739
- });
7740
- } else {
7741
- for (const asset of stmts.listAssets.all(operationId)) {
8852
+ };
8853
+
8854
+ const requestAttachments = [];
8855
+ if (operation.parent_node_id) {
8856
+ const parentNode = stmts.getNode.get(operation.parent_node_id);
8857
+ const parentMessage = parentNode
8858
+ ? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
8859
+ : null;
8860
+ const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
8861
+ String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
8862
+ ));
8863
+ const generatedImage = parentAttachment?.generatedImage;
8864
+ const parentPath = resolveGeneratedImageReadPath({
8865
+ resolveSessionStateRoot,
8866
+ sessionId: generatedImage?.sessionId,
8867
+ relativePath: generatedImage?.relativePath,
8868
+ });
8869
+ if (!parentPath) {
8870
+ return await fail({
8871
+ ...openAIImageRequestRefusal(400, 'IMAGE_PARENT_UNAVAILABLE', 'The parent generated image is unavailable'),
8872
+ upstreamStatus: 400,
8873
+ });
8874
+ }
7742
8875
  requestAttachments.push({
7743
- name: asset.original_name || 'reference-image',
7744
- type: asset.media_type,
7745
- path: uploadPathForSha(asset.content_sha256),
8876
+ name: parentAttachment.name,
8877
+ type: parentAttachment.type,
8878
+ path: parentPath,
7746
8879
  });
8880
+ } else {
8881
+ // In the order they were attached: the first is the base image.
8882
+ for (const asset of stmts.listAssets.all(operationId)) {
8883
+ requestAttachments.push({
8884
+ name: asset.original_name || 'reference-image',
8885
+ type: asset.media_type,
8886
+ path: uploadPathForSha(asset.content_sha256),
8887
+ });
8888
+ }
7747
8889
  }
7748
- }
7749
8890
 
7750
- let parameters = {};
7751
- try {
7752
- parameters = JSON.parse(operation.parameters_json || '{}');
7753
- } catch {}
7754
- const generationBody = {
7755
- messageId: operation.queue_message_id,
7756
- conversationId: operation.conversation_id,
7757
- model: operation.selected_image_model,
7758
- prompt: operation.prompt,
7759
- n: Number.isInteger(Number(parameters.count)) ? Number(parameters.count) : 1,
7760
- attachments: requestAttachments,
7761
- };
7762
- if (parameters.size === 'auto' || /^\d{2,5}x\d{2,5}$/i.test(String(parameters.size || ''))) {
7763
- generationBody.size = parameters.size;
7764
- }
7765
- if (/^(auto|standard|hd|low|medium|high)$/i.test(String(parameters.quality || ''))) {
7766
- generationBody.quality = parameters.quality;
7767
- }
7768
-
7769
- let generatedPayload = null;
7770
- try {
7771
- const generatedResponse = await fetch(`${selfBaseUrl}${remotePath}/api/openai/images/generate`, {
7772
- method: 'POST',
7773
- headers: selfHeaders,
7774
- body: JSON.stringify(generationBody),
7775
- });
7776
- generatedPayload = await generatedResponse.json().catch(() => ({}));
7777
- if (!generatedResponse.ok) {
7778
- const error = new Error(generatedPayload?.error || `OpenAI image request failed (${generatedResponse.status})`);
7779
- error.status = generatedResponse.status;
7780
- throw error;
7781
- }
7782
- const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
7783
- method: 'POST',
7784
- headers: selfHeaders,
7785
- body: JSON.stringify({
8891
+ // {size, quality, count, output_format, background}; any may be
8892
+ // missing, count is 1 then.
8893
+ let parameters = {};
8894
+ try {
8895
+ parameters = JSON.parse(operation.parameters_json || '{}') || {};
8896
+ } catch {}
8897
+ let requestBody;
8898
+ try {
8899
+ requestBody = normalizeOpenAIImageGenerateRequestBody({
7786
8900
  messageId: operation.queue_message_id,
7787
8901
  conversationId: operation.conversation_id,
7788
- text: '',
7789
- generatedImages: generatedPayload.generatedImages,
7790
- model: generatedPayload.model || operation.selected_image_model,
7791
- }),
7792
- });
7793
- const finalized = await finalizeResponse.json().catch(() => ({}));
7794
- if (!finalizeResponse.ok) {
7795
- const error = new Error(finalized?.error || `Image finalization failed (${finalizeResponse.status})`);
7796
- error.status = finalizeResponse.status;
7797
- throw error;
8902
+ model: operation.selected_image_model,
8903
+ prompt: operation.prompt,
8904
+ n: parameters.count ?? 1,
8905
+ size: parameters.size,
8906
+ quality: parameters.quality,
8907
+ output_format: parameters.output_format,
8908
+ background: parameters.background,
8909
+ attachments: requestAttachments,
8910
+ });
8911
+ } catch (error) {
8912
+ return await fail(openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', String(error?.message || 'Invalid image request')));
7798
8913
  }
7799
- return res.json({
7800
- ok: true,
7801
- outcome: 'completed',
7802
- operationId,
7803
- generatedImageCount: Array.isArray(generatedPayload.generatedImages)
7804
- ? generatedPayload.generatedImages.length
7805
- : 0,
7806
- });
7807
- } catch (error) {
7808
- const status = Number(error?.status || 0);
7809
- const terminal = status >= 400 && status < 500;
7810
- const uncertain = !!generatedPayload?.generatedImages;
7811
- const outcome = terminal ? 'terminal' : (uncertain ? 'uncertain' : 'retryable');
7812
- stmts.finishAttempt.run({
7813
- id: acquired.attempt.id,
7814
- status: terminal ? 'failed_terminal' : (uncertain ? 'uncertain' : 'failed_retryable'),
7815
- providerRequestId: null,
7816
- providerConversationId: null,
7817
- providerResponseId: null,
7818
- httpStatus: status || null,
7819
- errorCode: terminal ? 'IMAGE_PROVIDER_REJECTED' : (uncertain ? 'IMAGE_FINALIZATION_UNCERTAIN' : 'IMAGE_PROVIDER_UNAVAILABLE'),
7820
- errorMessage: String(error?.message || 'Image operation failed').slice(0, 500),
7821
- completedAt: new Date().toISOString(),
8914
+
8915
+ const result = await runOpenAIImageRequest(requestBody, {
8916
+ idempotencyKey: String(operation.idempotency_key || operation.id),
8917
+ preview: { conversationId: operation.conversation_id, messageId: operation.queue_message_id },
8918
+ scopeId: acquired.attempt.id,
7822
8919
  });
7823
- if (outcome === 'retryable') {
7824
- const requeueResponse = await fetch(`${selfBaseUrl}${remotePath}/api/requeue`, {
7825
- method: 'POST',
7826
- headers: selfHeaders,
7827
- body: JSON.stringify({ messageId: operation.queue_message_id }),
7828
- }).catch(() => null);
7829
- if (!requeueResponse?.ok) {
7830
- console.warn(`[${ts()}] IMAGE_OP failed to requeue retryable operation=${operationId}`);
7831
- }
7832
- } else {
7833
- const failure = {
7834
- code: outcome === 'uncertain' ? 'IMAGE_FINALIZATION_UNCERTAIN' : 'IMAGE_PROVIDER_REJECTED',
7835
- message: String(error?.message || 'Image operation failed').slice(0, 500),
7836
- retryable: false,
7837
- };
7838
- const failResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
8920
+ if (!result.ok) return await fail(result);
8921
+
8922
+ let finalizeError = null;
8923
+ try {
8924
+ const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
7839
8925
  method: 'POST',
7840
8926
  headers: selfHeaders,
7841
8927
  body: JSON.stringify({
7842
8928
  messageId: operation.queue_message_id,
7843
8929
  conversationId: operation.conversation_id,
7844
- terminalError: failure,
8930
+ text: '',
8931
+ generatedImages: result.generatedImages,
8932
+ model: result.model || operation.selected_image_model,
8933
+ // A first try that may have been billed too: the reply says so.
8934
+ ...(result.note ? { imageNote: result.note } : {}),
7845
8935
  }),
7846
- }).catch(() => null);
7847
- if (!failResponse?.ok) {
7848
- console.warn(`[${ts()}] IMAGE_OP failed to finalize ${outcome} operation=${operationId}`);
7849
- }
8936
+ });
8937
+ const finalized = await finalizeResponse.json().catch(() => ({}));
8938
+ if (!finalizeResponse.ok) finalizeError = finalized?.error || `Image finalization failed (${finalizeResponse.status})`;
8939
+ } catch (error) {
8940
+ finalizeError = error?.message || 'Image finalization failed';
7850
8941
  }
7851
- return res.status(outcome === 'retryable' ? 503 : 409).json({
7852
- ok: false,
7853
- outcome,
8942
+ if (finalizeError) {
8943
+ return await fail({
8944
+ ok: false,
8945
+ outcome: 'uncertain',
8946
+ code: 'IMAGE_FINALIZATION_UNCERTAIN',
8947
+ error: `The images OpenAI returned could not be saved (${finalizeError}). The generation was billed; the message was not retried.`,
8948
+ upstreamStatus: 200,
8949
+ usage: result.usage,
8950
+ });
8951
+ }
8952
+ return res.json({
8953
+ ok: true,
8954
+ outcome: 'completed',
7854
8955
  operationId,
7855
- error: String(error?.message || 'Image operation failed'),
8956
+ generatedImageCount: result.generatedImages.length,
8957
+ usage: result.usage,
8958
+ ...(result.note ? { note: result.note } : {}),
7856
8959
  });
8960
+ } finally {
8961
+ executingImageOperations.delete(operationId);
7857
8962
  }
7858
8963
  });
7859
8964
 
8965
+ // The relay's own workers call this on this machine for an image turn
8966
+ // without an operation (image continuity off). Like the execute route it
8967
+ // is local only: through the tunnel, or with a paired relay's token, it
8968
+ // would spend the owner's money outside the queue. One request per queue
8969
+ // message: a second call for the same message is refused.
7860
8970
  app.post('/api/openai/images/generate', auth, async (req, res) => {
7861
8971
  touchCli();
7862
- const settings = getOpenAIProviderSettings();
7863
- if (settings?.configured !== true || !String(settings?.apiKey || '').trim()) {
7864
- return res.status(503).json({ error: 'OpenAI BYOK is not configured' });
8972
+ if (!isTrulyLocalRequest(req)) {
8973
+ return res.status(403).json({ error: 'OpenAI image requests are reachable from this machine only' });
7865
8974
  }
7866
8975
 
7867
8976
  let requestBody;
7868
8977
  try {
7869
- requestBody = normalizeOpenAIImageGenerateRequestBody(req.body, {
7870
- maxImages: MAX_UPLOAD_ATTACHMENTS,
7871
- });
8978
+ requestBody = normalizeOpenAIImageGenerateRequestBody(req.body);
7872
8979
  } catch (error) {
7873
- return res.status(400).json({ error: error?.message || 'Invalid OpenAI image request' });
7874
- }
7875
-
7876
- const baseUrl = String(settings.baseUrl || '').trim().replace(/\/+$/, '');
7877
- if (!baseUrl) return res.status(503).json({ error: 'OpenAI base URL is not configured' });
7878
-
7879
- let editAttachment = null;
7880
- if (Array.isArray(requestBody.attachments) && requestBody.attachments.length > 0) {
7881
- try {
7882
- editAttachment = resolveOpenAIImageEditAttachment(requestBody.attachments, {
7883
- maxImageBytes: MAX_UPLOAD_BYTES,
7884
- allowedRootPath: uploadsDir,
7885
- allowedBlobRootPaths: uploadReadDirs.slice(1),
7886
- allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
7887
- });
7888
- } catch (error) {
7889
- return res.status(400).json({ error: error?.message || 'Invalid image attachment' });
7890
- }
7891
- }
7892
-
7893
- const options = {
7894
- model: requestBody.model,
7895
- prompt: requestBody.prompt,
7896
- };
7897
- const supportsResponseFormat = String(requestBody.model || '')
7898
- .trim()
7899
- .toLowerCase()
7900
- .replace(/^openai\//, '')
7901
- .startsWith('dall-e-');
7902
- if (supportsResponseFormat) options.response_format = 'b64_json';
7903
- if (requestBody.n !== undefined) options.n = requestBody.n;
7904
- if (requestBody.size) options.size = requestBody.size;
7905
- if (requestBody.quality) options.quality = requestBody.quality;
7906
-
7907
- const endpoint = editAttachment ? '/images/edits' : '/images/generations';
7908
- let response;
7909
- try {
7910
- if (editAttachment) {
7911
- const form = new FormData();
7912
- form.set('model', options.model);
7913
- form.set('prompt', options.prompt);
7914
- if (options.response_format) form.set('response_format', String(options.response_format));
7915
- if (options.n !== undefined) form.set('n', String(options.n));
7916
- if (options.size) form.set('size', String(options.size));
7917
- if (options.quality) form.set('quality', String(options.quality));
7918
- form.set(
7919
- 'image',
7920
- new Blob([editAttachment.bytes], { type: editAttachment.mimeType }),
7921
- editAttachment.name,
7922
- );
7923
- response = await fetch(`${baseUrl}${endpoint}`, {
7924
- method: 'POST',
7925
- headers: {
7926
- Authorization: `Bearer ${settings.apiKey}`,
7927
- },
7928
- body: form,
7929
- });
7930
- } else {
7931
- response = await fetch(`${baseUrl}${endpoint}`, {
7932
- method: 'POST',
7933
- headers: {
7934
- Authorization: `Bearer ${settings.apiKey}`,
7935
- 'Content-Type': 'application/json',
7936
- },
7937
- body: JSON.stringify(options),
7938
- });
7939
- }
7940
- } catch (error) {
7941
- return res.status(502).json({ error: `OpenAI image API request failed: ${error?.message || 'request error'}` });
7942
- }
7943
-
7944
- const rawText = String(await response.text().catch(() => '')).trim();
7945
- let payload = {};
7946
- if (rawText) {
7947
- try {
7948
- payload = JSON.parse(rawText);
7949
- } catch {
7950
- return res.status(502).json({ error: 'OpenAI image API returned invalid JSON' });
7951
- }
8980
+ return res.status(400).json({
8981
+ error: error?.message || 'Invalid OpenAI image request',
8982
+ code: 'IMAGE_REQUEST_INVALID',
8983
+ outcome: 'terminal',
8984
+ });
7952
8985
  }
7953
-
7954
- if (!response.ok) {
7955
- const detail = String(payload?.error?.message || payload?.message || rawText).trim().slice(0, 240);
7956
- const upstreamStatus = Number(response.status);
7957
- const relayStatus = Number.isFinite(upstreamStatus) && upstreamStatus >= 400 && upstreamStatus < 500
7958
- ? upstreamStatus
7959
- : 502;
7960
- return res.status(relayStatus).json({
7961
- error: `OpenAI image API failed (${response.status})${detail ? `: ${detail}` : ''}`,
7962
- upstreamStatus: Number.isFinite(upstreamStatus) ? upstreamStatus : null,
8986
+ if (dispatchedImageMessages.has(requestBody.messageId)) {
8987
+ return res.status(409).json({
8988
+ error: OPENAI_IMAGE_REPEATED_NOTE,
8989
+ code: 'IMAGE_ATTEMPT_REPEATED',
8990
+ outcome: 'uncertain',
7963
8991
  });
7964
8992
  }
7965
8993
 
7966
- let generatedImages = [];
8994
+ const dispatched = noteDispatchedImageMessage(requestBody.messageId);
8995
+ // One run of the message: the claim's attempt id (a restart claims the
8996
+ // row anew), so its tries never merge into an earlier run's rows.
8997
+ // Known gap: this path (image continuity off) keeps no attempt row, so a
8998
+ // try a relay restart cuts off is booked nowhere; the operation path's
8999
+ // recovery books such a try as an estimate. docs/providers.md says so.
9000
+ let claimId = '';
7967
9001
  try {
7968
- generatedImages = normalizeOpenAIImageApiGeneratedImages(payload, {
7969
- maxImages: MAX_UPLOAD_ATTACHMENTS,
9002
+ claimId = String(stmts.findQById?.get?.(requestBody.messageId)?.attempt_id || '').trim();
9003
+ } catch {}
9004
+ const result = await runOpenAIImageRequest(requestBody, {
9005
+ idempotencyKey: `oar-image-${requestBody.messageId}`,
9006
+ preview: { conversationId: requestBody.conversationId, messageId: requestBody.messageId },
9007
+ scopeId: claimId ? `claim_${claimId}` : '',
9008
+ });
9009
+ // A request that never reached OpenAI may be sent again.
9010
+ if (result.ok || result.sent) dispatched.running = false;
9011
+ else dispatchedImageMessages.delete(requestBody.messageId);
9012
+
9013
+ if (!result.ok) {
9014
+ return res.status(result.status).json({
9015
+ error: result.error,
9016
+ code: result.code,
9017
+ outcome: result.outcome,
9018
+ upstreamStatus: result.upstreamStatus,
9019
+ ...(result.detail ? { detail: result.detail } : {}),
9020
+ ...(result.relayProcessing ? { relayProcessing: true } : {}),
7970
9021
  });
7971
- } catch (error) {
7972
- return res.status(502).json({ error: error?.message || 'OpenAI image API returned no images' });
7973
9022
  }
7974
-
7975
9023
  return res.json({
7976
9024
  ok: true,
7977
- generatedImages,
7978
- model: requestBody.model,
7979
- endpoint,
9025
+ generatedImages: result.generatedImages,
9026
+ model: result.model,
9027
+ endpoint: result.endpoint,
9028
+ usage: result.usage,
9029
+ // The worker hands it to /api/response as `imageNote`.
9030
+ ...(result.note ? { note: result.note } : {}),
7980
9031
  });
7981
9032
  });
7982
9033
 
@@ -8075,12 +9126,16 @@ export function registerMessagesRoutes(app, deps) {
8075
9126
  console.log(`[${ts()}] FENCE ${messageId?.slice(0,8)} legacy unfenced response write`);
8076
9127
  }
8077
9128
 
9129
+ // The partial previews of an image turn go once its reply or its
9130
+ // failure note is stored (or was already).
8078
9131
  if (q && q.status === 'done') {
8079
9132
  console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_done`);
9133
+ imagePreviews.finish(messageId, 'done');
8080
9134
  return res.json({ ok: true, ignored: 'already_done' });
8081
9135
  }
8082
9136
  if (q && q.status === 'failed') {
8083
9137
  console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_failed`);
9138
+ imagePreviews.finish(messageId, 'failed');
8084
9139
  return res.json({ ok: true, ignored: 'already_failed' });
8085
9140
  }
8086
9141
 
@@ -8185,6 +9240,7 @@ export function registerMessagesRoutes(app, deps) {
8185
9240
  + `${terminalFailure.functionCallId ? ` call=${terminalFailure.functionCallId}` : ''}`
8186
9241
  + `${terminalFailure.requestId ? ` req=${terminalFailure.requestId}` : ''}`,
8187
9242
  );
9243
+ imagePreviews.finish(messageId, 'failed');
8188
9244
  return res.json({ ok: true, terminal: true, code: terminalFailure.stableCode });
8189
9245
  }
8190
9246
 
@@ -8192,7 +9248,7 @@ export function registerMessagesRoutes(app, deps) {
8192
9248
  const hasPendingQuestion = !!stmts.findPendingQuestionByMessage?.get(messageId);
8193
9249
  const hasRecentlyAnsweredQuestion = !!stmts.findRecentlyAnsweredQuestionByMessage?.get(messageId, holdCutoff);
8194
9250
  const conversation = stmts.getConvAnyStatus?.get?.(targetConversationId) || null;
8195
- const resolvedText = await resolveRelayResponseText({
9251
+ let resolvedText = await resolveRelayResponseText({
8196
9252
  text: trimmedText,
8197
9253
  conversation,
8198
9254
  queueTimestamp: q?.timestamp || null,
@@ -8275,6 +9331,29 @@ export function registerMessagesRoutes(app, deps) {
8275
9331
  } catch (error) {
8276
9332
  return res.status(500).json({ error: error?.message || 'Failed to persist generated images' });
8277
9333
  }
9334
+ // An image turn whose model wrote no text: the reply line says what came
9335
+ // back ("1 image · 1536x1024 · gpt-image-2"), so the chat and an agent
9336
+ // that reads text only see the same.
9337
+ const imageTurnModel = [resolvedAssistantModel, requestedModel].find((candidate) => isOpenAIImageModelId(candidate)) || '';
9338
+ const imageSummaryLine = Boolean(!resolvedText && generatedImageAttachments.length && (q?.image_operation_id || imageTurnModel));
9339
+ if (imageSummaryLine) {
9340
+ resolvedText = openAIImageReplyLine({
9341
+ images: generatedImageAttachments,
9342
+ model: imageTurnModel || resolvedAssistantModel || '',
9343
+ turnId: messageId,
9344
+ });
9345
+ }
9346
+ // The image request came through on its second try, and the first may
9347
+ // have been billed as well: the reply says so, on the reply line when
9348
+ // that is the text, so an agent that reads text only sees it too.
9349
+ const imageNote = generatedImageAttachments.length
9350
+ ? String(req.body.imageNote || '').trim().slice(0, OPENAI_IMAGE_NOTE_MAX)
9351
+ : '';
9352
+ if (imageNote) {
9353
+ resolvedText = imageSummaryLine
9354
+ ? `${resolvedText}. Note: ${imageNote}`
9355
+ : `${String(resolvedText || '').trimEnd()}${resolvedText ? '\n\n' : ''}Note: ${imageNote}`;
9356
+ }
8278
9357
  const assistantAttachments = generatedImageAttachments.length ? JSON.stringify(generatedImageAttachments) : null;
8279
9358
  const now = new Date().toISOString();
8280
9359
  const imageOperation = q?.image_operation_id
@@ -8405,8 +9484,10 @@ export function registerMessagesRoutes(app, deps) {
8405
9484
  // Ensure UI is updated even if the response was duplicate/late
8406
9485
  if (currentStatus === 'done') {
8407
9486
  io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
9487
+ imagePreviews.finish(messageId, 'done');
8408
9488
  } else if (currentStatus === 'failed') {
8409
9489
  io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'failed' });
9490
+ imagePreviews.finish(messageId, 'failed');
8410
9491
  }
8411
9492
  return res.json({ ok: true, ignored: 'not_pending_or_processing', actualStatus: currentStatus });
8412
9493
  }
@@ -8555,6 +9636,8 @@ export function registerMessagesRoutes(app, deps) {
8555
9636
  },
8556
9637
  });
8557
9638
  io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
9639
+ // The final images are in the chat: the previews go.
9640
+ imagePreviews.finish(messageId, generatedImageAttachments.length ? 'done' : 'failed');
8558
9641
  if (usageLimitPause) {
8559
9642
  try {
8560
9643
  usageLimitPauseService.schedulePause(usageLimitPause, { queueRow: q });
@@ -9459,7 +10542,13 @@ export function registerMessagesRoutes(app, deps) {
9459
10542
  });
9460
10543
  }
9461
10544
 
9462
- io.emit('relay_thought', {
10545
+ // An update in flight (done false) is volatile, as relay_stream: each one
10546
+ // carries the whole thought so far (up to MAX_THOUGHT_CHARS), so keeping
10547
+ // them in the connection-state-recovery buffer, or queueing them on a
10548
+ // socket that does not drain, holds a turn's worth of copies — gigabytes
10549
+ // for a few long reasoning turns at once. A reconnect resync reads the
10550
+ // stored snapshot instead; the final thought (done) is never dropped.
10551
+ (done ? io : io.volatile).emit('relay_thought', {
9463
10552
  messageId,
9464
10553
  conversationId,
9465
10554
  mode: relayMode,
@@ -9731,6 +10820,41 @@ export function registerMessagesRoutes(app, deps) {
9731
10820
  failConsumedQueueRow(q, { reason: 'requeue', attemptId: claimedAttemptId });
9732
10821
  return res.json({ ok: true, terminal: true, code: STEER_SETTLE_FAILED_STABLE_CODE });
9733
10822
  }
10823
+ if (q && q.status === 'processing') {
10824
+ // An image turn whose request went to OpenAI is never run again: a
10825
+ // second run is billed again. One still running here finishes the row
10826
+ // itself; one that ended, or was cut off, ends with the note.
10827
+ const imageStop = imageTurnRequeueStop(q);
10828
+ if (imageStop?.running) return res.json({ ok: true, held: 'image-request-running' });
10829
+ if (imageStop) {
10830
+ const failed = failQueueMessage({
10831
+ queueRow: q,
10832
+ messageId,
10833
+ conversationId: q.conversation_id,
10834
+ relayMode: normalizeRelayMode(q.relay_mode) || DEFAULT_RELAY_MODE,
10835
+ model: q.model || null,
10836
+ responseText: buildTerminalFailureTextForChat(imageStop.failure),
10837
+ failureRecord: {
10838
+ kind: 'terminal',
10839
+ error: 'image-not-requeued',
10840
+ code: normalizeTerminalErrorCode(imageStop.failure.code),
10841
+ stableCode: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}`,
10842
+ message: imageStop.failure.message,
10843
+ guidance: imageStop.failure.guidance,
10844
+ retryCount: Number(q.retry_count || 0),
10845
+ failedAt: new Date().toISOString(),
10846
+ },
10847
+ attemptId: claimedAttemptId,
10848
+ });
10849
+ if (failed) {
10850
+ settleRelayAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
10851
+ settleRelaySubagentAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
10852
+ console.warn(`[${ts()}] FAILED ${messageId?.slice(0,8)} reason=image-not-requeued code=${imageStop.failure.code}`);
10853
+ imagePreviews.finish(messageId, 'failed');
10854
+ }
10855
+ return res.json({ ok: true, terminal: true, code: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}` });
10856
+ }
10857
+ }
9734
10858
  if (q && q.status === 'processing' && String(req.body?.code || '').trim() === 'relay.provider-mismatch') {
9735
10859
  // A routing refusal, not a delivery failure: the legacy relay handed
9736
10860
  // back a turn that belongs to a session worker. No retry increment and