@oar-sh/oar 0.9.24 → 0.9.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/.github/extensions/web-relay/skills/reasoning-stream.mjs +3 -1
  2. package/CHANGELOG.md +284 -0
  3. package/README.md +12 -11
  4. package/package.json +5 -1
  5. package/server/claude-worker/claude-remote-relay-tool.mjs +15 -0
  6. package/server/claude-worker/claude-sdk-adapter.mjs +8 -4
  7. package/server/claude-worker/claude-session-process.mjs +4 -0
  8. package/server/claude-worker/claude-turn-publisher.mjs +25 -11
  9. package/server/claude-worker/sdk-message-normalizer.mjs +29 -1
  10. package/server/copilot-worker/copilot-prompt-context.mjs +2 -1
  11. package/server/copilot-worker/copilot-sdk-session-process.mjs +11 -9
  12. package/server/cursor-worker/cursor-turn-runner.mjs +3 -1
  13. package/server/cursor-worker/sdk-message-normalizer.mjs +13 -0
  14. package/server/db-schema.mjs +19 -0
  15. package/server/grok-worker/grok-turn-runner.mjs +3 -1
  16. package/server/grok-worker/sdk-message-normalizer.mjs +3 -0
  17. package/server/migrations/0039-media-index.mjs +167 -0
  18. package/server/migrations/0040-image-session-defaults.mjs +93 -0
  19. package/server/public/app/api-client.js +58 -0
  20. package/server/public/app/ask-user-view.js +2 -1
  21. package/server/public/app/attachments-view.js +66 -9
  22. package/server/public/app/bootstrap.js +85 -25
  23. package/server/public/app/chat-actions-menu-state.mjs +1 -0
  24. package/server/public/app/context-media-tab.js +143 -0
  25. package/server/public/app/context-usage-view.mjs +2 -2
  26. package/server/public/app/conversation-view.js +90 -5
  27. package/server/public/app/image-composer-ui.js +518 -0
  28. package/server/public/app/image-turn-placeholder.mjs +233 -0
  29. package/server/public/app/journal-view.js +12 -2
  30. package/server/public/app/live-activity-cap.mjs +3 -3
  31. package/server/public/app/markdown-math.mjs +471 -0
  32. package/server/public/app/media-gallery-model.mjs +202 -0
  33. package/server/public/app/media-gallery-ui.js +524 -0
  34. package/server/public/app/media-view.js +79 -0
  35. package/server/public/app/openai-image-composer.mjs +492 -0
  36. package/server/public/app/openai-image-cost-ui.js +426 -0
  37. package/server/public/app/openai-image-spend.mjs +443 -0
  38. package/server/public/app/plan-usage-view.mjs +38 -6
  39. package/server/public/app/pwa-install.js +25 -4
  40. package/server/public/app/relay-error-ctas.mjs +5 -0
  41. package/server/public/app/relay-prompt-sanitizer.mjs +3 -0
  42. package/server/public/app/relay-restart-ui.js +2 -2
  43. package/server/public/app/remote-relays-settings.js +33 -6
  44. package/server/public/app/router.js +49 -237
  45. package/server/public/app/session-history-ui.js +2 -0
  46. package/server/public/app/settings-modal.js +4 -0
  47. package/server/public/app/settings-tabs.js +10 -1
  48. package/server/public/app/socket-handlers.js +26 -0
  49. package/server/public/app/store.js +2 -0
  50. package/server/public/app/stream-view-preference.mjs +1 -1
  51. package/server/public/app/tab-strip-fade.mjs +1 -1
  52. package/server/public/app/thoughts-preference.mjs +1 -1
  53. package/server/public/app/tmux-inspector-view.js +29 -16
  54. package/server/public/app/viewer-gallery.mjs +23 -1
  55. package/server/public/index.html +517 -123
  56. package/server/relay-tools.md +1 -0
  57. package/server/repositories/image-conversation-repository.mjs +69 -0
  58. package/server/repositories/media-repository.mjs +305 -0
  59. package/server/routes/media-routes.mjs +361 -0
  60. package/server/routes/messages-routes.mjs +1410 -360
  61. package/server/routes/relay-board-routes.mjs +46 -1
  62. package/server/routes/remote-relay-routes.mjs +1 -1
  63. package/server/routes/sessions-routes.mjs +361 -14
  64. package/server/server-runtime.mjs +21 -0
  65. package/server/services/database-file.mjs +6 -4
  66. package/server/services/generated-image-cleanup-service.mjs +2 -1
  67. package/server/services/generated-image-files.mjs +110 -0
  68. package/server/services/image-input-paths.mjs +389 -0
  69. package/server/services/image-operation-service.mjs +6 -2
  70. package/server/services/media-extract.mjs +653 -0
  71. package/server/services/media-index-service.mjs +221 -0
  72. package/server/services/openai-image-preview-turns.mjs +330 -0
  73. package/server/services/openai-image-spend-service.mjs +516 -0
  74. package/server/services/openai-image-stream.mjs +170 -0
  75. package/server/services/plan-usage-openai-image.mjs +148 -0
  76. package/server/services/relay-prompt-sanitizer.mjs +7 -0
  77. package/server/services/relay-restart-queue.mjs +1 -1
  78. package/server/services/remote-relay-caller-context.mjs +6 -0
  79. package/server/services/remote-relay-dispatcher.mjs +409 -23
  80. package/server/services/remote-relay-registry-service.mjs +10 -2
  81. package/server/services/session-worker-stop-service.mjs +50 -18
  82. package/server/services/token-usage-service.mjs +15 -0
  83. package/shared/image-header.mjs +90 -0
  84. package/shared/math-instructions.mjs +30 -0
  85. package/shared/openai-image-options.mjs +299 -0
  86. package/shared/openai-image-presets.mjs +175 -0
  87. package/shared/openai-image-previews.mjs +40 -0
  88. package/shared/openai-image-prices.mjs +291 -0
  89. package/shared/openai-image-spend.mjs +90 -0
  90. package/shared/remote-relay-contract.mjs +126 -4
  91. package/shared/worker-runtime/direct-openai-image.mjs +96 -50
  92. package/server/public/app/math-dollars.mjs +0 -89
@@ -6,6 +6,7 @@ import { randomUUID } from 'crypto';
6
6
  import { stripRelayPromptContext } from '../services/relay-prompt-sanitizer.mjs';
7
7
  import { resolveUploadMimeType } from '../services/mime-sniffer.mjs';
8
8
  import { fileVersionToken, serveFileWithRangeSupport, withFileVersion } from '../services/file-serving.mjs';
9
+ import { registerMediaRoutes } from './media-routes.mjs';
9
10
  import { validateEntryName } from '../services/explorer-entry-name.mjs';
10
11
  import { countEntries, createDirectory, removeEntry, renameEntry } from '../services/explorer-mutations.mjs';
11
12
  import {
@@ -64,6 +65,47 @@ import {
64
65
  unknownProviderModelText,
65
66
  } from '../services/provider-model-selection.mjs';
66
67
  import { stripRemotePromptHeader } from '../../shared/remote-relay-contract.mjs';
68
+ import {
69
+ OPENAI_IMAGE_LIMITS,
70
+ OPENAI_IMAGE_MIME_BY_FORMAT,
71
+ OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
72
+ checkOpenAIImageSize,
73
+ formatOpenAIImageSummary,
74
+ isOpenAIImageModelId,
75
+ mergeOpenAIImageOptions,
76
+ openAIImageModelFamily,
77
+ openAIImageModelNeedsOpenAIText,
78
+ validateOpenAIImageOptions,
79
+ } from '../../shared/openai-image-options.mjs';
80
+ import { OPENAI_IMAGE_PARTIAL_IMAGES, OPENAI_IMAGE_PARTIALS_DIR } from '../../shared/openai-image-previews.mjs';
81
+ import {
82
+ SSE_EVENT_TOO_LARGE_CODE,
83
+ interpretOpenAIImageStreamEvent,
84
+ isEventStreamResponse,
85
+ readSseEvents,
86
+ } from '../services/openai-image-stream.mjs';
87
+ import { createOpenAIImagePreviewTurns } from '../services/openai-image-preview-turns.mjs';
88
+ import {
89
+ OPENAI_IMAGE_CAP_WARN_RATIO,
90
+ OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
91
+ OPENAI_IMAGE_USAGE_PATH,
92
+ formatOpenAIImageUsd,
93
+ } from '../../shared/openai-image-spend.mjs';
94
+ import { estimateOpenAIImageCost } from '../../shared/openai-image-prices.mjs';
95
+ import {
96
+ OPENAI_IMAGE_PRESETS_SETTING_KEY,
97
+ findOpenAIImagePreset,
98
+ parseOwnOpenAIImagePresets,
99
+ splitOpenAIImagePreset,
100
+ } from '../../shared/openai-image-presets.mjs';
101
+ import {
102
+ createOpenAIImageSpendService,
103
+ openAIImageCapRefusalText,
104
+ openAIImageTryCharge,
105
+ openAIImageTryRunId,
106
+ } from '../services/openai-image-spend-service.mjs';
107
+ import { IMAGE_INPUT_REFUSED_CODE, resolveImageInputRoots, resolveImageInputs } from '../services/image-input-paths.mjs';
108
+ import { IMAGE_HEADER_BYTES, readImageHeader } from '../../shared/image-header.mjs';
67
109
  import { stripModelTierSuffix } from '../../shared/claude-cloud/repo-url.mjs';
68
110
  import {
69
111
  createSubagentTimelineService,
@@ -206,35 +248,33 @@ function isAutoModel(model) {
206
248
  return String(model || '').trim().toLowerCase() === AUTO_MODEL_SENTINEL;
207
249
  }
208
250
 
209
- function isOpenAIImageModelId(model = '') {
210
- const normalized = String(model || '').trim().toLowerCase().replace(/^openai\//, '');
211
- return normalized.startsWith('gpt-image-') || normalized.startsWith('dall-e-');
212
- }
213
-
214
- const OPENAI_IMAGE_QUALITY_VALUES = new Set(['auto', 'low', 'medium', 'high']);
215
- const OPENAI_IMAGE_SIZE_VALUES = new Set([
216
- 'auto',
217
- '256x256',
218
- '512x512',
219
- '1024x1024',
220
- '1536x1024',
221
- '1024x1536',
222
- '1792x1024',
223
- '1024x1792',
224
- ]);
225
-
251
+ // A quality or size any image model family takes (shared/openai-image-options.mjs),
252
+ // or ''. The model's own family is checked where the request is built.
226
253
  function normalizeOpenAIImageQuality(value = '') {
227
- const normalized = String(value || '').trim().toLowerCase();
228
- return OPENAI_IMAGE_QUALITY_VALUES.has(normalized) ? normalized : '';
254
+ const checked = validateOpenAIImageOptions({ quality: value });
255
+ return checked.ok ? (checked.values.quality || '') : '';
229
256
  }
230
257
 
231
258
  function normalizeOpenAIImageSize(value = '') {
232
259
  const normalized = String(value || '').trim().toLowerCase();
233
- if (OPENAI_IMAGE_SIZE_VALUES.has(normalized)) return normalized;
260
+ // The composer's size select is the context-tier select: its own values
261
+ // mean the API's default size.
234
262
  if (normalized === 'default' || normalized === 'long_context') return 'auto';
235
- return '';
263
+ if (!normalized) return '';
264
+ return checkOpenAIImageSize(normalized).size || '';
236
265
  }
237
266
 
267
+ // How long one OpenAI image request may take, from sending it to the last
268
+ // byte of the reply; config.json `openaiImageGenerateTimeoutMs` and
269
+ // `openaiImageEditTimeoutMs` override them. Above 300 s undici's own header
270
+ // timeout ends a request first.
271
+ export const OPENAI_IMAGE_GENERATE_TIMEOUT_MS = 180_000;
272
+ export const OPENAI_IMAGE_EDIT_TIMEOUT_MS = 300_000;
273
+ // The pause before the one retry of a request that may pass the second time.
274
+ const OPENAI_IMAGE_RETRY_DELAY_MS = 2_000;
275
+ // OpenAI takes up to 16 input images on an edit.
276
+ const OPENAI_IMAGE_EDIT_INPUTS_MAX = 16;
277
+
238
278
  function normalizeSessionWorkerId(value) {
239
279
  const text = String(value || '').trim();
240
280
  return text || null;
@@ -1445,6 +1485,7 @@ export function buildDequeuedRelayMessage({
1445
1485
  try {
1446
1486
  messageKind = String(stmts.getMessageByConversation?.get?.(msg.id, msg.conversation_id)?.kind || '').trim();
1447
1487
  } catch {}
1488
+ const imageOptions = parseQueueImageOptions(msg.image_options_json);
1448
1489
  return {
1449
1490
  id: msg.id,
1450
1491
  conversationId: msg.conversation_id,
@@ -1477,10 +1518,23 @@ export function buildDequeuedRelayMessage({
1477
1518
  ownerAssignedAt: msg.owner_assigned_at || null,
1478
1519
  ownerLeaseExpiresAt: msg.owner_lease_expires_at || null,
1479
1520
  imageOperationId: String(msg.image_operation_id || '').trim() || null,
1521
+ // An image turn's options as the send settled them; a turn without an
1522
+ // operation sends them with its direct request.
1523
+ ...(imageOptions ? { imageOptions } : {}),
1480
1524
  ...(isRelayAutomatedMessageKind(messageKind) ? { messageKind } : {}),
1481
1525
  };
1482
1526
  }
1483
1527
 
1528
+ function parseQueueImageOptions(value) {
1529
+ if (!value) return null;
1530
+ try {
1531
+ const parsed = JSON.parse(String(value));
1532
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
1533
+ } catch {
1534
+ return null;
1535
+ }
1536
+ }
1537
+
1484
1538
  function buildAttachmentPromptContext(attachments) {
1485
1539
  if (!Array.isArray(attachments) || !attachments.length) return '';
1486
1540
  const lines = [
@@ -1631,49 +1685,35 @@ function parseOpenAIImageBase64Value(value) {
1631
1685
  return Buffer.from(raw, 'base64').toString('base64');
1632
1686
  }
1633
1687
 
1634
- function normalizeOpenAIImageOptionString(value, fieldName) {
1635
- if (value === undefined || value === null) return undefined;
1636
- const normalized = String(value).trim();
1637
- if (!normalized) return undefined;
1638
- if (!/^[a-z0-9._-]{2,32}$/i.test(normalized)) {
1639
- throw new Error(`Invalid OpenAI image option "${fieldName}"`);
1640
- }
1641
- return normalized;
1642
- }
1643
-
1644
- export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 10 } = {}) {
1688
+ /**
1689
+ * The body of one image request (the generate route's, or the one the
1690
+ * execute route builds from an operation), checked against the model's
1691
+ * family. `n` is the count; size and quality "auto" pass and are left out
1692
+ * for a dall-e model when the request is built. A prompt that came from an
1693
+ * agent on another relay loses its "[Remote prompt from …]" header: OpenAI
1694
+ * would draw it.
1695
+ */
1696
+ export function normalizeOpenAIImageGenerateRequestBody(rawBody) {
1645
1697
  const body = rawBody && typeof rawBody === 'object' ? rawBody : {};
1646
1698
  const messageId = String(body.messageId || '').trim();
1647
1699
  const conversationId = String(body.conversationId || '').trim();
1648
1700
  const model = String(body.model || '').trim();
1649
- const prompt = String(body.prompt || '').trim();
1701
+ const prompt = stripRemotePromptHeader(String(body.prompt || '')).trim();
1650
1702
  if (!messageId) throw new Error('Missing messageId');
1651
1703
  if (!conversationId) throw new Error('Missing conversationId');
1652
1704
  if (!model) throw new Error('Missing model');
1653
1705
  if (!prompt) throw new Error('Missing prompt');
1654
1706
 
1655
- const hasN = Object.prototype.hasOwnProperty.call(body, 'n');
1656
- let n = undefined;
1657
- if (hasN) {
1658
- const parsed = Number(body.n);
1659
- const maxN = Math.max(1, Math.trunc(Number(maxImages) || 10));
1660
- if (!Number.isInteger(parsed) || parsed < 1 || parsed > maxN) {
1661
- throw new Error(`Invalid n value (must be an integer between 1 and ${maxN})`);
1662
- }
1663
- n = parsed;
1664
- }
1707
+ const legacySize = String(body.size ?? '').trim().toLowerCase();
1708
+ const checked = validateOpenAIImageOptions({
1709
+ size: legacySize === 'default' || legacySize === 'long_context' ? 'auto' : body.size,
1710
+ quality: body.quality,
1711
+ count: body.n,
1712
+ output_format: body.output_format,
1713
+ background: body.background,
1714
+ }, { model, field: 'image' });
1715
+ if (!checked.ok) throw new Error(checked.error);
1665
1716
 
1666
- const sizeRaw = body.size;
1667
- let size = undefined;
1668
- if (sizeRaw !== undefined && sizeRaw !== null && String(sizeRaw).trim()) {
1669
- const normalizedSize = String(sizeRaw).trim().toLowerCase();
1670
- if (normalizedSize !== 'auto' && !/^\d{2,5}x\d{2,5}$/i.test(normalizedSize)) {
1671
- throw new Error('Invalid size value (expected "<width>x<height>" or "auto")');
1672
- }
1673
- size = normalizedSize;
1674
- }
1675
-
1676
- const quality = normalizeOpenAIImageOptionString(body.quality, 'quality');
1677
1717
  const attachments = body.attachments;
1678
1718
  if (attachments !== undefined && !Array.isArray(attachments)) {
1679
1719
  throw new Error('attachments must be an array when provided');
@@ -1684,13 +1724,288 @@ export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 1
1684
1724
  conversationId,
1685
1725
  model,
1686
1726
  prompt,
1687
- n,
1688
- size,
1689
- quality,
1727
+ n: checked.values.count,
1728
+ size: checked.values.size,
1729
+ quality: checked.values.quality,
1730
+ output_format: checked.values.output_format,
1731
+ background: checked.values.background,
1690
1732
  attachments: Array.isArray(attachments) ? attachments : [],
1691
1733
  };
1692
1734
  }
1693
1735
 
1736
+ /**
1737
+ * The fields OpenAI gets for one image request. dall-e models take no
1738
+ * output_format or background, and their "auto" size or quality is not sent
1739
+ * (the API's own default); they answer in b64_json only when asked to.
1740
+ */
1741
+ export function buildOpenAIImageRequestFields(requestBody = {}) {
1742
+ const family = openAIImageModelFamily(requestBody.model);
1743
+ const isGptImage = family === 'gpt-image';
1744
+ const fields = { model: requestBody.model, prompt: requestBody.prompt };
1745
+ if (!isGptImage) fields.response_format = 'b64_json';
1746
+ if (requestBody.n !== undefined) fields.n = requestBody.n;
1747
+ for (const key of ['size', 'quality']) {
1748
+ const value = requestBody[key];
1749
+ if (value && (isGptImage || value !== 'auto')) fields[key] = value;
1750
+ }
1751
+ if (isGptImage) {
1752
+ if (requestBody.output_format) fields.output_format = requestBody.output_format;
1753
+ if (requestBody.background) fields.background = requestBody.background;
1754
+ }
1755
+ return fields;
1756
+ }
1757
+
1758
+ const OPENAI_IMAGE_USAGE_DETAIL_KEYS = Object.freeze(['input_tokens_details', 'output_tokens_details']);
1759
+
1760
+ /**
1761
+ * The `usage` block of an OpenAI image reply, numbers only (top level and
1762
+ * the two token-detail blocks), or null when there is none.
1763
+ */
1764
+ export function normalizeOpenAIImageUsage(raw) {
1765
+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
1766
+ const numbersOf = (source) => {
1767
+ const out = {};
1768
+ for (const [key, value] of Object.entries(source || {})) {
1769
+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) out[key] = value;
1770
+ }
1771
+ return out;
1772
+ };
1773
+ const usage = numbersOf(raw);
1774
+ for (const key of OPENAI_IMAGE_USAGE_DETAIL_KEYS) {
1775
+ const detail = raw[key];
1776
+ if (detail && typeof detail === 'object' && !Array.isArray(detail)) {
1777
+ const numbers = numbersOf(detail);
1778
+ if (Object.keys(numbers).length) usage[key] = numbers;
1779
+ }
1780
+ }
1781
+ return Object.keys(usage).length ? usage : null;
1782
+ }
1783
+
1784
+ /**
1785
+ * Two image usages added up, field by field and detail by detail (a stream
1786
+ * of a count > 1 request reports one per completed image); null + null is
1787
+ * null.
1788
+ */
1789
+ export function addOpenAIImageUsage(a, b) {
1790
+ if (!a) return b || null;
1791
+ if (!b) return a;
1792
+ const sum = { ...a };
1793
+ for (const [key, value] of Object.entries(b)) {
1794
+ if (typeof value === 'number') sum[key] = (typeof sum[key] === 'number' ? sum[key] : 0) + value;
1795
+ else if (value && typeof value === 'object') sum[key] = addOpenAIImageUsage(sum[key] && typeof sum[key] === 'object' ? sum[key] : null, value);
1796
+ }
1797
+ return sum;
1798
+ }
1799
+
1800
+ // The base64 characters that decode to the first IMAGE_HEADER_BYTES bytes.
1801
+ const IMAGE_HEADER_BASE64_CHARS = Math.ceil(IMAGE_HEADER_BYTES / 3) * 4;
1802
+
1803
+ /** readImageHeader of a canonical base64 payload, decoding only its start. */
1804
+ function readBase64ImageHeader(base64) {
1805
+ const text = String(base64 || '');
1806
+ if (!text) return null;
1807
+ return readImageHeader(Buffer.from(text.slice(0, IMAGE_HEADER_BASE64_CHARS), 'base64'));
1808
+ }
1809
+
1810
+ // What one OpenAI image request came to, as `kind`:
1811
+ // - 'unsent': it never reached OpenAI (no connection, no name, no TLS);
1812
+ // nothing was billed.
1813
+ // - 'timeout', 'broken', 'server': no answer in time, the connection broke
1814
+ // off after the request may have gone out, or a 5xx; it may have been
1815
+ // billed.
1816
+ // - 'stream': a streamed reply that broke off, or ended without its image,
1817
+ // after OpenAI had started on it; it may have been billed.
1818
+ // - 'refused': a 4xx (429 included); nothing came of it, a retry is refused
1819
+ // the same way.
1820
+ // - 'unprocessed': a 2xx the relay could not read; billed.
1821
+ // The first five are tried once more; the last two never.
1822
+ const OPENAI_IMAGE_RETRY_KINDS = new Set(['unsent', 'timeout', 'broken', 'server', 'stream']);
1823
+ const OPENAI_IMAGE_AMBIGUOUS_KINDS = new Set(['timeout', 'broken', 'server', 'stream']);
1824
+ const OPENAI_IMAGE_UNSENT_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT']);
1825
+ const OPENAI_IMAGE_TIMEOUT_ERROR_CODES = new Set(['UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']);
1826
+
1827
+ export const OPENAI_IMAGE_INTERRUPTED_NOTE = 'The relay was interrupted while OpenAI was working on the image; the request may have been billed; retry the message.';
1828
+ export const OPENAI_IMAGE_REPEATED_NOTE = 'This image request was already sent to OpenAI once and was not sent again; the first request may have been billed; retry the message.';
1829
+ // The guidance under every image failure note: the note itself says what to do.
1830
+ const OPENAI_IMAGE_FAILURE_GUIDANCE = 'If this keeps failing, include the error code when you report it.';
1831
+
1832
+ /** The `code` of a thrown fetch error and of every cause under it. */
1833
+ function fetchErrorCodes(error) {
1834
+ const codes = [];
1835
+ const seen = new Set();
1836
+ const visit = (entry, depth) => {
1837
+ if (!entry || typeof entry !== 'object' || seen.has(entry) || depth > 4) return;
1838
+ seen.add(entry);
1839
+ if (entry.code) codes.push(String(entry.code));
1840
+ visit(entry.cause, depth + 1);
1841
+ if (Array.isArray(entry.errors)) for (const inner of entry.errors) visit(inner, depth + 1);
1842
+ };
1843
+ visit(error, 0);
1844
+ return codes;
1845
+ }
1846
+
1847
+ function isTlsFailureCode(code) {
1848
+ return code.startsWith('ERR_TLS_') || code.startsWith('ERR_SSL_') || code.includes('CERT') || code === 'EPROTO';
1849
+ }
1850
+
1851
+ /** 'unsent', 'timeout' or 'broken' for an error fetch threw (see OPENAI_IMAGE_RETRY_KINDS). */
1852
+ export function classifyOpenAIImageFetchError(error) {
1853
+ if (error?.name === 'TimeoutError' || error?.cause?.name === 'TimeoutError') return 'timeout';
1854
+ const codes = fetchErrorCodes(error);
1855
+ if (codes.some((code) => OPENAI_IMAGE_TIMEOUT_ERROR_CODES.has(code))) return 'timeout';
1856
+ if (codes.length && codes.every((code) => OPENAI_IMAGE_UNSENT_ERROR_CODES.has(code) || isTlsFailureCode(code))) return 'unsent';
1857
+ return 'broken';
1858
+ }
1859
+
1860
+ function describeFetchError(error) {
1861
+ const codes = [...new Set(fetchErrorCodes(error))];
1862
+ const message = String(error?.message || error || 'request error').trim();
1863
+ return codes.length ? `${message} (${codes.join(', ')})` : message;
1864
+ }
1865
+
1866
+ function formatNoteDuration(ms) {
1867
+ const value = Math.max(0, Number(ms) || 0);
1868
+ return value >= 1000 ? `${Math.round(value / 1000)} s` : `${Math.round(value)} ms`;
1869
+ }
1870
+
1871
+ function asSentence(text) {
1872
+ const value = String(text || '').trim();
1873
+ return !value || /[.!?]$/.test(value) ? value : `${value}.`;
1874
+ }
1875
+
1876
+ /**
1877
+ * The failure of an image request after its tries: the note the chat shows
1878
+ * (`error`), the attempt's code, `outcome` 'uncertain' when it may have been
1879
+ * billed (else 'terminal'), and the HTTP status the generate route answers.
1880
+ * A request that never reached OpenAI on its last try but may have been
1881
+ * billed on the first is told as the first.
1882
+ */
1883
+ export function openAIImageFailureResult(tries, { timeoutMs = 0, resent = false } = {}) {
1884
+ const list = Array.isArray(tries) && tries.length ? tries : [{ kind: 'broken' }];
1885
+ const last = list[list.length - 1];
1886
+ const ambiguous = list.find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry.kind)) || null;
1887
+ const deciding = last.kind === 'unsent' && ambiguous ? ambiguous : last;
1888
+ // A resend without stream fields is the second request of a turn whose
1889
+ // first was refused for them before any work.
1890
+ const triedText = resent ? 'sent twice, once without previews' : (list.length > 1 ? 'tried twice' : 'tried once');
1891
+ const maybeBilled = 'the request may have been billed; retry the message.';
1892
+ // A refusal or an unreadable reply after a first try that may have been
1893
+ // billed: the note says so too.
1894
+ const earlierBilled = ambiguous && !OPENAI_IMAGE_AMBIGUOUS_KINDS.has(deciding.kind) && deciding.kind !== 'unsent'
1895
+ ? ' The first try may have been billed as well.'
1896
+ : '';
1897
+ const base = {
1898
+ ok: false,
1899
+ tries: list.length,
1900
+ sent: list.some((entry) => entry.kind !== 'unsent'),
1901
+ ...(deciding.detail ? { detail: String(deciding.detail).slice(0, 500) } : {}),
1902
+ upstreamStatus: Number.isFinite(Number(deciding.status)) && Number(deciding.status) > 0 ? Number(deciding.status) : null,
1903
+ };
1904
+ switch (deciding.kind) {
1905
+ case 'unsent':
1906
+ return { ...base, status: 502, outcome: 'terminal', code: 'IMAGE_PROVIDER_UNREACHABLE', error: 'OpenAI could not be reached; nothing was billed; retry the message.' };
1907
+ case 'timeout': {
1908
+ const waited = deciding.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
1909
+ return { ...base, status: 504, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `OpenAI did not answer ${waited} (${triedText}); ${maybeBilled}` };
1910
+ }
1911
+ case 'server':
1912
+ return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_UNAVAILABLE', error: `OpenAI answered ${deciding.status || 'with a server error'} (${triedText}); ${maybeBilled}` };
1913
+ case 'stream':
1914
+ return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_STREAM_BROKEN', error: `OpenAI's image stream broke off before the image was complete (${triedText}); ${maybeBilled}` };
1915
+ case 'refused': {
1916
+ const message = String(deciding.message || '').trim().slice(0, 240);
1917
+ return {
1918
+ ...base,
1919
+ status: deciding.status >= 400 && deciding.status < 500 ? deciding.status : 502,
1920
+ outcome: 'terminal',
1921
+ code: 'IMAGE_PROVIDER_REJECTED',
1922
+ error: `${asSentence(`OpenAI refused the image request (${deciding.status})${message ? `: ${message}` : ''}`)}${earlierBilled}`,
1923
+ };
1924
+ }
1925
+ case 'unprocessed':
1926
+ return {
1927
+ ...base,
1928
+ status: 502,
1929
+ outcome: 'terminal',
1930
+ code: 'IMAGE_REPLY_UNPROCESSED',
1931
+ relayProcessing: true,
1932
+ error: `The relay could not process the image OpenAI returned (${deciding.detail || 'unknown error'}). The generation was billed; the message was not retried.${earlierBilled}`,
1933
+ };
1934
+ default:
1935
+ return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `The connection to OpenAI broke off (${triedText}); ${maybeBilled}` };
1936
+ }
1937
+ }
1938
+
1939
+ /**
1940
+ * The note on an image request that came through on its second try after a
1941
+ * first that may have reached OpenAI (OPENAI_IMAGE_AMBIGUOUS_KINDS): both may
1942
+ * have been billed. Empty when the earlier tries never went out.
1943
+ */
1944
+ export function openAIImageRetriedNote(tries, { timeoutMs = 0 } = {}) {
1945
+ const first = (Array.isArray(tries) ? tries : []).find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
1946
+ if (!first) return '';
1947
+ const tail = 'it was sent once more and the first try may have been billed as well.';
1948
+ switch (first.kind) {
1949
+ case 'timeout': {
1950
+ const waited = first.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
1951
+ return `OpenAI did not answer the first try ${waited}; ${tail}`;
1952
+ }
1953
+ case 'server':
1954
+ return `OpenAI answered the first try with ${first.status || 'a server error'}; ${tail}`;
1955
+ case 'stream':
1956
+ return `OpenAI's image stream broke off on the first try; ${tail}`;
1957
+ default:
1958
+ return `The connection to OpenAI broke off on the first try; ${tail}`;
1959
+ }
1960
+ }
1961
+
1962
+ /**
1963
+ * The failure of an image request whose next try the daily cap stopped
1964
+ * before it was sent (`refusal`: the spend service's reserveTry). Nothing of
1965
+ * that try reached OpenAI; an earlier try may have been billed, and the note
1966
+ * says so then.
1967
+ */
1968
+ export function openAIImageCapRefusalResult(tries, refusal = {}) {
1969
+ const list = Array.isArray(tries) ? tries : [];
1970
+ const earlierBilled = list.some((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
1971
+ return {
1972
+ ok: false,
1973
+ tries: list.length,
1974
+ sent: list.some((entry) => entry?.kind !== 'unsent'),
1975
+ status: 409,
1976
+ outcome: earlierBilled ? 'uncertain' : 'terminal',
1977
+ code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
1978
+ error: openAIImageCapRefusalText({ tryNumber: earlierBilled ? 2 : 1, refusal }),
1979
+ upstreamStatus: null,
1980
+ };
1981
+ }
1982
+
1983
+ // The file system the partial preview route serves through: its open
1984
+ // follows no link.
1985
+ const PARTIAL_PREVIEW_FS = Object.freeze({
1986
+ ...fs,
1987
+ openSync: (file) => fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)),
1988
+ });
1989
+
1990
+ // The longest image note /api/response takes into the reply text.
1991
+ const OPENAI_IMAGE_NOTE_MAX = 500;
1992
+
1993
+ /** A request refused before anything was sent to OpenAI. */
1994
+ function openAIImageRequestRefusal(status, code, error) {
1995
+ return { ok: false, tries: 0, sent: false, status, outcome: 'terminal', code, error, upstreamStatus: null };
1996
+ }
1997
+
1998
+ /** The `terminalError` /api/response gets for a failed image request. */
1999
+ function openAIImageTerminalError(result) {
2000
+ return {
2001
+ code: result.code,
2002
+ message: String(result.error || 'Image operation failed').slice(0, 500),
2003
+ ...(result.detail ? { detail: String(result.detail).slice(0, 500) } : {}),
2004
+ guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
2005
+ retryable: false,
2006
+ };
2007
+ }
2008
+
1694
2009
  function isOpenAIImageAttachment(raw) {
1695
2010
  if (!raw || typeof raw !== 'object') return false;
1696
2011
  const type = String(raw.type || '').trim().toLowerCase();
@@ -1699,7 +2014,26 @@ function isOpenAIImageAttachment(raw) {
1699
2014
  return dataUrl.startsWith('data:image/');
1700
2015
  }
1701
2016
 
1702
- export function resolveOpenAIImageEditAttachment(rawAttachments, {
2017
+ export function resolveOpenAIImageEditAttachment(rawAttachments, options = {}) {
2018
+ return resolveOpenAIImageEditAttachments(rawAttachments, { ...options, maxImages: 1 })[0] || null;
2019
+ }
2020
+
2021
+ /**
2022
+ * Every image attachment of an edit, in the order given (the first is the
2023
+ * base image), at most `maxImages`; the others are ignored.
2024
+ */
2025
+ export function resolveOpenAIImageEditAttachments(rawAttachments, {
2026
+ maxImages = OPENAI_IMAGE_EDIT_INPUTS_MAX,
2027
+ ...options
2028
+ } = {}) {
2029
+ const attachments = Array.isArray(rawAttachments) ? rawAttachments : [];
2030
+ return attachments
2031
+ .filter((entry) => isOpenAIImageAttachment(entry))
2032
+ .slice(0, Math.max(1, Math.trunc(Number(maxImages) || 1)))
2033
+ .map((entry) => resolveOneOpenAIImageEditAttachment(entry, options));
2034
+ }
2035
+
2036
+ function resolveOneOpenAIImageEditAttachment(image, {
1703
2037
  maxImageBytes = 20 * 1024 * 1024,
1704
2038
  allowedRootPath = '',
1705
2039
  // Folders of earlier versions that uploads are still read from: only a
@@ -1707,12 +2041,8 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1707
2041
  allowedBlobRootPaths = [],
1708
2042
  allowedGeneratedImagesRootPath = '',
1709
2043
  } = {}) {
1710
- const attachments = Array.isArray(rawAttachments) ? rawAttachments : [];
1711
- const firstImage = attachments.find((entry) => isOpenAIImageAttachment(entry));
1712
- if (!firstImage) return null;
1713
-
1714
- const dataUrl = String(firstImage.dataUrl || '').trim();
1715
- let mimeType = String(firstImage.type || '').trim().toLowerCase();
2044
+ const dataUrl = String(image.dataUrl || '').trim();
2045
+ let mimeType = String(image.type || '').trim().toLowerCase();
1716
2046
  let bytes = null;
1717
2047
 
1718
2048
  if (dataUrl.startsWith('data:')) {
@@ -1722,7 +2052,7 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1722
2052
  const payload = parseOpenAIImageBase64Value(parts.payload);
1723
2053
  bytes = Buffer.from(payload, 'base64');
1724
2054
  } else {
1725
- const filePath = String(firstImage.path || '').trim();
2055
+ const filePath = String(image.path || '').trim();
1726
2056
  if (!filePath) throw new Error('Image attachment is missing file path or data URL');
1727
2057
  const allowedRoot = String(allowedRootPath || '').trim();
1728
2058
  const blobRoots = (Array.isArray(allowedBlobRootPaths) ? allowedBlobRootPaths : [])
@@ -1756,6 +2086,8 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1756
2086
  bytes = fs.readFileSync(filePath);
1757
2087
  }
1758
2088
 
2089
+ // What the bytes are wins over what the attachment says they are.
2090
+ mimeType = (Buffer.isBuffer(bytes) && readImageHeader(bytes)?.mime) || mimeType;
1759
2091
  if (!mimeType.startsWith('image/')) {
1760
2092
  throw new Error('Image attachment must use image/* MIME type');
1761
2093
  }
@@ -1769,13 +2101,20 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
1769
2101
  return {
1770
2102
  bytes,
1771
2103
  mimeType,
1772
- name: String(firstImage.name || 'image').trim().slice(0, 120) || 'image',
2104
+ name: String(image.name || 'image').trim().slice(0, 120) || 'image',
1773
2105
  };
1774
2106
  }
1775
2107
 
1776
- export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8 } = {}) {
2108
+ /**
2109
+ * The images of an OpenAI image reply. Their type comes from the format
2110
+ * asked for (`outputFormat`), else the one the reply names, else PNG; the
2111
+ * image's own header has the last word.
2112
+ */
2113
+ export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8, outputFormat = '' } = {}) {
1777
2114
  const rows = Array.isArray(payload?.data) ? payload.data : [];
1778
2115
  const maxCount = Math.max(1, Math.trunc(Number(maxImages) || 8));
2116
+ const format = String(outputFormat || payload?.output_format || '').trim().toLowerCase();
2117
+ const declaredMime = OPENAI_IMAGE_MIME_BY_FORMAT[format] || OPENAI_IMAGE_MIME_BY_FORMAT.png;
1779
2118
  const generatedImages = [];
1780
2119
  for (const [index, row] of rows.entries()) {
1781
2120
  if (index >= maxCount) break;
@@ -1785,7 +2124,7 @@ export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8
1785
2124
  const revisedPrompt = String(row.revised_prompt || row.revisedPrompt || '').trim();
1786
2125
  generatedImages.push({
1787
2126
  data,
1788
- mimeType: 'image/png',
2127
+ mimeType: readBase64ImageHeader(data)?.mime || declaredMime,
1789
2128
  name: `generated-image-${index + 1}`,
1790
2129
  ...(revisedPrompt ? { revisedPrompt } : {}),
1791
2130
  });
@@ -1853,7 +2192,12 @@ function createGeneratedImageAttachment({
1853
2192
  resolveSessionStateRoot,
1854
2193
  }) {
1855
2194
  const imageId = `img-${String(imageIndex + 1).padStart(2, '0')}`;
1856
- const extension = resolveGeneratedImageExtension(image.mimeType);
2195
+ const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
2196
+ // The type and pixel size the image's own header gives; the type the
2197
+ // sender named only when the header is not one the reader knows.
2198
+ const header = readImageHeader(bytes);
2199
+ const mimeType = header?.mime || image.mimeType;
2200
+ const extension = resolveGeneratedImageExtension(mimeType);
1857
2201
  const location = resolveGeneratedImageStoragePath({
1858
2202
  resolveSessionStateRoot,
1859
2203
  sdkSessionId,
@@ -1864,7 +2208,6 @@ function createGeneratedImageAttachment({
1864
2208
  });
1865
2209
  fs.mkdirSync(location.absoluteDir, { recursive: true });
1866
2210
  const tmpPath = path.join(location.absoluteDir, `${location.fileName}.${process.pid}.${Date.now()}.tmp`);
1867
- const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
1868
2211
  fs.writeFileSync(tmpPath, bytes);
1869
2212
  try {
1870
2213
  fs.renameSync(tmpPath, location.absolutePath);
@@ -1876,8 +2219,9 @@ function createGeneratedImageAttachment({
1876
2219
  }
1877
2220
  return {
1878
2221
  name: `${String(image.name || 'generated-image').replace(/\.[a-z0-9]+$/i, '') || 'generated-image'}-${imageIndex + 1}.${extension}`,
1879
- type: image.mimeType,
2222
+ type: mimeType,
1880
2223
  size: bytes.length,
2224
+ ...(header ? { width: header.width, height: header.height } : {}),
1881
2225
  contentUrl: `/api/generated-image/${encodeURIComponent(conversationId)}/${encodeURIComponent(messageId)}/${encodeURIComponent(imageId)}/content`,
1882
2226
  generatedImage: {
1883
2227
  imageId,
@@ -2029,6 +2373,9 @@ export function registerMessagesRoutes(app, deps) {
2029
2373
  MAX_UPLOAD_ATTACHMENTS,
2030
2374
  uploadsDir,
2031
2375
  uploadReadDirs = [uploadsDir],
2376
+ // What a send's `imagePaths` never reads, beside the upload dirs: the
2377
+ // relay's data dir (`dirs`), its database and config files (`files`).
2378
+ imageInputDenyPaths = { dirs: [], files: [] },
2032
2379
  MAX_REPO_TREE_NODES,
2033
2380
  MAX_REQUEUE_RETRIES,
2034
2381
  MAX_IMAGE_DATA_URL_LENGTH,
@@ -2132,6 +2479,11 @@ export function registerMessagesRoutes(app, deps) {
2132
2479
  // (services/request-boundary.mjs). The runtime passes the check built
2133
2480
  // with the configured marker headers.
2134
2481
  isTrulyLocalRequest = createLocalRequestCheck().isTrulyLocal,
2482
+ // The pause before the one retry of an OpenAI image request.
2483
+ openAIImageRetryDelayMs = OPENAI_IMAGE_RETRY_DELAY_MS,
2484
+ // How long the partial previews of an image request that ended wait for
2485
+ // the turn's end (services/openai-image-preview-turns.mjs).
2486
+ openAIImagePreviewLeftoverMs,
2135
2487
  featureFlags,
2136
2488
  sessionWorkerRegistry,
2137
2489
  sessionWorkerSupervisor,
@@ -2154,6 +2506,10 @@ export function registerMessagesRoutes(app, deps) {
2154
2506
  claudeApiUsageService = null,
2155
2507
  // Tokens per turn for every provider (token-usage-service.mjs).
2156
2508
  tokenUsageService = null,
2509
+ // OpenAI image spend and its daily cap (openai-image-spend-service.mjs):
2510
+ // the runtime shares one instance with the settings routes. Without it
2511
+ // (tests) the routes make their own over tokenUsageService.
2512
+ openAIImageSpend = null,
2157
2513
  // The delivery fence's runtime half for GET /api/pending, as on the
2158
2514
  // socket: the Claude API claim check (null: no refusal) and what a fenced
2159
2515
  // row does (the swap, or fail closed).
@@ -2196,6 +2552,24 @@ export function registerMessagesRoutes(app, deps) {
2196
2552
  deepseekClaudeConfigDir = '',
2197
2553
  } = deps;
2198
2554
 
2555
+ // An app_settings value as text ('' when unset or unreadable).
2556
+ const readAppSettingText = (key) => {
2557
+ try {
2558
+ return String(stmts?.getAppSetting?.get?.(key)?.value ?? '').trim();
2559
+ } catch {
2560
+ return '';
2561
+ }
2562
+ };
2563
+ // OpenAI image spend: the cap check and reservation before each try, its
2564
+ // booking after, the send-time refusal and the spend state.
2565
+ const imageSpend = openAIImageSpend || createOpenAIImageSpendService({
2566
+ tokenUsageService,
2567
+ readSetting: readAppSettingText,
2568
+ db,
2569
+ emit: (event, payload) => io?.emit?.(event, payload),
2570
+ logger: console,
2571
+ });
2572
+
2199
2573
  // ─── Sub-agent runs: header, timeline, live events ─────────────────────────
2200
2574
  const subagentTimeline = createSubagentTimelineService({ db, stmts });
2201
2575
 
@@ -4059,6 +4433,19 @@ export function registerMessagesRoutes(app, deps) {
4059
4433
  });
4060
4434
  });
4061
4435
 
4436
+ // The media gallery (🧠 Media tab, global Media view) and its index.
4437
+ registerMediaRoutes(app, {
4438
+ auth,
4439
+ db,
4440
+ uploadPathForSha,
4441
+ fetchBrowsableDrives,
4442
+ resolveGeneratedImagePath: ({ sessionId, relativePath }) => resolveGeneratedImageReadPath({
4443
+ resolveSessionStateRoot,
4444
+ sessionId,
4445
+ relativePath,
4446
+ }),
4447
+ });
4448
+
4062
4449
  app.get('/api/files/{*splat}', auth, (req, res) => {
4063
4450
  const requestedPath = splatPath(req).trim();
4064
4451
  const rootOverride = resolveScopedWorkspaceRootPath(req);
@@ -4871,6 +5258,8 @@ export function registerMessagesRoutes(app, deps) {
4871
5258
  provider,
4872
5259
  attachments: rawAttachments,
4873
5260
  imageTarget: rawImageTarget,
5261
+ imagePaths: rawImagePaths,
5262
+ image: rawImageOptions,
4874
5263
  } = req.body;
4875
5264
  // A prompt from another relay's agent (body `origin` or the
4876
5265
  // X-OAR-Remote-Origin header) is refused while the inbound switch is off.
@@ -4907,7 +5296,7 @@ export function registerMessagesRoutes(app, deps) {
4907
5296
  Array.isArray(rawAttachments) ? rawAttachments.filter((item) => item && typeof item === 'object').length : (rawAttachments ? 1 : 0),
4908
5297
  );
4909
5298
  const referenceResolution = collectReferenceAttachmentsFromText(trimmedText);
4910
- const attachments = mergeMessageAttachments(normalizedAttachments, referenceResolution.attachments);
5299
+ let attachments = mergeMessageAttachments(normalizedAttachments, referenceResolution.attachments);
4911
5300
  const imageTarget = rawImageTarget && typeof rawImageTarget === 'object'
4912
5301
  ? {
4913
5302
  messageId: String(rawImageTarget.messageId || '').trim(),
@@ -4922,6 +5311,17 @@ export function registerMessagesRoutes(app, deps) {
4922
5311
  code: 'IMAGE_CONTINUITY_DISABLED',
4923
5312
  });
4924
5313
  }
5314
+ // Images named by path on this relay and the turn's own image options
5315
+ // (relay_tool send and create_session). Both go to an OpenAI image
5316
+ // session only; the paths are read before anything is stored, and one
5317
+ // that may not be read refuses the whole send.
5318
+ const refuseImageInput = (error) => res.status(400).json({ code: IMAGE_INPUT_REFUSED_CODE, error });
5319
+ const imagePathsGiven = rawImagePaths !== undefined && rawImagePaths !== null;
5320
+ const imageOptionsGiven = rawImageOptions !== undefined && rawImageOptions !== null;
5321
+ if (imagePathsGiven && (!Array.isArray(rawImagePaths) || rawImagePaths.some((entry) => typeof entry !== 'string'))) {
5322
+ return refuseImageInput('imagePaths must be a list of absolute file paths on this relay');
5323
+ }
5324
+ const imagePathList = imagePathsGiven ? rawImagePaths.filter((entry) => entry !== '') : [];
4925
5325
 
4926
5326
  if (trimmedText.toLowerCase() === '/compact') {
4927
5327
  if (attachments.length) return res.status(400).json({ error: 'Compact command does not accept attachments' });
@@ -5443,6 +5843,16 @@ export function registerMessagesRoutes(app, deps) {
5443
5843
  latestUserMessageModel: stmts.getLatestUserMessageModel?.get?.(conversationId)?.model || '',
5444
5844
  })
5445
5845
  : null;
5846
+ // Copilot lists image ids its runtime cannot serve: a Copilot chat on one
5847
+ // is refused with a pointer to an OpenAI Image chat, whoever sends.
5848
+ const copilotModelId = usesGitHubCatalog ? String(inheritedModel ? inheritedModel.model : (model || '')).trim() : '';
5849
+ if (copilotModelId && isOpenAIImageModelId(copilotModelId)) {
5850
+ return res.status(400).json({
5851
+ error: `${openAIImageModelNeedsOpenAIText(copilotModelId)} Nothing was sent.`,
5852
+ code: OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
5853
+ ...(conversationId ? { conversationId } : {}),
5854
+ });
5855
+ }
5446
5856
  const githubModelResolution = usesGitHubCatalog
5447
5857
  ? resolveRequestedModel(inheritedModel ? inheritedModel.model : model)
5448
5858
  : null;
@@ -5546,6 +5956,17 @@ export function registerMessagesRoutes(app, deps) {
5546
5956
  code: 'IMAGE_TARGET_REQUIRES_IMAGE_MODEL',
5547
5957
  });
5548
5958
  }
5959
+ if ((imagePathsGiven || imageOptionsGiven) && !openAIImageModel) {
5960
+ return refuseImageInput(`Images and image options go to an OpenAI image session; this session runs ${requestedModel || 'no image model'}`);
5961
+ }
5962
+ // An image turn from another relay's agent takes its images from
5963
+ // `imagePaths` alone: an @file: token resolves against this relay's
5964
+ // workspace root, not the session's folder, and carries no stored copy.
5965
+ let ignoredReferenceAttachments = 0;
5966
+ if (remoteRequest.remote && openAIImageModel && referenceResolution.attachments.length) {
5967
+ ignoredReferenceAttachments = referenceResolution.attachments.length;
5968
+ attachments = mergeMessageAttachments(normalizedAttachments, []);
5969
+ }
5549
5970
  // A send that names no effort runs with the conversation's own, clamped
5550
5971
  // to the nearest level the model has. Null when there is nothing to keep
5551
5972
  // or it neither matches nor clamps (an ultracode on a model without it):
@@ -5668,11 +6089,66 @@ export function registerMessagesRoutes(app, deps) {
5668
6089
  // is already 1M.
5669
6090
  normalizedContextTier = 'default';
5670
6091
  }
6092
+ // An image turn's options: the session's defaults (update_session
6093
+ // `image`), then the preset the turn names (`image.preset`), then the
6094
+ // turn's own. The owner's composer names quality and size in the effort
6095
+ // and context-tier fields, which are the turn's own; an agent's turn
6096
+ // names them in `image`, and its effort counts below the defaults. Only
6097
+ // what an agent sent is refused here: the composer's choice goes on as
6098
+ // before, checked against the model when it runs. The options are
6099
+ // resolved here, at queue time: a preset changed later does not change a
6100
+ // queued turn.
6101
+ let imageOptions = null;
6102
+ if (openAIImageModel) {
6103
+ const { preset: presetName, own: ownImageInput } = splitOpenAIImagePreset(rawImageOptions);
6104
+ const preset = findOpenAIImagePreset(presetName, parseOwnOpenAIImagePresets(readAppSettingText(OPENAI_IMAGE_PRESETS_SETTING_KEY)));
6105
+ if (!preset.ok) return refuseImageInput(`${asSentence(preset.error)} Nothing was sent.`);
6106
+ const ownImageOptions = validateOpenAIImageOptions(ownImageInput, { model: requestedModel });
6107
+ if (!ownImageOptions.ok) return refuseImageInput(ownImageOptions.error);
6108
+ const keptQuality = !explicitReasoningEffort && requestedReasoningEffort && !['auto', 'none'].includes(requestedReasoningEffort)
6109
+ ? { quality: requestedReasoningEffort }
6110
+ : {};
6111
+ const fieldOptions = {
6112
+ ...(explicitReasoningEffort ? { quality: requestedReasoningEffort } : {}),
6113
+ ...(String(contextTier || '').trim() && !['default', 'long_context'].includes(requestedContextTier) ? { size: normalizedContextTier } : {}),
6114
+ };
6115
+ let sessionDefaults = {};
6116
+ if (!shouldCreateConversation) {
6117
+ try {
6118
+ sessionDefaults = stmts.readConversationImageDefaults?.(conversationId) || {};
6119
+ } catch {}
6120
+ }
6121
+ const layers = remoteRequest.remote
6122
+ ? [keptQuality, fieldOptions, sessionDefaults, preset.values, ownImageOptions.values]
6123
+ : [keptQuality, sessionDefaults, preset.values, fieldOptions, ownImageOptions.values];
6124
+ const merged = layers.reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
6125
+ const checked = validateOpenAIImageOptions(merged, { model: requestedModel });
6126
+ if (checked.ok) {
6127
+ imageOptions = mergeOpenAIImageOptions(checked.values, null);
6128
+ } else if (imagePathsGiven || imageOptionsGiven) {
6129
+ return refuseImageInput(checked.error);
6130
+ } else {
6131
+ imageOptions = [keptQuality, fieldOptions].reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
6132
+ }
6133
+ // Today's image spend has reached the daily cap: refused for every
6134
+ // sender before anything is stored (the owner's override lifts it).
6135
+ let capRefusal = null;
6136
+ try {
6137
+ capRefusal = imageSpend.sendRefusal();
6138
+ } catch (error) {
6139
+ capRefusal = { code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE, error: `The relay could not check the daily cap of OpenAI images (${error?.message || error}). Nothing was sent.` };
6140
+ }
6141
+ if (capRefusal) {
6142
+ return res.status(409).json({ error: capRefusal.error, code: capRefusal.code, ...(conversationId ? { conversationId } : {}) });
6143
+ }
6144
+ }
5671
6145
  if (!requestedRelayMode) return res.status(400).json({ error: 'Unsupported relay mode' });
5672
6146
  const conversationWorkspaceState = (!shouldCreateConversation && typeof resolveConversationWorkspaceState === 'function')
5673
6147
  ? resolveConversationWorkspaceState({ conversationId })
5674
6148
  : null;
5675
- let workspaceRootUpdate = attachments.length === 0
6149
+ // Images by path count as attachments: a "cd …" in their prompt moves
6150
+ // no folder, and they are checked against the folder the session has now.
6151
+ let workspaceRootUpdate = attachments.length === 0 && imagePathList.length === 0
5676
6152
  ? maybeApplyWorkspaceRootFromMessage(
5677
6153
  trimmedText,
5678
6154
  conversationWorkspaceState?.currentWorkspaceRootPath || null,
@@ -5742,6 +6218,56 @@ export function registerMessagesRoutes(app, deps) {
5742
6218
  }
5743
6219
  }
5744
6220
 
6221
+ // The images by path: every one checked and read before anything is
6222
+ // stored, then kept in the upload store, so each becomes an attachment
6223
+ // with its bytes' hash and sniffed type (and an asset of the operation).
6224
+ // They come first: the first image is the base of an edit.
6225
+ let imageInputs = [];
6226
+ let imageInputsDropped = 0;
6227
+ if (imagePathsGiven) {
6228
+ if (shouldCreateConversation) {
6229
+ return refuseImageInput('imagePaths need an existing session: create it first, then send the images to it');
6230
+ }
6231
+ let sessionStateRoot = '';
6232
+ try {
6233
+ sessionStateRoot = String(resolveSessionStateRoot?.() || '');
6234
+ } catch {}
6235
+ const roots = resolveImageInputRoots({
6236
+ cwd: conversationWorkspaceState?.currentWorkspaceRootPath || '',
6237
+ sessionStateRoot,
6238
+ deniedDirs: [...(imageInputDenyPaths?.dirs || []), uploadsDir, ...(uploadReadDirs || [])],
6239
+ deniedFiles: imageInputDenyPaths?.files || [],
6240
+ });
6241
+ const resolvedImages = resolveImageInputs(imagePathList, {
6242
+ roots,
6243
+ isLiveConversation: (id) => {
6244
+ const row = stmts.getConvAnyStatus?.get?.(id) || null;
6245
+ return !!row && String(row.status || '').trim() !== 'deleted';
6246
+ },
6247
+ });
6248
+ if (!resolvedImages.ok) return refuseImageInput(resolvedImages.error);
6249
+ const resolvedHashes = new Set(resolvedImages.images.map((image) => image.sha256));
6250
+ const otherAttachments = attachments.filter((attachment) => !resolvedHashes.has(String(attachment?.sha256 || '').trim().toLowerCase()));
6251
+ const total = resolvedImages.images.length + otherAttachments.length;
6252
+ if (total > OPENAI_IMAGE_LIMITS.inputImagesMax) {
6253
+ return refuseImageInput(`An image turn takes at most ${OPENAI_IMAGE_LIMITS.inputImagesMax} images; this one would have ${total} with the message's other attachments`);
6254
+ }
6255
+ try {
6256
+ imageInputs = resolvedImages.images.map((image) => ({
6257
+ ...persistUploadBuffer(image.bytes, { name: image.name, type: image.mime }),
6258
+ // The type the bytes are, not the one an earlier upload of the
6259
+ // same bytes was stored with.
6260
+ type: image.mime,
6261
+ width: image.width,
6262
+ height: image.height,
6263
+ }));
6264
+ } catch (error) {
6265
+ return res.status(500).json({ code: 'IMAGE_INPUT_STORE_FAILED', error: `The images could not be stored on this relay: ${error?.message || error}` });
6266
+ }
6267
+ imageInputsDropped = imagePathList.length - imageInputs.length;
6268
+ attachments = [...imageInputs, ...otherAttachments];
6269
+ }
6270
+
5745
6271
  const convId = shouldCreateConversation ? uuidv4() : conversationId;
5746
6272
  if (shouldCreateConversation) {
5747
6273
  // An agent's prompt opens with the sending relay's header line, which
@@ -5978,6 +6504,7 @@ export function registerMessagesRoutes(app, deps) {
5978
6504
  null,
5979
6505
  null,
5980
6506
  );
6507
+ if (imageOptions) stmts.setQueueImageOptions?.run?.(JSON.stringify(imageOptions), msgId);
5981
6508
  if (imageContinuityEnabled && openAIImageModel) {
5982
6509
  if (!imageOperationService?.createEnqueuedOperation) {
5983
6510
  throw new Error('Image operation service is unavailable');
@@ -5989,11 +6516,7 @@ export function registerMessagesRoutes(app, deps) {
5989
6516
  prompt: trimmedText,
5990
6517
  selectedImageModel: requestedModel,
5991
6518
  executionMode: 'direct_images',
5992
- parameters: {
5993
- quality: requestedReasoningEffort,
5994
- size: normalizedContextTier,
5995
- count: 1,
5996
- },
6519
+ parameters: imageOptions || { count: 1 },
5997
6520
  attachments,
5998
6521
  imageTarget,
5999
6522
  createdAt: now,
@@ -6158,8 +6681,17 @@ export function registerMessagesRoutes(app, deps) {
6158
6681
  workspaceRootWarning: workspaceRootUpdate.error || null,
6159
6682
  workspaceRootChanged: !!workspaceRootUpdate.changed,
6160
6683
  ...workspaceRootPayload(),
6161
- referenceAttachmentCount: referenceResolution.attachments.length,
6684
+ referenceAttachmentCount: referenceResolution.attachments.length - ignoredReferenceAttachments,
6162
6685
  skippedReferenceAttachments: referenceResolution.skipped,
6686
+ ...(ignoredReferenceAttachments ? { ignoredReferenceAttachments } : {}),
6687
+ // How many images from imagePaths the turn has, and how many named
6688
+ // twice (the same file, or the same bytes) were kept once.
6689
+ ...(imagePathsGiven ? { imageInputs: imageInputs.length } : {}),
6690
+ ...(imageInputsDropped > 0 ? {
6691
+ imageInputsDropped,
6692
+ imageInputsNote: `${imageInputsDropped} of the ${imagePathList.length} image paths named an image already attached; it was attached once.`,
6693
+ } : {}),
6694
+ ...(imageOptions ? { imageOptions } : {}),
6163
6695
  // Attachment references that no longer resolve (a deleted upload) are
6164
6696
  // dropped silently by normalizeAttachments; the sender is told how many.
6165
6697
  droppedAttachmentCount: Math.max(0, requestedAttachmentCount - normalizedAttachments.length),
@@ -7652,8 +8184,516 @@ export function registerMessagesRoutes(app, deps) {
7652
8184
  return res.json({ ok: true, ...result });
7653
8185
  });
7654
8186
 
8187
+ // Image operations whose OpenAI request runs in this process, and the
8188
+ // direct image requests (no operation) this process sent, by queue
8189
+ // message: a second run of either would be billed again.
8190
+ const executingImageOperations = new Set();
8191
+ const dispatchedImageMessages = new Map();
8192
+ const DISPATCHED_IMAGE_MESSAGES_MAX = 500;
8193
+ const openAIImageTimeoutFromConfig = (key, fallback) => (Number(config?.[key]) > 0 ? Number(config[key]) : fallback);
8194
+ const openAIImageGenerateTimeoutMs = openAIImageTimeoutFromConfig('openaiImageGenerateTimeoutMs', OPENAI_IMAGE_GENERATE_TIMEOUT_MS);
8195
+ const openAIImageEditTimeoutMs = openAIImageTimeoutFromConfig('openaiImageEditTimeoutMs', OPENAI_IMAGE_EDIT_TIMEOUT_MS);
8196
+ // The partial previews of the running image turns: files beside the
8197
+ // turn's final images, events to the owner's clients (every socket is the
8198
+ // owner's: share views read over HTTP and have none).
8199
+ const imagePreviews = createOpenAIImagePreviewTurns({
8200
+ resolveSessionStateRoot,
8201
+ sessionIdFor: (conversationId) => String(stmts.getConvAnyStatus?.get?.(conversationId)?.sdk_session_id || '').trim() || conversationId,
8202
+ emit: (event, payload) => io?.emit?.(event, payload),
8203
+ ...(openAIImagePreviewLeftoverMs !== undefined ? { leftoverMs: openAIImagePreviewLeftoverMs } : {}),
8204
+ });
8205
+
8206
+ // A partial preview of a running image turn, for the owner's clients
8207
+ // only: only a file the turn wrote and still has is served (no path is
8208
+ // built from the request), never through a share link, and gone with the
8209
+ // turn.
8210
+ app.get(`/api/generated-image/:conversationId/:messageId/${OPENAI_IMAGE_PARTIALS_DIR}/:fileName`, auth, (req, res) => {
8211
+ const file = imagePreviews.partialFile({
8212
+ conversationId: String(req.params.conversationId || '').trim(),
8213
+ messageId: String(req.params.messageId || '').trim(),
8214
+ fileName: String(req.params.fileName || '').trim(),
8215
+ });
8216
+ // lstat: a link put in the turn's partials folder is not followed.
8217
+ let stat = null;
8218
+ try {
8219
+ stat = file ? fs.lstatSync(file.path) : null;
8220
+ } catch {}
8221
+ if (!stat?.isFile()) return res.status(404).json({ error: 'Preview not found' });
8222
+ return serveFileWithRangeSupport(req, res, file.path, { size: stat.size, contentType: file.mime }, {
8223
+ safeName: path.basename(file.path).replace(/"/g, ''),
8224
+ // And the open itself refuses a link swapped in after the lstat (POSIX;
8225
+ // Windows has no O_NOFOLLOW, the lstat is the guard there).
8226
+ fsImpl: PARTIAL_PREVIEW_FS,
8227
+ });
8228
+ });
8229
+
8230
+ function noteDispatchedImageMessage(messageId) {
8231
+ dispatchedImageMessages.delete(messageId);
8232
+ dispatchedImageMessages.set(messageId, { at: Date.now(), running: true });
8233
+ while (dispatchedImageMessages.size > DISPATCHED_IMAGE_MESSAGES_MAX) {
8234
+ dispatchedImageMessages.delete(dispatchedImageMessages.keys().next().value);
8235
+ }
8236
+ return dispatchedImageMessages.get(messageId);
8237
+ }
8238
+
8239
+ /**
8240
+ * One try of an OpenAI image request: `{ ok: true, generatedImages, usage }`
8241
+ * or `{ kind, status?, message?, detail? }` (see OPENAI_IMAGE_RETRY_KINDS).
8242
+ * The timeout covers the reply's body as well, a streamed one whole.
8243
+ * `fields` asking for a stream get one read event by event (a reply that is
8244
+ * no event stream is read as JSON); `onPartial` sees each partial image.
8245
+ */
8246
+ async function sendOpenAIImageRequestOnce({ url, headers, fields, editImages, family, timeoutMs, outputFormat, model, onPartial = null }) {
8247
+ // The images asked for: a stream's events past them are dropped.
8248
+ const expectedImages = Math.max(1, Math.trunc(Number(fields.n) || 1));
8249
+ let response;
8250
+ try {
8251
+ let body;
8252
+ if (editImages.length) {
8253
+ body = new FormData();
8254
+ for (const [key, value] of Object.entries(fields)) body.set(key, String(value));
8255
+ // gpt-image takes every image, the first being the base; dall-e-2
8256
+ // takes one.
8257
+ const images = family === 'gpt-image' ? editImages : editImages.slice(0, 1);
8258
+ for (const image of images) {
8259
+ body.append(
8260
+ family === 'gpt-image' ? 'image[]' : 'image',
8261
+ new Blob([image.bytes], { type: image.mimeType }),
8262
+ image.name,
8263
+ );
8264
+ }
8265
+ } else {
8266
+ body = JSON.stringify(fields);
8267
+ }
8268
+ response = await fetch(url, {
8269
+ method: 'POST',
8270
+ headers: editImages.length ? headers : { ...headers, 'Content-Type': 'application/json' },
8271
+ body,
8272
+ signal: AbortSignal.timeout(timeoutMs),
8273
+ });
8274
+ } catch (error) {
8275
+ const kind = classifyOpenAIImageFetchError(error);
8276
+ return { kind, ownTimeout: error?.name === 'TimeoutError', detail: describeFetchError(error) };
8277
+ }
8278
+
8279
+ const status = Number(response.status);
8280
+ if (response.ok && fields.stream === true && isEventStreamResponse(response)) {
8281
+ return readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages });
8282
+ }
8283
+ let rawText = '';
8284
+ try {
8285
+ rawText = String(await response.text()).trim();
8286
+ } catch (error) {
8287
+ // A 2xx whose body broke off is billed; an error status keeps its class.
8288
+ if (response.ok) return { kind: 'unprocessed', status, detail: `the reply broke off: ${describeFetchError(error)}` };
8289
+ return status >= 500 ? { kind: 'server', status } : { kind: 'refused', status };
8290
+ }
8291
+ let payload = {};
8292
+ if (rawText) {
8293
+ try {
8294
+ payload = JSON.parse(rawText);
8295
+ } catch {
8296
+ // After a 2xx the generation is billed; a retry pays again. An error
8297
+ // status with a body that is not JSON stays a provider failure.
8298
+ if (response.ok) return { kind: 'unprocessed', status, detail: 'OpenAI image API returned invalid JSON' };
8299
+ payload = null;
8300
+ }
8301
+ }
8302
+ if (!response.ok) {
8303
+ const message = String(payload?.error?.message || payload?.message || (payload ? '' : rawText)).trim().slice(0, 240);
8304
+ if (!Number.isFinite(status) || status >= 500) return { kind: 'server', status, message };
8305
+ return { kind: 'refused', status, message };
8306
+ }
8307
+ try {
8308
+ const generatedImages = normalizeOpenAIImageApiGeneratedImages(payload, {
8309
+ maxImages: MAX_UPLOAD_ATTACHMENTS,
8310
+ outputFormat,
8311
+ });
8312
+ return { ok: true, generatedImages, usage: normalizeOpenAIImageUsage(payload?.usage) };
8313
+ } catch (error) {
8314
+ // The usage the reply carried is what was billed.
8315
+ const usage = normalizeOpenAIImageUsage(payload?.usage);
8316
+ // The provider answered 2xx, so the generation is billed; what failed
8317
+ // is the relay's handling of the reply. Logged with the stack: a retry
8318
+ // pays again and fails the same way.
8319
+ console.error(
8320
+ `[${ts()}] IMAGE reply of ${model} not processed: ${error?.message || error} (body ${rawText.length} chars)\n${error?.stack || ''}`,
8321
+ );
8322
+ return { kind: 'unprocessed', status, detail: error?.message || 'OpenAI image API returned no images', ...(usage ? { usage } : {}) };
8323
+ }
8324
+ }
8325
+
8326
+ /**
8327
+ * The rest of a streamed reply (after a 2xx with an event stream): each
8328
+ * partial goes to `onPartial`, each completed event's image is kept in
8329
+ * order (a count > 1 turn may get several), and the completed events'
8330
+ * usages add up to the request's. A stream that breaks off, ends without an image
8331
+ * or reports a server error after OpenAI started may have been billed:
8332
+ * 'stream', 'timeout' or 'server', tried once more. An error event that
8333
+ * names a refusal is 'refused'; an image the relay cannot read, or an event
8334
+ * past the reader's cap, 'unprocessed'.
8335
+ *
8336
+ * Once every image asked for (`expectedImages`) is complete, a stream that
8337
+ * breaks off or times out before its end still answers them: they are
8338
+ * billed, and a retry would buy them again. Partials past
8339
+ * OPENAI_IMAGE_PARTIAL_IMAGES per image or past the images asked for, and
8340
+ * completed events past those images, are dropped unread.
8341
+ *
8342
+ * Every result carries `partials` (received) and, once a completed event
8343
+ * was read, `usage`: a try that fails after it books that usage
8344
+ * (openAIImageTryCharge).
8345
+ */
8346
+ async function readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages = 1 }) {
8347
+ const completed = [];
8348
+ let usage = null;
8349
+ let partials = 0;
8350
+ // What every result carries for the booking.
8351
+ const counted = (result) => ({ ...result, partials, ...(usage ? { usage } : {}) });
8352
+ const answer = () => {
8353
+ try {
8354
+ const generatedImages = normalizeOpenAIImageApiGeneratedImages({
8355
+ data: completed.map((payload) => ({ b64_json: payload.b64_json })),
8356
+ output_format: completed[0].output_format,
8357
+ }, { maxImages: MAX_UPLOAD_ATTACHMENTS, outputFormat });
8358
+ return counted({ ok: true, generatedImages });
8359
+ } catch (error) {
8360
+ console.error(`[${ts()}] IMAGE streamed reply of ${model} not processed: ${error?.message || error}\n${error?.stack || ''}`);
8361
+ return counted({ kind: 'unprocessed', status, detail: error?.message || 'OpenAI image stream returned no images' });
8362
+ }
8363
+ };
8364
+ try {
8365
+ for await (const sse of readSseEvents(response.body)) {
8366
+ const event = interpretOpenAIImageStreamEvent(sse);
8367
+ if (event.type === 'partial') {
8368
+ partials += 1;
8369
+ const partialIndex = Number.isInteger(event.payload.partial_image_index) ? event.payload.partial_image_index : partials - 1;
8370
+ const shown = completed.length < expectedImages && partialIndex >= 0 && partialIndex < OPENAI_IMAGE_PARTIAL_IMAGES;
8371
+ if (onPartial && shown) {
8372
+ try {
8373
+ await onPartial({
8374
+ imageIndex: completed.length,
8375
+ partialIndex,
8376
+ b64: event.payload.b64_json,
8377
+ outputFormat: event.payload.output_format || outputFormat,
8378
+ });
8379
+ } catch (error) {
8380
+ console.warn(`[${ts()}] IMAGE partial of ${model} not shown: ${error?.message || error}`);
8381
+ }
8382
+ }
8383
+ } else if (event.type === 'completed') {
8384
+ if (completed.length >= expectedImages) continue;
8385
+ completed.push(event.payload);
8386
+ usage = addOpenAIImageUsage(usage, normalizeOpenAIImageUsage(event.payload.usage));
8387
+ } else if (event.type === 'invalid' && sse.event.endsWith('.completed')) {
8388
+ return counted({ kind: 'unprocessed', status, detail: 'OpenAI image stream sent an unreadable image' });
8389
+ } else if (event.type === 'error') {
8390
+ const { message, type, code } = event.error;
8391
+ const named = `${type} ${code}`.toLowerCase();
8392
+ const detail = `error event in the stream${code ? ` (${code})` : ''} after ${partials} partial image(s)`;
8393
+ if (['server', 'internal', 'overloaded', 'timeout', 'unavailable'].some((word) => named.includes(word)) || !named.trim()) {
8394
+ return counted({ kind: 'server', message, detail });
8395
+ }
8396
+ return counted({ kind: 'refused', status: 400, message, detail, streamed: true });
8397
+ }
8398
+ }
8399
+ } catch (error) {
8400
+ if (error?.code === SSE_EVENT_TOO_LARGE_CODE) return counted({ kind: 'unprocessed', status, detail: error.message });
8401
+ if (completed.length >= expectedImages) {
8402
+ console.warn(`[${ts()}] IMAGE ${model} stream broke off after its ${completed.length} image(s): kept (${describeFetchError(error)})`);
8403
+ return answer();
8404
+ }
8405
+ const kind = classifyOpenAIImageFetchError(error);
8406
+ return counted({
8407
+ kind: kind === 'timeout' ? 'timeout' : 'stream',
8408
+ ownTimeout: error?.name === 'TimeoutError',
8409
+ detail: `the stream broke off after ${partials} partial image(s): ${describeFetchError(error)}`,
8410
+ });
8411
+ }
8412
+ if (!completed.length) return counted({ kind: 'stream', detail: `the stream ended without an image after ${partials} partial image(s)` });
8413
+ return answer();
8414
+ }
8415
+
8416
+ /**
8417
+ * The OpenAI image request of one turn, for the execute route and the
8418
+ * generate route alike: the request with every option and every input
8419
+ * image, a timeout per try (generate / edit), and one retry of a try that
8420
+ * may pass the second time, with the same Idempotency-Key (OpenAI may
8421
+ * ignore it; the note says the request may have been billed). Never more
8422
+ * than two requests: nothing else may send this turn again.
8423
+ *
8424
+ * `requestBody` is normalizeOpenAIImageGenerateRequestBody's. Resolves
8425
+ * `{ ok: true, generatedImages, model, endpoint, usage, tries, note? }`
8426
+ * (`note`: openAIImageRetriedNote's) or openAIImageFailureResult's failure.
8427
+ *
8428
+ * A gpt-image request is streamed with OPENAI_IMAGE_PARTIAL_IMAGES partials
8429
+ * per image (dall-e models do not stream). With `preview` ({ conversationId,
8430
+ * messageId }: the queue message the turn answers) the owner's clients see
8431
+ * each try start and each partial; a retry drops the first try's, and the
8432
+ * turn's end (/api/response) the rest.
8433
+ *
8434
+ * Each try is checked against the daily cap and reserves its estimate
8435
+ * right before it is sent (the resend without stream fields keeps the
8436
+ * reservation); the cap refuses a try with OPENAI_IMAGE_DAILY_CAP_REACHED
8437
+ * and sends nothing. When the try ends, its cost is booked under the queue
8438
+ * row (`scopeId`: the image attempt, or the claim, that runs it; run_id
8439
+ * image:<scopeId>:try<n>) and its reservation dropped, in one step.
8440
+ */
8441
+ async function runOpenAIImageRequest(requestBody, { idempotencyKey = '', preview = null, scopeId = '' } = {}) {
8442
+ const settings = getOpenAIProviderSettings();
8443
+ if (settings?.configured !== true || !String(settings?.apiKey || '').trim()) {
8444
+ return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI BYOK is not configured');
8445
+ }
8446
+ const baseUrl = String(settings.baseUrl || '').trim().replace(/\/+$/, '');
8447
+ if (!baseUrl) return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI base URL is not configured');
8448
+
8449
+ let editImages = [];
8450
+ try {
8451
+ editImages = resolveOpenAIImageEditAttachments(requestBody.attachments, {
8452
+ maxImageBytes: MAX_UPLOAD_BYTES,
8453
+ allowedRootPath: uploadsDir,
8454
+ allowedBlobRootPaths: uploadReadDirs.slice(1),
8455
+ allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
8456
+ });
8457
+ } catch (error) {
8458
+ return openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', error?.message || 'Invalid image attachment');
8459
+ }
8460
+
8461
+ const family = openAIImageModelFamily(requestBody.model);
8462
+ const fields = buildOpenAIImageRequestFields(requestBody);
8463
+ const endpoint = editImages.length ? '/images/edits' : '/images/generations';
8464
+ const timeoutMs = editImages.length ? openAIImageEditTimeoutMs : openAIImageGenerateTimeoutMs;
8465
+ const headers = {
8466
+ Authorization: `Bearer ${settings.apiKey}`,
8467
+ ...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
8468
+ };
8469
+ let streaming = family === 'gpt-image' && OPENAI_IMAGE_PARTIAL_IMAGES > 0;
8470
+ const previewTurn = preview
8471
+ ? imagePreviews.begin({
8472
+ conversationId: preview.conversationId,
8473
+ messageId: preview.messageId,
8474
+ size: requestBody.size,
8475
+ quality: requestBody.quality,
8476
+ count: requestBody.n,
8477
+ model: requestBody.model,
8478
+ })
8479
+ : null;
8480
+ const tries = [];
8481
+ // HTTP requests of this turn, the resend without stream fields included:
8482
+ // never more than two.
8483
+ let calls = 0;
8484
+ let resent = false;
8485
+ let result = null;
8486
+ // What a try of this request costs, and where it books.
8487
+ const spendRequest = {
8488
+ model: requestBody.model,
8489
+ size: requestBody.size,
8490
+ quality: requestBody.quality,
8491
+ count: requestBody.n,
8492
+ inputImages: editImages.length,
8493
+ };
8494
+ const bookingScope = String(scopeId || '').trim() || `claim_${randomUUID()}`;
8495
+ // The try in flight's reservation: held across the resend, released
8496
+ // (with its booking, or without one) when the try ends.
8497
+ let reservation = null;
8498
+ previewTurn?.started(1);
8499
+ try {
8500
+ for (;;) {
8501
+ if (!reservation) {
8502
+ const tryNumber = tries.length + 1;
8503
+ const reserved = imageSpend.reserveTry({ estimateUsd: estimateOpenAIImageCost(spendRequest).costUsd });
8504
+ if (!reserved.ok) {
8505
+ console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tryNumber} stopped by the daily cap`);
8506
+ result = openAIImageCapRefusalResult(tries, reserved);
8507
+ break;
8508
+ }
8509
+ reservation = reserved.reservation;
8510
+ }
8511
+ calls += 1;
8512
+ const tried = await sendOpenAIImageRequestOnce({
8513
+ url: `${baseUrl}${endpoint}`,
8514
+ // The resend's body differs from the first request's: its own key.
8515
+ headers: resent && idempotencyKey ? { ...headers, 'Idempotency-Key': `${idempotencyKey}-nostream` } : headers,
8516
+ fields: streaming ? { ...fields, stream: true, partial_images: OPENAI_IMAGE_PARTIAL_IMAGES } : fields,
8517
+ editImages,
8518
+ family,
8519
+ timeoutMs,
8520
+ outputFormat: requestBody.output_format,
8521
+ model: requestBody.model,
8522
+ onPartial: previewTurn ? (partial) => previewTurn.partial(partial) : null,
8523
+ });
8524
+ // The resend without stream fields (below) is the same try: it keeps
8525
+ // the reservation and books once.
8526
+ const resend = streaming && calls < 2 && tried.kind === 'refused' && Number(tried.status) === 400 && !tried.streamed
8527
+ && /\bstream|partial_image/i.test(String(tried.message || '').slice(0, 240));
8528
+ if (!resend) {
8529
+ imageSpend.settleTry(reservation, {
8530
+ conversationId: requestBody.conversationId,
8531
+ turnId: requestBody.messageId,
8532
+ runId: openAIImageTryRunId(bookingScope, tries.length + 1),
8533
+ model: requestBody.model,
8534
+ charge: openAIImageTryCharge({ tried, request: spendRequest }),
8535
+ });
8536
+ reservation = null;
8537
+ }
8538
+ if (tried.ok) {
8539
+ const note = openAIImageRetriedNote(tries, { timeoutMs });
8540
+ result = {
8541
+ ok: true,
8542
+ generatedImages: tried.generatedImages,
8543
+ model: requestBody.model,
8544
+ endpoint,
8545
+ usage: tried.usage,
8546
+ tries: tries.length + 1,
8547
+ ...(note ? { note } : {}),
8548
+ };
8549
+ break;
8550
+ }
8551
+ // An endpoint that takes no stream (an OpenAI-compatible one, or a
8552
+ // model OpenAI does not stream) refuses the fields with a 400 before
8553
+ // any work: the same try again without them, nothing billed. It
8554
+ // counts as one of the two requests, so a resend that fails is final.
8555
+ // Any other refusal (a 429 that names streaming, say) is no reason.
8556
+ if (resend) {
8557
+ console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} refused streaming; sent again without it`);
8558
+ streaming = false;
8559
+ resent = true;
8560
+ continue;
8561
+ }
8562
+ tries.push(tried);
8563
+ console.warn(
8564
+ `[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tries.length} ${tried.kind}`
8565
+ + `${tried.status ? ` status=${tried.status}` : ''}${tried.detail ? ` ${String(tried.detail).slice(0, 200)}` : ''}`,
8566
+ );
8567
+ if (calls >= 2 || !OPENAI_IMAGE_RETRY_KINDS.has(tried.kind)) break;
8568
+ await delay(openAIImageRetryDelayMs);
8569
+ previewTurn?.retry();
8570
+ previewTurn?.started(tries.length + 1);
8571
+ }
8572
+ } finally {
8573
+ // A throw or an abort: the try books nothing it did not book above.
8574
+ imageSpend.releaseTry(reservation);
8575
+ previewTurn?.ended({ ok: Boolean(result?.ok) });
8576
+ }
8577
+ return result || openAIImageFailureResult(tries, { timeoutMs, resent });
8578
+ }
8579
+
8580
+ /**
8581
+ * The reply line of an image turn: what came back, and what this turn's
8582
+ * tries were booked at ("~USD 0.21", marked "(estimate)" when any part is
8583
+ * one, "(price unknown, estimated)" for the stand-in); from 80 % of the
8584
+ * daily cap on, today's spend against it.
8585
+ */
8586
+ function openAIImageReplyLine({ images, model, turnId }) {
8587
+ let cost = null;
8588
+ let state = null;
8589
+ try {
8590
+ cost = imageSpend.turnCost(turnId);
8591
+ state = imageSpend.buildState();
8592
+ } catch (error) {
8593
+ console.warn(`[${ts()}] IMAGE the cost of ${String(turnId || '').slice(0, 8)} could not be read: ${error?.message || error}`);
8594
+ }
8595
+ let line = formatOpenAIImageSummary({ images, model, costUsd: cost ? cost.costUsd : null });
8596
+ if (cost?.unknownPrice) line += ' (price unknown, estimated)';
8597
+ else if (cost?.estimated) line += ' (estimate)';
8598
+ // The spend state's own rule (booked and in flight, `warning`), as the
8599
+ // banner reads it.
8600
+ if (state?.capUsd && state.warning) {
8601
+ const todayUsd = (Number(state.spentUsd) || 0) + (Number(state.reservedUsd) || 0);
8602
+ line += ` · today ${formatOpenAIImageUsd(todayUsd)} of ${state.capUsd.toFixed(2)} (${Math.round((todayUsd / state.capUsd) * 100)} %)`;
8603
+ }
8604
+ return line;
8605
+ }
8606
+
8607
+ // Today's OpenAI image spend against the daily cap (the Check Usage card,
8608
+ // the banner, a paired relay's check before create_session); with
8609
+ // `?conversationId=` also that chat's.
8610
+ app.get(OPENAI_IMAGE_USAGE_PATH, auth, (req, res) => {
8611
+ const conversationId = String(req.query?.conversationId || '').trim().slice(0, 200) || null;
8612
+ try {
8613
+ return res.json(imageSpend.buildState({ conversationId }));
8614
+ } catch (error) {
8615
+ return res.status(500).json({ error: `Image spend could not be read: ${error?.message || error}` });
8616
+ }
8617
+ });
8618
+
8619
+ /**
8620
+ * An image attempt the recovery found cut off while its request ran
8621
+ * ('started', now 'uncertain'): it may have been billed, so its estimate
8622
+ * is booked as image:<attemptId>:recovered, unless a try of that attempt
8623
+ * was booked already.
8624
+ */
8625
+ function bookRecoveredImageAttempt(operation, attempt) {
8626
+ if (!operation || !attempt?.id) return;
8627
+ let parameters = {};
8628
+ try {
8629
+ parameters = JSON.parse(operation.parameters_json || '{}') || {};
8630
+ } catch {}
8631
+ let inputImages = operation.parent_node_id ? 1 : 0;
8632
+ if (!operation.parent_node_id) {
8633
+ try {
8634
+ inputImages = (stmts.listAssets?.all?.(operation.id) || []).length;
8635
+ } catch {}
8636
+ }
8637
+ try {
8638
+ imageSpend.bookRecovered({
8639
+ conversationId: operation.conversation_id,
8640
+ turnId: operation.queue_message_id,
8641
+ attemptId: attempt.id,
8642
+ model: operation.selected_image_model,
8643
+ startedAt: attempt.started_at,
8644
+ request: { size: parameters.size, quality: parameters.quality, count: parameters.count ?? 1, inputImages },
8645
+ });
8646
+ } catch (error) {
8647
+ console.warn(`[${ts()}] IMAGE_OP the estimate of attempt=${attempt.id} was not booked: ${error?.message || error}`);
8648
+ }
8649
+ }
8650
+
8651
+ /**
8652
+ * Why a queue row may not go back to pending (/api/requeue): `{ running:
8653
+ * true }` while its image request runs in this process, `{ failure }`
8654
+ * (a terminalError) once its request went to OpenAI, or null for a row
8655
+ * whose request was never sent (requeueing it bills nothing).
8656
+ */
8657
+ function imageTurnRequeueStop(queueRow) {
8658
+ const operationId = String(queueRow?.image_operation_id || '').trim();
8659
+ if (operationId) {
8660
+ if (executingImageOperations.has(operationId)) return { running: true };
8661
+ const attempt = stmts.getLatestAttempt?.get?.(operationId) || null;
8662
+ if (!attempt) return null;
8663
+ if (attempt.status === 'started') {
8664
+ stmts.finishAttempt.run({
8665
+ id: attempt.id,
8666
+ status: 'uncertain',
8667
+ providerRequestId: null,
8668
+ providerConversationId: null,
8669
+ providerResponseId: null,
8670
+ httpStatus: null,
8671
+ errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
8672
+ errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
8673
+ completedAt: new Date().toISOString(),
8674
+ });
8675
+ bookRecoveredImageAttempt(stmts.getOperation?.get?.(operationId), attempt);
8676
+ return { failure: { code: 'IMAGE_ATTEMPT_INTERRUPTED', message: OPENAI_IMAGE_INTERRUPTED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
8677
+ }
8678
+ return {
8679
+ failure: {
8680
+ code: String(attempt.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED',
8681
+ message: String(attempt.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE,
8682
+ guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
8683
+ },
8684
+ };
8685
+ }
8686
+ const dispatched = dispatchedImageMessages.get(String(queueRow?.id || ''));
8687
+ if (!dispatched) return null;
8688
+ if (dispatched.running) return { running: true };
8689
+ return { failure: { code: 'IMAGE_ATTEMPT_REPEATED', message: OPENAI_IMAGE_REPEATED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
8690
+ }
8691
+
7655
8692
  // Called by the relay's own worker on this machine; nothing that came in
7656
8693
  // through a proxy or the tunnel may start a provider image request.
8694
+ // The request runs in this process (runOpenAIImageRequest): its timeout
8695
+ // and its one retry are the only ones. A failure ends the row with a note,
8696
+ // never with a requeue: a requeued image turn was billed again.
7657
8697
  app.post('/api/image-operations/:operationId/execute', auth, async (req, res) => {
7658
8698
  if (!isTrulyLocalRequest(req)) {
7659
8699
  return res.status(403).json({ error: 'Image operation execution is reachable from this machine only' });
@@ -7673,6 +8713,11 @@ export function registerMessagesRoutes(app, deps) {
7673
8713
  if (!queueRow || queueRow.image_operation_id !== operationId) {
7674
8714
  return res.status(409).json({ error: 'Image operation queue linkage is unavailable' });
7675
8715
  }
8716
+ // Delivered again while its request still runs here (a worker that
8717
+ // died and came back): that run finishes the row.
8718
+ if (executingImageOperations.has(operationId)) {
8719
+ return res.status(202).json({ ok: true, outcome: 'executing', operationId });
8720
+ }
7676
8721
 
7677
8722
  const now = new Date().toISOString();
7678
8723
  const selfHeaders = {
@@ -7680,6 +8725,23 @@ export function registerMessagesRoutes(app, deps) {
7680
8725
  'Content-Type': 'application/json',
7681
8726
  };
7682
8727
  const selfBaseUrl = `http://127.0.0.1:${config.port}`;
8728
+ const publishFailure = async (terminalError) => {
8729
+ // `terminalError` is the field /api/response reads; under any other
8730
+ // name the failure was refused as an empty reply and the row stayed
8731
+ // processing until the relay recovered and re-ran it.
8732
+ const response = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
8733
+ method: 'POST',
8734
+ headers: selfHeaders,
8735
+ body: JSON.stringify({
8736
+ messageId: operation.queue_message_id,
8737
+ conversationId: operation.conversation_id,
8738
+ terminalError,
8739
+ }),
8740
+ }).catch(() => null);
8741
+ if (!response?.ok) {
8742
+ console.warn(`[${ts()}] IMAGE_OP failed to publish the failure of operation=${operationId}`);
8743
+ }
8744
+ };
7683
8745
  const acquireAttempt = db.transaction(() => {
7684
8746
  const current = stmts.getLatestAttempt.get(operationId);
7685
8747
  if (current?.status === 'started') {
@@ -7697,15 +8759,19 @@ export function registerMessagesRoutes(app, deps) {
7697
8759
  providerResponseId: null,
7698
8760
  httpStatus: null,
7699
8761
  errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
7700
- errorMessage: 'Image execution was interrupted after provider dispatch may have begun',
8762
+ errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
7701
8763
  completedAt: now,
7702
8764
  });
7703
8765
  return { acquired: false, attempt: current, stale: true };
7704
8766
  }
8767
+ // An attempt that ended: its note was published, or publishing it
8768
+ // failed and the row came back. Either way the request is not sent
8769
+ // again.
8770
+ if (current) return { acquired: false, attempt: current, finished: true };
7705
8771
  const attempt = {
7706
8772
  id: `img_attempt_${uuidv4()}`,
7707
8773
  operationId,
7708
- attemptNumber: Math.max(1, Number(current?.attempt_number || 0) + 1),
8774
+ attemptNumber: 1,
7709
8775
  provider: operation.provider,
7710
8776
  capabilitySnapshotJson: JSON.stringify({
7711
8777
  executionMode: operation.execution_mode,
@@ -7718,339 +8784,250 @@ export function registerMessagesRoutes(app, deps) {
7718
8784
  });
7719
8785
  const acquired = acquireAttempt();
7720
8786
  if (acquired.stale) {
7721
- await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
7722
- method: 'POST',
7723
- headers: selfHeaders,
7724
- body: JSON.stringify({
7725
- messageId: operation.queue_message_id,
7726
- conversationId: operation.conversation_id,
7727
- // `terminalError` is the field /api/response reads; under any other
7728
- // name the failure was refused as an empty reply and the row stayed
7729
- // processing until the relay recovered and re-ran it.
7730
- terminalError: {
7731
- code: 'IMAGE_ATTEMPT_INTERRUPTED',
7732
- message: 'Image generation may have completed before the relay was interrupted. It was not retried to avoid duplicate images.',
7733
- retryable: false,
7734
- },
7735
- }),
7736
- }).catch(() => null);
8787
+ bookRecoveredImageAttempt(operation, acquired.attempt);
8788
+ await publishFailure({
8789
+ code: 'IMAGE_ATTEMPT_INTERRUPTED',
8790
+ message: OPENAI_IMAGE_INTERRUPTED_NOTE,
8791
+ guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
8792
+ retryable: false,
8793
+ });
7737
8794
  return res.status(409).json({
7738
8795
  ok: false,
7739
8796
  outcome: 'uncertain',
7740
8797
  operationId,
7741
- error: 'The previous image attempt ended in an uncertain state and was not retried',
8798
+ code: 'IMAGE_ATTEMPT_INTERRUPTED',
8799
+ error: OPENAI_IMAGE_INTERRUPTED_NOTE,
8800
+ });
8801
+ }
8802
+ if (acquired.finished) {
8803
+ const earlier = acquired.attempt;
8804
+ const message = String(earlier.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE;
8805
+ const code = String(earlier.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED';
8806
+ await publishFailure({ code, message, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE, retryable: false });
8807
+ return res.status(409).json({
8808
+ ok: false,
8809
+ outcome: ['failed_terminal', 'failed_retryable'].includes(earlier.status) ? 'terminal' : 'uncertain',
8810
+ operationId,
8811
+ code,
8812
+ error: message,
7742
8813
  });
7743
8814
  }
7744
8815
  if (!acquired.acquired) {
7745
8816
  return res.status(202).json({ ok: true, outcome: 'executing', operationId });
7746
8817
  }
7747
8818
 
7748
- const requestAttachments = [];
7749
- if (operation.parent_node_id) {
7750
- const parentNode = stmts.getNode.get(operation.parent_node_id);
7751
- const parentMessage = parentNode
7752
- ? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
7753
- : null;
7754
- const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
7755
- String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
7756
- ));
7757
- const generatedImage = parentAttachment?.generatedImage;
7758
- const parentPath = resolveGeneratedImageReadPath({
7759
- resolveSessionStateRoot,
7760
- sessionId: generatedImage?.sessionId,
7761
- relativePath: generatedImage?.relativePath,
7762
- });
7763
- if (!parentPath) {
8819
+ executingImageOperations.add(operationId);
8820
+ try {
8821
+ const finishAttempt = (result) => {
7764
8822
  stmts.finishAttempt.run({
7765
8823
  id: acquired.attempt.id,
7766
- status: 'failed_terminal',
8824
+ // A request that never reached OpenAI (unreachable, or stopped by
8825
+ // the daily cap before it was sent) closes as retryable: nothing
8826
+ // was billed, and the recovery books nothing for it.
8827
+ status: result.outcome === 'uncertain'
8828
+ ? 'uncertain'
8829
+ : ((result.code === 'IMAGE_PROVIDER_UNREACHABLE' || (result.code === OPENAI_IMAGE_DAILY_CAP_REACHED_CODE && !result.sent))
8830
+ ? 'failed_retryable'
8831
+ : 'failed_terminal'),
7767
8832
  providerRequestId: null,
7768
8833
  providerConversationId: null,
7769
8834
  providerResponseId: null,
7770
- httpStatus: 400,
7771
- errorCode: 'IMAGE_PARENT_UNAVAILABLE',
7772
- errorMessage: 'The parent generated image is unavailable',
8835
+ httpStatus: result.upstreamStatus || null,
8836
+ errorCode: result.code,
8837
+ errorMessage: String(result.error || '').slice(0, 500),
7773
8838
  completedAt: new Date().toISOString(),
7774
8839
  });
8840
+ };
8841
+ const fail = async (result) => {
8842
+ finishAttempt(result);
8843
+ await publishFailure(openAIImageTerminalError(result));
7775
8844
  return res.status(409).json({
7776
8845
  ok: false,
7777
- outcome: 'terminal',
7778
- code: 'IMAGE_PARENT_UNAVAILABLE',
7779
- error: 'The parent generated image is unavailable',
8846
+ outcome: result.outcome,
8847
+ operationId,
8848
+ code: result.code,
8849
+ error: result.error,
8850
+ ...(result.usage ? { usage: result.usage } : {}),
7780
8851
  });
7781
- }
7782
- requestAttachments.push({
7783
- name: parentAttachment.name,
7784
- type: parentAttachment.type,
7785
- path: parentPath,
7786
- });
7787
- } else {
7788
- for (const asset of stmts.listAssets.all(operationId)) {
8852
+ };
8853
+
8854
+ const requestAttachments = [];
8855
+ if (operation.parent_node_id) {
8856
+ const parentNode = stmts.getNode.get(operation.parent_node_id);
8857
+ const parentMessage = parentNode
8858
+ ? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
8859
+ : null;
8860
+ const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
8861
+ String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
8862
+ ));
8863
+ const generatedImage = parentAttachment?.generatedImage;
8864
+ const parentPath = resolveGeneratedImageReadPath({
8865
+ resolveSessionStateRoot,
8866
+ sessionId: generatedImage?.sessionId,
8867
+ relativePath: generatedImage?.relativePath,
8868
+ });
8869
+ if (!parentPath) {
8870
+ return await fail({
8871
+ ...openAIImageRequestRefusal(400, 'IMAGE_PARENT_UNAVAILABLE', 'The parent generated image is unavailable'),
8872
+ upstreamStatus: 400,
8873
+ });
8874
+ }
7789
8875
  requestAttachments.push({
7790
- name: asset.original_name || 'reference-image',
7791
- type: asset.media_type,
7792
- path: uploadPathForSha(asset.content_sha256),
8876
+ name: parentAttachment.name,
8877
+ type: parentAttachment.type,
8878
+ path: parentPath,
7793
8879
  });
8880
+ } else {
8881
+ // In the order they were attached: the first is the base image.
8882
+ for (const asset of stmts.listAssets.all(operationId)) {
8883
+ requestAttachments.push({
8884
+ name: asset.original_name || 'reference-image',
8885
+ type: asset.media_type,
8886
+ path: uploadPathForSha(asset.content_sha256),
8887
+ });
8888
+ }
7794
8889
  }
7795
- }
7796
-
7797
- let parameters = {};
7798
- try {
7799
- parameters = JSON.parse(operation.parameters_json || '{}');
7800
- } catch {}
7801
- const generationBody = {
7802
- messageId: operation.queue_message_id,
7803
- conversationId: operation.conversation_id,
7804
- model: operation.selected_image_model,
7805
- prompt: operation.prompt,
7806
- n: Number.isInteger(Number(parameters.count)) ? Number(parameters.count) : 1,
7807
- attachments: requestAttachments,
7808
- };
7809
- if (parameters.size === 'auto' || /^\d{2,5}x\d{2,5}$/i.test(String(parameters.size || ''))) {
7810
- generationBody.size = parameters.size;
7811
- }
7812
- if (/^(auto|standard|hd|low|medium|high)$/i.test(String(parameters.quality || ''))) {
7813
- generationBody.quality = parameters.quality;
7814
- }
7815
8890
 
7816
- let generatedPayload = null;
7817
- try {
7818
- const generatedResponse = await fetch(`${selfBaseUrl}${remotePath}/api/openai/images/generate`, {
7819
- method: 'POST',
7820
- headers: selfHeaders,
7821
- body: JSON.stringify(generationBody),
7822
- });
7823
- generatedPayload = await generatedResponse.json().catch(() => ({}));
7824
- if (!generatedResponse.ok) {
7825
- const error = new Error(generatedPayload?.error || `OpenAI image request failed (${generatedResponse.status})`);
7826
- error.status = generatedResponse.status;
7827
- throw error;
7828
- }
7829
- const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
7830
- method: 'POST',
7831
- headers: selfHeaders,
7832
- body: JSON.stringify({
8891
+ // {size, quality, count, output_format, background}; any may be
8892
+ // missing, count is 1 then.
8893
+ let parameters = {};
8894
+ try {
8895
+ parameters = JSON.parse(operation.parameters_json || '{}') || {};
8896
+ } catch {}
8897
+ let requestBody;
8898
+ try {
8899
+ requestBody = normalizeOpenAIImageGenerateRequestBody({
7833
8900
  messageId: operation.queue_message_id,
7834
8901
  conversationId: operation.conversation_id,
7835
- text: '',
7836
- generatedImages: generatedPayload.generatedImages,
7837
- model: generatedPayload.model || operation.selected_image_model,
7838
- }),
7839
- });
7840
- const finalized = await finalizeResponse.json().catch(() => ({}));
7841
- if (!finalizeResponse.ok) {
7842
- const error = new Error(finalized?.error || `Image finalization failed (${finalizeResponse.status})`);
7843
- error.status = finalizeResponse.status;
7844
- throw error;
8902
+ model: operation.selected_image_model,
8903
+ prompt: operation.prompt,
8904
+ n: parameters.count ?? 1,
8905
+ size: parameters.size,
8906
+ quality: parameters.quality,
8907
+ output_format: parameters.output_format,
8908
+ background: parameters.background,
8909
+ attachments: requestAttachments,
8910
+ });
8911
+ } catch (error) {
8912
+ return await fail(openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', String(error?.message || 'Invalid image request')));
7845
8913
  }
7846
- return res.json({
7847
- ok: true,
7848
- outcome: 'completed',
7849
- operationId,
7850
- generatedImageCount: Array.isArray(generatedPayload.generatedImages)
7851
- ? generatedPayload.generatedImages.length
7852
- : 0,
7853
- });
7854
- } catch (error) {
7855
- const status = Number(error?.status || 0);
7856
- // `relayProcessing`: the provider answered 2xx and the relay could not
7857
- // process the reply. The generation is billed, a retry pays again and
7858
- // fails the same way, so the row fails like a refusal.
7859
- const relayProcessing = generatedPayload?.relayProcessing === true;
7860
- const terminal = (status >= 400 && status < 500) || relayProcessing;
7861
- const uncertain = !!generatedPayload?.generatedImages;
7862
- const outcome = terminal ? 'terminal' : (uncertain ? 'uncertain' : 'retryable');
7863
- const errorMessage = relayProcessing
7864
- ? `The relay could not process the image OpenAI returned (${error?.message || 'unknown error'}). The generation was billed; the message was not retried.`
7865
- : String(error?.message || 'Image operation failed');
7866
- stmts.finishAttempt.run({
7867
- id: acquired.attempt.id,
7868
- status: terminal ? 'failed_terminal' : (uncertain ? 'uncertain' : 'failed_retryable'),
7869
- providerRequestId: null,
7870
- providerConversationId: null,
7871
- providerResponseId: null,
7872
- httpStatus: status || null,
7873
- errorCode: terminal
7874
- ? (relayProcessing ? 'IMAGE_REPLY_UNPROCESSED' : 'IMAGE_PROVIDER_REJECTED')
7875
- : (uncertain ? 'IMAGE_FINALIZATION_UNCERTAIN' : 'IMAGE_PROVIDER_UNAVAILABLE'),
7876
- errorMessage: errorMessage.slice(0, 500),
7877
- completedAt: new Date().toISOString(),
8914
+
8915
+ const result = await runOpenAIImageRequest(requestBody, {
8916
+ idempotencyKey: String(operation.idempotency_key || operation.id),
8917
+ preview: { conversationId: operation.conversation_id, messageId: operation.queue_message_id },
8918
+ scopeId: acquired.attempt.id,
7878
8919
  });
7879
- if (outcome === 'retryable') {
7880
- const requeueResponse = await fetch(`${selfBaseUrl}${remotePath}/api/requeue`, {
7881
- method: 'POST',
7882
- headers: selfHeaders,
7883
- body: JSON.stringify({ messageId: operation.queue_message_id }),
7884
- }).catch(() => null);
7885
- if (!requeueResponse?.ok) {
7886
- console.warn(`[${ts()}] IMAGE_OP failed to requeue retryable operation=${operationId}`);
7887
- }
7888
- } else {
7889
- const failure = {
7890
- code: outcome === 'uncertain'
7891
- ? 'IMAGE_FINALIZATION_UNCERTAIN'
7892
- : (relayProcessing ? 'IMAGE_REPLY_UNPROCESSED' : 'IMAGE_PROVIDER_REJECTED'),
7893
- message: errorMessage.slice(0, 500),
7894
- retryable: false,
7895
- };
7896
- const failResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
8920
+ if (!result.ok) return await fail(result);
8921
+
8922
+ let finalizeError = null;
8923
+ try {
8924
+ const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
7897
8925
  method: 'POST',
7898
8926
  headers: selfHeaders,
7899
8927
  body: JSON.stringify({
7900
8928
  messageId: operation.queue_message_id,
7901
8929
  conversationId: operation.conversation_id,
7902
- terminalError: failure,
8930
+ text: '',
8931
+ generatedImages: result.generatedImages,
8932
+ model: result.model || operation.selected_image_model,
8933
+ // A first try that may have been billed too: the reply says so.
8934
+ ...(result.note ? { imageNote: result.note } : {}),
7903
8935
  }),
7904
- }).catch(() => null);
7905
- if (!failResponse?.ok) {
7906
- console.warn(`[${ts()}] IMAGE_OP failed to finalize ${outcome} operation=${operationId}`);
7907
- }
8936
+ });
8937
+ const finalized = await finalizeResponse.json().catch(() => ({}));
8938
+ if (!finalizeResponse.ok) finalizeError = finalized?.error || `Image finalization failed (${finalizeResponse.status})`;
8939
+ } catch (error) {
8940
+ finalizeError = error?.message || 'Image finalization failed';
7908
8941
  }
7909
- return res.status(outcome === 'retryable' ? 503 : 409).json({
7910
- ok: false,
7911
- outcome,
8942
+ if (finalizeError) {
8943
+ return await fail({
8944
+ ok: false,
8945
+ outcome: 'uncertain',
8946
+ code: 'IMAGE_FINALIZATION_UNCERTAIN',
8947
+ error: `The images OpenAI returned could not be saved (${finalizeError}). The generation was billed; the message was not retried.`,
8948
+ upstreamStatus: 200,
8949
+ usage: result.usage,
8950
+ });
8951
+ }
8952
+ return res.json({
8953
+ ok: true,
8954
+ outcome: 'completed',
7912
8955
  operationId,
7913
- error: String(error?.message || 'Image operation failed'),
8956
+ generatedImageCount: result.generatedImages.length,
8957
+ usage: result.usage,
8958
+ ...(result.note ? { note: result.note } : {}),
7914
8959
  });
8960
+ } finally {
8961
+ executingImageOperations.delete(operationId);
7915
8962
  }
7916
8963
  });
7917
8964
 
8965
+ // The relay's own workers call this on this machine for an image turn
8966
+ // without an operation (image continuity off). Like the execute route it
8967
+ // is local only: through the tunnel, or with a paired relay's token, it
8968
+ // would spend the owner's money outside the queue. One request per queue
8969
+ // message: a second call for the same message is refused.
7918
8970
  app.post('/api/openai/images/generate', auth, async (req, res) => {
7919
8971
  touchCli();
7920
- const settings = getOpenAIProviderSettings();
7921
- if (settings?.configured !== true || !String(settings?.apiKey || '').trim()) {
7922
- return res.status(503).json({ error: 'OpenAI BYOK is not configured' });
8972
+ if (!isTrulyLocalRequest(req)) {
8973
+ return res.status(403).json({ error: 'OpenAI image requests are reachable from this machine only' });
7923
8974
  }
7924
8975
 
7925
8976
  let requestBody;
7926
8977
  try {
7927
- requestBody = normalizeOpenAIImageGenerateRequestBody(req.body, {
7928
- maxImages: MAX_UPLOAD_ATTACHMENTS,
7929
- });
8978
+ requestBody = normalizeOpenAIImageGenerateRequestBody(req.body);
7930
8979
  } catch (error) {
7931
- return res.status(400).json({ error: error?.message || 'Invalid OpenAI image request' });
7932
- }
7933
-
7934
- const baseUrl = String(settings.baseUrl || '').trim().replace(/\/+$/, '');
7935
- if (!baseUrl) return res.status(503).json({ error: 'OpenAI base URL is not configured' });
7936
-
7937
- let editAttachment = null;
7938
- if (Array.isArray(requestBody.attachments) && requestBody.attachments.length > 0) {
7939
- try {
7940
- editAttachment = resolveOpenAIImageEditAttachment(requestBody.attachments, {
7941
- maxImageBytes: MAX_UPLOAD_BYTES,
7942
- allowedRootPath: uploadsDir,
7943
- allowedBlobRootPaths: uploadReadDirs.slice(1),
7944
- allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
7945
- });
7946
- } catch (error) {
7947
- return res.status(400).json({ error: error?.message || 'Invalid image attachment' });
7948
- }
7949
- }
7950
-
7951
- const options = {
7952
- model: requestBody.model,
7953
- prompt: requestBody.prompt,
7954
- };
7955
- const supportsResponseFormat = String(requestBody.model || '')
7956
- .trim()
7957
- .toLowerCase()
7958
- .replace(/^openai\//, '')
7959
- .startsWith('dall-e-');
7960
- if (supportsResponseFormat) options.response_format = 'b64_json';
7961
- if (requestBody.n !== undefined) options.n = requestBody.n;
7962
- if (requestBody.size) options.size = requestBody.size;
7963
- if (requestBody.quality) options.quality = requestBody.quality;
7964
-
7965
- const endpoint = editAttachment ? '/images/edits' : '/images/generations';
7966
- let response;
7967
- try {
7968
- if (editAttachment) {
7969
- const form = new FormData();
7970
- form.set('model', options.model);
7971
- form.set('prompt', options.prompt);
7972
- if (options.response_format) form.set('response_format', String(options.response_format));
7973
- if (options.n !== undefined) form.set('n', String(options.n));
7974
- if (options.size) form.set('size', String(options.size));
7975
- if (options.quality) form.set('quality', String(options.quality));
7976
- form.set(
7977
- 'image',
7978
- new Blob([editAttachment.bytes], { type: editAttachment.mimeType }),
7979
- editAttachment.name,
7980
- );
7981
- response = await fetch(`${baseUrl}${endpoint}`, {
7982
- method: 'POST',
7983
- headers: {
7984
- Authorization: `Bearer ${settings.apiKey}`,
7985
- },
7986
- body: form,
7987
- });
7988
- } else {
7989
- response = await fetch(`${baseUrl}${endpoint}`, {
7990
- method: 'POST',
7991
- headers: {
7992
- Authorization: `Bearer ${settings.apiKey}`,
7993
- 'Content-Type': 'application/json',
7994
- },
7995
- body: JSON.stringify(options),
7996
- });
7997
- }
7998
- } catch (error) {
7999
- return res.status(502).json({ error: `OpenAI image API request failed: ${error?.message || 'request error'}` });
8000
- }
8001
-
8002
- const rawText = String(await response.text().catch(() => '')).trim();
8003
- let payload = {};
8004
- if (rawText) {
8005
- try {
8006
- payload = JSON.parse(rawText);
8007
- } catch {
8008
- // After a 2xx the generation is billed; the callers fail the row
8009
- // instead of paying for it again. An error status with a body that
8010
- // is not JSON stays a provider failure.
8011
- return res.status(502).json({
8012
- error: 'OpenAI image API returned invalid JSON',
8013
- ...(response.ok ? { relayProcessing: true } : {}),
8014
- });
8015
- }
8980
+ return res.status(400).json({
8981
+ error: error?.message || 'Invalid OpenAI image request',
8982
+ code: 'IMAGE_REQUEST_INVALID',
8983
+ outcome: 'terminal',
8984
+ });
8016
8985
  }
8017
-
8018
- if (!response.ok) {
8019
- const detail = String(payload?.error?.message || payload?.message || rawText).trim().slice(0, 240);
8020
- const upstreamStatus = Number(response.status);
8021
- const relayStatus = Number.isFinite(upstreamStatus) && upstreamStatus >= 400 && upstreamStatus < 500
8022
- ? upstreamStatus
8023
- : 502;
8024
- return res.status(relayStatus).json({
8025
- error: `OpenAI image API failed (${response.status})${detail ? `: ${detail}` : ''}`,
8026
- upstreamStatus: Number.isFinite(upstreamStatus) ? upstreamStatus : null,
8986
+ if (dispatchedImageMessages.has(requestBody.messageId)) {
8987
+ return res.status(409).json({
8988
+ error: OPENAI_IMAGE_REPEATED_NOTE,
8989
+ code: 'IMAGE_ATTEMPT_REPEATED',
8990
+ outcome: 'uncertain',
8027
8991
  });
8028
8992
  }
8029
8993
 
8030
- let generatedImages = [];
8994
+ const dispatched = noteDispatchedImageMessage(requestBody.messageId);
8995
+ // One run of the message: the claim's attempt id (a restart claims the
8996
+ // row anew), so its tries never merge into an earlier run's rows.
8997
+ // Known gap: this path (image continuity off) keeps no attempt row, so a
8998
+ // try a relay restart cuts off is booked nowhere; the operation path's
8999
+ // recovery books such a try as an estimate. docs/providers.md says so.
9000
+ let claimId = '';
8031
9001
  try {
8032
- generatedImages = normalizeOpenAIImageApiGeneratedImages(payload, {
8033
- maxImages: MAX_UPLOAD_ATTACHMENTS,
8034
- });
8035
- } catch (error) {
8036
- // The provider answered 2xx, so the generation is billed; what failed
8037
- // is the relay's handling of the reply. Logged with the stack, and
8038
- // flagged so the callers fail the row instead of requeueing it: a
8039
- // retry pays again and fails the same way.
8040
- console.error(
8041
- `[${ts()}] IMAGE reply of ${requestBody.model} not processed: ${error?.message || error} (body ${rawText.length} chars)\n${error?.stack || ''}`,
8042
- );
8043
- return res.status(502).json({
8044
- error: error?.message || 'OpenAI image API returned no images',
8045
- relayProcessing: true,
9002
+ claimId = String(stmts.findQById?.get?.(requestBody.messageId)?.attempt_id || '').trim();
9003
+ } catch {}
9004
+ const result = await runOpenAIImageRequest(requestBody, {
9005
+ idempotencyKey: `oar-image-${requestBody.messageId}`,
9006
+ preview: { conversationId: requestBody.conversationId, messageId: requestBody.messageId },
9007
+ scopeId: claimId ? `claim_${claimId}` : '',
9008
+ });
9009
+ // A request that never reached OpenAI may be sent again.
9010
+ if (result.ok || result.sent) dispatched.running = false;
9011
+ else dispatchedImageMessages.delete(requestBody.messageId);
9012
+
9013
+ if (!result.ok) {
9014
+ return res.status(result.status).json({
9015
+ error: result.error,
9016
+ code: result.code,
9017
+ outcome: result.outcome,
9018
+ upstreamStatus: result.upstreamStatus,
9019
+ ...(result.detail ? { detail: result.detail } : {}),
9020
+ ...(result.relayProcessing ? { relayProcessing: true } : {}),
8046
9021
  });
8047
9022
  }
8048
-
8049
9023
  return res.json({
8050
9024
  ok: true,
8051
- generatedImages,
8052
- model: requestBody.model,
8053
- endpoint,
9025
+ generatedImages: result.generatedImages,
9026
+ model: result.model,
9027
+ endpoint: result.endpoint,
9028
+ usage: result.usage,
9029
+ // The worker hands it to /api/response as `imageNote`.
9030
+ ...(result.note ? { note: result.note } : {}),
8054
9031
  });
8055
9032
  });
8056
9033
 
@@ -8149,12 +9126,16 @@ export function registerMessagesRoutes(app, deps) {
8149
9126
  console.log(`[${ts()}] FENCE ${messageId?.slice(0,8)} legacy unfenced response write`);
8150
9127
  }
8151
9128
 
9129
+ // The partial previews of an image turn go once its reply or its
9130
+ // failure note is stored (or was already).
8152
9131
  if (q && q.status === 'done') {
8153
9132
  console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_done`);
9133
+ imagePreviews.finish(messageId, 'done');
8154
9134
  return res.json({ ok: true, ignored: 'already_done' });
8155
9135
  }
8156
9136
  if (q && q.status === 'failed') {
8157
9137
  console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_failed`);
9138
+ imagePreviews.finish(messageId, 'failed');
8158
9139
  return res.json({ ok: true, ignored: 'already_failed' });
8159
9140
  }
8160
9141
 
@@ -8259,6 +9240,7 @@ export function registerMessagesRoutes(app, deps) {
8259
9240
  + `${terminalFailure.functionCallId ? ` call=${terminalFailure.functionCallId}` : ''}`
8260
9241
  + `${terminalFailure.requestId ? ` req=${terminalFailure.requestId}` : ''}`,
8261
9242
  );
9243
+ imagePreviews.finish(messageId, 'failed');
8262
9244
  return res.json({ ok: true, terminal: true, code: terminalFailure.stableCode });
8263
9245
  }
8264
9246
 
@@ -8266,7 +9248,7 @@ export function registerMessagesRoutes(app, deps) {
8266
9248
  const hasPendingQuestion = !!stmts.findPendingQuestionByMessage?.get(messageId);
8267
9249
  const hasRecentlyAnsweredQuestion = !!stmts.findRecentlyAnsweredQuestionByMessage?.get(messageId, holdCutoff);
8268
9250
  const conversation = stmts.getConvAnyStatus?.get?.(targetConversationId) || null;
8269
- const resolvedText = await resolveRelayResponseText({
9251
+ let resolvedText = await resolveRelayResponseText({
8270
9252
  text: trimmedText,
8271
9253
  conversation,
8272
9254
  queueTimestamp: q?.timestamp || null,
@@ -8349,6 +9331,29 @@ export function registerMessagesRoutes(app, deps) {
8349
9331
  } catch (error) {
8350
9332
  return res.status(500).json({ error: error?.message || 'Failed to persist generated images' });
8351
9333
  }
9334
+ // An image turn whose model wrote no text: the reply line says what came
9335
+ // back ("1 image · 1536x1024 · gpt-image-2"), so the chat and an agent
9336
+ // that reads text only see the same.
9337
+ const imageTurnModel = [resolvedAssistantModel, requestedModel].find((candidate) => isOpenAIImageModelId(candidate)) || '';
9338
+ const imageSummaryLine = Boolean(!resolvedText && generatedImageAttachments.length && (q?.image_operation_id || imageTurnModel));
9339
+ if (imageSummaryLine) {
9340
+ resolvedText = openAIImageReplyLine({
9341
+ images: generatedImageAttachments,
9342
+ model: imageTurnModel || resolvedAssistantModel || '',
9343
+ turnId: messageId,
9344
+ });
9345
+ }
9346
+ // The image request came through on its second try, and the first may
9347
+ // have been billed as well: the reply says so, on the reply line when
9348
+ // that is the text, so an agent that reads text only sees it too.
9349
+ const imageNote = generatedImageAttachments.length
9350
+ ? String(req.body.imageNote || '').trim().slice(0, OPENAI_IMAGE_NOTE_MAX)
9351
+ : '';
9352
+ if (imageNote) {
9353
+ resolvedText = imageSummaryLine
9354
+ ? `${resolvedText}. Note: ${imageNote}`
9355
+ : `${String(resolvedText || '').trimEnd()}${resolvedText ? '\n\n' : ''}Note: ${imageNote}`;
9356
+ }
8352
9357
  const assistantAttachments = generatedImageAttachments.length ? JSON.stringify(generatedImageAttachments) : null;
8353
9358
  const now = new Date().toISOString();
8354
9359
  const imageOperation = q?.image_operation_id
@@ -8479,8 +9484,10 @@ export function registerMessagesRoutes(app, deps) {
8479
9484
  // Ensure UI is updated even if the response was duplicate/late
8480
9485
  if (currentStatus === 'done') {
8481
9486
  io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
9487
+ imagePreviews.finish(messageId, 'done');
8482
9488
  } else if (currentStatus === 'failed') {
8483
9489
  io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'failed' });
9490
+ imagePreviews.finish(messageId, 'failed');
8484
9491
  }
8485
9492
  return res.json({ ok: true, ignored: 'not_pending_or_processing', actualStatus: currentStatus });
8486
9493
  }
@@ -8629,6 +9636,8 @@ export function registerMessagesRoutes(app, deps) {
8629
9636
  },
8630
9637
  });
8631
9638
  io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
9639
+ // The final images are in the chat: the previews go.
9640
+ imagePreviews.finish(messageId, generatedImageAttachments.length ? 'done' : 'failed');
8632
9641
  if (usageLimitPause) {
8633
9642
  try {
8634
9643
  usageLimitPauseService.schedulePause(usageLimitPause, { queueRow: q });
@@ -9533,7 +10542,13 @@ export function registerMessagesRoutes(app, deps) {
9533
10542
  });
9534
10543
  }
9535
10544
 
9536
- io.emit('relay_thought', {
10545
+ // An update in flight (done false) is volatile, as relay_stream: each one
10546
+ // carries the whole thought so far (up to MAX_THOUGHT_CHARS), so keeping
10547
+ // them in the connection-state-recovery buffer, or queueing them on a
10548
+ // socket that does not drain, holds a turn's worth of copies — gigabytes
10549
+ // for a few long reasoning turns at once. A reconnect resync reads the
10550
+ // stored snapshot instead; the final thought (done) is never dropped.
10551
+ (done ? io : io.volatile).emit('relay_thought', {
9537
10552
  messageId,
9538
10553
  conversationId,
9539
10554
  mode: relayMode,
@@ -9805,6 +10820,41 @@ export function registerMessagesRoutes(app, deps) {
9805
10820
  failConsumedQueueRow(q, { reason: 'requeue', attemptId: claimedAttemptId });
9806
10821
  return res.json({ ok: true, terminal: true, code: STEER_SETTLE_FAILED_STABLE_CODE });
9807
10822
  }
10823
+ if (q && q.status === 'processing') {
10824
+ // An image turn whose request went to OpenAI is never run again: a
10825
+ // second run is billed again. One still running here finishes the row
10826
+ // itself; one that ended, or was cut off, ends with the note.
10827
+ const imageStop = imageTurnRequeueStop(q);
10828
+ if (imageStop?.running) return res.json({ ok: true, held: 'image-request-running' });
10829
+ if (imageStop) {
10830
+ const failed = failQueueMessage({
10831
+ queueRow: q,
10832
+ messageId,
10833
+ conversationId: q.conversation_id,
10834
+ relayMode: normalizeRelayMode(q.relay_mode) || DEFAULT_RELAY_MODE,
10835
+ model: q.model || null,
10836
+ responseText: buildTerminalFailureTextForChat(imageStop.failure),
10837
+ failureRecord: {
10838
+ kind: 'terminal',
10839
+ error: 'image-not-requeued',
10840
+ code: normalizeTerminalErrorCode(imageStop.failure.code),
10841
+ stableCode: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}`,
10842
+ message: imageStop.failure.message,
10843
+ guidance: imageStop.failure.guidance,
10844
+ retryCount: Number(q.retry_count || 0),
10845
+ failedAt: new Date().toISOString(),
10846
+ },
10847
+ attemptId: claimedAttemptId,
10848
+ });
10849
+ if (failed) {
10850
+ settleRelayAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
10851
+ settleRelaySubagentAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
10852
+ console.warn(`[${ts()}] FAILED ${messageId?.slice(0,8)} reason=image-not-requeued code=${imageStop.failure.code}`);
10853
+ imagePreviews.finish(messageId, 'failed');
10854
+ }
10855
+ return res.json({ ok: true, terminal: true, code: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}` });
10856
+ }
10857
+ }
9808
10858
  if (q && q.status === 'processing' && String(req.body?.code || '').trim() === 'relay.provider-mismatch') {
9809
10859
  // A routing refusal, not a delivery failure: the legacy relay handed
9810
10860
  // back a turn that belongs to a session worker. No retry increment and