@oar-sh/oar 0.9.24 → 0.9.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/skills/reasoning-stream.mjs +3 -1
- package/CHANGELOG.md +284 -0
- package/README.md +12 -11
- package/package.json +5 -1
- package/server/claude-worker/claude-remote-relay-tool.mjs +15 -0
- package/server/claude-worker/claude-sdk-adapter.mjs +8 -4
- package/server/claude-worker/claude-session-process.mjs +4 -0
- package/server/claude-worker/claude-turn-publisher.mjs +25 -11
- package/server/claude-worker/sdk-message-normalizer.mjs +29 -1
- package/server/copilot-worker/copilot-prompt-context.mjs +2 -1
- package/server/copilot-worker/copilot-sdk-session-process.mjs +11 -9
- package/server/cursor-worker/cursor-turn-runner.mjs +3 -1
- package/server/cursor-worker/sdk-message-normalizer.mjs +13 -0
- package/server/db-schema.mjs +19 -0
- package/server/grok-worker/grok-turn-runner.mjs +3 -1
- package/server/grok-worker/sdk-message-normalizer.mjs +3 -0
- package/server/migrations/0039-media-index.mjs +167 -0
- package/server/migrations/0040-image-session-defaults.mjs +93 -0
- package/server/public/app/api-client.js +58 -0
- package/server/public/app/ask-user-view.js +2 -1
- package/server/public/app/attachments-view.js +66 -9
- package/server/public/app/bootstrap.js +85 -25
- package/server/public/app/chat-actions-menu-state.mjs +1 -0
- package/server/public/app/context-media-tab.js +143 -0
- package/server/public/app/context-usage-view.mjs +2 -2
- package/server/public/app/conversation-view.js +90 -5
- package/server/public/app/image-composer-ui.js +518 -0
- package/server/public/app/image-turn-placeholder.mjs +233 -0
- package/server/public/app/journal-view.js +12 -2
- package/server/public/app/live-activity-cap.mjs +3 -3
- package/server/public/app/markdown-math.mjs +471 -0
- package/server/public/app/media-gallery-model.mjs +202 -0
- package/server/public/app/media-gallery-ui.js +524 -0
- package/server/public/app/media-view.js +79 -0
- package/server/public/app/openai-image-composer.mjs +492 -0
- package/server/public/app/openai-image-cost-ui.js +426 -0
- package/server/public/app/openai-image-spend.mjs +443 -0
- package/server/public/app/plan-usage-view.mjs +38 -6
- package/server/public/app/pwa-install.js +25 -4
- package/server/public/app/relay-error-ctas.mjs +5 -0
- package/server/public/app/relay-prompt-sanitizer.mjs +3 -0
- package/server/public/app/relay-restart-ui.js +2 -2
- package/server/public/app/remote-relays-settings.js +33 -6
- package/server/public/app/router.js +49 -237
- package/server/public/app/session-history-ui.js +2 -0
- package/server/public/app/settings-modal.js +4 -0
- package/server/public/app/settings-tabs.js +10 -1
- package/server/public/app/socket-handlers.js +26 -0
- package/server/public/app/store.js +2 -0
- package/server/public/app/stream-view-preference.mjs +1 -1
- package/server/public/app/tab-strip-fade.mjs +1 -1
- package/server/public/app/thoughts-preference.mjs +1 -1
- package/server/public/app/tmux-inspector-view.js +29 -16
- package/server/public/app/viewer-gallery.mjs +23 -1
- package/server/public/index.html +517 -123
- package/server/relay-tools.md +1 -0
- package/server/repositories/image-conversation-repository.mjs +69 -0
- package/server/repositories/media-repository.mjs +305 -0
- package/server/routes/media-routes.mjs +361 -0
- package/server/routes/messages-routes.mjs +1410 -360
- package/server/routes/relay-board-routes.mjs +46 -1
- package/server/routes/remote-relay-routes.mjs +1 -1
- package/server/routes/sessions-routes.mjs +361 -14
- package/server/server-runtime.mjs +21 -0
- package/server/services/database-file.mjs +6 -4
- package/server/services/generated-image-cleanup-service.mjs +2 -1
- package/server/services/generated-image-files.mjs +110 -0
- package/server/services/image-input-paths.mjs +389 -0
- package/server/services/image-operation-service.mjs +6 -2
- package/server/services/media-extract.mjs +653 -0
- package/server/services/media-index-service.mjs +221 -0
- package/server/services/openai-image-preview-turns.mjs +330 -0
- package/server/services/openai-image-spend-service.mjs +516 -0
- package/server/services/openai-image-stream.mjs +170 -0
- package/server/services/plan-usage-openai-image.mjs +148 -0
- package/server/services/relay-prompt-sanitizer.mjs +7 -0
- package/server/services/relay-restart-queue.mjs +1 -1
- package/server/services/remote-relay-caller-context.mjs +6 -0
- package/server/services/remote-relay-dispatcher.mjs +409 -23
- package/server/services/remote-relay-registry-service.mjs +10 -2
- package/server/services/session-worker-stop-service.mjs +50 -18
- package/server/services/token-usage-service.mjs +15 -0
- package/shared/image-header.mjs +90 -0
- package/shared/math-instructions.mjs +30 -0
- package/shared/openai-image-options.mjs +299 -0
- package/shared/openai-image-presets.mjs +175 -0
- package/shared/openai-image-previews.mjs +40 -0
- package/shared/openai-image-prices.mjs +291 -0
- package/shared/openai-image-spend.mjs +90 -0
- package/shared/remote-relay-contract.mjs +126 -4
- package/shared/worker-runtime/direct-openai-image.mjs +96 -50
- package/server/public/app/math-dollars.mjs +0 -89
|
@@ -6,6 +6,7 @@ import { randomUUID } from 'crypto';
|
|
|
6
6
|
import { stripRelayPromptContext } from '../services/relay-prompt-sanitizer.mjs';
|
|
7
7
|
import { resolveUploadMimeType } from '../services/mime-sniffer.mjs';
|
|
8
8
|
import { fileVersionToken, serveFileWithRangeSupport, withFileVersion } from '../services/file-serving.mjs';
|
|
9
|
+
import { registerMediaRoutes } from './media-routes.mjs';
|
|
9
10
|
import { validateEntryName } from '../services/explorer-entry-name.mjs';
|
|
10
11
|
import { countEntries, createDirectory, removeEntry, renameEntry } from '../services/explorer-mutations.mjs';
|
|
11
12
|
import {
|
|
@@ -64,6 +65,47 @@ import {
|
|
|
64
65
|
unknownProviderModelText,
|
|
65
66
|
} from '../services/provider-model-selection.mjs';
|
|
66
67
|
import { stripRemotePromptHeader } from '../../shared/remote-relay-contract.mjs';
|
|
68
|
+
import {
|
|
69
|
+
OPENAI_IMAGE_LIMITS,
|
|
70
|
+
OPENAI_IMAGE_MIME_BY_FORMAT,
|
|
71
|
+
OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
|
|
72
|
+
checkOpenAIImageSize,
|
|
73
|
+
formatOpenAIImageSummary,
|
|
74
|
+
isOpenAIImageModelId,
|
|
75
|
+
mergeOpenAIImageOptions,
|
|
76
|
+
openAIImageModelFamily,
|
|
77
|
+
openAIImageModelNeedsOpenAIText,
|
|
78
|
+
validateOpenAIImageOptions,
|
|
79
|
+
} from '../../shared/openai-image-options.mjs';
|
|
80
|
+
import { OPENAI_IMAGE_PARTIAL_IMAGES, OPENAI_IMAGE_PARTIALS_DIR } from '../../shared/openai-image-previews.mjs';
|
|
81
|
+
import {
|
|
82
|
+
SSE_EVENT_TOO_LARGE_CODE,
|
|
83
|
+
interpretOpenAIImageStreamEvent,
|
|
84
|
+
isEventStreamResponse,
|
|
85
|
+
readSseEvents,
|
|
86
|
+
} from '../services/openai-image-stream.mjs';
|
|
87
|
+
import { createOpenAIImagePreviewTurns } from '../services/openai-image-preview-turns.mjs';
|
|
88
|
+
import {
|
|
89
|
+
OPENAI_IMAGE_CAP_WARN_RATIO,
|
|
90
|
+
OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
|
|
91
|
+
OPENAI_IMAGE_USAGE_PATH,
|
|
92
|
+
formatOpenAIImageUsd,
|
|
93
|
+
} from '../../shared/openai-image-spend.mjs';
|
|
94
|
+
import { estimateOpenAIImageCost } from '../../shared/openai-image-prices.mjs';
|
|
95
|
+
import {
|
|
96
|
+
OPENAI_IMAGE_PRESETS_SETTING_KEY,
|
|
97
|
+
findOpenAIImagePreset,
|
|
98
|
+
parseOwnOpenAIImagePresets,
|
|
99
|
+
splitOpenAIImagePreset,
|
|
100
|
+
} from '../../shared/openai-image-presets.mjs';
|
|
101
|
+
import {
|
|
102
|
+
createOpenAIImageSpendService,
|
|
103
|
+
openAIImageCapRefusalText,
|
|
104
|
+
openAIImageTryCharge,
|
|
105
|
+
openAIImageTryRunId,
|
|
106
|
+
} from '../services/openai-image-spend-service.mjs';
|
|
107
|
+
import { IMAGE_INPUT_REFUSED_CODE, resolveImageInputRoots, resolveImageInputs } from '../services/image-input-paths.mjs';
|
|
108
|
+
import { IMAGE_HEADER_BYTES, readImageHeader } from '../../shared/image-header.mjs';
|
|
67
109
|
import { stripModelTierSuffix } from '../../shared/claude-cloud/repo-url.mjs';
|
|
68
110
|
import {
|
|
69
111
|
createSubagentTimelineService,
|
|
@@ -206,35 +248,33 @@ function isAutoModel(model) {
|
|
|
206
248
|
return String(model || '').trim().toLowerCase() === AUTO_MODEL_SENTINEL;
|
|
207
249
|
}
|
|
208
250
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
return normalized.startsWith('gpt-image-') || normalized.startsWith('dall-e-');
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
const OPENAI_IMAGE_QUALITY_VALUES = new Set(['auto', 'low', 'medium', 'high']);
|
|
215
|
-
const OPENAI_IMAGE_SIZE_VALUES = new Set([
|
|
216
|
-
'auto',
|
|
217
|
-
'256x256',
|
|
218
|
-
'512x512',
|
|
219
|
-
'1024x1024',
|
|
220
|
-
'1536x1024',
|
|
221
|
-
'1024x1536',
|
|
222
|
-
'1792x1024',
|
|
223
|
-
'1024x1792',
|
|
224
|
-
]);
|
|
225
|
-
|
|
251
|
+
// A quality or size any image model family takes (shared/openai-image-options.mjs),
|
|
252
|
+
// or ''. The model's own family is checked where the request is built.
|
|
226
253
|
function normalizeOpenAIImageQuality(value = '') {
|
|
227
|
-
const
|
|
228
|
-
return
|
|
254
|
+
const checked = validateOpenAIImageOptions({ quality: value });
|
|
255
|
+
return checked.ok ? (checked.values.quality || '') : '';
|
|
229
256
|
}
|
|
230
257
|
|
|
231
258
|
function normalizeOpenAIImageSize(value = '') {
|
|
232
259
|
const normalized = String(value || '').trim().toLowerCase();
|
|
233
|
-
|
|
260
|
+
// The composer's size select is the context-tier select: its own values
|
|
261
|
+
// mean the API's default size.
|
|
234
262
|
if (normalized === 'default' || normalized === 'long_context') return 'auto';
|
|
235
|
-
return '';
|
|
263
|
+
if (!normalized) return '';
|
|
264
|
+
return checkOpenAIImageSize(normalized).size || '';
|
|
236
265
|
}
|
|
237
266
|
|
|
267
|
+
// How long one OpenAI image request may take, from sending it to the last
|
|
268
|
+
// byte of the reply; config.json `openaiImageGenerateTimeoutMs` and
|
|
269
|
+
// `openaiImageEditTimeoutMs` override them. Above 300 s undici's own header
|
|
270
|
+
// timeout ends a request first.
|
|
271
|
+
export const OPENAI_IMAGE_GENERATE_TIMEOUT_MS = 180_000;
|
|
272
|
+
export const OPENAI_IMAGE_EDIT_TIMEOUT_MS = 300_000;
|
|
273
|
+
// The pause before the one retry of a request that may pass the second time.
|
|
274
|
+
const OPENAI_IMAGE_RETRY_DELAY_MS = 2_000;
|
|
275
|
+
// OpenAI takes up to 16 input images on an edit.
|
|
276
|
+
const OPENAI_IMAGE_EDIT_INPUTS_MAX = 16;
|
|
277
|
+
|
|
238
278
|
function normalizeSessionWorkerId(value) {
|
|
239
279
|
const text = String(value || '').trim();
|
|
240
280
|
return text || null;
|
|
@@ -1445,6 +1485,7 @@ export function buildDequeuedRelayMessage({
|
|
|
1445
1485
|
try {
|
|
1446
1486
|
messageKind = String(stmts.getMessageByConversation?.get?.(msg.id, msg.conversation_id)?.kind || '').trim();
|
|
1447
1487
|
} catch {}
|
|
1488
|
+
const imageOptions = parseQueueImageOptions(msg.image_options_json);
|
|
1448
1489
|
return {
|
|
1449
1490
|
id: msg.id,
|
|
1450
1491
|
conversationId: msg.conversation_id,
|
|
@@ -1477,10 +1518,23 @@ export function buildDequeuedRelayMessage({
|
|
|
1477
1518
|
ownerAssignedAt: msg.owner_assigned_at || null,
|
|
1478
1519
|
ownerLeaseExpiresAt: msg.owner_lease_expires_at || null,
|
|
1479
1520
|
imageOperationId: String(msg.image_operation_id || '').trim() || null,
|
|
1521
|
+
// An image turn's options as the send settled them; a turn without an
|
|
1522
|
+
// operation sends them with its direct request.
|
|
1523
|
+
...(imageOptions ? { imageOptions } : {}),
|
|
1480
1524
|
...(isRelayAutomatedMessageKind(messageKind) ? { messageKind } : {}),
|
|
1481
1525
|
};
|
|
1482
1526
|
}
|
|
1483
1527
|
|
|
1528
|
+
function parseQueueImageOptions(value) {
|
|
1529
|
+
if (!value) return null;
|
|
1530
|
+
try {
|
|
1531
|
+
const parsed = JSON.parse(String(value));
|
|
1532
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
|
|
1533
|
+
} catch {
|
|
1534
|
+
return null;
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1484
1538
|
function buildAttachmentPromptContext(attachments) {
|
|
1485
1539
|
if (!Array.isArray(attachments) || !attachments.length) return '';
|
|
1486
1540
|
const lines = [
|
|
@@ -1631,49 +1685,35 @@ function parseOpenAIImageBase64Value(value) {
|
|
|
1631
1685
|
return Buffer.from(raw, 'base64').toString('base64');
|
|
1632
1686
|
}
|
|
1633
1687
|
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 10 } = {}) {
|
|
1688
|
+
/**
|
|
1689
|
+
* The body of one image request (the generate route's, or the one the
|
|
1690
|
+
* execute route builds from an operation), checked against the model's
|
|
1691
|
+
* family. `n` is the count; size and quality "auto" pass and are left out
|
|
1692
|
+
* for a dall-e model when the request is built. A prompt that came from an
|
|
1693
|
+
* agent on another relay loses its "[Remote prompt from …]" header: OpenAI
|
|
1694
|
+
* would draw it.
|
|
1695
|
+
*/
|
|
1696
|
+
export function normalizeOpenAIImageGenerateRequestBody(rawBody) {
|
|
1645
1697
|
const body = rawBody && typeof rawBody === 'object' ? rawBody : {};
|
|
1646
1698
|
const messageId = String(body.messageId || '').trim();
|
|
1647
1699
|
const conversationId = String(body.conversationId || '').trim();
|
|
1648
1700
|
const model = String(body.model || '').trim();
|
|
1649
|
-
const prompt = String(body.prompt || '').trim();
|
|
1701
|
+
const prompt = stripRemotePromptHeader(String(body.prompt || '')).trim();
|
|
1650
1702
|
if (!messageId) throw new Error('Missing messageId');
|
|
1651
1703
|
if (!conversationId) throw new Error('Missing conversationId');
|
|
1652
1704
|
if (!model) throw new Error('Missing model');
|
|
1653
1705
|
if (!prompt) throw new Error('Missing prompt');
|
|
1654
1706
|
|
|
1655
|
-
const
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
}
|
|
1707
|
+
const legacySize = String(body.size ?? '').trim().toLowerCase();
|
|
1708
|
+
const checked = validateOpenAIImageOptions({
|
|
1709
|
+
size: legacySize === 'default' || legacySize === 'long_context' ? 'auto' : body.size,
|
|
1710
|
+
quality: body.quality,
|
|
1711
|
+
count: body.n,
|
|
1712
|
+
output_format: body.output_format,
|
|
1713
|
+
background: body.background,
|
|
1714
|
+
}, { model, field: 'image' });
|
|
1715
|
+
if (!checked.ok) throw new Error(checked.error);
|
|
1665
1716
|
|
|
1666
|
-
const sizeRaw = body.size;
|
|
1667
|
-
let size = undefined;
|
|
1668
|
-
if (sizeRaw !== undefined && sizeRaw !== null && String(sizeRaw).trim()) {
|
|
1669
|
-
const normalizedSize = String(sizeRaw).trim().toLowerCase();
|
|
1670
|
-
if (normalizedSize !== 'auto' && !/^\d{2,5}x\d{2,5}$/i.test(normalizedSize)) {
|
|
1671
|
-
throw new Error('Invalid size value (expected "<width>x<height>" or "auto")');
|
|
1672
|
-
}
|
|
1673
|
-
size = normalizedSize;
|
|
1674
|
-
}
|
|
1675
|
-
|
|
1676
|
-
const quality = normalizeOpenAIImageOptionString(body.quality, 'quality');
|
|
1677
1717
|
const attachments = body.attachments;
|
|
1678
1718
|
if (attachments !== undefined && !Array.isArray(attachments)) {
|
|
1679
1719
|
throw new Error('attachments must be an array when provided');
|
|
@@ -1684,13 +1724,288 @@ export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 1
|
|
|
1684
1724
|
conversationId,
|
|
1685
1725
|
model,
|
|
1686
1726
|
prompt,
|
|
1687
|
-
n,
|
|
1688
|
-
size,
|
|
1689
|
-
quality,
|
|
1727
|
+
n: checked.values.count,
|
|
1728
|
+
size: checked.values.size,
|
|
1729
|
+
quality: checked.values.quality,
|
|
1730
|
+
output_format: checked.values.output_format,
|
|
1731
|
+
background: checked.values.background,
|
|
1690
1732
|
attachments: Array.isArray(attachments) ? attachments : [],
|
|
1691
1733
|
};
|
|
1692
1734
|
}
|
|
1693
1735
|
|
|
1736
|
+
/**
|
|
1737
|
+
* The fields OpenAI gets for one image request. dall-e models take no
|
|
1738
|
+
* output_format or background, and their "auto" size or quality is not sent
|
|
1739
|
+
* (the API's own default); they answer in b64_json only when asked to.
|
|
1740
|
+
*/
|
|
1741
|
+
export function buildOpenAIImageRequestFields(requestBody = {}) {
|
|
1742
|
+
const family = openAIImageModelFamily(requestBody.model);
|
|
1743
|
+
const isGptImage = family === 'gpt-image';
|
|
1744
|
+
const fields = { model: requestBody.model, prompt: requestBody.prompt };
|
|
1745
|
+
if (!isGptImage) fields.response_format = 'b64_json';
|
|
1746
|
+
if (requestBody.n !== undefined) fields.n = requestBody.n;
|
|
1747
|
+
for (const key of ['size', 'quality']) {
|
|
1748
|
+
const value = requestBody[key];
|
|
1749
|
+
if (value && (isGptImage || value !== 'auto')) fields[key] = value;
|
|
1750
|
+
}
|
|
1751
|
+
if (isGptImage) {
|
|
1752
|
+
if (requestBody.output_format) fields.output_format = requestBody.output_format;
|
|
1753
|
+
if (requestBody.background) fields.background = requestBody.background;
|
|
1754
|
+
}
|
|
1755
|
+
return fields;
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
const OPENAI_IMAGE_USAGE_DETAIL_KEYS = Object.freeze(['input_tokens_details', 'output_tokens_details']);
|
|
1759
|
+
|
|
1760
|
+
/**
|
|
1761
|
+
* The `usage` block of an OpenAI image reply, numbers only (top level and
|
|
1762
|
+
* the two token-detail blocks), or null when there is none.
|
|
1763
|
+
*/
|
|
1764
|
+
export function normalizeOpenAIImageUsage(raw) {
|
|
1765
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
|
|
1766
|
+
const numbersOf = (source) => {
|
|
1767
|
+
const out = {};
|
|
1768
|
+
for (const [key, value] of Object.entries(source || {})) {
|
|
1769
|
+
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) out[key] = value;
|
|
1770
|
+
}
|
|
1771
|
+
return out;
|
|
1772
|
+
};
|
|
1773
|
+
const usage = numbersOf(raw);
|
|
1774
|
+
for (const key of OPENAI_IMAGE_USAGE_DETAIL_KEYS) {
|
|
1775
|
+
const detail = raw[key];
|
|
1776
|
+
if (detail && typeof detail === 'object' && !Array.isArray(detail)) {
|
|
1777
|
+
const numbers = numbersOf(detail);
|
|
1778
|
+
if (Object.keys(numbers).length) usage[key] = numbers;
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
return Object.keys(usage).length ? usage : null;
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
/**
|
|
1785
|
+
* Two image usages added up, field by field and detail by detail (a stream
|
|
1786
|
+
* of a count > 1 request reports one per completed image); null + null is
|
|
1787
|
+
* null.
|
|
1788
|
+
*/
|
|
1789
|
+
export function addOpenAIImageUsage(a, b) {
|
|
1790
|
+
if (!a) return b || null;
|
|
1791
|
+
if (!b) return a;
|
|
1792
|
+
const sum = { ...a };
|
|
1793
|
+
for (const [key, value] of Object.entries(b)) {
|
|
1794
|
+
if (typeof value === 'number') sum[key] = (typeof sum[key] === 'number' ? sum[key] : 0) + value;
|
|
1795
|
+
else if (value && typeof value === 'object') sum[key] = addOpenAIImageUsage(sum[key] && typeof sum[key] === 'object' ? sum[key] : null, value);
|
|
1796
|
+
}
|
|
1797
|
+
return sum;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
// The base64 characters that decode to the first IMAGE_HEADER_BYTES bytes.
|
|
1801
|
+
const IMAGE_HEADER_BASE64_CHARS = Math.ceil(IMAGE_HEADER_BYTES / 3) * 4;
|
|
1802
|
+
|
|
1803
|
+
/** readImageHeader of a canonical base64 payload, decoding only its start. */
|
|
1804
|
+
function readBase64ImageHeader(base64) {
|
|
1805
|
+
const text = String(base64 || '');
|
|
1806
|
+
if (!text) return null;
|
|
1807
|
+
return readImageHeader(Buffer.from(text.slice(0, IMAGE_HEADER_BASE64_CHARS), 'base64'));
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// What one OpenAI image request came to, as `kind`:
|
|
1811
|
+
// - 'unsent': it never reached OpenAI (no connection, no name, no TLS);
|
|
1812
|
+
// nothing was billed.
|
|
1813
|
+
// - 'timeout', 'broken', 'server': no answer in time, the connection broke
|
|
1814
|
+
// off after the request may have gone out, or a 5xx; it may have been
|
|
1815
|
+
// billed.
|
|
1816
|
+
// - 'stream': a streamed reply that broke off, or ended without its image,
|
|
1817
|
+
// after OpenAI had started on it; it may have been billed.
|
|
1818
|
+
// - 'refused': a 4xx (429 included); nothing came of it, a retry is refused
|
|
1819
|
+
// the same way.
|
|
1820
|
+
// - 'unprocessed': a 2xx the relay could not read; billed.
|
|
1821
|
+
// The first five are tried once more; the last two never.
|
|
1822
|
+
const OPENAI_IMAGE_RETRY_KINDS = new Set(['unsent', 'timeout', 'broken', 'server', 'stream']);
|
|
1823
|
+
const OPENAI_IMAGE_AMBIGUOUS_KINDS = new Set(['timeout', 'broken', 'server', 'stream']);
|
|
1824
|
+
const OPENAI_IMAGE_UNSENT_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT']);
|
|
1825
|
+
const OPENAI_IMAGE_TIMEOUT_ERROR_CODES = new Set(['UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']);
|
|
1826
|
+
|
|
1827
|
+
export const OPENAI_IMAGE_INTERRUPTED_NOTE = 'The relay was interrupted while OpenAI was working on the image; the request may have been billed; retry the message.';
|
|
1828
|
+
export const OPENAI_IMAGE_REPEATED_NOTE = 'This image request was already sent to OpenAI once and was not sent again; the first request may have been billed; retry the message.';
|
|
1829
|
+
// The guidance under every image failure note: the note itself says what to do.
|
|
1830
|
+
const OPENAI_IMAGE_FAILURE_GUIDANCE = 'If this keeps failing, include the error code when you report it.';
|
|
1831
|
+
|
|
1832
|
+
/** The `code` of a thrown fetch error and of every cause under it. */
|
|
1833
|
+
function fetchErrorCodes(error) {
|
|
1834
|
+
const codes = [];
|
|
1835
|
+
const seen = new Set();
|
|
1836
|
+
const visit = (entry, depth) => {
|
|
1837
|
+
if (!entry || typeof entry !== 'object' || seen.has(entry) || depth > 4) return;
|
|
1838
|
+
seen.add(entry);
|
|
1839
|
+
if (entry.code) codes.push(String(entry.code));
|
|
1840
|
+
visit(entry.cause, depth + 1);
|
|
1841
|
+
if (Array.isArray(entry.errors)) for (const inner of entry.errors) visit(inner, depth + 1);
|
|
1842
|
+
};
|
|
1843
|
+
visit(error, 0);
|
|
1844
|
+
return codes;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
function isTlsFailureCode(code) {
|
|
1848
|
+
return code.startsWith('ERR_TLS_') || code.startsWith('ERR_SSL_') || code.includes('CERT') || code === 'EPROTO';
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
/** 'unsent', 'timeout' or 'broken' for an error fetch threw (see OPENAI_IMAGE_RETRY_KINDS). */
|
|
1852
|
+
export function classifyOpenAIImageFetchError(error) {
|
|
1853
|
+
if (error?.name === 'TimeoutError' || error?.cause?.name === 'TimeoutError') return 'timeout';
|
|
1854
|
+
const codes = fetchErrorCodes(error);
|
|
1855
|
+
if (codes.some((code) => OPENAI_IMAGE_TIMEOUT_ERROR_CODES.has(code))) return 'timeout';
|
|
1856
|
+
if (codes.length && codes.every((code) => OPENAI_IMAGE_UNSENT_ERROR_CODES.has(code) || isTlsFailureCode(code))) return 'unsent';
|
|
1857
|
+
return 'broken';
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
function describeFetchError(error) {
|
|
1861
|
+
const codes = [...new Set(fetchErrorCodes(error))];
|
|
1862
|
+
const message = String(error?.message || error || 'request error').trim();
|
|
1863
|
+
return codes.length ? `${message} (${codes.join(', ')})` : message;
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
function formatNoteDuration(ms) {
|
|
1867
|
+
const value = Math.max(0, Number(ms) || 0);
|
|
1868
|
+
return value >= 1000 ? `${Math.round(value / 1000)} s` : `${Math.round(value)} ms`;
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
function asSentence(text) {
|
|
1872
|
+
const value = String(text || '').trim();
|
|
1873
|
+
return !value || /[.!?]$/.test(value) ? value : `${value}.`;
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
/**
|
|
1877
|
+
* The failure of an image request after its tries: the note the chat shows
|
|
1878
|
+
* (`error`), the attempt's code, `outcome` 'uncertain' when it may have been
|
|
1879
|
+
* billed (else 'terminal'), and the HTTP status the generate route answers.
|
|
1880
|
+
* A request that never reached OpenAI on its last try but may have been
|
|
1881
|
+
* billed on the first is told as the first.
|
|
1882
|
+
*/
|
|
1883
|
+
export function openAIImageFailureResult(tries, { timeoutMs = 0, resent = false } = {}) {
|
|
1884
|
+
const list = Array.isArray(tries) && tries.length ? tries : [{ kind: 'broken' }];
|
|
1885
|
+
const last = list[list.length - 1];
|
|
1886
|
+
const ambiguous = list.find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry.kind)) || null;
|
|
1887
|
+
const deciding = last.kind === 'unsent' && ambiguous ? ambiguous : last;
|
|
1888
|
+
// A resend without stream fields is the second request of a turn whose
|
|
1889
|
+
// first was refused for them before any work.
|
|
1890
|
+
const triedText = resent ? 'sent twice, once without previews' : (list.length > 1 ? 'tried twice' : 'tried once');
|
|
1891
|
+
const maybeBilled = 'the request may have been billed; retry the message.';
|
|
1892
|
+
// A refusal or an unreadable reply after a first try that may have been
|
|
1893
|
+
// billed: the note says so too.
|
|
1894
|
+
const earlierBilled = ambiguous && !OPENAI_IMAGE_AMBIGUOUS_KINDS.has(deciding.kind) && deciding.kind !== 'unsent'
|
|
1895
|
+
? ' The first try may have been billed as well.'
|
|
1896
|
+
: '';
|
|
1897
|
+
const base = {
|
|
1898
|
+
ok: false,
|
|
1899
|
+
tries: list.length,
|
|
1900
|
+
sent: list.some((entry) => entry.kind !== 'unsent'),
|
|
1901
|
+
...(deciding.detail ? { detail: String(deciding.detail).slice(0, 500) } : {}),
|
|
1902
|
+
upstreamStatus: Number.isFinite(Number(deciding.status)) && Number(deciding.status) > 0 ? Number(deciding.status) : null,
|
|
1903
|
+
};
|
|
1904
|
+
switch (deciding.kind) {
|
|
1905
|
+
case 'unsent':
|
|
1906
|
+
return { ...base, status: 502, outcome: 'terminal', code: 'IMAGE_PROVIDER_UNREACHABLE', error: 'OpenAI could not be reached; nothing was billed; retry the message.' };
|
|
1907
|
+
case 'timeout': {
|
|
1908
|
+
const waited = deciding.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
|
|
1909
|
+
return { ...base, status: 504, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `OpenAI did not answer ${waited} (${triedText}); ${maybeBilled}` };
|
|
1910
|
+
}
|
|
1911
|
+
case 'server':
|
|
1912
|
+
return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_UNAVAILABLE', error: `OpenAI answered ${deciding.status || 'with a server error'} (${triedText}); ${maybeBilled}` };
|
|
1913
|
+
case 'stream':
|
|
1914
|
+
return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_STREAM_BROKEN', error: `OpenAI's image stream broke off before the image was complete (${triedText}); ${maybeBilled}` };
|
|
1915
|
+
case 'refused': {
|
|
1916
|
+
const message = String(deciding.message || '').trim().slice(0, 240);
|
|
1917
|
+
return {
|
|
1918
|
+
...base,
|
|
1919
|
+
status: deciding.status >= 400 && deciding.status < 500 ? deciding.status : 502,
|
|
1920
|
+
outcome: 'terminal',
|
|
1921
|
+
code: 'IMAGE_PROVIDER_REJECTED',
|
|
1922
|
+
error: `${asSentence(`OpenAI refused the image request (${deciding.status})${message ? `: ${message}` : ''}`)}${earlierBilled}`,
|
|
1923
|
+
};
|
|
1924
|
+
}
|
|
1925
|
+
case 'unprocessed':
|
|
1926
|
+
return {
|
|
1927
|
+
...base,
|
|
1928
|
+
status: 502,
|
|
1929
|
+
outcome: 'terminal',
|
|
1930
|
+
code: 'IMAGE_REPLY_UNPROCESSED',
|
|
1931
|
+
relayProcessing: true,
|
|
1932
|
+
error: `The relay could not process the image OpenAI returned (${deciding.detail || 'unknown error'}). The generation was billed; the message was not retried.${earlierBilled}`,
|
|
1933
|
+
};
|
|
1934
|
+
default:
|
|
1935
|
+
return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `The connection to OpenAI broke off (${triedText}); ${maybeBilled}` };
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
/**
|
|
1940
|
+
* The note on an image request that came through on its second try after a
|
|
1941
|
+
* first that may have reached OpenAI (OPENAI_IMAGE_AMBIGUOUS_KINDS): both may
|
|
1942
|
+
* have been billed. Empty when the earlier tries never went out.
|
|
1943
|
+
*/
|
|
1944
|
+
export function openAIImageRetriedNote(tries, { timeoutMs = 0 } = {}) {
|
|
1945
|
+
const first = (Array.isArray(tries) ? tries : []).find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
|
|
1946
|
+
if (!first) return '';
|
|
1947
|
+
const tail = 'it was sent once more and the first try may have been billed as well.';
|
|
1948
|
+
switch (first.kind) {
|
|
1949
|
+
case 'timeout': {
|
|
1950
|
+
const waited = first.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
|
|
1951
|
+
return `OpenAI did not answer the first try ${waited}; ${tail}`;
|
|
1952
|
+
}
|
|
1953
|
+
case 'server':
|
|
1954
|
+
return `OpenAI answered the first try with ${first.status || 'a server error'}; ${tail}`;
|
|
1955
|
+
case 'stream':
|
|
1956
|
+
return `OpenAI's image stream broke off on the first try; ${tail}`;
|
|
1957
|
+
default:
|
|
1958
|
+
return `The connection to OpenAI broke off on the first try; ${tail}`;
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
/**
|
|
1963
|
+
* The failure of an image request whose next try the daily cap stopped
|
|
1964
|
+
* before it was sent (`refusal`: the spend service's reserveTry). Nothing of
|
|
1965
|
+
* that try reached OpenAI; an earlier try may have been billed, and the note
|
|
1966
|
+
* says so then.
|
|
1967
|
+
*/
|
|
1968
|
+
export function openAIImageCapRefusalResult(tries, refusal = {}) {
|
|
1969
|
+
const list = Array.isArray(tries) ? tries : [];
|
|
1970
|
+
const earlierBilled = list.some((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
|
|
1971
|
+
return {
|
|
1972
|
+
ok: false,
|
|
1973
|
+
tries: list.length,
|
|
1974
|
+
sent: list.some((entry) => entry?.kind !== 'unsent'),
|
|
1975
|
+
status: 409,
|
|
1976
|
+
outcome: earlierBilled ? 'uncertain' : 'terminal',
|
|
1977
|
+
code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
|
|
1978
|
+
error: openAIImageCapRefusalText({ tryNumber: earlierBilled ? 2 : 1, refusal }),
|
|
1979
|
+
upstreamStatus: null,
|
|
1980
|
+
};
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
// The file system the partial preview route serves through: its open
|
|
1984
|
+
// follows no link.
|
|
1985
|
+
const PARTIAL_PREVIEW_FS = Object.freeze({
|
|
1986
|
+
...fs,
|
|
1987
|
+
openSync: (file) => fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)),
|
|
1988
|
+
});
|
|
1989
|
+
|
|
1990
|
+
// The longest image note /api/response takes into the reply text.
|
|
1991
|
+
const OPENAI_IMAGE_NOTE_MAX = 500;
|
|
1992
|
+
|
|
1993
|
+
/** A request refused before anything was sent to OpenAI. */
|
|
1994
|
+
function openAIImageRequestRefusal(status, code, error) {
|
|
1995
|
+
return { ok: false, tries: 0, sent: false, status, outcome: 'terminal', code, error, upstreamStatus: null };
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/** The `terminalError` /api/response gets for a failed image request. */
|
|
1999
|
+
function openAIImageTerminalError(result) {
|
|
2000
|
+
return {
|
|
2001
|
+
code: result.code,
|
|
2002
|
+
message: String(result.error || 'Image operation failed').slice(0, 500),
|
|
2003
|
+
...(result.detail ? { detail: String(result.detail).slice(0, 500) } : {}),
|
|
2004
|
+
guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
|
|
2005
|
+
retryable: false,
|
|
2006
|
+
};
|
|
2007
|
+
}
|
|
2008
|
+
|
|
1694
2009
|
function isOpenAIImageAttachment(raw) {
|
|
1695
2010
|
if (!raw || typeof raw !== 'object') return false;
|
|
1696
2011
|
const type = String(raw.type || '').trim().toLowerCase();
|
|
@@ -1699,7 +2014,26 @@ function isOpenAIImageAttachment(raw) {
|
|
|
1699
2014
|
return dataUrl.startsWith('data:image/');
|
|
1700
2015
|
}
|
|
1701
2016
|
|
|
1702
|
-
export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
2017
|
+
export function resolveOpenAIImageEditAttachment(rawAttachments, options = {}) {
|
|
2018
|
+
return resolveOpenAIImageEditAttachments(rawAttachments, { ...options, maxImages: 1 })[0] || null;
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
/**
|
|
2022
|
+
* Every image attachment of an edit, in the order given (the first is the
|
|
2023
|
+
* base image), at most `maxImages`; the others are ignored.
|
|
2024
|
+
*/
|
|
2025
|
+
export function resolveOpenAIImageEditAttachments(rawAttachments, {
|
|
2026
|
+
maxImages = OPENAI_IMAGE_EDIT_INPUTS_MAX,
|
|
2027
|
+
...options
|
|
2028
|
+
} = {}) {
|
|
2029
|
+
const attachments = Array.isArray(rawAttachments) ? rawAttachments : [];
|
|
2030
|
+
return attachments
|
|
2031
|
+
.filter((entry) => isOpenAIImageAttachment(entry))
|
|
2032
|
+
.slice(0, Math.max(1, Math.trunc(Number(maxImages) || 1)))
|
|
2033
|
+
.map((entry) => resolveOneOpenAIImageEditAttachment(entry, options));
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
function resolveOneOpenAIImageEditAttachment(image, {
|
|
1703
2037
|
maxImageBytes = 20 * 1024 * 1024,
|
|
1704
2038
|
allowedRootPath = '',
|
|
1705
2039
|
// Folders of earlier versions that uploads are still read from: only a
|
|
@@ -1707,12 +2041,8 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1707
2041
|
allowedBlobRootPaths = [],
|
|
1708
2042
|
allowedGeneratedImagesRootPath = '',
|
|
1709
2043
|
} = {}) {
|
|
1710
|
-
const
|
|
1711
|
-
|
|
1712
|
-
if (!firstImage) return null;
|
|
1713
|
-
|
|
1714
|
-
const dataUrl = String(firstImage.dataUrl || '').trim();
|
|
1715
|
-
let mimeType = String(firstImage.type || '').trim().toLowerCase();
|
|
2044
|
+
const dataUrl = String(image.dataUrl || '').trim();
|
|
2045
|
+
let mimeType = String(image.type || '').trim().toLowerCase();
|
|
1716
2046
|
let bytes = null;
|
|
1717
2047
|
|
|
1718
2048
|
if (dataUrl.startsWith('data:')) {
|
|
@@ -1722,7 +2052,7 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1722
2052
|
const payload = parseOpenAIImageBase64Value(parts.payload);
|
|
1723
2053
|
bytes = Buffer.from(payload, 'base64');
|
|
1724
2054
|
} else {
|
|
1725
|
-
const filePath = String(
|
|
2055
|
+
const filePath = String(image.path || '').trim();
|
|
1726
2056
|
if (!filePath) throw new Error('Image attachment is missing file path or data URL');
|
|
1727
2057
|
const allowedRoot = String(allowedRootPath || '').trim();
|
|
1728
2058
|
const blobRoots = (Array.isArray(allowedBlobRootPaths) ? allowedBlobRootPaths : [])
|
|
@@ -1756,6 +2086,8 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1756
2086
|
bytes = fs.readFileSync(filePath);
|
|
1757
2087
|
}
|
|
1758
2088
|
|
|
2089
|
+
// What the bytes are wins over what the attachment says they are.
|
|
2090
|
+
mimeType = (Buffer.isBuffer(bytes) && readImageHeader(bytes)?.mime) || mimeType;
|
|
1759
2091
|
if (!mimeType.startsWith('image/')) {
|
|
1760
2092
|
throw new Error('Image attachment must use image/* MIME type');
|
|
1761
2093
|
}
|
|
@@ -1769,13 +2101,20 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1769
2101
|
return {
|
|
1770
2102
|
bytes,
|
|
1771
2103
|
mimeType,
|
|
1772
|
-
name: String(
|
|
2104
|
+
name: String(image.name || 'image').trim().slice(0, 120) || 'image',
|
|
1773
2105
|
};
|
|
1774
2106
|
}
|
|
1775
2107
|
|
|
1776
|
-
|
|
2108
|
+
/**
|
|
2109
|
+
* The images of an OpenAI image reply. Their type comes from the format
|
|
2110
|
+
* asked for (`outputFormat`), else the one the reply names, else PNG; the
|
|
2111
|
+
* image's own header has the last word.
|
|
2112
|
+
*/
|
|
2113
|
+
export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8, outputFormat = '' } = {}) {
|
|
1777
2114
|
const rows = Array.isArray(payload?.data) ? payload.data : [];
|
|
1778
2115
|
const maxCount = Math.max(1, Math.trunc(Number(maxImages) || 8));
|
|
2116
|
+
const format = String(outputFormat || payload?.output_format || '').trim().toLowerCase();
|
|
2117
|
+
const declaredMime = OPENAI_IMAGE_MIME_BY_FORMAT[format] || OPENAI_IMAGE_MIME_BY_FORMAT.png;
|
|
1779
2118
|
const generatedImages = [];
|
|
1780
2119
|
for (const [index, row] of rows.entries()) {
|
|
1781
2120
|
if (index >= maxCount) break;
|
|
@@ -1785,7 +2124,7 @@ export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8
|
|
|
1785
2124
|
const revisedPrompt = String(row.revised_prompt || row.revisedPrompt || '').trim();
|
|
1786
2125
|
generatedImages.push({
|
|
1787
2126
|
data,
|
|
1788
|
-
mimeType:
|
|
2127
|
+
mimeType: readBase64ImageHeader(data)?.mime || declaredMime,
|
|
1789
2128
|
name: `generated-image-${index + 1}`,
|
|
1790
2129
|
...(revisedPrompt ? { revisedPrompt } : {}),
|
|
1791
2130
|
});
|
|
@@ -1853,7 +2192,12 @@ function createGeneratedImageAttachment({
|
|
|
1853
2192
|
resolveSessionStateRoot,
|
|
1854
2193
|
}) {
|
|
1855
2194
|
const imageId = `img-${String(imageIndex + 1).padStart(2, '0')}`;
|
|
1856
|
-
const
|
|
2195
|
+
const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
|
|
2196
|
+
// The type and pixel size the image's own header gives; the type the
|
|
2197
|
+
// sender named only when the header is not one the reader knows.
|
|
2198
|
+
const header = readImageHeader(bytes);
|
|
2199
|
+
const mimeType = header?.mime || image.mimeType;
|
|
2200
|
+
const extension = resolveGeneratedImageExtension(mimeType);
|
|
1857
2201
|
const location = resolveGeneratedImageStoragePath({
|
|
1858
2202
|
resolveSessionStateRoot,
|
|
1859
2203
|
sdkSessionId,
|
|
@@ -1864,7 +2208,6 @@ function createGeneratedImageAttachment({
|
|
|
1864
2208
|
});
|
|
1865
2209
|
fs.mkdirSync(location.absoluteDir, { recursive: true });
|
|
1866
2210
|
const tmpPath = path.join(location.absoluteDir, `${location.fileName}.${process.pid}.${Date.now()}.tmp`);
|
|
1867
|
-
const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
|
|
1868
2211
|
fs.writeFileSync(tmpPath, bytes);
|
|
1869
2212
|
try {
|
|
1870
2213
|
fs.renameSync(tmpPath, location.absolutePath);
|
|
@@ -1876,8 +2219,9 @@ function createGeneratedImageAttachment({
|
|
|
1876
2219
|
}
|
|
1877
2220
|
return {
|
|
1878
2221
|
name: `${String(image.name || 'generated-image').replace(/\.[a-z0-9]+$/i, '') || 'generated-image'}-${imageIndex + 1}.${extension}`,
|
|
1879
|
-
type:
|
|
2222
|
+
type: mimeType,
|
|
1880
2223
|
size: bytes.length,
|
|
2224
|
+
...(header ? { width: header.width, height: header.height } : {}),
|
|
1881
2225
|
contentUrl: `/api/generated-image/${encodeURIComponent(conversationId)}/${encodeURIComponent(messageId)}/${encodeURIComponent(imageId)}/content`,
|
|
1882
2226
|
generatedImage: {
|
|
1883
2227
|
imageId,
|
|
@@ -2029,6 +2373,9 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2029
2373
|
MAX_UPLOAD_ATTACHMENTS,
|
|
2030
2374
|
uploadsDir,
|
|
2031
2375
|
uploadReadDirs = [uploadsDir],
|
|
2376
|
+
// What a send's `imagePaths` never reads, beside the upload dirs: the
|
|
2377
|
+
// relay's data dir (`dirs`), its database and config files (`files`).
|
|
2378
|
+
imageInputDenyPaths = { dirs: [], files: [] },
|
|
2032
2379
|
MAX_REPO_TREE_NODES,
|
|
2033
2380
|
MAX_REQUEUE_RETRIES,
|
|
2034
2381
|
MAX_IMAGE_DATA_URL_LENGTH,
|
|
@@ -2132,6 +2479,11 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2132
2479
|
// (services/request-boundary.mjs). The runtime passes the check built
|
|
2133
2480
|
// with the configured marker headers.
|
|
2134
2481
|
isTrulyLocalRequest = createLocalRequestCheck().isTrulyLocal,
|
|
2482
|
+
// The pause before the one retry of an OpenAI image request.
|
|
2483
|
+
openAIImageRetryDelayMs = OPENAI_IMAGE_RETRY_DELAY_MS,
|
|
2484
|
+
// How long the partial previews of an image request that ended wait for
|
|
2485
|
+
// the turn's end (services/openai-image-preview-turns.mjs).
|
|
2486
|
+
openAIImagePreviewLeftoverMs,
|
|
2135
2487
|
featureFlags,
|
|
2136
2488
|
sessionWorkerRegistry,
|
|
2137
2489
|
sessionWorkerSupervisor,
|
|
@@ -2154,6 +2506,10 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2154
2506
|
claudeApiUsageService = null,
|
|
2155
2507
|
// Tokens per turn for every provider (token-usage-service.mjs).
|
|
2156
2508
|
tokenUsageService = null,
|
|
2509
|
+
// OpenAI image spend and its daily cap (openai-image-spend-service.mjs):
|
|
2510
|
+
// the runtime shares one instance with the settings routes. Without it
|
|
2511
|
+
// (tests) the routes make their own over tokenUsageService.
|
|
2512
|
+
openAIImageSpend = null,
|
|
2157
2513
|
// The delivery fence's runtime half for GET /api/pending, as on the
|
|
2158
2514
|
// socket: the Claude API claim check (null: no refusal) and what a fenced
|
|
2159
2515
|
// row does (the swap, or fail closed).
|
|
@@ -2196,6 +2552,24 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2196
2552
|
deepseekClaudeConfigDir = '',
|
|
2197
2553
|
} = deps;
|
|
2198
2554
|
|
|
2555
|
+
// An app_settings value as text ('' when unset or unreadable).
|
|
2556
|
+
const readAppSettingText = (key) => {
|
|
2557
|
+
try {
|
|
2558
|
+
return String(stmts?.getAppSetting?.get?.(key)?.value ?? '').trim();
|
|
2559
|
+
} catch {
|
|
2560
|
+
return '';
|
|
2561
|
+
}
|
|
2562
|
+
};
|
|
2563
|
+
// OpenAI image spend: the cap check and reservation before each try, its
|
|
2564
|
+
// booking after, the send-time refusal and the spend state.
|
|
2565
|
+
const imageSpend = openAIImageSpend || createOpenAIImageSpendService({
|
|
2566
|
+
tokenUsageService,
|
|
2567
|
+
readSetting: readAppSettingText,
|
|
2568
|
+
db,
|
|
2569
|
+
emit: (event, payload) => io?.emit?.(event, payload),
|
|
2570
|
+
logger: console,
|
|
2571
|
+
});
|
|
2572
|
+
|
|
2199
2573
|
// ─── Sub-agent runs: header, timeline, live events ─────────────────────────
|
|
2200
2574
|
const subagentTimeline = createSubagentTimelineService({ db, stmts });
|
|
2201
2575
|
|
|
@@ -4059,6 +4433,19 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4059
4433
|
});
|
|
4060
4434
|
});
|
|
4061
4435
|
|
|
4436
|
+
// The media gallery (🧠 Media tab, global Media view) and its index.
|
|
4437
|
+
registerMediaRoutes(app, {
|
|
4438
|
+
auth,
|
|
4439
|
+
db,
|
|
4440
|
+
uploadPathForSha,
|
|
4441
|
+
fetchBrowsableDrives,
|
|
4442
|
+
resolveGeneratedImagePath: ({ sessionId, relativePath }) => resolveGeneratedImageReadPath({
|
|
4443
|
+
resolveSessionStateRoot,
|
|
4444
|
+
sessionId,
|
|
4445
|
+
relativePath,
|
|
4446
|
+
}),
|
|
4447
|
+
});
|
|
4448
|
+
|
|
4062
4449
|
app.get('/api/files/{*splat}', auth, (req, res) => {
|
|
4063
4450
|
const requestedPath = splatPath(req).trim();
|
|
4064
4451
|
const rootOverride = resolveScopedWorkspaceRootPath(req);
|
|
@@ -4871,6 +5258,8 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4871
5258
|
provider,
|
|
4872
5259
|
attachments: rawAttachments,
|
|
4873
5260
|
imageTarget: rawImageTarget,
|
|
5261
|
+
imagePaths: rawImagePaths,
|
|
5262
|
+
image: rawImageOptions,
|
|
4874
5263
|
} = req.body;
|
|
4875
5264
|
// A prompt from another relay's agent (body `origin` or the
|
|
4876
5265
|
// X-OAR-Remote-Origin header) is refused while the inbound switch is off.
|
|
@@ -4907,7 +5296,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4907
5296
|
Array.isArray(rawAttachments) ? rawAttachments.filter((item) => item && typeof item === 'object').length : (rawAttachments ? 1 : 0),
|
|
4908
5297
|
);
|
|
4909
5298
|
const referenceResolution = collectReferenceAttachmentsFromText(trimmedText);
|
|
4910
|
-
|
|
5299
|
+
let attachments = mergeMessageAttachments(normalizedAttachments, referenceResolution.attachments);
|
|
4911
5300
|
const imageTarget = rawImageTarget && typeof rawImageTarget === 'object'
|
|
4912
5301
|
? {
|
|
4913
5302
|
messageId: String(rawImageTarget.messageId || '').trim(),
|
|
@@ -4922,6 +5311,17 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4922
5311
|
code: 'IMAGE_CONTINUITY_DISABLED',
|
|
4923
5312
|
});
|
|
4924
5313
|
}
|
|
5314
|
+
// Images named by path on this relay and the turn's own image options
|
|
5315
|
+
// (relay_tool send and create_session). Both go to an OpenAI image
|
|
5316
|
+
// session only; the paths are read before anything is stored, and one
|
|
5317
|
+
// that may not be read refuses the whole send.
|
|
5318
|
+
const refuseImageInput = (error) => res.status(400).json({ code: IMAGE_INPUT_REFUSED_CODE, error });
|
|
5319
|
+
const imagePathsGiven = rawImagePaths !== undefined && rawImagePaths !== null;
|
|
5320
|
+
const imageOptionsGiven = rawImageOptions !== undefined && rawImageOptions !== null;
|
|
5321
|
+
if (imagePathsGiven && (!Array.isArray(rawImagePaths) || rawImagePaths.some((entry) => typeof entry !== 'string'))) {
|
|
5322
|
+
return refuseImageInput('imagePaths must be a list of absolute file paths on this relay');
|
|
5323
|
+
}
|
|
5324
|
+
const imagePathList = imagePathsGiven ? rawImagePaths.filter((entry) => entry !== '') : [];
|
|
4925
5325
|
|
|
4926
5326
|
if (trimmedText.toLowerCase() === '/compact') {
|
|
4927
5327
|
if (attachments.length) return res.status(400).json({ error: 'Compact command does not accept attachments' });
|
|
@@ -5443,6 +5843,16 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5443
5843
|
latestUserMessageModel: stmts.getLatestUserMessageModel?.get?.(conversationId)?.model || '',
|
|
5444
5844
|
})
|
|
5445
5845
|
: null;
|
|
5846
|
+
// Copilot lists image ids its runtime cannot serve: a Copilot chat on one
|
|
5847
|
+
// is refused with a pointer to an OpenAI Image chat, whoever sends.
|
|
5848
|
+
const copilotModelId = usesGitHubCatalog ? String(inheritedModel ? inheritedModel.model : (model || '')).trim() : '';
|
|
5849
|
+
if (copilotModelId && isOpenAIImageModelId(copilotModelId)) {
|
|
5850
|
+
return res.status(400).json({
|
|
5851
|
+
error: `${openAIImageModelNeedsOpenAIText(copilotModelId)} Nothing was sent.`,
|
|
5852
|
+
code: OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
|
|
5853
|
+
...(conversationId ? { conversationId } : {}),
|
|
5854
|
+
});
|
|
5855
|
+
}
|
|
5446
5856
|
const githubModelResolution = usesGitHubCatalog
|
|
5447
5857
|
? resolveRequestedModel(inheritedModel ? inheritedModel.model : model)
|
|
5448
5858
|
: null;
|
|
@@ -5546,6 +5956,17 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5546
5956
|
code: 'IMAGE_TARGET_REQUIRES_IMAGE_MODEL',
|
|
5547
5957
|
});
|
|
5548
5958
|
}
|
|
5959
|
+
if ((imagePathsGiven || imageOptionsGiven) && !openAIImageModel) {
|
|
5960
|
+
return refuseImageInput(`Images and image options go to an OpenAI image session; this session runs ${requestedModel || 'no image model'}`);
|
|
5961
|
+
}
|
|
5962
|
+
// An image turn from another relay's agent takes its images from
|
|
5963
|
+
// `imagePaths` alone: an @file: token resolves against this relay's
|
|
5964
|
+
// workspace root, not the session's folder, and carries no stored copy.
|
|
5965
|
+
let ignoredReferenceAttachments = 0;
|
|
5966
|
+
if (remoteRequest.remote && openAIImageModel && referenceResolution.attachments.length) {
|
|
5967
|
+
ignoredReferenceAttachments = referenceResolution.attachments.length;
|
|
5968
|
+
attachments = mergeMessageAttachments(normalizedAttachments, []);
|
|
5969
|
+
}
|
|
5549
5970
|
// A send that names no effort runs with the conversation's own, clamped
|
|
5550
5971
|
// to the nearest level the model has. Null when there is nothing to keep
|
|
5551
5972
|
// or it neither matches nor clamps (an ultracode on a model without it):
|
|
@@ -5668,11 +6089,66 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5668
6089
|
// is already 1M.
|
|
5669
6090
|
normalizedContextTier = 'default';
|
|
5670
6091
|
}
|
|
6092
|
+
// An image turn's options: the session's defaults (update_session
|
|
6093
|
+
// `image`), then the preset the turn names (`image.preset`), then the
|
|
6094
|
+
// turn's own. The owner's composer names quality and size in the effort
|
|
6095
|
+
// and context-tier fields, which are the turn's own; an agent's turn
|
|
6096
|
+
// names them in `image`, and its effort counts below the defaults. Only
|
|
6097
|
+
// what an agent sent is refused here: the composer's choice goes on as
|
|
6098
|
+
// before, checked against the model when it runs. The options are
|
|
6099
|
+
// resolved here, at queue time: a preset changed later does not change a
|
|
6100
|
+
// queued turn.
|
|
6101
|
+
let imageOptions = null;
|
|
6102
|
+
if (openAIImageModel) {
|
|
6103
|
+
const { preset: presetName, own: ownImageInput } = splitOpenAIImagePreset(rawImageOptions);
|
|
6104
|
+
const preset = findOpenAIImagePreset(presetName, parseOwnOpenAIImagePresets(readAppSettingText(OPENAI_IMAGE_PRESETS_SETTING_KEY)));
|
|
6105
|
+
if (!preset.ok) return refuseImageInput(`${asSentence(preset.error)} Nothing was sent.`);
|
|
6106
|
+
const ownImageOptions = validateOpenAIImageOptions(ownImageInput, { model: requestedModel });
|
|
6107
|
+
if (!ownImageOptions.ok) return refuseImageInput(ownImageOptions.error);
|
|
6108
|
+
const keptQuality = !explicitReasoningEffort && requestedReasoningEffort && !['auto', 'none'].includes(requestedReasoningEffort)
|
|
6109
|
+
? { quality: requestedReasoningEffort }
|
|
6110
|
+
: {};
|
|
6111
|
+
const fieldOptions = {
|
|
6112
|
+
...(explicitReasoningEffort ? { quality: requestedReasoningEffort } : {}),
|
|
6113
|
+
...(String(contextTier || '').trim() && !['default', 'long_context'].includes(requestedContextTier) ? { size: normalizedContextTier } : {}),
|
|
6114
|
+
};
|
|
6115
|
+
let sessionDefaults = {};
|
|
6116
|
+
if (!shouldCreateConversation) {
|
|
6117
|
+
try {
|
|
6118
|
+
sessionDefaults = stmts.readConversationImageDefaults?.(conversationId) || {};
|
|
6119
|
+
} catch {}
|
|
6120
|
+
}
|
|
6121
|
+
const layers = remoteRequest.remote
|
|
6122
|
+
? [keptQuality, fieldOptions, sessionDefaults, preset.values, ownImageOptions.values]
|
|
6123
|
+
: [keptQuality, sessionDefaults, preset.values, fieldOptions, ownImageOptions.values];
|
|
6124
|
+
const merged = layers.reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
|
|
6125
|
+
const checked = validateOpenAIImageOptions(merged, { model: requestedModel });
|
|
6126
|
+
if (checked.ok) {
|
|
6127
|
+
imageOptions = mergeOpenAIImageOptions(checked.values, null);
|
|
6128
|
+
} else if (imagePathsGiven || imageOptionsGiven) {
|
|
6129
|
+
return refuseImageInput(checked.error);
|
|
6130
|
+
} else {
|
|
6131
|
+
imageOptions = [keptQuality, fieldOptions].reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
|
|
6132
|
+
}
|
|
6133
|
+
// Today's image spend has reached the daily cap: refused for every
|
|
6134
|
+
// sender before anything is stored (the owner's override lifts it).
|
|
6135
|
+
let capRefusal = null;
|
|
6136
|
+
try {
|
|
6137
|
+
capRefusal = imageSpend.sendRefusal();
|
|
6138
|
+
} catch (error) {
|
|
6139
|
+
capRefusal = { code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE, error: `The relay could not check the daily cap of OpenAI images (${error?.message || error}). Nothing was sent.` };
|
|
6140
|
+
}
|
|
6141
|
+
if (capRefusal) {
|
|
6142
|
+
return res.status(409).json({ error: capRefusal.error, code: capRefusal.code, ...(conversationId ? { conversationId } : {}) });
|
|
6143
|
+
}
|
|
6144
|
+
}
|
|
5671
6145
|
if (!requestedRelayMode) return res.status(400).json({ error: 'Unsupported relay mode' });
|
|
5672
6146
|
const conversationWorkspaceState = (!shouldCreateConversation && typeof resolveConversationWorkspaceState === 'function')
|
|
5673
6147
|
? resolveConversationWorkspaceState({ conversationId })
|
|
5674
6148
|
: null;
|
|
5675
|
-
|
|
6149
|
+
// Images by path count as attachments: a "cd …" in their prompt moves
|
|
6150
|
+
// no folder, and they are checked against the folder the session has now.
|
|
6151
|
+
let workspaceRootUpdate = attachments.length === 0 && imagePathList.length === 0
|
|
5676
6152
|
? maybeApplyWorkspaceRootFromMessage(
|
|
5677
6153
|
trimmedText,
|
|
5678
6154
|
conversationWorkspaceState?.currentWorkspaceRootPath || null,
|
|
@@ -5742,6 +6218,56 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5742
6218
|
}
|
|
5743
6219
|
}
|
|
5744
6220
|
|
|
6221
|
+
// The images by path: every one checked and read before anything is
|
|
6222
|
+
// stored, then kept in the upload store, so each becomes an attachment
|
|
6223
|
+
// with its bytes' hash and sniffed type (and an asset of the operation).
|
|
6224
|
+
// They come first: the first image is the base of an edit.
|
|
6225
|
+
let imageInputs = [];
|
|
6226
|
+
let imageInputsDropped = 0;
|
|
6227
|
+
if (imagePathsGiven) {
|
|
6228
|
+
if (shouldCreateConversation) {
|
|
6229
|
+
return refuseImageInput('imagePaths need an existing session: create it first, then send the images to it');
|
|
6230
|
+
}
|
|
6231
|
+
let sessionStateRoot = '';
|
|
6232
|
+
try {
|
|
6233
|
+
sessionStateRoot = String(resolveSessionStateRoot?.() || '');
|
|
6234
|
+
} catch {}
|
|
6235
|
+
const roots = resolveImageInputRoots({
|
|
6236
|
+
cwd: conversationWorkspaceState?.currentWorkspaceRootPath || '',
|
|
6237
|
+
sessionStateRoot,
|
|
6238
|
+
deniedDirs: [...(imageInputDenyPaths?.dirs || []), uploadsDir, ...(uploadReadDirs || [])],
|
|
6239
|
+
deniedFiles: imageInputDenyPaths?.files || [],
|
|
6240
|
+
});
|
|
6241
|
+
const resolvedImages = resolveImageInputs(imagePathList, {
|
|
6242
|
+
roots,
|
|
6243
|
+
isLiveConversation: (id) => {
|
|
6244
|
+
const row = stmts.getConvAnyStatus?.get?.(id) || null;
|
|
6245
|
+
return !!row && String(row.status || '').trim() !== 'deleted';
|
|
6246
|
+
},
|
|
6247
|
+
});
|
|
6248
|
+
if (!resolvedImages.ok) return refuseImageInput(resolvedImages.error);
|
|
6249
|
+
const resolvedHashes = new Set(resolvedImages.images.map((image) => image.sha256));
|
|
6250
|
+
const otherAttachments = attachments.filter((attachment) => !resolvedHashes.has(String(attachment?.sha256 || '').trim().toLowerCase()));
|
|
6251
|
+
const total = resolvedImages.images.length + otherAttachments.length;
|
|
6252
|
+
if (total > OPENAI_IMAGE_LIMITS.inputImagesMax) {
|
|
6253
|
+
return refuseImageInput(`An image turn takes at most ${OPENAI_IMAGE_LIMITS.inputImagesMax} images; this one would have ${total} with the message's other attachments`);
|
|
6254
|
+
}
|
|
6255
|
+
try {
|
|
6256
|
+
imageInputs = resolvedImages.images.map((image) => ({
|
|
6257
|
+
...persistUploadBuffer(image.bytes, { name: image.name, type: image.mime }),
|
|
6258
|
+
// The type the bytes are, not the one an earlier upload of the
|
|
6259
|
+
// same bytes was stored with.
|
|
6260
|
+
type: image.mime,
|
|
6261
|
+
width: image.width,
|
|
6262
|
+
height: image.height,
|
|
6263
|
+
}));
|
|
6264
|
+
} catch (error) {
|
|
6265
|
+
return res.status(500).json({ code: 'IMAGE_INPUT_STORE_FAILED', error: `The images could not be stored on this relay: ${error?.message || error}` });
|
|
6266
|
+
}
|
|
6267
|
+
imageInputsDropped = imagePathList.length - imageInputs.length;
|
|
6268
|
+
attachments = [...imageInputs, ...otherAttachments];
|
|
6269
|
+
}
|
|
6270
|
+
|
|
5745
6271
|
const convId = shouldCreateConversation ? uuidv4() : conversationId;
|
|
5746
6272
|
if (shouldCreateConversation) {
|
|
5747
6273
|
// An agent's prompt opens with the sending relay's header line, which
|
|
@@ -5978,6 +6504,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5978
6504
|
null,
|
|
5979
6505
|
null,
|
|
5980
6506
|
);
|
|
6507
|
+
if (imageOptions) stmts.setQueueImageOptions?.run?.(JSON.stringify(imageOptions), msgId);
|
|
5981
6508
|
if (imageContinuityEnabled && openAIImageModel) {
|
|
5982
6509
|
if (!imageOperationService?.createEnqueuedOperation) {
|
|
5983
6510
|
throw new Error('Image operation service is unavailable');
|
|
@@ -5989,11 +6516,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5989
6516
|
prompt: trimmedText,
|
|
5990
6517
|
selectedImageModel: requestedModel,
|
|
5991
6518
|
executionMode: 'direct_images',
|
|
5992
|
-
parameters: {
|
|
5993
|
-
quality: requestedReasoningEffort,
|
|
5994
|
-
size: normalizedContextTier,
|
|
5995
|
-
count: 1,
|
|
5996
|
-
},
|
|
6519
|
+
parameters: imageOptions || { count: 1 },
|
|
5997
6520
|
attachments,
|
|
5998
6521
|
imageTarget,
|
|
5999
6522
|
createdAt: now,
|
|
@@ -6158,8 +6681,17 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
6158
6681
|
workspaceRootWarning: workspaceRootUpdate.error || null,
|
|
6159
6682
|
workspaceRootChanged: !!workspaceRootUpdate.changed,
|
|
6160
6683
|
...workspaceRootPayload(),
|
|
6161
|
-
referenceAttachmentCount: referenceResolution.attachments.length,
|
|
6684
|
+
referenceAttachmentCount: referenceResolution.attachments.length - ignoredReferenceAttachments,
|
|
6162
6685
|
skippedReferenceAttachments: referenceResolution.skipped,
|
|
6686
|
+
...(ignoredReferenceAttachments ? { ignoredReferenceAttachments } : {}),
|
|
6687
|
+
// How many images from imagePaths the turn has, and how many named
|
|
6688
|
+
// twice (the same file, or the same bytes) were kept once.
|
|
6689
|
+
...(imagePathsGiven ? { imageInputs: imageInputs.length } : {}),
|
|
6690
|
+
...(imageInputsDropped > 0 ? {
|
|
6691
|
+
imageInputsDropped,
|
|
6692
|
+
imageInputsNote: `${imageInputsDropped} of the ${imagePathList.length} image paths named an image already attached; it was attached once.`,
|
|
6693
|
+
} : {}),
|
|
6694
|
+
...(imageOptions ? { imageOptions } : {}),
|
|
6163
6695
|
// Attachment references that no longer resolve (a deleted upload) are
|
|
6164
6696
|
// dropped silently by normalizeAttachments; the sender is told how many.
|
|
6165
6697
|
droppedAttachmentCount: Math.max(0, requestedAttachmentCount - normalizedAttachments.length),
|
|
@@ -7652,8 +8184,516 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7652
8184
|
return res.json({ ok: true, ...result });
|
|
7653
8185
|
});
|
|
7654
8186
|
|
|
8187
|
+
// Image operations whose OpenAI request runs in this process, and the
|
|
8188
|
+
// direct image requests (no operation) this process sent, by queue
|
|
8189
|
+
// message: a second run of either would be billed again.
|
|
8190
|
+
const executingImageOperations = new Set();
|
|
8191
|
+
const dispatchedImageMessages = new Map();
|
|
8192
|
+
const DISPATCHED_IMAGE_MESSAGES_MAX = 500;
|
|
8193
|
+
const openAIImageTimeoutFromConfig = (key, fallback) => (Number(config?.[key]) > 0 ? Number(config[key]) : fallback);
|
|
8194
|
+
const openAIImageGenerateTimeoutMs = openAIImageTimeoutFromConfig('openaiImageGenerateTimeoutMs', OPENAI_IMAGE_GENERATE_TIMEOUT_MS);
|
|
8195
|
+
const openAIImageEditTimeoutMs = openAIImageTimeoutFromConfig('openaiImageEditTimeoutMs', OPENAI_IMAGE_EDIT_TIMEOUT_MS);
|
|
8196
|
+
// The partial previews of the running image turns: files beside the
|
|
8197
|
+
// turn's final images, events to the owner's clients (every socket is the
|
|
8198
|
+
// owner's: share views read over HTTP and have none).
|
|
8199
|
+
const imagePreviews = createOpenAIImagePreviewTurns({
|
|
8200
|
+
resolveSessionStateRoot,
|
|
8201
|
+
sessionIdFor: (conversationId) => String(stmts.getConvAnyStatus?.get?.(conversationId)?.sdk_session_id || '').trim() || conversationId,
|
|
8202
|
+
emit: (event, payload) => io?.emit?.(event, payload),
|
|
8203
|
+
...(openAIImagePreviewLeftoverMs !== undefined ? { leftoverMs: openAIImagePreviewLeftoverMs } : {}),
|
|
8204
|
+
});
|
|
8205
|
+
|
|
8206
|
+
// A partial preview of a running image turn, for the owner's clients
|
|
8207
|
+
// only: only a file the turn wrote and still has is served (no path is
|
|
8208
|
+
// built from the request), never through a share link, and gone with the
|
|
8209
|
+
// turn.
|
|
8210
|
+
app.get(`/api/generated-image/:conversationId/:messageId/${OPENAI_IMAGE_PARTIALS_DIR}/:fileName`, auth, (req, res) => {
|
|
8211
|
+
const file = imagePreviews.partialFile({
|
|
8212
|
+
conversationId: String(req.params.conversationId || '').trim(),
|
|
8213
|
+
messageId: String(req.params.messageId || '').trim(),
|
|
8214
|
+
fileName: String(req.params.fileName || '').trim(),
|
|
8215
|
+
});
|
|
8216
|
+
// lstat: a link put in the turn's partials folder is not followed.
|
|
8217
|
+
let stat = null;
|
|
8218
|
+
try {
|
|
8219
|
+
stat = file ? fs.lstatSync(file.path) : null;
|
|
8220
|
+
} catch {}
|
|
8221
|
+
if (!stat?.isFile()) return res.status(404).json({ error: 'Preview not found' });
|
|
8222
|
+
return serveFileWithRangeSupport(req, res, file.path, { size: stat.size, contentType: file.mime }, {
|
|
8223
|
+
safeName: path.basename(file.path).replace(/"/g, ''),
|
|
8224
|
+
// And the open itself refuses a link swapped in after the lstat (POSIX;
|
|
8225
|
+
// Windows has no O_NOFOLLOW, the lstat is the guard there).
|
|
8226
|
+
fsImpl: PARTIAL_PREVIEW_FS,
|
|
8227
|
+
});
|
|
8228
|
+
});
|
|
8229
|
+
|
|
8230
|
+
function noteDispatchedImageMessage(messageId) {
|
|
8231
|
+
dispatchedImageMessages.delete(messageId);
|
|
8232
|
+
dispatchedImageMessages.set(messageId, { at: Date.now(), running: true });
|
|
8233
|
+
while (dispatchedImageMessages.size > DISPATCHED_IMAGE_MESSAGES_MAX) {
|
|
8234
|
+
dispatchedImageMessages.delete(dispatchedImageMessages.keys().next().value);
|
|
8235
|
+
}
|
|
8236
|
+
return dispatchedImageMessages.get(messageId);
|
|
8237
|
+
}
|
|
8238
|
+
|
|
8239
|
+
/**
|
|
8240
|
+
* One try of an OpenAI image request: `{ ok: true, generatedImages, usage }`
|
|
8241
|
+
* or `{ kind, status?, message?, detail? }` (see OPENAI_IMAGE_RETRY_KINDS).
|
|
8242
|
+
* The timeout covers the reply's body as well, a streamed one whole.
|
|
8243
|
+
* `fields` asking for a stream get one read event by event (a reply that is
|
|
8244
|
+
* no event stream is read as JSON); `onPartial` sees each partial image.
|
|
8245
|
+
*/
|
|
8246
|
+
async function sendOpenAIImageRequestOnce({ url, headers, fields, editImages, family, timeoutMs, outputFormat, model, onPartial = null }) {
|
|
8247
|
+
// The images asked for: a stream's events past them are dropped.
|
|
8248
|
+
const expectedImages = Math.max(1, Math.trunc(Number(fields.n) || 1));
|
|
8249
|
+
let response;
|
|
8250
|
+
try {
|
|
8251
|
+
let body;
|
|
8252
|
+
if (editImages.length) {
|
|
8253
|
+
body = new FormData();
|
|
8254
|
+
for (const [key, value] of Object.entries(fields)) body.set(key, String(value));
|
|
8255
|
+
// gpt-image takes every image, the first being the base; dall-e-2
|
|
8256
|
+
// takes one.
|
|
8257
|
+
const images = family === 'gpt-image' ? editImages : editImages.slice(0, 1);
|
|
8258
|
+
for (const image of images) {
|
|
8259
|
+
body.append(
|
|
8260
|
+
family === 'gpt-image' ? 'image[]' : 'image',
|
|
8261
|
+
new Blob([image.bytes], { type: image.mimeType }),
|
|
8262
|
+
image.name,
|
|
8263
|
+
);
|
|
8264
|
+
}
|
|
8265
|
+
} else {
|
|
8266
|
+
body = JSON.stringify(fields);
|
|
8267
|
+
}
|
|
8268
|
+
response = await fetch(url, {
|
|
8269
|
+
method: 'POST',
|
|
8270
|
+
headers: editImages.length ? headers : { ...headers, 'Content-Type': 'application/json' },
|
|
8271
|
+
body,
|
|
8272
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
8273
|
+
});
|
|
8274
|
+
} catch (error) {
|
|
8275
|
+
const kind = classifyOpenAIImageFetchError(error);
|
|
8276
|
+
return { kind, ownTimeout: error?.name === 'TimeoutError', detail: describeFetchError(error) };
|
|
8277
|
+
}
|
|
8278
|
+
|
|
8279
|
+
const status = Number(response.status);
|
|
8280
|
+
if (response.ok && fields.stream === true && isEventStreamResponse(response)) {
|
|
8281
|
+
return readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages });
|
|
8282
|
+
}
|
|
8283
|
+
let rawText = '';
|
|
8284
|
+
try {
|
|
8285
|
+
rawText = String(await response.text()).trim();
|
|
8286
|
+
} catch (error) {
|
|
8287
|
+
// A 2xx whose body broke off is billed; an error status keeps its class.
|
|
8288
|
+
if (response.ok) return { kind: 'unprocessed', status, detail: `the reply broke off: ${describeFetchError(error)}` };
|
|
8289
|
+
return status >= 500 ? { kind: 'server', status } : { kind: 'refused', status };
|
|
8290
|
+
}
|
|
8291
|
+
let payload = {};
|
|
8292
|
+
if (rawText) {
|
|
8293
|
+
try {
|
|
8294
|
+
payload = JSON.parse(rawText);
|
|
8295
|
+
} catch {
|
|
8296
|
+
// After a 2xx the generation is billed; a retry pays again. An error
|
|
8297
|
+
// status with a body that is not JSON stays a provider failure.
|
|
8298
|
+
if (response.ok) return { kind: 'unprocessed', status, detail: 'OpenAI image API returned invalid JSON' };
|
|
8299
|
+
payload = null;
|
|
8300
|
+
}
|
|
8301
|
+
}
|
|
8302
|
+
if (!response.ok) {
|
|
8303
|
+
const message = String(payload?.error?.message || payload?.message || (payload ? '' : rawText)).trim().slice(0, 240);
|
|
8304
|
+
if (!Number.isFinite(status) || status >= 500) return { kind: 'server', status, message };
|
|
8305
|
+
return { kind: 'refused', status, message };
|
|
8306
|
+
}
|
|
8307
|
+
try {
|
|
8308
|
+
const generatedImages = normalizeOpenAIImageApiGeneratedImages(payload, {
|
|
8309
|
+
maxImages: MAX_UPLOAD_ATTACHMENTS,
|
|
8310
|
+
outputFormat,
|
|
8311
|
+
});
|
|
8312
|
+
return { ok: true, generatedImages, usage: normalizeOpenAIImageUsage(payload?.usage) };
|
|
8313
|
+
} catch (error) {
|
|
8314
|
+
// The usage the reply carried is what was billed.
|
|
8315
|
+
const usage = normalizeOpenAIImageUsage(payload?.usage);
|
|
8316
|
+
// The provider answered 2xx, so the generation is billed; what failed
|
|
8317
|
+
// is the relay's handling of the reply. Logged with the stack: a retry
|
|
8318
|
+
// pays again and fails the same way.
|
|
8319
|
+
console.error(
|
|
8320
|
+
`[${ts()}] IMAGE reply of ${model} not processed: ${error?.message || error} (body ${rawText.length} chars)\n${error?.stack || ''}`,
|
|
8321
|
+
);
|
|
8322
|
+
return { kind: 'unprocessed', status, detail: error?.message || 'OpenAI image API returned no images', ...(usage ? { usage } : {}) };
|
|
8323
|
+
}
|
|
8324
|
+
}
|
|
8325
|
+
|
|
8326
|
+
/**
|
|
8327
|
+
* The rest of a streamed reply (after a 2xx with an event stream): each
|
|
8328
|
+
* partial goes to `onPartial`, each completed event's image is kept in
|
|
8329
|
+
* order (a count > 1 turn may get several), and the completed events'
|
|
8330
|
+
* usages add up to the request's. A stream that breaks off, ends without an image
|
|
8331
|
+
* or reports a server error after OpenAI started may have been billed:
|
|
8332
|
+
* 'stream', 'timeout' or 'server', tried once more. An error event that
|
|
8333
|
+
* names a refusal is 'refused'; an image the relay cannot read, or an event
|
|
8334
|
+
* past the reader's cap, 'unprocessed'.
|
|
8335
|
+
*
|
|
8336
|
+
* Once every image asked for (`expectedImages`) is complete, a stream that
|
|
8337
|
+
* breaks off or times out before its end still answers them: they are
|
|
8338
|
+
* billed, and a retry would buy them again. Partials past
|
|
8339
|
+
* OPENAI_IMAGE_PARTIAL_IMAGES per image or past the images asked for, and
|
|
8340
|
+
* completed events past those images, are dropped unread.
|
|
8341
|
+
*
|
|
8342
|
+
* Every result carries `partials` (received) and, once a completed event
|
|
8343
|
+
* was read, `usage`: a try that fails after it books that usage
|
|
8344
|
+
* (openAIImageTryCharge).
|
|
8345
|
+
*/
|
|
8346
|
+
async function readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages = 1 }) {
|
|
8347
|
+
const completed = [];
|
|
8348
|
+
let usage = null;
|
|
8349
|
+
let partials = 0;
|
|
8350
|
+
// What every result carries for the booking.
|
|
8351
|
+
const counted = (result) => ({ ...result, partials, ...(usage ? { usage } : {}) });
|
|
8352
|
+
const answer = () => {
|
|
8353
|
+
try {
|
|
8354
|
+
const generatedImages = normalizeOpenAIImageApiGeneratedImages({
|
|
8355
|
+
data: completed.map((payload) => ({ b64_json: payload.b64_json })),
|
|
8356
|
+
output_format: completed[0].output_format,
|
|
8357
|
+
}, { maxImages: MAX_UPLOAD_ATTACHMENTS, outputFormat });
|
|
8358
|
+
return counted({ ok: true, generatedImages });
|
|
8359
|
+
} catch (error) {
|
|
8360
|
+
console.error(`[${ts()}] IMAGE streamed reply of ${model} not processed: ${error?.message || error}\n${error?.stack || ''}`);
|
|
8361
|
+
return counted({ kind: 'unprocessed', status, detail: error?.message || 'OpenAI image stream returned no images' });
|
|
8362
|
+
}
|
|
8363
|
+
};
|
|
8364
|
+
try {
|
|
8365
|
+
for await (const sse of readSseEvents(response.body)) {
|
|
8366
|
+
const event = interpretOpenAIImageStreamEvent(sse);
|
|
8367
|
+
if (event.type === 'partial') {
|
|
8368
|
+
partials += 1;
|
|
8369
|
+
const partialIndex = Number.isInteger(event.payload.partial_image_index) ? event.payload.partial_image_index : partials - 1;
|
|
8370
|
+
const shown = completed.length < expectedImages && partialIndex >= 0 && partialIndex < OPENAI_IMAGE_PARTIAL_IMAGES;
|
|
8371
|
+
if (onPartial && shown) {
|
|
8372
|
+
try {
|
|
8373
|
+
await onPartial({
|
|
8374
|
+
imageIndex: completed.length,
|
|
8375
|
+
partialIndex,
|
|
8376
|
+
b64: event.payload.b64_json,
|
|
8377
|
+
outputFormat: event.payload.output_format || outputFormat,
|
|
8378
|
+
});
|
|
8379
|
+
} catch (error) {
|
|
8380
|
+
console.warn(`[${ts()}] IMAGE partial of ${model} not shown: ${error?.message || error}`);
|
|
8381
|
+
}
|
|
8382
|
+
}
|
|
8383
|
+
} else if (event.type === 'completed') {
|
|
8384
|
+
if (completed.length >= expectedImages) continue;
|
|
8385
|
+
completed.push(event.payload);
|
|
8386
|
+
usage = addOpenAIImageUsage(usage, normalizeOpenAIImageUsage(event.payload.usage));
|
|
8387
|
+
} else if (event.type === 'invalid' && sse.event.endsWith('.completed')) {
|
|
8388
|
+
return counted({ kind: 'unprocessed', status, detail: 'OpenAI image stream sent an unreadable image' });
|
|
8389
|
+
} else if (event.type === 'error') {
|
|
8390
|
+
const { message, type, code } = event.error;
|
|
8391
|
+
const named = `${type} ${code}`.toLowerCase();
|
|
8392
|
+
const detail = `error event in the stream${code ? ` (${code})` : ''} after ${partials} partial image(s)`;
|
|
8393
|
+
if (['server', 'internal', 'overloaded', 'timeout', 'unavailable'].some((word) => named.includes(word)) || !named.trim()) {
|
|
8394
|
+
return counted({ kind: 'server', message, detail });
|
|
8395
|
+
}
|
|
8396
|
+
return counted({ kind: 'refused', status: 400, message, detail, streamed: true });
|
|
8397
|
+
}
|
|
8398
|
+
}
|
|
8399
|
+
} catch (error) {
|
|
8400
|
+
if (error?.code === SSE_EVENT_TOO_LARGE_CODE) return counted({ kind: 'unprocessed', status, detail: error.message });
|
|
8401
|
+
if (completed.length >= expectedImages) {
|
|
8402
|
+
console.warn(`[${ts()}] IMAGE ${model} stream broke off after its ${completed.length} image(s): kept (${describeFetchError(error)})`);
|
|
8403
|
+
return answer();
|
|
8404
|
+
}
|
|
8405
|
+
const kind = classifyOpenAIImageFetchError(error);
|
|
8406
|
+
return counted({
|
|
8407
|
+
kind: kind === 'timeout' ? 'timeout' : 'stream',
|
|
8408
|
+
ownTimeout: error?.name === 'TimeoutError',
|
|
8409
|
+
detail: `the stream broke off after ${partials} partial image(s): ${describeFetchError(error)}`,
|
|
8410
|
+
});
|
|
8411
|
+
}
|
|
8412
|
+
if (!completed.length) return counted({ kind: 'stream', detail: `the stream ended without an image after ${partials} partial image(s)` });
|
|
8413
|
+
return answer();
|
|
8414
|
+
}
|
|
8415
|
+
|
|
8416
|
+
/**
|
|
8417
|
+
* The OpenAI image request of one turn, for the execute route and the
|
|
8418
|
+
* generate route alike: the request with every option and every input
|
|
8419
|
+
* image, a timeout per try (generate / edit), and one retry of a try that
|
|
8420
|
+
* may pass the second time, with the same Idempotency-Key (OpenAI may
|
|
8421
|
+
* ignore it; the note says the request may have been billed). Never more
|
|
8422
|
+
* than two requests: nothing else may send this turn again.
|
|
8423
|
+
*
|
|
8424
|
+
* `requestBody` is normalizeOpenAIImageGenerateRequestBody's. Resolves
|
|
8425
|
+
* `{ ok: true, generatedImages, model, endpoint, usage, tries, note? }`
|
|
8426
|
+
* (`note`: openAIImageRetriedNote's) or openAIImageFailureResult's failure.
|
|
8427
|
+
*
|
|
8428
|
+
* A gpt-image request is streamed with OPENAI_IMAGE_PARTIAL_IMAGES partials
|
|
8429
|
+
* per image (dall-e models do not stream). With `preview` ({ conversationId,
|
|
8430
|
+
* messageId }: the queue message the turn answers) the owner's clients see
|
|
8431
|
+
* each try start and each partial; a retry drops the first try's, and the
|
|
8432
|
+
* turn's end (/api/response) the rest.
|
|
8433
|
+
*
|
|
8434
|
+
* Each try is checked against the daily cap and reserves its estimate
|
|
8435
|
+
* right before it is sent (the resend without stream fields keeps the
|
|
8436
|
+
* reservation); the cap refuses a try with OPENAI_IMAGE_DAILY_CAP_REACHED
|
|
8437
|
+
* and sends nothing. When the try ends, its cost is booked under the queue
|
|
8438
|
+
* row (`scopeId`: the image attempt, or the claim, that runs it; run_id
|
|
8439
|
+
* image:<scopeId>:try<n>) and its reservation dropped, in one step.
|
|
8440
|
+
*/
|
|
8441
|
+
async function runOpenAIImageRequest(requestBody, { idempotencyKey = '', preview = null, scopeId = '' } = {}) {
|
|
8442
|
+
const settings = getOpenAIProviderSettings();
|
|
8443
|
+
if (settings?.configured !== true || !String(settings?.apiKey || '').trim()) {
|
|
8444
|
+
return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI BYOK is not configured');
|
|
8445
|
+
}
|
|
8446
|
+
const baseUrl = String(settings.baseUrl || '').trim().replace(/\/+$/, '');
|
|
8447
|
+
if (!baseUrl) return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI base URL is not configured');
|
|
8448
|
+
|
|
8449
|
+
let editImages = [];
|
|
8450
|
+
try {
|
|
8451
|
+
editImages = resolveOpenAIImageEditAttachments(requestBody.attachments, {
|
|
8452
|
+
maxImageBytes: MAX_UPLOAD_BYTES,
|
|
8453
|
+
allowedRootPath: uploadsDir,
|
|
8454
|
+
allowedBlobRootPaths: uploadReadDirs.slice(1),
|
|
8455
|
+
allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
|
|
8456
|
+
});
|
|
8457
|
+
} catch (error) {
|
|
8458
|
+
return openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', error?.message || 'Invalid image attachment');
|
|
8459
|
+
}
|
|
8460
|
+
|
|
8461
|
+
const family = openAIImageModelFamily(requestBody.model);
|
|
8462
|
+
const fields = buildOpenAIImageRequestFields(requestBody);
|
|
8463
|
+
const endpoint = editImages.length ? '/images/edits' : '/images/generations';
|
|
8464
|
+
const timeoutMs = editImages.length ? openAIImageEditTimeoutMs : openAIImageGenerateTimeoutMs;
|
|
8465
|
+
const headers = {
|
|
8466
|
+
Authorization: `Bearer ${settings.apiKey}`,
|
|
8467
|
+
...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
|
|
8468
|
+
};
|
|
8469
|
+
let streaming = family === 'gpt-image' && OPENAI_IMAGE_PARTIAL_IMAGES > 0;
|
|
8470
|
+
const previewTurn = preview
|
|
8471
|
+
? imagePreviews.begin({
|
|
8472
|
+
conversationId: preview.conversationId,
|
|
8473
|
+
messageId: preview.messageId,
|
|
8474
|
+
size: requestBody.size,
|
|
8475
|
+
quality: requestBody.quality,
|
|
8476
|
+
count: requestBody.n,
|
|
8477
|
+
model: requestBody.model,
|
|
8478
|
+
})
|
|
8479
|
+
: null;
|
|
8480
|
+
const tries = [];
|
|
8481
|
+
// HTTP requests of this turn, the resend without stream fields included:
|
|
8482
|
+
// never more than two.
|
|
8483
|
+
let calls = 0;
|
|
8484
|
+
let resent = false;
|
|
8485
|
+
let result = null;
|
|
8486
|
+
// What a try of this request costs, and where it books.
|
|
8487
|
+
const spendRequest = {
|
|
8488
|
+
model: requestBody.model,
|
|
8489
|
+
size: requestBody.size,
|
|
8490
|
+
quality: requestBody.quality,
|
|
8491
|
+
count: requestBody.n,
|
|
8492
|
+
inputImages: editImages.length,
|
|
8493
|
+
};
|
|
8494
|
+
const bookingScope = String(scopeId || '').trim() || `claim_${randomUUID()}`;
|
|
8495
|
+
// The try in flight's reservation: held across the resend, released
|
|
8496
|
+
// (with its booking, or without one) when the try ends.
|
|
8497
|
+
let reservation = null;
|
|
8498
|
+
previewTurn?.started(1);
|
|
8499
|
+
try {
|
|
8500
|
+
for (;;) {
|
|
8501
|
+
if (!reservation) {
|
|
8502
|
+
const tryNumber = tries.length + 1;
|
|
8503
|
+
const reserved = imageSpend.reserveTry({ estimateUsd: estimateOpenAIImageCost(spendRequest).costUsd });
|
|
8504
|
+
if (!reserved.ok) {
|
|
8505
|
+
console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tryNumber} stopped by the daily cap`);
|
|
8506
|
+
result = openAIImageCapRefusalResult(tries, reserved);
|
|
8507
|
+
break;
|
|
8508
|
+
}
|
|
8509
|
+
reservation = reserved.reservation;
|
|
8510
|
+
}
|
|
8511
|
+
calls += 1;
|
|
8512
|
+
const tried = await sendOpenAIImageRequestOnce({
|
|
8513
|
+
url: `${baseUrl}${endpoint}`,
|
|
8514
|
+
// The resend's body differs from the first request's: its own key.
|
|
8515
|
+
headers: resent && idempotencyKey ? { ...headers, 'Idempotency-Key': `${idempotencyKey}-nostream` } : headers,
|
|
8516
|
+
fields: streaming ? { ...fields, stream: true, partial_images: OPENAI_IMAGE_PARTIAL_IMAGES } : fields,
|
|
8517
|
+
editImages,
|
|
8518
|
+
family,
|
|
8519
|
+
timeoutMs,
|
|
8520
|
+
outputFormat: requestBody.output_format,
|
|
8521
|
+
model: requestBody.model,
|
|
8522
|
+
onPartial: previewTurn ? (partial) => previewTurn.partial(partial) : null,
|
|
8523
|
+
});
|
|
8524
|
+
// The resend without stream fields (below) is the same try: it keeps
|
|
8525
|
+
// the reservation and books once.
|
|
8526
|
+
const resend = streaming && calls < 2 && tried.kind === 'refused' && Number(tried.status) === 400 && !tried.streamed
|
|
8527
|
+
&& /\bstream|partial_image/i.test(String(tried.message || '').slice(0, 240));
|
|
8528
|
+
if (!resend) {
|
|
8529
|
+
imageSpend.settleTry(reservation, {
|
|
8530
|
+
conversationId: requestBody.conversationId,
|
|
8531
|
+
turnId: requestBody.messageId,
|
|
8532
|
+
runId: openAIImageTryRunId(bookingScope, tries.length + 1),
|
|
8533
|
+
model: requestBody.model,
|
|
8534
|
+
charge: openAIImageTryCharge({ tried, request: spendRequest }),
|
|
8535
|
+
});
|
|
8536
|
+
reservation = null;
|
|
8537
|
+
}
|
|
8538
|
+
if (tried.ok) {
|
|
8539
|
+
const note = openAIImageRetriedNote(tries, { timeoutMs });
|
|
8540
|
+
result = {
|
|
8541
|
+
ok: true,
|
|
8542
|
+
generatedImages: tried.generatedImages,
|
|
8543
|
+
model: requestBody.model,
|
|
8544
|
+
endpoint,
|
|
8545
|
+
usage: tried.usage,
|
|
8546
|
+
tries: tries.length + 1,
|
|
8547
|
+
...(note ? { note } : {}),
|
|
8548
|
+
};
|
|
8549
|
+
break;
|
|
8550
|
+
}
|
|
8551
|
+
// An endpoint that takes no stream (an OpenAI-compatible one, or a
|
|
8552
|
+
// model OpenAI does not stream) refuses the fields with a 400 before
|
|
8553
|
+
// any work: the same try again without them, nothing billed. It
|
|
8554
|
+
// counts as one of the two requests, so a resend that fails is final.
|
|
8555
|
+
// Any other refusal (a 429 that names streaming, say) is no reason.
|
|
8556
|
+
if (resend) {
|
|
8557
|
+
console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} refused streaming; sent again without it`);
|
|
8558
|
+
streaming = false;
|
|
8559
|
+
resent = true;
|
|
8560
|
+
continue;
|
|
8561
|
+
}
|
|
8562
|
+
tries.push(tried);
|
|
8563
|
+
console.warn(
|
|
8564
|
+
`[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tries.length} ${tried.kind}`
|
|
8565
|
+
+ `${tried.status ? ` status=${tried.status}` : ''}${tried.detail ? ` ${String(tried.detail).slice(0, 200)}` : ''}`,
|
|
8566
|
+
);
|
|
8567
|
+
if (calls >= 2 || !OPENAI_IMAGE_RETRY_KINDS.has(tried.kind)) break;
|
|
8568
|
+
await delay(openAIImageRetryDelayMs);
|
|
8569
|
+
previewTurn?.retry();
|
|
8570
|
+
previewTurn?.started(tries.length + 1);
|
|
8571
|
+
}
|
|
8572
|
+
} finally {
|
|
8573
|
+
// A throw or an abort: the try books nothing it did not book above.
|
|
8574
|
+
imageSpend.releaseTry(reservation);
|
|
8575
|
+
previewTurn?.ended({ ok: Boolean(result?.ok) });
|
|
8576
|
+
}
|
|
8577
|
+
return result || openAIImageFailureResult(tries, { timeoutMs, resent });
|
|
8578
|
+
}
|
|
8579
|
+
|
|
8580
|
+
/**
|
|
8581
|
+
* The reply line of an image turn: what came back, and what this turn's
|
|
8582
|
+
* tries were booked at ("~USD 0.21", marked "(estimate)" when any part is
|
|
8583
|
+
* one, "(price unknown, estimated)" for the stand-in); from 80 % of the
|
|
8584
|
+
* daily cap on, today's spend against it.
|
|
8585
|
+
*/
|
|
8586
|
+
function openAIImageReplyLine({ images, model, turnId }) {
|
|
8587
|
+
let cost = null;
|
|
8588
|
+
let state = null;
|
|
8589
|
+
try {
|
|
8590
|
+
cost = imageSpend.turnCost(turnId);
|
|
8591
|
+
state = imageSpend.buildState();
|
|
8592
|
+
} catch (error) {
|
|
8593
|
+
console.warn(`[${ts()}] IMAGE the cost of ${String(turnId || '').slice(0, 8)} could not be read: ${error?.message || error}`);
|
|
8594
|
+
}
|
|
8595
|
+
let line = formatOpenAIImageSummary({ images, model, costUsd: cost ? cost.costUsd : null });
|
|
8596
|
+
if (cost?.unknownPrice) line += ' (price unknown, estimated)';
|
|
8597
|
+
else if (cost?.estimated) line += ' (estimate)';
|
|
8598
|
+
// The spend state's own rule (booked and in flight, `warning`), as the
|
|
8599
|
+
// banner reads it.
|
|
8600
|
+
if (state?.capUsd && state.warning) {
|
|
8601
|
+
const todayUsd = (Number(state.spentUsd) || 0) + (Number(state.reservedUsd) || 0);
|
|
8602
|
+
line += ` · today ${formatOpenAIImageUsd(todayUsd)} of ${state.capUsd.toFixed(2)} (${Math.round((todayUsd / state.capUsd) * 100)} %)`;
|
|
8603
|
+
}
|
|
8604
|
+
return line;
|
|
8605
|
+
}
|
|
8606
|
+
|
|
8607
|
+
// Today's OpenAI image spend against the daily cap (the Check Usage card,
|
|
8608
|
+
// the banner, a paired relay's check before create_session); with
|
|
8609
|
+
// `?conversationId=` also that chat's.
|
|
8610
|
+
app.get(OPENAI_IMAGE_USAGE_PATH, auth, (req, res) => {
|
|
8611
|
+
const conversationId = String(req.query?.conversationId || '').trim().slice(0, 200) || null;
|
|
8612
|
+
try {
|
|
8613
|
+
return res.json(imageSpend.buildState({ conversationId }));
|
|
8614
|
+
} catch (error) {
|
|
8615
|
+
return res.status(500).json({ error: `Image spend could not be read: ${error?.message || error}` });
|
|
8616
|
+
}
|
|
8617
|
+
});
|
|
8618
|
+
|
|
8619
|
+
/**
|
|
8620
|
+
* An image attempt the recovery found cut off while its request ran
|
|
8621
|
+
* ('started', now 'uncertain'): it may have been billed, so its estimate
|
|
8622
|
+
* is booked as image:<attemptId>:recovered, unless a try of that attempt
|
|
8623
|
+
* was booked already.
|
|
8624
|
+
*/
|
|
8625
|
+
function bookRecoveredImageAttempt(operation, attempt) {
|
|
8626
|
+
if (!operation || !attempt?.id) return;
|
|
8627
|
+
let parameters = {};
|
|
8628
|
+
try {
|
|
8629
|
+
parameters = JSON.parse(operation.parameters_json || '{}') || {};
|
|
8630
|
+
} catch {}
|
|
8631
|
+
let inputImages = operation.parent_node_id ? 1 : 0;
|
|
8632
|
+
if (!operation.parent_node_id) {
|
|
8633
|
+
try {
|
|
8634
|
+
inputImages = (stmts.listAssets?.all?.(operation.id) || []).length;
|
|
8635
|
+
} catch {}
|
|
8636
|
+
}
|
|
8637
|
+
try {
|
|
8638
|
+
imageSpend.bookRecovered({
|
|
8639
|
+
conversationId: operation.conversation_id,
|
|
8640
|
+
turnId: operation.queue_message_id,
|
|
8641
|
+
attemptId: attempt.id,
|
|
8642
|
+
model: operation.selected_image_model,
|
|
8643
|
+
startedAt: attempt.started_at,
|
|
8644
|
+
request: { size: parameters.size, quality: parameters.quality, count: parameters.count ?? 1, inputImages },
|
|
8645
|
+
});
|
|
8646
|
+
} catch (error) {
|
|
8647
|
+
console.warn(`[${ts()}] IMAGE_OP the estimate of attempt=${attempt.id} was not booked: ${error?.message || error}`);
|
|
8648
|
+
}
|
|
8649
|
+
}
|
|
8650
|
+
|
|
8651
|
+
/**
|
|
8652
|
+
* Why a queue row may not go back to pending (/api/requeue): `{ running:
|
|
8653
|
+
* true }` while its image request runs in this process, `{ failure }`
|
|
8654
|
+
* (a terminalError) once its request went to OpenAI, or null for a row
|
|
8655
|
+
* whose request was never sent (requeueing it bills nothing).
|
|
8656
|
+
*/
|
|
8657
|
+
function imageTurnRequeueStop(queueRow) {
|
|
8658
|
+
const operationId = String(queueRow?.image_operation_id || '').trim();
|
|
8659
|
+
if (operationId) {
|
|
8660
|
+
if (executingImageOperations.has(operationId)) return { running: true };
|
|
8661
|
+
const attempt = stmts.getLatestAttempt?.get?.(operationId) || null;
|
|
8662
|
+
if (!attempt) return null;
|
|
8663
|
+
if (attempt.status === 'started') {
|
|
8664
|
+
stmts.finishAttempt.run({
|
|
8665
|
+
id: attempt.id,
|
|
8666
|
+
status: 'uncertain',
|
|
8667
|
+
providerRequestId: null,
|
|
8668
|
+
providerConversationId: null,
|
|
8669
|
+
providerResponseId: null,
|
|
8670
|
+
httpStatus: null,
|
|
8671
|
+
errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
8672
|
+
errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
8673
|
+
completedAt: new Date().toISOString(),
|
|
8674
|
+
});
|
|
8675
|
+
bookRecoveredImageAttempt(stmts.getOperation?.get?.(operationId), attempt);
|
|
8676
|
+
return { failure: { code: 'IMAGE_ATTEMPT_INTERRUPTED', message: OPENAI_IMAGE_INTERRUPTED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
|
|
8677
|
+
}
|
|
8678
|
+
return {
|
|
8679
|
+
failure: {
|
|
8680
|
+
code: String(attempt.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED',
|
|
8681
|
+
message: String(attempt.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE,
|
|
8682
|
+
guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
|
|
8683
|
+
},
|
|
8684
|
+
};
|
|
8685
|
+
}
|
|
8686
|
+
const dispatched = dispatchedImageMessages.get(String(queueRow?.id || ''));
|
|
8687
|
+
if (!dispatched) return null;
|
|
8688
|
+
if (dispatched.running) return { running: true };
|
|
8689
|
+
return { failure: { code: 'IMAGE_ATTEMPT_REPEATED', message: OPENAI_IMAGE_REPEATED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
|
|
8690
|
+
}
|
|
8691
|
+
|
|
7655
8692
|
// Called by the relay's own worker on this machine; nothing that came in
|
|
7656
8693
|
// through a proxy or the tunnel may start a provider image request.
|
|
8694
|
+
// The request runs in this process (runOpenAIImageRequest): its timeout
|
|
8695
|
+
// and its one retry are the only ones. A failure ends the row with a note,
|
|
8696
|
+
// never with a requeue: a requeued image turn was billed again.
|
|
7657
8697
|
app.post('/api/image-operations/:operationId/execute', auth, async (req, res) => {
|
|
7658
8698
|
if (!isTrulyLocalRequest(req)) {
|
|
7659
8699
|
return res.status(403).json({ error: 'Image operation execution is reachable from this machine only' });
|
|
@@ -7673,6 +8713,11 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7673
8713
|
if (!queueRow || queueRow.image_operation_id !== operationId) {
|
|
7674
8714
|
return res.status(409).json({ error: 'Image operation queue linkage is unavailable' });
|
|
7675
8715
|
}
|
|
8716
|
+
// Delivered again while its request still runs here (a worker that
|
|
8717
|
+
// died and came back): that run finishes the row.
|
|
8718
|
+
if (executingImageOperations.has(operationId)) {
|
|
8719
|
+
return res.status(202).json({ ok: true, outcome: 'executing', operationId });
|
|
8720
|
+
}
|
|
7676
8721
|
|
|
7677
8722
|
const now = new Date().toISOString();
|
|
7678
8723
|
const selfHeaders = {
|
|
@@ -7680,6 +8725,23 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7680
8725
|
'Content-Type': 'application/json',
|
|
7681
8726
|
};
|
|
7682
8727
|
const selfBaseUrl = `http://127.0.0.1:${config.port}`;
|
|
8728
|
+
const publishFailure = async (terminalError) => {
|
|
8729
|
+
// `terminalError` is the field /api/response reads; under any other
|
|
8730
|
+
// name the failure was refused as an empty reply and the row stayed
|
|
8731
|
+
// processing until the relay recovered and re-ran it.
|
|
8732
|
+
const response = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
8733
|
+
method: 'POST',
|
|
8734
|
+
headers: selfHeaders,
|
|
8735
|
+
body: JSON.stringify({
|
|
8736
|
+
messageId: operation.queue_message_id,
|
|
8737
|
+
conversationId: operation.conversation_id,
|
|
8738
|
+
terminalError,
|
|
8739
|
+
}),
|
|
8740
|
+
}).catch(() => null);
|
|
8741
|
+
if (!response?.ok) {
|
|
8742
|
+
console.warn(`[${ts()}] IMAGE_OP failed to publish the failure of operation=${operationId}`);
|
|
8743
|
+
}
|
|
8744
|
+
};
|
|
7683
8745
|
const acquireAttempt = db.transaction(() => {
|
|
7684
8746
|
const current = stmts.getLatestAttempt.get(operationId);
|
|
7685
8747
|
if (current?.status === 'started') {
|
|
@@ -7697,15 +8759,19 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7697
8759
|
providerResponseId: null,
|
|
7698
8760
|
httpStatus: null,
|
|
7699
8761
|
errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
7700
|
-
errorMessage:
|
|
8762
|
+
errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
7701
8763
|
completedAt: now,
|
|
7702
8764
|
});
|
|
7703
8765
|
return { acquired: false, attempt: current, stale: true };
|
|
7704
8766
|
}
|
|
8767
|
+
// An attempt that ended: its note was published, or publishing it
|
|
8768
|
+
// failed and the row came back. Either way the request is not sent
|
|
8769
|
+
// again.
|
|
8770
|
+
if (current) return { acquired: false, attempt: current, finished: true };
|
|
7705
8771
|
const attempt = {
|
|
7706
8772
|
id: `img_attempt_${uuidv4()}`,
|
|
7707
8773
|
operationId,
|
|
7708
|
-
attemptNumber:
|
|
8774
|
+
attemptNumber: 1,
|
|
7709
8775
|
provider: operation.provider,
|
|
7710
8776
|
capabilitySnapshotJson: JSON.stringify({
|
|
7711
8777
|
executionMode: operation.execution_mode,
|
|
@@ -7718,339 +8784,250 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7718
8784
|
});
|
|
7719
8785
|
const acquired = acquireAttempt();
|
|
7720
8786
|
if (acquired.stale) {
|
|
7721
|
-
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
// name the failure was refused as an empty reply and the row stayed
|
|
7729
|
-
// processing until the relay recovered and re-ran it.
|
|
7730
|
-
terminalError: {
|
|
7731
|
-
code: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
7732
|
-
message: 'Image generation may have completed before the relay was interrupted. It was not retried to avoid duplicate images.',
|
|
7733
|
-
retryable: false,
|
|
7734
|
-
},
|
|
7735
|
-
}),
|
|
7736
|
-
}).catch(() => null);
|
|
8787
|
+
bookRecoveredImageAttempt(operation, acquired.attempt);
|
|
8788
|
+
await publishFailure({
|
|
8789
|
+
code: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
8790
|
+
message: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
8791
|
+
guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
|
|
8792
|
+
retryable: false,
|
|
8793
|
+
});
|
|
7737
8794
|
return res.status(409).json({
|
|
7738
8795
|
ok: false,
|
|
7739
8796
|
outcome: 'uncertain',
|
|
7740
8797
|
operationId,
|
|
7741
|
-
|
|
8798
|
+
code: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
8799
|
+
error: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
8800
|
+
});
|
|
8801
|
+
}
|
|
8802
|
+
if (acquired.finished) {
|
|
8803
|
+
const earlier = acquired.attempt;
|
|
8804
|
+
const message = String(earlier.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE;
|
|
8805
|
+
const code = String(earlier.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED';
|
|
8806
|
+
await publishFailure({ code, message, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE, retryable: false });
|
|
8807
|
+
return res.status(409).json({
|
|
8808
|
+
ok: false,
|
|
8809
|
+
outcome: ['failed_terminal', 'failed_retryable'].includes(earlier.status) ? 'terminal' : 'uncertain',
|
|
8810
|
+
operationId,
|
|
8811
|
+
code,
|
|
8812
|
+
error: message,
|
|
7742
8813
|
});
|
|
7743
8814
|
}
|
|
7744
8815
|
if (!acquired.acquired) {
|
|
7745
8816
|
return res.status(202).json({ ok: true, outcome: 'executing', operationId });
|
|
7746
8817
|
}
|
|
7747
8818
|
|
|
7748
|
-
|
|
7749
|
-
|
|
7750
|
-
const
|
|
7751
|
-
const parentMessage = parentNode
|
|
7752
|
-
? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
|
|
7753
|
-
: null;
|
|
7754
|
-
const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
|
|
7755
|
-
String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
|
|
7756
|
-
));
|
|
7757
|
-
const generatedImage = parentAttachment?.generatedImage;
|
|
7758
|
-
const parentPath = resolveGeneratedImageReadPath({
|
|
7759
|
-
resolveSessionStateRoot,
|
|
7760
|
-
sessionId: generatedImage?.sessionId,
|
|
7761
|
-
relativePath: generatedImage?.relativePath,
|
|
7762
|
-
});
|
|
7763
|
-
if (!parentPath) {
|
|
8819
|
+
executingImageOperations.add(operationId);
|
|
8820
|
+
try {
|
|
8821
|
+
const finishAttempt = (result) => {
|
|
7764
8822
|
stmts.finishAttempt.run({
|
|
7765
8823
|
id: acquired.attempt.id,
|
|
7766
|
-
|
|
8824
|
+
// A request that never reached OpenAI (unreachable, or stopped by
|
|
8825
|
+
// the daily cap before it was sent) closes as retryable: nothing
|
|
8826
|
+
// was billed, and the recovery books nothing for it.
|
|
8827
|
+
status: result.outcome === 'uncertain'
|
|
8828
|
+
? 'uncertain'
|
|
8829
|
+
: ((result.code === 'IMAGE_PROVIDER_UNREACHABLE' || (result.code === OPENAI_IMAGE_DAILY_CAP_REACHED_CODE && !result.sent))
|
|
8830
|
+
? 'failed_retryable'
|
|
8831
|
+
: 'failed_terminal'),
|
|
7767
8832
|
providerRequestId: null,
|
|
7768
8833
|
providerConversationId: null,
|
|
7769
8834
|
providerResponseId: null,
|
|
7770
|
-
httpStatus:
|
|
7771
|
-
errorCode:
|
|
7772
|
-
errorMessage:
|
|
8835
|
+
httpStatus: result.upstreamStatus || null,
|
|
8836
|
+
errorCode: result.code,
|
|
8837
|
+
errorMessage: String(result.error || '').slice(0, 500),
|
|
7773
8838
|
completedAt: new Date().toISOString(),
|
|
7774
8839
|
});
|
|
8840
|
+
};
|
|
8841
|
+
const fail = async (result) => {
|
|
8842
|
+
finishAttempt(result);
|
|
8843
|
+
await publishFailure(openAIImageTerminalError(result));
|
|
7775
8844
|
return res.status(409).json({
|
|
7776
8845
|
ok: false,
|
|
7777
|
-
outcome:
|
|
7778
|
-
|
|
7779
|
-
|
|
8846
|
+
outcome: result.outcome,
|
|
8847
|
+
operationId,
|
|
8848
|
+
code: result.code,
|
|
8849
|
+
error: result.error,
|
|
8850
|
+
...(result.usage ? { usage: result.usage } : {}),
|
|
7780
8851
|
});
|
|
7781
|
-
}
|
|
7782
|
-
|
|
7783
|
-
|
|
7784
|
-
|
|
7785
|
-
|
|
7786
|
-
|
|
7787
|
-
|
|
7788
|
-
|
|
8852
|
+
};
|
|
8853
|
+
|
|
8854
|
+
const requestAttachments = [];
|
|
8855
|
+
if (operation.parent_node_id) {
|
|
8856
|
+
const parentNode = stmts.getNode.get(operation.parent_node_id);
|
|
8857
|
+
const parentMessage = parentNode
|
|
8858
|
+
? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
|
|
8859
|
+
: null;
|
|
8860
|
+
const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
|
|
8861
|
+
String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
|
|
8862
|
+
));
|
|
8863
|
+
const generatedImage = parentAttachment?.generatedImage;
|
|
8864
|
+
const parentPath = resolveGeneratedImageReadPath({
|
|
8865
|
+
resolveSessionStateRoot,
|
|
8866
|
+
sessionId: generatedImage?.sessionId,
|
|
8867
|
+
relativePath: generatedImage?.relativePath,
|
|
8868
|
+
});
|
|
8869
|
+
if (!parentPath) {
|
|
8870
|
+
return await fail({
|
|
8871
|
+
...openAIImageRequestRefusal(400, 'IMAGE_PARENT_UNAVAILABLE', 'The parent generated image is unavailable'),
|
|
8872
|
+
upstreamStatus: 400,
|
|
8873
|
+
});
|
|
8874
|
+
}
|
|
7789
8875
|
requestAttachments.push({
|
|
7790
|
-
name:
|
|
7791
|
-
type:
|
|
7792
|
-
path:
|
|
8876
|
+
name: parentAttachment.name,
|
|
8877
|
+
type: parentAttachment.type,
|
|
8878
|
+
path: parentPath,
|
|
7793
8879
|
});
|
|
8880
|
+
} else {
|
|
8881
|
+
// In the order they were attached: the first is the base image.
|
|
8882
|
+
for (const asset of stmts.listAssets.all(operationId)) {
|
|
8883
|
+
requestAttachments.push({
|
|
8884
|
+
name: asset.original_name || 'reference-image',
|
|
8885
|
+
type: asset.media_type,
|
|
8886
|
+
path: uploadPathForSha(asset.content_sha256),
|
|
8887
|
+
});
|
|
8888
|
+
}
|
|
7794
8889
|
}
|
|
7795
|
-
}
|
|
7796
|
-
|
|
7797
|
-
let parameters = {};
|
|
7798
|
-
try {
|
|
7799
|
-
parameters = JSON.parse(operation.parameters_json || '{}');
|
|
7800
|
-
} catch {}
|
|
7801
|
-
const generationBody = {
|
|
7802
|
-
messageId: operation.queue_message_id,
|
|
7803
|
-
conversationId: operation.conversation_id,
|
|
7804
|
-
model: operation.selected_image_model,
|
|
7805
|
-
prompt: operation.prompt,
|
|
7806
|
-
n: Number.isInteger(Number(parameters.count)) ? Number(parameters.count) : 1,
|
|
7807
|
-
attachments: requestAttachments,
|
|
7808
|
-
};
|
|
7809
|
-
if (parameters.size === 'auto' || /^\d{2,5}x\d{2,5}$/i.test(String(parameters.size || ''))) {
|
|
7810
|
-
generationBody.size = parameters.size;
|
|
7811
|
-
}
|
|
7812
|
-
if (/^(auto|standard|hd|low|medium|high)$/i.test(String(parameters.quality || ''))) {
|
|
7813
|
-
generationBody.quality = parameters.quality;
|
|
7814
|
-
}
|
|
7815
8890
|
|
|
7816
|
-
|
|
7817
|
-
|
|
7818
|
-
|
|
7819
|
-
|
|
7820
|
-
|
|
7821
|
-
|
|
7822
|
-
|
|
7823
|
-
|
|
7824
|
-
|
|
7825
|
-
const error = new Error(generatedPayload?.error || `OpenAI image request failed (${generatedResponse.status})`);
|
|
7826
|
-
error.status = generatedResponse.status;
|
|
7827
|
-
throw error;
|
|
7828
|
-
}
|
|
7829
|
-
const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
7830
|
-
method: 'POST',
|
|
7831
|
-
headers: selfHeaders,
|
|
7832
|
-
body: JSON.stringify({
|
|
8891
|
+
// {size, quality, count, output_format, background}; any may be
|
|
8892
|
+
// missing, count is 1 then.
|
|
8893
|
+
let parameters = {};
|
|
8894
|
+
try {
|
|
8895
|
+
parameters = JSON.parse(operation.parameters_json || '{}') || {};
|
|
8896
|
+
} catch {}
|
|
8897
|
+
let requestBody;
|
|
8898
|
+
try {
|
|
8899
|
+
requestBody = normalizeOpenAIImageGenerateRequestBody({
|
|
7833
8900
|
messageId: operation.queue_message_id,
|
|
7834
8901
|
conversationId: operation.conversation_id,
|
|
7835
|
-
|
|
7836
|
-
|
|
7837
|
-
|
|
7838
|
-
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
8902
|
+
model: operation.selected_image_model,
|
|
8903
|
+
prompt: operation.prompt,
|
|
8904
|
+
n: parameters.count ?? 1,
|
|
8905
|
+
size: parameters.size,
|
|
8906
|
+
quality: parameters.quality,
|
|
8907
|
+
output_format: parameters.output_format,
|
|
8908
|
+
background: parameters.background,
|
|
8909
|
+
attachments: requestAttachments,
|
|
8910
|
+
});
|
|
8911
|
+
} catch (error) {
|
|
8912
|
+
return await fail(openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', String(error?.message || 'Invalid image request')));
|
|
7845
8913
|
}
|
|
7846
|
-
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
7850
|
-
|
|
7851
|
-
? generatedPayload.generatedImages.length
|
|
7852
|
-
: 0,
|
|
7853
|
-
});
|
|
7854
|
-
} catch (error) {
|
|
7855
|
-
const status = Number(error?.status || 0);
|
|
7856
|
-
// `relayProcessing`: the provider answered 2xx and the relay could not
|
|
7857
|
-
// process the reply. The generation is billed, a retry pays again and
|
|
7858
|
-
// fails the same way, so the row fails like a refusal.
|
|
7859
|
-
const relayProcessing = generatedPayload?.relayProcessing === true;
|
|
7860
|
-
const terminal = (status >= 400 && status < 500) || relayProcessing;
|
|
7861
|
-
const uncertain = !!generatedPayload?.generatedImages;
|
|
7862
|
-
const outcome = terminal ? 'terminal' : (uncertain ? 'uncertain' : 'retryable');
|
|
7863
|
-
const errorMessage = relayProcessing
|
|
7864
|
-
? `The relay could not process the image OpenAI returned (${error?.message || 'unknown error'}). The generation was billed; the message was not retried.`
|
|
7865
|
-
: String(error?.message || 'Image operation failed');
|
|
7866
|
-
stmts.finishAttempt.run({
|
|
7867
|
-
id: acquired.attempt.id,
|
|
7868
|
-
status: terminal ? 'failed_terminal' : (uncertain ? 'uncertain' : 'failed_retryable'),
|
|
7869
|
-
providerRequestId: null,
|
|
7870
|
-
providerConversationId: null,
|
|
7871
|
-
providerResponseId: null,
|
|
7872
|
-
httpStatus: status || null,
|
|
7873
|
-
errorCode: terminal
|
|
7874
|
-
? (relayProcessing ? 'IMAGE_REPLY_UNPROCESSED' : 'IMAGE_PROVIDER_REJECTED')
|
|
7875
|
-
: (uncertain ? 'IMAGE_FINALIZATION_UNCERTAIN' : 'IMAGE_PROVIDER_UNAVAILABLE'),
|
|
7876
|
-
errorMessage: errorMessage.slice(0, 500),
|
|
7877
|
-
completedAt: new Date().toISOString(),
|
|
8914
|
+
|
|
8915
|
+
const result = await runOpenAIImageRequest(requestBody, {
|
|
8916
|
+
idempotencyKey: String(operation.idempotency_key || operation.id),
|
|
8917
|
+
preview: { conversationId: operation.conversation_id, messageId: operation.queue_message_id },
|
|
8918
|
+
scopeId: acquired.attempt.id,
|
|
7878
8919
|
});
|
|
7879
|
-
if (
|
|
7880
|
-
|
|
7881
|
-
|
|
7882
|
-
|
|
7883
|
-
|
|
7884
|
-
}).catch(() => null);
|
|
7885
|
-
if (!requeueResponse?.ok) {
|
|
7886
|
-
console.warn(`[${ts()}] IMAGE_OP failed to requeue retryable operation=${operationId}`);
|
|
7887
|
-
}
|
|
7888
|
-
} else {
|
|
7889
|
-
const failure = {
|
|
7890
|
-
code: outcome === 'uncertain'
|
|
7891
|
-
? 'IMAGE_FINALIZATION_UNCERTAIN'
|
|
7892
|
-
: (relayProcessing ? 'IMAGE_REPLY_UNPROCESSED' : 'IMAGE_PROVIDER_REJECTED'),
|
|
7893
|
-
message: errorMessage.slice(0, 500),
|
|
7894
|
-
retryable: false,
|
|
7895
|
-
};
|
|
7896
|
-
const failResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
8920
|
+
if (!result.ok) return await fail(result);
|
|
8921
|
+
|
|
8922
|
+
let finalizeError = null;
|
|
8923
|
+
try {
|
|
8924
|
+
const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
7897
8925
|
method: 'POST',
|
|
7898
8926
|
headers: selfHeaders,
|
|
7899
8927
|
body: JSON.stringify({
|
|
7900
8928
|
messageId: operation.queue_message_id,
|
|
7901
8929
|
conversationId: operation.conversation_id,
|
|
7902
|
-
|
|
8930
|
+
text: '',
|
|
8931
|
+
generatedImages: result.generatedImages,
|
|
8932
|
+
model: result.model || operation.selected_image_model,
|
|
8933
|
+
// A first try that may have been billed too: the reply says so.
|
|
8934
|
+
...(result.note ? { imageNote: result.note } : {}),
|
|
7903
8935
|
}),
|
|
7904
|
-
})
|
|
7905
|
-
|
|
7906
|
-
|
|
7907
|
-
|
|
8936
|
+
});
|
|
8937
|
+
const finalized = await finalizeResponse.json().catch(() => ({}));
|
|
8938
|
+
if (!finalizeResponse.ok) finalizeError = finalized?.error || `Image finalization failed (${finalizeResponse.status})`;
|
|
8939
|
+
} catch (error) {
|
|
8940
|
+
finalizeError = error?.message || 'Image finalization failed';
|
|
7908
8941
|
}
|
|
7909
|
-
|
|
7910
|
-
|
|
7911
|
-
|
|
8942
|
+
if (finalizeError) {
|
|
8943
|
+
return await fail({
|
|
8944
|
+
ok: false,
|
|
8945
|
+
outcome: 'uncertain',
|
|
8946
|
+
code: 'IMAGE_FINALIZATION_UNCERTAIN',
|
|
8947
|
+
error: `The images OpenAI returned could not be saved (${finalizeError}). The generation was billed; the message was not retried.`,
|
|
8948
|
+
upstreamStatus: 200,
|
|
8949
|
+
usage: result.usage,
|
|
8950
|
+
});
|
|
8951
|
+
}
|
|
8952
|
+
return res.json({
|
|
8953
|
+
ok: true,
|
|
8954
|
+
outcome: 'completed',
|
|
7912
8955
|
operationId,
|
|
7913
|
-
|
|
8956
|
+
generatedImageCount: result.generatedImages.length,
|
|
8957
|
+
usage: result.usage,
|
|
8958
|
+
...(result.note ? { note: result.note } : {}),
|
|
7914
8959
|
});
|
|
8960
|
+
} finally {
|
|
8961
|
+
executingImageOperations.delete(operationId);
|
|
7915
8962
|
}
|
|
7916
8963
|
});
|
|
7917
8964
|
|
|
8965
|
+
// The relay's own workers call this on this machine for an image turn
|
|
8966
|
+
// without an operation (image continuity off). Like the execute route it
|
|
8967
|
+
// is local only: through the tunnel, or with a paired relay's token, it
|
|
8968
|
+
// would spend the owner's money outside the queue. One request per queue
|
|
8969
|
+
// message: a second call for the same message is refused.
|
|
7918
8970
|
app.post('/api/openai/images/generate', auth, async (req, res) => {
|
|
7919
8971
|
touchCli();
|
|
7920
|
-
|
|
7921
|
-
|
|
7922
|
-
return res.status(503).json({ error: 'OpenAI BYOK is not configured' });
|
|
8972
|
+
if (!isTrulyLocalRequest(req)) {
|
|
8973
|
+
return res.status(403).json({ error: 'OpenAI image requests are reachable from this machine only' });
|
|
7923
8974
|
}
|
|
7924
8975
|
|
|
7925
8976
|
let requestBody;
|
|
7926
8977
|
try {
|
|
7927
|
-
requestBody = normalizeOpenAIImageGenerateRequestBody(req.body
|
|
7928
|
-
maxImages: MAX_UPLOAD_ATTACHMENTS,
|
|
7929
|
-
});
|
|
8978
|
+
requestBody = normalizeOpenAIImageGenerateRequestBody(req.body);
|
|
7930
8979
|
} catch (error) {
|
|
7931
|
-
return res.status(400).json({
|
|
7932
|
-
|
|
7933
|
-
|
|
7934
|
-
|
|
7935
|
-
|
|
7936
|
-
|
|
7937
|
-
let editAttachment = null;
|
|
7938
|
-
if (Array.isArray(requestBody.attachments) && requestBody.attachments.length > 0) {
|
|
7939
|
-
try {
|
|
7940
|
-
editAttachment = resolveOpenAIImageEditAttachment(requestBody.attachments, {
|
|
7941
|
-
maxImageBytes: MAX_UPLOAD_BYTES,
|
|
7942
|
-
allowedRootPath: uploadsDir,
|
|
7943
|
-
allowedBlobRootPaths: uploadReadDirs.slice(1),
|
|
7944
|
-
allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
|
|
7945
|
-
});
|
|
7946
|
-
} catch (error) {
|
|
7947
|
-
return res.status(400).json({ error: error?.message || 'Invalid image attachment' });
|
|
7948
|
-
}
|
|
7949
|
-
}
|
|
7950
|
-
|
|
7951
|
-
const options = {
|
|
7952
|
-
model: requestBody.model,
|
|
7953
|
-
prompt: requestBody.prompt,
|
|
7954
|
-
};
|
|
7955
|
-
const supportsResponseFormat = String(requestBody.model || '')
|
|
7956
|
-
.trim()
|
|
7957
|
-
.toLowerCase()
|
|
7958
|
-
.replace(/^openai\//, '')
|
|
7959
|
-
.startsWith('dall-e-');
|
|
7960
|
-
if (supportsResponseFormat) options.response_format = 'b64_json';
|
|
7961
|
-
if (requestBody.n !== undefined) options.n = requestBody.n;
|
|
7962
|
-
if (requestBody.size) options.size = requestBody.size;
|
|
7963
|
-
if (requestBody.quality) options.quality = requestBody.quality;
|
|
7964
|
-
|
|
7965
|
-
const endpoint = editAttachment ? '/images/edits' : '/images/generations';
|
|
7966
|
-
let response;
|
|
7967
|
-
try {
|
|
7968
|
-
if (editAttachment) {
|
|
7969
|
-
const form = new FormData();
|
|
7970
|
-
form.set('model', options.model);
|
|
7971
|
-
form.set('prompt', options.prompt);
|
|
7972
|
-
if (options.response_format) form.set('response_format', String(options.response_format));
|
|
7973
|
-
if (options.n !== undefined) form.set('n', String(options.n));
|
|
7974
|
-
if (options.size) form.set('size', String(options.size));
|
|
7975
|
-
if (options.quality) form.set('quality', String(options.quality));
|
|
7976
|
-
form.set(
|
|
7977
|
-
'image',
|
|
7978
|
-
new Blob([editAttachment.bytes], { type: editAttachment.mimeType }),
|
|
7979
|
-
editAttachment.name,
|
|
7980
|
-
);
|
|
7981
|
-
response = await fetch(`${baseUrl}${endpoint}`, {
|
|
7982
|
-
method: 'POST',
|
|
7983
|
-
headers: {
|
|
7984
|
-
Authorization: `Bearer ${settings.apiKey}`,
|
|
7985
|
-
},
|
|
7986
|
-
body: form,
|
|
7987
|
-
});
|
|
7988
|
-
} else {
|
|
7989
|
-
response = await fetch(`${baseUrl}${endpoint}`, {
|
|
7990
|
-
method: 'POST',
|
|
7991
|
-
headers: {
|
|
7992
|
-
Authorization: `Bearer ${settings.apiKey}`,
|
|
7993
|
-
'Content-Type': 'application/json',
|
|
7994
|
-
},
|
|
7995
|
-
body: JSON.stringify(options),
|
|
7996
|
-
});
|
|
7997
|
-
}
|
|
7998
|
-
} catch (error) {
|
|
7999
|
-
return res.status(502).json({ error: `OpenAI image API request failed: ${error?.message || 'request error'}` });
|
|
8000
|
-
}
|
|
8001
|
-
|
|
8002
|
-
const rawText = String(await response.text().catch(() => '')).trim();
|
|
8003
|
-
let payload = {};
|
|
8004
|
-
if (rawText) {
|
|
8005
|
-
try {
|
|
8006
|
-
payload = JSON.parse(rawText);
|
|
8007
|
-
} catch {
|
|
8008
|
-
// After a 2xx the generation is billed; the callers fail the row
|
|
8009
|
-
// instead of paying for it again. An error status with a body that
|
|
8010
|
-
// is not JSON stays a provider failure.
|
|
8011
|
-
return res.status(502).json({
|
|
8012
|
-
error: 'OpenAI image API returned invalid JSON',
|
|
8013
|
-
...(response.ok ? { relayProcessing: true } : {}),
|
|
8014
|
-
});
|
|
8015
|
-
}
|
|
8980
|
+
return res.status(400).json({
|
|
8981
|
+
error: error?.message || 'Invalid OpenAI image request',
|
|
8982
|
+
code: 'IMAGE_REQUEST_INVALID',
|
|
8983
|
+
outcome: 'terminal',
|
|
8984
|
+
});
|
|
8016
8985
|
}
|
|
8017
|
-
|
|
8018
|
-
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
|
|
8022
|
-
? upstreamStatus
|
|
8023
|
-
: 502;
|
|
8024
|
-
return res.status(relayStatus).json({
|
|
8025
|
-
error: `OpenAI image API failed (${response.status})${detail ? `: ${detail}` : ''}`,
|
|
8026
|
-
upstreamStatus: Number.isFinite(upstreamStatus) ? upstreamStatus : null,
|
|
8986
|
+
if (dispatchedImageMessages.has(requestBody.messageId)) {
|
|
8987
|
+
return res.status(409).json({
|
|
8988
|
+
error: OPENAI_IMAGE_REPEATED_NOTE,
|
|
8989
|
+
code: 'IMAGE_ATTEMPT_REPEATED',
|
|
8990
|
+
outcome: 'uncertain',
|
|
8027
8991
|
});
|
|
8028
8992
|
}
|
|
8029
8993
|
|
|
8030
|
-
|
|
8994
|
+
const dispatched = noteDispatchedImageMessage(requestBody.messageId);
|
|
8995
|
+
// One run of the message: the claim's attempt id (a restart claims the
|
|
8996
|
+
// row anew), so its tries never merge into an earlier run's rows.
|
|
8997
|
+
// Known gap: this path (image continuity off) keeps no attempt row, so a
|
|
8998
|
+
// try a relay restart cuts off is booked nowhere; the operation path's
|
|
8999
|
+
// recovery books such a try as an estimate. docs/providers.md says so.
|
|
9000
|
+
let claimId = '';
|
|
8031
9001
|
try {
|
|
8032
|
-
|
|
8033
|
-
|
|
8034
|
-
|
|
8035
|
-
|
|
8036
|
-
|
|
8037
|
-
|
|
8038
|
-
|
|
8039
|
-
|
|
8040
|
-
|
|
8041
|
-
|
|
8042
|
-
|
|
8043
|
-
|
|
8044
|
-
|
|
8045
|
-
|
|
9002
|
+
claimId = String(stmts.findQById?.get?.(requestBody.messageId)?.attempt_id || '').trim();
|
|
9003
|
+
} catch {}
|
|
9004
|
+
const result = await runOpenAIImageRequest(requestBody, {
|
|
9005
|
+
idempotencyKey: `oar-image-${requestBody.messageId}`,
|
|
9006
|
+
preview: { conversationId: requestBody.conversationId, messageId: requestBody.messageId },
|
|
9007
|
+
scopeId: claimId ? `claim_${claimId}` : '',
|
|
9008
|
+
});
|
|
9009
|
+
// A request that never reached OpenAI may be sent again.
|
|
9010
|
+
if (result.ok || result.sent) dispatched.running = false;
|
|
9011
|
+
else dispatchedImageMessages.delete(requestBody.messageId);
|
|
9012
|
+
|
|
9013
|
+
if (!result.ok) {
|
|
9014
|
+
return res.status(result.status).json({
|
|
9015
|
+
error: result.error,
|
|
9016
|
+
code: result.code,
|
|
9017
|
+
outcome: result.outcome,
|
|
9018
|
+
upstreamStatus: result.upstreamStatus,
|
|
9019
|
+
...(result.detail ? { detail: result.detail } : {}),
|
|
9020
|
+
...(result.relayProcessing ? { relayProcessing: true } : {}),
|
|
8046
9021
|
});
|
|
8047
9022
|
}
|
|
8048
|
-
|
|
8049
9023
|
return res.json({
|
|
8050
9024
|
ok: true,
|
|
8051
|
-
generatedImages,
|
|
8052
|
-
model:
|
|
8053
|
-
endpoint,
|
|
9025
|
+
generatedImages: result.generatedImages,
|
|
9026
|
+
model: result.model,
|
|
9027
|
+
endpoint: result.endpoint,
|
|
9028
|
+
usage: result.usage,
|
|
9029
|
+
// The worker hands it to /api/response as `imageNote`.
|
|
9030
|
+
...(result.note ? { note: result.note } : {}),
|
|
8054
9031
|
});
|
|
8055
9032
|
});
|
|
8056
9033
|
|
|
@@ -8149,12 +9126,16 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8149
9126
|
console.log(`[${ts()}] FENCE ${messageId?.slice(0,8)} legacy unfenced response write`);
|
|
8150
9127
|
}
|
|
8151
9128
|
|
|
9129
|
+
// The partial previews of an image turn go once its reply or its
|
|
9130
|
+
// failure note is stored (or was already).
|
|
8152
9131
|
if (q && q.status === 'done') {
|
|
8153
9132
|
console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_done`);
|
|
9133
|
+
imagePreviews.finish(messageId, 'done');
|
|
8154
9134
|
return res.json({ ok: true, ignored: 'already_done' });
|
|
8155
9135
|
}
|
|
8156
9136
|
if (q && q.status === 'failed') {
|
|
8157
9137
|
console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_failed`);
|
|
9138
|
+
imagePreviews.finish(messageId, 'failed');
|
|
8158
9139
|
return res.json({ ok: true, ignored: 'already_failed' });
|
|
8159
9140
|
}
|
|
8160
9141
|
|
|
@@ -8259,6 +9240,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8259
9240
|
+ `${terminalFailure.functionCallId ? ` call=${terminalFailure.functionCallId}` : ''}`
|
|
8260
9241
|
+ `${terminalFailure.requestId ? ` req=${terminalFailure.requestId}` : ''}`,
|
|
8261
9242
|
);
|
|
9243
|
+
imagePreviews.finish(messageId, 'failed');
|
|
8262
9244
|
return res.json({ ok: true, terminal: true, code: terminalFailure.stableCode });
|
|
8263
9245
|
}
|
|
8264
9246
|
|
|
@@ -8266,7 +9248,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8266
9248
|
const hasPendingQuestion = !!stmts.findPendingQuestionByMessage?.get(messageId);
|
|
8267
9249
|
const hasRecentlyAnsweredQuestion = !!stmts.findRecentlyAnsweredQuestionByMessage?.get(messageId, holdCutoff);
|
|
8268
9250
|
const conversation = stmts.getConvAnyStatus?.get?.(targetConversationId) || null;
|
|
8269
|
-
|
|
9251
|
+
let resolvedText = await resolveRelayResponseText({
|
|
8270
9252
|
text: trimmedText,
|
|
8271
9253
|
conversation,
|
|
8272
9254
|
queueTimestamp: q?.timestamp || null,
|
|
@@ -8349,6 +9331,29 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8349
9331
|
} catch (error) {
|
|
8350
9332
|
return res.status(500).json({ error: error?.message || 'Failed to persist generated images' });
|
|
8351
9333
|
}
|
|
9334
|
+
// An image turn whose model wrote no text: the reply line says what came
|
|
9335
|
+
// back ("1 image · 1536x1024 · gpt-image-2"), so the chat and an agent
|
|
9336
|
+
// that reads text only see the same.
|
|
9337
|
+
const imageTurnModel = [resolvedAssistantModel, requestedModel].find((candidate) => isOpenAIImageModelId(candidate)) || '';
|
|
9338
|
+
const imageSummaryLine = Boolean(!resolvedText && generatedImageAttachments.length && (q?.image_operation_id || imageTurnModel));
|
|
9339
|
+
if (imageSummaryLine) {
|
|
9340
|
+
resolvedText = openAIImageReplyLine({
|
|
9341
|
+
images: generatedImageAttachments,
|
|
9342
|
+
model: imageTurnModel || resolvedAssistantModel || '',
|
|
9343
|
+
turnId: messageId,
|
|
9344
|
+
});
|
|
9345
|
+
}
|
|
9346
|
+
// The image request came through on its second try, and the first may
|
|
9347
|
+
// have been billed as well: the reply says so, on the reply line when
|
|
9348
|
+
// that is the text, so an agent that reads text only sees it too.
|
|
9349
|
+
const imageNote = generatedImageAttachments.length
|
|
9350
|
+
? String(req.body.imageNote || '').trim().slice(0, OPENAI_IMAGE_NOTE_MAX)
|
|
9351
|
+
: '';
|
|
9352
|
+
if (imageNote) {
|
|
9353
|
+
resolvedText = imageSummaryLine
|
|
9354
|
+
? `${resolvedText}. Note: ${imageNote}`
|
|
9355
|
+
: `${String(resolvedText || '').trimEnd()}${resolvedText ? '\n\n' : ''}Note: ${imageNote}`;
|
|
9356
|
+
}
|
|
8352
9357
|
const assistantAttachments = generatedImageAttachments.length ? JSON.stringify(generatedImageAttachments) : null;
|
|
8353
9358
|
const now = new Date().toISOString();
|
|
8354
9359
|
const imageOperation = q?.image_operation_id
|
|
@@ -8479,8 +9484,10 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8479
9484
|
// Ensure UI is updated even if the response was duplicate/late
|
|
8480
9485
|
if (currentStatus === 'done') {
|
|
8481
9486
|
io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
|
|
9487
|
+
imagePreviews.finish(messageId, 'done');
|
|
8482
9488
|
} else if (currentStatus === 'failed') {
|
|
8483
9489
|
io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'failed' });
|
|
9490
|
+
imagePreviews.finish(messageId, 'failed');
|
|
8484
9491
|
}
|
|
8485
9492
|
return res.json({ ok: true, ignored: 'not_pending_or_processing', actualStatus: currentStatus });
|
|
8486
9493
|
}
|
|
@@ -8629,6 +9636,8 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8629
9636
|
},
|
|
8630
9637
|
});
|
|
8631
9638
|
io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
|
|
9639
|
+
// The final images are in the chat: the previews go.
|
|
9640
|
+
imagePreviews.finish(messageId, generatedImageAttachments.length ? 'done' : 'failed');
|
|
8632
9641
|
if (usageLimitPause) {
|
|
8633
9642
|
try {
|
|
8634
9643
|
usageLimitPauseService.schedulePause(usageLimitPause, { queueRow: q });
|
|
@@ -9533,7 +10542,13 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
9533
10542
|
});
|
|
9534
10543
|
}
|
|
9535
10544
|
|
|
9536
|
-
|
|
10545
|
+
// An update in flight (done false) is volatile, as relay_stream: each one
|
|
10546
|
+
// carries the whole thought so far (up to MAX_THOUGHT_CHARS), so keeping
|
|
10547
|
+
// them in the connection-state-recovery buffer, or queueing them on a
|
|
10548
|
+
// socket that does not drain, holds a turn's worth of copies — gigabytes
|
|
10549
|
+
// for a few long reasoning turns at once. A reconnect resync reads the
|
|
10550
|
+
// stored snapshot instead; the final thought (done) is never dropped.
|
|
10551
|
+
(done ? io : io.volatile).emit('relay_thought', {
|
|
9537
10552
|
messageId,
|
|
9538
10553
|
conversationId,
|
|
9539
10554
|
mode: relayMode,
|
|
@@ -9805,6 +10820,41 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
9805
10820
|
failConsumedQueueRow(q, { reason: 'requeue', attemptId: claimedAttemptId });
|
|
9806
10821
|
return res.json({ ok: true, terminal: true, code: STEER_SETTLE_FAILED_STABLE_CODE });
|
|
9807
10822
|
}
|
|
10823
|
+
if (q && q.status === 'processing') {
|
|
10824
|
+
// An image turn whose request went to OpenAI is never run again: a
|
|
10825
|
+
// second run is billed again. One still running here finishes the row
|
|
10826
|
+
// itself; one that ended, or was cut off, ends with the note.
|
|
10827
|
+
const imageStop = imageTurnRequeueStop(q);
|
|
10828
|
+
if (imageStop?.running) return res.json({ ok: true, held: 'image-request-running' });
|
|
10829
|
+
if (imageStop) {
|
|
10830
|
+
const failed = failQueueMessage({
|
|
10831
|
+
queueRow: q,
|
|
10832
|
+
messageId,
|
|
10833
|
+
conversationId: q.conversation_id,
|
|
10834
|
+
relayMode: normalizeRelayMode(q.relay_mode) || DEFAULT_RELAY_MODE,
|
|
10835
|
+
model: q.model || null,
|
|
10836
|
+
responseText: buildTerminalFailureTextForChat(imageStop.failure),
|
|
10837
|
+
failureRecord: {
|
|
10838
|
+
kind: 'terminal',
|
|
10839
|
+
error: 'image-not-requeued',
|
|
10840
|
+
code: normalizeTerminalErrorCode(imageStop.failure.code),
|
|
10841
|
+
stableCode: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}`,
|
|
10842
|
+
message: imageStop.failure.message,
|
|
10843
|
+
guidance: imageStop.failure.guidance,
|
|
10844
|
+
retryCount: Number(q.retry_count || 0),
|
|
10845
|
+
failedAt: new Date().toISOString(),
|
|
10846
|
+
},
|
|
10847
|
+
attemptId: claimedAttemptId,
|
|
10848
|
+
});
|
|
10849
|
+
if (failed) {
|
|
10850
|
+
settleRelayAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
|
|
10851
|
+
settleRelaySubagentAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
|
|
10852
|
+
console.warn(`[${ts()}] FAILED ${messageId?.slice(0,8)} reason=image-not-requeued code=${imageStop.failure.code}`);
|
|
10853
|
+
imagePreviews.finish(messageId, 'failed');
|
|
10854
|
+
}
|
|
10855
|
+
return res.json({ ok: true, terminal: true, code: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}` });
|
|
10856
|
+
}
|
|
10857
|
+
}
|
|
9808
10858
|
if (q && q.status === 'processing' && String(req.body?.code || '').trim() === 'relay.provider-mismatch') {
|
|
9809
10859
|
// A routing refusal, not a delivery failure: the legacy relay handed
|
|
9810
10860
|
// back a turn that belongs to a session worker. No retry increment and
|