@oar-sh/oar 0.9.23 → 0.9.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/skills/reasoning-stream.mjs +3 -1
- package/CHANGELOG.md +307 -0
- package/README.md +12 -11
- package/bin/oar.js +54 -4
- package/package.json +11 -7
- package/server/agent-sdk-loader.mjs +116 -0
- package/server/agent-sdks/claude/entry.mjs +4 -0
- package/server/agent-sdks/claude/package-lock.json +1438 -0
- package/server/agent-sdks/claude/package.json +13 -0
- package/server/agent-sdks/cursor/entry-sqlite.mjs +2 -0
- package/server/agent-sdks/cursor/entry.mjs +3 -0
- package/server/agent-sdks/cursor/package-lock.json +171 -0
- package/server/agent-sdks/cursor/package.json +11 -0
- package/server/agent-sdks/koffi/entry.cjs +2 -0
- package/server/agent-sdks/koffi/package-lock.json +368 -0
- package/server/agent-sdks/koffi/package.json +10 -0
- package/server/claude-worker/claude-preview-tool.mjs +5 -2
- package/server/claude-worker/claude-remote-relay-tool.mjs +20 -2
- package/server/claude-worker/claude-sdk-adapter.mjs +4 -1
- package/server/claude-worker/sdk-message-normalizer.mjs +29 -1
- package/server/copilot-worker/copilot-sdk-session-process.mjs +13 -8
- package/server/cursor-worker/cursor-sdk-adapter.mjs +5 -3
- package/server/cursor-worker/sdk-message-normalizer.mjs +13 -0
- package/server/db-schema.mjs +19 -0
- package/server/grok-worker/sdk-message-normalizer.mjs +3 -0
- package/server/migrations/0039-media-index.mjs +167 -0
- package/server/migrations/0040-image-session-defaults.mjs +93 -0
- package/server/public/app/agent-sdk-ui.js +349 -0
- package/server/public/app/api-client.js +149 -8
- package/server/public/app/ask-user-view.js +2 -1
- package/server/public/app/attachments-view.js +66 -9
- package/server/public/app/bootstrap.js +289 -94
- package/server/public/app/chat-actions-menu-state.mjs +1 -0
- package/server/public/app/context-media-tab.js +143 -0
- package/server/public/app/conversation-view.js +90 -5
- package/server/public/app/image-composer-ui.js +518 -0
- package/server/public/app/image-turn-placeholder.mjs +233 -0
- package/server/public/app/journal-view.js +12 -2
- package/server/public/app/media-gallery-model.mjs +202 -0
- package/server/public/app/media-gallery-ui.js +524 -0
- package/server/public/app/media-view.js +79 -0
- package/server/public/app/model-refresh-ui.mjs +413 -0
- package/server/public/app/no-conversation-state.mjs +5 -2
- package/server/public/app/openai-image-composer.mjs +492 -0
- package/server/public/app/openai-image-cost-ui.js +426 -0
- package/server/public/app/openai-image-spend.mjs +443 -0
- package/server/public/app/plan-usage-view.mjs +38 -6
- package/server/public/app/provider-readiness.mjs +67 -4
- package/server/public/app/relay-error-ctas.mjs +5 -0
- package/server/public/app/session-history-ui.js +2 -0
- package/server/public/app/settings-modal.js +171 -0
- package/server/public/app/socket-handlers.js +47 -0
- package/server/public/app/tab-strip-fade.mjs +1 -1
- package/server/public/app/viewer-gallery.mjs +23 -1
- package/server/public/index.html +478 -9
- package/server/relay-tools.md +1 -0
- package/server/repositories/image-conversation-repository.mjs +69 -0
- package/server/repositories/media-repository.mjs +305 -0
- package/server/routes/agent-sdk-routes.mjs +73 -0
- package/server/routes/media-routes.mjs +361 -0
- package/server/routes/messages-routes.mjs +1467 -343
- package/server/routes/sessions-routes.mjs +596 -123
- package/server/server-runtime.mjs +414 -81
- package/server/services/about-service.mjs +26 -6
- package/server/services/agent-sdk-launch-gate.mjs +124 -0
- package/server/services/agent-sdk-paths.mjs +307 -0
- package/server/services/agent-sdk-store.mjs +1211 -0
- package/server/services/claude-api-settings-service.mjs +54 -17
- package/server/services/cli-install-service.mjs +3 -2
- package/server/services/copilot-model-discovery-service.mjs +7 -1
- package/server/services/database-file.mjs +23 -5
- package/server/services/deepseek-settings-service.mjs +58 -20
- package/server/services/earlier-database-service.mjs +4 -2
- package/server/services/generated-image-cleanup-service.mjs +2 -1
- package/server/services/generated-image-files.mjs +110 -0
- package/server/services/image-input-paths.mjs +389 -0
- package/server/services/image-operation-service.mjs +6 -2
- package/server/services/media-extract.mjs +653 -0
- package/server/services/media-index-service.mjs +221 -0
- package/server/services/model-refresh-scheduler.mjs +153 -0
- package/server/services/model-variant-catalog-service.mjs +141 -9
- package/server/services/oar-cli-helpers.mjs +72 -9
- package/server/services/oar-state-migration-service.mjs +2 -12
- package/server/services/oar-state-root.mjs +17 -0
- package/server/services/openai-image-preview-turns.mjs +330 -0
- package/server/services/openai-image-spend-service.mjs +516 -0
- package/server/services/openai-image-stream.mjs +170 -0
- package/server/services/plan-usage-openai-image.mjs +148 -0
- package/server/services/provider-model-refresh.mjs +326 -0
- package/server/services/provider-model-state.mjs +395 -0
- package/server/services/relay-singleton-guard.mjs +72 -10
- package/server/services/remote-relay-caller-context.mjs +6 -0
- package/server/services/remote-relay-dispatcher.mjs +386 -22
- package/server/services/remote-relay-registry-service.mjs +3 -1
- package/server/services/session-worker-launch-service.mjs +13 -0
- package/server/services/session-worker-supervisor-service.mjs +19 -1
- package/server/services/session-worker-websocket-service.mjs +15 -0
- package/server/services/token-usage-service.mjs +15 -0
- package/server/services/update-install-service.mjs +2 -2
- package/server/services/worker-launch-note-service.mjs +14 -2
- package/server/services/worker-recycle-service.mjs +145 -11
- package/shared/image-header.mjs +90 -0
- package/shared/model-refresh-schedule.mjs +120 -0
- package/shared/openai-image-options.mjs +299 -0
- package/shared/openai-image-presets.mjs +175 -0
- package/shared/openai-image-previews.mjs +40 -0
- package/shared/openai-image-prices.mjs +291 -0
- package/shared/openai-image-spend.mjs +90 -0
- package/shared/remote-relay-contract.mjs +126 -4
- package/shared/worker-runtime/direct-openai-image.mjs +98 -48
|
@@ -6,6 +6,7 @@ import { randomUUID } from 'crypto';
|
|
|
6
6
|
import { stripRelayPromptContext } from '../services/relay-prompt-sanitizer.mjs';
|
|
7
7
|
import { resolveUploadMimeType } from '../services/mime-sniffer.mjs';
|
|
8
8
|
import { fileVersionToken, serveFileWithRangeSupport, withFileVersion } from '../services/file-serving.mjs';
|
|
9
|
+
import { registerMediaRoutes } from './media-routes.mjs';
|
|
9
10
|
import { validateEntryName } from '../services/explorer-entry-name.mjs';
|
|
10
11
|
import { countEntries, createDirectory, removeEntry, renameEntry } from '../services/explorer-mutations.mjs';
|
|
11
12
|
import {
|
|
@@ -64,6 +65,47 @@ import {
|
|
|
64
65
|
unknownProviderModelText,
|
|
65
66
|
} from '../services/provider-model-selection.mjs';
|
|
66
67
|
import { stripRemotePromptHeader } from '../../shared/remote-relay-contract.mjs';
|
|
68
|
+
import {
|
|
69
|
+
OPENAI_IMAGE_LIMITS,
|
|
70
|
+
OPENAI_IMAGE_MIME_BY_FORMAT,
|
|
71
|
+
OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
|
|
72
|
+
checkOpenAIImageSize,
|
|
73
|
+
formatOpenAIImageSummary,
|
|
74
|
+
isOpenAIImageModelId,
|
|
75
|
+
mergeOpenAIImageOptions,
|
|
76
|
+
openAIImageModelFamily,
|
|
77
|
+
openAIImageModelNeedsOpenAIText,
|
|
78
|
+
validateOpenAIImageOptions,
|
|
79
|
+
} from '../../shared/openai-image-options.mjs';
|
|
80
|
+
import { OPENAI_IMAGE_PARTIAL_IMAGES, OPENAI_IMAGE_PARTIALS_DIR } from '../../shared/openai-image-previews.mjs';
|
|
81
|
+
import {
|
|
82
|
+
SSE_EVENT_TOO_LARGE_CODE,
|
|
83
|
+
interpretOpenAIImageStreamEvent,
|
|
84
|
+
isEventStreamResponse,
|
|
85
|
+
readSseEvents,
|
|
86
|
+
} from '../services/openai-image-stream.mjs';
|
|
87
|
+
import { createOpenAIImagePreviewTurns } from '../services/openai-image-preview-turns.mjs';
|
|
88
|
+
import {
|
|
89
|
+
OPENAI_IMAGE_CAP_WARN_RATIO,
|
|
90
|
+
OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
|
|
91
|
+
OPENAI_IMAGE_USAGE_PATH,
|
|
92
|
+
formatOpenAIImageUsd,
|
|
93
|
+
} from '../../shared/openai-image-spend.mjs';
|
|
94
|
+
import { estimateOpenAIImageCost } from '../../shared/openai-image-prices.mjs';
|
|
95
|
+
import {
|
|
96
|
+
OPENAI_IMAGE_PRESETS_SETTING_KEY,
|
|
97
|
+
findOpenAIImagePreset,
|
|
98
|
+
parseOwnOpenAIImagePresets,
|
|
99
|
+
splitOpenAIImagePreset,
|
|
100
|
+
} from '../../shared/openai-image-presets.mjs';
|
|
101
|
+
import {
|
|
102
|
+
createOpenAIImageSpendService,
|
|
103
|
+
openAIImageCapRefusalText,
|
|
104
|
+
openAIImageTryCharge,
|
|
105
|
+
openAIImageTryRunId,
|
|
106
|
+
} from '../services/openai-image-spend-service.mjs';
|
|
107
|
+
import { IMAGE_INPUT_REFUSED_CODE, resolveImageInputRoots, resolveImageInputs } from '../services/image-input-paths.mjs';
|
|
108
|
+
import { IMAGE_HEADER_BYTES, readImageHeader } from '../../shared/image-header.mjs';
|
|
67
109
|
import { stripModelTierSuffix } from '../../shared/claude-cloud/repo-url.mjs';
|
|
68
110
|
import {
|
|
69
111
|
createSubagentTimelineService,
|
|
@@ -206,35 +248,33 @@ function isAutoModel(model) {
|
|
|
206
248
|
return String(model || '').trim().toLowerCase() === AUTO_MODEL_SENTINEL;
|
|
207
249
|
}
|
|
208
250
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
return normalized.startsWith('gpt-image-') || normalized.startsWith('dall-e-');
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
const OPENAI_IMAGE_QUALITY_VALUES = new Set(['auto', 'low', 'medium', 'high']);
|
|
215
|
-
const OPENAI_IMAGE_SIZE_VALUES = new Set([
|
|
216
|
-
'auto',
|
|
217
|
-
'256x256',
|
|
218
|
-
'512x512',
|
|
219
|
-
'1024x1024',
|
|
220
|
-
'1536x1024',
|
|
221
|
-
'1024x1536',
|
|
222
|
-
'1792x1024',
|
|
223
|
-
'1024x1792',
|
|
224
|
-
]);
|
|
225
|
-
|
|
251
|
+
// A quality or size any image model family takes (shared/openai-image-options.mjs),
|
|
252
|
+
// or ''. The model's own family is checked where the request is built.
|
|
226
253
|
function normalizeOpenAIImageQuality(value = '') {
|
|
227
|
-
const
|
|
228
|
-
return
|
|
254
|
+
const checked = validateOpenAIImageOptions({ quality: value });
|
|
255
|
+
return checked.ok ? (checked.values.quality || '') : '';
|
|
229
256
|
}
|
|
230
257
|
|
|
231
258
|
function normalizeOpenAIImageSize(value = '') {
|
|
232
259
|
const normalized = String(value || '').trim().toLowerCase();
|
|
233
|
-
|
|
260
|
+
// The composer's size select is the context-tier select: its own values
|
|
261
|
+
// mean the API's default size.
|
|
234
262
|
if (normalized === 'default' || normalized === 'long_context') return 'auto';
|
|
235
|
-
return '';
|
|
263
|
+
if (!normalized) return '';
|
|
264
|
+
return checkOpenAIImageSize(normalized).size || '';
|
|
236
265
|
}
|
|
237
266
|
|
|
267
|
+
// How long one OpenAI image request may take, from sending it to the last
|
|
268
|
+
// byte of the reply; config.json `openaiImageGenerateTimeoutMs` and
|
|
269
|
+
// `openaiImageEditTimeoutMs` override them. Above 300 s undici's own header
|
|
270
|
+
// timeout ends a request first.
|
|
271
|
+
export const OPENAI_IMAGE_GENERATE_TIMEOUT_MS = 180_000;
|
|
272
|
+
export const OPENAI_IMAGE_EDIT_TIMEOUT_MS = 300_000;
|
|
273
|
+
// The pause before the one retry of a request that may pass the second time.
|
|
274
|
+
const OPENAI_IMAGE_RETRY_DELAY_MS = 2_000;
|
|
275
|
+
// OpenAI takes up to 16 input images on an edit.
|
|
276
|
+
const OPENAI_IMAGE_EDIT_INPUTS_MAX = 16;
|
|
277
|
+
|
|
238
278
|
function normalizeSessionWorkerId(value) {
|
|
239
279
|
const text = String(value || '').trim();
|
|
240
280
|
return text || null;
|
|
@@ -1445,6 +1485,7 @@ export function buildDequeuedRelayMessage({
|
|
|
1445
1485
|
try {
|
|
1446
1486
|
messageKind = String(stmts.getMessageByConversation?.get?.(msg.id, msg.conversation_id)?.kind || '').trim();
|
|
1447
1487
|
} catch {}
|
|
1488
|
+
const imageOptions = parseQueueImageOptions(msg.image_options_json);
|
|
1448
1489
|
return {
|
|
1449
1490
|
id: msg.id,
|
|
1450
1491
|
conversationId: msg.conversation_id,
|
|
@@ -1477,10 +1518,23 @@ export function buildDequeuedRelayMessage({
|
|
|
1477
1518
|
ownerAssignedAt: msg.owner_assigned_at || null,
|
|
1478
1519
|
ownerLeaseExpiresAt: msg.owner_lease_expires_at || null,
|
|
1479
1520
|
imageOperationId: String(msg.image_operation_id || '').trim() || null,
|
|
1521
|
+
// An image turn's options as the send settled them; a turn without an
|
|
1522
|
+
// operation sends them with its direct request.
|
|
1523
|
+
...(imageOptions ? { imageOptions } : {}),
|
|
1480
1524
|
...(isRelayAutomatedMessageKind(messageKind) ? { messageKind } : {}),
|
|
1481
1525
|
};
|
|
1482
1526
|
}
|
|
1483
1527
|
|
|
1528
|
+
function parseQueueImageOptions(value) {
|
|
1529
|
+
if (!value) return null;
|
|
1530
|
+
try {
|
|
1531
|
+
const parsed = JSON.parse(String(value));
|
|
1532
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
|
|
1533
|
+
} catch {
|
|
1534
|
+
return null;
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1484
1538
|
function buildAttachmentPromptContext(attachments) {
|
|
1485
1539
|
if (!Array.isArray(attachments) || !attachments.length) return '';
|
|
1486
1540
|
const lines = [
|
|
@@ -1526,14 +1580,61 @@ function resolveGeneratedImageExtension(mimeType) {
|
|
|
1526
1580
|
return GENERATED_IMAGE_EXTENSIONS[normalized] || 'png';
|
|
1527
1581
|
}
|
|
1528
1582
|
|
|
1583
|
+
// Base64 payloads are checked with a character loop, never with a regular
|
|
1584
|
+
// expression over the whole string. V8 compiles a regexp without
|
|
1585
|
+
// optimisation once the process has generated more than 1 MB of regexp code
|
|
1586
|
+
// and holds more than 16 MB of executable memory (TooMuchRegExpCode in
|
|
1587
|
+
// regexp.cc), which a relay reaches after some uptime. An unoptimised
|
|
1588
|
+
// `[...]+` then pushes one backtrack frame per character and overflows the
|
|
1589
|
+
// 64 MiB regexp stack at 4,194,304 characters, a PNG of about 3 MB, with
|
|
1590
|
+
// "Maximum call stack size exceeded": every gpt-image reply above that size
|
|
1591
|
+
// was dropped after the provider had billed it, and the row was retried four
|
|
1592
|
+
// more times (2026-11-07). A fresh process optimises the same regexp and
|
|
1593
|
+
// never shows it.
|
|
1594
|
+
// An empty string is not base64 here: every caller checks for a missing
|
|
1595
|
+
// payload first and answers it with its own message.
|
|
1596
|
+
export function isBase64Text(value) {
|
|
1597
|
+
const text = String(value || '');
|
|
1598
|
+
if (!text) return false;
|
|
1599
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
1600
|
+
const c = text.charCodeAt(i);
|
|
1601
|
+
if (
|
|
1602
|
+
(c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122)
|
|
1603
|
+
|| c === 43 || c === 47 || c === 61 // + / =
|
|
1604
|
+
|| c === 32 || c === 9 || c === 10 || c === 13 || c === 11 || c === 12 // ASCII whitespace
|
|
1605
|
+
) continue;
|
|
1606
|
+
return false;
|
|
1607
|
+
}
|
|
1608
|
+
return true;
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
/**
|
|
1612
|
+
* `data:<mime>;base64,<payload>` taken apart without a regexp over the
|
|
1613
|
+
* payload: `{ mimeType, payload }`, or null when the string is not that
|
|
1614
|
+
* shape (no comma, no `;base64` right before it, a parameter in the type,
|
|
1615
|
+
* an empty payload).
|
|
1616
|
+
*/
|
|
1617
|
+
export function splitBase64DataUrl(dataUrl) {
|
|
1618
|
+
const text = String(dataUrl || '');
|
|
1619
|
+
if (!/^data:/i.test(text)) return null;
|
|
1620
|
+
const comma = text.indexOf(',');
|
|
1621
|
+
if (comma < 0) return null;
|
|
1622
|
+
const header = text.slice('data:'.length, comma);
|
|
1623
|
+
const payload = text.slice(comma + 1);
|
|
1624
|
+
if (!/;base64$/i.test(header) || !payload) return null;
|
|
1625
|
+
const mimeType = header.slice(0, -';base64'.length);
|
|
1626
|
+
if (!mimeType || mimeType.includes(';') || mimeType.includes(',')) return null;
|
|
1627
|
+
return { mimeType, payload };
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1529
1630
|
function parseGeneratedImageData(raw) {
|
|
1530
1631
|
const dataUrl = String(raw?.dataUrl || raw?.data_url || '').trim();
|
|
1531
1632
|
if (dataUrl.startsWith('data:')) {
|
|
1532
|
-
const
|
|
1533
|
-
if (!
|
|
1633
|
+
const parts = splitBase64DataUrl(dataUrl);
|
|
1634
|
+
if (!parts) throw new Error('Invalid generated image data URL');
|
|
1534
1635
|
return {
|
|
1535
|
-
mimeType: String(raw?.mimeType || raw?.mime_type ||
|
|
1536
|
-
dataBase64:
|
|
1636
|
+
mimeType: String(raw?.mimeType || raw?.mime_type || parts.mimeType || '').trim().toLowerCase(),
|
|
1637
|
+
dataBase64: parts.payload.trim(),
|
|
1537
1638
|
};
|
|
1538
1639
|
}
|
|
1539
1640
|
return {
|
|
@@ -1552,7 +1653,7 @@ export function normalizeGeneratedImageResponses(rawGeneratedImages, { maxImages
|
|
|
1552
1653
|
const mimeType = parsed.mimeType || 'image/png';
|
|
1553
1654
|
if (!mimeType.startsWith('image/')) throw new Error('Generated image type must be image/*');
|
|
1554
1655
|
if (!parsed.dataBase64) throw new Error('Generated image payload is missing base64 data');
|
|
1555
|
-
if (
|
|
1656
|
+
if (!isBase64Text(parsed.dataBase64)) {
|
|
1556
1657
|
throw new Error('Generated image payload is not valid base64');
|
|
1557
1658
|
}
|
|
1558
1659
|
const buffer = Buffer.from(parsed.dataBase64, 'base64');
|
|
@@ -1578,55 +1679,41 @@ export function shouldAcceptAssistantResponsePayload({ text = '', terminalFailur
|
|
|
1578
1679
|
function parseOpenAIImageBase64Value(value) {
|
|
1579
1680
|
const raw = String(value || '').trim();
|
|
1580
1681
|
if (!raw) return '';
|
|
1581
|
-
if (
|
|
1682
|
+
if (!isBase64Text(raw)) {
|
|
1582
1683
|
throw new Error('OpenAI image payload contains invalid base64 data');
|
|
1583
1684
|
}
|
|
1584
1685
|
return Buffer.from(raw, 'base64').toString('base64');
|
|
1585
1686
|
}
|
|
1586
1687
|
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 10 } = {}) {
|
|
1688
|
+
/**
|
|
1689
|
+
* The body of one image request (the generate route's, or the one the
|
|
1690
|
+
* execute route builds from an operation), checked against the model's
|
|
1691
|
+
* family. `n` is the count; size and quality "auto" pass and are left out
|
|
1692
|
+
* for a dall-e model when the request is built. A prompt that came from an
|
|
1693
|
+
* agent on another relay loses its "[Remote prompt from …]" header: OpenAI
|
|
1694
|
+
* would draw it.
|
|
1695
|
+
*/
|
|
1696
|
+
export function normalizeOpenAIImageGenerateRequestBody(rawBody) {
|
|
1598
1697
|
const body = rawBody && typeof rawBody === 'object' ? rawBody : {};
|
|
1599
1698
|
const messageId = String(body.messageId || '').trim();
|
|
1600
1699
|
const conversationId = String(body.conversationId || '').trim();
|
|
1601
1700
|
const model = String(body.model || '').trim();
|
|
1602
|
-
const prompt = String(body.prompt || '').trim();
|
|
1701
|
+
const prompt = stripRemotePromptHeader(String(body.prompt || '')).trim();
|
|
1603
1702
|
if (!messageId) throw new Error('Missing messageId');
|
|
1604
1703
|
if (!conversationId) throw new Error('Missing conversationId');
|
|
1605
1704
|
if (!model) throw new Error('Missing model');
|
|
1606
1705
|
if (!prompt) throw new Error('Missing prompt');
|
|
1607
1706
|
|
|
1608
|
-
const
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
}
|
|
1618
|
-
|
|
1619
|
-
const sizeRaw = body.size;
|
|
1620
|
-
let size = undefined;
|
|
1621
|
-
if (sizeRaw !== undefined && sizeRaw !== null && String(sizeRaw).trim()) {
|
|
1622
|
-
const normalizedSize = String(sizeRaw).trim().toLowerCase();
|
|
1623
|
-
if (normalizedSize !== 'auto' && !/^\d{2,5}x\d{2,5}$/i.test(normalizedSize)) {
|
|
1624
|
-
throw new Error('Invalid size value (expected "<width>x<height>" or "auto")');
|
|
1625
|
-
}
|
|
1626
|
-
size = normalizedSize;
|
|
1627
|
-
}
|
|
1707
|
+
const legacySize = String(body.size ?? '').trim().toLowerCase();
|
|
1708
|
+
const checked = validateOpenAIImageOptions({
|
|
1709
|
+
size: legacySize === 'default' || legacySize === 'long_context' ? 'auto' : body.size,
|
|
1710
|
+
quality: body.quality,
|
|
1711
|
+
count: body.n,
|
|
1712
|
+
output_format: body.output_format,
|
|
1713
|
+
background: body.background,
|
|
1714
|
+
}, { model, field: 'image' });
|
|
1715
|
+
if (!checked.ok) throw new Error(checked.error);
|
|
1628
1716
|
|
|
1629
|
-
const quality = normalizeOpenAIImageOptionString(body.quality, 'quality');
|
|
1630
1717
|
const attachments = body.attachments;
|
|
1631
1718
|
if (attachments !== undefined && !Array.isArray(attachments)) {
|
|
1632
1719
|
throw new Error('attachments must be an array when provided');
|
|
@@ -1637,13 +1724,288 @@ export function normalizeOpenAIImageGenerateRequestBody(rawBody, { maxImages = 1
|
|
|
1637
1724
|
conversationId,
|
|
1638
1725
|
model,
|
|
1639
1726
|
prompt,
|
|
1640
|
-
n,
|
|
1641
|
-
size,
|
|
1642
|
-
quality,
|
|
1727
|
+
n: checked.values.count,
|
|
1728
|
+
size: checked.values.size,
|
|
1729
|
+
quality: checked.values.quality,
|
|
1730
|
+
output_format: checked.values.output_format,
|
|
1731
|
+
background: checked.values.background,
|
|
1643
1732
|
attachments: Array.isArray(attachments) ? attachments : [],
|
|
1644
1733
|
};
|
|
1645
1734
|
}
|
|
1646
1735
|
|
|
1736
|
+
/**
|
|
1737
|
+
* The fields OpenAI gets for one image request. dall-e models take no
|
|
1738
|
+
* output_format or background, and their "auto" size or quality is not sent
|
|
1739
|
+
* (the API's own default); they answer in b64_json only when asked to.
|
|
1740
|
+
*/
|
|
1741
|
+
export function buildOpenAIImageRequestFields(requestBody = {}) {
|
|
1742
|
+
const family = openAIImageModelFamily(requestBody.model);
|
|
1743
|
+
const isGptImage = family === 'gpt-image';
|
|
1744
|
+
const fields = { model: requestBody.model, prompt: requestBody.prompt };
|
|
1745
|
+
if (!isGptImage) fields.response_format = 'b64_json';
|
|
1746
|
+
if (requestBody.n !== undefined) fields.n = requestBody.n;
|
|
1747
|
+
for (const key of ['size', 'quality']) {
|
|
1748
|
+
const value = requestBody[key];
|
|
1749
|
+
if (value && (isGptImage || value !== 'auto')) fields[key] = value;
|
|
1750
|
+
}
|
|
1751
|
+
if (isGptImage) {
|
|
1752
|
+
if (requestBody.output_format) fields.output_format = requestBody.output_format;
|
|
1753
|
+
if (requestBody.background) fields.background = requestBody.background;
|
|
1754
|
+
}
|
|
1755
|
+
return fields;
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
const OPENAI_IMAGE_USAGE_DETAIL_KEYS = Object.freeze(['input_tokens_details', 'output_tokens_details']);
|
|
1759
|
+
|
|
1760
|
+
/**
|
|
1761
|
+
* The `usage` block of an OpenAI image reply, numbers only (top level and
|
|
1762
|
+
* the two token-detail blocks), or null when there is none.
|
|
1763
|
+
*/
|
|
1764
|
+
export function normalizeOpenAIImageUsage(raw) {
|
|
1765
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
|
|
1766
|
+
const numbersOf = (source) => {
|
|
1767
|
+
const out = {};
|
|
1768
|
+
for (const [key, value] of Object.entries(source || {})) {
|
|
1769
|
+
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) out[key] = value;
|
|
1770
|
+
}
|
|
1771
|
+
return out;
|
|
1772
|
+
};
|
|
1773
|
+
const usage = numbersOf(raw);
|
|
1774
|
+
for (const key of OPENAI_IMAGE_USAGE_DETAIL_KEYS) {
|
|
1775
|
+
const detail = raw[key];
|
|
1776
|
+
if (detail && typeof detail === 'object' && !Array.isArray(detail)) {
|
|
1777
|
+
const numbers = numbersOf(detail);
|
|
1778
|
+
if (Object.keys(numbers).length) usage[key] = numbers;
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
return Object.keys(usage).length ? usage : null;
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
/**
|
|
1785
|
+
* Two image usages added up, field by field and detail by detail (a stream
|
|
1786
|
+
* of a count > 1 request reports one per completed image); null + null is
|
|
1787
|
+
* null.
|
|
1788
|
+
*/
|
|
1789
|
+
export function addOpenAIImageUsage(a, b) {
|
|
1790
|
+
if (!a) return b || null;
|
|
1791
|
+
if (!b) return a;
|
|
1792
|
+
const sum = { ...a };
|
|
1793
|
+
for (const [key, value] of Object.entries(b)) {
|
|
1794
|
+
if (typeof value === 'number') sum[key] = (typeof sum[key] === 'number' ? sum[key] : 0) + value;
|
|
1795
|
+
else if (value && typeof value === 'object') sum[key] = addOpenAIImageUsage(sum[key] && typeof sum[key] === 'object' ? sum[key] : null, value);
|
|
1796
|
+
}
|
|
1797
|
+
return sum;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
// The base64 characters that decode to the first IMAGE_HEADER_BYTES bytes.
|
|
1801
|
+
const IMAGE_HEADER_BASE64_CHARS = Math.ceil(IMAGE_HEADER_BYTES / 3) * 4;
|
|
1802
|
+
|
|
1803
|
+
/** readImageHeader of a canonical base64 payload, decoding only its start. */
|
|
1804
|
+
function readBase64ImageHeader(base64) {
|
|
1805
|
+
const text = String(base64 || '');
|
|
1806
|
+
if (!text) return null;
|
|
1807
|
+
return readImageHeader(Buffer.from(text.slice(0, IMAGE_HEADER_BASE64_CHARS), 'base64'));
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// What one OpenAI image request came to, as `kind`:
|
|
1811
|
+
// - 'unsent': it never reached OpenAI (no connection, no name, no TLS);
|
|
1812
|
+
// nothing was billed.
|
|
1813
|
+
// - 'timeout', 'broken', 'server': no answer in time, the connection broke
|
|
1814
|
+
// off after the request may have gone out, or a 5xx; it may have been
|
|
1815
|
+
// billed.
|
|
1816
|
+
// - 'stream': a streamed reply that broke off, or ended without its image,
|
|
1817
|
+
// after OpenAI had started on it; it may have been billed.
|
|
1818
|
+
// - 'refused': a 4xx (429 included); nothing came of it, a retry is refused
|
|
1819
|
+
// the same way.
|
|
1820
|
+
// - 'unprocessed': a 2xx the relay could not read; billed.
|
|
1821
|
+
// The first five are tried once more; the last two never.
|
|
1822
|
+
const OPENAI_IMAGE_RETRY_KINDS = new Set(['unsent', 'timeout', 'broken', 'server', 'stream']);
|
|
1823
|
+
const OPENAI_IMAGE_AMBIGUOUS_KINDS = new Set(['timeout', 'broken', 'server', 'stream']);
|
|
1824
|
+
const OPENAI_IMAGE_UNSENT_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT']);
|
|
1825
|
+
const OPENAI_IMAGE_TIMEOUT_ERROR_CODES = new Set(['UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']);
|
|
1826
|
+
|
|
1827
|
+
export const OPENAI_IMAGE_INTERRUPTED_NOTE = 'The relay was interrupted while OpenAI was working on the image; the request may have been billed; retry the message.';
|
|
1828
|
+
export const OPENAI_IMAGE_REPEATED_NOTE = 'This image request was already sent to OpenAI once and was not sent again; the first request may have been billed; retry the message.';
|
|
1829
|
+
// The guidance under every image failure note: the note itself says what to do.
|
|
1830
|
+
const OPENAI_IMAGE_FAILURE_GUIDANCE = 'If this keeps failing, include the error code when you report it.';
|
|
1831
|
+
|
|
1832
|
+
/** The `code` of a thrown fetch error and of every cause under it. */
|
|
1833
|
+
function fetchErrorCodes(error) {
|
|
1834
|
+
const codes = [];
|
|
1835
|
+
const seen = new Set();
|
|
1836
|
+
const visit = (entry, depth) => {
|
|
1837
|
+
if (!entry || typeof entry !== 'object' || seen.has(entry) || depth > 4) return;
|
|
1838
|
+
seen.add(entry);
|
|
1839
|
+
if (entry.code) codes.push(String(entry.code));
|
|
1840
|
+
visit(entry.cause, depth + 1);
|
|
1841
|
+
if (Array.isArray(entry.errors)) for (const inner of entry.errors) visit(inner, depth + 1);
|
|
1842
|
+
};
|
|
1843
|
+
visit(error, 0);
|
|
1844
|
+
return codes;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
function isTlsFailureCode(code) {
|
|
1848
|
+
return code.startsWith('ERR_TLS_') || code.startsWith('ERR_SSL_') || code.includes('CERT') || code === 'EPROTO';
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
/** 'unsent', 'timeout' or 'broken' for an error fetch threw (see OPENAI_IMAGE_RETRY_KINDS). */
|
|
1852
|
+
export function classifyOpenAIImageFetchError(error) {
|
|
1853
|
+
if (error?.name === 'TimeoutError' || error?.cause?.name === 'TimeoutError') return 'timeout';
|
|
1854
|
+
const codes = fetchErrorCodes(error);
|
|
1855
|
+
if (codes.some((code) => OPENAI_IMAGE_TIMEOUT_ERROR_CODES.has(code))) return 'timeout';
|
|
1856
|
+
if (codes.length && codes.every((code) => OPENAI_IMAGE_UNSENT_ERROR_CODES.has(code) || isTlsFailureCode(code))) return 'unsent';
|
|
1857
|
+
return 'broken';
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
function describeFetchError(error) {
|
|
1861
|
+
const codes = [...new Set(fetchErrorCodes(error))];
|
|
1862
|
+
const message = String(error?.message || error || 'request error').trim();
|
|
1863
|
+
return codes.length ? `${message} (${codes.join(', ')})` : message;
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
function formatNoteDuration(ms) {
|
|
1867
|
+
const value = Math.max(0, Number(ms) || 0);
|
|
1868
|
+
return value >= 1000 ? `${Math.round(value / 1000)} s` : `${Math.round(value)} ms`;
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
function asSentence(text) {
|
|
1872
|
+
const value = String(text || '').trim();
|
|
1873
|
+
return !value || /[.!?]$/.test(value) ? value : `${value}.`;
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
/**
|
|
1877
|
+
* The failure of an image request after its tries: the note the chat shows
|
|
1878
|
+
* (`error`), the attempt's code, `outcome` 'uncertain' when it may have been
|
|
1879
|
+
* billed (else 'terminal'), and the HTTP status the generate route answers.
|
|
1880
|
+
* A request that never reached OpenAI on its last try but may have been
|
|
1881
|
+
* billed on the first is told as the first.
|
|
1882
|
+
*/
|
|
1883
|
+
export function openAIImageFailureResult(tries, { timeoutMs = 0, resent = false } = {}) {
|
|
1884
|
+
const list = Array.isArray(tries) && tries.length ? tries : [{ kind: 'broken' }];
|
|
1885
|
+
const last = list[list.length - 1];
|
|
1886
|
+
const ambiguous = list.find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry.kind)) || null;
|
|
1887
|
+
const deciding = last.kind === 'unsent' && ambiguous ? ambiguous : last;
|
|
1888
|
+
// A resend without stream fields is the second request of a turn whose
|
|
1889
|
+
// first was refused for them before any work.
|
|
1890
|
+
const triedText = resent ? 'sent twice, once without previews' : (list.length > 1 ? 'tried twice' : 'tried once');
|
|
1891
|
+
const maybeBilled = 'the request may have been billed; retry the message.';
|
|
1892
|
+
// A refusal or an unreadable reply after a first try that may have been
|
|
1893
|
+
// billed: the note says so too.
|
|
1894
|
+
const earlierBilled = ambiguous && !OPENAI_IMAGE_AMBIGUOUS_KINDS.has(deciding.kind) && deciding.kind !== 'unsent'
|
|
1895
|
+
? ' The first try may have been billed as well.'
|
|
1896
|
+
: '';
|
|
1897
|
+
const base = {
|
|
1898
|
+
ok: false,
|
|
1899
|
+
tries: list.length,
|
|
1900
|
+
sent: list.some((entry) => entry.kind !== 'unsent'),
|
|
1901
|
+
...(deciding.detail ? { detail: String(deciding.detail).slice(0, 500) } : {}),
|
|
1902
|
+
upstreamStatus: Number.isFinite(Number(deciding.status)) && Number(deciding.status) > 0 ? Number(deciding.status) : null,
|
|
1903
|
+
};
|
|
1904
|
+
switch (deciding.kind) {
|
|
1905
|
+
case 'unsent':
|
|
1906
|
+
return { ...base, status: 502, outcome: 'terminal', code: 'IMAGE_PROVIDER_UNREACHABLE', error: 'OpenAI could not be reached; nothing was billed; retry the message.' };
|
|
1907
|
+
case 'timeout': {
|
|
1908
|
+
const waited = deciding.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
|
|
1909
|
+
return { ...base, status: 504, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `OpenAI did not answer ${waited} (${triedText}); ${maybeBilled}` };
|
|
1910
|
+
}
|
|
1911
|
+
case 'server':
|
|
1912
|
+
return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_UNAVAILABLE', error: `OpenAI answered ${deciding.status || 'with a server error'} (${triedText}); ${maybeBilled}` };
|
|
1913
|
+
case 'stream':
|
|
1914
|
+
return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_STREAM_BROKEN', error: `OpenAI's image stream broke off before the image was complete (${triedText}); ${maybeBilled}` };
|
|
1915
|
+
case 'refused': {
|
|
1916
|
+
const message = String(deciding.message || '').trim().slice(0, 240);
|
|
1917
|
+
return {
|
|
1918
|
+
...base,
|
|
1919
|
+
status: deciding.status >= 400 && deciding.status < 500 ? deciding.status : 502,
|
|
1920
|
+
outcome: 'terminal',
|
|
1921
|
+
code: 'IMAGE_PROVIDER_REJECTED',
|
|
1922
|
+
error: `${asSentence(`OpenAI refused the image request (${deciding.status})${message ? `: ${message}` : ''}`)}${earlierBilled}`,
|
|
1923
|
+
};
|
|
1924
|
+
}
|
|
1925
|
+
case 'unprocessed':
|
|
1926
|
+
return {
|
|
1927
|
+
...base,
|
|
1928
|
+
status: 502,
|
|
1929
|
+
outcome: 'terminal',
|
|
1930
|
+
code: 'IMAGE_REPLY_UNPROCESSED',
|
|
1931
|
+
relayProcessing: true,
|
|
1932
|
+
error: `The relay could not process the image OpenAI returned (${deciding.detail || 'unknown error'}). The generation was billed; the message was not retried.${earlierBilled}`,
|
|
1933
|
+
};
|
|
1934
|
+
default:
|
|
1935
|
+
return { ...base, status: 502, outcome: 'uncertain', code: 'IMAGE_PROVIDER_NO_ANSWER', error: `The connection to OpenAI broke off (${triedText}); ${maybeBilled}` };
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
/**
|
|
1940
|
+
* The note on an image request that came through on its second try after a
|
|
1941
|
+
* first that may have reached OpenAI (OPENAI_IMAGE_AMBIGUOUS_KINDS): both may
|
|
1942
|
+
* have been billed. Empty when the earlier tries never went out.
|
|
1943
|
+
*/
|
|
1944
|
+
export function openAIImageRetriedNote(tries, { timeoutMs = 0 } = {}) {
|
|
1945
|
+
const first = (Array.isArray(tries) ? tries : []).find((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
|
|
1946
|
+
if (!first) return '';
|
|
1947
|
+
const tail = 'it was sent once more and the first try may have been billed as well.';
|
|
1948
|
+
switch (first.kind) {
|
|
1949
|
+
case 'timeout': {
|
|
1950
|
+
const waited = first.ownTimeout === false ? 'in time' : `within ${formatNoteDuration(timeoutMs)}`;
|
|
1951
|
+
return `OpenAI did not answer the first try ${waited}; ${tail}`;
|
|
1952
|
+
}
|
|
1953
|
+
case 'server':
|
|
1954
|
+
return `OpenAI answered the first try with ${first.status || 'a server error'}; ${tail}`;
|
|
1955
|
+
case 'stream':
|
|
1956
|
+
return `OpenAI's image stream broke off on the first try; ${tail}`;
|
|
1957
|
+
default:
|
|
1958
|
+
return `The connection to OpenAI broke off on the first try; ${tail}`;
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
/**
|
|
1963
|
+
* The failure of an image request whose next try the daily cap stopped
|
|
1964
|
+
* before it was sent (`refusal`: the spend service's reserveTry). Nothing of
|
|
1965
|
+
* that try reached OpenAI; an earlier try may have been billed, and the note
|
|
1966
|
+
* says so then.
|
|
1967
|
+
*/
|
|
1968
|
+
export function openAIImageCapRefusalResult(tries, refusal = {}) {
|
|
1969
|
+
const list = Array.isArray(tries) ? tries : [];
|
|
1970
|
+
const earlierBilled = list.some((entry) => OPENAI_IMAGE_AMBIGUOUS_KINDS.has(entry?.kind));
|
|
1971
|
+
return {
|
|
1972
|
+
ok: false,
|
|
1973
|
+
tries: list.length,
|
|
1974
|
+
sent: list.some((entry) => entry?.kind !== 'unsent'),
|
|
1975
|
+
status: 409,
|
|
1976
|
+
outcome: earlierBilled ? 'uncertain' : 'terminal',
|
|
1977
|
+
code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE,
|
|
1978
|
+
error: openAIImageCapRefusalText({ tryNumber: earlierBilled ? 2 : 1, refusal }),
|
|
1979
|
+
upstreamStatus: null,
|
|
1980
|
+
};
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
// The file system the partial preview route serves through: its open
|
|
1984
|
+
// follows no link.
|
|
1985
|
+
const PARTIAL_PREVIEW_FS = Object.freeze({
|
|
1986
|
+
...fs,
|
|
1987
|
+
openSync: (file) => fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)),
|
|
1988
|
+
});
|
|
1989
|
+
|
|
1990
|
+
// The longest image note /api/response takes into the reply text.
|
|
1991
|
+
const OPENAI_IMAGE_NOTE_MAX = 500;
|
|
1992
|
+
|
|
1993
|
+
/** A request refused before anything was sent to OpenAI. */
|
|
1994
|
+
function openAIImageRequestRefusal(status, code, error) {
|
|
1995
|
+
return { ok: false, tries: 0, sent: false, status, outcome: 'terminal', code, error, upstreamStatus: null };
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/** The `terminalError` /api/response gets for a failed image request. */
|
|
1999
|
+
function openAIImageTerminalError(result) {
|
|
2000
|
+
return {
|
|
2001
|
+
code: result.code,
|
|
2002
|
+
message: String(result.error || 'Image operation failed').slice(0, 500),
|
|
2003
|
+
...(result.detail ? { detail: String(result.detail).slice(0, 500) } : {}),
|
|
2004
|
+
guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
|
|
2005
|
+
retryable: false,
|
|
2006
|
+
};
|
|
2007
|
+
}
|
|
2008
|
+
|
|
1647
2009
|
function isOpenAIImageAttachment(raw) {
|
|
1648
2010
|
if (!raw || typeof raw !== 'object') return false;
|
|
1649
2011
|
const type = String(raw.type || '').trim().toLowerCase();
|
|
@@ -1652,7 +2014,26 @@ function isOpenAIImageAttachment(raw) {
|
|
|
1652
2014
|
return dataUrl.startsWith('data:image/');
|
|
1653
2015
|
}
|
|
1654
2016
|
|
|
1655
|
-
export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
2017
|
+
export function resolveOpenAIImageEditAttachment(rawAttachments, options = {}) {
|
|
2018
|
+
return resolveOpenAIImageEditAttachments(rawAttachments, { ...options, maxImages: 1 })[0] || null;
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
/**
|
|
2022
|
+
* Every image attachment of an edit, in the order given (the first is the
|
|
2023
|
+
* base image), at most `maxImages`; the others are ignored.
|
|
2024
|
+
*/
|
|
2025
|
+
export function resolveOpenAIImageEditAttachments(rawAttachments, {
|
|
2026
|
+
maxImages = OPENAI_IMAGE_EDIT_INPUTS_MAX,
|
|
2027
|
+
...options
|
|
2028
|
+
} = {}) {
|
|
2029
|
+
const attachments = Array.isArray(rawAttachments) ? rawAttachments : [];
|
|
2030
|
+
return attachments
|
|
2031
|
+
.filter((entry) => isOpenAIImageAttachment(entry))
|
|
2032
|
+
.slice(0, Math.max(1, Math.trunc(Number(maxImages) || 1)))
|
|
2033
|
+
.map((entry) => resolveOneOpenAIImageEditAttachment(entry, options));
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
function resolveOneOpenAIImageEditAttachment(image, {
|
|
1656
2037
|
maxImageBytes = 20 * 1024 * 1024,
|
|
1657
2038
|
allowedRootPath = '',
|
|
1658
2039
|
// Folders of earlier versions that uploads are still read from: only a
|
|
@@ -1660,22 +2041,18 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1660
2041
|
allowedBlobRootPaths = [],
|
|
1661
2042
|
allowedGeneratedImagesRootPath = '',
|
|
1662
2043
|
} = {}) {
|
|
1663
|
-
const
|
|
1664
|
-
|
|
1665
|
-
if (!firstImage) return null;
|
|
1666
|
-
|
|
1667
|
-
const dataUrl = String(firstImage.dataUrl || '').trim();
|
|
1668
|
-
let mimeType = String(firstImage.type || '').trim().toLowerCase();
|
|
2044
|
+
const dataUrl = String(image.dataUrl || '').trim();
|
|
2045
|
+
let mimeType = String(image.type || '').trim().toLowerCase();
|
|
1669
2046
|
let bytes = null;
|
|
1670
2047
|
|
|
1671
2048
|
if (dataUrl.startsWith('data:')) {
|
|
1672
|
-
const
|
|
1673
|
-
if (!
|
|
1674
|
-
mimeType = mimeType || String(
|
|
1675
|
-
const payload = parseOpenAIImageBase64Value(
|
|
2049
|
+
const parts = splitBase64DataUrl(dataUrl);
|
|
2050
|
+
if (!parts) throw new Error('Invalid image attachment data URL');
|
|
2051
|
+
mimeType = mimeType || String(parts.mimeType || '').trim().toLowerCase();
|
|
2052
|
+
const payload = parseOpenAIImageBase64Value(parts.payload);
|
|
1676
2053
|
bytes = Buffer.from(payload, 'base64');
|
|
1677
2054
|
} else {
|
|
1678
|
-
const filePath = String(
|
|
2055
|
+
const filePath = String(image.path || '').trim();
|
|
1679
2056
|
if (!filePath) throw new Error('Image attachment is missing file path or data URL');
|
|
1680
2057
|
const allowedRoot = String(allowedRootPath || '').trim();
|
|
1681
2058
|
const blobRoots = (Array.isArray(allowedBlobRootPaths) ? allowedBlobRootPaths : [])
|
|
@@ -1709,6 +2086,8 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1709
2086
|
bytes = fs.readFileSync(filePath);
|
|
1710
2087
|
}
|
|
1711
2088
|
|
|
2089
|
+
// What the bytes are wins over what the attachment says they are.
|
|
2090
|
+
mimeType = (Buffer.isBuffer(bytes) && readImageHeader(bytes)?.mime) || mimeType;
|
|
1712
2091
|
if (!mimeType.startsWith('image/')) {
|
|
1713
2092
|
throw new Error('Image attachment must use image/* MIME type');
|
|
1714
2093
|
}
|
|
@@ -1722,13 +2101,20 @@ export function resolveOpenAIImageEditAttachment(rawAttachments, {
|
|
|
1722
2101
|
return {
|
|
1723
2102
|
bytes,
|
|
1724
2103
|
mimeType,
|
|
1725
|
-
name: String(
|
|
2104
|
+
name: String(image.name || 'image').trim().slice(0, 120) || 'image',
|
|
1726
2105
|
};
|
|
1727
2106
|
}
|
|
1728
2107
|
|
|
1729
|
-
|
|
2108
|
+
/**
|
|
2109
|
+
* The images of an OpenAI image reply. Their type comes from the format
|
|
2110
|
+
* asked for (`outputFormat`), else the one the reply names, else PNG; the
|
|
2111
|
+
* image's own header has the last word.
|
|
2112
|
+
*/
|
|
2113
|
+
export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8, outputFormat = '' } = {}) {
|
|
1730
2114
|
const rows = Array.isArray(payload?.data) ? payload.data : [];
|
|
1731
2115
|
const maxCount = Math.max(1, Math.trunc(Number(maxImages) || 8));
|
|
2116
|
+
const format = String(outputFormat || payload?.output_format || '').trim().toLowerCase();
|
|
2117
|
+
const declaredMime = OPENAI_IMAGE_MIME_BY_FORMAT[format] || OPENAI_IMAGE_MIME_BY_FORMAT.png;
|
|
1732
2118
|
const generatedImages = [];
|
|
1733
2119
|
for (const [index, row] of rows.entries()) {
|
|
1734
2120
|
if (index >= maxCount) break;
|
|
@@ -1738,7 +2124,7 @@ export function normalizeOpenAIImageApiGeneratedImages(payload, { maxImages = 8
|
|
|
1738
2124
|
const revisedPrompt = String(row.revised_prompt || row.revisedPrompt || '').trim();
|
|
1739
2125
|
generatedImages.push({
|
|
1740
2126
|
data,
|
|
1741
|
-
mimeType:
|
|
2127
|
+
mimeType: readBase64ImageHeader(data)?.mime || declaredMime,
|
|
1742
2128
|
name: `generated-image-${index + 1}`,
|
|
1743
2129
|
...(revisedPrompt ? { revisedPrompt } : {}),
|
|
1744
2130
|
});
|
|
@@ -1806,7 +2192,12 @@ function createGeneratedImageAttachment({
|
|
|
1806
2192
|
resolveSessionStateRoot,
|
|
1807
2193
|
}) {
|
|
1808
2194
|
const imageId = `img-${String(imageIndex + 1).padStart(2, '0')}`;
|
|
1809
|
-
const
|
|
2195
|
+
const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
|
|
2196
|
+
// The type and pixel size the image's own header gives; the type the
|
|
2197
|
+
// sender named only when the header is not one the reader knows.
|
|
2198
|
+
const header = readImageHeader(bytes);
|
|
2199
|
+
const mimeType = header?.mime || image.mimeType;
|
|
2200
|
+
const extension = resolveGeneratedImageExtension(mimeType);
|
|
1810
2201
|
const location = resolveGeneratedImageStoragePath({
|
|
1811
2202
|
resolveSessionStateRoot,
|
|
1812
2203
|
sdkSessionId,
|
|
@@ -1817,7 +2208,6 @@ function createGeneratedImageAttachment({
|
|
|
1817
2208
|
});
|
|
1818
2209
|
fs.mkdirSync(location.absoluteDir, { recursive: true });
|
|
1819
2210
|
const tmpPath = path.join(location.absoluteDir, `${location.fileName}.${process.pid}.${Date.now()}.tmp`);
|
|
1820
|
-
const bytes = Buffer.from(String(image.dataBase64 || ''), 'base64');
|
|
1821
2211
|
fs.writeFileSync(tmpPath, bytes);
|
|
1822
2212
|
try {
|
|
1823
2213
|
fs.renameSync(tmpPath, location.absolutePath);
|
|
@@ -1829,8 +2219,9 @@ function createGeneratedImageAttachment({
|
|
|
1829
2219
|
}
|
|
1830
2220
|
return {
|
|
1831
2221
|
name: `${String(image.name || 'generated-image').replace(/\.[a-z0-9]+$/i, '') || 'generated-image'}-${imageIndex + 1}.${extension}`,
|
|
1832
|
-
type:
|
|
2222
|
+
type: mimeType,
|
|
1833
2223
|
size: bytes.length,
|
|
2224
|
+
...(header ? { width: header.width, height: header.height } : {}),
|
|
1834
2225
|
contentUrl: `/api/generated-image/${encodeURIComponent(conversationId)}/${encodeURIComponent(messageId)}/${encodeURIComponent(imageId)}/content`,
|
|
1835
2226
|
generatedImage: {
|
|
1836
2227
|
imageId,
|
|
@@ -1982,6 +2373,9 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
1982
2373
|
MAX_UPLOAD_ATTACHMENTS,
|
|
1983
2374
|
uploadsDir,
|
|
1984
2375
|
uploadReadDirs = [uploadsDir],
|
|
2376
|
+
// What a send's `imagePaths` never reads, beside the upload dirs: the
|
|
2377
|
+
// relay's data dir (`dirs`), its database and config files (`files`).
|
|
2378
|
+
imageInputDenyPaths = { dirs: [], files: [] },
|
|
1985
2379
|
MAX_REPO_TREE_NODES,
|
|
1986
2380
|
MAX_REQUEUE_RETRIES,
|
|
1987
2381
|
MAX_IMAGE_DATA_URL_LENGTH,
|
|
@@ -2085,6 +2479,11 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2085
2479
|
// (services/request-boundary.mjs). The runtime passes the check built
|
|
2086
2480
|
// with the configured marker headers.
|
|
2087
2481
|
isTrulyLocalRequest = createLocalRequestCheck().isTrulyLocal,
|
|
2482
|
+
// The pause before the one retry of an OpenAI image request.
|
|
2483
|
+
openAIImageRetryDelayMs = OPENAI_IMAGE_RETRY_DELAY_MS,
|
|
2484
|
+
// How long the partial previews of an image request that ended wait for
|
|
2485
|
+
// the turn's end (services/openai-image-preview-turns.mjs).
|
|
2486
|
+
openAIImagePreviewLeftoverMs,
|
|
2088
2487
|
featureFlags,
|
|
2089
2488
|
sessionWorkerRegistry,
|
|
2090
2489
|
sessionWorkerSupervisor,
|
|
@@ -2107,6 +2506,10 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2107
2506
|
claudeApiUsageService = null,
|
|
2108
2507
|
// Tokens per turn for every provider (token-usage-service.mjs).
|
|
2109
2508
|
tokenUsageService = null,
|
|
2509
|
+
// OpenAI image spend and its daily cap (openai-image-spend-service.mjs):
|
|
2510
|
+
// the runtime shares one instance with the settings routes. Without it
|
|
2511
|
+
// (tests) the routes make their own over tokenUsageService.
|
|
2512
|
+
openAIImageSpend = null,
|
|
2110
2513
|
// The delivery fence's runtime half for GET /api/pending, as on the
|
|
2111
2514
|
// socket: the Claude API claim check (null: no refusal) and what a fenced
|
|
2112
2515
|
// row does (the swap, or fail closed).
|
|
@@ -2149,6 +2552,24 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
2149
2552
|
deepseekClaudeConfigDir = '',
|
|
2150
2553
|
} = deps;
|
|
2151
2554
|
|
|
2555
|
+
// An app_settings value as text ('' when unset or unreadable).
|
|
2556
|
+
const readAppSettingText = (key) => {
|
|
2557
|
+
try {
|
|
2558
|
+
return String(stmts?.getAppSetting?.get?.(key)?.value ?? '').trim();
|
|
2559
|
+
} catch {
|
|
2560
|
+
return '';
|
|
2561
|
+
}
|
|
2562
|
+
};
|
|
2563
|
+
// OpenAI image spend: the cap check and reservation before each try, its
|
|
2564
|
+
// booking after, the send-time refusal and the spend state.
|
|
2565
|
+
const imageSpend = openAIImageSpend || createOpenAIImageSpendService({
|
|
2566
|
+
tokenUsageService,
|
|
2567
|
+
readSetting: readAppSettingText,
|
|
2568
|
+
db,
|
|
2569
|
+
emit: (event, payload) => io?.emit?.(event, payload),
|
|
2570
|
+
logger: console,
|
|
2571
|
+
});
|
|
2572
|
+
|
|
2152
2573
|
// ─── Sub-agent runs: header, timeline, live events ─────────────────────────
|
|
2153
2574
|
const subagentTimeline = createSubagentTimelineService({ db, stmts });
|
|
2154
2575
|
|
|
@@ -4012,6 +4433,19 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4012
4433
|
});
|
|
4013
4434
|
});
|
|
4014
4435
|
|
|
4436
|
+
// The media gallery (🧠 Media tab, global Media view) and its index.
|
|
4437
|
+
registerMediaRoutes(app, {
|
|
4438
|
+
auth,
|
|
4439
|
+
db,
|
|
4440
|
+
uploadPathForSha,
|
|
4441
|
+
fetchBrowsableDrives,
|
|
4442
|
+
resolveGeneratedImagePath: ({ sessionId, relativePath }) => resolveGeneratedImageReadPath({
|
|
4443
|
+
resolveSessionStateRoot,
|
|
4444
|
+
sessionId,
|
|
4445
|
+
relativePath,
|
|
4446
|
+
}),
|
|
4447
|
+
});
|
|
4448
|
+
|
|
4015
4449
|
app.get('/api/files/{*splat}', auth, (req, res) => {
|
|
4016
4450
|
const requestedPath = splatPath(req).trim();
|
|
4017
4451
|
const rootOverride = resolveScopedWorkspaceRootPath(req);
|
|
@@ -4824,6 +5258,8 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4824
5258
|
provider,
|
|
4825
5259
|
attachments: rawAttachments,
|
|
4826
5260
|
imageTarget: rawImageTarget,
|
|
5261
|
+
imagePaths: rawImagePaths,
|
|
5262
|
+
image: rawImageOptions,
|
|
4827
5263
|
} = req.body;
|
|
4828
5264
|
// A prompt from another relay's agent (body `origin` or the
|
|
4829
5265
|
// X-OAR-Remote-Origin header) is refused while the inbound switch is off.
|
|
@@ -4860,7 +5296,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4860
5296
|
Array.isArray(rawAttachments) ? rawAttachments.filter((item) => item && typeof item === 'object').length : (rawAttachments ? 1 : 0),
|
|
4861
5297
|
);
|
|
4862
5298
|
const referenceResolution = collectReferenceAttachmentsFromText(trimmedText);
|
|
4863
|
-
|
|
5299
|
+
let attachments = mergeMessageAttachments(normalizedAttachments, referenceResolution.attachments);
|
|
4864
5300
|
const imageTarget = rawImageTarget && typeof rawImageTarget === 'object'
|
|
4865
5301
|
? {
|
|
4866
5302
|
messageId: String(rawImageTarget.messageId || '').trim(),
|
|
@@ -4875,6 +5311,17 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
4875
5311
|
code: 'IMAGE_CONTINUITY_DISABLED',
|
|
4876
5312
|
});
|
|
4877
5313
|
}
|
|
5314
|
+
// Images named by path on this relay and the turn's own image options
|
|
5315
|
+
// (relay_tool send and create_session). Both go to an OpenAI image
|
|
5316
|
+
// session only; the paths are read before anything is stored, and one
|
|
5317
|
+
// that may not be read refuses the whole send.
|
|
5318
|
+
const refuseImageInput = (error) => res.status(400).json({ code: IMAGE_INPUT_REFUSED_CODE, error });
|
|
5319
|
+
const imagePathsGiven = rawImagePaths !== undefined && rawImagePaths !== null;
|
|
5320
|
+
const imageOptionsGiven = rawImageOptions !== undefined && rawImageOptions !== null;
|
|
5321
|
+
if (imagePathsGiven && (!Array.isArray(rawImagePaths) || rawImagePaths.some((entry) => typeof entry !== 'string'))) {
|
|
5322
|
+
return refuseImageInput('imagePaths must be a list of absolute file paths on this relay');
|
|
5323
|
+
}
|
|
5324
|
+
const imagePathList = imagePathsGiven ? rawImagePaths.filter((entry) => entry !== '') : [];
|
|
4878
5325
|
|
|
4879
5326
|
if (trimmedText.toLowerCase() === '/compact') {
|
|
4880
5327
|
if (attachments.length) return res.status(400).json({ error: 'Compact command does not accept attachments' });
|
|
@@ -5396,6 +5843,16 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5396
5843
|
latestUserMessageModel: stmts.getLatestUserMessageModel?.get?.(conversationId)?.model || '',
|
|
5397
5844
|
})
|
|
5398
5845
|
: null;
|
|
5846
|
+
// Copilot lists image ids its runtime cannot serve: a Copilot chat on one
|
|
5847
|
+
// is refused with a pointer to an OpenAI Image chat, whoever sends.
|
|
5848
|
+
const copilotModelId = usesGitHubCatalog ? String(inheritedModel ? inheritedModel.model : (model || '')).trim() : '';
|
|
5849
|
+
if (copilotModelId && isOpenAIImageModelId(copilotModelId)) {
|
|
5850
|
+
return res.status(400).json({
|
|
5851
|
+
error: `${openAIImageModelNeedsOpenAIText(copilotModelId)} Nothing was sent.`,
|
|
5852
|
+
code: OPENAI_IMAGE_MODEL_NEEDS_OPENAI_CODE,
|
|
5853
|
+
...(conversationId ? { conversationId } : {}),
|
|
5854
|
+
});
|
|
5855
|
+
}
|
|
5399
5856
|
const githubModelResolution = usesGitHubCatalog
|
|
5400
5857
|
? resolveRequestedModel(inheritedModel ? inheritedModel.model : model)
|
|
5401
5858
|
: null;
|
|
@@ -5499,6 +5956,17 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5499
5956
|
code: 'IMAGE_TARGET_REQUIRES_IMAGE_MODEL',
|
|
5500
5957
|
});
|
|
5501
5958
|
}
|
|
5959
|
+
if ((imagePathsGiven || imageOptionsGiven) && !openAIImageModel) {
|
|
5960
|
+
return refuseImageInput(`Images and image options go to an OpenAI image session; this session runs ${requestedModel || 'no image model'}`);
|
|
5961
|
+
}
|
|
5962
|
+
// An image turn from another relay's agent takes its images from
|
|
5963
|
+
// `imagePaths` alone: an @file: token resolves against this relay's
|
|
5964
|
+
// workspace root, not the session's folder, and carries no stored copy.
|
|
5965
|
+
let ignoredReferenceAttachments = 0;
|
|
5966
|
+
if (remoteRequest.remote && openAIImageModel && referenceResolution.attachments.length) {
|
|
5967
|
+
ignoredReferenceAttachments = referenceResolution.attachments.length;
|
|
5968
|
+
attachments = mergeMessageAttachments(normalizedAttachments, []);
|
|
5969
|
+
}
|
|
5502
5970
|
// A send that names no effort runs with the conversation's own, clamped
|
|
5503
5971
|
// to the nearest level the model has. Null when there is nothing to keep
|
|
5504
5972
|
// or it neither matches nor clamps (an ultracode on a model without it):
|
|
@@ -5621,11 +6089,66 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5621
6089
|
// is already 1M.
|
|
5622
6090
|
normalizedContextTier = 'default';
|
|
5623
6091
|
}
|
|
6092
|
+
// An image turn's options: the session's defaults (update_session
|
|
6093
|
+
// `image`), then the preset the turn names (`image.preset`), then the
|
|
6094
|
+
// turn's own. The owner's composer names quality and size in the effort
|
|
6095
|
+
// and context-tier fields, which are the turn's own; an agent's turn
|
|
6096
|
+
// names them in `image`, and its effort counts below the defaults. Only
|
|
6097
|
+
// what an agent sent is refused here: the composer's choice goes on as
|
|
6098
|
+
// before, checked against the model when it runs. The options are
|
|
6099
|
+
// resolved here, at queue time: a preset changed later does not change a
|
|
6100
|
+
// queued turn.
|
|
6101
|
+
let imageOptions = null;
|
|
6102
|
+
if (openAIImageModel) {
|
|
6103
|
+
const { preset: presetName, own: ownImageInput } = splitOpenAIImagePreset(rawImageOptions);
|
|
6104
|
+
const preset = findOpenAIImagePreset(presetName, parseOwnOpenAIImagePresets(readAppSettingText(OPENAI_IMAGE_PRESETS_SETTING_KEY)));
|
|
6105
|
+
if (!preset.ok) return refuseImageInput(`${asSentence(preset.error)} Nothing was sent.`);
|
|
6106
|
+
const ownImageOptions = validateOpenAIImageOptions(ownImageInput, { model: requestedModel });
|
|
6107
|
+
if (!ownImageOptions.ok) return refuseImageInput(ownImageOptions.error);
|
|
6108
|
+
const keptQuality = !explicitReasoningEffort && requestedReasoningEffort && !['auto', 'none'].includes(requestedReasoningEffort)
|
|
6109
|
+
? { quality: requestedReasoningEffort }
|
|
6110
|
+
: {};
|
|
6111
|
+
const fieldOptions = {
|
|
6112
|
+
...(explicitReasoningEffort ? { quality: requestedReasoningEffort } : {}),
|
|
6113
|
+
...(String(contextTier || '').trim() && !['default', 'long_context'].includes(requestedContextTier) ? { size: normalizedContextTier } : {}),
|
|
6114
|
+
};
|
|
6115
|
+
let sessionDefaults = {};
|
|
6116
|
+
if (!shouldCreateConversation) {
|
|
6117
|
+
try {
|
|
6118
|
+
sessionDefaults = stmts.readConversationImageDefaults?.(conversationId) || {};
|
|
6119
|
+
} catch {}
|
|
6120
|
+
}
|
|
6121
|
+
const layers = remoteRequest.remote
|
|
6122
|
+
? [keptQuality, fieldOptions, sessionDefaults, preset.values, ownImageOptions.values]
|
|
6123
|
+
: [keptQuality, sessionDefaults, preset.values, fieldOptions, ownImageOptions.values];
|
|
6124
|
+
const merged = layers.reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
|
|
6125
|
+
const checked = validateOpenAIImageOptions(merged, { model: requestedModel });
|
|
6126
|
+
if (checked.ok) {
|
|
6127
|
+
imageOptions = mergeOpenAIImageOptions(checked.values, null);
|
|
6128
|
+
} else if (imagePathsGiven || imageOptionsGiven) {
|
|
6129
|
+
return refuseImageInput(checked.error);
|
|
6130
|
+
} else {
|
|
6131
|
+
imageOptions = [keptQuality, fieldOptions].reduce((options, layer) => mergeOpenAIImageOptions(options, layer), {});
|
|
6132
|
+
}
|
|
6133
|
+
// Today's image spend has reached the daily cap: refused for every
|
|
6134
|
+
// sender before anything is stored (the owner's override lifts it).
|
|
6135
|
+
let capRefusal = null;
|
|
6136
|
+
try {
|
|
6137
|
+
capRefusal = imageSpend.sendRefusal();
|
|
6138
|
+
} catch (error) {
|
|
6139
|
+
capRefusal = { code: OPENAI_IMAGE_DAILY_CAP_REACHED_CODE, error: `The relay could not check the daily cap of OpenAI images (${error?.message || error}). Nothing was sent.` };
|
|
6140
|
+
}
|
|
6141
|
+
if (capRefusal) {
|
|
6142
|
+
return res.status(409).json({ error: capRefusal.error, code: capRefusal.code, ...(conversationId ? { conversationId } : {}) });
|
|
6143
|
+
}
|
|
6144
|
+
}
|
|
5624
6145
|
if (!requestedRelayMode) return res.status(400).json({ error: 'Unsupported relay mode' });
|
|
5625
6146
|
const conversationWorkspaceState = (!shouldCreateConversation && typeof resolveConversationWorkspaceState === 'function')
|
|
5626
6147
|
? resolveConversationWorkspaceState({ conversationId })
|
|
5627
6148
|
: null;
|
|
5628
|
-
|
|
6149
|
+
// Images by path count as attachments: a "cd …" in their prompt moves
|
|
6150
|
+
// no folder, and they are checked against the folder the session has now.
|
|
6151
|
+
let workspaceRootUpdate = attachments.length === 0 && imagePathList.length === 0
|
|
5629
6152
|
? maybeApplyWorkspaceRootFromMessage(
|
|
5630
6153
|
trimmedText,
|
|
5631
6154
|
conversationWorkspaceState?.currentWorkspaceRootPath || null,
|
|
@@ -5695,6 +6218,56 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5695
6218
|
}
|
|
5696
6219
|
}
|
|
5697
6220
|
|
|
6221
|
+
// The images by path: every one checked and read before anything is
|
|
6222
|
+
// stored, then kept in the upload store, so each becomes an attachment
|
|
6223
|
+
// with its bytes' hash and sniffed type (and an asset of the operation).
|
|
6224
|
+
// They come first: the first image is the base of an edit.
|
|
6225
|
+
let imageInputs = [];
|
|
6226
|
+
let imageInputsDropped = 0;
|
|
6227
|
+
if (imagePathsGiven) {
|
|
6228
|
+
if (shouldCreateConversation) {
|
|
6229
|
+
return refuseImageInput('imagePaths need an existing session: create it first, then send the images to it');
|
|
6230
|
+
}
|
|
6231
|
+
let sessionStateRoot = '';
|
|
6232
|
+
try {
|
|
6233
|
+
sessionStateRoot = String(resolveSessionStateRoot?.() || '');
|
|
6234
|
+
} catch {}
|
|
6235
|
+
const roots = resolveImageInputRoots({
|
|
6236
|
+
cwd: conversationWorkspaceState?.currentWorkspaceRootPath || '',
|
|
6237
|
+
sessionStateRoot,
|
|
6238
|
+
deniedDirs: [...(imageInputDenyPaths?.dirs || []), uploadsDir, ...(uploadReadDirs || [])],
|
|
6239
|
+
deniedFiles: imageInputDenyPaths?.files || [],
|
|
6240
|
+
});
|
|
6241
|
+
const resolvedImages = resolveImageInputs(imagePathList, {
|
|
6242
|
+
roots,
|
|
6243
|
+
isLiveConversation: (id) => {
|
|
6244
|
+
const row = stmts.getConvAnyStatus?.get?.(id) || null;
|
|
6245
|
+
return !!row && String(row.status || '').trim() !== 'deleted';
|
|
6246
|
+
},
|
|
6247
|
+
});
|
|
6248
|
+
if (!resolvedImages.ok) return refuseImageInput(resolvedImages.error);
|
|
6249
|
+
const resolvedHashes = new Set(resolvedImages.images.map((image) => image.sha256));
|
|
6250
|
+
const otherAttachments = attachments.filter((attachment) => !resolvedHashes.has(String(attachment?.sha256 || '').trim().toLowerCase()));
|
|
6251
|
+
const total = resolvedImages.images.length + otherAttachments.length;
|
|
6252
|
+
if (total > OPENAI_IMAGE_LIMITS.inputImagesMax) {
|
|
6253
|
+
return refuseImageInput(`An image turn takes at most ${OPENAI_IMAGE_LIMITS.inputImagesMax} images; this one would have ${total} with the message's other attachments`);
|
|
6254
|
+
}
|
|
6255
|
+
try {
|
|
6256
|
+
imageInputs = resolvedImages.images.map((image) => ({
|
|
6257
|
+
...persistUploadBuffer(image.bytes, { name: image.name, type: image.mime }),
|
|
6258
|
+
// The type the bytes are, not the one an earlier upload of the
|
|
6259
|
+
// same bytes was stored with.
|
|
6260
|
+
type: image.mime,
|
|
6261
|
+
width: image.width,
|
|
6262
|
+
height: image.height,
|
|
6263
|
+
}));
|
|
6264
|
+
} catch (error) {
|
|
6265
|
+
return res.status(500).json({ code: 'IMAGE_INPUT_STORE_FAILED', error: `The images could not be stored on this relay: ${error?.message || error}` });
|
|
6266
|
+
}
|
|
6267
|
+
imageInputsDropped = imagePathList.length - imageInputs.length;
|
|
6268
|
+
attachments = [...imageInputs, ...otherAttachments];
|
|
6269
|
+
}
|
|
6270
|
+
|
|
5698
6271
|
const convId = shouldCreateConversation ? uuidv4() : conversationId;
|
|
5699
6272
|
if (shouldCreateConversation) {
|
|
5700
6273
|
// An agent's prompt opens with the sending relay's header line, which
|
|
@@ -5931,6 +6504,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5931
6504
|
null,
|
|
5932
6505
|
null,
|
|
5933
6506
|
);
|
|
6507
|
+
if (imageOptions) stmts.setQueueImageOptions?.run?.(JSON.stringify(imageOptions), msgId);
|
|
5934
6508
|
if (imageContinuityEnabled && openAIImageModel) {
|
|
5935
6509
|
if (!imageOperationService?.createEnqueuedOperation) {
|
|
5936
6510
|
throw new Error('Image operation service is unavailable');
|
|
@@ -5942,11 +6516,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
5942
6516
|
prompt: trimmedText,
|
|
5943
6517
|
selectedImageModel: requestedModel,
|
|
5944
6518
|
executionMode: 'direct_images',
|
|
5945
|
-
parameters: {
|
|
5946
|
-
quality: requestedReasoningEffort,
|
|
5947
|
-
size: normalizedContextTier,
|
|
5948
|
-
count: 1,
|
|
5949
|
-
},
|
|
6519
|
+
parameters: imageOptions || { count: 1 },
|
|
5950
6520
|
attachments,
|
|
5951
6521
|
imageTarget,
|
|
5952
6522
|
createdAt: now,
|
|
@@ -6111,8 +6681,17 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
6111
6681
|
workspaceRootWarning: workspaceRootUpdate.error || null,
|
|
6112
6682
|
workspaceRootChanged: !!workspaceRootUpdate.changed,
|
|
6113
6683
|
...workspaceRootPayload(),
|
|
6114
|
-
referenceAttachmentCount: referenceResolution.attachments.length,
|
|
6684
|
+
referenceAttachmentCount: referenceResolution.attachments.length - ignoredReferenceAttachments,
|
|
6115
6685
|
skippedReferenceAttachments: referenceResolution.skipped,
|
|
6686
|
+
...(ignoredReferenceAttachments ? { ignoredReferenceAttachments } : {}),
|
|
6687
|
+
// How many images from imagePaths the turn has, and how many named
|
|
6688
|
+
// twice (the same file, or the same bytes) were kept once.
|
|
6689
|
+
...(imagePathsGiven ? { imageInputs: imageInputs.length } : {}),
|
|
6690
|
+
...(imageInputsDropped > 0 ? {
|
|
6691
|
+
imageInputsDropped,
|
|
6692
|
+
imageInputsNote: `${imageInputsDropped} of the ${imagePathList.length} image paths named an image already attached; it was attached once.`,
|
|
6693
|
+
} : {}),
|
|
6694
|
+
...(imageOptions ? { imageOptions } : {}),
|
|
6116
6695
|
// Attachment references that no longer resolve (a deleted upload) are
|
|
6117
6696
|
// dropped silently by normalizeAttachments; the sender is told how many.
|
|
6118
6697
|
droppedAttachmentCount: Math.max(0, requestedAttachmentCount - normalizedAttachments.length),
|
|
@@ -7605,8 +8184,516 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7605
8184
|
return res.json({ ok: true, ...result });
|
|
7606
8185
|
});
|
|
7607
8186
|
|
|
8187
|
+
// Image operations whose OpenAI request runs in this process, and the
|
|
8188
|
+
// direct image requests (no operation) this process sent, by queue
|
|
8189
|
+
// message: a second run of either would be billed again.
|
|
8190
|
+
const executingImageOperations = new Set();
|
|
8191
|
+
const dispatchedImageMessages = new Map();
|
|
8192
|
+
const DISPATCHED_IMAGE_MESSAGES_MAX = 500;
|
|
8193
|
+
const openAIImageTimeoutFromConfig = (key, fallback) => (Number(config?.[key]) > 0 ? Number(config[key]) : fallback);
|
|
8194
|
+
const openAIImageGenerateTimeoutMs = openAIImageTimeoutFromConfig('openaiImageGenerateTimeoutMs', OPENAI_IMAGE_GENERATE_TIMEOUT_MS);
|
|
8195
|
+
const openAIImageEditTimeoutMs = openAIImageTimeoutFromConfig('openaiImageEditTimeoutMs', OPENAI_IMAGE_EDIT_TIMEOUT_MS);
|
|
8196
|
+
// The partial previews of the running image turns: files beside the
|
|
8197
|
+
// turn's final images, events to the owner's clients (every socket is the
|
|
8198
|
+
// owner's: share views read over HTTP and have none).
|
|
8199
|
+
const imagePreviews = createOpenAIImagePreviewTurns({
|
|
8200
|
+
resolveSessionStateRoot,
|
|
8201
|
+
sessionIdFor: (conversationId) => String(stmts.getConvAnyStatus?.get?.(conversationId)?.sdk_session_id || '').trim() || conversationId,
|
|
8202
|
+
emit: (event, payload) => io?.emit?.(event, payload),
|
|
8203
|
+
...(openAIImagePreviewLeftoverMs !== undefined ? { leftoverMs: openAIImagePreviewLeftoverMs } : {}),
|
|
8204
|
+
});
|
|
8205
|
+
|
|
8206
|
+
// A partial preview of a running image turn, for the owner's clients
|
|
8207
|
+
// only: only a file the turn wrote and still has is served (no path is
|
|
8208
|
+
// built from the request), never through a share link, and gone with the
|
|
8209
|
+
// turn.
|
|
8210
|
+
app.get(`/api/generated-image/:conversationId/:messageId/${OPENAI_IMAGE_PARTIALS_DIR}/:fileName`, auth, (req, res) => {
|
|
8211
|
+
const file = imagePreviews.partialFile({
|
|
8212
|
+
conversationId: String(req.params.conversationId || '').trim(),
|
|
8213
|
+
messageId: String(req.params.messageId || '').trim(),
|
|
8214
|
+
fileName: String(req.params.fileName || '').trim(),
|
|
8215
|
+
});
|
|
8216
|
+
// lstat: a link put in the turn's partials folder is not followed.
|
|
8217
|
+
let stat = null;
|
|
8218
|
+
try {
|
|
8219
|
+
stat = file ? fs.lstatSync(file.path) : null;
|
|
8220
|
+
} catch {}
|
|
8221
|
+
if (!stat?.isFile()) return res.status(404).json({ error: 'Preview not found' });
|
|
8222
|
+
return serveFileWithRangeSupport(req, res, file.path, { size: stat.size, contentType: file.mime }, {
|
|
8223
|
+
safeName: path.basename(file.path).replace(/"/g, ''),
|
|
8224
|
+
// And the open itself refuses a link swapped in after the lstat (POSIX;
|
|
8225
|
+
// Windows has no O_NOFOLLOW, the lstat is the guard there).
|
|
8226
|
+
fsImpl: PARTIAL_PREVIEW_FS,
|
|
8227
|
+
});
|
|
8228
|
+
});
|
|
8229
|
+
|
|
8230
|
+
function noteDispatchedImageMessage(messageId) {
|
|
8231
|
+
dispatchedImageMessages.delete(messageId);
|
|
8232
|
+
dispatchedImageMessages.set(messageId, { at: Date.now(), running: true });
|
|
8233
|
+
while (dispatchedImageMessages.size > DISPATCHED_IMAGE_MESSAGES_MAX) {
|
|
8234
|
+
dispatchedImageMessages.delete(dispatchedImageMessages.keys().next().value);
|
|
8235
|
+
}
|
|
8236
|
+
return dispatchedImageMessages.get(messageId);
|
|
8237
|
+
}
|
|
8238
|
+
|
|
8239
|
+
/**
|
|
8240
|
+
* One try of an OpenAI image request: `{ ok: true, generatedImages, usage }`
|
|
8241
|
+
* or `{ kind, status?, message?, detail? }` (see OPENAI_IMAGE_RETRY_KINDS).
|
|
8242
|
+
* The timeout covers the reply's body as well, a streamed one whole.
|
|
8243
|
+
* `fields` asking for a stream get one read event by event (a reply that is
|
|
8244
|
+
* no event stream is read as JSON); `onPartial` sees each partial image.
|
|
8245
|
+
*/
|
|
8246
|
+
async function sendOpenAIImageRequestOnce({ url, headers, fields, editImages, family, timeoutMs, outputFormat, model, onPartial = null }) {
|
|
8247
|
+
// The images asked for: a stream's events past them are dropped.
|
|
8248
|
+
const expectedImages = Math.max(1, Math.trunc(Number(fields.n) || 1));
|
|
8249
|
+
let response;
|
|
8250
|
+
try {
|
|
8251
|
+
let body;
|
|
8252
|
+
if (editImages.length) {
|
|
8253
|
+
body = new FormData();
|
|
8254
|
+
for (const [key, value] of Object.entries(fields)) body.set(key, String(value));
|
|
8255
|
+
// gpt-image takes every image, the first being the base; dall-e-2
|
|
8256
|
+
// takes one.
|
|
8257
|
+
const images = family === 'gpt-image' ? editImages : editImages.slice(0, 1);
|
|
8258
|
+
for (const image of images) {
|
|
8259
|
+
body.append(
|
|
8260
|
+
family === 'gpt-image' ? 'image[]' : 'image',
|
|
8261
|
+
new Blob([image.bytes], { type: image.mimeType }),
|
|
8262
|
+
image.name,
|
|
8263
|
+
);
|
|
8264
|
+
}
|
|
8265
|
+
} else {
|
|
8266
|
+
body = JSON.stringify(fields);
|
|
8267
|
+
}
|
|
8268
|
+
response = await fetch(url, {
|
|
8269
|
+
method: 'POST',
|
|
8270
|
+
headers: editImages.length ? headers : { ...headers, 'Content-Type': 'application/json' },
|
|
8271
|
+
body,
|
|
8272
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
8273
|
+
});
|
|
8274
|
+
} catch (error) {
|
|
8275
|
+
const kind = classifyOpenAIImageFetchError(error);
|
|
8276
|
+
return { kind, ownTimeout: error?.name === 'TimeoutError', detail: describeFetchError(error) };
|
|
8277
|
+
}
|
|
8278
|
+
|
|
8279
|
+
const status = Number(response.status);
|
|
8280
|
+
if (response.ok && fields.stream === true && isEventStreamResponse(response)) {
|
|
8281
|
+
return readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages });
|
|
8282
|
+
}
|
|
8283
|
+
let rawText = '';
|
|
8284
|
+
try {
|
|
8285
|
+
rawText = String(await response.text()).trim();
|
|
8286
|
+
} catch (error) {
|
|
8287
|
+
// A 2xx whose body broke off is billed; an error status keeps its class.
|
|
8288
|
+
if (response.ok) return { kind: 'unprocessed', status, detail: `the reply broke off: ${describeFetchError(error)}` };
|
|
8289
|
+
return status >= 500 ? { kind: 'server', status } : { kind: 'refused', status };
|
|
8290
|
+
}
|
|
8291
|
+
let payload = {};
|
|
8292
|
+
if (rawText) {
|
|
8293
|
+
try {
|
|
8294
|
+
payload = JSON.parse(rawText);
|
|
8295
|
+
} catch {
|
|
8296
|
+
// After a 2xx the generation is billed; a retry pays again. An error
|
|
8297
|
+
// status with a body that is not JSON stays a provider failure.
|
|
8298
|
+
if (response.ok) return { kind: 'unprocessed', status, detail: 'OpenAI image API returned invalid JSON' };
|
|
8299
|
+
payload = null;
|
|
8300
|
+
}
|
|
8301
|
+
}
|
|
8302
|
+
if (!response.ok) {
|
|
8303
|
+
const message = String(payload?.error?.message || payload?.message || (payload ? '' : rawText)).trim().slice(0, 240);
|
|
8304
|
+
if (!Number.isFinite(status) || status >= 500) return { kind: 'server', status, message };
|
|
8305
|
+
return { kind: 'refused', status, message };
|
|
8306
|
+
}
|
|
8307
|
+
try {
|
|
8308
|
+
const generatedImages = normalizeOpenAIImageApiGeneratedImages(payload, {
|
|
8309
|
+
maxImages: MAX_UPLOAD_ATTACHMENTS,
|
|
8310
|
+
outputFormat,
|
|
8311
|
+
});
|
|
8312
|
+
return { ok: true, generatedImages, usage: normalizeOpenAIImageUsage(payload?.usage) };
|
|
8313
|
+
} catch (error) {
|
|
8314
|
+
// The usage the reply carried is what was billed.
|
|
8315
|
+
const usage = normalizeOpenAIImageUsage(payload?.usage);
|
|
8316
|
+
// The provider answered 2xx, so the generation is billed; what failed
|
|
8317
|
+
// is the relay's handling of the reply. Logged with the stack: a retry
|
|
8318
|
+
// pays again and fails the same way.
|
|
8319
|
+
console.error(
|
|
8320
|
+
`[${ts()}] IMAGE reply of ${model} not processed: ${error?.message || error} (body ${rawText.length} chars)\n${error?.stack || ''}`,
|
|
8321
|
+
);
|
|
8322
|
+
return { kind: 'unprocessed', status, detail: error?.message || 'OpenAI image API returned no images', ...(usage ? { usage } : {}) };
|
|
8323
|
+
}
|
|
8324
|
+
}
|
|
8325
|
+
|
|
8326
|
+
/**
|
|
8327
|
+
* The rest of a streamed reply (after a 2xx with an event stream): each
|
|
8328
|
+
* partial goes to `onPartial`, each completed event's image is kept in
|
|
8329
|
+
* order (a count > 1 turn may get several), and the completed events'
|
|
8330
|
+
* usages add up to the request's. A stream that breaks off, ends without an image
|
|
8331
|
+
* or reports a server error after OpenAI started may have been billed:
|
|
8332
|
+
* 'stream', 'timeout' or 'server', tried once more. An error event that
|
|
8333
|
+
* names a refusal is 'refused'; an image the relay cannot read, or an event
|
|
8334
|
+
* past the reader's cap, 'unprocessed'.
|
|
8335
|
+
*
|
|
8336
|
+
* Once every image asked for (`expectedImages`) is complete, a stream that
|
|
8337
|
+
* breaks off or times out before its end still answers them: they are
|
|
8338
|
+
* billed, and a retry would buy them again. Partials past
|
|
8339
|
+
* OPENAI_IMAGE_PARTIAL_IMAGES per image or past the images asked for, and
|
|
8340
|
+
* completed events past those images, are dropped unread.
|
|
8341
|
+
*
|
|
8342
|
+
* Every result carries `partials` (received) and, once a completed event
|
|
8343
|
+
* was read, `usage`: a try that fails after it books that usage
|
|
8344
|
+
* (openAIImageTryCharge).
|
|
8345
|
+
*/
|
|
8346
|
+
async function readOpenAIImageStream(response, { status, outputFormat, model, onPartial, expectedImages = 1 }) {
|
|
8347
|
+
const completed = [];
|
|
8348
|
+
let usage = null;
|
|
8349
|
+
let partials = 0;
|
|
8350
|
+
// What every result carries for the booking.
|
|
8351
|
+
const counted = (result) => ({ ...result, partials, ...(usage ? { usage } : {}) });
|
|
8352
|
+
const answer = () => {
|
|
8353
|
+
try {
|
|
8354
|
+
const generatedImages = normalizeOpenAIImageApiGeneratedImages({
|
|
8355
|
+
data: completed.map((payload) => ({ b64_json: payload.b64_json })),
|
|
8356
|
+
output_format: completed[0].output_format,
|
|
8357
|
+
}, { maxImages: MAX_UPLOAD_ATTACHMENTS, outputFormat });
|
|
8358
|
+
return counted({ ok: true, generatedImages });
|
|
8359
|
+
} catch (error) {
|
|
8360
|
+
console.error(`[${ts()}] IMAGE streamed reply of ${model} not processed: ${error?.message || error}\n${error?.stack || ''}`);
|
|
8361
|
+
return counted({ kind: 'unprocessed', status, detail: error?.message || 'OpenAI image stream returned no images' });
|
|
8362
|
+
}
|
|
8363
|
+
};
|
|
8364
|
+
try {
|
|
8365
|
+
for await (const sse of readSseEvents(response.body)) {
|
|
8366
|
+
const event = interpretOpenAIImageStreamEvent(sse);
|
|
8367
|
+
if (event.type === 'partial') {
|
|
8368
|
+
partials += 1;
|
|
8369
|
+
const partialIndex = Number.isInteger(event.payload.partial_image_index) ? event.payload.partial_image_index : partials - 1;
|
|
8370
|
+
const shown = completed.length < expectedImages && partialIndex >= 0 && partialIndex < OPENAI_IMAGE_PARTIAL_IMAGES;
|
|
8371
|
+
if (onPartial && shown) {
|
|
8372
|
+
try {
|
|
8373
|
+
await onPartial({
|
|
8374
|
+
imageIndex: completed.length,
|
|
8375
|
+
partialIndex,
|
|
8376
|
+
b64: event.payload.b64_json,
|
|
8377
|
+
outputFormat: event.payload.output_format || outputFormat,
|
|
8378
|
+
});
|
|
8379
|
+
} catch (error) {
|
|
8380
|
+
console.warn(`[${ts()}] IMAGE partial of ${model} not shown: ${error?.message || error}`);
|
|
8381
|
+
}
|
|
8382
|
+
}
|
|
8383
|
+
} else if (event.type === 'completed') {
|
|
8384
|
+
if (completed.length >= expectedImages) continue;
|
|
8385
|
+
completed.push(event.payload);
|
|
8386
|
+
usage = addOpenAIImageUsage(usage, normalizeOpenAIImageUsage(event.payload.usage));
|
|
8387
|
+
} else if (event.type === 'invalid' && sse.event.endsWith('.completed')) {
|
|
8388
|
+
return counted({ kind: 'unprocessed', status, detail: 'OpenAI image stream sent an unreadable image' });
|
|
8389
|
+
} else if (event.type === 'error') {
|
|
8390
|
+
const { message, type, code } = event.error;
|
|
8391
|
+
const named = `${type} ${code}`.toLowerCase();
|
|
8392
|
+
const detail = `error event in the stream${code ? ` (${code})` : ''} after ${partials} partial image(s)`;
|
|
8393
|
+
if (['server', 'internal', 'overloaded', 'timeout', 'unavailable'].some((word) => named.includes(word)) || !named.trim()) {
|
|
8394
|
+
return counted({ kind: 'server', message, detail });
|
|
8395
|
+
}
|
|
8396
|
+
return counted({ kind: 'refused', status: 400, message, detail, streamed: true });
|
|
8397
|
+
}
|
|
8398
|
+
}
|
|
8399
|
+
} catch (error) {
|
|
8400
|
+
if (error?.code === SSE_EVENT_TOO_LARGE_CODE) return counted({ kind: 'unprocessed', status, detail: error.message });
|
|
8401
|
+
if (completed.length >= expectedImages) {
|
|
8402
|
+
console.warn(`[${ts()}] IMAGE ${model} stream broke off after its ${completed.length} image(s): kept (${describeFetchError(error)})`);
|
|
8403
|
+
return answer();
|
|
8404
|
+
}
|
|
8405
|
+
const kind = classifyOpenAIImageFetchError(error);
|
|
8406
|
+
return counted({
|
|
8407
|
+
kind: kind === 'timeout' ? 'timeout' : 'stream',
|
|
8408
|
+
ownTimeout: error?.name === 'TimeoutError',
|
|
8409
|
+
detail: `the stream broke off after ${partials} partial image(s): ${describeFetchError(error)}`,
|
|
8410
|
+
});
|
|
8411
|
+
}
|
|
8412
|
+
if (!completed.length) return counted({ kind: 'stream', detail: `the stream ended without an image after ${partials} partial image(s)` });
|
|
8413
|
+
return answer();
|
|
8414
|
+
}
|
|
8415
|
+
|
|
8416
|
+
/**
|
|
8417
|
+
* The OpenAI image request of one turn, for the execute route and the
|
|
8418
|
+
* generate route alike: the request with every option and every input
|
|
8419
|
+
* image, a timeout per try (generate / edit), and one retry of a try that
|
|
8420
|
+
* may pass the second time, with the same Idempotency-Key (OpenAI may
|
|
8421
|
+
* ignore it; the note says the request may have been billed). Never more
|
|
8422
|
+
* than two requests: nothing else may send this turn again.
|
|
8423
|
+
*
|
|
8424
|
+
* `requestBody` is normalizeOpenAIImageGenerateRequestBody's. Resolves
|
|
8425
|
+
* `{ ok: true, generatedImages, model, endpoint, usage, tries, note? }`
|
|
8426
|
+
* (`note`: openAIImageRetriedNote's) or openAIImageFailureResult's failure.
|
|
8427
|
+
*
|
|
8428
|
+
* A gpt-image request is streamed with OPENAI_IMAGE_PARTIAL_IMAGES partials
|
|
8429
|
+
* per image (dall-e models do not stream). With `preview` ({ conversationId,
|
|
8430
|
+
* messageId }: the queue message the turn answers) the owner's clients see
|
|
8431
|
+
* each try start and each partial; a retry drops the first try's, and the
|
|
8432
|
+
* turn's end (/api/response) the rest.
|
|
8433
|
+
*
|
|
8434
|
+
* Each try is checked against the daily cap and reserves its estimate
|
|
8435
|
+
* right before it is sent (the resend without stream fields keeps the
|
|
8436
|
+
* reservation); the cap refuses a try with OPENAI_IMAGE_DAILY_CAP_REACHED
|
|
8437
|
+
* and sends nothing. When the try ends, its cost is booked under the queue
|
|
8438
|
+
* row (`scopeId`: the image attempt, or the claim, that runs it; run_id
|
|
8439
|
+
* image:<scopeId>:try<n>) and its reservation dropped, in one step.
|
|
8440
|
+
*/
|
|
8441
|
+
async function runOpenAIImageRequest(requestBody, { idempotencyKey = '', preview = null, scopeId = '' } = {}) {
|
|
8442
|
+
const settings = getOpenAIProviderSettings();
|
|
8443
|
+
if (settings?.configured !== true || !String(settings?.apiKey || '').trim()) {
|
|
8444
|
+
return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI BYOK is not configured');
|
|
8445
|
+
}
|
|
8446
|
+
const baseUrl = String(settings.baseUrl || '').trim().replace(/\/+$/, '');
|
|
8447
|
+
if (!baseUrl) return openAIImageRequestRefusal(503, 'IMAGE_PROVIDER_NOT_CONFIGURED', 'OpenAI base URL is not configured');
|
|
8448
|
+
|
|
8449
|
+
let editImages = [];
|
|
8450
|
+
try {
|
|
8451
|
+
editImages = resolveOpenAIImageEditAttachments(requestBody.attachments, {
|
|
8452
|
+
maxImageBytes: MAX_UPLOAD_BYTES,
|
|
8453
|
+
allowedRootPath: uploadsDir,
|
|
8454
|
+
allowedBlobRootPaths: uploadReadDirs.slice(1),
|
|
8455
|
+
allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
|
|
8456
|
+
});
|
|
8457
|
+
} catch (error) {
|
|
8458
|
+
return openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', error?.message || 'Invalid image attachment');
|
|
8459
|
+
}
|
|
8460
|
+
|
|
8461
|
+
const family = openAIImageModelFamily(requestBody.model);
|
|
8462
|
+
const fields = buildOpenAIImageRequestFields(requestBody);
|
|
8463
|
+
const endpoint = editImages.length ? '/images/edits' : '/images/generations';
|
|
8464
|
+
const timeoutMs = editImages.length ? openAIImageEditTimeoutMs : openAIImageGenerateTimeoutMs;
|
|
8465
|
+
const headers = {
|
|
8466
|
+
Authorization: `Bearer ${settings.apiKey}`,
|
|
8467
|
+
...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
|
|
8468
|
+
};
|
|
8469
|
+
let streaming = family === 'gpt-image' && OPENAI_IMAGE_PARTIAL_IMAGES > 0;
|
|
8470
|
+
const previewTurn = preview
|
|
8471
|
+
? imagePreviews.begin({
|
|
8472
|
+
conversationId: preview.conversationId,
|
|
8473
|
+
messageId: preview.messageId,
|
|
8474
|
+
size: requestBody.size,
|
|
8475
|
+
quality: requestBody.quality,
|
|
8476
|
+
count: requestBody.n,
|
|
8477
|
+
model: requestBody.model,
|
|
8478
|
+
})
|
|
8479
|
+
: null;
|
|
8480
|
+
const tries = [];
|
|
8481
|
+
// HTTP requests of this turn, the resend without stream fields included:
|
|
8482
|
+
// never more than two.
|
|
8483
|
+
let calls = 0;
|
|
8484
|
+
let resent = false;
|
|
8485
|
+
let result = null;
|
|
8486
|
+
// What a try of this request costs, and where it books.
|
|
8487
|
+
const spendRequest = {
|
|
8488
|
+
model: requestBody.model,
|
|
8489
|
+
size: requestBody.size,
|
|
8490
|
+
quality: requestBody.quality,
|
|
8491
|
+
count: requestBody.n,
|
|
8492
|
+
inputImages: editImages.length,
|
|
8493
|
+
};
|
|
8494
|
+
const bookingScope = String(scopeId || '').trim() || `claim_${randomUUID()}`;
|
|
8495
|
+
// The try in flight's reservation: held across the resend, released
|
|
8496
|
+
// (with its booking, or without one) when the try ends.
|
|
8497
|
+
let reservation = null;
|
|
8498
|
+
previewTurn?.started(1);
|
|
8499
|
+
try {
|
|
8500
|
+
for (;;) {
|
|
8501
|
+
if (!reservation) {
|
|
8502
|
+
const tryNumber = tries.length + 1;
|
|
8503
|
+
const reserved = imageSpend.reserveTry({ estimateUsd: estimateOpenAIImageCost(spendRequest).costUsd });
|
|
8504
|
+
if (!reserved.ok) {
|
|
8505
|
+
console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tryNumber} stopped by the daily cap`);
|
|
8506
|
+
result = openAIImageCapRefusalResult(tries, reserved);
|
|
8507
|
+
break;
|
|
8508
|
+
}
|
|
8509
|
+
reservation = reserved.reservation;
|
|
8510
|
+
}
|
|
8511
|
+
calls += 1;
|
|
8512
|
+
const tried = await sendOpenAIImageRequestOnce({
|
|
8513
|
+
url: `${baseUrl}${endpoint}`,
|
|
8514
|
+
// The resend's body differs from the first request's: its own key.
|
|
8515
|
+
headers: resent && idempotencyKey ? { ...headers, 'Idempotency-Key': `${idempotencyKey}-nostream` } : headers,
|
|
8516
|
+
fields: streaming ? { ...fields, stream: true, partial_images: OPENAI_IMAGE_PARTIAL_IMAGES } : fields,
|
|
8517
|
+
editImages,
|
|
8518
|
+
family,
|
|
8519
|
+
timeoutMs,
|
|
8520
|
+
outputFormat: requestBody.output_format,
|
|
8521
|
+
model: requestBody.model,
|
|
8522
|
+
onPartial: previewTurn ? (partial) => previewTurn.partial(partial) : null,
|
|
8523
|
+
});
|
|
8524
|
+
// The resend without stream fields (below) is the same try: it keeps
|
|
8525
|
+
// the reservation and books once.
|
|
8526
|
+
const resend = streaming && calls < 2 && tried.kind === 'refused' && Number(tried.status) === 400 && !tried.streamed
|
|
8527
|
+
&& /\bstream|partial_image/i.test(String(tried.message || '').slice(0, 240));
|
|
8528
|
+
if (!resend) {
|
|
8529
|
+
imageSpend.settleTry(reservation, {
|
|
8530
|
+
conversationId: requestBody.conversationId,
|
|
8531
|
+
turnId: requestBody.messageId,
|
|
8532
|
+
runId: openAIImageTryRunId(bookingScope, tries.length + 1),
|
|
8533
|
+
model: requestBody.model,
|
|
8534
|
+
charge: openAIImageTryCharge({ tried, request: spendRequest }),
|
|
8535
|
+
});
|
|
8536
|
+
reservation = null;
|
|
8537
|
+
}
|
|
8538
|
+
if (tried.ok) {
|
|
8539
|
+
const note = openAIImageRetriedNote(tries, { timeoutMs });
|
|
8540
|
+
result = {
|
|
8541
|
+
ok: true,
|
|
8542
|
+
generatedImages: tried.generatedImages,
|
|
8543
|
+
model: requestBody.model,
|
|
8544
|
+
endpoint,
|
|
8545
|
+
usage: tried.usage,
|
|
8546
|
+
tries: tries.length + 1,
|
|
8547
|
+
...(note ? { note } : {}),
|
|
8548
|
+
};
|
|
8549
|
+
break;
|
|
8550
|
+
}
|
|
8551
|
+
// An endpoint that takes no stream (an OpenAI-compatible one, or a
|
|
8552
|
+
// model OpenAI does not stream) refuses the fields with a 400 before
|
|
8553
|
+
// any work: the same try again without them, nothing billed. It
|
|
8554
|
+
// counts as one of the two requests, so a resend that fails is final.
|
|
8555
|
+
// Any other refusal (a 429 that names streaming, say) is no reason.
|
|
8556
|
+
if (resend) {
|
|
8557
|
+
console.warn(`[${ts()}] IMAGE ${requestBody.model} ${endpoint} refused streaming; sent again without it`);
|
|
8558
|
+
streaming = false;
|
|
8559
|
+
resent = true;
|
|
8560
|
+
continue;
|
|
8561
|
+
}
|
|
8562
|
+
tries.push(tried);
|
|
8563
|
+
console.warn(
|
|
8564
|
+
`[${ts()}] IMAGE ${requestBody.model} ${endpoint} try=${tries.length} ${tried.kind}`
|
|
8565
|
+
+ `${tried.status ? ` status=${tried.status}` : ''}${tried.detail ? ` ${String(tried.detail).slice(0, 200)}` : ''}`,
|
|
8566
|
+
);
|
|
8567
|
+
if (calls >= 2 || !OPENAI_IMAGE_RETRY_KINDS.has(tried.kind)) break;
|
|
8568
|
+
await delay(openAIImageRetryDelayMs);
|
|
8569
|
+
previewTurn?.retry();
|
|
8570
|
+
previewTurn?.started(tries.length + 1);
|
|
8571
|
+
}
|
|
8572
|
+
} finally {
|
|
8573
|
+
// A throw or an abort: the try books nothing it did not book above.
|
|
8574
|
+
imageSpend.releaseTry(reservation);
|
|
8575
|
+
previewTurn?.ended({ ok: Boolean(result?.ok) });
|
|
8576
|
+
}
|
|
8577
|
+
return result || openAIImageFailureResult(tries, { timeoutMs, resent });
|
|
8578
|
+
}
|
|
8579
|
+
|
|
8580
|
+
/**
|
|
8581
|
+
* The reply line of an image turn: what came back, and what this turn's
|
|
8582
|
+
* tries were booked at ("~USD 0.21", marked "(estimate)" when any part is
|
|
8583
|
+
* one, "(price unknown, estimated)" for the stand-in); from 80 % of the
|
|
8584
|
+
* daily cap on, today's spend against it.
|
|
8585
|
+
*/
|
|
8586
|
+
function openAIImageReplyLine({ images, model, turnId }) {
|
|
8587
|
+
let cost = null;
|
|
8588
|
+
let state = null;
|
|
8589
|
+
try {
|
|
8590
|
+
cost = imageSpend.turnCost(turnId);
|
|
8591
|
+
state = imageSpend.buildState();
|
|
8592
|
+
} catch (error) {
|
|
8593
|
+
console.warn(`[${ts()}] IMAGE the cost of ${String(turnId || '').slice(0, 8)} could not be read: ${error?.message || error}`);
|
|
8594
|
+
}
|
|
8595
|
+
let line = formatOpenAIImageSummary({ images, model, costUsd: cost ? cost.costUsd : null });
|
|
8596
|
+
if (cost?.unknownPrice) line += ' (price unknown, estimated)';
|
|
8597
|
+
else if (cost?.estimated) line += ' (estimate)';
|
|
8598
|
+
// The spend state's own rule (booked and in flight, `warning`), as the
|
|
8599
|
+
// banner reads it.
|
|
8600
|
+
if (state?.capUsd && state.warning) {
|
|
8601
|
+
const todayUsd = (Number(state.spentUsd) || 0) + (Number(state.reservedUsd) || 0);
|
|
8602
|
+
line += ` · today ${formatOpenAIImageUsd(todayUsd)} of ${state.capUsd.toFixed(2)} (${Math.round((todayUsd / state.capUsd) * 100)} %)`;
|
|
8603
|
+
}
|
|
8604
|
+
return line;
|
|
8605
|
+
}
|
|
8606
|
+
|
|
8607
|
+
// Today's OpenAI image spend against the daily cap (the Check Usage card,
|
|
8608
|
+
// the banner, a paired relay's check before create_session); with
|
|
8609
|
+
// `?conversationId=` also that chat's.
|
|
8610
|
+
app.get(OPENAI_IMAGE_USAGE_PATH, auth, (req, res) => {
|
|
8611
|
+
const conversationId = String(req.query?.conversationId || '').trim().slice(0, 200) || null;
|
|
8612
|
+
try {
|
|
8613
|
+
return res.json(imageSpend.buildState({ conversationId }));
|
|
8614
|
+
} catch (error) {
|
|
8615
|
+
return res.status(500).json({ error: `Image spend could not be read: ${error?.message || error}` });
|
|
8616
|
+
}
|
|
8617
|
+
});
|
|
8618
|
+
|
|
8619
|
+
/**
|
|
8620
|
+
* An image attempt the recovery found cut off while its request ran
|
|
8621
|
+
* ('started', now 'uncertain'): it may have been billed, so its estimate
|
|
8622
|
+
* is booked as image:<attemptId>:recovered, unless a try of that attempt
|
|
8623
|
+
* was booked already.
|
|
8624
|
+
*/
|
|
8625
|
+
function bookRecoveredImageAttempt(operation, attempt) {
|
|
8626
|
+
if (!operation || !attempt?.id) return;
|
|
8627
|
+
let parameters = {};
|
|
8628
|
+
try {
|
|
8629
|
+
parameters = JSON.parse(operation.parameters_json || '{}') || {};
|
|
8630
|
+
} catch {}
|
|
8631
|
+
let inputImages = operation.parent_node_id ? 1 : 0;
|
|
8632
|
+
if (!operation.parent_node_id) {
|
|
8633
|
+
try {
|
|
8634
|
+
inputImages = (stmts.listAssets?.all?.(operation.id) || []).length;
|
|
8635
|
+
} catch {}
|
|
8636
|
+
}
|
|
8637
|
+
try {
|
|
8638
|
+
imageSpend.bookRecovered({
|
|
8639
|
+
conversationId: operation.conversation_id,
|
|
8640
|
+
turnId: operation.queue_message_id,
|
|
8641
|
+
attemptId: attempt.id,
|
|
8642
|
+
model: operation.selected_image_model,
|
|
8643
|
+
startedAt: attempt.started_at,
|
|
8644
|
+
request: { size: parameters.size, quality: parameters.quality, count: parameters.count ?? 1, inputImages },
|
|
8645
|
+
});
|
|
8646
|
+
} catch (error) {
|
|
8647
|
+
console.warn(`[${ts()}] IMAGE_OP the estimate of attempt=${attempt.id} was not booked: ${error?.message || error}`);
|
|
8648
|
+
}
|
|
8649
|
+
}
|
|
8650
|
+
|
|
8651
|
+
/**
|
|
8652
|
+
* Why a queue row may not go back to pending (/api/requeue): `{ running:
|
|
8653
|
+
* true }` while its image request runs in this process, `{ failure }`
|
|
8654
|
+
* (a terminalError) once its request went to OpenAI, or null for a row
|
|
8655
|
+
* whose request was never sent (requeueing it bills nothing).
|
|
8656
|
+
*/
|
|
8657
|
+
function imageTurnRequeueStop(queueRow) {
|
|
8658
|
+
const operationId = String(queueRow?.image_operation_id || '').trim();
|
|
8659
|
+
if (operationId) {
|
|
8660
|
+
if (executingImageOperations.has(operationId)) return { running: true };
|
|
8661
|
+
const attempt = stmts.getLatestAttempt?.get?.(operationId) || null;
|
|
8662
|
+
if (!attempt) return null;
|
|
8663
|
+
if (attempt.status === 'started') {
|
|
8664
|
+
stmts.finishAttempt.run({
|
|
8665
|
+
id: attempt.id,
|
|
8666
|
+
status: 'uncertain',
|
|
8667
|
+
providerRequestId: null,
|
|
8668
|
+
providerConversationId: null,
|
|
8669
|
+
providerResponseId: null,
|
|
8670
|
+
httpStatus: null,
|
|
8671
|
+
errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
8672
|
+
errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
8673
|
+
completedAt: new Date().toISOString(),
|
|
8674
|
+
});
|
|
8675
|
+
bookRecoveredImageAttempt(stmts.getOperation?.get?.(operationId), attempt);
|
|
8676
|
+
return { failure: { code: 'IMAGE_ATTEMPT_INTERRUPTED', message: OPENAI_IMAGE_INTERRUPTED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
|
|
8677
|
+
}
|
|
8678
|
+
return {
|
|
8679
|
+
failure: {
|
|
8680
|
+
code: String(attempt.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED',
|
|
8681
|
+
message: String(attempt.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE,
|
|
8682
|
+
guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
|
|
8683
|
+
},
|
|
8684
|
+
};
|
|
8685
|
+
}
|
|
8686
|
+
const dispatched = dispatchedImageMessages.get(String(queueRow?.id || ''));
|
|
8687
|
+
if (!dispatched) return null;
|
|
8688
|
+
if (dispatched.running) return { running: true };
|
|
8689
|
+
return { failure: { code: 'IMAGE_ATTEMPT_REPEATED', message: OPENAI_IMAGE_REPEATED_NOTE, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE } };
|
|
8690
|
+
}
|
|
8691
|
+
|
|
7608
8692
|
// Called by the relay's own worker on this machine; nothing that came in
|
|
7609
8693
|
// through a proxy or the tunnel may start a provider image request.
|
|
8694
|
+
// The request runs in this process (runOpenAIImageRequest): its timeout
|
|
8695
|
+
// and its one retry are the only ones. A failure ends the row with a note,
|
|
8696
|
+
// never with a requeue: a requeued image turn was billed again.
|
|
7610
8697
|
app.post('/api/image-operations/:operationId/execute', auth, async (req, res) => {
|
|
7611
8698
|
if (!isTrulyLocalRequest(req)) {
|
|
7612
8699
|
return res.status(403).json({ error: 'Image operation execution is reachable from this machine only' });
|
|
@@ -7626,6 +8713,11 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7626
8713
|
if (!queueRow || queueRow.image_operation_id !== operationId) {
|
|
7627
8714
|
return res.status(409).json({ error: 'Image operation queue linkage is unavailable' });
|
|
7628
8715
|
}
|
|
8716
|
+
// Delivered again while its request still runs here (a worker that
|
|
8717
|
+
// died and came back): that run finishes the row.
|
|
8718
|
+
if (executingImageOperations.has(operationId)) {
|
|
8719
|
+
return res.status(202).json({ ok: true, outcome: 'executing', operationId });
|
|
8720
|
+
}
|
|
7629
8721
|
|
|
7630
8722
|
const now = new Date().toISOString();
|
|
7631
8723
|
const selfHeaders = {
|
|
@@ -7633,6 +8725,23 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7633
8725
|
'Content-Type': 'application/json',
|
|
7634
8726
|
};
|
|
7635
8727
|
const selfBaseUrl = `http://127.0.0.1:${config.port}`;
|
|
8728
|
+
const publishFailure = async (terminalError) => {
|
|
8729
|
+
// `terminalError` is the field /api/response reads; under any other
|
|
8730
|
+
// name the failure was refused as an empty reply and the row stayed
|
|
8731
|
+
// processing until the relay recovered and re-ran it.
|
|
8732
|
+
const response = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
8733
|
+
method: 'POST',
|
|
8734
|
+
headers: selfHeaders,
|
|
8735
|
+
body: JSON.stringify({
|
|
8736
|
+
messageId: operation.queue_message_id,
|
|
8737
|
+
conversationId: operation.conversation_id,
|
|
8738
|
+
terminalError,
|
|
8739
|
+
}),
|
|
8740
|
+
}).catch(() => null);
|
|
8741
|
+
if (!response?.ok) {
|
|
8742
|
+
console.warn(`[${ts()}] IMAGE_OP failed to publish the failure of operation=${operationId}`);
|
|
8743
|
+
}
|
|
8744
|
+
};
|
|
7636
8745
|
const acquireAttempt = db.transaction(() => {
|
|
7637
8746
|
const current = stmts.getLatestAttempt.get(operationId);
|
|
7638
8747
|
if (current?.status === 'started') {
|
|
@@ -7650,15 +8759,19 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7650
8759
|
providerResponseId: null,
|
|
7651
8760
|
httpStatus: null,
|
|
7652
8761
|
errorCode: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
7653
|
-
errorMessage:
|
|
8762
|
+
errorMessage: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
7654
8763
|
completedAt: now,
|
|
7655
8764
|
});
|
|
7656
8765
|
return { acquired: false, attempt: current, stale: true };
|
|
7657
8766
|
}
|
|
8767
|
+
// An attempt that ended: its note was published, or publishing it
|
|
8768
|
+
// failed and the row came back. Either way the request is not sent
|
|
8769
|
+
// again.
|
|
8770
|
+
if (current) return { acquired: false, attempt: current, finished: true };
|
|
7658
8771
|
const attempt = {
|
|
7659
8772
|
id: `img_attempt_${uuidv4()}`,
|
|
7660
8773
|
operationId,
|
|
7661
|
-
attemptNumber:
|
|
8774
|
+
attemptNumber: 1,
|
|
7662
8775
|
provider: operation.provider,
|
|
7663
8776
|
capabilitySnapshotJson: JSON.stringify({
|
|
7664
8777
|
executionMode: operation.execution_mode,
|
|
@@ -7671,312 +8784,250 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
7671
8784
|
});
|
|
7672
8785
|
const acquired = acquireAttempt();
|
|
7673
8786
|
if (acquired.stale) {
|
|
7674
|
-
|
|
7675
|
-
|
|
7676
|
-
|
|
7677
|
-
|
|
7678
|
-
|
|
7679
|
-
|
|
7680
|
-
|
|
7681
|
-
// name the failure was refused as an empty reply and the row stayed
|
|
7682
|
-
// processing until the relay recovered and re-ran it.
|
|
7683
|
-
terminalError: {
|
|
7684
|
-
code: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
7685
|
-
message: 'Image generation may have completed before the relay was interrupted. It was not retried to avoid duplicate images.',
|
|
7686
|
-
retryable: false,
|
|
7687
|
-
},
|
|
7688
|
-
}),
|
|
7689
|
-
}).catch(() => null);
|
|
8787
|
+
bookRecoveredImageAttempt(operation, acquired.attempt);
|
|
8788
|
+
await publishFailure({
|
|
8789
|
+
code: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
8790
|
+
message: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
8791
|
+
guidance: OPENAI_IMAGE_FAILURE_GUIDANCE,
|
|
8792
|
+
retryable: false,
|
|
8793
|
+
});
|
|
7690
8794
|
return res.status(409).json({
|
|
7691
8795
|
ok: false,
|
|
7692
8796
|
outcome: 'uncertain',
|
|
7693
8797
|
operationId,
|
|
7694
|
-
|
|
8798
|
+
code: 'IMAGE_ATTEMPT_INTERRUPTED',
|
|
8799
|
+
error: OPENAI_IMAGE_INTERRUPTED_NOTE,
|
|
8800
|
+
});
|
|
8801
|
+
}
|
|
8802
|
+
if (acquired.finished) {
|
|
8803
|
+
const earlier = acquired.attempt;
|
|
8804
|
+
const message = String(earlier.error_message || '').trim() || OPENAI_IMAGE_REPEATED_NOTE;
|
|
8805
|
+
const code = String(earlier.error_code || '').trim() || 'IMAGE_ATTEMPT_REPEATED';
|
|
8806
|
+
await publishFailure({ code, message, guidance: OPENAI_IMAGE_FAILURE_GUIDANCE, retryable: false });
|
|
8807
|
+
return res.status(409).json({
|
|
8808
|
+
ok: false,
|
|
8809
|
+
outcome: ['failed_terminal', 'failed_retryable'].includes(earlier.status) ? 'terminal' : 'uncertain',
|
|
8810
|
+
operationId,
|
|
8811
|
+
code,
|
|
8812
|
+
error: message,
|
|
7695
8813
|
});
|
|
7696
8814
|
}
|
|
7697
8815
|
if (!acquired.acquired) {
|
|
7698
8816
|
return res.status(202).json({ ok: true, outcome: 'executing', operationId });
|
|
7699
8817
|
}
|
|
7700
8818
|
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
const
|
|
7704
|
-
const parentMessage = parentNode
|
|
7705
|
-
? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
|
|
7706
|
-
: null;
|
|
7707
|
-
const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
|
|
7708
|
-
String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
|
|
7709
|
-
));
|
|
7710
|
-
const generatedImage = parentAttachment?.generatedImage;
|
|
7711
|
-
const parentPath = resolveGeneratedImageReadPath({
|
|
7712
|
-
resolveSessionStateRoot,
|
|
7713
|
-
sessionId: generatedImage?.sessionId,
|
|
7714
|
-
relativePath: generatedImage?.relativePath,
|
|
7715
|
-
});
|
|
7716
|
-
if (!parentPath) {
|
|
8819
|
+
executingImageOperations.add(operationId);
|
|
8820
|
+
try {
|
|
8821
|
+
const finishAttempt = (result) => {
|
|
7717
8822
|
stmts.finishAttempt.run({
|
|
7718
8823
|
id: acquired.attempt.id,
|
|
7719
|
-
|
|
8824
|
+
// A request that never reached OpenAI (unreachable, or stopped by
|
|
8825
|
+
// the daily cap before it was sent) closes as retryable: nothing
|
|
8826
|
+
// was billed, and the recovery books nothing for it.
|
|
8827
|
+
status: result.outcome === 'uncertain'
|
|
8828
|
+
? 'uncertain'
|
|
8829
|
+
: ((result.code === 'IMAGE_PROVIDER_UNREACHABLE' || (result.code === OPENAI_IMAGE_DAILY_CAP_REACHED_CODE && !result.sent))
|
|
8830
|
+
? 'failed_retryable'
|
|
8831
|
+
: 'failed_terminal'),
|
|
7720
8832
|
providerRequestId: null,
|
|
7721
8833
|
providerConversationId: null,
|
|
7722
8834
|
providerResponseId: null,
|
|
7723
|
-
httpStatus:
|
|
7724
|
-
errorCode:
|
|
7725
|
-
errorMessage:
|
|
8835
|
+
httpStatus: result.upstreamStatus || null,
|
|
8836
|
+
errorCode: result.code,
|
|
8837
|
+
errorMessage: String(result.error || '').slice(0, 500),
|
|
7726
8838
|
completedAt: new Date().toISOString(),
|
|
7727
8839
|
});
|
|
8840
|
+
};
|
|
8841
|
+
const fail = async (result) => {
|
|
8842
|
+
finishAttempt(result);
|
|
8843
|
+
await publishFailure(openAIImageTerminalError(result));
|
|
7728
8844
|
return res.status(409).json({
|
|
7729
8845
|
ok: false,
|
|
7730
|
-
outcome:
|
|
7731
|
-
|
|
7732
|
-
|
|
8846
|
+
outcome: result.outcome,
|
|
8847
|
+
operationId,
|
|
8848
|
+
code: result.code,
|
|
8849
|
+
error: result.error,
|
|
8850
|
+
...(result.usage ? { usage: result.usage } : {}),
|
|
7733
8851
|
});
|
|
7734
|
-
}
|
|
7735
|
-
|
|
7736
|
-
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
|
|
8852
|
+
};
|
|
8853
|
+
|
|
8854
|
+
const requestAttachments = [];
|
|
8855
|
+
if (operation.parent_node_id) {
|
|
8856
|
+
const parentNode = stmts.getNode.get(operation.parent_node_id);
|
|
8857
|
+
const parentMessage = parentNode
|
|
8858
|
+
? db.prepare(`SELECT attachments FROM messages WHERE id = ?`).get(parentNode.assistant_message_id)
|
|
8859
|
+
: null;
|
|
8860
|
+
const parentAttachment = parseAttachments(parentMessage?.attachments).find((attachment) => (
|
|
8861
|
+
String(attachment?.generatedImage?.imageId || '').trim() === parentNode?.attachment_image_id
|
|
8862
|
+
));
|
|
8863
|
+
const generatedImage = parentAttachment?.generatedImage;
|
|
8864
|
+
const parentPath = resolveGeneratedImageReadPath({
|
|
8865
|
+
resolveSessionStateRoot,
|
|
8866
|
+
sessionId: generatedImage?.sessionId,
|
|
8867
|
+
relativePath: generatedImage?.relativePath,
|
|
8868
|
+
});
|
|
8869
|
+
if (!parentPath) {
|
|
8870
|
+
return await fail({
|
|
8871
|
+
...openAIImageRequestRefusal(400, 'IMAGE_PARENT_UNAVAILABLE', 'The parent generated image is unavailable'),
|
|
8872
|
+
upstreamStatus: 400,
|
|
8873
|
+
});
|
|
8874
|
+
}
|
|
7742
8875
|
requestAttachments.push({
|
|
7743
|
-
name:
|
|
7744
|
-
type:
|
|
7745
|
-
path:
|
|
8876
|
+
name: parentAttachment.name,
|
|
8877
|
+
type: parentAttachment.type,
|
|
8878
|
+
path: parentPath,
|
|
7746
8879
|
});
|
|
8880
|
+
} else {
|
|
8881
|
+
// In the order they were attached: the first is the base image.
|
|
8882
|
+
for (const asset of stmts.listAssets.all(operationId)) {
|
|
8883
|
+
requestAttachments.push({
|
|
8884
|
+
name: asset.original_name || 'reference-image',
|
|
8885
|
+
type: asset.media_type,
|
|
8886
|
+
path: uploadPathForSha(asset.content_sha256),
|
|
8887
|
+
});
|
|
8888
|
+
}
|
|
7747
8889
|
}
|
|
7748
|
-
}
|
|
7749
8890
|
|
|
7750
|
-
|
|
7751
|
-
|
|
7752
|
-
parameters =
|
|
7753
|
-
|
|
7754
|
-
|
|
7755
|
-
|
|
7756
|
-
|
|
7757
|
-
|
|
7758
|
-
|
|
7759
|
-
n: Number.isInteger(Number(parameters.count)) ? Number(parameters.count) : 1,
|
|
7760
|
-
attachments: requestAttachments,
|
|
7761
|
-
};
|
|
7762
|
-
if (parameters.size === 'auto' || /^\d{2,5}x\d{2,5}$/i.test(String(parameters.size || ''))) {
|
|
7763
|
-
generationBody.size = parameters.size;
|
|
7764
|
-
}
|
|
7765
|
-
if (/^(auto|standard|hd|low|medium|high)$/i.test(String(parameters.quality || ''))) {
|
|
7766
|
-
generationBody.quality = parameters.quality;
|
|
7767
|
-
}
|
|
7768
|
-
|
|
7769
|
-
let generatedPayload = null;
|
|
7770
|
-
try {
|
|
7771
|
-
const generatedResponse = await fetch(`${selfBaseUrl}${remotePath}/api/openai/images/generate`, {
|
|
7772
|
-
method: 'POST',
|
|
7773
|
-
headers: selfHeaders,
|
|
7774
|
-
body: JSON.stringify(generationBody),
|
|
7775
|
-
});
|
|
7776
|
-
generatedPayload = await generatedResponse.json().catch(() => ({}));
|
|
7777
|
-
if (!generatedResponse.ok) {
|
|
7778
|
-
const error = new Error(generatedPayload?.error || `OpenAI image request failed (${generatedResponse.status})`);
|
|
7779
|
-
error.status = generatedResponse.status;
|
|
7780
|
-
throw error;
|
|
7781
|
-
}
|
|
7782
|
-
const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
7783
|
-
method: 'POST',
|
|
7784
|
-
headers: selfHeaders,
|
|
7785
|
-
body: JSON.stringify({
|
|
8891
|
+
// {size, quality, count, output_format, background}; any may be
|
|
8892
|
+
// missing, count is 1 then.
|
|
8893
|
+
let parameters = {};
|
|
8894
|
+
try {
|
|
8895
|
+
parameters = JSON.parse(operation.parameters_json || '{}') || {};
|
|
8896
|
+
} catch {}
|
|
8897
|
+
let requestBody;
|
|
8898
|
+
try {
|
|
8899
|
+
requestBody = normalizeOpenAIImageGenerateRequestBody({
|
|
7786
8900
|
messageId: operation.queue_message_id,
|
|
7787
8901
|
conversationId: operation.conversation_id,
|
|
7788
|
-
|
|
7789
|
-
|
|
7790
|
-
|
|
7791
|
-
|
|
7792
|
-
|
|
7793
|
-
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
|
|
8902
|
+
model: operation.selected_image_model,
|
|
8903
|
+
prompt: operation.prompt,
|
|
8904
|
+
n: parameters.count ?? 1,
|
|
8905
|
+
size: parameters.size,
|
|
8906
|
+
quality: parameters.quality,
|
|
8907
|
+
output_format: parameters.output_format,
|
|
8908
|
+
background: parameters.background,
|
|
8909
|
+
attachments: requestAttachments,
|
|
8910
|
+
});
|
|
8911
|
+
} catch (error) {
|
|
8912
|
+
return await fail(openAIImageRequestRefusal(400, 'IMAGE_REQUEST_INVALID', String(error?.message || 'Invalid image request')));
|
|
7798
8913
|
}
|
|
7799
|
-
|
|
7800
|
-
|
|
7801
|
-
|
|
7802
|
-
|
|
7803
|
-
|
|
7804
|
-
? generatedPayload.generatedImages.length
|
|
7805
|
-
: 0,
|
|
7806
|
-
});
|
|
7807
|
-
} catch (error) {
|
|
7808
|
-
const status = Number(error?.status || 0);
|
|
7809
|
-
const terminal = status >= 400 && status < 500;
|
|
7810
|
-
const uncertain = !!generatedPayload?.generatedImages;
|
|
7811
|
-
const outcome = terminal ? 'terminal' : (uncertain ? 'uncertain' : 'retryable');
|
|
7812
|
-
stmts.finishAttempt.run({
|
|
7813
|
-
id: acquired.attempt.id,
|
|
7814
|
-
status: terminal ? 'failed_terminal' : (uncertain ? 'uncertain' : 'failed_retryable'),
|
|
7815
|
-
providerRequestId: null,
|
|
7816
|
-
providerConversationId: null,
|
|
7817
|
-
providerResponseId: null,
|
|
7818
|
-
httpStatus: status || null,
|
|
7819
|
-
errorCode: terminal ? 'IMAGE_PROVIDER_REJECTED' : (uncertain ? 'IMAGE_FINALIZATION_UNCERTAIN' : 'IMAGE_PROVIDER_UNAVAILABLE'),
|
|
7820
|
-
errorMessage: String(error?.message || 'Image operation failed').slice(0, 500),
|
|
7821
|
-
completedAt: new Date().toISOString(),
|
|
8914
|
+
|
|
8915
|
+
const result = await runOpenAIImageRequest(requestBody, {
|
|
8916
|
+
idempotencyKey: String(operation.idempotency_key || operation.id),
|
|
8917
|
+
preview: { conversationId: operation.conversation_id, messageId: operation.queue_message_id },
|
|
8918
|
+
scopeId: acquired.attempt.id,
|
|
7822
8919
|
});
|
|
7823
|
-
if (
|
|
7824
|
-
|
|
7825
|
-
|
|
7826
|
-
|
|
7827
|
-
|
|
7828
|
-
}).catch(() => null);
|
|
7829
|
-
if (!requeueResponse?.ok) {
|
|
7830
|
-
console.warn(`[${ts()}] IMAGE_OP failed to requeue retryable operation=${operationId}`);
|
|
7831
|
-
}
|
|
7832
|
-
} else {
|
|
7833
|
-
const failure = {
|
|
7834
|
-
code: outcome === 'uncertain' ? 'IMAGE_FINALIZATION_UNCERTAIN' : 'IMAGE_PROVIDER_REJECTED',
|
|
7835
|
-
message: String(error?.message || 'Image operation failed').slice(0, 500),
|
|
7836
|
-
retryable: false,
|
|
7837
|
-
};
|
|
7838
|
-
const failResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
8920
|
+
if (!result.ok) return await fail(result);
|
|
8921
|
+
|
|
8922
|
+
let finalizeError = null;
|
|
8923
|
+
try {
|
|
8924
|
+
const finalizeResponse = await fetch(`${selfBaseUrl}${remotePath}/api/response`, {
|
|
7839
8925
|
method: 'POST',
|
|
7840
8926
|
headers: selfHeaders,
|
|
7841
8927
|
body: JSON.stringify({
|
|
7842
8928
|
messageId: operation.queue_message_id,
|
|
7843
8929
|
conversationId: operation.conversation_id,
|
|
7844
|
-
|
|
8930
|
+
text: '',
|
|
8931
|
+
generatedImages: result.generatedImages,
|
|
8932
|
+
model: result.model || operation.selected_image_model,
|
|
8933
|
+
// A first try that may have been billed too: the reply says so.
|
|
8934
|
+
...(result.note ? { imageNote: result.note } : {}),
|
|
7845
8935
|
}),
|
|
7846
|
-
})
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
8936
|
+
});
|
|
8937
|
+
const finalized = await finalizeResponse.json().catch(() => ({}));
|
|
8938
|
+
if (!finalizeResponse.ok) finalizeError = finalized?.error || `Image finalization failed (${finalizeResponse.status})`;
|
|
8939
|
+
} catch (error) {
|
|
8940
|
+
finalizeError = error?.message || 'Image finalization failed';
|
|
7850
8941
|
}
|
|
7851
|
-
|
|
7852
|
-
|
|
7853
|
-
|
|
8942
|
+
if (finalizeError) {
|
|
8943
|
+
return await fail({
|
|
8944
|
+
ok: false,
|
|
8945
|
+
outcome: 'uncertain',
|
|
8946
|
+
code: 'IMAGE_FINALIZATION_UNCERTAIN',
|
|
8947
|
+
error: `The images OpenAI returned could not be saved (${finalizeError}). The generation was billed; the message was not retried.`,
|
|
8948
|
+
upstreamStatus: 200,
|
|
8949
|
+
usage: result.usage,
|
|
8950
|
+
});
|
|
8951
|
+
}
|
|
8952
|
+
return res.json({
|
|
8953
|
+
ok: true,
|
|
8954
|
+
outcome: 'completed',
|
|
7854
8955
|
operationId,
|
|
7855
|
-
|
|
8956
|
+
generatedImageCount: result.generatedImages.length,
|
|
8957
|
+
usage: result.usage,
|
|
8958
|
+
...(result.note ? { note: result.note } : {}),
|
|
7856
8959
|
});
|
|
8960
|
+
} finally {
|
|
8961
|
+
executingImageOperations.delete(operationId);
|
|
7857
8962
|
}
|
|
7858
8963
|
});
|
|
7859
8964
|
|
|
8965
|
+
// The relay's own workers call this on this machine for an image turn
|
|
8966
|
+
// without an operation (image continuity off). Like the execute route it
|
|
8967
|
+
// is local only: through the tunnel, or with a paired relay's token, it
|
|
8968
|
+
// would spend the owner's money outside the queue. One request per queue
|
|
8969
|
+
// message: a second call for the same message is refused.
|
|
7860
8970
|
app.post('/api/openai/images/generate', auth, async (req, res) => {
|
|
7861
8971
|
touchCli();
|
|
7862
|
-
|
|
7863
|
-
|
|
7864
|
-
return res.status(503).json({ error: 'OpenAI BYOK is not configured' });
|
|
8972
|
+
if (!isTrulyLocalRequest(req)) {
|
|
8973
|
+
return res.status(403).json({ error: 'OpenAI image requests are reachable from this machine only' });
|
|
7865
8974
|
}
|
|
7866
8975
|
|
|
7867
8976
|
let requestBody;
|
|
7868
8977
|
try {
|
|
7869
|
-
requestBody = normalizeOpenAIImageGenerateRequestBody(req.body
|
|
7870
|
-
maxImages: MAX_UPLOAD_ATTACHMENTS,
|
|
7871
|
-
});
|
|
8978
|
+
requestBody = normalizeOpenAIImageGenerateRequestBody(req.body);
|
|
7872
8979
|
} catch (error) {
|
|
7873
|
-
return res.status(400).json({
|
|
7874
|
-
|
|
7875
|
-
|
|
7876
|
-
|
|
7877
|
-
|
|
7878
|
-
|
|
7879
|
-
let editAttachment = null;
|
|
7880
|
-
if (Array.isArray(requestBody.attachments) && requestBody.attachments.length > 0) {
|
|
7881
|
-
try {
|
|
7882
|
-
editAttachment = resolveOpenAIImageEditAttachment(requestBody.attachments, {
|
|
7883
|
-
maxImageBytes: MAX_UPLOAD_BYTES,
|
|
7884
|
-
allowedRootPath: uploadsDir,
|
|
7885
|
-
allowedBlobRootPaths: uploadReadDirs.slice(1),
|
|
7886
|
-
allowedGeneratedImagesRootPath: resolveSessionStateRoot?.() || '',
|
|
7887
|
-
});
|
|
7888
|
-
} catch (error) {
|
|
7889
|
-
return res.status(400).json({ error: error?.message || 'Invalid image attachment' });
|
|
7890
|
-
}
|
|
7891
|
-
}
|
|
7892
|
-
|
|
7893
|
-
const options = {
|
|
7894
|
-
model: requestBody.model,
|
|
7895
|
-
prompt: requestBody.prompt,
|
|
7896
|
-
};
|
|
7897
|
-
const supportsResponseFormat = String(requestBody.model || '')
|
|
7898
|
-
.trim()
|
|
7899
|
-
.toLowerCase()
|
|
7900
|
-
.replace(/^openai\//, '')
|
|
7901
|
-
.startsWith('dall-e-');
|
|
7902
|
-
if (supportsResponseFormat) options.response_format = 'b64_json';
|
|
7903
|
-
if (requestBody.n !== undefined) options.n = requestBody.n;
|
|
7904
|
-
if (requestBody.size) options.size = requestBody.size;
|
|
7905
|
-
if (requestBody.quality) options.quality = requestBody.quality;
|
|
7906
|
-
|
|
7907
|
-
const endpoint = editAttachment ? '/images/edits' : '/images/generations';
|
|
7908
|
-
let response;
|
|
7909
|
-
try {
|
|
7910
|
-
if (editAttachment) {
|
|
7911
|
-
const form = new FormData();
|
|
7912
|
-
form.set('model', options.model);
|
|
7913
|
-
form.set('prompt', options.prompt);
|
|
7914
|
-
if (options.response_format) form.set('response_format', String(options.response_format));
|
|
7915
|
-
if (options.n !== undefined) form.set('n', String(options.n));
|
|
7916
|
-
if (options.size) form.set('size', String(options.size));
|
|
7917
|
-
if (options.quality) form.set('quality', String(options.quality));
|
|
7918
|
-
form.set(
|
|
7919
|
-
'image',
|
|
7920
|
-
new Blob([editAttachment.bytes], { type: editAttachment.mimeType }),
|
|
7921
|
-
editAttachment.name,
|
|
7922
|
-
);
|
|
7923
|
-
response = await fetch(`${baseUrl}${endpoint}`, {
|
|
7924
|
-
method: 'POST',
|
|
7925
|
-
headers: {
|
|
7926
|
-
Authorization: `Bearer ${settings.apiKey}`,
|
|
7927
|
-
},
|
|
7928
|
-
body: form,
|
|
7929
|
-
});
|
|
7930
|
-
} else {
|
|
7931
|
-
response = await fetch(`${baseUrl}${endpoint}`, {
|
|
7932
|
-
method: 'POST',
|
|
7933
|
-
headers: {
|
|
7934
|
-
Authorization: `Bearer ${settings.apiKey}`,
|
|
7935
|
-
'Content-Type': 'application/json',
|
|
7936
|
-
},
|
|
7937
|
-
body: JSON.stringify(options),
|
|
7938
|
-
});
|
|
7939
|
-
}
|
|
7940
|
-
} catch (error) {
|
|
7941
|
-
return res.status(502).json({ error: `OpenAI image API request failed: ${error?.message || 'request error'}` });
|
|
7942
|
-
}
|
|
7943
|
-
|
|
7944
|
-
const rawText = String(await response.text().catch(() => '')).trim();
|
|
7945
|
-
let payload = {};
|
|
7946
|
-
if (rawText) {
|
|
7947
|
-
try {
|
|
7948
|
-
payload = JSON.parse(rawText);
|
|
7949
|
-
} catch {
|
|
7950
|
-
return res.status(502).json({ error: 'OpenAI image API returned invalid JSON' });
|
|
7951
|
-
}
|
|
8980
|
+
return res.status(400).json({
|
|
8981
|
+
error: error?.message || 'Invalid OpenAI image request',
|
|
8982
|
+
code: 'IMAGE_REQUEST_INVALID',
|
|
8983
|
+
outcome: 'terminal',
|
|
8984
|
+
});
|
|
7952
8985
|
}
|
|
7953
|
-
|
|
7954
|
-
|
|
7955
|
-
|
|
7956
|
-
|
|
7957
|
-
|
|
7958
|
-
? upstreamStatus
|
|
7959
|
-
: 502;
|
|
7960
|
-
return res.status(relayStatus).json({
|
|
7961
|
-
error: `OpenAI image API failed (${response.status})${detail ? `: ${detail}` : ''}`,
|
|
7962
|
-
upstreamStatus: Number.isFinite(upstreamStatus) ? upstreamStatus : null,
|
|
8986
|
+
if (dispatchedImageMessages.has(requestBody.messageId)) {
|
|
8987
|
+
return res.status(409).json({
|
|
8988
|
+
error: OPENAI_IMAGE_REPEATED_NOTE,
|
|
8989
|
+
code: 'IMAGE_ATTEMPT_REPEATED',
|
|
8990
|
+
outcome: 'uncertain',
|
|
7963
8991
|
});
|
|
7964
8992
|
}
|
|
7965
8993
|
|
|
7966
|
-
|
|
8994
|
+
const dispatched = noteDispatchedImageMessage(requestBody.messageId);
|
|
8995
|
+
// One run of the message: the claim's attempt id (a restart claims the
|
|
8996
|
+
// row anew), so its tries never merge into an earlier run's rows.
|
|
8997
|
+
// Known gap: this path (image continuity off) keeps no attempt row, so a
|
|
8998
|
+
// try a relay restart cuts off is booked nowhere; the operation path's
|
|
8999
|
+
// recovery books such a try as an estimate. docs/providers.md says so.
|
|
9000
|
+
let claimId = '';
|
|
7967
9001
|
try {
|
|
7968
|
-
|
|
7969
|
-
|
|
9002
|
+
claimId = String(stmts.findQById?.get?.(requestBody.messageId)?.attempt_id || '').trim();
|
|
9003
|
+
} catch {}
|
|
9004
|
+
const result = await runOpenAIImageRequest(requestBody, {
|
|
9005
|
+
idempotencyKey: `oar-image-${requestBody.messageId}`,
|
|
9006
|
+
preview: { conversationId: requestBody.conversationId, messageId: requestBody.messageId },
|
|
9007
|
+
scopeId: claimId ? `claim_${claimId}` : '',
|
|
9008
|
+
});
|
|
9009
|
+
// A request that never reached OpenAI may be sent again.
|
|
9010
|
+
if (result.ok || result.sent) dispatched.running = false;
|
|
9011
|
+
else dispatchedImageMessages.delete(requestBody.messageId);
|
|
9012
|
+
|
|
9013
|
+
if (!result.ok) {
|
|
9014
|
+
return res.status(result.status).json({
|
|
9015
|
+
error: result.error,
|
|
9016
|
+
code: result.code,
|
|
9017
|
+
outcome: result.outcome,
|
|
9018
|
+
upstreamStatus: result.upstreamStatus,
|
|
9019
|
+
...(result.detail ? { detail: result.detail } : {}),
|
|
9020
|
+
...(result.relayProcessing ? { relayProcessing: true } : {}),
|
|
7970
9021
|
});
|
|
7971
|
-
} catch (error) {
|
|
7972
|
-
return res.status(502).json({ error: error?.message || 'OpenAI image API returned no images' });
|
|
7973
9022
|
}
|
|
7974
|
-
|
|
7975
9023
|
return res.json({
|
|
7976
9024
|
ok: true,
|
|
7977
|
-
generatedImages,
|
|
7978
|
-
model:
|
|
7979
|
-
endpoint,
|
|
9025
|
+
generatedImages: result.generatedImages,
|
|
9026
|
+
model: result.model,
|
|
9027
|
+
endpoint: result.endpoint,
|
|
9028
|
+
usage: result.usage,
|
|
9029
|
+
// The worker hands it to /api/response as `imageNote`.
|
|
9030
|
+
...(result.note ? { note: result.note } : {}),
|
|
7980
9031
|
});
|
|
7981
9032
|
});
|
|
7982
9033
|
|
|
@@ -8075,12 +9126,16 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8075
9126
|
console.log(`[${ts()}] FENCE ${messageId?.slice(0,8)} legacy unfenced response write`);
|
|
8076
9127
|
}
|
|
8077
9128
|
|
|
9129
|
+
// The partial previews of an image turn go once its reply or its
|
|
9130
|
+
// failure note is stored (or was already).
|
|
8078
9131
|
if (q && q.status === 'done') {
|
|
8079
9132
|
console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_done`);
|
|
9133
|
+
imagePreviews.finish(messageId, 'done');
|
|
8080
9134
|
return res.json({ ok: true, ignored: 'already_done' });
|
|
8081
9135
|
}
|
|
8082
9136
|
if (q && q.status === 'failed') {
|
|
8083
9137
|
console.log(`[${ts()}] RESPONSE ${messageId?.slice(0,8)} ignored=already_failed`);
|
|
9138
|
+
imagePreviews.finish(messageId, 'failed');
|
|
8084
9139
|
return res.json({ ok: true, ignored: 'already_failed' });
|
|
8085
9140
|
}
|
|
8086
9141
|
|
|
@@ -8185,6 +9240,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8185
9240
|
+ `${terminalFailure.functionCallId ? ` call=${terminalFailure.functionCallId}` : ''}`
|
|
8186
9241
|
+ `${terminalFailure.requestId ? ` req=${terminalFailure.requestId}` : ''}`,
|
|
8187
9242
|
);
|
|
9243
|
+
imagePreviews.finish(messageId, 'failed');
|
|
8188
9244
|
return res.json({ ok: true, terminal: true, code: terminalFailure.stableCode });
|
|
8189
9245
|
}
|
|
8190
9246
|
|
|
@@ -8192,7 +9248,7 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8192
9248
|
const hasPendingQuestion = !!stmts.findPendingQuestionByMessage?.get(messageId);
|
|
8193
9249
|
const hasRecentlyAnsweredQuestion = !!stmts.findRecentlyAnsweredQuestionByMessage?.get(messageId, holdCutoff);
|
|
8194
9250
|
const conversation = stmts.getConvAnyStatus?.get?.(targetConversationId) || null;
|
|
8195
|
-
|
|
9251
|
+
let resolvedText = await resolveRelayResponseText({
|
|
8196
9252
|
text: trimmedText,
|
|
8197
9253
|
conversation,
|
|
8198
9254
|
queueTimestamp: q?.timestamp || null,
|
|
@@ -8275,6 +9331,29 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8275
9331
|
} catch (error) {
|
|
8276
9332
|
return res.status(500).json({ error: error?.message || 'Failed to persist generated images' });
|
|
8277
9333
|
}
|
|
9334
|
+
// An image turn whose model wrote no text: the reply line says what came
|
|
9335
|
+
// back ("1 image · 1536x1024 · gpt-image-2"), so the chat and an agent
|
|
9336
|
+
// that reads text only see the same.
|
|
9337
|
+
const imageTurnModel = [resolvedAssistantModel, requestedModel].find((candidate) => isOpenAIImageModelId(candidate)) || '';
|
|
9338
|
+
const imageSummaryLine = Boolean(!resolvedText && generatedImageAttachments.length && (q?.image_operation_id || imageTurnModel));
|
|
9339
|
+
if (imageSummaryLine) {
|
|
9340
|
+
resolvedText = openAIImageReplyLine({
|
|
9341
|
+
images: generatedImageAttachments,
|
|
9342
|
+
model: imageTurnModel || resolvedAssistantModel || '',
|
|
9343
|
+
turnId: messageId,
|
|
9344
|
+
});
|
|
9345
|
+
}
|
|
9346
|
+
// The image request came through on its second try, and the first may
|
|
9347
|
+
// have been billed as well: the reply says so, on the reply line when
|
|
9348
|
+
// that is the text, so an agent that reads text only sees it too.
|
|
9349
|
+
const imageNote = generatedImageAttachments.length
|
|
9350
|
+
? String(req.body.imageNote || '').trim().slice(0, OPENAI_IMAGE_NOTE_MAX)
|
|
9351
|
+
: '';
|
|
9352
|
+
if (imageNote) {
|
|
9353
|
+
resolvedText = imageSummaryLine
|
|
9354
|
+
? `${resolvedText}. Note: ${imageNote}`
|
|
9355
|
+
: `${String(resolvedText || '').trimEnd()}${resolvedText ? '\n\n' : ''}Note: ${imageNote}`;
|
|
9356
|
+
}
|
|
8278
9357
|
const assistantAttachments = generatedImageAttachments.length ? JSON.stringify(generatedImageAttachments) : null;
|
|
8279
9358
|
const now = new Date().toISOString();
|
|
8280
9359
|
const imageOperation = q?.image_operation_id
|
|
@@ -8405,8 +9484,10 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8405
9484
|
// Ensure UI is updated even if the response was duplicate/late
|
|
8406
9485
|
if (currentStatus === 'done') {
|
|
8407
9486
|
io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
|
|
9487
|
+
imagePreviews.finish(messageId, 'done');
|
|
8408
9488
|
} else if (currentStatus === 'failed') {
|
|
8409
9489
|
io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'failed' });
|
|
9490
|
+
imagePreviews.finish(messageId, 'failed');
|
|
8410
9491
|
}
|
|
8411
9492
|
return res.json({ ok: true, ignored: 'not_pending_or_processing', actualStatus: currentStatus });
|
|
8412
9493
|
}
|
|
@@ -8555,6 +9636,8 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
8555
9636
|
},
|
|
8556
9637
|
});
|
|
8557
9638
|
io.emit('message_status', { messageId, conversationId: targetConversationId, status: 'done' });
|
|
9639
|
+
// The final images are in the chat: the previews go.
|
|
9640
|
+
imagePreviews.finish(messageId, generatedImageAttachments.length ? 'done' : 'failed');
|
|
8558
9641
|
if (usageLimitPause) {
|
|
8559
9642
|
try {
|
|
8560
9643
|
usageLimitPauseService.schedulePause(usageLimitPause, { queueRow: q });
|
|
@@ -9459,7 +10542,13 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
9459
10542
|
});
|
|
9460
10543
|
}
|
|
9461
10544
|
|
|
9462
|
-
|
|
10545
|
+
// An update in flight (done false) is volatile, as relay_stream: each one
|
|
10546
|
+
// carries the whole thought so far (up to MAX_THOUGHT_CHARS), so keeping
|
|
10547
|
+
// them in the connection-state-recovery buffer, or queueing them on a
|
|
10548
|
+
// socket that does not drain, holds a turn's worth of copies — gigabytes
|
|
10549
|
+
// for a few long reasoning turns at once. A reconnect resync reads the
|
|
10550
|
+
// stored snapshot instead; the final thought (done) is never dropped.
|
|
10551
|
+
(done ? io : io.volatile).emit('relay_thought', {
|
|
9463
10552
|
messageId,
|
|
9464
10553
|
conversationId,
|
|
9465
10554
|
mode: relayMode,
|
|
@@ -9731,6 +10820,41 @@ export function registerMessagesRoutes(app, deps) {
|
|
|
9731
10820
|
failConsumedQueueRow(q, { reason: 'requeue', attemptId: claimedAttemptId });
|
|
9732
10821
|
return res.json({ ok: true, terminal: true, code: STEER_SETTLE_FAILED_STABLE_CODE });
|
|
9733
10822
|
}
|
|
10823
|
+
if (q && q.status === 'processing') {
|
|
10824
|
+
// An image turn whose request went to OpenAI is never run again: a
|
|
10825
|
+
// second run is billed again. One still running here finishes the row
|
|
10826
|
+
// itself; one that ended, or was cut off, ends with the note.
|
|
10827
|
+
const imageStop = imageTurnRequeueStop(q);
|
|
10828
|
+
if (imageStop?.running) return res.json({ ok: true, held: 'image-request-running' });
|
|
10829
|
+
if (imageStop) {
|
|
10830
|
+
const failed = failQueueMessage({
|
|
10831
|
+
queueRow: q,
|
|
10832
|
+
messageId,
|
|
10833
|
+
conversationId: q.conversation_id,
|
|
10834
|
+
relayMode: normalizeRelayMode(q.relay_mode) || DEFAULT_RELAY_MODE,
|
|
10835
|
+
model: q.model || null,
|
|
10836
|
+
responseText: buildTerminalFailureTextForChat(imageStop.failure),
|
|
10837
|
+
failureRecord: {
|
|
10838
|
+
kind: 'terminal',
|
|
10839
|
+
error: 'image-not-requeued',
|
|
10840
|
+
code: normalizeTerminalErrorCode(imageStop.failure.code),
|
|
10841
|
+
stableCode: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}`,
|
|
10842
|
+
message: imageStop.failure.message,
|
|
10843
|
+
guidance: imageStop.failure.guidance,
|
|
10844
|
+
retryCount: Number(q.retry_count || 0),
|
|
10845
|
+
failedAt: new Date().toISOString(),
|
|
10846
|
+
},
|
|
10847
|
+
attemptId: claimedAttemptId,
|
|
10848
|
+
});
|
|
10849
|
+
if (failed) {
|
|
10850
|
+
settleRelayAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
|
|
10851
|
+
settleRelaySubagentAbortControlsForQueueMessage(messageId, { ok: false, error: 'queue-terminal-failure' });
|
|
10852
|
+
console.warn(`[${ts()}] FAILED ${messageId?.slice(0,8)} reason=image-not-requeued code=${imageStop.failure.code}`);
|
|
10853
|
+
imagePreviews.finish(messageId, 'failed');
|
|
10854
|
+
}
|
|
10855
|
+
return res.json({ ok: true, terminal: true, code: `relay.${normalizeTerminalErrorCode(imageStop.failure.code)}` });
|
|
10856
|
+
}
|
|
10857
|
+
}
|
|
9734
10858
|
if (q && q.status === 'processing' && String(req.body?.code || '').trim() === 'relay.provider-mismatch') {
|
|
9735
10859
|
// A routing refusal, not a delivery failure: the legacy relay handed
|
|
9736
10860
|
// back a turn that belongs to a session worker. No retry increment and
|