mixdog 0.9.69 → 0.9.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -1
- package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +18 -4
- package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +45 -12
- package/src/runtime/agent/orchestrator/session/store/listing.mjs +1 -0
- package/src/runtime/agent/orchestrator/session/store/paths-heartbeat.mjs +51 -18
- package/src/runtime/agent/orchestrator/session/store.mjs +3 -0
- package/src/runtime/channels/lib/owned-runtime.mjs +8 -1
- package/src/runtime/media/adapters/codex-image.mjs +109 -0
- package/src/runtime/media/adapters/gemini-image.mjs +68 -0
- package/src/runtime/media/adapters/gemini-video.mjs +119 -0
- package/src/runtime/media/adapters/xai-media.mjs +116 -0
- package/src/runtime/media/auth.mjs +51 -0
- package/src/runtime/media/index.mjs +18 -0
- package/src/runtime/media/jobs.mjs +174 -0
- package/src/runtime/media/lanes.mjs +230 -0
- package/src/runtime/media/renditions.mjs +203 -0
- package/src/runtime/media/store.mjs +356 -0
- package/src/runtime/media/store.test.mjs +103 -0
- package/src/runtime/media/upstream-error.mjs +39 -0
- package/src/session-runtime/channel-config-api.mjs +12 -1
- package/src/session-runtime/lifecycle-api.mjs +13 -4
- package/src/session-runtime/media-api.mjs +50 -0
- package/src/session-runtime/model-route-api.mjs +4 -1
- package/src/session-runtime/runtime-core.mjs +17 -1
- package/src/session-runtime/session-text.mjs +0 -1
- package/src/session-runtime/tool-catalog.mjs +32 -0
- package/src/session-runtime/warmup-schedulers.mjs +14 -1
- package/src/session-runtime/workflow-agents-api.mjs +29 -4
- package/src/tui/dist/index.mjs +21 -2
- package/src/tui/engine/session-api-ext.mjs +15 -0
- package/src/tui/engine.mjs +39 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixdog",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.71",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Standalone mixdog coding-agent CLI/TUI workspace.",
|
|
@@ -85,6 +85,9 @@
|
|
|
85
85
|
"test:session": "node --test scripts/session-orphan-sweep-test.mjs scripts/interrupted-turn-history-test.mjs scripts/session-heartbeat-lifecycle-test.mjs scripts/remote-transition-order-test.mjs",
|
|
86
86
|
"test:rebindtail": "node --test scripts/forwarder-rebind-tail-test.mjs",
|
|
87
87
|
"test:workflow-editor": "node --test scripts/workflow-id-test.mjs scripts/workflow-pack-editor-test.mjs",
|
|
88
|
+
"test:route-scope": "node --test scripts/route-scope-isolation-test.mjs",
|
|
89
|
+
"test:schedule-reload": "node --test scripts/schedule-reload-arm-test.mjs",
|
|
90
|
+
"test:media": "node --test src/runtime/media/store.test.mjs",
|
|
88
91
|
"failures": "node scripts/tool-failures.mjs",
|
|
89
92
|
"trace:llm": "node scripts/llm-trace-summary.mjs",
|
|
90
93
|
"diag:sessions": "node scripts/session-diag.mjs",
|
|
@@ -706,8 +706,22 @@ export function enqueueRemotePendingMessage(sessionId, message) {
|
|
|
706
706
|
}
|
|
707
707
|
|
|
708
708
|
// Spool-file mtime gate so the owner's idle poller costs one stat per tick,
|
|
709
|
-
// not a locked read-modify-write.
|
|
710
|
-
|
|
709
|
+
// not a locked read-modify-write. Keyed PER SESSION: one process can own
|
|
710
|
+
// several sessions (desktop tabs, TUI + engine hosts) and a single shared
|
|
711
|
+
// counter let the first drain of a tick swallow the mtime bump for every
|
|
712
|
+
// other session, stranding their foreign submits until the next spool write.
|
|
713
|
+
const FOREIGN_SPOOL_SCAN_LIMIT = 64;
|
|
714
|
+
const _foreignSpoolScanMtimes = new Map();
|
|
715
|
+
|
|
716
|
+
function _rememberForeignSpoolScan(sessionId, mtime) {
|
|
717
|
+
_foreignSpoolScanMtimes.delete(sessionId);
|
|
718
|
+
_foreignSpoolScanMtimes.set(sessionId, mtime);
|
|
719
|
+
while (_foreignSpoolScanMtimes.size > FOREIGN_SPOOL_SCAN_LIMIT) {
|
|
720
|
+
const oldest = _foreignSpoolScanMtimes.keys().next().value;
|
|
721
|
+
if (oldest === undefined) break;
|
|
722
|
+
_foreignSpoolScanMtimes.delete(oldest);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
711
725
|
|
|
712
726
|
/**
|
|
713
727
|
* Owner-side drain of FOREIGN user injections for a session this process
|
|
@@ -721,8 +735,8 @@ export function drainForeignUserInjections(sessionId) {
|
|
|
721
735
|
if (!isValidPendingSessionId(sessionId)) return [];
|
|
722
736
|
let mtime = 0;
|
|
723
737
|
try { mtime = statSync(pendingMessagesPath()).mtimeMs || 0; } catch { return []; }
|
|
724
|
-
if (
|
|
725
|
-
|
|
738
|
+
if (_foreignSpoolScanMtimes.get(sessionId) === mtime) return [];
|
|
739
|
+
_rememberForeignSpoolScan(sessionId, mtime);
|
|
726
740
|
const localIds = new Set();
|
|
727
741
|
for (const map of [_sessionPendingMessages, _pendingPersistBuffers, _hydratedPendingMessages]) {
|
|
728
742
|
for (const entry of map.get(sessionId) || []) {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import { getProvider } from '../../providers/registry.mjs';
|
|
7
7
|
import { normalizeCompactType, DEFAULT_COMPACT_TYPE } from '../compact.mjs';
|
|
8
8
|
import { collectPromptSkillsCached, buildSkillManifest, composeSystemPrompt } from '../../context/collect.mjs';
|
|
9
|
-
import { saveSession, saveSessionAsync, saveSessionAsyncDeferred, loadSession, setLiveSession, readSessionHeartbeatMtime, readSessionPresenceMtime, isSessionPresenceOwnerDead, deleteSessionPresence } from '../store.mjs';
|
|
9
|
+
import { saveSession, saveSessionAsync, saveSessionAsyncDeferred, loadSession, setLiveSession, readSessionHeartbeatMtime, readSessionPresenceMtime, isSessionPresenceOwnerDead, deleteSessionPresence, isSessionHeartbeatOwnerDead, readSessionHeartbeatOwnerPid, deleteHeartbeat, isProcessAlive } from '../store.mjs';
|
|
10
10
|
import { _getRuntimeEntry } from './runtime-liveness.mjs';
|
|
11
11
|
import { isAgentOwner } from '../../agent-owner.mjs';
|
|
12
12
|
import { getHiddenAgent } from '../../internal-agents.mjs';
|
|
@@ -547,6 +547,18 @@ export function prefetchSession(sessionId, preset = 'full') {
|
|
|
547
547
|
return true;
|
|
548
548
|
}
|
|
549
549
|
|
|
550
|
+
// Owner-pid hint for liveness signals that carry NO pid of their own:
|
|
551
|
+
// `session.lastHeartbeatAt` is persisted in the session file and therefore
|
|
552
|
+
// survives its writer forever, and pre-pid `.hb` sidecars only hold a
|
|
553
|
+
// timestamp. The recorded client host is the process that created/claimed the
|
|
554
|
+
// runtime for this session; the session-id prefix is the legacy fallback.
|
|
555
|
+
function _recordedOwnerPid(session, sessionId) {
|
|
556
|
+
const recorded = Number(session?.clientHostPid) || 0;
|
|
557
|
+
if (recorded > 0) return recorded;
|
|
558
|
+
const match = /^sess_(\d+)_/.exec(String(sessionId || ''));
|
|
559
|
+
return Number(match?.[1]) || 0;
|
|
560
|
+
}
|
|
561
|
+
|
|
550
562
|
function _isActivelyOwnedElsewhere(session, sessionId) {
|
|
551
563
|
// This process already owns the runtime for the id — switching back to
|
|
552
564
|
// one of our own sessions (desktop tab switch, TUI /resume) never attaches.
|
|
@@ -560,19 +572,40 @@ function _isActivelyOwnedElsewhere(session, sessionId) {
|
|
|
560
572
|
deleteSessionPresence(sessionId);
|
|
561
573
|
return false;
|
|
562
574
|
}
|
|
575
|
+
const now = Date.now();
|
|
576
|
+
// Presence (`.own`, pid-verified just above) covers the idle gaps between
|
|
577
|
+
// turns: a live interactive surface keeps refreshing it (~20s) for its
|
|
578
|
+
// CURRENT session, so cross-opening an idle-but-open session still
|
|
579
|
+
// attaches as a viewer instead of splitting ownership into two writers
|
|
580
|
+
// that clobber each other's saves.
|
|
581
|
+
const presenceAt = Number(readSessionPresenceMtime(sessionId)) || 0;
|
|
582
|
+
if (presenceAt > 0 && now - presenceAt <= ACTIVE_OWNER_HB_FRESH_MS) return true;
|
|
563
583
|
// Heartbeats publish only while a turn is running (≤5s cadence) and the
|
|
564
584
|
// sidecar is deleted on detach/close, so freshness here means another
|
|
565
|
-
// process is mid-conversation on this session right now
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
//
|
|
569
|
-
//
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
);
|
|
575
|
-
|
|
585
|
+
// process is mid-conversation on this session right now — PROVIDED that
|
|
586
|
+
// process still exists. Without the pid check a force-killed owner (app
|
|
587
|
+
// upgrade restart, crash) kept looking live for the whole freshness
|
|
588
|
+
// window, so every cross-open attached as a viewer and the user's
|
|
589
|
+
// messages spooled to a queue nobody drains (silently dropped 30m later).
|
|
590
|
+
if (isSessionHeartbeatOwnerDead(sessionId)) {
|
|
591
|
+
void deleteHeartbeat(sessionId);
|
|
592
|
+
return false;
|
|
593
|
+
}
|
|
594
|
+
const sidecarAt = Number(readSessionHeartbeatMtime(sessionId)) || 0;
|
|
595
|
+
if (sidecarAt > 0 && now - sidecarAt <= ACTIVE_OWNER_HB_FRESH_MS
|
|
596
|
+
&& readSessionHeartbeatOwnerPid(sessionId) > 0) {
|
|
597
|
+
// Fresh sidecar whose recorded pid is alive: a real owner is driving
|
|
598
|
+
// this session right now, whatever the session file remembers.
|
|
599
|
+
return true;
|
|
600
|
+
}
|
|
601
|
+
const heartbeatAt = Math.max(sidecarAt, Number(session.lastHeartbeatAt) || 0);
|
|
602
|
+
if (!(heartbeatAt > 0 && now - heartbeatAt <= ACTIVE_OWNER_HB_FRESH_MS)) return false;
|
|
603
|
+
// Pid-less evidence only (persisted field / legacy sidecar): fall back to
|
|
604
|
+
// the recorded client-host pid. A dead host means no owner; an unknown pid
|
|
605
|
+
// keeps the conservative attach.
|
|
606
|
+
const ownerPid = _recordedOwnerPid(session, sessionId);
|
|
607
|
+
if (ownerPid > 0 && !isProcessAlive(ownerPid)) return false;
|
|
608
|
+
return true;
|
|
576
609
|
}
|
|
577
610
|
|
|
578
611
|
// Viewer self-heal probe: true when a re-resume of this session would NO
|
|
@@ -10,6 +10,7 @@ import { resolveAgentTerminalReapMs } from '../../../../../session-runtime/confi
|
|
|
10
10
|
import { getStoreDir, sessionPath } from './paths-heartbeat.mjs';
|
|
11
11
|
import { isCancelledWrite as _isCancelledWrite } from './write-guards.mjs';
|
|
12
12
|
import {
|
|
13
|
+
SESSION_SUMMARY_INDEX_VERSION,
|
|
13
14
|
summaryIndexPath,
|
|
14
15
|
_sessionSummary,
|
|
15
16
|
_normalizeSummaryIndex,
|
|
@@ -19,13 +19,40 @@ export function sessionPath(id) {
|
|
|
19
19
|
return join(getStoreDir(), `${id}.json`);
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
// ── Sidecar owner-pid helpers ─────────────────────────────
|
|
23
|
+
// Liveness sidecars record the publishing process id on their SECOND line so
|
|
24
|
+
// a reader can tell a genuinely live owner from a force-killed one whose
|
|
25
|
+
// timestamp still looks fresh. Legacy (pid-less) sidecars read back as 0.
|
|
26
|
+
export function isProcessAlive(pid) {
|
|
27
|
+
const target = Number(pid) || 0;
|
|
28
|
+
if (target <= 0) return false;
|
|
29
|
+
try {
|
|
30
|
+
process.kill(target, 0);
|
|
31
|
+
return true;
|
|
32
|
+
} catch (error) {
|
|
33
|
+
// ESRCH: no such process. EPERM means the pid exists (alive).
|
|
34
|
+
return error?.code !== 'ESRCH';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function _readSidecarOwnerPid(path) {
|
|
39
|
+
try {
|
|
40
|
+
if (!existsSync(path)) return 0;
|
|
41
|
+
return Number(String(readFileSync(path, 'utf8')).split('\n')[1]) || 0;
|
|
42
|
+
} catch {
|
|
43
|
+
return 0;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
22
47
|
// ── Heartbeat publish ─────────────────────────────────────
|
|
23
48
|
// Lightweight per-session timestamp file (`<id>.hb`) consumed by the
|
|
24
49
|
// status aggregator for fresh-session detection. Decoupled from the
|
|
25
50
|
// full session JSON save so it can fire at a tight cadence (≤5s)
|
|
26
|
-
// without serialising the whole payload. The .hb file holds
|
|
27
|
-
//
|
|
28
|
-
//
|
|
51
|
+
// without serialising the whole payload. The .hb file holds
|
|
52
|
+
// `<msTimestamp>\n<pid>\n` — readers that only need liveness use the file
|
|
53
|
+
// mtime, while the attach-on-resume guard verifies the recorded pid.
|
|
54
|
+
// Aggregator scans the same sessions/ directory and matches `<id>.hb` to
|
|
55
|
+
// `<id>.json`.
|
|
29
56
|
const _HEARTBEAT_THROTTLE_MS = 5_000;
|
|
30
57
|
const _hbLastAt = new Map();
|
|
31
58
|
const _hbOperations = new Map();
|
|
@@ -56,7 +83,7 @@ export function publishHeartbeat(id, ts) {
|
|
|
56
83
|
}
|
|
57
84
|
const target = _heartbeatPath(id);
|
|
58
85
|
_hbLastAt.set(id, now);
|
|
59
|
-
return _queueHeartbeatOperation(id, () => fsp.writeFile(target, `${now}\n`, 'utf8'));
|
|
86
|
+
return _queueHeartbeatOperation(id, () => fsp.writeFile(target, `${now}\n${process.pid}\n`, 'utf8'));
|
|
60
87
|
}
|
|
61
88
|
|
|
62
89
|
export function deleteHeartbeat(id) {
|
|
@@ -91,6 +118,24 @@ export function listSessionHeartbeatMtimes() {
|
|
|
91
118
|
return result;
|
|
92
119
|
}
|
|
93
120
|
|
|
121
|
+
// Owner pid recorded in the `<id>.hb` sidecar (0 when absent or written by a
|
|
122
|
+
// pre-pid build).
|
|
123
|
+
export function readSessionHeartbeatOwnerPid(id) {
|
|
124
|
+
if (!id) return 0;
|
|
125
|
+
try { return _readSidecarOwnerPid(_heartbeatPath(id)); } catch { return 0; }
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// A killed owner never deletes its `.hb` sidecar, so bare mtime freshness
|
|
129
|
+
// would keep the session "actively driven" for the whole freshness window and
|
|
130
|
+
// every cross-open would attach as a viewer whose submits spool to nobody.
|
|
131
|
+
// The recorded pid is authoritative proof; pid-less legacy sidecars return
|
|
132
|
+
// false (unknown, not dead).
|
|
133
|
+
export function isSessionHeartbeatOwnerDead(id) {
|
|
134
|
+
const pid = readSessionHeartbeatOwnerPid(id);
|
|
135
|
+
if (!pid) return false;
|
|
136
|
+
return !isProcessAlive(pid);
|
|
137
|
+
}
|
|
138
|
+
|
|
94
139
|
// ── Interactive presence publish ──────────────────────────
|
|
95
140
|
// `<id>.own` marks a live interactive surface (TUI/desktop engine) HOLDING
|
|
96
141
|
// the session open — including idle time between turns. Kept separate from
|
|
@@ -144,19 +189,7 @@ export function readSessionPresenceMtime(id) {
|
|
|
144
189
|
export function isSessionPresenceOwnerDead(id) {
|
|
145
190
|
if (!id) return false;
|
|
146
191
|
let pid = 0;
|
|
147
|
-
try {
|
|
148
|
-
const path = _presencePath(id);
|
|
149
|
-
if (!existsSync(path)) return false;
|
|
150
|
-
pid = Number(String(readFileSync(path, 'utf8')).split('\n')[1]) || 0;
|
|
151
|
-
} catch {
|
|
152
|
-
return false;
|
|
153
|
-
}
|
|
192
|
+
try { pid = _readSidecarOwnerPid(_presencePath(id)); } catch { return false; }
|
|
154
193
|
if (!pid) return false;
|
|
155
|
-
|
|
156
|
-
process.kill(pid, 0);
|
|
157
|
-
return false;
|
|
158
|
-
} catch (error) {
|
|
159
|
-
// ESRCH: no such process. EPERM means the pid exists (alive).
|
|
160
|
-
return error?.code === 'ESRCH';
|
|
161
|
-
}
|
|
194
|
+
return !isProcessAlive(pid);
|
|
162
195
|
}
|
|
@@ -54,6 +54,9 @@ export {
|
|
|
54
54
|
deleteSessionPresence,
|
|
55
55
|
readSessionPresenceMtime,
|
|
56
56
|
isSessionPresenceOwnerDead,
|
|
57
|
+
readSessionHeartbeatOwnerPid,
|
|
58
|
+
isSessionHeartbeatOwnerDead,
|
|
59
|
+
isProcessAlive,
|
|
57
60
|
} from './store/paths-heartbeat.mjs';
|
|
58
61
|
import { _readStoredSessionCached } from './store/load-cache.mjs';
|
|
59
62
|
import { _sessionForDisk, _renameWithRetrySync, _ensureLifecycleFields, _storedSessionFromFile } from './store/serialize.mjs';
|
|
@@ -477,7 +477,14 @@ async function reloadRuntimeConfig() {
|
|
|
477
477
|
getConfig().interactive ?? [],
|
|
478
478
|
// Single resolved main-channel id used for the schedule `channel` flag.
|
|
479
479
|
getConfig().channelId,
|
|
480
|
-
|
|
480
|
+
// The scheduler must be RE-ARMED by the reload whenever it is supposed to
|
|
481
|
+
// be running. `restart: false` only destroys the cron/one-shot bindings and
|
|
482
|
+
// hands lifecycle back to the caller — but on an automation-only install
|
|
483
|
+
// (no messaging backend) startAutomationRuntime() is already past its
|
|
484
|
+
// `automationRunning` guard, so nobody would ever call scheduler.start()
|
|
485
|
+
// again and every saved/edited schedule stayed silently disarmed until the
|
|
486
|
+
// daemon restarted.
|
|
487
|
+
{ restart: automationRunning || getBridgeRuntimeConnected() }
|
|
481
488
|
);
|
|
482
489
|
const nextBackend = createBackend(getConfig());
|
|
483
490
|
const backendTypeChanged = (nextBackend?.name || "") !== previousBackendName;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ChatGPT OAuth image adapter (`openai-oauth` lane).
|
|
3
|
+
*
|
|
4
|
+
* Runs the hosted `image_generation` tool on the Codex responses backend — the
|
|
5
|
+
* same endpoint/headers the chat provider uses, so the subscription session is
|
|
6
|
+
* the only credential involved. The final image arrives as base64 on the
|
|
7
|
+
* image_generation_call item; partial frames are kept as a fallback.
|
|
8
|
+
*/
|
|
9
|
+
import { randomBytes } from 'crypto';
|
|
10
|
+
import { resolveCodexAuth } from '../auth.mjs';
|
|
11
|
+
import { mediaError } from '../lanes.mjs';
|
|
12
|
+
import { upstreamError } from '../upstream-error.mjs';
|
|
13
|
+
import { CODEX_OAUTH_ORIGINATOR, CODEX_RESPONSES_URL } from '../../agent/orchestrator/providers/openai-oauth.mjs';
|
|
14
|
+
|
|
15
|
+
const REQUEST_TIMEOUT_MS = 400_000;
|
|
16
|
+
|
|
17
|
+
function imageTool(options = {}) {
|
|
18
|
+
const tool = { type: 'image_generation' };
|
|
19
|
+
const size = String(options.size || 'auto');
|
|
20
|
+
if (size && size !== 'auto') tool.size = size;
|
|
21
|
+
const quality = String(options.quality || 'auto');
|
|
22
|
+
if (quality && quality !== 'auto') tool.quality = quality;
|
|
23
|
+
return tool;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function collectImage(event, state) {
|
|
27
|
+
const item = event?.item;
|
|
28
|
+
if (event?.type === 'response.output_item.done' && item?.type === 'image_generation_call') {
|
|
29
|
+
if (typeof item.result === 'string' && item.result.length > 0) state.final = item.result;
|
|
30
|
+
if (typeof item.revised_prompt === 'string') state.revisedPrompt = item.revised_prompt;
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (event?.type === 'response.image_generation_call.partial_image') {
|
|
34
|
+
const partial = event?.partial_image_b64;
|
|
35
|
+
if (typeof partial === 'string' && partial.length > (state.partial?.length || 0)) state.partial = partial;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function generateImage({ model, prompt, options = {}, references = [], signal }) {
|
|
40
|
+
const auth = await resolveCodexAuth();
|
|
41
|
+
// Reference images ride as input_image parts on the user turn — the same
|
|
42
|
+
// shape the chat path uses for pasted images.
|
|
43
|
+
const content = [
|
|
44
|
+
...references.map((ref) => ({
|
|
45
|
+
type: 'input_image',
|
|
46
|
+
image_url: `data:${ref.mime || 'image/png'};base64,${ref.base64}`,
|
|
47
|
+
})),
|
|
48
|
+
{ type: 'input_text', text: prompt },
|
|
49
|
+
];
|
|
50
|
+
const body = {
|
|
51
|
+
model,
|
|
52
|
+
stream: true,
|
|
53
|
+
store: false,
|
|
54
|
+
instructions: 'You generate images with the image_generation tool. Call the tool once for the user request; do not ask follow-up questions.',
|
|
55
|
+
input: [{ type: 'message', role: 'user', content }],
|
|
56
|
+
tools: [imageTool(options)],
|
|
57
|
+
tool_choice: 'auto',
|
|
58
|
+
};
|
|
59
|
+
const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
|
|
60
|
+
const res = await fetch(CODEX_RESPONSES_URL, {
|
|
61
|
+
method: 'POST',
|
|
62
|
+
headers: {
|
|
63
|
+
Authorization: `Bearer ${auth.access_token}`,
|
|
64
|
+
'Content-Type': 'application/json',
|
|
65
|
+
Accept: 'text/event-stream',
|
|
66
|
+
'OpenAI-Beta': 'responses=experimental',
|
|
67
|
+
originator: CODEX_OAUTH_ORIGINATOR,
|
|
68
|
+
'chatgpt-account-id': auth.account_id || '',
|
|
69
|
+
'x-client-request-id': randomBytes(16).toString('hex'),
|
|
70
|
+
},
|
|
71
|
+
body: JSON.stringify(body),
|
|
72
|
+
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
|
|
73
|
+
});
|
|
74
|
+
if (!res.ok || !res.body) throw upstreamError('ChatGPT image', res.status, await res.text().catch(() => ''));
|
|
75
|
+
|
|
76
|
+
const reader = res.body.getReader();
|
|
77
|
+
const decoder = new TextDecoder();
|
|
78
|
+
const state = { final: '', partial: '', revisedPrompt: null };
|
|
79
|
+
let buffer = '';
|
|
80
|
+
let failure = null;
|
|
81
|
+
try {
|
|
82
|
+
for (;;) {
|
|
83
|
+
const { done, value } = await reader.read();
|
|
84
|
+
if (done) break;
|
|
85
|
+
buffer += decoder.decode(value, { stream: true });
|
|
86
|
+
const lines = buffer.split('\n');
|
|
87
|
+
buffer = lines.pop() || '';
|
|
88
|
+
for (const line of lines) {
|
|
89
|
+
if (!line.startsWith('data:')) continue;
|
|
90
|
+
const raw = line.slice(5).trim();
|
|
91
|
+
if (!raw || raw === '[DONE]') continue;
|
|
92
|
+
let event;
|
|
93
|
+
try { event = JSON.parse(raw); } catch { continue; }
|
|
94
|
+
if (event?.type === 'response.failed' || event?.type === 'error') {
|
|
95
|
+
failure = event?.response?.error?.message || event?.message || 'stream failed';
|
|
96
|
+
}
|
|
97
|
+
collectImage(event, state);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
} finally {
|
|
101
|
+
try { reader.releaseLock(); } catch {}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const b64 = state.final || state.partial;
|
|
105
|
+
if (!b64) {
|
|
106
|
+
throw mediaError(`ChatGPT returned no image data${failure ? `: ${failure}` : ''}`, 'MEDIA_EMPTY_RESULT', 502);
|
|
107
|
+
}
|
|
108
|
+
return { bytes: Buffer.from(b64, 'base64'), mime: 'image/png', revisedPrompt: state.revisedPrompt };
|
|
109
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gemini image adapter (API key lane).
|
|
3
|
+
*
|
|
4
|
+
* Calls the Generative Language `generateContent` route and pulls the first
|
|
5
|
+
* inline image part out of the candidate. Aspect ratio rides in `imageConfig`,
|
|
6
|
+
* which older image models reject — a 400 there retries once without it so a
|
|
7
|
+
* control the model does not know never fails the whole generation.
|
|
8
|
+
*/
|
|
9
|
+
import { resolveGeminiKey } from '../auth.mjs';
|
|
10
|
+
import { mediaError } from '../lanes.mjs';
|
|
11
|
+
import { upstreamError } from '../upstream-error.mjs';
|
|
12
|
+
|
|
13
|
+
const BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/models';
|
|
14
|
+
const REQUEST_TIMEOUT_MS = 180_000;
|
|
15
|
+
|
|
16
|
+
function requestBody(prompt, options, withImageConfig, references = []) {
|
|
17
|
+
const body = {
|
|
18
|
+
contents: [{
|
|
19
|
+
role: 'user',
|
|
20
|
+
parts: [
|
|
21
|
+
...references.map((ref) => ({
|
|
22
|
+
inlineData: { mimeType: ref.mime || 'image/png', data: ref.base64 },
|
|
23
|
+
})),
|
|
24
|
+
{ text: prompt },
|
|
25
|
+
],
|
|
26
|
+
}],
|
|
27
|
+
};
|
|
28
|
+
const aspect = String(options?.aspectRatio || 'auto');
|
|
29
|
+
if (withImageConfig && aspect !== 'auto') {
|
|
30
|
+
body.generationConfig = { imageConfig: { aspectRatio: aspect } };
|
|
31
|
+
}
|
|
32
|
+
return body;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function post(model, key, body, signal) {
|
|
36
|
+
return await fetch(`${BASE_URL}/${encodeURIComponent(model)}:generateContent`, {
|
|
37
|
+
method: 'POST',
|
|
38
|
+
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': key },
|
|
39
|
+
body: JSON.stringify(body),
|
|
40
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)].filter(Boolean)),
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function generateImage({ model, prompt, options = {}, references = [], signal }) {
|
|
45
|
+
const key = resolveGeminiKey();
|
|
46
|
+
const wantsImageConfig = String(options.aspectRatio || 'auto') !== 'auto';
|
|
47
|
+
let res = await post(model, key, requestBody(prompt, options, wantsImageConfig, references), signal);
|
|
48
|
+
if (!res.ok && res.status === 400 && wantsImageConfig) {
|
|
49
|
+
res = await post(model, key, requestBody(prompt, options, false, references), signal);
|
|
50
|
+
}
|
|
51
|
+
if (!res.ok) throw upstreamError('Gemini image', res.status, await res.text().catch(() => ''));
|
|
52
|
+
const data = await res.json();
|
|
53
|
+
const parts = data?.candidates?.[0]?.content?.parts || [];
|
|
54
|
+
const image = parts.find((part) => part?.inlineData?.data);
|
|
55
|
+
if (!image) {
|
|
56
|
+
const refusal = parts.find((part) => typeof part?.text === 'string')?.text || '';
|
|
57
|
+
throw mediaError(
|
|
58
|
+
`Gemini returned no image data${refusal ? `: ${refusal.slice(0, 200)}` : ''}`,
|
|
59
|
+
'MEDIA_EMPTY_RESULT',
|
|
60
|
+
502,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
bytes: Buffer.from(image.inlineData.data, 'base64'),
|
|
65
|
+
mime: image.inlineData.mimeType || 'image/png',
|
|
66
|
+
revisedPrompt: null,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gemini video adapter (API key lane).
|
|
3
|
+
*
|
|
4
|
+
* Two different upstream shapes live behind one lane:
|
|
5
|
+
* - Omni Flash answers on the Interactions API in a single call, with the
|
|
6
|
+
* mp4 inline as base64 on a `model_output` step.
|
|
7
|
+
* - Veo 3.1 is a long-running predict: submit, poll the operation, then pull
|
|
8
|
+
* the sample URI with `alt=media`.
|
|
9
|
+
*/
|
|
10
|
+
import { resolveGeminiKey } from '../auth.mjs';
|
|
11
|
+
import { mediaError } from '../lanes.mjs';
|
|
12
|
+
import { upstreamError } from '../upstream-error.mjs';
|
|
13
|
+
|
|
14
|
+
const BASE_URL = 'https://generativelanguage.googleapis.com/v1beta';
|
|
15
|
+
const OMNI_TIMEOUT_MS = 600_000;
|
|
16
|
+
const POLL_INTERVAL_MS = 8_000;
|
|
17
|
+
const TOTAL_TIMEOUT_MS = 900_000;
|
|
18
|
+
|
|
19
|
+
export function isOmniModel(model) {
|
|
20
|
+
return /omni/i.test(String(model || ''));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function aspectFor(options) {
|
|
24
|
+
const aspect = String(options?.aspectRatio || 'auto');
|
|
25
|
+
return aspect === 'auto' ? '16:9' : aspect;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function generateViaOmni({ model, prompt, options, references = [], signal, key }) {
|
|
29
|
+
// Omni takes a multimodal input array; a reference image switches the task
|
|
30
|
+
// from text-to-video to image-to-video.
|
|
31
|
+
const input = references.length
|
|
32
|
+
? [
|
|
33
|
+
...references.map((ref) => ({ type: 'image', data: ref.base64, mime_type: ref.mime || 'image/png' })),
|
|
34
|
+
{ type: 'text', text: prompt },
|
|
35
|
+
]
|
|
36
|
+
: prompt;
|
|
37
|
+
const res = await fetch(`${BASE_URL}/interactions`, {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': key },
|
|
40
|
+
body: JSON.stringify({
|
|
41
|
+
model,
|
|
42
|
+
input,
|
|
43
|
+
response_format: { type: 'video', aspect_ratio: aspectFor(options) },
|
|
44
|
+
generation_config: {
|
|
45
|
+
video_config: { task: references.length ? 'image_to_video' : 'text_to_video' },
|
|
46
|
+
},
|
|
47
|
+
}),
|
|
48
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(OMNI_TIMEOUT_MS)].filter(Boolean)),
|
|
49
|
+
});
|
|
50
|
+
if (!res.ok) throw upstreamError('Gemini Omni video', res.status, await res.text().catch(() => ''));
|
|
51
|
+
const data = await res.json();
|
|
52
|
+
for (const step of data?.steps || []) {
|
|
53
|
+
const content = Array.isArray(step?.content) ? step.content : [step?.content].filter(Boolean);
|
|
54
|
+
const video = content.find((item) => item?.type === 'video' && typeof item?.data === 'string');
|
|
55
|
+
if (video) {
|
|
56
|
+
return { bytes: Buffer.from(video.data, 'base64'), mime: video.mime_type || 'video/mp4' };
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
throw mediaError('Gemini Omni returned no video data', 'MEDIA_EMPTY_RESULT', 502);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function generateViaVeo({ model, prompt, options, references = [], signal, onProgress, key }) {
|
|
63
|
+
const headers = { 'Content-Type': 'application/json', 'x-goog-api-key': key };
|
|
64
|
+
const parameters = { aspectRatio: aspectFor(options) };
|
|
65
|
+
const instance = { prompt };
|
|
66
|
+
// Veo accepts a single seed image for image-to-video.
|
|
67
|
+
if (references[0]) {
|
|
68
|
+
instance.image = { bytesBase64Encoded: references[0].base64, mimeType: references[0].mime || 'image/png' };
|
|
69
|
+
}
|
|
70
|
+
const resolution = String(options?.resolution || '');
|
|
71
|
+
if (resolution === '720p' || resolution === '1080p') parameters.resolution = resolution;
|
|
72
|
+
// Veo takes discrete clip lengths; anything else is rejected upstream, so an
|
|
73
|
+
// out-of-contract value is dropped rather than forwarded.
|
|
74
|
+
const duration = Math.trunc(Number(options?.duration) || 0);
|
|
75
|
+
if ([4, 6, 8].includes(duration)) parameters.durationSeconds = duration;
|
|
76
|
+
|
|
77
|
+
const started = await fetch(`${BASE_URL}/models/${encodeURIComponent(model)}:predictLongRunning`, {
|
|
78
|
+
method: 'POST',
|
|
79
|
+
headers,
|
|
80
|
+
body: JSON.stringify({ instances: [instance], parameters }),
|
|
81
|
+
signal,
|
|
82
|
+
});
|
|
83
|
+
if (!started.ok) throw upstreamError('Veo video', started.status, await started.text().catch(() => ''));
|
|
84
|
+
const operation = await started.json();
|
|
85
|
+
if (!operation?.name) throw mediaError('Veo returned no operation name', 'MEDIA_UPSTREAM_FAILED', 502);
|
|
86
|
+
|
|
87
|
+
const deadline = Date.now() + TOTAL_TIMEOUT_MS;
|
|
88
|
+
for (;;) {
|
|
89
|
+
if (signal?.aborted) throw mediaError('canceled', 'MEDIA_CANCELED', 499);
|
|
90
|
+
if (Date.now() > deadline) throw mediaError('Veo poll budget exceeded', 'MEDIA_TIMEOUT', 504);
|
|
91
|
+
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
92
|
+
const poll = await fetch(`${BASE_URL}/${operation.name}`, { headers, signal });
|
|
93
|
+
if (!poll.ok) throw upstreamError('Veo poll', poll.status, await poll.text().catch(() => ''));
|
|
94
|
+
const data = await poll.json();
|
|
95
|
+
if (!data?.done) {
|
|
96
|
+
// The operation exposes no percentage; keep the UI moving with a coarse
|
|
97
|
+
// heartbeat instead of a fake number.
|
|
98
|
+
if (typeof onProgress === 'function') onProgress(50);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (data.error) {
|
|
102
|
+
throw mediaError(`Veo generation failed: ${data.error.message || 'unknown error'}`, 'MEDIA_UPSTREAM_FAILED', 502);
|
|
103
|
+
}
|
|
104
|
+
const sample = data?.response?.generateVideoResponse?.generatedSamples?.[0]
|
|
105
|
+
|| data?.response?.generatedVideos?.[0];
|
|
106
|
+
const uri = sample?.video?.uri || sample?.video?.fileUri;
|
|
107
|
+
if (!uri) throw mediaError('Veo finished without a video URI', 'MEDIA_EMPTY_RESULT', 502);
|
|
108
|
+
const file = await fetch(`${uri}${uri.includes('?') ? '&' : '?'}alt=media`, { headers, signal });
|
|
109
|
+
if (!file.ok) throw upstreamError('Veo download', file.status, await file.text().catch(() => ''));
|
|
110
|
+
return { bytes: Buffer.from(await file.arrayBuffer()), mime: 'video/mp4' };
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export async function generateVideo({ model, prompt, options = {}, references = [], signal, onProgress }) {
|
|
115
|
+
const key = resolveGeminiKey();
|
|
116
|
+
return isOmniModel(model)
|
|
117
|
+
? await generateViaOmni({ model, prompt, options, references, signal, key })
|
|
118
|
+
: await generateViaVeo({ model, prompt, options, references, signal, onProgress, key });
|
|
119
|
+
}
|