mixdog 0.9.69 → 0.9.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/package.json +4 -1
  2. package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +18 -4
  3. package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +45 -12
  4. package/src/runtime/agent/orchestrator/session/store/listing.mjs +1 -0
  5. package/src/runtime/agent/orchestrator/session/store/paths-heartbeat.mjs +51 -18
  6. package/src/runtime/agent/orchestrator/session/store.mjs +3 -0
  7. package/src/runtime/channels/lib/owned-runtime.mjs +8 -1
  8. package/src/runtime/media/adapters/codex-image.mjs +109 -0
  9. package/src/runtime/media/adapters/gemini-image.mjs +68 -0
  10. package/src/runtime/media/adapters/gemini-video.mjs +119 -0
  11. package/src/runtime/media/adapters/xai-media.mjs +116 -0
  12. package/src/runtime/media/auth.mjs +51 -0
  13. package/src/runtime/media/index.mjs +18 -0
  14. package/src/runtime/media/jobs.mjs +174 -0
  15. package/src/runtime/media/lanes.mjs +230 -0
  16. package/src/runtime/media/renditions.mjs +203 -0
  17. package/src/runtime/media/store.mjs +356 -0
  18. package/src/runtime/media/store.test.mjs +103 -0
  19. package/src/runtime/media/upstream-error.mjs +39 -0
  20. package/src/session-runtime/channel-config-api.mjs +12 -1
  21. package/src/session-runtime/lifecycle-api.mjs +13 -4
  22. package/src/session-runtime/media-api.mjs +50 -0
  23. package/src/session-runtime/model-route-api.mjs +4 -1
  24. package/src/session-runtime/runtime-core.mjs +17 -1
  25. package/src/session-runtime/session-text.mjs +0 -1
  26. package/src/session-runtime/tool-catalog.mjs +32 -0
  27. package/src/session-runtime/warmup-schedulers.mjs +14 -1
  28. package/src/session-runtime/workflow-agents-api.mjs +29 -4
  29. package/src/tui/dist/index.mjs +21 -2
  30. package/src/tui/engine/session-api-ext.mjs +15 -0
  31. package/src/tui/engine.mjs +39 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mixdog",
3
- "version": "0.9.69",
3
+ "version": "0.9.71",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "Standalone mixdog coding-agent CLI/TUI workspace.",
@@ -85,6 +85,9 @@
85
85
  "test:session": "node --test scripts/session-orphan-sweep-test.mjs scripts/interrupted-turn-history-test.mjs scripts/session-heartbeat-lifecycle-test.mjs scripts/remote-transition-order-test.mjs",
86
86
  "test:rebindtail": "node --test scripts/forwarder-rebind-tail-test.mjs",
87
87
  "test:workflow-editor": "node --test scripts/workflow-id-test.mjs scripts/workflow-pack-editor-test.mjs",
88
+ "test:route-scope": "node --test scripts/route-scope-isolation-test.mjs",
89
+ "test:schedule-reload": "node --test scripts/schedule-reload-arm-test.mjs",
90
+ "test:media": "node --test src/runtime/media/store.test.mjs",
88
91
  "failures": "node scripts/tool-failures.mjs",
89
92
  "trace:llm": "node scripts/llm-trace-summary.mjs",
90
93
  "diag:sessions": "node scripts/session-diag.mjs",
@@ -706,8 +706,22 @@ export function enqueueRemotePendingMessage(sessionId, message) {
706
706
  }
707
707
 
708
708
  // Spool-file mtime gate so the owner's idle poller costs one stat per tick,
709
- // not a locked read-modify-write.
710
- let _foreignSpoolScanMtime = 0;
709
+ // not a locked read-modify-write. Keyed PER SESSION: one process can own
710
+ // several sessions (desktop tabs, TUI + engine hosts) and a single shared
711
+ // counter let the first drain of a tick swallow the mtime bump for every
712
+ // other session, stranding their foreign submits until the next spool write.
713
+ const FOREIGN_SPOOL_SCAN_LIMIT = 64;
714
+ const _foreignSpoolScanMtimes = new Map();
715
+
716
+ function _rememberForeignSpoolScan(sessionId, mtime) {
717
+ _foreignSpoolScanMtimes.delete(sessionId);
718
+ _foreignSpoolScanMtimes.set(sessionId, mtime);
719
+ while (_foreignSpoolScanMtimes.size > FOREIGN_SPOOL_SCAN_LIMIT) {
720
+ const oldest = _foreignSpoolScanMtimes.keys().next().value;
721
+ if (oldest === undefined) break;
722
+ _foreignSpoolScanMtimes.delete(oldest);
723
+ }
724
+ }
711
725
 
712
726
  /**
713
727
  * Owner-side drain of FOREIGN user injections for a session this process
@@ -721,8 +735,8 @@ export function drainForeignUserInjections(sessionId) {
721
735
  if (!isValidPendingSessionId(sessionId)) return [];
722
736
  let mtime = 0;
723
737
  try { mtime = statSync(pendingMessagesPath()).mtimeMs || 0; } catch { return []; }
724
- if (mtime === _foreignSpoolScanMtime) return [];
725
- _foreignSpoolScanMtime = mtime;
738
+ if (_foreignSpoolScanMtimes.get(sessionId) === mtime) return [];
739
+ _rememberForeignSpoolScan(sessionId, mtime);
726
740
  const localIds = new Set();
727
741
  for (const map of [_sessionPendingMessages, _pendingPersistBuffers, _hydratedPendingMessages]) {
728
742
  for (const entry of map.get(sessionId) || []) {
@@ -6,7 +6,7 @@
6
6
  import { getProvider } from '../../providers/registry.mjs';
7
7
  import { normalizeCompactType, DEFAULT_COMPACT_TYPE } from '../compact.mjs';
8
8
  import { collectPromptSkillsCached, buildSkillManifest, composeSystemPrompt } from '../../context/collect.mjs';
9
- import { saveSession, saveSessionAsync, saveSessionAsyncDeferred, loadSession, setLiveSession, readSessionHeartbeatMtime, readSessionPresenceMtime, isSessionPresenceOwnerDead, deleteSessionPresence } from '../store.mjs';
9
+ import { saveSession, saveSessionAsync, saveSessionAsyncDeferred, loadSession, setLiveSession, readSessionHeartbeatMtime, readSessionPresenceMtime, isSessionPresenceOwnerDead, deleteSessionPresence, isSessionHeartbeatOwnerDead, readSessionHeartbeatOwnerPid, deleteHeartbeat, isProcessAlive } from '../store.mjs';
10
10
  import { _getRuntimeEntry } from './runtime-liveness.mjs';
11
11
  import { isAgentOwner } from '../../agent-owner.mjs';
12
12
  import { getHiddenAgent } from '../../internal-agents.mjs';
@@ -547,6 +547,18 @@ export function prefetchSession(sessionId, preset = 'full') {
547
547
  return true;
548
548
  }
549
549
 
550
+ // Owner-pid hint for liveness signals that carry NO pid of their own:
551
+ // `session.lastHeartbeatAt` is persisted in the session file and therefore
552
+ // survives its writer forever, and pre-pid `.hb` sidecars only hold a
553
+ // timestamp. The recorded client host is the process that created/claimed the
554
+ // runtime for this session; the session-id prefix is the legacy fallback.
555
+ function _recordedOwnerPid(session, sessionId) {
556
+ const recorded = Number(session?.clientHostPid) || 0;
557
+ if (recorded > 0) return recorded;
558
+ const match = /^sess_(\d+)_/.exec(String(sessionId || ''));
559
+ return Number(match?.[1]) || 0;
560
+ }
561
+
550
562
  function _isActivelyOwnedElsewhere(session, sessionId) {
551
563
  // This process already owns the runtime for the id — switching back to
552
564
  // one of our own sessions (desktop tab switch, TUI /resume) never attaches.
@@ -560,19 +572,40 @@ function _isActivelyOwnedElsewhere(session, sessionId) {
560
572
  deleteSessionPresence(sessionId);
561
573
  return false;
562
574
  }
575
+ const now = Date.now();
576
+ // Presence (`.own`, pid-verified just above) covers the idle gaps between
577
+ // turns: a live interactive surface keeps refreshing it (~20s) for its
578
+ // CURRENT session, so cross-opening an idle-but-open session still
579
+ // attaches as a viewer instead of splitting ownership into two writers
580
+ // that clobber each other's saves.
581
+ const presenceAt = Number(readSessionPresenceMtime(sessionId)) || 0;
582
+ if (presenceAt > 0 && now - presenceAt <= ACTIVE_OWNER_HB_FRESH_MS) return true;
563
583
  // Heartbeats publish only while a turn is running (≤5s cadence) and the
564
584
  // sidecar is deleted on detach/close, so freshness here means another
565
- // process is mid-conversation on this session right now. Presence (`.own`)
566
- // covers the idle gaps between turns: a live interactive surface keeps
567
- // refreshing it (~20s) for its CURRENT session, so cross-opening an
568
- // idle-but-open session still attaches as a viewer instead of splitting
569
- // ownership into two writers that clobber each other's saves.
570
- const lastHb = Math.max(
571
- Number(readSessionHeartbeatMtime(sessionId)) || 0,
572
- Number(session.lastHeartbeatAt) || 0,
573
- Number(readSessionPresenceMtime(sessionId)) || 0,
574
- );
575
- return lastHb > 0 && Date.now() - lastHb <= ACTIVE_OWNER_HB_FRESH_MS;
585
+ // process is mid-conversation on this session right now PROVIDED that
586
+ // process still exists. Without the pid check a force-killed owner (app
587
+ // upgrade restart, crash) kept looking live for the whole freshness
588
+ // window, so every cross-open attached as a viewer and the user's
589
+ // messages spooled to a queue nobody drains (silently dropped 30m later).
590
+ if (isSessionHeartbeatOwnerDead(sessionId)) {
591
+ void deleteHeartbeat(sessionId);
592
+ return false;
593
+ }
594
+ const sidecarAt = Number(readSessionHeartbeatMtime(sessionId)) || 0;
595
+ if (sidecarAt > 0 && now - sidecarAt <= ACTIVE_OWNER_HB_FRESH_MS
596
+ && readSessionHeartbeatOwnerPid(sessionId) > 0) {
597
+ // Fresh sidecar whose recorded pid is alive: a real owner is driving
598
+ // this session right now, whatever the session file remembers.
599
+ return true;
600
+ }
601
+ const heartbeatAt = Math.max(sidecarAt, Number(session.lastHeartbeatAt) || 0);
602
+ if (!(heartbeatAt > 0 && now - heartbeatAt <= ACTIVE_OWNER_HB_FRESH_MS)) return false;
603
+ // Pid-less evidence only (persisted field / legacy sidecar): fall back to
604
+ // the recorded client-host pid. A dead host means no owner; an unknown pid
605
+ // keeps the conservative attach.
606
+ const ownerPid = _recordedOwnerPid(session, sessionId);
607
+ if (ownerPid > 0 && !isProcessAlive(ownerPid)) return false;
608
+ return true;
576
609
  }
577
610
 
578
611
  // Viewer self-heal probe: true when a re-resume of this session would NO
@@ -10,6 +10,7 @@ import { resolveAgentTerminalReapMs } from '../../../../../session-runtime/confi
10
10
  import { getStoreDir, sessionPath } from './paths-heartbeat.mjs';
11
11
  import { isCancelledWrite as _isCancelledWrite } from './write-guards.mjs';
12
12
  import {
13
+ SESSION_SUMMARY_INDEX_VERSION,
13
14
  summaryIndexPath,
14
15
  _sessionSummary,
15
16
  _normalizeSummaryIndex,
@@ -19,13 +19,40 @@ export function sessionPath(id) {
19
19
  return join(getStoreDir(), `${id}.json`);
20
20
  }
21
21
 
22
+ // ── Sidecar owner-pid helpers ─────────────────────────────
23
+ // Liveness sidecars record the publishing process id on their SECOND line so
24
+ // a reader can tell a genuinely live owner from a force-killed one whose
25
+ // timestamp still looks fresh. Legacy (pid-less) sidecars read back as 0.
26
+ export function isProcessAlive(pid) {
27
+ const target = Number(pid) || 0;
28
+ if (target <= 0) return false;
29
+ try {
30
+ process.kill(target, 0);
31
+ return true;
32
+ } catch (error) {
33
+ // ESRCH: no such process. EPERM means the pid exists (alive).
34
+ return error?.code !== 'ESRCH';
35
+ }
36
+ }
37
+
38
+ function _readSidecarOwnerPid(path) {
39
+ try {
40
+ if (!existsSync(path)) return 0;
41
+ return Number(String(readFileSync(path, 'utf8')).split('\n')[1]) || 0;
42
+ } catch {
43
+ return 0;
44
+ }
45
+ }
46
+
22
47
  // ── Heartbeat publish ─────────────────────────────────────
23
48
  // Lightweight per-session timestamp file (`<id>.hb`) consumed by the
24
49
  // status aggregator for fresh-session detection. Decoupled from the
25
50
  // full session JSON save so it can fire at a tight cadence (≤5s)
26
- // without serialising the whole payload. The .hb file holds a single
27
- // ASCII line: `<msTimestamp>\n`. Aggregator scans the same sessions/
28
- // directory and matches `<id>.hb` to `<id>.json`.
51
+ // without serialising the whole payload. The .hb file holds
52
+ // `<msTimestamp>\n<pid>\n` readers that only need liveness use the file
53
+ // mtime, while the attach-on-resume guard verifies the recorded pid.
54
+ // Aggregator scans the same sessions/ directory and matches `<id>.hb` to
55
+ // `<id>.json`.
29
56
  const _HEARTBEAT_THROTTLE_MS = 5_000;
30
57
  const _hbLastAt = new Map();
31
58
  const _hbOperations = new Map();
@@ -56,7 +83,7 @@ export function publishHeartbeat(id, ts) {
56
83
  }
57
84
  const target = _heartbeatPath(id);
58
85
  _hbLastAt.set(id, now);
59
- return _queueHeartbeatOperation(id, () => fsp.writeFile(target, `${now}\n`, 'utf8'));
86
+ return _queueHeartbeatOperation(id, () => fsp.writeFile(target, `${now}\n${process.pid}\n`, 'utf8'));
60
87
  }
61
88
 
62
89
  export function deleteHeartbeat(id) {
@@ -91,6 +118,24 @@ export function listSessionHeartbeatMtimes() {
91
118
  return result;
92
119
  }
93
120
 
121
+ // Owner pid recorded in the `<id>.hb` sidecar (0 when absent or written by a
122
+ // pre-pid build).
123
+ export function readSessionHeartbeatOwnerPid(id) {
124
+ if (!id) return 0;
125
+ try { return _readSidecarOwnerPid(_heartbeatPath(id)); } catch { return 0; }
126
+ }
127
+
128
+ // A killed owner never deletes its `.hb` sidecar, so bare mtime freshness
129
+ // would keep the session "actively driven" for the whole freshness window and
130
+ // every cross-open would attach as a viewer whose submits spool to nobody.
131
+ // The recorded pid is authoritative proof; pid-less legacy sidecars return
132
+ // false (unknown, not dead).
133
+ export function isSessionHeartbeatOwnerDead(id) {
134
+ const pid = readSessionHeartbeatOwnerPid(id);
135
+ if (!pid) return false;
136
+ return !isProcessAlive(pid);
137
+ }
138
+
94
139
  // ── Interactive presence publish ──────────────────────────
95
140
  // `<id>.own` marks a live interactive surface (TUI/desktop engine) HOLDING
96
141
  // the session open — including idle time between turns. Kept separate from
@@ -144,19 +189,7 @@ export function readSessionPresenceMtime(id) {
144
189
  export function isSessionPresenceOwnerDead(id) {
145
190
  if (!id) return false;
146
191
  let pid = 0;
147
- try {
148
- const path = _presencePath(id);
149
- if (!existsSync(path)) return false;
150
- pid = Number(String(readFileSync(path, 'utf8')).split('\n')[1]) || 0;
151
- } catch {
152
- return false;
153
- }
192
+ try { pid = _readSidecarOwnerPid(_presencePath(id)); } catch { return false; }
154
193
  if (!pid) return false;
155
- try {
156
- process.kill(pid, 0);
157
- return false;
158
- } catch (error) {
159
- // ESRCH: no such process. EPERM means the pid exists (alive).
160
- return error?.code === 'ESRCH';
161
- }
194
+ return !isProcessAlive(pid);
162
195
  }
@@ -54,6 +54,9 @@ export {
54
54
  deleteSessionPresence,
55
55
  readSessionPresenceMtime,
56
56
  isSessionPresenceOwnerDead,
57
+ readSessionHeartbeatOwnerPid,
58
+ isSessionHeartbeatOwnerDead,
59
+ isProcessAlive,
57
60
  } from './store/paths-heartbeat.mjs';
58
61
  import { _readStoredSessionCached } from './store/load-cache.mjs';
59
62
  import { _sessionForDisk, _renameWithRetrySync, _ensureLifecycleFields, _storedSessionFromFile } from './store/serialize.mjs';
@@ -477,7 +477,14 @@ async function reloadRuntimeConfig() {
477
477
  getConfig().interactive ?? [],
478
478
  // Single resolved main-channel id used for the schedule `channel` flag.
479
479
  getConfig().channelId,
480
- { restart: getBridgeRuntimeConnected() }
480
+ // The scheduler must be RE-ARMED by the reload whenever it is supposed to
481
+ // be running. `restart: false` only destroys the cron/one-shot bindings and
482
+ // hands lifecycle back to the caller — but on an automation-only install
483
+ // (no messaging backend) startAutomationRuntime() is already past its
484
+ // `automationRunning` guard, so nobody would ever call scheduler.start()
485
+ // again and every saved/edited schedule stayed silently disarmed until the
486
+ // daemon restarted.
487
+ { restart: automationRunning || getBridgeRuntimeConnected() }
481
488
  );
482
489
  const nextBackend = createBackend(getConfig());
483
490
  const backendTypeChanged = (nextBackend?.name || "") !== previousBackendName;
@@ -0,0 +1,109 @@
1
+ /**
2
+ * ChatGPT OAuth image adapter (`openai-oauth` lane).
3
+ *
4
+ * Runs the hosted `image_generation` tool on the Codex responses backend — the
5
+ * same endpoint/headers the chat provider uses, so the subscription session is
6
+ * the only credential involved. The final image arrives as base64 on the
7
+ * image_generation_call item; partial frames are kept as a fallback.
8
+ */
9
+ import { randomBytes } from 'crypto';
10
+ import { resolveCodexAuth } from '../auth.mjs';
11
+ import { mediaError } from '../lanes.mjs';
12
+ import { upstreamError } from '../upstream-error.mjs';
13
+ import { CODEX_OAUTH_ORIGINATOR, CODEX_RESPONSES_URL } from '../../agent/orchestrator/providers/openai-oauth.mjs';
14
+
15
+ const REQUEST_TIMEOUT_MS = 400_000;
16
+
17
+ function imageTool(options = {}) {
18
+ const tool = { type: 'image_generation' };
19
+ const size = String(options.size || 'auto');
20
+ if (size && size !== 'auto') tool.size = size;
21
+ const quality = String(options.quality || 'auto');
22
+ if (quality && quality !== 'auto') tool.quality = quality;
23
+ return tool;
24
+ }
25
+
26
+ function collectImage(event, state) {
27
+ const item = event?.item;
28
+ if (event?.type === 'response.output_item.done' && item?.type === 'image_generation_call') {
29
+ if (typeof item.result === 'string' && item.result.length > 0) state.final = item.result;
30
+ if (typeof item.revised_prompt === 'string') state.revisedPrompt = item.revised_prompt;
31
+ return;
32
+ }
33
+ if (event?.type === 'response.image_generation_call.partial_image') {
34
+ const partial = event?.partial_image_b64;
35
+ if (typeof partial === 'string' && partial.length > (state.partial?.length || 0)) state.partial = partial;
36
+ }
37
+ }
38
+
39
+ export async function generateImage({ model, prompt, options = {}, references = [], signal }) {
40
+ const auth = await resolveCodexAuth();
41
+ // Reference images ride as input_image parts on the user turn — the same
42
+ // shape the chat path uses for pasted images.
43
+ const content = [
44
+ ...references.map((ref) => ({
45
+ type: 'input_image',
46
+ image_url: `data:${ref.mime || 'image/png'};base64,${ref.base64}`,
47
+ })),
48
+ { type: 'input_text', text: prompt },
49
+ ];
50
+ const body = {
51
+ model,
52
+ stream: true,
53
+ store: false,
54
+ instructions: 'You generate images with the image_generation tool. Call the tool once for the user request; do not ask follow-up questions.',
55
+ input: [{ type: 'message', role: 'user', content }],
56
+ tools: [imageTool(options)],
57
+ tool_choice: 'auto',
58
+ };
59
+ const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
60
+ const res = await fetch(CODEX_RESPONSES_URL, {
61
+ method: 'POST',
62
+ headers: {
63
+ Authorization: `Bearer ${auth.access_token}`,
64
+ 'Content-Type': 'application/json',
65
+ Accept: 'text/event-stream',
66
+ 'OpenAI-Beta': 'responses=experimental',
67
+ originator: CODEX_OAUTH_ORIGINATOR,
68
+ 'chatgpt-account-id': auth.account_id || '',
69
+ 'x-client-request-id': randomBytes(16).toString('hex'),
70
+ },
71
+ body: JSON.stringify(body),
72
+ signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
73
+ });
74
+ if (!res.ok || !res.body) throw upstreamError('ChatGPT image', res.status, await res.text().catch(() => ''));
75
+
76
+ const reader = res.body.getReader();
77
+ const decoder = new TextDecoder();
78
+ const state = { final: '', partial: '', revisedPrompt: null };
79
+ let buffer = '';
80
+ let failure = null;
81
+ try {
82
+ for (;;) {
83
+ const { done, value } = await reader.read();
84
+ if (done) break;
85
+ buffer += decoder.decode(value, { stream: true });
86
+ const lines = buffer.split('\n');
87
+ buffer = lines.pop() || '';
88
+ for (const line of lines) {
89
+ if (!line.startsWith('data:')) continue;
90
+ const raw = line.slice(5).trim();
91
+ if (!raw || raw === '[DONE]') continue;
92
+ let event;
93
+ try { event = JSON.parse(raw); } catch { continue; }
94
+ if (event?.type === 'response.failed' || event?.type === 'error') {
95
+ failure = event?.response?.error?.message || event?.message || 'stream failed';
96
+ }
97
+ collectImage(event, state);
98
+ }
99
+ }
100
+ } finally {
101
+ try { reader.releaseLock(); } catch {}
102
+ }
103
+
104
+ const b64 = state.final || state.partial;
105
+ if (!b64) {
106
+ throw mediaError(`ChatGPT returned no image data${failure ? `: ${failure}` : ''}`, 'MEDIA_EMPTY_RESULT', 502);
107
+ }
108
+ return { bytes: Buffer.from(b64, 'base64'), mime: 'image/png', revisedPrompt: state.revisedPrompt };
109
+ }
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Gemini image adapter (API key lane).
3
+ *
4
+ * Calls the Generative Language `generateContent` route and pulls the first
5
+ * inline image part out of the candidate. Aspect ratio rides in `imageConfig`,
6
+ * which older image models reject — a 400 there retries once without it so a
7
+ * control the model does not know never fails the whole generation.
8
+ */
9
+ import { resolveGeminiKey } from '../auth.mjs';
10
+ import { mediaError } from '../lanes.mjs';
11
+ import { upstreamError } from '../upstream-error.mjs';
12
+
13
+ const BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/models';
14
+ const REQUEST_TIMEOUT_MS = 180_000;
15
+
16
+ function requestBody(prompt, options, withImageConfig, references = []) {
17
+ const body = {
18
+ contents: [{
19
+ role: 'user',
20
+ parts: [
21
+ ...references.map((ref) => ({
22
+ inlineData: { mimeType: ref.mime || 'image/png', data: ref.base64 },
23
+ })),
24
+ { text: prompt },
25
+ ],
26
+ }],
27
+ };
28
+ const aspect = String(options?.aspectRatio || 'auto');
29
+ if (withImageConfig && aspect !== 'auto') {
30
+ body.generationConfig = { imageConfig: { aspectRatio: aspect } };
31
+ }
32
+ return body;
33
+ }
34
+
35
+ async function post(model, key, body, signal) {
36
+ return await fetch(`${BASE_URL}/${encodeURIComponent(model)}:generateContent`, {
37
+ method: 'POST',
38
+ headers: { 'Content-Type': 'application/json', 'x-goog-api-key': key },
39
+ body: JSON.stringify(body),
40
+ signal: AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)].filter(Boolean)),
41
+ });
42
+ }
43
+
44
+ export async function generateImage({ model, prompt, options = {}, references = [], signal }) {
45
+ const key = resolveGeminiKey();
46
+ const wantsImageConfig = String(options.aspectRatio || 'auto') !== 'auto';
47
+ let res = await post(model, key, requestBody(prompt, options, wantsImageConfig, references), signal);
48
+ if (!res.ok && res.status === 400 && wantsImageConfig) {
49
+ res = await post(model, key, requestBody(prompt, options, false, references), signal);
50
+ }
51
+ if (!res.ok) throw upstreamError('Gemini image', res.status, await res.text().catch(() => ''));
52
+ const data = await res.json();
53
+ const parts = data?.candidates?.[0]?.content?.parts || [];
54
+ const image = parts.find((part) => part?.inlineData?.data);
55
+ if (!image) {
56
+ const refusal = parts.find((part) => typeof part?.text === 'string')?.text || '';
57
+ throw mediaError(
58
+ `Gemini returned no image data${refusal ? `: ${refusal.slice(0, 200)}` : ''}`,
59
+ 'MEDIA_EMPTY_RESULT',
60
+ 502,
61
+ );
62
+ }
63
+ return {
64
+ bytes: Buffer.from(image.inlineData.data, 'base64'),
65
+ mime: image.inlineData.mimeType || 'image/png',
66
+ revisedPrompt: null,
67
+ };
68
+ }
@@ -0,0 +1,119 @@
1
+ /**
2
+ * Gemini video adapter (API key lane).
3
+ *
4
+ * Two different upstream shapes live behind one lane:
5
+ * - Omni Flash answers on the Interactions API in a single call, with the
6
+ * mp4 inline as base64 on a `model_output` step.
7
+ * - Veo 3.1 is a long-running predict: submit, poll the operation, then pull
8
+ * the sample URI with `alt=media`.
9
+ */
10
+ import { resolveGeminiKey } from '../auth.mjs';
11
+ import { mediaError } from '../lanes.mjs';
12
+ import { upstreamError } from '../upstream-error.mjs';
13
+
14
+ const BASE_URL = 'https://generativelanguage.googleapis.com/v1beta';
15
+ const OMNI_TIMEOUT_MS = 600_000;
16
+ const POLL_INTERVAL_MS = 8_000;
17
+ const TOTAL_TIMEOUT_MS = 900_000;
18
+
19
+ export function isOmniModel(model) {
20
+ return /omni/i.test(String(model || ''));
21
+ }
22
+
23
+ function aspectFor(options) {
24
+ const aspect = String(options?.aspectRatio || 'auto');
25
+ return aspect === 'auto' ? '16:9' : aspect;
26
+ }
27
+
28
+ async function generateViaOmni({ model, prompt, options, references = [], signal, key }) {
29
+ // Omni takes a multimodal input array; a reference image switches the task
30
+ // from text-to-video to image-to-video.
31
+ const input = references.length
32
+ ? [
33
+ ...references.map((ref) => ({ type: 'image', data: ref.base64, mime_type: ref.mime || 'image/png' })),
34
+ { type: 'text', text: prompt },
35
+ ]
36
+ : prompt;
37
+ const res = await fetch(`${BASE_URL}/interactions`, {
38
+ method: 'POST',
39
+ headers: { 'Content-Type': 'application/json', 'x-goog-api-key': key },
40
+ body: JSON.stringify({
41
+ model,
42
+ input,
43
+ response_format: { type: 'video', aspect_ratio: aspectFor(options) },
44
+ generation_config: {
45
+ video_config: { task: references.length ? 'image_to_video' : 'text_to_video' },
46
+ },
47
+ }),
48
+ signal: AbortSignal.any([signal, AbortSignal.timeout(OMNI_TIMEOUT_MS)].filter(Boolean)),
49
+ });
50
+ if (!res.ok) throw upstreamError('Gemini Omni video', res.status, await res.text().catch(() => ''));
51
+ const data = await res.json();
52
+ for (const step of data?.steps || []) {
53
+ const content = Array.isArray(step?.content) ? step.content : [step?.content].filter(Boolean);
54
+ const video = content.find((item) => item?.type === 'video' && typeof item?.data === 'string');
55
+ if (video) {
56
+ return { bytes: Buffer.from(video.data, 'base64'), mime: video.mime_type || 'video/mp4' };
57
+ }
58
+ }
59
+ throw mediaError('Gemini Omni returned no video data', 'MEDIA_EMPTY_RESULT', 502);
60
+ }
61
+
62
+ async function generateViaVeo({ model, prompt, options, references = [], signal, onProgress, key }) {
63
+ const headers = { 'Content-Type': 'application/json', 'x-goog-api-key': key };
64
+ const parameters = { aspectRatio: aspectFor(options) };
65
+ const instance = { prompt };
66
+ // Veo accepts a single seed image for image-to-video.
67
+ if (references[0]) {
68
+ instance.image = { bytesBase64Encoded: references[0].base64, mimeType: references[0].mime || 'image/png' };
69
+ }
70
+ const resolution = String(options?.resolution || '');
71
+ if (resolution === '720p' || resolution === '1080p') parameters.resolution = resolution;
72
+ // Veo takes discrete clip lengths; anything else is rejected upstream, so an
73
+ // out-of-contract value is dropped rather than forwarded.
74
+ const duration = Math.trunc(Number(options?.duration) || 0);
75
+ if ([4, 6, 8].includes(duration)) parameters.durationSeconds = duration;
76
+
77
+ const started = await fetch(`${BASE_URL}/models/${encodeURIComponent(model)}:predictLongRunning`, {
78
+ method: 'POST',
79
+ headers,
80
+ body: JSON.stringify({ instances: [instance], parameters }),
81
+ signal,
82
+ });
83
+ if (!started.ok) throw upstreamError('Veo video', started.status, await started.text().catch(() => ''));
84
+ const operation = await started.json();
85
+ if (!operation?.name) throw mediaError('Veo returned no operation name', 'MEDIA_UPSTREAM_FAILED', 502);
86
+
87
+ const deadline = Date.now() + TOTAL_TIMEOUT_MS;
88
+ for (;;) {
89
+ if (signal?.aborted) throw mediaError('canceled', 'MEDIA_CANCELED', 499);
90
+ if (Date.now() > deadline) throw mediaError('Veo poll budget exceeded', 'MEDIA_TIMEOUT', 504);
91
+ await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
92
+ const poll = await fetch(`${BASE_URL}/${operation.name}`, { headers, signal });
93
+ if (!poll.ok) throw upstreamError('Veo poll', poll.status, await poll.text().catch(() => ''));
94
+ const data = await poll.json();
95
+ if (!data?.done) {
96
+ // The operation exposes no percentage; keep the UI moving with a coarse
97
+ // heartbeat instead of a fake number.
98
+ if (typeof onProgress === 'function') onProgress(50);
99
+ continue;
100
+ }
101
+ if (data.error) {
102
+ throw mediaError(`Veo generation failed: ${data.error.message || 'unknown error'}`, 'MEDIA_UPSTREAM_FAILED', 502);
103
+ }
104
+ const sample = data?.response?.generateVideoResponse?.generatedSamples?.[0]
105
+ || data?.response?.generatedVideos?.[0];
106
+ const uri = sample?.video?.uri || sample?.video?.fileUri;
107
+ if (!uri) throw mediaError('Veo finished without a video URI', 'MEDIA_EMPTY_RESULT', 502);
108
+ const file = await fetch(`${uri}${uri.includes('?') ? '&' : '?'}alt=media`, { headers, signal });
109
+ if (!file.ok) throw upstreamError('Veo download', file.status, await file.text().catch(() => ''));
110
+ return { bytes: Buffer.from(await file.arrayBuffer()), mime: 'video/mp4' };
111
+ }
112
+ }
113
+
114
+ export async function generateVideo({ model, prompt, options = {}, references = [], signal, onProgress }) {
115
+ const key = resolveGeminiKey();
116
+ return isOmniModel(model)
117
+ ? await generateViaOmni({ model, prompt, options, references, signal, key })
118
+ : await generateViaVeo({ model, prompt, options, references, signal, onProgress, key });
119
+ }