mixdog 0.9.69 → 0.9.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -1
- package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +18 -4
- package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +45 -12
- package/src/runtime/agent/orchestrator/session/store/listing.mjs +1 -0
- package/src/runtime/agent/orchestrator/session/store/paths-heartbeat.mjs +51 -18
- package/src/runtime/agent/orchestrator/session/store.mjs +3 -0
- package/src/runtime/channels/lib/owned-runtime.mjs +8 -1
- package/src/runtime/media/adapters/codex-image.mjs +109 -0
- package/src/runtime/media/adapters/gemini-image.mjs +68 -0
- package/src/runtime/media/adapters/gemini-video.mjs +119 -0
- package/src/runtime/media/adapters/xai-media.mjs +116 -0
- package/src/runtime/media/auth.mjs +51 -0
- package/src/runtime/media/index.mjs +18 -0
- package/src/runtime/media/jobs.mjs +174 -0
- package/src/runtime/media/lanes.mjs +230 -0
- package/src/runtime/media/renditions.mjs +203 -0
- package/src/runtime/media/store.mjs +356 -0
- package/src/runtime/media/store.test.mjs +103 -0
- package/src/runtime/media/upstream-error.mjs +39 -0
- package/src/session-runtime/channel-config-api.mjs +12 -1
- package/src/session-runtime/lifecycle-api.mjs +13 -4
- package/src/session-runtime/media-api.mjs +50 -0
- package/src/session-runtime/model-route-api.mjs +4 -1
- package/src/session-runtime/runtime-core.mjs +17 -1
- package/src/session-runtime/session-text.mjs +0 -1
- package/src/session-runtime/tool-catalog.mjs +32 -0
- package/src/session-runtime/warmup-schedulers.mjs +14 -1
- package/src/session-runtime/workflow-agents-api.mjs +29 -4
- package/src/tui/dist/index.mjs +21 -2
- package/src/tui/engine/session-api-ext.mjs +15 -0
- package/src/tui/engine.mjs +39 -1
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Derived renditions for the media gallery (thumb / display / video poster).
|
|
3
|
+
*
|
|
4
|
+
* A gallery tile is ~512px and a detail view never exceeds the viewport, so
|
|
5
|
+
* the original bytes are the wrong unit of transfer: over the remote link one
|
|
6
|
+
* full-size still costs more than the entire visible grid. Renditions are
|
|
7
|
+
* generated once, cached on disk beside the assets, and afterwards read back
|
|
8
|
+
* as small files.
|
|
9
|
+
*
|
|
10
|
+
* sharp (stills) and ffmpeg (video posters) are OPTIONAL. When neither can
|
|
11
|
+
* produce a rendition this module returns null and the caller reduces the
|
|
12
|
+
* feature (glyph tile) or explicitly asks for the original. It never silently
|
|
13
|
+
* substitutes full-size bytes for a thumbnail — that hidden path is exactly
|
|
14
|
+
* what made the remote gallery slow.
|
|
15
|
+
*/
|
|
16
|
+
import { spawn } from 'child_process';
|
|
17
|
+
import { existsSync, mkdirSync, renameSync, statSync, unlinkSync, writeFileSync } from 'fs';
|
|
18
|
+
import { dirname, join } from 'path';
|
|
19
|
+
|
|
20
|
+
import { resolvePluginData } from '../shared/plugin-paths.mjs';
|
|
21
|
+
|
|
22
|
+
// maxEdge is the long edge in pixels; quality is the webp setting.
|
|
23
|
+
const SPECS = {
|
|
24
|
+
thumb: { maxEdge: 512, quality: 70 },
|
|
25
|
+
display: { maxEdge: 2048, quality: 82 },
|
|
26
|
+
};
|
|
27
|
+
// Probe order when reading a cached rendition: sharp writes webp, the
|
|
28
|
+
// ffmpeg-only poster path writes png.
|
|
29
|
+
const EXTENSION_MIME = { '.webp': 'image/webp', '.png': 'image/png' };
|
|
30
|
+
const POSTER_TIMEOUT_MS = 20_000;
|
|
31
|
+
const MAX_POSTER_BYTES = 32 * 1024 * 1024;
|
|
32
|
+
|
|
33
|
+
export const MEDIA_VARIANTS = Object.keys(SPECS);
|
|
34
|
+
|
|
35
|
+
/** Spec for a variant name, or null when the caller asked for the original. */
|
|
36
|
+
export function renditionSpec(variant) {
|
|
37
|
+
return SPECS[String(variant || '')] || null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let sharpPromise;
|
|
41
|
+
async function loadSharp() {
|
|
42
|
+
sharpPromise ??= import('sharp').then((mod) => mod?.default || mod || null, () => null);
|
|
43
|
+
return sharpPromise;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let ffmpegPromise;
|
|
47
|
+
// The managed voice runtime already installs ffmpeg; reuse it rather than
|
|
48
|
+
// shipping a second copy. MIXDOG_FFMPEG_PATH overrides for hosts that have
|
|
49
|
+
// their own build.
|
|
50
|
+
async function resolveFfmpeg() {
|
|
51
|
+
ffmpegPromise ??= (async () => {
|
|
52
|
+
const explicit = String(process.env.MIXDOG_FFMPEG_PATH || '').trim();
|
|
53
|
+
if (explicit && existsSync(explicit)) return explicit;
|
|
54
|
+
try {
|
|
55
|
+
const { resolveVoiceRuntime } = await import('../channels/lib/voice-runtime-fetcher.mjs');
|
|
56
|
+
const runtime = resolveVoiceRuntime(resolvePluginData());
|
|
57
|
+
return runtime?.ffmpegPath && existsSync(runtime.ffmpegPath) ? runtime.ffmpegPath : null;
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
})();
|
|
62
|
+
return ffmpegPromise;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function renditionPath(cacheDir, variant, id, extension) {
|
|
66
|
+
return join(cacheDir, variant, `${id}${extension}`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function cachedRendition(cacheDir, variant, id) {
|
|
70
|
+
for (const [extension, mime] of Object.entries(EXTENSION_MIME)) {
|
|
71
|
+
const path = renditionPath(cacheDir, variant, id, extension);
|
|
72
|
+
if (existsSync(path)) return { path, mime, bytes: statSync(path).size };
|
|
73
|
+
}
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Drop every cached rendition of one asset (called when the asset dies). */
|
|
78
|
+
export function removeRenditions(cacheDir, id) {
|
|
79
|
+
for (const variant of MEDIA_VARIANTS) {
|
|
80
|
+
for (const extension of Object.keys(EXTENSION_MIME)) {
|
|
81
|
+
try { unlinkSync(renditionPath(cacheDir, variant, id, extension)); } catch { /* absent */ }
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function writeRendition(cacheDir, variant, id, extension, buffer) {
|
|
87
|
+
const path = renditionPath(cacheDir, variant, id, extension);
|
|
88
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
89
|
+
// Stage-then-rename: a reader must never observe a half-written cache file.
|
|
90
|
+
const staging = `${path}.${process.pid}.tmp`;
|
|
91
|
+
writeFileSync(staging, buffer);
|
|
92
|
+
renameSync(staging, path);
|
|
93
|
+
return { path, mime: EXTENSION_MIME[extension], bytes: buffer.length };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Downscale a still (path or buffer) into a webp; null without sharp. */
|
|
97
|
+
async function encodeStill(input, spec) {
|
|
98
|
+
const sharp = await loadSharp();
|
|
99
|
+
if (!sharp) return null;
|
|
100
|
+
try {
|
|
101
|
+
const buffer = await sharp(input)
|
|
102
|
+
// EXIF-rotated phone photos would otherwise land sideways in the grid.
|
|
103
|
+
.rotate()
|
|
104
|
+
.resize(spec.maxEdge, spec.maxEdge, { fit: 'inside', withoutEnlargement: true })
|
|
105
|
+
.webp({ quality: spec.quality })
|
|
106
|
+
.toBuffer();
|
|
107
|
+
return { extension: '.webp', buffer };
|
|
108
|
+
} catch {
|
|
109
|
+
// sharp present but the source is not a decodable image.
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function parseDurationSeconds(text) {
|
|
115
|
+
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(text || '');
|
|
116
|
+
if (!match) return 0;
|
|
117
|
+
const seconds = Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]);
|
|
118
|
+
return Number.isFinite(seconds) ? Math.round(seconds) : 0;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** First frame + duration of a clip, straight from ffmpeg; null when the
|
|
122
|
+
* managed runtime is not installed. */
|
|
123
|
+
async function grabVideoFrame(sourcePath, spec) {
|
|
124
|
+
const ffmpeg = await resolveFfmpeg();
|
|
125
|
+
if (!ffmpeg) return null;
|
|
126
|
+
return new Promise((resolve) => {
|
|
127
|
+
const child = spawn(ffmpeg, [
|
|
128
|
+
'-hide_banner', '-nostdin',
|
|
129
|
+
'-i', sourcePath,
|
|
130
|
+
'-frames:v', '1',
|
|
131
|
+
// Never upscale a small clip: -2 keeps the height even for the encoder.
|
|
132
|
+
'-vf', `scale='min(${spec.maxEdge},iw)':-2`,
|
|
133
|
+
'-f', 'image2', '-vcodec', 'png', 'pipe:1',
|
|
134
|
+
], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
135
|
+
const chunks = [];
|
|
136
|
+
let total = 0;
|
|
137
|
+
let stderr = '';
|
|
138
|
+
let settled = false;
|
|
139
|
+
const finish = (value) => {
|
|
140
|
+
if (settled) return;
|
|
141
|
+
settled = true;
|
|
142
|
+
clearTimeout(timer);
|
|
143
|
+
try { child.kill(); } catch { /* already gone */ }
|
|
144
|
+
resolve(value);
|
|
145
|
+
};
|
|
146
|
+
const timer = setTimeout(() => finish(null), POSTER_TIMEOUT_MS);
|
|
147
|
+
timer.unref?.();
|
|
148
|
+
child.stdout.on('data', (chunk) => {
|
|
149
|
+
total += chunk.length;
|
|
150
|
+
// A corrupt clip could stream frames forever; the poster is one image.
|
|
151
|
+
if (total > MAX_POSTER_BYTES) {
|
|
152
|
+
finish(null);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
chunks.push(chunk);
|
|
156
|
+
});
|
|
157
|
+
child.stderr.on('data', (chunk) => { stderr += String(chunk); });
|
|
158
|
+
child.on('error', () => finish(null));
|
|
159
|
+
child.on('close', () => {
|
|
160
|
+
const buffer = Buffer.concat(chunks);
|
|
161
|
+
finish(buffer.length ? { buffer, durationSeconds: parseDurationSeconds(stderr) } : null);
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Two tiles asking for the same missing rendition must not both encode it.
|
|
167
|
+
const inflight = new Map();
|
|
168
|
+
|
|
169
|
+
async function generate({ id, kind, sourcePath, variant, spec, cacheDir }) {
|
|
170
|
+
if (kind === 'video') {
|
|
171
|
+
// Only the tile-sized poster is worth extracting: a "display" frame would
|
|
172
|
+
// be a still pretending to be a clip.
|
|
173
|
+
if (variant !== 'thumb') return null;
|
|
174
|
+
const frame = await grabVideoFrame(sourcePath, spec);
|
|
175
|
+
if (!frame) return null;
|
|
176
|
+
const encoded = await encodeStill(frame.buffer, spec);
|
|
177
|
+
const written = encoded
|
|
178
|
+
? writeRendition(cacheDir, variant, id, encoded.extension, encoded.buffer)
|
|
179
|
+
: writeRendition(cacheDir, variant, id, '.png', frame.buffer);
|
|
180
|
+
return { ...written, durationSeconds: frame.durationSeconds };
|
|
181
|
+
}
|
|
182
|
+
const encoded = await encodeStill(sourcePath, spec);
|
|
183
|
+
return encoded ? writeRendition(cacheDir, variant, id, encoded.extension, encoded.buffer) : null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Cached rendition for one asset, generating it on first use.
|
|
188
|
+
* Returns `{ path, mime, bytes, durationSeconds? }`, or null when this host
|
|
189
|
+
* cannot produce it (no sharp / no ffmpeg / undecodable source).
|
|
190
|
+
*/
|
|
191
|
+
export async function ensureRendition({ id, kind, sourcePath, variant, cacheDir }) {
|
|
192
|
+
const spec = renditionSpec(variant);
|
|
193
|
+
if (!spec) return null;
|
|
194
|
+
const cached = cachedRendition(cacheDir, variant, id);
|
|
195
|
+
if (cached) return cached;
|
|
196
|
+
const key = `${variant}:${id}`;
|
|
197
|
+
const pending = inflight.get(key)
|
|
198
|
+
?? generate({ id, kind, sourcePath, variant, spec, cacheDir })
|
|
199
|
+
.catch(() => null)
|
|
200
|
+
.finally(() => { inflight.delete(key); });
|
|
201
|
+
inflight.set(key, pending);
|
|
202
|
+
return pending;
|
|
203
|
+
}
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* On-disk store for generated media.
|
|
3
|
+
*
|
|
4
|
+
* Assets live as real files under <data>/media/assets so nothing large is kept
|
|
5
|
+
* in memory or in a session transcript; index.json holds the metadata list
|
|
6
|
+
* (newest first) and is written atomically under a file lock because the
|
|
7
|
+
* desktop and CLI can generate concurrently.
|
|
8
|
+
*/
|
|
9
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, statSync } from 'fs';
|
|
10
|
+
import { dirname, join, resolve, sep } from 'path';
|
|
11
|
+
import { randomUUID } from 'crypto';
|
|
12
|
+
import { spawn } from 'child_process';
|
|
13
|
+
import { resolvePluginData } from '../shared/plugin-paths.mjs';
|
|
14
|
+
import { writeJsonAtomicSync, withFileLockSync } from '../shared/atomic-file.mjs';
|
|
15
|
+
import { ensureRendition, removeRenditions, renditionSpec } from './renditions.mjs';
|
|
16
|
+
|
|
17
|
+
const MAX_INDEX_ENTRIES = 2_000;
|
|
18
|
+
// Renderer transport is base64 over IPC: refuse to inline anything larger.
|
|
19
|
+
const MAX_INLINE_BYTES = 48 * 1024 * 1024;
|
|
20
|
+
|
|
21
|
+
const EXTENSIONS = {
|
|
22
|
+
'image/png': 'png',
|
|
23
|
+
'image/jpeg': 'jpg',
|
|
24
|
+
'image/webp': 'webp',
|
|
25
|
+
'video/mp4': 'mp4',
|
|
26
|
+
};
|
|
27
|
+
let layoutMigrationChecked = false;
|
|
28
|
+
|
|
29
|
+
function mediaDir() {
|
|
30
|
+
return join(resolvePluginData(), 'media');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function assetsDir() {
|
|
34
|
+
return join(mediaDir(), 'assets');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Derived tile/detail renditions live beside the assets: they are a rebuildable
|
|
38
|
+
// cache, so they never share the asset tree the user browses.
|
|
39
|
+
function renditionsDir() {
|
|
40
|
+
return join(mediaDir(), 'renditions');
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function indexPath() {
|
|
44
|
+
return join(mediaDir(), 'index.json');
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function indexLockPath() {
|
|
48
|
+
return `${indexPath()}.lock`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function ensureDirs() {
|
|
52
|
+
mkdirSync(assetsDir(), { recursive: true });
|
|
53
|
+
if (!layoutMigrationChecked) {
|
|
54
|
+
migrateAssetLayout();
|
|
55
|
+
layoutMigrationChecked = true;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function readIndex() {
|
|
60
|
+
try {
|
|
61
|
+
const raw = readFileSync(indexPath(), 'utf8');
|
|
62
|
+
const parsed = JSON.parse(raw);
|
|
63
|
+
return Array.isArray(parsed?.assets) ? parsed.assets : [];
|
|
64
|
+
} catch {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function writeIndex(assets) {
|
|
70
|
+
writeJsonAtomicSync(indexPath(), { version: 2, assets });
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function extensionFor(mime) {
|
|
74
|
+
return EXTENSIONS[String(mime || '').toLowerCase()] || 'bin';
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function folderSegment(value, fallback) {
|
|
78
|
+
const normalized = String(value || '')
|
|
79
|
+
.normalize('NFKC')
|
|
80
|
+
.replace(/[<>:"/\\|?*\u0000-\u001f]/g, '-')
|
|
81
|
+
.replace(/\s+/g, ' ')
|
|
82
|
+
.trim()
|
|
83
|
+
.replace(/[. ]+$/g, '')
|
|
84
|
+
.slice(0, 80);
|
|
85
|
+
if (!normalized || /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i.test(normalized)) {
|
|
86
|
+
return fallback;
|
|
87
|
+
}
|
|
88
|
+
return normalized;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function localDateFolder(createdAt) {
|
|
92
|
+
const date = new Date(Number(createdAt) || Date.now());
|
|
93
|
+
const year = date.getFullYear();
|
|
94
|
+
const month = String(date.getMonth() + 1).padStart(2, '0');
|
|
95
|
+
const day = String(date.getDate()).padStart(2, '0');
|
|
96
|
+
return `${year}-${month}-${day}`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function organizedAssetFile(entry) {
|
|
100
|
+
const kind = entry.kind === 'video' ? 'videos' : 'images';
|
|
101
|
+
const lane = folderSegment(entry.lane, 'unknown-provider');
|
|
102
|
+
const model = folderSegment(entry.model, 'unknown-model');
|
|
103
|
+
const date = localDateFolder(entry.createdAt);
|
|
104
|
+
const id = folderSegment(entry.id, 'asset');
|
|
105
|
+
return [kind, lane, model, date, `${id}.${extensionFor(entry.mime)}`].join('/');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function storedAssetPath(file) {
|
|
109
|
+
const root = resolve(assetsDir());
|
|
110
|
+
const path = resolve(root, String(file || ''));
|
|
111
|
+
return path === root || path.startsWith(`${root}${sep}`) ? path : null;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Move flat v1 assets into kind/provider/model/date folders without losing old indexes. */
|
|
115
|
+
function migrateAssetLayout() {
|
|
116
|
+
withFileLockSync(indexLockPath(), () => {
|
|
117
|
+
const current = readIndex();
|
|
118
|
+
let changed = false;
|
|
119
|
+
const assets = current.map((entry) => {
|
|
120
|
+
const targetFile = organizedAssetFile(entry);
|
|
121
|
+
const sourcePath = storedAssetPath(entry.file);
|
|
122
|
+
const targetPath = storedAssetPath(targetFile);
|
|
123
|
+
if (!targetPath) return entry;
|
|
124
|
+
if (String(entry.file || '').replace(/\\/g, '/') === targetFile) return entry;
|
|
125
|
+
if (existsSync(targetPath)) {
|
|
126
|
+
changed = true;
|
|
127
|
+
return { ...entry, file: targetFile };
|
|
128
|
+
}
|
|
129
|
+
if (!sourcePath || !existsSync(sourcePath)) return entry;
|
|
130
|
+
try {
|
|
131
|
+
mkdirSync(dirname(targetPath), { recursive: true });
|
|
132
|
+
renameSync(sourcePath, targetPath);
|
|
133
|
+
changed = true;
|
|
134
|
+
return { ...entry, file: targetFile };
|
|
135
|
+
} catch {
|
|
136
|
+
return entry;
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
if (changed) writeIndex(assets);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Persist one generated artifact and return its index entry. */
|
|
144
|
+
export function saveMediaAsset({ kind, lane, model, prompt, options = {}, mime, bytes, meta = {} }) {
|
|
145
|
+
ensureDirs();
|
|
146
|
+
const id = randomUUID();
|
|
147
|
+
const createdAt = Date.now();
|
|
148
|
+
const entry = {
|
|
149
|
+
id,
|
|
150
|
+
kind,
|
|
151
|
+
lane,
|
|
152
|
+
model,
|
|
153
|
+
prompt: String(prompt || '').slice(0, 4_000),
|
|
154
|
+
options,
|
|
155
|
+
mime,
|
|
156
|
+
bytes: bytes.length,
|
|
157
|
+
createdAt,
|
|
158
|
+
...meta,
|
|
159
|
+
};
|
|
160
|
+
const file = organizedAssetFile(entry);
|
|
161
|
+
const path = storedAssetPath(file);
|
|
162
|
+
if (!path) throw new Error('invalid media asset path');
|
|
163
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
164
|
+
writeFileSync(path, bytes);
|
|
165
|
+
entry.file = file;
|
|
166
|
+
withFileLockSync(indexLockPath(), () => {
|
|
167
|
+
const assets = [entry, ...readIndex()];
|
|
168
|
+
const kept = assets.slice(0, MAX_INDEX_ENTRIES);
|
|
169
|
+
for (const dropped of assets.slice(MAX_INDEX_ENTRIES)) {
|
|
170
|
+
const droppedPath = storedAssetPath(dropped.file);
|
|
171
|
+
try { if (droppedPath) unlinkSync(droppedPath); } catch {}
|
|
172
|
+
}
|
|
173
|
+
writeIndex(kept);
|
|
174
|
+
});
|
|
175
|
+
// The first gallery paint after a generation would otherwise be the one that
|
|
176
|
+
// pays for the tile rendition; build it now, off the caller's path.
|
|
177
|
+
void ensureRendition({
|
|
178
|
+
id: entry.id,
|
|
179
|
+
kind: entry.kind,
|
|
180
|
+
sourcePath: path,
|
|
181
|
+
variant: 'thumb',
|
|
182
|
+
cacheDir: renditionsDir(),
|
|
183
|
+
});
|
|
184
|
+
return entry;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function listMediaAssets({ limit = 60, offset = 0, kind = null } = {}) {
|
|
188
|
+
ensureDirs();
|
|
189
|
+
const all = readIndex().filter((entry) => (kind ? entry.kind === kind : true));
|
|
190
|
+
const start = Math.max(0, Math.trunc(Number(offset) || 0));
|
|
191
|
+
const count = Math.min(200, Math.max(1, Math.trunc(Number(limit) || 60)));
|
|
192
|
+
const assets = all.slice(start, start + count);
|
|
193
|
+
// Assets generated before this cache existed have no tile rendition, so the
|
|
194
|
+
// first gallery paint used to wait on an encode per visible tile. Listing is
|
|
195
|
+
// the earliest moment the host knows WHICH tiles are coming: warm them here,
|
|
196
|
+
// off the caller's path (ensureRendition dedupes the concurrent request).
|
|
197
|
+
void prewarmThumbs(assets);
|
|
198
|
+
return { total: all.length, assets };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Bounded: the encoder is shared with generation, and a large page must not
|
|
202
|
+
// turn one listing into dozens of parallel sharp/ffmpeg jobs.
|
|
203
|
+
const PREWARM_LIMIT = 24;
|
|
204
|
+
const PREWARM_CONCURRENCY = 2;
|
|
205
|
+
|
|
206
|
+
async function prewarmThumbs(assets) {
|
|
207
|
+
const queue = assets.slice(0, PREWARM_LIMIT);
|
|
208
|
+
const workers = Array.from({ length: PREWARM_CONCURRENCY }, async () => {
|
|
209
|
+
for (let entry = queue.shift(); entry; entry = queue.shift()) {
|
|
210
|
+
const path = storedAssetPath(entry.file);
|
|
211
|
+
if (!path || !existsSync(path)) continue;
|
|
212
|
+
const rendition = await ensureRendition({
|
|
213
|
+
id: entry.id,
|
|
214
|
+
kind: entry.kind,
|
|
215
|
+
sourcePath: path,
|
|
216
|
+
variant: 'thumb',
|
|
217
|
+
cacheDir: renditionsDir(),
|
|
218
|
+
}).catch(() => null);
|
|
219
|
+
if (rendition?.durationSeconds) rememberDuration(entry, rendition.durationSeconds);
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
await Promise.all(workers).catch(() => {});
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export function getMediaAsset(id) {
|
|
226
|
+
ensureDirs();
|
|
227
|
+
return readIndex().find((entry) => entry.id === id) || null;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Persist a duration probed while building a poster, so the gallery badge
|
|
231
|
+
* survives a restart without decoding the clip again. */
|
|
232
|
+
function rememberDuration(entry, durationSeconds) {
|
|
233
|
+
if (!durationSeconds || entry.durationSeconds === durationSeconds) return;
|
|
234
|
+
withFileLockSync(indexLockPath(), () => {
|
|
235
|
+
const assets = readIndex();
|
|
236
|
+
const row = assets.find((item) => item.id === entry.id);
|
|
237
|
+
if (!row || row.durationSeconds === durationSeconds) return;
|
|
238
|
+
row.durationSeconds = durationSeconds;
|
|
239
|
+
writeIndex(assets);
|
|
240
|
+
});
|
|
241
|
+
entry.durationSeconds = durationSeconds;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Base64 for the inline (RPC) path. Large files are refused: bytes that big
|
|
245
|
+
* belong on the file route, not in a JSON frame. */
|
|
246
|
+
function inlineBase64(path) {
|
|
247
|
+
const size = statSync(path).size;
|
|
248
|
+
if (size > MAX_INLINE_BYTES) {
|
|
249
|
+
const err = new Error('media asset is too large to preview');
|
|
250
|
+
err.code = 'MEDIA_ASSET_TOO_LARGE';
|
|
251
|
+
throw err;
|
|
252
|
+
}
|
|
253
|
+
return readFileSync(path).toString('base64');
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Locate an asset (or one of its renditions) as a FILE — no inlining.
|
|
258
|
+
*
|
|
259
|
+
* This is the unit the media routes serve: the client receives a cacheable,
|
|
260
|
+
* range-able response instead of a base64 payload riding the RPC lane.
|
|
261
|
+
* `available: false` means this host cannot build the rendition (no sharp /
|
|
262
|
+
* no ffmpeg); the caller reduces the feature instead of substituting the
|
|
263
|
+
* original behind the client's back.
|
|
264
|
+
*/
|
|
265
|
+
export async function resolveMediaFile(id, { variant = 'original' } = {}) {
|
|
266
|
+
const entry = getMediaAsset(id);
|
|
267
|
+
if (!entry) return null;
|
|
268
|
+
const path = storedAssetPath(entry.file);
|
|
269
|
+
if (!path || !existsSync(path)) return null;
|
|
270
|
+
if (variant === 'original') return { ...entry, variant: 'original', path, available: true };
|
|
271
|
+
if (!renditionSpec(variant)) throw new Error(`unknown media variant: ${variant}`);
|
|
272
|
+
const rendition = await ensureRendition({
|
|
273
|
+
id: entry.id,
|
|
274
|
+
kind: entry.kind,
|
|
275
|
+
sourcePath: path,
|
|
276
|
+
variant,
|
|
277
|
+
cacheDir: renditionsDir(),
|
|
278
|
+
});
|
|
279
|
+
if (!rendition) return { ...entry, variant, path: '', available: false };
|
|
280
|
+
rememberDuration(entry, rendition.durationSeconds);
|
|
281
|
+
return {
|
|
282
|
+
...entry,
|
|
283
|
+
...(rendition.durationSeconds ? { durationSeconds: rendition.durationSeconds } : {}),
|
|
284
|
+
variant,
|
|
285
|
+
path: rendition.path,
|
|
286
|
+
mime: rendition.mime,
|
|
287
|
+
bytes: rendition.bytes,
|
|
288
|
+
available: true,
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Inline an asset for the renderer (base64 + mime).
|
|
294
|
+
*
|
|
295
|
+
* `variant` selects a derived rendition ('thumb' | 'display'); the original is
|
|
296
|
+
* read only when it is asked for. A host that cannot build the rendition says
|
|
297
|
+
* so (`available: false`) instead of quietly shipping full-size bytes — the
|
|
298
|
+
* caller that can afford them (local IPC) opts in with `allowOriginal`.
|
|
299
|
+
*/
|
|
300
|
+
export async function readMediaAsset(id, { variant = 'original', allowOriginal = false } = {}) {
|
|
301
|
+
const file = await resolveMediaFile(id, { variant });
|
|
302
|
+
if (!file) return null;
|
|
303
|
+
if (file.available) return { ...file, base64: inlineBase64(file.path) };
|
|
304
|
+
if (!allowOriginal) return { ...file, base64: '' };
|
|
305
|
+
const original = await resolveMediaFile(id, { variant: 'original' });
|
|
306
|
+
if (!original) return null;
|
|
307
|
+
return { ...original, downgraded: true, base64: inlineBase64(original.path) };
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
export function mediaAssetPath(id) {
|
|
311
|
+
const entry = getMediaAsset(id);
|
|
312
|
+
return entry ? storedAssetPath(entry.file) : null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Hand an asset to the OS viewer. The path is resolved from our own index, so
|
|
317
|
+
* no caller-supplied path ever reaches the shell.
|
|
318
|
+
*/
|
|
319
|
+
export function openMediaAsset(id) {
|
|
320
|
+
const path = mediaAssetPath(id);
|
|
321
|
+
if (!path || !existsSync(path)) return { id, opened: false };
|
|
322
|
+
return { id, opened: openWithOs(path) };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** Open the assets directory itself (Studio → folder button). */
|
|
326
|
+
export function openMediaFolder() {
|
|
327
|
+
ensureDirs();
|
|
328
|
+
return { path: assetsDir(), opened: openWithOs(assetsDir()) };
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function openWithOs(path) {
|
|
332
|
+
const [command, args] = process.platform === 'win32'
|
|
333
|
+
? ['cmd', ['/c', 'start', '', path]]
|
|
334
|
+
: process.platform === 'darwin'
|
|
335
|
+
? ['open', [path]]
|
|
336
|
+
: ['xdg-open', [path]];
|
|
337
|
+
const child = spawn(command, args, { detached: true, stdio: 'ignore', windowsHide: true });
|
|
338
|
+
child.unref();
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
export function deleteMediaAsset(id) {
|
|
343
|
+
ensureDirs();
|
|
344
|
+
let removed = false;
|
|
345
|
+
withFileLockSync(indexLockPath(), () => {
|
|
346
|
+
const assets = readIndex();
|
|
347
|
+
const entry = assets.find((row) => row.id === id);
|
|
348
|
+
if (!entry) return;
|
|
349
|
+
const path = storedAssetPath(entry.file);
|
|
350
|
+
try { if (path) unlinkSync(path); } catch {}
|
|
351
|
+
removeRenditions(renditionsDir(), id);
|
|
352
|
+
writeIndex(assets.filter((row) => row.id !== id));
|
|
353
|
+
removed = true;
|
|
354
|
+
});
|
|
355
|
+
return { id, removed };
|
|
356
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import {
|
|
3
|
+
existsSync,
|
|
4
|
+
mkdirSync,
|
|
5
|
+
mkdtempSync,
|
|
6
|
+
readFileSync,
|
|
7
|
+
rmSync,
|
|
8
|
+
writeFileSync,
|
|
9
|
+
} from 'node:fs';
|
|
10
|
+
import { tmpdir } from 'node:os';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { test } from 'node:test';
|
|
13
|
+
|
|
14
|
+
test('media assets are organized by kind, provider, model, and local date while flat files migrate', async () => {
|
|
15
|
+
const root = mkdtempSync(join(tmpdir(), 'mixdog-media-store-'));
|
|
16
|
+
const previousDataDir = process.env.MIXDOG_DATA_DIR;
|
|
17
|
+
process.env.MIXDOG_DATA_DIR = root;
|
|
18
|
+
const assetsDir = join(root, 'media', 'assets');
|
|
19
|
+
const createdAt = new Date(2026, 6, 5, 12, 0, 0).getTime();
|
|
20
|
+
mkdirSync(assetsDir, { recursive: true });
|
|
21
|
+
writeFileSync(join(assetsDir, 'legacy-image.jpg'), Buffer.from('image'));
|
|
22
|
+
writeFileSync(join(assetsDir, 'legacy-video.mp4'), Buffer.from('video'));
|
|
23
|
+
writeFileSync(join(root, 'media', 'index.json'), JSON.stringify({
|
|
24
|
+
version: 1,
|
|
25
|
+
assets: [
|
|
26
|
+
{
|
|
27
|
+
id: 'legacy-image',
|
|
28
|
+
file: 'legacy-image.jpg',
|
|
29
|
+
kind: 'image',
|
|
30
|
+
lane: 'gemini',
|
|
31
|
+
model: 'image-alpha',
|
|
32
|
+
prompt: 'image',
|
|
33
|
+
options: { aspectRatio: '4:3' },
|
|
34
|
+
mime: 'image/jpeg',
|
|
35
|
+
bytes: 5,
|
|
36
|
+
createdAt,
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
id: 'legacy-video',
|
|
40
|
+
file: 'legacy-video.mp4',
|
|
41
|
+
kind: 'video',
|
|
42
|
+
lane: 'grok',
|
|
43
|
+
model: 'video-beta',
|
|
44
|
+
prompt: 'video',
|
|
45
|
+
options: { aspectRatio: '16:9' },
|
|
46
|
+
mime: 'video/mp4',
|
|
47
|
+
bytes: 5,
|
|
48
|
+
createdAt,
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
}));
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
const store = await import(`./store.mjs?test=${Date.now()}`);
|
|
55
|
+
const listed = store.listMediaAssets({ limit: 10 }).assets;
|
|
56
|
+
assert.equal(listed[0].file, 'images/gemini/image-alpha/2026-07-05/legacy-image.jpg');
|
|
57
|
+
assert.equal(listed[1].file, 'videos/grok/video-beta/2026-07-05/legacy-video.mp4');
|
|
58
|
+
assert.equal(existsSync(join(assetsDir, ...listed[0].file.split('/'))), true);
|
|
59
|
+
assert.equal(existsSync(join(assetsDir, ...listed[1].file.split('/'))), true);
|
|
60
|
+
assert.equal(existsSync(join(assetsDir, 'legacy-image.jpg')), false);
|
|
61
|
+
assert.equal(
|
|
62
|
+
(await store.readMediaAsset('legacy-video')).base64,
|
|
63
|
+
Buffer.from('video').toString('base64'),
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
// A tile-sized rendition is impossible for these stub bytes (and video
|
|
67
|
+
// needs ffmpeg): the read must REPORT the miss instead of quietly
|
|
68
|
+
// shipping the original, which is what made the remote gallery slow.
|
|
69
|
+
const missing = await store.readMediaAsset('legacy-image', { variant: 'thumb' });
|
|
70
|
+
assert.equal(missing.available, false);
|
|
71
|
+
assert.equal(missing.base64, '');
|
|
72
|
+
// Callers that can afford full-size bytes (local IPC) opt in explicitly
|
|
73
|
+
// and get them labelled as a downgrade.
|
|
74
|
+
const downgraded = await store.readMediaAsset('legacy-image', {
|
|
75
|
+
variant: 'thumb',
|
|
76
|
+
allowOriginal: true,
|
|
77
|
+
});
|
|
78
|
+
assert.equal(downgraded.variant, 'original');
|
|
79
|
+
assert.equal(downgraded.downgraded, true);
|
|
80
|
+
assert.equal(downgraded.base64, Buffer.from('image').toString('base64'));
|
|
81
|
+
|
|
82
|
+
const saved = store.saveMediaAsset({
|
|
83
|
+
kind: 'video',
|
|
84
|
+
lane: 'gemini',
|
|
85
|
+
model: 'veo:3/preview',
|
|
86
|
+
prompt: 'new clip',
|
|
87
|
+
options: { aspectRatio: '16:9' },
|
|
88
|
+
mime: 'video/mp4',
|
|
89
|
+
bytes: Buffer.from('new-video'),
|
|
90
|
+
});
|
|
91
|
+
assert.match(saved.file, /^videos\/gemini\/veo-3-preview\/\d{4}-\d{2}-\d{2}\/[0-9a-f-]+\.mp4$/);
|
|
92
|
+
assert.equal(existsSync(join(assetsDir, ...saved.file.split('/'))), true);
|
|
93
|
+
assert.equal(store.deleteMediaAsset(saved.id).removed, true);
|
|
94
|
+
assert.equal(existsSync(join(assetsDir, ...saved.file.split('/'))), false);
|
|
95
|
+
|
|
96
|
+
const persisted = JSON.parse(readFileSync(join(root, 'media', 'index.json'), 'utf8'));
|
|
97
|
+
assert.equal(persisted.version, 2);
|
|
98
|
+
} finally {
|
|
99
|
+
if (previousDataDir === undefined) delete process.env.MIXDOG_DATA_DIR;
|
|
100
|
+
else process.env.MIXDOG_DATA_DIR = previousDataDir;
|
|
101
|
+
rmSync(root, { recursive: true, force: true });
|
|
102
|
+
}
|
|
103
|
+
});
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Upstream failure translation for media lanes.
|
|
3
|
+
*
|
|
4
|
+
* A credential can be present and still unusable (exhausted credits, billing
|
|
5
|
+
* caps, per-account gating). Those come back as raw provider JSON, so the lane
|
|
6
|
+
* maps them to one legible sentence instead of leaking a wall of upstream text
|
|
7
|
+
* into the Studio.
|
|
8
|
+
*/
|
|
9
|
+
import { mediaError } from './lanes.mjs';
|
|
10
|
+
|
|
11
|
+
const CREDIT_HINTS = /credit|spending limit|billing|quota|insufficient|payment/i;
|
|
12
|
+
|
|
13
|
+
function reason(text) {
|
|
14
|
+
try {
|
|
15
|
+
const parsed = JSON.parse(text);
|
|
16
|
+
const message = parsed?.error?.message || parsed?.error || parsed?.message;
|
|
17
|
+
return typeof message === 'string' ? message : '';
|
|
18
|
+
} catch {
|
|
19
|
+
return '';
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Build the error a lane throws for a non-OK upstream response. */
|
|
24
|
+
export function upstreamError(label, status, text) {
|
|
25
|
+
const detail = reason(text) || String(text || '').slice(0, 300);
|
|
26
|
+
if (status === 401 || status === 403) {
|
|
27
|
+
if (CREDIT_HINTS.test(detail)) {
|
|
28
|
+
return mediaError(`${label}: account has no usable balance — ${detail}`, 'MEDIA_BILLING_BLOCKED', status);
|
|
29
|
+
}
|
|
30
|
+
return mediaError(`${label}: authentication rejected — sign in again in Settings → Providers`, 'MEDIA_AUTH_REJECTED', status);
|
|
31
|
+
}
|
|
32
|
+
if (status === 402 || CREDIT_HINTS.test(detail)) {
|
|
33
|
+
return mediaError(`${label}: account has no usable balance — ${detail}`, 'MEDIA_BILLING_BLOCKED', status || 402);
|
|
34
|
+
}
|
|
35
|
+
if (status === 429) {
|
|
36
|
+
return mediaError(`${label}: rate limited — retry in a moment`, 'MEDIA_RATE_LIMITED', 429);
|
|
37
|
+
}
|
|
38
|
+
return mediaError(`${label} failed (${status})${detail ? `: ${detail}` : ''}`, 'MEDIA_UPSTREAM_FAILED', status);
|
|
39
|
+
}
|