dsh-cliproxyapi-pro 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.en.md +138 -9
- package/README.md +138 -9
- package/lib/client.js +3368 -904
- package/lib/index.js +2042 -33
- package/lib/media.js +2528 -0
- package/package.json +3 -2
package/lib/media.js
ADDED
|
@@ -0,0 +1,2528 @@
|
|
|
1
|
+
// Internal module: CPA image / video generation support.
|
|
2
|
+
//
|
|
3
|
+
// Classifies catalog models as chat / image / video, owns the independent
|
|
4
|
+
// `media-settings.json` preferences, `media-sessions.json` (per-session picks
|
|
5
|
+
// and run records) and `media-jobs.json` (video jobs that outlive a reload),
|
|
6
|
+
// runs a read-only diagnostic, and talks to CPA's image and video routes.
|
|
7
|
+
// Generation happens only when the host calls it for an explicit user action.
|
|
8
|
+
// Keys are passed in per call and never stored or returned.
|
|
9
|
+
|
|
10
|
+
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
11
|
+
import { randomBytes } from "node:crypto";
|
|
12
|
+
import { inflateSync } from "node:zlib";
|
|
13
|
+
import { dirname, isAbsolute, join } from "node:path";
|
|
14
|
+
|
|
15
|
+
// #region Classification
|
|
16
|
+
|
|
17
|
+
export const MEDIA_KINDS = ["chat", "image", "video"];
|
|
18
|
+
|
|
19
|
+
// Families CLIProxyAPI routes to its image / video handlers
|
|
20
|
+
// (sdk/api/handlers/openai/openai_{images,videos}_handlers.go), matched by
|
|
21
|
+
// prefix: a catalog may list variants CPA does not accept yet (BCPA lists
|
|
22
|
+
// `grok-imagine-video-1.5-lite`; CPA 9e79552 accepts only the base, `-1.5`
|
|
23
|
+
// and `-1.5-preview`). Such a model still cannot chat, so it is a media
|
|
24
|
+
// model, and CPA's own error explains a refused generation. Compare against
|
|
25
|
+
// those handlers when CPA adds a family.
|
|
26
|
+
const CPA_IMAGE_PREFIXES = ["grok-imagine-image", "gpt-image-"];
|
|
27
|
+
const CPA_VIDEO_PREFIXES = ["grok-imagine-video"];
|
|
28
|
+
const SORA_ALIAS_TARGET = "grok-imagine-video";
|
|
29
|
+
|
|
30
|
+
/** The model id without an account / vendor prefix, lower-cased. */
|
|
31
|
+
export function bareModelId(id) {
|
|
32
|
+
return String(id ?? "").trim().toLowerCase().split("/").pop() ?? "";
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function isSoraAlias(bare) {
|
|
36
|
+
return bare === "sora-2" || bare.startsWith("sora-2-");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function lowerList(value) {
|
|
40
|
+
return Array.isArray(value)
|
|
41
|
+
? value.filter((v) => typeof v === "string").map((v) => v.trim().toLowerCase()).filter(Boolean)
|
|
42
|
+
: [];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Kind of a parsed catalog model, from catalog fields first and CPA's own
|
|
47
|
+
* model families second. `catalogType` is the CPA registry `type`
|
|
48
|
+
* (`openai-image` marks image endpoints); `outputModalities` the declared
|
|
49
|
+
* output modalities.
|
|
50
|
+
* @returns one of {@link MEDIA_KINDS}.
|
|
51
|
+
*/
|
|
52
|
+
export function detectMediaKind(model) {
|
|
53
|
+
const type = String(model?.catalogType ?? "").trim().toLowerCase();
|
|
54
|
+
if (type === "openai-image" || /(^|[-_])image($|[-_])/.test(type)) return "image";
|
|
55
|
+
if (/(^|[-_])video($|[-_])/.test(type)) return "video";
|
|
56
|
+
const out = lowerList(model?.outputModalities);
|
|
57
|
+
if (out.includes("video")) return "video";
|
|
58
|
+
if (out.includes("image") && !out.includes("text")) return "image";
|
|
59
|
+
const bare = bareModelId(model?.id);
|
|
60
|
+
if (CPA_VIDEO_PREFIXES.some((p) => bare.startsWith(p)) || isSoraAlias(bare)) return "video";
|
|
61
|
+
if (CPA_IMAGE_PREFIXES.some((p) => bare.startsWith(p))) return "image";
|
|
62
|
+
return "chat";
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The model CPA really calls for an alias id (`sora-2*` → Grok video), or undefined. */
|
|
66
|
+
export function mediaAliasTarget(id) {
|
|
67
|
+
return isSoraAlias(bareModelId(id)) ? SORA_ALIAS_TARGET : undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Effective kind: a valid user override wins over detection. */
|
|
71
|
+
export function effectiveKind(model, overrides) {
|
|
72
|
+
const forced = overrides?.[model?.id];
|
|
73
|
+
if (MEDIA_KINDS.includes(forced)) return forced;
|
|
74
|
+
if (MEDIA_KINDS.includes(model?.detectedKind)) return model.detectedKind;
|
|
75
|
+
return detectMediaKind(model);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// #endregion
|
|
79
|
+
|
|
80
|
+
// #region Preferences (media-settings.json)
|
|
81
|
+
|
|
82
|
+
export const MEDIA_SETTINGS_VERSION = 1;
|
|
83
|
+
export const VIDEO_SECONDS = { min: 1, max: 15 };
|
|
84
|
+
export const VIDEO_RESOLUTIONS = ["480p", "720p"];
|
|
85
|
+
export const BUDGET_SECONDS = { min: 1, max: 3600 };
|
|
86
|
+
export const CONFIRM_MODES = ["ask", "auto"];
|
|
87
|
+
const MAX_OUTPUT_DIR = 260;
|
|
88
|
+
const MAX_LIST = 500;
|
|
89
|
+
const ACCOUNT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/;
|
|
90
|
+
|
|
91
|
+
export class MediaError extends Error {
|
|
92
|
+
constructor(code, detail) {
|
|
93
|
+
super(code);
|
|
94
|
+
this.name = "MediaError";
|
|
95
|
+
this.code = code;
|
|
96
|
+
if (detail !== undefined) this.detail = detail;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function defaultMediaSettings() {
|
|
101
|
+
return {
|
|
102
|
+
version: MEDIA_SETTINGS_VERSION,
|
|
103
|
+
agentTool: false,
|
|
104
|
+
outputDir: "generated",
|
|
105
|
+
defaultVideoSeconds: 6,
|
|
106
|
+
defaultVideoResolution: "720p",
|
|
107
|
+
imageConfirm: "auto",
|
|
108
|
+
videoConfirm: "ask",
|
|
109
|
+
videoBudget: { mode: "limited", seconds: 60 },
|
|
110
|
+
lastSelection: {},
|
|
111
|
+
accounts: {},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function plain(value) {
|
|
116
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function intIn(value, range) {
|
|
120
|
+
return Number.isInteger(value) && value >= range.min && value <= range.max ? value : undefined;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function validOutputDir(value) {
|
|
124
|
+
if (typeof value !== "string") return undefined;
|
|
125
|
+
const trimmed = value.trim();
|
|
126
|
+
if (trimmed === "" || trimmed.length > MAX_OUTPUT_DIR || /[\u0000-\u001f]/.test(trimmed)) return undefined;
|
|
127
|
+
return trimmed;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function validBudget(value) {
|
|
131
|
+
const raw = plain(value);
|
|
132
|
+
if (raw?.mode === "unlimited") return { mode: "unlimited" };
|
|
133
|
+
if (raw?.mode === "limited") {
|
|
134
|
+
const seconds = intIn(raw.seconds, BUDGET_SECONDS);
|
|
135
|
+
if (seconds !== undefined) return { mode: "limited", seconds };
|
|
136
|
+
}
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function validSelectionRef(value) {
|
|
141
|
+
if (typeof value !== "string") return undefined;
|
|
142
|
+
const trimmed = value.trim();
|
|
143
|
+
const slash = trimmed.indexOf("/");
|
|
144
|
+
if (slash <= 0 || slash === trimmed.length - 1 || trimmed.length > 300) return undefined;
|
|
145
|
+
return ACCOUNT_ID_PATTERN.test(trimmed.slice(0, slash)) ? trimmed : undefined;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function validModelList(value) {
|
|
149
|
+
if (!Array.isArray(value)) return undefined;
|
|
150
|
+
const out = [];
|
|
151
|
+
for (const item of value) {
|
|
152
|
+
if (typeof item !== "string") continue;
|
|
153
|
+
const trimmed = item.trim();
|
|
154
|
+
if (trimmed !== "" && trimmed.length <= 200 && !out.includes(trimmed)) out.push(trimmed);
|
|
155
|
+
if (out.length >= MAX_LIST) break;
|
|
156
|
+
}
|
|
157
|
+
return out;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function validKindOverrides(value) {
|
|
161
|
+
const raw = plain(value);
|
|
162
|
+
if (raw === undefined) return {};
|
|
163
|
+
const out = {};
|
|
164
|
+
for (const [id, kind] of Object.entries(raw)) {
|
|
165
|
+
const key = id.trim();
|
|
166
|
+
if (key === "" || key.length > 200 || !MEDIA_KINDS.includes(kind)) continue;
|
|
167
|
+
out[key] = kind;
|
|
168
|
+
if (Object.keys(out).length >= MAX_LIST) break;
|
|
169
|
+
}
|
|
170
|
+
return out;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Per account: the media models switched off (new models start switched on)
|
|
174
|
+
// and the user's chat / image / video reclassifications.
|
|
175
|
+
function normalizeAccountMedia(value) {
|
|
176
|
+
const raw = plain(value) ?? {};
|
|
177
|
+
return {
|
|
178
|
+
disabledModels: validModelList(raw.disabledModels) ?? [],
|
|
179
|
+
kindOverrides: validKindOverrides(raw.kindOverrides),
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Tolerant normalization: every invalid or missing field takes its default,
|
|
185
|
+
* unknown fields are dropped. The result always has `version: 1`.
|
|
186
|
+
*/
|
|
187
|
+
export function normalizeMediaSettings(raw) {
|
|
188
|
+
const input = plain(raw) ?? {};
|
|
189
|
+
const defaults = defaultMediaSettings();
|
|
190
|
+
const accounts = {};
|
|
191
|
+
for (const [id, entry] of Object.entries(plain(input.accounts) ?? {})) {
|
|
192
|
+
if (ACCOUNT_ID_PATTERN.test(id)) accounts[id] = normalizeAccountMedia(entry);
|
|
193
|
+
}
|
|
194
|
+
const selection = plain(input.lastSelection) ?? {};
|
|
195
|
+
const lastSelection = {};
|
|
196
|
+
for (const kind of ["image", "video"]) {
|
|
197
|
+
const ref = validSelectionRef(selection[kind]);
|
|
198
|
+
if (ref !== undefined) lastSelection[kind] = ref;
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
version: MEDIA_SETTINGS_VERSION,
|
|
202
|
+
agentTool: input.agentTool === true,
|
|
203
|
+
outputDir: validOutputDir(input.outputDir) ?? defaults.outputDir,
|
|
204
|
+
defaultVideoSeconds: intIn(input.defaultVideoSeconds, VIDEO_SECONDS) ?? defaults.defaultVideoSeconds,
|
|
205
|
+
defaultVideoResolution: VIDEO_RESOLUTIONS.includes(input.defaultVideoResolution)
|
|
206
|
+
? input.defaultVideoResolution
|
|
207
|
+
: defaults.defaultVideoResolution,
|
|
208
|
+
imageConfirm: CONFIRM_MODES.includes(input.imageConfirm) ? input.imageConfirm : defaults.imageConfirm,
|
|
209
|
+
videoConfirm: CONFIRM_MODES.includes(input.videoConfirm) ? input.videoConfirm : defaults.videoConfirm,
|
|
210
|
+
videoBudget: validBudget(input.videoBudget) ?? defaults.videoBudget,
|
|
211
|
+
lastSelection,
|
|
212
|
+
accounts,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Read preferences. A missing file yields defaults; an unreadable or
|
|
218
|
+
* malformed one also yields defaults but reports `error: "invalid-file"` so
|
|
219
|
+
* the caller can log it and the next save keeps a copy of the damaged file.
|
|
220
|
+
*/
|
|
221
|
+
export function loadMediaSettings(filePath) {
|
|
222
|
+
if (!existsSync(filePath)) return { settings: defaultMediaSettings() };
|
|
223
|
+
try {
|
|
224
|
+
const raw = JSON.parse(readFileSync(filePath, "utf8"));
|
|
225
|
+
if (plain(raw) === undefined) return { settings: defaultMediaSettings(), error: "invalid-file" };
|
|
226
|
+
return { settings: normalizeMediaSettings(raw) };
|
|
227
|
+
} catch {
|
|
228
|
+
return { settings: defaultMediaSettings(), error: "invalid-file" };
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Atomic write; a damaged existing file is first renamed to `*.corrupt-<time>.json`. */
|
|
233
|
+
export function saveMediaSettings(filePath, settings) {
|
|
234
|
+
mkdirSync(dirname(filePath), { recursive: true });
|
|
235
|
+
if (existsSync(filePath) && loadMediaSettings(filePath).error !== undefined) {
|
|
236
|
+
renameSync(filePath, filePath.replace(/\.json$/, `.corrupt-${Date.now().toString(36)}.json`));
|
|
237
|
+
}
|
|
238
|
+
const tmp = `${filePath}.${Date.now().toString(36)}.tmp`;
|
|
239
|
+
writeFileSync(tmp, `${JSON.stringify(normalizeMediaSettings(settings), null, 2)}\n`, "utf8");
|
|
240
|
+
renameSync(tmp, filePath);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const PATCH_FIELDS = [
|
|
244
|
+
"agentTool",
|
|
245
|
+
"outputDir",
|
|
246
|
+
"defaultVideoSeconds",
|
|
247
|
+
"defaultVideoResolution",
|
|
248
|
+
"imageConfirm",
|
|
249
|
+
"videoConfirm",
|
|
250
|
+
"videoBudget",
|
|
251
|
+
"account",
|
|
252
|
+
];
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Apply a validated patch. Unlike {@link normalizeMediaSettings}, an invalid
|
|
256
|
+
* value is an error (`MediaError` with a stable code), never a silent default.
|
|
257
|
+
* `account.kindOverrides` merges per model; a `null` value removes an override.
|
|
258
|
+
* `account.disabledModels` replaces the account's list of switched-off models.
|
|
259
|
+
* @returns `{ settings, kindsChanged }` — `kindsChanged` names the account
|
|
260
|
+
* whose model kinds changed (its routes need a re-sync).
|
|
261
|
+
*/
|
|
262
|
+
export function applyMediaPatch(current, patch) {
|
|
263
|
+
const input = plain(patch);
|
|
264
|
+
if (input === undefined) throw new MediaError("invalid-request");
|
|
265
|
+
const unknown = Object.keys(input).filter((key) => !PATCH_FIELDS.includes(key) && key !== "accountId");
|
|
266
|
+
if (unknown.length > 0) throw new MediaError("invalid-request", `unknown field(s): ${unknown.join(", ")}`);
|
|
267
|
+
const next = normalizeMediaSettings(current);
|
|
268
|
+
if ("agentTool" in input) {
|
|
269
|
+
if (typeof input.agentTool !== "boolean") throw new MediaError("invalid-agent-tool");
|
|
270
|
+
next.agentTool = input.agentTool;
|
|
271
|
+
}
|
|
272
|
+
if ("outputDir" in input) {
|
|
273
|
+
const dir = validOutputDir(input.outputDir);
|
|
274
|
+
if (dir === undefined) throw new MediaError("invalid-output-dir");
|
|
275
|
+
next.outputDir = dir;
|
|
276
|
+
}
|
|
277
|
+
if ("defaultVideoSeconds" in input) {
|
|
278
|
+
const seconds = intIn(input.defaultVideoSeconds, VIDEO_SECONDS);
|
|
279
|
+
if (seconds === undefined) throw new MediaError("invalid-video-seconds");
|
|
280
|
+
next.defaultVideoSeconds = seconds;
|
|
281
|
+
}
|
|
282
|
+
if ("defaultVideoResolution" in input) {
|
|
283
|
+
if (!VIDEO_RESOLUTIONS.includes(input.defaultVideoResolution)) throw new MediaError("invalid-video-resolution");
|
|
284
|
+
next.defaultVideoResolution = input.defaultVideoResolution;
|
|
285
|
+
}
|
|
286
|
+
for (const field of ["imageConfirm", "videoConfirm"]) {
|
|
287
|
+
if (!(field in input)) continue;
|
|
288
|
+
if (!CONFIRM_MODES.includes(input[field])) throw new MediaError("invalid-confirm-mode");
|
|
289
|
+
next[field] = input[field];
|
|
290
|
+
}
|
|
291
|
+
if ("videoBudget" in input) {
|
|
292
|
+
const budget = validBudget(input.videoBudget);
|
|
293
|
+
if (budget === undefined) throw new MediaError("invalid-video-budget");
|
|
294
|
+
next.videoBudget = budget;
|
|
295
|
+
}
|
|
296
|
+
let kindsChanged;
|
|
297
|
+
if ("account" in input) {
|
|
298
|
+
const account = plain(input.account);
|
|
299
|
+
const id = typeof account?.accountId === "string" ? account.accountId.trim() : "";
|
|
300
|
+
if (!ACCOUNT_ID_PATTERN.test(id)) throw new MediaError("invalid-account");
|
|
301
|
+
const entry = normalizeAccountMedia(next.accounts[id]);
|
|
302
|
+
if ("disabledModels" in account) {
|
|
303
|
+
const list = validModelList(account.disabledModels);
|
|
304
|
+
if (list === undefined) throw new MediaError("invalid-model-list");
|
|
305
|
+
entry.disabledModels = list;
|
|
306
|
+
}
|
|
307
|
+
if ("kindOverrides" in account) {
|
|
308
|
+
const raw = plain(account.kindOverrides);
|
|
309
|
+
if (raw === undefined) throw new MediaError("invalid-kind");
|
|
310
|
+
const merged = { ...entry.kindOverrides };
|
|
311
|
+
for (const [modelId, kind] of Object.entries(raw)) {
|
|
312
|
+
const key = modelId.trim();
|
|
313
|
+
if (key === "" || key.length > 200) throw new MediaError("invalid-kind");
|
|
314
|
+
if (kind === null) delete merged[key];
|
|
315
|
+
else if (MEDIA_KINDS.includes(kind)) merged[key] = kind;
|
|
316
|
+
else throw new MediaError("invalid-kind");
|
|
317
|
+
}
|
|
318
|
+
if (JSON.stringify(merged) !== JSON.stringify(entry.kindOverrides)) kindsChanged = id;
|
|
319
|
+
entry.kindOverrides = merged;
|
|
320
|
+
}
|
|
321
|
+
next.accounts[id] = entry;
|
|
322
|
+
}
|
|
323
|
+
return { settings: normalizeMediaSettings(next), kindsChanged };
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** Kind overrides of one account (empty when none). */
|
|
327
|
+
export function accountKindOverrides(settings, accountId) {
|
|
328
|
+
return settings?.accounts?.[accountId]?.kindOverrides ?? {};
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** Whether a media model is offered for an account (on unless switched off). */
|
|
332
|
+
export function mediaModelEnabled(settings, accountId, modelId) {
|
|
333
|
+
const list = settings?.accounts?.[accountId]?.disabledModels;
|
|
334
|
+
return !Array.isArray(list) || !list.includes(modelId);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// #endregion
|
|
338
|
+
|
|
339
|
+
// #region Read-only diagnostic
|
|
340
|
+
|
|
341
|
+
const DIAG_MAX_BYTES = 8 * 1024 * 1024;
|
|
342
|
+
const DIAG_FIELDS = [
|
|
343
|
+
"type",
|
|
344
|
+
"object",
|
|
345
|
+
"owned_by",
|
|
346
|
+
"supportedOutputModalities",
|
|
347
|
+
"output_modalities",
|
|
348
|
+
"outputModalities",
|
|
349
|
+
"input_modalities",
|
|
350
|
+
"supportedInputModalities",
|
|
351
|
+
"display_name",
|
|
352
|
+
];
|
|
353
|
+
const MEDIA_HINT = /(image|imagine|video|sora|veo|dall-?e)/i;
|
|
354
|
+
const DIAG_PROBE_ID = "dsh-diagnostic-probe";
|
|
355
|
+
|
|
356
|
+
async function readBoundedText(response, max = DIAG_MAX_BYTES) {
|
|
357
|
+
const declared = Number(response.headers?.get?.("content-length"));
|
|
358
|
+
const limit = `response is larger than ${Math.round(max / 1024 / 1024)} MiB`;
|
|
359
|
+
if (Number.isFinite(declared) && declared > max) throw new Error(limit);
|
|
360
|
+
const text = await response.text();
|
|
361
|
+
if (new TextEncoder().encode(text).byteLength > max) throw new Error(limit);
|
|
362
|
+
return text;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
function short(value) {
|
|
366
|
+
if (typeof value === "string") return value.slice(0, 120);
|
|
367
|
+
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
368
|
+
if (Array.isArray(value)) return value.filter((v) => typeof v === "string").slice(0, 12).map((v) => v.slice(0, 40));
|
|
369
|
+
return undefined;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function catalogEntries(payload) {
|
|
373
|
+
if (Array.isArray(payload?.models)) return payload.models;
|
|
374
|
+
if (Array.isArray(payload?.data)) return payload.data;
|
|
375
|
+
return undefined;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** Media-looking catalog entries with a fixed set of descriptive fields (no other values). */
|
|
379
|
+
export function summarizeCatalogForMedia(payload) {
|
|
380
|
+
const entries = catalogEntries(payload);
|
|
381
|
+
if (entries === undefined) return undefined;
|
|
382
|
+
const media = [];
|
|
383
|
+
for (const item of entries) {
|
|
384
|
+
if (plain(item) === undefined) continue;
|
|
385
|
+
const id = typeof (item.slug ?? item.id) === "string" ? String(item.slug ?? item.id).trim() : "";
|
|
386
|
+
if (id === "") continue;
|
|
387
|
+
const kind = detectMediaKind({
|
|
388
|
+
id,
|
|
389
|
+
catalogType: item.type,
|
|
390
|
+
outputModalities: item.supportedOutputModalities ?? item.output_modalities ?? item.outputModalities,
|
|
391
|
+
});
|
|
392
|
+
if (kind === "chat" && !MEDIA_HINT.test(id)) continue;
|
|
393
|
+
const fields = {};
|
|
394
|
+
for (const field of DIAG_FIELDS) {
|
|
395
|
+
const value = short(item[field]);
|
|
396
|
+
if (value !== undefined) fields[field] = value;
|
|
397
|
+
}
|
|
398
|
+
media.push({ id, kind, keys: Object.keys(item).sort().slice(0, 40), fields });
|
|
399
|
+
if (media.length >= 50) break;
|
|
400
|
+
}
|
|
401
|
+
return { count: entries.length, media };
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Read-only CPA check for media support. Sends only GET requests:
|
|
406
|
+
* - the enriched (`?client_version=pi`) and plain `/v1/models` catalogs,
|
|
407
|
+
* with the account key, reporting media-looking entries and their fields;
|
|
408
|
+
* - two route probes **without** the key, so CPA answers 401 for a route
|
|
409
|
+
* that exists and its router's plain-text 404 for one that does not,
|
|
410
|
+
* without selecting a credential or contacting an upstream.
|
|
411
|
+
* Never returns the key, response bodies or upstream URLs.
|
|
412
|
+
*/
|
|
413
|
+
export async function diagnoseMedia({ fetch: fetchImpl = fetch, rootURL, apiKey, timeoutMs = 8000, signal }) {
|
|
414
|
+
const timed = () => {
|
|
415
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
416
|
+
return signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
|
|
417
|
+
};
|
|
418
|
+
const authHeaders = {
|
|
419
|
+
Accept: "application/json",
|
|
420
|
+
...(typeof apiKey === "string" && apiKey !== "" ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
421
|
+
};
|
|
422
|
+
const catalog = async (path) => {
|
|
423
|
+
try {
|
|
424
|
+
const response = await fetchImpl(`${rootURL}${path}`, { method: "GET", headers: authHeaders, signal: timed() });
|
|
425
|
+
if (!response.ok) return { ok: false, status: response.status };
|
|
426
|
+
let payload;
|
|
427
|
+
try {
|
|
428
|
+
payload = JSON.parse(await readBoundedText(response));
|
|
429
|
+
} catch {
|
|
430
|
+
return { ok: false, status: response.status, error: "invalid-json" };
|
|
431
|
+
}
|
|
432
|
+
const summary = summarizeCatalogForMedia(payload);
|
|
433
|
+
return summary === undefined
|
|
434
|
+
? { ok: false, status: response.status, error: "no-model-list" }
|
|
435
|
+
: { ok: true, status: response.status, ...summary };
|
|
436
|
+
} catch (error) {
|
|
437
|
+
if (signal?.aborted) throw signal.reason;
|
|
438
|
+
return { ok: false, error: error?.name === "TimeoutError" ? "timeout" : "network" };
|
|
439
|
+
}
|
|
440
|
+
};
|
|
441
|
+
const probe = async (label, path) => {
|
|
442
|
+
try {
|
|
443
|
+
const response = await fetchImpl(`${rootURL}${path}`, {
|
|
444
|
+
method: "GET",
|
|
445
|
+
headers: { Accept: "application/json" },
|
|
446
|
+
signal: timed(),
|
|
447
|
+
});
|
|
448
|
+
let text = "";
|
|
449
|
+
try {
|
|
450
|
+
text = (await readBoundedText(response)).trim();
|
|
451
|
+
} catch {}
|
|
452
|
+
const verdict =
|
|
453
|
+
response.status === 401 || response.status === 403
|
|
454
|
+
? "present"
|
|
455
|
+
: response.status === 404 && /^404 page not found$/i.test(text)
|
|
456
|
+
? "missing"
|
|
457
|
+
: "unknown";
|
|
458
|
+
return { route: label, status: response.status, verdict };
|
|
459
|
+
} catch (error) {
|
|
460
|
+
if (signal?.aborted) throw signal.reason;
|
|
461
|
+
return { route: label, verdict: "unreachable", error: error?.name === "TimeoutError" ? "timeout" : "network" };
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
const [enriched, listing, nativeVideo, videoContent] = await Promise.all([
|
|
465
|
+
catalog("/v1/models?client_version=pi"),
|
|
466
|
+
catalog("/v1/models"),
|
|
467
|
+
probe("GET /v1/videos/{id}", `/v1/videos/${DIAG_PROBE_ID}`),
|
|
468
|
+
probe("GET /openai/v1/videos/{id}/content", `/openai/v1/videos/${DIAG_PROBE_ID}/content`),
|
|
469
|
+
]);
|
|
470
|
+
return {
|
|
471
|
+
endpoint: rootURL,
|
|
472
|
+
checkedAt: new Date().toISOString(),
|
|
473
|
+
catalogs: { enriched, listing },
|
|
474
|
+
routes: [nativeVideo, videoContent],
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// #endregion
|
|
479
|
+
|
|
480
|
+
// #region Model choice
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Generation models a user may pick: enabled models of the given kind on
|
|
484
|
+
* enabled accounts. `ref` is `<accountId>/<modelId>` (an account id never
|
|
485
|
+
* contains a slash; a model id may).
|
|
486
|
+
* @param accounts - `controller.mediaAccounts()` entries.
|
|
487
|
+
*/
|
|
488
|
+
export function mediaOptions(accounts, settings, kind) {
|
|
489
|
+
const out = [];
|
|
490
|
+
for (const account of accounts ?? []) {
|
|
491
|
+
if (account.enabled !== true) continue;
|
|
492
|
+
for (const model of account.models ?? []) {
|
|
493
|
+
if (model.kind !== kind || !mediaModelEnabled(settings, account.id, model.id)) continue;
|
|
494
|
+
out.push({
|
|
495
|
+
ref: `${account.id}/${model.id}`,
|
|
496
|
+
accountId: account.id,
|
|
497
|
+
accountName: account.displayName || account.id,
|
|
498
|
+
modelId: model.id,
|
|
499
|
+
...(model.name === undefined ? {} : { name: model.name }),
|
|
500
|
+
...(model.aliasOf === undefined ? {} : { aliasOf: model.aliasOf }),
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
return out;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** Split a selection ref at its first slash. */
|
|
508
|
+
export function splitMediaRef(ref) {
|
|
509
|
+
const value = typeof ref === "string" ? ref.trim() : "";
|
|
510
|
+
const slash = value.indexOf("/");
|
|
511
|
+
return slash > 0 && slash < value.length - 1
|
|
512
|
+
? { accountId: value.slice(0, slash), modelId: value.slice(slash + 1) }
|
|
513
|
+
: undefined;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Match what a user typed (`-m`): a full ref, then an exact model id, then a
|
|
518
|
+
* unique case-insensitive substring of the model id. Never guesses between
|
|
519
|
+
* several candidates.
|
|
520
|
+
* @throws MediaError `model-not-found` / `model-ambiguous` (detail lists refs).
|
|
521
|
+
*/
|
|
522
|
+
export function matchMediaModel(options, requested) {
|
|
523
|
+
const wanted = String(requested ?? "").trim();
|
|
524
|
+
const lower = wanted.toLowerCase();
|
|
525
|
+
const byRef = options.find((o) => o.ref === wanted);
|
|
526
|
+
if (byRef !== undefined) return byRef;
|
|
527
|
+
const pick = (list) => {
|
|
528
|
+
if (list.length === 1) return list[0];
|
|
529
|
+
if (list.length > 1) throw new MediaError("model-ambiguous", list.map((o) => o.ref).join(", "));
|
|
530
|
+
return undefined;
|
|
531
|
+
};
|
|
532
|
+
const exact = pick(options.filter((o) => o.modelId.toLowerCase() === lower));
|
|
533
|
+
if (exact !== undefined) return exact;
|
|
534
|
+
const partial = pick(lower === "" ? [] : options.filter((o) => o.modelId.toLowerCase().includes(lower)));
|
|
535
|
+
if (partial !== undefined) return partial;
|
|
536
|
+
throw new MediaError("model-not-found", options.map((o) => o.ref).join(", "));
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* The model a generation uses: an explicit request, else the session's pick,
|
|
541
|
+
* else the last pick anywhere, else the only option.
|
|
542
|
+
* @returns `{ option, source }` with source `request | session | last | only`.
|
|
543
|
+
* @throws MediaError `no-media-model` or `choose-model` (several, none picked).
|
|
544
|
+
*/
|
|
545
|
+
export function chooseMediaModel({ options, requested, sessionRef, lastRef }) {
|
|
546
|
+
if (typeof requested === "string" && requested.trim() !== "") {
|
|
547
|
+
return { option: matchMediaModel(options, requested), source: "request" };
|
|
548
|
+
}
|
|
549
|
+
const session = options.find((o) => o.ref === sessionRef);
|
|
550
|
+
if (session !== undefined) return { option: session, source: "session" };
|
|
551
|
+
const last = options.find((o) => o.ref === lastRef);
|
|
552
|
+
if (last !== undefined) return { option: last, source: "last" };
|
|
553
|
+
if (options.length === 1) return { option: options[0], source: "only" };
|
|
554
|
+
if (options.length === 0) throw new MediaError("no-media-model");
|
|
555
|
+
throw new MediaError("choose-model", options.map((o) => o.ref).join(", "));
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// #endregion
|
|
559
|
+
|
|
560
|
+
// #region Session store (media-sessions.json)
|
|
561
|
+
|
|
562
|
+
export const MEDIA_SESSIONS_VERSION = 1;
|
|
563
|
+
const MAX_SESSIONS = 500;
|
|
564
|
+
// Enough to keep the images of a session's recent commands visible in chat.
|
|
565
|
+
export const MAX_RECENT = 48;
|
|
566
|
+
const SESSION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/;
|
|
567
|
+
const COMMAND_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$/;
|
|
568
|
+
|
|
569
|
+
export function validSessionId(value) {
|
|
570
|
+
return typeof value === "string" && SESSION_ID_PATTERN.test(value) ? value : undefined;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** A DSH command run id (`cmd-<token>-<seq>`), or undefined. */
|
|
574
|
+
export function validCommandId(value) {
|
|
575
|
+
return typeof value === "string" && COMMAND_ID_PATTERN.test(value) ? value : undefined;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
function normalizeRecent(value) {
|
|
579
|
+
if (!Array.isArray(value)) return [];
|
|
580
|
+
const out = [];
|
|
581
|
+
for (const item of value) {
|
|
582
|
+
const entry = plain(item);
|
|
583
|
+
if (entry === undefined) continue;
|
|
584
|
+
const { id, kind, file, model, createdAt, commandId } = entry;
|
|
585
|
+
if (typeof id !== "string" || !/^[a-f0-9]{8,32}$/.test(id)) continue;
|
|
586
|
+
if (!["image", "video"].includes(kind) || typeof file !== "string" || !isAbsolute(file)) continue;
|
|
587
|
+
out.push({
|
|
588
|
+
id,
|
|
589
|
+
kind,
|
|
590
|
+
file,
|
|
591
|
+
model: typeof model === "string" ? model.slice(0, 300) : "",
|
|
592
|
+
createdAt: typeof createdAt === "string" ? createdAt.slice(0, 40) : "",
|
|
593
|
+
...(validCommandId(commandId) === undefined ? {} : { commandId }),
|
|
594
|
+
});
|
|
595
|
+
if (out.length >= MAX_RECENT) break;
|
|
596
|
+
}
|
|
597
|
+
return out;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
export const MAX_RUNS = 48;
|
|
601
|
+
export const VIDEO_MODES = ["text", "first-frame", "reference", "first-last", "last-frame", "edit", "extend"];
|
|
602
|
+
const RUN_STATUSES = ["running", "done", "error"];
|
|
603
|
+
const clip = (value, max) => (typeof value === "string" ? value.slice(0, max) : "");
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* One generation run as its chat row shows it: what was asked, by which model,
|
|
607
|
+
* and how it ended. Never paths, keys or upstream bodies; `folder` is a display
|
|
608
|
+
* label (relative to the working directory, or the folder's own name).
|
|
609
|
+
*/
|
|
610
|
+
export function normalizeRun(value) {
|
|
611
|
+
const run = plain(value);
|
|
612
|
+
if (run === undefined || validCommandId(run.commandId) === undefined) return undefined;
|
|
613
|
+
if (!["image", "video"].includes(run.kind) || !RUN_STATUSES.includes(run.status)) return undefined;
|
|
614
|
+
const count = Number.isInteger(run.count) && run.count >= 1 && run.count <= 4 ? run.count : 1;
|
|
615
|
+
const references = Number.isInteger(run.references) && run.references >= 0 && run.references <= 14 ? run.references : 0;
|
|
616
|
+
const files = Number.isInteger(run.files) && run.files >= 0 && run.files <= 4 ? run.files : 0;
|
|
617
|
+
return {
|
|
618
|
+
commandId: run.commandId,
|
|
619
|
+
kind: run.kind,
|
|
620
|
+
status: run.status,
|
|
621
|
+
model: clip(run.model, 300),
|
|
622
|
+
modelName: clip(run.modelName, 300),
|
|
623
|
+
accountName: clip(run.accountName, 120),
|
|
624
|
+
prompt: clip(run.prompt, 4000),
|
|
625
|
+
...(typeof run.aspect === "string" && IMAGE_ASPECTS.includes(run.aspect) ? { aspect: run.aspect } : {}),
|
|
626
|
+
...(run.kind === "video" && intIn(run.seconds, VIDEO_SECONDS) !== undefined ? { seconds: run.seconds } : {}),
|
|
627
|
+
...(run.kind === "video" && VIDEO_RESOLUTIONS.includes(run.resolution) ? { resolution: run.resolution } : {}),
|
|
628
|
+
...(run.kind === "video" && VIDEO_MODES.includes(run.mode) ? { mode: run.mode } : {}),
|
|
629
|
+
...(run.kind === "video" && validCommandId(run.sourceCommandId) !== undefined ? { sourceCommandId: run.sourceCommandId } : {}),
|
|
630
|
+
// Prompt words the edit / extension route cannot use, taken out of the prompt.
|
|
631
|
+
...(run.kind === "video" && Array.isArray(run.ignored) && run.ignored.length > 0
|
|
632
|
+
? { ignored: run.ignored.filter((w) => typeof w === "string" && w !== "").slice(0, 4).map((w) => w.slice(0, 20)) }
|
|
633
|
+
: {}),
|
|
634
|
+
// Parameters the session's chat model read from the prompt (or that it was asked and failed).
|
|
635
|
+
...(plain(run.interpreted) !== undefined && typeof run.interpreted.by === "string" && run.interpreted.by !== ""
|
|
636
|
+
? {
|
|
637
|
+
interpreted: {
|
|
638
|
+
by: run.interpreted.by.slice(0, 160),
|
|
639
|
+
items: Array.isArray(run.interpreted.items) ? run.interpreted.items.filter((w) => typeof w === "string" && w !== "").slice(0, 8).map((w) => w.slice(0, 30)) : [],
|
|
640
|
+
...(run.interpreted.failed === true ? { failed: true } : {}),
|
|
641
|
+
...(run.interpreted.prompt === true ? { prompt: true } : {}),
|
|
642
|
+
},
|
|
643
|
+
}
|
|
644
|
+
: {}),
|
|
645
|
+
// How an edit / extension got its source: xAI's own result link, an inline upload, or a public URL.
|
|
646
|
+
...(run.kind === "video" && ["link", "upload", "url"].includes(run.sourceKind) ? { sourceKind: run.sourceKind } : {}),
|
|
647
|
+
// A local file's name or a URL's host, never a path or a full address.
|
|
648
|
+
...(run.kind === "video" && typeof run.sourceName === "string" && run.sourceName.trim() !== "" ? { sourceName: run.sourceName.trim().slice(0, 200) } : {}),
|
|
649
|
+
...(run.kind === "video" && typeof run.videoId === "string" && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/.test(run.videoId) ? { videoId: run.videoId } : {}),
|
|
650
|
+
count,
|
|
651
|
+
references,
|
|
652
|
+
files,
|
|
653
|
+
startedAt: clip(run.startedAt, 40),
|
|
654
|
+
...(typeof run.finishedAt === "string" ? { finishedAt: run.finishedAt.slice(0, 40) } : {}),
|
|
655
|
+
...(typeof run.folder === "string" && run.folder !== "" ? { folder: run.folder.slice(0, 300) } : {}),
|
|
656
|
+
...(typeof run.revisedPrompt === "string" && run.revisedPrompt !== "" ? { revisedPrompt: run.revisedPrompt.slice(0, 4000) } : {}),
|
|
657
|
+
...(typeof run.errorCode === "string" && /^[a-z][a-z-]{0,59}$/.test(run.errorCode) ? { errorCode: run.errorCode } : {}),
|
|
658
|
+
...(typeof run.error === "string" && run.error !== "" ? { error: run.error.slice(0, 500) } : {}),
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function normalizeRuns(value) {
|
|
663
|
+
if (!Array.isArray(value)) return [];
|
|
664
|
+
const out = [];
|
|
665
|
+
const seen = new Set();
|
|
666
|
+
for (const item of value) {
|
|
667
|
+
const run = normalizeRun(item);
|
|
668
|
+
if (run === undefined || seen.has(run.commandId)) continue;
|
|
669
|
+
seen.add(run.commandId);
|
|
670
|
+
out.push(run);
|
|
671
|
+
if (out.length >= MAX_RUNS) break;
|
|
672
|
+
}
|
|
673
|
+
return out;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function normalizeSessionEntry(value) {
|
|
677
|
+
const raw = plain(value) ?? {};
|
|
678
|
+
const out = {};
|
|
679
|
+
for (const kind of ["image", "video"]) {
|
|
680
|
+
const ref = validSelectionRef(raw[kind]);
|
|
681
|
+
if (ref !== undefined) out[kind] = ref;
|
|
682
|
+
}
|
|
683
|
+
// Agent-tool overrides for this session (absent = follow the global setting).
|
|
684
|
+
if (CONFIRM_MODES.includes(raw.imageConfirm)) out.imageConfirm = raw.imageConfirm;
|
|
685
|
+
if (CONFIRM_MODES.includes(raw.videoConfirm)) out.videoConfirm = raw.videoConfirm;
|
|
686
|
+
const budget = validBudget(raw.videoBudget);
|
|
687
|
+
if (budget !== undefined) out.videoBudget = budget;
|
|
688
|
+
// Video seconds the agent submitted, against the limit named by `budgetKey`.
|
|
689
|
+
if (Number.isInteger(raw.usedSeconds) && raw.usedSeconds > 0 && raw.usedSeconds <= 1_000_000) out.usedSeconds = raw.usedSeconds;
|
|
690
|
+
if (typeof raw.budgetKey === "string" && /^(unlimited|limited:\d{1,4})$/.test(raw.budgetKey)) out.budgetKey = raw.budgetKey;
|
|
691
|
+
if (CONFIRM_MODES.includes(raw.lastConfirm)) out.lastConfirm = raw.lastConfirm;
|
|
692
|
+
out.recent = normalizeRecent(raw.recent);
|
|
693
|
+
out.runs = normalizeRuns(raw.runs);
|
|
694
|
+
out.updatedAt = typeof raw.updatedAt === "string" ? raw.updatedAt.slice(0, 40) : "";
|
|
695
|
+
return out;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
/** The identity of a seconds limit: a different limit starts the count again. */
|
|
699
|
+
export function budgetKey(budget) {
|
|
700
|
+
return budget?.mode === "limited" ? `limited:${budget.seconds}` : "unlimited";
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* The agent tool's effective rules for one (root) session: confirmation per
|
|
705
|
+
* kind and the video seconds limit, each from the session's override or the
|
|
706
|
+
* global setting, and the seconds already used under that limit. The count
|
|
707
|
+
* starts again when the limit changed or the video confirmation went from
|
|
708
|
+
* `ask` to `auto` since the last charge (design §8).
|
|
709
|
+
*/
|
|
710
|
+
export function agentPolicy(settings, entry = {}) {
|
|
711
|
+
const imageConfirm = entry.imageConfirm ?? settings.imageConfirm;
|
|
712
|
+
const videoConfirm = entry.videoConfirm ?? settings.videoConfirm;
|
|
713
|
+
const budget = entry.videoBudget ?? settings.videoBudget;
|
|
714
|
+
const key = budgetKey(budget);
|
|
715
|
+
const stale = (entry.budgetKey !== undefined && entry.budgetKey !== key) || (entry.lastConfirm === "ask" && videoConfirm === "auto");
|
|
716
|
+
const used = stale ? 0 : entry.usedSeconds ?? 0;
|
|
717
|
+
return {
|
|
718
|
+
imageConfirm,
|
|
719
|
+
videoConfirm,
|
|
720
|
+
budget,
|
|
721
|
+
key,
|
|
722
|
+
used,
|
|
723
|
+
limit: budget.mode === "limited" ? budget.seconds : undefined,
|
|
724
|
+
sources: {
|
|
725
|
+
imageConfirm: entry.imageConfirm === undefined ? "global" : "session",
|
|
726
|
+
videoConfirm: entry.videoConfirm === undefined ? "global" : "session",
|
|
727
|
+
videoBudget: entry.videoBudget === undefined ? "global" : "session",
|
|
728
|
+
},
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Apply a session's agent-tool overrides from the composer button:
|
|
734
|
+
* `imageConfirm` / `videoConfirm` `ask` | `auto` | `inherit`, `videoBudget`
|
|
735
|
+
* `{ mode, seconds? }` | `inherit`. The seconds count starts again when the
|
|
736
|
+
* limit changes or video confirmation turns from `ask` to `auto`. Mutates `entry`.
|
|
737
|
+
* @throws MediaError `invalid-confirm-mode` / `invalid-video-budget` / `invalid-request`.
|
|
738
|
+
*/
|
|
739
|
+
export function applySessionPolicy(entry, input, settings) {
|
|
740
|
+
const before = agentPolicy(settings, entry);
|
|
741
|
+
let changed = false;
|
|
742
|
+
for (const field of ["imageConfirm", "videoConfirm"]) {
|
|
743
|
+
if (!(field in input)) continue;
|
|
744
|
+
if (input[field] === "inherit") delete entry[field];
|
|
745
|
+
else if (CONFIRM_MODES.includes(input[field])) entry[field] = input[field];
|
|
746
|
+
else throw new MediaError("invalid-confirm-mode");
|
|
747
|
+
changed = true;
|
|
748
|
+
}
|
|
749
|
+
if ("videoBudget" in input) {
|
|
750
|
+
if (input.videoBudget === "inherit") delete entry.videoBudget;
|
|
751
|
+
else {
|
|
752
|
+
const budget = validBudget(input.videoBudget);
|
|
753
|
+
if (budget === undefined) throw new MediaError("invalid-video-budget");
|
|
754
|
+
entry.videoBudget = budget;
|
|
755
|
+
}
|
|
756
|
+
changed = true;
|
|
757
|
+
}
|
|
758
|
+
if (!changed) throw new MediaError("invalid-request");
|
|
759
|
+
const after = agentPolicy(settings, entry);
|
|
760
|
+
if (after.key !== before.key || (before.videoConfirm === "ask" && after.videoConfirm === "auto")) {
|
|
761
|
+
delete entry.usedSeconds;
|
|
762
|
+
delete entry.budgetKey;
|
|
763
|
+
delete entry.lastConfirm;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Charge (`seconds` > 0) or refund (< 0) agent video seconds on a session
|
|
769
|
+
* entry, under the policy's current limit. Mutates `entry`.
|
|
770
|
+
*/
|
|
771
|
+
export function chargeSeconds(entry, policy, seconds) {
|
|
772
|
+
entry.usedSeconds = Math.max(0, policy.used + seconds);
|
|
773
|
+
entry.budgetKey = policy.key;
|
|
774
|
+
entry.lastConfirm = policy.videoConfirm;
|
|
775
|
+
if (entry.usedSeconds === 0) delete entry.usedSeconds;
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/** Tolerant normalization; keeps the {@link MAX_SESSIONS} most recently updated sessions. */
|
|
779
|
+
export function normalizeMediaSessions(raw) {
|
|
780
|
+
const sessions = Object.entries(plain(plain(raw)?.sessions) ?? {})
|
|
781
|
+
.filter(([id]) => validSessionId(id) !== undefined)
|
|
782
|
+
.map(([id, entry]) => [id, normalizeSessionEntry(entry)])
|
|
783
|
+
.sort((a, b) => String(b[1].updatedAt).localeCompare(String(a[1].updatedAt)))
|
|
784
|
+
.slice(0, MAX_SESSIONS);
|
|
785
|
+
return { version: MEDIA_SESSIONS_VERSION, sessions: Object.fromEntries(sessions) };
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Per-session generation state (model picks, recent outputs) in its own file,
|
|
790
|
+
* re-read on every access. Writes are atomic; a damaged file loads as empty
|
|
791
|
+
* and is kept as `*.corrupt-*.json` by the next write.
|
|
792
|
+
*/
|
|
793
|
+
export function createMediaSessionStore(filePath, { now = () => new Date() } = {}) {
|
|
794
|
+
const read = () => {
|
|
795
|
+
if (!existsSync(filePath)) return { data: normalizeMediaSessions({}) };
|
|
796
|
+
try {
|
|
797
|
+
const raw = JSON.parse(readFileSync(filePath, "utf8"));
|
|
798
|
+
if (plain(raw) === undefined) return { data: normalizeMediaSessions({}), damaged: true };
|
|
799
|
+
return { data: normalizeMediaSessions(raw) };
|
|
800
|
+
} catch {
|
|
801
|
+
return { data: normalizeMediaSessions({}), damaged: true };
|
|
802
|
+
}
|
|
803
|
+
};
|
|
804
|
+
return {
|
|
805
|
+
get(sessionId) {
|
|
806
|
+
const id = validSessionId(sessionId);
|
|
807
|
+
return id === undefined ? normalizeSessionEntry({}) : read().data.sessions[id] ?? normalizeSessionEntry({});
|
|
808
|
+
},
|
|
809
|
+
/** Apply `mutate(entry)` to one session's entry and persist it. */
|
|
810
|
+
update(sessionId, mutate) {
|
|
811
|
+
const id = validSessionId(sessionId);
|
|
812
|
+
if (id === undefined) throw new MediaError("invalid-session");
|
|
813
|
+
const { data, damaged } = read();
|
|
814
|
+
const entry = normalizeSessionEntry(data.sessions[id]);
|
|
815
|
+
mutate(entry);
|
|
816
|
+
entry.updatedAt = now().toISOString();
|
|
817
|
+
data.sessions[id] = normalizeSessionEntry(entry);
|
|
818
|
+
const next = normalizeMediaSessions(data);
|
|
819
|
+
mkdirSync(dirname(filePath), { recursive: true });
|
|
820
|
+
if (damaged && existsSync(filePath)) {
|
|
821
|
+
renameSync(filePath, filePath.replace(/\.json$/, `.corrupt-${Date.now().toString(36)}.json`));
|
|
822
|
+
}
|
|
823
|
+
const tmp = `${filePath}.${Date.now().toString(36)}.tmp`;
|
|
824
|
+
writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`, "utf8");
|
|
825
|
+
renameSync(tmp, filePath);
|
|
826
|
+
return next.sessions[id];
|
|
827
|
+
},
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/**
|
|
832
|
+
* A recent-output record for a file this plugin just wrote; `commandId` ties
|
|
833
|
+
* it to the slash-command run that produced it (its chat row shows it).
|
|
834
|
+
*/
|
|
835
|
+
export function recentEntry(kind, file, model, createdAt = new Date(), commandId) {
|
|
836
|
+
return {
|
|
837
|
+
id: randomBytes(8).toString("hex"),
|
|
838
|
+
kind,
|
|
839
|
+
file,
|
|
840
|
+
model,
|
|
841
|
+
createdAt: createdAt.toISOString(),
|
|
842
|
+
...(validCommandId(commandId) === undefined ? {} : { commandId }),
|
|
843
|
+
};
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
// #endregion
|
|
847
|
+
|
|
848
|
+
// #region Image generation
|
|
849
|
+
|
|
850
|
+
export const IMAGE_ASPECTS = ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "9:20", "20:9"];
|
|
851
|
+
export const IMAGE_COUNT = { min: 1, max: 4 };
|
|
852
|
+
export const MAX_REFERENCE_IMAGES = 4;
|
|
853
|
+
const IMAGE_TIMEOUT_MS = 180_000;
|
|
854
|
+
const MAX_IMAGE_RESPONSE_BYTES = 96 * 1024 * 1024;
|
|
855
|
+
const MAX_IMAGE_DOWNLOAD_BYTES = 32 * 1024 * 1024;
|
|
856
|
+
// gpt-image accepts only these sizes; xAI reads `aspect_ratio` instead.
|
|
857
|
+
const GPT_IMAGE_SIZES = { "1:1": "1024x1024", "3:2": "1536x1024", "2:3": "1024x1536" };
|
|
858
|
+
const IMAGE_EXTENSIONS = { "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif" };
|
|
859
|
+
|
|
860
|
+
// Plain-language options at the start or end of a prompt, written as their
|
|
861
|
+
// own words ("10 秒 480p 竖屏 …", "… 16:9 2张"): a token counts only when a
|
|
862
|
+
// space, comma or the end follows it, so "3 秒后爆炸" stays prompt text.
|
|
863
|
+
const WORD_END = "(?=[\\s,,、;;]|$)";
|
|
864
|
+
const NATURAL_TOKENS = [
|
|
865
|
+
{ key: "seconds", kinds: ["video"], re: new RegExp(`^(\\d{1,3})\\s*(?:秒钟?|s|secs?|seconds?)${WORD_END}`, "i"), value: (m) => Number(m[1]) },
|
|
866
|
+
// "edit" / "extend": words xAI's edit and extension routes cannot use (the
|
|
867
|
+
// output keeps the source's resolution and ratio; an edit keeps its length).
|
|
868
|
+
{ key: "seconds", kinds: ["edit"], re: new RegExp(`^(\\d{1,3})\\s*(?:秒钟?|s|secs?|seconds?)${WORD_END}`, "i"), value: (m) => Number(m[1]) },
|
|
869
|
+
{ key: "resolution", kinds: ["video", "edit", "extend"], re: new RegExp(`^(480|720|1080)\\s*p${WORD_END}`, "i"), value: (m) => `${m[1]}p` },
|
|
870
|
+
{ key: "count", kinds: ["image"], re: new RegExp(`^(\\d{1,2})\\s*(?:张|幅|images?|pics?)${WORD_END}`, "i"), value: (m) => Number(m[1]) },
|
|
871
|
+
{ key: "aspect", kinds: ["image", "video", "edit", "extend"], re: new RegExp(`^(\\d{1,2})\\s*[::]\\s*(\\d{1,2})${WORD_END}`), value: (m) => `${m[1]}:${m[2]}` },
|
|
872
|
+
{ key: "aspect", kinds: ["image", "video", "edit", "extend"], re: new RegExp(`^(?:竖屏|竖版|portrait)${WORD_END}`, "i"), value: () => "9:16" },
|
|
873
|
+
{ key: "aspect", kinds: ["image", "video", "edit", "extend"], re: new RegExp(`^(?:横屏|横版|landscape)${WORD_END}`, "i"), value: () => "16:9" },
|
|
874
|
+
{ key: "aspect", kinds: ["image", "video", "edit", "extend"], re: new RegExp(`^(?:方形|正方形|square)${WORD_END}`, "i"), value: () => "1:1" },
|
|
875
|
+
{ key: "frames", kinds: ["video"], re: new RegExp(`^(?:参考图|参考图模式|references?)${WORD_END}`, "i"), value: () => "reference" },
|
|
876
|
+
{ key: "frames", kinds: ["video"], re: new RegExp(`^(?:首尾帧|首帧尾帧|first-last)${WORD_END}`, "i"), value: () => "first-last" },
|
|
877
|
+
{ key: "frames", kinds: ["video"], re: new RegExp(`^(?:尾帧|last-frame)${WORD_END}`, "i"), value: () => "last" },
|
|
878
|
+
{ key: "seconds", kinds: ["extend"], re: new RegExp(`^(\\d{1,3})\\s*(?:秒钟?|s|secs?|seconds?)${WORD_END}`, "i"), value: (m) => Number(m[1]) },
|
|
879
|
+
];
|
|
880
|
+
const SEPARATORS = /^[\s,,、;;]+/;
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Take plain-language options from the start and the end of `text`.
|
|
884
|
+
* Explicit flags already in `out` win; a recognized token is removed either way.
|
|
885
|
+
* @returns the remaining prompt; values land in `out` (validated by the caller).
|
|
886
|
+
*/
|
|
887
|
+
function takeNaturalOptions(text, kind, out) {
|
|
888
|
+
const tokens = NATURAL_TOKENS.filter((t) => t.kinds.includes(kind));
|
|
889
|
+
const take = (match, token) => {
|
|
890
|
+
if (out[token.key] === undefined) out[token.key] = token.value(match);
|
|
891
|
+
};
|
|
892
|
+
let rest = text;
|
|
893
|
+
for (let guard = 0; guard < 8; guard += 1) {
|
|
894
|
+
const hit = tokens.map((token) => [token, token.re.exec(rest)]).find(([, m]) => m !== null);
|
|
895
|
+
if (hit === undefined) break;
|
|
896
|
+
take(hit[1], hit[0]);
|
|
897
|
+
rest = rest.slice(hit[1][0].length).replace(SEPARATORS, "");
|
|
898
|
+
}
|
|
899
|
+
// The end: peel whole words off the tail while they are options.
|
|
900
|
+
for (let guard = 0; guard < 8; guard += 1) {
|
|
901
|
+
const words = rest.split(/([\s,,、;;]+)/);
|
|
902
|
+
let matched = false;
|
|
903
|
+
for (let n = Math.min(3, words.length); n >= 1 && !matched; n -= 1) {
|
|
904
|
+
const tail = words.slice(-(2 * n - 1)).join("");
|
|
905
|
+
for (const token of tokens) {
|
|
906
|
+
const m = token.re.exec(tail);
|
|
907
|
+
if (m !== null && m[0].length === tail.length) {
|
|
908
|
+
take(m, token);
|
|
909
|
+
rest = words.slice(0, words.length - (2 * n - 1)).join("").replace(/[\s,,、;;]+$/, "");
|
|
910
|
+
matched = true;
|
|
911
|
+
break;
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
if (!matched) break;
|
|
916
|
+
}
|
|
917
|
+
return rest.trim();
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* Parse `/cpa-image` input: leading options, then the prompt verbatim.
|
|
922
|
+
* Options: `-m|--model <id>`, `--aspect <ratio>`, `-n|--count <1-4>`; `--`
|
|
923
|
+
* ends options.
|
|
924
|
+
* @throws MediaError `missing-prompt` / `invalid-option` / `invalid-aspect` / `invalid-count`.
|
|
925
|
+
*/
|
|
926
|
+
export function parseImageCommand(rawInput, { literal = false } = {}) {
|
|
927
|
+
let rest = String(rawInput ?? "").trim();
|
|
928
|
+
const out = {};
|
|
929
|
+
for (;;) {
|
|
930
|
+
if (rest.startsWith("-- ") || rest === "--") {
|
|
931
|
+
rest = rest.slice(2).trim();
|
|
932
|
+
break;
|
|
933
|
+
}
|
|
934
|
+
const match = rest.match(/^(-m|--model|--aspect|-n|--count)(?:=|\s+)(\S+)(?:\s+|$)/);
|
|
935
|
+
if (match === null) {
|
|
936
|
+
if (/^--?[A-Za-z]/.test(rest)) throw new MediaError("invalid-option", rest.split(/\s/, 1)[0]);
|
|
937
|
+
break;
|
|
938
|
+
}
|
|
939
|
+
const [whole, flag, value] = match;
|
|
940
|
+
if (flag === "-m" || flag === "--model") out.model = value;
|
|
941
|
+
else if (flag === "--aspect") {
|
|
942
|
+
if (!IMAGE_ASPECTS.includes(value)) throw new MediaError("invalid-aspect", value);
|
|
943
|
+
out.aspect = value;
|
|
944
|
+
} else {
|
|
945
|
+
const count = /^\d+$/.test(value) ? Number(value) : NaN;
|
|
946
|
+
if (intIn(count, IMAGE_COUNT) === undefined) throw new MediaError("invalid-count", value);
|
|
947
|
+
out.count = count;
|
|
948
|
+
}
|
|
949
|
+
rest = rest.slice(whole.length).trim();
|
|
950
|
+
}
|
|
951
|
+
// `literal` (agent tool calls): the prompt is taken as written.
|
|
952
|
+
const natural = {};
|
|
953
|
+
if (!literal) rest = takeNaturalOptions(rest, "image", natural);
|
|
954
|
+
if (out.aspect === undefined && natural.aspect !== undefined) {
|
|
955
|
+
if (!IMAGE_ASPECTS.includes(natural.aspect)) throw new MediaError("invalid-aspect", natural.aspect);
|
|
956
|
+
out.aspect = natural.aspect;
|
|
957
|
+
}
|
|
958
|
+
if (out.count === undefined && natural.count !== undefined) {
|
|
959
|
+
if (intIn(natural.count, IMAGE_COUNT) === undefined) throw new MediaError("invalid-count", String(natural.count));
|
|
960
|
+
out.count = natural.count;
|
|
961
|
+
}
|
|
962
|
+
if (rest === "") throw new MediaError("missing-prompt");
|
|
963
|
+
return { ...out, prompt: rest };
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/** Media type from magic bytes (PNG, JPEG, WebP, GIF), or undefined. */
|
|
967
|
+
export function sniffImageType(bytes) {
|
|
968
|
+
const b = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes ?? []);
|
|
969
|
+
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
|
|
970
|
+
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
|
971
|
+
if (b.length >= 12 && String.fromCharCode(...b.subarray(0, 4)) === "RIFF" && String.fromCharCode(...b.subarray(8, 12)) === "WEBP") return "image/webp";
|
|
972
|
+
if (b.length >= 6 && String.fromCharCode(...b.subarray(0, 4)) === "GIF8") return "image/gif";
|
|
973
|
+
return undefined;
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/** A short, URL-free upstream error message (never a body dump). */
|
|
977
|
+
export function upstreamMessage(text, status) {
|
|
978
|
+
let message = "";
|
|
979
|
+
try {
|
|
980
|
+
const payload = JSON.parse(text);
|
|
981
|
+
message = payload?.error?.message ?? payload?.message ?? (typeof payload?.error === "string" ? payload.error : "");
|
|
982
|
+
} catch {
|
|
983
|
+
message = text;
|
|
984
|
+
}
|
|
985
|
+
message = String(message ?? "").replace(/https?:\/\/\S+/g, "[url]").replace(/\s+/g, " ").trim().slice(0, 300);
|
|
986
|
+
return message === "" ? `HTTP ${status}` : `HTTP ${status}: ${message}`;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
async function readBoundedBytes(response, max) {
|
|
990
|
+
const declared = Number(response.headers?.get?.("content-length"));
|
|
991
|
+
if (Number.isFinite(declared) && declared > max) throw new MediaError("image-too-large");
|
|
992
|
+
if (typeof response.body?.getReader !== "function") {
|
|
993
|
+
const buffer = new Uint8Array(await response.arrayBuffer());
|
|
994
|
+
if (buffer.byteLength > max) throw new MediaError("image-too-large");
|
|
995
|
+
return buffer;
|
|
996
|
+
}
|
|
997
|
+
const reader = response.body.getReader();
|
|
998
|
+
const chunks = [];
|
|
999
|
+
let size = 0;
|
|
1000
|
+
for (;;) {
|
|
1001
|
+
const { done, value } = await reader.read();
|
|
1002
|
+
if (done) break;
|
|
1003
|
+
size += value.byteLength;
|
|
1004
|
+
if (size > max) {
|
|
1005
|
+
await reader.cancel().catch(() => {});
|
|
1006
|
+
throw new MediaError("image-too-large");
|
|
1007
|
+
}
|
|
1008
|
+
chunks.push(value);
|
|
1009
|
+
}
|
|
1010
|
+
return new Uint8Array(Buffer.concat(chunks));
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/** Bytes behind a result `url`: a data URL, or an HTTPS download without credentials. */
|
|
1014
|
+
async function imageFromUrl(fetchImpl, url, signal) {
|
|
1015
|
+
const dataMatch = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(url);
|
|
1016
|
+
if (dataMatch !== null) {
|
|
1017
|
+
if (dataMatch[2] === undefined) throw new MediaError("invalid-image");
|
|
1018
|
+
const bytes = new Uint8Array(Buffer.from(dataMatch[3], "base64"));
|
|
1019
|
+
if (bytes.byteLength > MAX_IMAGE_DOWNLOAD_BYTES) throw new MediaError("image-too-large");
|
|
1020
|
+
return bytes;
|
|
1021
|
+
}
|
|
1022
|
+
let parsed;
|
|
1023
|
+
try {
|
|
1024
|
+
parsed = new URL(url);
|
|
1025
|
+
} catch {
|
|
1026
|
+
throw new MediaError("invalid-image");
|
|
1027
|
+
}
|
|
1028
|
+
if (parsed.protocol !== "https:" || parsed.username || parsed.password) throw new MediaError("invalid-image");
|
|
1029
|
+
const response = await fetchImpl(parsed.toString(), { method: "GET", headers: { Accept: "image/*" }, signal });
|
|
1030
|
+
if (!response.ok) throw new MediaError("download-failed", `HTTP ${response.status}`);
|
|
1031
|
+
return readBoundedBytes(response, MAX_IMAGE_DOWNLOAD_BYTES);
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/**
|
|
1035
|
+
* Generate (no references) or edit (with references) images through CPA's
|
|
1036
|
+
* OpenAI-compatible endpoints. Asks for `b64_json`; a result that carries a
|
|
1037
|
+
* URL instead is fetched (data URL or HTTPS only, size-bounded, no key).
|
|
1038
|
+
* @param references - `[{ mediaType, data: Uint8Array }]`, at most 4.
|
|
1039
|
+
* @returns `[{ bytes, mediaType, ext, revisedPrompt? }]`.
|
|
1040
|
+
* @throws MediaError `upstream-error` (detail: short message), `invalid-response`,
|
|
1041
|
+
* `no-image`, `invalid-image`, `image-too-large`, `download-failed`, `timeout`.
|
|
1042
|
+
*/
|
|
1043
|
+
export async function generateImages({
|
|
1044
|
+
fetch: fetchImpl = fetch,
|
|
1045
|
+
rootURL,
|
|
1046
|
+
apiKey,
|
|
1047
|
+
model,
|
|
1048
|
+
prompt,
|
|
1049
|
+
aspect,
|
|
1050
|
+
count = 1,
|
|
1051
|
+
references = [],
|
|
1052
|
+
signal,
|
|
1053
|
+
timeoutMs = IMAGE_TIMEOUT_MS,
|
|
1054
|
+
}) {
|
|
1055
|
+
if (references.length > MAX_REFERENCE_IMAGES) throw new MediaError("too-many-references");
|
|
1056
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
1057
|
+
const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
|
|
1058
|
+
const edit = references.length > 0;
|
|
1059
|
+
const body = { model, prompt, n: count, response_format: "b64_json" };
|
|
1060
|
+
if (aspect !== undefined) {
|
|
1061
|
+
body.aspect_ratio = aspect;
|
|
1062
|
+
if (GPT_IMAGE_SIZES[aspect] !== undefined) body.size = GPT_IMAGE_SIZES[aspect];
|
|
1063
|
+
}
|
|
1064
|
+
if (edit) {
|
|
1065
|
+
body.images = references.map((ref) => ({
|
|
1066
|
+
image_url: `data:${ref.mediaType};base64,${Buffer.from(ref.data).toString("base64")}`,
|
|
1067
|
+
}));
|
|
1068
|
+
}
|
|
1069
|
+
let response;
|
|
1070
|
+
let text;
|
|
1071
|
+
try {
|
|
1072
|
+
response = await fetchImpl(`${rootURL}${edit ? "/v1/images/edits" : "/v1/images/generations"}`, {
|
|
1073
|
+
method: "POST",
|
|
1074
|
+
headers: {
|
|
1075
|
+
"Content-Type": "application/json",
|
|
1076
|
+
Accept: "application/json",
|
|
1077
|
+
...(typeof apiKey === "string" && apiKey !== "" ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
1078
|
+
},
|
|
1079
|
+
body: JSON.stringify(body),
|
|
1080
|
+
signal: combined,
|
|
1081
|
+
});
|
|
1082
|
+
text = await readBoundedText(response, MAX_IMAGE_RESPONSE_BYTES);
|
|
1083
|
+
} catch (error) {
|
|
1084
|
+
if (signal?.aborted) throw signal.reason;
|
|
1085
|
+
if (timeout.aborted) throw new MediaError("timeout");
|
|
1086
|
+
if (error instanceof MediaError) throw error;
|
|
1087
|
+
throw new MediaError("network", error instanceof Error ? error.message.slice(0, 200) : undefined);
|
|
1088
|
+
}
|
|
1089
|
+
if (!response.ok) throw new MediaError("upstream-error", upstreamMessage(text, response.status));
|
|
1090
|
+
let payload;
|
|
1091
|
+
try {
|
|
1092
|
+
payload = JSON.parse(text);
|
|
1093
|
+
} catch {
|
|
1094
|
+
throw new MediaError("invalid-response");
|
|
1095
|
+
}
|
|
1096
|
+
const items = Array.isArray(payload?.data) ? payload.data : [];
|
|
1097
|
+
const out = [];
|
|
1098
|
+
for (const item of items) {
|
|
1099
|
+
let bytes;
|
|
1100
|
+
if (typeof item?.b64_json === "string" && item.b64_json !== "") {
|
|
1101
|
+
bytes = new Uint8Array(Buffer.from(item.b64_json, "base64"));
|
|
1102
|
+
if (bytes.byteLength > MAX_IMAGE_DOWNLOAD_BYTES) throw new MediaError("image-too-large");
|
|
1103
|
+
} else if (typeof item?.url === "string" && item.url !== "") {
|
|
1104
|
+
try {
|
|
1105
|
+
bytes = await imageFromUrl(fetchImpl, item.url, combined);
|
|
1106
|
+
} catch (error) {
|
|
1107
|
+
if (signal?.aborted) throw signal.reason;
|
|
1108
|
+
if (timeout.aborted) throw new MediaError("timeout");
|
|
1109
|
+
if (error instanceof MediaError) throw error;
|
|
1110
|
+
throw new MediaError("download-failed");
|
|
1111
|
+
}
|
|
1112
|
+
} else {
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
const mediaType = sniffImageType(bytes);
|
|
1116
|
+
if (mediaType === undefined) throw new MediaError("invalid-image");
|
|
1117
|
+
out.push({
|
|
1118
|
+
bytes,
|
|
1119
|
+
mediaType,
|
|
1120
|
+
ext: IMAGE_EXTENSIONS[mediaType],
|
|
1121
|
+
...(typeof item.revised_prompt === "string" && item.revised_prompt.trim() !== ""
|
|
1122
|
+
? { revisedPrompt: item.revised_prompt.trim().slice(0, 500) }
|
|
1123
|
+
: {}),
|
|
1124
|
+
});
|
|
1125
|
+
}
|
|
1126
|
+
if (out.length === 0) throw new MediaError("no-image");
|
|
1127
|
+
return out;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
// #endregion
|
|
1131
|
+
|
|
1132
|
+
// #region Output files
|
|
1133
|
+
|
|
1134
|
+
/** The output directory for a session: absolute settings are used as is, relative ones under `cwd`. */
|
|
1135
|
+
export function outputDirectory(cwd, outputDir) {
|
|
1136
|
+
return isAbsolute(outputDir) ? outputDir : join(cwd, outputDir);
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
function stamp(date) {
|
|
1140
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
1141
|
+
return `${date.getFullYear()}${p(date.getMonth() + 1)}${p(date.getDate())}-${p(date.getHours())}${p(date.getMinutes())}${p(date.getSeconds())}`;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
/** File stem `YYYYMMDD-HHmmss-<model>-<prompt slug>`; safe on Windows, macOS and Linux. */
|
|
1145
|
+
export function outputStem(model, prompt, date = new Date()) {
|
|
1146
|
+
const modelPart = bareModelId(model).replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "model";
|
|
1147
|
+
const slug =
|
|
1148
|
+
String(prompt ?? "")
|
|
1149
|
+
.normalize("NFKC")
|
|
1150
|
+
.replace(/[^\p{L}\p{N}]+/gu, "-")
|
|
1151
|
+
.replace(/^-+|-+$/g, "")
|
|
1152
|
+
.slice(0, 24)
|
|
1153
|
+
.replace(/-+$/g, "") || "media";
|
|
1154
|
+
return `${stamp(date)}-${modelPart}-${slug}`;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/**
|
|
1158
|
+
* Write each file next to the others without ever replacing an existing
|
|
1159
|
+
* file (`wx`); a taken name gets the next free `-N` suffix.
|
|
1160
|
+
* @param files - `[{ bytes, ext }]`.
|
|
1161
|
+
* @returns absolute paths, in order.
|
|
1162
|
+
*/
|
|
1163
|
+
export function writeOutputFiles(dir, stem, files) {
|
|
1164
|
+
mkdirSync(dir, { recursive: true });
|
|
1165
|
+
const paths = [];
|
|
1166
|
+
files.forEach((file, index) => {
|
|
1167
|
+
const base = files.length > 1 ? `${stem}-${index + 1}` : stem;
|
|
1168
|
+
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
1169
|
+
const name = attempt === 0 ? `${base}.${file.ext}` : `${base}-${attempt + 1}.${file.ext}`;
|
|
1170
|
+
const path = join(dir, name);
|
|
1171
|
+
try {
|
|
1172
|
+
writeFileSync(path, file.bytes, { flag: "wx" });
|
|
1173
|
+
paths.push(path);
|
|
1174
|
+
return;
|
|
1175
|
+
} catch (error) {
|
|
1176
|
+
if (error?.code !== "EEXIST") throw new MediaError("write-failed", error?.code ?? undefined);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
throw new MediaError("write-failed", "EEXIST");
|
|
1180
|
+
});
|
|
1181
|
+
return paths;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// #endregion
|
|
1185
|
+
|
|
1186
|
+
// #region Video generation
|
|
1187
|
+
|
|
1188
|
+
export const VIDEO_ASPECTS = ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"];
|
|
1189
|
+
export const MAX_VIDEO_BYTES = 200 * 1024 * 1024;
|
|
1190
|
+
const VIDEO_REQUEST_TIMEOUT_MS = 60_000;
|
|
1191
|
+
const VIDEO_DOWNLOAD_TIMEOUT_MS = 10 * 60_000;
|
|
1192
|
+
const MAX_VIDEO_JSON_BYTES = 1024 * 1024;
|
|
1193
|
+
/** CPA keeps the request → credential binding this long (`defaultVideoAuthBindingTTL`). */
|
|
1194
|
+
export const VIDEO_JOB_TTL_MS = 3 * 60 * 60 * 1000;
|
|
1195
|
+
const REQUEST_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/;
|
|
1196
|
+
/**
|
|
1197
|
+
* Request body limits measured on a live CPA → xAI path (Oct 2026, with
|
|
1198
|
+
* requests that fail validation, so nothing was generated):
|
|
1199
|
+
* `POST /v1/videos/generations` read 1 MiB but refused 1.2 MiB ("Failed to
|
|
1200
|
+
* read request body: length limit exceeded"); `/v1/videos/edits` read 20 MiB.
|
|
1201
|
+
*/
|
|
1202
|
+
export const VIDEO_GENERATION_BODY_MAX_BYTES = 1024 * 1024;
|
|
1203
|
+
export const VIDEO_UPLOAD_BODY_MAX_BYTES = 20 * 1024 * 1024;
|
|
1204
|
+
/** Largest local video sent inline as a data URI (base64 adds a third). */
|
|
1205
|
+
export const VIDEO_UPLOAD_MAX_BYTES = 14 * 1024 * 1024;
|
|
1206
|
+
|
|
1207
|
+
/**
|
|
1208
|
+
* Raw bytes each of `count` attached images may take so that a
|
|
1209
|
+
* `/v1/videos/generations` body with this prompt stays within
|
|
1210
|
+
* {@link VIDEO_GENERATION_BODY_MAX_BYTES} once base64-encoded.
|
|
1211
|
+
*/
|
|
1212
|
+
export function videoImageBudget(prompt, count) {
|
|
1213
|
+
if (!Number.isInteger(count) || count <= 0) return 0;
|
|
1214
|
+
const fixed = 1024 + Buffer.byteLength(JSON.stringify(String(prompt ?? "")));
|
|
1215
|
+
const room = VIDEO_GENERATION_BODY_MAX_BYTES - fixed - 64 * count;
|
|
1216
|
+
return Math.max(0, Math.floor(room / count / 4) * 3);
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
const asBuffer = (bytes) => (Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
|
1220
|
+
|
|
1221
|
+
/** Whether the bytes start like an ISO base media file (MP4 / MOV: an `ftyp` box first). */
|
|
1222
|
+
export function isMp4(bytes) {
|
|
1223
|
+
return bytes instanceof Uint8Array && bytes.byteLength >= 12 && asBuffer(bytes).toString("latin1", 4, 8) === "ftyp";
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
/**
|
|
1227
|
+
* A video's length from its `moov/mvhd` box, in seconds (two decimals), or
|
|
1228
|
+
* undefined when the file does not carry one this parser understands.
|
|
1229
|
+
*/
|
|
1230
|
+
export function mp4Duration(bytes) {
|
|
1231
|
+
if (!isMp4(bytes)) return undefined;
|
|
1232
|
+
const buf = asBuffer(bytes);
|
|
1233
|
+
const find = (start, end, type) => {
|
|
1234
|
+
let at = start;
|
|
1235
|
+
while (at + 8 <= end) {
|
|
1236
|
+
let size = buf.readUInt32BE(at);
|
|
1237
|
+
let header = 8;
|
|
1238
|
+
if (size === 1) {
|
|
1239
|
+
if (at + 16 > end) return undefined;
|
|
1240
|
+
const large = buf.readBigUInt64BE(at + 8);
|
|
1241
|
+
if (large > BigInt(end - at)) return undefined;
|
|
1242
|
+
size = Number(large);
|
|
1243
|
+
header = 16;
|
|
1244
|
+
} else if (size === 0) size = end - at;
|
|
1245
|
+
if (size < header || at + size > end) return undefined;
|
|
1246
|
+
if (buf.toString("latin1", at + 4, at + 8) === type) return [at + header, at + size];
|
|
1247
|
+
at += size;
|
|
1248
|
+
}
|
|
1249
|
+
return undefined;
|
|
1250
|
+
};
|
|
1251
|
+
const moov = find(0, buf.length, "moov");
|
|
1252
|
+
const mvhd = moov === undefined ? undefined : find(moov[0], moov[1], "mvhd");
|
|
1253
|
+
if (mvhd === undefined) return undefined;
|
|
1254
|
+
const [c, end] = mvhd;
|
|
1255
|
+
const v1 = buf[c] === 1;
|
|
1256
|
+
if (c + (v1 ? 32 : 20) > end) return undefined;
|
|
1257
|
+
const timescale = buf.readUInt32BE(c + (v1 ? 20 : 12));
|
|
1258
|
+
const duration = v1 ? Number(buf.readBigUInt64BE(c + 24)) : buf.readUInt32BE(c + 16);
|
|
1259
|
+
if (timescale === 0 || !Number.isFinite(duration) || duration <= 0) return undefined;
|
|
1260
|
+
return Math.round((duration / timescale) * 100) / 100;
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
/**
|
|
1264
|
+
* Parse `/cpa-video` input: leading options, then the prompt verbatim.
|
|
1265
|
+
* Options: `-m|--model <id>`, `--aspect <ratio>`, `-s|--seconds <1-15>`,
|
|
1266
|
+
* `-r|--resolution <480p|720p>`; `--` ends options.
|
|
1267
|
+
* @throws MediaError `missing-prompt` / `invalid-option` / `invalid-aspect` /
|
|
1268
|
+
* `invalid-video-seconds` / `invalid-video-resolution`.
|
|
1269
|
+
*/
|
|
1270
|
+
export function parseVideoCommand(rawInput, { literal = false } = {}) {
|
|
1271
|
+
let rest = String(rawInput ?? "").trim();
|
|
1272
|
+
const out = {};
|
|
1273
|
+
for (;;) {
|
|
1274
|
+
if (rest.startsWith("-- ") || rest === "--") {
|
|
1275
|
+
rest = rest.slice(2).trim();
|
|
1276
|
+
break;
|
|
1277
|
+
}
|
|
1278
|
+
const toggle = rest.match(/^(--ref|--first-last|--last)(?:\s+|$)/);
|
|
1279
|
+
if (toggle !== null) {
|
|
1280
|
+
out.frames = { "--ref": "reference", "--first-last": "first-last", "--last": "last" }[toggle[1]];
|
|
1281
|
+
rest = rest.slice(toggle[0].length).trim();
|
|
1282
|
+
continue;
|
|
1283
|
+
}
|
|
1284
|
+
const match = rest.match(/^(-m|--model|--aspect|-s|--seconds|-r|--resolution)(?:=|\s+)(\S+)(?:\s+|$)/);
|
|
1285
|
+
if (match === null) {
|
|
1286
|
+
if (/^--?[A-Za-z]/.test(rest)) throw new MediaError("invalid-option", rest.split(/\s/, 1)[0]);
|
|
1287
|
+
break;
|
|
1288
|
+
}
|
|
1289
|
+
const [whole, flag, value] = match;
|
|
1290
|
+
if (flag === "-m" || flag === "--model") out.model = value;
|
|
1291
|
+
else if (flag === "--aspect") {
|
|
1292
|
+
if (!VIDEO_ASPECTS.includes(value)) throw new MediaError("invalid-aspect", value);
|
|
1293
|
+
out.aspect = value;
|
|
1294
|
+
} else if (flag === "-s" || flag === "--seconds") {
|
|
1295
|
+
const seconds = /^\d+$/.test(value) ? Number(value) : NaN;
|
|
1296
|
+
if (intIn(seconds, VIDEO_SECONDS) === undefined) throw new MediaError("invalid-video-seconds", value);
|
|
1297
|
+
out.seconds = seconds;
|
|
1298
|
+
} else {
|
|
1299
|
+
const resolution = value.toLowerCase();
|
|
1300
|
+
if (!VIDEO_RESOLUTIONS.includes(resolution)) throw new MediaError("invalid-video-resolution", value);
|
|
1301
|
+
out.resolution = resolution;
|
|
1302
|
+
}
|
|
1303
|
+
rest = rest.slice(whole.length).trim();
|
|
1304
|
+
}
|
|
1305
|
+
const natural = {};
|
|
1306
|
+
if (!literal) rest = takeNaturalOptions(rest, "video", natural);
|
|
1307
|
+
if (out.seconds === undefined && natural.seconds !== undefined) {
|
|
1308
|
+
if (intIn(natural.seconds, VIDEO_SECONDS) === undefined) throw new MediaError("invalid-video-seconds", String(natural.seconds));
|
|
1309
|
+
out.seconds = natural.seconds;
|
|
1310
|
+
}
|
|
1311
|
+
if (out.resolution === undefined && natural.resolution !== undefined) {
|
|
1312
|
+
if (!VIDEO_RESOLUTIONS.includes(natural.resolution)) throw new MediaError("invalid-video-resolution", natural.resolution);
|
|
1313
|
+
out.resolution = natural.resolution;
|
|
1314
|
+
}
|
|
1315
|
+
if (out.aspect === undefined && natural.aspect !== undefined) {
|
|
1316
|
+
if (!VIDEO_ASPECTS.includes(natural.aspect)) throw new MediaError("invalid-aspect", natural.aspect);
|
|
1317
|
+
out.aspect = natural.aspect;
|
|
1318
|
+
}
|
|
1319
|
+
if (out.frames === undefined && natural.frames !== undefined) out.frames = natural.frames;
|
|
1320
|
+
if (rest === "") throw new MediaError("missing-prompt");
|
|
1321
|
+
return { ...out, prompt: rest };
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
export const EXTEND_SECONDS = { min: 2, max: 10 };
|
|
1325
|
+
/** xAI's limits on a source video: edits take at most 8.7 s, extensions 2–15 s. */
|
|
1326
|
+
export const EDIT_SOURCE_MAX_SECONDS = 8.7;
|
|
1327
|
+
export const EXTEND_SOURCE_SECONDS = { min: 2, max: 15 };
|
|
1328
|
+
/** xAI runs edits and extensions only on this model. */
|
|
1329
|
+
export const VIDEO_EDIT_MODEL = "grok-imagine-video";
|
|
1330
|
+
|
|
1331
|
+
/**
|
|
1332
|
+
* Parse `/cpa-video-edit` and `/cpa-video-extend` input. One source at most:
|
|
1333
|
+
* `--from <command id>` (a video this plugin made in the session),
|
|
1334
|
+
* `--video <path>` (a local MP4; quote a path with spaces), `--url <http(s) URL>`
|
|
1335
|
+
* (a public video xAI downloads itself) — or an attached MP4, or by default the
|
|
1336
|
+
* session's newest video. `-m|--model <id>` picks the account for a local or
|
|
1337
|
+
* URL source; extensions take `-s|--seconds <2-10>` (also as a plain word).
|
|
1338
|
+
* @throws MediaError `missing-prompt` / `invalid-option` / `invalid-extend-seconds` /
|
|
1339
|
+
* `invalid-video-url` / `one-video-source`.
|
|
1340
|
+
*/
|
|
1341
|
+
export function parseVideoEditCommand(rawInput, mode, { literal = false } = {}) {
|
|
1342
|
+
let rest = String(rawInput ?? "").trim();
|
|
1343
|
+
const out = {};
|
|
1344
|
+
for (;;) {
|
|
1345
|
+
if (rest.startsWith("-- ") || rest === "--") {
|
|
1346
|
+
rest = rest.slice(2).trim();
|
|
1347
|
+
break;
|
|
1348
|
+
}
|
|
1349
|
+
const match = rest.match(/^(--from|--video|--url|-m|--model|-s|--seconds)(?:=|\s+)(?:"([^"]+)"|(\S+))(?:\s+|$)/);
|
|
1350
|
+
if (match === null) {
|
|
1351
|
+
if (/^--?[A-Za-z]/.test(rest)) throw new MediaError("invalid-option", rest.split(/\s/, 1)[0]);
|
|
1352
|
+
break;
|
|
1353
|
+
}
|
|
1354
|
+
const [whole, flag, quoted, bare] = match;
|
|
1355
|
+
const value = quoted ?? bare;
|
|
1356
|
+
if (flag === "--from" || flag === "--video" || flag === "--url") {
|
|
1357
|
+
if (out.from !== undefined || out.video !== undefined || out.url !== undefined) throw new MediaError("one-video-source");
|
|
1358
|
+
}
|
|
1359
|
+
if (flag === "--from") {
|
|
1360
|
+
if (validCommandId(value) === undefined) throw new MediaError("invalid-option", `--from ${value}`);
|
|
1361
|
+
out.from = value;
|
|
1362
|
+
} else if (flag === "--video") {
|
|
1363
|
+
out.video = value;
|
|
1364
|
+
} else if (flag === "--url") {
|
|
1365
|
+
let url;
|
|
1366
|
+
try {
|
|
1367
|
+
url = new URL(value);
|
|
1368
|
+
} catch {
|
|
1369
|
+
throw new MediaError("invalid-video-url");
|
|
1370
|
+
}
|
|
1371
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username !== "" || url.password !== "") throw new MediaError("invalid-video-url");
|
|
1372
|
+
out.url = url.href;
|
|
1373
|
+
} else if (flag === "-m" || flag === "--model") {
|
|
1374
|
+
out.model = value;
|
|
1375
|
+
} else {
|
|
1376
|
+
if (mode !== "extend") throw new MediaError("invalid-option", flag);
|
|
1377
|
+
const seconds = /^\d+$/.test(value) ? Number(value) : NaN;
|
|
1378
|
+
if (intIn(seconds, EXTEND_SECONDS) === undefined) throw new MediaError("invalid-extend-seconds", value);
|
|
1379
|
+
out.seconds = seconds;
|
|
1380
|
+
}
|
|
1381
|
+
rest = rest.slice(whole.length).trim();
|
|
1382
|
+
}
|
|
1383
|
+
// A video path written straight into the prompt (with or without quotes or
|
|
1384
|
+
// DSH's `@` file mention) is the source, as if passed with --video.
|
|
1385
|
+
const named = () => out.from !== undefined || out.video !== undefined || out.url !== undefined;
|
|
1386
|
+
const takePath = (re) => {
|
|
1387
|
+
if (named()) return;
|
|
1388
|
+
const m = re.exec(rest);
|
|
1389
|
+
if (m === null) return;
|
|
1390
|
+
out.video = m[1] ?? m[2] ?? m[3];
|
|
1391
|
+
rest = (rest.slice(0, m.index) + rest.slice(m.index + m[0].length)).replace(SEPARATORS, "").replace(/[\s,,、;;]+$/, "");
|
|
1392
|
+
};
|
|
1393
|
+
const natural = {};
|
|
1394
|
+
const kind = mode === "extend" ? "extend" : "edit";
|
|
1395
|
+
if (!literal) {
|
|
1396
|
+
takePath(VIDEO_PATH_HEAD);
|
|
1397
|
+
rest = takeNaturalOptions(rest, kind, natural);
|
|
1398
|
+
takePath(VIDEO_PATH_HEAD);
|
|
1399
|
+
takePath(VIDEO_PATH_TAIL);
|
|
1400
|
+
rest = takeNaturalOptions(rest, kind, natural);
|
|
1401
|
+
}
|
|
1402
|
+
if (mode === "extend" && out.seconds === undefined && natural.seconds !== undefined) {
|
|
1403
|
+
if (intIn(natural.seconds, EXTEND_SECONDS) === undefined) throw new MediaError("invalid-extend-seconds", String(natural.seconds));
|
|
1404
|
+
out.seconds = natural.seconds;
|
|
1405
|
+
}
|
|
1406
|
+
// Words the edit / extension routes cannot use are taken out of the prompt and reported.
|
|
1407
|
+
const ignored = [
|
|
1408
|
+
...(mode === "edit" && natural.seconds !== undefined ? [`${natural.seconds}s`] : []),
|
|
1409
|
+
...(natural.resolution !== undefined ? [natural.resolution] : []),
|
|
1410
|
+
...(natural.aspect !== undefined ? [natural.aspect] : []),
|
|
1411
|
+
];
|
|
1412
|
+
if (ignored.length > 0) out.ignored = ignored;
|
|
1413
|
+
if (rest === "") throw new MediaError("missing-prompt");
|
|
1414
|
+
return { ...out, prompt: rest };
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
const VIDEO_EXT = "(?:mp4|m4v|mov)";
|
|
1418
|
+
// Quoted path; unquoted absolute path (may hold spaces, up to the first `.mp4`
|
|
1419
|
+
// followed by a space); or one unquoted word ending in a video extension.
|
|
1420
|
+
const VIDEO_PATH_HEAD = new RegExp(
|
|
1421
|
+
`^@?(?:"([^"]+\\.${VIDEO_EXT})"|((?:[A-Za-z]:[\\\\/]|\\\\\\\\|/)[^\\n]*?\\.${VIDEO_EXT})|([^\\s"]+\\.${VIDEO_EXT}))(?=[\\s,,、;;]|$)`,
|
|
1422
|
+
"i"
|
|
1423
|
+
);
|
|
1424
|
+
const VIDEO_PATH_TAIL = new RegExp(
|
|
1425
|
+
`(?:^|[\\s,,、;;])@?(?:"([^"]+\\.${VIDEO_EXT})"|((?:[A-Za-z]:[\\\\/]|\\\\\\\\)[^\\n"]*?\\.${VIDEO_EXT})|([^\\s"]+\\.${VIDEO_EXT}))$`,
|
|
1426
|
+
"i"
|
|
1427
|
+
);
|
|
1428
|
+
|
|
1429
|
+
// #region Parameter interpretation by the session's chat model
|
|
1430
|
+
//
|
|
1431
|
+
// The rule parsers read flags and a few whole words; anything else written in
|
|
1432
|
+
// plain language ("十秒钟", "竖版的", "第二张当结尾") is handed to the session's
|
|
1433
|
+
// own chat model, which returns the parameters as JSON. Explicit flags always
|
|
1434
|
+
// win, every value is validated here, and the prompt may only lose words
|
|
1435
|
+
// (never be rewritten): `interpretedPrompt()` keeps the original otherwise.
|
|
1436
|
+
|
|
1437
|
+
const INTERPRET_FIELDS = {
|
|
1438
|
+
image: ["aspect", "count"],
|
|
1439
|
+
video: ["seconds", "resolution", "aspect", "frames"],
|
|
1440
|
+
edit: [],
|
|
1441
|
+
extend: ["seconds"],
|
|
1442
|
+
};
|
|
1443
|
+
|
|
1444
|
+
/**
|
|
1445
|
+
* The system and user text asking a chat model to read parameters out of a
|
|
1446
|
+
* generation request.
|
|
1447
|
+
* @param command - `image` / `video` / `edit` / `extend`.
|
|
1448
|
+
* @param text - the prompt as the rule parser left it.
|
|
1449
|
+
* @param images - attached image count.
|
|
1450
|
+
* @param fixed - field names already set by explicit options (not to be read again).
|
|
1451
|
+
*/
|
|
1452
|
+
export function interpretationRequest({ command, text, images = 0, fixed = [] }) {
|
|
1453
|
+
const fields = INTERPRET_FIELDS[command] ?? [];
|
|
1454
|
+
const lines = {
|
|
1455
|
+
seconds: command === "extend" ? "seconds: integer 2-10, how many seconds to ADD to the video" : "seconds: integer 1-15, video length",
|
|
1456
|
+
resolution: 'resolution: "480p" or "720p" (high definition / 高清 = "720p")',
|
|
1457
|
+
aspect: `aspect: one of ${(command === "image" ? IMAGE_ASPECTS : VIDEO_ASPECTS).join(", ")} (portrait / 竖屏 = "9:16", landscape / 横屏 = "16:9", square = "1:1")`,
|
|
1458
|
+
count: "count: integer 1-4, number of images",
|
|
1459
|
+
frames:
|
|
1460
|
+
'frames: how the attached images are used — "first-frame" (one image starts the video), "reference" (images show people / objects to include, no fixed frame), "first-last" (image 1 starts, image 2 ends the video), "last-frame" (the video ends on the image)',
|
|
1461
|
+
};
|
|
1462
|
+
const wanted = fields.filter((f) => !fixed.includes(f) && (f !== "frames" || images > 0));
|
|
1463
|
+
const system = [
|
|
1464
|
+
"You read generation parameters out of a user's request to an image / video generator.",
|
|
1465
|
+
"Reply with exactly one JSON object and nothing else.",
|
|
1466
|
+
wanted.length > 0 ? `Fields (omit a field the request does not state):\n- ${wanted.map((f) => lines[f]).join("\n- ")}` : "There are no parameter fields to read.",
|
|
1467
|
+
'- prompt: the request text with ONLY the words that state these parameters removed. Copy every other character exactly as written: do not translate, rephrase, fix, shorten or add anything. If no such words exist, return the text unchanged.',
|
|
1468
|
+
].join("\n");
|
|
1469
|
+
const user = `${images > 0 ? `Attached images: ${images}.\n` : ""}Request:\n${text}`;
|
|
1470
|
+
return { system, user, fields: wanted };
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
/** Whether `short` can be made from `long` by deleting characters (order kept). */
|
|
1474
|
+
function isSubsequence(short, long) {
|
|
1475
|
+
let at = 0;
|
|
1476
|
+
for (const ch of long) if (at < short.length && ch === short[at]) at += 1;
|
|
1477
|
+
return at === short.length;
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
/**
|
|
1481
|
+
* The model's prompt when it only removed characters and kept a fair part of
|
|
1482
|
+
* the text (at least 30 %, unless it removed 12 characters or fewer),
|
|
1483
|
+
* otherwise the original.
|
|
1484
|
+
*/
|
|
1485
|
+
export function interpretedPrompt(candidate, original) {
|
|
1486
|
+
if (typeof candidate !== "string") return original;
|
|
1487
|
+
const prompt = candidate.replace(/\s+/g, " ").trim();
|
|
1488
|
+
const base = original.replace(/\s+/g, " ").trim();
|
|
1489
|
+
if (prompt === "" || !isSubsequence(prompt, base)) return original;
|
|
1490
|
+
if (prompt.length < base.length * 0.3 && base.length - prompt.length > 12) return original;
|
|
1491
|
+
return prompt;
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
/**
|
|
1495
|
+
* Validate a chat model's JSON answer against the command's fields.
|
|
1496
|
+
* @returns `{ values, prompt }`: only valid, wanted fields; the prompt checked by
|
|
1497
|
+
* {@link interpretedPrompt}. Undefined when the answer holds no JSON object.
|
|
1498
|
+
*/
|
|
1499
|
+
export function parseInterpretation(answer, { command, text, fields, images = 0 }) {
|
|
1500
|
+
const raw = String(answer ?? "");
|
|
1501
|
+
const start = raw.indexOf("{");
|
|
1502
|
+
const end = raw.lastIndexOf("}");
|
|
1503
|
+
if (start < 0 || end <= start) return undefined;
|
|
1504
|
+
let data;
|
|
1505
|
+
try {
|
|
1506
|
+
data = JSON.parse(raw.slice(start, end + 1));
|
|
1507
|
+
} catch {
|
|
1508
|
+
return undefined;
|
|
1509
|
+
}
|
|
1510
|
+
if (data === null || typeof data !== "object" || Array.isArray(data)) return undefined;
|
|
1511
|
+
const values = {};
|
|
1512
|
+
const allowed = new Set(fields ?? INTERPRET_FIELDS[command] ?? []);
|
|
1513
|
+
const int = (v) => (typeof v === "number" ? v : typeof v === "string" && /^\s*\d+\s*$/.test(v) ? Number(v) : NaN);
|
|
1514
|
+
if (allowed.has("seconds")) {
|
|
1515
|
+
const range = command === "extend" ? EXTEND_SECONDS : VIDEO_SECONDS;
|
|
1516
|
+
const seconds = intIn(int(data.seconds), range);
|
|
1517
|
+
if (seconds !== undefined) values.seconds = seconds;
|
|
1518
|
+
}
|
|
1519
|
+
if (allowed.has("resolution") && typeof data.resolution === "string") {
|
|
1520
|
+
const resolution = data.resolution.trim().toLowerCase();
|
|
1521
|
+
if (VIDEO_RESOLUTIONS.includes(resolution)) values.resolution = resolution;
|
|
1522
|
+
}
|
|
1523
|
+
if (allowed.has("aspect") && typeof data.aspect === "string") {
|
|
1524
|
+
const aspect = data.aspect.replace(/[:\s]/g, (c) => (c === ":" ? ":" : ""));
|
|
1525
|
+
if ((command === "image" ? IMAGE_ASPECTS : VIDEO_ASPECTS).includes(aspect)) values.aspect = aspect;
|
|
1526
|
+
}
|
|
1527
|
+
if (allowed.has("count")) {
|
|
1528
|
+
const count = intIn(int(data.count), IMAGE_COUNT);
|
|
1529
|
+
if (count !== undefined) values.count = count;
|
|
1530
|
+
}
|
|
1531
|
+
if (allowed.has("frames") && images > 0) {
|
|
1532
|
+
// The parser's words: undefined = default (first frame for one image, references for more).
|
|
1533
|
+
const frames = { "first-frame": images === 1 ? "first" : undefined, reference: "reference", "first-last": "first-last", "last-frame": "last" }[data.frames];
|
|
1534
|
+
if (frames !== undefined) values.frames = frames;
|
|
1535
|
+
}
|
|
1536
|
+
return { values, prompt: interpretedPrompt(data.prompt, text) };
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
/** Short labels of interpreted values for the chat row ("10s", "720p", "9:16", "2 images", "reference"). */
|
|
1540
|
+
export function interpretationLabels(values) {
|
|
1541
|
+
return [
|
|
1542
|
+
...(values.seconds !== undefined ? [`${values.seconds}s`] : []),
|
|
1543
|
+
...(values.resolution !== undefined ? [values.resolution] : []),
|
|
1544
|
+
...(values.aspect !== undefined ? [values.aspect] : []),
|
|
1545
|
+
...(values.count !== undefined ? [`×${values.count}`] : []),
|
|
1546
|
+
...(values.frames !== undefined ? [values.frames] : []),
|
|
1547
|
+
];
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
// #endregion
|
|
1551
|
+
|
|
1552
|
+
/** The model id CPA's native video route accepts: `sora-2*` is an alias of Grok's video model there. */
|
|
1553
|
+
export function videoRequestModel(modelId) {
|
|
1554
|
+
return mediaAliasTarget(modelId) ?? modelId;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
function authHeaders(apiKey, accept = "application/json") {
|
|
1558
|
+
return {
|
|
1559
|
+
Accept: accept,
|
|
1560
|
+
...(typeof apiKey === "string" && apiKey !== "" ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
async function videoJson(fetchImpl, url, init, signal, timeoutMs) {
|
|
1565
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
1566
|
+
const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
|
|
1567
|
+
let response;
|
|
1568
|
+
let text;
|
|
1569
|
+
try {
|
|
1570
|
+
response = await fetchImpl(url, { ...init, signal: combined });
|
|
1571
|
+
text = await readBoundedText(response, MAX_VIDEO_JSON_BYTES);
|
|
1572
|
+
} catch (error) {
|
|
1573
|
+
if (signal?.aborted) throw signal.reason;
|
|
1574
|
+
if (timeout.aborted) throw new MediaError("timeout");
|
|
1575
|
+
if (error instanceof MediaError) throw error;
|
|
1576
|
+
throw new MediaError("network", error instanceof Error ? error.message.slice(0, 200) : undefined);
|
|
1577
|
+
}
|
|
1578
|
+
let payload;
|
|
1579
|
+
try {
|
|
1580
|
+
payload = JSON.parse(text);
|
|
1581
|
+
} catch {
|
|
1582
|
+
payload = undefined;
|
|
1583
|
+
}
|
|
1584
|
+
return { response, text, payload };
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
/**
|
|
1588
|
+
* Submit one video through CPA's native xAI route `POST /v1/videos/generations`
|
|
1589
|
+
* (fields `model`, `prompt`, `duration`, `aspect_ratio`, `resolution`,
|
|
1590
|
+
* `image.url`; CPA passes them to xAI). An omitted ratio lets xAI choose
|
|
1591
|
+
* (16:9, or the first frame's ratio). A submitted video cannot be cancelled.
|
|
1592
|
+
* @param image - optional first frame `{ mediaType, data }`.
|
|
1593
|
+
* @returns `{ requestId }`.
|
|
1594
|
+
* @throws MediaError `upstream-error` (CPA refused; nothing was submitted), `invalid-response`, `network`, `timeout`.
|
|
1595
|
+
*/
|
|
1596
|
+
const dataUrl = (image) => `data:${image.mediaType};base64,${Buffer.from(image.data).toString("base64")}`;
|
|
1597
|
+
|
|
1598
|
+
export async function submitVideo({ fetch: fetchImpl = fetch, rootURL, apiKey, model, prompt, seconds, aspect, resolution, image, lastFrame, references, signal }) {
|
|
1599
|
+
const body = { model: videoRequestModel(model), prompt, duration: seconds };
|
|
1600
|
+
if (aspect !== undefined) body.aspect_ratio = aspect;
|
|
1601
|
+
if (resolution !== undefined) body.resolution = resolution;
|
|
1602
|
+
if (image !== undefined) body.image = { url: dataUrl(image) };
|
|
1603
|
+
if (lastFrame !== undefined) body.last_frame = { url: dataUrl(lastFrame) };
|
|
1604
|
+
if (Array.isArray(references) && references.length > 0) body.reference_images = references.map((ref) => ({ url: dataUrl(ref) }));
|
|
1605
|
+
return postVideoJob(fetchImpl, `${rootURL}/v1/videos/generations`, apiKey, body, signal, VIDEO_GENERATION_BODY_MAX_BYTES);
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
/**
|
|
1609
|
+
* Edit (`POST /v1/videos/edits`) or extend (`POST /v1/videos/extensions`) any
|
|
1610
|
+
* MP4: by URL (`sourceUrl` — xAI's own short-lived result link or a public
|
|
1611
|
+
* http(s) address; xAI downloads it) or inline (`sourceData`, the file's bytes
|
|
1612
|
+
* as a base64 data URI). Extensions take `duration` (2–10 s, the added part
|
|
1613
|
+
* only); edits keep the source's length.
|
|
1614
|
+
* @returns `{ requestId }`.
|
|
1615
|
+
* @throws MediaError `request-too-large`, `route-unsupported`, `upstream-error`, …
|
|
1616
|
+
*/
|
|
1617
|
+
export async function submitVideoEdit({ fetch: fetchImpl = fetch, rootURL, apiKey, mode, sourceUrl, sourceData, prompt, seconds, signal }) {
|
|
1618
|
+
const url = sourceData !== undefined ? dataUrl({ mediaType: "video/mp4", data: sourceData }) : sourceUrl;
|
|
1619
|
+
const body = { model: VIDEO_EDIT_MODEL, prompt, video: { url } };
|
|
1620
|
+
if (mode === "extend" && seconds !== undefined) body.duration = seconds;
|
|
1621
|
+
return postVideoJob(
|
|
1622
|
+
fetchImpl,
|
|
1623
|
+
`${rootURL}/v1/videos/${mode === "extend" ? "extensions" : "edits"}`,
|
|
1624
|
+
apiKey,
|
|
1625
|
+
body,
|
|
1626
|
+
signal,
|
|
1627
|
+
VIDEO_UPLOAD_BODY_MAX_BYTES,
|
|
1628
|
+
sourceData !== undefined ? VIDEO_UPLOAD_TIMEOUT_MS : VIDEO_REQUEST_TIMEOUT_MS
|
|
1629
|
+
);
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
const VIDEO_UPLOAD_TIMEOUT_MS = 5 * 60_000;
|
|
1633
|
+
|
|
1634
|
+
async function postVideoJob(fetchImpl, url, apiKey, body, signal, maxBodyBytes, timeoutMs = VIDEO_REQUEST_TIMEOUT_MS) {
|
|
1635
|
+
const json = JSON.stringify(body);
|
|
1636
|
+
const bytes = Buffer.byteLength(json);
|
|
1637
|
+
// Refused here rather than upstream: CPA cools the credential down after some upstream errors.
|
|
1638
|
+
if (maxBodyBytes !== undefined && bytes > maxBodyBytes) throw new MediaError("request-too-large", String(bytes));
|
|
1639
|
+
const { response, text, payload } = await videoJson(
|
|
1640
|
+
fetchImpl,
|
|
1641
|
+
url,
|
|
1642
|
+
{ method: "POST", headers: { ...authHeaders(apiKey), "Content-Type": "application/json" }, body: json },
|
|
1643
|
+
signal,
|
|
1644
|
+
timeoutMs
|
|
1645
|
+
);
|
|
1646
|
+
// The upstream has no such route (seen for /v1/videos/extensions behind some CPA servers).
|
|
1647
|
+
if (response.status === 405) throw new MediaError("route-unsupported", url.slice(url.lastIndexOf("/") + 1));
|
|
1648
|
+
if ((response.status === 400 || response.status === 413) && /length limit exceeded|request body too large/i.test(text)) {
|
|
1649
|
+
throw new MediaError("request-too-large", String(bytes));
|
|
1650
|
+
}
|
|
1651
|
+
if (!response.ok) throw new MediaError("upstream-error", upstreamMessage(text, response.status));
|
|
1652
|
+
const requestId = String(payload?.request_id ?? payload?.id ?? "").trim();
|
|
1653
|
+
if (!REQUEST_ID_PATTERN.test(requestId)) throw new MediaError("invalid-response");
|
|
1654
|
+
return { requestId };
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
/** xAI / OpenAI video status words → `queued` / `running` / `done` / `failed`, or undefined. */
|
|
1658
|
+
export function videoState(status) {
|
|
1659
|
+
switch (String(status ?? "").trim().toLowerCase()) {
|
|
1660
|
+
case "queued":
|
|
1661
|
+
case "pending":
|
|
1662
|
+
return "queued";
|
|
1663
|
+
case "in_progress":
|
|
1664
|
+
case "processing":
|
|
1665
|
+
case "running":
|
|
1666
|
+
return "running";
|
|
1667
|
+
case "completed":
|
|
1668
|
+
case "done":
|
|
1669
|
+
case "succeeded":
|
|
1670
|
+
case "success":
|
|
1671
|
+
return "done";
|
|
1672
|
+
case "failed":
|
|
1673
|
+
case "error":
|
|
1674
|
+
case "expired":
|
|
1675
|
+
case "cancelled":
|
|
1676
|
+
case "canceled":
|
|
1677
|
+
return "failed";
|
|
1678
|
+
default:
|
|
1679
|
+
return undefined;
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
function videoFailure(payload) {
|
|
1684
|
+
const error = payload?.error;
|
|
1685
|
+
const message = typeof error === "string" ? error : typeof error?.message === "string" ? error.message : typeof payload?.code === "string" ? payload.code : "";
|
|
1686
|
+
return String(message).replace(/https?:\/\/\S+/g, "[url]").replace(/\s+/g, " ").trim().slice(0, 300);
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
/**
|
|
1690
|
+
* Ask CPA once about a submitted video (`GET /v1/videos/{request_id}`, the
|
|
1691
|
+
* native xAI shape; CPA routes it to the credential that accepted it while
|
|
1692
|
+
* its 3-hour binding lasts). A transient HTTP failure is reported as such so
|
|
1693
|
+
* the caller can retry. A finished video's own short-lived HTTPS URL comes
|
|
1694
|
+
* back as `sourceUrl` for edits / extensions only: the host keeps it in
|
|
1695
|
+
* memory and never stores, logs or returns it.
|
|
1696
|
+
* @returns `{ state, progress?, error?, seconds?, sourceUrl? }`; `state` is `queued` / `running` / `done` / `failed`.
|
|
1697
|
+
* @throws MediaError `upstream-error` (with `status`), `invalid-response`, `network`, `timeout`.
|
|
1698
|
+
*/
|
|
1699
|
+
export async function retrieveVideo({ fetch: fetchImpl = fetch, rootURL, apiKey, requestId, signal }) {
|
|
1700
|
+
const { response, text, payload } = await videoJson(
|
|
1701
|
+
fetchImpl,
|
|
1702
|
+
`${rootURL}/v1/videos/${encodeURIComponent(requestId)}`,
|
|
1703
|
+
{ method: "GET", headers: authHeaders(apiKey) },
|
|
1704
|
+
signal,
|
|
1705
|
+
VIDEO_REQUEST_TIMEOUT_MS
|
|
1706
|
+
);
|
|
1707
|
+
if (!response.ok) {
|
|
1708
|
+
const error = new MediaError("upstream-error", upstreamMessage(text, response.status));
|
|
1709
|
+
error.status = response.status;
|
|
1710
|
+
throw error;
|
|
1711
|
+
}
|
|
1712
|
+
if (payload === undefined || payload === null || typeof payload !== "object") throw new MediaError("invalid-response");
|
|
1713
|
+
const failure = videoFailure(payload);
|
|
1714
|
+
let state = videoState(payload.status);
|
|
1715
|
+
if (state === undefined && typeof payload.video?.url === "string") state = "done";
|
|
1716
|
+
if (failure !== "" && state !== "done") state = "failed";
|
|
1717
|
+
if (state === undefined) state = "queued";
|
|
1718
|
+
const progress = Number(payload.progress);
|
|
1719
|
+
const seconds = Number(payload.video?.duration ?? payload.seconds);
|
|
1720
|
+
let sourceUrl;
|
|
1721
|
+
if (state === "done" && typeof payload.video?.url === "string") {
|
|
1722
|
+
try {
|
|
1723
|
+
const parsed = new URL(payload.video.url);
|
|
1724
|
+
if (parsed.protocol === "https:" && !parsed.username && !parsed.password) sourceUrl = parsed.toString();
|
|
1725
|
+
} catch {}
|
|
1726
|
+
}
|
|
1727
|
+
return {
|
|
1728
|
+
state,
|
|
1729
|
+
...(sourceUrl === undefined ? {} : { sourceUrl }),
|
|
1730
|
+
...(Number.isFinite(progress) && progress >= 0 && progress <= 100 ? { progress: Math.round(progress) } : {}),
|
|
1731
|
+
...(state === "failed" ? { error: failure || "the video could not be generated" } : {}),
|
|
1732
|
+
...(Number.isFinite(seconds) && seconds > 0 ? { seconds: Math.round(seconds * 10) / 10 } : {}),
|
|
1733
|
+
};
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
/** Container from magic bytes: MP4 / MOV (`ftyp`) or WebM / Matroska (EBML), or undefined. */
|
|
1737
|
+
export function sniffVideoType(bytes) {
|
|
1738
|
+
const b = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes ?? []);
|
|
1739
|
+
if (b.length >= 12 && String.fromCharCode(...b.subarray(4, 8)) === "ftyp") {
|
|
1740
|
+
return String.fromCharCode(...b.subarray(8, 10)) === "qt" ? { mediaType: "video/quicktime", ext: "mov" } : { mediaType: "video/mp4", ext: "mp4" };
|
|
1741
|
+
}
|
|
1742
|
+
if (b.length >= 4 && b[0] === 0x1a && b[1] === 0x45 && b[2] === 0xdf && b[3] === 0xa3) return { mediaType: "video/webm", ext: "webm" };
|
|
1743
|
+
return undefined;
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
/**
|
|
1747
|
+
* Download a finished video through CPA (`GET /openai/v1/videos/{id}/content`,
|
|
1748
|
+
* which fetches xAI's file with the credential's own proxy; the plugin never
|
|
1749
|
+
* contacts the upstream host) and stream it into `dir` without replacing a
|
|
1750
|
+
* file. At most `maxBytes`; a partial file is removed.
|
|
1751
|
+
* @returns the absolute path.
|
|
1752
|
+
* @throws MediaError `upstream-error`, `video-too-large`, `invalid-video`, `write-failed`, `network`, `timeout`.
|
|
1753
|
+
*/
|
|
1754
|
+
export async function downloadVideo({ fetch: fetchImpl = fetch, rootURL, apiKey, requestId, dir, stem, signal, maxBytes = MAX_VIDEO_BYTES, timeoutMs = VIDEO_DOWNLOAD_TIMEOUT_MS }) {
|
|
1755
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
1756
|
+
const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
|
|
1757
|
+
let response;
|
|
1758
|
+
try {
|
|
1759
|
+
response = await fetchImpl(`${rootURL}/openai/v1/videos/${encodeURIComponent(requestId)}/content`, {
|
|
1760
|
+
method: "GET",
|
|
1761
|
+
headers: authHeaders(apiKey, "video/*, application/octet-stream"),
|
|
1762
|
+
signal: combined,
|
|
1763
|
+
});
|
|
1764
|
+
} catch (error) {
|
|
1765
|
+
if (signal?.aborted) throw signal.reason;
|
|
1766
|
+
if (timeout.aborted) throw new MediaError("timeout");
|
|
1767
|
+
throw new MediaError("network", error instanceof Error ? error.message.slice(0, 200) : undefined);
|
|
1768
|
+
}
|
|
1769
|
+
if (!response.ok) {
|
|
1770
|
+
let text = "";
|
|
1771
|
+
try {
|
|
1772
|
+
text = await readBoundedText(response, 64 * 1024);
|
|
1773
|
+
} catch {}
|
|
1774
|
+
throw new MediaError("upstream-error", upstreamMessage(text, response.status));
|
|
1775
|
+
}
|
|
1776
|
+
const declared = Number(response.headers?.get?.("content-length"));
|
|
1777
|
+
if (Number.isFinite(declared) && declared > maxBytes) throw new MediaError("video-too-large");
|
|
1778
|
+
mkdirSync(dir, { recursive: true });
|
|
1779
|
+
const reader = typeof response.body?.getReader === "function" ? response.body.getReader() : undefined;
|
|
1780
|
+
let first;
|
|
1781
|
+
let pending = [];
|
|
1782
|
+
try {
|
|
1783
|
+
if (reader === undefined) {
|
|
1784
|
+
first = new Uint8Array(await response.arrayBuffer());
|
|
1785
|
+
} else {
|
|
1786
|
+
// Read until the container header can be recognized.
|
|
1787
|
+
let size = 0;
|
|
1788
|
+
while (size < 16) {
|
|
1789
|
+
const { done, value } = await reader.read();
|
|
1790
|
+
if (done) break;
|
|
1791
|
+
pending.push(value);
|
|
1792
|
+
size += value.byteLength;
|
|
1793
|
+
}
|
|
1794
|
+
first = new Uint8Array(Buffer.concat(pending));
|
|
1795
|
+
pending = [];
|
|
1796
|
+
}
|
|
1797
|
+
} catch (error) {
|
|
1798
|
+
if (signal?.aborted) throw signal.reason;
|
|
1799
|
+
if (timeout.aborted) throw new MediaError("timeout");
|
|
1800
|
+
throw new MediaError("network", error instanceof Error ? error.message.slice(0, 200) : undefined);
|
|
1801
|
+
}
|
|
1802
|
+
const type = sniffVideoType(first);
|
|
1803
|
+
if (type === undefined) {
|
|
1804
|
+
await reader?.cancel().catch(() => {});
|
|
1805
|
+
throw new MediaError("invalid-video");
|
|
1806
|
+
}
|
|
1807
|
+
let fd;
|
|
1808
|
+
let path;
|
|
1809
|
+
for (let attempt = 0; attempt < 100 && fd === undefined; attempt += 1) {
|
|
1810
|
+
path = join(dir, attempt === 0 ? `${stem}.${type.ext}` : `${stem}-${attempt + 1}.${type.ext}`);
|
|
1811
|
+
try {
|
|
1812
|
+
fd = openSync(path, "wx");
|
|
1813
|
+
} catch (error) {
|
|
1814
|
+
if (error?.code !== "EEXIST") {
|
|
1815
|
+
await reader?.cancel().catch(() => {});
|
|
1816
|
+
throw new MediaError("write-failed", error?.code ?? undefined);
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
if (fd === undefined) {
|
|
1821
|
+
await reader?.cancel().catch(() => {});
|
|
1822
|
+
throw new MediaError("write-failed", "EEXIST");
|
|
1823
|
+
}
|
|
1824
|
+
let size = 0;
|
|
1825
|
+
const write = (chunk) => {
|
|
1826
|
+
size += chunk.byteLength;
|
|
1827
|
+
if (size > maxBytes) throw new MediaError("video-too-large");
|
|
1828
|
+
writeSync(fd, chunk);
|
|
1829
|
+
};
|
|
1830
|
+
try {
|
|
1831
|
+
write(first);
|
|
1832
|
+
if (reader !== undefined) {
|
|
1833
|
+
for (;;) {
|
|
1834
|
+
const { done, value } = await reader.read();
|
|
1835
|
+
if (done) break;
|
|
1836
|
+
write(value);
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
closeSync(fd);
|
|
1840
|
+
fd = undefined;
|
|
1841
|
+
return path;
|
|
1842
|
+
} catch (error) {
|
|
1843
|
+
await reader?.cancel().catch(() => {});
|
|
1844
|
+
if (fd !== undefined) closeSync(fd);
|
|
1845
|
+
try {
|
|
1846
|
+
unlinkSync(path);
|
|
1847
|
+
} catch {}
|
|
1848
|
+
if (signal?.aborted) throw signal.reason;
|
|
1849
|
+
if (timeout.aborted) throw new MediaError("timeout");
|
|
1850
|
+
if (error instanceof MediaError) throw error;
|
|
1851
|
+
if (typeof error?.code === "string") throw new MediaError("write-failed", error.code);
|
|
1852
|
+
throw new MediaError("network", error instanceof Error ? error.message.slice(0, 200) : undefined);
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
// #endregion
|
|
1857
|
+
|
|
1858
|
+
// #region Video job store (media-jobs.json)
|
|
1859
|
+
|
|
1860
|
+
export const MEDIA_JOBS_VERSION = 1;
|
|
1861
|
+
const JOB_STATES = ["submitted", "done", "failed", "expired"];
|
|
1862
|
+
const MAX_JOBS = 200;
|
|
1863
|
+
const FINISHED_JOB_KEEP_MS = 7 * 24 * 60 * 60 * 1000;
|
|
1864
|
+
|
|
1865
|
+
/**
|
|
1866
|
+
* One submitted video, enough to resume polling and saving after a reload or
|
|
1867
|
+
* restart: CPA's request id, the account and model, where to save, and the
|
|
1868
|
+
* chat row it belongs to. Never keys or upstream URLs.
|
|
1869
|
+
*/
|
|
1870
|
+
export function normalizeJob(value) {
|
|
1871
|
+
const job = plain(value);
|
|
1872
|
+
if (job === undefined || typeof job.id !== "string" || !REQUEST_ID_PATTERN.test(job.id)) return undefined;
|
|
1873
|
+
if (!JOB_STATES.includes(job.state) || typeof job.accountId !== "string" || !ACCOUNT_ID_PATTERN.test(job.accountId)) return undefined;
|
|
1874
|
+
if (typeof job.modelId !== "string" || job.modelId === "" || typeof job.cwd !== "string" || !isAbsolute(job.cwd)) return undefined;
|
|
1875
|
+
if (typeof job.submittedAt !== "string" || !Number.isFinite(Date.parse(job.submittedAt))) return undefined;
|
|
1876
|
+
return {
|
|
1877
|
+
id: job.id,
|
|
1878
|
+
state: job.state,
|
|
1879
|
+
accountId: job.accountId,
|
|
1880
|
+
modelId: job.modelId.slice(0, 300),
|
|
1881
|
+
ref: clip(job.ref, 400),
|
|
1882
|
+
...(validSessionId(job.sessionId) === undefined ? {} : { sessionId: job.sessionId }),
|
|
1883
|
+
...(validCommandId(job.commandId) === undefined ? {} : { commandId: job.commandId }),
|
|
1884
|
+
cwd: job.cwd,
|
|
1885
|
+
outputDir: typeof job.outputDir === "string" && job.outputDir !== "" ? job.outputDir.slice(0, MAX_OUTPUT_DIR) : "generated",
|
|
1886
|
+
prompt: clip(job.prompt, 4000),
|
|
1887
|
+
seconds: intIn(job.seconds, VIDEO_SECONDS) ?? 6,
|
|
1888
|
+
submittedAt: job.submittedAt.slice(0, 40),
|
|
1889
|
+
...(typeof job.finishedAt === "string" ? { finishedAt: job.finishedAt.slice(0, 40) } : {}),
|
|
1890
|
+
};
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
export function normalizeMediaJobs(raw, now = Date.now()) {
|
|
1894
|
+
const jobs = [];
|
|
1895
|
+
const seen = new Set();
|
|
1896
|
+
for (const item of Array.isArray(plain(raw)?.jobs) ? raw.jobs : []) {
|
|
1897
|
+
const job = normalizeJob(item);
|
|
1898
|
+
if (job === undefined || seen.has(job.id)) continue;
|
|
1899
|
+
if (job.state !== "submitted" && now - Date.parse(job.finishedAt ?? job.submittedAt) > FINISHED_JOB_KEEP_MS) continue;
|
|
1900
|
+
seen.add(job.id);
|
|
1901
|
+
jobs.push(job);
|
|
1902
|
+
if (jobs.length >= MAX_JOBS) break;
|
|
1903
|
+
}
|
|
1904
|
+
return { version: MEDIA_JOBS_VERSION, jobs };
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
/**
|
|
1908
|
+
* Video jobs in their own file, re-read on every access; atomic writes; a
|
|
1909
|
+
* damaged file loads as empty and is kept as `*.corrupt-*.json` by the next write.
|
|
1910
|
+
*/
|
|
1911
|
+
export function createMediaJobStore(filePath, { now = () => Date.now() } = {}) {
|
|
1912
|
+
const read = () => {
|
|
1913
|
+
if (!existsSync(filePath)) return { data: normalizeMediaJobs({}, now()) };
|
|
1914
|
+
try {
|
|
1915
|
+
const raw = JSON.parse(readFileSync(filePath, "utf8"));
|
|
1916
|
+
if (plain(raw) === undefined) return { data: normalizeMediaJobs({}, now()), damaged: true };
|
|
1917
|
+
return { data: normalizeMediaJobs(raw, now()) };
|
|
1918
|
+
} catch {
|
|
1919
|
+
return { data: normalizeMediaJobs({}, now()), damaged: true };
|
|
1920
|
+
}
|
|
1921
|
+
};
|
|
1922
|
+
const write = (data, damaged) => {
|
|
1923
|
+
const next = normalizeMediaJobs(data, now());
|
|
1924
|
+
mkdirSync(dirname(filePath), { recursive: true });
|
|
1925
|
+
if (damaged && existsSync(filePath)) renameSync(filePath, filePath.replace(/\.json$/, `.corrupt-${Date.now().toString(36)}.json`));
|
|
1926
|
+
const tmp = `${filePath}.${Date.now().toString(36)}.tmp`;
|
|
1927
|
+
writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`, "utf8");
|
|
1928
|
+
renameSync(tmp, filePath);
|
|
1929
|
+
return next;
|
|
1930
|
+
};
|
|
1931
|
+
return {
|
|
1932
|
+
list() {
|
|
1933
|
+
return read().data.jobs;
|
|
1934
|
+
},
|
|
1935
|
+
get(id) {
|
|
1936
|
+
return read().data.jobs.find((job) => job.id === id);
|
|
1937
|
+
},
|
|
1938
|
+
/** Insert or merge one job (newest first). */
|
|
1939
|
+
put(job) {
|
|
1940
|
+
const { data, damaged } = read();
|
|
1941
|
+
const index = data.jobs.findIndex((j) => j.id === job.id);
|
|
1942
|
+
if (index >= 0) data.jobs[index] = { ...data.jobs[index], ...job };
|
|
1943
|
+
else data.jobs.unshift(job);
|
|
1944
|
+
return write(data, damaged).jobs.find((j) => j.id === job.id);
|
|
1945
|
+
},
|
|
1946
|
+
};
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
// #endregion
|
|
1950
|
+
// #region Reference images (size fitting without dependencies)
|
|
1951
|
+
//
|
|
1952
|
+
// Upstreams cap the request body (xAI answers "Failed to read request body:
|
|
1953
|
+
// length limit exceeded" near 2 MiB), and references travel base64-encoded
|
|
1954
|
+
// inside the JSON. A reference over the byte budget is shrunk here: PNG is
|
|
1955
|
+
// decoded with node:zlib, flattened onto white, scaled to fit, and encoded as
|
|
1956
|
+
// baseline JPEG. Other formats over the budget are left to the caller (DSH's
|
|
1957
|
+
// attachment store can derive a smaller version).
|
|
1958
|
+
|
|
1959
|
+
export const REFERENCE_MAX_EDGE = 1280;
|
|
1960
|
+
|
|
1961
|
+
/** Decode a non-interlaced PNG (any color type, 1–16-bit) to RGBA8, or undefined when unsupported. */
|
|
1962
|
+
export function decodePng(bytes) {
|
|
1963
|
+
const b = Buffer.from(bytes);
|
|
1964
|
+
if (b.length < 33 || sniffImageType(b) !== "image/png") return undefined;
|
|
1965
|
+
let offset = 8;
|
|
1966
|
+
let width = 0;
|
|
1967
|
+
let height = 0;
|
|
1968
|
+
let depth = 0;
|
|
1969
|
+
let type = 0;
|
|
1970
|
+
let interlace = 0;
|
|
1971
|
+
let palette;
|
|
1972
|
+
let trns;
|
|
1973
|
+
const idat = [];
|
|
1974
|
+
while (offset + 8 <= b.length) {
|
|
1975
|
+
const length = b.readUInt32BE(offset);
|
|
1976
|
+
const name = b.toString("latin1", offset + 4, offset + 8);
|
|
1977
|
+
const data = b.subarray(offset + 8, offset + 8 + length);
|
|
1978
|
+
offset += 12 + length;
|
|
1979
|
+
if (name === "IHDR") {
|
|
1980
|
+
width = data.readUInt32BE(0);
|
|
1981
|
+
height = data.readUInt32BE(4);
|
|
1982
|
+
depth = data[8];
|
|
1983
|
+
type = data[9];
|
|
1984
|
+
interlace = data[12];
|
|
1985
|
+
} else if (name === "PLTE") palette = data;
|
|
1986
|
+
else if (name === "tRNS") trns = data;
|
|
1987
|
+
else if (name === "IDAT") idat.push(data);
|
|
1988
|
+
else if (name === "IEND") break;
|
|
1989
|
+
}
|
|
1990
|
+
if (width === 0 || height === 0 || interlace !== 0 || width * height > 40_000_000) return undefined;
|
|
1991
|
+
const channels = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 }[type];
|
|
1992
|
+
if (channels === undefined || ![1, 2, 4, 8, 16].includes(depth) || (type === 3 && palette === undefined)) return undefined;
|
|
1993
|
+
let raw;
|
|
1994
|
+
try {
|
|
1995
|
+
raw = inflateSync(Buffer.concat(idat));
|
|
1996
|
+
} catch {
|
|
1997
|
+
return undefined;
|
|
1998
|
+
}
|
|
1999
|
+
const bpp = Math.max(1, (channels * depth) / 8);
|
|
2000
|
+
const stride = Math.ceil((width * channels * depth) / 8);
|
|
2001
|
+
if (raw.length < height * (stride + 1)) return undefined;
|
|
2002
|
+
const lines = Buffer.alloc(height * stride);
|
|
2003
|
+
for (let y = 0; y < height; y += 1) {
|
|
2004
|
+
const filter = raw[y * (stride + 1)];
|
|
2005
|
+
const src = y * (stride + 1) + 1;
|
|
2006
|
+
const dst = y * stride;
|
|
2007
|
+
for (let x = 0; x < stride; x += 1) {
|
|
2008
|
+
const a = x >= bpp ? lines[dst + x - bpp] : 0;
|
|
2009
|
+
const up = y > 0 ? lines[dst - stride + x] : 0;
|
|
2010
|
+
const c = x >= bpp && y > 0 ? lines[dst - stride + x - bpp] : 0;
|
|
2011
|
+
let value = raw[src + x];
|
|
2012
|
+
if (filter === 1) value += a;
|
|
2013
|
+
else if (filter === 2) value += up;
|
|
2014
|
+
else if (filter === 3) value += (a + up) >> 1;
|
|
2015
|
+
else if (filter === 4) {
|
|
2016
|
+
const p = a + up - c;
|
|
2017
|
+
const pa = Math.abs(p - a);
|
|
2018
|
+
const pb = Math.abs(p - up);
|
|
2019
|
+
const pc = Math.abs(p - c);
|
|
2020
|
+
value += pa <= pb && pa <= pc ? a : pb <= pc ? up : c;
|
|
2021
|
+
} else if (filter !== 0) return undefined;
|
|
2022
|
+
lines[dst + x] = value & 0xff;
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
const rgba = new Uint8Array(width * height * 4);
|
|
2026
|
+
const sample = (row, index) => {
|
|
2027
|
+
if (depth === 8) return lines[row + index];
|
|
2028
|
+
if (depth === 16) return lines[row + index * 2];
|
|
2029
|
+
const perByte = 8 / depth;
|
|
2030
|
+
const byte = lines[row + Math.floor(index / perByte)];
|
|
2031
|
+
const shift = 8 - depth * ((index % perByte) + 1);
|
|
2032
|
+
return (byte >> shift) & ((1 << depth) - 1);
|
|
2033
|
+
};
|
|
2034
|
+
const scale = depth < 8 && type !== 3 ? 255 / ((1 << depth) - 1) : 1;
|
|
2035
|
+
for (let y = 0; y < height; y += 1) {
|
|
2036
|
+
const row = y * stride;
|
|
2037
|
+
for (let x = 0; x < width; x += 1) {
|
|
2038
|
+
const o = (y * width + x) * 4;
|
|
2039
|
+
if (type === 3) {
|
|
2040
|
+
const i = sample(row, x);
|
|
2041
|
+
rgba[o] = palette[i * 3] ?? 0;
|
|
2042
|
+
rgba[o + 1] = palette[i * 3 + 1] ?? 0;
|
|
2043
|
+
rgba[o + 2] = palette[i * 3 + 2] ?? 0;
|
|
2044
|
+
rgba[o + 3] = trns !== undefined && i < trns.length ? trns[i] : 255;
|
|
2045
|
+
} else if (type === 0 || type === 4) {
|
|
2046
|
+
const g = Math.round(sample(row, x * channels) * scale);
|
|
2047
|
+
rgba[o] = rgba[o + 1] = rgba[o + 2] = g;
|
|
2048
|
+
rgba[o + 3] = type === 4 ? sample(row, x * channels + 1) : 255;
|
|
2049
|
+
} else {
|
|
2050
|
+
rgba[o] = sample(row, x * channels);
|
|
2051
|
+
rgba[o + 1] = sample(row, x * channels + 1);
|
|
2052
|
+
rgba[o + 2] = sample(row, x * channels + 2);
|
|
2053
|
+
rgba[o + 3] = type === 6 ? sample(row, x * channels + 3) : 255;
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
return { width, height, rgba };
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
/** Flatten onto white and scale down (area average) so the long edge is at most `maxEdge`; returns RGB8. */
|
|
2061
|
+
export function fitRgb({ width, height, rgba }, maxEdge) {
|
|
2062
|
+
const scale = Math.min(1, maxEdge / Math.max(width, height));
|
|
2063
|
+
const w = Math.max(1, Math.round(width * scale));
|
|
2064
|
+
const h = Math.max(1, Math.round(height * scale));
|
|
2065
|
+
const rgb = new Uint8Array(w * h * 3);
|
|
2066
|
+
const fx = width / w;
|
|
2067
|
+
const fy = height / h;
|
|
2068
|
+
for (let y = 0; y < h; y += 1) {
|
|
2069
|
+
const y0 = Math.floor(y * fy);
|
|
2070
|
+
const y1 = Math.max(y0 + 1, Math.min(height, Math.floor((y + 1) * fy)));
|
|
2071
|
+
for (let x = 0; x < w; x += 1) {
|
|
2072
|
+
const x0 = Math.floor(x * fx);
|
|
2073
|
+
const x1 = Math.max(x0 + 1, Math.min(width, Math.floor((x + 1) * fx)));
|
|
2074
|
+
let r = 0;
|
|
2075
|
+
let g = 0;
|
|
2076
|
+
let bl = 0;
|
|
2077
|
+
let n = 0;
|
|
2078
|
+
for (let yy = y0; yy < y1; yy += 1) {
|
|
2079
|
+
for (let xx = x0; xx < x1; xx += 1) {
|
|
2080
|
+
const o = (yy * width + xx) * 4;
|
|
2081
|
+
const a = rgba[o + 3] / 255;
|
|
2082
|
+
r += rgba[o] * a + 255 * (1 - a);
|
|
2083
|
+
g += rgba[o + 1] * a + 255 * (1 - a);
|
|
2084
|
+
bl += rgba[o + 2] * a + 255 * (1 - a);
|
|
2085
|
+
n += 1;
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
const d = (y * w + x) * 3;
|
|
2089
|
+
rgb[d] = Math.round(r / n);
|
|
2090
|
+
rgb[d + 1] = Math.round(g / n);
|
|
2091
|
+
rgb[d + 2] = Math.round(bl / n);
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
return { width: w, height: h, rgb };
|
|
2095
|
+
}
|
|
2096
|
+
|
|
2097
|
+
const ZIGZAG = [0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63];
|
|
2098
|
+
const STD_LUMA_Q = [16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19, 26, 58, 60, 55, 14, 13, 16, 24, 40, 57, 69, 56, 14, 17, 22, 29, 51, 87, 80, 62, 18, 22, 37, 56, 68, 109, 103, 77, 24, 35, 55, 64, 81, 104, 113, 92, 49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98, 112, 100, 103, 99];
|
|
2099
|
+
const STD_CHROMA_Q = [17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, 99, 99, 99, 24, 26, 56, 99, 99, 99, 99, 99, 47, 66, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99];
|
|
2100
|
+
// Annex K standard Huffman tables: code counts per length 1..16, then symbols.
|
|
2101
|
+
const HUFF = {
|
|
2102
|
+
dcL: [[0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]],
|
|
2103
|
+
dcC: [[0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]],
|
|
2104
|
+
acL: [[0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d], [
|
|
2105
|
+
0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08,
|
|
2106
|
+
0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28,
|
|
2107
|
+
0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
|
|
2108
|
+
0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89,
|
|
2109
|
+
0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
|
|
2110
|
+
0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2,
|
|
2111
|
+
0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa,
|
|
2112
|
+
]],
|
|
2113
|
+
acC: [[0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77], [
|
|
2114
|
+
0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
|
|
2115
|
+
0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26,
|
|
2116
|
+
0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
|
|
2117
|
+
0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
|
|
2118
|
+
0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4,
|
|
2119
|
+
0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda,
|
|
2120
|
+
0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa,
|
|
2121
|
+
]],
|
|
2122
|
+
};
|
|
2123
|
+
|
|
2124
|
+
function huffmanCodes([counts, symbols]) {
|
|
2125
|
+
const codes = new Array(256);
|
|
2126
|
+
let code = 0;
|
|
2127
|
+
let k = 0;
|
|
2128
|
+
for (let length = 1; length <= 16; length += 1) {
|
|
2129
|
+
for (let i = 0; i < counts[length - 1]; i += 1) codes[symbols[k++]] = [code++, length];
|
|
2130
|
+
code <<= 1;
|
|
2131
|
+
}
|
|
2132
|
+
return codes;
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
const COS = Array.from({ length: 8 }, (_, x) => Array.from({ length: 8 }, (_, u) => Math.cos(((2 * x + 1) * u * Math.PI) / 16)));
|
|
2136
|
+
const C0 = 1 / Math.SQRT2;
|
|
2137
|
+
|
|
2138
|
+
/** Baseline JPEG (4:4:4, standard tables) from RGB8. */
|
|
2139
|
+
export function encodeJpeg({ width, height, rgb }, quality = 85) {
|
|
2140
|
+
const q = Math.min(100, Math.max(1, Math.round(quality)));
|
|
2141
|
+
const factor = q < 50 ? 5000 / q : 200 - q * 2;
|
|
2142
|
+
const table = (base) => base.map((v) => Math.min(255, Math.max(1, Math.floor((v * factor + 50) / 100))));
|
|
2143
|
+
const qL = table(STD_LUMA_Q);
|
|
2144
|
+
const qC = table(STD_CHROMA_Q);
|
|
2145
|
+
const codes = { dcL: huffmanCodes(HUFF.dcL), dcC: huffmanCodes(HUFF.dcC), acL: huffmanCodes(HUFF.acL), acC: huffmanCodes(HUFF.acC) };
|
|
2146
|
+
const out = [];
|
|
2147
|
+
const bytes = (...values) => out.push(...values);
|
|
2148
|
+
const marker = (code, payload) => {
|
|
2149
|
+
bytes(0xff, code, ((payload.length + 2) >> 8) & 0xff, (payload.length + 2) & 0xff);
|
|
2150
|
+
for (const v of payload) out.push(v);
|
|
2151
|
+
};
|
|
2152
|
+
bytes(0xff, 0xd8);
|
|
2153
|
+
marker(0xe0, [0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0]);
|
|
2154
|
+
marker(0xdb, [0, ...ZIGZAG.map((i) => qL[i]), 1, ...ZIGZAG.map((i) => qC[i])]);
|
|
2155
|
+
marker(0xc0, [8, (height >> 8) & 0xff, height & 0xff, (width >> 8) & 0xff, width & 0xff, 3, 1, 0x11, 0, 2, 0x11, 1, 3, 0x11, 1]);
|
|
2156
|
+
const dht = [];
|
|
2157
|
+
for (const [id, key] of [[0x00, "dcL"], [0x10, "acL"], [0x01, "dcC"], [0x11, "acC"]]) dht.push(id, ...HUFF[key][0], ...HUFF[key][1]);
|
|
2158
|
+
marker(0xc4, dht);
|
|
2159
|
+
marker(0xda, [3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0]);
|
|
2160
|
+
|
|
2161
|
+
let buffer = 0;
|
|
2162
|
+
let bits = 0;
|
|
2163
|
+
const writeBits = (value, length) => {
|
|
2164
|
+
buffer = (buffer << length) | (value & ((1 << length) - 1));
|
|
2165
|
+
bits += length;
|
|
2166
|
+
while (bits >= 8) {
|
|
2167
|
+
const byte = (buffer >> (bits - 8)) & 0xff;
|
|
2168
|
+
out.push(byte);
|
|
2169
|
+
if (byte === 0xff) out.push(0);
|
|
2170
|
+
bits -= 8;
|
|
2171
|
+
}
|
|
2172
|
+
buffer &= (1 << bits) - 1;
|
|
2173
|
+
};
|
|
2174
|
+
const category = (v) => {
|
|
2175
|
+
let a = Math.abs(v);
|
|
2176
|
+
let n = 0;
|
|
2177
|
+
while (a > 0) {
|
|
2178
|
+
n += 1;
|
|
2179
|
+
a >>= 1;
|
|
2180
|
+
}
|
|
2181
|
+
return n;
|
|
2182
|
+
};
|
|
2183
|
+
const encodeValue = (v, n) => writeBits(v < 0 ? v + (1 << n) - 1 : v, n);
|
|
2184
|
+
const block = new Float64Array(64);
|
|
2185
|
+
const tmp = new Float64Array(64);
|
|
2186
|
+
const coef = new Int32Array(64);
|
|
2187
|
+
const prev = [0, 0, 0];
|
|
2188
|
+
const encodeBlock = (component, quant, dcCodes, acCodes) => {
|
|
2189
|
+
// Separable 2-D DCT-II.
|
|
2190
|
+
for (let y = 0; y < 8; y += 1) {
|
|
2191
|
+
for (let u = 0; u < 8; u += 1) {
|
|
2192
|
+
let sum = 0;
|
|
2193
|
+
for (let x = 0; x < 8; x += 1) sum += block[y * 8 + x] * COS[x][u];
|
|
2194
|
+
tmp[y * 8 + u] = sum * (u === 0 ? C0 : 1) / 2;
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
for (let u = 0; u < 8; u += 1) {
|
|
2198
|
+
for (let v = 0; v < 8; v += 1) {
|
|
2199
|
+
let sum = 0;
|
|
2200
|
+
for (let y = 0; y < 8; y += 1) sum += tmp[y * 8 + u] * COS[y][v];
|
|
2201
|
+
coef[v * 8 + u] = Math.round((sum * (v === 0 ? C0 : 1)) / 2 / quant[v * 8 + u]);
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
const dc = coef[0] - prev[component];
|
|
2205
|
+
prev[component] = coef[0];
|
|
2206
|
+
const dn = category(dc);
|
|
2207
|
+
writeBits(dcCodes[dn][0], dcCodes[dn][1]);
|
|
2208
|
+
if (dn > 0) encodeValue(dc, dn);
|
|
2209
|
+
let run = 0;
|
|
2210
|
+
for (let k = 1; k < 64; k += 1) {
|
|
2211
|
+
const value = coef[ZIGZAG[k]];
|
|
2212
|
+
if (value === 0) {
|
|
2213
|
+
run += 1;
|
|
2214
|
+
continue;
|
|
2215
|
+
}
|
|
2216
|
+
while (run > 15) {
|
|
2217
|
+
writeBits(acCodes[0xf0][0], acCodes[0xf0][1]);
|
|
2218
|
+
run -= 16;
|
|
2219
|
+
}
|
|
2220
|
+
const n = category(value);
|
|
2221
|
+
const symbol = (run << 4) | n;
|
|
2222
|
+
writeBits(acCodes[symbol][0], acCodes[symbol][1]);
|
|
2223
|
+
encodeValue(value, n);
|
|
2224
|
+
run = 0;
|
|
2225
|
+
}
|
|
2226
|
+
if (run > 0) writeBits(acCodes[0][0], acCodes[0][1]);
|
|
2227
|
+
};
|
|
2228
|
+
for (let by = 0; by < height; by += 8) {
|
|
2229
|
+
for (let bx = 0; bx < width; bx += 8) {
|
|
2230
|
+
for (let component = 0; component < 3; component += 1) {
|
|
2231
|
+
for (let y = 0; y < 8; y += 1) {
|
|
2232
|
+
const sy = Math.min(height - 1, by + y);
|
|
2233
|
+
for (let x = 0; x < 8; x += 1) {
|
|
2234
|
+
const o = (sy * width + Math.min(width - 1, bx + x)) * 3;
|
|
2235
|
+
const r = rgb[o];
|
|
2236
|
+
const g = rgb[o + 1];
|
|
2237
|
+
const b = rgb[o + 2];
|
|
2238
|
+
const value =
|
|
2239
|
+
component === 0 ? 0.299 * r + 0.587 * g + 0.114 * b
|
|
2240
|
+
: component === 1 ? -0.168736 * r - 0.331264 * g + 0.5 * b + 128
|
|
2241
|
+
: 0.5 * r - 0.418688 * g - 0.081312 * b + 128;
|
|
2242
|
+
block[y * 8 + x] = value - 128;
|
|
2243
|
+
}
|
|
2244
|
+
}
|
|
2245
|
+
if (component === 0) encodeBlock(0, qL, codes.dcL, codes.acL);
|
|
2246
|
+
else encodeBlock(component, qC, codes.dcC, codes.acC);
|
|
2247
|
+
}
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
if (bits > 0) writeBits(0x7f, 8 - bits);
|
|
2251
|
+
bytes(0xff, 0xd9);
|
|
2252
|
+
return new Uint8Array(out);
|
|
2253
|
+
}
|
|
2254
|
+
|
|
2255
|
+
/**
|
|
2256
|
+
* A reference image within `maxBytes`: unchanged when it already fits; a PNG
|
|
2257
|
+
* over the budget is re-encoded as JPEG (flattened onto white, long edge at
|
|
2258
|
+
* most `maxEdge`, lower quality / size until it fits). Returns undefined for
|
|
2259
|
+
* other formats over the budget (the caller may ask DSH for a smaller version).
|
|
2260
|
+
* @param image - `{ mediaType, data }`.
|
|
2261
|
+
* @returns `{ mediaType, data, resized }` or undefined.
|
|
2262
|
+
*/
|
|
2263
|
+
export function fitReferenceImage(image, maxBytes, maxEdge = REFERENCE_MAX_EDGE) {
|
|
2264
|
+
const data = image.data instanceof Uint8Array ? image.data : new Uint8Array(image.data);
|
|
2265
|
+
if (data.byteLength <= maxBytes) return { mediaType: image.mediaType ?? sniffImageType(data) ?? "image/png", data, resized: false };
|
|
2266
|
+
const decoded = sniffImageType(data) === "image/png" ? decodePng(data) : undefined;
|
|
2267
|
+
if (decoded === undefined) return undefined;
|
|
2268
|
+
for (const edge of [maxEdge, Math.round(maxEdge * 0.75), Math.round(maxEdge * 0.5), Math.round(maxEdge * 0.375)]) {
|
|
2269
|
+
const fitted = fitRgb(decoded, edge);
|
|
2270
|
+
for (const quality of [88, 78, 65]) {
|
|
2271
|
+
const jpeg = encodeJpeg(fitted, quality);
|
|
2272
|
+
if (jpeg.byteLength <= maxBytes) return { mediaType: "image/jpeg", data: jpeg, resized: true };
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
return undefined;
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2278
|
+
// #endregion
|
|
2279
|
+
// #region Agent tool (generate_media)
|
|
2280
|
+
//
|
|
2281
|
+
// Pure parts of the optional agent tool: its static definition (no model
|
|
2282
|
+
// lists), argument checks, the command line it runs through the slash-command
|
|
2283
|
+
// handlers, the run id derived from the tool call id, and the approval text.
|
|
2284
|
+
|
|
2285
|
+
export const GENERATE_MEDIA = "generate_media";
|
|
2286
|
+
|
|
2287
|
+
const MEDIA_ASPECTS_ALL = [...new Set([...IMAGE_ASPECTS, ...VIDEO_ASPECTS])];
|
|
2288
|
+
|
|
2289
|
+
// Responses strict schemas may require every property: null keeps optional and
|
|
2290
|
+
// inapplicable arguments expressible without inventing a model, source or mode.
|
|
2291
|
+
const optionalMediaArg = (schema, description) => ({
|
|
2292
|
+
oneOf: [schema, { type: "null" }],
|
|
2293
|
+
...(description === undefined ? {} : { description }),
|
|
2294
|
+
});
|
|
2295
|
+
|
|
2296
|
+
/** The tool's model-facing description and parameter schema (raw JSON Schema subset). */
|
|
2297
|
+
export const GENERATE_MEDIA_TOOL = {
|
|
2298
|
+
name: GENERATE_MEDIA,
|
|
2299
|
+
description:
|
|
2300
|
+
"Generate an image or video with the user's CLIProxyAPI media models, or edit / extend a video; saves the files under the working directory and returns their paths. Pass the user's prompt in their own words and level of detail: never expand it into a longer scene description or add camera, lighting, style, character or environment details they did not ask for. Videos are billed, cannot be cancelled and may need the user's approval; never retry one that was declined or over budget.",
|
|
2301
|
+
parameters: {
|
|
2302
|
+
type: "object",
|
|
2303
|
+
properties: {
|
|
2304
|
+
kind: { type: "string", enum: ["image", "video"] },
|
|
2305
|
+
prompt: { type: "string", description: "The user's prompt, verbatim: their own words and level of detail. Do not rewrite, expand, translate or add style/camera/scene details." },
|
|
2306
|
+
mode: optionalMediaArg({ type: "string", enum: ["generate", "edit", "extend"] }, "Video only; null = generate."),
|
|
2307
|
+
model: optionalMediaArg({ type: "string" }, "Model id or unique part; null = user's pick."),
|
|
2308
|
+
images: optionalMediaArg({ type: "array", items: { type: "string" } }, "Image paths, or null. Image: up to 4 to edit/combine. Video: 1 = first frame, 2+ = references."),
|
|
2309
|
+
frames: optionalMediaArg({ type: "string", enum: ["first-frame", "reference", "first-last", "last-frame"] }, "Video images only; otherwise null. Pinned first-last/last-frame need grok-imagine-video-1.5."),
|
|
2310
|
+
video: optionalMediaArg({ type: "string" }, "Edit/extend source: MP4 path or public URL; null = session's newest video. Other modes: null."),
|
|
2311
|
+
aspect: optionalMediaArg({ type: "string", enum: MEDIA_ASPECTS_ALL }, "Image/video generation ratio, or null for default; edit/extend: null."),
|
|
2312
|
+
seconds: optionalMediaArg({ type: "integer" }, "Video length 1–15, extend adds 2–10; null = default. Image/edit: null."),
|
|
2313
|
+
resolution: optionalMediaArg({ type: "string", enum: ["480p", "720p"] }, "Video generation only; null = default or unused."),
|
|
2314
|
+
count: optionalMediaArg({ type: "integer" }, "Images 1–4; null = 1. Video: null."),
|
|
2315
|
+
},
|
|
2316
|
+
required: ["kind", "prompt"],
|
|
2317
|
+
},
|
|
2318
|
+
};
|
|
2319
|
+
|
|
2320
|
+
/** The tool's result value schema (validated by DSH's tool registry). */
|
|
2321
|
+
export const GENERATE_MEDIA_OUTPUT = {
|
|
2322
|
+
type: "object",
|
|
2323
|
+
additionalProperties: false,
|
|
2324
|
+
properties: {
|
|
2325
|
+
status: { type: "string", enum: ["done", "running"] },
|
|
2326
|
+
kind: { type: "string" },
|
|
2327
|
+
mode: { type: "string" },
|
|
2328
|
+
model: { type: "string" },
|
|
2329
|
+
account: { type: "string" },
|
|
2330
|
+
run: { type: "string" },
|
|
2331
|
+
files: { type: "array", items: { type: "string" } },
|
|
2332
|
+
folder: { type: "string" },
|
|
2333
|
+
seconds: { type: "integer" },
|
|
2334
|
+
resolution: { type: "string" },
|
|
2335
|
+
aspect: { type: "string" },
|
|
2336
|
+
revisedPrompt: { type: "string" },
|
|
2337
|
+
note: { type: "string" },
|
|
2338
|
+
commandRow: { type: "boolean" },
|
|
2339
|
+
previews: {
|
|
2340
|
+
type: "array",
|
|
2341
|
+
items: {
|
|
2342
|
+
type: "object",
|
|
2343
|
+
additionalProperties: false,
|
|
2344
|
+
properties: {
|
|
2345
|
+
attachmentId: { type: "string" },
|
|
2346
|
+
mediaType: { type: "string" },
|
|
2347
|
+
bytes: { type: "integer" },
|
|
2348
|
+
width: { type: "integer" },
|
|
2349
|
+
height: { type: "integer" },
|
|
2350
|
+
name: { type: "string" },
|
|
2351
|
+
originalDimensions: {
|
|
2352
|
+
type: "object",
|
|
2353
|
+
additionalProperties: false,
|
|
2354
|
+
properties: { width: { type: "integer" }, height: { type: "integer" } },
|
|
2355
|
+
required: ["width", "height"],
|
|
2356
|
+
},
|
|
2357
|
+
},
|
|
2358
|
+
required: ["attachmentId", "mediaType", "bytes", "width", "height"],
|
|
2359
|
+
},
|
|
2360
|
+
},
|
|
2361
|
+
},
|
|
2362
|
+
required: ["status", "kind", "model", "run", "files"],
|
|
2363
|
+
};
|
|
2364
|
+
|
|
2365
|
+
/**
|
|
2366
|
+
* Check one `generate_media` call's arguments.
|
|
2367
|
+
* @returns the normalized arguments.
|
|
2368
|
+
* @throws Error with a model-facing message.
|
|
2369
|
+
*/
|
|
2370
|
+
export function parseToolArgs(args) {
|
|
2371
|
+
const input = plain(args);
|
|
2372
|
+
if (input === undefined) throw new Error("arguments must be an object");
|
|
2373
|
+
const known = new Set(Object.keys(GENERATE_MEDIA_TOOL.parameters.properties));
|
|
2374
|
+
for (const key of Object.keys(input)) if (!known.has(key)) throw new Error(`unknown argument "${key}"`);
|
|
2375
|
+
// Canonical strict-schema calls use null for absent optional fields. Also
|
|
2376
|
+
// accept empty strings from older all-fields calls, without changing prompt.
|
|
2377
|
+
const a = Object.fromEntries(Object.entries(input).filter(([key, value]) =>
|
|
2378
|
+
key === "kind" || key === "prompt" || (value !== null && value !== undefined && !(typeof value === "string" && value.trim() === ""))
|
|
2379
|
+
));
|
|
2380
|
+
if (a.kind !== "image" && a.kind !== "video") throw new Error('kind must be "image" or "video"');
|
|
2381
|
+
if (typeof a.prompt !== "string" || a.prompt.trim() === "") throw new Error("prompt must be a non-empty string");
|
|
2382
|
+
if (a.prompt.length > 4000) throw new Error("prompt is longer than 4000 characters");
|
|
2383
|
+
const str = (key) => {
|
|
2384
|
+
if (a[key] === undefined) return undefined;
|
|
2385
|
+
if (typeof a[key] !== "string" || a[key].trim() === "") throw new Error(`${key} must be a non-empty string`);
|
|
2386
|
+
return a[key].trim();
|
|
2387
|
+
};
|
|
2388
|
+
const int = (key) => {
|
|
2389
|
+
if (a[key] === undefined) return undefined;
|
|
2390
|
+
if (!Number.isInteger(a[key])) throw new Error(`${key} must be an integer`);
|
|
2391
|
+
return a[key];
|
|
2392
|
+
};
|
|
2393
|
+
const out = { kind: a.kind, prompt: a.prompt.trim() };
|
|
2394
|
+
const model = str("model");
|
|
2395
|
+
if (model !== undefined) {
|
|
2396
|
+
if (/\s/.test(model)) throw new Error("model must not contain spaces");
|
|
2397
|
+
out.model = model;
|
|
2398
|
+
}
|
|
2399
|
+
if (a.images !== undefined) {
|
|
2400
|
+
if (!Array.isArray(a.images) || a.images.some((p) => typeof p !== "string" || p.trim() === "")) throw new Error("images must be an array of paths");
|
|
2401
|
+
if (a.images.length > 14) throw new Error("at most 14 images");
|
|
2402
|
+
if (a.images.length > 0) out.images = a.images.map((p) => p.trim());
|
|
2403
|
+
}
|
|
2404
|
+
const aspect = str("aspect");
|
|
2405
|
+
if (aspect !== undefined) out.aspect = aspect;
|
|
2406
|
+
if (a.kind === "image") {
|
|
2407
|
+
// Older strict calls supplied these neutral defaults even for images.
|
|
2408
|
+
if (a.mode === "generate") delete a.mode;
|
|
2409
|
+
if (a.frames === "first-frame") delete a.frames;
|
|
2410
|
+
for (const key of ["mode", "frames", "video", "seconds", "resolution"]) if (a[key] !== undefined) throw new Error(`${key} is for videos only`);
|
|
2411
|
+
const count = int("count");
|
|
2412
|
+
if (count !== undefined && (count < 1 || count > 4)) throw new Error("count must be 1–4 for images");
|
|
2413
|
+
if (count !== undefined) out.count = count;
|
|
2414
|
+
if (aspect !== undefined && !IMAGE_ASPECTS.includes(aspect)) throw new Error(`aspect must be one of ${IMAGE_ASPECTS.join(", ")} for images`);
|
|
2415
|
+
if ((out.images?.length ?? 0) > MAX_REFERENCE_IMAGES) throw new Error(`at most ${MAX_REFERENCE_IMAGES} images for an image`);
|
|
2416
|
+
return out;
|
|
2417
|
+
}
|
|
2418
|
+
// One video is already the only supported behavior: legacy count: 1 is a
|
|
2419
|
+
// harmless default, but an actual request for multiple videos is rejected.
|
|
2420
|
+
if (a.count !== undefined && a.count !== 1) throw new Error("count is for images only; use null for a video (one video per call)");
|
|
2421
|
+
const mode = str("mode") ?? "generate";
|
|
2422
|
+
if (!["generate", "edit", "extend"].includes(mode)) throw new Error('mode must be "generate", "edit" or "extend"');
|
|
2423
|
+
out.mode = mode;
|
|
2424
|
+
const seconds = int("seconds");
|
|
2425
|
+
if (seconds !== undefined) out.seconds = seconds;
|
|
2426
|
+
if (mode === "generate") {
|
|
2427
|
+
if (a.video !== undefined) throw new Error("video is for mode edit / extend");
|
|
2428
|
+
if (seconds !== undefined && (seconds < 1 || seconds > 15)) throw new Error("seconds must be 1–15 for video generation");
|
|
2429
|
+
if (aspect !== undefined && !VIDEO_ASPECTS.includes(aspect)) throw new Error(`aspect must be one of ${VIDEO_ASPECTS.join(", ")} for videos`);
|
|
2430
|
+
const resolution = str("resolution");
|
|
2431
|
+
if (resolution !== undefined && !["480p", "720p"].includes(resolution)) throw new Error("resolution must be 480p or 720p");
|
|
2432
|
+
if (resolution !== undefined) out.resolution = resolution;
|
|
2433
|
+
const frames = str("frames");
|
|
2434
|
+
if (frames !== undefined) {
|
|
2435
|
+
if (!["first-frame", "reference", "first-last", "last-frame"].includes(frames)) throw new Error("frames must be first-frame, reference, first-last or last-frame");
|
|
2436
|
+
const count = out.images?.length ?? 0;
|
|
2437
|
+
// A legacy all-fields text-video call supplied first-frame with no images.
|
|
2438
|
+
// It is still text-to-video, not an implicit image or a guessed reference.
|
|
2439
|
+
if (count === 0 && frames !== "first-frame") throw new Error("frames needs images; use null without images");
|
|
2440
|
+
if (count > 0) {
|
|
2441
|
+
if ((frames === "first-frame" || frames === "last-frame") && count !== 1) throw new Error(`${frames} needs exactly one image`);
|
|
2442
|
+
if (frames === "first-last" && count !== 2) throw new Error("first-last needs exactly two images");
|
|
2443
|
+
out.frames = frames;
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
return out;
|
|
2447
|
+
}
|
|
2448
|
+
if (a.images?.length === 0) delete a.images;
|
|
2449
|
+
if (a.frames === "first-frame") delete a.frames;
|
|
2450
|
+
for (const key of ["images", "frames", "aspect", "resolution"]) if (a[key] !== undefined) throw new Error(`${key} is not used by mode ${mode} (the output keeps the source's format)`);
|
|
2451
|
+
if (mode === "edit" && seconds !== undefined) throw new Error("an edit keeps the source's length; seconds is for extend");
|
|
2452
|
+
if (mode === "extend" && seconds !== undefined && (seconds < 2 || seconds > 10)) throw new Error("seconds must be 2–10 for extend");
|
|
2453
|
+
const video = str("video");
|
|
2454
|
+
if (video !== undefined) out.video = video;
|
|
2455
|
+
return out;
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
/** The slash-command line (after the command name) that runs these arguments, taken literally. */
|
|
2459
|
+
export function toolCommandLine(t) {
|
|
2460
|
+
const parts = [];
|
|
2461
|
+
if (t.model !== undefined) parts.push(`-m ${t.model}`);
|
|
2462
|
+
if (t.kind === "image") {
|
|
2463
|
+
if (t.aspect !== undefined) parts.push(`--aspect ${t.aspect}`);
|
|
2464
|
+
if (t.count !== undefined) parts.push(`-n ${t.count}`);
|
|
2465
|
+
} else if (t.mode === "generate") {
|
|
2466
|
+
if (t.aspect !== undefined) parts.push(`--aspect ${t.aspect}`);
|
|
2467
|
+
if (t.seconds !== undefined) parts.push(`-s ${t.seconds}`);
|
|
2468
|
+
if (t.resolution !== undefined) parts.push(`-r ${t.resolution}`);
|
|
2469
|
+
const flag = { reference: "--ref", "first-last": "--first-last", "last-frame": "--last" }[t.frames];
|
|
2470
|
+
if (flag !== undefined) parts.push(flag);
|
|
2471
|
+
} else {
|
|
2472
|
+
if (t.video !== undefined) {
|
|
2473
|
+
if (/^https?:\/\//i.test(t.video)) parts.push(`--url ${t.video.replace(/\s/g, "%20")}`);
|
|
2474
|
+
else {
|
|
2475
|
+
if (t.video.includes('"')) throw new Error("video path must not contain a double quote");
|
|
2476
|
+
parts.push(`--video "${t.video}"`);
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
if (t.mode === "extend" && t.seconds !== undefined) parts.push(`-s ${t.seconds}`);
|
|
2480
|
+
}
|
|
2481
|
+
parts.push("--", t.prompt);
|
|
2482
|
+
return parts.join(" ");
|
|
2483
|
+
}
|
|
2484
|
+
|
|
2485
|
+
/** The run id (a chat-row key) for one tool call: the call id with other characters replaced. */
|
|
2486
|
+
export function toolRunId(callId) {
|
|
2487
|
+
const raw = String(callId ?? "").replace(/[^A-Za-z0-9._:-]/g, "_").replace(/^[^A-Za-z0-9]+/, "").slice(0, 119);
|
|
2488
|
+
return validCommandId(raw);
|
|
2489
|
+
}
|
|
2490
|
+
|
|
2491
|
+
/**
|
|
2492
|
+
* What the approval prompt says about one generation (both languages).
|
|
2493
|
+
* @param plan - from the command's gate: kind, mode, option, prompt, seconds, resolution, aspect, references, count, sourceName.
|
|
2494
|
+
* @param policy - `agentPolicy()` of the root session (videos show the budget).
|
|
2495
|
+
*/
|
|
2496
|
+
export function approvalText(plan, policy) {
|
|
2497
|
+
const prompt = plan.prompt.length > 80 ? `${plan.prompt.slice(0, 80)}…` : plan.prompt;
|
|
2498
|
+
const model = `${plan.option.modelId} (${plan.option.accountName})`;
|
|
2499
|
+
const modelZh = `${plan.option.modelId}(${plan.option.accountName})`;
|
|
2500
|
+
if (plan.kind === "image") {
|
|
2501
|
+
const n = plan.count ?? 1;
|
|
2502
|
+
return {
|
|
2503
|
+
reason: `generate ${n} image(s) with ${plan.option.ref}`,
|
|
2504
|
+
en: `${plan.references > 0 ? `Edit ${plan.references} image(s) into` : "Generate"} ${n} image(s) with ${model}: "${prompt}".`,
|
|
2505
|
+
zh: `用 ${modelZh}${plan.references > 0 ? `改 ${plan.references} 张图,` : ""}生成 ${n} 张图片:「${prompt}」。`,
|
|
2506
|
+
};
|
|
2507
|
+
}
|
|
2508
|
+
const used = policy.limit === undefined ? `${policy.used} s (no limit)` : `${policy.used} of ${policy.limit} s`;
|
|
2509
|
+
const usedZh = policy.limit === undefined ? `${policy.used} 秒(不限)` : `${policy.used} / ${policy.limit} 秒`;
|
|
2510
|
+
const shape = [plan.resolution, plan.aspect].filter(Boolean).join(", ");
|
|
2511
|
+
const what =
|
|
2512
|
+
plan.mode === "edit" ? `Edit the video${plan.sourceName ? ` ${plan.sourceName}` : ""}`
|
|
2513
|
+
: plan.mode === "extend" ? `Extend the video${plan.sourceName ? ` ${plan.sourceName}` : ""} by ${plan.seconds} s`
|
|
2514
|
+
: `Generate a ${plan.seconds} s video${shape ? ` (${shape})` : ""}`;
|
|
2515
|
+
const whatZh =
|
|
2516
|
+
plan.mode === "edit" ? `编辑视频${plan.sourceName ? ` ${plan.sourceName}` : ""}`
|
|
2517
|
+
: plan.mode === "extend" ? `把视频${plan.sourceName ? ` ${plan.sourceName}` : ""}延长 ${plan.seconds} 秒`
|
|
2518
|
+
: `生成 ${plan.seconds} 秒视频${shape ? `(${shape})` : ""}`;
|
|
2519
|
+
const refs = plan.references > 0 ? ` with ${plan.references} attached image(s)` : "";
|
|
2520
|
+
const refsZh = plan.references > 0 ? `,附 ${plan.references} 张图` : "";
|
|
2521
|
+
return {
|
|
2522
|
+
reason: `${plan.mode ?? "generate"} a ${plan.seconds} s video with ${plan.option.ref}`,
|
|
2523
|
+
en: `${what} with ${model}${refs}: "${prompt}". It is billed and cannot be cancelled once submitted. Agent videos in this session so far: ${used}.`,
|
|
2524
|
+
zh: `用 ${modelZh}${whatZh}${refsZh}:「${prompt}」。提交后无法取消,会照常计费。本会话 agent 已用视频 ${usedZh}。`,
|
|
2525
|
+
};
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
// #endregion
|