dsh-cliproxyapi-pro 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +123 -59
- package/README.en.md +183 -70
- package/README.md +184 -69
- package/lib/client.js +5196 -618
- package/lib/index.js +2186 -204
- package/lib/media.js +2528 -0
- package/lib/quota.js +1646 -0
- package/lib/updater.js +84 -20
- package/locale/en.json +2 -2
- package/locale/zh.json +2 -2
- package/package.json +10 -4
package/lib/index.js
CHANGED
|
@@ -1,13 +1,32 @@
|
|
|
1
1
|
// Zero-external-dependency host plugin for DSH CLIProxyAPI auto-discovery, model selection & sync.
|
|
2
|
-
// Uses only Node.js built-ins and Cordis context services (`ctx.settings`, `ctx.credentials`, `ctx.commands`, `ctx.
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
// Uses only Node.js built-ins and Cordis context services (`ctx.settings`, `ctx.credentials`, `ctx.commands`, `ctx.webServer`).
|
|
3
|
+
// It deliberately registers no agent tool: accounts are managed in the Settings
|
|
4
|
+
// page, HTTP routes and slash commands, so nothing adds to model context.
|
|
5
|
+
// Image / video models (lib/media.js) are classified and kept out of the
|
|
6
|
+
// chat routes; their preferences live in media-settings.json.
|
|
7
|
+
|
|
8
|
+
import { createReadStream, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { randomUUID } from "node:crypto";
|
|
5
10
|
import { homedir } from "node:os";
|
|
6
|
-
import { dirname, join } from "node:path";
|
|
11
|
+
import { basename, dirname, extname, isAbsolute, join, relative, sep } from "node:path";
|
|
7
12
|
import {
|
|
8
13
|
createPluginUpdater, loadUpdateSettings, saveUpdateSettings, storeUpdatePackage,
|
|
9
14
|
UPDATE_INTERVAL_MS, UpdateError,
|
|
10
15
|
} from "./updater.js";
|
|
16
|
+
import { createQuotaController } from "./quota.js";
|
|
17
|
+
import {
|
|
18
|
+
accountKindOverrides, applyMediaPatch, chooseMediaModel, createMediaSessionStore, detectMediaKind,
|
|
19
|
+
diagnoseMedia, effectiveKind, generateImages, IMAGE_ASPECTS, loadMediaSettings, MAX_REFERENCE_IMAGES,
|
|
20
|
+
mediaAliasTarget, MediaError, mediaModelEnabled, mediaOptions, normalizeMediaSettings, outputDirectory,
|
|
21
|
+
outputStem, parseImageCommand, recentEntry, saveMediaSettings, validCommandId, validSessionId, writeOutputFiles,
|
|
22
|
+
createMediaJobStore, downloadVideo, MAX_VIDEO_BYTES, parseVideoCommand, retrieveVideo, submitVideo, VIDEO_ASPECTS, VIDEO_JOB_TTL_MS,
|
|
23
|
+
fitReferenceImage, REFERENCE_MAX_EDGE,
|
|
24
|
+
EDIT_SOURCE_MAX_SECONDS, EXTEND_SOURCE_SECONDS, parseVideoEditCommand, submitVideoEdit, VIDEO_EDIT_MODEL,
|
|
25
|
+
isMp4, mp4Duration, videoImageBudget, VIDEO_UPLOAD_MAX_BYTES,
|
|
26
|
+
interpretationLabels, interpretationRequest, parseInterpretation,
|
|
27
|
+
agentPolicy, applySessionPolicy, approvalText, chargeSeconds, GENERATE_MEDIA, GENERATE_MEDIA_OUTPUT, GENERATE_MEDIA_TOOL, parseToolArgs,
|
|
28
|
+
sniffImageType, toolCommandLine, toolRunId,
|
|
29
|
+
} from "./media.js";
|
|
11
30
|
|
|
12
31
|
// #region Constants & Config Resolution
|
|
13
32
|
|
|
@@ -372,7 +391,7 @@ export function newAccountDefaults(accounts) {
|
|
|
372
391
|
}
|
|
373
392
|
|
|
374
393
|
/**
|
|
375
|
-
* Apply an account patch from the UI
|
|
394
|
+
* Apply an account patch from the UI. `selectedModels` and
|
|
376
395
|
* `modelOverrides` replace the stored value when present; the id never changes.
|
|
377
396
|
*/
|
|
378
397
|
export function patchAccount(account, patch) {
|
|
@@ -554,6 +573,10 @@ export function parseCatalog(payload) {
|
|
|
554
573
|
const inputModalities = stringArray(item.input_modalities ?? item.inputModalities).map((v) =>
|
|
555
574
|
v.toLowerCase()
|
|
556
575
|
);
|
|
576
|
+
const outputModalities = stringArray(
|
|
577
|
+
item.supportedOutputModalities ?? item.output_modalities ?? item.outputModalities
|
|
578
|
+
).map((v) => v.toLowerCase());
|
|
579
|
+
const catalogType = typeof item.type === "string" && item.type.trim() !== "" ? item.type.trim() : undefined;
|
|
557
580
|
const rawLevels = item.supported_reasoning_levels ?? item.reasoning_levels;
|
|
558
581
|
const levels = parseReasoningLevels(rawLevels);
|
|
559
582
|
const reasoningWire = parseReasoningWire(rawLevels);
|
|
@@ -581,9 +604,13 @@ export function parseCatalog(payload) {
|
|
|
581
604
|
...(contextWindow === undefined ? {} : { contextWindow }),
|
|
582
605
|
...(maxTokens === undefined ? {} : { maxTokens }),
|
|
583
606
|
inputModalities: inputModalities.length > 0 ? inputModalities : ["text"],
|
|
607
|
+
...(outputModalities.length === 0 ? {} : { outputModalities }),
|
|
608
|
+
...(catalogType === undefined ? {} : { catalogType }),
|
|
584
609
|
reasoning: item.reasoning === true || levels.some((level) => level !== "off"),
|
|
585
610
|
reasoningLevels: levels,
|
|
586
611
|
...(Object.keys(reasoningWire).length === 0 ? {} : { reasoningWire }),
|
|
612
|
+
// chat | image | video, before user overrides (see lib/media.js).
|
|
613
|
+
detectedKind: detectMediaKind({ id, catalogType, outputModalities }),
|
|
587
614
|
});
|
|
588
615
|
}
|
|
589
616
|
return [...models.values()];
|
|
@@ -605,9 +632,20 @@ async function readBoundedJson(response) {
|
|
|
605
632
|
}
|
|
606
633
|
}
|
|
607
634
|
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
635
|
+
/**
|
|
636
|
+
* The plain `/v1/models` listing, parsed, by id. It carries models the
|
|
637
|
+
* enriched catalog omits (the registry `type` that marks image models, and
|
|
638
|
+
* media models only this listing knows), so it gets its own deadline instead
|
|
639
|
+
* of the enriched request's remaining time and one retry: a slow first
|
|
640
|
+
* response must never silently hide every image model until the next
|
|
641
|
+
* five-minute refresh.
|
|
642
|
+
* @returns `{ models, ok }`; `ok` is false when the listing could not be read.
|
|
643
|
+
*/
|
|
644
|
+
async function fetchListing(fetchImpl, rootURL, options) {
|
|
645
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
|
646
|
+
const attempt = async () => {
|
|
647
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
648
|
+
const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]);
|
|
611
649
|
const response = await fetchImpl(`${rootURL}/v1/models`, {
|
|
612
650
|
method: "GET",
|
|
613
651
|
headers: {
|
|
@@ -618,17 +656,46 @@ async function fetchOwners(fetchImpl, rootURL, options, signal) {
|
|
|
618
656
|
},
|
|
619
657
|
signal,
|
|
620
658
|
});
|
|
621
|
-
if (!response.ok) return
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
if (id !== "" && owner !== "") owners.set(id, owner);
|
|
627
|
-
}
|
|
659
|
+
if (!response.ok) return { models: new Map(), ok: false };
|
|
660
|
+
return { models: new Map(parseCatalog(await readBoundedJson(response)).map((m) => [m.id, m])), ok: true };
|
|
661
|
+
};
|
|
662
|
+
try {
|
|
663
|
+
return await attempt();
|
|
628
664
|
} catch (error) {
|
|
629
665
|
if (options.signal?.aborted) throw options.signal.reason;
|
|
666
|
+
if (error instanceof Error && /catalog is larger|not valid JSON|does not contain/.test(error.message)) {
|
|
667
|
+
return { models: new Map(), ok: false };
|
|
668
|
+
}
|
|
669
|
+
try {
|
|
670
|
+
return await attempt();
|
|
671
|
+
} catch (retryError) {
|
|
672
|
+
if (options.signal?.aborted) throw options.signal.reason;
|
|
673
|
+
return { models: new Map(), ok: false };
|
|
674
|
+
}
|
|
630
675
|
}
|
|
631
|
-
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
/**
|
|
679
|
+
* Fill catalog gaps from the plain listing: `owned_by` (native protocol),
|
|
680
|
+
* the registry `type` and output modalities (media kind). A media model that
|
|
681
|
+
* only the plain listing carries is added; plain-only chat models are not.
|
|
682
|
+
*/
|
|
683
|
+
function mergeListing(models, listing) {
|
|
684
|
+
const merged = models.map((m) => {
|
|
685
|
+
const plainEntry = listing.get(m.id);
|
|
686
|
+
if (plainEntry === undefined) return m;
|
|
687
|
+
const next = { ...m };
|
|
688
|
+
for (const field of ["ownedBy", "catalogType", "outputModalities"]) {
|
|
689
|
+
if (next[field] === undefined && plainEntry[field] !== undefined) next[field] = plainEntry[field];
|
|
690
|
+
}
|
|
691
|
+
next.detectedKind = detectMediaKind(next);
|
|
692
|
+
return next;
|
|
693
|
+
});
|
|
694
|
+
const known = new Set(merged.map((m) => m.id));
|
|
695
|
+
for (const entry of listing.values()) {
|
|
696
|
+
if (!known.has(entry.id) && entry.detectedKind !== "chat") merged.push(entry);
|
|
697
|
+
}
|
|
698
|
+
return merged;
|
|
632
699
|
}
|
|
633
700
|
|
|
634
701
|
export async function discoverCpa(options) {
|
|
@@ -640,6 +707,9 @@ export async function discoverCpa(options) {
|
|
|
640
707
|
const timeout = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS);
|
|
641
708
|
const signal =
|
|
642
709
|
options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]);
|
|
710
|
+
// Start the plain listing together with the enriched catalog; both carry
|
|
711
|
+
// different fields, and the listing alone knows some image models.
|
|
712
|
+
const listingPromise = fetchListing(fetchImpl, rootURL, options);
|
|
643
713
|
try {
|
|
644
714
|
const response = await fetchImpl(modelsURL, {
|
|
645
715
|
method: "GET",
|
|
@@ -652,21 +722,18 @@ export async function discoverCpa(options) {
|
|
|
652
722
|
signal,
|
|
653
723
|
});
|
|
654
724
|
if (!response.ok) {
|
|
725
|
+
void listingPromise.catch(() => {});
|
|
655
726
|
failures.push(`${rootURL}: HTTP ${response.status}`);
|
|
656
727
|
continue;
|
|
657
728
|
}
|
|
658
729
|
const models = parseCatalog(await readBoundedJson(response));
|
|
659
730
|
// The enriched catalog omits `owned_by`; the plain listing carries the
|
|
660
|
-
// model family each upstream reports, which decides its native protocol
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
modelsURL,
|
|
665
|
-
models: models.map((m) =>
|
|
666
|
-
m.ownedBy === undefined && owners.has(m.id) ? { ...m, ownedBy: owners.get(m.id) } : m
|
|
667
|
-
),
|
|
668
|
-
};
|
|
731
|
+
// model family each upstream reports, which decides its native protocol,
|
|
732
|
+
// and the registry type that marks image models.
|
|
733
|
+
const listing = await listingPromise;
|
|
734
|
+
return { rootURL, modelsURL, models: mergeListing(models, listing.models), listingOk: listing.ok };
|
|
669
735
|
} catch (error) {
|
|
736
|
+
void listingPromise.catch(() => {});
|
|
670
737
|
if (options.signal?.aborted) throw options.signal.reason;
|
|
671
738
|
failures.push(`${rootURL}: ${error instanceof Error ? error.message : String(error)}`);
|
|
672
739
|
}
|
|
@@ -896,16 +963,23 @@ function toPiAiModelProfile(model, protocol) {
|
|
|
896
963
|
* route key to its profile, in protocol order, and every ordered model
|
|
897
964
|
* carries its `protocol` and `route`.
|
|
898
965
|
*/
|
|
966
|
+
/** Whether a model belongs in chat routes: `kind` (with overrides) or detection says chat. */
|
|
967
|
+
export function isChatModel(model) {
|
|
968
|
+
return (model.kind ?? model.detectedKind ?? "chat") === "chat";
|
|
969
|
+
}
|
|
970
|
+
|
|
899
971
|
export function buildPiAiRoutes(config, discovery, hasApiKey) {
|
|
900
972
|
const allOrdered = sortModelsNewestFirst(discovery.models);
|
|
901
|
-
|
|
973
|
+
// Image / video models are served by CPA's media endpoints, never by chat routes.
|
|
974
|
+
const chatOrdered = allOrdered.filter(isChatModel);
|
|
975
|
+
const filtered = filterModels(chatOrdered, config).map((model) => {
|
|
902
976
|
const effective = applyModelOverride(model, config.modelOverrides?.[model.id]);
|
|
903
977
|
const protocol = modelProtocol(model, config);
|
|
904
978
|
return { ...effective, protocol, route: routeKeyFor(config.provider, protocol) };
|
|
905
979
|
});
|
|
906
980
|
if (filtered.length === 0) {
|
|
907
981
|
throw new Error(
|
|
908
|
-
`dsh-cliproxyapi: none of the selected models matched the ${
|
|
982
|
+
`dsh-cliproxyapi: none of the selected models matched the ${chatOrdered.length} chat model(s) offered by ${discovery.rootURL}. Select models in Settings → CLIProxyAPI → Models.`
|
|
909
983
|
);
|
|
910
984
|
}
|
|
911
985
|
const auth = hasApiKey
|
|
@@ -981,6 +1055,7 @@ function formatModelView(m, selectedSet, config) {
|
|
|
981
1055
|
const effective = applyModelOverride(m, override);
|
|
982
1056
|
const protocol = config === undefined ? nativeProtocol(m) : modelProtocol(m, config);
|
|
983
1057
|
return {
|
|
1058
|
+
kind: m.kind ?? m.detectedKind ?? "chat",
|
|
984
1059
|
protocol,
|
|
985
1060
|
nativeProtocol: nativeProtocol(m),
|
|
986
1061
|
...(m.ownedBy === undefined ? {} : { ownedBy: m.ownedBy }),
|
|
@@ -1013,6 +1088,12 @@ export class CpaRuntime {
|
|
|
1013
1088
|
this.controller = new AbortController();
|
|
1014
1089
|
this.allDiscoveredModels = [];
|
|
1015
1090
|
this.syncedModels = [];
|
|
1091
|
+
// Whether the last successful discovery could read the plain `/v1/models`
|
|
1092
|
+
// listing, which carries models the enriched catalog omits (image models
|
|
1093
|
+
// for some servers). A failed read is retried shortly, not in five minutes.
|
|
1094
|
+
this.listingOk = true;
|
|
1095
|
+
this.listingRetries = 0;
|
|
1096
|
+
this.listingRetryTimer = undefined;
|
|
1016
1097
|
this.currentStatus = {
|
|
1017
1098
|
state: "idle",
|
|
1018
1099
|
modelCount: 0,
|
|
@@ -1020,17 +1101,36 @@ export class CpaRuntime {
|
|
|
1020
1101
|
};
|
|
1021
1102
|
}
|
|
1022
1103
|
|
|
1104
|
+
/**
|
|
1105
|
+
* Models with their effective `kind` under the account's current kind
|
|
1106
|
+
* overrides (read fresh, so an override applies without a new probe).
|
|
1107
|
+
*/
|
|
1108
|
+
withKinds(models = this.allDiscoveredModels) {
|
|
1109
|
+
let overrides = {};
|
|
1110
|
+
try {
|
|
1111
|
+
overrides = this.options.kindOverrides?.() ?? {};
|
|
1112
|
+
} catch {}
|
|
1113
|
+
return models.map((m) => ({ ...m, kind: effectiveKind(m, overrides) }));
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
/** Discovered chat models (what the picker, recommendations and seeding use). */
|
|
1117
|
+
chatModels(models = this.allDiscoveredModels) {
|
|
1118
|
+
return this.withKinds(models).filter(isChatModel);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1023
1121
|
status() {
|
|
1024
1122
|
const syncedSet = new Set(this.syncedModels.map((m) => m.id));
|
|
1025
1123
|
const config = this.options.config();
|
|
1026
|
-
const
|
|
1124
|
+
const kinded = this.withKinds();
|
|
1125
|
+
const byId = new Map(kinded.map((m) => [m.id, m]));
|
|
1027
1126
|
return {
|
|
1028
1127
|
...this.currentStatus,
|
|
1128
|
+
listingOk: this.listingOk,
|
|
1029
1129
|
models: this.syncedModels.map((m) => ({
|
|
1030
1130
|
...formatModelView(byId.get(m.id) ?? m, syncedSet, config),
|
|
1031
1131
|
route: m.route,
|
|
1032
1132
|
})),
|
|
1033
|
-
allModels:
|
|
1133
|
+
allModels: kinded.map((m) => formatModelView(m, syncedSet, config)),
|
|
1034
1134
|
};
|
|
1035
1135
|
}
|
|
1036
1136
|
|
|
@@ -1054,6 +1154,7 @@ export class CpaRuntime {
|
|
|
1054
1154
|
dispose() {
|
|
1055
1155
|
this.controller.abort(new Error("dsh-cliproxyapi disposed"));
|
|
1056
1156
|
if (this.hmrRetryTimer !== undefined) clearTimeout(this.hmrRetryTimer);
|
|
1157
|
+
if (this.listingRetryTimer !== undefined) clearTimeout(this.listingRetryTimer);
|
|
1057
1158
|
}
|
|
1058
1159
|
|
|
1059
1160
|
/** Stop syncing and remove every route this account wrote. */
|
|
@@ -1178,6 +1279,28 @@ export class CpaRuntime {
|
|
|
1178
1279
|
}, 3000 * this.hmrRetries);
|
|
1179
1280
|
}
|
|
1180
1281
|
|
|
1282
|
+
/**
|
|
1283
|
+
* A failed plain listing must not leave image models hidden until the next
|
|
1284
|
+
* five-minute refresh. Re-read soon, at most twice per failure streak.
|
|
1285
|
+
*/
|
|
1286
|
+
scheduleListingRetry() {
|
|
1287
|
+
if (this.listingOk !== false) {
|
|
1288
|
+
this.listingRetries = 0;
|
|
1289
|
+
if (this.listingRetryTimer !== undefined) {
|
|
1290
|
+
clearTimeout(this.listingRetryTimer);
|
|
1291
|
+
this.listingRetryTimer = undefined;
|
|
1292
|
+
}
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
if (this.listingRetries >= 2 || this.controller.signal.aborted || this.listingRetryTimer !== undefined) return;
|
|
1296
|
+
this.listingRetries += 1;
|
|
1297
|
+
const schedule = this.options.setTimeout ?? setTimeout;
|
|
1298
|
+
this.listingRetryTimer = schedule(() => {
|
|
1299
|
+
this.listingRetryTimer = undefined;
|
|
1300
|
+
if (!this.controller.signal.aborted) void this.refresh();
|
|
1301
|
+
}, 4000);
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1181
1304
|
async syncOnce() {
|
|
1182
1305
|
const config = this.options.config();
|
|
1183
1306
|
const probe = this.probeRequested === true;
|
|
@@ -1204,7 +1327,8 @@ export class CpaRuntime {
|
|
|
1204
1327
|
try {
|
|
1205
1328
|
const { result } = await this.discover(config);
|
|
1206
1329
|
this.allDiscoveredModels = result.models;
|
|
1207
|
-
this.
|
|
1330
|
+
this.listingOk = result.listingOk !== false;
|
|
1331
|
+
this.seedSelection(config, this.chatModels(result.models));
|
|
1208
1332
|
endpoint = result.rootURL;
|
|
1209
1333
|
} catch (error) {
|
|
1210
1334
|
if (this.controller.signal.aborted) throw error;
|
|
@@ -1230,7 +1354,9 @@ export class CpaRuntime {
|
|
|
1230
1354
|
const { result, apiKey, effectiveApiKeyEnv } = await this.discover(config);
|
|
1231
1355
|
// Keep the catalog visible in the panel even if the selection matches nothing.
|
|
1232
1356
|
this.allDiscoveredModels = result.models;
|
|
1233
|
-
|
|
1357
|
+
this.listingOk = result.listingOk !== false;
|
|
1358
|
+
this.scheduleListingRetry();
|
|
1359
|
+
const seeded = this.seedSelection(config, this.chatModels(result.models));
|
|
1234
1360
|
|
|
1235
1361
|
const effectiveConfig = {
|
|
1236
1362
|
...config,
|
|
@@ -1238,7 +1364,11 @@ export class CpaRuntime {
|
|
|
1238
1364
|
apiKeyEnv: effectiveApiKeyEnv,
|
|
1239
1365
|
};
|
|
1240
1366
|
|
|
1241
|
-
const built = buildPiAiRoutes(
|
|
1367
|
+
const built = buildPiAiRoutes(
|
|
1368
|
+
effectiveConfig,
|
|
1369
|
+
{ ...result, models: this.withKinds(result.models) },
|
|
1370
|
+
apiKey !== undefined
|
|
1371
|
+
);
|
|
1242
1372
|
this.syncedModels = built.orderedModels;
|
|
1243
1373
|
|
|
1244
1374
|
// Each managed route keeps unrelated fields a user set by hand (retry
|
|
@@ -1379,6 +1509,8 @@ export function createCpaController(deps) {
|
|
|
1379
1509
|
resolveApiKey: deps.resolveApiKey,
|
|
1380
1510
|
readExistingPiAiRoute: deps.readExistingPiAiRoute,
|
|
1381
1511
|
readDefaultModel: deps.readDefaultModel,
|
|
1512
|
+
// User reclassifications (chat / image / video) from media-settings.json.
|
|
1513
|
+
kindOverrides: () => deps.mediaKindOverrides?.(id) ?? {},
|
|
1382
1514
|
// Persist a first-run selection unless one was saved in the meantime.
|
|
1383
1515
|
seedSelection: (ids) => {
|
|
1384
1516
|
const state = deps.readState();
|
|
@@ -1470,7 +1602,7 @@ export function createCpaController(deps) {
|
|
|
1470
1602
|
source: keyInfo.source ?? null,
|
|
1471
1603
|
},
|
|
1472
1604
|
// One model per family from this account's catalog (the panel's reset button).
|
|
1473
|
-
recommendedModels: recommendModels(runtime.
|
|
1605
|
+
recommendedModels: recommendModels(runtime.chatModels()),
|
|
1474
1606
|
status: st,
|
|
1475
1607
|
};
|
|
1476
1608
|
};
|
|
@@ -1589,6 +1721,34 @@ export function createCpaController(deps) {
|
|
|
1589
1721
|
return statusPayload({ removedId: account.id });
|
|
1590
1722
|
},
|
|
1591
1723
|
|
|
1724
|
+
/**
|
|
1725
|
+
* Probe a CPA address with unsaved form values, without saving or writing
|
|
1726
|
+
* routes. A blank `apiKey` uses the account's stored key; a blank
|
|
1727
|
+
* `baseURL` follows the usual fallbacks. Never returns the key.
|
|
1728
|
+
*/
|
|
1729
|
+
async testConnection(ref, input = {}) {
|
|
1730
|
+
const state = deps.readState();
|
|
1731
|
+
const account = ref === undefined || ref === null || String(ref).trim() === "" ? undefined : findAccount(state, ref);
|
|
1732
|
+
const config = account === undefined ? undefined : accountConfig(state, account.id);
|
|
1733
|
+
const draftKey = typeof input.apiKey === "string" ? input.apiKey.trim() : "";
|
|
1734
|
+
const apiKey = draftKey !== "" ? draftKey : config === undefined ? undefined : await deps.resolveApiKey(effectiveKeyEnv(config));
|
|
1735
|
+
const baseURL = typeof input.baseURL === "string" ? input.baseURL.trim() : "";
|
|
1736
|
+
try {
|
|
1737
|
+
const result = await discoverCpa({
|
|
1738
|
+
baseURL,
|
|
1739
|
+
autoDiscover: baseURL === "",
|
|
1740
|
+
fallbackBaseURL: config === undefined ? undefined : deps.readExistingPiAiRoute?.(config.provider)?.baseURL,
|
|
1741
|
+
environmentBaseURL: deps.environmentBaseURL?.(),
|
|
1742
|
+
apiKey,
|
|
1743
|
+
timeoutMs: state.probeTimeoutMs,
|
|
1744
|
+
...(deps.fetch === undefined ? {} : { fetch: deps.fetch }),
|
|
1745
|
+
});
|
|
1746
|
+
return { ok: true, test: { ok: true, endpoint: result.rootURL, modelCount: result.models.length } };
|
|
1747
|
+
} catch (error) {
|
|
1748
|
+
return { ok: true, test: { ok: false, error: error instanceof Error ? error.message : String(error) } };
|
|
1749
|
+
}
|
|
1750
|
+
},
|
|
1751
|
+
|
|
1592
1752
|
/** Accounts whose credential reference is `ref`. */
|
|
1593
1753
|
accountsUsingKey(ref) {
|
|
1594
1754
|
const state = deps.readState();
|
|
@@ -1600,6 +1760,112 @@ export function createCpaController(deps) {
|
|
|
1600
1760
|
|
|
1601
1761
|
refreshAccount,
|
|
1602
1762
|
|
|
1763
|
+
/**
|
|
1764
|
+
* Account facts the quota engine needs: the CPA address it talks to and
|
|
1765
|
+
* the DSH route keys whose models belong to this account. A conflicting
|
|
1766
|
+
* account owns no routes, so none of its models map to quota.
|
|
1767
|
+
*/
|
|
1768
|
+
quotaAccounts() {
|
|
1769
|
+
const state = deps.readState();
|
|
1770
|
+
const conflicts = accountConflicts(state.accounts);
|
|
1771
|
+
return state.accounts.map((account) => {
|
|
1772
|
+
const config = accountConfig(state, account.id);
|
|
1773
|
+
const runtime = runtimes.get(account.id);
|
|
1774
|
+
const st = runtime?.currentStatus ?? {};
|
|
1775
|
+
const existingRoute = deps.readExistingPiAiRoute?.(config.provider);
|
|
1776
|
+
const owners = new Map((runtime?.allDiscoveredModels ?? []).map((m) => [m.id, m.ownedBy]));
|
|
1777
|
+
return {
|
|
1778
|
+
id: account.id,
|
|
1779
|
+
displayName: config.displayName,
|
|
1780
|
+
provider: config.provider,
|
|
1781
|
+
enabled: config.enabled === true,
|
|
1782
|
+
baseURL: config.baseURL || "",
|
|
1783
|
+
endpoint: st.endpoint || existingRoute?.baseURL || "",
|
|
1784
|
+
routes: conflicts.has(account.id) ? [] : managedRouteKeys(config.provider),
|
|
1785
|
+
// Models this account currently puts in the DSH picker.
|
|
1786
|
+
pickerModels: (runtime?.syncedModels ?? []).map((m) => ({
|
|
1787
|
+
id: m.id,
|
|
1788
|
+
route: m.route,
|
|
1789
|
+
...(typeof owners.get(m.id) === "string" ? { ownedBy: owners.get(m.id) } : {}),
|
|
1790
|
+
})),
|
|
1791
|
+
};
|
|
1792
|
+
});
|
|
1793
|
+
},
|
|
1794
|
+
|
|
1795
|
+
/**
|
|
1796
|
+
* Media models per account, from the last catalog each runtime saw, with
|
|
1797
|
+
* their effective and detected kind. `chatModelIds` lets the panel mark a
|
|
1798
|
+
* chat model as media by hand. No network access.
|
|
1799
|
+
*/
|
|
1800
|
+
mediaAccounts() {
|
|
1801
|
+
const state = deps.readState();
|
|
1802
|
+
return state.accounts.map((account) => {
|
|
1803
|
+
const config = accountConfig(state, account.id);
|
|
1804
|
+
const runtime = ensureRuntime(account.id);
|
|
1805
|
+
const st = runtime.currentStatus ?? {};
|
|
1806
|
+
const kinded = runtime.withKinds();
|
|
1807
|
+
return {
|
|
1808
|
+
id: account.id,
|
|
1809
|
+
displayName: config.displayName,
|
|
1810
|
+
enabled: config.enabled === true,
|
|
1811
|
+
state: config.enabled === true ? st.state ?? "idle" : "disabled",
|
|
1812
|
+
endpoint: st.endpoint || "",
|
|
1813
|
+
discovered: kinded.length,
|
|
1814
|
+
listingOk: runtime.listingOk !== false,
|
|
1815
|
+
catalogIncomplete: runtime.listingOk === false,
|
|
1816
|
+
models: kinded
|
|
1817
|
+
.filter((m) => !isChatModel(m))
|
|
1818
|
+
.map((m) => ({
|
|
1819
|
+
id: m.id,
|
|
1820
|
+
...(m.name === undefined ? {} : { name: m.name }),
|
|
1821
|
+
kind: m.kind,
|
|
1822
|
+
detectedKind: m.detectedKind ?? "chat",
|
|
1823
|
+
...(mediaAliasTarget(m.id) === undefined ? {} : { aliasOf: mediaAliasTarget(m.id) }),
|
|
1824
|
+
})),
|
|
1825
|
+
chatModelIds: kinded.filter(isChatModel).map((m) => m.id),
|
|
1826
|
+
};
|
|
1827
|
+
});
|
|
1828
|
+
},
|
|
1829
|
+
|
|
1830
|
+
/**
|
|
1831
|
+
* Read-only media diagnostic for one account (see diagnoseMedia in
|
|
1832
|
+
* lib/media.js). Uses the account's key on the host; never returns it.
|
|
1833
|
+
*/
|
|
1834
|
+
/**
|
|
1835
|
+
* The CPA root and key a media request of this account uses: the address
|
|
1836
|
+
* the last sync connected to, else the configured or route address.
|
|
1837
|
+
* Host-only; the key is never returned to a page.
|
|
1838
|
+
* @throws MediaError `no-endpoint`.
|
|
1839
|
+
*/
|
|
1840
|
+
async mediaEndpoint(accountId) {
|
|
1841
|
+
const state = deps.readState();
|
|
1842
|
+
const account = state.accounts.find((a) => a.id === accountId);
|
|
1843
|
+
if (account === undefined) throw new MediaError("invalid-account");
|
|
1844
|
+
const config = accountConfig(state, account.id);
|
|
1845
|
+
const runtime = ensureRuntime(account.id);
|
|
1846
|
+
const known = runtime.currentStatus?.endpoint || config.baseURL || deps.readExistingPiAiRoute?.(config.provider)?.baseURL || "";
|
|
1847
|
+
let rootURL;
|
|
1848
|
+
try {
|
|
1849
|
+
rootURL = normalizeCpaRoot(known);
|
|
1850
|
+
} catch {
|
|
1851
|
+
throw new MediaError("no-endpoint");
|
|
1852
|
+
}
|
|
1853
|
+
return { rootURL, apiKey: await deps.resolveApiKey(effectiveKeyEnv(config)), probeTimeoutMs: state.probeTimeoutMs };
|
|
1854
|
+
},
|
|
1855
|
+
|
|
1856
|
+
async diagnoseMedia(ref) {
|
|
1857
|
+
const state = deps.readState();
|
|
1858
|
+
const account = findAccount(state, ref);
|
|
1859
|
+
const { rootURL, apiKey } = await this.mediaEndpoint(account.id);
|
|
1860
|
+
const diagnostic = await diagnoseMedia({
|
|
1861
|
+
rootURL,
|
|
1862
|
+
apiKey,
|
|
1863
|
+
timeoutMs: Math.max(state.probeTimeoutMs, 8000),
|
|
1864
|
+
...(deps.fetch === undefined ? {} : { fetch: deps.fetch }),
|
|
1865
|
+
});
|
|
1866
|
+
return { accountId: account.id, keyConfigured: apiKey !== undefined, ...diagnostic };
|
|
1867
|
+
},
|
|
1868
|
+
|
|
1603
1869
|
dispose() {
|
|
1604
1870
|
for (const runtime of runtimes.values()) runtime.dispose();
|
|
1605
1871
|
runtimes.clear();
|
|
@@ -1617,10 +1883,22 @@ export const ROUTES = {
|
|
|
1617
1883
|
save: "/api/dsh-cliproxyapi/save",
|
|
1618
1884
|
addAccount: "/api/dsh-cliproxyapi/accounts/add",
|
|
1619
1885
|
removeAccount: "/api/dsh-cliproxyapi/accounts/remove",
|
|
1886
|
+
testAccount: "/api/dsh-cliproxyapi/accounts/test",
|
|
1620
1887
|
updateStatus: "/api/dsh-cliproxyapi/updates/status",
|
|
1621
1888
|
updateCheck: "/api/dsh-cliproxyapi/updates/check",
|
|
1622
1889
|
updateSave: "/api/dsh-cliproxyapi/updates/save",
|
|
1623
1890
|
updateDownload: "/api/dsh-cliproxyapi/updates/download",
|
|
1891
|
+
quotaStatus: "/api/dsh-cliproxyapi/quota/status",
|
|
1892
|
+
quotaRefresh: "/api/dsh-cliproxyapi/quota/refresh",
|
|
1893
|
+
quotaSave: "/api/dsh-cliproxyapi/quota/save",
|
|
1894
|
+
mediaStatus: "/api/dsh-cliproxyapi/media/status",
|
|
1895
|
+
mediaSave: "/api/dsh-cliproxyapi/media/save",
|
|
1896
|
+
mediaDiagnose: "/api/dsh-cliproxyapi/media/diagnose",
|
|
1897
|
+
mediaSession: "/api/dsh-cliproxyapi/media/session",
|
|
1898
|
+
mediaSessionSave: "/api/dsh-cliproxyapi/media/session/save",
|
|
1899
|
+
mediaFile: "/api/dsh-cliproxyapi/media/file",
|
|
1900
|
+
mediaCommand: "/api/dsh-cliproxyapi/media/command",
|
|
1901
|
+
mediaConfirm: "/api/dsh-cliproxyapi/media/confirm",
|
|
1624
1902
|
};
|
|
1625
1903
|
|
|
1626
1904
|
function sendJson(res, status, payload) {
|
|
@@ -1835,6 +2113,11 @@ function renderAccountStatus(account) {
|
|
|
1835
2113
|
.map((m) => m.id)
|
|
1836
2114
|
.join(", ")}${status.models.length > 12 ? ` ... (+${status.models.length - 12} more)` : ""}`
|
|
1837
2115
|
: undefined;
|
|
2116
|
+
const media = (status.allModels ?? []).filter((m) => m.kind === "image" || m.kind === "video");
|
|
2117
|
+
const mediaLine =
|
|
2118
|
+
media.length > 0
|
|
2119
|
+
? `Media Models (not in the picker): ${media.filter((m) => m.kind === "image").length} image, ${media.filter((m) => m.kind === "video").length} video`
|
|
2120
|
+
: undefined;
|
|
1838
2121
|
const counts = status.protocolCounts ?? {};
|
|
1839
2122
|
const protocolLine =
|
|
1840
2123
|
cfg.protocolMode === "fixed"
|
|
@@ -1855,16 +2138,44 @@ function renderAccountStatus(account) {
|
|
|
1855
2138
|
status.totalDiscovered ? ` (of ${status.totalDiscovered} discovered)` : ""
|
|
1856
2139
|
}`,
|
|
1857
2140
|
...(modelListPreview ? [modelListPreview] : []),
|
|
2141
|
+
...(mediaLine ? [mediaLine] : []),
|
|
1858
2142
|
...(status.refreshedAt ? [`Last Refreshed: ${status.refreshedAt}`] : []),
|
|
1859
2143
|
...(status.error ? [`Error: ${status.error}`] : []),
|
|
1860
2144
|
];
|
|
1861
2145
|
return lines.join("\n");
|
|
1862
2146
|
}
|
|
1863
2147
|
|
|
2148
|
+
/** One line per account: management key, quota state and what the picker models map to. Never includes keys. */
|
|
2149
|
+
export function renderQuotaText(snapshot, onlyAccountId) {
|
|
2150
|
+
const accounts = (snapshot?.accounts ?? []).filter((a) => onlyAccountId === undefined || a.accountId === onlyAccountId);
|
|
2151
|
+
if (accounts.length === 0) return "";
|
|
2152
|
+
const lines = ["Quota (CPA management API):"];
|
|
2153
|
+
for (const a of accounts) {
|
|
2154
|
+
if (!a.keyConfigured) {
|
|
2155
|
+
lines.push(`- ${a.displayName} [${a.accountId}]: no management key (${a.keyRef})`);
|
|
2156
|
+
continue;
|
|
2157
|
+
}
|
|
2158
|
+
const creds = Array.isArray(a.credentials) ? a.credentials : [];
|
|
2159
|
+
const state = a.state + (a.errorCode ? ` (${a.errorCode}${a.errorStatus ? ` ${a.errorStatus}` : ""})` : "") + (a.refreshing ? ", refreshing" : "");
|
|
2160
|
+
lines.push(`- ${a.displayName} [${a.accountId}]: ${state}; ${creds.length} credential(s)${a.fetchedAt ? `; fetched ${a.fetchedAt}` : ""}; routes ${(a.routes ?? []).join(", ") || "none"}`);
|
|
2161
|
+
for (const m of a.pickerModels ?? []) {
|
|
2162
|
+
const ids = a.modelIndex?.[m.id] ?? [];
|
|
2163
|
+
const names = ids.map((id) => {
|
|
2164
|
+
const c = creds.find((x) => x.id === id);
|
|
2165
|
+
if (!c) return id;
|
|
2166
|
+
const pct = (c.windows ?? []).map((w) => w.remainingPercent).filter((v) => typeof v === "number");
|
|
2167
|
+
return `${c.provider}:${c.label || c.id}${pct.length ? ` ${Math.round(Math.min(...pct))}%` : ""}${c.status === "active" ? "" : ` (${c.status})`}`;
|
|
2168
|
+
});
|
|
2169
|
+
lines.push(` ${m.route}/${m.id} -> ${names.length ? names.join(", ") : "no credential"} [${a.modelMatch?.[m.id] ?? "none"}]`);
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
return lines.join("\n");
|
|
2173
|
+
}
|
|
2174
|
+
|
|
1864
2175
|
export function renderStatusText(payload, onlyAccountId) {
|
|
1865
2176
|
const accounts = payload.accounts ?? [];
|
|
1866
2177
|
if (accounts.length === 0) {
|
|
1867
|
-
return "No CLIProxyAPI account is configured. Add one
|
|
2178
|
+
return "No CLIProxyAPI account is configured. Add one in Settings → CLIProxyAPI.";
|
|
1868
2179
|
}
|
|
1869
2180
|
const shown =
|
|
1870
2181
|
onlyAccountId === undefined ? accounts : accounts.filter((a) => a.id === onlyAccountId);
|
|
@@ -1873,22 +2184,7 @@ export function renderStatusText(payload, onlyAccountId) {
|
|
|
1873
2184
|
return [header, ...shown.map(renderAccountStatus)].join("\n\n");
|
|
1874
2185
|
}
|
|
1875
2186
|
|
|
1876
|
-
|
|
1877
|
-
const st = account.status;
|
|
1878
|
-
return {
|
|
1879
|
-
id: account.id,
|
|
1880
|
-
displayName: account.config.displayName,
|
|
1881
|
-
provider: account.config.provider,
|
|
1882
|
-
enabled: account.config.enabled,
|
|
1883
|
-
state: st.state,
|
|
1884
|
-
...(st.endpoint ? { endpoint: st.endpoint } : {}),
|
|
1885
|
-
modelCount: st.modelCount ?? 0,
|
|
1886
|
-
totalDiscovered: st.totalDiscovered ?? 0,
|
|
1887
|
-
...(st.error ? { error: st.error } : {}),
|
|
1888
|
-
};
|
|
1889
|
-
}
|
|
1890
|
-
|
|
1891
|
-
/** Account fields the tool and the save route accept, besides key handling. */
|
|
2187
|
+
/** Account fields the save and add routes accept, besides key handling. */
|
|
1892
2188
|
function accountPatchFromInput(input) {
|
|
1893
2189
|
const patch = {};
|
|
1894
2190
|
for (const field of ["baseURL", "provider", "displayName", "protocol", "protocolMode", "includePrefix", "excludePrefix", "apiKeyEnv"]) {
|
|
@@ -2003,6 +2299,7 @@ function applyImpl(ctx, config) {
|
|
|
2003
2299
|
writeSettings: (preferences) => saveUpdateSettings(updateSettingsPath, preferences),
|
|
2004
2300
|
storePackage: (version, bytes) => storeUpdatePackage(join(updateDirectory, "updates"), version, bytes),
|
|
2005
2301
|
onSettingsChanged: scheduleUpdates,
|
|
2302
|
+
logger: ctx.logger,
|
|
2006
2303
|
});
|
|
2007
2304
|
ctx.effect(() => {
|
|
2008
2305
|
scheduleUpdates(updater.status());
|
|
@@ -2050,15 +2347,33 @@ function applyImpl(ctx, config) {
|
|
|
2050
2347
|
if (timer !== undefined) clearInterval(timer);
|
|
2051
2348
|
timer = outsideHmrTransaction(ctx, () =>
|
|
2052
2349
|
setInterval(() => {
|
|
2053
|
-
void controller.refreshAll();
|
|
2350
|
+
void controller.refreshAll().finally(() => syncAgentTool());
|
|
2054
2351
|
}, readState().refreshIntervalSeconds * 1000)
|
|
2055
2352
|
);
|
|
2056
2353
|
};
|
|
2057
2354
|
|
|
2355
|
+
// Media preferences (lib/media.js) live in their own file next to the
|
|
2356
|
+
// account state, never inside it, so no state migration is involved.
|
|
2357
|
+
const mediaSettingsPath = join(updateDirectory, "media-settings.json");
|
|
2358
|
+
let mediaWarned = false;
|
|
2359
|
+
const readMedia = () => {
|
|
2360
|
+
const loaded = loadMediaSettings(mediaSettingsPath);
|
|
2361
|
+
if (loaded.error === undefined) {
|
|
2362
|
+
mediaWarned = false;
|
|
2363
|
+
} else if (!mediaWarned) {
|
|
2364
|
+
mediaWarned = true;
|
|
2365
|
+
ctx.logger?.warn?.(
|
|
2366
|
+
`dsh-cliproxyapi: ${mediaSettingsPath} is unreadable; default media settings apply until it is saved again (the damaged file is then kept as *.corrupt-*.json)`
|
|
2367
|
+
);
|
|
2368
|
+
}
|
|
2369
|
+
return loaded;
|
|
2370
|
+
};
|
|
2371
|
+
|
|
2058
2372
|
const controller = createCpaController({
|
|
2059
2373
|
settings: ctx.settings,
|
|
2060
2374
|
readState,
|
|
2061
2375
|
writeState,
|
|
2376
|
+
mediaKindOverrides: (id) => accountKindOverrides(readMedia().settings, id),
|
|
2062
2377
|
resolveApiKey,
|
|
2063
2378
|
describeApiKey,
|
|
2064
2379
|
setCredential: async (ref, value) => {
|
|
@@ -2079,11 +2394,1585 @@ function applyImpl(ctx, config) {
|
|
|
2079
2394
|
onGlobalsChanged: () => reschedule(),
|
|
2080
2395
|
});
|
|
2081
2396
|
|
|
2397
|
+
// Quota ("额度" tab + composer badge): management keys live in the
|
|
2398
|
+
// credentials service under their own refs, never in account state.
|
|
2399
|
+
const quota = createQuotaController({
|
|
2400
|
+
listAccounts: () => controller.quotaAccounts(),
|
|
2401
|
+
resolveKey: (ref) => resolveApiKey(ref),
|
|
2402
|
+
describeKey: (ref) => describeApiKey(ref),
|
|
2403
|
+
setKey: async (ref, value) => {
|
|
2404
|
+
const credentials = ctx.get("credentials");
|
|
2405
|
+
if (credentials === undefined) {
|
|
2406
|
+
throw new Error("Credentials service is not available on this host");
|
|
2407
|
+
}
|
|
2408
|
+
await credentials.set(ref, value);
|
|
2409
|
+
},
|
|
2410
|
+
unsetKey: async (ref) => {
|
|
2411
|
+
await ctx.get("credentials")?.unset(ref);
|
|
2412
|
+
},
|
|
2413
|
+
logger: ctx.logger,
|
|
2414
|
+
});
|
|
2415
|
+
ctx.effect(() => () => quota.dispose(), "cpa: quota");
|
|
2416
|
+
|
|
2417
|
+
/** Media page payload: preferences (without per-account maps) and each account's media models. */
|
|
2418
|
+
const mediaStatus = () => {
|
|
2419
|
+
const { settings, error } = readMedia();
|
|
2420
|
+
const { accounts: perAccount, ...global } = settings;
|
|
2421
|
+
return {
|
|
2422
|
+
settings: global,
|
|
2423
|
+
...(error === undefined ? {} : { settingsError: error }),
|
|
2424
|
+
accounts: controller.mediaAccounts().map((account) => {
|
|
2425
|
+
const overrides = accountKindOverrides(settings, account.id);
|
|
2426
|
+
return {
|
|
2427
|
+
...account,
|
|
2428
|
+
disabledModels: perAccount[account.id]?.disabledModels ?? [],
|
|
2429
|
+
kindOverrides: overrides,
|
|
2430
|
+
models: account.models.map((m) => ({
|
|
2431
|
+
...m,
|
|
2432
|
+
enabled: mediaModelEnabled(settings, account.id, m.id),
|
|
2433
|
+
overridden: overrides[m.id] !== undefined,
|
|
2434
|
+
})),
|
|
2435
|
+
};
|
|
2436
|
+
}),
|
|
2437
|
+
};
|
|
2438
|
+
};
|
|
2439
|
+
|
|
2440
|
+
const saveMedia = async (body) => {
|
|
2441
|
+
const accountId = body?.account?.accountId;
|
|
2442
|
+
if (accountId !== undefined && !controller.mediaAccounts().some((a) => a.id === accountId)) {
|
|
2443
|
+
throw new MediaError("invalid-account");
|
|
2444
|
+
}
|
|
2445
|
+
const { settings, kindsChanged } = applyMediaPatch(readMedia().settings, body);
|
|
2446
|
+
saveMediaSettings(mediaSettingsPath, settings);
|
|
2447
|
+
// A reclassified model enters or leaves the chat routes on the next sync.
|
|
2448
|
+
if (kindsChanged === undefined) return { ok: true, media: mediaStatus() };
|
|
2449
|
+
await controller.refreshAccount(kindsChanged);
|
|
2450
|
+
return { ok: true, media: mediaStatus(), status: await controller.statusPayload() };
|
|
2451
|
+
};
|
|
2452
|
+
|
|
2453
|
+
// Per-session model picks and recent outputs, in their own file.
|
|
2454
|
+
const mediaSessions = createMediaSessionStore(join(updateDirectory, "media-sessions.json"));
|
|
2455
|
+
|
|
2456
|
+
const mediaChoices = () => {
|
|
2457
|
+
const { settings } = readMedia();
|
|
2458
|
+
const accounts = controller.mediaAccounts();
|
|
2459
|
+
return {
|
|
2460
|
+
settings,
|
|
2461
|
+
accounts,
|
|
2462
|
+
options: { image: mediaOptions(accounts, settings, "image"), video: mediaOptions(accounts, settings, "video") },
|
|
2463
|
+
};
|
|
2464
|
+
};
|
|
2465
|
+
|
|
2466
|
+
/**
|
|
2467
|
+
* Choices for a generation command. When a needed kind is not offered and an
|
|
2468
|
+
* enabled account either has not read its catalog yet (just after a start
|
|
2469
|
+
* or reload) or reported a failed plain listing, sync once and look again
|
|
2470
|
+
* instead of failing on a timing race. An incomplete-catalog retry is
|
|
2471
|
+
* limited to once per 30 s so a permanently missing listing is not fetched
|
|
2472
|
+
* on every request.
|
|
2473
|
+
*/
|
|
2474
|
+
let catalogRetryAt = 0;
|
|
2475
|
+
const mediaChoicesFor = async (kind) => {
|
|
2476
|
+
let choices = mediaChoices();
|
|
2477
|
+
const missing = (k) => choices.options[k].length === 0;
|
|
2478
|
+
const someMissing = kind === "any" ? missing("image") || missing("video") : missing(kind);
|
|
2479
|
+
const cold = choices.accounts.some((a) => a.enabled && a.discovered === 0);
|
|
2480
|
+
const incomplete = choices.accounts.some((a) => a.enabled && a.catalogIncomplete === true);
|
|
2481
|
+
const retryIncomplete = incomplete && Date.now() - catalogRetryAt >= 30_000;
|
|
2482
|
+
if (someMissing && (cold || retryIncomplete)) {
|
|
2483
|
+
catalogRetryAt = Date.now();
|
|
2484
|
+
await controller.refreshAll();
|
|
2485
|
+
choices = mediaChoices();
|
|
2486
|
+
}
|
|
2487
|
+
return choices;
|
|
2488
|
+
};
|
|
2489
|
+
|
|
2490
|
+
/** One line per account for an empty choice: state and catalog counts, never keys or URLs. */
|
|
2491
|
+
const mediaAccountsSummary = (accounts, kind) =>
|
|
2492
|
+
accounts
|
|
2493
|
+
.map((a) => `${a.displayName || a.id}: ${a.enabled ? a.state : "disabled"}, ${a.discovered} model(s), ${a.models.filter((m) => m.kind === kind).length} ${kind}`)
|
|
2494
|
+
.join("; ") || "no account";
|
|
2495
|
+
|
|
2496
|
+
/** What the composer button shows for one session (no network access). */
|
|
2497
|
+
const sessionView = (sessionId) => {
|
|
2498
|
+
const { settings, options, accounts } = mediaChoices();
|
|
2499
|
+
const entry = sessionId === undefined ? { recent: [] } : mediaSessions.get(sessionId);
|
|
2500
|
+
const current = {};
|
|
2501
|
+
for (const kind of ["image", "video"]) {
|
|
2502
|
+
try {
|
|
2503
|
+
const { option, source } = chooseMediaModel({
|
|
2504
|
+
options: options[kind],
|
|
2505
|
+
sessionRef: entry[kind],
|
|
2506
|
+
lastRef: settings.lastSelection[kind],
|
|
2507
|
+
});
|
|
2508
|
+
current[kind] = { ref: option.ref, source };
|
|
2509
|
+
} catch {
|
|
2510
|
+
current[kind] = null;
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
const policy = agentPolicy(settings, entry);
|
|
2514
|
+
return {
|
|
2515
|
+
sessionId: sessionId ?? null,
|
|
2516
|
+
options,
|
|
2517
|
+
current,
|
|
2518
|
+
// Compatibility composer fallback; native questions never enter this map.
|
|
2519
|
+
pendingConfirmations: sessionId === undefined ? [] : [...pendingConfirms.entries()]
|
|
2520
|
+
.filter(([, waiting]) => waiting.sessionId === sessionId)
|
|
2521
|
+
.map(([commandId, { kind, en, zh, expiresAt }]) => ({ commandId, kind, en, zh, expiresAt })),
|
|
2522
|
+
// True while an enabled account's last catalog read missed the plain
|
|
2523
|
+
// listing; the composer button retries instead of keeping an old,
|
|
2524
|
+
// image-less list.
|
|
2525
|
+
catalogIncomplete: accounts.some((a) => a.enabled === true && a.catalogIncomplete === true),
|
|
2526
|
+
// The agent tool's rules for this session (shown while the tool is switched on).
|
|
2527
|
+
agent: {
|
|
2528
|
+
enabled: settings.agentTool === true,
|
|
2529
|
+
imageConfirm: policy.imageConfirm,
|
|
2530
|
+
videoConfirm: policy.videoConfirm,
|
|
2531
|
+
videoBudget: policy.budget,
|
|
2532
|
+
used: policy.used,
|
|
2533
|
+
sources: policy.sources,
|
|
2534
|
+
global: { imageConfirm: settings.imageConfirm, videoConfirm: settings.videoConfirm, videoBudget: settings.videoBudget },
|
|
2535
|
+
},
|
|
2536
|
+
recent: entry.recent
|
|
2537
|
+
.filter((r) => existsSync(r.file))
|
|
2538
|
+
.slice(0, 12)
|
|
2539
|
+
.map((r) => ({ id: r.id, kind: r.kind, name: basename(r.file), model: r.model, createdAt: r.createdAt })),
|
|
2540
|
+
};
|
|
2541
|
+
};
|
|
2542
|
+
|
|
2543
|
+
/** Pick a model for one session (and remember it as the last pick); `ref: null` clears the session's pick. */
|
|
2544
|
+
const saveSessionPick = (body) => {
|
|
2545
|
+
// Agent-tool overrides for one session (no model pick in the same request).
|
|
2546
|
+
if (body?.kind === undefined && ["imageConfirm", "videoConfirm", "videoBudget"].some((k) => k in (body ?? {}))) {
|
|
2547
|
+
const sessionId = validSessionId(body.sessionId);
|
|
2548
|
+
if (sessionId === undefined) throw new MediaError("invalid-session");
|
|
2549
|
+
const { settings } = mediaChoices();
|
|
2550
|
+
mediaSessions.update(sessionId, (entry) => applySessionPolicy(entry, body, settings));
|
|
2551
|
+
return { ok: true, session: sessionView(sessionId) };
|
|
2552
|
+
}
|
|
2553
|
+
const kind = body?.kind;
|
|
2554
|
+
if (kind !== "image" && kind !== "video") throw new MediaError("invalid-request");
|
|
2555
|
+
const sessionId = body.sessionId === undefined || body.sessionId === null ? undefined : validSessionId(body.sessionId);
|
|
2556
|
+
if (body.sessionId !== undefined && body.sessionId !== null && sessionId === undefined) throw new MediaError("invalid-session");
|
|
2557
|
+
const ref = body.ref === null ? null : typeof body.ref === "string" ? body.ref : undefined;
|
|
2558
|
+
if (ref === undefined) throw new MediaError("invalid-model");
|
|
2559
|
+
const { settings, options } = mediaChoices();
|
|
2560
|
+
if (ref !== null && !options[kind].some((o) => o.ref === ref)) throw new MediaError("invalid-model");
|
|
2561
|
+
if (sessionId !== undefined) {
|
|
2562
|
+
mediaSessions.update(sessionId, (entry) => {
|
|
2563
|
+
if (ref === null) delete entry[kind];
|
|
2564
|
+
else entry[kind] = ref;
|
|
2565
|
+
});
|
|
2566
|
+
}
|
|
2567
|
+
if (ref !== null && settings.lastSelection[kind] !== ref) {
|
|
2568
|
+
saveMediaSettings(mediaSettingsPath, normalizeMediaSettings({ ...settings, lastSelection: { ...settings.lastSelection, [kind]: ref } }));
|
|
2569
|
+
}
|
|
2570
|
+
return { ok: true, session: sessionView(sessionId) };
|
|
2571
|
+
};
|
|
2572
|
+
|
|
2573
|
+
const MEDIA_FILE_TYPES = {
|
|
2574
|
+
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif",
|
|
2575
|
+
".mp4": "video/mp4", ".webm": "video/webm", ".mov": "video/quicktime",
|
|
2576
|
+
};
|
|
2577
|
+
|
|
2578
|
+
/**
|
|
2579
|
+
* One `Range: bytes=a-b` request against a file of `size` bytes: `[start, end]`,
|
|
2580
|
+
* `undefined` without a (usable single) range, `null` when unsatisfiable.
|
|
2581
|
+
*/
|
|
2582
|
+
const byteRange = (header, size) => {
|
|
2583
|
+
if (typeof header !== "string" || header === "") return undefined;
|
|
2584
|
+
const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
|
2585
|
+
if (match === null || (match[1] === "" && match[2] === "")) return undefined;
|
|
2586
|
+
let start;
|
|
2587
|
+
let end;
|
|
2588
|
+
if (match[1] === "") {
|
|
2589
|
+
const suffix = Number(match[2]);
|
|
2590
|
+
if (suffix === 0) return null;
|
|
2591
|
+
start = Math.max(0, size - suffix);
|
|
2592
|
+
end = size - 1;
|
|
2593
|
+
} else {
|
|
2594
|
+
start = Number(match[1]);
|
|
2595
|
+
end = match[2] === "" ? size - 1 : Math.min(Number(match[2]), size - 1);
|
|
2596
|
+
}
|
|
2597
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end || start >= size) return null;
|
|
2598
|
+
return [start, end];
|
|
2599
|
+
};
|
|
2600
|
+
// Request bodies carry references base64-encoded and upstreams cap the body
|
|
2601
|
+
// (xAI: "Failed to read request body: length limit exceeded"). Video
|
|
2602
|
+
// requests use the measured `videoImageBudget()`; image edits keep this
|
|
2603
|
+
// unmeasured share.
|
|
2604
|
+
const FIRST_FRAME_MAX_BYTES = 1_000_000;
|
|
2605
|
+
const REFERENCES_MAX_BYTES = 1_400_000;
|
|
2606
|
+
|
|
2607
|
+
/**
|
|
2608
|
+
* One attached image within `maxBytes`: as stored when it fits; a PNG is
|
|
2609
|
+
* re-encoded here (JPEG on white, long edge ≤ 1280); another format asks
|
|
2610
|
+
* DSH's attachment store for a smaller request version.
|
|
2611
|
+
* @throws MediaError `reference-too-large`.
|
|
2612
|
+
*/
|
|
2613
|
+
const readReference = async (store, block, maxBytes, signal) => {
|
|
2614
|
+
const stored = await store.readImage(block.attachment, signal);
|
|
2615
|
+
const image = { mediaType: stored.ref?.mediaType ?? block.attachment?.mediaType, data: stored.data };
|
|
2616
|
+
const fitted = fitReferenceImage(image, maxBytes);
|
|
2617
|
+
if (fitted !== undefined) return { mediaType: fitted.mediaType, data: fitted.data };
|
|
2618
|
+
const width = Number(stored.ref?.width ?? block.attachment?.width);
|
|
2619
|
+
const height = Number(stored.ref?.height ?? block.attachment?.height);
|
|
2620
|
+
if (typeof store.readImageRequest === "function" && width > 0 && height > 0) {
|
|
2621
|
+
const scale = Math.min(1, REFERENCE_MAX_EDGE / Math.max(width, height));
|
|
2622
|
+
try {
|
|
2623
|
+
const smaller = await store.readImageRequest(
|
|
2624
|
+
block.attachment,
|
|
2625
|
+
{ width: Math.max(1, Math.round(width * scale)), height: Math.max(1, Math.round(height * scale)), maxBytes },
|
|
2626
|
+
signal
|
|
2627
|
+
);
|
|
2628
|
+
if (smaller?.data?.byteLength <= maxBytes) return { mediaType: smaller.mediaType, data: smaller.data };
|
|
2629
|
+
} catch (error) {
|
|
2630
|
+
if (signal?.aborted) throw error;
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
throw new MediaError("reference-too-large", String(Math.round(maxBytes / 1000)));
|
|
2634
|
+
};
|
|
2635
|
+
|
|
2636
|
+
// #region Parameter interpretation (the session's chat model)
|
|
2637
|
+
const INTERPRET_TIMEOUT_MS = 30_000;
|
|
2638
|
+
const INTERPRET_MAX_TOKENS = 2000;
|
|
2639
|
+
|
|
2640
|
+
/** The provider / model the session's agent last used (else its options, else the default model). */
|
|
2641
|
+
const sessionChatTarget = (agent) => {
|
|
2642
|
+
const valid = (t) => (t && typeof t.provider === "string" && t.provider !== "" && typeof t.model === "string" && t.model !== "" ? { provider: t.provider, model: t.model } : undefined);
|
|
2643
|
+
try {
|
|
2644
|
+
const header = valid(agent?.session?.requestHeader?.()?.config);
|
|
2645
|
+
if (header !== undefined) return header;
|
|
2646
|
+
} catch {
|
|
2647
|
+
// fall through
|
|
2648
|
+
}
|
|
2649
|
+
return valid(agent?.options) ?? valid(ctx.get?.("agentDefaultModel")?.currentSelection?.());
|
|
2650
|
+
};
|
|
2651
|
+
|
|
2652
|
+
/**
|
|
2653
|
+
* Ask the session's chat model which parameters the prompt states.
|
|
2654
|
+
* Never throws: without an `llm` service or a model it returns undefined;
|
|
2655
|
+
* a failed call returns `{ failed: true, by }`.
|
|
2656
|
+
* @param fixed - fields the rule parser already set (they win).
|
|
2657
|
+
* @returns `{ by, values, prompt, items, failed? }` or undefined.
|
|
2658
|
+
*/
|
|
2659
|
+
const interpretParams = async ({ agent, command, text, images = 0, fixed = [], signal }) => {
|
|
2660
|
+
const llm = ctx.get?.("llm");
|
|
2661
|
+
const target = sessionChatTarget(agent);
|
|
2662
|
+
if (typeof llm?.stream !== "function" || target === undefined) return undefined;
|
|
2663
|
+
const by = `${target.provider}/${target.model}`;
|
|
2664
|
+
const request = interpretationRequest({ command, text, images, fixed });
|
|
2665
|
+
const timeout = AbortSignal.timeout(INTERPRET_TIMEOUT_MS);
|
|
2666
|
+
const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
|
|
2667
|
+
try {
|
|
2668
|
+
const parts = new Map();
|
|
2669
|
+
for await (const chunk of llm.stream({
|
|
2670
|
+
provider: target.provider,
|
|
2671
|
+
model: target.model,
|
|
2672
|
+
system: request.system,
|
|
2673
|
+
messages: [{ role: "user", content: [{ type: "text", text: request.user }] }],
|
|
2674
|
+
maxTokens: INTERPRET_MAX_TOKENS,
|
|
2675
|
+
...(agent?.session?.id === undefined ? {} : { sessionId: agent.session.id }),
|
|
2676
|
+
purpose: "cpa-media-params",
|
|
2677
|
+
signal: combined,
|
|
2678
|
+
})) {
|
|
2679
|
+
if (chunk?.type === "text-delta") parts.set(chunk.index, (parts.get(chunk.index) ?? "") + chunk.text);
|
|
2680
|
+
else if (chunk?.type === "block-end" && chunk.block?.type === "text") parts.set(chunk.index, chunk.block.text);
|
|
2681
|
+
}
|
|
2682
|
+
const answer = [...parts.values()].join("");
|
|
2683
|
+
const parsed = parseInterpretation(answer, { command, text, fields: request.fields, images });
|
|
2684
|
+
if (parsed === undefined) throw new Error("no JSON in the answer");
|
|
2685
|
+
return { by, ...parsed, items: interpretationLabels(parsed.values) };
|
|
2686
|
+
} catch (error) {
|
|
2687
|
+
if (signal?.aborted) throw signal.reason;
|
|
2688
|
+
ctx.logger?.warn?.(`cpa: parameter interpretation with ${by} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2689
|
+
return { by, failed: true, values: {}, prompt: text, items: [] };
|
|
2690
|
+
}
|
|
2691
|
+
};
|
|
2692
|
+
|
|
2693
|
+
/** The run-record field describing an interpretation. */
|
|
2694
|
+
const interpretedRecord = (result, original) =>
|
|
2695
|
+
result === undefined
|
|
2696
|
+
? {}
|
|
2697
|
+
: { interpreted: { by: result.by, items: result.items, ...(result.failed ? { failed: true } : {}), ...(result.prompt !== original ? { prompt: true } : {}) } };
|
|
2698
|
+
// #endregion
|
|
2699
|
+
|
|
2700
|
+
/** Plain-English host text for a media failure (slash command output and run records). */
|
|
2701
|
+
const mediaFailureText = (error, kind = "image") => {
|
|
2702
|
+
if (!(error instanceof MediaError)) return error instanceof Error ? error.message : String(error);
|
|
2703
|
+
const detail = error.detail;
|
|
2704
|
+
const video = kind === "video";
|
|
2705
|
+
const noun = video ? "video" : "image";
|
|
2706
|
+
switch (error.code) {
|
|
2707
|
+
case "missing-prompt": return video ? "add a prompt, for example: /cpa-video waves rolling onto a beach at sunset" : "add a prompt, for example: /cpa-image a red fox in the snow";
|
|
2708
|
+
case "invalid-option": return video
|
|
2709
|
+
? `unknown option ${detail}; use -m <model>, --aspect <ratio>, -s <1-15>, -r <480p|720p>`
|
|
2710
|
+
: `unknown option ${detail}; use -m <model>, --aspect <ratio>, -n <1-4>`;
|
|
2711
|
+
case "invalid-aspect": return `--aspect must be one of ${(video ? VIDEO_ASPECTS : IMAGE_ASPECTS).join(", ")}`;
|
|
2712
|
+
case "invalid-count": return "-n must be 1 to 4";
|
|
2713
|
+
case "invalid-video-seconds": return "-s / --seconds must be a whole number from 1 to 15";
|
|
2714
|
+
case "invalid-video-resolution": return "-r / --resolution must be 480p or 720p";
|
|
2715
|
+
case "no-media-model": return `no ${noun} model is available (${detail || "no account"}); check Settings → CLIProxyAPI → Generation, or run /cpa-refresh if a catalog has not been read yet`;
|
|
2716
|
+
case "choose-model": return `several ${noun} models are available (${detail}); pick one with the button next to the composer or pass -m <model>`;
|
|
2717
|
+
case "model-not-found": return `no enabled ${noun} model matches; available: ${detail || "none"}`;
|
|
2718
|
+
case "model-ambiguous": return `several ${noun} models match (${detail}); be more specific`;
|
|
2719
|
+
case "workspace-not-local": return `the session's working directory is not a local folder, so the ${noun} cannot be saved`;
|
|
2720
|
+
case "unsupported-attachment": return detail === "edit"
|
|
2721
|
+
? "attach one MP4 video as the source (images cannot be used here)"
|
|
2722
|
+
: video ? "only image attachments can be used for /cpa-video (first frame, references, first / last frames)" : "only image attachments can be used as references";
|
|
2723
|
+
case "too-many-references": return video ? `attach at most ${detail ?? "14"} images for this mode (more than 2 images are references)` : `attach at most ${MAX_REFERENCE_IMAGES} reference images`;
|
|
2724
|
+
case "frames-need-images": return "--ref / --last / --first-last need attached images: one for a reference or a last frame, two for first and last frame";
|
|
2725
|
+
case "needs-video-15": return "pinned first / last frames need grok-imagine-video-1.5; pick it with -m or the model button";
|
|
2726
|
+
case "invalid-extend-seconds": return "an extension adds 2 to 10 seconds (-s 2-10)";
|
|
2727
|
+
case "no-video-source": return "no source video: attach an MP4, or pass --video <path>, --url <public URL> or --from <command id>";
|
|
2728
|
+
case "video-source-expired": return "that video's file is gone and its short-lived xAI link has expired; attach the MP4 or pass --video <path>";
|
|
2729
|
+
case "one-video-source": return "use one source video: an attachment, --video, --url or --from";
|
|
2730
|
+
case "invalid-video-url": return "--url must be a public http(s) address without a user name or password";
|
|
2731
|
+
case "video-not-found": return `no video file at ${detail}`;
|
|
2732
|
+
case "video-not-mp4": return "the source video must be an MP4 file (H.264, H.265 or AV1)";
|
|
2733
|
+
case "source-too-large": return `the source video is ${detail} MiB; videos up to ${Math.round(VIDEO_UPLOAD_MAX_BYTES / 1024 / 1024)} MiB can be sent inline — shorten or compress it, or pass a public link with --url`;
|
|
2734
|
+
case "request-too-large": return video
|
|
2735
|
+
? `the request is too large for this server (${detail ? `${Math.round(Number(detail) / 1024)} KB` : "body limit"}); attach smaller or fewer images`
|
|
2736
|
+
: "the request is too large for this server";
|
|
2737
|
+
case "route-unsupported": return `this server's upstream does not offer ${detail === "extensions" ? "video extension" : detail === "edits" ? "video editing" : detail} (HTTP 405); CPA may pause the credential for this model for about a minute`;
|
|
2738
|
+
case "edit-source-too-long": return `xAI edits videos of at most ${EDIT_SOURCE_MAX_SECONDS} seconds (this one has ${detail} s); extend it instead`;
|
|
2739
|
+
case "extend-source-length": return `xAI extends videos of ${EXTEND_SOURCE_SECONDS.min} to ${EXTEND_SOURCE_SECONDS.max} seconds (this one has ${detail} s)`;
|
|
2740
|
+
case "reference-too-large": return `an attached image could not be made smaller than ${detail ?? "1000"} KB; attach a smaller image`;
|
|
2741
|
+
case "upstream-error": return `CPA rejected the request (${detail})`;
|
|
2742
|
+
case "timeout": return video ? "CPA did not answer in time" : "CPA did not answer within 3 minutes";
|
|
2743
|
+
case "network": return "could not reach CPA";
|
|
2744
|
+
case "no-image": return "CPA returned no image";
|
|
2745
|
+
case "invalid-image":
|
|
2746
|
+
case "invalid-response": return video ? "CPA returned an answer this plugin does not understand" : "CPA returned data that is not a PNG, JPEG, WebP or GIF image";
|
|
2747
|
+
case "image-too-large": return "an image is larger than 32 MiB";
|
|
2748
|
+
case "video-too-large": return `the video is larger than ${Math.round(MAX_VIDEO_BYTES / 1024 / 1024)} MiB`;
|
|
2749
|
+
case "invalid-video": return "CPA returned data that is not an MP4, MOV or WebM video";
|
|
2750
|
+
case "video-failed": return `the video could not be generated${detail ? ` (${detail})` : ""}`;
|
|
2751
|
+
case "expired": return "CPA no longer knows this video: more than 3 hours have passed since it was submitted";
|
|
2752
|
+
case "download-failed": return `could not download the generated ${noun}`;
|
|
2753
|
+
case "write-failed": return `could not write the ${noun} file${detail ? ` (${detail})` : ""}`;
|
|
2754
|
+
case "no-endpoint": return "this account has no usable server address yet";
|
|
2755
|
+
case "invalid-account": return "the model's account no longer exists";
|
|
2756
|
+
case "interrupted": return "stopped because the plugin was reloaded or DSH closed";
|
|
2757
|
+
default: return error.code;
|
|
2758
|
+
}
|
|
2759
|
+
};
|
|
2760
|
+
// #region Background generation jobs and the session's "running" state
|
|
2761
|
+
//
|
|
2762
|
+
// `/cpa-image` returns as soon as the request is admitted, so the composer
|
|
2763
|
+
// clears the draft at once; the generation continues here and its chat row
|
|
2764
|
+
// follows it through `media/command`. DSH counts only agent turns as
|
|
2765
|
+
// running, so while a job runs this plugin publishes the host's own
|
|
2766
|
+
// `api-session/status` event for that session (the workspace list shows it
|
|
2767
|
+
// as running, then completed). It is republished every few seconds and
|
|
2768
|
+
// after the agent itself goes idle, because those reset it to idle.
|
|
2769
|
+
|
|
2770
|
+
const mediaJobs = new Map(); // job key → { sessionId, abort }
|
|
2771
|
+
// job key → the promise of a job of this activation, until it settles (the agent tool waits on it).
|
|
2772
|
+
const jobPromises = new Map();
|
|
2773
|
+
const settleWith = (key, promise) => {
|
|
2774
|
+
jobPromises.set(key, promise);
|
|
2775
|
+
void promise.finally(() => {
|
|
2776
|
+
if (jobPromises.get(key) === promise) jobPromises.delete(key);
|
|
2777
|
+
});
|
|
2778
|
+
return promise;
|
|
2779
|
+
};
|
|
2780
|
+
let mediaJobSeq = 0;
|
|
2781
|
+
let mediaHeartbeat;
|
|
2782
|
+
const MEDIA_HEARTBEAT_MS = 5000;
|
|
2783
|
+
const jobSessions = () => new Set([...mediaJobs.values()].map((job) => job.sessionId).filter(Boolean));
|
|
2784
|
+
const agentRunning = (sessionId) => {
|
|
2785
|
+
try {
|
|
2786
|
+
return ctx.get?.("agents")?.get?.(sessionId)?.status === "running";
|
|
2787
|
+
} catch {
|
|
2788
|
+
return false;
|
|
2789
|
+
}
|
|
2790
|
+
};
|
|
2791
|
+
const publishSessionStatus = (sessionId, running) => {
|
|
2792
|
+
if (typeof ctx.emit !== "function") return;
|
|
2793
|
+
try {
|
|
2794
|
+
ctx.emit("api-session/status", sessionId, running);
|
|
2795
|
+
} catch (error) {
|
|
2796
|
+
ctx.logger?.warn?.(`cpa: could not publish the session status: ${error instanceof Error ? error.message : String(error)}`);
|
|
2797
|
+
}
|
|
2798
|
+
};
|
|
2799
|
+
const settleSessionStatus = (sessionId) => {
|
|
2800
|
+
if (sessionId === undefined || jobSessions().has(sessionId) || agentRunning(sessionId)) return;
|
|
2801
|
+
publishSessionStatus(sessionId, false);
|
|
2802
|
+
};
|
|
2803
|
+
const syncHeartbeat = () => {
|
|
2804
|
+
if (mediaJobs.size > 0 && mediaHeartbeat === undefined) {
|
|
2805
|
+
mediaHeartbeat = setInterval(() => {
|
|
2806
|
+
for (const sessionId of jobSessions()) publishSessionStatus(sessionId, true);
|
|
2807
|
+
}, MEDIA_HEARTBEAT_MS);
|
|
2808
|
+
mediaHeartbeat.unref?.();
|
|
2809
|
+
} else if (mediaJobs.size === 0 && mediaHeartbeat !== undefined) {
|
|
2810
|
+
clearInterval(mediaHeartbeat);
|
|
2811
|
+
mediaHeartbeat = undefined;
|
|
2812
|
+
}
|
|
2813
|
+
};
|
|
2814
|
+
ctx.on?.("agent/status", (event) => {
|
|
2815
|
+
const sessionId = event?.agent?.id;
|
|
2816
|
+
if (event?.status === "running" || !jobSessions().has(sessionId)) return;
|
|
2817
|
+
// After the host's own idle notice for this transition.
|
|
2818
|
+
setTimeout(() => {
|
|
2819
|
+
if (jobSessions().has(sessionId)) publishSessionStatus(sessionId, true);
|
|
2820
|
+
}, 0);
|
|
2821
|
+
});
|
|
2822
|
+
ctx.effect(
|
|
2823
|
+
() => () => {
|
|
2824
|
+
const sessions = jobSessions();
|
|
2825
|
+
for (const job of mediaJobs.values()) job.abort.abort(new MediaError("interrupted"));
|
|
2826
|
+
mediaJobs.clear();
|
|
2827
|
+
syncHeartbeat();
|
|
2828
|
+
for (const sessionId of sessions) settleSessionStatus(sessionId);
|
|
2829
|
+
},
|
|
2830
|
+
"cpa: generation jobs"
|
|
2831
|
+
);
|
|
2832
|
+
|
|
2833
|
+
/** Update one run record (and prepend new recent outputs) in the session store; never throws. */
|
|
2834
|
+
const recordRun = (sessionId, commandId, patch, recent = []) => {
|
|
2835
|
+
if (sessionId === undefined) return;
|
|
2836
|
+
try {
|
|
2837
|
+
mediaSessions.update(sessionId, (entry) => {
|
|
2838
|
+
if (recent.length > 0) entry.recent = [...recent, ...entry.recent];
|
|
2839
|
+
if (commandId === undefined) return;
|
|
2840
|
+
const index = entry.runs.findIndex((run) => run.commandId === commandId);
|
|
2841
|
+
if (index >= 0) entry.runs[index] = { ...entry.runs[index], ...patch };
|
|
2842
|
+
else entry.runs = [{ commandId, ...patch }, ...entry.runs];
|
|
2843
|
+
});
|
|
2844
|
+
} catch (error) {
|
|
2845
|
+
ctx.logger?.warn?.(`cpa: could not record a generation result: ${error instanceof Error ? error.message : String(error)}`);
|
|
2846
|
+
}
|
|
2847
|
+
};
|
|
2848
|
+
|
|
2849
|
+
/** How the chat row names the save folder: relative to the working directory when inside it. */
|
|
2850
|
+
const folderLabel = (cwd, dir) => {
|
|
2851
|
+
const rel = relative(cwd, dir);
|
|
2852
|
+
return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel) ? rel.split(sep).join("/") : basename(dir);
|
|
2853
|
+
};
|
|
2854
|
+
|
|
2855
|
+
/** Run one admitted image request in the background; resolves when it has settled and been recorded. */
|
|
2856
|
+
const startImageJob = ({ sessionId, commandId, option, args: admitted, references, cwd, outputDir, endpoint, interpret }) => {
|
|
2857
|
+
const key = commandId ?? `job-${++mediaJobSeq}`;
|
|
2858
|
+
const abort = new AbortController();
|
|
2859
|
+
mediaJobs.set(key, { sessionId, abort });
|
|
2860
|
+
syncHeartbeat();
|
|
2861
|
+
if (sessionId !== undefined) publishSessionStatus(sessionId, true);
|
|
2862
|
+
const started = Date.now();
|
|
2863
|
+
return settleWith(key, (async () => {
|
|
2864
|
+
try {
|
|
2865
|
+
let args = admitted;
|
|
2866
|
+
if (interpret !== undefined) {
|
|
2867
|
+
if (commandId !== undefined) videoLive.set(commandId, { phase: "interpreting" });
|
|
2868
|
+
const result = await interpret(abort.signal);
|
|
2869
|
+
if (commandId !== undefined) videoLive.delete(commandId);
|
|
2870
|
+
if (result !== undefined) {
|
|
2871
|
+
args = { ...admitted, ...result.values, prompt: result.prompt };
|
|
2872
|
+
recordRun(sessionId, commandId, {
|
|
2873
|
+
prompt: args.prompt,
|
|
2874
|
+
...(args.aspect === undefined ? {} : { aspect: args.aspect }),
|
|
2875
|
+
count: args.count ?? 1,
|
|
2876
|
+
...interpretedRecord(result, admitted.prompt),
|
|
2877
|
+
});
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
const images = await generateImages({
|
|
2881
|
+
rootURL: endpoint.rootURL,
|
|
2882
|
+
apiKey: endpoint.apiKey,
|
|
2883
|
+
model: option.modelId,
|
|
2884
|
+
prompt: args.prompt,
|
|
2885
|
+
...(args.aspect === undefined ? {} : { aspect: args.aspect }),
|
|
2886
|
+
count: args.count ?? 1,
|
|
2887
|
+
references,
|
|
2888
|
+
signal: abort.signal,
|
|
2889
|
+
});
|
|
2890
|
+
const dir = outputDirectory(cwd, outputDir);
|
|
2891
|
+
const paths = writeOutputFiles(dir, outputStem(option.modelId, args.prompt), images);
|
|
2892
|
+
const revised = images.find((image) => image.revisedPrompt !== undefined)?.revisedPrompt;
|
|
2893
|
+
recordRun(
|
|
2894
|
+
sessionId,
|
|
2895
|
+
commandId,
|
|
2896
|
+
{
|
|
2897
|
+
status: "done",
|
|
2898
|
+
files: paths.length,
|
|
2899
|
+
folder: folderLabel(cwd, dir),
|
|
2900
|
+
finishedAt: new Date().toISOString(),
|
|
2901
|
+
...(revised === undefined ? {} : { revisedPrompt: revised }),
|
|
2902
|
+
},
|
|
2903
|
+
paths.map((p) => recentEntry("image", p, option.ref, new Date(), commandId))
|
|
2904
|
+
);
|
|
2905
|
+
ctx.logger?.info?.(`cpa: /cpa-image wrote ${paths.length} image(s) with ${option.modelId} in ${Math.round((Date.now() - started) / 1000)} s`);
|
|
2906
|
+
} catch (thrown) {
|
|
2907
|
+
const error = abort.signal.aborted ? new MediaError("interrupted") : thrown;
|
|
2908
|
+
recordRun(sessionId, commandId, {
|
|
2909
|
+
status: "error",
|
|
2910
|
+
errorCode: error instanceof MediaError ? error.code : "failed",
|
|
2911
|
+
error: mediaFailureText(error),
|
|
2912
|
+
finishedAt: new Date().toISOString(),
|
|
2913
|
+
});
|
|
2914
|
+
ctx.logger?.warn?.(`cpa: /cpa-image failed: ${mediaFailureText(error)}`);
|
|
2915
|
+
} finally {
|
|
2916
|
+
if (commandId !== undefined) videoLive.delete(commandId);
|
|
2917
|
+
mediaJobs.delete(key);
|
|
2918
|
+
syncHeartbeat();
|
|
2919
|
+
settleSessionStatus(sessionId);
|
|
2920
|
+
}
|
|
2921
|
+
})());
|
|
2922
|
+
};
|
|
2923
|
+
|
|
2924
|
+
// #endregion
|
|
2925
|
+
|
|
2926
|
+
/**
|
|
2927
|
+
* `/cpa-image`: generate or (with attached images) edit, save into the
|
|
2928
|
+
* session's folder. Checks everything it can up front (options, model,
|
|
2929
|
+
* folder, references, endpoint), then returns while the generation runs as
|
|
2930
|
+
* a background job; the chat row shows its progress and result.
|
|
2931
|
+
*/
|
|
2932
|
+
// Internal callers (the agent tool) add: `literal` (no plain-language
|
|
2933
|
+
// parsing or interpretation), `pickSession` (whose model pick applies) and
|
|
2934
|
+
// `gate(plan)` (confirmation / budget, run after every check and before
|
|
2935
|
+
// anything is recorded or sent; a thrown `gateError` passes through).
|
|
2936
|
+
const runImageCommand = async ({ agent, rawInput, attachments = [], signal, commandId, literal = false, pickSession, gate }) => {
|
|
2937
|
+
try {
|
|
2938
|
+
const args = parseImageCommand(rawInput, { literal });
|
|
2939
|
+
const blocks = Array.isArray(attachments) ? attachments : [];
|
|
2940
|
+
if (blocks.some((block) => block?.type !== "image")) throw new MediaError("unsupported-attachment");
|
|
2941
|
+
if (blocks.length > MAX_REFERENCE_IMAGES) throw new MediaError("too-many-references");
|
|
2942
|
+
const { settings, options, accounts } = await mediaChoicesFor("image");
|
|
2943
|
+
const sessionId = validSessionId(agent?.id);
|
|
2944
|
+
const pickId = validSessionId(pickSession) ?? sessionId;
|
|
2945
|
+
const entry = pickId === undefined ? {} : mediaSessions.get(pickId);
|
|
2946
|
+
let option;
|
|
2947
|
+
try {
|
|
2948
|
+
({ option } = chooseMediaModel({
|
|
2949
|
+
options: options.image,
|
|
2950
|
+
requested: args.model,
|
|
2951
|
+
sessionRef: entry.image,
|
|
2952
|
+
lastRef: settings.lastSelection.image,
|
|
2953
|
+
}));
|
|
2954
|
+
} catch (error) {
|
|
2955
|
+
if (error instanceof MediaError && error.code === "no-media-model") {
|
|
2956
|
+
throw new MediaError("no-media-model", mediaAccountsSummary(accounts, "image"));
|
|
2957
|
+
}
|
|
2958
|
+
throw error;
|
|
2959
|
+
}
|
|
2960
|
+
const workingDirectory = ctx.get?.("workingDirectory");
|
|
2961
|
+
const cwd = agent !== undefined && workingDirectory !== undefined ? await workingDirectory.ensure(agent, signal) : undefined;
|
|
2962
|
+
if (typeof cwd !== "string" || !isAbsolute(cwd) || !existsSync(cwd)) throw new MediaError("workspace-not-local");
|
|
2963
|
+
const references = [];
|
|
2964
|
+
if (blocks.length > 0) {
|
|
2965
|
+
const store = ctx.get?.("attachments");
|
|
2966
|
+
if (store === undefined) throw new MediaError("unsupported-attachment");
|
|
2967
|
+
const each = Math.floor(REFERENCES_MAX_BYTES / blocks.length);
|
|
2968
|
+
for (const block of blocks) references.push(await readReference(store, block, each, signal));
|
|
2969
|
+
}
|
|
2970
|
+
const endpoint = await controller.mediaEndpoint(option.accountId);
|
|
2971
|
+
const id = validCommandId(commandId);
|
|
2972
|
+
const count = args.count ?? 1;
|
|
2973
|
+
if (gate !== undefined) {
|
|
2974
|
+
await gate({ kind: "image", mode: references.length > 0 ? "image-edit" : "image-text", option, prompt: args.prompt, count, references: references.length, aspect: args.aspect, cwd, outputDir: settings.outputDir });
|
|
2975
|
+
}
|
|
2976
|
+
recordRun(sessionId, id, {
|
|
2977
|
+
kind: "image",
|
|
2978
|
+
status: "running",
|
|
2979
|
+
model: option.ref,
|
|
2980
|
+
modelName: option.name || option.modelId,
|
|
2981
|
+
accountName: option.accountName,
|
|
2982
|
+
prompt: args.prompt,
|
|
2983
|
+
...(args.aspect === undefined ? {} : { aspect: args.aspect }),
|
|
2984
|
+
count,
|
|
2985
|
+
references: references.length,
|
|
2986
|
+
files: 0,
|
|
2987
|
+
startedAt: new Date().toISOString(),
|
|
2988
|
+
});
|
|
2989
|
+
void startImageJob({
|
|
2990
|
+
sessionId,
|
|
2991
|
+
commandId: id,
|
|
2992
|
+
option,
|
|
2993
|
+
args,
|
|
2994
|
+
references,
|
|
2995
|
+
cwd,
|
|
2996
|
+
outputDir: settings.outputDir,
|
|
2997
|
+
endpoint,
|
|
2998
|
+
...(literal
|
|
2999
|
+
? {}
|
|
3000
|
+
: {
|
|
3001
|
+
interpret: (jobSignal) =>
|
|
3002
|
+
interpretParams({ agent, command: "image", text: args.prompt, images: references.length, fixed: ["aspect", "count"].filter((k) => args[k] !== undefined), signal: jobSignal }),
|
|
3003
|
+
}),
|
|
3004
|
+
});
|
|
3005
|
+
const dir = outputDirectory(cwd, settings.outputDir);
|
|
3006
|
+
return {
|
|
3007
|
+
kind: "success",
|
|
3008
|
+
text: `${references.length > 0 ? "Editing" : "Generating"} ${count} image(s) with ${option.modelId} (${option.accountName}); files go to ${dir}`,
|
|
3009
|
+
};
|
|
3010
|
+
} catch (error) {
|
|
3011
|
+
if (signal?.aborted || error?.gateError === true) throw error;
|
|
3012
|
+
return { kind: "error", text: `/cpa-image: ${mediaFailureText(error)}` };
|
|
3013
|
+
}
|
|
3014
|
+
};
|
|
3015
|
+
|
|
3016
|
+
// #region Video jobs (/cpa-video)
|
|
3017
|
+
//
|
|
3018
|
+
// A video is submitted once and cannot be cancelled (CPA and xAI have no
|
|
3019
|
+
// cancel route). The job is stored in media-jobs.json as soon as CPA
|
|
3020
|
+
// returns its request id, polled (5 s, then 10 s, then 15 s) and downloaded
|
|
3021
|
+
// through CPA into the session's folder. A reload or restart only pauses
|
|
3022
|
+
// the polling: the next activation resumes every stored job younger than
|
|
3023
|
+
// CPA's 3-hour request binding, and marks older ones expired.
|
|
3024
|
+
|
|
3025
|
+
const mediaJobStore = createMediaJobStore(join(updateDirectory, "media-jobs.json"));
|
|
3026
|
+
// commandId → { phase: "submitting" | "queued" | "running" | "downloading", progress? } for the chat row.
|
|
3027
|
+
const videoLive = new Map();
|
|
3028
|
+
// commandId → { url, accountId, accountName, seconds, sessionId, commandId, at }: the short-lived
|
|
3029
|
+
// result URL of a finished video, for /cpa-video-edit and /cpa-video-extend.
|
|
3030
|
+
// In memory only (never stored, logged or sent to the page); a reload forgets it.
|
|
3031
|
+
const videoSources = new Map();
|
|
3032
|
+
// xAI calls its result URLs ephemeral without a number; offer them for an hour.
|
|
3033
|
+
const VIDEO_SOURCE_TTL_MS = 60 * 60 * 1000;
|
|
3034
|
+
const MAX_VIDEO_SOURCES = 100;
|
|
3035
|
+
const rememberSource = (job, status) => {
|
|
3036
|
+
if (job.commandId === undefined || status.sourceUrl === undefined) return;
|
|
3037
|
+
videoSources.delete(job.commandId);
|
|
3038
|
+
videoSources.set(job.commandId, {
|
|
3039
|
+
url: status.sourceUrl,
|
|
3040
|
+
accountId: job.accountId,
|
|
3041
|
+
seconds: status.seconds ?? job.seconds,
|
|
3042
|
+
sessionId: job.sessionId,
|
|
3043
|
+
commandId: job.commandId,
|
|
3044
|
+
at: Date.now(),
|
|
3045
|
+
});
|
|
3046
|
+
while (videoSources.size > MAX_VIDEO_SOURCES) videoSources.delete(videoSources.keys().next().value);
|
|
3047
|
+
};
|
|
3048
|
+
const usableSource = (commandId) => {
|
|
3049
|
+
const source = videoSources.get(commandId);
|
|
3050
|
+
return source !== undefined && Date.now() - source.at <= VIDEO_SOURCE_TTL_MS ? source : undefined;
|
|
3051
|
+
};
|
|
3052
|
+
|
|
3053
|
+
const sleep = (ms, signal) =>
|
|
3054
|
+
new Promise((resolve, reject) => {
|
|
3055
|
+
if (signal.aborted) return reject(signal.reason);
|
|
3056
|
+
const timer = setTimeout(() => {
|
|
3057
|
+
signal.removeEventListener("abort", onAbort);
|
|
3058
|
+
resolve();
|
|
3059
|
+
}, ms);
|
|
3060
|
+
timer.unref?.();
|
|
3061
|
+
const onAbort = () => {
|
|
3062
|
+
clearTimeout(timer);
|
|
3063
|
+
reject(signal.reason);
|
|
3064
|
+
};
|
|
3065
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
3066
|
+
});
|
|
3067
|
+
|
|
3068
|
+
const pollDelay = (elapsedMs) => (elapsedMs < 60_000 ? 5000 : elapsedMs < 180_000 ? 10_000 : 15_000);
|
|
3069
|
+
const MAX_POLL_FAILURES = 12;
|
|
3070
|
+
|
|
3071
|
+
/** Poll, download and record one submitted video; a dispose abort leaves it stored for the next activation. */
|
|
3072
|
+
const followVideoJob = async (job, abort) => {
|
|
3073
|
+
const live = (patch) => {
|
|
3074
|
+
if (job.commandId !== undefined) videoLive.set(job.commandId, { ...(videoLive.get(job.commandId) ?? {}), ...patch });
|
|
3075
|
+
};
|
|
3076
|
+
const finish = (state, runPatch, recent = []) => {
|
|
3077
|
+
mediaJobStore.put({ ...job, state, finishedAt: new Date().toISOString() });
|
|
3078
|
+
recordRun(job.sessionId, job.commandId, { ...runPatch, finishedAt: new Date().toISOString() }, recent);
|
|
3079
|
+
};
|
|
3080
|
+
const fail = (error) => {
|
|
3081
|
+
const code = error instanceof MediaError ? error.code : "failed";
|
|
3082
|
+
finish(code === "expired" ? "expired" : "failed", { status: "error", errorCode: code, error: mediaFailureText(error, "video") });
|
|
3083
|
+
ctx.logger?.warn?.(`cpa: /cpa-video ${job.id} failed: ${mediaFailureText(error, "video")}`);
|
|
3084
|
+
};
|
|
3085
|
+
const submitted = Date.parse(job.submittedAt);
|
|
3086
|
+
let failures = 0;
|
|
3087
|
+
live({ phase: "queued" });
|
|
3088
|
+
try {
|
|
3089
|
+
for (;;) {
|
|
3090
|
+
if (Date.now() - submitted > VIDEO_JOB_TTL_MS) throw new MediaError("expired");
|
|
3091
|
+
let endpoint;
|
|
3092
|
+
let status;
|
|
3093
|
+
try {
|
|
3094
|
+
endpoint = await controller.mediaEndpoint(job.accountId);
|
|
3095
|
+
status = await retrieveVideo({ rootURL: endpoint.rootURL, apiKey: endpoint.apiKey, requestId: job.id, signal: abort.signal });
|
|
3096
|
+
failures = 0;
|
|
3097
|
+
} catch (error) {
|
|
3098
|
+
if (abort.signal.aborted) throw error;
|
|
3099
|
+
// A missing account or a refusal that keeps coming back ends the job;
|
|
3100
|
+
// network trouble and short CPA hiccups are retried.
|
|
3101
|
+
if (error instanceof MediaError && error.code === "invalid-account") throw error;
|
|
3102
|
+
failures += 1;
|
|
3103
|
+
if (failures >= MAX_POLL_FAILURES) throw error;
|
|
3104
|
+
await sleep(pollDelay(Date.now() - submitted), abort.signal);
|
|
3105
|
+
continue;
|
|
3106
|
+
}
|
|
3107
|
+
if (status.state === "failed") throw new MediaError("video-failed", status.error);
|
|
3108
|
+
if (status.state === "done") {
|
|
3109
|
+
live({ phase: "downloading", progress: 100 });
|
|
3110
|
+
const dir = outputDirectory(job.cwd, job.outputDir);
|
|
3111
|
+
const path = await downloadVideo({
|
|
3112
|
+
rootURL: endpoint.rootURL,
|
|
3113
|
+
apiKey: endpoint.apiKey,
|
|
3114
|
+
requestId: job.id,
|
|
3115
|
+
dir,
|
|
3116
|
+
stem: outputStem(job.modelId, job.prompt),
|
|
3117
|
+
signal: abort.signal,
|
|
3118
|
+
});
|
|
3119
|
+
finish("done", { status: "done", files: 1, folder: folderLabel(job.cwd, dir) }, [recentEntry("video", path, job.ref, new Date(), job.commandId)]);
|
|
3120
|
+
rememberSource(job, status);
|
|
3121
|
+
ctx.logger?.info?.(`cpa: /cpa-video ${job.id} saved after ${Math.round((Date.now() - submitted) / 1000)} s`);
|
|
3122
|
+
return;
|
|
3123
|
+
}
|
|
3124
|
+
live({ phase: status.state, ...(status.progress === undefined ? {} : { progress: status.progress }) });
|
|
3125
|
+
await sleep(pollDelay(Date.now() - submitted), abort.signal);
|
|
3126
|
+
}
|
|
3127
|
+
} catch (error) {
|
|
3128
|
+
if (abort.signal.aborted) return; // paused by a reload; resumed by the next activation
|
|
3129
|
+
fail(error);
|
|
3130
|
+
}
|
|
3131
|
+
};
|
|
3132
|
+
|
|
3133
|
+
/** Track one job in the session's running state until it settles. */
|
|
3134
|
+
const trackVideoJob = (key, sessionId, run) => {
|
|
3135
|
+
const abort = new AbortController();
|
|
3136
|
+
mediaJobs.set(key, { sessionId, abort });
|
|
3137
|
+
syncHeartbeat();
|
|
3138
|
+
if (sessionId !== undefined) publishSessionStatus(sessionId, true);
|
|
3139
|
+
return settleWith(key, (async () => {
|
|
3140
|
+
try {
|
|
3141
|
+
await run(abort);
|
|
3142
|
+
} finally {
|
|
3143
|
+
if (mediaJobs.get(key)?.abort === abort) mediaJobs.delete(key);
|
|
3144
|
+
videoLive.delete(key);
|
|
3145
|
+
syncHeartbeat();
|
|
3146
|
+
settleSessionStatus(sessionId);
|
|
3147
|
+
}
|
|
3148
|
+
})());
|
|
3149
|
+
};
|
|
3150
|
+
|
|
3151
|
+
/**
|
|
3152
|
+
* Submit one admitted video through `submit(signal)` (generation, edit or
|
|
3153
|
+
* extension), store the job, then follow it.
|
|
3154
|
+
*/
|
|
3155
|
+
const startVideoJob = ({ sessionId, commandId, accountId, modelId, ref, prompt, seconds, cwd, outputDir, submit, label }) => {
|
|
3156
|
+
const key = commandId ?? `job-${++mediaJobSeq}`;
|
|
3157
|
+
if (commandId !== undefined) videoLive.set(commandId, { phase: "submitting" });
|
|
3158
|
+
return trackVideoJob(key, sessionId, async (abort) => {
|
|
3159
|
+
let requestId;
|
|
3160
|
+
try {
|
|
3161
|
+
// `submit` may settle the final prompt / length first (parameter interpretation).
|
|
3162
|
+
const submitted = await submit(abort.signal);
|
|
3163
|
+
requestId = submitted.requestId;
|
|
3164
|
+
if (typeof submitted.prompt === "string") prompt = submitted.prompt;
|
|
3165
|
+
if (Number.isInteger(submitted.seconds)) seconds = submitted.seconds;
|
|
3166
|
+
} catch (thrown) {
|
|
3167
|
+
// Cut off before CPA answered: whether it was submitted is unknown.
|
|
3168
|
+
const error = abort.signal.aborted ? new MediaError("interrupted") : thrown;
|
|
3169
|
+
recordRun(sessionId, commandId, {
|
|
3170
|
+
status: "error",
|
|
3171
|
+
errorCode: error instanceof MediaError ? error.code : "failed",
|
|
3172
|
+
error: mediaFailureText(error, "video"),
|
|
3173
|
+
finishedAt: new Date().toISOString(),
|
|
3174
|
+
});
|
|
3175
|
+
ctx.logger?.warn?.(`cpa: ${label} was not submitted: ${mediaFailureText(error, "video")}`);
|
|
3176
|
+
return;
|
|
3177
|
+
}
|
|
3178
|
+
const job = mediaJobStore.put({
|
|
3179
|
+
id: requestId,
|
|
3180
|
+
state: "submitted",
|
|
3181
|
+
accountId,
|
|
3182
|
+
modelId,
|
|
3183
|
+
ref,
|
|
3184
|
+
...(sessionId === undefined ? {} : { sessionId }),
|
|
3185
|
+
...(commandId === undefined ? {} : { commandId }),
|
|
3186
|
+
cwd,
|
|
3187
|
+
outputDir,
|
|
3188
|
+
prompt,
|
|
3189
|
+
seconds,
|
|
3190
|
+
submittedAt: new Date().toISOString(),
|
|
3191
|
+
});
|
|
3192
|
+
recordRun(sessionId, commandId, { videoId: requestId });
|
|
3193
|
+
ctx.logger?.info?.(`cpa: ${label} submitted ${requestId} (${modelId}, ${seconds} s)`);
|
|
3194
|
+
await followVideoJob(job, abort);
|
|
3195
|
+
});
|
|
3196
|
+
};
|
|
3197
|
+
/** Resume every stored job that is still running (after a reload or restart). */
|
|
3198
|
+
const resumeVideoJobs = () => {
|
|
3199
|
+
let jobs;
|
|
3200
|
+
try {
|
|
3201
|
+
jobs = mediaJobStore.list().filter((job) => job.state === "submitted");
|
|
3202
|
+
} catch (error) {
|
|
3203
|
+
ctx.logger?.warn?.(`cpa: could not read media-jobs.json: ${error instanceof Error ? error.message : String(error)}`);
|
|
3204
|
+
return;
|
|
3205
|
+
}
|
|
3206
|
+
for (const job of jobs) {
|
|
3207
|
+
const key = job.commandId ?? `video-${job.id}`;
|
|
3208
|
+
if (mediaJobs.has(key)) continue;
|
|
3209
|
+
void trackVideoJob(key, job.sessionId, (abort) => followVideoJob(job, abort));
|
|
3210
|
+
}
|
|
3211
|
+
};
|
|
3212
|
+
|
|
3213
|
+
/** Grok's 1.5 video family takes more references and pinned first / last frames. */
|
|
3214
|
+
const isVideo15 = (modelId) => /(^|\/)grok-imagine-video-1\.5/i.test(String(modelId ?? ""));
|
|
3215
|
+
|
|
3216
|
+
/**
|
|
3217
|
+
* How attached images are used: none = text only; one = first frame
|
|
3218
|
+
* (`--ref`: a reference, `--last`: the last frame); two = references
|
|
3219
|
+
* (`--first-last` / `--last`: first and last frame); more = references.
|
|
3220
|
+
* @throws MediaError `frames-need-images` / `too-many-references` / `needs-video-15`.
|
|
3221
|
+
*/
|
|
3222
|
+
const videoMode = (count, frames, modelId) => {
|
|
3223
|
+
if (count === 0) {
|
|
3224
|
+
if (frames !== undefined) throw new MediaError("frames-need-images");
|
|
3225
|
+
return "text";
|
|
3226
|
+
}
|
|
3227
|
+
let mode;
|
|
3228
|
+
if (count === 1) mode = frames === "reference" ? "reference" : frames === "last" ? "last-frame" : frames === "first-last" ? undefined : "first-frame";
|
|
3229
|
+
else if (count === 2) mode = frames === "first-last" || frames === "last" ? "first-last" : "reference";
|
|
3230
|
+
else mode = frames === undefined || frames === "reference" ? "reference" : undefined;
|
|
3231
|
+
if (mode === undefined) throw new MediaError(count === 1 ? "frames-need-images" : "too-many-references", count === 1 ? "2" : "2");
|
|
3232
|
+
const maxRefs = isVideo15(modelId) ? 14 : 7;
|
|
3233
|
+
if (mode === "reference" && count > maxRefs) throw new MediaError("too-many-references", String(maxRefs));
|
|
3234
|
+
if ((mode === "first-last" || mode === "last-frame") && !isVideo15(modelId)) throw new MediaError("needs-video-15");
|
|
3235
|
+
return mode;
|
|
3236
|
+
};
|
|
3237
|
+
|
|
3238
|
+
/**
|
|
3239
|
+
* `/cpa-video`: submit a video — text only, from a first frame, from
|
|
3240
|
+
* reference images, or between pinned first / last frames — and return at
|
|
3241
|
+
* once; the chat row follows the background job.
|
|
3242
|
+
*/
|
|
3243
|
+
const runVideoCommand = async ({ agent, rawInput, attachments = [], signal, commandId, literal = false, pickSession, gate }) => {
|
|
3244
|
+
try {
|
|
3245
|
+
const parsed = parseVideoCommand(rawInput, { literal });
|
|
3246
|
+
const blocks = Array.isArray(attachments) ? attachments : [];
|
|
3247
|
+
if (blocks.some((block) => block?.type !== "image")) throw new MediaError("unsupported-attachment");
|
|
3248
|
+
if (blocks.length > 14) throw new MediaError("too-many-references", "14");
|
|
3249
|
+
const { settings, options, accounts } = await mediaChoicesFor("video");
|
|
3250
|
+
const sessionId = validSessionId(agent?.id);
|
|
3251
|
+
const pickId = validSessionId(pickSession) ?? sessionId;
|
|
3252
|
+
const entry = pickId === undefined ? {} : mediaSessions.get(pickId);
|
|
3253
|
+
let option;
|
|
3254
|
+
try {
|
|
3255
|
+
({ option } = chooseMediaModel({ options: options.video, requested: parsed.model, sessionRef: entry.video, lastRef: settings.lastSelection.video }));
|
|
3256
|
+
} catch (error) {
|
|
3257
|
+
if (error instanceof MediaError && error.code === "no-media-model") throw new MediaError("no-media-model", mediaAccountsSummary(accounts, "video"));
|
|
3258
|
+
throw error;
|
|
3259
|
+
}
|
|
3260
|
+
const mode = videoMode(blocks.length, parsed.frames, option.modelId);
|
|
3261
|
+
const workingDirectory = ctx.get?.("workingDirectory");
|
|
3262
|
+
const cwd = agent !== undefined && workingDirectory !== undefined ? await workingDirectory.ensure(agent, signal) : undefined;
|
|
3263
|
+
if (typeof cwd !== "string" || !isAbsolute(cwd) || !existsSync(cwd)) throw new MediaError("workspace-not-local");
|
|
3264
|
+
const images = [];
|
|
3265
|
+
if (blocks.length > 0) {
|
|
3266
|
+
const store = ctx.get?.("attachments");
|
|
3267
|
+
if (store === undefined) throw new MediaError("unsupported-attachment");
|
|
3268
|
+
// The generation route reads about 1 MiB of body (measured), so all images share that.
|
|
3269
|
+
const each = Math.min(FIRST_FRAME_MAX_BYTES, videoImageBudget(parsed.prompt, blocks.length));
|
|
3270
|
+
for (const block of blocks) images.push(await readReference(store, block, each, signal));
|
|
3271
|
+
}
|
|
3272
|
+
const framesFor = (m) =>
|
|
3273
|
+
m === "first-frame" ? { image: images[0] }
|
|
3274
|
+
: m === "last-frame" ? { lastFrame: images[0] }
|
|
3275
|
+
: m === "first-last" ? { image: images[0], lastFrame: images[1] }
|
|
3276
|
+
: m === "reference" ? { references: images }
|
|
3277
|
+
: {};
|
|
3278
|
+
const endpoint = await controller.mediaEndpoint(option.accountId);
|
|
3279
|
+
const args = { ...parsed, seconds: parsed.seconds ?? settings.defaultVideoSeconds, resolution: parsed.resolution ?? settings.defaultVideoResolution };
|
|
3280
|
+
const id = validCommandId(commandId);
|
|
3281
|
+
if (gate !== undefined) {
|
|
3282
|
+
await gate({ kind: "video", mode, option, prompt: args.prompt, seconds: args.seconds, resolution: args.resolution, aspect: args.aspect, references: images.length, cwd, outputDir: settings.outputDir });
|
|
3283
|
+
}
|
|
3284
|
+
recordRun(sessionId, id, {
|
|
3285
|
+
kind: "video",
|
|
3286
|
+
status: "running",
|
|
3287
|
+
mode,
|
|
3288
|
+
model: option.ref,
|
|
3289
|
+
modelName: option.name || option.modelId,
|
|
3290
|
+
accountName: option.accountName,
|
|
3291
|
+
prompt: args.prompt,
|
|
3292
|
+
...(args.aspect === undefined ? {} : { aspect: args.aspect }),
|
|
3293
|
+
seconds: args.seconds,
|
|
3294
|
+
resolution: args.resolution,
|
|
3295
|
+
count: 1,
|
|
3296
|
+
references: images.length,
|
|
3297
|
+
files: 0,
|
|
3298
|
+
startedAt: new Date().toISOString(),
|
|
3299
|
+
});
|
|
3300
|
+
void startVideoJob({
|
|
3301
|
+
sessionId,
|
|
3302
|
+
commandId: id,
|
|
3303
|
+
accountId: option.accountId,
|
|
3304
|
+
modelId: option.modelId,
|
|
3305
|
+
ref: option.ref,
|
|
3306
|
+
prompt: args.prompt,
|
|
3307
|
+
seconds: args.seconds,
|
|
3308
|
+
cwd,
|
|
3309
|
+
outputDir: settings.outputDir,
|
|
3310
|
+
label: "/cpa-video",
|
|
3311
|
+
submit: async (jobSignal) => {
|
|
3312
|
+
// The session's chat model reads what the rules did not (explicit options and recognized words win).
|
|
3313
|
+
if (id !== undefined && !literal) videoLive.set(id, { phase: "interpreting" });
|
|
3314
|
+
const fixed = ["seconds", "resolution", "aspect", "frames"].filter((k) => parsed[k] !== undefined);
|
|
3315
|
+
const result = literal ? undefined : await interpretParams({ agent, command: "video", text: parsed.prompt, images: images.length, fixed, signal: jobSignal });
|
|
3316
|
+
let final = args;
|
|
3317
|
+
let finalMode = mode;
|
|
3318
|
+
if (result !== undefined) {
|
|
3319
|
+
final = { ...args, ...result.values, prompt: result.prompt };
|
|
3320
|
+
if (result.values.frames !== undefined) finalMode = videoMode(images.length, result.values.frames, option.modelId);
|
|
3321
|
+
recordRun(sessionId, id, {
|
|
3322
|
+
mode: finalMode,
|
|
3323
|
+
prompt: final.prompt,
|
|
3324
|
+
...(final.aspect === undefined ? {} : { aspect: final.aspect }),
|
|
3325
|
+
seconds: final.seconds,
|
|
3326
|
+
resolution: final.resolution,
|
|
3327
|
+
...interpretedRecord(result, parsed.prompt),
|
|
3328
|
+
});
|
|
3329
|
+
}
|
|
3330
|
+
if (id !== undefined) videoLive.set(id, { phase: "submitting" });
|
|
3331
|
+
const submitted = await submitVideo({
|
|
3332
|
+
rootURL: endpoint.rootURL,
|
|
3333
|
+
apiKey: endpoint.apiKey,
|
|
3334
|
+
model: option.modelId,
|
|
3335
|
+
prompt: final.prompt,
|
|
3336
|
+
seconds: final.seconds,
|
|
3337
|
+
...(final.aspect === undefined ? {} : { aspect: final.aspect }),
|
|
3338
|
+
resolution: final.resolution,
|
|
3339
|
+
...framesFor(finalMode),
|
|
3340
|
+
signal: jobSignal,
|
|
3341
|
+
});
|
|
3342
|
+
return { ...submitted, prompt: final.prompt, seconds: final.seconds };
|
|
3343
|
+
},
|
|
3344
|
+
});
|
|
3345
|
+
return {
|
|
3346
|
+
kind: "success",
|
|
3347
|
+
text: `Submitting a ${args.seconds} s ${args.resolution} video (${mode}) to ${option.modelId} (${option.accountName}); a submitted video cannot be cancelled. Files go to ${outputDirectory(cwd, settings.outputDir)}`,
|
|
3348
|
+
};
|
|
3349
|
+
} catch (error) {
|
|
3350
|
+
if (signal?.aborted || error?.gateError === true) throw error;
|
|
3351
|
+
return { kind: "error", text: `/cpa-video: ${mediaFailureText(error, "video")}` };
|
|
3352
|
+
}
|
|
3353
|
+
};
|
|
3354
|
+
|
|
3355
|
+
// `${accountId}/${route}` the upstream answered 405 for in this activation:
|
|
3356
|
+
// asked again only after a reload (CPA pauses the credential after a 405).
|
|
3357
|
+
const unsupportedRoutes = new Set();
|
|
3358
|
+
const editRoute = (mode) => (mode === "extend" ? "extensions" : "edits");
|
|
3359
|
+
|
|
3360
|
+
/** The newest existing video file the session recorded (for one command, or any). */
|
|
3361
|
+
const sessionVideoFile = (sessionId, commandId) => {
|
|
3362
|
+
if (sessionId === undefined) return undefined;
|
|
3363
|
+
return mediaSessions.get(sessionId).recent.find((r) => r.kind === "video" && (commandId === undefined || r.commandId === commandId) && existsSync(r.file));
|
|
3364
|
+
};
|
|
3365
|
+
/** The account a recorded run used (`accountId/modelId`), if it still exists. */
|
|
3366
|
+
const runAccountId = (sessionId, commandId, accounts) => {
|
|
3367
|
+
if (sessionId === undefined || commandId === undefined) return undefined;
|
|
3368
|
+
const run = mediaSessions.get(sessionId).runs.find((r) => r.commandId === commandId);
|
|
3369
|
+
const id = typeof run?.model === "string" ? run.model.split("/", 1)[0] : undefined;
|
|
3370
|
+
return accounts.some((a) => a.id === id) ? id : undefined;
|
|
3371
|
+
};
|
|
3372
|
+
const checkVideoBytes = (data) => {
|
|
3373
|
+
if (!isMp4(data)) throw new MediaError("video-not-mp4");
|
|
3374
|
+
return mp4Duration(data);
|
|
3375
|
+
};
|
|
3376
|
+
const tooLarge = (bytes) => new MediaError("source-too-large", (bytes / 1024 / 1024).toFixed(1));
|
|
3377
|
+
/** A local MP4 read whole (at most {@link VIDEO_UPLOAD_MAX_BYTES}) for an inline upload. */
|
|
3378
|
+
const readLocalVideo = (path, name = basename(path)) => {
|
|
3379
|
+
let info;
|
|
3380
|
+
try {
|
|
3381
|
+
info = statSync(path);
|
|
3382
|
+
} catch {
|
|
3383
|
+
info = undefined;
|
|
3384
|
+
}
|
|
3385
|
+
if (info === undefined || !info.isFile()) throw new MediaError("video-not-found", name);
|
|
3386
|
+
if (info.size > VIDEO_UPLOAD_MAX_BYTES) throw tooLarge(info.size);
|
|
3387
|
+
const data = new Uint8Array(readFileSync(path));
|
|
3388
|
+
return { data, seconds: checkVideoBytes(data), name };
|
|
3389
|
+
};
|
|
3390
|
+
/** An MP4 attached to the command (a DSH file attachment `{ attachmentId, name, bytes }`). */
|
|
3391
|
+
const readAttachedVideo = async (store, ref, signal) => {
|
|
3392
|
+
const name = basename(String(ref?.name ?? "video.mp4"));
|
|
3393
|
+
if (Number(ref?.bytes) > VIDEO_UPLOAD_MAX_BYTES) throw tooLarge(Number(ref.bytes));
|
|
3394
|
+
let hostPath;
|
|
3395
|
+
try {
|
|
3396
|
+
hostPath = store.fileHostPath?.(ref);
|
|
3397
|
+
} catch {
|
|
3398
|
+
hostPath = undefined;
|
|
3399
|
+
}
|
|
3400
|
+
if (typeof hostPath === "string" && existsSync(hostPath)) return readLocalVideo(hostPath, name);
|
|
3401
|
+
if (typeof store.readFileStream !== "function") throw new MediaError("unsupported-attachment", "edit");
|
|
3402
|
+
const chunks = [];
|
|
3403
|
+
let size = 0;
|
|
3404
|
+
for await (const chunk of store.readFileStream(ref, signal)) {
|
|
3405
|
+
size += chunk.byteLength;
|
|
3406
|
+
if (size > VIDEO_UPLOAD_MAX_BYTES) throw tooLarge(size);
|
|
3407
|
+
chunks.push(chunk);
|
|
3408
|
+
}
|
|
3409
|
+
const data = new Uint8Array(Buffer.concat(chunks.map((c) => Buffer.from(c.buffer, c.byteOffset, c.byteLength))));
|
|
3410
|
+
return { data, seconds: checkVideoBytes(data), name };
|
|
3411
|
+
};
|
|
3412
|
+
|
|
3413
|
+
/**
|
|
3414
|
+
* `/cpa-video-edit` and `/cpa-video-extend`: change or continue any MP4.
|
|
3415
|
+
* Sources: an attached MP4, `--video <path>`, `--url <public URL>`,
|
|
3416
|
+
* `--from <command id>`, or by default the session's newest video. A video
|
|
3417
|
+
* this plugin made goes by xAI's own short-lived link while it is
|
|
3418
|
+
* remembered (about an hour, this activation), otherwise its file is sent
|
|
3419
|
+
* inline; local files go inline as a base64 data URI (CPA has no file
|
|
3420
|
+
* upload route; the edit routes read bodies of 20 MiB, measured), a URL is
|
|
3421
|
+
* downloaded by xAI itself.
|
|
3422
|
+
*/
|
|
3423
|
+
const runVideoEditCommand = (mode) => async ({ agent, rawInput, attachments = [], signal, commandId, literal = false, pickSession, gate }) => {
|
|
3424
|
+
const name = mode === "extend" ? "/cpa-video-extend" : "/cpa-video-edit";
|
|
3425
|
+
try {
|
|
3426
|
+
const parsed = parseVideoEditCommand(rawInput, mode, { literal });
|
|
3427
|
+
const blocks = Array.isArray(attachments) ? attachments : [];
|
|
3428
|
+
if (blocks.some((block) => block?.type !== "file")) throw new MediaError("unsupported-attachment", "edit");
|
|
3429
|
+
const named = [parsed.from, parsed.video, parsed.url].filter((v) => v !== undefined).length;
|
|
3430
|
+
if (blocks.length > 1 || (blocks.length === 1 && named > 0)) throw new MediaError("one-video-source");
|
|
3431
|
+
const sessionId = validSessionId(agent?.id);
|
|
3432
|
+
const workingDirectory = ctx.get?.("workingDirectory");
|
|
3433
|
+
const cwd = agent !== undefined && workingDirectory !== undefined ? await workingDirectory.ensure(agent, signal) : undefined;
|
|
3434
|
+
if (typeof cwd !== "string" || !isAbsolute(cwd) || !existsSync(cwd)) throw new MediaError("workspace-not-local");
|
|
3435
|
+
const { settings, options, accounts } = await mediaChoicesFor("video");
|
|
3436
|
+
|
|
3437
|
+
// source: { url? | data?, seconds?, accountId?, commandId?, name? }
|
|
3438
|
+
let source;
|
|
3439
|
+
const linkOf = (id) => {
|
|
3440
|
+
const link = usableSource(id);
|
|
3441
|
+
return link !== undefined && link.sessionId === sessionId ? link : undefined;
|
|
3442
|
+
};
|
|
3443
|
+
const fromLink = (link) => ({ url: link.url, seconds: link.seconds, accountId: link.accountId, commandId: link.commandId });
|
|
3444
|
+
const fromFile = (record) => ({ ...readLocalVideo(record.file), commandId: record.commandId, accountId: runAccountId(sessionId, record.commandId, accounts) });
|
|
3445
|
+
if (blocks.length === 1) {
|
|
3446
|
+
const store = ctx.get?.("attachments");
|
|
3447
|
+
if (store === undefined) throw new MediaError("unsupported-attachment", "edit");
|
|
3448
|
+
source = await readAttachedVideo(store, blocks[0].attachment, signal);
|
|
3449
|
+
} else if (parsed.video !== undefined) {
|
|
3450
|
+
source = readLocalVideo(isAbsolute(parsed.video) ? parsed.video : join(cwd, parsed.video));
|
|
3451
|
+
} else if (parsed.url !== undefined) {
|
|
3452
|
+
source = { url: parsed.url, name: new URL(parsed.url).hostname };
|
|
3453
|
+
} else if (parsed.from !== undefined) {
|
|
3454
|
+
const link = linkOf(parsed.from);
|
|
3455
|
+
const record = link === undefined ? sessionVideoFile(sessionId, parsed.from) : undefined;
|
|
3456
|
+
if (link !== undefined) source = fromLink(link);
|
|
3457
|
+
else if (record !== undefined) source = fromFile(record);
|
|
3458
|
+
else throw new MediaError(videoSources.has(parsed.from) ? "video-source-expired" : "no-video-source");
|
|
3459
|
+
} else {
|
|
3460
|
+
const record = sessionVideoFile(sessionId);
|
|
3461
|
+
const link = record !== undefined ? linkOf(record.commandId) : [...videoSources.values()].reverse().map((s) => linkOf(s.commandId)).find(Boolean);
|
|
3462
|
+
if (link !== undefined) source = fromLink(link);
|
|
3463
|
+
else if (record !== undefined) source = fromFile(record);
|
|
3464
|
+
else throw new MediaError("no-video-source");
|
|
3465
|
+
}
|
|
3466
|
+
if (source.seconds !== undefined) {
|
|
3467
|
+
if (mode === "edit" && source.seconds > EDIT_SOURCE_MAX_SECONDS) throw new MediaError("edit-source-too-long", String(source.seconds));
|
|
3468
|
+
if (mode === "extend" && (source.seconds < EXTEND_SOURCE_SECONDS.min || source.seconds > EXTEND_SOURCE_SECONDS.max)) {
|
|
3469
|
+
throw new MediaError("extend-source-length", String(source.seconds));
|
|
3470
|
+
}
|
|
3471
|
+
}
|
|
3472
|
+
|
|
3473
|
+
// The account: the source video's own, unless -m names a model; else the session's video model.
|
|
3474
|
+
let accountId = parsed.model === undefined ? source.accountId : undefined;
|
|
3475
|
+
if (accountId === undefined) {
|
|
3476
|
+
const pickId = validSessionId(pickSession) ?? sessionId;
|
|
3477
|
+
const entry = pickId === undefined ? {} : mediaSessions.get(pickId);
|
|
3478
|
+
try {
|
|
3479
|
+
({ option: { accountId } } = chooseMediaModel({ options: options.video, requested: parsed.model, sessionRef: entry.video, lastRef: settings.lastSelection.video }));
|
|
3480
|
+
} catch (error) {
|
|
3481
|
+
if (error instanceof MediaError && error.code === "no-media-model") throw new MediaError("no-media-model", mediaAccountsSummary(accounts, "video"));
|
|
3482
|
+
throw error;
|
|
3483
|
+
}
|
|
3484
|
+
}
|
|
3485
|
+
const account = accounts.find((a) => a.id === accountId);
|
|
3486
|
+
if (account === undefined) throw new MediaError("invalid-account");
|
|
3487
|
+
const route = editRoute(mode);
|
|
3488
|
+
if (unsupportedRoutes.has(`${accountId}/${route}`)) throw new MediaError("route-unsupported", route);
|
|
3489
|
+
const endpoint = await controller.mediaEndpoint(accountId);
|
|
3490
|
+
const known = source.seconds === undefined ? undefined : Math.max(1, Math.min(15, Math.round(source.seconds)));
|
|
3491
|
+
const seconds = mode === "extend" ? parsed.seconds ?? 6 : known;
|
|
3492
|
+
const id = validCommandId(commandId);
|
|
3493
|
+
const accountName = account.displayName || account.id;
|
|
3494
|
+
if (gate !== undefined) {
|
|
3495
|
+
await gate({
|
|
3496
|
+
kind: "video",
|
|
3497
|
+
mode,
|
|
3498
|
+
option: { ref: `${accountId}/${VIDEO_EDIT_MODEL}`, modelId: VIDEO_EDIT_MODEL, accountId, accountName },
|
|
3499
|
+
prompt: parsed.prompt,
|
|
3500
|
+
// Billed as the new footage: an edit re-renders the source, an extension adds `seconds`.
|
|
3501
|
+
seconds: seconds ?? 6,
|
|
3502
|
+
sourceName: source.name ?? (source.commandId !== undefined ? source.commandId : undefined),
|
|
3503
|
+
references: 0,
|
|
3504
|
+
cwd,
|
|
3505
|
+
outputDir: settings.outputDir,
|
|
3506
|
+
});
|
|
3507
|
+
}
|
|
3508
|
+
recordRun(sessionId, id, {
|
|
3509
|
+
kind: "video",
|
|
3510
|
+
status: "running",
|
|
3511
|
+
mode,
|
|
3512
|
+
...(source.commandId === undefined ? {} : { sourceCommandId: source.commandId }),
|
|
3513
|
+
...(source.name === undefined ? {} : { sourceName: source.name }),
|
|
3514
|
+
sourceKind: source.data !== undefined ? "upload" : source.commandId !== undefined ? "link" : "url",
|
|
3515
|
+
...(Array.isArray(parsed.ignored) ? { ignored: parsed.ignored } : {}),
|
|
3516
|
+
model: `${accountId}/${VIDEO_EDIT_MODEL}`,
|
|
3517
|
+
modelName: VIDEO_EDIT_MODEL,
|
|
3518
|
+
accountName,
|
|
3519
|
+
prompt: parsed.prompt,
|
|
3520
|
+
...(seconds === undefined ? {} : { seconds }),
|
|
3521
|
+
count: 1,
|
|
3522
|
+
references: 0,
|
|
3523
|
+
files: 0,
|
|
3524
|
+
startedAt: new Date().toISOString(),
|
|
3525
|
+
});
|
|
3526
|
+
const { url: sourceUrl, data: sourceData } = source;
|
|
3527
|
+
void startVideoJob({
|
|
3528
|
+
sessionId,
|
|
3529
|
+
commandId: id,
|
|
3530
|
+
accountId,
|
|
3531
|
+
modelId: VIDEO_EDIT_MODEL,
|
|
3532
|
+
ref: `${accountId}/${VIDEO_EDIT_MODEL}`,
|
|
3533
|
+
prompt: parsed.prompt,
|
|
3534
|
+
// An extension returns the source plus the new part.
|
|
3535
|
+
seconds: mode === "extend" ? Math.min(15, (known ?? 0) + seconds) : seconds ?? 6,
|
|
3536
|
+
cwd,
|
|
3537
|
+
outputDir: settings.outputDir,
|
|
3538
|
+
label: name,
|
|
3539
|
+
submit: async (jobSignal) => {
|
|
3540
|
+
// The session's chat model reads an extension length or other wording the rules missed.
|
|
3541
|
+
if (id !== undefined && !literal) videoLive.set(id, { phase: "interpreting" });
|
|
3542
|
+
const fixed = mode === "extend" && parsed.seconds !== undefined ? ["seconds"] : [];
|
|
3543
|
+
const result = literal ? undefined : await interpretParams({ agent, command: mode, text: parsed.prompt, fixed, signal: jobSignal });
|
|
3544
|
+
let prompt = parsed.prompt;
|
|
3545
|
+
let added = seconds;
|
|
3546
|
+
if (result !== undefined) {
|
|
3547
|
+
prompt = result.prompt;
|
|
3548
|
+
if (mode === "extend" && result.values.seconds !== undefined) added = result.values.seconds;
|
|
3549
|
+
recordRun(sessionId, id, { prompt, ...(added === undefined ? {} : { seconds: added }), ...interpretedRecord(result, parsed.prompt) });
|
|
3550
|
+
}
|
|
3551
|
+
if (id !== undefined) videoLive.set(id, { phase: "submitting" });
|
|
3552
|
+
try {
|
|
3553
|
+
const submitted = await submitVideoEdit({
|
|
3554
|
+
rootURL: endpoint.rootURL,
|
|
3555
|
+
apiKey: endpoint.apiKey,
|
|
3556
|
+
mode,
|
|
3557
|
+
...(sourceData !== undefined ? { sourceData } : { sourceUrl }),
|
|
3558
|
+
prompt,
|
|
3559
|
+
...(mode === "extend" ? { seconds: added } : {}),
|
|
3560
|
+
signal: jobSignal,
|
|
3561
|
+
});
|
|
3562
|
+
return { ...submitted, prompt, ...(mode === "extend" ? { seconds: Math.min(15, (known ?? 0) + added) } : {}) };
|
|
3563
|
+
} catch (error) {
|
|
3564
|
+
if (error instanceof MediaError && error.code === "route-unsupported") unsupportedRoutes.add(`${accountId}/${route}`);
|
|
3565
|
+
throw error;
|
|
3566
|
+
}
|
|
3567
|
+
},
|
|
3568
|
+
});
|
|
3569
|
+
const skipped = Array.isArray(parsed.ignored) && parsed.ignored.length > 0
|
|
3570
|
+
? ` Ignored ${parsed.ignored.join(", ")}: ${mode === "extend" ? "an extension keeps the source's resolution and ratio" : "an edit keeps the source's length, resolution and ratio"}.`
|
|
3571
|
+
: "";
|
|
3572
|
+
const how = sourceData !== undefined ? `uploading ${source.name} (${(sourceData.byteLength / 1024 / 1024).toFixed(1)} MiB)` : source.name !== undefined ? `from ${source.name}` : "from xAI's link";
|
|
3573
|
+
return {
|
|
3574
|
+
kind: "success",
|
|
3575
|
+
text: `${mode === "extend" ? `Extending the video by ${seconds} s` : "Editing the video"} with ${VIDEO_EDIT_MODEL} (${accountName}), ${how}; a submitted video cannot be cancelled. Files go to ${outputDirectory(cwd, settings.outputDir)}.${skipped}`,
|
|
3576
|
+
};
|
|
3577
|
+
} catch (error) {
|
|
3578
|
+
if (signal?.aborted || error?.gateError === true) throw error;
|
|
3579
|
+
return { kind: "error", text: `${name}: ${mediaFailureText(error, "video")}` };
|
|
3580
|
+
}
|
|
3581
|
+
};
|
|
3582
|
+
/** `/cpa-video-status`: this session's video jobs, newest first (text). */
|
|
3583
|
+
const runVideoStatusCommand = async ({ agent }) => {
|
|
3584
|
+
const sessionId = validSessionId(agent?.id);
|
|
3585
|
+
const jobs = mediaJobStore.list().filter((job) => job.sessionId !== undefined && job.sessionId === sessionId).slice(0, 20);
|
|
3586
|
+
if (jobs.length === 0) return { kind: "success", text: "No video jobs in this session." };
|
|
3587
|
+
const lines = jobs.map((job) => {
|
|
3588
|
+
const live = job.commandId === undefined ? undefined : videoLive.get(job.commandId);
|
|
3589
|
+
const state = job.state === "submitted" ? (live?.phase ?? "waiting") + (live?.progress === undefined ? "" : ` ${live.progress}%`) : job.state;
|
|
3590
|
+
const prompt = job.prompt.length > 40 ? `${job.prompt.slice(0, 40)}…` : job.prompt;
|
|
3591
|
+
return `${job.submittedAt.slice(0, 19).replace("T", " ")} ${state.padEnd(14)} ${job.modelId}, ${job.seconds} s ${prompt} (${job.id})`;
|
|
3592
|
+
});
|
|
3593
|
+
return { kind: "success", text: lines.join("\n") };
|
|
3594
|
+
};
|
|
3595
|
+
|
|
3596
|
+
// #endregion
|
|
3597
|
+
|
|
3598
|
+
// #region Agent tool generate_media (off by default; design §5, §7, §8)
|
|
3599
|
+
//
|
|
3600
|
+
// One optional tool, registered only while the Generation tab's switch is
|
|
3601
|
+
// on and some enabled account offers a generation model. A call runs the
|
|
3602
|
+
// same admission as the slash commands (literally: no plain-language
|
|
3603
|
+
// parsing or interpretation), with a gate between admission and the
|
|
3604
|
+
// request that asks for approval and enforces the video seconds budget of
|
|
3605
|
+
// the root session, then waits up to 10 minutes for the background job.
|
|
3606
|
+
const TOOL_WAIT_MS = 10 * 60_000;
|
|
3607
|
+
const TOOL_IMAGE_MAX_BYTES = 32 * 1024 * 1024;
|
|
3608
|
+
const TOOL_PREVIEWS = 4;
|
|
3609
|
+
let toolsCtx;
|
|
3610
|
+
let toolOff;
|
|
3611
|
+
|
|
3612
|
+
const gateError = (message) => Object.assign(new Error(message), { gateError: true });
|
|
3613
|
+
|
|
3614
|
+
/** The top-level agent a (sub-)agent was created under: its session holds picks and the budget. */
|
|
3615
|
+
const rootAgentOf = (agent) => {
|
|
3616
|
+
const agents = ctx.get?.("agents");
|
|
3617
|
+
let current = agent;
|
|
3618
|
+
if (typeof agents?.list !== "function" || typeof agents.isOwnedBy !== "function") return current;
|
|
3619
|
+
for (let depth = 0; depth < 16; depth += 1) {
|
|
3620
|
+
const owner = agents.list().find((candidate) => candidate !== current && agents.isOwnedBy(current.id, candidate));
|
|
3621
|
+
if (owner === undefined) break;
|
|
3622
|
+
current = owner;
|
|
3623
|
+
}
|
|
3624
|
+
return current;
|
|
3625
|
+
};
|
|
3626
|
+
|
|
3627
|
+
// Legacy fallback only when the native userQuestions service is absent. Cost
|
|
3628
|
+
// confirmation is independent of DSH's execution-approval permission policy.
|
|
3629
|
+
// runId → { sessionId, kind, en, zh, expiresAt, settle(outcome) }, in memory only.
|
|
3630
|
+
const pendingConfirms = new Map();
|
|
3631
|
+
const pendingNativeConfirms = new Set();
|
|
3632
|
+
const CONFIRM_WAIT_MS = 10 * 60_000;
|
|
3633
|
+
const CONFIRM_ALLOW = "Allow / 允许";
|
|
3634
|
+
const CONFIRM_DECLINE = "Decline / 拒绝";
|
|
3635
|
+
const cancelConfirms = () => {
|
|
3636
|
+
for (const cancel of [...pendingNativeConfirms]) cancel();
|
|
3637
|
+
for (const waiting of [...pendingConfirms.values()]) waiting.settle("cancelled");
|
|
3638
|
+
};
|
|
3639
|
+
|
|
3640
|
+
/** Compatibility composer/row consent without userQuestions (`allowed` / `rejected` / `timeout` / `cancelled`). */
|
|
3641
|
+
const askInRow = (exec, runId, sessionId, text, kind) =>
|
|
3642
|
+
new Promise((resolve) => {
|
|
3643
|
+
if (exec.signal?.aborted || toolOff === undefined || toolsCtx === undefined) return resolve("cancelled");
|
|
3644
|
+
const timer = setTimeout(() => settle("timeout"), CONFIRM_WAIT_MS);
|
|
3645
|
+
timer.unref?.();
|
|
3646
|
+
const onAbort = () => settle("cancelled");
|
|
3647
|
+
const settle = (outcome) => {
|
|
3648
|
+
if (pendingConfirms.get(runId)?.settle !== settle) return;
|
|
3649
|
+
pendingConfirms.delete(runId);
|
|
3650
|
+
clearTimeout(timer);
|
|
3651
|
+
exec.signal?.removeEventListener("abort", onAbort);
|
|
3652
|
+
resolve(outcome);
|
|
3653
|
+
};
|
|
3654
|
+
exec.signal?.addEventListener("abort", onAbort, { once: true });
|
|
3655
|
+
pendingConfirms.set(runId, { sessionId, kind, en: text.en, zh: text.zh, expiresAt: new Date(Date.now() + CONFIRM_WAIT_MS).toISOString(), settle });
|
|
3656
|
+
});
|
|
3657
|
+
|
|
3658
|
+
/** Anonymous questions block in the native composer, not in a folded tool call. */
|
|
3659
|
+
const askNativeConfirm = (questions, exec, text, kind, runId) =>
|
|
3660
|
+
new Promise((resolve) => {
|
|
3661
|
+
if (exec.signal?.aborted || toolOff === undefined || toolsCtx === undefined) return resolve("cancelled");
|
|
3662
|
+
const controller = new AbortController();
|
|
3663
|
+
const questionId = `media-confirm-${runId}`;
|
|
3664
|
+
let settled = false;
|
|
3665
|
+
const onAbort = () => finish("cancelled");
|
|
3666
|
+
const cancel = () => finish("cancelled");
|
|
3667
|
+
const timer = setTimeout(() => finish("timeout"), CONFIRM_WAIT_MS);
|
|
3668
|
+
timer.unref?.();
|
|
3669
|
+
const finish = (outcome) => {
|
|
3670
|
+
if (settled) return;
|
|
3671
|
+
settled = true;
|
|
3672
|
+
clearTimeout(timer);
|
|
3673
|
+
exec.signal?.removeEventListener("abort", onAbort);
|
|
3674
|
+
pendingNativeConfirms.delete(cancel);
|
|
3675
|
+
// Remove the composer's pending request on every settlement, even if
|
|
3676
|
+
// an answerer ignores cancellation or rejects after the local deadline.
|
|
3677
|
+
controller.abort();
|
|
3678
|
+
resolve(outcome);
|
|
3679
|
+
};
|
|
3680
|
+
pendingNativeConfirms.add(cancel);
|
|
3681
|
+
exec.signal?.addEventListener("abort", onAbort, { once: true });
|
|
3682
|
+
Promise.resolve().then(() => {
|
|
3683
|
+
if (settled) return;
|
|
3684
|
+
return questions.ask({
|
|
3685
|
+
agent: rootAgentOf(exec.agent),
|
|
3686
|
+
signal: controller.signal,
|
|
3687
|
+
questions: [{
|
|
3688
|
+
id: questionId,
|
|
3689
|
+
header: kind === "video" ? "Video confirmation / 视频确认" : "Image confirmation / 图像确认",
|
|
3690
|
+
question: `${text.zh}\n\n${text.en}`,
|
|
3691
|
+
options: [{ label: CONFIRM_ALLOW }, { label: CONFIRM_DECLINE }],
|
|
3692
|
+
multiSelect: false,
|
|
3693
|
+
}],
|
|
3694
|
+
// Do not supply wait/callId: anonymous requests publish the native
|
|
3695
|
+
// blocking PendingQuestion instead of a continued tool-call card.
|
|
3696
|
+
});
|
|
3697
|
+
}).then((answer) => {
|
|
3698
|
+
const item = Array.isArray(answer?.answers) && answer.answers.length === 1 ? answer.answers[0] : undefined;
|
|
3699
|
+
const allowed = item?.id === questionId && item.custom === undefined &&
|
|
3700
|
+
Array.isArray(item.selected) && item.selected.length === 1 && item.selected[0] === CONFIRM_ALLOW;
|
|
3701
|
+
finish(allowed ? "allowed" : "rejected");
|
|
3702
|
+
}, () => finish("unavailable"));
|
|
3703
|
+
});
|
|
3704
|
+
|
|
3705
|
+
/** Cost consent is not an execution approval and never falls back after a native error. */
|
|
3706
|
+
const askApproval = async (exec, text, kind, runId, sessionId) => {
|
|
3707
|
+
const questions = ctx.get?.("userQuestions");
|
|
3708
|
+
const what = kind === "video" ? "this video" : "these images";
|
|
3709
|
+
let answer;
|
|
3710
|
+
if (questions === undefined || questions === null) {
|
|
3711
|
+
const ownerId = validSessionId(rootAgentOf(exec.agent)?.id) ?? sessionId;
|
|
3712
|
+
if (ownerId === undefined) throw gateError(`${what} needs the user's confirmation, but this call has no session to ask in`);
|
|
3713
|
+
answer = await askInRow(exec, runId, ownerId, text, kind);
|
|
3714
|
+
} else {
|
|
3715
|
+
answer = await askNativeConfirm(questions, exec, text, kind, runId);
|
|
3716
|
+
}
|
|
3717
|
+
if (answer === "allowed") return;
|
|
3718
|
+
if (answer === "rejected") throw gateError(`the user declined ${what}; do not retry unless the user asks`);
|
|
3719
|
+
if (answer === "timeout") throw gateError(`the user did not answer the confirmation for ${what} within 10 minutes; ask the user before trying again`);
|
|
3720
|
+
if (answer === "cancelled") throw gateError(`the confirmation for ${what} was cancelled`);
|
|
3721
|
+
throw gateError(`the confirmation for ${what} could not be shown; ask the user before trying again`);
|
|
3722
|
+
};
|
|
3723
|
+
|
|
3724
|
+
/** Attached-image blocks for local image paths (saved in DSH's attachment store, like composer images). */
|
|
3725
|
+
const toolImageBlocks = async (paths, cwd, signal) => {
|
|
3726
|
+
if (paths === undefined) return [];
|
|
3727
|
+
const store = ctx.get?.("attachments");
|
|
3728
|
+
if (typeof store?.saveImage !== "function") throw new Error("images cannot be read: DSH's attachment store is not available");
|
|
3729
|
+
const blocks = [];
|
|
3730
|
+
for (const raw of paths) {
|
|
3731
|
+
const path = isAbsolute(raw) ? raw : join(cwd, raw);
|
|
3732
|
+
let info;
|
|
3733
|
+
try {
|
|
3734
|
+
info = statSync(path);
|
|
3735
|
+
} catch {
|
|
3736
|
+
info = undefined;
|
|
3737
|
+
}
|
|
3738
|
+
if (info === undefined || !info.isFile()) throw new Error(`image not found: ${raw}`);
|
|
3739
|
+
if (info.size > TOOL_IMAGE_MAX_BYTES) throw new Error(`image is larger than 32 MiB: ${raw}`);
|
|
3740
|
+
const data = new Uint8Array(readFileSync(path));
|
|
3741
|
+
const mediaType = sniffImageType(data);
|
|
3742
|
+
if (mediaType === undefined) throw new Error(`not a PNG, JPEG, WebP or GIF image: ${raw}`);
|
|
3743
|
+
signal?.throwIfAborted();
|
|
3744
|
+
const ref = await store.saveImage({ data, mediaType, name: basename(path) });
|
|
3745
|
+
blocks.push({ type: "image", attachment: ref });
|
|
3746
|
+
}
|
|
3747
|
+
return blocks;
|
|
3748
|
+
};
|
|
3749
|
+
|
|
3750
|
+
/** Whether the calling route declares image input (as DSH's own read_image checks). */
|
|
3751
|
+
const routeSeesImages = async (agent, signal) => {
|
|
3752
|
+
const target = sessionChatTarget(agent);
|
|
3753
|
+
const llm = ctx.get?.("llm");
|
|
3754
|
+
if (target === undefined || typeof llm?.resolveModelInfo !== "function") return false;
|
|
3755
|
+
try {
|
|
3756
|
+
const info = await llm.resolveModelInfo(target.provider, target.model, signal);
|
|
3757
|
+
return Array.isArray(info?.inputModalities) && info.inputModalities.includes("image");
|
|
3758
|
+
} catch {
|
|
3759
|
+
return false;
|
|
3760
|
+
}
|
|
3761
|
+
};
|
|
3762
|
+
|
|
3763
|
+
/** Smaller copies of generated images for the model (DSH's attachment store normalizes them). */
|
|
3764
|
+
const imagePreviews = async (paths, agent, signal) => {
|
|
3765
|
+
const store = ctx.get?.("attachments");
|
|
3766
|
+
if (paths.length === 0 || typeof store?.saveImage !== "function" || !(await routeSeesImages(agent, signal))) return [];
|
|
3767
|
+
const previews = [];
|
|
3768
|
+
for (const path of paths.slice(0, TOOL_PREVIEWS)) {
|
|
3769
|
+
try {
|
|
3770
|
+
const data = new Uint8Array(readFileSync(path));
|
|
3771
|
+
const mediaType = sniffImageType(data);
|
|
3772
|
+
if (mediaType === undefined) continue;
|
|
3773
|
+
const ref = await store.saveImage({ data, mediaType, name: basename(path) });
|
|
3774
|
+
previews.push({
|
|
3775
|
+
attachmentId: String(ref.attachmentId),
|
|
3776
|
+
mediaType: ref.mediaType,
|
|
3777
|
+
bytes: ref.bytes,
|
|
3778
|
+
width: ref.width,
|
|
3779
|
+
height: ref.height,
|
|
3780
|
+
...(ref.name === undefined ? {} : { name: ref.name }),
|
|
3781
|
+
...(ref.originalDimensions === undefined ? {} : { originalDimensions: { width: ref.originalDimensions.width, height: ref.originalDimensions.height } }),
|
|
3782
|
+
});
|
|
3783
|
+
} catch (error) {
|
|
3784
|
+
if (signal?.aborted) throw error;
|
|
3785
|
+
ctx.logger?.warn?.(`cpa: could not attach a preview of ${basename(path)}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3786
|
+
}
|
|
3787
|
+
}
|
|
3788
|
+
return previews;
|
|
3789
|
+
};
|
|
3790
|
+
|
|
3791
|
+
/** Wait for a job of this activation: "settled", "timeout" or "aborted" (the job keeps running). */
|
|
3792
|
+
const waitForJob = (key, signal, ms) => {
|
|
3793
|
+
const promise = jobPromises.get(key);
|
|
3794
|
+
if (promise === undefined) return Promise.resolve("settled");
|
|
3795
|
+
return new Promise((resolve) => {
|
|
3796
|
+
const timer = setTimeout(() => finish("timeout"), ms);
|
|
3797
|
+
timer.unref?.();
|
|
3798
|
+
const onAbort = () => finish("aborted");
|
|
3799
|
+
const finish = (how) => {
|
|
3800
|
+
clearTimeout(timer);
|
|
3801
|
+
signal?.removeEventListener("abort", onAbort);
|
|
3802
|
+
resolve(how);
|
|
3803
|
+
};
|
|
3804
|
+
if (signal?.aborted) return finish("aborted");
|
|
3805
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
3806
|
+
promise.then(() => finish("settled"), () => finish("settled"));
|
|
3807
|
+
});
|
|
3808
|
+
};
|
|
3809
|
+
|
|
3810
|
+
// Not submitted (CPA refused it at once): the seconds go back to the budget.
|
|
3811
|
+
const NOT_SUBMITTED = new Set(["upstream-error", "request-too-large", "route-unsupported", "no-endpoint", "invalid-account"]);
|
|
3812
|
+
|
|
3813
|
+
const executeGenerateMedia = async (args, exec) => {
|
|
3814
|
+
const t = parseToolArgs(args);
|
|
3815
|
+
if (exec.agent === undefined) throw new Error("generate_media needs an agent session");
|
|
3816
|
+
const agent = exec.agent;
|
|
3817
|
+
const sessionId = validSessionId(agent.id);
|
|
3818
|
+
const root = rootAgentOf(agent);
|
|
3819
|
+
const rootId = validSessionId(root?.id) ?? sessionId;
|
|
3820
|
+
const runId = toolRunId(exec.callId) ?? `tool-${randomUUID()}`;
|
|
3821
|
+
const workingDirectory = ctx.get?.("workingDirectory");
|
|
3822
|
+
const cwd = workingDirectory === undefined ? undefined : await workingDirectory.ensure(agent, exec.signal);
|
|
3823
|
+
if (typeof cwd !== "string" || !isAbsolute(cwd) || !existsSync(cwd)) throw new Error(mediaFailureText(new MediaError("workspace-not-local"), t.kind));
|
|
3824
|
+
const blocks = await toolImageBlocks(t.images, cwd, exec.signal);
|
|
3825
|
+
const line = toolCommandLine(t);
|
|
3826
|
+
const handler = t.kind === "image" ? runImageCommand : t.mode === "generate" ? runVideoCommand : runVideoEditCommand(t.mode);
|
|
3827
|
+
// The client groups native tool events at turn level; never impersonate
|
|
3828
|
+
// a user slash command. Retain the deprecated output flag for old clients.
|
|
3829
|
+
const commandRow = false;
|
|
3830
|
+
let charged = 0;
|
|
3831
|
+
let planned;
|
|
3832
|
+
const gate = async (plan) => {
|
|
3833
|
+
planned = plan;
|
|
3834
|
+
const { settings } = mediaChoices();
|
|
3835
|
+
const policy = agentPolicy(settings, rootId === undefined ? {} : mediaSessions.get(rootId));
|
|
3836
|
+
if (plan.kind === "video") {
|
|
3837
|
+
if (policy.videoConfirm === "auto" && policy.limit !== undefined && policy.used + plan.seconds > policy.limit) {
|
|
3838
|
+
throw gateError(`this session's agent video budget is ${policy.limit} s and ${policy.used} s are used; this video needs ${plan.seconds} s. Ask the user to raise the limit (the media button next to the composer) or to confirm videos one by one`);
|
|
3839
|
+
}
|
|
3840
|
+
if (policy.videoConfirm === "ask") await askApproval(exec, approvalText(plan, policy), "video", runId, sessionId);
|
|
3841
|
+
if (rootId !== undefined) {
|
|
3842
|
+
mediaSessions.update(rootId, (entry) => chargeSeconds(entry, agentPolicy(settings, entry), plan.seconds));
|
|
3843
|
+
charged = plan.seconds;
|
|
3844
|
+
}
|
|
3845
|
+
} else if (policy.imageConfirm === "ask") {
|
|
3846
|
+
await askApproval(exec, approvalText(plan, policy), "image", runId, sessionId);
|
|
3847
|
+
}
|
|
3848
|
+
};
|
|
3849
|
+
const refund = () => {
|
|
3850
|
+
if (charged === 0 || rootId === undefined) return;
|
|
3851
|
+
const { settings } = mediaChoices();
|
|
3852
|
+
mediaSessions.update(rootId, (entry) => chargeSeconds(entry, agentPolicy(settings, entry), -charged));
|
|
3853
|
+
charged = 0;
|
|
3854
|
+
};
|
|
3855
|
+
const outcome = await handler({ agent, rawInput: line, attachments: blocks, signal: exec.signal, commandId: runId, literal: true, pickSession: rootId, gate });
|
|
3856
|
+
if (outcome?.kind !== "success") {
|
|
3857
|
+
refund();
|
|
3858
|
+
throw new Error(String(outcome?.text ?? "the request was refused").replace(/^\/cpa-[a-z-]+:\s*/, ""));
|
|
3859
|
+
}
|
|
3860
|
+
const waited = await waitForJob(runId, exec.signal, TOOL_WAIT_MS);
|
|
3861
|
+
const entry = sessionId === undefined ? { runs: [], recent: [] } : mediaSessions.get(sessionId);
|
|
3862
|
+
const run = entry.runs.find((r) => r.commandId === runId);
|
|
3863
|
+
const files = entry.recent.filter((r) => r.commandId === runId && existsSync(r.file)).map((r) => r.file);
|
|
3864
|
+
const option = planned?.option;
|
|
3865
|
+
const base = {
|
|
3866
|
+
kind: t.kind,
|
|
3867
|
+
...(t.kind === "video" ? { mode: t.mode } : {}),
|
|
3868
|
+
model: option?.modelId ?? run?.modelName ?? "",
|
|
3869
|
+
...(option?.accountName ? { account: option.accountName } : {}),
|
|
3870
|
+
run: runId,
|
|
3871
|
+
commandRow,
|
|
3872
|
+
files,
|
|
3873
|
+
folder: outputDirectory(cwd, planned?.outputDir ?? mediaChoices().settings.outputDir),
|
|
3874
|
+
...(Number.isInteger(run?.seconds) ? { seconds: run.seconds } : {}),
|
|
3875
|
+
...(typeof run?.resolution === "string" ? { resolution: run.resolution } : {}),
|
|
3876
|
+
...(typeof run?.aspect === "string" ? { aspect: run.aspect } : {}),
|
|
3877
|
+
};
|
|
3878
|
+
if (waited !== "settled" || run?.status === "running") {
|
|
3879
|
+
return {
|
|
3880
|
+
status: "running",
|
|
3881
|
+
...base,
|
|
3882
|
+
note:
|
|
3883
|
+
waited === "aborted"
|
|
3884
|
+
? "Stopped waiting. The generation keeps running in the background (and is billed); the file will be saved in the folder above and shown in this call's row."
|
|
3885
|
+
: "Still generating after 10 minutes. It keeps running in the background; the file will be saved in the folder above and shown in this call's row.",
|
|
3886
|
+
};
|
|
3887
|
+
}
|
|
3888
|
+
if (run === undefined || run.status === "error") {
|
|
3889
|
+
if (run === undefined || (run.videoId === undefined && NOT_SUBMITTED.has(run.errorCode))) refund();
|
|
3890
|
+
throw new Error(run?.error ?? "the generation failed");
|
|
3891
|
+
}
|
|
3892
|
+
const previews = t.kind === "image" ? await imagePreviews(files, agent, exec.signal) : [];
|
|
3893
|
+
return {
|
|
3894
|
+
status: "done",
|
|
3895
|
+
...base,
|
|
3896
|
+
...(typeof run.revisedPrompt === "string" ? { revisedPrompt: run.revisedPrompt } : {}),
|
|
3897
|
+
...(previews.length > 0 ? { previews } : {}),
|
|
3898
|
+
};
|
|
3899
|
+
};
|
|
3900
|
+
|
|
3901
|
+
/** The model-facing content: a short summary, then any image previews. */
|
|
3902
|
+
const renderGenerateMedia = (_args, value) => {
|
|
3903
|
+
const lines = [
|
|
3904
|
+
`${value.status === "done" ? "Saved" : "Running"}: ${value.kind}${value.mode && value.mode !== "generate" ? ` (${value.mode})` : ""} with ${value.model}${value.account ? ` (${value.account})` : ""}${value.seconds ? `, ${value.seconds} s` : ""}${value.resolution ? `, ${value.resolution}` : ""}${value.aspect ? `, ${value.aspect}` : ""}`,
|
|
3905
|
+
...value.files.map((file) => file),
|
|
3906
|
+
...(value.files.length === 0 ? [`Folder: ${value.folder}`] : []),
|
|
3907
|
+
...(value.revisedPrompt ? [`Revised prompt: ${value.revisedPrompt}`] : []),
|
|
3908
|
+
...(value.note ? [value.note] : []),
|
|
3909
|
+
];
|
|
3910
|
+
return [
|
|
3911
|
+
{ type: "text", text: lines.join("\n") },
|
|
3912
|
+
...(value.previews ?? []).map((p) => ({
|
|
3913
|
+
type: "image",
|
|
3914
|
+
attachment: {
|
|
3915
|
+
attachmentId: p.attachmentId,
|
|
3916
|
+
mediaType: p.mediaType,
|
|
3917
|
+
bytes: p.bytes,
|
|
3918
|
+
width: p.width,
|
|
3919
|
+
height: p.height,
|
|
3920
|
+
...(p.name === undefined ? {} : { name: p.name }),
|
|
3921
|
+
...(p.originalDimensions === undefined ? {} : { originalDimensions: { ...p.originalDimensions } }),
|
|
3922
|
+
},
|
|
3923
|
+
})),
|
|
3924
|
+
];
|
|
3925
|
+
};
|
|
3926
|
+
|
|
3927
|
+
const generateMediaTool = {
|
|
3928
|
+
name: GENERATE_MEDIA,
|
|
3929
|
+
description: GENERATE_MEDIA_TOOL.description,
|
|
3930
|
+
parameters: GENERATE_MEDIA_TOOL.parameters,
|
|
3931
|
+
output: {
|
|
3932
|
+
schema: GENERATE_MEDIA_OUTPUT,
|
|
3933
|
+
render: renderGenerateMedia,
|
|
3934
|
+
presentationMeta: (_args, value) => ({ run: value.run, status: value.status, commandRow: value.commandRow === true }),
|
|
3935
|
+
},
|
|
3936
|
+
execute: executeGenerateMedia,
|
|
3937
|
+
};
|
|
3938
|
+
|
|
3939
|
+
/** Register or unregister the tool to match the switch and the offered models. */
|
|
3940
|
+
const syncAgentTool = () => {
|
|
3941
|
+
let wanted = false;
|
|
3942
|
+
try {
|
|
3943
|
+
const { settings, options } = mediaChoices();
|
|
3944
|
+
wanted = settings.agentTool === true && options.image.length + options.video.length > 0;
|
|
3945
|
+
} catch {
|
|
3946
|
+
wanted = false;
|
|
3947
|
+
}
|
|
3948
|
+
if (wanted && toolOff === undefined && toolsCtx !== undefined) {
|
|
3949
|
+
try {
|
|
3950
|
+
const off = toolsCtx.tools.register(generateMediaTool);
|
|
3951
|
+
toolOff = typeof off === "function" ? off : () => {};
|
|
3952
|
+
ctx.logger?.info?.("cpa: generate_media tool registered");
|
|
3953
|
+
} catch (error) {
|
|
3954
|
+
ctx.logger?.warn?.(`cpa: could not register generate_media: ${error instanceof Error ? error.message : String(error)}`);
|
|
3955
|
+
}
|
|
3956
|
+
} else if (!wanted && toolOff !== undefined) {
|
|
3957
|
+
cancelConfirms();
|
|
3958
|
+
const off = toolOff;
|
|
3959
|
+
toolOff = undefined;
|
|
3960
|
+
try {
|
|
3961
|
+
off();
|
|
3962
|
+
} catch {
|
|
3963
|
+
// already gone with its scope
|
|
3964
|
+
}
|
|
3965
|
+
ctx.logger?.info?.("cpa: generate_media tool unregistered");
|
|
3966
|
+
}
|
|
3967
|
+
};
|
|
3968
|
+
// #endregion
|
|
3969
|
+
|
|
2082
3970
|
reschedule();
|
|
3971
|
+
resumeVideoJobs();
|
|
2083
3972
|
startupTimeout = outsideHmrTransaction(ctx, () =>
|
|
2084
3973
|
setTimeout(() => {
|
|
2085
3974
|
startupTimeout = undefined;
|
|
2086
|
-
void controller.refreshAll();
|
|
3975
|
+
void controller.refreshAll().finally(() => syncAgentTool());
|
|
2087
3976
|
}, 600)
|
|
2088
3977
|
);
|
|
2089
3978
|
|
|
@@ -2110,7 +3999,8 @@ function applyImpl(ctx, config) {
|
|
|
2110
3999
|
description: "Show CLIProxyAPI discovery and model synchronization status of every account.",
|
|
2111
4000
|
handler: async () => {
|
|
2112
4001
|
const payload = await controller.statusPayload();
|
|
2113
|
-
|
|
4002
|
+
const quotaText = renderQuotaText(await quota.snapshot());
|
|
4003
|
+
return { kind: "success", text: [renderStatusText(payload), quotaText].filter(Boolean).join("\n\n") };
|
|
2114
4004
|
},
|
|
2115
4005
|
});
|
|
2116
4006
|
const offRefresh = cmdCtx.commands.register({
|
|
@@ -2118,167 +4008,69 @@ function applyImpl(ctx, config) {
|
|
|
2118
4008
|
description: "Probe every CLIProxyAPI account now and refresh the synchronized DSH model catalog.",
|
|
2119
4009
|
handler: async () => {
|
|
2120
4010
|
const payload = await controller.refresh(undefined);
|
|
4011
|
+
syncAgentTool();
|
|
2121
4012
|
const failed = payload.accounts.some((a) => a.status.state === "error");
|
|
2122
4013
|
return { kind: failed ? "error" : "success", text: renderStatusText(payload) };
|
|
2123
4014
|
},
|
|
2124
4015
|
});
|
|
4016
|
+
const offVideo = cmdCtx.commands.register({
|
|
4017
|
+
name: "cpa-video",
|
|
4018
|
+
description: "Generate a video with a CLIProxyAPI video model: text only, one image as the first frame, reference images (--ref, or 2+ images), or pinned first / last frames (--first-last, --last). A submitted video cannot be cancelled.",
|
|
4019
|
+
input: { hint: "[-m <model>] [--aspect 16:9] [-s 1-15] [-r 480p|720p] [--ref|--first-last|--last] <prompt>", attachments: true },
|
|
4020
|
+
handler: runVideoCommand,
|
|
4021
|
+
});
|
|
4022
|
+
const offVideoEdit = cmdCtx.commands.register({
|
|
4023
|
+
name: "cpa-video-edit",
|
|
4024
|
+
description: "Edit any MP4 of up to 8.7 s with grok-imagine-video: attach it, or pass --video <path>, --url <public URL> or --from <command id> (default: this session's newest video). A submitted video cannot be cancelled.",
|
|
4025
|
+
input: { hint: "[--video <path> | --url <URL> | --from <command id>] [-m <model>] <what to change>", attachments: true },
|
|
4026
|
+
handler: runVideoEditCommand("edit"),
|
|
4027
|
+
});
|
|
4028
|
+
const offVideoExtend = cmdCtx.commands.register({
|
|
4029
|
+
name: "cpa-video-extend",
|
|
4030
|
+
description: "Continue any MP4 of 2-15 s by 2-10 seconds: attach it, or pass --video <path>, --url <public URL> or --from <command id> (default: this session's newest video). A submitted video cannot be cancelled.",
|
|
4031
|
+
input: { hint: "[--video <path> | --url <URL> | --from <command id>] [-m <model>] [-s 2-10] <what happens next>", attachments: true },
|
|
4032
|
+
handler: runVideoEditCommand("extend"),
|
|
4033
|
+
});
|
|
4034
|
+
const offVideoStatus = cmdCtx.commands.register({
|
|
4035
|
+
name: "cpa-video-status",
|
|
4036
|
+
description: "List this session's CLIProxyAPI video jobs and their state.",
|
|
4037
|
+
handler: runVideoStatusCommand,
|
|
4038
|
+
});
|
|
4039
|
+
const offImage = cmdCtx.commands.register({
|
|
4040
|
+
name: "cpa-image",
|
|
4041
|
+
description: "Generate an image with a CLIProxyAPI image model; attach images to edit them. Files go to the session's working directory.",
|
|
4042
|
+
input: { hint: "[-m <model>] [--aspect 16:9] [-n 1-4] <prompt>", attachments: true },
|
|
4043
|
+
handler: runImageCommand,
|
|
4044
|
+
});
|
|
2125
4045
|
return () => {
|
|
2126
4046
|
offStatus?.();
|
|
2127
4047
|
offRefresh?.();
|
|
4048
|
+
offImage?.();
|
|
4049
|
+
offVideo?.();
|
|
4050
|
+
offVideoStatus?.();
|
|
4051
|
+
offVideoEdit?.();
|
|
4052
|
+
offVideoExtend?.();
|
|
2128
4053
|
};
|
|
2129
4054
|
}, "cpa: commands");
|
|
2130
4055
|
});
|
|
2131
4056
|
|
|
2132
|
-
//
|
|
2133
|
-
ctx.inject(["tools"], (
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
},
|
|
2150
|
-
account: {
|
|
2151
|
-
type: "string",
|
|
2152
|
-
description:
|
|
2153
|
-
"Target account id (or its provider route ID / display name) for status, refresh, configure and remove_account. Defaults to the first account; refresh without it refreshes all.",
|
|
2154
|
-
},
|
|
2155
|
-
enabled: {
|
|
2156
|
-
type: "boolean",
|
|
2157
|
-
description:
|
|
2158
|
-
"Enable or disable the account. A disabled account keeps its settings but its models leave the picker. add_account defaults to enabled when baseURL is given.",
|
|
2159
|
-
},
|
|
2160
|
-
baseURL: {
|
|
2161
|
-
type: "string",
|
|
2162
|
-
description:
|
|
2163
|
-
"CLIProxyAPI base URL (e.g. http://127.0.0.1:8317). Empty means auto-discover locally.",
|
|
2164
|
-
},
|
|
2165
|
-
provider: {
|
|
2166
|
-
type: "string",
|
|
2167
|
-
description:
|
|
2168
|
-
"Provider route ID in llm-pi-ai; must be unique per account (default: cli-proxy, then cli-proxy-2, ...).",
|
|
2169
|
-
},
|
|
2170
|
-
displayName: {
|
|
2171
|
-
type: "string",
|
|
2172
|
-
description: "Account name shown in the DSH model picker.",
|
|
2173
|
-
},
|
|
2174
|
-
protocolMode: {
|
|
2175
|
-
type: "string",
|
|
2176
|
-
enum: ["auto", "fixed"],
|
|
2177
|
-
description:
|
|
2178
|
-
"'auto' (default) sends each model over its own family's protocol: Claude via Anthropic Messages, GPT/Codex/Grok via OpenAI Responses, others (incl. Gemini) via OpenAI Chat. 'fixed' sends every model over `protocol`.",
|
|
2179
|
-
},
|
|
2180
|
-
protocol: {
|
|
2181
|
-
type: "string",
|
|
2182
|
-
enum: ["openai-completions", "openai-responses", "anthropic-messages"],
|
|
2183
|
-
description: "Wire protocol used for every model when protocolMode is 'fixed'.",
|
|
2184
|
-
},
|
|
2185
|
-
selectedModels: {
|
|
2186
|
-
type: "array",
|
|
2187
|
-
items: { type: "string" },
|
|
2188
|
-
description:
|
|
2189
|
-
"Exact model IDs to show in the DSH model picker. Pass an empty array [] to show all discovered models.",
|
|
2190
|
-
},
|
|
2191
|
-
includePrefix: {
|
|
2192
|
-
type: "string",
|
|
2193
|
-
description: "Comma-separated model ID substrings/prefixes to include.",
|
|
2194
|
-
},
|
|
2195
|
-
excludePrefix: {
|
|
2196
|
-
type: "string",
|
|
2197
|
-
description: "Comma-separated model ID substrings/prefixes to exclude.",
|
|
2198
|
-
},
|
|
2199
|
-
apiKey: {
|
|
2200
|
-
type: "string",
|
|
2201
|
-
description:
|
|
2202
|
-
"CLIProxyAPI Bearer key, stored in the DSH credentials service under the account's apiKeyEnv.",
|
|
2203
|
-
},
|
|
2204
|
-
apiKeyEnv: {
|
|
2205
|
-
type: "string",
|
|
2206
|
-
description:
|
|
2207
|
-
"Credential reference name holding the account's key (default: CLI_PROXY_API_KEY, then CLI_PROXY_API_KEY_2, ...).",
|
|
2208
|
-
},
|
|
2209
|
-
modelOverrides: {
|
|
2210
|
-
type: "object",
|
|
2211
|
-
description:
|
|
2212
|
-
"Per-model overrides keyed by model ID, replacing the account's whole map: { [modelId]: { contextWindow?: number, maxTokens?: number, image?: boolean, reasoningLevels?: string[], protocol?: 'openai-completions' | 'openai-responses' | 'anthropic-messages' } }.",
|
|
2213
|
-
},
|
|
2214
|
-
},
|
|
2215
|
-
},
|
|
2216
|
-
output: {
|
|
2217
|
-
schema: {
|
|
2218
|
-
type: "object",
|
|
2219
|
-
additionalProperties: false,
|
|
2220
|
-
required: ["accounts", "summary"],
|
|
2221
|
-
properties: {
|
|
2222
|
-
account: { type: "string" },
|
|
2223
|
-
accounts: {
|
|
2224
|
-
type: "array",
|
|
2225
|
-
items: {
|
|
2226
|
-
type: "object",
|
|
2227
|
-
additionalProperties: false,
|
|
2228
|
-
required: ["id", "displayName", "provider", "enabled", "state", "modelCount", "totalDiscovered"],
|
|
2229
|
-
properties: {
|
|
2230
|
-
id: { type: "string" },
|
|
2231
|
-
displayName: { type: "string" },
|
|
2232
|
-
provider: { type: "string" },
|
|
2233
|
-
enabled: { type: "boolean" },
|
|
2234
|
-
state: { type: "string" },
|
|
2235
|
-
endpoint: { type: "string" },
|
|
2236
|
-
modelCount: { type: "integer" },
|
|
2237
|
-
totalDiscovered: { type: "integer" },
|
|
2238
|
-
error: { type: "string" },
|
|
2239
|
-
},
|
|
2240
|
-
},
|
|
2241
|
-
},
|
|
2242
|
-
summary: { type: "string" },
|
|
2243
|
-
},
|
|
2244
|
-
},
|
|
2245
|
-
render: (_args, value) => [{ type: "text", text: value.summary }],
|
|
2246
|
-
},
|
|
2247
|
-
async execute(args) {
|
|
2248
|
-
const input = args ?? {};
|
|
2249
|
-
const ref = typeof input.account === "string" && input.account.trim() !== "" ? input.account.trim() : undefined;
|
|
2250
|
-
let target;
|
|
2251
|
-
if (input.action === "refresh") {
|
|
2252
|
-
target = (await controller.refresh(ref)).account;
|
|
2253
|
-
} else if (input.action === "configure") {
|
|
2254
|
-
target = (await controller.updateAccount(ref, accountPatchFromInput(input))).account;
|
|
2255
|
-
} else if (input.action === "add_account") {
|
|
2256
|
-
const patch = accountPatchFromInput(input);
|
|
2257
|
-
if (patch.enabled === undefined) {
|
|
2258
|
-
patch.enabled = typeof patch.baseURL === "string" && patch.baseURL.trim() !== "";
|
|
2259
|
-
}
|
|
2260
|
-
target = (await controller.addAccount(patch)).createdId;
|
|
2261
|
-
} else if (input.action === "remove_account") {
|
|
2262
|
-
if (ref === undefined) throw new Error("remove_account needs `account`");
|
|
2263
|
-
await controller.removeAccount(ref);
|
|
2264
|
-
} else if (ref !== undefined) {
|
|
2265
|
-
target = controller.findAccount(ref).id;
|
|
2266
|
-
}
|
|
2267
|
-
const payload = await controller.statusPayload();
|
|
2268
|
-
return {
|
|
2269
|
-
...(target === undefined ? {} : { account: target }),
|
|
2270
|
-
accounts: payload.accounts.map(summarizeAccount),
|
|
2271
|
-
summary: renderStatusText(payload, input.action === "status" ? target : undefined),
|
|
2272
|
-
};
|
|
2273
|
-
},
|
|
2274
|
-
presentCall: (args) => ({
|
|
2275
|
-
card: "generic",
|
|
2276
|
-
title: `CLIProxyAPI: ${args?.action ?? "status"}${args?.account ? ` (${args.account})` : ""}`,
|
|
2277
|
-
kind: args?.action === "status" ? "read" : "execute",
|
|
2278
|
-
rawInput: args,
|
|
2279
|
-
}),
|
|
2280
|
-
});
|
|
2281
|
-
}, "cpa: tool");
|
|
4057
|
+
// The optional agent tool (registered only while switched on; see syncAgentTool).
|
|
4058
|
+
ctx.inject(["tools"], (scope) => {
|
|
4059
|
+
scope.effect(() => {
|
|
4060
|
+
toolsCtx = scope;
|
|
4061
|
+
syncAgentTool();
|
|
4062
|
+
return () => {
|
|
4063
|
+
cancelConfirms();
|
|
4064
|
+
const off = toolOff;
|
|
4065
|
+
toolOff = undefined;
|
|
4066
|
+
toolsCtx = undefined;
|
|
4067
|
+
try {
|
|
4068
|
+
off?.();
|
|
4069
|
+
} catch {
|
|
4070
|
+
// disposed with its scope
|
|
4071
|
+
}
|
|
4072
|
+
};
|
|
4073
|
+
}, "cpa: agent tool");
|
|
2282
4074
|
});
|
|
2283
4075
|
|
|
2284
4076
|
// Register HTTP API Routes on webServer for the Web UI
|
|
@@ -2292,7 +4084,10 @@ function applyImpl(ctx, config) {
|
|
|
2292
4084
|
if (!admitRequest(ctx, req, res, method)) return;
|
|
2293
4085
|
try {
|
|
2294
4086
|
const body = method === "POST" ? (await readJsonBody(req)) ?? {} : {};
|
|
2295
|
-
|
|
4087
|
+
const result = await run(body, req);
|
|
4088
|
+
// Account, model and Generation-tab changes may switch the agent tool on or off.
|
|
4089
|
+
if (method === "POST") syncAgentTool();
|
|
4090
|
+
sendJson(res, 200, result);
|
|
2296
4091
|
} catch (error) {
|
|
2297
4092
|
sendJson(res, errorStatus, {
|
|
2298
4093
|
ok: false,
|
|
@@ -2341,6 +4136,193 @@ function applyImpl(ctx, config) {
|
|
|
2341
4136
|
),
|
|
2342
4137
|
route(ROUTES.addAccount, "POST", (body) => controller.addAccount(accountPatchFromInput(body)), 400),
|
|
2343
4138
|
route(ROUTES.removeAccount, "POST", (body) => controller.removeAccount(body.accountId), 400),
|
|
4139
|
+
route(ROUTES.testAccount, "POST", (body) => controller.testConnection(body.accountId, { baseURL: body.baseURL, apiKey: body.apiKey }), 400),
|
|
4140
|
+
route(ROUTES.quotaStatus, "GET", async () => ({ ok: true, quota: await quota.snapshot() })),
|
|
4141
|
+
route(ROUTES.quotaRefresh, "POST", async (body) => ({
|
|
4142
|
+
ok: true,
|
|
4143
|
+
quota: await quota.refresh(typeof body.accountId === "string" && body.accountId !== "" ? body.accountId : undefined, { wait: false }),
|
|
4144
|
+
})),
|
|
4145
|
+
route(
|
|
4146
|
+
ROUTES.quotaSave,
|
|
4147
|
+
"POST",
|
|
4148
|
+
async (body) => {
|
|
4149
|
+
if (typeof body.accountId !== "string" || body.accountId === "") {
|
|
4150
|
+
throw new Error("dsh-cliproxyapi: saving a management key needs accountId");
|
|
4151
|
+
}
|
|
4152
|
+
const managementKey = typeof body.managementKey === "string" ? body.managementKey.trim() : "";
|
|
4153
|
+
return {
|
|
4154
|
+
ok: true,
|
|
4155
|
+
quota: await quota.saveKey(body.accountId, {
|
|
4156
|
+
...(managementKey === "" ? {} : { managementKey }),
|
|
4157
|
+
...(body.clear === true ? { clear: true } : {}),
|
|
4158
|
+
}, { wait: false }),
|
|
4159
|
+
};
|
|
4160
|
+
},
|
|
4161
|
+
400
|
|
4162
|
+
),
|
|
4163
|
+
route(ROUTES.mediaStatus, "GET", async () => ({ ok: true, media: mediaStatus() })),
|
|
4164
|
+
route(ROUTES.mediaSave, "POST", saveMedia, 400),
|
|
4165
|
+
route(
|
|
4166
|
+
ROUTES.mediaDiagnose,
|
|
4167
|
+
"POST",
|
|
4168
|
+
async (body) => {
|
|
4169
|
+
if (typeof body.accountId !== "string" || body.accountId === "") throw new MediaError("invalid-account");
|
|
4170
|
+
return { ok: true, diagnostic: await controller.diagnoseMedia(body.accountId) };
|
|
4171
|
+
},
|
|
4172
|
+
400
|
|
4173
|
+
),
|
|
4174
|
+
route(
|
|
4175
|
+
ROUTES.mediaSession,
|
|
4176
|
+
"GET",
|
|
4177
|
+
async (_body, req) => {
|
|
4178
|
+
const raw = new URL(req.url ?? "/", "http://host").searchParams.get("sessionId");
|
|
4179
|
+
const sessionId = raw === null || raw === "" ? undefined : validSessionId(raw);
|
|
4180
|
+
if (raw !== null && raw !== "" && sessionId === undefined) throw new MediaError("invalid-session");
|
|
4181
|
+
// Right after a start the catalogs may not be read yet; sync once
|
|
4182
|
+
// instead of letting the composer button wait for its next retry.
|
|
4183
|
+
await mediaChoicesFor("any");
|
|
4184
|
+
return { ok: true, session: sessionView(sessionId) };
|
|
4185
|
+
},
|
|
4186
|
+
400
|
|
4187
|
+
),
|
|
4188
|
+
route(ROUTES.mediaSessionSave, "POST", async (body) => saveSessionPick(body), 400),
|
|
4189
|
+
// The user's answer to a generate_media confirmation shown in its chat row.
|
|
4190
|
+
route(
|
|
4191
|
+
ROUTES.mediaConfirm,
|
|
4192
|
+
"POST",
|
|
4193
|
+
async (body) => {
|
|
4194
|
+
const sessionId = validSessionId(body.sessionId);
|
|
4195
|
+
const commandId = validCommandId(body.commandId);
|
|
4196
|
+
if (sessionId === undefined || commandId === undefined || typeof body.allow !== "boolean") throw new MediaError("invalid-request");
|
|
4197
|
+
const waiting = pendingConfirms.get(commandId);
|
|
4198
|
+
if (waiting === undefined || waiting.sessionId !== sessionId) throw new MediaError("confirm-expired");
|
|
4199
|
+
waiting.settle(body.allow ? "allowed" : "rejected");
|
|
4200
|
+
return { ok: true };
|
|
4201
|
+
},
|
|
4202
|
+
400
|
|
4203
|
+
),
|
|
4204
|
+
// One slash-command run's record and files (its chat row shows them).
|
|
4205
|
+
// The absolute paths let the page open or reveal a file through DSH's
|
|
4206
|
+
// own Session Remote (`session.openWorkspacePath`), which re-verifies
|
|
4207
|
+
// them on the Host.
|
|
4208
|
+
route(
|
|
4209
|
+
ROUTES.mediaCommand,
|
|
4210
|
+
"GET",
|
|
4211
|
+
async (_body, req) => {
|
|
4212
|
+
const params = new URL(req.url ?? "/", "http://host").searchParams;
|
|
4213
|
+
const sessionId = validSessionId(params.get("sessionId") ?? "");
|
|
4214
|
+
const commandId = validCommandId(params.get("commandId") ?? "");
|
|
4215
|
+
if (sessionId === undefined || commandId === undefined) throw new MediaError("invalid-request");
|
|
4216
|
+
const entry = mediaSessions.get(sessionId);
|
|
4217
|
+
const files = entry.recent
|
|
4218
|
+
.filter((r) => r.commandId === commandId && existsSync(r.file))
|
|
4219
|
+
.map((r) => ({ id: r.id, kind: r.kind, name: basename(r.file), path: r.file }));
|
|
4220
|
+
let run = entry.runs.find((r) => r.commandId === commandId) ?? null;
|
|
4221
|
+
// A run recorded as running that no job of this activation owns was
|
|
4222
|
+
// cut off by a reload or a restart.
|
|
4223
|
+
if (run?.status === "running" && !mediaJobs.has(commandId)) {
|
|
4224
|
+
run = { ...run, status: "error", errorCode: "interrupted", error: mediaFailureText(new MediaError("interrupted"), run.kind) };
|
|
4225
|
+
} else if (run?.status === "running" && videoLive.has(commandId)) {
|
|
4226
|
+
// A video's phase (submitting / queued / running / downloading) and CPA's progress, if it reports one.
|
|
4227
|
+
run = { ...run, ...videoLive.get(commandId) };
|
|
4228
|
+
}
|
|
4229
|
+
// Which file manager "show in folder" opens, to name the menu item.
|
|
4230
|
+
const desktop = process.platform === "win32" ? "explorer" : process.platform === "darwin" ? "finder" : "filemanager";
|
|
4231
|
+
// Edit / extend this video: by xAI's short-lived link while it is
|
|
4232
|
+
// remembered, else by sending its file inline when small enough.
|
|
4233
|
+
let actions;
|
|
4234
|
+
if (run?.kind === "video" && run.status === "done") {
|
|
4235
|
+
const source = usableSource(commandId);
|
|
4236
|
+
const video = files.find((f) => f.kind === "video");
|
|
4237
|
+
let size;
|
|
4238
|
+
try {
|
|
4239
|
+
size = video === undefined ? undefined : statSync(video.path).size;
|
|
4240
|
+
} catch {
|
|
4241
|
+
size = undefined;
|
|
4242
|
+
}
|
|
4243
|
+
if (source !== undefined || (size !== undefined && size <= VIDEO_UPLOAD_MAX_BYTES)) {
|
|
4244
|
+
const seconds = source?.seconds ?? run.seconds;
|
|
4245
|
+
const accountId = source?.accountId ?? String(run.model).split("/", 1)[0];
|
|
4246
|
+
actions = {
|
|
4247
|
+
edit: seconds === undefined || seconds <= EDIT_SOURCE_MAX_SECONDS,
|
|
4248
|
+
extend:
|
|
4249
|
+
(seconds === undefined || (seconds >= EXTEND_SOURCE_SECONDS.min && seconds <= EXTEND_SOURCE_SECONDS.max)) &&
|
|
4250
|
+
!unsupportedRoutes.has(`${accountId}/extensions`),
|
|
4251
|
+
...(source === undefined ? {} : { expiresAt: new Date(source.at + VIDEO_SOURCE_TTL_MS).toISOString() }),
|
|
4252
|
+
};
|
|
4253
|
+
}
|
|
4254
|
+
}
|
|
4255
|
+
// A generate_media call waiting for the user's confirmation in this row.
|
|
4256
|
+
const waiting = pendingConfirms.get(commandId);
|
|
4257
|
+
const confirm = waiting !== undefined && waiting.sessionId === sessionId ? { kind: waiting.kind, en: waiting.en, zh: waiting.zh, expiresAt: waiting.expiresAt } : undefined;
|
|
4258
|
+
return {
|
|
4259
|
+
ok: true,
|
|
4260
|
+
run,
|
|
4261
|
+
files,
|
|
4262
|
+
desktop,
|
|
4263
|
+
...(files.length > 0 ? { folderPath: dirname(files[0].path) } : {}),
|
|
4264
|
+
...(actions === undefined ? {} : { actions }),
|
|
4265
|
+
...(confirm === undefined ? {} : { confirm }),
|
|
4266
|
+
};
|
|
4267
|
+
},
|
|
4268
|
+
400
|
|
4269
|
+
),
|
|
4270
|
+
// Serves only files this plugin wrote and recorded for that session.
|
|
4271
|
+
webCtx.webServer.register({
|
|
4272
|
+
kind: "exact",
|
|
4273
|
+
path: ROUTES.mediaFile,
|
|
4274
|
+
handler: async (req, res) => {
|
|
4275
|
+
if (!admitRequest(ctx, req, res, "GET")) return;
|
|
4276
|
+
const params = new URL(req.url ?? "/", "http://host").searchParams;
|
|
4277
|
+
const sessionId = validSessionId(params.get("sessionId") ?? "");
|
|
4278
|
+
const record = sessionId === undefined ? undefined : mediaSessions.get(sessionId).recent.find((r) => r.id === params.get("id"));
|
|
4279
|
+
const type = record === undefined ? undefined : MEDIA_FILE_TYPES[extname(record.file).toLowerCase()];
|
|
4280
|
+
if (record === undefined || type === undefined || !existsSync(record.file)) {
|
|
4281
|
+
sendJson(res, 404, { ok: false, error: "not-found" });
|
|
4282
|
+
return;
|
|
4283
|
+
}
|
|
4284
|
+
try {
|
|
4285
|
+
if (!type.startsWith("video/")) {
|
|
4286
|
+
const bytes = readFileSync(record.file);
|
|
4287
|
+
res.writeHead(200, {
|
|
4288
|
+
"cache-control": "no-store",
|
|
4289
|
+
"content-type": type,
|
|
4290
|
+
"content-length": String(bytes.byteLength),
|
|
4291
|
+
"x-content-type-options": "nosniff",
|
|
4292
|
+
});
|
|
4293
|
+
res.end(bytes);
|
|
4294
|
+
return;
|
|
4295
|
+
}
|
|
4296
|
+
// Videos are streamed, with single byte ranges so <video> can seek.
|
|
4297
|
+
const size = statSync(record.file).size;
|
|
4298
|
+
const range = byteRange(req.headers?.range, size);
|
|
4299
|
+
if (range === null) {
|
|
4300
|
+
res.writeHead(416, { "content-range": `bytes */${size}` });
|
|
4301
|
+
res.end();
|
|
4302
|
+
return;
|
|
4303
|
+
}
|
|
4304
|
+
const [start, end] = range ?? [0, size - 1];
|
|
4305
|
+
res.writeHead(range === undefined ? 200 : 206, {
|
|
4306
|
+
"cache-control": "no-store",
|
|
4307
|
+
"content-type": type,
|
|
4308
|
+
"accept-ranges": "bytes",
|
|
4309
|
+
"content-length": String(size === 0 ? 0 : end - start + 1),
|
|
4310
|
+
...(range === undefined ? {} : { "content-range": `bytes ${start}-${end}/${size}` }),
|
|
4311
|
+
"x-content-type-options": "nosniff",
|
|
4312
|
+
});
|
|
4313
|
+
if (size === 0) {
|
|
4314
|
+
res.end();
|
|
4315
|
+
return;
|
|
4316
|
+
}
|
|
4317
|
+
const stream = createReadStream(record.file, { start, end });
|
|
4318
|
+
stream.on("error", () => res.destroy?.());
|
|
4319
|
+
res.on?.("close", () => stream.destroy());
|
|
4320
|
+
stream.pipe(res);
|
|
4321
|
+
} catch {
|
|
4322
|
+
sendJson(res, 404, { ok: false, error: "not-found" });
|
|
4323
|
+
}
|
|
4324
|
+
},
|
|
4325
|
+
}),
|
|
2344
4326
|
];
|
|
2345
4327
|
return () => {
|
|
2346
4328
|
for (const dispose of disposers) dispose?.();
|