@vibedev-si/dsh-vibedev 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -10
- package/THIRD-PARTY-NOTICES.txt +1 -1
- package/client/client.js +56 -5
- package/lib/account/index.d.ts +50 -1
- package/lib/audio/request.d.ts +141 -0
- package/lib/gateway/assets.d.ts +2 -2
- package/lib/gateway/catalog.d.ts +75 -4
- package/lib/gateway/http.d.ts +5 -0
- package/lib/index.d.ts +7 -2
- package/lib/index.js +1211 -160
- package/lib/llm/adapter.d.ts +2 -0
- package/lib/llm/index.d.ts +2 -0
- package/lib/pricing.d.ts +45 -9
- package/lib/runtime.d.ts +24 -1
- package/lib/service.d.ts +81 -7
- package/lib/tasks/remote.d.ts +13 -0
- package/lib/tasks/store.d.ts +17 -0
- package/lib/tasks/tracker.d.ts +15 -0
- package/lib/tools/audio.d.ts +3 -2
- package/lib/tools/models.d.ts +7 -0
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -19,15 +19,41 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
|
19
19
|
//#region src/account/index.ts
|
|
20
20
|
/** Path prefix of the plugin's Host routes. */
|
|
21
21
|
const ACCOUNT_ROUTE_PREFIX = "/api/dsh-vibedev/account";
|
|
22
|
-
/**
|
|
23
|
-
|
|
22
|
+
/**
|
|
23
|
+
* How long a balance read is reused as it is. After that the cached value is
|
|
24
|
+
* still answered at once while a new read runs in the background (the page
|
|
25
|
+
* hears the result through the long poll), so a page never waits on the gateway
|
|
26
|
+
* when it already has a balance.
|
|
27
|
+
*/
|
|
28
|
+
const BALANCE_TTL_MS = 3e4;
|
|
29
|
+
/** How long `?since=` holds a request open when nothing changes. */
|
|
30
|
+
const WAIT_MS = 25e3;
|
|
31
|
+
/**
|
|
32
|
+
* When the balance is read again after a charge: the gateway settles a charge
|
|
33
|
+
* shortly after the request ends, so it is read a moment later, and once more
|
|
34
|
+
* if the first read still shows the old amount.
|
|
35
|
+
*/
|
|
36
|
+
const SETTLE_READS_MS = [1500, 8e3];
|
|
37
|
+
/** Reads after charges are at least this far apart. */
|
|
38
|
+
const MIN_READ_SPACING_MS = 5e3;
|
|
24
39
|
const record$8 = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
25
40
|
const text$8 = (value) => typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
26
41
|
/** The account state behind the pages. */
|
|
27
42
|
var AccountService = class {
|
|
28
43
|
options;
|
|
29
44
|
balanceCache;
|
|
45
|
+
/** A background read of a stale balance, so only one runs at a time. */
|
|
46
|
+
revalidating;
|
|
30
47
|
now;
|
|
48
|
+
revision = 0;
|
|
49
|
+
/** What the last read showed, to tell whether the next one changed anything. */
|
|
50
|
+
lastSeen;
|
|
51
|
+
waiters = /* @__PURE__ */ new Set();
|
|
52
|
+
settleTimers = [];
|
|
53
|
+
/** Grows with every credential change; a read started before one is dropped. */
|
|
54
|
+
epoch = 0;
|
|
55
|
+
/** When the last read after a charge started, to space them out. */
|
|
56
|
+
lastSpendReadAt = Number.NEGATIVE_INFINITY;
|
|
31
57
|
/**
|
|
32
58
|
* @param options - origin, credential chain, the plugin's sign-in and the route's model state.
|
|
33
59
|
*/
|
|
@@ -35,6 +61,73 @@ var AccountService = class {
|
|
|
35
61
|
this.options = options;
|
|
36
62
|
this.now = options.now ?? Date.now;
|
|
37
63
|
}
|
|
64
|
+
timeout(callback, ms) {
|
|
65
|
+
return (this.options.setTimeout ?? ((run, delay) => {
|
|
66
|
+
const handle = globalThis.setTimeout(run, delay);
|
|
67
|
+
handle.unref?.();
|
|
68
|
+
return handle;
|
|
69
|
+
}))(callback, ms);
|
|
70
|
+
}
|
|
71
|
+
cancelTimeout(handle) {
|
|
72
|
+
(this.options.clearTimeout ?? ((value) => {
|
|
73
|
+
globalThis.clearTimeout(value);
|
|
74
|
+
}))(handle);
|
|
75
|
+
}
|
|
76
|
+
/** Something the pages show changed: answer every waiting `?since=` request. */
|
|
77
|
+
changed() {
|
|
78
|
+
this.revision++;
|
|
79
|
+
const waiting = [...this.waiters];
|
|
80
|
+
this.waiters.clear();
|
|
81
|
+
for (const wake of waiting) wake();
|
|
82
|
+
}
|
|
83
|
+
/** The account changed hands (sign-in, sign-out, switch): forget the balance and tell the pages. */
|
|
84
|
+
credentialChanged() {
|
|
85
|
+
this.epoch++;
|
|
86
|
+
this.balanceCache = void 0;
|
|
87
|
+
this.revalidating = void 0;
|
|
88
|
+
this.lastSeen = void 0;
|
|
89
|
+
this.changed();
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* A request that may have cost money ended (a chat reply, a generation, a
|
|
93
|
+
* transcription): read the balance again shortly, and once more later if the
|
|
94
|
+
* charge had not settled yet. Repeated calls in a burst share the reads.
|
|
95
|
+
*/
|
|
96
|
+
spent() {
|
|
97
|
+
for (const handle of this.settleTimers) this.cancelTimeout(handle);
|
|
98
|
+
const before = this.lastSeen;
|
|
99
|
+
const spacing = Math.max(0, this.lastSpendReadAt + MIN_READ_SPACING_MS - this.now());
|
|
100
|
+
this.settleTimers = SETTLE_READS_MS.map((delay, index) => this.timeout(() => {
|
|
101
|
+
if (index > 0 && this.lastSeen !== before) return;
|
|
102
|
+
this.readNow();
|
|
103
|
+
}, Math.max(delay, spacing + (index === 0 ? 0 : delay - SETTLE_READS_MS[0]))));
|
|
104
|
+
}
|
|
105
|
+
/** Read the balance of the current credential now (the pages hear about a change through the long poll). */
|
|
106
|
+
async readNow() {
|
|
107
|
+
this.lastSpendReadAt = this.now();
|
|
108
|
+
const credential = await this.options.chain.resolve().catch(() => void 0);
|
|
109
|
+
if (credential !== void 0) await this.me(credential.token, true);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Wait until the view changes past `since`, or a while.
|
|
113
|
+
* @param since - the revision the page shows.
|
|
114
|
+
* @param signal - the page went away.
|
|
115
|
+
* @returns once there is something new, or the wait ran out.
|
|
116
|
+
*/
|
|
117
|
+
async wait(since, signal) {
|
|
118
|
+
if (since !== this.revision || signal?.aborted === true) return;
|
|
119
|
+
await new Promise((resolve) => {
|
|
120
|
+
const done = () => {
|
|
121
|
+
this.waiters.delete(done);
|
|
122
|
+
this.cancelTimeout(timer);
|
|
123
|
+
signal?.removeEventListener("abort", done);
|
|
124
|
+
resolve();
|
|
125
|
+
};
|
|
126
|
+
const timer = this.timeout(done, WAIT_MS);
|
|
127
|
+
this.waiters.add(done);
|
|
128
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
129
|
+
});
|
|
130
|
+
}
|
|
38
131
|
/** Links to the gateway's pages. */
|
|
39
132
|
links() {
|
|
40
133
|
const origin = this.options.origin;
|
|
@@ -62,20 +155,39 @@ var AccountService = class {
|
|
|
62
155
|
...pending === void 0 ? {} : { pending },
|
|
63
156
|
...attempt === void 0 ? {} : { attempt }
|
|
64
157
|
};
|
|
65
|
-
if (credential === void 0) return
|
|
66
|
-
|
|
67
|
-
|
|
158
|
+
if (credential === void 0) return {
|
|
159
|
+
...base,
|
|
160
|
+
revision: this.revision
|
|
161
|
+
};
|
|
162
|
+
const [grantUser, me] = await Promise.all([credential.kind === "plugin" ? this.options.login.user().catch(() => void 0) : Promise.resolve(void 0), this.me(credential.token, fresh)]);
|
|
68
163
|
const user = me.user ?? userOf(grantUser);
|
|
69
164
|
return {
|
|
70
165
|
...base,
|
|
71
166
|
...user === void 0 ? {} : { user },
|
|
72
|
-
...me.balance === void 0 ? {} : { balance: me.balance }
|
|
167
|
+
...me.balance === void 0 ? {} : { balance: me.balance },
|
|
168
|
+
revision: this.revision
|
|
73
169
|
};
|
|
74
170
|
}
|
|
75
|
-
/**
|
|
171
|
+
/**
|
|
172
|
+
* `GET /v1/account/auth/me` for one token. A read younger than the TTL is
|
|
173
|
+
* reused; an older one is answered at once while a new read runs behind it.
|
|
174
|
+
*/
|
|
76
175
|
me(token, fresh) {
|
|
77
176
|
const cached = this.balanceCache;
|
|
78
|
-
if (!fresh && cached !== void 0 && cached.token === token
|
|
177
|
+
if (!fresh && cached !== void 0 && cached.token === token) {
|
|
178
|
+
if (this.now() - cached.at >= BALANCE_TTL_MS && this.revalidating === void 0) {
|
|
179
|
+
const next = this.read(token);
|
|
180
|
+
this.revalidating = next;
|
|
181
|
+
next.finally(() => {
|
|
182
|
+
if (this.revalidating === next) this.revalidating = void 0;
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
return cached.value;
|
|
186
|
+
}
|
|
187
|
+
return this.read(token);
|
|
188
|
+
}
|
|
189
|
+
read(token) {
|
|
190
|
+
const epoch = this.epoch;
|
|
79
191
|
const value = (async () => {
|
|
80
192
|
try {
|
|
81
193
|
const response = await (this.options.fetch ?? globalThis.fetch)(`${this.options.origin}/v1/account/auth/me`, {
|
|
@@ -111,13 +223,28 @@ var AccountService = class {
|
|
|
111
223
|
return {};
|
|
112
224
|
}
|
|
113
225
|
})();
|
|
114
|
-
this.balanceCache
|
|
226
|
+
const previous = this.balanceCache;
|
|
227
|
+
if (previous === void 0 || previous.token !== token) this.balanceCache = {
|
|
115
228
|
token,
|
|
116
229
|
at: this.now(),
|
|
117
230
|
value
|
|
118
231
|
};
|
|
119
232
|
value.then((read) => {
|
|
120
|
-
if (
|
|
233
|
+
if (epoch !== this.epoch) return;
|
|
234
|
+
if (read.balance === void 0) {
|
|
235
|
+
if (this.balanceCache?.value === value) this.balanceCache = void 0;
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
this.balanceCache = {
|
|
239
|
+
token,
|
|
240
|
+
at: this.now(),
|
|
241
|
+
value
|
|
242
|
+
};
|
|
243
|
+
const seen = JSON.stringify(read);
|
|
244
|
+
if (this.lastSeen !== void 0 && this.lastSeen !== seen) {
|
|
245
|
+
this.lastSeen = seen;
|
|
246
|
+
this.changed();
|
|
247
|
+
} else this.lastSeen = seen;
|
|
121
248
|
});
|
|
122
249
|
return value;
|
|
123
250
|
}
|
|
@@ -145,9 +272,17 @@ var AccountService = class {
|
|
|
145
272
|
async signOut() {
|
|
146
273
|
if ((await this.options.chain.resolve().catch(() => void 0))?.kind === "account") return false;
|
|
147
274
|
await this.options.login.signOut();
|
|
148
|
-
this.
|
|
275
|
+
this.credentialChanged();
|
|
149
276
|
return true;
|
|
150
277
|
}
|
|
278
|
+
/** Stop the pending reads after a charge (the plugin is going away). */
|
|
279
|
+
dispose() {
|
|
280
|
+
for (const handle of this.settleTimers) this.cancelTimeout(handle);
|
|
281
|
+
this.settleTimers = [];
|
|
282
|
+
const waiting = [...this.waiters];
|
|
283
|
+
this.waiters.clear();
|
|
284
|
+
for (const wake of waiting) wake();
|
|
285
|
+
}
|
|
151
286
|
};
|
|
152
287
|
function userOf(user) {
|
|
153
288
|
if (user === void 0) return void 0;
|
|
@@ -189,7 +324,10 @@ function accountRoutes(account) {
|
|
|
189
324
|
requestBody: "buffered",
|
|
190
325
|
fetch: async (request) => {
|
|
191
326
|
try {
|
|
192
|
-
|
|
327
|
+
const query = new URL(request.url, "http://host").searchParams;
|
|
328
|
+
const since = query.get("since");
|
|
329
|
+
if (since !== null && /^\d{1,15}$/.test(since)) await account.wait(Number(since), request.signal);
|
|
330
|
+
return json(200, await account.view(query.get("fresh") === "1"));
|
|
193
331
|
} catch (error) {
|
|
194
332
|
return failure(error);
|
|
195
333
|
}
|
|
@@ -2761,6 +2899,7 @@ var GatewayAdapter = class extends LlmAdapter {
|
|
|
2761
2899
|
const upstream = options.signal === void 0 ? consumer.signal : AbortSignal.any([options.signal, consumer.signal]);
|
|
2762
2900
|
const streamIdleTimeoutMs = this.config.streamIdleTimeoutMs ?? 3e5;
|
|
2763
2901
|
const watchdog = _usingCtx$1.u(idleWatchdog(upstream, streamIdleTimeoutMs, "LLM_STREAM_IDLE_TIMEOUT"));
|
|
2902
|
+
let answered = false;
|
|
2764
2903
|
try {
|
|
2765
2904
|
const containsImage = options.messages.some((message) => contentHasImage(message.content));
|
|
2766
2905
|
if (containsImage && !model.input.includes("image")) throw new LlmError(`VibeDev model "${model.id}" does not support image input`, "UNSUPPORTED_CONTENT");
|
|
@@ -2797,6 +2936,7 @@ var GatewayAdapter = class extends LlmAdapter {
|
|
|
2797
2936
|
...init,
|
|
2798
2937
|
redirect: "error"
|
|
2799
2938
|
});
|
|
2939
|
+
answered = true;
|
|
2800
2940
|
response = {
|
|
2801
2941
|
status: received.status,
|
|
2802
2942
|
headers: Object.fromEntries(received.headers.entries())
|
|
@@ -2882,6 +3022,9 @@ var GatewayAdapter = class extends LlmAdapter {
|
|
|
2882
3022
|
throw error;
|
|
2883
3023
|
} finally {
|
|
2884
3024
|
consumer.abort("gateway stream consumer stopped");
|
|
3025
|
+
if (answered) try {
|
|
3026
|
+
this.config.onRequestEnded?.();
|
|
3027
|
+
} catch {}
|
|
2885
3028
|
}
|
|
2886
3029
|
} catch (_) {
|
|
2887
3030
|
_usingCtx$1.e = _;
|
|
@@ -3034,7 +3177,8 @@ function installGatewayModels(ctx, options) {
|
|
|
3034
3177
|
resolveImageAccess: (attachments, ref) => resolveImageAttachmentAccess(attachments, (hostPath) => ctx.get("fs")?.processPathFromHostPath(hostPath), ref),
|
|
3035
3178
|
onReplayDegrade: ({ provider, model, reason }) => {
|
|
3036
3179
|
ctx.logger.warn(`dsh-vibedev: unusable replay state on assistant history for "${provider}/${model}"; sending provider-neutral content (${reason})`);
|
|
3037
|
-
}
|
|
3180
|
+
},
|
|
3181
|
+
...options.onRequestEnded === void 0 ? {} : { onRequestEnded: options.onRequestEnded }
|
|
3038
3182
|
});
|
|
3039
3183
|
const registration = ctx.llm.registerAdapter([VIBEDEV_ROUTE], adapter);
|
|
3040
3184
|
ctx.inject(["settings"], (child) => {
|
|
@@ -3380,12 +3524,21 @@ var GatewayHttp = class {
|
|
|
3380
3524
|
*/
|
|
3381
3525
|
async send(path, request = {}) {
|
|
3382
3526
|
const url = this.url(path);
|
|
3383
|
-
|
|
3527
|
+
const authenticate = request.anonymous !== true && this.isGatewayUrl(url);
|
|
3528
|
+
let credential = authenticate ? await this.credential() : void 0;
|
|
3384
3529
|
let refreshed = false;
|
|
3385
3530
|
let busyRetries = request.busyRetries ?? 2;
|
|
3386
3531
|
for (;;) {
|
|
3387
3532
|
const response = await this.attempt(url, request, credential);
|
|
3388
|
-
if (response.ok)
|
|
3533
|
+
if (response.ok) {
|
|
3534
|
+
if (authenticate) try {
|
|
3535
|
+
this.options.onAccepted?.({
|
|
3536
|
+
method: request.method ?? "GET",
|
|
3537
|
+
path: new URL(url).pathname
|
|
3538
|
+
});
|
|
3539
|
+
} catch {}
|
|
3540
|
+
return response;
|
|
3541
|
+
}
|
|
3389
3542
|
const failure = parseGatewayFailure(await response.clone().json().catch(() => void 0));
|
|
3390
3543
|
if (response.status === 401 && credential !== void 0 && credential.kind !== "key" && !refreshed) {
|
|
3391
3544
|
refreshed = true;
|
|
@@ -3514,6 +3667,13 @@ const VIDEO_INPUTS = [
|
|
|
3514
3667
|
"referenceVideos",
|
|
3515
3668
|
"referenceAudios"
|
|
3516
3669
|
];
|
|
3670
|
+
/** What an audio model does, as the catalog declares it (`audio.generation_kind`). */
|
|
3671
|
+
const AUDIO_GENERATION_KINDS = [
|
|
3672
|
+
"music",
|
|
3673
|
+
"podcast",
|
|
3674
|
+
"speech",
|
|
3675
|
+
"transcription"
|
|
3676
|
+
];
|
|
3517
3677
|
const record$3 = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
3518
3678
|
const text$4 = (value) => typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
3519
3679
|
const strings = (value) => Array.isArray(value) && value.every((item) => typeof item === "string") ? value : void 0;
|
|
@@ -3747,10 +3907,144 @@ function effectiveVideoModes(video) {
|
|
|
3747
3907
|
}
|
|
3748
3908
|
return modes;
|
|
3749
3909
|
}
|
|
3750
|
-
|
|
3910
|
+
const PRICE_UNITS = [
|
|
3911
|
+
"generation",
|
|
3912
|
+
"second",
|
|
3913
|
+
"original_second",
|
|
3914
|
+
"minute"
|
|
3915
|
+
];
|
|
3916
|
+
/** The audio block: `entry.audio`, else `capabilities.audio`, else the fields flattened into `capabilities`. */
|
|
3917
|
+
function audioBlock(entry, capabilities) {
|
|
3918
|
+
const own = record$3(entry.audio) ?? record$3(capabilities.audio);
|
|
3919
|
+
if (own !== void 0) return own;
|
|
3920
|
+
return field(capabilities, "generation_kind") !== void 0 ? capabilities : void 0;
|
|
3921
|
+
}
|
|
3922
|
+
function stringChoices(raw) {
|
|
3923
|
+
const choices = record$3(raw);
|
|
3924
|
+
if (choices === void 0) return void 0;
|
|
3925
|
+
const out = {};
|
|
3926
|
+
for (const [name, values] of Object.entries(choices)) {
|
|
3927
|
+
const items = strings(values)?.map((item) => item.trim()).filter((item) => item !== "");
|
|
3928
|
+
if (items !== void 0 && items.length > 0) out[name] = items;
|
|
3929
|
+
}
|
|
3930
|
+
return Object.keys(out).length === 0 ? void 0 : out;
|
|
3931
|
+
}
|
|
3932
|
+
function parseReferenceLimits(raw) {
|
|
3933
|
+
if (raw === void 0) return void 0;
|
|
3934
|
+
const max = count(field(raw, "max_count")) ?? count(raw.max);
|
|
3935
|
+
if (max === void 0 || max === 0) return void 0;
|
|
3936
|
+
const min = count(field(raw, "min_count")) ?? count(raw.min) ?? 0;
|
|
3937
|
+
const limits = {
|
|
3938
|
+
min: Math.min(min, max),
|
|
3939
|
+
max
|
|
3940
|
+
};
|
|
3941
|
+
for (const [key, ...names] of [
|
|
3942
|
+
[
|
|
3943
|
+
"maxBytes",
|
|
3944
|
+
"max_bytes",
|
|
3945
|
+
"max_size_bytes"
|
|
3946
|
+
],
|
|
3947
|
+
[
|
|
3948
|
+
"minSeconds",
|
|
3949
|
+
"min_duration_seconds",
|
|
3950
|
+
"min_seconds"
|
|
3951
|
+
],
|
|
3952
|
+
[
|
|
3953
|
+
"maxSeconds",
|
|
3954
|
+
"max_duration_seconds",
|
|
3955
|
+
"max_seconds"
|
|
3956
|
+
],
|
|
3957
|
+
["maxTotalSeconds", "max_total_duration_seconds"]
|
|
3958
|
+
]) {
|
|
3959
|
+
const value = names.map((name) => positive(field(raw, name))).find((found) => found !== void 0);
|
|
3960
|
+
if (value !== void 0) limits[key] = value;
|
|
3961
|
+
}
|
|
3962
|
+
const mimes = list(raw, "allowed_mimes") ?? list(raw, "mimes");
|
|
3963
|
+
if (mimes !== void 0 && mimes.length > 0) limits.mimes = mimes;
|
|
3964
|
+
const exclusive = list(raw, "mutually_exclusive_with") ?? list(raw, "exclusive_with");
|
|
3965
|
+
if (exclusive !== void 0 && exclusive.length > 0) limits.exclusiveWith = exclusive;
|
|
3966
|
+
const purpose = text$4(field(raw, "asset_purpose")) ?? text$4(raw.purpose);
|
|
3967
|
+
if (purpose !== void 0) limits.purpose = purpose;
|
|
3968
|
+
return limits;
|
|
3969
|
+
}
|
|
3970
|
+
/**
|
|
3971
|
+
* The output formats, in either declared shape: `output_formats: [{format, sample_rates}]`,
|
|
3972
|
+
* or `output_formats: ['wav','mp3']` with `sample_rates: {wav: [...], mp3: [...]}`.
|
|
3973
|
+
*/
|
|
3974
|
+
function parseOutputFormats(raw) {
|
|
3975
|
+
const declared = field(raw, "output_formats");
|
|
3976
|
+
if (!Array.isArray(declared)) return void 0;
|
|
3977
|
+
const rates = record$3(field(raw, "sample_rates"));
|
|
3978
|
+
const formats = [];
|
|
3979
|
+
for (const item of declared) {
|
|
3980
|
+
const entry = record$3(item);
|
|
3981
|
+
const format = (entry === void 0 ? text$4(item) : text$4(entry.format))?.toLowerCase();
|
|
3982
|
+
if (format === void 0 || formats.some((known) => known.format === format)) continue;
|
|
3983
|
+
const sampleRates = (entry === void 0 ? void 0 : field(entry, "sample_rates")) ?? rates?.[format];
|
|
3984
|
+
const valid = Array.isArray(sampleRates) ? sampleRates.filter((rate) => positive(rate) !== void 0 && Number.isInteger(rate)) : [];
|
|
3985
|
+
const fallback = entry === void 0 ? void 0 : positive(field(entry, "default_sample_rate"));
|
|
3986
|
+
formats.push({
|
|
3987
|
+
format,
|
|
3988
|
+
sampleRates: valid,
|
|
3989
|
+
...fallback !== void 0 && valid.includes(fallback) ? { defaultSampleRate: fallback } : {}
|
|
3990
|
+
});
|
|
3991
|
+
}
|
|
3992
|
+
return formats.length === 0 ? void 0 : formats;
|
|
3993
|
+
}
|
|
3994
|
+
/**
|
|
3995
|
+
* Parse an audio capability block.
|
|
3996
|
+
* @param raw - the block.
|
|
3997
|
+
* @returns the declared capabilities; `generationKind` is absent unless declared.
|
|
3998
|
+
*/
|
|
3999
|
+
function parseAudioCapabilities(raw) {
|
|
4000
|
+
const kindRaw = text$4(field(raw, "generation_kind"))?.toLowerCase();
|
|
4001
|
+
const generationKind = AUDIO_GENERATION_KINDS.includes(kindRaw ?? "") ? kindRaw : void 0;
|
|
4002
|
+
const billingRaw = text$4(field(raw, "billing_unit"))?.toLowerCase();
|
|
4003
|
+
const billingUnit = billingRaw === "generation" || billingRaw === "original_second" || billingRaw === "minute" ? billingRaw : void 0;
|
|
4004
|
+
const promptField = text$4(field(raw, "prompt_field"));
|
|
4005
|
+
const requestFields = list(raw, "request_fields") ?? [];
|
|
4006
|
+
const references = parseReferenceLimits(record$3(field(raw, "references")) ?? record$3(field(raw, "reference_audio")));
|
|
4007
|
+
const outputFormats = parseOutputFormats(raw);
|
|
4008
|
+
const defaultFormat = text$4(field(raw, "default_format"))?.toLowerCase();
|
|
4009
|
+
const choices = stringChoices(field(raw, "choices") ?? field(raw, "field_options"));
|
|
4010
|
+
const result = { requestFields };
|
|
4011
|
+
if (generationKind !== void 0) result.generationKind = generationKind;
|
|
4012
|
+
if (promptField !== void 0) result.promptField = promptField;
|
|
4013
|
+
if (billingUnit !== void 0) result.billingUnit = billingUnit;
|
|
4014
|
+
if (references !== void 0) result.references = references;
|
|
4015
|
+
if (outputFormats !== void 0) result.outputFormats = outputFormats;
|
|
4016
|
+
if (defaultFormat !== void 0 && outputFormats?.some((format) => format.format === defaultFormat) === true) result.defaultFormat = defaultFormat;
|
|
4017
|
+
if (choices !== void 0) result.choices = choices;
|
|
4018
|
+
for (const [key, ...names] of [
|
|
4019
|
+
[
|
|
4020
|
+
"maxPromptChars",
|
|
4021
|
+
"max_prompt_chars",
|
|
4022
|
+
"max_text_chars"
|
|
4023
|
+
],
|
|
4024
|
+
[
|
|
4025
|
+
"maxInputBytes",
|
|
4026
|
+
"max_input_bytes",
|
|
4027
|
+
"max_file_bytes"
|
|
4028
|
+
],
|
|
4029
|
+
[
|
|
4030
|
+
"maxInputSeconds",
|
|
4031
|
+
"max_input_seconds",
|
|
4032
|
+
"max_duration_seconds"
|
|
4033
|
+
]
|
|
4034
|
+
]) {
|
|
4035
|
+
const value = names.map((name) => positive(field(raw, name))).find((found) => found !== void 0);
|
|
4036
|
+
if (value !== void 0) result[key] = value;
|
|
4037
|
+
}
|
|
4038
|
+
const inputMimes = list(raw, "input_mimes") ?? list(raw, "allowed_input_mimes");
|
|
4039
|
+
if (inputMimes !== void 0 && inputMimes.length > 0) result.inputMimes = inputMimes;
|
|
4040
|
+
const languages = list(raw, "languages");
|
|
4041
|
+
if (languages !== void 0 && languages.length > 0) result.languages = languages;
|
|
4042
|
+
return result;
|
|
4043
|
+
}
|
|
4044
|
+
function parsePricing(value, audioUnit) {
|
|
3751
4045
|
const raw = record$3(value);
|
|
3752
4046
|
if (raw === void 0) return void 0;
|
|
3753
|
-
const unit = (item) => item
|
|
4047
|
+
const unit = (item) => PRICE_UNITS.includes(item) ? item : item === void 0 && audioUnit !== void 0 ? audioUnit : void 0;
|
|
3754
4048
|
const tiers = [];
|
|
3755
4049
|
if (Array.isArray(raw.tiers)) for (const entry of raw.tiers) {
|
|
3756
4050
|
const tier = record$3(entry);
|
|
@@ -3777,8 +4071,10 @@ function parsePricing(value) {
|
|
|
3777
4071
|
};
|
|
3778
4072
|
}
|
|
3779
4073
|
/** Which kind of media model a catalog entry is, or undefined for a chat or internal model. */
|
|
3780
|
-
function kindOf(entry, capabilities) {
|
|
4074
|
+
function kindOf(entry, capabilities, audio) {
|
|
3781
4075
|
if (text$4(entry.internal_role) !== void 0) return void 0;
|
|
4076
|
+
if (audio?.generationKind === "transcription") return "transcription";
|
|
4077
|
+
if (audio?.generationKind !== void 0) return "audio";
|
|
3782
4078
|
const mediaType = text$4(entry.media_type);
|
|
3783
4079
|
if (mediaType === "video" || mediaType === "image" || mediaType === "audio" || mediaType === "transcription") return mediaType;
|
|
3784
4080
|
if (capabilities.transcription === true || capabilities.realtime_transcription === true) return "transcription";
|
|
@@ -3807,29 +4103,35 @@ function parseMediaCatalog(body) {
|
|
|
3807
4103
|
const id = text$4(entry?.id);
|
|
3808
4104
|
if (entry === void 0 || id === void 0 || seen.has(id)) continue;
|
|
3809
4105
|
const capabilities = record$3(entry.capabilities) ?? {};
|
|
3810
|
-
const
|
|
4106
|
+
const audioRaw = audioBlock(entry, capabilities);
|
|
4107
|
+
const audio = audioRaw === void 0 ? void 0 : parseAudioCapabilities(audioRaw);
|
|
4108
|
+
const kind = kindOf(entry, capabilities, audio);
|
|
3811
4109
|
if (kind === void 0) continue;
|
|
3812
4110
|
seen.add(id);
|
|
3813
4111
|
const name = text$4(entry.display_name) ?? id;
|
|
3814
4112
|
const description = text$4(entry.description);
|
|
3815
|
-
const pricing = parsePricing(entry.media_pricing);
|
|
4113
|
+
const pricing = parsePricing(entry.media_pricing, audio?.billingUnit);
|
|
3816
4114
|
const inputModalities = strings(capabilities.input_modalities) ?? strings(entry.input_modalities) ?? [];
|
|
3817
4115
|
const endpoints = strings(capabilities.allowed_endpoints) ?? [];
|
|
4116
|
+
const aliases = (strings(entry.aliases) ?? []).map((alias) => alias.trim()).filter((alias) => alias !== "" && alias !== id);
|
|
3818
4117
|
let video;
|
|
3819
4118
|
if (kind === "video") video = {
|
|
3820
4119
|
...parseVideoCapabilities(capabilities),
|
|
3821
4120
|
...parseVideoCapabilities(record$3(entry.video) ?? {})
|
|
3822
4121
|
};
|
|
4122
|
+
const audioKind = audio?.generationKind === void 0 || audio.generationKind === "transcription" ? void 0 : audio.generationKind;
|
|
3823
4123
|
models.push({
|
|
3824
4124
|
id,
|
|
3825
4125
|
name,
|
|
3826
4126
|
kind,
|
|
3827
4127
|
endpoints,
|
|
3828
4128
|
inputModalities,
|
|
4129
|
+
...aliases.length === 0 ? {} : { aliases },
|
|
3829
4130
|
...description === void 0 ? {} : { description },
|
|
3830
4131
|
...pricing === void 0 ? {} : { pricing },
|
|
3831
4132
|
...video === void 0 ? {} : { video },
|
|
3832
|
-
...
|
|
4133
|
+
...audio === void 0 ? {} : { audio },
|
|
4134
|
+
...audioKind === void 0 ? {} : { audioKind }
|
|
3833
4135
|
});
|
|
3834
4136
|
}
|
|
3835
4137
|
return models;
|
|
@@ -4121,6 +4423,27 @@ async function loadChatReference(reference, want, messages, store, maxBytes, sig
|
|
|
4121
4423
|
* spending confirmation.
|
|
4122
4424
|
* @module dsh-vibedev/runtime
|
|
4123
4425
|
*/
|
|
4426
|
+
/**
|
|
4427
|
+
* Whether a model serves a slot. Audio models serve the kind their catalog
|
|
4428
|
+
* entry declares, and only that: an undeclared kind serves none.
|
|
4429
|
+
* @param model - the model.
|
|
4430
|
+
* @param slot - the slot.
|
|
4431
|
+
* @returns whether the model can be chosen for it.
|
|
4432
|
+
*/
|
|
4433
|
+
function servesSlot(model, slot) {
|
|
4434
|
+
if (slot === "music" || slot === "podcast" || slot === "speech") return model.kind === "audio" && model.audioKind === slot;
|
|
4435
|
+
return model.kind === slot;
|
|
4436
|
+
}
|
|
4437
|
+
/**
|
|
4438
|
+
* The model a name refers to: its id, else one of its aliases (case-insensitive).
|
|
4439
|
+
* @param models - the catalog.
|
|
4440
|
+
* @param wanted - the id or alias.
|
|
4441
|
+
* @returns the model, or undefined.
|
|
4442
|
+
*/
|
|
4443
|
+
function modelNamed(models, wanted) {
|
|
4444
|
+
const lower = wanted.toLowerCase();
|
|
4445
|
+
return models.find((model) => model.id === wanted) ?? models.find((model) => model.id.toLowerCase() === lower) ?? models.find((model) => (model.aliases ?? []).some((alias) => alias.toLowerCase() === lower));
|
|
4446
|
+
}
|
|
4124
4447
|
const CATALOG_TTL_MS = 12e4;
|
|
4125
4448
|
/** Wait for a shared promise under one caller's signal without cancelling it for the others. */
|
|
4126
4449
|
function abortable$1(promise, signal) {
|
|
@@ -4224,20 +4547,36 @@ var MediaRuntime = class {
|
|
|
4224
4547
|
*/
|
|
4225
4548
|
async pickModel(kind, slot, requested, accept, signal) {
|
|
4226
4549
|
const models = await this.models(signal);
|
|
4227
|
-
const ofKind = models.filter((model) => model.kind === kind && (
|
|
4550
|
+
const ofKind = models.filter((model) => model.kind === kind && servesSlot(model, slot));
|
|
4228
4551
|
const wanted = requested?.trim();
|
|
4229
4552
|
if (wanted !== void 0 && wanted !== "") {
|
|
4230
|
-
const found =
|
|
4553
|
+
const found = modelNamed(models, wanted);
|
|
4231
4554
|
if (found === void 0) throw new MediaError(`There is no model "${wanted}" on this VibeDev account. ${slotLabel(slot)} models: ${ofKind.map((model) => model.id).join(", ") || "none"}.`, "MODEL_NOT_FOUND", { field: "model" });
|
|
4232
|
-
if (!ofKind.includes(found))
|
|
4555
|
+
if (!ofKind.includes(found)) {
|
|
4556
|
+
if (found.kind === "audio" && found.audioKind === void 0) throw new MediaError(`${found.id} does not declare what kind of audio it makes, so it cannot be used for ${slotLabel(slot).toLowerCase()}.`, "AUDIO_KIND_UNDECLARED", { field: "model" });
|
|
4557
|
+
throw new MediaError(`${found.id} is not a ${slotLabel(slot).toLowerCase()} model. ${slotLabel(slot)} models: ${ofKind.map((model) => model.id).join(", ") || "none"}.`, "MODEL_KIND_MISMATCH", { field: "model" });
|
|
4558
|
+
}
|
|
4233
4559
|
return found;
|
|
4234
4560
|
}
|
|
4235
4561
|
if (ofKind.length === 0) throw new MediaError(`This VibeDev account has no ${slotLabel(slot).toLowerCase()} model available right now.`, "NO_MODEL_AVAILABLE");
|
|
4236
|
-
const
|
|
4237
|
-
const
|
|
4562
|
+
const pinnedName = this.settings.defaultModel(slot).trim();
|
|
4563
|
+
const pinned = pinnedName === "" ? void 0 : modelNamed(ofKind, pinnedName);
|
|
4564
|
+
const ordered = pinned === void 0 ? ofKind : [pinned, ...ofKind.filter((model) => model !== pinned)];
|
|
4238
4565
|
return (accept === void 0 ? void 0 : ordered.find((model) => accept(model))) ?? ordered[0];
|
|
4239
4566
|
}
|
|
4240
4567
|
/**
|
|
4568
|
+
* The model a name refers to, whatever its kind.
|
|
4569
|
+
* @param requested - an id or alias.
|
|
4570
|
+
* @param signal - cancels the catalog request.
|
|
4571
|
+
* @returns the model.
|
|
4572
|
+
* @throws {@link MediaError} when the account has no such model.
|
|
4573
|
+
*/
|
|
4574
|
+
async findModel(requested, signal) {
|
|
4575
|
+
const found = modelNamed(await this.models(signal), requested.trim());
|
|
4576
|
+
if (found === void 0) throw new MediaError(`There is no model "${requested}" on this VibeDev account.`, "MODEL_NOT_FOUND", { field: "model" });
|
|
4577
|
+
return found;
|
|
4578
|
+
}
|
|
4579
|
+
/**
|
|
4241
4580
|
* The workspace of the calling session.
|
|
4242
4581
|
* @param call - the tool call or host call.
|
|
4243
4582
|
* @returns its absolute working directory, when it has one.
|
|
@@ -4374,6 +4713,7 @@ function slotLabel(slot) {
|
|
|
4374
4713
|
video: "Video",
|
|
4375
4714
|
music: "Music",
|
|
4376
4715
|
podcast: "Podcast",
|
|
4716
|
+
speech: "Speech",
|
|
4377
4717
|
transcription: "Transcription"
|
|
4378
4718
|
}[slot];
|
|
4379
4719
|
}
|
|
@@ -4617,14 +4957,23 @@ var GatewaySearchProvider = class {
|
|
|
4617
4957
|
};
|
|
4618
4958
|
//#endregion
|
|
4619
4959
|
//#region src/pricing.ts
|
|
4960
|
+
/** Shown wherever the catalog gives no price. */
|
|
4961
|
+
const UNPRICED = {
|
|
4962
|
+
en: "not priced",
|
|
4963
|
+
zh: "未定价"
|
|
4964
|
+
};
|
|
4620
4965
|
/** Seconds the gateway bills when a request names no duration. */
|
|
4621
4966
|
const DEFAULT_BILLED_SECONDS = 8;
|
|
4622
4967
|
/**
|
|
4623
|
-
* Format an amount in yuan.
|
|
4968
|
+
* Format an amount in yuan: two decimals, more when a small unit price needs them (`¥0.015`).
|
|
4624
4969
|
* @param amount - the amount.
|
|
4625
4970
|
* @returns `¥4.95`.
|
|
4626
4971
|
*/
|
|
4627
4972
|
function yuan(amount) {
|
|
4973
|
+
if (amount !== 0 && Math.abs(amount) < .1) {
|
|
4974
|
+
const precise = amount.toFixed(4).replace(/0+$/, "");
|
|
4975
|
+
if (precise.split(".")[1].length > 2) return `¥${precise}`;
|
|
4976
|
+
}
|
|
4628
4977
|
return `¥${amount.toFixed(2)}`;
|
|
4629
4978
|
}
|
|
4630
4979
|
function tierFor(pricing, wanted) {
|
|
@@ -4636,17 +4985,47 @@ function tierFor(pricing, wanted) {
|
|
|
4636
4985
|
return pricing.tiers.length === 1 ? pricing.tiers[0] : void 0;
|
|
4637
4986
|
}
|
|
4638
4987
|
/**
|
|
4639
|
-
*
|
|
4988
|
+
* The unit after a price: `/s`, `/原始秒`, `/次`.
|
|
4989
|
+
* @param unit - the price unit.
|
|
4990
|
+
* @param kind - the kind of model, which names a generation (an image, a video, a request).
|
|
4991
|
+
* @returns the suffix in both languages.
|
|
4992
|
+
*/
|
|
4993
|
+
function unitSuffix(unit, kind) {
|
|
4994
|
+
switch (unit) {
|
|
4995
|
+
case "second": return {
|
|
4996
|
+
en: "/s",
|
|
4997
|
+
zh: "/秒"
|
|
4998
|
+
};
|
|
4999
|
+
case "original_second": return {
|
|
5000
|
+
en: "/s of original audio",
|
|
5001
|
+
zh: "/原始秒"
|
|
5002
|
+
};
|
|
5003
|
+
case "minute": return {
|
|
5004
|
+
en: "/min",
|
|
5005
|
+
zh: "/分钟"
|
|
5006
|
+
};
|
|
5007
|
+
default: return kind === "image" ? {
|
|
5008
|
+
en: " per image",
|
|
5009
|
+
zh: "/张"
|
|
5010
|
+
} : {
|
|
5011
|
+
en: " per generation",
|
|
5012
|
+
zh: "/次"
|
|
5013
|
+
};
|
|
5014
|
+
}
|
|
5015
|
+
}
|
|
5016
|
+
/**
|
|
5017
|
+
* A one-line price for listings: `¥0.10 per image`, `¥0.99/s (720p)`; `not priced` without a price.
|
|
4640
5018
|
* @param model - the model.
|
|
4641
|
-
* @
|
|
5019
|
+
* @param lang - the language.
|
|
5020
|
+
* @returns the price line.
|
|
4642
5021
|
*/
|
|
4643
|
-
function describePrice(model) {
|
|
5022
|
+
function describePrice(model, lang = "en") {
|
|
4644
5023
|
const pricing = model.pricing;
|
|
4645
|
-
if (pricing === void 0) return
|
|
4646
|
-
const per = (unit) => unit
|
|
5024
|
+
if (pricing === void 0) return UNPRICED[lang];
|
|
5025
|
+
const per = (unit) => unitSuffix(unit, model.kind)[lang];
|
|
4647
5026
|
const parts = pricing.tiers.map((tier) => `${yuan(tier.amount)}${per(tier.unit)} (${tier.tier})`);
|
|
4648
|
-
if (pricing.default !== void 0 && (pricing.tiers.length === 0 || !pricing.tiers.some((tier) => tier.amount === pricing.default?.amount && tier.unit === pricing.default.unit))) parts.push(`${yuan(pricing.default.amount)}${per(pricing.default.unit)}${pricing.tiers.length === 0 ? "" : " otherwise"}`);
|
|
4649
|
-
return parts.length === 0 ?
|
|
5027
|
+
if (pricing.default !== void 0 && (pricing.tiers.length === 0 || !pricing.tiers.some((tier) => tier.amount === pricing.default?.amount && tier.unit === pricing.default.unit))) parts.push(`${yuan(pricing.default.amount)}${per(pricing.default.unit)}${pricing.tiers.length === 0 ? "" : lang === "zh" ? "(其他)" : " otherwise"}`);
|
|
5028
|
+
return parts.length === 0 ? UNPRICED[lang] : parts.join(", ");
|
|
4650
5029
|
}
|
|
4651
5030
|
/**
|
|
4652
5031
|
* Estimate an image request.
|
|
@@ -4656,7 +5035,7 @@ function describePrice(model) {
|
|
|
4656
5035
|
*/
|
|
4657
5036
|
function estimateImages(model, count) {
|
|
4658
5037
|
const price = model.pricing === void 0 ? void 0 : tierFor(model.pricing, void 0);
|
|
4659
|
-
if (price === void 0) return void 0;
|
|
5038
|
+
if (price === void 0 || price.unit !== "generation") return void 0;
|
|
4660
5039
|
return {
|
|
4661
5040
|
amountCny: price.amount * count,
|
|
4662
5041
|
basis: count === 1 ? `${yuan(price.amount)} per image` : `${yuan(price.amount)} × ${count} images`
|
|
@@ -4679,6 +5058,7 @@ function estimateVideo(model, duration, resolution, referenceMs = 0) {
|
|
|
4679
5058
|
amountCny: price.amount,
|
|
4680
5059
|
basis: `${yuan(price.amount)} per video`
|
|
4681
5060
|
};
|
|
5061
|
+
if (price.unit !== "second") return void 0;
|
|
4682
5062
|
const output = duration ?? DEFAULT_BILLED_SECONDS;
|
|
4683
5063
|
const reference = Math.ceil(referenceMs / 1e3);
|
|
4684
5064
|
const seconds = output + reference;
|
|
@@ -4688,16 +5068,429 @@ function estimateVideo(model, duration, resolution, referenceMs = 0) {
|
|
|
4688
5068
|
};
|
|
4689
5069
|
}
|
|
4690
5070
|
/**
|
|
4691
|
-
*
|
|
4692
|
-
* @param model - the audio model.
|
|
4693
|
-
* @
|
|
5071
|
+
* Quote an audio request: music, podcast and speech generation, or a transcription.
|
|
5072
|
+
* @param model - the audio or transcription model.
|
|
5073
|
+
* @param inputSeconds - for a transcription, the measured length of the input, when known.
|
|
5074
|
+
* @returns the quote; `priced: false` (未定价) when the catalog gives no usable price.
|
|
4694
5075
|
*/
|
|
4695
|
-
function
|
|
5076
|
+
function quoteAudio(model, inputSeconds) {
|
|
4696
5077
|
const price = model.pricing === void 0 ? void 0 : tierFor(model.pricing, void 0);
|
|
4697
|
-
|
|
4698
|
-
return {
|
|
5078
|
+
const unit = price?.unit ?? model.audio?.billingUnit;
|
|
5079
|
+
if (price === void 0 || unit === void 0 || unit === "second") return {
|
|
5080
|
+
priced: false,
|
|
5081
|
+
...unit === void 0 ? {} : { unit },
|
|
5082
|
+
label: UNPRICED,
|
|
5083
|
+
basis: {
|
|
5084
|
+
en: "The catalog states no price for this model; the gateway receipt shows the charge.",
|
|
5085
|
+
zh: "目录未给出价格,以网关结算为准。"
|
|
5086
|
+
}
|
|
5087
|
+
};
|
|
5088
|
+
const suffix = unitSuffix(unit, model.kind);
|
|
5089
|
+
const label = {
|
|
5090
|
+
en: `${yuan(price.amount)}${suffix.en}`,
|
|
5091
|
+
zh: `${yuan(price.amount)}${suffix.zh}`
|
|
5092
|
+
};
|
|
5093
|
+
const base = {
|
|
5094
|
+
priced: true,
|
|
5095
|
+
unit,
|
|
5096
|
+
unitPriceCny: price.amount,
|
|
5097
|
+
label
|
|
5098
|
+
};
|
|
5099
|
+
if (unit === "generation") return {
|
|
5100
|
+
...base,
|
|
4699
5101
|
amountCny: price.amount,
|
|
4700
|
-
basis:
|
|
5102
|
+
basis: {
|
|
5103
|
+
en: `${label.en}, charged only if it succeeds`,
|
|
5104
|
+
zh: `${label.zh},成功才扣费`
|
|
5105
|
+
}
|
|
5106
|
+
};
|
|
5107
|
+
if (unit === "original_second") return {
|
|
5108
|
+
...base,
|
|
5109
|
+
basis: {
|
|
5110
|
+
en: `${label.en}: billed on the original length the provider reports, which can differ from the playback length`,
|
|
5111
|
+
zh: `${label.zh}:按服务商返回的原始时长计费,可能与播放时长不同`
|
|
5112
|
+
}
|
|
5113
|
+
};
|
|
5114
|
+
if (inputSeconds !== void 0 && inputSeconds > 0) {
|
|
5115
|
+
const minutes = inputSeconds / 60;
|
|
5116
|
+
return {
|
|
5117
|
+
...base,
|
|
5118
|
+
amountCny: price.amount * minutes,
|
|
5119
|
+
basis: {
|
|
5120
|
+
en: `${label.en} × ${minutes.toFixed(2)} min`,
|
|
5121
|
+
zh: `${label.zh} × ${minutes.toFixed(2)} 分钟`
|
|
5122
|
+
}
|
|
5123
|
+
};
|
|
5124
|
+
}
|
|
5125
|
+
return {
|
|
5126
|
+
...base,
|
|
5127
|
+
basis: {
|
|
5128
|
+
en: `${label.en} of input audio`,
|
|
5129
|
+
zh: `${label.zh},按音频时长计费`
|
|
5130
|
+
}
|
|
5131
|
+
};
|
|
5132
|
+
}
|
|
5133
|
+
//#endregion
|
|
5134
|
+
//#region src/audio/request.ts
|
|
5135
|
+
/**
|
|
5136
|
+
* Audio generation requests — music, podcasts and speech — checked against what
|
|
5137
|
+
* the chosen model declares in the catalog's `audio` block, and turned into the
|
|
5138
|
+
* body of `POST /v1/audio/generations`.
|
|
5139
|
+
*
|
|
5140
|
+
* Each kind sends only its own fields:
|
|
5141
|
+
*
|
|
5142
|
+
* | kind | wire fields |
|
|
5143
|
+
* | ------- | --------------------------------------------------------------- |
|
|
5144
|
+
* | music | model, prompt, lyrics, genre, mood, gender |
|
|
5145
|
+
* | podcast | model, prompt, source_url |
|
|
5146
|
+
* | speech | as declared; Seed: model, text_prompt, references, audio_config, watermark |
|
|
5147
|
+
*
|
|
5148
|
+
* The main text goes in the model's `prompt_field`. A field the model does not
|
|
5149
|
+
* declare in `request_fields` is refused before anything is uploaded or charged;
|
|
5150
|
+
* music and podcasts never take speech references, output formats, sample rates
|
|
5151
|
+
* or a duration. Speech references, formats and sample rates come from the
|
|
5152
|
+
* catalog only (count, size, length, mutual exclusions, the sample rates of
|
|
5153
|
+
* each format), never from a fixed list.
|
|
5154
|
+
* @module dsh-vibedev/audio/request
|
|
5155
|
+
*/
|
|
5156
|
+
const AUDIO_MAKE_KINDS = [
|
|
5157
|
+
"music",
|
|
5158
|
+
"podcast",
|
|
5159
|
+
"speech"
|
|
5160
|
+
];
|
|
5161
|
+
/** The standard wire fields of a kind, for a model that declares none (speech declares its own). */
|
|
5162
|
+
const STANDARD_FIELDS = {
|
|
5163
|
+
music: [
|
|
5164
|
+
"model",
|
|
5165
|
+
"prompt",
|
|
5166
|
+
"lyrics",
|
|
5167
|
+
"genre",
|
|
5168
|
+
"mood",
|
|
5169
|
+
"gender"
|
|
5170
|
+
],
|
|
5171
|
+
podcast: [
|
|
5172
|
+
"model",
|
|
5173
|
+
"prompt",
|
|
5174
|
+
"source_url"
|
|
5175
|
+
],
|
|
5176
|
+
speech: []
|
|
5177
|
+
};
|
|
5178
|
+
/** Fields that belong to speech only: references, the output format and sample rate, a duration. */
|
|
5179
|
+
const SPEECH_ONLY = [
|
|
5180
|
+
"references",
|
|
5181
|
+
"format",
|
|
5182
|
+
"sampleRate",
|
|
5183
|
+
"duration"
|
|
5184
|
+
];
|
|
5185
|
+
/** Default limit on one reference file when the catalog states none. */
|
|
5186
|
+
const DEFAULT_REFERENCE_BYTES = 20971520;
|
|
5187
|
+
const OPERATION_ID = /^[\w-]{1,80}$/;
|
|
5188
|
+
const has = (value) => value !== void 0 && value.trim() !== "";
|
|
5189
|
+
/**
|
|
5190
|
+
* What an audio model makes, from its declaration only.
|
|
5191
|
+
* @param model - the model.
|
|
5192
|
+
* @returns the kind, or undefined when the catalog does not declare it.
|
|
5193
|
+
*/
|
|
5194
|
+
function audioKindOf(model) {
|
|
5195
|
+
return model.kind === "audio" ? model.audioKind : void 0;
|
|
5196
|
+
}
|
|
5197
|
+
/** The wire fields a model takes. */
|
|
5198
|
+
function requestFieldsOf(model) {
|
|
5199
|
+
const kind = audioKindOf(model);
|
|
5200
|
+
const declared = model.audio?.requestFields ?? [];
|
|
5201
|
+
if (declared.length > 0) return declared;
|
|
5202
|
+
return kind === void 0 ? [] : STANDARD_FIELDS[kind];
|
|
5203
|
+
}
|
|
5204
|
+
/** The field the main text goes in. */
|
|
5205
|
+
function promptFieldOf(model) {
|
|
5206
|
+
const declared = model.audio?.promptField;
|
|
5207
|
+
if (declared !== void 0) return declared;
|
|
5208
|
+
const fields = requestFieldsOf(model);
|
|
5209
|
+
return [
|
|
5210
|
+
"prompt",
|
|
5211
|
+
"text_prompt",
|
|
5212
|
+
"text",
|
|
5213
|
+
"input"
|
|
5214
|
+
].find((name) => fields.includes(name));
|
|
5215
|
+
}
|
|
5216
|
+
function refuse(message, field) {
|
|
5217
|
+
throw new MediaError(message, "AUDIO_OPTION_INVALID", field === void 0 ? {} : { field });
|
|
5218
|
+
}
|
|
5219
|
+
/** Check one value against the model's declared choices for a field. */
|
|
5220
|
+
function checkChoice(audio, wire, value) {
|
|
5221
|
+
const choices = audio?.choices?.[wire];
|
|
5222
|
+
if (choices !== void 0 && !choices.some((choice) => choice.toLowerCase() === value.toLowerCase())) refuse(`${wire} "${value}" is not offered by this model; it takes ${choices.join(", ")}.`, wire);
|
|
5223
|
+
}
|
|
5224
|
+
/** The output format and sample rate a speech request uses, from the declared formats only. */
|
|
5225
|
+
function audioConfigOf(model, format, sampleRate) {
|
|
5226
|
+
const formats = model.audio?.outputFormats ?? [];
|
|
5227
|
+
if (formats.length === 0) {
|
|
5228
|
+
if (format !== void 0 || sampleRate !== void 0) refuse(`${model.id} declares no output formats, so format and sample_rate cannot be chosen.`, "format");
|
|
5229
|
+
return;
|
|
5230
|
+
}
|
|
5231
|
+
const wanted = format?.trim().toLowerCase();
|
|
5232
|
+
const chosen = wanted === void 0 || wanted === "" ? formats.find((item) => item.format === model.audio?.defaultFormat) ?? formats[0] : formats.find((item) => item.format === wanted);
|
|
5233
|
+
if (chosen === void 0) refuse(`${model.id} writes ${formats.map((item) => item.format).join(", ")}, not "${format}".`, "format");
|
|
5234
|
+
if (sampleRate !== void 0) {
|
|
5235
|
+
if (chosen.sampleRates.length > 0 && !chosen.sampleRates.includes(sampleRate)) refuse(`${chosen.format.toUpperCase()} from ${model.id} takes a sample rate of ${chosen.sampleRates.join(", ")} Hz, not ${sampleRate}.`, "sample_rate");
|
|
5236
|
+
return {
|
|
5237
|
+
format: chosen.format,
|
|
5238
|
+
sample_rate: sampleRate
|
|
5239
|
+
};
|
|
5240
|
+
}
|
|
5241
|
+
const fallback = chosen.defaultSampleRate;
|
|
5242
|
+
return {
|
|
5243
|
+
format: chosen.format,
|
|
5244
|
+
...fallback === void 0 ? {} : { sample_rate: fallback }
|
|
5245
|
+
};
|
|
5246
|
+
}
|
|
5247
|
+
/**
|
|
5248
|
+
* Check a request against the model and build the body, without references
|
|
5249
|
+
* (they are uploaded first; their URLs are added by {@link prepareAudioTask}).
|
|
5250
|
+
* @param model - the chosen model.
|
|
5251
|
+
* @param request - what to generate.
|
|
5252
|
+
* @returns the body without references, and the wire fields the request uses.
|
|
5253
|
+
* @throws {@link MediaError} for a field the model does not take, a missing text, or a value outside the declared choices.
|
|
5254
|
+
*/
|
|
5255
|
+
function audioRequestBody(model, request) {
|
|
5256
|
+
const kind = audioKindOf(model);
|
|
5257
|
+
if (kind === void 0) throw new MediaError(`${model.id} does not declare what kind of audio it makes, so it cannot be used for generation.`, "AUDIO_KIND_UNDECLARED", { field: "model" });
|
|
5258
|
+
const audio = model.audio;
|
|
5259
|
+
const fields = requestFieldsOf(model);
|
|
5260
|
+
if (fields.length === 0) throw new MediaError(`${model.id} declares no request fields, so no request can be built for it.`, "AUDIO_FIELDS_UNDECLARED", { field: "model" });
|
|
5261
|
+
if (kind !== "speech") for (const name of SPEECH_ONLY) {
|
|
5262
|
+
const value = request[name];
|
|
5263
|
+
if (value !== void 0 && !(Array.isArray(value) && value.length === 0)) refuse(`${name === "sampleRate" ? "sample_rate" : name} is for speech; ${kind === "music" ? "music" : "a podcast"} does not take it.`, name);
|
|
5264
|
+
}
|
|
5265
|
+
const takes = (wire) => fields.includes(wire);
|
|
5266
|
+
const body = { model: model.id };
|
|
5267
|
+
const promptField = promptFieldOf(model);
|
|
5268
|
+
const prompt = request.prompt?.trim();
|
|
5269
|
+
if (has(prompt)) {
|
|
5270
|
+
if (promptField === void 0) refuse(`${model.id} takes no text prompt.`, "prompt");
|
|
5271
|
+
if (audio?.maxPromptChars !== void 0 && [...prompt].length > audio.maxPromptChars) refuse(`The text is ${[...prompt].length} characters; ${model.id} takes at most ${audio.maxPromptChars}.`, promptField);
|
|
5272
|
+
body[promptField] = prompt;
|
|
5273
|
+
}
|
|
5274
|
+
for (const [key, wire] of [
|
|
5275
|
+
["lyrics", "lyrics"],
|
|
5276
|
+
["genre", "genre"],
|
|
5277
|
+
["mood", "mood"],
|
|
5278
|
+
["gender", "gender"],
|
|
5279
|
+
["sourceUrl", "source_url"]
|
|
5280
|
+
]) {
|
|
5281
|
+
const value = request[key]?.trim();
|
|
5282
|
+
if (!has(value)) continue;
|
|
5283
|
+
if (!takes(wire)) refuse(`${model.id} does not take ${wire}; it takes ${fields.filter((name) => name !== "model").join(", ")}.`, wire);
|
|
5284
|
+
checkChoice(audio, wire, value);
|
|
5285
|
+
if (wire === "source_url" && !/^https?:\/\//i.test(value)) refuse("source_url must be an http(s) link.", wire);
|
|
5286
|
+
body[wire] = value;
|
|
5287
|
+
}
|
|
5288
|
+
if (request.duration !== void 0) {
|
|
5289
|
+
if (!takes("duration")) refuse(`${model.id} does not take a duration.`, "duration");
|
|
5290
|
+
body.duration = request.duration;
|
|
5291
|
+
}
|
|
5292
|
+
if (request.watermark !== void 0) {
|
|
5293
|
+
if (!takes("watermark")) refuse(`${model.id} does not take watermark.`, "watermark");
|
|
5294
|
+
body.watermark = request.watermark;
|
|
5295
|
+
}
|
|
5296
|
+
if (request.format !== void 0 || request.sampleRate !== void 0 || takes("audio_config")) {
|
|
5297
|
+
if (!takes("audio_config")) refuse(`${model.id} does not take an output format or sample rate.`, "format");
|
|
5298
|
+
const config = audioConfigOf(model, request.format, request.sampleRate);
|
|
5299
|
+
if (config !== void 0) body.audio_config = config;
|
|
5300
|
+
}
|
|
5301
|
+
if ((request.references?.length ?? 0) > 0 && !takes("references")) refuse(`${model.id} does not take reference audio.`, "references");
|
|
5302
|
+
if (kind === "music" && !has(prompt)) throw new MediaError("Music needs a prompt describing the song.", "AUDIO_PROMPT_REQUIRED", { field: "prompt" });
|
|
5303
|
+
if (kind === "podcast" && !has(prompt) && body.source_url === void 0) throw new MediaError("A podcast needs a prompt (the topic), a source_url, or both.", "AUDIO_PROMPT_REQUIRED", { field: "prompt" });
|
|
5304
|
+
if (kind === "speech" && !has(prompt)) throw new MediaError("Speech needs the text to speak.", "AUDIO_PROMPT_REQUIRED", { field: promptField ?? "prompt" });
|
|
5305
|
+
return body;
|
|
5306
|
+
}
|
|
5307
|
+
/** Load, check and upload the reference audio of a speech request; their measured lengths are checked too. */
|
|
5308
|
+
async function uploadReferences(runtime, call, model, sources, body) {
|
|
5309
|
+
const limits = model.audio?.references;
|
|
5310
|
+
if (limits === void 0) refuse(`${model.id} does not take reference audio.`, "references");
|
|
5311
|
+
if (sources.length < limits.min || sources.length > limits.max) refuse(limits.min === limits.max ? `${model.id} takes exactly ${limits.max} reference audio file(s); ${sources.length} were given.` : `${model.id} takes ${limits.min}–${limits.max} reference audio file(s); ${sources.length} were given.`, "references");
|
|
5312
|
+
const clash = (limits.exclusiveWith ?? []).filter((name) => body[name] !== void 0);
|
|
5313
|
+
if (clash.length > 0) refuse(`Reference audio cannot be combined with ${clash.join(", ")} on ${model.id}.`, "references");
|
|
5314
|
+
const maxBytes = limits.maxBytes ?? DEFAULT_REFERENCE_BYTES;
|
|
5315
|
+
const loaded = [];
|
|
5316
|
+
for (const source of sources) {
|
|
5317
|
+
const media = await runtime.load(call, source, "audio", maxBytes);
|
|
5318
|
+
if (limits.mimes !== void 0 && !limits.mimes.includes(media.mime)) throw new MediaError(`"${media.source}" is ${media.mime}; ${model.id} takes reference audio as ${limits.mimes.join(", ")}.`, "REFERENCE_MEDIA_TYPE_UNSUPPORTED", { field: "references" });
|
|
5319
|
+
loaded.push(media);
|
|
5320
|
+
}
|
|
5321
|
+
const checked = [];
|
|
5322
|
+
let totalSeconds = 0;
|
|
5323
|
+
for (const media of loaded) {
|
|
5324
|
+
const asset = await runtime.library.upload(media, limits.purpose ?? "audio_reference", call.signal);
|
|
5325
|
+
const seconds = asset.durationMs === void 0 ? void 0 : asset.durationMs / 1e3;
|
|
5326
|
+
if (seconds === void 0 && (limits.minSeconds !== void 0 || limits.maxSeconds !== void 0 || limits.maxTotalSeconds !== void 0)) throw new MediaError(`The gateway could not measure the length of "${media.source}". Export it as a standard WAV or MP3 and try again.`, "REFERENCE_DURATION_UNKNOWN", { field: "references" });
|
|
5327
|
+
if (seconds !== void 0) {
|
|
5328
|
+
if (limits.minSeconds !== void 0 && seconds < limits.minSeconds) throw new MediaError(`"${media.source}" is ${seconds.toFixed(1)} s; ${model.id} needs reference audio of at least ${limits.minSeconds} s.`, "REFERENCE_AUDIO_TOO_SHORT", { field: "references" });
|
|
5329
|
+
if (limits.maxSeconds !== void 0 && seconds > limits.maxSeconds) throw new MediaError(`"${media.source}" is ${seconds.toFixed(1)} s; ${model.id} takes reference audio of at most ${limits.maxSeconds} s.`, "REFERENCE_AUDIO_TOO_LONG", { field: "references" });
|
|
5330
|
+
totalSeconds += seconds;
|
|
5331
|
+
}
|
|
5332
|
+
checked.push({
|
|
5333
|
+
source: media.source,
|
|
5334
|
+
asset
|
|
5335
|
+
});
|
|
5336
|
+
}
|
|
5337
|
+
if (limits.maxTotalSeconds !== void 0 && totalSeconds > limits.maxTotalSeconds) throw new MediaError(`The reference audio adds up to ${totalSeconds.toFixed(1)} s; ${model.id} takes at most ${limits.maxTotalSeconds} s in total.`, "REFERENCE_TOTAL_TOO_LONG", { field: "references" });
|
|
5338
|
+
return checked;
|
|
5339
|
+
}
|
|
5340
|
+
/**
|
|
5341
|
+
* Choose the model for a request: the one named (an id or an alias), else the
|
|
5342
|
+
* pinned or first model of the kind. A named model must make the kind asked for.
|
|
5343
|
+
*/
|
|
5344
|
+
async function chooseAudioModel(runtime, request, signal) {
|
|
5345
|
+
if (request.kind === void 0 && !has(request.model)) throw new MediaError("Say what to make (music, podcast or speech), or name a model.", "AUDIO_KIND_REQUIRED", { field: "type" });
|
|
5346
|
+
if (request.kind !== void 0) return runtime.pickModel("audio", request.kind, request.model, void 0, signal);
|
|
5347
|
+
const model = await runtime.findModel(request.model, signal);
|
|
5348
|
+
if (model.kind !== "audio") throw new MediaError(`${model.id} is not an audio generation model.`, "MODEL_KIND_MISMATCH", { field: "model" });
|
|
5349
|
+
if (audioKindOf(model) === void 0) throw new MediaError(`${model.id} does not declare what kind of audio it makes, so it cannot be used for generation.`, "AUDIO_KIND_UNDECLARED", { field: "model" });
|
|
5350
|
+
return model;
|
|
5351
|
+
}
|
|
5352
|
+
/**
|
|
5353
|
+
* Everything before an audio submission: choose the model, check every field
|
|
5354
|
+
* against its declaration, upload and measure speech references, price it,
|
|
5355
|
+
* confirm the spending, and build the task under the operation's key.
|
|
5356
|
+
* @param runtime - the media runtime.
|
|
5357
|
+
* @param call - the tool call or host call.
|
|
5358
|
+
* @param request - what to generate.
|
|
5359
|
+
* @param save - where the result goes.
|
|
5360
|
+
* @returns the prepared task.
|
|
5361
|
+
*/
|
|
5362
|
+
async function prepareAudioTask(runtime, call, request, save) {
|
|
5363
|
+
if (request.operationId !== void 0 && !OPERATION_ID.test(request.operationId)) throw new MediaError("operationId may hold only letters, digits, \"-\" and \"_\" (at most 80).", "AUDIO_OPTION_INVALID", { field: "operationId" });
|
|
5364
|
+
const model = await chooseAudioModel(runtime, request, call.signal);
|
|
5365
|
+
const kind = audioKindOf(model);
|
|
5366
|
+
const body = audioRequestBody(model, request);
|
|
5367
|
+
const references = request.references ?? [];
|
|
5368
|
+
if (references.length > 0) body.references = (await uploadReferences(runtime, call, model, references, body)).map((item) => item.asset.referenceUrl);
|
|
5369
|
+
else if ((model.audio?.references?.min ?? 0) > 0) refuse(`${model.id} needs at least ${model.audio?.references?.min} reference audio file(s).`, "references");
|
|
5370
|
+
const quote = quoteAudio(model);
|
|
5371
|
+
const what = {
|
|
5372
|
+
music: {
|
|
5373
|
+
en: "a song",
|
|
5374
|
+
zh: "一首音乐"
|
|
5375
|
+
},
|
|
5376
|
+
podcast: {
|
|
5377
|
+
en: "a podcast episode",
|
|
5378
|
+
zh: "一期播客"
|
|
5379
|
+
},
|
|
5380
|
+
speech: {
|
|
5381
|
+
en: "speech",
|
|
5382
|
+
zh: "一段语音"
|
|
5383
|
+
}
|
|
5384
|
+
}[kind];
|
|
5385
|
+
await runtime.confirmSpending(call, "audio_generate", {
|
|
5386
|
+
en: `Generate ${what.en} with ${model.id}: ${quote.basis.en}.`,
|
|
5387
|
+
zh: `用 ${model.id} 生成${what.zh}:${quote.basis.zh}。`
|
|
5388
|
+
});
|
|
5389
|
+
const now = runtime.now();
|
|
5390
|
+
const prompt = request.prompt?.trim();
|
|
5391
|
+
return {
|
|
5392
|
+
model,
|
|
5393
|
+
kind,
|
|
5394
|
+
quote,
|
|
5395
|
+
draft: {
|
|
5396
|
+
id: request.operationId ?? randomUUID(),
|
|
5397
|
+
kind: "audio",
|
|
5398
|
+
model: model.id,
|
|
5399
|
+
label: excerptOf(prompt ?? request.sourceUrl ?? kind),
|
|
5400
|
+
createdAt: now,
|
|
5401
|
+
updatedAt: now,
|
|
5402
|
+
...call.agent === void 0 ? {} : { owner: call.agent.id },
|
|
5403
|
+
outputDir: save.folder,
|
|
5404
|
+
stem: save.stem,
|
|
5405
|
+
endpoint: "/v1/audio/generations",
|
|
5406
|
+
body,
|
|
5407
|
+
status: "submitting",
|
|
5408
|
+
audioKind: kind,
|
|
5409
|
+
...quote.amountCny === void 0 ? {} : { estimatedCny: quote.amountCny.toFixed(2) },
|
|
5410
|
+
...quote.unit === void 0 ? {} : { billing: {
|
|
5411
|
+
unit: quote.unit,
|
|
5412
|
+
...quote.unitPriceCny === void 0 ? {} : { unitPriceCny: String(quote.unitPriceCny) }
|
|
5413
|
+
} }
|
|
5414
|
+
}
|
|
5415
|
+
};
|
|
5416
|
+
}
|
|
5417
|
+
function excerptOf(text, max = 60) {
|
|
5418
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
5419
|
+
const chars = [...flat];
|
|
5420
|
+
return chars.length <= max ? flat : `${chars.slice(0, max - 1).join("")}…`;
|
|
5421
|
+
}
|
|
5422
|
+
/**
|
|
5423
|
+
* The form an interface shows for one model: only the fields the model
|
|
5424
|
+
* declares, with their declared choices and limits.
|
|
5425
|
+
* @param model - the model.
|
|
5426
|
+
* @returns the form, or undefined for a model that cannot generate audio.
|
|
5427
|
+
*/
|
|
5428
|
+
function audioFormOf(model) {
|
|
5429
|
+
const kind = audioKindOf(model);
|
|
5430
|
+
if (kind === void 0) return void 0;
|
|
5431
|
+
const fields = requestFieldsOf(model);
|
|
5432
|
+
if (fields.length === 0) return void 0;
|
|
5433
|
+
const audio = model.audio;
|
|
5434
|
+
const out = [];
|
|
5435
|
+
const promptField = promptFieldOf(model);
|
|
5436
|
+
if (promptField !== void 0) out.push({
|
|
5437
|
+
key: "prompt",
|
|
5438
|
+
wire: promptField,
|
|
5439
|
+
required: kind !== "podcast",
|
|
5440
|
+
...audio?.maxPromptChars === void 0 ? {} : { maxChars: audio.maxPromptChars }
|
|
5441
|
+
});
|
|
5442
|
+
for (const [key, wire] of [
|
|
5443
|
+
["lyrics", "lyrics"],
|
|
5444
|
+
["genre", "genre"],
|
|
5445
|
+
["mood", "mood"],
|
|
5446
|
+
["gender", "gender"],
|
|
5447
|
+
["sourceUrl", "source_url"]
|
|
5448
|
+
]) {
|
|
5449
|
+
if (!fields.includes(wire)) continue;
|
|
5450
|
+
const choices = audio?.choices?.[wire];
|
|
5451
|
+
out.push({
|
|
5452
|
+
key,
|
|
5453
|
+
wire,
|
|
5454
|
+
required: false,
|
|
5455
|
+
...choices === void 0 ? {} : { choices }
|
|
5456
|
+
});
|
|
5457
|
+
}
|
|
5458
|
+
if (kind === "speech") {
|
|
5459
|
+
if (fields.includes("references") && audio?.references !== void 0) out.push({
|
|
5460
|
+
key: "references",
|
|
5461
|
+
wire: "references",
|
|
5462
|
+
required: audio.references.min > 0,
|
|
5463
|
+
limits: audio.references
|
|
5464
|
+
});
|
|
5465
|
+
const formats = audio?.outputFormats ?? [];
|
|
5466
|
+
if (fields.includes("audio_config") && formats.length > 0) {
|
|
5467
|
+
out.push({
|
|
5468
|
+
key: "format",
|
|
5469
|
+
wire: "audio_config.format",
|
|
5470
|
+
required: false,
|
|
5471
|
+
choices: formats.map((item) => item.format)
|
|
5472
|
+
});
|
|
5473
|
+
const rates = Object.fromEntries(formats.filter((item) => item.sampleRates.length > 0).map((item) => [item.format, item.sampleRates]));
|
|
5474
|
+
if (Object.keys(rates).length > 0) out.push({
|
|
5475
|
+
key: "sampleRate",
|
|
5476
|
+
wire: "audio_config.sample_rate",
|
|
5477
|
+
required: false,
|
|
5478
|
+
sampleRates: rates
|
|
5479
|
+
});
|
|
5480
|
+
}
|
|
5481
|
+
if (fields.includes("watermark")) out.push({
|
|
5482
|
+
key: "watermark",
|
|
5483
|
+
wire: "watermark",
|
|
5484
|
+
required: false
|
|
5485
|
+
});
|
|
5486
|
+
}
|
|
5487
|
+
return {
|
|
5488
|
+
model: model.id,
|
|
5489
|
+
name: model.name,
|
|
5490
|
+
kind,
|
|
5491
|
+
fields: out,
|
|
5492
|
+
price: quoteAudio(model),
|
|
5493
|
+
...audio?.defaultFormat === void 0 ? {} : { defaultFormat: audio.defaultFormat }
|
|
4701
5494
|
};
|
|
4702
5495
|
}
|
|
4703
5496
|
//#endregion
|
|
@@ -4721,10 +5514,12 @@ function excerpt(prompt, max = 60) {
|
|
|
4721
5514
|
* @returns `Charged ¥4.95.`, an estimate, or the empty string.
|
|
4722
5515
|
*/
|
|
4723
5516
|
function moneyLine(record) {
|
|
4724
|
-
if (record.status !== "completed") return record.status === "failed" ? " Failed tasks are not charged." : "";
|
|
4725
|
-
|
|
4726
|
-
if (record.
|
|
4727
|
-
return
|
|
5517
|
+
if (record.status !== "completed") return record.status === "failed" ? " Failed and cancelled tasks are not charged." : "";
|
|
5518
|
+
const billed = record.billing?.originalSeconds === void 0 ? "" : ` Billed on ${record.billing.originalSeconds.toFixed(1)} s of original audio.`;
|
|
5519
|
+
if (record.settlement === "pending") return ` The result is saved; the charge is still being settled (it is never generated again — media_tasks shows the charge once settled).${billed}`;
|
|
5520
|
+
if (record.chargedCny !== void 0) return ` Charged ${yuan(Number(record.chargedCny))}.${billed}`;
|
|
5521
|
+
if (record.estimatedCny !== void 0) return ` Estimated ${yuan(Number(record.estimatedCny))} (the gateway settles the charge shortly).${billed}`;
|
|
5522
|
+
return billed;
|
|
4728
5523
|
}
|
|
4729
5524
|
/**
|
|
4730
5525
|
* The model-facing account of a task's current state.
|
|
@@ -5796,19 +6591,44 @@ function videoGenerateTool(runtime) {
|
|
|
5796
6591
|
}
|
|
5797
6592
|
//#endregion
|
|
5798
6593
|
//#region src/service.ts
|
|
6594
|
+
/**
|
|
6595
|
+
* The media service other plugins call, registered as `vibedevMedia`: the
|
|
6596
|
+
* film workbench's storyboard generates through it when someone presses a
|
|
6597
|
+
* generate button. It takes the same paths as the agent tools — model
|
|
6598
|
+
* choice, reference checks and uploads, idempotent submission, a background
|
|
6599
|
+
* follower that saves the result — with the caller naming the workspace and
|
|
6600
|
+
* where results go. The caller has already shown the price and the user chose
|
|
6601
|
+
* to go ahead, so the spending confirmation setting does not ask again.
|
|
6602
|
+
*
|
|
6603
|
+
* Every task, video or audio, is read the same way: {@link MediaHostService.task}
|
|
6604
|
+
* (what is recorded), {@link MediaHostService.refreshTask} (ask the gateway again:
|
|
6605
|
+
* fresh links, settlement — a query only, it never submits),
|
|
6606
|
+
* {@link MediaHostService.onTask} (each change) and
|
|
6607
|
+
* {@link MediaHostService.result} (the files to play or download).
|
|
6608
|
+
* @module dsh-vibedev/service
|
|
6609
|
+
*/
|
|
5799
6610
|
const view = (record) => ({
|
|
5800
6611
|
id: record.id,
|
|
5801
6612
|
kind: record.kind,
|
|
6613
|
+
...record.audioKind === void 0 ? {} : { audioKind: record.audioKind },
|
|
5802
6614
|
model: record.model,
|
|
5803
6615
|
status: record.status,
|
|
5804
6616
|
...record.progress === void 0 ? {} : { progress: record.progress },
|
|
5805
|
-
...record.outputs === void 0 ? {} : { outputs: record.outputs.map((output) => ({ ...output })) },
|
|
6617
|
+
...record.outputs === void 0 ? {} : { outputs: record.outputs.map(({ index: _index, bytes: _bytes, ...output }) => ({ ...output })) },
|
|
5806
6618
|
...record.error === void 0 ? {} : { error: { ...record.error } },
|
|
5807
6619
|
...record.estimatedCny === void 0 ? {} : { estimatedCny: record.estimatedCny },
|
|
5808
6620
|
...record.chargedCny === void 0 ? {} : { chargedCny: record.chargedCny },
|
|
6621
|
+
...record.settlement === void 0 ? {} : { settlement: record.settlement },
|
|
6622
|
+
...record.billing === void 0 ? {} : { billing: { ...record.billing } },
|
|
5809
6623
|
createdAt: record.createdAt,
|
|
5810
6624
|
updatedAt: record.updatedAt
|
|
5811
6625
|
});
|
|
6626
|
+
/** The parts of a request that decide what is generated, to tell a repeated operation from a different one. */
|
|
6627
|
+
function sameRequest(record, draft) {
|
|
6628
|
+
const { references: _a, ...left } = record.body;
|
|
6629
|
+
const { references: _b, ...right } = draft.body;
|
|
6630
|
+
return record.kind === draft.kind && record.model === draft.model && JSON.stringify(left) === JSON.stringify(right);
|
|
6631
|
+
}
|
|
5812
6632
|
var MediaHostService = class {
|
|
5813
6633
|
runtime;
|
|
5814
6634
|
version;
|
|
@@ -5824,7 +6644,7 @@ var MediaHostService = class {
|
|
|
5824
6644
|
};
|
|
5825
6645
|
}
|
|
5826
6646
|
/**
|
|
5827
|
-
* The media models this account can use (cached briefly).
|
|
6647
|
+
* The media models this account can use (cached briefly; renewed after a sign-in, sign-out or account switch).
|
|
5828
6648
|
* @param signal - cancels the request.
|
|
5829
6649
|
* @returns the models in catalog order.
|
|
5830
6650
|
*/
|
|
@@ -5832,6 +6652,28 @@ var MediaHostService = class {
|
|
|
5832
6652
|
return this.runtime.models(signal);
|
|
5833
6653
|
}
|
|
5834
6654
|
/**
|
|
6655
|
+
* The forms to show for the audio models this account can use: one per
|
|
6656
|
+
* model that declares what it makes, with only the fields it takes, their
|
|
6657
|
+
* declared choices and limits, and its price.
|
|
6658
|
+
* @param signal - cancels the catalog request.
|
|
6659
|
+
* @returns the forms, in catalog order.
|
|
6660
|
+
*/
|
|
6661
|
+
async audioForms(signal) {
|
|
6662
|
+
return (await this.runtime.models(signal)).flatMap((model) => {
|
|
6663
|
+
const form = audioFormOf(model);
|
|
6664
|
+
return form === void 0 ? [] : [form];
|
|
6665
|
+
});
|
|
6666
|
+
}
|
|
6667
|
+
/**
|
|
6668
|
+
* The price of one audio or transcription model: `¥0.50/次`, `¥0.02/原始秒`, `¥0.30/分钟`, or 未定价.
|
|
6669
|
+
* @param model - the model id or alias.
|
|
6670
|
+
* @param signal - cancels the catalog request.
|
|
6671
|
+
* @returns the quote.
|
|
6672
|
+
*/
|
|
6673
|
+
async audioPrice(model, signal) {
|
|
6674
|
+
return quoteAudio(await this.runtime.findModel(model, signal));
|
|
6675
|
+
}
|
|
6676
|
+
/**
|
|
5835
6677
|
* Generate images now and save them.
|
|
5836
6678
|
* @param request - what to generate.
|
|
5837
6679
|
* @param target - the workspace and where the files go.
|
|
@@ -5864,18 +6706,53 @@ var MediaHostService = class {
|
|
|
5864
6706
|
folder: target.folder,
|
|
5865
6707
|
stem: target.stem
|
|
5866
6708
|
});
|
|
6709
|
+
return this.submit(prepared.draft, signal);
|
|
6710
|
+
}
|
|
6711
|
+
/**
|
|
6712
|
+
* Submit music, a podcast episode or speech. Fields are checked against
|
|
6713
|
+
* what the model declares before anything is uploaded or charged; speech
|
|
6714
|
+
* references are uploaded to the gateway media library first.
|
|
6715
|
+
*
|
|
6716
|
+
* One user operation, one task: pass the same `operationId` when the same
|
|
6717
|
+
* click is retried and the task already recorded under it is returned —
|
|
6718
|
+
* nothing is submitted twice. Without one, every call is a new operation.
|
|
6719
|
+
* @param request - what to generate.
|
|
6720
|
+
* @param target - the workspace and where the audio goes.
|
|
6721
|
+
* @param signal - cancels the preparation and the submission.
|
|
6722
|
+
* @returns the task.
|
|
6723
|
+
*/
|
|
6724
|
+
async startAudio(request, target, signal) {
|
|
6725
|
+
if (request.operationId !== void 0) {
|
|
6726
|
+
const earlier = await this.runtime.store.get(request.operationId).catch(() => void 0);
|
|
6727
|
+
if (earlier !== void 0) {
|
|
6728
|
+
if (earlier.kind !== "audio") throw new MediaError(`Operation ${request.operationId} already belongs to a ${earlier.kind} task.`, "OPERATION_CONFLICT", { field: "operationId" });
|
|
6729
|
+
return view(earlier);
|
|
6730
|
+
}
|
|
6731
|
+
}
|
|
6732
|
+
const prepared = await prepareAudioTask(this.runtime, this.call(target, signal), request, {
|
|
6733
|
+
folder: target.folder,
|
|
6734
|
+
stem: target.stem
|
|
6735
|
+
});
|
|
6736
|
+
const raced = await this.runtime.store.get(prepared.draft.id).catch(() => void 0);
|
|
6737
|
+
if (raced !== void 0) {
|
|
6738
|
+
if (!sameRequest(raced, prepared.draft)) throw new MediaError(`Operation ${prepared.draft.id} already belongs to a different request.`, "OPERATION_CONFLICT", { field: "operationId" });
|
|
6739
|
+
return view(raced);
|
|
6740
|
+
}
|
|
6741
|
+
return this.submit(prepared.draft, signal);
|
|
6742
|
+
}
|
|
6743
|
+
async submit(draft, signal) {
|
|
5867
6744
|
try {
|
|
5868
|
-
return view(await this.runtime.tracker.submit(
|
|
6745
|
+
return view(await this.runtime.tracker.submit(draft, signal));
|
|
5869
6746
|
} catch (error) {
|
|
5870
6747
|
if (error instanceof MediaError && error.code === "SUBMISSION_UNCONFIRMED") {
|
|
5871
|
-
const recorded = await this.runtime.store.get(
|
|
6748
|
+
const recorded = await this.runtime.store.get(draft.id);
|
|
5872
6749
|
if (recorded !== void 0) return view(recorded);
|
|
5873
6750
|
}
|
|
5874
6751
|
throw error;
|
|
5875
6752
|
}
|
|
5876
6753
|
}
|
|
5877
6754
|
/**
|
|
5878
|
-
* A task's current state.
|
|
6755
|
+
* A task's current state, as recorded (no request to the gateway).
|
|
5879
6756
|
* @param id - the task id.
|
|
5880
6757
|
* @returns the task, or `undefined` when there is none.
|
|
5881
6758
|
*/
|
|
@@ -5884,6 +6761,31 @@ var MediaHostService = class {
|
|
|
5884
6761
|
return record === void 0 ? void 0 : view(record);
|
|
5885
6762
|
}
|
|
5886
6763
|
/**
|
|
6764
|
+
* Ask the gateway about a task again — a query, never a submission: a
|
|
6765
|
+
* finished task gets fresh links and its missing files are downloaded
|
|
6766
|
+
* again, and a pending settlement is read again. A running task is followed.
|
|
6767
|
+
* @param id - the task id.
|
|
6768
|
+
* @param signal - cancels the query.
|
|
6769
|
+
* @returns the task after the query, or `undefined` when there is none.
|
|
6770
|
+
*/
|
|
6771
|
+
async refreshTask(id, signal) {
|
|
6772
|
+
if (await this.runtime.store.get(id) === void 0) return void 0;
|
|
6773
|
+
return view(await this.runtime.tracker.refresh(id, signal));
|
|
6774
|
+
}
|
|
6775
|
+
/**
|
|
6776
|
+
* The files of a finished task, ready to play or download: saved files by
|
|
6777
|
+
* path, and links renewed by a query when one is needed (a missing file, an
|
|
6778
|
+
* expired link). Never generates again.
|
|
6779
|
+
* @param id - the task id.
|
|
6780
|
+
* @param signal - cancels the query.
|
|
6781
|
+
* @returns the outputs, or `undefined` while the task has none.
|
|
6782
|
+
*/
|
|
6783
|
+
async result(id, signal) {
|
|
6784
|
+
const record = await this.runtime.store.get(id);
|
|
6785
|
+
if (record === void 0 || record.status !== "completed") return void 0;
|
|
6786
|
+
return (((record.outputs ?? []).length > 0 && (record.outputs ?? []).every((output) => output.path !== void 0) ? record : await this.runtime.tracker.refresh(id, signal).catch(() => record)).outputs ?? []).map(({ index: _index, bytes: _bytes, ...output }) => ({ ...output }));
|
|
6787
|
+
}
|
|
6788
|
+
/**
|
|
5887
6789
|
* Hear about every change to one task.
|
|
5888
6790
|
* @param id - the task id.
|
|
5889
6791
|
* @param listener - called with the task after each change.
|
|
@@ -6095,7 +6997,8 @@ const COMPLETED = /* @__PURE__ */ new Set([
|
|
|
6095
6997
|
"succeeded",
|
|
6096
6998
|
"success",
|
|
6097
6999
|
"done",
|
|
6098
|
-
"finished"
|
|
7000
|
+
"finished",
|
|
7001
|
+
"settlement_pending"
|
|
6099
7002
|
]);
|
|
6100
7003
|
const FAILED = /* @__PURE__ */ new Set([
|
|
6101
7004
|
"failed",
|
|
@@ -6106,12 +7009,40 @@ const FAILED = /* @__PURE__ */ new Set([
|
|
|
6106
7009
|
"expired",
|
|
6107
7010
|
"rejected"
|
|
6108
7011
|
]);
|
|
7012
|
+
const CANCELLED = /* @__PURE__ */ new Set(["cancelled", "canceled"]);
|
|
7013
|
+
/** The result exists; only the charge is still being settled. */
|
|
7014
|
+
const SETTLEMENT_PENDING = "settlement_pending";
|
|
6109
7015
|
function stateOf(status) {
|
|
6110
7016
|
const word = status.toLowerCase();
|
|
6111
7017
|
if (COMPLETED.has(word)) return "completed";
|
|
6112
7018
|
if (FAILED.has(word)) return "failed";
|
|
6113
7019
|
return "pending";
|
|
6114
7020
|
}
|
|
7021
|
+
const seconds = (value) => {
|
|
7022
|
+
const number = typeof value === "string" ? Number(value) : value;
|
|
7023
|
+
return typeof number === "number" && Number.isFinite(number) && number > 0 ? number : void 0;
|
|
7024
|
+
};
|
|
7025
|
+
/** The billing facts of a task: whether the charge settled, and the billed length. */
|
|
7026
|
+
function billing(raw, status) {
|
|
7027
|
+
const blocks = [
|
|
7028
|
+
record$1(raw.billing),
|
|
7029
|
+
record$1(raw.usage),
|
|
7030
|
+
record$1(raw.effective),
|
|
7031
|
+
raw
|
|
7032
|
+
].filter((block) => block !== void 0);
|
|
7033
|
+
const pick = (...names) => {
|
|
7034
|
+
for (const block of blocks) for (const name of names) if (block[name] !== void 0) return block[name];
|
|
7035
|
+
};
|
|
7036
|
+
const originalSeconds = seconds(pick("original_seconds", "billed_original_seconds", "original_duration_seconds"));
|
|
7037
|
+
const minutes = seconds(pick("billed_minutes", "minutes"));
|
|
7038
|
+
const stateRaw = text$1(pick("settlement_state", "settlement_status"))?.toLowerCase();
|
|
7039
|
+
const settlement = status.toLowerCase() === SETTLEMENT_PENDING || stateRaw === "pending" ? "pending" : stateRaw === "settled" || stateRaw === "completed" ? "settled" : void 0;
|
|
7040
|
+
return {
|
|
7041
|
+
...settlement === void 0 ? {} : { settlement },
|
|
7042
|
+
...originalSeconds === void 0 ? {} : { billedOriginalSeconds: originalSeconds },
|
|
7043
|
+
...minutes === void 0 ? {} : { billedMinutes: minutes }
|
|
7044
|
+
};
|
|
7045
|
+
}
|
|
6115
7046
|
function errorOf(raw) {
|
|
6116
7047
|
const nested = record$1(raw.error);
|
|
6117
7048
|
const message = text$1(nested?.message) ?? text$1(raw.error) ?? text$1(raw.failure_reason) ?? text$1(raw.error_message) ?? text$1(raw.fail_reason);
|
|
@@ -6191,19 +7122,25 @@ function parseAudioTask(body) {
|
|
|
6191
7122
|
const url = httpUrl(output.content_url) ?? httpUrl(output.url);
|
|
6192
7123
|
const contentType = text$1(output.content_type);
|
|
6193
7124
|
const title = text$1(output.title);
|
|
6194
|
-
const duration =
|
|
7125
|
+
const duration = seconds(output.duration_seconds);
|
|
7126
|
+
const original = seconds(output.original_duration_seconds) ?? seconds(output.original_seconds);
|
|
6195
7127
|
outputs.push({
|
|
6196
7128
|
index,
|
|
6197
7129
|
...url === void 0 ? {} : { url },
|
|
6198
7130
|
...contentType === void 0 ? {} : { contentType },
|
|
6199
7131
|
...title === void 0 ? {} : { title },
|
|
6200
|
-
...duration === void 0 ? {} : { durationSeconds: duration }
|
|
7132
|
+
...duration === void 0 ? {} : { durationSeconds: duration },
|
|
7133
|
+
...original === void 0 ? {} : { originalSeconds: original }
|
|
6201
7134
|
});
|
|
6202
7135
|
}
|
|
6203
|
-
const
|
|
7136
|
+
const cancelled = CANCELLED.has(status.toLowerCase());
|
|
7137
|
+
const error = state === "failed" ? errorOf(raw) ?? (cancelled ? {
|
|
7138
|
+
code: "CANCELLED",
|
|
7139
|
+
message: "The task was cancelled; cancelled tasks are not charged."
|
|
7140
|
+
} : {
|
|
6204
7141
|
code: "GENERATION_FAILED",
|
|
6205
7142
|
message: `the task ended with status "${status}"`
|
|
6206
|
-
} : void 0;
|
|
7143
|
+
}) : void 0;
|
|
6207
7144
|
const progress = progressOf(raw.progress);
|
|
6208
7145
|
return {
|
|
6209
7146
|
id,
|
|
@@ -6212,6 +7149,7 @@ function parseAudioTask(body) {
|
|
|
6212
7149
|
...progress === void 0 ? {} : { progress },
|
|
6213
7150
|
...error === void 0 ? {} : { error },
|
|
6214
7151
|
...receipt(raw),
|
|
7152
|
+
...billing(raw, status),
|
|
6215
7153
|
outputs: state === "completed" ? outputs : []
|
|
6216
7154
|
};
|
|
6217
7155
|
}
|
|
@@ -6568,11 +7506,15 @@ var TaskTracker = class {
|
|
|
6568
7506
|
}
|
|
6569
7507
|
async finish(id, patch) {
|
|
6570
7508
|
this.downloadFailures.delete(id);
|
|
6571
|
-
|
|
7509
|
+
const record = await this.update(id, {
|
|
6572
7510
|
...patch,
|
|
6573
7511
|
progress: void 0,
|
|
6574
7512
|
finishedAt: this.now()
|
|
6575
7513
|
});
|
|
7514
|
+
try {
|
|
7515
|
+
this.options.onFinished?.(record);
|
|
7516
|
+
} catch {}
|
|
7517
|
+
return record;
|
|
6576
7518
|
}
|
|
6577
7519
|
/** Hold (or keep) this process's claim on a task, renewing it while held. */
|
|
6578
7520
|
async holdClaim(id) {
|
|
@@ -6717,12 +7659,25 @@ var TaskTracker = class {
|
|
|
6717
7659
|
}
|
|
6718
7660
|
return Math.min(6e4, 5e3 * 2 ** Math.min(failures - 1, 4));
|
|
6719
7661
|
}
|
|
6720
|
-
/**
|
|
6721
|
-
|
|
6722
|
-
const
|
|
7662
|
+
/** The money and billing facts of a gateway answer, merged into what the record already knows. */
|
|
7663
|
+
moneyOf(record, task) {
|
|
7664
|
+
const charged = task.settlement === "pending" ? void 0 : task.chargedCny;
|
|
7665
|
+
const settlement = task.settlement ?? (charged !== void 0 ? "settled" : void 0);
|
|
7666
|
+
const billing = task.billedOriginalSeconds !== void 0 || task.billedMinutes !== void 0 ? { billing: {
|
|
7667
|
+
...record.billing,
|
|
7668
|
+
...task.billedOriginalSeconds === void 0 ? {} : { originalSeconds: task.billedOriginalSeconds },
|
|
7669
|
+
...task.billedMinutes === void 0 ? {} : { minutes: task.billedMinutes }
|
|
7670
|
+
} } : {};
|
|
7671
|
+
return {
|
|
6723
7672
|
...task.estimatedCny === void 0 ? {} : { estimatedCny: task.estimatedCny },
|
|
6724
|
-
...
|
|
7673
|
+
...charged === void 0 ? {} : { chargedCny: charged },
|
|
7674
|
+
...settlement === void 0 ? {} : { settlement },
|
|
7675
|
+
...billing
|
|
6725
7676
|
};
|
|
7677
|
+
}
|
|
7678
|
+
/** Record what the gateway says about a task; save the outputs of a finished one. */
|
|
7679
|
+
async apply(record, task) {
|
|
7680
|
+
const money = this.moneyOf(record, task);
|
|
6726
7681
|
if (task.state === "pending") return this.update(record.id, {
|
|
6727
7682
|
gatewayId: task.id,
|
|
6728
7683
|
status: "pending",
|
|
@@ -6748,14 +7703,14 @@ var TaskTracker = class {
|
|
|
6748
7703
|
if (attempts < (this.options.downloadAttempts ?? 5)) throw new MediaError(`${missing} result file(s) could not be downloaded yet.`, "DOWNLOAD_RETRY", { retryable: true });
|
|
6749
7704
|
}
|
|
6750
7705
|
let charged = money;
|
|
6751
|
-
if (
|
|
7706
|
+
if (money.chargedCny === void 0 && task.settlement !== "pending") {
|
|
6752
7707
|
const settled = await this.options.http.json(SPECS[record.kind].poll(task.id), {
|
|
6753
7708
|
timeoutMs: 15e3,
|
|
6754
7709
|
signal: this.signal
|
|
6755
7710
|
}).then((body) => SPECS[record.kind].parse(body), () => void 0);
|
|
6756
|
-
if (settled
|
|
7711
|
+
if (settled !== void 0) charged = {
|
|
6757
7712
|
...money,
|
|
6758
|
-
|
|
7713
|
+
...this.moneyOf(current, settled)
|
|
6759
7714
|
};
|
|
6760
7715
|
}
|
|
6761
7716
|
return this.finish(record.id, {
|
|
@@ -6770,6 +7725,50 @@ var TaskTracker = class {
|
|
|
6770
7725
|
});
|
|
6771
7726
|
}
|
|
6772
7727
|
/**
|
|
7728
|
+
* Ask the gateway about a task again, by query only: a finished task gets
|
|
7729
|
+
* fresh links (signed links expire), files that are missing locally are
|
|
7730
|
+
* downloaded again, and a pending settlement is read again. Nothing is ever
|
|
7731
|
+
* submitted from here, so a failed query or download cannot create a task.
|
|
7732
|
+
* A task still running is followed as usual.
|
|
7733
|
+
* @param id - the local task id.
|
|
7734
|
+
* @param signal - cancels the query.
|
|
7735
|
+
* @returns the task after the query.
|
|
7736
|
+
*/
|
|
7737
|
+
async refresh(id, signal) {
|
|
7738
|
+
const record = await this.options.store.get(id);
|
|
7739
|
+
if (record === void 0) throw new MediaError(`dsh-vibedev has no task ${id}.`, "UNKNOWN_TASK");
|
|
7740
|
+
if (!isFinished(record)) {
|
|
7741
|
+
this.followQuietly(id);
|
|
7742
|
+
return record;
|
|
7743
|
+
}
|
|
7744
|
+
if (record.gatewayId === void 0) return record;
|
|
7745
|
+
const spec = SPECS[record.kind];
|
|
7746
|
+
const task = spec.parse(await this.options.http.json(spec.poll(record.gatewayId), {
|
|
7747
|
+
timeoutMs: 3e4,
|
|
7748
|
+
...signal === void 0 ? {} : { signal }
|
|
7749
|
+
}));
|
|
7750
|
+
if (task === void 0) throw new MediaError("The VibeDev gateway returned an unreadable task.", "GATEWAY_BAD_RESPONSE");
|
|
7751
|
+
const money = this.moneyOf(record, task);
|
|
7752
|
+
if (task.state !== "completed") return this.update(id, money);
|
|
7753
|
+
const fresh = new Map(task.outputs.map((output) => [output.index, output]));
|
|
7754
|
+
const known = (record.outputs ?? []).filter((output) => output.path !== void 0 && output.index !== void 0).map((output) => ({
|
|
7755
|
+
...output,
|
|
7756
|
+
...fresh.get(output.index)?.url === void 0 ? {} : { url: fresh.get(output.index)?.url }
|
|
7757
|
+
}));
|
|
7758
|
+
const { outputs, missing } = await this.saveOutputs({
|
|
7759
|
+
...record,
|
|
7760
|
+
outputs: known
|
|
7761
|
+
}, task);
|
|
7762
|
+
return this.update(id, {
|
|
7763
|
+
...money,
|
|
7764
|
+
outputs,
|
|
7765
|
+
error: missing === 0 ? void 0 : {
|
|
7766
|
+
code: "DOWNLOAD_FAILED",
|
|
7767
|
+
message: `${missing} of ${outputs.length} result file(s) could not be downloaded; the gateway links are listed instead.`
|
|
7768
|
+
}
|
|
7769
|
+
});
|
|
7770
|
+
}
|
|
7771
|
+
/**
|
|
6773
7772
|
* Save every output not saved yet, recording each one as it lands. A
|
|
6774
7773
|
* shutdown mid-download throws, so the task stays open and is resumed.
|
|
6775
7774
|
*/
|
|
@@ -6789,7 +7788,8 @@ var TaskTracker = class {
|
|
|
6789
7788
|
...output.url === void 0 ? {} : { url: output.url },
|
|
6790
7789
|
...output.contentType === void 0 ? {} : { contentType: output.contentType },
|
|
6791
7790
|
...output.title === void 0 ? {} : { title: output.title },
|
|
6792
|
-
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds }
|
|
7791
|
+
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds },
|
|
7792
|
+
...output.originalSeconds === void 0 ? {} : { originalSeconds: output.originalSeconds }
|
|
6793
7793
|
};
|
|
6794
7794
|
const fromUrl = urlExtension(output.url);
|
|
6795
7795
|
const extension = output.contentType === void 0 ? fromUrl ?? extensionFor(spec.fallbackType) : extensionFor(output.contentType, fromUrl ?? extensionFor(spec.fallbackType));
|
|
@@ -6946,8 +7946,9 @@ function mediaAccountTool(runtime, chain, login) {
|
|
|
6946
7946
|
//#endregion
|
|
6947
7947
|
//#region src/tools/audio.ts
|
|
6948
7948
|
/**
|
|
6949
|
-
* `audio_generate`:
|
|
6950
|
-
* results are saved in the workspace.
|
|
7949
|
+
* `audio_generate`: music, a podcast episode or speech, as a background task
|
|
7950
|
+
* whose results are saved in the workspace. Every field is checked against
|
|
7951
|
+
* what the chosen model declares (see `audio/request`).
|
|
6951
7952
|
* @module dsh-vibedev/tools/audio
|
|
6952
7953
|
*/
|
|
6953
7954
|
/**
|
|
@@ -6958,17 +7959,17 @@ function mediaAccountTool(runtime, chain, login) {
|
|
|
6958
7959
|
function audioGenerateTool(runtime) {
|
|
6959
7960
|
return defineTool({
|
|
6960
7961
|
name: "audio_generate",
|
|
6961
|
-
description: "Generate music (a song with vocals or an instrumental)
|
|
7962
|
+
description: "Generate music (a song with vocals or an instrumental), a podcast episode (a spoken conversation about a topic or a web page) or speech (text read aloud, optionally in the voice of reference audio) with a VibeDev audio model. It runs as a background job: this call returns once the gateway accepts the task, you are notified when it finishes, and the files are saved in the workspace (media/audio/). Each model takes only the fields it declares; media_models lists them with prices (music and podcasts per request, speech per second of original audio). Charged only when it succeeds.",
|
|
6962
7963
|
parameters: {
|
|
6963
7964
|
type: {
|
|
6964
7965
|
type: "string",
|
|
6965
|
-
enum: [
|
|
7966
|
+
enum: [...AUDIO_MAKE_KINDS],
|
|
6966
7967
|
required: true,
|
|
6967
7968
|
description: "What to make."
|
|
6968
7969
|
},
|
|
6969
7970
|
prompt: {
|
|
6970
7971
|
type: "string",
|
|
6971
|
-
description: "Music: style, mood, instruments, tempo, theme. Podcast: the topic and angle
|
|
7972
|
+
description: "Music: style, mood, instruments, tempo, theme (required). Podcast: the topic and angle; give a prompt, a source_url, or both. Speech: the exact text to speak (required)."
|
|
6972
7973
|
},
|
|
6973
7974
|
lyrics: {
|
|
6974
7975
|
type: "string",
|
|
@@ -6984,16 +7985,32 @@ function audioGenerateTool(runtime) {
|
|
|
6984
7985
|
},
|
|
6985
7986
|
gender: {
|
|
6986
7987
|
type: "string",
|
|
6987
|
-
|
|
6988
|
-
description: "Music only: the singer's voice."
|
|
7988
|
+
description: "Music only: the singer's voice, such as male or female."
|
|
6989
7989
|
},
|
|
6990
7990
|
source_url: {
|
|
6991
7991
|
type: "string",
|
|
6992
7992
|
description: "Podcast only: an http(s) page for the hosts to discuss."
|
|
6993
7993
|
},
|
|
7994
|
+
references: {
|
|
7995
|
+
type: "array",
|
|
7996
|
+
items: { type: "string" },
|
|
7997
|
+
description: `Speech only: reference audio whose voice to imitate, each ${SOURCE_HELP}. The model states how many, how large and how long.`
|
|
7998
|
+
},
|
|
7999
|
+
format: {
|
|
8000
|
+
type: "string",
|
|
8001
|
+
description: "Speech only: an output format the model offers (media_models lists them, such as wav or mp3)."
|
|
8002
|
+
},
|
|
8003
|
+
sample_rate: {
|
|
8004
|
+
type: "integer",
|
|
8005
|
+
description: "Speech only: a sample rate the model offers for that format (Hz)."
|
|
8006
|
+
},
|
|
8007
|
+
watermark: {
|
|
8008
|
+
type: "boolean",
|
|
8009
|
+
description: "Speech only, when the model takes it: add the provider's audio watermark."
|
|
8010
|
+
},
|
|
6994
8011
|
model: {
|
|
6995
8012
|
type: "string",
|
|
6996
|
-
description: "Audio model id from media_models. Omit to use the default for the type."
|
|
8013
|
+
description: "Audio model id (or alias) from media_models. Omit to use the default for the type."
|
|
6997
8014
|
},
|
|
6998
8015
|
filename: {
|
|
6999
8016
|
type: "string",
|
|
@@ -7019,6 +8036,7 @@ function audioGenerateTool(runtime) {
|
|
|
7019
8036
|
},
|
|
7020
8037
|
jobId: { type: "string" },
|
|
7021
8038
|
estimatedCny: { type: "string" },
|
|
8039
|
+
price: { type: "string" },
|
|
7022
8040
|
outputs: {
|
|
7023
8041
|
type: "array",
|
|
7024
8042
|
items: {
|
|
@@ -7028,7 +8046,8 @@ function audioGenerateTool(runtime) {
|
|
|
7028
8046
|
path: { type: "string" },
|
|
7029
8047
|
url: { type: "string" },
|
|
7030
8048
|
title: { type: "string" },
|
|
7031
|
-
durationSeconds: { type: "number" }
|
|
8049
|
+
durationSeconds: { type: "number" },
|
|
8050
|
+
originalSeconds: { type: "number" }
|
|
7032
8051
|
}
|
|
7033
8052
|
}
|
|
7034
8053
|
},
|
|
@@ -7044,75 +8063,44 @@ function audioGenerateTool(runtime) {
|
|
|
7044
8063
|
}]
|
|
7045
8064
|
},
|
|
7046
8065
|
async execute(args, exec) {
|
|
8066
|
+
const kind = args.type;
|
|
7047
8067
|
const prompt = args.prompt?.trim();
|
|
7048
|
-
const
|
|
7049
|
-
|
|
7050
|
-
|
|
7051
|
-
|
|
7052
|
-
|
|
7053
|
-
|
|
7054
|
-
|
|
7055
|
-
|
|
7056
|
-
|
|
7057
|
-
|
|
7058
|
-
|
|
7059
|
-
}
|
|
7060
|
-
|
|
7061
|
-
|
|
7062
|
-
|
|
7063
|
-
|
|
7064
|
-
body = {
|
|
7065
|
-
...prompt ? { prompt } : {},
|
|
7066
|
-
...sourceUrl ? { source_url: sourceUrl } : {}
|
|
7067
|
-
};
|
|
7068
|
-
}
|
|
7069
|
-
const model = await runtime.pickModel("audio", args.type, args.model, void 0, exec.signal);
|
|
7070
|
-
const estimate = estimateAudio(model);
|
|
7071
|
-
const what = args.type === "music" ? {
|
|
7072
|
-
en: "a song",
|
|
7073
|
-
zh: "一首音乐"
|
|
7074
|
-
} : {
|
|
7075
|
-
en: "a podcast episode",
|
|
7076
|
-
zh: "一期播客"
|
|
7077
|
-
};
|
|
7078
|
-
await runtime.confirmSpending(exec, "audio_generate", {
|
|
7079
|
-
en: `Generate ${what.en} with ${model.id}${estimate === void 0 ? "" : `, about ${yuan(estimate.amountCny)}`}. Charged only if it succeeds.`,
|
|
7080
|
-
zh: `用 ${model.id} 生成${what.zh}${estimate === void 0 ? "" : `,约 ${yuan(estimate.amountCny)}`},成功才扣费。`
|
|
7081
|
-
});
|
|
7082
|
-
const now = runtime.now();
|
|
7083
|
-
const label = excerpt(prompt ?? sourceUrl);
|
|
7084
|
-
const { record: accepted, jobId, unconfirmed } = await submitTask(runtime, exec, {
|
|
7085
|
-
id: randomUUID(),
|
|
7086
|
-
kind: "audio",
|
|
7087
|
-
model: model.id,
|
|
7088
|
-
label,
|
|
7089
|
-
createdAt: now,
|
|
7090
|
-
updatedAt: now,
|
|
7091
|
-
...exec.agent === void 0 ? {} : { owner: exec.agent.id },
|
|
7092
|
-
outputDir: runtime.outputFolder(exec, "audio"),
|
|
7093
|
-
stem: runtime.stem(args.filename, prompt ?? args.type),
|
|
7094
|
-
endpoint: "/v1/audio/generations",
|
|
7095
|
-
body: {
|
|
7096
|
-
model: model.id,
|
|
7097
|
-
...body
|
|
7098
|
-
},
|
|
7099
|
-
status: "submitting",
|
|
7100
|
-
...estimate === void 0 ? {} : { estimatedCny: estimate.amountCny.toFixed(2) }
|
|
8068
|
+
const prepared = await prepareAudioTask(runtime, exec, {
|
|
8069
|
+
kind,
|
|
8070
|
+
...args.model === void 0 ? {} : { model: args.model },
|
|
8071
|
+
...prompt === void 0 || prompt === "" ? {} : { prompt },
|
|
8072
|
+
...args.lyrics === void 0 ? {} : { lyrics: args.lyrics },
|
|
8073
|
+
...args.genre === void 0 ? {} : { genre: args.genre },
|
|
8074
|
+
...args.mood === void 0 ? {} : { mood: args.mood },
|
|
8075
|
+
...args.gender === void 0 ? {} : { gender: args.gender },
|
|
8076
|
+
...args.source_url === void 0 ? {} : { sourceUrl: args.source_url },
|
|
8077
|
+
...args.references === void 0 ? {} : { references: args.references },
|
|
8078
|
+
...args.format === void 0 ? {} : { format: args.format },
|
|
8079
|
+
...args.sample_rate === void 0 ? {} : { sampleRate: args.sample_rate },
|
|
8080
|
+
...args.watermark === void 0 ? {} : { watermark: args.watermark }
|
|
8081
|
+
}, {
|
|
8082
|
+
folder: runtime.outputFolder(exec, "audio"),
|
|
8083
|
+
stem: runtime.stem(args.filename, prompt ?? args.source_url ?? kind)
|
|
7101
8084
|
});
|
|
8085
|
+
const { model, quote } = prepared;
|
|
8086
|
+
const { record: accepted, jobId, unconfirmed } = await submitTask(runtime, exec, prepared.draft);
|
|
7102
8087
|
const base = {
|
|
7103
8088
|
taskId: accepted.id,
|
|
7104
8089
|
model: model.id,
|
|
8090
|
+
price: quote.label.en,
|
|
7105
8091
|
...accepted.estimatedCny === void 0 ? {} : { estimatedCny: accepted.estimatedCny }
|
|
7106
8092
|
};
|
|
7107
|
-
|
|
7108
|
-
|
|
7109
|
-
|
|
7110
|
-
|
|
7111
|
-
|
|
7112
|
-
|
|
7113
|
-
|
|
7114
|
-
|
|
7115
|
-
|
|
8093
|
+
const name = {
|
|
8094
|
+
music: "Music",
|
|
8095
|
+
podcast: "Podcast",
|
|
8096
|
+
speech: "Speech"
|
|
8097
|
+
}[kind];
|
|
8098
|
+
if (jobId !== void 0) return {
|
|
8099
|
+
...base,
|
|
8100
|
+
status: accepted.status,
|
|
8101
|
+
jobId,
|
|
8102
|
+
message: `${unconfirmed ? `${name} task ${accepted.id} (${model.id}): ${UNCONFIRMED_NOTE}` : `${name} task submitted to ${model.id}.`} Background job ${jobId} reports when it finishes; files are saved under ${runtime.display(exec, accepted.outputDir)}. ${accepted.estimatedCny === void 0 ? `Price: ${quote.basis.en}. ` : `Estimated ${yuan(Number(accepted.estimatedCny))}, charged only if it succeeds. `}Carry on with other work meanwhile; there is no need to poll.`
|
|
8103
|
+
};
|
|
7116
8104
|
const final = await waitForTask(runtime, exec, accepted);
|
|
7117
8105
|
return {
|
|
7118
8106
|
...base,
|
|
@@ -7121,7 +8109,8 @@ function audioGenerateTool(runtime) {
|
|
|
7121
8109
|
...output.path === void 0 ? {} : { path: runtime.display(exec, output.path) },
|
|
7122
8110
|
...output.url === void 0 ? {} : { url: output.url },
|
|
7123
8111
|
...output.title === void 0 ? {} : { title: output.title },
|
|
7124
|
-
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds }
|
|
8112
|
+
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds },
|
|
8113
|
+
...output.originalSeconds === void 0 ? {} : { originalSeconds: output.originalSeconds }
|
|
7125
8114
|
})) },
|
|
7126
8115
|
message: describeTask(runtime, exec, final)
|
|
7127
8116
|
};
|
|
@@ -7138,14 +8127,47 @@ function audioGenerateTool(runtime) {
|
|
|
7138
8127
|
const KIND_LABEL = {
|
|
7139
8128
|
image: "Image models (image_generate)",
|
|
7140
8129
|
video: "Video models (video_generate)",
|
|
7141
|
-
audio: "Music and
|
|
8130
|
+
audio: "Music, podcast and speech models (audio_generate)",
|
|
7142
8131
|
transcription: "Transcription models (audio_transcribe)"
|
|
7143
8132
|
};
|
|
7144
8133
|
function slotOf(model) {
|
|
7145
|
-
if (model.kind === "audio") return model.audioKind
|
|
8134
|
+
if (model.kind === "audio") return model.audioKind;
|
|
7146
8135
|
return model.kind;
|
|
7147
8136
|
}
|
|
7148
8137
|
/**
|
|
8138
|
+
* What an audio model takes, from its declaration: the fields, speech references, output formats and sample rates.
|
|
8139
|
+
* @param model - an audio or transcription model.
|
|
8140
|
+
* @returns one line, or undefined when nothing is declared.
|
|
8141
|
+
*/
|
|
8142
|
+
function describeAudioModel(model) {
|
|
8143
|
+
const audio = model.audio;
|
|
8144
|
+
if (model.kind === "transcription") {
|
|
8145
|
+
const facts = [
|
|
8146
|
+
audio?.maxInputBytes === void 0 ? void 0 : `at most ${(audio.maxInputBytes / 1e6).toFixed(0)} MB`,
|
|
8147
|
+
audio?.maxInputSeconds === void 0 ? void 0 : `at most ${Math.round(audio.maxInputSeconds / 60)} min`,
|
|
8148
|
+
audio?.languages === void 0 ? void 0 : `languages ${audio.languages.join(", ")}`
|
|
8149
|
+
].filter(Boolean);
|
|
8150
|
+
return facts.length === 0 ? void 0 : facts.join("; ");
|
|
8151
|
+
}
|
|
8152
|
+
if (model.audioKind === void 0) return "does not declare what kind of audio it makes; it cannot be used for generation";
|
|
8153
|
+
const fields = requestFieldsOf(model).filter((name) => name !== "model");
|
|
8154
|
+
const parts = [`fields: ${fields.join(", ") || "none declared"}`];
|
|
8155
|
+
const prompt = promptFieldOf(model);
|
|
8156
|
+
if (prompt !== void 0 && prompt !== "prompt") parts.push(`the text goes in ${prompt}`);
|
|
8157
|
+
const references = audio?.references;
|
|
8158
|
+
if (references !== void 0 && fields.includes("references")) {
|
|
8159
|
+
const count = references.min === references.max ? `${references.max}` : `${references.min}-${references.max}`;
|
|
8160
|
+
parts.push([
|
|
8161
|
+
`${count} reference audio`,
|
|
8162
|
+
references.maxBytes === void 0 ? void 0 : `each at most ${(references.maxBytes / 1048576).toFixed(0)} MiB`,
|
|
8163
|
+
references.minSeconds === void 0 && references.maxSeconds === void 0 ? void 0 : `each ${references.minSeconds ?? 0}-${references.maxSeconds ?? "∞"} s`,
|
|
8164
|
+
references.exclusiveWith === void 0 ? void 0 : `not with ${references.exclusiveWith.join(", ")}`
|
|
8165
|
+
].filter(Boolean).join(", "));
|
|
8166
|
+
}
|
|
8167
|
+
for (const format of audio?.outputFormats ?? []) parts.push(`format ${format.format}${format.sampleRates.length === 0 ? "" : ` at ${format.sampleRates.join("/")} Hz`}`);
|
|
8168
|
+
return parts.join("; ");
|
|
8169
|
+
}
|
|
8170
|
+
/**
|
|
7149
8171
|
* Build the tool.
|
|
7150
8172
|
* @param runtime - the media runtime.
|
|
7151
8173
|
* @returns the tool definition.
|
|
@@ -7153,7 +8175,7 @@ function slotOf(model) {
|
|
|
7153
8175
|
function mediaModelsTool(runtime) {
|
|
7154
8176
|
return defineTool({
|
|
7155
8177
|
name: "media_models",
|
|
7156
|
-
description: "List the image, video, music/podcast and transcription models this VibeDev account can use, with prices. For video models it lists each generation mode and the inputs it accepts (reference images, videos and audio, first and last frames), durations, aspect ratios and resolutions. Use it to choose a model when a request needs particular inputs or options; the generation tools pick a suitable model themselves when none is named.",
|
|
8178
|
+
description: "List the image, video, music/podcast/speech and transcription models this VibeDev account can use, with prices (\"not priced\" when the catalog states none). For video models it lists each generation mode and the inputs it accepts (reference images, videos and audio, first and last frames), durations, aspect ratios and resolutions; for audio models the fields each takes, speech references, output formats and sample rates. Use it to choose a model when a request needs particular inputs or options; the generation tools pick a suitable model themselves when none is named.",
|
|
7157
8179
|
parameters: { kind: {
|
|
7158
8180
|
type: "string",
|
|
7159
8181
|
enum: [
|
|
@@ -7232,16 +8254,20 @@ function mediaModelsTool(runtime) {
|
|
|
7232
8254
|
isConcurrencySafe: () => true,
|
|
7233
8255
|
async execute(args, exec) {
|
|
7234
8256
|
return { models: (await runtime.models(exec.signal)).filter((model) => args.kind === void 0 || model.kind === args.kind).map((model) => {
|
|
7235
|
-
const
|
|
7236
|
-
const
|
|
8257
|
+
const audioLike = model.kind === "audio" || model.kind === "transcription";
|
|
8258
|
+
const price = audioLike ? quoteAudio(model).label.en : describePrice(model);
|
|
8259
|
+
const slot = slotOf(model);
|
|
8260
|
+
const pinned = slot !== void 0 && runtime.settings.defaultModel(slot).trim() === model.id;
|
|
8261
|
+
const audioFacts = audioLike ? describeAudioModel(model) : void 0;
|
|
7237
8262
|
return {
|
|
7238
8263
|
id: model.id,
|
|
7239
8264
|
name: model.name,
|
|
7240
8265
|
kind: model.kind,
|
|
7241
|
-
...model.
|
|
8266
|
+
...model.kind !== "audio" ? {} : { produces: model.audioKind ?? "undeclared" },
|
|
7242
8267
|
...model.description === void 0 ? {} : { description: model.description },
|
|
7243
|
-
|
|
8268
|
+
price,
|
|
7244
8269
|
...model.kind === "video" ? { capabilities: describeVideoModel(model) } : {},
|
|
8270
|
+
...audioFacts === void 0 ? {} : { capabilities: audioFacts },
|
|
7245
8271
|
...pinned ? { pinned } : {}
|
|
7246
8272
|
};
|
|
7247
8273
|
}) };
|
|
@@ -7329,7 +8355,7 @@ function mediaTasksTool(runtime) {
|
|
|
7329
8355
|
const limit = Math.min(50, Math.max(1, Math.trunc(args.limit ?? 10)));
|
|
7330
8356
|
const shown = tasks.slice(0, limit);
|
|
7331
8357
|
for (const task of shown) if (!isFinished(task)) runtime.tracker.follow(task.id).catch(() => void 0);
|
|
7332
|
-
return { tasks: shown.map((task) => ({
|
|
8358
|
+
return { tasks: (await Promise.all(shown.map((task) => task.status === "completed" && task.settlement === "pending" ? runtime.tracker.refresh(task.id, exec.signal).catch(() => task) : task))).map((task) => ({
|
|
7333
8359
|
id: task.id,
|
|
7334
8360
|
kind: task.kind,
|
|
7335
8361
|
status: task.status,
|
|
@@ -7411,7 +8437,7 @@ function sleep(ms, signal) {
|
|
|
7411
8437
|
function audioTranscribeTool(runtime) {
|
|
7412
8438
|
return defineTool({
|
|
7413
8439
|
name: "audio_transcribe",
|
|
7414
|
-
description: "Transcribe speech in an audio file to text with a VibeDev transcription model
|
|
8440
|
+
description: "Transcribe speech in an audio file to text with a VibeDev transcription model (uploaded as a multipart file). Each model states its languages and limits (media_models; commonly Mandarin Chinese, at most 20 MB and 10 minutes); 16 kHz mono WAV works best. Charged per minute of audio.",
|
|
7415
8441
|
parameters: {
|
|
7416
8442
|
file: {
|
|
7417
8443
|
type: "string",
|
|
@@ -7462,15 +8488,19 @@ function audioTranscribeTool(runtime) {
|
|
|
7462
8488
|
async execute(args, exec) {
|
|
7463
8489
|
const language = (args.language ?? "zh").trim().toLowerCase() || "zh";
|
|
7464
8490
|
const model = await runtime.pickModel("transcription", "transcription", args.model, void 0, exec.signal);
|
|
7465
|
-
const
|
|
7466
|
-
|
|
8491
|
+
const declared = model.audio;
|
|
8492
|
+
const maxBytes = declared?.maxInputBytes ?? MAX_BYTES;
|
|
8493
|
+
const maxSeconds = declared?.maxInputSeconds ?? MAX_SECONDS;
|
|
8494
|
+
const accepted = declared?.inputMimes ?? ACCEPTED;
|
|
8495
|
+
if (declared?.languages !== void 0 && !declared.languages.includes(language)) throw new MediaError(`${model.id} transcribes ${declared.languages.join(", ")}, not "${language}".`, "AUDIO_OPTION_INVALID", { field: "language" });
|
|
8496
|
+
const media = await runtime.load(exec, args.file, "audio", maxBytes);
|
|
8497
|
+
if (!accepted.includes(media.mime)) throw new MediaError(`"${media.source}" is ${media.mime}, not an audio file the gateway transcribes; convert it to 16 kHz mono WAV first.`, "AUDIO_FORMAT_UNSUPPORTED", { field: "file" });
|
|
7467
8498
|
const seconds = media.mime === "audio/wav" ? wavSeconds(media.data) : void 0;
|
|
7468
|
-
if (seconds !== void 0 && seconds >
|
|
7469
|
-
const
|
|
7470
|
-
const estimate = tier?.unit === "second" && seconds !== void 0 ? tier.amount * seconds : void 0;
|
|
8499
|
+
if (seconds !== void 0 && seconds > maxSeconds) throw new MediaError(`"${media.source}" is ${(seconds / 60).toFixed(1)} minutes long; the limit is ${(maxSeconds / 60).toFixed(0)} minutes. Split it first.`, "AUDIO_TOO_LONG", { field: "file" });
|
|
8500
|
+
const quote = quoteAudio(model, seconds);
|
|
7471
8501
|
await runtime.confirmSpending(exec, "audio_transcribe", {
|
|
7472
|
-
en: `Transcribe ${media.name} with ${model.id}${
|
|
7473
|
-
zh: `用 ${model.id} 转写 ${media.name}
|
|
8502
|
+
en: `Transcribe ${media.name} with ${model.id}: ${quote.amountCny === void 0 ? quote.basis.en : `about ${yuan(quote.amountCny)} (${quote.basis.en})`}.`,
|
|
8503
|
+
zh: `用 ${model.id} 转写 ${media.name}:${quote.amountCny === void 0 ? quote.basis.zh : `约 ${yuan(quote.amountCny)}(${quote.basis.zh})`}。`
|
|
7474
8504
|
});
|
|
7475
8505
|
const body = new FormData();
|
|
7476
8506
|
body.append("file", new Blob([media.data], { type: media.mime }), media.name);
|
|
@@ -7542,6 +8572,8 @@ function audioTranscribeTool(runtime) {
|
|
|
7542
8572
|
* ```
|
|
7543
8573
|
* @module dsh-vibedev
|
|
7544
8574
|
*/
|
|
8575
|
+
/** Requests that are charged: generations and transcriptions (uploads and reads are free). */
|
|
8576
|
+
const PAID_PATH = /^\/v1\/(?:images|videos|audio)(?:\/|$)/;
|
|
7545
8577
|
/** The package version, sent in the `User-Agent` the gateway attributes plugin traffic by. */
|
|
7546
8578
|
const version = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
7547
8579
|
const name = "dsh-vibedev";
|
|
@@ -7557,6 +8589,7 @@ const Config = Schema.object({
|
|
|
7557
8589
|
videoModel: Schema.string().default("").volatile(),
|
|
7558
8590
|
musicModel: Schema.string().default("").volatile(),
|
|
7559
8591
|
podcastModel: Schema.string().default("").volatile(),
|
|
8592
|
+
speechModel: Schema.string().default("").volatile(),
|
|
7560
8593
|
transcriptionModel: Schema.string().default("").volatile(),
|
|
7561
8594
|
gatewayOrigin: Schema.string().default(DEFAULT_GATEWAY_ORIGIN),
|
|
7562
8595
|
client: Schema.string().default("vibedev-plugin"),
|
|
@@ -7575,6 +8608,7 @@ const Config = Schema.object({
|
|
|
7575
8608
|
videoModel: "默认视频模型(留空由 Agent 按目录选择)",
|
|
7576
8609
|
musicModel: "默认音乐模型(留空自动选择)",
|
|
7577
8610
|
podcastModel: "默认播客模型(留空自动选择)",
|
|
8611
|
+
speechModel: "默认语音合成模型(留空自动选择)",
|
|
7578
8612
|
transcriptionModel: "默认语音转写模型(留空自动选择)",
|
|
7579
8613
|
gatewayOrigin: "VibeDev 网关地址",
|
|
7580
8614
|
client: "登录时报给网关的客户端名称(决定用量记在哪个密钥下)",
|
|
@@ -7593,6 +8627,7 @@ const Config = Schema.object({
|
|
|
7593
8627
|
videoModel: "Default video model (empty: the agent chooses from the catalog)",
|
|
7594
8628
|
musicModel: "Default music model (empty: chosen automatically)",
|
|
7595
8629
|
podcastModel: "Default podcast model (empty: chosen automatically)",
|
|
8630
|
+
speechModel: "Default speech model (empty: chosen automatically)",
|
|
7596
8631
|
transcriptionModel: "Default transcription model (empty: chosen automatically)",
|
|
7597
8632
|
gatewayOrigin: "VibeDev gateway address",
|
|
7598
8633
|
client: "Client name reported to the gateway at sign-in (decides which key the usage is recorded under)",
|
|
@@ -7601,7 +8636,7 @@ const Config = Schema.object({
|
|
|
7601
8636
|
stateDir: "Plugin data folder (empty for the default)"
|
|
7602
8637
|
}
|
|
7603
8638
|
});
|
|
7604
|
-
const GUIDANCE = "VibeDev media tools: image_generate, video_generate, audio_generate (music
|
|
8639
|
+
const GUIDANCE = "VibeDev media tools: image_generate, video_generate, audio_generate (music, a podcast or speech) and audio_transcribe create and transcribe media with the user's VibeDev account; media_models lists models and prices. Use them on your own judgment whenever generated media serves the request (illustrations, product shots, storyboard frames, short clips, background music, a podcast episode, a voice-over), not only when a tool is named. They spend the user's balance, so make what the request needs rather than speculative variations. Results are saved in the workspace under media/. Pass files the user attached in this chat as chat:1 (the most recent), chat:2, and so on. Video and audio run as background jobs: carry on and let the job notification report the result.";
|
|
7605
8640
|
function apply(ctx, config) {
|
|
7606
8641
|
const origin = normalizeGatewayOrigin(config.gatewayOrigin);
|
|
7607
8642
|
const stateDir = stateDirectory(config.stateDir);
|
|
@@ -7633,23 +8668,32 @@ function apply(ctx, config) {
|
|
|
7633
8668
|
account: () => ctx.get("deepseekAccount"),
|
|
7634
8669
|
apiKey: () => developmentKey(origin, config.apiKeyEnv)
|
|
7635
8670
|
});
|
|
8671
|
+
let account;
|
|
8672
|
+
const spent = () => {
|
|
8673
|
+
account?.spent();
|
|
8674
|
+
};
|
|
7636
8675
|
const http = new GatewayHttp({
|
|
7637
8676
|
origin,
|
|
7638
8677
|
userAgent,
|
|
7639
8678
|
resolveCredential: () => chain.resolve(),
|
|
7640
|
-
rejectToken: (credential) => chain.reject(credential)
|
|
8679
|
+
rejectToken: (credential) => chain.reject(credential),
|
|
8680
|
+
onAccepted: ({ method, path }) => {
|
|
8681
|
+
if (method === "POST" && PAID_PATH.test(path)) spent();
|
|
8682
|
+
}
|
|
7641
8683
|
});
|
|
7642
8684
|
const store = new TaskStore(gatewayStateDirectory(stateDir, origin));
|
|
7643
8685
|
const tracker = new TaskTracker({
|
|
7644
8686
|
http,
|
|
7645
8687
|
store,
|
|
7646
|
-
log: warn
|
|
8688
|
+
log: warn,
|
|
8689
|
+
onFinished: spent
|
|
7647
8690
|
});
|
|
7648
8691
|
const slots = {
|
|
7649
8692
|
image: config.imageModel,
|
|
7650
8693
|
video: config.videoModel,
|
|
7651
8694
|
music: config.musicModel,
|
|
7652
8695
|
podcast: config.podcastModel,
|
|
8696
|
+
speech: config.speechModel,
|
|
7653
8697
|
transcription: config.transcriptionModel
|
|
7654
8698
|
};
|
|
7655
8699
|
const runtime = new MediaRuntime({
|
|
@@ -7696,7 +8740,8 @@ function apply(ctx, config) {
|
|
|
7696
8740
|
deviceId: () => storage.deviceId(),
|
|
7697
8741
|
preferredModels: () => config.preferredModels,
|
|
7698
8742
|
onCredentialChange: (listener) => login.onChange(listener),
|
|
7699
|
-
catalogRefreshMinutes: config.catalogRefreshMinutes
|
|
8743
|
+
catalogRefreshMinutes: config.catalogRefreshMinutes,
|
|
8744
|
+
onRequestEnded: spent
|
|
7700
8745
|
});
|
|
7701
8746
|
llmCtx.effect(() => () => {
|
|
7702
8747
|
route = void 0;
|
|
@@ -7713,7 +8758,7 @@ function apply(ctx, config) {
|
|
|
7713
8758
|
maxRetryWaitMs: 1e4
|
|
7714
8759
|
}));
|
|
7715
8760
|
});
|
|
7716
|
-
const
|
|
8761
|
+
const accountService = new AccountService({
|
|
7717
8762
|
origin,
|
|
7718
8763
|
userAgent,
|
|
7719
8764
|
chain,
|
|
@@ -7724,8 +8769,9 @@ function apply(ctx, config) {
|
|
|
7724
8769
|
...route?.hidden() === void 0 ? {} : { hidden: route.hidden() }
|
|
7725
8770
|
})
|
|
7726
8771
|
});
|
|
8772
|
+
account = accountService;
|
|
7727
8773
|
ctx.inject(["connection"], (scoped) => {
|
|
7728
|
-
for (const accountRoute of accountRoutes(
|
|
8774
|
+
for (const accountRoute of accountRoutes(accountService)) scoped.effect(() => scoped.connection.fetch.register(accountRoute), `dsh-vibedev: ${accountRoute.path}`);
|
|
7729
8775
|
});
|
|
7730
8776
|
ctx.effect(() => {
|
|
7731
8777
|
tracker.resume().catch((error) => {
|
|
@@ -7733,9 +8779,11 @@ function apply(ctx, config) {
|
|
|
7733
8779
|
});
|
|
7734
8780
|
const unsubscribe = login.onChange(() => {
|
|
7735
8781
|
runtime.invalidateCatalog();
|
|
8782
|
+
accountService.credentialChanged();
|
|
7736
8783
|
});
|
|
7737
8784
|
return () => {
|
|
7738
8785
|
unsubscribe();
|
|
8786
|
+
accountService.dispose();
|
|
7739
8787
|
tracker.dispose();
|
|
7740
8788
|
login.dispose();
|
|
7741
8789
|
};
|
|
@@ -7745,7 +8793,10 @@ function apply(ctx, config) {
|
|
|
7745
8793
|
const controller = new AbortController();
|
|
7746
8794
|
(async () => {
|
|
7747
8795
|
try {
|
|
7748
|
-
for await (const _view of accountCtx.deepseekAccount.watch(controller.signal))
|
|
8796
|
+
for await (const _view of accountCtx.deepseekAccount.watch(controller.signal)) {
|
|
8797
|
+
runtime.invalidateCatalog();
|
|
8798
|
+
accountService.credentialChanged();
|
|
8799
|
+
}
|
|
7749
8800
|
} catch {}
|
|
7750
8801
|
})();
|
|
7751
8802
|
return () => {
|