@vibedev-si/dsh-vibedev 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -10
- package/THIRD-PARTY-NOTICES.txt +1 -1
- package/client/client.js +3 -0
- package/lib/audio/request.d.ts +141 -0
- package/lib/gateway/assets.d.ts +2 -2
- package/lib/gateway/catalog.d.ts +75 -4
- package/lib/index.d.ts +7 -2
- package/lib/index.js +1015 -138
- package/lib/pricing.d.ts +45 -9
- package/lib/runtime.d.ts +24 -1
- package/lib/service.d.ts +81 -7
- package/lib/tasks/remote.d.ts +13 -0
- package/lib/tasks/store.d.ts +17 -0
- package/lib/tasks/tracker.d.ts +13 -0
- package/lib/tools/audio.d.ts +3 -2
- package/lib/tools/models.d.ts +7 -0
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -3514,6 +3514,13 @@ const VIDEO_INPUTS = [
|
|
|
3514
3514
|
"referenceVideos",
|
|
3515
3515
|
"referenceAudios"
|
|
3516
3516
|
];
|
|
3517
|
+
/** What an audio model does, as the catalog declares it (`audio.generation_kind`). */
|
|
3518
|
+
const AUDIO_GENERATION_KINDS = [
|
|
3519
|
+
"music",
|
|
3520
|
+
"podcast",
|
|
3521
|
+
"speech",
|
|
3522
|
+
"transcription"
|
|
3523
|
+
];
|
|
3517
3524
|
const record$3 = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
3518
3525
|
const text$4 = (value) => typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
3519
3526
|
const strings = (value) => Array.isArray(value) && value.every((item) => typeof item === "string") ? value : void 0;
|
|
@@ -3747,10 +3754,144 @@ function effectiveVideoModes(video) {
|
|
|
3747
3754
|
}
|
|
3748
3755
|
return modes;
|
|
3749
3756
|
}
|
|
3750
|
-
|
|
3757
|
+
const PRICE_UNITS = [
|
|
3758
|
+
"generation",
|
|
3759
|
+
"second",
|
|
3760
|
+
"original_second",
|
|
3761
|
+
"minute"
|
|
3762
|
+
];
|
|
3763
|
+
/** The audio block: `entry.audio`, else `capabilities.audio`, else the fields flattened into `capabilities`. */
|
|
3764
|
+
function audioBlock(entry, capabilities) {
|
|
3765
|
+
const own = record$3(entry.audio) ?? record$3(capabilities.audio);
|
|
3766
|
+
if (own !== void 0) return own;
|
|
3767
|
+
return field(capabilities, "generation_kind") !== void 0 ? capabilities : void 0;
|
|
3768
|
+
}
|
|
3769
|
+
function stringChoices(raw) {
|
|
3770
|
+
const choices = record$3(raw);
|
|
3771
|
+
if (choices === void 0) return void 0;
|
|
3772
|
+
const out = {};
|
|
3773
|
+
for (const [name, values] of Object.entries(choices)) {
|
|
3774
|
+
const items = strings(values)?.map((item) => item.trim()).filter((item) => item !== "");
|
|
3775
|
+
if (items !== void 0 && items.length > 0) out[name] = items;
|
|
3776
|
+
}
|
|
3777
|
+
return Object.keys(out).length === 0 ? void 0 : out;
|
|
3778
|
+
}
|
|
3779
|
+
function parseReferenceLimits(raw) {
|
|
3780
|
+
if (raw === void 0) return void 0;
|
|
3781
|
+
const max = count(field(raw, "max_count")) ?? count(raw.max);
|
|
3782
|
+
if (max === void 0 || max === 0) return void 0;
|
|
3783
|
+
const min = count(field(raw, "min_count")) ?? count(raw.min) ?? 0;
|
|
3784
|
+
const limits = {
|
|
3785
|
+
min: Math.min(min, max),
|
|
3786
|
+
max
|
|
3787
|
+
};
|
|
3788
|
+
for (const [key, ...names] of [
|
|
3789
|
+
[
|
|
3790
|
+
"maxBytes",
|
|
3791
|
+
"max_bytes",
|
|
3792
|
+
"max_size_bytes"
|
|
3793
|
+
],
|
|
3794
|
+
[
|
|
3795
|
+
"minSeconds",
|
|
3796
|
+
"min_duration_seconds",
|
|
3797
|
+
"min_seconds"
|
|
3798
|
+
],
|
|
3799
|
+
[
|
|
3800
|
+
"maxSeconds",
|
|
3801
|
+
"max_duration_seconds",
|
|
3802
|
+
"max_seconds"
|
|
3803
|
+
],
|
|
3804
|
+
["maxTotalSeconds", "max_total_duration_seconds"]
|
|
3805
|
+
]) {
|
|
3806
|
+
const value = names.map((name) => positive(field(raw, name))).find((found) => found !== void 0);
|
|
3807
|
+
if (value !== void 0) limits[key] = value;
|
|
3808
|
+
}
|
|
3809
|
+
const mimes = list(raw, "allowed_mimes") ?? list(raw, "mimes");
|
|
3810
|
+
if (mimes !== void 0 && mimes.length > 0) limits.mimes = mimes;
|
|
3811
|
+
const exclusive = list(raw, "mutually_exclusive_with") ?? list(raw, "exclusive_with");
|
|
3812
|
+
if (exclusive !== void 0 && exclusive.length > 0) limits.exclusiveWith = exclusive;
|
|
3813
|
+
const purpose = text$4(field(raw, "asset_purpose")) ?? text$4(raw.purpose);
|
|
3814
|
+
if (purpose !== void 0) limits.purpose = purpose;
|
|
3815
|
+
return limits;
|
|
3816
|
+
}
|
|
3817
|
+
/**
|
|
3818
|
+
* The output formats, in either declared shape: `output_formats: [{format, sample_rates}]`,
|
|
3819
|
+
* or `output_formats: ['wav','mp3']` with `sample_rates: {wav: [...], mp3: [...]}`.
|
|
3820
|
+
*/
|
|
3821
|
+
function parseOutputFormats(raw) {
|
|
3822
|
+
const declared = field(raw, "output_formats");
|
|
3823
|
+
if (!Array.isArray(declared)) return void 0;
|
|
3824
|
+
const rates = record$3(field(raw, "sample_rates"));
|
|
3825
|
+
const formats = [];
|
|
3826
|
+
for (const item of declared) {
|
|
3827
|
+
const entry = record$3(item);
|
|
3828
|
+
const format = (entry === void 0 ? text$4(item) : text$4(entry.format))?.toLowerCase();
|
|
3829
|
+
if (format === void 0 || formats.some((known) => known.format === format)) continue;
|
|
3830
|
+
const sampleRates = (entry === void 0 ? void 0 : field(entry, "sample_rates")) ?? rates?.[format];
|
|
3831
|
+
const valid = Array.isArray(sampleRates) ? sampleRates.filter((rate) => positive(rate) !== void 0 && Number.isInteger(rate)) : [];
|
|
3832
|
+
const fallback = entry === void 0 ? void 0 : positive(field(entry, "default_sample_rate"));
|
|
3833
|
+
formats.push({
|
|
3834
|
+
format,
|
|
3835
|
+
sampleRates: valid,
|
|
3836
|
+
...fallback !== void 0 && valid.includes(fallback) ? { defaultSampleRate: fallback } : {}
|
|
3837
|
+
});
|
|
3838
|
+
}
|
|
3839
|
+
return formats.length === 0 ? void 0 : formats;
|
|
3840
|
+
}
|
|
3841
|
+
/**
|
|
3842
|
+
* Parse an audio capability block.
|
|
3843
|
+
* @param raw - the block.
|
|
3844
|
+
* @returns the declared capabilities; `generationKind` is absent unless declared.
|
|
3845
|
+
*/
|
|
3846
|
+
function parseAudioCapabilities(raw) {
|
|
3847
|
+
const kindRaw = text$4(field(raw, "generation_kind"))?.toLowerCase();
|
|
3848
|
+
const generationKind = AUDIO_GENERATION_KINDS.includes(kindRaw ?? "") ? kindRaw : void 0;
|
|
3849
|
+
const billingRaw = text$4(field(raw, "billing_unit"))?.toLowerCase();
|
|
3850
|
+
const billingUnit = billingRaw === "generation" || billingRaw === "original_second" || billingRaw === "minute" ? billingRaw : void 0;
|
|
3851
|
+
const promptField = text$4(field(raw, "prompt_field"));
|
|
3852
|
+
const requestFields = list(raw, "request_fields") ?? [];
|
|
3853
|
+
const references = parseReferenceLimits(record$3(field(raw, "references")) ?? record$3(field(raw, "reference_audio")));
|
|
3854
|
+
const outputFormats = parseOutputFormats(raw);
|
|
3855
|
+
const defaultFormat = text$4(field(raw, "default_format"))?.toLowerCase();
|
|
3856
|
+
const choices = stringChoices(field(raw, "choices") ?? field(raw, "field_options"));
|
|
3857
|
+
const result = { requestFields };
|
|
3858
|
+
if (generationKind !== void 0) result.generationKind = generationKind;
|
|
3859
|
+
if (promptField !== void 0) result.promptField = promptField;
|
|
3860
|
+
if (billingUnit !== void 0) result.billingUnit = billingUnit;
|
|
3861
|
+
if (references !== void 0) result.references = references;
|
|
3862
|
+
if (outputFormats !== void 0) result.outputFormats = outputFormats;
|
|
3863
|
+
if (defaultFormat !== void 0 && outputFormats?.some((format) => format.format === defaultFormat) === true) result.defaultFormat = defaultFormat;
|
|
3864
|
+
if (choices !== void 0) result.choices = choices;
|
|
3865
|
+
for (const [key, ...names] of [
|
|
3866
|
+
[
|
|
3867
|
+
"maxPromptChars",
|
|
3868
|
+
"max_prompt_chars",
|
|
3869
|
+
"max_text_chars"
|
|
3870
|
+
],
|
|
3871
|
+
[
|
|
3872
|
+
"maxInputBytes",
|
|
3873
|
+
"max_input_bytes",
|
|
3874
|
+
"max_file_bytes"
|
|
3875
|
+
],
|
|
3876
|
+
[
|
|
3877
|
+
"maxInputSeconds",
|
|
3878
|
+
"max_input_seconds",
|
|
3879
|
+
"max_duration_seconds"
|
|
3880
|
+
]
|
|
3881
|
+
]) {
|
|
3882
|
+
const value = names.map((name) => positive(field(raw, name))).find((found) => found !== void 0);
|
|
3883
|
+
if (value !== void 0) result[key] = value;
|
|
3884
|
+
}
|
|
3885
|
+
const inputMimes = list(raw, "input_mimes") ?? list(raw, "allowed_input_mimes");
|
|
3886
|
+
if (inputMimes !== void 0 && inputMimes.length > 0) result.inputMimes = inputMimes;
|
|
3887
|
+
const languages = list(raw, "languages");
|
|
3888
|
+
if (languages !== void 0 && languages.length > 0) result.languages = languages;
|
|
3889
|
+
return result;
|
|
3890
|
+
}
|
|
3891
|
+
function parsePricing(value, audioUnit) {
|
|
3751
3892
|
const raw = record$3(value);
|
|
3752
3893
|
if (raw === void 0) return void 0;
|
|
3753
|
-
const unit = (item) => item
|
|
3894
|
+
const unit = (item) => PRICE_UNITS.includes(item) ? item : item === void 0 && audioUnit !== void 0 ? audioUnit : void 0;
|
|
3754
3895
|
const tiers = [];
|
|
3755
3896
|
if (Array.isArray(raw.tiers)) for (const entry of raw.tiers) {
|
|
3756
3897
|
const tier = record$3(entry);
|
|
@@ -3777,8 +3918,10 @@ function parsePricing(value) {
|
|
|
3777
3918
|
};
|
|
3778
3919
|
}
|
|
3779
3920
|
/** Which kind of media model a catalog entry is, or undefined for a chat or internal model. */
|
|
3780
|
-
function kindOf(entry, capabilities) {
|
|
3921
|
+
function kindOf(entry, capabilities, audio) {
|
|
3781
3922
|
if (text$4(entry.internal_role) !== void 0) return void 0;
|
|
3923
|
+
if (audio?.generationKind === "transcription") return "transcription";
|
|
3924
|
+
if (audio?.generationKind !== void 0) return "audio";
|
|
3782
3925
|
const mediaType = text$4(entry.media_type);
|
|
3783
3926
|
if (mediaType === "video" || mediaType === "image" || mediaType === "audio" || mediaType === "transcription") return mediaType;
|
|
3784
3927
|
if (capabilities.transcription === true || capabilities.realtime_transcription === true) return "transcription";
|
|
@@ -3807,29 +3950,35 @@ function parseMediaCatalog(body) {
|
|
|
3807
3950
|
const id = text$4(entry?.id);
|
|
3808
3951
|
if (entry === void 0 || id === void 0 || seen.has(id)) continue;
|
|
3809
3952
|
const capabilities = record$3(entry.capabilities) ?? {};
|
|
3810
|
-
const
|
|
3953
|
+
const audioRaw = audioBlock(entry, capabilities);
|
|
3954
|
+
const audio = audioRaw === void 0 ? void 0 : parseAudioCapabilities(audioRaw);
|
|
3955
|
+
const kind = kindOf(entry, capabilities, audio);
|
|
3811
3956
|
if (kind === void 0) continue;
|
|
3812
3957
|
seen.add(id);
|
|
3813
3958
|
const name = text$4(entry.display_name) ?? id;
|
|
3814
3959
|
const description = text$4(entry.description);
|
|
3815
|
-
const pricing = parsePricing(entry.media_pricing);
|
|
3960
|
+
const pricing = parsePricing(entry.media_pricing, audio?.billingUnit);
|
|
3816
3961
|
const inputModalities = strings(capabilities.input_modalities) ?? strings(entry.input_modalities) ?? [];
|
|
3817
3962
|
const endpoints = strings(capabilities.allowed_endpoints) ?? [];
|
|
3963
|
+
const aliases = (strings(entry.aliases) ?? []).map((alias) => alias.trim()).filter((alias) => alias !== "" && alias !== id);
|
|
3818
3964
|
let video;
|
|
3819
3965
|
if (kind === "video") video = {
|
|
3820
3966
|
...parseVideoCapabilities(capabilities),
|
|
3821
3967
|
...parseVideoCapabilities(record$3(entry.video) ?? {})
|
|
3822
3968
|
};
|
|
3969
|
+
const audioKind = audio?.generationKind === void 0 || audio.generationKind === "transcription" ? void 0 : audio.generationKind;
|
|
3823
3970
|
models.push({
|
|
3824
3971
|
id,
|
|
3825
3972
|
name,
|
|
3826
3973
|
kind,
|
|
3827
3974
|
endpoints,
|
|
3828
3975
|
inputModalities,
|
|
3976
|
+
...aliases.length === 0 ? {} : { aliases },
|
|
3829
3977
|
...description === void 0 ? {} : { description },
|
|
3830
3978
|
...pricing === void 0 ? {} : { pricing },
|
|
3831
3979
|
...video === void 0 ? {} : { video },
|
|
3832
|
-
...
|
|
3980
|
+
...audio === void 0 ? {} : { audio },
|
|
3981
|
+
...audioKind === void 0 ? {} : { audioKind }
|
|
3833
3982
|
});
|
|
3834
3983
|
}
|
|
3835
3984
|
return models;
|
|
@@ -4121,6 +4270,27 @@ async function loadChatReference(reference, want, messages, store, maxBytes, sig
|
|
|
4121
4270
|
* spending confirmation.
|
|
4122
4271
|
* @module dsh-vibedev/runtime
|
|
4123
4272
|
*/
|
|
4273
|
+
/**
|
|
4274
|
+
* Whether a model serves a slot. Audio models serve the kind their catalog
|
|
4275
|
+
* entry declares, and only that: an undeclared kind serves none.
|
|
4276
|
+
* @param model - the model.
|
|
4277
|
+
* @param slot - the slot.
|
|
4278
|
+
* @returns whether the model can be chosen for it.
|
|
4279
|
+
*/
|
|
4280
|
+
function servesSlot(model, slot) {
|
|
4281
|
+
if (slot === "music" || slot === "podcast" || slot === "speech") return model.kind === "audio" && model.audioKind === slot;
|
|
4282
|
+
return model.kind === slot;
|
|
4283
|
+
}
|
|
4284
|
+
/**
|
|
4285
|
+
* The model a name refers to: its id, else one of its aliases (case-insensitive).
|
|
4286
|
+
* @param models - the catalog.
|
|
4287
|
+
* @param wanted - the id or alias.
|
|
4288
|
+
* @returns the model, or undefined.
|
|
4289
|
+
*/
|
|
4290
|
+
function modelNamed(models, wanted) {
|
|
4291
|
+
const lower = wanted.toLowerCase();
|
|
4292
|
+
return models.find((model) => model.id === wanted) ?? models.find((model) => model.id.toLowerCase() === lower) ?? models.find((model) => (model.aliases ?? []).some((alias) => alias.toLowerCase() === lower));
|
|
4293
|
+
}
|
|
4124
4294
|
const CATALOG_TTL_MS = 12e4;
|
|
4125
4295
|
/** Wait for a shared promise under one caller's signal without cancelling it for the others. */
|
|
4126
4296
|
function abortable$1(promise, signal) {
|
|
@@ -4224,20 +4394,36 @@ var MediaRuntime = class {
|
|
|
4224
4394
|
*/
|
|
4225
4395
|
async pickModel(kind, slot, requested, accept, signal) {
|
|
4226
4396
|
const models = await this.models(signal);
|
|
4227
|
-
const ofKind = models.filter((model) => model.kind === kind && (
|
|
4397
|
+
const ofKind = models.filter((model) => model.kind === kind && servesSlot(model, slot));
|
|
4228
4398
|
const wanted = requested?.trim();
|
|
4229
4399
|
if (wanted !== void 0 && wanted !== "") {
|
|
4230
|
-
const found =
|
|
4400
|
+
const found = modelNamed(models, wanted);
|
|
4231
4401
|
if (found === void 0) throw new MediaError(`There is no model "${wanted}" on this VibeDev account. ${slotLabel(slot)} models: ${ofKind.map((model) => model.id).join(", ") || "none"}.`, "MODEL_NOT_FOUND", { field: "model" });
|
|
4232
|
-
if (!ofKind.includes(found))
|
|
4402
|
+
if (!ofKind.includes(found)) {
|
|
4403
|
+
if (found.kind === "audio" && found.audioKind === void 0) throw new MediaError(`${found.id} does not declare what kind of audio it makes, so it cannot be used for ${slotLabel(slot).toLowerCase()}.`, "AUDIO_KIND_UNDECLARED", { field: "model" });
|
|
4404
|
+
throw new MediaError(`${found.id} is not a ${slotLabel(slot).toLowerCase()} model. ${slotLabel(slot)} models: ${ofKind.map((model) => model.id).join(", ") || "none"}.`, "MODEL_KIND_MISMATCH", { field: "model" });
|
|
4405
|
+
}
|
|
4233
4406
|
return found;
|
|
4234
4407
|
}
|
|
4235
4408
|
if (ofKind.length === 0) throw new MediaError(`This VibeDev account has no ${slotLabel(slot).toLowerCase()} model available right now.`, "NO_MODEL_AVAILABLE");
|
|
4236
|
-
const
|
|
4237
|
-
const
|
|
4409
|
+
const pinnedName = this.settings.defaultModel(slot).trim();
|
|
4410
|
+
const pinned = pinnedName === "" ? void 0 : modelNamed(ofKind, pinnedName);
|
|
4411
|
+
const ordered = pinned === void 0 ? ofKind : [pinned, ...ofKind.filter((model) => model !== pinned)];
|
|
4238
4412
|
return (accept === void 0 ? void 0 : ordered.find((model) => accept(model))) ?? ordered[0];
|
|
4239
4413
|
}
|
|
4240
4414
|
/**
|
|
4415
|
+
* The model a name refers to, whatever its kind.
|
|
4416
|
+
* @param requested - an id or alias.
|
|
4417
|
+
* @param signal - cancels the catalog request.
|
|
4418
|
+
* @returns the model.
|
|
4419
|
+
* @throws {@link MediaError} when the account has no such model.
|
|
4420
|
+
*/
|
|
4421
|
+
async findModel(requested, signal) {
|
|
4422
|
+
const found = modelNamed(await this.models(signal), requested.trim());
|
|
4423
|
+
if (found === void 0) throw new MediaError(`There is no model "${requested}" on this VibeDev account.`, "MODEL_NOT_FOUND", { field: "model" });
|
|
4424
|
+
return found;
|
|
4425
|
+
}
|
|
4426
|
+
/**
|
|
4241
4427
|
* The workspace of the calling session.
|
|
4242
4428
|
* @param call - the tool call or host call.
|
|
4243
4429
|
* @returns its absolute working directory, when it has one.
|
|
@@ -4374,6 +4560,7 @@ function slotLabel(slot) {
|
|
|
4374
4560
|
video: "Video",
|
|
4375
4561
|
music: "Music",
|
|
4376
4562
|
podcast: "Podcast",
|
|
4563
|
+
speech: "Speech",
|
|
4377
4564
|
transcription: "Transcription"
|
|
4378
4565
|
}[slot];
|
|
4379
4566
|
}
|
|
@@ -4617,14 +4804,23 @@ var GatewaySearchProvider = class {
|
|
|
4617
4804
|
};
|
|
4618
4805
|
//#endregion
|
|
4619
4806
|
//#region src/pricing.ts
|
|
4807
|
+
/** Shown wherever the catalog gives no price. */
|
|
4808
|
+
const UNPRICED = {
|
|
4809
|
+
en: "not priced",
|
|
4810
|
+
zh: "未定价"
|
|
4811
|
+
};
|
|
4620
4812
|
/** Seconds the gateway bills when a request names no duration. */
|
|
4621
4813
|
const DEFAULT_BILLED_SECONDS = 8;
|
|
4622
4814
|
/**
|
|
4623
|
-
* Format an amount in yuan.
|
|
4815
|
+
* Format an amount in yuan: two decimals, more when a small unit price needs them (`¥0.015`).
|
|
4624
4816
|
* @param amount - the amount.
|
|
4625
4817
|
* @returns `¥4.95`.
|
|
4626
4818
|
*/
|
|
4627
4819
|
function yuan(amount) {
|
|
4820
|
+
if (amount !== 0 && Math.abs(amount) < .1) {
|
|
4821
|
+
const precise = amount.toFixed(4).replace(/0+$/, "");
|
|
4822
|
+
if (precise.split(".")[1].length > 2) return `¥${precise}`;
|
|
4823
|
+
}
|
|
4628
4824
|
return `¥${amount.toFixed(2)}`;
|
|
4629
4825
|
}
|
|
4630
4826
|
function tierFor(pricing, wanted) {
|
|
@@ -4636,17 +4832,47 @@ function tierFor(pricing, wanted) {
|
|
|
4636
4832
|
return pricing.tiers.length === 1 ? pricing.tiers[0] : void 0;
|
|
4637
4833
|
}
|
|
4638
4834
|
/**
|
|
4639
|
-
*
|
|
4835
|
+
* The unit after a price: `/s`, `/原始秒`, `/次`.
|
|
4836
|
+
* @param unit - the price unit.
|
|
4837
|
+
* @param kind - the kind of model, which names a generation (an image, a video, a request).
|
|
4838
|
+
* @returns the suffix in both languages.
|
|
4839
|
+
*/
|
|
4840
|
+
function unitSuffix(unit, kind) {
|
|
4841
|
+
switch (unit) {
|
|
4842
|
+
case "second": return {
|
|
4843
|
+
en: "/s",
|
|
4844
|
+
zh: "/秒"
|
|
4845
|
+
};
|
|
4846
|
+
case "original_second": return {
|
|
4847
|
+
en: "/s of original audio",
|
|
4848
|
+
zh: "/原始秒"
|
|
4849
|
+
};
|
|
4850
|
+
case "minute": return {
|
|
4851
|
+
en: "/min",
|
|
4852
|
+
zh: "/分钟"
|
|
4853
|
+
};
|
|
4854
|
+
default: return kind === "image" ? {
|
|
4855
|
+
en: " per image",
|
|
4856
|
+
zh: "/张"
|
|
4857
|
+
} : {
|
|
4858
|
+
en: " per generation",
|
|
4859
|
+
zh: "/次"
|
|
4860
|
+
};
|
|
4861
|
+
}
|
|
4862
|
+
}
|
|
4863
|
+
/**
|
|
4864
|
+
* A one-line price for listings: `¥0.10 per image`, `¥0.99/s (720p)`; `not priced` without a price.
|
|
4640
4865
|
* @param model - the model.
|
|
4641
|
-
* @
|
|
4866
|
+
* @param lang - the language.
|
|
4867
|
+
* @returns the price line.
|
|
4642
4868
|
*/
|
|
4643
|
-
function describePrice(model) {
|
|
4869
|
+
function describePrice(model, lang = "en") {
|
|
4644
4870
|
const pricing = model.pricing;
|
|
4645
|
-
if (pricing === void 0) return
|
|
4646
|
-
const per = (unit) => unit
|
|
4871
|
+
if (pricing === void 0) return UNPRICED[lang];
|
|
4872
|
+
const per = (unit) => unitSuffix(unit, model.kind)[lang];
|
|
4647
4873
|
const parts = pricing.tiers.map((tier) => `${yuan(tier.amount)}${per(tier.unit)} (${tier.tier})`);
|
|
4648
|
-
if (pricing.default !== void 0 && (pricing.tiers.length === 0 || !pricing.tiers.some((tier) => tier.amount === pricing.default?.amount && tier.unit === pricing.default.unit))) parts.push(`${yuan(pricing.default.amount)}${per(pricing.default.unit)}${pricing.tiers.length === 0 ? "" : " otherwise"}`);
|
|
4649
|
-
return parts.length === 0 ?
|
|
4874
|
+
if (pricing.default !== void 0 && (pricing.tiers.length === 0 || !pricing.tiers.some((tier) => tier.amount === pricing.default?.amount && tier.unit === pricing.default.unit))) parts.push(`${yuan(pricing.default.amount)}${per(pricing.default.unit)}${pricing.tiers.length === 0 ? "" : lang === "zh" ? "(其他)" : " otherwise"}`);
|
|
4875
|
+
return parts.length === 0 ? UNPRICED[lang] : parts.join(", ");
|
|
4650
4876
|
}
|
|
4651
4877
|
/**
|
|
4652
4878
|
* Estimate an image request.
|
|
@@ -4656,7 +4882,7 @@ function describePrice(model) {
|
|
|
4656
4882
|
*/
|
|
4657
4883
|
function estimateImages(model, count) {
|
|
4658
4884
|
const price = model.pricing === void 0 ? void 0 : tierFor(model.pricing, void 0);
|
|
4659
|
-
if (price === void 0) return void 0;
|
|
4885
|
+
if (price === void 0 || price.unit !== "generation") return void 0;
|
|
4660
4886
|
return {
|
|
4661
4887
|
amountCny: price.amount * count,
|
|
4662
4888
|
basis: count === 1 ? `${yuan(price.amount)} per image` : `${yuan(price.amount)} × ${count} images`
|
|
@@ -4679,6 +4905,7 @@ function estimateVideo(model, duration, resolution, referenceMs = 0) {
|
|
|
4679
4905
|
amountCny: price.amount,
|
|
4680
4906
|
basis: `${yuan(price.amount)} per video`
|
|
4681
4907
|
};
|
|
4908
|
+
if (price.unit !== "second") return void 0;
|
|
4682
4909
|
const output = duration ?? DEFAULT_BILLED_SECONDS;
|
|
4683
4910
|
const reference = Math.ceil(referenceMs / 1e3);
|
|
4684
4911
|
const seconds = output + reference;
|
|
@@ -4688,16 +4915,429 @@ function estimateVideo(model, duration, resolution, referenceMs = 0) {
|
|
|
4688
4915
|
};
|
|
4689
4916
|
}
|
|
4690
4917
|
/**
|
|
4691
|
-
*
|
|
4692
|
-
* @param model - the audio model.
|
|
4693
|
-
* @
|
|
4918
|
+
* Quote an audio request: music, podcast and speech generation, or a transcription.
|
|
4919
|
+
* @param model - the audio or transcription model.
|
|
4920
|
+
* @param inputSeconds - for a transcription, the measured length of the input, when known.
|
|
4921
|
+
* @returns the quote; `priced: false` (未定价) when the catalog gives no usable price.
|
|
4694
4922
|
*/
|
|
4695
|
-
function
|
|
4923
|
+
function quoteAudio(model, inputSeconds) {
|
|
4696
4924
|
const price = model.pricing === void 0 ? void 0 : tierFor(model.pricing, void 0);
|
|
4697
|
-
|
|
4698
|
-
return {
|
|
4925
|
+
const unit = price?.unit ?? model.audio?.billingUnit;
|
|
4926
|
+
if (price === void 0 || unit === void 0 || unit === "second") return {
|
|
4927
|
+
priced: false,
|
|
4928
|
+
...unit === void 0 ? {} : { unit },
|
|
4929
|
+
label: UNPRICED,
|
|
4930
|
+
basis: {
|
|
4931
|
+
en: "The catalog states no price for this model; the gateway receipt shows the charge.",
|
|
4932
|
+
zh: "目录未给出价格,以网关结算为准。"
|
|
4933
|
+
}
|
|
4934
|
+
};
|
|
4935
|
+
const suffix = unitSuffix(unit, model.kind);
|
|
4936
|
+
const label = {
|
|
4937
|
+
en: `${yuan(price.amount)}${suffix.en}`,
|
|
4938
|
+
zh: `${yuan(price.amount)}${suffix.zh}`
|
|
4939
|
+
};
|
|
4940
|
+
const base = {
|
|
4941
|
+
priced: true,
|
|
4942
|
+
unit,
|
|
4943
|
+
unitPriceCny: price.amount,
|
|
4944
|
+
label
|
|
4945
|
+
};
|
|
4946
|
+
if (unit === "generation") return {
|
|
4947
|
+
...base,
|
|
4699
4948
|
amountCny: price.amount,
|
|
4700
|
-
basis:
|
|
4949
|
+
basis: {
|
|
4950
|
+
en: `${label.en}, charged only if it succeeds`,
|
|
4951
|
+
zh: `${label.zh},成功才扣费`
|
|
4952
|
+
}
|
|
4953
|
+
};
|
|
4954
|
+
if (unit === "original_second") return {
|
|
4955
|
+
...base,
|
|
4956
|
+
basis: {
|
|
4957
|
+
en: `${label.en}: billed on the original length the provider reports, which can differ from the playback length`,
|
|
4958
|
+
zh: `${label.zh}:按服务商返回的原始时长计费,可能与播放时长不同`
|
|
4959
|
+
}
|
|
4960
|
+
};
|
|
4961
|
+
if (inputSeconds !== void 0 && inputSeconds > 0) {
|
|
4962
|
+
const minutes = inputSeconds / 60;
|
|
4963
|
+
return {
|
|
4964
|
+
...base,
|
|
4965
|
+
amountCny: price.amount * minutes,
|
|
4966
|
+
basis: {
|
|
4967
|
+
en: `${label.en} × ${minutes.toFixed(2)} min`,
|
|
4968
|
+
zh: `${label.zh} × ${minutes.toFixed(2)} 分钟`
|
|
4969
|
+
}
|
|
4970
|
+
};
|
|
4971
|
+
}
|
|
4972
|
+
return {
|
|
4973
|
+
...base,
|
|
4974
|
+
basis: {
|
|
4975
|
+
en: `${label.en} of input audio`,
|
|
4976
|
+
zh: `${label.zh},按音频时长计费`
|
|
4977
|
+
}
|
|
4978
|
+
};
|
|
4979
|
+
}
|
|
4980
|
+
//#endregion
|
|
4981
|
+
//#region src/audio/request.ts
|
|
4982
|
+
/**
|
|
4983
|
+
* Audio generation requests — music, podcasts and speech — checked against what
|
|
4984
|
+
* the chosen model declares in the catalog's `audio` block, and turned into the
|
|
4985
|
+
* body of `POST /v1/audio/generations`.
|
|
4986
|
+
*
|
|
4987
|
+
* Each kind sends only its own fields:
|
|
4988
|
+
*
|
|
4989
|
+
* | kind | wire fields |
|
|
4990
|
+
* | ------- | --------------------------------------------------------------- |
|
|
4991
|
+
* | music | model, prompt, lyrics, genre, mood, gender |
|
|
4992
|
+
* | podcast | model, prompt, source_url |
|
|
4993
|
+
* | speech | as declared; Seed: model, text_prompt, references, audio_config, watermark |
|
|
4994
|
+
*
|
|
4995
|
+
* The main text goes in the model's `prompt_field`. A field the model does not
|
|
4996
|
+
* declare in `request_fields` is refused before anything is uploaded or charged;
|
|
4997
|
+
* music and podcasts never take speech references, output formats, sample rates
|
|
4998
|
+
* or a duration. Speech references, formats and sample rates come from the
|
|
4999
|
+
* catalog only (count, size, length, mutual exclusions, the sample rates of
|
|
5000
|
+
* each format), never from a fixed list.
|
|
5001
|
+
* @module dsh-vibedev/audio/request
|
|
5002
|
+
*/
|
|
5003
|
+
const AUDIO_MAKE_KINDS = [
|
|
5004
|
+
"music",
|
|
5005
|
+
"podcast",
|
|
5006
|
+
"speech"
|
|
5007
|
+
];
|
|
5008
|
+
/** The standard wire fields of a kind, for a model that declares none (speech declares its own). */
|
|
5009
|
+
const STANDARD_FIELDS = {
|
|
5010
|
+
music: [
|
|
5011
|
+
"model",
|
|
5012
|
+
"prompt",
|
|
5013
|
+
"lyrics",
|
|
5014
|
+
"genre",
|
|
5015
|
+
"mood",
|
|
5016
|
+
"gender"
|
|
5017
|
+
],
|
|
5018
|
+
podcast: [
|
|
5019
|
+
"model",
|
|
5020
|
+
"prompt",
|
|
5021
|
+
"source_url"
|
|
5022
|
+
],
|
|
5023
|
+
speech: []
|
|
5024
|
+
};
|
|
5025
|
+
/** Fields that belong to speech only: references, the output format and sample rate, a duration. */
|
|
5026
|
+
const SPEECH_ONLY = [
|
|
5027
|
+
"references",
|
|
5028
|
+
"format",
|
|
5029
|
+
"sampleRate",
|
|
5030
|
+
"duration"
|
|
5031
|
+
];
|
|
5032
|
+
/** Default limit on one reference file when the catalog states none. */
|
|
5033
|
+
const DEFAULT_REFERENCE_BYTES = 20971520;
|
|
5034
|
+
const OPERATION_ID = /^[\w-]{1,80}$/;
|
|
5035
|
+
const has = (value) => value !== void 0 && value.trim() !== "";
|
|
5036
|
+
/**
|
|
5037
|
+
* What an audio model makes, from its declaration only.
|
|
5038
|
+
* @param model - the model.
|
|
5039
|
+
* @returns the kind, or undefined when the catalog does not declare it.
|
|
5040
|
+
*/
|
|
5041
|
+
function audioKindOf(model) {
|
|
5042
|
+
return model.kind === "audio" ? model.audioKind : void 0;
|
|
5043
|
+
}
|
|
5044
|
+
/** The wire fields a model takes. */
|
|
5045
|
+
function requestFieldsOf(model) {
|
|
5046
|
+
const kind = audioKindOf(model);
|
|
5047
|
+
const declared = model.audio?.requestFields ?? [];
|
|
5048
|
+
if (declared.length > 0) return declared;
|
|
5049
|
+
return kind === void 0 ? [] : STANDARD_FIELDS[kind];
|
|
5050
|
+
}
|
|
5051
|
+
/** The field the main text goes in. */
|
|
5052
|
+
function promptFieldOf(model) {
|
|
5053
|
+
const declared = model.audio?.promptField;
|
|
5054
|
+
if (declared !== void 0) return declared;
|
|
5055
|
+
const fields = requestFieldsOf(model);
|
|
5056
|
+
return [
|
|
5057
|
+
"prompt",
|
|
5058
|
+
"text_prompt",
|
|
5059
|
+
"text",
|
|
5060
|
+
"input"
|
|
5061
|
+
].find((name) => fields.includes(name));
|
|
5062
|
+
}
|
|
5063
|
+
function refuse(message, field) {
|
|
5064
|
+
throw new MediaError(message, "AUDIO_OPTION_INVALID", field === void 0 ? {} : { field });
|
|
5065
|
+
}
|
|
5066
|
+
/** Check one value against the model's declared choices for a field. */
|
|
5067
|
+
function checkChoice(audio, wire, value) {
|
|
5068
|
+
const choices = audio?.choices?.[wire];
|
|
5069
|
+
if (choices !== void 0 && !choices.some((choice) => choice.toLowerCase() === value.toLowerCase())) refuse(`${wire} "${value}" is not offered by this model; it takes ${choices.join(", ")}.`, wire);
|
|
5070
|
+
}
|
|
5071
|
+
/** The output format and sample rate a speech request uses, from the declared formats only. */
|
|
5072
|
+
function audioConfigOf(model, format, sampleRate) {
|
|
5073
|
+
const formats = model.audio?.outputFormats ?? [];
|
|
5074
|
+
if (formats.length === 0) {
|
|
5075
|
+
if (format !== void 0 || sampleRate !== void 0) refuse(`${model.id} declares no output formats, so format and sample_rate cannot be chosen.`, "format");
|
|
5076
|
+
return;
|
|
5077
|
+
}
|
|
5078
|
+
const wanted = format?.trim().toLowerCase();
|
|
5079
|
+
const chosen = wanted === void 0 || wanted === "" ? formats.find((item) => item.format === model.audio?.defaultFormat) ?? formats[0] : formats.find((item) => item.format === wanted);
|
|
5080
|
+
if (chosen === void 0) refuse(`${model.id} writes ${formats.map((item) => item.format).join(", ")}, not "${format}".`, "format");
|
|
5081
|
+
if (sampleRate !== void 0) {
|
|
5082
|
+
if (chosen.sampleRates.length > 0 && !chosen.sampleRates.includes(sampleRate)) refuse(`${chosen.format.toUpperCase()} from ${model.id} takes a sample rate of ${chosen.sampleRates.join(", ")} Hz, not ${sampleRate}.`, "sample_rate");
|
|
5083
|
+
return {
|
|
5084
|
+
format: chosen.format,
|
|
5085
|
+
sample_rate: sampleRate
|
|
5086
|
+
};
|
|
5087
|
+
}
|
|
5088
|
+
const fallback = chosen.defaultSampleRate;
|
|
5089
|
+
return {
|
|
5090
|
+
format: chosen.format,
|
|
5091
|
+
...fallback === void 0 ? {} : { sample_rate: fallback }
|
|
5092
|
+
};
|
|
5093
|
+
}
|
|
5094
|
+
/**
|
|
5095
|
+
* Check a request against the model and build the body, without references
|
|
5096
|
+
* (they are uploaded first; their URLs are added by {@link prepareAudioTask}).
|
|
5097
|
+
* @param model - the chosen model.
|
|
5098
|
+
* @param request - what to generate.
|
|
5099
|
+
* @returns the body without references, and the wire fields the request uses.
|
|
5100
|
+
* @throws {@link MediaError} for a field the model does not take, a missing text, or a value outside the declared choices.
|
|
5101
|
+
*/
|
|
5102
|
+
function audioRequestBody(model, request) {
|
|
5103
|
+
const kind = audioKindOf(model);
|
|
5104
|
+
if (kind === void 0) throw new MediaError(`${model.id} does not declare what kind of audio it makes, so it cannot be used for generation.`, "AUDIO_KIND_UNDECLARED", { field: "model" });
|
|
5105
|
+
const audio = model.audio;
|
|
5106
|
+
const fields = requestFieldsOf(model);
|
|
5107
|
+
if (fields.length === 0) throw new MediaError(`${model.id} declares no request fields, so no request can be built for it.`, "AUDIO_FIELDS_UNDECLARED", { field: "model" });
|
|
5108
|
+
if (kind !== "speech") for (const name of SPEECH_ONLY) {
|
|
5109
|
+
const value = request[name];
|
|
5110
|
+
if (value !== void 0 && !(Array.isArray(value) && value.length === 0)) refuse(`${name === "sampleRate" ? "sample_rate" : name} is for speech; ${kind === "music" ? "music" : "a podcast"} does not take it.`, name);
|
|
5111
|
+
}
|
|
5112
|
+
const takes = (wire) => fields.includes(wire);
|
|
5113
|
+
const body = { model: model.id };
|
|
5114
|
+
const promptField = promptFieldOf(model);
|
|
5115
|
+
const prompt = request.prompt?.trim();
|
|
5116
|
+
if (has(prompt)) {
|
|
5117
|
+
if (promptField === void 0) refuse(`${model.id} takes no text prompt.`, "prompt");
|
|
5118
|
+
if (audio?.maxPromptChars !== void 0 && [...prompt].length > audio.maxPromptChars) refuse(`The text is ${[...prompt].length} characters; ${model.id} takes at most ${audio.maxPromptChars}.`, promptField);
|
|
5119
|
+
body[promptField] = prompt;
|
|
5120
|
+
}
|
|
5121
|
+
for (const [key, wire] of [
|
|
5122
|
+
["lyrics", "lyrics"],
|
|
5123
|
+
["genre", "genre"],
|
|
5124
|
+
["mood", "mood"],
|
|
5125
|
+
["gender", "gender"],
|
|
5126
|
+
["sourceUrl", "source_url"]
|
|
5127
|
+
]) {
|
|
5128
|
+
const value = request[key]?.trim();
|
|
5129
|
+
if (!has(value)) continue;
|
|
5130
|
+
if (!takes(wire)) refuse(`${model.id} does not take ${wire}; it takes ${fields.filter((name) => name !== "model").join(", ")}.`, wire);
|
|
5131
|
+
checkChoice(audio, wire, value);
|
|
5132
|
+
if (wire === "source_url" && !/^https?:\/\//i.test(value)) refuse("source_url must be an http(s) link.", wire);
|
|
5133
|
+
body[wire] = value;
|
|
5134
|
+
}
|
|
5135
|
+
if (request.duration !== void 0) {
|
|
5136
|
+
if (!takes("duration")) refuse(`${model.id} does not take a duration.`, "duration");
|
|
5137
|
+
body.duration = request.duration;
|
|
5138
|
+
}
|
|
5139
|
+
if (request.watermark !== void 0) {
|
|
5140
|
+
if (!takes("watermark")) refuse(`${model.id} does not take watermark.`, "watermark");
|
|
5141
|
+
body.watermark = request.watermark;
|
|
5142
|
+
}
|
|
5143
|
+
if (request.format !== void 0 || request.sampleRate !== void 0 || takes("audio_config")) {
|
|
5144
|
+
if (!takes("audio_config")) refuse(`${model.id} does not take an output format or sample rate.`, "format");
|
|
5145
|
+
const config = audioConfigOf(model, request.format, request.sampleRate);
|
|
5146
|
+
if (config !== void 0) body.audio_config = config;
|
|
5147
|
+
}
|
|
5148
|
+
if ((request.references?.length ?? 0) > 0 && !takes("references")) refuse(`${model.id} does not take reference audio.`, "references");
|
|
5149
|
+
if (kind === "music" && !has(prompt)) throw new MediaError("Music needs a prompt describing the song.", "AUDIO_PROMPT_REQUIRED", { field: "prompt" });
|
|
5150
|
+
if (kind === "podcast" && !has(prompt) && body.source_url === void 0) throw new MediaError("A podcast needs a prompt (the topic), a source_url, or both.", "AUDIO_PROMPT_REQUIRED", { field: "prompt" });
|
|
5151
|
+
if (kind === "speech" && !has(prompt)) throw new MediaError("Speech needs the text to speak.", "AUDIO_PROMPT_REQUIRED", { field: promptField ?? "prompt" });
|
|
5152
|
+
return body;
|
|
5153
|
+
}
|
|
5154
|
+
/** Load, check and upload the reference audio of a speech request; their measured lengths are checked too. */
|
|
5155
|
+
async function uploadReferences(runtime, call, model, sources, body) {
|
|
5156
|
+
const limits = model.audio?.references;
|
|
5157
|
+
if (limits === void 0) refuse(`${model.id} does not take reference audio.`, "references");
|
|
5158
|
+
if (sources.length < limits.min || sources.length > limits.max) refuse(limits.min === limits.max ? `${model.id} takes exactly ${limits.max} reference audio file(s); ${sources.length} were given.` : `${model.id} takes ${limits.min}–${limits.max} reference audio file(s); ${sources.length} were given.`, "references");
|
|
5159
|
+
const clash = (limits.exclusiveWith ?? []).filter((name) => body[name] !== void 0);
|
|
5160
|
+
if (clash.length > 0) refuse(`Reference audio cannot be combined with ${clash.join(", ")} on ${model.id}.`, "references");
|
|
5161
|
+
const maxBytes = limits.maxBytes ?? DEFAULT_REFERENCE_BYTES;
|
|
5162
|
+
const loaded = [];
|
|
5163
|
+
for (const source of sources) {
|
|
5164
|
+
const media = await runtime.load(call, source, "audio", maxBytes);
|
|
5165
|
+
if (limits.mimes !== void 0 && !limits.mimes.includes(media.mime)) throw new MediaError(`"${media.source}" is ${media.mime}; ${model.id} takes reference audio as ${limits.mimes.join(", ")}.`, "REFERENCE_MEDIA_TYPE_UNSUPPORTED", { field: "references" });
|
|
5166
|
+
loaded.push(media);
|
|
5167
|
+
}
|
|
5168
|
+
const checked = [];
|
|
5169
|
+
let totalSeconds = 0;
|
|
5170
|
+
for (const media of loaded) {
|
|
5171
|
+
const asset = await runtime.library.upload(media, limits.purpose ?? "audio_reference", call.signal);
|
|
5172
|
+
const seconds = asset.durationMs === void 0 ? void 0 : asset.durationMs / 1e3;
|
|
5173
|
+
if (seconds === void 0 && (limits.minSeconds !== void 0 || limits.maxSeconds !== void 0 || limits.maxTotalSeconds !== void 0)) throw new MediaError(`The gateway could not measure the length of "${media.source}". Export it as a standard WAV or MP3 and try again.`, "REFERENCE_DURATION_UNKNOWN", { field: "references" });
|
|
5174
|
+
if (seconds !== void 0) {
|
|
5175
|
+
if (limits.minSeconds !== void 0 && seconds < limits.minSeconds) throw new MediaError(`"${media.source}" is ${seconds.toFixed(1)} s; ${model.id} needs reference audio of at least ${limits.minSeconds} s.`, "REFERENCE_AUDIO_TOO_SHORT", { field: "references" });
|
|
5176
|
+
if (limits.maxSeconds !== void 0 && seconds > limits.maxSeconds) throw new MediaError(`"${media.source}" is ${seconds.toFixed(1)} s; ${model.id} takes reference audio of at most ${limits.maxSeconds} s.`, "REFERENCE_AUDIO_TOO_LONG", { field: "references" });
|
|
5177
|
+
totalSeconds += seconds;
|
|
5178
|
+
}
|
|
5179
|
+
checked.push({
|
|
5180
|
+
source: media.source,
|
|
5181
|
+
asset
|
|
5182
|
+
});
|
|
5183
|
+
}
|
|
5184
|
+
if (limits.maxTotalSeconds !== void 0 && totalSeconds > limits.maxTotalSeconds) throw new MediaError(`The reference audio adds up to ${totalSeconds.toFixed(1)} s; ${model.id} takes at most ${limits.maxTotalSeconds} s in total.`, "REFERENCE_TOTAL_TOO_LONG", { field: "references" });
|
|
5185
|
+
return checked;
|
|
5186
|
+
}
|
|
5187
|
+
/**
|
|
5188
|
+
* Choose the model for a request: the one named (an id or an alias), else the
|
|
5189
|
+
* pinned or first model of the kind. A named model must make the kind asked for.
|
|
5190
|
+
*/
|
|
5191
|
+
async function chooseAudioModel(runtime, request, signal) {
|
|
5192
|
+
if (request.kind === void 0 && !has(request.model)) throw new MediaError("Say what to make (music, podcast or speech), or name a model.", "AUDIO_KIND_REQUIRED", { field: "type" });
|
|
5193
|
+
if (request.kind !== void 0) return runtime.pickModel("audio", request.kind, request.model, void 0, signal);
|
|
5194
|
+
const model = await runtime.findModel(request.model, signal);
|
|
5195
|
+
if (model.kind !== "audio") throw new MediaError(`${model.id} is not an audio generation model.`, "MODEL_KIND_MISMATCH", { field: "model" });
|
|
5196
|
+
if (audioKindOf(model) === void 0) throw new MediaError(`${model.id} does not declare what kind of audio it makes, so it cannot be used for generation.`, "AUDIO_KIND_UNDECLARED", { field: "model" });
|
|
5197
|
+
return model;
|
|
5198
|
+
}
|
|
5199
|
+
/**
|
|
5200
|
+
* Everything before an audio submission: choose the model, check every field
|
|
5201
|
+
* against its declaration, upload and measure speech references, price it,
|
|
5202
|
+
* confirm the spending, and build the task under the operation's key.
|
|
5203
|
+
* @param runtime - the media runtime.
|
|
5204
|
+
* @param call - the tool call or host call.
|
|
5205
|
+
* @param request - what to generate.
|
|
5206
|
+
* @param save - where the result goes.
|
|
5207
|
+
* @returns the prepared task.
|
|
5208
|
+
*/
|
|
5209
|
+
async function prepareAudioTask(runtime, call, request, save) {
|
|
5210
|
+
if (request.operationId !== void 0 && !OPERATION_ID.test(request.operationId)) throw new MediaError("operationId may hold only letters, digits, \"-\" and \"_\" (at most 80).", "AUDIO_OPTION_INVALID", { field: "operationId" });
|
|
5211
|
+
const model = await chooseAudioModel(runtime, request, call.signal);
|
|
5212
|
+
const kind = audioKindOf(model);
|
|
5213
|
+
const body = audioRequestBody(model, request);
|
|
5214
|
+
const references = request.references ?? [];
|
|
5215
|
+
if (references.length > 0) body.references = (await uploadReferences(runtime, call, model, references, body)).map((item) => item.asset.referenceUrl);
|
|
5216
|
+
else if ((model.audio?.references?.min ?? 0) > 0) refuse(`${model.id} needs at least ${model.audio?.references?.min} reference audio file(s).`, "references");
|
|
5217
|
+
const quote = quoteAudio(model);
|
|
5218
|
+
const what = {
|
|
5219
|
+
music: {
|
|
5220
|
+
en: "a song",
|
|
5221
|
+
zh: "一首音乐"
|
|
5222
|
+
},
|
|
5223
|
+
podcast: {
|
|
5224
|
+
en: "a podcast episode",
|
|
5225
|
+
zh: "一期播客"
|
|
5226
|
+
},
|
|
5227
|
+
speech: {
|
|
5228
|
+
en: "speech",
|
|
5229
|
+
zh: "一段语音"
|
|
5230
|
+
}
|
|
5231
|
+
}[kind];
|
|
5232
|
+
await runtime.confirmSpending(call, "audio_generate", {
|
|
5233
|
+
en: `Generate ${what.en} with ${model.id}: ${quote.basis.en}.`,
|
|
5234
|
+
zh: `用 ${model.id} 生成${what.zh}:${quote.basis.zh}。`
|
|
5235
|
+
});
|
|
5236
|
+
const now = runtime.now();
|
|
5237
|
+
const prompt = request.prompt?.trim();
|
|
5238
|
+
return {
|
|
5239
|
+
model,
|
|
5240
|
+
kind,
|
|
5241
|
+
quote,
|
|
5242
|
+
draft: {
|
|
5243
|
+
id: request.operationId ?? randomUUID(),
|
|
5244
|
+
kind: "audio",
|
|
5245
|
+
model: model.id,
|
|
5246
|
+
label: excerptOf(prompt ?? request.sourceUrl ?? kind),
|
|
5247
|
+
createdAt: now,
|
|
5248
|
+
updatedAt: now,
|
|
5249
|
+
...call.agent === void 0 ? {} : { owner: call.agent.id },
|
|
5250
|
+
outputDir: save.folder,
|
|
5251
|
+
stem: save.stem,
|
|
5252
|
+
endpoint: "/v1/audio/generations",
|
|
5253
|
+
body,
|
|
5254
|
+
status: "submitting",
|
|
5255
|
+
audioKind: kind,
|
|
5256
|
+
...quote.amountCny === void 0 ? {} : { estimatedCny: quote.amountCny.toFixed(2) },
|
|
5257
|
+
...quote.unit === void 0 ? {} : { billing: {
|
|
5258
|
+
unit: quote.unit,
|
|
5259
|
+
...quote.unitPriceCny === void 0 ? {} : { unitPriceCny: String(quote.unitPriceCny) }
|
|
5260
|
+
} }
|
|
5261
|
+
}
|
|
5262
|
+
};
|
|
5263
|
+
}
|
|
5264
|
+
function excerptOf(text, max = 60) {
|
|
5265
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
5266
|
+
const chars = [...flat];
|
|
5267
|
+
return chars.length <= max ? flat : `${chars.slice(0, max - 1).join("")}…`;
|
|
5268
|
+
}
|
|
5269
|
+
/**
|
|
5270
|
+
* The form an interface shows for one model: only the fields the model
|
|
5271
|
+
* declares, with their declared choices and limits.
|
|
5272
|
+
* @param model - the model.
|
|
5273
|
+
* @returns the form, or undefined for a model that cannot generate audio.
|
|
5274
|
+
*/
|
|
5275
|
+
function audioFormOf(model) {
|
|
5276
|
+
const kind = audioKindOf(model);
|
|
5277
|
+
if (kind === void 0) return void 0;
|
|
5278
|
+
const fields = requestFieldsOf(model);
|
|
5279
|
+
if (fields.length === 0) return void 0;
|
|
5280
|
+
const audio = model.audio;
|
|
5281
|
+
const out = [];
|
|
5282
|
+
const promptField = promptFieldOf(model);
|
|
5283
|
+
if (promptField !== void 0) out.push({
|
|
5284
|
+
key: "prompt",
|
|
5285
|
+
wire: promptField,
|
|
5286
|
+
required: kind !== "podcast",
|
|
5287
|
+
...audio?.maxPromptChars === void 0 ? {} : { maxChars: audio.maxPromptChars }
|
|
5288
|
+
});
|
|
5289
|
+
for (const [key, wire] of [
|
|
5290
|
+
["lyrics", "lyrics"],
|
|
5291
|
+
["genre", "genre"],
|
|
5292
|
+
["mood", "mood"],
|
|
5293
|
+
["gender", "gender"],
|
|
5294
|
+
["sourceUrl", "source_url"]
|
|
5295
|
+
]) {
|
|
5296
|
+
if (!fields.includes(wire)) continue;
|
|
5297
|
+
const choices = audio?.choices?.[wire];
|
|
5298
|
+
out.push({
|
|
5299
|
+
key,
|
|
5300
|
+
wire,
|
|
5301
|
+
required: false,
|
|
5302
|
+
...choices === void 0 ? {} : { choices }
|
|
5303
|
+
});
|
|
5304
|
+
}
|
|
5305
|
+
if (kind === "speech") {
|
|
5306
|
+
if (fields.includes("references") && audio?.references !== void 0) out.push({
|
|
5307
|
+
key: "references",
|
|
5308
|
+
wire: "references",
|
|
5309
|
+
required: audio.references.min > 0,
|
|
5310
|
+
limits: audio.references
|
|
5311
|
+
});
|
|
5312
|
+
const formats = audio?.outputFormats ?? [];
|
|
5313
|
+
if (fields.includes("audio_config") && formats.length > 0) {
|
|
5314
|
+
out.push({
|
|
5315
|
+
key: "format",
|
|
5316
|
+
wire: "audio_config.format",
|
|
5317
|
+
required: false,
|
|
5318
|
+
choices: formats.map((item) => item.format)
|
|
5319
|
+
});
|
|
5320
|
+
const rates = Object.fromEntries(formats.filter((item) => item.sampleRates.length > 0).map((item) => [item.format, item.sampleRates]));
|
|
5321
|
+
if (Object.keys(rates).length > 0) out.push({
|
|
5322
|
+
key: "sampleRate",
|
|
5323
|
+
wire: "audio_config.sample_rate",
|
|
5324
|
+
required: false,
|
|
5325
|
+
sampleRates: rates
|
|
5326
|
+
});
|
|
5327
|
+
}
|
|
5328
|
+
if (fields.includes("watermark")) out.push({
|
|
5329
|
+
key: "watermark",
|
|
5330
|
+
wire: "watermark",
|
|
5331
|
+
required: false
|
|
5332
|
+
});
|
|
5333
|
+
}
|
|
5334
|
+
return {
|
|
5335
|
+
model: model.id,
|
|
5336
|
+
name: model.name,
|
|
5337
|
+
kind,
|
|
5338
|
+
fields: out,
|
|
5339
|
+
price: quoteAudio(model),
|
|
5340
|
+
...audio?.defaultFormat === void 0 ? {} : { defaultFormat: audio.defaultFormat }
|
|
4701
5341
|
};
|
|
4702
5342
|
}
|
|
4703
5343
|
//#endregion
|
|
@@ -4721,10 +5361,12 @@ function excerpt(prompt, max = 60) {
|
|
|
4721
5361
|
* @returns `Charged ¥4.95.`, an estimate, or the empty string.
|
|
4722
5362
|
*/
|
|
4723
5363
|
function moneyLine(record) {
|
|
4724
|
-
if (record.status !== "completed") return record.status === "failed" ? " Failed tasks are not charged." : "";
|
|
4725
|
-
|
|
4726
|
-
if (record.
|
|
4727
|
-
return
|
|
5364
|
+
if (record.status !== "completed") return record.status === "failed" ? " Failed and cancelled tasks are not charged." : "";
|
|
5365
|
+
const billed = record.billing?.originalSeconds === void 0 ? "" : ` Billed on ${record.billing.originalSeconds.toFixed(1)} s of original audio.`;
|
|
5366
|
+
if (record.settlement === "pending") return ` The result is saved; the charge is still being settled (it is never generated again — media_tasks shows the charge once settled).${billed}`;
|
|
5367
|
+
if (record.chargedCny !== void 0) return ` Charged ${yuan(Number(record.chargedCny))}.${billed}`;
|
|
5368
|
+
if (record.estimatedCny !== void 0) return ` Estimated ${yuan(Number(record.estimatedCny))} (the gateway settles the charge shortly).${billed}`;
|
|
5369
|
+
return billed;
|
|
4728
5370
|
}
|
|
4729
5371
|
/**
|
|
4730
5372
|
* The model-facing account of a task's current state.
|
|
@@ -5796,19 +6438,44 @@ function videoGenerateTool(runtime) {
|
|
|
5796
6438
|
}
|
|
5797
6439
|
//#endregion
|
|
5798
6440
|
//#region src/service.ts
|
|
6441
|
+
/**
|
|
6442
|
+
* The media service other plugins call, registered as `vibedevMedia`: the
|
|
6443
|
+
* film workbench's storyboard generates through it when someone presses a
|
|
6444
|
+
* generate button. It takes the same paths as the agent tools — model
|
|
6445
|
+
* choice, reference checks and uploads, idempotent submission, a background
|
|
6446
|
+
* follower that saves the result — with the caller naming the workspace and
|
|
6447
|
+
* where results go. The caller has already shown the price and the user chose
|
|
6448
|
+
* to go ahead, so the spending confirmation setting does not ask again.
|
|
6449
|
+
*
|
|
6450
|
+
* Every task, video or audio, is read the same way: {@link MediaHostService.task}
|
|
6451
|
+
* (what is recorded), {@link MediaHostService.refreshTask} (ask the gateway again:
|
|
6452
|
+
* fresh links, settlement — a query only, it never submits),
|
|
6453
|
+
* {@link MediaHostService.onTask} (each change) and
|
|
6454
|
+
* {@link MediaHostService.result} (the files to play or download).
|
|
6455
|
+
* @module dsh-vibedev/service
|
|
6456
|
+
*/
|
|
5799
6457
|
const view = (record) => ({
|
|
5800
6458
|
id: record.id,
|
|
5801
6459
|
kind: record.kind,
|
|
6460
|
+
...record.audioKind === void 0 ? {} : { audioKind: record.audioKind },
|
|
5802
6461
|
model: record.model,
|
|
5803
6462
|
status: record.status,
|
|
5804
6463
|
...record.progress === void 0 ? {} : { progress: record.progress },
|
|
5805
|
-
...record.outputs === void 0 ? {} : { outputs: record.outputs.map((output) => ({ ...output })) },
|
|
6464
|
+
...record.outputs === void 0 ? {} : { outputs: record.outputs.map(({ index: _index, bytes: _bytes, ...output }) => ({ ...output })) },
|
|
5806
6465
|
...record.error === void 0 ? {} : { error: { ...record.error } },
|
|
5807
6466
|
...record.estimatedCny === void 0 ? {} : { estimatedCny: record.estimatedCny },
|
|
5808
6467
|
...record.chargedCny === void 0 ? {} : { chargedCny: record.chargedCny },
|
|
6468
|
+
...record.settlement === void 0 ? {} : { settlement: record.settlement },
|
|
6469
|
+
...record.billing === void 0 ? {} : { billing: { ...record.billing } },
|
|
5809
6470
|
createdAt: record.createdAt,
|
|
5810
6471
|
updatedAt: record.updatedAt
|
|
5811
6472
|
});
|
|
6473
|
+
/** The parts of a request that decide what is generated, to tell a repeated operation from a different one. */
|
|
6474
|
+
function sameRequest(record, draft) {
|
|
6475
|
+
const { references: _a, ...left } = record.body;
|
|
6476
|
+
const { references: _b, ...right } = draft.body;
|
|
6477
|
+
return record.kind === draft.kind && record.model === draft.model && JSON.stringify(left) === JSON.stringify(right);
|
|
6478
|
+
}
|
|
5812
6479
|
var MediaHostService = class {
|
|
5813
6480
|
runtime;
|
|
5814
6481
|
version;
|
|
@@ -5824,7 +6491,7 @@ var MediaHostService = class {
|
|
|
5824
6491
|
};
|
|
5825
6492
|
}
|
|
5826
6493
|
/**
|
|
5827
|
-
* The media models this account can use (cached briefly).
|
|
6494
|
+
* The media models this account can use (cached briefly; renewed after a sign-in, sign-out or account switch).
|
|
5828
6495
|
* @param signal - cancels the request.
|
|
5829
6496
|
* @returns the models in catalog order.
|
|
5830
6497
|
*/
|
|
@@ -5832,6 +6499,28 @@ var MediaHostService = class {
|
|
|
5832
6499
|
return this.runtime.models(signal);
|
|
5833
6500
|
}
|
|
5834
6501
|
/**
|
|
6502
|
+
* The forms to show for the audio models this account can use: one per
|
|
6503
|
+
* model that declares what it makes, with only the fields it takes, their
|
|
6504
|
+
* declared choices and limits, and its price.
|
|
6505
|
+
* @param signal - cancels the catalog request.
|
|
6506
|
+
* @returns the forms, in catalog order.
|
|
6507
|
+
*/
|
|
6508
|
+
async audioForms(signal) {
|
|
6509
|
+
return (await this.runtime.models(signal)).flatMap((model) => {
|
|
6510
|
+
const form = audioFormOf(model);
|
|
6511
|
+
return form === void 0 ? [] : [form];
|
|
6512
|
+
});
|
|
6513
|
+
}
|
|
6514
|
+
/**
|
|
6515
|
+
* The price of one audio or transcription model: `¥0.50/次`, `¥0.02/原始秒`, `¥0.30/分钟`, or 未定价.
|
|
6516
|
+
* @param model - the model id or alias.
|
|
6517
|
+
* @param signal - cancels the catalog request.
|
|
6518
|
+
* @returns the quote.
|
|
6519
|
+
*/
|
|
6520
|
+
async audioPrice(model, signal) {
|
|
6521
|
+
return quoteAudio(await this.runtime.findModel(model, signal));
|
|
6522
|
+
}
|
|
6523
|
+
/**
|
|
5835
6524
|
* Generate images now and save them.
|
|
5836
6525
|
* @param request - what to generate.
|
|
5837
6526
|
* @param target - the workspace and where the files go.
|
|
@@ -5864,18 +6553,53 @@ var MediaHostService = class {
|
|
|
5864
6553
|
folder: target.folder,
|
|
5865
6554
|
stem: target.stem
|
|
5866
6555
|
});
|
|
6556
|
+
return this.submit(prepared.draft, signal);
|
|
6557
|
+
}
|
|
6558
|
+
/**
|
|
6559
|
+
* Submit music, a podcast episode or speech. Fields are checked against
|
|
6560
|
+
* what the model declares before anything is uploaded or charged; speech
|
|
6561
|
+
* references are uploaded to the gateway media library first.
|
|
6562
|
+
*
|
|
6563
|
+
* One user operation, one task: pass the same `operationId` when the same
|
|
6564
|
+
* click is retried and the task already recorded under it is returned —
|
|
6565
|
+
* nothing is submitted twice. Without one, every call is a new operation.
|
|
6566
|
+
* @param request - what to generate.
|
|
6567
|
+
* @param target - the workspace and where the audio goes.
|
|
6568
|
+
* @param signal - cancels the preparation and the submission.
|
|
6569
|
+
* @returns the task.
|
|
6570
|
+
*/
|
|
6571
|
+
async startAudio(request, target, signal) {
|
|
6572
|
+
if (request.operationId !== void 0) {
|
|
6573
|
+
const earlier = await this.runtime.store.get(request.operationId).catch(() => void 0);
|
|
6574
|
+
if (earlier !== void 0) {
|
|
6575
|
+
if (earlier.kind !== "audio") throw new MediaError(`Operation ${request.operationId} already belongs to a ${earlier.kind} task.`, "OPERATION_CONFLICT", { field: "operationId" });
|
|
6576
|
+
return view(earlier);
|
|
6577
|
+
}
|
|
6578
|
+
}
|
|
6579
|
+
const prepared = await prepareAudioTask(this.runtime, this.call(target, signal), request, {
|
|
6580
|
+
folder: target.folder,
|
|
6581
|
+
stem: target.stem
|
|
6582
|
+
});
|
|
6583
|
+
const raced = await this.runtime.store.get(prepared.draft.id).catch(() => void 0);
|
|
6584
|
+
if (raced !== void 0) {
|
|
6585
|
+
if (!sameRequest(raced, prepared.draft)) throw new MediaError(`Operation ${prepared.draft.id} already belongs to a different request.`, "OPERATION_CONFLICT", { field: "operationId" });
|
|
6586
|
+
return view(raced);
|
|
6587
|
+
}
|
|
6588
|
+
return this.submit(prepared.draft, signal);
|
|
6589
|
+
}
|
|
6590
|
+
async submit(draft, signal) {
|
|
5867
6591
|
try {
|
|
5868
|
-
return view(await this.runtime.tracker.submit(
|
|
6592
|
+
return view(await this.runtime.tracker.submit(draft, signal));
|
|
5869
6593
|
} catch (error) {
|
|
5870
6594
|
if (error instanceof MediaError && error.code === "SUBMISSION_UNCONFIRMED") {
|
|
5871
|
-
const recorded = await this.runtime.store.get(
|
|
6595
|
+
const recorded = await this.runtime.store.get(draft.id);
|
|
5872
6596
|
if (recorded !== void 0) return view(recorded);
|
|
5873
6597
|
}
|
|
5874
6598
|
throw error;
|
|
5875
6599
|
}
|
|
5876
6600
|
}
|
|
5877
6601
|
/**
|
|
5878
|
-
* A task's current state.
|
|
6602
|
+
* A task's current state, as recorded (no request to the gateway).
|
|
5879
6603
|
* @param id - the task id.
|
|
5880
6604
|
* @returns the task, or `undefined` when there is none.
|
|
5881
6605
|
*/
|
|
@@ -5884,6 +6608,31 @@ var MediaHostService = class {
|
|
|
5884
6608
|
return record === void 0 ? void 0 : view(record);
|
|
5885
6609
|
}
|
|
5886
6610
|
/**
|
|
6611
|
+
* Ask the gateway about a task again — a query, never a submission: a
|
|
6612
|
+
* finished task gets fresh links and its missing files are downloaded
|
|
6613
|
+
* again, and a pending settlement is read again. A running task is followed.
|
|
6614
|
+
* @param id - the task id.
|
|
6615
|
+
* @param signal - cancels the query.
|
|
6616
|
+
* @returns the task after the query, or `undefined` when there is none.
|
|
6617
|
+
*/
|
|
6618
|
+
async refreshTask(id, signal) {
|
|
6619
|
+
if (await this.runtime.store.get(id) === void 0) return void 0;
|
|
6620
|
+
return view(await this.runtime.tracker.refresh(id, signal));
|
|
6621
|
+
}
|
|
6622
|
+
/**
|
|
6623
|
+
* The files of a finished task, ready to play or download: saved files by
|
|
6624
|
+
* path, and links renewed by a query when one is needed (a missing file, an
|
|
6625
|
+
* expired link). Never generates again.
|
|
6626
|
+
* @param id - the task id.
|
|
6627
|
+
* @param signal - cancels the query.
|
|
6628
|
+
* @returns the outputs, or `undefined` while the task has none.
|
|
6629
|
+
*/
|
|
6630
|
+
async result(id, signal) {
|
|
6631
|
+
const record = await this.runtime.store.get(id);
|
|
6632
|
+
if (record === void 0 || record.status !== "completed") return void 0;
|
|
6633
|
+
return (((record.outputs ?? []).length > 0 && (record.outputs ?? []).every((output) => output.path !== void 0) ? record : await this.runtime.tracker.refresh(id, signal).catch(() => record)).outputs ?? []).map(({ index: _index, bytes: _bytes, ...output }) => ({ ...output }));
|
|
6634
|
+
}
|
|
6635
|
+
/**
|
|
5887
6636
|
* Hear about every change to one task.
|
|
5888
6637
|
* @param id - the task id.
|
|
5889
6638
|
* @param listener - called with the task after each change.
|
|
@@ -6095,7 +6844,8 @@ const COMPLETED = /* @__PURE__ */ new Set([
|
|
|
6095
6844
|
"succeeded",
|
|
6096
6845
|
"success",
|
|
6097
6846
|
"done",
|
|
6098
|
-
"finished"
|
|
6847
|
+
"finished",
|
|
6848
|
+
"settlement_pending"
|
|
6099
6849
|
]);
|
|
6100
6850
|
const FAILED = /* @__PURE__ */ new Set([
|
|
6101
6851
|
"failed",
|
|
@@ -6106,12 +6856,40 @@ const FAILED = /* @__PURE__ */ new Set([
|
|
|
6106
6856
|
"expired",
|
|
6107
6857
|
"rejected"
|
|
6108
6858
|
]);
|
|
6859
|
+
const CANCELLED = /* @__PURE__ */ new Set(["cancelled", "canceled"]);
|
|
6860
|
+
/** The result exists; only the charge is still being settled. */
|
|
6861
|
+
const SETTLEMENT_PENDING = "settlement_pending";
|
|
6109
6862
|
function stateOf(status) {
|
|
6110
6863
|
const word = status.toLowerCase();
|
|
6111
6864
|
if (COMPLETED.has(word)) return "completed";
|
|
6112
6865
|
if (FAILED.has(word)) return "failed";
|
|
6113
6866
|
return "pending";
|
|
6114
6867
|
}
|
|
6868
|
+
const seconds = (value) => {
|
|
6869
|
+
const number = typeof value === "string" ? Number(value) : value;
|
|
6870
|
+
return typeof number === "number" && Number.isFinite(number) && number > 0 ? number : void 0;
|
|
6871
|
+
};
|
|
6872
|
+
/** The billing facts of a task: whether the charge settled, and the billed length. */
|
|
6873
|
+
function billing(raw, status) {
|
|
6874
|
+
const blocks = [
|
|
6875
|
+
record$1(raw.billing),
|
|
6876
|
+
record$1(raw.usage),
|
|
6877
|
+
record$1(raw.effective),
|
|
6878
|
+
raw
|
|
6879
|
+
].filter((block) => block !== void 0);
|
|
6880
|
+
const pick = (...names) => {
|
|
6881
|
+
for (const block of blocks) for (const name of names) if (block[name] !== void 0) return block[name];
|
|
6882
|
+
};
|
|
6883
|
+
const originalSeconds = seconds(pick("original_seconds", "billed_original_seconds", "original_duration_seconds"));
|
|
6884
|
+
const minutes = seconds(pick("billed_minutes", "minutes"));
|
|
6885
|
+
const stateRaw = text$1(pick("settlement_state", "settlement_status"))?.toLowerCase();
|
|
6886
|
+
const settlement = status.toLowerCase() === SETTLEMENT_PENDING || stateRaw === "pending" ? "pending" : stateRaw === "settled" || stateRaw === "completed" ? "settled" : void 0;
|
|
6887
|
+
return {
|
|
6888
|
+
...settlement === void 0 ? {} : { settlement },
|
|
6889
|
+
...originalSeconds === void 0 ? {} : { billedOriginalSeconds: originalSeconds },
|
|
6890
|
+
...minutes === void 0 ? {} : { billedMinutes: minutes }
|
|
6891
|
+
};
|
|
6892
|
+
}
|
|
6115
6893
|
function errorOf(raw) {
|
|
6116
6894
|
const nested = record$1(raw.error);
|
|
6117
6895
|
const message = text$1(nested?.message) ?? text$1(raw.error) ?? text$1(raw.failure_reason) ?? text$1(raw.error_message) ?? text$1(raw.fail_reason);
|
|
@@ -6191,19 +6969,25 @@ function parseAudioTask(body) {
|
|
|
6191
6969
|
const url = httpUrl(output.content_url) ?? httpUrl(output.url);
|
|
6192
6970
|
const contentType = text$1(output.content_type);
|
|
6193
6971
|
const title = text$1(output.title);
|
|
6194
|
-
const duration =
|
|
6972
|
+
const duration = seconds(output.duration_seconds);
|
|
6973
|
+
const original = seconds(output.original_duration_seconds) ?? seconds(output.original_seconds);
|
|
6195
6974
|
outputs.push({
|
|
6196
6975
|
index,
|
|
6197
6976
|
...url === void 0 ? {} : { url },
|
|
6198
6977
|
...contentType === void 0 ? {} : { contentType },
|
|
6199
6978
|
...title === void 0 ? {} : { title },
|
|
6200
|
-
...duration === void 0 ? {} : { durationSeconds: duration }
|
|
6979
|
+
...duration === void 0 ? {} : { durationSeconds: duration },
|
|
6980
|
+
...original === void 0 ? {} : { originalSeconds: original }
|
|
6201
6981
|
});
|
|
6202
6982
|
}
|
|
6203
|
-
const
|
|
6983
|
+
const cancelled = CANCELLED.has(status.toLowerCase());
|
|
6984
|
+
const error = state === "failed" ? errorOf(raw) ?? (cancelled ? {
|
|
6985
|
+
code: "CANCELLED",
|
|
6986
|
+
message: "The task was cancelled; cancelled tasks are not charged."
|
|
6987
|
+
} : {
|
|
6204
6988
|
code: "GENERATION_FAILED",
|
|
6205
6989
|
message: `the task ended with status "${status}"`
|
|
6206
|
-
} : void 0;
|
|
6990
|
+
}) : void 0;
|
|
6207
6991
|
const progress = progressOf(raw.progress);
|
|
6208
6992
|
return {
|
|
6209
6993
|
id,
|
|
@@ -6212,6 +6996,7 @@ function parseAudioTask(body) {
|
|
|
6212
6996
|
...progress === void 0 ? {} : { progress },
|
|
6213
6997
|
...error === void 0 ? {} : { error },
|
|
6214
6998
|
...receipt(raw),
|
|
6999
|
+
...billing(raw, status),
|
|
6215
7000
|
outputs: state === "completed" ? outputs : []
|
|
6216
7001
|
};
|
|
6217
7002
|
}
|
|
@@ -6717,12 +7502,25 @@ var TaskTracker = class {
|
|
|
6717
7502
|
}
|
|
6718
7503
|
return Math.min(6e4, 5e3 * 2 ** Math.min(failures - 1, 4));
|
|
6719
7504
|
}
|
|
6720
|
-
/**
|
|
6721
|
-
|
|
6722
|
-
const
|
|
7505
|
+
/** The money and billing facts of a gateway answer, merged into what the record already knows. */
|
|
7506
|
+
moneyOf(record, task) {
|
|
7507
|
+
const charged = task.settlement === "pending" ? void 0 : task.chargedCny;
|
|
7508
|
+
const settlement = task.settlement ?? (charged !== void 0 ? "settled" : void 0);
|
|
7509
|
+
const billing = task.billedOriginalSeconds !== void 0 || task.billedMinutes !== void 0 ? { billing: {
|
|
7510
|
+
...record.billing,
|
|
7511
|
+
...task.billedOriginalSeconds === void 0 ? {} : { originalSeconds: task.billedOriginalSeconds },
|
|
7512
|
+
...task.billedMinutes === void 0 ? {} : { minutes: task.billedMinutes }
|
|
7513
|
+
} } : {};
|
|
7514
|
+
return {
|
|
6723
7515
|
...task.estimatedCny === void 0 ? {} : { estimatedCny: task.estimatedCny },
|
|
6724
|
-
...
|
|
7516
|
+
...charged === void 0 ? {} : { chargedCny: charged },
|
|
7517
|
+
...settlement === void 0 ? {} : { settlement },
|
|
7518
|
+
...billing
|
|
6725
7519
|
};
|
|
7520
|
+
}
|
|
7521
|
+
/** Record what the gateway says about a task; save the outputs of a finished one. */
|
|
7522
|
+
async apply(record, task) {
|
|
7523
|
+
const money = this.moneyOf(record, task);
|
|
6726
7524
|
if (task.state === "pending") return this.update(record.id, {
|
|
6727
7525
|
gatewayId: task.id,
|
|
6728
7526
|
status: "pending",
|
|
@@ -6748,14 +7546,14 @@ var TaskTracker = class {
|
|
|
6748
7546
|
if (attempts < (this.options.downloadAttempts ?? 5)) throw new MediaError(`${missing} result file(s) could not be downloaded yet.`, "DOWNLOAD_RETRY", { retryable: true });
|
|
6749
7547
|
}
|
|
6750
7548
|
let charged = money;
|
|
6751
|
-
if (
|
|
7549
|
+
if (money.chargedCny === void 0 && task.settlement !== "pending") {
|
|
6752
7550
|
const settled = await this.options.http.json(SPECS[record.kind].poll(task.id), {
|
|
6753
7551
|
timeoutMs: 15e3,
|
|
6754
7552
|
signal: this.signal
|
|
6755
7553
|
}).then((body) => SPECS[record.kind].parse(body), () => void 0);
|
|
6756
|
-
if (settled
|
|
7554
|
+
if (settled !== void 0) charged = {
|
|
6757
7555
|
...money,
|
|
6758
|
-
|
|
7556
|
+
...this.moneyOf(current, settled)
|
|
6759
7557
|
};
|
|
6760
7558
|
}
|
|
6761
7559
|
return this.finish(record.id, {
|
|
@@ -6770,6 +7568,50 @@ var TaskTracker = class {
|
|
|
6770
7568
|
});
|
|
6771
7569
|
}
|
|
6772
7570
|
/**
|
|
7571
|
+
* Ask the gateway about a task again, by query only: a finished task gets
|
|
7572
|
+
* fresh links (signed links expire), files that are missing locally are
|
|
7573
|
+
* downloaded again, and a pending settlement is read again. Nothing is ever
|
|
7574
|
+
* submitted from here, so a failed query or download cannot create a task.
|
|
7575
|
+
* A task still running is followed as usual.
|
|
7576
|
+
* @param id - the local task id.
|
|
7577
|
+
* @param signal - cancels the query.
|
|
7578
|
+
* @returns the task after the query.
|
|
7579
|
+
*/
|
|
7580
|
+
async refresh(id, signal) {
|
|
7581
|
+
const record = await this.options.store.get(id);
|
|
7582
|
+
if (record === void 0) throw new MediaError(`dsh-vibedev has no task ${id}.`, "UNKNOWN_TASK");
|
|
7583
|
+
if (!isFinished(record)) {
|
|
7584
|
+
this.followQuietly(id);
|
|
7585
|
+
return record;
|
|
7586
|
+
}
|
|
7587
|
+
if (record.gatewayId === void 0) return record;
|
|
7588
|
+
const spec = SPECS[record.kind];
|
|
7589
|
+
const task = spec.parse(await this.options.http.json(spec.poll(record.gatewayId), {
|
|
7590
|
+
timeoutMs: 3e4,
|
|
7591
|
+
...signal === void 0 ? {} : { signal }
|
|
7592
|
+
}));
|
|
7593
|
+
if (task === void 0) throw new MediaError("The VibeDev gateway returned an unreadable task.", "GATEWAY_BAD_RESPONSE");
|
|
7594
|
+
const money = this.moneyOf(record, task);
|
|
7595
|
+
if (task.state !== "completed") return this.update(id, money);
|
|
7596
|
+
const fresh = new Map(task.outputs.map((output) => [output.index, output]));
|
|
7597
|
+
const known = (record.outputs ?? []).filter((output) => output.path !== void 0 && output.index !== void 0).map((output) => ({
|
|
7598
|
+
...output,
|
|
7599
|
+
...fresh.get(output.index)?.url === void 0 ? {} : { url: fresh.get(output.index)?.url }
|
|
7600
|
+
}));
|
|
7601
|
+
const { outputs, missing } = await this.saveOutputs({
|
|
7602
|
+
...record,
|
|
7603
|
+
outputs: known
|
|
7604
|
+
}, task);
|
|
7605
|
+
return this.update(id, {
|
|
7606
|
+
...money,
|
|
7607
|
+
outputs,
|
|
7608
|
+
error: missing === 0 ? void 0 : {
|
|
7609
|
+
code: "DOWNLOAD_FAILED",
|
|
7610
|
+
message: `${missing} of ${outputs.length} result file(s) could not be downloaded; the gateway links are listed instead.`
|
|
7611
|
+
}
|
|
7612
|
+
});
|
|
7613
|
+
}
|
|
7614
|
+
/**
|
|
6773
7615
|
* Save every output not saved yet, recording each one as it lands. A
|
|
6774
7616
|
* shutdown mid-download throws, so the task stays open and is resumed.
|
|
6775
7617
|
*/
|
|
@@ -6789,7 +7631,8 @@ var TaskTracker = class {
|
|
|
6789
7631
|
...output.url === void 0 ? {} : { url: output.url },
|
|
6790
7632
|
...output.contentType === void 0 ? {} : { contentType: output.contentType },
|
|
6791
7633
|
...output.title === void 0 ? {} : { title: output.title },
|
|
6792
|
-
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds }
|
|
7634
|
+
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds },
|
|
7635
|
+
...output.originalSeconds === void 0 ? {} : { originalSeconds: output.originalSeconds }
|
|
6793
7636
|
};
|
|
6794
7637
|
const fromUrl = urlExtension(output.url);
|
|
6795
7638
|
const extension = output.contentType === void 0 ? fromUrl ?? extensionFor(spec.fallbackType) : extensionFor(output.contentType, fromUrl ?? extensionFor(spec.fallbackType));
|
|
@@ -6946,8 +7789,9 @@ function mediaAccountTool(runtime, chain, login) {
|
|
|
6946
7789
|
//#endregion
|
|
6947
7790
|
//#region src/tools/audio.ts
|
|
6948
7791
|
/**
|
|
6949
|
-
* `audio_generate`:
|
|
6950
|
-
* results are saved in the workspace.
|
|
7792
|
+
* `audio_generate`: music, a podcast episode or speech, as a background task
|
|
7793
|
+
* whose results are saved in the workspace. Every field is checked against
|
|
7794
|
+
* what the chosen model declares (see `audio/request`).
|
|
6951
7795
|
* @module dsh-vibedev/tools/audio
|
|
6952
7796
|
*/
|
|
6953
7797
|
/**
|
|
@@ -6958,17 +7802,17 @@ function mediaAccountTool(runtime, chain, login) {
|
|
|
6958
7802
|
function audioGenerateTool(runtime) {
|
|
6959
7803
|
return defineTool({
|
|
6960
7804
|
name: "audio_generate",
|
|
6961
|
-
description: "Generate music (a song with vocals or an instrumental)
|
|
7805
|
+
description: "Generate music (a song with vocals or an instrumental), a podcast episode (a spoken conversation about a topic or a web page) or speech (text read aloud, optionally in the voice of reference audio) with a VibeDev audio model. It runs as a background job: this call returns once the gateway accepts the task, you are notified when it finishes, and the files are saved in the workspace (media/audio/). Each model takes only the fields it declares; media_models lists them with prices (music and podcasts per request, speech per second of original audio). Charged only when it succeeds.",
|
|
6962
7806
|
parameters: {
|
|
6963
7807
|
type: {
|
|
6964
7808
|
type: "string",
|
|
6965
|
-
enum: [
|
|
7809
|
+
enum: [...AUDIO_MAKE_KINDS],
|
|
6966
7810
|
required: true,
|
|
6967
7811
|
description: "What to make."
|
|
6968
7812
|
},
|
|
6969
7813
|
prompt: {
|
|
6970
7814
|
type: "string",
|
|
6971
|
-
description: "Music: style, mood, instruments, tempo, theme. Podcast: the topic and angle
|
|
7815
|
+
description: "Music: style, mood, instruments, tempo, theme (required). Podcast: the topic and angle; give a prompt, a source_url, or both. Speech: the exact text to speak (required)."
|
|
6972
7816
|
},
|
|
6973
7817
|
lyrics: {
|
|
6974
7818
|
type: "string",
|
|
@@ -6984,16 +7828,32 @@ function audioGenerateTool(runtime) {
|
|
|
6984
7828
|
},
|
|
6985
7829
|
gender: {
|
|
6986
7830
|
type: "string",
|
|
6987
|
-
|
|
6988
|
-
description: "Music only: the singer's voice."
|
|
7831
|
+
description: "Music only: the singer's voice, such as male or female."
|
|
6989
7832
|
},
|
|
6990
7833
|
source_url: {
|
|
6991
7834
|
type: "string",
|
|
6992
7835
|
description: "Podcast only: an http(s) page for the hosts to discuss."
|
|
6993
7836
|
},
|
|
7837
|
+
references: {
|
|
7838
|
+
type: "array",
|
|
7839
|
+
items: { type: "string" },
|
|
7840
|
+
description: `Speech only: reference audio whose voice to imitate, each ${SOURCE_HELP}. The model states how many, how large and how long.`
|
|
7841
|
+
},
|
|
7842
|
+
format: {
|
|
7843
|
+
type: "string",
|
|
7844
|
+
description: "Speech only: an output format the model offers (media_models lists them, such as wav or mp3)."
|
|
7845
|
+
},
|
|
7846
|
+
sample_rate: {
|
|
7847
|
+
type: "integer",
|
|
7848
|
+
description: "Speech only: a sample rate the model offers for that format (Hz)."
|
|
7849
|
+
},
|
|
7850
|
+
watermark: {
|
|
7851
|
+
type: "boolean",
|
|
7852
|
+
description: "Speech only, when the model takes it: add the provider's audio watermark."
|
|
7853
|
+
},
|
|
6994
7854
|
model: {
|
|
6995
7855
|
type: "string",
|
|
6996
|
-
description: "Audio model id from media_models. Omit to use the default for the type."
|
|
7856
|
+
description: "Audio model id (or alias) from media_models. Omit to use the default for the type."
|
|
6997
7857
|
},
|
|
6998
7858
|
filename: {
|
|
6999
7859
|
type: "string",
|
|
@@ -7019,6 +7879,7 @@ function audioGenerateTool(runtime) {
|
|
|
7019
7879
|
},
|
|
7020
7880
|
jobId: { type: "string" },
|
|
7021
7881
|
estimatedCny: { type: "string" },
|
|
7882
|
+
price: { type: "string" },
|
|
7022
7883
|
outputs: {
|
|
7023
7884
|
type: "array",
|
|
7024
7885
|
items: {
|
|
@@ -7028,7 +7889,8 @@ function audioGenerateTool(runtime) {
|
|
|
7028
7889
|
path: { type: "string" },
|
|
7029
7890
|
url: { type: "string" },
|
|
7030
7891
|
title: { type: "string" },
|
|
7031
|
-
durationSeconds: { type: "number" }
|
|
7892
|
+
durationSeconds: { type: "number" },
|
|
7893
|
+
originalSeconds: { type: "number" }
|
|
7032
7894
|
}
|
|
7033
7895
|
}
|
|
7034
7896
|
},
|
|
@@ -7044,75 +7906,44 @@ function audioGenerateTool(runtime) {
|
|
|
7044
7906
|
}]
|
|
7045
7907
|
},
|
|
7046
7908
|
async execute(args, exec) {
|
|
7909
|
+
const kind = args.type;
|
|
7047
7910
|
const prompt = args.prompt?.trim();
|
|
7048
|
-
const
|
|
7049
|
-
|
|
7050
|
-
|
|
7051
|
-
|
|
7052
|
-
|
|
7053
|
-
|
|
7054
|
-
|
|
7055
|
-
|
|
7056
|
-
|
|
7057
|
-
|
|
7058
|
-
|
|
7059
|
-
}
|
|
7060
|
-
|
|
7061
|
-
|
|
7062
|
-
|
|
7063
|
-
|
|
7064
|
-
body = {
|
|
7065
|
-
...prompt ? { prompt } : {},
|
|
7066
|
-
...sourceUrl ? { source_url: sourceUrl } : {}
|
|
7067
|
-
};
|
|
7068
|
-
}
|
|
7069
|
-
const model = await runtime.pickModel("audio", args.type, args.model, void 0, exec.signal);
|
|
7070
|
-
const estimate = estimateAudio(model);
|
|
7071
|
-
const what = args.type === "music" ? {
|
|
7072
|
-
en: "a song",
|
|
7073
|
-
zh: "一首音乐"
|
|
7074
|
-
} : {
|
|
7075
|
-
en: "a podcast episode",
|
|
7076
|
-
zh: "一期播客"
|
|
7077
|
-
};
|
|
7078
|
-
await runtime.confirmSpending(exec, "audio_generate", {
|
|
7079
|
-
en: `Generate ${what.en} with ${model.id}${estimate === void 0 ? "" : `, about ${yuan(estimate.amountCny)}`}. Charged only if it succeeds.`,
|
|
7080
|
-
zh: `用 ${model.id} 生成${what.zh}${estimate === void 0 ? "" : `,约 ${yuan(estimate.amountCny)}`},成功才扣费。`
|
|
7081
|
-
});
|
|
7082
|
-
const now = runtime.now();
|
|
7083
|
-
const label = excerpt(prompt ?? sourceUrl);
|
|
7084
|
-
const { record: accepted, jobId, unconfirmed } = await submitTask(runtime, exec, {
|
|
7085
|
-
id: randomUUID(),
|
|
7086
|
-
kind: "audio",
|
|
7087
|
-
model: model.id,
|
|
7088
|
-
label,
|
|
7089
|
-
createdAt: now,
|
|
7090
|
-
updatedAt: now,
|
|
7091
|
-
...exec.agent === void 0 ? {} : { owner: exec.agent.id },
|
|
7092
|
-
outputDir: runtime.outputFolder(exec, "audio"),
|
|
7093
|
-
stem: runtime.stem(args.filename, prompt ?? args.type),
|
|
7094
|
-
endpoint: "/v1/audio/generations",
|
|
7095
|
-
body: {
|
|
7096
|
-
model: model.id,
|
|
7097
|
-
...body
|
|
7098
|
-
},
|
|
7099
|
-
status: "submitting",
|
|
7100
|
-
...estimate === void 0 ? {} : { estimatedCny: estimate.amountCny.toFixed(2) }
|
|
7911
|
+
const prepared = await prepareAudioTask(runtime, exec, {
|
|
7912
|
+
kind,
|
|
7913
|
+
...args.model === void 0 ? {} : { model: args.model },
|
|
7914
|
+
...prompt === void 0 || prompt === "" ? {} : { prompt },
|
|
7915
|
+
...args.lyrics === void 0 ? {} : { lyrics: args.lyrics },
|
|
7916
|
+
...args.genre === void 0 ? {} : { genre: args.genre },
|
|
7917
|
+
...args.mood === void 0 ? {} : { mood: args.mood },
|
|
7918
|
+
...args.gender === void 0 ? {} : { gender: args.gender },
|
|
7919
|
+
...args.source_url === void 0 ? {} : { sourceUrl: args.source_url },
|
|
7920
|
+
...args.references === void 0 ? {} : { references: args.references },
|
|
7921
|
+
...args.format === void 0 ? {} : { format: args.format },
|
|
7922
|
+
...args.sample_rate === void 0 ? {} : { sampleRate: args.sample_rate },
|
|
7923
|
+
...args.watermark === void 0 ? {} : { watermark: args.watermark }
|
|
7924
|
+
}, {
|
|
7925
|
+
folder: runtime.outputFolder(exec, "audio"),
|
|
7926
|
+
stem: runtime.stem(args.filename, prompt ?? args.source_url ?? kind)
|
|
7101
7927
|
});
|
|
7928
|
+
const { model, quote } = prepared;
|
|
7929
|
+
const { record: accepted, jobId, unconfirmed } = await submitTask(runtime, exec, prepared.draft);
|
|
7102
7930
|
const base = {
|
|
7103
7931
|
taskId: accepted.id,
|
|
7104
7932
|
model: model.id,
|
|
7933
|
+
price: quote.label.en,
|
|
7105
7934
|
...accepted.estimatedCny === void 0 ? {} : { estimatedCny: accepted.estimatedCny }
|
|
7106
7935
|
};
|
|
7107
|
-
|
|
7108
|
-
|
|
7109
|
-
|
|
7110
|
-
|
|
7111
|
-
|
|
7112
|
-
|
|
7113
|
-
|
|
7114
|
-
|
|
7115
|
-
|
|
7936
|
+
const name = {
|
|
7937
|
+
music: "Music",
|
|
7938
|
+
podcast: "Podcast",
|
|
7939
|
+
speech: "Speech"
|
|
7940
|
+
}[kind];
|
|
7941
|
+
if (jobId !== void 0) return {
|
|
7942
|
+
...base,
|
|
7943
|
+
status: accepted.status,
|
|
7944
|
+
jobId,
|
|
7945
|
+
message: `${unconfirmed ? `${name} task ${accepted.id} (${model.id}): ${UNCONFIRMED_NOTE}` : `${name} task submitted to ${model.id}.`} Background job ${jobId} reports when it finishes; files are saved under ${runtime.display(exec, accepted.outputDir)}. ${accepted.estimatedCny === void 0 ? `Price: ${quote.basis.en}. ` : `Estimated ${yuan(Number(accepted.estimatedCny))}, charged only if it succeeds. `}Carry on with other work meanwhile; there is no need to poll.`
|
|
7946
|
+
};
|
|
7116
7947
|
const final = await waitForTask(runtime, exec, accepted);
|
|
7117
7948
|
return {
|
|
7118
7949
|
...base,
|
|
@@ -7121,7 +7952,8 @@ function audioGenerateTool(runtime) {
|
|
|
7121
7952
|
...output.path === void 0 ? {} : { path: runtime.display(exec, output.path) },
|
|
7122
7953
|
...output.url === void 0 ? {} : { url: output.url },
|
|
7123
7954
|
...output.title === void 0 ? {} : { title: output.title },
|
|
7124
|
-
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds }
|
|
7955
|
+
...output.durationSeconds === void 0 ? {} : { durationSeconds: output.durationSeconds },
|
|
7956
|
+
...output.originalSeconds === void 0 ? {} : { originalSeconds: output.originalSeconds }
|
|
7125
7957
|
})) },
|
|
7126
7958
|
message: describeTask(runtime, exec, final)
|
|
7127
7959
|
};
|
|
@@ -7138,14 +7970,47 @@ function audioGenerateTool(runtime) {
|
|
|
7138
7970
|
const KIND_LABEL = {
|
|
7139
7971
|
image: "Image models (image_generate)",
|
|
7140
7972
|
video: "Video models (video_generate)",
|
|
7141
|
-
audio: "Music and
|
|
7973
|
+
audio: "Music, podcast and speech models (audio_generate)",
|
|
7142
7974
|
transcription: "Transcription models (audio_transcribe)"
|
|
7143
7975
|
};
|
|
7144
7976
|
function slotOf(model) {
|
|
7145
|
-
if (model.kind === "audio") return model.audioKind
|
|
7977
|
+
if (model.kind === "audio") return model.audioKind;
|
|
7146
7978
|
return model.kind;
|
|
7147
7979
|
}
|
|
7148
7980
|
/**
|
|
7981
|
+
* What an audio model takes, from its declaration: the fields, speech references, output formats and sample rates.
|
|
7982
|
+
* @param model - an audio or transcription model.
|
|
7983
|
+
* @returns one line, or undefined when nothing is declared.
|
|
7984
|
+
*/
|
|
7985
|
+
function describeAudioModel(model) {
|
|
7986
|
+
const audio = model.audio;
|
|
7987
|
+
if (model.kind === "transcription") {
|
|
7988
|
+
const facts = [
|
|
7989
|
+
audio?.maxInputBytes === void 0 ? void 0 : `at most ${(audio.maxInputBytes / 1e6).toFixed(0)} MB`,
|
|
7990
|
+
audio?.maxInputSeconds === void 0 ? void 0 : `at most ${Math.round(audio.maxInputSeconds / 60)} min`,
|
|
7991
|
+
audio?.languages === void 0 ? void 0 : `languages ${audio.languages.join(", ")}`
|
|
7992
|
+
].filter(Boolean);
|
|
7993
|
+
return facts.length === 0 ? void 0 : facts.join("; ");
|
|
7994
|
+
}
|
|
7995
|
+
if (model.audioKind === void 0) return "does not declare what kind of audio it makes; it cannot be used for generation";
|
|
7996
|
+
const fields = requestFieldsOf(model).filter((name) => name !== "model");
|
|
7997
|
+
const parts = [`fields: ${fields.join(", ") || "none declared"}`];
|
|
7998
|
+
const prompt = promptFieldOf(model);
|
|
7999
|
+
if (prompt !== void 0 && prompt !== "prompt") parts.push(`the text goes in ${prompt}`);
|
|
8000
|
+
const references = audio?.references;
|
|
8001
|
+
if (references !== void 0 && fields.includes("references")) {
|
|
8002
|
+
const count = references.min === references.max ? `${references.max}` : `${references.min}-${references.max}`;
|
|
8003
|
+
parts.push([
|
|
8004
|
+
`${count} reference audio`,
|
|
8005
|
+
references.maxBytes === void 0 ? void 0 : `each at most ${(references.maxBytes / 1048576).toFixed(0)} MiB`,
|
|
8006
|
+
references.minSeconds === void 0 && references.maxSeconds === void 0 ? void 0 : `each ${references.minSeconds ?? 0}-${references.maxSeconds ?? "∞"} s`,
|
|
8007
|
+
references.exclusiveWith === void 0 ? void 0 : `not with ${references.exclusiveWith.join(", ")}`
|
|
8008
|
+
].filter(Boolean).join(", "));
|
|
8009
|
+
}
|
|
8010
|
+
for (const format of audio?.outputFormats ?? []) parts.push(`format ${format.format}${format.sampleRates.length === 0 ? "" : ` at ${format.sampleRates.join("/")} Hz`}`);
|
|
8011
|
+
return parts.join("; ");
|
|
8012
|
+
}
|
|
8013
|
+
/**
|
|
7149
8014
|
* Build the tool.
|
|
7150
8015
|
* @param runtime - the media runtime.
|
|
7151
8016
|
* @returns the tool definition.
|
|
@@ -7153,7 +8018,7 @@ function slotOf(model) {
|
|
|
7153
8018
|
function mediaModelsTool(runtime) {
|
|
7154
8019
|
return defineTool({
|
|
7155
8020
|
name: "media_models",
|
|
7156
|
-
description: "List the image, video, music/podcast and transcription models this VibeDev account can use, with prices. For video models it lists each generation mode and the inputs it accepts (reference images, videos and audio, first and last frames), durations, aspect ratios and resolutions. Use it to choose a model when a request needs particular inputs or options; the generation tools pick a suitable model themselves when none is named.",
|
|
8021
|
+
description: "List the image, video, music/podcast/speech and transcription models this VibeDev account can use, with prices (\"not priced\" when the catalog states none). For video models it lists each generation mode and the inputs it accepts (reference images, videos and audio, first and last frames), durations, aspect ratios and resolutions; for audio models the fields each takes, speech references, output formats and sample rates. Use it to choose a model when a request needs particular inputs or options; the generation tools pick a suitable model themselves when none is named.",
|
|
7157
8022
|
parameters: { kind: {
|
|
7158
8023
|
type: "string",
|
|
7159
8024
|
enum: [
|
|
@@ -7232,16 +8097,20 @@ function mediaModelsTool(runtime) {
|
|
|
7232
8097
|
isConcurrencySafe: () => true,
|
|
7233
8098
|
async execute(args, exec) {
|
|
7234
8099
|
return { models: (await runtime.models(exec.signal)).filter((model) => args.kind === void 0 || model.kind === args.kind).map((model) => {
|
|
7235
|
-
const
|
|
7236
|
-
const
|
|
8100
|
+
const audioLike = model.kind === "audio" || model.kind === "transcription";
|
|
8101
|
+
const price = audioLike ? quoteAudio(model).label.en : describePrice(model);
|
|
8102
|
+
const slot = slotOf(model);
|
|
8103
|
+
const pinned = slot !== void 0 && runtime.settings.defaultModel(slot).trim() === model.id;
|
|
8104
|
+
const audioFacts = audioLike ? describeAudioModel(model) : void 0;
|
|
7237
8105
|
return {
|
|
7238
8106
|
id: model.id,
|
|
7239
8107
|
name: model.name,
|
|
7240
8108
|
kind: model.kind,
|
|
7241
|
-
...model.
|
|
8109
|
+
...model.kind !== "audio" ? {} : { produces: model.audioKind ?? "undeclared" },
|
|
7242
8110
|
...model.description === void 0 ? {} : { description: model.description },
|
|
7243
|
-
|
|
8111
|
+
price,
|
|
7244
8112
|
...model.kind === "video" ? { capabilities: describeVideoModel(model) } : {},
|
|
8113
|
+
...audioFacts === void 0 ? {} : { capabilities: audioFacts },
|
|
7245
8114
|
...pinned ? { pinned } : {}
|
|
7246
8115
|
};
|
|
7247
8116
|
}) };
|
|
@@ -7329,7 +8198,7 @@ function mediaTasksTool(runtime) {
|
|
|
7329
8198
|
const limit = Math.min(50, Math.max(1, Math.trunc(args.limit ?? 10)));
|
|
7330
8199
|
const shown = tasks.slice(0, limit);
|
|
7331
8200
|
for (const task of shown) if (!isFinished(task)) runtime.tracker.follow(task.id).catch(() => void 0);
|
|
7332
|
-
return { tasks: shown.map((task) => ({
|
|
8201
|
+
return { tasks: (await Promise.all(shown.map((task) => task.status === "completed" && task.settlement === "pending" ? runtime.tracker.refresh(task.id, exec.signal).catch(() => task) : task))).map((task) => ({
|
|
7333
8202
|
id: task.id,
|
|
7334
8203
|
kind: task.kind,
|
|
7335
8204
|
status: task.status,
|
|
@@ -7411,7 +8280,7 @@ function sleep(ms, signal) {
|
|
|
7411
8280
|
function audioTranscribeTool(runtime) {
|
|
7412
8281
|
return defineTool({
|
|
7413
8282
|
name: "audio_transcribe",
|
|
7414
|
-
description: "Transcribe speech in an audio file to text with a VibeDev transcription model
|
|
8283
|
+
description: "Transcribe speech in an audio file to text with a VibeDev transcription model (uploaded as a multipart file). Each model states its languages and limits (media_models; commonly Mandarin Chinese, at most 20 MB and 10 minutes); 16 kHz mono WAV works best. Charged per minute of audio.",
|
|
7415
8284
|
parameters: {
|
|
7416
8285
|
file: {
|
|
7417
8286
|
type: "string",
|
|
@@ -7462,15 +8331,19 @@ function audioTranscribeTool(runtime) {
|
|
|
7462
8331
|
async execute(args, exec) {
|
|
7463
8332
|
const language = (args.language ?? "zh").trim().toLowerCase() || "zh";
|
|
7464
8333
|
const model = await runtime.pickModel("transcription", "transcription", args.model, void 0, exec.signal);
|
|
7465
|
-
const
|
|
7466
|
-
|
|
8334
|
+
const declared = model.audio;
|
|
8335
|
+
const maxBytes = declared?.maxInputBytes ?? MAX_BYTES;
|
|
8336
|
+
const maxSeconds = declared?.maxInputSeconds ?? MAX_SECONDS;
|
|
8337
|
+
const accepted = declared?.inputMimes ?? ACCEPTED;
|
|
8338
|
+
if (declared?.languages !== void 0 && !declared.languages.includes(language)) throw new MediaError(`${model.id} transcribes ${declared.languages.join(", ")}, not "${language}".`, "AUDIO_OPTION_INVALID", { field: "language" });
|
|
8339
|
+
const media = await runtime.load(exec, args.file, "audio", maxBytes);
|
|
8340
|
+
if (!accepted.includes(media.mime)) throw new MediaError(`"${media.source}" is ${media.mime}, not an audio file the gateway transcribes; convert it to 16 kHz mono WAV first.`, "AUDIO_FORMAT_UNSUPPORTED", { field: "file" });
|
|
7467
8341
|
const seconds = media.mime === "audio/wav" ? wavSeconds(media.data) : void 0;
|
|
7468
|
-
if (seconds !== void 0 && seconds >
|
|
7469
|
-
const
|
|
7470
|
-
const estimate = tier?.unit === "second" && seconds !== void 0 ? tier.amount * seconds : void 0;
|
|
8342
|
+
if (seconds !== void 0 && seconds > maxSeconds) throw new MediaError(`"${media.source}" is ${(seconds / 60).toFixed(1)} minutes long; the limit is ${(maxSeconds / 60).toFixed(0)} minutes. Split it first.`, "AUDIO_TOO_LONG", { field: "file" });
|
|
8343
|
+
const quote = quoteAudio(model, seconds);
|
|
7471
8344
|
await runtime.confirmSpending(exec, "audio_transcribe", {
|
|
7472
|
-
en: `Transcribe ${media.name} with ${model.id}${
|
|
7473
|
-
zh: `用 ${model.id} 转写 ${media.name}
|
|
8345
|
+
en: `Transcribe ${media.name} with ${model.id}: ${quote.amountCny === void 0 ? quote.basis.en : `about ${yuan(quote.amountCny)} (${quote.basis.en})`}.`,
|
|
8346
|
+
zh: `用 ${model.id} 转写 ${media.name}:${quote.amountCny === void 0 ? quote.basis.zh : `约 ${yuan(quote.amountCny)}(${quote.basis.zh})`}。`
|
|
7474
8347
|
});
|
|
7475
8348
|
const body = new FormData();
|
|
7476
8349
|
body.append("file", new Blob([media.data], { type: media.mime }), media.name);
|
|
@@ -7557,6 +8430,7 @@ const Config = Schema.object({
|
|
|
7557
8430
|
videoModel: Schema.string().default("").volatile(),
|
|
7558
8431
|
musicModel: Schema.string().default("").volatile(),
|
|
7559
8432
|
podcastModel: Schema.string().default("").volatile(),
|
|
8433
|
+
speechModel: Schema.string().default("").volatile(),
|
|
7560
8434
|
transcriptionModel: Schema.string().default("").volatile(),
|
|
7561
8435
|
gatewayOrigin: Schema.string().default(DEFAULT_GATEWAY_ORIGIN),
|
|
7562
8436
|
client: Schema.string().default("vibedev-plugin"),
|
|
@@ -7575,6 +8449,7 @@ const Config = Schema.object({
|
|
|
7575
8449
|
videoModel: "默认视频模型(留空由 Agent 按目录选择)",
|
|
7576
8450
|
musicModel: "默认音乐模型(留空自动选择)",
|
|
7577
8451
|
podcastModel: "默认播客模型(留空自动选择)",
|
|
8452
|
+
speechModel: "默认语音合成模型(留空自动选择)",
|
|
7578
8453
|
transcriptionModel: "默认语音转写模型(留空自动选择)",
|
|
7579
8454
|
gatewayOrigin: "VibeDev 网关地址",
|
|
7580
8455
|
client: "登录时报给网关的客户端名称(决定用量记在哪个密钥下)",
|
|
@@ -7593,6 +8468,7 @@ const Config = Schema.object({
|
|
|
7593
8468
|
videoModel: "Default video model (empty: the agent chooses from the catalog)",
|
|
7594
8469
|
musicModel: "Default music model (empty: chosen automatically)",
|
|
7595
8470
|
podcastModel: "Default podcast model (empty: chosen automatically)",
|
|
8471
|
+
speechModel: "Default speech model (empty: chosen automatically)",
|
|
7596
8472
|
transcriptionModel: "Default transcription model (empty: chosen automatically)",
|
|
7597
8473
|
gatewayOrigin: "VibeDev gateway address",
|
|
7598
8474
|
client: "Client name reported to the gateway at sign-in (decides which key the usage is recorded under)",
|
|
@@ -7601,7 +8477,7 @@ const Config = Schema.object({
|
|
|
7601
8477
|
stateDir: "Plugin data folder (empty for the default)"
|
|
7602
8478
|
}
|
|
7603
8479
|
});
|
|
7604
|
-
const GUIDANCE = "VibeDev media tools: image_generate, video_generate, audio_generate (music
|
|
8480
|
+
const GUIDANCE = "VibeDev media tools: image_generate, video_generate, audio_generate (music, a podcast or speech) and audio_transcribe create and transcribe media with the user's VibeDev account; media_models lists models and prices. Use them on your own judgment whenever generated media serves the request (illustrations, product shots, storyboard frames, short clips, background music, a podcast episode, a voice-over), not only when a tool is named. They spend the user's balance, so make what the request needs rather than speculative variations. Results are saved in the workspace under media/. Pass files the user attached in this chat as chat:1 (the most recent), chat:2, and so on. Video and audio run as background jobs: carry on and let the job notification report the result.";
|
|
7605
8481
|
function apply(ctx, config) {
|
|
7606
8482
|
const origin = normalizeGatewayOrigin(config.gatewayOrigin);
|
|
7607
8483
|
const stateDir = stateDirectory(config.stateDir);
|
|
@@ -7650,6 +8526,7 @@ function apply(ctx, config) {
|
|
|
7650
8526
|
video: config.videoModel,
|
|
7651
8527
|
music: config.musicModel,
|
|
7652
8528
|
podcast: config.podcastModel,
|
|
8529
|
+
speech: config.speechModel,
|
|
7653
8530
|
transcription: config.transcriptionModel
|
|
7654
8531
|
};
|
|
7655
8532
|
const runtime = new MediaRuntime({
|