@mindstudio-ai/agent 0.1.167 → 0.1.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/cli.js +8 -8
- package/dist/index.d.ts +6 -6
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/postinstall.js +8 -8
- package/llms.txt +6 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,16 @@ What changed in `@mindstudio-ai/agent`, newest first, each version under its own
|
|
|
4
4
|
|
|
5
5
|
Versions before 0.1.163 aren't recorded here.
|
|
6
6
|
|
|
7
|
+
## 0.1.168
|
|
8
|
+
|
|
9
|
+
### New
|
|
10
|
+
|
|
11
|
+
- `transcribeAudio` can run on Mistral's Voxtral 3: `transcriptionModelOverride.model` set to `voxtral-mini-transcribe-3` or `voxtral-small-transcribe-3`. Both label speakers, take a language and key terms, and return segments on every transcript, but neither times words. A recording past 40 minutes is transcribed in pieces, and refused with speakers on. `textToSpeech` can run on `voxtral-tts-3`, whose text takes `[emotion=happy]` and `[pace=slow]` tags. The usage notes say so. App code isn't affected.
|
|
12
|
+
|
|
13
|
+
### Changed behavior
|
|
14
|
+
|
|
15
|
+
- `voxtral-mini-transcribe-2` takes a language together with word timings or subtitles, which it refused before, and returns segments when it's given a language. App code isn't affected.
|
|
16
|
+
|
|
7
17
|
## 0.1.167
|
|
8
18
|
|
|
9
19
|
### New
|
package/dist/cli.js
CHANGED
|
@@ -1055,14 +1055,14 @@ var init_metadata = __esm({
|
|
|
1055
1055
|
"textToSpeech": {
|
|
1056
1056
|
stepType: "textToSpeech",
|
|
1057
1057
|
description: "Generate an audio file from provided text using a speech model.",
|
|
1058
|
-
usageNotes: "- The text field contains the exact words to be spoken (not instructions).\n- Voice, delivery and format are set through `speechModelOverride.config`, whose keys are the model's own (look a model up with its id). Delivery direction, where a model takes one, is never spoken: `styleInstruction` on gemini-3.8-flash-tts and gemini-3.8-flash-lite-tts, `instructions` on gpt-realtime-2.1-mini-tts, `voice_instruction` on seed-speech-tts-2.0, `emotion` on cartesia-sonic-3.6 and the MiniMax models.\n- Tags in the text act at that point, in each vendor's own syntax: `[laughs]` or `[whispers]` on elevenlabs-tts with Eleven v4 or v3; `<laugh>`, `<sigh>` or `<whispers>` on the Gemini 3.8 models; `[laugh]`, `[pause]` or `<whisper>\u2026</whisper>` on grok-voice-tts; `[laughter]` or `<emotion value=\"excited\"/>` on cartesia-sonic-3.6; `[be happy and upbeat]` on inworld-tts-2.\n- A dialogue between voices: set `$mode: 'dialogue'` and name each speaker (`speaker1_name`, `speaker1_voice`, `speaker2_name`, \u2026), then write the text as a script, one `Name: line` per turn. Supported by elevenlabs-tts (Eleven v4 or v3, up to 4 speakers) and the Gemini 3.8 models (exactly 2).\n- A text longer than a model takes in one request is spoken in pieces and joined, and then comes back as MP3 whatever format was asked for.\n- In foreground mode, the audio is displayed to the user. In background mode, the URL is saved to a variable.",
|
|
1058
|
+
usageNotes: "- The text field contains the exact words to be spoken (not instructions).\n- Voice, delivery and format are set through `speechModelOverride.config`, whose keys are the model's own (look a model up with its id). Delivery direction, where a model takes one, is never spoken: `styleInstruction` on gemini-3.8-flash-tts and gemini-3.8-flash-lite-tts, `instructions` on gpt-realtime-2.1-mini-tts, `voice_instruction` on seed-speech-tts-2.0, `emotion` on cartesia-sonic-3.6 and the MiniMax models.\n- Tags in the text act at that point, in each vendor's own syntax: `[laughs]` or `[whispers]` on elevenlabs-tts with Eleven v4 or v3; `<laugh>`, `<sigh>` or `<whispers>` on the Gemini 3.8 models; `[laugh]`, `[pause]` or `<whisper>\u2026</whisper>` on grok-voice-tts; `[laughter]` or `<emotion value=\"excited\"/>` on cartesia-sonic-3.6; `[be happy and upbeat]` on inworld-tts-2; `[emotion=happy]` or `[pace=slow]` on voxtral-tts-3, until the next tag.\n- A dialogue between voices: set `$mode: 'dialogue'` and name each speaker (`speaker1_name`, `speaker1_voice`, `speaker2_name`, \u2026), then write the text as a script, one `Name: line` per turn. Supported by elevenlabs-tts (Eleven v4 or v3, up to 4 speakers) and the Gemini 3.8 models (exactly 2).\n- A text longer than a model takes in one request is spoken in pieces and joined, and then comes back as MP3 whatever format was asked for.\n- In foreground mode, the audio is displayed to the user. In background mode, the URL is saved to a variable.",
|
|
1059
1059
|
inputSchema: { "type": "object", "properties": { "text": { "type": "string", "description": "The text to convert to speech" }, "intermediateAsset": { "type": "boolean" }, "speechModelOverride": { "type": "object", "properties": { "model": { "type": "string", "description": "Speech synthesis model identifier" }, "config": { "type": "object", "properties": {}, "required": [], "description": "Additional model-specific configuration" } }, "required": ["model"], "description": "Optional model configuration override. Uses the workflow's default speech model if not specified" } }, "required": ["text"] },
|
|
1060
1060
|
outputSchema: { "type": "object", "properties": { "audioUrl": { "type": "string", "description": "URL of the generated audio file" } }, "required": ["audioUrl"] }
|
|
1061
1061
|
},
|
|
1062
1062
|
"transcribeAudio": {
|
|
1063
1063
|
stepType: "transcribeAudio",
|
|
1064
1064
|
description: "Convert an audio file to text using a transcription model.",
|
|
1065
|
-
usageNotes: "- The default model is elevenlabs-scribe-v2: speakers, word timings, key terms and redaction, at any length.\n- The prompt field gives context (speaker names, domain) to OpenAI's and DeepInfra's models. Other models ignore it; give them `keyterms` instead.\n- Options go in `transcriptionModelOverride.config`, and support varies by model:\n - `language` \u2014 the spoken language as a code such as `en` or `es`; left out, the model detects it. Every model takes it. deepgram-nova-3 assumes `en` unless told, and also takes `multi` for speech that switches language (billed at its multilingual rate).\n - `detectLanguage: 'yes'` \u2014 detect the language instead of assuming English (deepgram-nova-3 only).\n - `includeSpeakers: 'yes'` \u2014 label each passage with a speaker. Supported by elevenlabs-scribe-v2, elevenlabs-scribe-v2-medical, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe; gpt-4o-transcribe-diarize always does. Scribe and Voxtral also switch `text` to a timestamped, speaker-labelled block format; Deepgram, Gemini and gpt-4o-transcribe-diarize to `Speaker N:` lines.\n - `numSpeakers` \u2014 the most people talking, to help tell them apart (Scribe).\n - `keyterms` \u2014 a list of names, products and terms the audio contains, so they're spelled right. Supported by gpt-transcribe, Scribe, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Billed as an add-on on Scribe and Deepgram.\n - `redactPii: 'yes'` \u2014 replace names, phone numbers, addresses and other personal information with a label of what was there (Scribe, deepgram-nova-3). Billed as an add-on.\n - `includeWordTimings: 'yes'` \u2014 add per-word timings and confidence inside each segment. Supported by Scribe, deepgram-nova-3, whisper-1, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Off by default because an hour of speech is roughly ten thousand words.\n - `subtitles: 'srt' | 'vtt'` \u2014 also return the transcript as a subtitle file, in the `subtitles` output, timed by the segments. Any model that returns segments.\n- gemini-3.5-transcribe can't take `keyterms` together with speakers, word timings or subtitles
|
|
1065
|
+
usageNotes: "- The default model is elevenlabs-scribe-v2: speakers, word timings, key terms and redaction, at any length.\n- The prompt field gives context (speaker names, domain) to OpenAI's and DeepInfra's models. Other models ignore it; give them `keyterms` instead.\n- Options go in `transcriptionModelOverride.config`, and support varies by model:\n - `language` \u2014 the spoken language as a code such as `en` or `es`; left out, the model detects it. Every model takes it. deepgram-nova-3 assumes `en` unless told, and also takes `multi` for speech that switches language (billed at its multilingual rate).\n - `detectLanguage: 'yes'` \u2014 detect the language instead of assuming English (deepgram-nova-3 only).\n - `includeSpeakers: 'yes'` \u2014 label each passage with a speaker. Supported by elevenlabs-scribe-v2, elevenlabs-scribe-v2-medical, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3 and gemini-3.5-transcribe; gpt-4o-transcribe-diarize always does. Scribe and Voxtral also switch `text` to a timestamped, speaker-labelled block format; Deepgram, Gemini and gpt-4o-transcribe-diarize to `Speaker N:` lines.\n - `numSpeakers` \u2014 the most people talking, to help tell them apart (Scribe).\n - `keyterms` \u2014 a list of names, products and terms the audio contains, so they're spelled right. Supported by gpt-transcribe, Scribe, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3 and gemini-3.5-transcribe. Billed as an add-on on Scribe and Deepgram.\n - `redactPii: 'yes'` \u2014 replace names, phone numbers, addresses and other personal information with a label of what was there (Scribe, deepgram-nova-3). Billed as an add-on.\n - `includeWordTimings: 'yes'` \u2014 add per-word timings and confidence inside each segment. Supported by Scribe, deepgram-nova-3, whisper-1, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Off by default because an hour of speech is roughly ten thousand words.\n - `subtitles: 'srt' | 'vtt'` \u2014 also return the transcript as a subtitle file, in the `subtitles` output, timed by the segments. Any model that returns segments.\n- gemini-3.5-transcribe can't take `keyterms` together with speakers, word timings or subtitles; asked for both, it's refused.\n- The `segments` output carries timings, confidence and speakers as structured data, so there is no need to slice the audio up to recover them. It is populated by Scribe, deepgram-nova-3, grok-stt, whisper-1, gpt-4o-transcribe-diarize, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3, gemini-3.5-transcribe (with word timings or subtitles) and the DeepInfra-served whisper and qwen models, and absent for models that return nothing but text (gpt-transcribe, gpt-4o-transcribe, gpt-4o-mini-transcribe, the DeepInfra-served voxtral models).\n- In a workflow, `destinationVar` receives the transcript text. Name `segmentsVar` to also keep the `segments` array. A model that returns no segments leaves that variable unset.\n- A recording of any length can be transcribed. One longer than a model takes at once (OpenAI's and DeepInfra's models, Gemini, Voxtral 3) is transcribed in pieces and joined, its timings running from the start of the recording. Speakers can't be followed from one piece to the next, so with speakers on, such a recording is refused; Scribe and Deepgram label speakers at any length.\n- `confidence` is always 0..1 with higher meaning more confident, whichever model produced it.",
|
|
1066
1066
|
inputSchema: { "type": "object", "properties": { "audioUrl": { "type": "string", "description": "URL of the audio file to transcribe" }, "prompt": { "type": "string", "description": "Optional context to improve transcription accuracy (e.g. language, speaker names, domain terms)" }, "transcriptionModelOverride": { "type": "object", "properties": { "model": { "type": "string", "description": "Audio transcription model identifier" }, "config": { "type": "object", "properties": {}, "required": [], "description": "Additional model-specific configuration" } }, "required": ["model"], "description": "Optional model configuration override. Uses the workflow's default transcription model if not specified" } }, "required": ["audioUrl", "prompt"] },
|
|
1067
1067
|
outputSchema: { "type": "object", "properties": { "text": { "type": "string", "description": "The transcribed text from the audio file" }, "segments": { "type": "array", "items": { "type": "object", "properties": { "start": { "type": "number", "description": "Seconds from the start of the audio." }, "end": { "type": "number" }, "text": { "type": "string", "description": "The passage, spacing normalized." }, "confidence": { "type": "number", "description": "How sure the model was, always `0..1` with higher meaning more confident \u2014 whatever the provider reported.\n\nNormalizing this is the whole point of the type. Providers disagree on both the scale and the direction: Deepgram reports a probability in `0..1`, ElevenLabs and Whisper report a log probability in `(-inf, 0]`. Handing those through untouched would make `confidence < 0.5` mean opposite things depending on which model the caller happened to pick. Log probabilities are converted with `Math.exp`; probabilities pass through." }, "speaker": { "type": "string", "description": "Present only when diarization was asked for and the provider supports it." }, "language": { "type": "string", "description": "Present only for providers that report language per passage rather than per request. Most report it once for the whole audio \u2014 see `TranscriptionResponse.language`." }, "words": { "type": "array", "items": { "type": "object", "properties": { "text": { "type": "string" }, "start": { "type": "number", "description": "Seconds from the start of the audio." }, "end": { "type": "number" }, "confidence": { "type": "number", "description": "See the confidence convention on `TranscriptSegment`." }, "speaker": { "type": "string", "description": "Provider-assigned speaker id, present only when diarization was asked for." } }, "required": ["text"], "description": "One recognized token.\n\n`text` rather than `word` because an entry is not always a word \u2014 providers emit punctuation, spacing and non-speech audio events (laughter, footsteps) through the same channel." }, "description": "Word-level detail, when the caller asked for it (`includeWordTimings`). Off by default: an hour of speech is roughly ten thousand words, an order of magnitude more payload than the segments that contain them." } }, "required": ["start", "end", "text"], "description": "A contiguous passage of one speaker's speech, bounded in time.\n\nThe unit a review UI works in: short enough to seek to and highlight, coarse enough that an hour of audio is a few hundred entries rather than ten thousand." }, "description": "Timed passages with per-segment confidence, speaker attribution and optional word-level detail \u2014 see the step's usage notes for which models populate it. Absent for models that return nothing but text." }, "language": { "type": "string", "description": "Detected or declared language of the audio, when the model reports it." }, "durationSeconds": { "type": "number", "description": "Length of the transcribed audio, when the model reports it." }, "subtitles": { "type": "string", "description": "The transcript as an SRT or WebVTT file, when `subtitles` was asked for in the model's config and the model returned segments to time it by." } }, "required": ["text"] }
|
|
1068
1068
|
},
|
|
@@ -9467,7 +9467,7 @@ async function startMcpServer(options) {
|
|
|
9467
9467
|
capabilities: { tools: {} },
|
|
9468
9468
|
serverInfo: {
|
|
9469
9469
|
name: "mindstudio-agent",
|
|
9470
|
-
version: "0.1.
|
|
9470
|
+
version: "0.1.168"
|
|
9471
9471
|
},
|
|
9472
9472
|
instructions: 'Welcome to MindStudio \u2014 a platform with 200+ AI models, 850+ third-party integrations, and pre-built agents.\n\nGetting started:\n1. Call `ask` with any question about the SDK \u2014 it knows every action, model, and connector and returns working code with real model IDs and config options. Examples: ask("generate an image with FLUX"), ask("what models support vision?"), ask("how do I send a Slack message?").\n2. Call `changeName` to set your display name \u2014 use your name or whatever your user calls you. This is how you\'ll appear in MindStudio request logs.\n3. If you have a profile picture, pass its public image URL to `changeProfilePicture`.\n4. For manual browsing, call `listActions` to discover all available actions.\n\nThen use the tools to generate text, images, video, audio, search the web, work with data sources, run agents, and more.\n\nImportant:\n- AI-powered actions (text generation, image generation, video, audio, etc.) cost money. Before running these, call `estimateActionCost` and confirm with the user before proceeding \u2014 unless they\'ve explicitly told you to go ahead.\n- Not all agents from `listAgents` are configured for API use. Do not try to run an agent just because it appears in the list \u2014 it will likely fail. Only run agents the user specifically asks you to run.'
|
|
9473
9473
|
});
|
|
@@ -10507,7 +10507,7 @@ function isNewerVersion(current, latest) {
|
|
|
10507
10507
|
return false;
|
|
10508
10508
|
}
|
|
10509
10509
|
async function checkForUpdate() {
|
|
10510
|
-
const currentVersion = "0.1.
|
|
10510
|
+
const currentVersion = "0.1.168";
|
|
10511
10511
|
if (!currentVersion || process.env.MINDSTUDIO_NO_UPDATE_CHECK) {
|
|
10512
10512
|
return null;
|
|
10513
10513
|
}
|
|
@@ -10542,7 +10542,7 @@ async function checkForUpdate() {
|
|
|
10542
10542
|
}
|
|
10543
10543
|
}
|
|
10544
10544
|
function printUpdateNotice(latestVersion) {
|
|
10545
|
-
const currentVersion = "0.1.
|
|
10545
|
+
const currentVersion = "0.1.168";
|
|
10546
10546
|
process.stderr.write(
|
|
10547
10547
|
`
|
|
10548
10548
|
${ansi2.cyanBright("Update available")} ${ansi2.gray(currentVersion + " \u2192")} ${ansi2.cyanBold(latestVersion)}
|
|
@@ -10555,7 +10555,7 @@ function isStandaloneBinary() {
|
|
|
10555
10555
|
return !argv1.includes("node_modules");
|
|
10556
10556
|
}
|
|
10557
10557
|
async function cmdUpdate() {
|
|
10558
|
-
const currentVersion = "0.1.
|
|
10558
|
+
const currentVersion = "0.1.168";
|
|
10559
10559
|
process.stderr.write(
|
|
10560
10560
|
` ${ansi2.gray("Current version:")} ${currentVersion}
|
|
10561
10561
|
`
|
|
@@ -10696,7 +10696,7 @@ async function cmdLogin(options) {
|
|
|
10696
10696
|
process.stderr.write("\n");
|
|
10697
10697
|
printLogo();
|
|
10698
10698
|
process.stderr.write("\n");
|
|
10699
|
-
const ver = "0.1.
|
|
10699
|
+
const ver = "0.1.168";
|
|
10700
10700
|
process.stderr.write(
|
|
10701
10701
|
` ${ansi2.bold("MindStudio Agent")} ${ver ? " " + ansi2.gray("v" + ver) : ""}
|
|
10702
10702
|
`
|
|
@@ -11024,7 +11024,7 @@ async function main() {
|
|
|
11024
11024
|
const updatePromise = command !== "mcp" && command !== "login" && command !== "update" ? checkForUpdate() : Promise.resolve(null);
|
|
11025
11025
|
try {
|
|
11026
11026
|
if (command === "version" || command === "-v") {
|
|
11027
|
-
process.stdout.write("0.1.
|
|
11027
|
+
process.stdout.write("0.1.168\n");
|
|
11028
11028
|
return;
|
|
11029
11029
|
}
|
|
11030
11030
|
if (command === "login") {
|
package/dist/index.d.ts
CHANGED
|
@@ -11717,7 +11717,7 @@ interface StepMethods {
|
|
|
11717
11717
|
* @remarks
|
|
11718
11718
|
* - The text field contains the exact words to be spoken (not instructions).
|
|
11719
11719
|
* - Voice, delivery and format are set through `speechModelOverride.config`, whose keys are the model's own (look a model up with its id). Delivery direction, where a model takes one, is never spoken: `styleInstruction` on gemini-3.8-flash-tts and gemini-3.8-flash-lite-tts, `instructions` on gpt-realtime-2.1-mini-tts, `voice_instruction` on seed-speech-tts-2.0, `emotion` on cartesia-sonic-3.6 and the MiniMax models.
|
|
11720
|
-
* - Tags in the text act at that point, in each vendor's own syntax: `[laughs]` or `[whispers]` on elevenlabs-tts with Eleven v4 or v3; `<laugh>`, `<sigh>` or `<whispers>` on the Gemini 3.8 models; `[laugh]`, `[pause]` or `<whisper>…</whisper>` on grok-voice-tts; `[laughter]` or `<emotion value="excited"/>` on cartesia-sonic-3.6; `[be happy and upbeat]` on inworld-tts-2.
|
|
11720
|
+
* - Tags in the text act at that point, in each vendor's own syntax: `[laughs]` or `[whispers]` on elevenlabs-tts with Eleven v4 or v3; `<laugh>`, `<sigh>` or `<whispers>` on the Gemini 3.8 models; `[laugh]`, `[pause]` or `<whisper>…</whisper>` on grok-voice-tts; `[laughter]` or `<emotion value="excited"/>` on cartesia-sonic-3.6; `[be happy and upbeat]` on inworld-tts-2; `[emotion=happy]` or `[pace=slow]` on voxtral-tts-3, until the next tag.
|
|
11721
11721
|
* - A dialogue between voices: set `$mode: 'dialogue'` and name each speaker (`speaker1_name`, `speaker1_voice`, `speaker2_name`, …), then write the text as a script, one `Name: line` per turn. Supported by elevenlabs-tts (Eleven v4 or v3, up to 4 speakers) and the Gemini 3.8 models (exactly 2).
|
|
11722
11722
|
* - A text longer than a model takes in one request is spoken in pieces and joined, and then comes back as MP3 whatever format was asked for.
|
|
11723
11723
|
* - In foreground mode, the audio is displayed to the user. In background mode, the URL is saved to a variable.
|
|
@@ -11739,16 +11739,16 @@ interface StepMethods {
|
|
|
11739
11739
|
* - Options go in `transcriptionModelOverride.config`, and support varies by model:
|
|
11740
11740
|
* - `language` — the spoken language as a code such as `en` or `es`; left out, the model detects it. Every model takes it. deepgram-nova-3 assumes `en` unless told, and also takes `multi` for speech that switches language (billed at its multilingual rate).
|
|
11741
11741
|
* - `detectLanguage: 'yes'` — detect the language instead of assuming English (deepgram-nova-3 only).
|
|
11742
|
-
* - `includeSpeakers: 'yes'` — label each passage with a speaker. Supported by elevenlabs-scribe-v2, elevenlabs-scribe-v2-medical, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe; gpt-4o-transcribe-diarize always does. Scribe and Voxtral also switch `text` to a timestamped, speaker-labelled block format; Deepgram, Gemini and gpt-4o-transcribe-diarize to `Speaker N:` lines.
|
|
11742
|
+
* - `includeSpeakers: 'yes'` — label each passage with a speaker. Supported by elevenlabs-scribe-v2, elevenlabs-scribe-v2-medical, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3 and gemini-3.5-transcribe; gpt-4o-transcribe-diarize always does. Scribe and Voxtral also switch `text` to a timestamped, speaker-labelled block format; Deepgram, Gemini and gpt-4o-transcribe-diarize to `Speaker N:` lines.
|
|
11743
11743
|
* - `numSpeakers` — the most people talking, to help tell them apart (Scribe).
|
|
11744
|
-
* - `keyterms` — a list of names, products and terms the audio contains, so they're spelled right. Supported by gpt-transcribe, Scribe, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Billed as an add-on on Scribe and Deepgram.
|
|
11744
|
+
* - `keyterms` — a list of names, products and terms the audio contains, so they're spelled right. Supported by gpt-transcribe, Scribe, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3 and gemini-3.5-transcribe. Billed as an add-on on Scribe and Deepgram.
|
|
11745
11745
|
* - `redactPii: 'yes'` — replace names, phone numbers, addresses and other personal information with a label of what was there (Scribe, deepgram-nova-3). Billed as an add-on.
|
|
11746
11746
|
* - `includeWordTimings: 'yes'` — add per-word timings and confidence inside each segment. Supported by Scribe, deepgram-nova-3, whisper-1, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Off by default because an hour of speech is roughly ten thousand words.
|
|
11747
11747
|
* - `subtitles: 'srt' | 'vtt'` — also return the transcript as a subtitle file, in the `subtitles` output, timed by the segments. Any model that returns segments.
|
|
11748
|
-
* - gemini-3.5-transcribe can't take `keyterms` together with speakers, word timings or subtitles
|
|
11749
|
-
* - The `segments` output carries timings, confidence and speakers as structured data, so there is no need to slice the audio up to recover them. It is populated by Scribe, deepgram-nova-3, grok-stt, whisper-1, gpt-4o-transcribe-diarize, voxtral-mini-transcribe-2
|
|
11748
|
+
* - gemini-3.5-transcribe can't take `keyterms` together with speakers, word timings or subtitles; asked for both, it's refused.
|
|
11749
|
+
* - The `segments` output carries timings, confidence and speakers as structured data, so there is no need to slice the audio up to recover them. It is populated by Scribe, deepgram-nova-3, grok-stt, whisper-1, gpt-4o-transcribe-diarize, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3, gemini-3.5-transcribe (with word timings or subtitles) and the DeepInfra-served whisper and qwen models, and absent for models that return nothing but text (gpt-transcribe, gpt-4o-transcribe, gpt-4o-mini-transcribe, the DeepInfra-served voxtral models).
|
|
11750
11750
|
* - In a workflow, `destinationVar` receives the transcript text. Name `segmentsVar` to also keep the `segments` array. A model that returns no segments leaves that variable unset.
|
|
11751
|
-
* - A recording of any length can be transcribed. One longer than a model takes at once (OpenAI's and DeepInfra's models, Gemini) is transcribed in pieces and joined, its timings running from the start of the recording. Speakers can't be followed from one piece to the next, so with speakers on, such a recording is refused; Scribe and Deepgram label speakers at any length.
|
|
11751
|
+
* - A recording of any length can be transcribed. One longer than a model takes at once (OpenAI's and DeepInfra's models, Gemini, Voxtral 3) is transcribed in pieces and joined, its timings running from the start of the recording. Speakers can't be followed from one piece to the next, so with speakers on, such a recording is refused; Scribe and Deepgram label speakers at any length.
|
|
11752
11752
|
* - `confidence` is always 0..1 with higher meaning more confident, whichever model produced it.
|
|
11753
11753
|
*
|
|
11754
11754
|
* @example
|
package/dist/index.js
CHANGED
|
@@ -5337,14 +5337,14 @@ var stepMetadata = {
|
|
|
5337
5337
|
"textToSpeech": {
|
|
5338
5338
|
stepType: "textToSpeech",
|
|
5339
5339
|
description: "Generate an audio file from provided text using a speech model.",
|
|
5340
|
-
usageNotes: "- The text field contains the exact words to be spoken (not instructions).\n- Voice, delivery and format are set through `speechModelOverride.config`, whose keys are the model's own (look a model up with its id). Delivery direction, where a model takes one, is never spoken: `styleInstruction` on gemini-3.8-flash-tts and gemini-3.8-flash-lite-tts, `instructions` on gpt-realtime-2.1-mini-tts, `voice_instruction` on seed-speech-tts-2.0, `emotion` on cartesia-sonic-3.6 and the MiniMax models.\n- Tags in the text act at that point, in each vendor's own syntax: `[laughs]` or `[whispers]` on elevenlabs-tts with Eleven v4 or v3; `<laugh>`, `<sigh>` or `<whispers>` on the Gemini 3.8 models; `[laugh]`, `[pause]` or `<whisper>\u2026</whisper>` on grok-voice-tts; `[laughter]` or `<emotion value=\"excited\"/>` on cartesia-sonic-3.6; `[be happy and upbeat]` on inworld-tts-2.\n- A dialogue between voices: set `$mode: 'dialogue'` and name each speaker (`speaker1_name`, `speaker1_voice`, `speaker2_name`, \u2026), then write the text as a script, one `Name: line` per turn. Supported by elevenlabs-tts (Eleven v4 or v3, up to 4 speakers) and the Gemini 3.8 models (exactly 2).\n- A text longer than a model takes in one request is spoken in pieces and joined, and then comes back as MP3 whatever format was asked for.\n- In foreground mode, the audio is displayed to the user. In background mode, the URL is saved to a variable.",
|
|
5340
|
+
usageNotes: "- The text field contains the exact words to be spoken (not instructions).\n- Voice, delivery and format are set through `speechModelOverride.config`, whose keys are the model's own (look a model up with its id). Delivery direction, where a model takes one, is never spoken: `styleInstruction` on gemini-3.8-flash-tts and gemini-3.8-flash-lite-tts, `instructions` on gpt-realtime-2.1-mini-tts, `voice_instruction` on seed-speech-tts-2.0, `emotion` on cartesia-sonic-3.6 and the MiniMax models.\n- Tags in the text act at that point, in each vendor's own syntax: `[laughs]` or `[whispers]` on elevenlabs-tts with Eleven v4 or v3; `<laugh>`, `<sigh>` or `<whispers>` on the Gemini 3.8 models; `[laugh]`, `[pause]` or `<whisper>\u2026</whisper>` on grok-voice-tts; `[laughter]` or `<emotion value=\"excited\"/>` on cartesia-sonic-3.6; `[be happy and upbeat]` on inworld-tts-2; `[emotion=happy]` or `[pace=slow]` on voxtral-tts-3, until the next tag.\n- A dialogue between voices: set `$mode: 'dialogue'` and name each speaker (`speaker1_name`, `speaker1_voice`, `speaker2_name`, \u2026), then write the text as a script, one `Name: line` per turn. Supported by elevenlabs-tts (Eleven v4 or v3, up to 4 speakers) and the Gemini 3.8 models (exactly 2).\n- A text longer than a model takes in one request is spoken in pieces and joined, and then comes back as MP3 whatever format was asked for.\n- In foreground mode, the audio is displayed to the user. In background mode, the URL is saved to a variable.",
|
|
5341
5341
|
inputSchema: { "type": "object", "properties": { "text": { "type": "string", "description": "The text to convert to speech" }, "intermediateAsset": { "type": "boolean" }, "speechModelOverride": { "type": "object", "properties": { "model": { "type": "string", "description": "Speech synthesis model identifier" }, "config": { "type": "object", "properties": {}, "required": [], "description": "Additional model-specific configuration" } }, "required": ["model"], "description": "Optional model configuration override. Uses the workflow's default speech model if not specified" } }, "required": ["text"] },
|
|
5342
5342
|
outputSchema: { "type": "object", "properties": { "audioUrl": { "type": "string", "description": "URL of the generated audio file" } }, "required": ["audioUrl"] }
|
|
5343
5343
|
},
|
|
5344
5344
|
"transcribeAudio": {
|
|
5345
5345
|
stepType: "transcribeAudio",
|
|
5346
5346
|
description: "Convert an audio file to text using a transcription model.",
|
|
5347
|
-
usageNotes: "- The default model is elevenlabs-scribe-v2: speakers, word timings, key terms and redaction, at any length.\n- The prompt field gives context (speaker names, domain) to OpenAI's and DeepInfra's models. Other models ignore it; give them `keyterms` instead.\n- Options go in `transcriptionModelOverride.config`, and support varies by model:\n - `language` \u2014 the spoken language as a code such as `en` or `es`; left out, the model detects it. Every model takes it. deepgram-nova-3 assumes `en` unless told, and also takes `multi` for speech that switches language (billed at its multilingual rate).\n - `detectLanguage: 'yes'` \u2014 detect the language instead of assuming English (deepgram-nova-3 only).\n - `includeSpeakers: 'yes'` \u2014 label each passage with a speaker. Supported by elevenlabs-scribe-v2, elevenlabs-scribe-v2-medical, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe; gpt-4o-transcribe-diarize always does. Scribe and Voxtral also switch `text` to a timestamped, speaker-labelled block format; Deepgram, Gemini and gpt-4o-transcribe-diarize to `Speaker N:` lines.\n - `numSpeakers` \u2014 the most people talking, to help tell them apart (Scribe).\n - `keyterms` \u2014 a list of names, products and terms the audio contains, so they're spelled right. Supported by gpt-transcribe, Scribe, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Billed as an add-on on Scribe and Deepgram.\n - `redactPii: 'yes'` \u2014 replace names, phone numbers, addresses and other personal information with a label of what was there (Scribe, deepgram-nova-3). Billed as an add-on.\n - `includeWordTimings: 'yes'` \u2014 add per-word timings and confidence inside each segment. Supported by Scribe, deepgram-nova-3, whisper-1, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Off by default because an hour of speech is roughly ten thousand words.\n - `subtitles: 'srt' | 'vtt'` \u2014 also return the transcript as a subtitle file, in the `subtitles` output, timed by the segments. Any model that returns segments.\n- gemini-3.5-transcribe can't take `keyterms` together with speakers, word timings or subtitles
|
|
5347
|
+
usageNotes: "- The default model is elevenlabs-scribe-v2: speakers, word timings, key terms and redaction, at any length.\n- The prompt field gives context (speaker names, domain) to OpenAI's and DeepInfra's models. Other models ignore it; give them `keyterms` instead.\n- Options go in `transcriptionModelOverride.config`, and support varies by model:\n - `language` \u2014 the spoken language as a code such as `en` or `es`; left out, the model detects it. Every model takes it. deepgram-nova-3 assumes `en` unless told, and also takes `multi` for speech that switches language (billed at its multilingual rate).\n - `detectLanguage: 'yes'` \u2014 detect the language instead of assuming English (deepgram-nova-3 only).\n - `includeSpeakers: 'yes'` \u2014 label each passage with a speaker. Supported by elevenlabs-scribe-v2, elevenlabs-scribe-v2-medical, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3 and gemini-3.5-transcribe; gpt-4o-transcribe-diarize always does. Scribe and Voxtral also switch `text` to a timestamped, speaker-labelled block format; Deepgram, Gemini and gpt-4o-transcribe-diarize to `Speaker N:` lines.\n - `numSpeakers` \u2014 the most people talking, to help tell them apart (Scribe).\n - `keyterms` \u2014 a list of names, products and terms the audio contains, so they're spelled right. Supported by gpt-transcribe, Scribe, deepgram-nova-3, grok-stt, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3 and gemini-3.5-transcribe. Billed as an add-on on Scribe and Deepgram.\n - `redactPii: 'yes'` \u2014 replace names, phone numbers, addresses and other personal information with a label of what was there (Scribe, deepgram-nova-3). Billed as an add-on.\n - `includeWordTimings: 'yes'` \u2014 add per-word timings and confidence inside each segment. Supported by Scribe, deepgram-nova-3, whisper-1, grok-stt, voxtral-mini-transcribe-2 and gemini-3.5-transcribe. Off by default because an hour of speech is roughly ten thousand words.\n - `subtitles: 'srt' | 'vtt'` \u2014 also return the transcript as a subtitle file, in the `subtitles` output, timed by the segments. Any model that returns segments.\n- gemini-3.5-transcribe can't take `keyterms` together with speakers, word timings or subtitles; asked for both, it's refused.\n- The `segments` output carries timings, confidence and speakers as structured data, so there is no need to slice the audio up to recover them. It is populated by Scribe, deepgram-nova-3, grok-stt, whisper-1, gpt-4o-transcribe-diarize, voxtral-mini-transcribe-2, voxtral-mini-transcribe-3, voxtral-small-transcribe-3, gemini-3.5-transcribe (with word timings or subtitles) and the DeepInfra-served whisper and qwen models, and absent for models that return nothing but text (gpt-transcribe, gpt-4o-transcribe, gpt-4o-mini-transcribe, the DeepInfra-served voxtral models).\n- In a workflow, `destinationVar` receives the transcript text. Name `segmentsVar` to also keep the `segments` array. A model that returns no segments leaves that variable unset.\n- A recording of any length can be transcribed. One longer than a model takes at once (OpenAI's and DeepInfra's models, Gemini, Voxtral 3) is transcribed in pieces and joined, its timings running from the start of the recording. Speakers can't be followed from one piece to the next, so with speakers on, such a recording is refused; Scribe and Deepgram label speakers at any length.\n- `confidence` is always 0..1 with higher meaning more confident, whichever model produced it.",
|
|
5348
5348
|
inputSchema: { "type": "object", "properties": { "audioUrl": { "type": "string", "description": "URL of the audio file to transcribe" }, "prompt": { "type": "string", "description": "Optional context to improve transcription accuracy (e.g. language, speaker names, domain terms)" }, "transcriptionModelOverride": { "type": "object", "properties": { "model": { "type": "string", "description": "Audio transcription model identifier" }, "config": { "type": "object", "properties": {}, "required": [], "description": "Additional model-specific configuration" } }, "required": ["model"], "description": "Optional model configuration override. Uses the workflow's default transcription model if not specified" } }, "required": ["audioUrl", "prompt"] },
|
|
5349
5349
|
outputSchema: { "type": "object", "properties": { "text": { "type": "string", "description": "The transcribed text from the audio file" }, "segments": { "type": "array", "items": { "type": "object", "properties": { "start": { "type": "number", "description": "Seconds from the start of the audio." }, "end": { "type": "number" }, "text": { "type": "string", "description": "The passage, spacing normalized." }, "confidence": { "type": "number", "description": "How sure the model was, always `0..1` with higher meaning more confident \u2014 whatever the provider reported.\n\nNormalizing this is the whole point of the type. Providers disagree on both the scale and the direction: Deepgram reports a probability in `0..1`, ElevenLabs and Whisper report a log probability in `(-inf, 0]`. Handing those through untouched would make `confidence < 0.5` mean opposite things depending on which model the caller happened to pick. Log probabilities are converted with `Math.exp`; probabilities pass through." }, "speaker": { "type": "string", "description": "Present only when diarization was asked for and the provider supports it." }, "language": { "type": "string", "description": "Present only for providers that report language per passage rather than per request. Most report it once for the whole audio \u2014 see `TranscriptionResponse.language`." }, "words": { "type": "array", "items": { "type": "object", "properties": { "text": { "type": "string" }, "start": { "type": "number", "description": "Seconds from the start of the audio." }, "end": { "type": "number" }, "confidence": { "type": "number", "description": "See the confidence convention on `TranscriptSegment`." }, "speaker": { "type": "string", "description": "Provider-assigned speaker id, present only when diarization was asked for." } }, "required": ["text"], "description": "One recognized token.\n\n`text` rather than `word` because an entry is not always a word \u2014 providers emit punctuation, spacing and non-speech audio events (laughter, footsteps) through the same channel." }, "description": "Word-level detail, when the caller asked for it (`includeWordTimings`). Off by default: an hour of speech is roughly ten thousand words, an order of magnitude more payload than the segments that contain them." } }, "required": ["start", "end", "text"], "description": "A contiguous passage of one speaker's speech, bounded in time.\n\nThe unit a review UI works in: short enough to seek to and highlight, coarse enough that an hour of audio is a few hundred entries rather than ten thousand." }, "description": "Timed passages with per-segment confidence, speaker attribution and optional word-level detail \u2014 see the step's usage notes for which models populate it. Absent for models that return nothing but text." }, "language": { "type": "string", "description": "Detected or declared language of the audio, when the model reports it." }, "durationSeconds": { "type": "number", "description": "Length of the transcribed audio, when the model reports it." }, "subtitles": { "type": "string", "description": "The transcript as an SRT or WebVTT file, when `subtitles` was asked for in the model's config and the model returned segments to time it by." } }, "required": ["text"] }
|
|
5350
5350
|
},
|