@jambonz/schema 0.4.24 → 0.4.26

package/components/recognizer-mistralOptions.schema.json

@@ -0,0 +1,86 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://jambonz.org/schema/components/recognizer-mistralOptions",
+  "title": "Mistral Recognizer Options",
+  "description": "Mistral Voxtral realtime STT options. Only applies when recognizer vendor is 'mistral'. Voxtral streams text as it is recognized (sent as interim results when interim is enabled); with turn detection on (the default) a final transcript is produced when Voxtral detects the end of the speaker's turn, and in the agent verb each final ends the user's turn. Transcripts are reported with confidence 1.0. The language is sent as a two-letter code (en-US becomes en); leave it unset or 'any' for automatic detection.",
+  "type": "object",
+  "properties": {
+    "apiKey": {
+      "type": "string",
+      "description": "Mistral API key. Overrides the key configured in jambonz."
+    },
+    "model": {
+      "type": "string",
+      "description": "Realtime transcription model. Defaults to voxtral-mini-transcribe-realtime-3.",
+      "examples": ["voxtral-mini-transcribe-realtime-3"]
+    },
+    "streamingDelayMs": {
+      "type": "integer",
+      "description": "Target streaming delay in ms: how far the model looks ahead before emitting text. Lower is faster, higher is more accurate. Any value from 80 to 1200, or 2400. Voxtral's default is 480.",
+      "anyOf": [
+        { "minimum": 80, "maximum": 1200 },
+        { "const": 2400 }
+      ]
+    },
+    "verbatim": {
+      "type": "boolean",
+      "description": "Ask Voxtral for a verbatim transcript (sent as verbatim). Not in Mistral's docs yet; the realtime API accepts it."
+    },
+    "contextBias": {
+      "type": "array",
+      "items": { "type": "string" },
+      "maxItems": 100,
+      "description": "Words or phrases to favor (names, brands, jargon). When omitted, recognizer.hints are used."
+    },
+    "diarize": {
+      "type": "boolean",
+      "description": "Label each transcript with a speaker. Voxtral cannot diarize and detect turns at once, so this turns off turn detection: a final transcript closes at each sentence end or change of speaker. recognizer.diarization has the same effect. Ignored in the agent verb, which needs turn detection."
+    },
+    "speakerReferences": {
+      "type": "array",
+      "maxItems": 8,
+      "description": "Reference clips (3 to 15 seconds each) that let diarization name known speakers. Passed to Mistral as speaker_references. Requires diarize.",
+      "items": { "type": "object" }
+    },
+    "turnDetection": {
+      "type": "object",
+      "description": "Tunes when Voxtral ends a turn. Ignored when diarize is on. Thresholds must satisfy startThreshold >= eagerEndThreshold > endThreshold.",
+      "properties": {
+        "startThreshold": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Speech probability that starts a turn. Default 0.5."
+        },
+        "eagerEndThreshold": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Probability at which Voxtral signals a likely end of turn early; the agent verb uses it for early generation. Default 0.5."
+        },
+        "endThreshold": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Probability below which silence counts toward ending the turn. Default 0.33."
+        },
+        "minSilenceMs": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Silence in ms before a turn can end. Default 500; raised to streamingDelayMs when that is longer, since Voxtral's text lags the audio by about that much."
+        },
+        "endTimeoutMs": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Silence in ms after which the turn ends regardless of the model. Default 3000; must be at least minSilenceMs."
+        }
+      },
+      "additionalProperties": false
+    },
+    "mistralSttUri": {
+      "type": "string",
+      "description": "Custom host to connect to instead of api.mistral.ai, e.g. a proxy. A ws:// or http:// prefix disables TLS."
+    }
+  },
+  "additionalProperties": false
+}

package/components/recognizer.schema.json

@@ -8,7 +8,7 @@
     "vendor": {
       "type": "string",
       "description": "The STT vendor to use. Must match a vendor configured in the jambonz platform.",
-      "examples": ["google", "aws", "microsoft", "deepgram", "ibm", "nvidia", "soniox", "xai", "modulate", "zoom", "gradium", "assemblyai", "speechmatics", "openai", "houndify", "gladia", "elevenlabs", "custom"]
+      "examples": ["google", "aws", "microsoft", "deepgram", "ibm", "nvidia", "soniox", "xai", "modulate", "zoom", "gradium", "mistral", "assemblyai", "speechmatics", "openai", "houndify", "gladia", "elevenlabs", "custom"]
     },
     "label": {
       "type": "string",
@@ -202,6 +202,7 @@
     "modulateOptions": { "$ref": "recognizer-modulateOptions" },
     "zoomOptions": { "$ref": "recognizer-zoomOptions" },
     "gradiumOptions": { "$ref": "recognizer-gradiumOptions" },
+    "mistralOptions": { "$ref": "recognizer-mistralOptions" },
     "assemblyAiOptions": { "$ref": "recognizer-assemblyAiOptions" },
     "speechmaticsOptions": { "$ref": "recognizer-speechmaticsOptions" },
     "openaiOptions": { "$ref": "recognizer-openaiOptions" },

package/components/synthesizer.schema.json

@@ -8,7 +8,7 @@
     "vendor": {
       "type": "string",
       "description": "The TTS vendor to use. Must match a vendor configured in the jambonz platform.",
-      "examples": ["google", "aws", "microsoft", "elevenlabs", "cartesia", "fishaudio", "gradium", "kugelaudio", "nineninesix", "speechify", "deepgram", "deepgramflux", "ibm", "nvidia", "wellsaid", "whisper", "custom"]
+      "examples": ["google", "aws", "microsoft", "elevenlabs", "cartesia", "fishaudio", "gradium", "kugelaudio", "mistral", "nineninesix", "speechify", "deepgram", "deepgramflux", "ibm", "nvidia", "wellsaid", "whisper", "custom"]
     },
     "label": {
       "type": "string",

package/docs/verbs/agent.md

@@ -331,6 +331,35 @@ For Anthropic models, use `"vendor": "anthropic"` and structure messages accordi
 
 By default (`greeting: true`), the agent prompts the LLM to generate an initial greeting before the user speaks. Set `greeting: false` if you want the agent to wait silently for the user to speak first.
 
+## Controlling what is spoken
+
+Everything the LLM writes is normally sent to TTS. Two options change that. Neither changes what the LLM sees: conversation history, `llm_response` events and the `turn_end` transcript always hold the model's raw text.
+
+**`speechCleanup`** (default `true`) drops text that models write but don't mean to be heard:
+
+- a paragraph that is only a stage direction in `()`, `[]` or `**`, such as `(silence)`, `(empty response)` or `[No response needed - the call has ended]`. Models write these when they are told to stay silent. A bracketed aside inside a sentence, such as "PPO (including Anthem)", is still spoken, and so is a span with no letters in it, such as a phone number's area code.
+- `<think>`, `<thinking>` and `<reasoning>` blocks, wherever they appear.
+
+The cleanup runs on the streaming tokens and holds back only text that could still turn out to be one of these, so ordinary speech is not delayed. If nothing is left to say, the reply ends without speaking, the same way as a reply that only called tools. Set `speechCleanup: false` to send the text to TTS exactly as written.
+
+**`speak: false`** keeps all of the agent's text out of TTS. Use it for an agent that must never talk, such as one that navigates a DTMF-only phone menu by calling tools. The LLM's replies are still kept, so the agent can take notes for itself. With `speak: false`, `toolFiller` and `earlyGeneration` are turned off.
+
+```json
+{
+  "verb": "agent",
+  "speak": false,
+  "llm": {
+    "vendor": "anthropic",
+    "model": "claude-haiku-4-5",
+    "llmOptions": {
+      "messages": [
+        { "role": "system", "content": "You navigate phone menus by pressing keys with the send_dtmf tool." }
+      ]
+    }
+  }
+}
+```
+
 ## Complete example (TypeScript)
 
 A voice agent using Deepgram STT, OpenAI LLM, and Cartesia TTS with Krisp turn detection. Exposes multiple endpoints with different STT/TTS combinations:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/schema",
-  "version": "0.4.24",
+  "version": "0.4.26",
   "description": "JSON Schema definitions and validation for jambonz verb applications",
   "main": "index.js",
   "scripts": {

package/verbs/agent.schema.json

@@ -26,12 +26,12 @@
         { "type": "boolean" },
         { "type": "string", "enum": ["always"] }
       ],
-      "description": "When using Deepgram Flux Multilingual, automatically adjust STT language hints and switch TTS voice based on detected language. Values: false (disabled), true (lock on first utterance), 'always' (continuously adapt on every turn). Default: false.",
+      "description": "Switch the TTS to the language the caller speaks, using the matching languageConfig entry. Works with any STT that reports the detected language: Deepgram (language 'multi', or Flux Multilingual), AssemblyAI (languageDetection), Google and Azure (altLanguages), Soniox (enableLanguageIdentification), Gladia and Modulate. Values: false (disabled), true (lock on the first utterance whose language is detected), 'always' (re-evaluate every turn). With Deepgram Flux, true also locks the STT language hints to the detected language. Default: false.",
       "default": false
     },
     "languageConfig": {
       "type": "object",
-      "description": "Per-language overrides for TTS. Keys are BCP-47 language codes. When autoLockLanguage detects a language switch, the agent uses the corresponding config.",
+      "description": "Per-language TTS overrides, used when autoLockLanguage detects a language. Keys are language codes such as 'es' or 'es-MX'. A detected language matches an exact key first (case-insensitive), then a key with the same primary language, so 'es' matches a detected 'es-US'. A detected language with no entry keeps the current TTS. The entry's tts is merged over the agent's tts; set its language to the code the TTS vendor expects.",
       "additionalProperties": {
         "type": "object",
         "properties": {
@@ -231,6 +231,16 @@
       "description": "Enable speculative LLM prompting before end-of-turn is confirmed. When using Krisp turn detection, set this to true to speculatively prompt the LLM before Krisp confirms the turn has ended. If the transcript matches when turn ends, buffered tokens are released immediately — reducing response latency. Note: Deepgram Flux performs early generation automatically via its native EagerEndOfTurn signal regardless of this setting. Default: false.",
       "default": false
     },
+    "speak": {
+      "type": "boolean",
+      "description": "Whether the agent's text is spoken. With false the agent never talks (for example while it drives a DTMF-only phone menu with its tools): the LLM's text is still kept in conversation history, llm_response events and the transcript, but is never sent to TTS, and toolFiller and earlyGeneration are turned off. Each reply ends when the LLM finishes, as for a reply that only called tools. Default: true.",
+      "default": true
+    },
+    "speechCleanup": {
+      "type": "boolean",
+      "description": "Keep text the model was not meant to say out of TTS: a paragraph that is only a stage direction in (), [] or ** (such as '(silence)' or '[No response needed]'), and <think>, <thinking> or <reasoning> blocks. Asides inside a sentence are still spoken. Applied as the reply streams; ordinary speech is not delayed. A reply with nothing left to say ends without speaking. History, llm_response events and the transcript keep the model's raw text. Default: true.",
+      "default": true
+    },
     "noiseIsolation": {
       "oneOf": [
         {