@absolutejs/voice 0.0.22-beta.632 → 0.0.22-beta.634

package/dist/manifest.json

@@ -0,0 +1,385 @@
+{
+	"contract": 1,
+	"identity": {
+		"accent": "#f43f5e",
+		"category": "voice",
+		"description": "Self-hosted voice assistant runtime for Elysia: browser and phone-call sessions over WebSockets, pluggable speech-to-text (`voice/stt`), text-to-speech (`voice/tts`), and speech-to-speech (`voice/realtime`) vendor adapters, turn detection, barge-in, greetings, reconnect recovery, session stores (`voice/session-store`), traces, evals, and live-monitor surfaces — the orchestration hosted voice platforms keep behind their dashboards, inside your own server.",
+		"docsUrl": "https://github.com/absolutejs/voice",
+		"name": "@absolutejs/voice",
+		"tagline": "Let people talk to your site and get spoken answers."
+	},
+	"implements": [
+		{
+			"contract": "voice/session-store",
+			"factory": "createVoiceMemoryStore",
+			"from": "@absolutejs/voice",
+			"title": "In memory (development only — calls are forgotten on restart)",
+			"wiring": {
+				"code": "createVoiceMemoryStore()",
+				"imports": [
+					{
+						"from": "@absolutejs/voice",
+						"names": [
+							"createVoiceMemoryStore"
+						]
+					}
+				]
+			}
+		},
+		{
+			"contract": "voice/session-store",
+			"factory": "createVoiceSQLiteSessionStore",
+			"from": "@absolutejs/voice",
+			"settings": {
+				"type": "object",
+				"required": [
+					"path"
+				],
+				"properties": {
+					"path": {
+						"description": "Database file on this machine where call sessions are kept. Created if missing.",
+						"examples": [
+							"./var/voice-sessions.sqlite"
+						],
+						"title": "Database file",
+						"type": "string"
+					},
+					"tableName": {
+						"description": "Table the sessions are stored in.",
+						"title": "Table name",
+						"type": "string"
+					}
+				}
+			},
+			"title": "SQLite file on this machine",
+			"wiring": {
+				"code": "createVoiceSQLiteSessionStore(${settings})",
+				"imports": [
+					{
+						"from": "@absolutejs/voice",
+						"names": [
+							"createVoiceSQLiteSessionStore"
+						]
+					}
+				]
+			}
+		},
+		{
+			"contract": "voice/session-store",
+			"factory": "createVoicePostgresSessionStore",
+			"from": "@absolutejs/voice",
+			"requires": {
+				"env": [
+					{
+						"description": "Postgres connection string (call sessions are stored here)",
+						"example": "postgres://user:pass@host/db",
+						"key": "DATABASE_URL",
+						"secret": true
+					}
+				],
+				"services": [
+					{
+						"description": "Stores call sessions durably",
+						"id": "postgres"
+					}
+				]
+			},
+			"settings": {
+				"type": "object",
+				"properties": {
+					"schemaName": {
+						"description": "Postgres schema the table lives in.",
+						"title": "Schema name",
+						"type": "string"
+					},
+					"tableName": {
+						"description": "Table the sessions are stored in.",
+						"title": "Table name",
+						"type": "string"
+					}
+				}
+			},
+			"title": "Postgres",
+			"wiring": {
+				"code": "createVoicePostgresSessionStore({ connectionString: ${env.DATABASE_URL} ?? \"\", ...${settings} })",
+				"imports": [
+					{
+						"from": "@absolutejs/voice",
+						"names": [
+							"createVoicePostgresSessionStore"
+						]
+					}
+				]
+			}
+		},
+		{
+			"contract": "voice/session-store",
+			"factory": "createVoiceFileSessionStore",
+			"from": "@absolutejs/voice",
+			"settings": {
+				"type": "object",
+				"required": [
+					"directory"
+				],
+				"properties": {
+					"directory": {
+						"description": "Folder on this machine where each call session is kept as a JSON file. Created if missing.",
+						"examples": [
+							"./var/voice-sessions"
+						],
+						"title": "Storage folder",
+						"type": "string"
+					}
+				}
+			},
+			"title": "JSON files on this machine",
+			"wiring": {
+				"code": "createVoiceFileSessionStore(${settings})",
+				"imports": [
+					{
+						"from": "@absolutejs/voice",
+						"names": [
+							"createVoiceFileSessionStore"
+						]
+					}
+				]
+			}
+		}
+	],
+	"requires": {
+		"peers": [
+			{
+				"name": "elysia",
+				"range": ">= 1.4.18",
+				"reason": "plugin host"
+			}
+		]
+	},
+	"settings": {
+		"type": "object",
+		"properties": {
+			"backchannelBargeInGuard": {
+				"description": "Keep the assistant talking through listening cues like \"mm-hm\" or \"right\" instead of treating them as interruptions.",
+				"title": "Ignore listening cues",
+				"type": "boolean"
+			},
+			"greeting": {
+				"description": "Spoken by the assistant the moment a call connects, before the caller says anything.",
+				"examples": [
+					"Hi! How can I help you today?"
+				],
+				"title": "Opening greeting",
+				"type": "string"
+			},
+			"normalizeNumbers": {
+				"description": "Rewrite spoken numbers to digits in transcripts (\"ten million\" becomes \"10 million\") so downstream systems read them consistently.",
+				"title": "Write numbers as digits",
+				"type": "boolean"
+			},
+			"path": {
+				"default": "/voice/realtime",
+				"description": "The WebSocket route callers connect to.",
+				"title": "Call route",
+				"type": "string"
+			},
+			"preset": {
+				"description": "Tuned bundle of turn-taking and latency defaults for the kind of calls you take — casual chat, guided intake, dictation, noisy rooms, or phone lines.",
+				"title": "Conversation preset",
+				"anyOf": [
+					{
+						"const": "default",
+						"type": "string"
+					},
+					{
+						"const": "chat",
+						"type": "string"
+					},
+					{
+						"const": "guided-intake",
+						"type": "string"
+					},
+					{
+						"const": "dictation",
+						"type": "string"
+					},
+					{
+						"const": "noisy-room",
+						"type": "string"
+					},
+					{
+						"const": "pstn-balanced",
+						"type": "string"
+					},
+					{
+						"const": "pstn-fast",
+						"type": "string"
+					},
+					{
+						"const": "reliability",
+						"type": "string"
+					}
+				]
+			},
+			"resumeGreeting": {
+				"description": "Spoken when a dropped call reconnects, so the caller knows they are back.",
+				"examples": [
+					"Sorry, we got cut off — where were we?"
+				],
+				"title": "Reconnect greeting",
+				"type": "string"
+			}
+		}
+	},
+	"slots": {
+		"realtime": {
+			"configPath": "realtime",
+			"contract": "voice/realtime",
+			"description": "One speech-to-speech model that listens and talks in a single stream — an alternative to picking separate speech-to-text and text-to-speech services.",
+			"known": [
+				"@absolutejs/voice-openai",
+				"@absolutejs/voice-gemini"
+			]
+		},
+		"session": {
+			"configPath": "session",
+			"contract": "voice/session-store",
+			"description": "Where each call's transcript and state is kept while it runs (and after, for review)",
+			"known": [
+				"@absolutejs/voice#memory",
+				"@absolutejs/voice#sqlite",
+				"@absolutejs/voice#postgres",
+				"@absolutejs/voice#file"
+			],
+			"required": true
+		},
+		"stt": {
+			"configPath": "stt",
+			"contract": "voice/stt",
+			"description": "The service that turns the caller's speech into text. Not needed when a speech-to-speech (realtime) adapter is chosen instead.",
+			"known": [
+				"@absolutejs/voice-deepgram",
+				"@absolutejs/voice-assemblyai",
+				"@absolutejs/voice-gladia",
+				"@absolutejs/voice-soniox",
+				"@absolutejs/voice-speechmatics",
+				"@absolutejs/voice-azure",
+				"@absolutejs/voice-google-speech",
+				"@absolutejs/voice-openai-whisper"
+			]
+		},
+		"tts": {
+			"configPath": "tts",
+			"contract": "voice/tts",
+			"description": "The service that speaks the assistant's replies out loud. Not needed when a speech-to-speech (realtime) adapter is chosen instead.",
+			"known": [
+				"@absolutejs/voice-elevenlabs",
+				"@absolutejs/voice-cartesia",
+				"@absolutejs/voice-azure",
+				"@absolutejs/voice-lmnt",
+				"@absolutejs/voice-playht",
+				"@absolutejs/voice-rime",
+				"@absolutejs/voice-smallest",
+				"@absolutejs/voice-neets"
+			]
+		}
+	},
+	"wiring": [
+		{
+			"description": "Pick a speech-to-text and a text-to-speech service, choose where call sessions are stored, and write the onTurn handler — the assistant's brain.",
+			"id": "default",
+			"server": {
+				"code": "// Every call's transcript and state lives in the session store.\nconst voiceSessions = ${slot.session};\n\n// Mount with .use(voiceAssistant); callers connect over WebSocket at `path`.\nconst voiceAssistant = voice({\n\tonTurn: async (session, turn) => {\n\t\t// TODO: generate the assistant's reply (call your LLM here).\n\t\treturn { assistantText: \"You said: \" + turn.text };\n\t},\n\tsession: voiceSessions,\n\tstt: ${slot.stt},\n\ttts: ${slot.tts},\n\t...${settings}\n});",
+				"imports": [
+					{
+						"from": "@absolutejs/voice",
+						"names": [
+							"voice"
+						]
+					}
+				],
+				"placement": "module-scope"
+			},
+			"title": "Voice assistant with separate STT + TTS"
+		},
+		{
+			"description": "Use one speech-to-speech model (OpenAI Realtime, Gemini Live) that listens and talks in a single stream instead of separate STT and TTS services.",
+			"id": "realtime",
+			"server": {
+				"code": "// Every call's transcript and state lives in the session store.\nconst voiceSessions = ${slot.session};\n\n// Mount with .use(voiceAssistant); callers connect over WebSocket at `path`.\nconst voiceAssistant = voice({\n\tonTurn: async (session, turn) => {\n\t\t// TODO: generate the assistant's reply (call your LLM here).\n\t\treturn { assistantText: \"You said: \" + turn.text };\n\t},\n\trealtime: ${slot.realtime},\n\tsession: voiceSessions,\n\t...${settings}\n});",
+				"imports": [
+					{
+						"from": "@absolutejs/voice",
+						"names": [
+							"voice"
+						]
+					}
+				],
+				"placement": "module-scope"
+			},
+			"title": "Voice assistant with a speech-to-speech model"
+		}
+	],
+	"tools": {
+		"delete_session": {
+			"annotations": {
+				"destructiveHint": true,
+				"idempotentHint": true
+			},
+			"description": "Delete one call session (its transcript and state) by id. Deleting an unknown id succeeds.",
+			"input": {
+				"type": "object",
+				"required": [
+					"id"
+				],
+				"properties": {
+					"id": {
+						"minLength": 1,
+						"type": "string"
+					}
+				}
+			},
+			"kind": "runtime"
+		},
+		"get_session": {
+			"annotations": {
+				"readOnlyHint": true
+			},
+			"description": "Fetch one call session by id: status, timing, and the committed conversation turns (caller text and assistant reply), most recent last.",
+			"input": {
+				"type": "object",
+				"required": [
+					"id"
+				],
+				"properties": {
+					"id": {
+						"minLength": 1,
+						"type": "string"
+					},
+					"maxTurns": {
+						"maximum": 200,
+						"minimum": 1,
+						"type": "integer"
+					}
+				}
+			},
+			"kind": "runtime"
+		},
+		"list_sessions": {
+			"annotations": {
+				"readOnlyHint": true
+			},
+			"description": "List call sessions with status, creation time, last activity, and turn count — most recently active first.",
+			"input": {
+				"type": "object",
+				"properties": {
+					"limit": {
+						"maximum": 200,
+						"minimum": 1,
+						"type": "integer"
+					}
+				}
+			},
+			"kind": "runtime"
+		}
+	}
+}

package/dist/testing/index.js

@@ -6408,7 +6408,10 @@ var createEmptyCurrentTurn = () => ({
   silenceStartedAt: undefined,
   transcripts: []
 });
-var cloneTranscript = (transcript) => ({ ...transcript });
+var cloneTranscript = (transcript) => ({
+  ...transcript,
+  words: transcript.words?.map((word) => ({ ...word }))
+});
 var encodeBase64 = (chunk) => Buffer2.from(chunk).toString("base64");
 var countWords2 = (text) => text.trim().split(/\s+/).filter(Boolean).length;
 var normalizeText2 = (text) => text.trim().replace(/\s+/g, " ");
@@ -6453,9 +6456,12 @@ var calculateMeanConfidence = (transcripts) => {
   }
   return sum / total;
 };
+var collectTranscriptWords = (transcripts) => transcripts.flatMap((transcript) => transcript.words ?? []);
+var collectWordConfidences = (transcripts) => collectTranscriptWords(transcripts).map((word) => word.confidence).filter((confidence) => typeof confidence === "number" && Number.isFinite(confidence));
 var createTurnQuality = (transcripts, source, fallbackUsed, fallbackDiagnostics, correctionDiagnostics, costEstimate) => {
   const sampledTranscripts = transcripts.filter((transcript) => typeof transcript.confidence === "number");
   const confidenceSampleCount = sampledTranscripts.length;
+  const wordConfidences = collectWordConfidences(transcripts);
   return {
     averageConfidence: confidenceSampleCount > 0 ? sampledTranscripts.reduce((sum, transcript) => sum + transcript.confidence, 0) / confidenceSampleCount : undefined,
     confidenceSampleCount,
@@ -6466,7 +6472,9 @@ var createTurnQuality = (transcripts, source, fallbackUsed, fallbackDiagnostics,
     finalTranscriptCount: transcripts.filter((transcript) => transcript.isFinal).length,
     partialTranscriptCount: transcripts.filter((transcript) => !transcript.isFinal).length,
     selectedTranscriptCount: transcripts.length,
-    source
+    source,
+    lowestWordConfidence: wordConfidences.length > 0 ? Math.min(...wordConfidences) : undefined,
+    wordConfidenceSampleCount: wordConfidences.length
   };
 };
 var createTurnCostEstimate = (input) => {
@@ -6483,20 +6491,38 @@ var createTurnCostEstimate = (input) => {
   };
 };
 var normalizeCorrectionText = (text) => normalizeText2(text);
-var isFallbackNeeded = (candidate, config) => {
+var evaluateFallbackNeed = (candidate, config) => {
   const trimmed = normalizeText2(candidate.text);
   const wordCount = countWords2(trimmed);
+  const averageConfidence = calculateMeanConfidence(candidate.transcripts);
+  const words = collectTranscriptWords(candidate.transcripts);
+  const wordConfidences = collectWordConfidences(candidate.transcripts);
+  const lowestWordConfidence = wordConfidences.length > 0 ? Math.min(...wordConfidences) : undefined;
+  const resolvedCandidate = {
+    averageConfidence,
+    lowestWordConfidence,
+    text: candidate.text,
+    transcripts: candidate.transcripts,
+    wordCount,
+    words
+  };
   if (config.trigger === "always") {
-    return true;
+    return { needed: true, reason: "always" };
   }
-  if (config.trigger === "empty-turn") {
-    return wordCount < config.minTextLength;
+  if (wordCount < config.minTextLength && (config.trigger === "empty-turn" || config.trigger === "empty-or-low-confidence")) {
+    return { needed: true, reason: "empty-turn" };
   }
-  const averageConfidence = calculateMeanConfidence(candidate.transcripts);
-  if (config.trigger === "low-confidence") {
-    return averageConfidence > 0 && averageConfidence < config.confidenceThreshold;
+  if (config.riskPolicy?.(resolvedCandidate)) {
+    return { needed: true, reason: "risk-policy" };
+  }
+  if (config.wordConfidenceThreshold !== undefined && lowestWordConfidence !== undefined && lowestWordConfidence < config.wordConfidenceThreshold) {
+    return { needed: true, reason: "word-confidence" };
+  }
+  const checksConfidence = config.trigger === "low-confidence" || config.trigger === "empty-or-low-confidence";
+  if (checksConfidence && averageConfidence > 0 && averageConfidence < config.confidenceThreshold) {
+    return { needed: true, reason: "transcript-confidence" };
   }
-  return averageConfidence > 0 && averageConfidence < config.confidenceThreshold || wordCount < config.minTextLength;
+  return { needed: false, reason: undefined };
 };
 var selectBetterTurnText = (candidate, fallback) => {
   if (!fallback.text) {
@@ -6615,7 +6641,9 @@ var createVoiceSession = (options) => {
     minTextLength: options.sttFallback.minTextLength ?? DEFAULT_FALLBACK_MIN_TEXT_LENGTH,
     replayWindowMs: options.sttFallback.replayWindowMs ?? DEFAULT_FALLBACK_REPLAY_MS,
     settleMs: options.sttFallback.settleMs ?? DEFAULT_FALLBACK_SETTLE_MS,
-    trigger: options.sttFallback.trigger ?? "empty-or-low-confidence"
+    trigger: options.sttFallback.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: options.sttFallback.wordConfidenceThreshold,
+    riskPolicy: options.sttFallback.riskPolicy
   } : undefined;
   const appendTrace = async (input) => {
     await options.trace?.append({
@@ -7762,7 +7790,8 @@ var createVoiceSession = (options) => {
       text: primaryText,
       transcripts: primaryTranscripts
     };
-    if (!isFallbackNeeded(candidate, sttFallback)) {
+    const fallbackNeed = evaluateFallbackNeed(candidate, sttFallback);
+    if (!fallbackNeed.needed) {
       return null;
     }
     fallbackAttemptsForCurrentTurn += 1;
@@ -7889,7 +7918,8 @@ var createVoiceSession = (options) => {
       primaryWordCount: primaryCandidate.wordCount,
       selected: selection.winner.text === fallbackCandidate.text,
       selectionReason: selection.reason,
-      trigger: sttFallback.trigger
+      trigger: sttFallback.trigger,
+      triggerReason: fallbackNeed.reason
     };
     if (selection.winner.text === primaryCandidate.text) {
       return {
@@ -10251,7 +10281,9 @@ var resolveBenchmarkFallbackConfig = (config) => {
     minTextLength: config.minTextLength ?? 2,
     replayWindowMs: config.replayWindowMs ?? 8000,
     settleMs: config.settleMs ?? 220,
-    trigger: config.trigger ?? "empty-or-low-confidence"
+    trigger: config.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: config.wordConfidenceThreshold,
+    riskPolicy: config.riskPolicy
   };
 };
 var chunkAudio2 = (audio, bytesPerChunk) => {
@@ -14426,7 +14458,9 @@ var resolveSTTFallbackConfig = (config) => {
     minTextLength: config.minTextLength ?? 2,
     replayWindowMs: config.replayWindowMs ?? 8000,
     settleMs: config.settleMs ?? 220,
-    trigger: config.trigger ?? "empty-or-low-confidence"
+    trigger: config.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: config.wordConfidenceThreshold,
+    riskPolicy: config.riskPolicy
   };
 };
 var normalizePhraseHints = (hints) => (hints ?? []).map((hint) => ({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.632",
+  "version": "0.0.22-beta.634",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",
@@ -19,7 +19,7 @@
   "license": "BSL-1.1",
   "author": "Alex Kahn",
   "scripts": {
-    "build": "rm -rf dist && bun build ./src/index.ts ./src/client/index.ts ./src/react/index.ts ./src/vue/index.ts ./src/svelte/index.ts ./src/angular/index.ts ./src/testing/index.ts ./src/drizzle/index.ts --outdir dist --target bun --external elysia --external react --external vue --external @angular/core --external @absolutejs/absolute --external @absolutejs/ai --external @absolutejs/media --external drizzle-orm && bun build ./src/client/htmxBootstrap.ts --outdir dist/client --target browser --format esm && bun build ./src/embed/index.ts --outfile dist/embed/voice-widget.js --target browser --format iife --minify && bun build ./src/embed/index.ts --outdir dist/embed --target browser --format esm && tsc --emitDeclarationOnly --project tsconfig.json",
+    "build": "rm -rf dist && bun build ./src/index.ts ./src/manifest.ts ./src/client/index.ts ./src/react/index.ts ./src/vue/index.ts ./src/svelte/index.ts ./src/angular/index.ts ./src/testing/index.ts ./src/drizzle/index.ts --outdir dist --root ./src --target bun --external elysia --external react --external vue --external @angular/core --external @absolutejs/absolute --external @absolutejs/ai --external @absolutejs/media --external drizzle-orm && bun build ./src/client/htmxBootstrap.ts --outdir dist/client --target browser --format esm && bun build ./src/embed/index.ts --outfile dist/embed/voice-widget.js --target browser --format iife --minify && bun build ./src/embed/index.ts --outdir dist/embed --target browser --format esm && tsc --emitDeclarationOnly --project tsconfig.json && absolute-manifest emit",
     "config": "absolute config",
     "format": "absolute prettier --write",
     "knip": "knip",
@@ -75,7 +75,12 @@
       "import": "./dist/embed/index.js",
       "types": "./dist/embed/index.d.ts"
     },
-    "./embed/voice-widget.js": "./dist/embed/voice-widget.js"
+    "./embed/voice-widget.js": "./dist/embed/voice-widget.js",
+    "./manifest": {
+      "import": "./dist/manifest.js",
+      "types": "./dist/manifest.d.ts"
+    },
+    "./manifest.json": "./dist/manifest.json"
   },
   "typesVersions": {
     "*": {
@@ -102,6 +107,9 @@
       ],
       "embed": [
         "dist/embed/index.d.ts"
+      ],
+      "manifest": [
+        "dist/manifest.d.ts"
       ]
     }
   },
@@ -132,7 +140,9 @@
     }
   },
   "dependencies": {
-    "@absolutejs/media": "0.0.1-beta.19"
+    "@absolutejs/manifest": "^0.1.0",
+    "@absolutejs/media": "0.0.1-beta.19",
+    "@sinclair/typebox": "^0.34.0"
   },
   "devDependencies": {
     "@absolutejs/absolute": "0.19.0-beta.1009",
@@ -157,5 +167,8 @@
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.57.2",
     "vue": "3.5.27"
+  },
+  "absolutejs": {
+    "manifestContract": 1
   }
 }