@absolutejs/voice 0.0.22-beta.495 → 0.0.22-beta.497

package/dist/agent.d.ts

@@ -3,7 +3,19 @@ import type { VoiceTraceEventStore } from "./trace";
 import type { VoiceToolRuntime } from "./toolRuntime";
 import { type VoiceAuditEventStore, type VoiceAuditLogger } from "./audit";
 export type VoiceAgentMessageRole = "assistant" | "system" | "tool" | "user";
+export type VoiceAgentImageMediaType = "image/gif" | "image/jpeg" | "image/png" | "image/webp";
+export type VoiceAgentMessageAttachment = {
+    data: string;
+    kind: "image";
+    mediaType: VoiceAgentImageMediaType;
+} | {
+    data: string;
+    kind: "document";
+    mediaType: "application/pdf";
+    name?: string;
+};
 export type VoiceAgentMessage = {
+    attachments?: VoiceAgentMessageAttachment[];
     content: string;
     metadata?: Record<string, unknown>;
     name?: string;

package/dist/index.js

@@ -3702,6 +3702,7 @@ var createVoiceSession = (options) => {
   let activeAdapterGeneration = 0;
   let activeTTSTurnId;
   const currentTurnAudio = [];
+  const pendingUserAttachments = [];
   let fallbackAttemptsForCurrentTurn = 0;
   let fallbackReplayAudioMsForCurrentTurn = 0;
   const amdDetector = options.amd;
@@ -4915,6 +4916,7 @@ var createVoiceSession = (options) => {
     ttsSessionPromise = (async () => {
       const openedSession = await ttsAdapter.open({
         lexicon,
+        prosody: options.prosody,
         sessionId: options.id
       });
       ttsSession = openedSession;
@@ -5226,7 +5228,9 @@ var createVoiceSession = (options) => {
         audioMs: costEstimate.totalBillableAudioMs
       });
     }
+    const drainedAttachments = pendingUserAttachments.length > 0 ? pendingUserAttachments.splice(0, pendingUserAttachments.length) : undefined;
     const turn = {
+      attachments: drainedAttachments,
       committedAt: Date.now(),
       id: createId(),
       text: finalText,
@@ -5520,6 +5524,9 @@ var createVoiceSession = (options) => {
   };
   const api = {
     id: options.id,
+    attachUserMedia: async (attachment) => {
+      pendingUserAttachments.push(attachment);
+    },
     close: async (reason) => {
       await runSerial("api.close", async () => {
         const disposition = reason === "silence-timeout" ? "silence-timeout" : "closed";
@@ -8559,8 +8566,10 @@ var createHistoryMessages = (session, turn) => {
     if (previousTurn.id === turn.id) {
       continue;
     }
-    if (previousTurn.text.trim()) {
+    const previousAttachments = previousTurn.attachments && previousTurn.attachments.length > 0 ? previousTurn.attachments : undefined;
+    if (previousTurn.text.trim() || previousAttachments) {
       messages.push({
+        attachments: previousAttachments,
         content: previousTurn.text,
         role: "user"
       });
@@ -8572,7 +8581,9 @@ var createHistoryMessages = (session, turn) => {
       });
     }
   }
+  const currentAttachments = turn.attachments && turn.attachments.length > 0 ? turn.attachments : undefined;
   messages.push({
+    attachments: currentAttachments,
     content: turn.text,
     role: "user"
   });
@@ -34968,6 +34979,36 @@ var toProviderMessages = (messages) => {
       out.push({ content: message.content, role: "user" });
       continue;
     }
+    if (message.role === "user" && message.attachments && message.attachments.length > 0) {
+      const blocks = [];
+      if (message.content) {
+        blocks.push({ content: message.content, type: "text" });
+      }
+      for (const attachment of message.attachments) {
+        if (attachment.kind === "image") {
+          blocks.push({
+            source: {
+              data: attachment.data,
+              media_type: attachment.mediaType,
+              type: "base64"
+            },
+            type: "image"
+          });
+        } else if (attachment.kind === "document") {
+          blocks.push({
+            name: attachment.name,
+            source: {
+              data: attachment.data,
+              media_type: attachment.mediaType,
+              type: "base64"
+            },
+            type: "document"
+          });
+        }
+      }
+      out.push({ content: blocks, role: "user" });
+      continue;
+    }
     out.push({ content: message.content, role: message.role });
   }
   return out;
@@ -36748,6 +36789,7 @@ var toHandoffExpectation = (event) => ({
   targetAgentId: getPayloadString2(event, "targetAgentId")
 });
 var createContractApi = (session) => ({
+  attachUserMedia: async () => {},
   close: async () => {},
   commitTurn: async () => {},
   complete: async () => {},

package/dist/testing/index.js

@@ -5670,6 +5670,7 @@ var createVoiceSession = (options) => {
   let activeAdapterGeneration = 0;
   let activeTTSTurnId;
   const currentTurnAudio = [];
+  const pendingUserAttachments = [];
   let fallbackAttemptsForCurrentTurn = 0;
   let fallbackReplayAudioMsForCurrentTurn = 0;
   const amdDetector = options.amd;
@@ -6883,6 +6884,7 @@ var createVoiceSession = (options) => {
     ttsSessionPromise = (async () => {
       const openedSession = await ttsAdapter.open({
         lexicon,
+        prosody: options.prosody,
         sessionId: options.id
       });
       ttsSession = openedSession;
@@ -7194,7 +7196,9 @@ var createVoiceSession = (options) => {
         audioMs: costEstimate.totalBillableAudioMs
       });
     }
+    const drainedAttachments = pendingUserAttachments.length > 0 ? pendingUserAttachments.splice(0, pendingUserAttachments.length) : undefined;
     const turn = {
+      attachments: drainedAttachments,
       committedAt: Date.now(),
       id: createId(),
       text: finalText,
@@ -7488,6 +7492,9 @@ var createVoiceSession = (options) => {
   };
   const api = {
     id: options.id,
+    attachUserMedia: async (attachment) => {
+      pendingUserAttachments.push(attachment);
+    },
     close: async (reason) => {
       await runSerial("api.close", async () => {
         const disposition = reason === "silence-timeout" ? "silence-timeout" : "closed";

package/dist/types.d.ts

@@ -49,12 +49,18 @@ export type VoiceLexiconEntry = {
     metadata?: Record<string, unknown>;
     pronunciation?: string;
 };
+export type VoiceTranscriptSentiment = {
+    label: "negative" | "neutral" | "positive" | (string & {});
+    metadata?: Record<string, unknown>;
+    score?: number;
+};
 export type Transcript = {
     id: string;
     text: string;
     isFinal: boolean;
     confidence?: number;
     language?: string;
+    sentiment?: VoiceTranscriptSentiment;
     speaker?: string | number;
     startedAtMs?: number;
     endedAtMs?: number;
@@ -172,9 +178,16 @@ export type TTSAdapterSession = {
 export declare const ttsAdapterSessionCanCancel: (session: TTSAdapterSession) => session is TTSAdapterSession & {
     cancel: (reason?: string) => Promise<void>;
 };
+export type VoiceTTSProsody = {
+    emphasis?: number;
+    pitch?: number;
+    speed?: number;
+    style?: string;
+};
 export type TTSAdapterOpenOptions = {
     sessionId: string;
     lexicon?: VoiceLexiconEntry[];
+    prosody?: VoiceTTSProsody;
     signal?: AbortSignal;
 };
 export type TTSAdapter<TOptions extends TTSAdapterOpenOptions = TTSAdapterOpenOptions> = {
@@ -220,6 +233,7 @@ export type VoiceTurnRecord<TResult = unknown> = {
     quality?: VoiceTranscriptQuality;
     transcripts: Transcript[];
     assistantText?: string;
+    attachments?: import("./agent").VoiceAgentMessageAttachment[];
     committedAt: number;
     result?: TResult;
 };
@@ -436,6 +450,7 @@ export type VoiceSessionHandle<TContext = unknown, TSession extends VoiceSession
     id: string;
     connect: (socket: VoiceSocket) => Promise<void>;
     receiveAudio: (audio: AudioChunk) => Promise<void>;
+    attachUserMedia: (attachment: import("./agent").VoiceAgentMessageAttachment) => Promise<void>;
     commitTurn: (reason?: VoiceEndOfTurnEvent["reason"]) => Promise<void>;
     disconnect: (event?: VoiceCloseEvent) => Promise<void>;
     complete: (result?: TResult) => Promise<void>;
@@ -737,6 +752,7 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     semanticTurnDetector?: import("./semanticTurn").VoiceSemanticTurnDetector;
     assistantMode?: import("./assistantMode").VoiceAssistantMode;
     modalities?: ReadonlyArray<"audio" | "text">;
+    prosody?: VoiceTTSProsody;
     reconnect: Required<VoiceReconnectConfig>;
     phraseHints?: VoicePhraseHint[];
     sessionMetadata?: Record<string, unknown>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.495",
+  "version": "0.0.22-beta.497",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",