@ouro.bot/cli 0.1.0-alpha.887 → 0.1.0-alpha.889

package/dist/senses/voice/twilio-phone.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TwilioAudioStreamJobStore = exports.TwilioOpenAIRealtimeMediaStreamSession = exports.TwilioMediaStreamSession = exports.DEFAULT_TWILIO_MEDIA_MAX_UTTERANCE_MS = exports.DEFAULT_TWILIO_MEDIA_MIN_SPEECH_MS = exports.DEFAULT_TWILIO_MEDIA_SILENCE_END_MS = exports.DEFAULT_TWILIO_MEDIA_SPEECH_RMS_THRESHOLD = exports.DEFAULT_TWILIO_PHONE_TRANSPORT_MODE = exports.DEFAULT_TWILIO_PHONE_PLAYBACK_MODE = exports.TWILIO_PHONE_WEBHOOK_BASE_PATH = exports.DEFAULT_TWILIO_GREETING_PREBUFFER_MS = exports.DEFAULT_TWILIO_RECORD_MAX_LENGTH_SECONDS = exports.DEFAULT_TWILIO_RECORD_TIMEOUT_SECONDS = exports.DEFAULT_TWILIO_PHONE_PORT = exports.normalizeTwilioE164PhoneNumber = exports.PendingVoiceCalls = void 0;
+exports.TwilioAudioStreamJobStore = exports.TwilioOpenAIRealtimeMediaStreamSession = exports.LOCAL_AUDIO_TOOL_ALLOWLIST = exports.TwilioMediaStreamSession = exports.DEFAULT_TWILIO_MEDIA_MAX_UTTERANCE_MS = exports.DEFAULT_TWILIO_MEDIA_MIN_SPEECH_MS = exports.DEFAULT_TWILIO_MEDIA_SILENCE_END_MS = exports.DEFAULT_TWILIO_MEDIA_SPEECH_RMS_THRESHOLD = exports.DEFAULT_TWILIO_PHONE_TRANSPORT_MODE = exports.DEFAULT_TWILIO_PHONE_PLAYBACK_MODE = exports.TWILIO_PHONE_WEBHOOK_BASE_PATH = exports.DEFAULT_TWILIO_GREETING_PREBUFFER_MS = exports.DEFAULT_TWILIO_RECORD_MAX_LENGTH_SECONDS = exports.DEFAULT_TWILIO_RECORD_TIMEOUT_SECONDS = exports.DEFAULT_TWILIO_PHONE_PORT = exports.normalizeTwilioE164PhoneNumber = exports.PendingVoiceCalls = void 0;
 exports.resolveTwilioPhoneAgentRoot = resolveTwilioPhoneAgentRoot;
 exports.normalizeTwilioPhoneBasePath = normalizeTwilioPhoneBasePath;
 exports.normalizeTwilioPhonePlaybackMode = normalizeTwilioPhonePlaybackMode;
@@ -50,6 +50,7 @@ exports.twilioPhoneVoiceSessionKey = twilioPhoneVoiceSessionKey;
 exports.outboundCallAnsweredPrompt = outboundCallAnsweredPrompt;
 exports.computeOpenAIWebhookSignature = computeOpenAIWebhookSignature;
 exports.validateOpenAIWebhookSignature = validateOpenAIWebhookSignature;
+exports.tonesOnly = tonesOnly;
 exports.parseToolArguments = parseToolArguments;
 exports.computeTwilioSignature = computeTwilioSignature;
 exports.validateTwilioSignature = validateTwilioSignature;
@@ -83,6 +84,9 @@ const phone_1 = require("./phone");
 const audio_playback_1 = require("./audio-playback");
 const floor_controller_1 = require("./floor-controller");
 const call_auth_1 = require("./call-auth");
+const local_audio_identity_1 = require("./local-audio-identity");
+const local_audio_disclosure_1 = require("./local-audio-disclosure");
+const local_audio_prompts_1 = require("./local-audio-prompts");
 var call_auth_2 = require("./call-auth");
 Object.defineProperty(exports, "PendingVoiceCalls", { enumerable: true, get: function () { return call_auth_2.PendingVoiceCalls; } });
 var phone_2 = require("./phone");
@@ -1500,10 +1504,29 @@ function realtimeOutputAudioConfig(realtime, format) {
         ...(speed === undefined ? {} : { speed }),
     };
 }
+/**
+ * A local audio room is several unidentified people on the owner's Mac, so no trust level (not even
+ * family through an owner-alone join) unlocks files, shell, credentials or MCP: the whole session is
+ * limited to these two tools, in what it advertises and in what it will run.
+ */
+exports.LOCAL_AUDIO_TOOL_ALLOWLIST = new Set(["voice_end_call", "voice_play_audio"]);
+/**
+ * In a local audio room anyone in the room can ask for a clip, so only generated tones play: a URL
+ * clip would make the Mac fetch any address (including the LAN), and a file clip reads local files.
+ */
+function tonesOnly(play) {
+    return async (request) => {
+        if ((request.source ?? "tone") !== "tone")
+            throw new Error("only tones can play in a local audio room; clips from a URL or file are not allowed here");
+        return play(request);
+    };
+}
 function selectRealtimeToolContext(context, mcpManager) {
+    const allowlist = context.toolAllowlist;
     const selectCurrentTools = () => {
         const selected = (0, tools_1.selectToolsForChannel)((0, friends_1.getChannelCapabilities)("voice"), context.context?.friend.toolPreferences, context.context, undefined, mcpManager, undefined, context);
-        return Object.freeze({ ordinary: selected.ordinary, engine: Object.freeze([]) });
+        const ordinary = allowlist ? selected.ordinary.filter(({ tool }) => allowlist.has(tool.function.name)) : selected.ordinary;
+        return Object.freeze({ ordinary, engine: Object.freeze([]) });
     };
     return { ...context, toolSelection: selectCurrentTools(), selectCurrentTools };
 }
@@ -1669,6 +1692,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
     started = false;
     openaiReady = false;
     greetingSent = false;
+    disclosure;
     hangupRequested = false;
     pendingAudioPayloads = [];
     openaiWs = null;
@@ -1771,20 +1795,29 @@ class TwilioOpenAIRealtimeMediaStreamSession {
         this.from = this.identity.from;
         this.to = this.identity.to;
         try {
-            const voiceContext = await resolveVoiceFriendContext(this.options, {
-                friendId: this.direction === "outbound" ? this.identity.friendId : undefined,
-                remotePhone: this.from || undefined,
-                callSid: this.callSid,
-            });
+            const local = this.identity.local;
+            const voiceContext = local
+                ? await (0, local_audio_identity_1.resolveLocalAudioFriend)({
+                    agentRoot: resolveTwilioPhoneAgentRoot(this.options),
+                    friendId: this.identity.friendId,
+                    ownerAlone: local.ownerAlone,
+                })
+                : await resolveVoiceFriendContext(this.options, {
+                    friendId: this.direction === "outbound" ? this.identity.friendId : undefined,
+                    remotePhone: this.from || undefined,
+                    callSid: this.callSid,
+                });
             this.friendId = voiceContext.friendId;
             this.friendStore = voiceContext.friendStore;
             this.resolvedContext = voiceContext.resolved;
-            this.sessionKey = twilioPhoneVoiceSessionKey({
-                defaultFriendId: this.friendId,
-                from: this.from,
-                to: this.to,
-                callSid: this.callSid,
-            });
+            this.sessionKey = local
+                ? `local-audio-${safeSegment(this.callSid)}`
+                : twilioPhoneVoiceSessionKey({
+                    defaultFriendId: this.friendId,
+                    from: this.from,
+                    to: this.to,
+                    callSid: this.callSid,
+                });
             this.lifecycle?.onIdentityChange?.(this, { callSid: this.callSid, outboundId: this.outboundId });
             this.floor = new floor_controller_1.VoiceFloorController({
                 transport: "twilio-media-stream",
@@ -1884,9 +1917,9 @@ class TwilioOpenAIRealtimeMediaStreamSession {
         ];
         if (!toolContext || this.closed)
             return;
-        this.sendOpenAIRealtimeSessionUpdate(realtime, instructions, toolContext);
+        this.sendOpenAIRealtimeSessionUpdate(realtime, this.localizeInstructions(instructions), toolContext);
         this.flushPendingAudio();
-        this.sendInitialGreeting();
+        this.sendGreeting();
         if (!usedBootstrap)
             return;
         Promise.all([instructionsPromise, toolsPromise])
@@ -1898,7 +1931,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
                 type: "session.update",
                 session: {
                     type: "realtime",
-                    instructions: fullInstructions,
+                    instructions: this.localizeInstructions(fullInstructions),
                     tools: realtimeToolsFromSelection(fullContext.toolSelection, this.toolAdvertiser),
                     tool_choice: "auto",
                 },
@@ -1913,6 +1946,64 @@ class TwilioOpenAIRealtimeMediaStreamSession {
             this.end();
         });
     }
+    /** Local audio rooms get transport-aware wording and the room facts; phone calls are unchanged. */
+    localizeInstructions(instructions) {
+        const local = this.identity.local;
+        return local
+            ? (0, local_audio_prompts_1.localizeRealtimeInstructions)(instructions, this.options.agentName, local, { disclosureSpoken: this.disclosure?.isSpoken() === true })
+            : instructions;
+    }
+    /**
+     * A spoken local join opens with the disclosure notice and tracks it until it has really been
+     * spoken (retrying a lost, cancelled or cut attempt); a consented silent join has no first turn.
+     */
+    sendGreeting() {
+        const local = this.identity.local;
+        if (!local) {
+            this.sendInitialGreeting();
+            return;
+        }
+        // Configured once per Realtime connection, so the opening turn is requested exactly once here.
+        this.greetingSent = true;
+        const prompt = (0, local_audio_prompts_1.localAudioGreetingPrompt)(this.options.agentName, local);
+        if (!prompt)
+            return;
+        const body = { instructions: prompt };
+        this.disclosure = new local_audio_disclosure_1.DisclosureTracker({
+            body,
+            host: {
+                request: (response) => this.requestRealtimeResponse(response),
+                isQueued: (response) => this.holdsRealtimeResponse(response),
+                sendMark: (name) => this.sendDisclosureMark(name),
+                onSpoken: (atMs) => this.noteDisclosureSpoken(atMs),
+                onFailed: () => this.lifecycle?.onDisclosureFailed?.(),
+            },
+        });
+        this.disclosure.start();
+    }
+    /** True while a response.create for this body is held by the floor gate, the hold timer or the wire. */
+    holdsRealtimeResponse(response) {
+        return this.queuedGatedRealtimeRequest?.response === response
+            || this.pendingRealtimeResponse?.response === response
+            || this.realtimeResponseCreateInFlight?.response === response;
+    }
+    sendDisclosureMark(name) {
+        if (this.closed || !this.streamSid || !this.socket.isOpen())
+            return;
+        this.socket.send(JSON.stringify({ event: "mark", streamSid: this.streamSid, mark: { name } }));
+    }
+    noteDisclosureSpoken(atMs) {
+        // The instructions said "do not claim you announced yourself until a system note confirms it".
+        this.sendOpenAI({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "system",
+                content: [{ type: "input_text", text: "System note: your disclosure notice has now been spoken aloud and heard. You have announced yourself." }],
+            },
+        });
+        this.lifecycle?.onDisclosureSpoken?.(atMs);
+    }
     sendOpenAIRealtimeSessionUpdate(realtime, instructions, toolContext) {
         this.toolContext = toolContext;
         this.sendOpenAI({
@@ -1974,6 +2065,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
     ensureVoiceToolContext() {
         if (this.toolContext)
             return selectRealtimeToolContext(this.toolContext);
+        const play = (request) => this.playPreparedAudio(request);
         const context = selectRealtimeToolContext({
             signin: async () => undefined,
             agentName: this.options.agentName,
@@ -1982,8 +2074,9 @@ class TwilioOpenAIRealtimeMediaStreamSession {
             friendStore: this.friendStore,
             voiceCall: {
                 requestEnd: () => this.requestHangupFromTool(),
-                playAudio: (request) => this.playPreparedAudio(request),
+                playAudio: this.identity.local ? tonesOnly(play) : play,
             },
+            ...(this.identity.local ? { toolAllowlist: exports.LOCAL_AUDIO_TOOL_ALLOWLIST } : {}),
         });
         this.toolContext = { ...context, toolSelection: Object.freeze({ ordinary: Object.freeze([]), engine: Object.freeze([]) }) };
         return context;
@@ -2041,6 +2134,8 @@ class TwilioOpenAIRealtimeMediaStreamSession {
         const name = stringField(mark?.name);
         if (!name)
             return;
+        if (this.disclosure?.noteMark(name))
+            return;
         const playback = this.playbackMarks.get(name);
         if (!playback)
             return;
@@ -2060,6 +2155,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
             return;
         }
         const type = typeof event.type === "string" ? event.type : "";
+        this.disclosure?.handleRealtimeEvent(event);
         if (type === "response.created") {
             this.noteRealtimeResponseCreated(event);
             return;
@@ -2518,6 +2614,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
     }
     sendRealtimeResponseCreate(request) {
         this.realtimeResponseCreateInFlight = request;
+        this.disclosure?.noteSent(request.response);
         this.sendOpenAI({
             type: "response.create",
             ...(request.response ? { response: request.response } : {}),
@@ -2638,6 +2735,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
     sendTwilioClear() {
         if (this.closed || !this.streamSid || !this.socket.isOpen())
             return;
+        this.disclosure?.noteCleared();
         this.socket.send(JSON.stringify({ event: "clear", streamSid: this.streamSid }));
     }
     appendTranscript(role, text) {
@@ -2674,6 +2772,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
         if (this.closed)
             return;
         this.closed = true;
+        this.disclosure?.stop();
         if (this.openaiWs && (this.openaiWs.readyState === ws_1.WebSocket.OPEN || this.openaiWs.readyState === ws_1.WebSocket.CONNECTING)) {
             this.openaiWs.close();
         }

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.887",
+  "version": "0.1.0-alpha.889",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@ouro.bot/cli",
-      "version": "0.1.0-alpha.887",
+      "version": "0.1.0-alpha.889",
       "dependencies": {
         "@anthropic-ai/sdk": "^0.78.0",
         "@azure/identity": "^4.13.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.887",
+  "version": "0.1.0-alpha.889",
   "engines": {
     "node": ">=22"
   },