@pouchy_ai/companion-sdk 0.67.0 → 0.69.0

package/src/call.ts

@@ -11,7 +11,13 @@
 // Browser-only (needs WebRTC + getUserMedia). Returns a handle to hang up and to
 // inject live context mid-call (the bridge the server uses for voiceRelevant).
 
-import type { CallCredentials, CallRecordingUnavailable, CallWorldBrief } from './client';
+import type {
+	CallCredentials,
+	CallPrivateStory,
+	CallRecordingUnavailable,
+	CallTrimmed,
+	CallWorldBrief
+} from './client';
 import { CompanionError } from './errors';
 import { VOICE_TOOL_VARIABLE, VOICE_TOOL_VARIABLE_PLAIN } from './voice-tool';
 
@@ -197,14 +203,57 @@ export interface CompanionCallOptions {
 	 *  "说一句就断了" field report). Enable only where AEC actually holds:
 	 *  headphones, desktop, or devices you have verified. */
 	bargeIn?: boolean;
+	/** Open the call with the reader's microphone MUTED (0.69.0); see
+	 *  `CompanionCall.setMuted`. On `openai-realtime` the track is disabled
+	 *  before it is attached, so nothing is ever sent. On `elevenlabs-convai`
+	 *  the mute is applied the moment the session is up; the provider client
+	 *  has no start-muted option, so the connect handshake itself (before the
+	 *  agent's first line) can carry audio. */
+	muted?: boolean;
 }
 
 export interface CompanionCall {
 	/** Hang up. Idempotent. */
 	close: () => void;
 	/** Inject an out-of-band context line. `speak: true` makes the companion react
-	 *  now; false stages it for the next turn. */
+	 *  now; false stages it for the next turn.
+	 *
+	 *  NOT the reader's words: the line does not reach `onTranscript` or the
+	 *  memory `/end` consolidates, and `connectCall` uses this same lever for
+	 *  server-pushed world moments. For something the reader TYPED, use
+	 *  `sendText`. */
 	injectEvent: (text: string, speak?: boolean) => void;
+	/** Send a line the READER typed, as their turn in the call (0.69.0). The
+	 *  companion answers it as something the reader said; it is reported to
+	 *  `onTranscript` as `role: 'user'` (once, by the SDK, the moment it is
+	 *  sent) and so reaches the memory `connectCall` hands to `/end` at hang-up,
+	 *  in order with the spoken lines.
+	 *
+	 *  It takes the floor the way speech does: an utterance the companion is
+	 *  in the middle of is cut, then it replies to this line.
+	 *
+	 *  Nothing is sent before this call — the provider sees the text only when
+	 *  you pass it here, so a check you run first runs before anyone reads it.
+	 *  Returns `false` (and sends nothing) for blank text, after `close()`, or
+	 *  when the call's channel is not open; `true` once it was handed to the
+	 *  provider. Same behaviour on both providers. */
+	sendText: (text: string) => boolean;
+	/** The reader's own mute (0.69.0). `true` stops the microphone sending
+	 *  audio until `setMuted(false)`. Distinct from the SDK's half-duplex mute
+	 *  (the mic is muted while the companion speaks, unless `bargeIn`): a
+	 *  reader who muted stays muted when the companion finishes a line. A
+	 *  no-op after `close()`. Check `canMute` before showing a mute control. */
+	setMuted: (muted: boolean) => void;
+	/** The reader's mute as last set (0.69.0) — the `setMuted` / `muted`
+	 *  state, not whether the half-duplex mute is momentarily engaged. A
+	 *  method, not a field, so it stays live on a spread copy of the handle. */
+	isMuted: () => boolean;
+	/** Whether `setMuted` actually controls the microphone on this call
+	 *  (0.69.0). `true` on `openai-realtime`; on `elevenlabs-convai` it is
+	 *  `true` when the installed `@elevenlabs/client` exposes `setMicMuted`
+	 *  (every version the peer range allows does). When `false`, `setMuted`
+	 *  only records the state: do not show a mute control. */
+	canMute: boolean;
 	/** SILENT CUT (0.36.0): stop the utterance currently being spoken WITHOUT
 	 *  generating any new speech — the "stale commentary at a beat boundary"
 	 *  case (field request: fast-paced game, last hand's commentary still
@@ -235,6 +284,11 @@ export interface CompanionCall {
 	/** Pouchy World role sessions (0.67.0): whether the call carries the
 	 *  story's current state. See `CallWorldBrief`. */
 	worldBrief?: CallWorldBrief;
+	/** Pouchy World role sessions (0.68.0): whether the role can reach its
+	 *  private layer on this call. See `CallPrivateStory`. */
+	privateStory?: CallPrivateStory;
+	/** What the server cut to fit the realtime cap (0.68.0). See `CallTrimmed`. */
+	trimmed?: CallTrimmed;
 }
 
 const OPENAI_REALTIME_CALLS = 'https://api.openai.com/v1/realtime/calls';
@@ -348,9 +402,16 @@ async function openOpenAICall(
 	// the companion interrupting itself.
 	const MIC_UNMUTE_TAIL_MS = 350;
 	let micUnmuteTimer: ReturnType<typeof setTimeout> | null = null;
-	const setMicEnabled = (enabled: boolean) => {
+	// Two independent reasons the mic is off (0.69.0): the READER muted
+	// (setMuted), or the companion is speaking (half-duplex). The track is on
+	// only when neither holds, so the end-of-speech re-open can never undo a
+	// reader's mute — which is what made a mute layered on top of this SDK a
+	// button that switched itself back on.
+	let userMuted = opts.muted === true;
+	let speechMuted = false;
+	const applyMic = () => {
 		try {
-			for (const tr of mic.getAudioTracks()) tr.enabled = enabled;
+			for (const tr of mic.getAudioTracks()) tr.enabled = !userMuted && !speechMuted;
 		} catch {
 			/* track gone — ignore */
 		}
@@ -360,13 +421,16 @@ async function openOpenAICall(
 			clearTimeout(micUnmuteTimer);
 			micUnmuteTimer = null;
 		}
-		setMicEnabled(false);
+		speechMuted = true;
+		applyMic();
 	};
 	const scheduleMicUnmute = () => {
 		if (micUnmuteTimer) clearTimeout(micUnmuteTimer);
 		micUnmuteTimer = setTimeout(() => {
 			micUnmuteTimer = null;
-			if (!closed) setMicEnabled(true);
+			if (closed) return;
+			speechMuted = false;
+			applyMic();
 		}, MIC_UNMUTE_TAIL_MS);
 	};
 
@@ -375,6 +439,15 @@ async function openOpenAICall(
 	// active response makes the server emit a benign-but-noisy error event, so
 	// an idle interrupt() must be a true no-op.
 	let oaiSpeaking = false;
+	// Whether a response is in flight (response.created → response.done) —
+	// broader than `oaiSpeaking`, which starts only with audio. `sendText`
+	// cancels an in-flight response before creating its own: the server
+	// refuses a second concurrent response, which would leave the typed line
+	// in the conversation with no reply.
+	let oaiResponseActive = false;
+	// A reader who asked to start muted never sends a frame: the track is
+	// disabled before the offer is even created.
+	applyMic();
 
 	const dc = pc.createDataChannel('oai-events');
 	// `response.function_call_arguments.done` carries the call_id + args but often
@@ -475,6 +548,8 @@ async function openOpenAICall(
 			msg.transcript
 		) {
 			opts.onTranscript?.({ role: 'user', text: msg.transcript });
+		} else if (msg.type === 'response.created') {
+			oaiResponseActive = true;
 		} else if (msg.type === 'output_audio_buffer.started') {
 			oaiSpeaking = true;
 			opts.onSpeakingChange?.(true);
@@ -493,6 +568,7 @@ async function openOpenAICall(
 			// audio doesn't get captured the instant we unmute. Several event names
 			// can end a turn; any of them re-arms the mic.
 			oaiSpeaking = false;
+			if (msg.type === 'response.done') oaiResponseActive = false;
 			opts.onSpeakingChange?.(false);
 			if (opts.bargeIn !== true) scheduleMicUnmute();
 		}
@@ -664,6 +740,36 @@ async function openOpenAICall(
 			);
 			if (speak) dc.send(JSON.stringify({ type: 'response.create' }));
 		},
+		sendText: (text) => {
+			const line = typeof text === 'string' ? text.trim() : '';
+			if (closed || dc.readyState !== 'open' || !line) return false;
+			try {
+				if (oaiResponseActive) {
+					dc.send(JSON.stringify({ type: 'response.cancel' }));
+					if (oaiSpeaking) dc.send(JSON.stringify({ type: 'output_audio_buffer.clear' }));
+				}
+				dc.send(
+					JSON.stringify({
+						type: 'conversation.item.create',
+						item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: line }] }
+					})
+				);
+				dc.send(JSON.stringify({ type: 'response.create' }));
+			} catch {
+				return false;
+			}
+			// A typed item produces no input-audio transcription event, so the SDK
+			// reports it itself — that is what puts it in the caller's transcript
+			// and in the memory connectCall consolidates at hang-up.
+			opts.onTranscript?.({ role: 'user', text: line });
+			return true;
+		},
+		setMuted: (muted) => {
+			userMuted = muted === true;
+			if (!closed) applyMic();
+		},
+		isMuted: () => userMuted,
+		canMute: true,
 		interrupt: () => {
 			// Silent cut: stop generating (response.cancel) AND flush what's
 			// already buffered for playback (output_audio_buffer.clear — the
@@ -729,6 +835,15 @@ async function openConvaiCall(
 			/* not supported on this client build — ignore */
 		}
 	};
+	// The reader's mute and the half-duplex mute are separate reasons (0.69.0,
+	// see the OpenAI leg); the mic opens only when neither holds.
+	let userMuted = opts.muted === true;
+	let speechMuted = false;
+	const applyMic = () => setConvMicMuted(userMuted || speechMuted);
+	// Lines `sendText` already reported. Should the provider ever echo a typed
+	// user_message back as a user transcript, it must not be reported twice
+	// (and land in memory twice). Matched by exact text, consumed once.
+	const typedEcho: string[] = [];
 	const conv = await mod.Conversation.startSession({
 		conversationToken: creds.token,
 		connectionType: 'webrtc',
@@ -777,12 +892,15 @@ async function openConvaiCall(
 					clearTimeout(micUnmuteTimer);
 					micUnmuteTimer = null;
 				}
-				setConvMicMuted(true);
+				speechMuted = true;
+				applyMic();
 			} else {
 				if (micUnmuteTimer) clearTimeout(micUnmuteTimer);
 				micUnmuteTimer = setTimeout(() => {
 					micUnmuteTimer = null;
-					if (!closed) setConvMicMuted(false);
+					if (closed) return;
+					speechMuted = false;
+					applyMic();
 				}, MIC_UNMUTE_TAIL_MS);
 			}
 		},
@@ -792,6 +910,13 @@ async function openConvaiCall(
 			const raw = (message ?? '').trim();
 			const text = creds.audioTags && role === 'agent' ? stripAudioTags(raw) : raw;
 			if (!closed && text) {
+				if (role !== 'agent') {
+					const echo = typedEcho.indexOf(text);
+					if (echo !== -1) {
+						typedEcho.splice(echo, 1);
+						return;
+					}
+				}
 				opts.onTranscript?.({ role: role === 'agent' ? 'assistant' : 'user', text });
 			}
 		},
@@ -834,6 +959,10 @@ async function openConvaiCall(
 		}
 	});
 
+	// The session is up: honour a start-muted request now (the EL client has no
+	// start-muted option — see CompanionCallOptions.muted).
+	if (userMuted) applyMic();
+
 	return {
 		provider: 'elevenlabs-convai',
 		close: () => {
@@ -854,6 +983,27 @@ async function openConvaiCall(
 				/* channel not ready */
 			}
 		},
+		sendText: (text) => {
+			const line = typeof text === 'string' ? text.trim() : '';
+			if (closed || !line) return false;
+			try {
+				// A `user_message`: the agent answers it as the user's turn, and
+				// the server's own turn-taking cuts an utterance in progress.
+				conv.sendUserMessage(line);
+			} catch {
+				return false;
+			}
+			typedEcho.push(line);
+			if (typedEcho.length > 20) typedEcho.shift();
+			opts.onTranscript?.({ role: 'user', text: line });
+			return true;
+		},
+		setMuted: (muted) => {
+			userMuted = muted === true;
+			if (!closed) applyMic();
+		},
+		isMuted: () => userMuted,
+		canMute: typeof (conv as unknown as { setMicMuted?: unknown }).setMicMuted === 'function',
 		interrupt: () => {
 			// BEST-EFFORT silent cut — the EL client has no public "stop playback"
 			// API (interruption is normally server-driven: a server `interruption`

package/src/client.ts

@@ -274,6 +274,13 @@ export type CallCredentials =
 			 *  the story's current state (see `CallWorldBrief`). ABSENT on an
 			 *  ordinary session. */
 			worldBrief?: CallWorldBrief;
+			/** Pouchy World role sessions only (0.68.0): whether the role can
+			 *  reach its private layer during this call (see `CallPrivateStory`).
+			 *  ABSENT on an ordinary session. */
+			privateStory?: CallPrivateStory;
+			/** What the server cut to keep `instructions` within the realtime cap
+			 *  (0.68.0; see `CallTrimmed`). ABSENT when nothing was cut. */
+			trimmed?: CallTrimmed;
 	  }
 	| {
 			provider: 'openai-realtime';
@@ -290,19 +297,45 @@ export type CallCredentials =
 			recordingUnavailable?: CallRecordingUnavailable;
 			/** See the elevenlabs-convai branch (0.67.0). */
 			worldBrief?: CallWorldBrief;
+			/** Always `'none'` on a world session here (0.68.0): the OpenAI
+			 *  Realtime fallback has no webhook tools. */
+			privateStory?: CallPrivateStory;
+			/** See the elevenlabs-convai branch (0.68.0). */
+			trimmed?: CallTrimmed;
 	  };
 
 /** Whether a Pouchy World role's call carries the story (0.67.0).
- *  - `included` — the call's instructions hold where the story is, as THIS
- *    role sees it (scene, clock, flags, the facts and notes it may know) and
- *    its own last beats. Nothing said on the call changes the story: fold the
- *    call back in as a beat on the world turn door (`trigger.context` +
- *    `proposedPatches`).
+ *  - `included` — the call's instructions hold the story's PUBLIC state (scene,
+ *    story clock, flags, the facts already revealed) and this role's own last
+ *    beats. Never its secrets, notes, goals or unrevealed facts: `instructions`
+ *    reach the reader's device (companion API 1.5.3). Nothing said on the call
+ *    changes the story: fold the call back in as a beat on the world turn door
+ *    (`trigger.context` + `proposedPatches`).
  *  - `no_story` — the world pins no story package.
  *  - `unavailable` — the story could not be read; the call went ahead without
  *    it. */
 export type CallWorldBrief = 'included' | 'no_story' | 'unavailable';
 
+/** Whether a Pouchy World role's call can reach the role's private layer
+ *  (0.68.0; companion API 1.6.1).
+ *  - `tool` — the call is told about, and carries the credential for, the
+ *    in-call `recall_private_story` tool: ElevenLabs asks Pouchy for the role's
+ *    secrets and unrevealed facts server-to-server, only when the conversation
+ *    needs them.
+ *  - `none` — it is not (the OpenAI Realtime fallback, a session with no memory
+ *    read scope, or the operator's switch is off): the role keeps its private
+ *    knowledge vague on this call. */
+export type CallPrivateStory = 'tool' | 'none';
+
+/** Characters the server removed to keep a call's `instructions` within the
+ *  realtime cap (0.68.0; companion API 1.5.2). `story` comes off the end of the
+ *  Pouchy World story block first, then `context` off the end of the persona /
+ *  memory context. Your `brief` and the call framing are never cut. */
+export interface CallTrimmed {
+	story?: number;
+	context?: number;
+}
+
 /** Why a call that asked for `record: true` is not recorded (0.65.0).
  *  - `not_enabled` — this agent's recording switch is off (the token lacks
  *    `call.record`; turn it on in the dashboard, then mint a new token).
@@ -2150,6 +2183,8 @@ export class CompanionClient {
 			...(creds.recording !== undefined ? { recording: creds.recording } : {}),
 			...(creds.recordingUnavailable ? { recordingUnavailable: creds.recordingUnavailable } : {}),
 			...(creds.worldBrief ? { worldBrief: creds.worldBrief } : {}),
+			...(creds.privateStory ? { privateStory: creds.privateStory } : {}),
+			...(creds.trimmed ? { trimmed: creds.trimmed } : {}),
 			close: () => {
 				close();
 				cleanup();

package/src/index.ts

@@ -31,6 +31,8 @@ export type {
 	CallRecordingUnavailable,
 	CallRecordingResult,
 	CallWorldBrief,
+	CallPrivateStory,
+	CallTrimmed,
 	CompanionToolDecl,
 	ToolCallEvent,
 	WorldStateInput,