@p8n.ai/pi-listens 0.1.0 → 0.1.1

package/CHANGELOG.md

@@ -6,6 +6,14 @@ This project follows [Semantic Versioning](https://semver.org/).
 
 ## [Unreleased]
 
+## [0.1.1] - 2026-05-09
+
+### Fixed
+
+- Return Sarvam STT results faster after flushing microphone audio.
+- Stop current speech playback before starting a new listen, without cancelling the new recording.
+- Keep spoken auto-summaries concise and avoid headings, hashtags, bullet lists, and boilerplate recaps.
+
 ## [0.1.0] - 2026-05-09
 
 ### Added
@@ -23,5 +31,6 @@ This project follows [Semantic Versioning](https://semver.org/).
 - Stop active audio capture/playback subprocesses when voice mode is closed or the Pi session shuts down.
 - Clean up generated audio files when spoken playback is interrupted.
 
-[Unreleased]: https://github.com/p8n-ai/pi-listens/compare/v0.1.0...HEAD
+[Unreleased]: https://github.com/p8n-ai/pi-listens/compare/v0.1.1...HEAD
 [0.1.0]: https://github.com/p8n-ai/pi-listens/releases/tag/v0.1.0
+[0.1.1]: https://github.com/p8n-ai/pi-listens/releases/tag/v0.1.1

package/README.md

@@ -80,9 +80,9 @@ The package registers these tools for Pi's agent:
 The extension also injects voice guidance into the system prompt:
 
 - use `voice_ask` whenever user input is needed in voice-first sessions
-- use `voice_output` for short spoken status or response snippets
+- use `voice_output` only for short spoken status or response snippets
+- keep spoken replies to 1-2 short sentences with no headings, hashtags, bullet lists, boilerplate recaps, or full task summaries
 - do not speak code blocks, logs, diffs, stack traces, or long explanations
-- keep spoken questions concise and answerable in a short response
 
 ## Commands
 
@@ -95,10 +95,10 @@ The extension also injects voice guidance into the system prompt:
 | `/voice-status` | Show setup and voice-mode status. |
 
 Voice panel controls in interactive mode:
-- R: listen now; press again while listening to stop listening
+- R: listen now; press again while listening to stop listening; if Pi is speaking, R stops playback before listening
 - A: auto-listen on/off (listen again after each assistant reply)
 - S: read aloud on/off (speak assistant replies)
-- Q: close the panel (and stop listening first if needed)
+- Q: close the panel and stop any active listening or speaking
 - Click the orb: visual ripple feedback (terminals with mouse reporting)
 
 ## Headless/RPC behavior
@@ -144,7 +144,7 @@ Example config file:
   "ttsOutputCodec": "wav",
   "textFallback": true,
   "autoSpeakAssistant": false,
-  "maxAutoSpeakChars": 900
+  "maxAutoSpeakChars": 320
 }
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@p8n.ai/pi-listens",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Pi package for speech-first interaction using Sarvam AI speech-to-text and text-to-speech.",
   "author": "Ravindra Barthwal",
   "license": "MIT",
@@ -33,7 +33,7 @@
   ],
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "npm run typecheck"
+    "test": "npm run typecheck && node --import tsx --test test/**/*.test.ts"
   },
   "pi": {
     "extensions": [

package/skills/pi-listens/SKILL.md

@@ -19,10 +19,11 @@ This Pi package provides voice tools backed by Sarvam AI.
 
 1. When you need user input, clarification, or confirmation, use `voice_ask` instead of asking only in text.
 2. Before using `voice_input`, make sure the user already knows you are listening. If not, use `voice_ask`.
-3. Use `voice_output` for concise spoken status updates or spoken summaries that matter to the user.
-4. Do not speak code blocks, diffs, stack traces, logs, long tables, or lengthy explanations. Summarize briefly and leave details in text.
-5. Treat transcripts returned by `voice_input` or `voice_ask` as user input, while allowing for speech-recognition mistakes. If the transcript is ambiguous, ask a short follow-up with `voice_ask`.
-6. If speech is not recognized, rely on the tool's text fallback when available, or ask again with a shorter prompt.
+3. Use `voice_output` only for concise spoken status updates or spoken summaries that matter to the user.
+4. Spoken output must be brief: 1-2 short sentences, no markdown headings, no hashtags, no bullet lists, no boilerplate recap, and no full task summaries. Leave details in text.
+5. Do not speak code blocks, diffs, stack traces, logs, long tables, or lengthy explanations. Summarize briefly and leave details in text.
+6. Treat transcripts returned by `voice_input` or `voice_ask` as user input, while allowing for speech-recognition mistakes. If the transcript is ambiguous, ask a short follow-up with `voice_ask`.
+7. If speech is not recognized, rely on the tool's text fallback when available, or ask again with a shorter prompt.
 
 ## Good voice question style
 

package/src/audio.ts

@@ -11,6 +11,7 @@ export interface AudioRuntime {
 	streamPcm(signal?: AbortSignal): AsyncIterable<Buffer>;
 	play(path: string, signal?: AbortSignal): Promise<void>;
 	cleanup(path: string): Promise<void>;
+	stopPlayback(): void;
 	stopAll(): void;
 	describe(): { recorder: string; player: string };
 }
@@ -41,7 +42,7 @@ export function createAudioRuntime(config: PiListensConfig): AudioRuntime {
 				: useUtteranceMode
 					? utteranceRecorderCommand(recorder, path, config.recordSampleRate, config.silenceStartSeconds, config.silenceStopSeconds, config.silenceThreshold)
 					: recorderCommand(recorder, path, seconds, config.recordSampleRate);
-			await run(command.command, command.args, signal, useUtteranceMode ? { timeoutMs: seconds * 1000, resolveOnTimeout: true } : undefined);
+			await run(command.command, command.args, signal, { ...(useUtteranceMode ? { timeoutMs: seconds * 1000, resolveOnTimeout: true } : {}), kind: "record" });
 			return path;
 		},
 
@@ -54,7 +55,7 @@ export function createAudioRuntime(config: PiListensConfig): AudioRuntime {
 			const command = config.streamCommand
 				? customCommand(config.streamCommand, { sampleRate: config.recordSampleRate })
 				: pcmStreamCommand(recorder, config.recordSampleRate);
-			return streamCommandOutput(command.command, command.args, signal);
+			return streamCommandOutput(command.command, command.args, signal, "record");
 		},
 
 		async play(path: string, signal?: AbortSignal): Promise<void> {
@@ -64,7 +65,7 @@ export function createAudioRuntime(config: PiListensConfig): AudioRuntime {
 				);
 			}
 			const command = config.playCommand ? customCommand(config.playCommand, { path }) : playerCommand(player, path);
-			await run(command.command, command.args, signal);
+			await run(command.command, command.args, signal, { kind: "play" });
 		},
 
 		async cleanup(path: string): Promise<void> {
@@ -72,6 +73,10 @@ export function createAudioRuntime(config: PiListensConfig): AudioRuntime {
 			await rm(path, { force: true }).catch(() => undefined);
 		},
 
+		stopPlayback(): void {
+			stopActiveAudioProcesses({ kind: "play" });
+		},
+
 		stopAll(): void {
 			stopActiveAudioProcesses();
 		},
@@ -213,14 +218,14 @@ function isCommandAvailable(command: string): boolean {
 	return false;
 }
 
-function run(command: string, args: string[], signal?: AbortSignal, options: { timeoutMs?: number; resolveOnTimeout?: boolean } = {}): Promise<void> {
+function run(command: string, args: string[], signal?: AbortSignal, options: { timeoutMs?: number; resolveOnTimeout?: boolean; kind?: AudioProcessKind } = {}): Promise<void> {
 	return new Promise((resolve, reject) => {
 		if (signal?.aborted) {
 			reject(new Error("Cancelled"));
 			return;
 		}
 
-		const child = spawnManaged(command, args);
+		const child = spawnManaged(command, args, options.kind ?? "other");
 		let stderr = "";
 		let stdout = "";
 		let timedOut = false;
@@ -265,9 +270,9 @@ function run(command: string, args: string[], signal?: AbortSignal, options: { t
 	});
 }
 
-async function* streamCommandOutput(command: string, args: string[], signal?: AbortSignal): AsyncIterable<Buffer> {
+async function* streamCommandOutput(command: string, args: string[], signal?: AbortSignal, kind: AudioProcessKind = "other"): AsyncIterable<Buffer> {
 	if (signal?.aborted) throw new Error("Cancelled");
-	const child = spawnManaged(command, args);
+	const child = spawnManaged(command, args, kind);
 	let stderr = "";
 	let exitCode: number | null = null;
 	let exitSignal: NodeJS.Signals | null = null;
@@ -298,23 +303,29 @@ async function* streamCommandOutput(command: string, args: string[], signal?: Ab
 	}
 }
 
+type AudioProcessKind = "record" | "play" | "other";
+
 type ManagedChild = ReturnType<typeof spawn>;
 
 const activeChildren = new Set<ManagedChild>();
+const childKinds = new WeakMap<ManagedChild, AudioProcessKind>();
 const terminatingChildren = new WeakSet<ManagedChild>();
 let processExitCleanupInstalled = false;
 
-export function stopActiveAudioProcesses(force = false): void {
-	for (const child of [...activeChildren]) terminateChild(child, force);
+export function stopActiveAudioProcesses(options: { kind?: AudioProcessKind; force?: boolean } = {}): void {
+	for (const child of [...activeChildren]) {
+		if (!options.kind || childKinds.get(child) === options.kind) terminateChild(child, options.force);
+	}
 }
 
-function spawnManaged(command: string, args: string[]): ManagedChild {
+function spawnManaged(command: string, args: string[], kind: AudioProcessKind): ManagedChild {
 	installProcessExitCleanup();
 	const child = spawn(command, args, {
 		stdio: ["ignore", "pipe", "pipe"],
 		detached: process.platform !== "win32",
 	});
 	activeChildren.add(child);
+	childKinds.set(child, kind);
 	const untrack = () => activeChildren.delete(child);
 	child.once("close", untrack);
 	child.once("error", untrack);
@@ -324,7 +335,7 @@ function spawnManaged(command: string, args: string[]): ManagedChild {
 function installProcessExitCleanup(): void {
 	if (processExitCleanupInstalled) return;
 	processExitCleanupInstalled = true;
-	process.once("exit", () => stopActiveAudioProcesses(true));
+	process.once("exit", () => stopActiveAudioProcesses({ force: true }));
 }
 
 function terminateChild(child: ManagedChild, force = false): void {

package/src/commands.ts

@@ -121,6 +121,7 @@ async function listenAndSend(
 		state.listenAbortController?.abort();
 		return;
 	}
+	stopSpeaking(services, state);
 	state.recordSeconds = seconds ?? services.getConfig().recordSeconds;
 	state.silenceStopSeconds = services.getConfig().silenceStopSeconds;
 	state.isListening = true;
@@ -253,6 +254,13 @@ function isCancelled(err: unknown): boolean {
 	return err instanceof Error && /cancelled|aborted/i.test(err.message);
 }
 
+function stopSpeaking(services: VoiceToolServices, state: VoiceModeState) {
+	const speakAbortController = state.speakAbortController;
+	state.speakAbortController = undefined;
+	speakAbortController?.abort();
+	services.getAudio().stopPlayback();
+}
+
 export function stopVoiceMode(services: VoiceToolServices, state: VoiceModeState, ctx?: ExtensionContext | ExtensionCommandContext) {
 	state.enabled = false;
 	state.autoListen = false;
@@ -264,10 +272,7 @@ export function stopVoiceMode(services: VoiceToolServices, state: VoiceModeState
 	state.listenAbortController = undefined;
 	listenAbortController?.abort();
 
-	const speakAbortController = state.speakAbortController;
-	state.speakAbortController = undefined;
-	speakAbortController?.abort();
-
+	stopSpeaking(services, state);
 	services.getAudio().stopAll();
 
 	if (ctx) uninstallVoiceUi(ctx, state);

package/src/config.ts

@@ -60,7 +60,7 @@ const DEFAULT_CONFIG: PiListensConfig = {
 	deleteAudio: true,
 	textFallback: true,
 	autoSpeakAssistant: false,
-	maxAutoSpeakChars: 900,
+	maxAutoSpeakChars: 320,
 };
 
 type RawConfig = Partial<PiListensConfig>;

package/src/index.ts

@@ -65,7 +65,7 @@ export default function piListensExtension(pi: ExtensionAPI) {
 
 	pi.on("before_agent_start", async (event) => {
 		return {
-			systemPrompt: `${event.systemPrompt}\n\nPi Listens voice guidance:\n- The user may primarily interact by speech through Sarvam AI. Text input is still possible.\n- When voice mode is active, treat it as a hands-free conversation: listen only while the voice UI/input tool is active, then pause listening while you work.\n- Use voice_output for concise spoken progress, completion, or status updates that matter to the user.\n- When you need clarification, confirmation, or any user input, prefer voice_ask with a concise spoken question instead of asking only in text.\n- Use voice_input only after the user already knows you are listening.\n- Do not speak code blocks, logs, diffs, stack traces, or long explanations; summarize them briefly and leave detail in text.`,
+			systemPrompt: `${event.systemPrompt}\n\nPi Listens voice guidance:\n- The user may primarily interact by speech through Sarvam AI. Text input is still possible.\n- When voice mode is active, treat it as a hands-free conversation: listen only while the voice UI/input tool is active, then pause listening while you work.\n- Use voice_output only for concise spoken progress, completion, or status updates that matter to the user.\n- Spoken replies must be brief: 1-2 short sentences, no headings, no hashtags, no bullet lists, no boilerplate recap, and no full task summaries. Leave details in text.\n- When you need clarification, confirmation, or any user input, prefer voice_ask with a concise spoken question instead of asking only in text.\n- Use voice_input only after the user already knows you are listening.\n- Do not speak code blocks, logs, diffs, stack traces, or long explanations; summarize briefly and leave detail in text.`,
 		};
 	});
 

package/src/sarvam.ts

@@ -1,5 +1,4 @@
 import { readFile, writeFile } from "node:fs/promises";
-import { setTimeout as delay } from "node:timers/promises";
 import { SarvamAIClient } from "sarvamai";
 import type { AudioRuntime } from "./audio.js";
 import type { PiListensConfig, SttMode } from "./config.js";
@@ -145,35 +144,46 @@ export class SarvamSpeechClient {
 		let languageProbability: number | undefined;
 		let streamError: Error | undefined;
 		let lastMessageAt = Date.now();
-
+		const messageWaiters = new Set<() => void>();
 		const socket = connectStreamingSocket(config, mode ?? (config.translateInputToEnglish ? "translate" : config.sttMode), inputAudioCodec);
 
 		const closeOnAbort = () => socket.close();
 		signal?.addEventListener("abort", closeOnAbort, { once: true });
+		const notifyMessageWaiters = () => {
+			const waiters = [...messageWaiters];
+			messageWaiters.clear();
+			for (const waiter of waiters) waiter();
+		};
 		socket.onMessage((message: StreamingResponse) => {
 			lastMessageAt = Date.now();
-			if (message.type === "error") {
-				streamError = new Error(message.data?.error ?? message.data?.code ?? "Sarvam streaming STT failed");
-				return;
+			try {
+				if (message.type === "error") {
+					streamError = new Error(message.data?.error ?? message.data?.code ?? "Sarvam streaming STT failed");
+					return;
+				}
+				if (message.type !== "data") return;
+				const data = message.data;
+				if (!data) return;
+				transcript = mergeTranscript(transcript, data.transcript ?? "");
+				requestId = data.request_id ?? requestId;
+				languageCode = data.language_code ?? languageCode;
+				languageProbability = data.language_probability ?? languageProbability;
+			} finally {
+				notifyMessageWaiters();
 			}
-			if (message.type !== "data") return;
-			const data = message.data;
-			if (!data) return;
-			transcript = mergeTranscript(transcript, data.transcript ?? "");
-			requestId = data.request_id ?? requestId;
-			languageCode = data.language_code ?? languageCode;
-			languageProbability = data.language_probability ?? languageProbability;
 		});
-		socket.onError((error: Error) => { streamError = error; });
+		socket.onError((error: Error) => { streamError = error; notifyMessageWaiters(); });
 
 		try {
 			await socket.waitForOpen();
 			await streamAudio(socket, async () => {
 				const startedWaitingAt = Date.now();
-				while (Date.now() - startedWaitingAt < 3000) {
+				const maxWaitMs = transcript.trim() ? 900 : 1600;
+				const settleMs = 250;
+				while (Date.now() - startedWaitingAt < maxWaitMs) {
 					if (streamError) throw streamError;
-					if (Date.now() - lastMessageAt > 850 && transcript.trim()) break;
-					await delay(100, undefined, { signal }).catch((err) => { throw err; });
+					if (transcript.trim() && Date.now() - lastMessageAt >= settleMs) break;
+					await waitForMessageOrTimeout(messageWaiters, 50, signal);
 				}
 			});
 			if (streamError) throw streamError;
@@ -288,6 +298,27 @@ function connectStreamingSocket(config: PiListensConfig, mode: SttMode, inputAud
 	};
 }
 
+function waitForMessageOrTimeout(waiters: Set<() => void>, timeoutMs: number, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve, reject) => {
+		if (signal?.aborted) {
+			reject(new Error("Cancelled"));
+			return;
+		}
+
+		const done = () => { cleanup(); resolve(); };
+		const onAbort = () => { cleanup(); reject(new Error("Cancelled")); };
+		const timeout = setTimeout(done, timeoutMs);
+		const cleanup = () => {
+			clearTimeout(timeout);
+			waiters.delete(done);
+			signal?.removeEventListener("abort", onAbort);
+		};
+
+		waiters.add(done);
+		signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}
+
 type CombinedSignal = { signal?: AbortSignal; cleanup: () => void };
 
 function combineSignals(...signals: Array<AbortSignal | undefined>): CombinedSignal {

package/src/text.ts

@@ -16,17 +16,32 @@ export function firstTextContent(message: unknown): string {
 
 export function prepareSpokenText(text: string, maxChars: number): string {
 	let prepared = text
-		.replace(/```[\s\S]*?```/g, " I am skipping a code block. ")
+		.replace(/```[\s\S]*?```/g, " I skipped a code block. ")
+		.replace(/^\s{0,3}#{1,6}\s+/gm, "")
+		.replace(/^\s*[-*+]\s+/gm, "")
+		.replace(/^\s*\d+[.)]\s+/gm, "")
 		.replace(/`([^`]+)`/g, "$1")
 		.replace(/https?:\/\/\S+/g, "link")
+		.replace(/[#*_>~|]+/g, " ")
 		.replace(/\s+/g, " ")
 		.trim();
+
+	prepared = conciseSpokenSummary(prepared);
 	if (prepared.length > maxChars) {
-		prepared = `${prepared.slice(0, Math.max(0, maxChars - 80)).trim()}… I have more details on screen.`;
+		prepared = `${prepared.slice(0, Math.max(0, maxChars - 32)).trim()}… More on screen.`;
 	}
 	return prepared;
 }
 
+function conciseSpokenSummary(text: string): string {
+	const sentences = text.match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(Boolean) ?? [];
+	if (sentences.length === 0) return text;
+
+	const useful = sentences.filter((sentence) => !/^(sure|here('|’)s|summary|in summary|done|completed|i('|’)ve|i have)\b/i.test(sentence));
+	const picked = (useful.length ? useful : sentences).slice(0, 2).join(" ").trim();
+	return picked || text;
+}
+
 export function conciseTranscript(transcript: string): string {
 	const trimmed = transcript.trim();
 	return trimmed.length === 0 ? "(no speech recognized)" : trimmed;

package/src/tools.ts

@@ -49,8 +49,8 @@ export function registerVoiceTools(pi: ExtensionAPI, services: VoiceToolServices
 		description: "Speak a short message to the user using Sarvam AI text-to-speech and local audio playback.",
 		promptSnippet: "Speak short user-facing messages with Sarvam AI TTS",
 		promptGuidelines: [
-			"Use voice_output when a spoken user-facing message matters, especially before waiting for voice input.",
-			"Keep voice_output text brief and conversational; do not speak code blocks, command output, stack traces, or long explanations.",
+			"Use voice_output only when a spoken user-facing message matters, especially before waiting for voice input.",
+			"Keep voice_output to 1-2 short conversational sentences. Do not speak headings, hashtags, bullet lists, boilerplate recaps, code, command output, stack traces, or long explanations.",
 		],
 		parameters: VoiceOutputParams,
 		async execute(_toolCallId, params: VoiceOutputInput, signal, onUpdate) {