@oh-my-pi/pi-coding-agent 3.15.0 → 3.20.0

package/src/core/voice.ts

@@ -0,0 +1,308 @@
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { completeSimple, type Model } from "@oh-my-pi/pi-ai";
+import voiceSummaryPrompt from "../prompts/voice-summary.md" with { type: "text" };
+import { logger } from "./logger";
+import type { ModelRegistry } from "./model-registry";
+import { findSmolModel } from "./model-resolver";
+import type { VoiceSettings } from "./settings-manager";
+
+const DEFAULT_SAMPLE_RATE = 16000;
+const DEFAULT_CHANNELS = 1;
+const DEFAULT_BITS = 16;
+const SUMMARY_MAX_CHARS = 6000;
+
+export interface VoiceRecordingHandle {
+	filePath: string;
+	stop: () => Promise<void>;
+	cancel: () => Promise<void>;
+	cleanup: () => void;
+}
+
+export interface VoiceTranscriptionResult {
+	text: string;
+}
+
+export interface VoiceSynthesisResult {
+	audio: Uint8Array;
+	format: "wav" | "mp3" | "opus" | "aac" | "flac";
+}
+
+function buildRecordingCommand(filePath: string, sampleRate: number, channels: number): string[] | null {
+	const soxPath = Bun.which("sox") ?? Bun.which("rec");
+	if (soxPath) {
+		return [soxPath, "-d", "-r", String(sampleRate), "-c", String(channels), "-b", String(DEFAULT_BITS), filePath];
+	}
+
+	const arecordPath = Bun.which("arecord");
+	if (arecordPath) {
+		return [arecordPath, "-f", "S16_LE", "-r", String(sampleRate), "-c", String(channels), filePath];
+	}
+
+	const ffmpegPath = Bun.which("ffmpeg");
+	if (ffmpegPath) {
+		const platform = process.platform;
+		if (platform === "darwin") {
+			// avfoundation default input device; users can override by installing sox for reliability.
+			return [
+				ffmpegPath,
+				"-f",
+				"avfoundation",
+				"-i",
+				":0",
+				"-ac",
+				String(channels),
+				"-ar",
+				String(sampleRate),
+				"-y",
+				filePath,
+			];
+		}
+		if (platform === "linux") {
+			// alsa default input device (commonly "default").
+			return [
+				ffmpegPath,
+				"-f",
+				"alsa",
+				"-i",
+				"default",
+				"-ac",
+				String(channels),
+				"-ar",
+				String(sampleRate),
+				"-y",
+				filePath,
+			];
+		}
+		if (platform === "win32") {
+			// dshow default input device name varies; "audio=default" is a best-effort fallback.
+			return [
+				ffmpegPath,
+				"-f",
+				"dshow",
+				"-i",
+				"audio=default",
+				"-ac",
+				String(channels),
+				"-ar",
+				String(sampleRate),
+				"-y",
+				filePath,
+			];
+		}
+	}
+
+	return null;
+}
+
+export async function startVoiceRecording(_settings: VoiceSettings): Promise<VoiceRecordingHandle> {
+	const sampleRate = DEFAULT_SAMPLE_RATE;
+	const channels = DEFAULT_CHANNELS;
+	const filePath = join(tmpdir(), `omp-voice-${Date.now()}.wav`);
+	const command = buildRecordingCommand(filePath, sampleRate, channels);
+	if (!command) {
+		throw new Error("No audio recorder found (install sox, arecord, or ffmpeg).");
+	}
+
+	logger.debug("voice: starting recorder", { command });
+	const proc = Bun.spawn(command, {
+		stdin: "ignore",
+		stdout: "ignore",
+		stderr: "pipe",
+	});
+
+	const stop = async (): Promise<void> => {
+		try {
+			proc.kill();
+		} catch {
+			// ignore
+		}
+		await proc.exited;
+	};
+
+	const cleanup = (): void => {
+		try {
+			unlinkSync(filePath);
+		} catch {
+			// ignore cleanup errors
+		}
+	};
+
+	const cancel = async (): Promise<void> => {
+		await stop();
+		cleanup();
+	};
+
+	return { filePath, stop, cancel, cleanup };
+}
+
+export async function transcribeAudio(
+	filePath: string,
+	apiKey: string,
+	settings: VoiceSettings,
+): Promise<VoiceTranscriptionResult> {
+	const file = Bun.file(filePath);
+	const buffer = await file.arrayBuffer();
+	const blob = new File([buffer], "speech.wav", { type: "audio/wav" });
+	const form = new FormData();
+	form.append("file", blob);
+	form.append("model", settings.transcriptionModel ?? "whisper-1");
+	if (settings.transcriptionLanguage) {
+		form.append("language", settings.transcriptionLanguage);
+	}
+
+	const response = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+		method: "POST",
+		headers: { Authorization: `Bearer ${apiKey}` },
+		body: form,
+	});
+
+	if (!response.ok) {
+		const errText = await response.text();
+		throw new Error(`Whisper transcription failed: ${response.status} ${errText}`);
+	}
+
+	const data = (await response.json()) as { text?: string };
+	return { text: (data.text ?? "").trim() };
+}
+
+export async function synthesizeSpeech(
+	text: string,
+	apiKey: string,
+	settings: VoiceSettings,
+): Promise<VoiceSynthesisResult> {
+	const format = settings.ttsFormat ?? "wav";
+	const response = await fetch("https://api.openai.com/v1/audio/speech", {
+		method: "POST",
+		headers: {
+			Authorization: `Bearer ${apiKey}`,
+			"Content-Type": "application/json",
+		},
+		body: JSON.stringify({
+			model: settings.ttsModel ?? "tts-1",
+			voice: settings.ttsVoice ?? "alloy",
+			format,
+			input: text,
+		}),
+	});
+
+	if (!response.ok) {
+		const errText = await response.text();
+		throw new Error(`TTS synthesis failed: ${response.status} ${errText}`);
+	}
+
+	const audio = new Uint8Array(await response.arrayBuffer());
+	return { audio, format };
+}
+
+function getPlayerCommand(filePath: string, format: VoiceSynthesisResult["format"]): string[] | null {
+	const platform = process.platform;
+	if (platform === "darwin") {
+		const afplay = Bun.which("afplay");
+		if (afplay) return [afplay, filePath];
+	}
+
+	if (platform === "linux") {
+		const paplay = Bun.which("paplay");
+		if (paplay) return [paplay, filePath];
+		const aplay = Bun.which("aplay");
+		if (aplay) return [aplay, filePath];
+		const ffplay = Bun.which("ffplay");
+		if (ffplay) return [ffplay, "-autoexit", "-nodisp", filePath];
+		const play = Bun.which("play");
+		if (play) return [play, filePath];
+	}
+
+	if (platform === "win32") {
+		if (format !== "wav") {
+			return null;
+		}
+		const ps = Bun.which("powershell");
+		if (ps) {
+			return [
+				ps,
+				"-NoProfile",
+				"-Command",
+				`(New-Object Media.SoundPlayer '${filePath.replace(/'/g, "''")}').PlaySync()`,
+			];
+		}
+	}
+
+	return null;
+}
+
+export async function playAudio(audio: Uint8Array, format: VoiceSynthesisResult["format"]): Promise<void> {
+	const filePath = join(tmpdir(), `omp-voice-tts-${Date.now()}.${format}`);
+	await Bun.write(filePath, audio);
+
+	const command = getPlayerCommand(filePath, format);
+	if (!command) {
+		throw new Error("No audio player available for playback.");
+	}
+
+	const proc = Bun.spawn(command, {
+		stdin: "ignore",
+		stdout: "ignore",
+		stderr: "pipe",
+	});
+	await proc.exited;
+
+	try {
+		unlinkSync(filePath);
+	} catch {
+		// ignore cleanup errors
+	}
+}
+
+function extractTextFromResponse(response: { content: Array<{ type: string; text?: string }> }): string {
+	let text = "";
+	for (const content of response.content) {
+		if (content.type === "text" && content.text) {
+			text += content.text;
+		}
+	}
+	return text.trim();
+}
+
+export async function summarizeForVoice(
+	text: string,
+	registry: ModelRegistry,
+	savedSmolModel?: string,
+): Promise<string | null> {
+	const model = await findSmolModel(registry, savedSmolModel);
+	if (!model) {
+		logger.debug("voice: no smol model found for summary");
+		return null;
+	}
+
+	const apiKey = await registry.getApiKey(model);
+	if (!apiKey) {
+		logger.debug("voice: no API key for summary model", { provider: model.provider, id: model.id });
+		return null;
+	}
+
+	const truncated = text.length > SUMMARY_MAX_CHARS ? `${text.slice(0, SUMMARY_MAX_CHARS)}...` : text;
+	const request = {
+		model: `${model.provider}/${model.id}`,
+		systemPrompt: voiceSummaryPrompt,
+		userMessage: `<assistant_response>\n${truncated}\n</assistant_response>`,
+	};
+	logger.debug("voice: summary request", request);
+
+	try {
+		const response = await completeSimple(
+			model as Model<any>,
+			{
+				systemPrompt: request.systemPrompt,
+				messages: [{ role: "user", content: request.userMessage, timestamp: Date.now() }],
+			},
+			{ apiKey, maxTokens: 120 },
+		);
+		const summary = extractTextFromResponse(response);
+		return summary || null;
+	} catch (error) {
+		logger.debug("voice: summary error", { error: error instanceof Error ? error.message : String(error) });
+		return null;
+	}
+}

package/src/discovery/builtin.ts

@@ -5,9 +5,10 @@
  * .pi is an alias for backwards compatibility.
  */
 
-import { basename, dirname, join } from "node:path";
+import { basename, dirname, isAbsolute, join, resolve } from "path";
 import { type ContextFile, contextFileCapability } from "../capability/context-file";
 import { type Extension, type ExtensionManifest, extensionCapability } from "../capability/extension";
+import { type ExtensionModule, extensionModuleCapability } from "../capability/extension-module";
 import { type Hook, hookCapability } from "../capability/hook";
 import { registerProvider } from "../capability/index";
 import { type Instruction, instructionCapability } from "../capability/instruction";
@@ -22,7 +23,9 @@ import { type CustomTool, toolCapability } from "../capability/tool";
 import type { LoadContext, LoadResult } from "../capability/types";
 import {
 	createSourceMeta,
+	discoverExtensionModulePaths,
 	expandEnvVarsDeep,
+	getExtensionNameFromPath,
 	loadFilesFromDir,
 	parseFrontmatter,
 	parseJSON,
@@ -361,6 +364,77 @@ registerProvider<Prompt>(promptCapability.id, {
 	load: loadPrompts,
 });
 
+// Extension Modules
+function loadExtensionModules(ctx: LoadContext): LoadResult<ExtensionModule> {
+	const items: ExtensionModule[] = [];
+	const warnings: string[] = [];
+
+	const resolveExtensionPath = (rawPath: string): string => {
+		if (rawPath.startsWith("~/")) {
+			return join(ctx.home, rawPath.slice(2));
+		}
+		if (rawPath.startsWith("~")) {
+			return join(ctx.home, rawPath.slice(1));
+		}
+		if (isAbsolute(rawPath)) {
+			return rawPath;
+		}
+		return resolve(ctx.cwd, rawPath);
+	};
+
+	const addExtensionPath = (extPath: string, level: "user" | "project"): void => {
+		items.push({
+			name: getExtensionNameFromPath(extPath),
+			path: extPath,
+			level,
+			_source: createSourceMeta(PROVIDER_ID, extPath, level),
+		});
+	};
+
+	for (const { dir, level } of getConfigDirs(ctx)) {
+		const extensionsDir = join(dir, "extensions");
+		const discovered = discoverExtensionModulePaths(ctx, extensionsDir);
+		for (const extPath of discovered) {
+			addExtensionPath(extPath, level);
+		}
+
+		const settingsPath = join(dir, "settings.json");
+		const settingsContent = ctx.fs.readFile(settingsPath);
+		if (settingsContent) {
+			const settingsData = parseJSON<{ extensions?: unknown }>(settingsContent);
+			const extensions = settingsData?.extensions;
+			if (Array.isArray(extensions)) {
+				for (const entry of extensions) {
+					if (typeof entry !== "string") {
+						warnings.push(`Invalid extension path in ${settingsPath}: ${String(entry)}`);
+						continue;
+					}
+					const resolvedPath = resolveExtensionPath(entry);
+					if (ctx.fs.isDir(resolvedPath)) {
+						for (const extPath of discoverExtensionModulePaths(ctx, resolvedPath)) {
+							addExtensionPath(extPath, level);
+						}
+					} else if (ctx.fs.isFile(resolvedPath)) {
+						addExtensionPath(resolvedPath, level);
+					} else {
+						warnings.push(`Extension path not found: ${resolvedPath}`);
+					}
+				}
+			}
+		}
+	}
+
+	return { items, warnings };
+}
+
+registerProvider<ExtensionModule>(extensionModuleCapability.id, {
+	id: PROVIDER_ID,
+	displayName: DISPLAY_NAME,
+	description: DESCRIPTION,
+	priority: PRIORITY,
+	load: loadExtensionModules,
+});
+
 // Extensions
 function loadExtensions(ctx: LoadContext): LoadResult<Extension> {
 	const items: Extension[] = [];

package/src/discovery/claude.ts

@@ -5,8 +5,9 @@
  * Priority: 80 (tool-specific, below builtin but above shared standards)
  */
 
-import { dirname, join, sep } from "node:path";
+import { dirname, join, sep } from "path";
 import { type ContextFile, contextFileCapability } from "../capability/context-file";
+import { type ExtensionModule, extensionModuleCapability } from "../capability/extension-module";
 import { type Hook, hookCapability } from "../capability/hook";
 import { registerProvider } from "../capability/index";
 import { type MCPServer, mcpCapability } from "../capability/mcp";
@@ -19,7 +20,9 @@ import type { LoadContext, LoadResult } from "../capability/types";
 import {
 	calculateDepth,
 	createSourceMeta,
+	discoverExtensionModulePaths,
 	expandEnvVarsDeep,
+	getExtensionNameFromPath,
 	loadFilesFromDir,
 	parseFrontmatter,
 	parseJSON,
@@ -292,6 +295,41 @@ function loadSkills(ctx: LoadContext): LoadResult<Skill> {
 	return { items, warnings };
 }
 
+// =============================================================================
+// Extension Modules
+// =============================================================================
+
+function loadExtensionModules(ctx: LoadContext): LoadResult<ExtensionModule> {
+	const items: ExtensionModule[] = [];
+	const warnings: string[] = [];
+
+	const userBase = getUserClaude(ctx);
+	const userExtensionsDir = join(userBase, "extensions");
+	for (const extPath of discoverExtensionModulePaths(ctx, userExtensionsDir)) {
+		items.push({
+			name: getExtensionNameFromPath(extPath),
+			path: extPath,
+			level: "user",
+			_source: createSourceMeta(PROVIDER_ID, extPath, "user"),
+		});
+	}
+
+	const projectBase = getProjectClaude(ctx);
+	if (projectBase) {
+		const projectExtensionsDir = join(projectBase, "extensions");
+		for (const extPath of discoverExtensionModulePaths(ctx, projectExtensionsDir)) {
+			items.push({
+				name: getExtensionNameFromPath(extPath),
+				path: extPath,
+				level: "project",
+				_source: createSourceMeta(PROVIDER_ID, extPath, "project"),
+			});
+		}
+	}
+
+	return { items, warnings };
+}
+
 // =============================================================================
 // Slash Commands
 // =============================================================================
@@ -582,6 +620,14 @@ registerProvider<Skill>(skillCapability.id, {
 	load: loadSkills,
 });
 
+registerProvider<ExtensionModule>(extensionModuleCapability.id, {
+	id: PROVIDER_ID,
+	displayName: DISPLAY_NAME,
+	description: "Load extension modules from .claude/extensions",
+	priority: PRIORITY,
+	load: loadExtensionModules,
+});
+
 registerProvider<SlashCommand>(slashCommandCapability.id, {
 	id: PROVIDER_ID,
 	displayName: DISPLAY_NAME,

package/src/discovery/codex.ts

@@ -7,10 +7,11 @@
  * User directory: ~/.codex
  */
 
-import { join } from "node:path";
+import { join } from "path";
 import { parse as parseToml } from "smol-toml";
 import type { ContextFile } from "../capability/context-file";
 import { contextFileCapability } from "../capability/context-file";
+import { type ExtensionModule, extensionModuleCapability } from "../capability/extension-module";
 import type { Hook } from "../capability/hook";
 import { hookCapability } from "../capability/hook";
 import { registerProvider } from "../capability/index";
@@ -27,7 +28,14 @@ import { slashCommandCapability } from "../capability/slash-command";
 import type { CustomTool } from "../capability/tool";
 import { toolCapability } from "../capability/tool";
 import type { LoadContext, LoadResult } from "../capability/types";
-import { createSourceMeta, loadFilesFromDir, parseFrontmatter, SOURCE_PATHS } from "./helpers";
+import {
+	createSourceMeta,
+	discoverExtensionModulePaths,
+	getExtensionNameFromPath,
+	loadFilesFromDir,
+	parseFrontmatter,
+	SOURCE_PATHS,
+} from "./helpers";
 
 const PROVIDER_ID = "codex";
 const DISPLAY_NAME = "OpenAI Codex";
@@ -251,6 +259,42 @@ function loadSkills(ctx: LoadContext): LoadResult<Skill> {
 	return { items, warnings };
 }
 
+// =============================================================================
+// Extension Modules (extensions/)
+// =============================================================================
+
+function loadExtensionModules(ctx: LoadContext): LoadResult<ExtensionModule> {
+	const items: ExtensionModule[] = [];
+	const warnings: string[] = [];
+
+	// User level: ~/.codex/extensions/
+	const userExtensionsDir = join(ctx.home, SOURCE_PATHS.codex.userBase, "extensions");
+	for (const extPath of discoverExtensionModulePaths(ctx, userExtensionsDir)) {
+		items.push({
+			name: getExtensionNameFromPath(extPath),
+			path: extPath,
+			level: "user",
+			_source: createSourceMeta(PROVIDER_ID, extPath, "user"),
+		});
+	}
+
+	// Project level: .codex/extensions/
+	const codexDir = ctx.fs.walkUp(".codex", { dir: true });
+	if (codexDir) {
+		const projectExtensionsDir = join(codexDir, "extensions");
+		for (const extPath of discoverExtensionModulePaths(ctx, projectExtensionsDir)) {
+			items.push({
+				name: getExtensionNameFromPath(extPath),
+				path: extPath,
+				level: "project",
+				_source: createSourceMeta(PROVIDER_ID, extPath, "project"),
+			});
+		}
+	}
+
+	return { items, warnings };
+}
+
 // =============================================================================
 // Slash Commands (commands/)
 // =============================================================================
@@ -530,6 +574,14 @@ registerProvider<Skill>(skillCapability.id, {
 	load: loadSkills,
 });
 
+registerProvider<ExtensionModule>(extensionModuleCapability.id, {
+	id: PROVIDER_ID,
+	displayName: DISPLAY_NAME,
+	description: "Load extension modules from ~/.codex/extensions and .codex/extensions/",
+	priority: PRIORITY,
+	load: loadExtensionModules,
+});
+
 registerProvider<SlashCommand>(slashCommandCapability.id, {
 	id: PROVIDER_ID,
 	displayName: DISPLAY_NAME,

package/src/discovery/gemini.ts

@@ -16,15 +16,25 @@
  * - settings: From settings.json
  */
 
-import { dirname, join, sep } from "node:path";
+import { dirname, join, sep } from "path";
 import { type ContextFile, contextFileCapability } from "../capability/context-file";
 import { type Extension, type ExtensionManifest, extensionCapability } from "../capability/extension";
+import { type ExtensionModule, extensionModuleCapability } from "../capability/extension-module";
 import { registerProvider } from "../capability/index";
 import { type MCPServer, mcpCapability } from "../capability/mcp";
 import { type Settings, settingsCapability } from "../capability/settings";
 import { type SystemPrompt, systemPromptCapability } from "../capability/system-prompt";
 import type { LoadContext, LoadResult } from "../capability/types";
-import { calculateDepth, createSourceMeta, expandEnvVarsDeep, getProjectPath, getUserPath, parseJSON } from "./helpers";
+import {
+	calculateDepth,
+	createSourceMeta,
+	discoverExtensionModulePaths,
+	expandEnvVarsDeep,
+	getExtensionNameFromPath,
+	getProjectPath,
+	getUserPath,
+	parseJSON,
+} from "./helpers";
 
 const PROVIDER_ID = "gemini";
 const DISPLAY_NAME = "Gemini CLI";
@@ -236,6 +246,41 @@ function loadExtensionsFromDir(
 	return { items, warnings };
 }
 
+// =============================================================================
+// Extension Modules
+// =============================================================================
+
+function loadExtensionModules(ctx: LoadContext): LoadResult<ExtensionModule> {
+	const items: ExtensionModule[] = [];
+	const warnings: string[] = [];
+
+	const userExtensionsDir = getUserPath(ctx, "gemini", "extensions");
+	if (userExtensionsDir) {
+		for (const extPath of discoverExtensionModulePaths(ctx, userExtensionsDir)) {
+			items.push({
+				name: getExtensionNameFromPath(extPath),
+				path: extPath,
+				level: "user",
+				_source: createSourceMeta(PROVIDER_ID, extPath, "user"),
+			});
+		}
+	}
+
+	const projectExtensionsDir = getProjectPath(ctx, "gemini", "extensions");
+	if (projectExtensionsDir) {
+		for (const extPath of discoverExtensionModulePaths(ctx, projectExtensionsDir)) {
+			items.push({
+				name: getExtensionNameFromPath(extPath),
+				path: extPath,
+				level: "project",
+				_source: createSourceMeta(PROVIDER_ID, extPath, "project"),
+			});
+		}
+	}
+
+	return { items, warnings };
+}
+
 // =============================================================================
 // Settings
 // =============================================================================
@@ -359,6 +404,14 @@ registerProvider(extensionCapability.id, {
 	load: loadExtensions,
 });
 
+registerProvider(extensionModuleCapability.id, {
+	id: PROVIDER_ID,
+	displayName: DISPLAY_NAME,
+	description: "Load extension modules from ~/.gemini/extensions/ and .gemini/extensions/",
+	priority: PRIORITY,
+	load: loadExtensionModules,
+});
+
 registerProvider(settingsCapability.id, {
 	id: PROVIDER_ID,
 	displayName: DISPLAY_NAME,