@agentprojectcontext/apx 1.50.1 → 1.51.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentprojectcontext/apx",
-  "version": "1.50.1",
+  "version": "1.51.1",
   "description": "APX — unified CLI + daemon for the Agent Project Context (APC) standard.",
   "publishConfig": {
     "access": "public"

package/src/core/agent/prompt-builder.js

@@ -21,6 +21,7 @@ import { agentsMdFile } from "../apc/paths.js";
 import { readSelfMemoryForPrompt } from "./self-memory.js";
 import { buildSkillsHintBlock } from "./skills/catalog.js";
 import { CHANNELS } from "#core/constants/channels.js";
+import { activeEmotionGuide, buildEmotionGuide } from "../voice/emotions.js";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const PROMPTS_DIR = path.join(__dirname, "prompts");
@@ -99,13 +100,16 @@ export function buildChannelContextBlock(channel, meta = {}) {
   return renderPromptTemplate(loadPrompt(rel), meta);
 }
 
-export function buildVoiceModeBlock(active) {
+export function buildVoiceModeBlock(active, emotionGuide = "") {
   if (!active) return "";
+  let base = "";
   try {
-    return loadPrompt(VOICE_MODE_FILE);
+    base = loadPrompt(VOICE_MODE_FILE);
   } catch {
-    return "";
+    base = "";
   }
+  if (!emotionGuide) return base;
+  return base ? `${base}\n\n${emotionGuide}` : emotionGuide;
 }
 
 // Pick the right segmenting discipline for the channel (and whether voice
@@ -287,7 +291,14 @@ export function buildSuperAgentSystem({
 
   const channelBlock = buildChannelContextBlock(channel, channelMeta);
   const extraContext = [channelBlock, contextNote].filter(Boolean).join("\n\n");
-  const voiceBlock = buildVoiceModeBlock(voice);
+  // In voice mode, if the engine that will speak supports inline emotion tags
+  // (a per-engine config toggle), teach the agent the syntax. channelMeta
+  // .ttsProvider optionally forces which engine's capability to honor.
+  const emotion = voice ? activeEmotionGuide(globalConfig, channelMeta?.ttsProvider) : null;
+  const voiceBlock = buildVoiceModeBlock(
+    voice,
+    emotion ? buildEmotionGuide(emotion.tags) : ""
+  );
   const segmentDiscipline = buildSegmentDiscipline({ channel: channelLow, voice });
 
   return [

package/src/core/voice/emotions.js

@@ -0,0 +1,108 @@
+// Generic "emotion tags" capability for TTS engines.
+//
+// Some speech backends (e.g. a local Qwen3-TTS / QVox server, reached through
+// the OpenAI-compatible `openai` adapter pointed at a custom base_url) accept
+// inline [tag] markers in the text and switch the speaking emotion per segment
+// while keeping the same base voice. QVox's tag set is the canonical default
+// (see qwen3-tts-api `_app.py` TAG_INSTRUCTS).
+//
+// This capability is NOT hard-coded into any adapter. It's a per-engine config
+// toggle (`voice.tts.<id>.emotions.enabled`) so it can be ADDED to whichever
+// engine the user actually points at an emotion-aware backend — the custom
+// OpenAI endpoint today, anything tomorrow. Two responsibilities live here:
+//   1. activeEmotionGuide() — what the prompt builder injects (voice mode only)
+//      so the agent learns the tag syntax, but ONLY when the engine that will
+//      speak supports tags.
+//   2. stripEmotionTags() — a safety net so that if a turn ends up on an engine
+//      WITHOUT tag support, stray markers are scrubbed and never read aloud.
+
+import { resolveMode, resolveChainOrder, isCustomId, CUSTOM_PREFIX } from "./engines/index.js";
+
+// Canonical tag set (mirrors QVox TAG_INSTRUCTS keys, de-duplicated).
+export const DEFAULT_EMOTION_TAGS = [
+  "happy", "sad", "excited", "angry", "calm",
+  "whisper", "shout", "laugh", "cry", "narrator", "neutral",
+];
+
+const TAG_RE = /\[[a-zA-Z]{2,12}\]/g;
+
+function ttsCfg(globalConfig) {
+  return globalConfig?.voice?.tts || {};
+}
+
+// Per-engine config block (built-in id → voice.tts.<id>; custom:<slug> →
+// voice.tts.custom.<slug>).
+function providerBlock(globalConfig, providerId) {
+  const tts = ttsCfg(globalConfig);
+  if (isCustomId(providerId)) return tts?.custom?.[providerId.slice(CUSTOM_PREFIX.length)] || {};
+  return tts?.[providerId] || {};
+}
+
+function enabledOf(tts, id) {
+  if (isCustomId(id)) return tts?.custom?.[id.slice(CUSTOM_PREFIX.length)]?.enabled !== false;
+  return tts?.[id]?.enabled !== false;
+}
+
+/**
+ * Emotion capability declared for one engine id. `enabled` is false when the
+ * block is missing or turned off; `tags` falls back to the canonical set.
+ */
+export function emotionConfigFor(globalConfig, providerId) {
+  const e = providerBlock(globalConfig, providerId)?.emotions;
+  const enabled = !!e?.enabled;
+  const tags = Array.isArray(e?.tags) && e.tags.length
+    ? e.tags.map((s) => String(s).trim().toLowerCase()).filter(Boolean)
+    : DEFAULT_EMOTION_TAGS;
+  return { enabled, tags };
+}
+
+/**
+ * Best-effort, SYNCHRONOUS resolution of which engine will speak. Used at
+ * prompt-build time (which is sync and must not probe isAvailable()). Mirrors
+ * the intent of selectTtsEngine without the async availability probes:
+ *   - explicit provider arg wins
+ *   - single mode → voice.tts.provider
+ *   - chain mode → the FIRST enabled engine in order (what selectTtsEngine
+ *     would speak with). We deliberately do NOT prefer an emotion-capable
+ *     engine here: the guide must reflect the engine that will actually speak,
+ *     otherwise the agent emits tags a different engine never asked for.
+ */
+export function resolveSpeakingProvider(globalConfig, provider) {
+  if (provider && provider !== "auto") return provider;
+  const cfg = ttsCfg(globalConfig);
+  const mode = resolveMode(cfg);
+  if (mode === "single" && cfg.provider && cfg.provider !== "auto") return cfg.provider;
+  const order = resolveChainOrder(cfg).filter(
+    (id) => id !== "mock" && enabledOf(cfg, id)
+  );
+  return order[0] || cfg.provider || undefined;
+}
+
+/**
+ * The emotion guide to inject for a voice-mode turn, or null when the engine
+ * that will speak does not support tags. `provider` optionally forces a
+ * specific engine (e.g. a tester override).
+ */
+export function activeEmotionGuide(globalConfig, provider) {
+  const id = resolveSpeakingProvider(globalConfig, provider);
+  if (!id) return null;
+  const { enabled, tags } = emotionConfigFor(globalConfig, id);
+  return enabled ? { provider: id, tags } : null;
+}
+
+/** Markdown block teaching the inline-tag syntax. Appended to modes/voice.md. */
+export function buildEmotionGuide(tags = DEFAULT_EMOTION_TAGS) {
+  const list = (Array.isArray(tags) && tags.length ? tags : DEFAULT_EMOTION_TAGS).join("] [");
+  return [
+    "## Emotion tags (spoken delivery)",
+    "The voice engine for this turn understands inline emotion tags. You MAY drop them into your spoken reply to color the delivery — a tag affects the words that follow it, until the next tag.",
+    `Available tags: [${list}]`,
+    'Write the tag in square brackets right before the phrase it colors (e.g. "[excited] ¡Listo! [calm] Lo dejé anotado."). Use them sparingly — at most one or two per reply, only when the emotion genuinely helps. The tags are removed before synthesis (never spoken). Never invent tags outside the list above.',
+  ].join("\n");
+}
+
+/** Remove stray [tag] markers so a non-tag engine never reads them aloud. */
+export function stripEmotionTags(text) {
+  if (typeof text !== "string") return text;
+  return text.replace(TAG_RE, "").replace(/[ \t]{2,}/g, " ").trim();
+}

package/src/core/voice/engines/index.js

@@ -28,7 +28,28 @@ export const TTS_ENGINE_IDS = Object.keys(ADAPTERS);
 
 export const AUTO_PREFERENCE = ["piper", "elevenlabs", "openai", "gemini", "mock"];
 
+// ── Custom providers ────────────────────────────────────────────────────────
+// Users can add any number of OpenAI-compatible endpoints (e.g. a local QVox /
+// Qwen3-TTS server). They live under voice.tts.custom.<slug> and surface with
+// engine id "custom:<slug>". They're all backed by the openai adapter.
+export const CUSTOM_PREFIX = "custom:";
+
+export function isCustomId(id) {
+  return typeof id === "string" && id.startsWith(CUSTOM_PREFIX);
+}
+function slugOf(id) {
+  return isCustomId(id) ? id.slice(CUSTOM_PREFIX.length) : id;
+}
+function customEngineIds(ttsCfg) {
+  return Object.keys(ttsCfg?.custom || {}).map((slug) => CUSTOM_PREFIX + slug);
+}
+function knownIds(ttsCfg) {
+  return [...TTS_ENGINE_IDS, ...customEngineIds(ttsCfg)];
+}
+
 export function getTtsAdapter(provider) {
+  // All custom providers are OpenAI-compatible → openai adapter.
+  if (isCustomId(provider)) return ADAPTERS.openai;
   const a = ADAPTERS[provider];
   if (!a) {
     throw new Error(
@@ -43,10 +64,13 @@ function ttsConfig(globalConfig) {
 }
 
 function providerConfig(globalConfig, provider) {
-  return ttsConfig(globalConfig)?.[provider] || {};
+  const tts = ttsConfig(globalConfig);
+  if (isCustomId(provider)) return tts?.custom?.[slugOf(provider)] || {};
+  return tts?.[provider] || {};
 }
 
 function isEnabled(ttsCfg, id) {
+  if (isCustomId(id)) return ttsCfg?.custom?.[slugOf(id)]?.enabled !== false;
   return ttsCfg?.[id]?.enabled !== false;
 }
 
@@ -63,11 +87,14 @@ export function resolveMode(ttsCfg) {
  * so the UI can render + reorder every row; filtering happens at selection time.
  */
 export function resolveChainOrder(ttsCfg) {
-  const custom = Array.isArray(ttsCfg?.order)
-    ? ttsCfg.order.filter((id) => TTS_ENGINE_IDS.includes(id))
+  const known = knownIds(ttsCfg);
+  const ordered = Array.isArray(ttsCfg?.order)
+    ? ttsCfg.order.filter((id) => known.includes(id))
     : [];
-  const rest = AUTO_PREFERENCE.filter((id) => !custom.includes(id));
-  const full = [...custom, ...rest];
+  const rest = [...AUTO_PREFERENCE, ...customEngineIds(ttsCfg)].filter(
+    (id) => !ordered.includes(id)
+  );
+  const full = [...ordered, ...rest];
   // Guarantee mock is present as the ultimate fallback.
   if (!full.includes("mock")) full.push("mock");
   return full;
@@ -101,7 +128,7 @@ export async function selectTtsEngine({ globalConfig, provider }) {
   // 3. Chain mode: probe the (enabled) order, first available wins.
   for (const id of resolveChainOrder(ttsCfg)) {
     if (id !== "mock" && !isEnabled(ttsCfg, id)) continue;
-    const adapter = ADAPTERS[id];
+    const adapter = getTtsAdapter(id);
     const cfg = providerConfig(globalConfig, id);
     try {
       if (await adapter.isAvailable(cfg, globalConfig?.engines)) {
@@ -124,20 +151,23 @@ export async function selectTtsEngine({ globalConfig, provider }) {
 export async function listAvailableTtsEngines(globalConfig) {
   const ttsCfg = ttsConfig(globalConfig);
   const out = [];
-  for (const id of TTS_ENGINE_IDS) {
-    const adapter = ADAPTERS[id];
+  for (const id of knownIds(ttsCfg)) {
+    const adapter = getTtsAdapter(id);
     const cfg = providerConfig(globalConfig, id);
     let available = false;
     try {
       available = await adapter.isAvailable(cfg, globalConfig?.engines);
     } catch { available = false; }
+    const custom = isCustomId(id);
     out.push({
       id,
       available,
       // `enabled` is a routing flag, not real config — exclude it from the
       // "configured" heuristic so toggling on/off doesn't fake-mark an engine.
-      configured: Object.keys(cfg).filter((k) => k !== "enabled").length > 0,
+      // For custom engines `label` is descriptive metadata, not config either.
+      configured: Object.keys(cfg).filter((k) => k !== "enabled" && k !== "label").length > 0,
       enabled: isEnabled(ttsCfg, id),
+      ...(custom ? { custom: true, label: cfg.label || slugOf(id), note: cfg.base_url || "" } : {}),
     });
   }
   return out;

package/src/core/voice/engines/openai.js

@@ -1,18 +1,29 @@
-// OpenAI TTS adapter (tts-1 / tts-1-hd).
+// OpenAI TTS adapter (tts-1 / tts-1-hd) — and any OpenAI-compatible endpoint.
 // Docs: https://platform.openai.com/docs/api-reference/audio/createSpeech
 //
 // Reuses engines.openai.api_key from ~/.apx/config.json. Per-engine voice
 // config (~/.apx/config.json → voice.tts.openai) can override model/voice.
+//
+// Custom endpoint ("QVox custom"): set voice.tts.openai.base_url to a local
+// OpenAI-compatible speech server (e.g. a Qwen3-TTS / QVox daemon at
+// http://127.0.0.1:5111/v1). When base_url is set we additionally forward the
+// non-OpenAI fields that server understands — `instruct` (the base voice, from
+// the `style` arg), `language` and `temperature`. These extras are NEVER sent
+// to stock OpenAI (only when base_url is present), so the standard path stays
+// byte-for-byte compatible.
 
 import fs from "node:fs";
 import path from "node:path";
 import { randomUUID } from "node:crypto";
 
-const API_URL = "https://api.openai.com/v1/audio/speech";
+const DEFAULT_API_URL = "https://api.openai.com/v1/audio/speech";
 const DEFAULT_MODEL = "tts-1";
 const DEFAULT_VOICE = "alloy";          // alloy|echo|fable|onyx|nova|shimmer
 
 function getKey(config, parentEnginesCfg) {
+  // A custom endpoint uses ONLY its own key (often none); never leak the stock
+  // OpenAI engine key / OPENAI_API_KEY env to a third-party server.
+  if (config.base_url) return config.api_key || "";
   return (
     config.api_key ||
     parentEnginesCfg?.openai?.api_key ||
@@ -21,6 +32,13 @@ function getKey(config, parentEnginesCfg) {
   );
 }
 
+function endpoint(config) {
+  if (config.base_url) {
+    return config.base_url.replace(/\/+$/, "") + "/audio/speech";
+  }
+  return DEFAULT_API_URL;
+}
+
 function mimeFor(format) {
   return {
     mp3: "audio/mpeg",
@@ -36,30 +54,48 @@ export default {
   id: "openai",
 
   async isAvailable(config = {}, parentEnginesCfg) {
-    return Boolean(getKey(config, parentEnginesCfg));
+    // A custom endpoint is assumed reachable (it may be keyless/open like QVox);
+    // stock OpenAI needs a key.
+    return Boolean(config.base_url) || Boolean(getKey(config, parentEnginesCfg));
   },
 
-  async synthesize({ text, voice, outDir, config = {}, format, signal, parentEnginesCfg }) {
+  async synthesize({ text, voice, language, style, outDir, config = {}, format, signal, parentEnginesCfg }) {
     if (!text) throw new Error("openai-tts: empty text");
+    const isCustom = Boolean(config.base_url);
     const key = getKey(config, parentEnginesCfg);
-    if (!key) throw new Error("openai-tts: no api_key (set OPENAI_API_KEY or engines.openai.api_key)");
+    if (!isCustom && !key) {
+      throw new Error("openai-tts: no api_key (set OPENAI_API_KEY or engines.openai.api_key)");
+    }
 
-    const model = config.model || DEFAULT_MODEL;
-    const chosenVoice = voice || config.voice || DEFAULT_VOICE;
-    const responseFormat = format || config.format || "mp3";
+    const url = endpoint(config);
+    const model = config.model || (isCustom ? undefined : DEFAULT_MODEL);
+    const chosenVoice = voice || config.voice || (isCustom ? undefined : DEFAULT_VOICE);
+    const responseFormat = format || config.format || (isCustom ? "wav" : "mp3");
+    const styleHint = String(style ?? config.style ?? "").trim();
+
+    const body = { input: text, response_format: responseFormat };
+    if (model) body.model = model;
+    if (chosenVoice) body.voice = chosenVoice;
+    if (isCustom) {
+      // QVox / Qwen3-TTS extras (ignored by stock OpenAI, so only sent here).
+      if (styleHint) body.instruct = styleHint;
+      if (language) body.language = language;
+      if (config.temperature != null) body.temperature = config.temperature;
+    } else if (styleHint && /gpt-4o.*tts/i.test(model || "")) {
+      // Stock OpenAI's newer TTS models accept a natural-language `instructions`.
+      body.instructions = styleHint;
+    }
+
+    const headers = { "content-type": "application/json" };
+    if (key) {
+      headers.authorization = `Bearer ${key}`;
+      if (isCustom) headers["x-api-key"] = key; // QVox accepts either header.
+    }
 
-    const res = await fetch(API_URL, {
+    const res = await fetch(url, {
       method: "POST",
-      headers: {
-        authorization: `Bearer ${key}`,
-        "content-type": "application/json",
-      },
-      body: JSON.stringify({
-        model,
-        voice: chosenVoice,
-        input: text,
-        response_format: responseFormat,
-      }),
+      headers,
+      body: JSON.stringify(body),
       signal,
     });
     if (!res.ok) {

package/src/core/voice/tts.js

@@ -17,6 +17,7 @@ import {
   resolveMode,
   resolveChainOrder,
 } from "./engines/index.js";
+import { emotionConfigFor, stripEmotionTags } from "./emotions.js";
 
 export const TTS_TMP_DIR = path.join(os.homedir(), ".apx", "tmp", "tts");
 
@@ -61,9 +62,15 @@ export async function synthesize({
     provider,
   });
 
+  // Safety net: if the engine that will speak does NOT support inline emotion
+  // tags, scrub any stray [tag] markers so they're never read aloud literally.
+  const speakText = emotionConfigFor(cfg, selectedProvider).enabled
+    ? text
+    : stripEmotionTags(text);
+
   const outDir = ensureTtsTmpDir();
   return adapter.synthesize({
-    text,
+    text: speakText,
     voice,
     language,
     format,
@@ -72,7 +79,7 @@ export async function synthesize({
     config: engineConfig,
     parentEnginesCfg: cfg.engines,
     signal,
-  }).then((r) => ({ ...r, provider: r.provider || selectedProvider }));
+  }).then((r) => ({ ...r, provider: selectedProvider || r.provider }));
 }
 
 /** List engines and whether they look usable right now. */

package/src/host/daemon/api/voice.js

@@ -22,6 +22,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { readConfig } from "#core/config/index.js";
 import { synthesize } from "#core/voice/tts.js";
+import { stripEmotionTags } from "#core/voice/emotions.js";
 import { transcribe } from "#core/voice/transcription.js";
 import { decodeAudioInput } from "#core/voice/audio-decode.js";
 import { runSuperAgent, isSuperAgentEnabled } from "#core/agent/super-agent.js";
@@ -174,6 +175,12 @@ export function register(app, { projects, plugins, registries }) {
         replyText = userText;
       }
 
+      // Emotion tags ([excited], [whisper], …) are a TTS-only signal: keep them
+      // for synthesis (engines that support them split on them), but strip them
+      // from everything the user reads — the chat bubble, history, RAG.
+      const audioText = replyText;
+      replyText = stripEmotionTags(replyText);
+
       // Persist the turn to the cross-channel store (feeds RAG index,
       // search_messages, and the "active threads" block). channelCtx.channel is
       // the resolved surface ("deck"/"desktop"). Best-effort.
@@ -190,10 +197,11 @@ export function register(app, { projects, plugins, registries }) {
       if (replyText) {
         try {
           tts = await synthesize({
-            text: replyText,
+            text: audioText,
             voice: body.voice,
             language: body.language,
             provider: body.provider,
+            style: body.style,
             format: body.format_out,
             globalConfig: cfg,
           });