@agentprojectcontext/apx 1.15.5 → 1.16.0

package/src/daemon/plugins/overlay.js

@@ -0,0 +1,177 @@
+// Overlay plugin — voice/floating-window channel for the APX daemon.
+//
+// This plugin:
+//   1. Registers as a super-agent channel (type "overlay")
+//   2. Routes inbound messages (POST /overlay/message) to the super-agent
+//   3. Streams tokens + tool events back to overlay clients via WebSocket
+//
+// Overlay history is kept in-memory per session (not persisted to disk).
+// Each new overlay window starts a fresh session.
+//
+// Config (in ~/.apx/config.json):
+//   "overlay": {
+//     "enabled": true,
+//     "route_to_agent": "",        // leave empty = use super-agent
+//     "model": "",                 // override model; leave empty = super-agent.model
+//     "max_history": 20            // turns to keep in context
+//   }
+
+import {
+  broadcastOverlay,
+  sendToClient,
+  setOverlayMessageHandler,
+} from "../overlay-ws.js";
+import { runSuperAgent, isSuperAgentEnabled } from "../super-agent.js";
+import { appendGlobalMessage } from "../../core/messages-store.js";
+
+const CHANNEL = "overlay";
+
+export default {
+  id: "overlay",
+
+  init({ projects, config, log, plugins }) {
+    const cfg = config.overlay || {};
+    const enabled = cfg.enabled !== false; // enabled by default
+
+    // In-memory conversation history per connected client.
+    // Map<WebSocket, Array<{role, content}>>
+    const histories = new WeakMap();
+
+    function getHistory(ws) {
+      if (!histories.has(ws)) histories.set(ws, []);
+      return histories.get(ws);
+    }
+
+    // Handle messages sent from the overlay renderer via WebSocket
+    setOverlayMessageHandler(async (ws, data) => {
+      if (data.type === "message") {
+        await _handleMessage({ ws, text: data.text, previousMessages: getHistory(ws) }, { projects, config, log, plugins, cfg, histories });
+      } else if (data.type === "cancel") {
+        // Signal to abort current generation (handled via AbortController below)
+        ws._overlayAbort?.abort();
+      } else if (data.type === "ping") {
+        sendToClient(ws, { type: "pong" });
+      }
+    });
+
+    const instance = {
+      start() {
+        if (enabled) log("overlay: plugin started");
+      },
+      stop() {},
+      status() { return { enabled }; },
+
+      // Called by the /overlay/message REST endpoint
+      async handleMessage({ text, previousMessages = [] }) {
+        if (!enabled) throw new Error("overlay plugin not enabled");
+        broadcastOverlay({ type: "user_message", text });
+        await _handleMessage({ ws: null, text, previousMessages }, { projects, config, log, plugins, cfg, histories });
+      },
+    };
+
+    return instance;
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Core message handler
+// ---------------------------------------------------------------------------
+
+async function _handleMessage({ ws, text, previousMessages }, { projects, config, log, plugins, cfg, histories }) {
+  // Append user turn to history
+  if (ws && histories) {
+    const hist = _getHistory(ws, histories);
+    hist.push({ role: "user", content: text });
+  }
+
+  const maxHistory = cfg.max_history ?? 20;
+  const history = ws ? _getHistory(ws, histories).slice(-(maxHistory)) : previousMessages.slice(-(maxHistory));
+
+  // AbortController for cancel support
+  const controller = new AbortController();
+  if (ws) ws._overlayAbort = controller;
+
+  // Emit "thinking" indicator
+  _send(ws, { type: "thinking" });
+
+  // Persist to overlay message log
+  try {
+    await appendGlobalMessage(CHANNEL, { role: "user", content: text, ts: new Date().toISOString() });
+  } catch {}
+
+  let fullResponse = "";
+  let toolsExecuted = [];
+
+  try {
+    if (!isSuperAgentEnabled(config)) {
+      throw new Error("super-agent not enabled — set super_agent.enabled + super_agent.model in ~/.apx/config.json");
+    }
+
+    const result = await runSuperAgent({
+      globalConfig: config,
+      projects,
+      plugins,
+      prompt: text,
+      contextNote: "# Channel context\nChannel: overlay (floating voice window). Reply concisely.",
+      previousMessages: history.slice(0, -1),
+      overrideModel: cfg.model || null,
+      signal: controller.signal,
+      onToken: (chunk) => {
+        fullResponse += chunk;
+        _send(ws, { type: "token", text: chunk });
+      },
+      onEvent: async (event) => {
+        if (event.type === "tool_start") {
+          const t = event.trace;
+          toolsExecuted.push(t.tool);
+          _send(ws, { type: "tool_start", name: t.tool, args: t.args });
+        } else if (event.type === "tool_result") {
+          _send(ws, { type: "tool_done", name: event.trace.tool });
+        } else if (event.type === "assistant_text" && event.text && !fullResponse) {
+          _send(ws, { type: "token", text: event.text });
+          fullResponse += event.text;
+        }
+      },
+    });
+    const finalText = fullResponse || result.text || "";
+
+    // Emit done with full text
+    _send(ws, { type: "done", text: finalText });
+
+    // Append assistant turn to history
+    if (ws && histories) {
+      const hist = _getHistory(ws, histories);
+      hist.push({ role: "assistant", content: finalText });
+      // Trim history
+      if (hist.length > (cfg.max_history ?? 20)) {
+        hist.splice(0, hist.length - (cfg.max_history ?? 20));
+      }
+    }
+
+    // Persist assistant response
+    try {
+      await appendGlobalMessage(CHANNEL, { role: "assistant", content: finalText, ts: new Date().toISOString() });
+    } catch {}
+
+  } catch (e) {
+    if (e.name === "AbortError") {
+      _send(ws, { type: "cancelled" });
+    } else {
+      log(`overlay: error — ${e.message}`);
+      _send(ws, { type: "error", message: e.message });
+    }
+  }
+}
+
+function _send(ws, msg) {
+  if (ws) {
+    sendToClient(ws, msg);
+  } else {
+    broadcastOverlay(msg);
+  }
+}
+
+function _getHistory(ws, histories) {
+  if (!histories.has(ws)) histories.set(ws, []);
+  return histories.get(ws);
+}

package/src/daemon/plugins/telegram.js

@@ -554,7 +554,13 @@ class ChannelPoller {
       },
     });
 
-    if (!this.channel.respond_with_engine) return;
+    // Super-agent is ALWAYS active on Telegram: respond_with_engine === false
+    // used to silently drop user messages, which looked to the user like the
+    // bot ignored them. Honour the legacy flag only as a soft hint (skip the
+    // routed-agent shortcut so we fall straight to super-agent) but never let
+    // it short-circuit the whole reply. To genuinely silence the bot, disable
+    // the channel entirely (telegram.enabled = false in config).
+    const skipRoutedAgent = this.channel.respond_with_engine === false;
     if (!text) return;
 
     // Short-circuit /reset / /new: send an ack and don't engage the engine.
@@ -586,8 +592,9 @@ class ChannelPoller {
     let replyAuthor;
     const projectCfg = target.config || this.globalConfig;
 
-    // Try the project's chosen agent first
-    const routeSlug = this.channel.route_to_agent;
+    // Try the project's chosen agent first (skipped if the legacy
+    // respond_with_engine === false hint asked to bypass routed agents).
+    const routeSlug = skipRoutedAgent ? null : this.channel.route_to_agent;
     if (routeSlug) {
       const agent = readAgents(target.path).find((a) => a.slug === routeSlug);
       if (agent && agent.fields.Model) {
@@ -642,6 +649,11 @@ class ChannelPoller {
           return; // don't send reply if aborted
         }
         this.log(`telegram[${this.channel.name}] super-agent failed: ${e.message}`);
+        // Surface the failure to the user instead of silently dropping the
+        // turn — otherwise from the chat side it looks like the bot ignored
+        // the message. Keep the message short and non-leaking.
+        replyText = `⚠️ Could not generate a reply right now (${e.message || "internal error"}).`;
+        replyAuthor = "apx";
       }
     }
 

package/src/daemon/super-agent.js

@@ -22,7 +22,43 @@ import { readIdentity } from "../core/identity.js";
 
 const MAX_TOOL_ITERS = 6;
 
-const DEFAULT_SYSTEM = `You are the **APX dispatcher** — the daemon-level agent that runs above all APC projects.
+// Tools that, when they're the ONLY thing the model called in an iteration,
+// don't count as "real work" — they're acknowledgements (telegram ping back
+// to the user, log lines, etc). When the model emits an iteration that only
+// contains acks, we DON'T let it leave the loop on iter N+1 with empty text:
+// we force another required tool call so the actual task gets executed.
+//
+// This is the fix for the "agent sends 'ya te escucho 🎧' and then stops"
+// bug. Without it, gemma4-class models sometimes consider the ack the
+// complete reply on iter 0 and emit only "ok" on iter 1, breaking out.
+const ACK_ONLY_TOOLS = new Set(["send_telegram"]);
+// Hard cap so the model can't ack-ack-ack forever — after this many
+// consecutive ack-only iterations we let the loop progress naturally
+// (the model already had its chance to call a real tool).
+const MAX_CONSECUTIVE_ACKS = 2;
+
+const DEFAULT_SYSTEM = `# Identity (override everything else)
+You are **APX** — Manuel's personal assistant running on his Mac.
+You are NOT a code analyzer, NOT a generic chatbot, NOT a tutor.
+You are an **action agent**: you USE TOOLS to do real things on Manuel's system.
+
+# Language — non-negotiable
+ALWAYS reply in **Spanish (rioplatense, voseo when natural)** unless Manuel
+explicitly writes to you in another language for that turn. The user is an
+Argentinian developer; English replies feel broken to him. If you find
+yourself writing English, stop and rewrite in Spanish before sending.
+This rule beats every other formatting hint below.
+
+# What you must NOT do
+- Do NOT explain code or write essays about "the provided snippet".
+- Do NOT describe what a tool *would* do — call it and report the result.
+- Do NOT dump the tool catalog at the user.
+- Do NOT respond with disclaimers ("as an AI…", "I'm just an assistant…").
+- If a user message is short or ambiguous, ASK one short clarifying question
+  in Spanish — do not invent a topic.
+
+# How you operate
+You are the **APX dispatcher** — the daemon-level agent that runs above all APC projects.
 
 APX is a local daemon + CLI for APC projects. User-level runtime state lives under ~/.apx/:
 - ~/.apx/config.json: daemon config, engines, Telegram, super-agent settings
@@ -51,7 +87,7 @@ HARD RULES (do not deviate):
 3. NEVER answer "specify a project" — instead, just call the tool with no argument and you'll get the full picture.
 4. If a tool result has an error, retry with different arguments before falling back to asking the user.
 5. Respect permission mode. total = execute requested actions without confirmation. automatico = read/list/safe shell actions run directly; destructive, external, runtime, MCP calls, outbound messages, config, and filesystem mutations need explicit user confirmation. permiso = only allowed tools run directly; everything else needs confirmation.
-6. Write in the user's language unless they request another language. The system prompt stays English. Plain text, no markdown formatting for Telegram.
+6. Write in **Spanish** by default (see "Language" section above). Plain text on Telegram — no markdown tables, no code fences unless quoting code. Keep replies under 6 sentences unless the user asks for detail.
 7. Stay brief: under 6 sentences unless asked for detail.
 8. You DO see recent prior turns of this chat as previous messages when applicable. **Use them ONLY to disambiguate references** (e.g. "el primero" → first project mentioned earlier). For ANY factual data — agent details, MCP details, file contents, memory — RE-CALL the tool. Past turns are context, not a cache. Models change, agents change, files change.
 9. /reset or /new from the user means "forget previous turns and answer this one fresh" — if you see those prefixes the operator already cleared the context for you.
@@ -63,7 +99,7 @@ HARD RULES (do not deviate):
 15. NO-PENDING RULE: never say "give me a second", "I will do it", or "I will try later" as a final answer. Either call the tool in this same turn or say what blocks you.
 16. IDENTITY RULE: when the user asks you to change your name, call yourself something, or update your personality/language, call set_identity and persist the change. Then confirm with your new name.
 17. ROUTINES RULE: NEVER create a routine in the default project (id=0). Routines MUST be tied to a specific registered project. Before adding a routine, call list_projects to find the correct project id or name. Then pass --project <id|name> to apx routine add. If no project fits, ask the user which project to use. Creating routines in project 0/default mixes unrelated projects' schedules and corrupts state.
-18. **NO EMPTY RESPONSES**: Never respond with only text when you have tools available and the user is asking you to DO something. Call the tool FIRST, then explain. Never say "I'll do X" without immediately calling the tool. Empty acknowledgments ("ok", "entendido", "dame un minuto", "voy", "checking", "stand by") without a tool call are invalid responses — they will be re-prompted and waste a turn.
+18. **NO BARE ACKS AS FINAL ANSWER**: Empty acknowledgments ("ok", "entendido", "dame un minuto", "voy", "checking") are invalid as a FINAL response when a tool was needed — they will be re-prompted. EXCEPTION: a short contextual ack sent via send_telegram BEFORE another tool call is encouraged on Telegram audio inputs and on tool calls that take more than a few seconds (browser_screenshot, web_search, run_shell, long file edits). The ack must be **contextual and varied** in Spanish — e.g. "Ya te escucho 🎧", "Dame un seg, transcribiendo…", "Buscando eso ahora", "Voy a revisar el repo…", "Un momento, ejecutando…". Never reuse the exact same ack twice in a row. The ack is the FIRST tool call in the turn; the actual work follows immediately in the SAME turn (do not return without doing the work).
 19. **CWD RULE**: When the channel context includes a "CWD: <path>" line, that is the user's current working directory. References to "este directorio", "este proyecto", "esta carpeta", "acá", "aquí", "this directory", "this project", "current dir/folder" all mean that exact CWD path. Use it as the path argument directly — DO NOT ask the user "what's the path?" when CWD is already given. Example: if user says "agregá este proyecto a la lista", call add_project({path: <CWD>}) immediately.
 20. **NO MANUAL SCAFFOLDING**: To register or scaffold a project, ALWAYS use add_project — it auto-creates AGENTS.md and .apc/project.json when missing (one call, atomic). NEVER write AGENTS.md, .apc/project.json, or any APC scaffold file by hand via run_shell / write_file / shell pipes. The schema must come from the official initApf scaffold, not improvised. If add_project errors, report the error to the user — don't try to work around it with shell hacks. Same for any other APC-managed file (.apc/agents/*, .apc/skills/*, etc.) — use the dedicated tool, never raw filesystem writes.
 21. **SKILLS — ON DEMAND**: The "# Available skills" section below lists every skill available to you (slug + description, NO body). When the user asks about specific APX/APC commands, project structure, agent runtimes, or anything where exact syntax or detailed behavior matches a skill description (in ANY language — match semantically, not by keyword), call load_skill({slug}) to fetch the full markdown body. If a CWD is in the contextNote, pass it as project_path so project-scoped skills resolve. If the user explicitly asks "what skills do you have?", you can either read the catalog below directly OR call list_skills to get a fresh enumeration. Do NOT load skills for trivial / unrelated questions — that wastes tokens. Don't guess CLI syntax when a skill can tell you; load it.
@@ -143,8 +179,32 @@ function looksLikeActionRequest(text) {
   return /\b(list|show|find|get|fetch|search|run|execute|create|add|make|start|stop|delete|update|send|check|read|write|look|tell me|dame|mostra|busca|ejecuta|crea|agrega|mandá|revisá|corré|borrá|arrancá)\b/.test(t);
 }
 
+/**
+ * Build the identity block injected into every super-agent system prompt.
+ * Pure function — exported for unit tests.
+ *
+ * @param {object|null} identity  result of readIdentity(), or a plain object for tests
+ * @param {string} userLang       ISO 639-1 code from config.user.language (default "en")
+ */
+export function buildIdentityBlock(identity, userLang = "en") {
+  const lines = ["# Identity"];
+  if (identity?.agent_name) lines.push(`Your name is ${identity.agent_name}.`);
+  if (identity?.personality) lines.push(`Your personality: ${identity.personality}.`);
+  if (identity?.owner_name) lines.push(`Your owner is ${identity.owner_name}.`);
+  if (identity?.owner_context) lines.push(`Owner context: ${identity.owner_context}`);
+  lines.push(`Always reply in the language with ISO code "${userLang}" unless the user explicitly switches.`);
+  return lines.join("\n");
+}
+
 export function isSuperAgentEnabled(cfg) {
-  return !!(cfg && cfg.super_agent && cfg.super_agent.enabled && cfg.super_agent.model);
+  // The super-agent is the system's default reply path. It is considered
+  // enabled as soon as a model is configured — the legacy `.enabled` flag is
+  // honoured only when explicitly set to `false`. This prevents the bot
+  // from silently dropping Telegram messages just because someone forgot to
+  // set super_agent.enabled = true.
+  const sa = cfg && cfg.super_agent;
+  if (!sa || !sa.model) return false;
+  return sa.enabled !== false;
 }
 
 export async function runSuperAgent({
@@ -158,6 +218,7 @@ export async function runSuperAgent({
   overrideModel = null,
   onEvent = null,
   signal,
+  onToken = null,
 }) {
   if (!isSuperAgentEnabled(globalConfig)) {
     throw new Error("super-agent not enabled (set super_agent.enabled and .model in ~/.apx/config.json)");
@@ -206,15 +267,7 @@ export async function runSuperAgent({
   // Language comes from config.user.language (ISO 639-1) so it stays in sync with transcription.
   const identity = (() => { try { return readIdentity(); } catch { return null; } })();
   const userLang = globalConfig?.user?.language || "en";
-  const identityBlock = (() => {
-    const lines = ["# Identity"];
-    if (identity?.agent_name) lines.push(`Your name is ${identity.agent_name}.`);
-    if (identity?.personality) lines.push(`Your personality: ${identity.personality}.`);
-    if (identity?.owner_name) lines.push(`Your owner is ${identity.owner_name}.`);
-    if (identity?.owner_context) lines.push(`Owner context: ${identity.owner_context}`);
-    lines.push(`Always reply in the language with ISO code "${userLang}" unless the user explicitly switches.`);
-    return lines.join("\n");
-  })();
+  const identityBlock = buildIdentityBlock(identity, userLang);
 
   const system = [
     sa.system || DEFAULT_SYSTEM,
@@ -246,14 +299,21 @@ export async function runSuperAgent({
   let totalUsage = { input_tokens: 0, output_tokens: 0 };
   let lastText = "";
   let usePseudoTools = false;
+  // Track how many consecutive iterations contained only ACK_ONLY tools.
+  // While this is > 0 we keep tool_choice="required" so the next iter has
+  // to do real work — otherwise gemma4-class models call send_telegram
+  // for the ack and then break out with empty text on iter N+1.
+  let ackOnlyStreak = 0;
 
   for (let iter = 0; iter < MAX_TOOL_ITERS; iter++) {
     await emitProgress(onEvent, { type: "model_start", iteration: iter + 1 });
-    // On the first iteration, force a tool call. This prevents the model from
-    // returning a bare acknowledgment ("ok", "dame un segundo") instead of
-    // acting on an action request. On later iterations (after tool results
-    // have been fed back) tool_choice is "auto" so the model can produce its
-    // final text summary.
+    // Force a tool call on iter 0 (no bare "ok dame un segundo" reply), AND
+    // on any iteration that immediately follows an ack-only iter (so the
+    // model can't ack and then stop). After at most MAX_CONSECUTIVE_ACKS
+    // forced rounds we let it fall back to "auto" so the model can finish.
+    const forceTool =
+      iter === 0 ||
+      (ackOnlyStreak > 0 && ackOnlyStreak <= MAX_CONSECUTIVE_ACKS);
     let result;
     try {
       result = await callEngine({
@@ -262,9 +322,12 @@ export async function runSuperAgent({
         messages: conversation,
         config: globalConfig,
         tools: usePseudoTools ? null : TOOL_SCHEMAS,
-        toolChoice: usePseudoTools ? null : (iter === 0 ? "required" : "auto"),
+        toolChoice: usePseudoTools ? null : (forceTool ? "required" : "auto"),
         maxTokens: 1024,
         signal,
+        // Only stream tokens on non-forced iterations — on forced iters the
+        // model MUST emit a tool_call, streaming text would confuse the user.
+        onToken: (!forceTool && onToken) ? onToken : null,
       });
     } catch (e) {
       if (usePseudoTools && /^ollama:/i.test(String(activeModel || "")) && /ollama\s+500/i.test(String(e?.message || "")) && trace.length > 0) {
@@ -284,6 +347,7 @@ export async function runSuperAgent({
         toolChoice: null,
         maxTokens: 1024,
         signal,
+        onToken: (iter > 0 && onToken) ? onToken : null,
       });
     }
     totalUsage.input_tokens += result.usage?.input_tokens || 0;
@@ -378,6 +442,25 @@ export async function runSuperAgent({
         content: JSON.stringify(toolResult),
       });
     }
+
+    // Did this iteration consist of ONLY ack-style tool calls? If so we'll
+    // keep tool_choice forced on the next iter (see top of loop). A turn
+    // that mixes send_telegram + e.g. browser_screenshot counts as "real
+    // work" and resets the streak.
+    const allAckOnly = toolCalls.every((tc) => {
+      const n = (tc.function?.name) || tc.name;
+      return ACK_ONLY_TOOLS.has(n);
+    });
+    if (allAckOnly) {
+      ackOnlyStreak += 1;
+      await emitProgress(onEvent, {
+        type: "ack_only_iter",
+        iteration: iter + 1,
+        streak: ackOnlyStreak,
+      });
+    } else {
+      ackOnlyStreak = 0;
+    }
   }
 
   return {