@agentprojectcontext/apx 1.81.0 → 1.89.1

package/src/host/daemon/events-ws.js

@@ -0,0 +1,273 @@
+// The live event feed: one WebSocket that tells every open surface "this
+// conversation just moved".
+//
+// The problem it solves: APX is one agent reachable from several places at
+// once — Telegram on the phone, the panel on the laptop, the deck on a tablet.
+// Until now each surface only saw what it did itself. A turn that arrived on
+// Telegram was invisible to a browser sitting on that very thread until someone
+// reloaded, and two browsers open on the same inbox drifted apart within
+// seconds. The daemon already knew — it wrote the row — it just never said so.
+//
+// SIGNAL, NOT DATA. A frame says which thread moved, never what was said. The
+// client re-fetches through the routes it already uses. That is deliberate:
+//   - one rendering path, not two. The record→bubble mapping lives in the API
+//     and in useChat; a second copy on the wire is a second thing to keep true.
+//   - a re-fetch is idempotent. Appending a pushed row is not: the device that
+//     SENT the message already painted it, and would show it twice.
+//   - nothing sensitive rides the socket beyond "channel X moved".
+//
+// Fan-out is per PROCESS and that is enough: the daemon owns the HTTP API, the
+// Telegram poller and the agent loop, so every write anyone makes happens here.
+// See core/events/bus.js for the one case it does not cover.
+import { onMessageEvent, onMobilityAlert, onRoutineEvent } from "#core/events/bus.js";
+import { mascotNoticesFromEvents } from "#core/events/mascot-notify.js";
+import { resolveSuperAgentBlob } from "#core/apc/agent-identity.js";
+import { apiPath } from "./api/prefix.js";
+
+const _clients = new Set(); // Set<WebSocket>
+
+export const eventsClients = _clients;
+
+/** The feed's upgrade path. Under /api like every other route (rule 9). */
+export const EVENTS_WS_PATH = apiPath("/events/ws");
+
+/** Path-gate: is this upgrade for the live event feed? */
+export function isEventsUpgradePath(url) {
+  let pathname = url || "";
+  try { pathname = new URL(url, "http://localhost").pathname; } catch { /* keep raw */ }
+  return pathname === EVENTS_WS_PATH;
+}
+
+// A streamed Telegram answer writes one ledger row per chunk, so a chatty turn
+// can emit a dozen events in a second. Collapsing them into one frame per
+// window turns that into one re-fetch per window per device instead of a dozen.
+// Short enough that it still reads as instant.
+const FLUSH_MS = 250;
+
+// Dead sockets do not always close: a phone that walks out of the tailnet
+// leaves a half-open connection that accepts writes into nothing. Ping on an
+// interval and drop whatever failed to answer the last one.
+const PING_MS = 30_000;
+
+/** Register a connected client. Sends a hello so the page can show it is live. */
+export function registerEventsClient(ws, config = {}) {
+  _clients.add(ws);
+  ws.isAlive = true;
+  ws.on("pong", () => { ws.isAlive = true; });
+  ws.on("close", () => _clients.delete(ws));
+  ws.on("error", () => _clients.delete(ws));
+  // The feed is one-directional. A client has nothing to say here — it acts
+  // through the HTTP API — so anything it sends is ignored rather than parsed.
+  ws.on("message", () => {});
+  send(ws, {
+    type: "hello",
+    ts: new Date().toISOString(),
+    settings: { super_agent: { icon: resolveSuperAgentBlob(config) } },
+  });
+}
+
+/** Publish a hot-reloaded super-agent avatar to every connected surface. */
+export function broadcastSuperAgentAvatar(config) {
+  broadcastEvents({
+    type: "settings",
+    settings: { super_agent: { icon: resolveSuperAgentBlob(config) } },
+  });
+}
+
+function send(ws, msg) {
+  try {
+    if (ws.readyState === 1) ws.send(JSON.stringify(msg)); // 1 = OPEN
+  } catch { /* the close handler will drop it */ }
+}
+
+/** Fan one frame out to every connected client. */
+export function broadcastEvents(msg) {
+  for (const ws of _clients) send(ws, msg);
+}
+
+/** Push one live-turn frame (start / delta / final / error) to every surface —
+ *  the token stream that used to belong only to the sending tab. Sent straight,
+ *  NOT through the 250ms message batch: tokens must arrive as they are written.
+ *  Unlike a "messages" frame this DOES carry data (the delta), on purpose — it
+ *  is the one thing the signal-only feed cannot express, and losing it to a
+ *  dropped connection is exactly the bug this fixes. */
+export function broadcastTurn(frame) {
+  broadcastEvents({ type: "turn", ...frame });
+}
+
+/** Push one routine-run frame (start / progress / end). Like a turn frame and
+ *  unlike a message frame this CARRIES the data — a run's steps are not in the
+ *  ledger until it ends, so there is nothing for a client to re-fetch while it
+ *  is still going. Sent straight, not through the 250ms batch: the whole point
+ *  is watching a run move. */
+/**
+ * A proximity alert, as its own frame.
+ *
+ * NOT collapsed into the 250 ms message window below: that window exists to
+ * turn a chatty streamed turn into one re-fetch, and an alert is neither
+ * chatty nor a re-fetch — it is one card, complete, that has to reach the
+ * phone while the car is still near the place. It also carries its payload
+ * rather than a "go look" signal, for the same reason.
+ */
+export function broadcastMobilityAlert(card) {
+  broadcastEvents({ type: "mobility_alert", ts: new Date().toISOString(), alert: card });
+}
+
+export function broadcastRoutineRun(frame) {
+  broadcastEvents({ type: "routine", ...frame });
+}
+
+/** Which project a write belongs to, as an id the panel can match on.
+ *  A global write already carries one in its meta; a project or conversation
+ *  write carries the storage path, which only the daemon's registry resolves. */
+function projectIdOf(event, projects) {
+  if (!event.project_root) return event.project_id ?? null;
+  if (!projects?.list) return null;
+  for (const entry of projects.list()) {
+    try {
+      if (projects.get(entry.id)?.storagePath === event.project_root) return entry.id;
+    } catch { /* an unreadable project is not a match */ }
+  }
+  return null;
+}
+
+/** The public shape of one event. `project_root` never leaves the daemon —
+ *  it is a path on this machine and the panel has no use for it. */
+function publicEvent(event, projects) {
+  return {
+    scope: event.scope,
+    channel: event.channel || null,
+    thread: event.thread || null,
+    project_id: projectIdOf(event, projects),
+    agent_slug: event.agent_slug || null,
+    // Only a conversation write has one; a channel thread is addressed by day.
+    conversation_id: event.conversation_id || null,
+    direction: event.direction || null,
+    type: event.type || null,
+    // A conversation write (a project agent's chat file) has no direction and
+    // no type — it carries the ROLE it was appended under. Without it every
+    // such write looks identical on the wire, and a surface that only wants to
+    // hear about the agent SPEAKING has to re-fetch on the owner's own send and
+    // on every tool row too. See lib/notify.ts.
+    role: event.role || null,
+    author: event.author || null,
+    // How the row was produced, when it matters to a subscriber — "routine_delivery"
+    // marks an agent reaching the owner, which the mascot surfaces on its own.
+    via: event.via || null,
+    // A ≤100-char headline for a delivery, so the mascot bubble can say what
+    // arrived. A notice, not the message body — "signal, not data" holds.
+    notify: event.notify || null,
+    // Closing vs mid-turn chunk. The pet only bubbles an agent's launched
+    // final on Telegram / group / A2A — never the owner's send.
+    final: event.final === true ? true : null,
+    streamed: event.streamed === true ? true : null,
+    // Who an a2a row was addressed to, so the bubble can name both ends
+    // ("de magui a roby") instead of just the channel. A name, not an address:
+    // see appendMessageToFs, which only lifts it for a2a. Null everywhere else.
+    to: event.to || null,
+    ts: event.ts || null,
+  };
+}
+
+/** Two events about the same thread in the same window are one re-fetch. */
+function keyOf(e) {
+  return [e.scope, e.channel, e.thread, e.project_id, e.agent_slug, e.conversation_id].join("|");
+}
+
+/**
+ * Subscribe the hub to the core bus. Called once at daemon startup.
+ * Returns a stop() that unsubscribes and clears the timers.
+ */
+export function startEventsBridge({ projects } = {}) {
+  const pending = new Map();
+  let flushTimer = null;
+
+  const flush = () => {
+    flushTimer = null;
+    if (!pending.size) return;
+    const events = [...pending.values()];
+    pending.clear();
+    // Nobody listening is the normal case (no panel open). Skip the work.
+    if (_clients.size) {
+      // Computed once here so desktop and the phone cannot drift: both pets
+      // just render the lines. Empty means "this burst is not news" (the
+      // owner sending, a stream chunk, a tool row).
+      const notices = mascotNoticesFromEvents(events);
+      broadcastEvents({
+        type: "messages",
+        events,
+        notifications: notices.map((notice) => notice.text),
+        // The same lines with the channel each one is about, so a device can
+        // answer "which of these may ring me" — the phone mutes Telegram
+        // because Telegram is installed on it. Sent alongside rather than
+        // instead: an APK from before this shipped still reads `notifications`.
+        notices,
+      });
+    }
+  };
+
+  // A routine run moving is its own signal: it is not a ledger write, and the
+  // batch that collapses ledger writes would make a step list arrive in clumps.
+  const unsubscribeRoutines = onRoutineEvent((event) => {
+    if (!_clients.size) return;
+    let projectId = null;
+    try {
+      projectId = projectIdOf(event, projects);
+    } catch {
+      return; // an unresolvable project is not worth taking the daemon down for
+    }
+    broadcastRoutineRun({
+      phase: event.phase,
+      project_id: projectId,
+      routine: event.routine,
+      run: event.run,
+    });
+  });
+
+  // Proximity alerts. Straight out, no batching: see broadcastMobilityAlert.
+  const unsubscribeMobility = onMobilityAlert((card) => {
+    if (!_clients.size) return;
+    broadcastMobilityAlert(card);
+  });
+
+  const unsubscribe = onMessageEvent((event) => {
+    let pub;
+    try {
+      pub = publicEvent(event, projects);
+    } catch {
+      return; // a malformed event must not take the daemon down
+    }
+    pending.set(keyOf(pub), pub);
+    if (!flushTimer) {
+      flushTimer = setTimeout(flush, FLUSH_MS);
+      // A pending frame must never be the reason the process stays up.
+      flushTimer.unref?.();
+    }
+  });
+
+  const pinger = setInterval(() => {
+    for (const ws of _clients) {
+      if (ws.isAlive === false) {
+        _clients.delete(ws);
+        try { ws.terminate(); } catch { /* already gone */ }
+        continue;
+      }
+      ws.isAlive = false;
+      try { ws.ping(); } catch { _clients.delete(ws); }
+    }
+  }, PING_MS);
+  pinger.unref?.();
+
+  return function stop() {
+    unsubscribe();
+    unsubscribeRoutines();
+    unsubscribeMobility();
+    clearInterval(pinger);
+    if (flushTimer) clearTimeout(flushTimer);
+    pending.clear();
+    for (const ws of _clients) {
+      try { ws.close(); } catch { /* already gone */ }
+    }
+    _clients.clear();
+  };
+}

package/src/host/daemon/index.js

@@ -29,11 +29,15 @@ import { ProjectManager } from "./db.js";
 import { McpRegistry } from "#core/mcp/runner.js";
 import { PluginManager } from "./plugins/index.js";
 import { RoutineScheduler } from "./routines-scheduler.js";
+import { startDeliverySweep } from "./delivery-sweep.js";
 import { startCallbackReconciler } from "./callback-reconciler.js";
 import { buildApi } from "./api.js";
 import { createTokenStore } from "./token-store.js";
 import { triggerWakeup } from "./wakeup.js";
-import { registerDesktopClient, isDesktopUpgradePath, isDesktopUpgradeAuthorized } from "./desktop-ws.js";
+import { registerDesktopClient, isDesktopUpgradePath } from "./desktop-ws.js";
+import { isTerminalUpgradePath, startTerminalSession } from "./terminal-ws.js";
+import { isEventsUpgradePath, registerEventsClient, startEventsBridge } from "./events-ws.js";
+import { isWsUpgradeAuthorized } from "./ws-auth.js";
 import { log as logToUnified } from "#core/logging.js";
 import { initMemory, stopMemory } from "#core/memory/index.js";
 
@@ -48,7 +52,7 @@ const PKG = JSON.parse(
 // process.stdout.write reaches the file once. In foreground (npm start), it
 // still prints to the console. No double-append.
 //
-// Beyond the legacy stdout sink we also fan out every line to the unified
+// Beyond the stdout sink we also fan out every line to the unified
 // ~/.apx/logs/apx.log via core/logging.js so `apx log` and `apx log -f`
 // see everything that any plugin/module writes through the daemon's log fn.
 //
@@ -222,6 +226,7 @@ async function main() {
     tokenStore,
     version: PKG.version,
     startedAt,
+    log,
     addProjectGlobally: (absPath) => {
       try {
         const fresh = readConfig();
@@ -235,6 +240,7 @@ async function main() {
   plugins.installRoutes(app);
 
   let callbackReconciler = null;
+  let eventsBridge = null;
 
   // Loopback is ALWAYS bound, whatever `host` says.
   //
@@ -254,6 +260,7 @@ async function main() {
       // one is already up and is the one everything depends on.
       const s2 = http.createServer(app);
       s2.on("error", (e) => log(`bind ${h}:${port} failed (${e.code || e.message}) — local access unaffected`));
+      s2.on("upgrade", handleUpgrade);
       s2.listen(port, h, () => log(`apx-daemon also listening on http://${h}:${port}`));
       secondary.push(s2);
     }
@@ -265,6 +272,14 @@ async function main() {
     // daemon died before it could (crash, pull, or a task that restarted the
     // daemon). Runs once now to recover prior IOUs, then on an interval.
     callbackReconciler = startCallbackReconciler({ plugins, log });
+    // Live event feed: turn every ledger write into a frame on /api/events/ws so
+    // an open panel — on any device — sees a conversation move the moment it
+    // moves, whichever channel produced the turn.
+    eventsBridge = startEventsBridge({ projects });
+    // Delivery grace sweep: notify ordinary deliveries that have sat unread past
+    // their grace window, so "notify unless he reads it first" actually fires.
+    // Priority deliveries are pinged immediately by the runner, not here.
+    startDeliverySweep({ projects, plugins, registries, config: cfg });
     // Cross-channel memory: ensure ~/.apx/memory.md exists, open the vector
     // store, and start the incremental RAG indexer. Best-effort — never blocks
     // boot and never throws into the daemon.
@@ -290,20 +305,41 @@ async function main() {
     setTimeout(() => triggerWakeup(cfg, log), 3000);
     // Preload whisper-server in the background so first desktop transcription is fast.
     // Adopts an existing one if already on the port; otherwise spawns fresh.
-    import("./whisper-server.js").then(({ preloadWhisperServer }) => {
-      preloadWhisperServer((m) => log(m));
+    import("./whisper-server.js").then(({ preloadWhisperServer, startWhisperKeepWarm }) => {
+      preloadWhisperServer((m) => log(m)).then(() => startWhisperKeepWarm((m) => log(m)));
+    }).catch(() => {});
+    // Same treatment for the voice engine. Warming it only when the microphone
+    // opens is too late: on a machine short on RAM the weights get compressed
+    // out while idle, and decompressing them takes far longer than anyone
+    // spends speaking — so the reply queues behind a warm-up still in flight.
+    // Only ever pings an engine running on this machine or the LAN.
+    import("./tts-keepwarm.js").then(({ startTtsKeepWarm }) => {
+      startTtsKeepWarm((m) => log(m));
     }).catch(() => {});
   });
 
-  // Attach WebSocket upgrade for the desktop channel on /api/desktop/ws.
-  server.on("upgrade", async (req, socket, head) => {
-    if (!isDesktopUpgradePath(req.url)) { socket.destroy(); return; }
+  // Attach WebSocket upgrades: the desktop channel on /api/desktop/ws, a
+  // terminal on /api/terminal/ws that reopens one session in its own CLI, and
+  // the live event feed on /api/events/ws that tells every open panel when a
+  // conversation moved.
+  //
+  // Attached to EVERY listener, not just the first. A specific `host` means the
+  // daemon runs two servers on one app (see the bind block above), and an
+  // upgrade handler on only one of them is how the phone on the tailnet gets a
+  // panel whose HTTP works and whose live feed silently never connects.
+  async function handleUpgrade(req, socket, head) {
+    const isTerminal = isTerminalUpgradePath(req.url);
+    const isEvents = isEventsUpgradePath(req.url);
+    if (!isTerminal && !isEvents && !isDesktopUpgradePath(req.url)) { socket.destroy(); return; }
     // Auth: the WS upgrade must carry a valid token (master or paired client),
-    // matching the HTTP /api/desktop/* routes. Without this, any client that can
-    // reach the daemon (host binds 0.0.0.0 → the LAN) could open the desktop
-    // channel and drive the super-agent (permission_mode "total"). The
-    // legitimate desktop window already sends the bearer token. See QA BUG-WS-AUTH.
-    if (!isDesktopUpgradeAuthorized(req, tokenStore)) {
+    // matching the HTTP /api/* routes. Without this, any client that can reach
+    // the daemon (host binds 0.0.0.0 → the LAN) could open the desktop channel
+    // and drive the super-agent (permission_mode "total"). The legitimate
+    // desktop window already sends the bearer token; a browser passes ?token=.
+    // See QA BUG-WS-AUTH. The terminal channel spawns a CLI on this machine and
+    // the event feed reports who is talking to whom, so all three are gated the
+    // same way — the check is about who may open a channel, not which one.
+    if (!isWsUpgradeAuthorized(req, tokenStore)) {
       socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
       socket.destroy();
       return;
@@ -315,9 +351,12 @@ async function main() {
     }
     const wss = new WebSocketServer({ noServer: true });
     wss.handleUpgrade(req, socket, head, (ws) => {
-      registerDesktopClient(ws);
+      if (isTerminal) startTerminalSession(ws, req);
+      else if (isEvents) registerEventsClient(ws, cfg);
+      else registerDesktopClient(ws);
     });
-  });
+  }
+  server.on("upgrade", handleUpgrade);
 
   server.on("error", (e) => {
     log(`fatal: listen ${host}:${port} failed: ${e.message}`);
@@ -331,13 +370,18 @@ async function main() {
     log(`received ${signal}, shutting down...`);
     scheduler.stop();
     callbackReconciler?.stop();
+    eventsBridge?.();
     plugins.stopAll();
     stopMemory();
     registries.shutdown();
     // Best-effort shutdown of whisper-server subprocess.
-    import("./whisper-server.js").then(({ shutdownWhisperServer }) => {
+    import("./whisper-server.js").then(({ shutdownWhisperServer, stopWhisperKeepWarm }) => {
+      stopWhisperKeepWarm();
       shutdownWhisperServer().catch(() => {});
     }).catch(() => {});
+    // The voice engine is a separate service with its own lifetime — nothing to
+    // shut down here, only our timer to stop pinging it.
+    import("./tts-keepwarm.js").then(({ stopTtsKeepWarm }) => stopTtsKeepWarm()).catch(() => {});
     // Close the LAN listener(s) too, or the port stays held after SIGTERM and
     // the next `apx restart` fails with "already running".
     for (const s2 of secondary) {

package/src/host/daemon/plugins/desktop/index.js

@@ -15,7 +15,6 @@
 //     "model": "",                 // override model; leave empty = super-agent.model
 //     "max_history": 20            // turns to keep in context
 //   }
-// (legacy "overlay" block is still read as a fallback)
 
 import {
   broadcastDesktop,
@@ -25,6 +24,7 @@ import {
 import { runSuperAgent, isSuperAgentEnabled } from "#core/agent/super-agent.js";
 import { appendGlobalMessage } from "#core/stores/messages.js";
 import { stripEmotionTags } from "#core/voice/emotions.js";
+import { stripEmoji } from "#core/voice/pronounceable.js";
 import { stripReasoning } from "#core/util/thinking.js";
 import { CHANNELS } from "#core/constants/channels.js";
 import { tryResolveSkillCommand } from "#core/agent/skills/trigger.js";
@@ -35,7 +35,7 @@ export default {
   id: "desktop",
 
   init({ projects, config, log, plugins }) {
-    const cfg = config.desktop || config.overlay || {};
+    const cfg = config.desktop || {};
     const enabled = cfg.enabled !== false; // enabled by default
 
     // In-memory conversation history per connected client.
@@ -127,7 +127,11 @@ async function _handleMessage({ ws, text, previousMessages }, { projects, config
     // Desktop is voice-first: an unstripped reasoning dump would be spoken
     // aloud, at length, in the wrong language. Drop the segment entirely
     // rather than narrate the model's notes.
-    const spoken = stripReasoning(seg).answer;
+    // Desktop is a voice capsule, so emoji come out of both halves: the TTS
+    // hums for two seconds trying to pronounce one, and a bubble that keeps it
+    // would no longer match what was said. A segment that was *only* an emoji
+    // leaves nothing to say and is dropped, rather than spoken as a shrug.
+    const spoken = stripEmoji(stripReasoning(seg).answer);
     if (!spoken.trim()) return;
     _send(ws, { type: "segment", seq: ++segSeq, text: stripEmotionTags(spoken), speak: spoken });
   };
@@ -157,7 +161,16 @@ async function _handleMessage({ ws, text, previousMessages }, { projects, config
       previousMessages: ws ? history.slice(0, -1) : history,
       overrideModel: cfg.model || null,
       signal: controller.signal,
-      onToken: (chunk) => { liveBuf += chunk; },
+      onToken: (chunk) => {
+        liveBuf += chunk;
+        // Ship the token onward too. `liveBuf` alone meant the window sat on
+        // "Pensando…" for the whole turn and then painted the finished reply in
+        // one go, even though the tokens had been arriving all along. The
+        // renderer paints these into a provisional bubble that the matching
+        // `segment` then replaces — deltas are for reading, segments are what
+        // gets spoken and kept.
+        _send(ws, { type: "delta", text: chunk });
+      },
       onEvent: async (event) => {
         if (event.type === "tool_start") {
           const t = event.trace;