@agentprojectcontext/apx 1.80.2 → 1.89.1

package/src/core/stores/conversations.js

@@ -3,6 +3,9 @@
 
 import fs from "node:fs";
 import path from "node:path";
+import { parseFrontmatter } from "#core/apc/frontmatter.js";
+import { emitMessageEvent } from "#core/events/bus.js";
+import { mediaFromMeta } from "#core/stores/messages.js";
 
 const nowIso = () => new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
 
@@ -27,10 +30,28 @@ export function conversationPath(storagePath, agentSlug, idOrFilename) {
   return path.join(storagePath, "agents", agentSlug, "conversations", filename);
 }
 
-export function startConversation({ storagePath, agentSlug, engine, system, channel }) {
+/** Read a conversation file path back into what it addresses. The inverse of
+ *  `conversationPath`, and it lives next to it so the two cannot drift: the
+ *  live event feed has a file path and needs the project/agent/conversation it
+ *  belongs to. Returns null for anything that is not a conversation file. */
+export function parseConversationPath(filePath) {
+  const parts = String(filePath || "").split(path.sep);
+  const i = parts.lastIndexOf("conversations");
+  if (i < 2 || parts[i - 2] !== "agents") return null;
+  return {
+    project_root: parts.slice(0, i - 2).join(path.sep),
+    agent_slug: parts[i - 1],
+    conversation_id: (parts[i + 1] || "").replace(/\.md$/, ""),
+  };
+}
+
+export function startConversation({ storagePath, agentSlug, engine, system, channel, title, id: idArg }) {
   const dir = path.join(storagePath, "agents", agentSlug, "conversations");
   fs.mkdirSync(dir, { recursive: true });
-  const id = generateConversationId(storagePath, agentSlug);
+  // A caller may pin a STABLE id (e.g. an agent's one persistent "web-main"
+  // coaching chat, reused across routine runs) instead of the auto-incrementing
+  // date-NN. Auto-increment stays the default so nothing else changes.
+  const id = idArg || generateConversationId(storagePath, agentSlug);
   const file = path.join(dir, `${id}.md`);
   const started = nowIso();
   const fm =
@@ -39,6 +60,7 @@ export function startConversation({ storagePath, agentSlug, engine, system, chan
     `agent: ${agentSlug}\n` +
     `engine: ${engine}\n` +
     (channel ? `channel: ${channel}\n` : "") +
+    (title ? `title: ${JSON.stringify(String(title))}\n` : "") +
     `started: ${started}\n` +
     `last_turn: \n` +
     `status: open\n` +
@@ -48,30 +70,41 @@ export function startConversation({ storagePath, agentSlug, engine, system, chan
   return { id, filename: `${id}.md`, path: file, started };
 }
 
-export function appendTurn({ filePath, role, content }) {
+/**
+ * Append one turn.
+ *
+ * `meta` is the turn's ATTRIBUTION — which agent answered, on which model, what
+ * it cost, what its tools did — and it is written as a JSON object on the turn
+ * header, after the timestamp. The ledger has carried this since
+ * `readGlobalThread`; the conversation file did not, so anything reopened from
+ * a file rendered "0 tok" and no model even when the run had recorded both. A
+ * routine is the case where that is ALL you get: nobody watched it stream.
+ *
+ * The content itself is untouched — meta lives on the header, so a body that
+ * happens to start with a brace is still a body. Files written before this
+ * simply have no meta, and parse exactly as they did.
+ */
+export function appendTurn({ filePath, role, content, meta }) {
   const ts = nowIso();
-  const block = `## ${role} — ${ts}\n${content}\n\n`;
+  const head = meta && Object.keys(meta).length ? ` ${JSON.stringify(meta)}` : "";
+  const block = `## ${role} — ${ts}${head}\n${content}\n\n`;
   fs.appendFileSync(filePath, block);
   // Update last_turn in frontmatter (in-place)
   let text = fs.readFileSync(filePath, "utf8");
   text = text.replace(/^last_turn:.*$/m, `last_turn: ${ts}`);
   fs.writeFileSync(filePath, text);
+  // Announce it, the same way a ledger write does: a routine or an `apx exec`
+  // appending here is a conversation moving, and an open panel should see it
+  // without being reloaded. See core/events/bus.js.
+  const where = parseConversationPath(filePath);
+  if (where) emitMessageEvent({ scope: "conversation", ...where, role, ts });
   return { ts };
 }
 
 // Parse a conversation file into structured turns. Tolerant — anything that
 // doesn't look like a turn header is ignored.
 export function parseConversation(text) {
-  const fmEnd = text.indexOf("\n---", 4);
-  const fm = {};
-  let body = text;
-  if (text.startsWith("---\n") && fmEnd !== -1) {
-    for (const line of text.slice(4, fmEnd).split("\n")) {
-      const m = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
-      if (m) fm[m[1]] = m[2].trim();
-    }
-    body = text.slice(fmEnd + 4);
-  }
+  const { fm, body } = parseFrontmatter(text);
   const turns = [];
   // The terminator is "the next turn header, or the true end of input".
   //
@@ -79,13 +112,23 @@ export function parseConversation(text) {
   // — so the lazy body stopped at the first newline and every multi-line turn
   // was silently truncated to its first line. `(?![\s\S])` is end-of-input and
   // nothing else. /m is still needed for the `^` on the header.
-  const re = /^##\s+(user|assistant|system|tool|compact)\s+—\s+(\S+)\s*\n([\s\S]*?)(?=\n##\s+(?:user|assistant|system|tool|compact)\s+—\s|\s*(?![\s\S]))/gm;
+  //
+  // The optional `{…}` after the timestamp is the turn's attribution (see
+  // `appendTurn`). It is single-line by construction — JSON.stringify escapes
+  // newlines — so `.*` cannot run into the body.
+  const re = /^##\s+(user|assistant|system|tool|compact)\s+—\s+(\S+)[ \t]*(\{.*\})?[ \t]*\n([\s\S]*?)(?=\n##\s+(?:user|assistant|system|tool|compact)\s+—\s|\s*(?![\s\S]))/gm;
   let m;
   while ((m = re.exec(body)) !== null) {
+    let meta = null;
+    if (m[3]) {
+      // A header we cannot parse costs the turn its attribution, never its text.
+      try { meta = JSON.parse(m[3]); } catch { meta = null; }
+    }
     turns.push({
       role: m[1],
       ts: m[2],
-      content: m[3].trim(),
+      content: m[4].trim(),
+      ...(meta && typeof meta === "object" ? { meta } : {}),
     });
   }
   return { fm, turns };
@@ -97,6 +140,118 @@ export function readConversation(storagePath, agentSlug, idOrFilename) {
   return { ...parseConversation(fs.readFileSync(p, "utf8")), path: p };
 }
 
+/** Shape a parsed turn for the web chat viewer (tool rows carry structured args).
+ *
+ *  Assistant rows hand out the attribution stored on their header under the
+ *  SAME field names `readGlobalThread` uses — agent / agent_name / model /
+ *  usage / tool_summary — because the web client shapes both through one
+ *  `threadToChatMsgs`. A conversation opened from a file and the same turn read
+ *  back from the ledger must not be two different things to the viewer. */
+export function shapeConversationMessage(t) {
+  const base = { role: t.role, content: t.content, ts: t.ts };
+  // The file this turn carried, in EITHER direction and under the same field
+  // names `readGlobalThread` hands out — one viewer shapes both. A conversation
+  // file used to surface no media at all: a photo dropped in the composer of a
+  // project agent's chat was sent to the model and then read back as the raw
+  // "[image attached — saved to /Users/…]" marker, and an image the agent
+  // attached had nowhere to be recorded at all.
+  const media = mediaFromMeta(t.meta);
+  const mediaFields = {
+    ...(media ? { media } : {}),
+    ...(Array.isArray(t.meta?.media) && t.meta.media.length ? { media_list: t.meta.media } : {}),
+  };
+  if (t.role === "user") return { ...base, ...mediaFields };
+  if (t.role === "assistant") {
+    const meta = t.meta || {};
+    const usage = meta.usage;
+    return {
+      ...base,
+      ...mediaFields,
+      ...(meta.agent ? { agent: meta.agent } : {}),
+      ...(meta.agent_name ? { agent_name: meta.agent_name } : {}),
+      ...(meta.actor_kind ? { actor_kind: meta.actor_kind } : {}),
+      ...(meta.model ? { model: meta.model } : {}),
+      ...(usage && typeof usage === "object" ? { usage } : {}),
+      ...(meta.tool_summary ? { tool_summary: meta.tool_summary } : {}),
+      // Legacy exec rows wrote `{ tools: N }` before tool rows existed — surface
+      // a compact count so reopened threads don't look tool-less.
+      ...(!meta.tool_summary && Number(meta.tools) > 0
+        ? { tool_summary: { total: Number(meta.tools), failed: 0, tools: [{ name: "tool", count: Number(meta.tools), failed: 0 }] } }
+        : {}),
+    };
+  }
+  if (t.role !== "tool") return base;
+  let parsed = null;
+  try { parsed = JSON.parse(t.content); } catch { parsed = null; }
+  if (parsed && typeof parsed === "object" && parsed.tool) {
+    return {
+      ...base,
+      tool: parsed.tool,
+      args: parsed.args,
+      result: parsed.result,
+    };
+  }
+  return { ...base, tool: "tool" };
+}
+
+// Rewind a conversation, dropping turns. This is what "regenerate" and "edit &
+// resend" stand on: the pane rewinds to a turn and the FILE has to rewind with
+// it, or the daemon (which rebuilds a project agent's history from this file)
+// would answer with the stale turns still appended.
+//
+// Cutoff is by a COUNT OF VISIBLE turns, not a timestamp: a just-sent turn's
+// on-screen ts (stamped in the browser) differs from the ts appendTurn writes
+// on disk, so a ts cutoff could misalign. `keepVisible: K` keeps turns through
+// the K-th user/assistant turn, carrying any interleaved system/compact context
+// with them, and drops everything after — which lines up exactly with the
+// viewer, where the pane's messages ARE the user/assistant turns in order (the
+// viewer hides system/compact; project agents have no tool turns). Returns false
+// when the file is missing.
+//
+// The frontmatter block is preserved verbatim except last_turn, and each kept
+// turn is re-emitted in appendTurn's exact format so a round-trip is stable.
+export function truncateConversation(storagePath, agentSlug, idOrFilename, opts = {}) {
+  const { keepVisible = null } = typeof opts === "number" ? { keepVisible: opts } : opts || {};
+  const p = conversationPath(storagePath, agentSlug, idOrFilename);
+  if (!fs.existsSync(p)) return false;
+  const text = fs.readFileSync(p, "utf8");
+  const { turns } = parseConversation(text);
+
+  // Keep turns up to and including the K-th visible (user/assistant) turn — any
+  // system/compact turns before it ride along as context. If K reaches or
+  // exceeds the visible turns present, nothing is dropped (a no-op rewind).
+  const isVisible = (t) => t.role === "user" || t.role === "assistant";
+  const k = Math.max(0, Number(keepVisible) || 0);
+  let kept = [];
+  if (k > 0) {
+    let seen = 0;
+    let lastIdx = -1;
+    for (let i = 0; i < turns.length; i++) {
+      if (isVisible(turns[i]) && ++seen === k) { lastIdx = i; break; }
+    }
+    kept = lastIdx >= 0 ? turns.slice(0, lastIdx + 1) : turns.slice();
+  }
+
+  // Preserve the frontmatter block (`---\n…\n---\n`) exactly, only rewriting
+  // last_turn — re-serialising parsed fm would reorder keys and drop unknowns.
+  const fmEnd = text.startsWith("---\n") ? text.indexOf("\n---\n", 4) : -1;
+  const header = fmEnd !== -1 ? text.slice(0, fmEnd + 5) : "";
+
+  const body = kept
+    .map((t) => {
+      const head = t.meta && Object.keys(t.meta).length ? ` ${JSON.stringify(t.meta)}` : "";
+      return `## ${t.role} — ${t.ts}${head}\n${t.content}\n\n`;
+    })
+    .join("");
+  const lastTs = kept.length ? kept[kept.length - 1].ts : "";
+  const head = header.replace(/^last_turn:.*$/m, `last_turn: ${lastTs}`);
+  fs.writeFileSync(p, `${head}\n${body}`);
+
+  const where = parseConversationPath(p);
+  if (where) emitMessageEvent({ scope: "conversation", ...where, role: "system", ts: nowIso() });
+  return true;
+}
+
 // Delete a conversation file. Filesystem is source of truth, so unlinking the
 // markdown removes it from the sidebar list on the next fetch. Returns false
 // when there is nothing to delete (already gone / bad id).
@@ -107,7 +262,10 @@ export function deleteConversation(storagePath, agentSlug, idOrFilename) {
   return true;
 }
 
-export function listConversations(storagePath, agentSlug) {
+/** Newest first. Archived ones are left out unless asked for — that is what
+ *  archiving is FOR — but they are never dropped from disk, so the session
+ *  picker can offer them back under their own heading. */
+export function listConversations(storagePath, agentSlug, { includeArchived = false } = {}) {
   const dir = path.join(storagePath, "agents", agentSlug, "conversations");
   if (!fs.existsSync(dir)) return [];
   return fs
@@ -116,7 +274,8 @@ export function listConversations(storagePath, agentSlug) {
     .sort()
     .reverse()
     .map((f) => summarizeConversation(path.join(dir, f), agentSlug, f))
-    .filter(Boolean);
+    .filter(Boolean)
+    .filter((c) => includeArchived || !c.archived);
 }
 
 // Lightweight summary used by the chat list sidebar — reads frontmatter and
@@ -127,9 +286,12 @@ function summarizeConversation(filePath, agentSlug, filename) {
   let text;
   try { text = fs.readFileSync(filePath, "utf8"); } catch { return null; }
   const { fm, turns } = parseConversation(text);
-  const messages = turns.filter((t) => t.role !== "system" && t.role !== "compact").length;
+  const messages = turns.filter((t) => t.role === "user" || t.role === "assistant").length;
   const firstUser = turns.find((t) => t.role === "user");
-  const title = (firstUser?.content || "").split("\n")[0].slice(0, 80).trim() || undefined;
+  const title =
+    (typeof fm.title === "string" && fm.title.trim()) ||
+    (firstUser?.content || "").split("\n")[0].slice(0, 80).trim() ||
+    undefined;
 
   // What the AGENT last said, not what the user last asked. An inbox row that
   // echoes your own prompt back tells you nothing; the reply is the thing you
@@ -149,6 +311,7 @@ function summarizeConversation(filePath, agentSlug, filename) {
     last_turn_at: fm.last_turn || fm.started || "",
     ended_at: fm.status === "closed" ? (fm.last_turn || undefined) : undefined,
     channel: fm.channel || undefined,
+    archived: String(fm.archived) === "true" || undefined,
     messages,
     title,
     preview,
@@ -156,6 +319,43 @@ function summarizeConversation(filePath, agentSlug, filename) {
   };
 }
 
+/**
+ * Patch a conversation's frontmatter in place: what it is called, and whether
+ * it has been put away.
+ *
+ * Only the keys present in `patch` are touched, and a key already in the file
+ * is rewritten rather than duplicated — a second `title:` line would win or
+ * lose depending on which parser read it. New keys are inserted just before the
+ * closing fence so the block keeps its shape.
+ *
+ * Archiving is deliberately a FIELD and not a move to another directory: the
+ * file stays exactly where every id, link and reader already expects it, and
+ * putting a conversation away stays a decision you can take back.
+ */
+export function setConversationMeta(storagePath, agentSlug, idOrFilename, patch = {}) {
+  const p = conversationPath(storagePath, agentSlug, idOrFilename);
+  if (!fs.existsSync(p)) return false;
+  let text = fs.readFileSync(p, "utf8");
+  const fence = text.indexOf("\n---", 4);
+  if (!text.startsWith("---\n") || fence < 0) return false;
+
+  const write = (key, value) => {
+    const line = value === undefined ? null : `${key}: ${JSON.stringify(String(value))}`;
+    const re = new RegExp(`^${key}:.*$`, "m");
+    const head = text.slice(0, fence);
+    if (re.test(head)) {
+      text = head.replace(re, line ?? "").replace(/\n\n+/g, "\n") + text.slice(fence);
+      return;
+    }
+    if (line) text = head + `\n${line}` + text.slice(fence);
+  };
+
+  if ("title" in patch) write("title", patch.title || undefined);
+  if ("archived" in patch) write("archived", patch.archived ? "true" : undefined);
+  fs.writeFileSync(p, text);
+  return true;
+}
+
 export function setStatus(filePath, status) {
   let text = fs.readFileSync(filePath, "utf8");
   text = text.replace(/^status:.*$/m, `status: ${status}`);

package/src/core/stores/deliveries.js

@@ -0,0 +1,177 @@
+// The delivery queue — the "famoso delivery" made visible.
+//
+// When a routine run by a non-Roby agent leaves a message in the agent's own web
+// chat, it also drops a DELIVERY here: a lightweight record that Manu has
+// something waiting. This replaced the old a2a-message hack (an agent "chatting"
+// with Roby cluttered the inbox and leaked the `super_agent` slug into the UI).
+//
+// A delivery is not a chat. It is a small, foldable event log you can look at to
+// see what is piling up and what has been crossed off:
+//
+//   pending   — the agent left something; nobody has told Manu yet.
+//   notified  — Roby reached Manu about it (immediately for a priority/anchor
+//               delivery, or later from the sweep for an ordinary one).
+//   answered  — Manu replied in the agent's chat (the delivery is done).
+//   held      — the interruption budget deliberately withheld the notify.
+//
+// Append-only JSONL, one event per line; `listDeliveries` folds it to the
+// current state per id. Appending never rewrites, so two writers (a routine run
+// and the sweep) cannot race a file rewrite.
+import fs from "node:fs";
+import path from "node:path";
+import { nowIso } from "#core/util/time.js";
+
+const DELIVERIES_FILE = "deliveries.jsonl";
+
+/** The one status a fold keeps per id, newest event wins. */
+export const DELIVERY_STATUS = Object.freeze({
+  PENDING: "pending",
+  NOTIFIED: "notified",
+  ANSWERED: "answered",
+  HELD: "held",
+});
+
+function deliveriesPath(storagePath) {
+  return path.join(storagePath, DELIVERIES_FILE);
+}
+
+function shortId(ts) {
+  // Stable-ish, human-legible, and unique enough for a per-project queue: the
+  // second-precision timestamp plus a short random suffix. crypto is a runtime
+  // global; a failure just falls back to the timestamp alone.
+  let suffix = "";
+  try { suffix = crypto.randomUUID().slice(0, 6); } catch { suffix = String(Math.floor((Date.parse(ts) || 0) % 1e6)); }
+  return `d_${suffix}`;
+}
+
+function appendEvent(storagePath, event) {
+  if (!storagePath) return null;
+  try {
+    const file = deliveriesPath(storagePath);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.appendFileSync(file, JSON.stringify(event) + "\n");
+    return event;
+  } catch {
+    return null; // the queue is best-effort — losing a row must not fail the run
+  }
+}
+
+/**
+ * Record a new pending delivery. Returns its id (or null if it could not be
+ * written) so the caller can mark it `notified`/`held` in the same run.
+ */
+export function recordDelivery(storagePath, { agent, agentName, routine, routineId, notify, priority = false, project_id = null }) {
+  const ts = nowIso();
+  const id = shortId(ts);
+  const ok = appendEvent(storagePath, {
+    id,
+    ev: DELIVERY_STATUS.PENDING,
+    ts,
+    agent,
+    agent_name: agentName || agent,
+    routine: routine || "",
+    routine_id: routineId || "",
+    notify: notify || "",
+    priority: !!priority,
+    project_id,
+  });
+  return ok ? id : null;
+}
+
+/** Cross a delivery off (or move it along): notified / answered / held. */
+export function markDelivery(storagePath, id, status, extra = {}) {
+  if (!id) return false;
+  return !!appendEvent(storagePath, { id, ev: status, ts: nowIso(), ...extra });
+}
+
+/**
+ * Manu engaged with an agent (replied in its chat) → close that agent's open
+ * deliveries. A `pending` one being answered CANCELS its grace-window notify
+ * before it fires; a `notified` one just gets crossed off. Returns how many were
+ * closed, so a caller can skip the notify when it already had one in flight.
+ */
+export function answerDeliveries(storagePath, agentSlug) {
+  if (!storagePath || !agentSlug) return 0;
+  const open = listDeliveries(storagePath).filter(
+    (d) => d.agent === agentSlug &&
+      (d.status === DELIVERY_STATUS.PENDING || d.status === DELIVERY_STATUS.NOTIFIED),
+  );
+  for (const d of open) markDelivery(storagePath, d.id, DELIVERY_STATUS.ANSWERED);
+  return open.length;
+}
+
+/**
+ * The current queue, newest first. Folds the event log so each id appears once
+ * with its latest status and the fields from its `pending` event. `status`
+ * filters to one state; `limit` caps the result.
+ */
+export function listDeliveries(storagePath, { status, limit = 100 } = {}) {
+  if (!storagePath) return [];
+  let lines = [];
+  try {
+    lines = fs.readFileSync(deliveriesPath(storagePath), "utf8").split("\n").filter(Boolean);
+  } catch {
+    return [];
+  }
+  // Fold events per id. The `pending` event carries the descriptive fields;
+  // every later event only moves the status. Events can arrive in any order, so
+  // merge onto whatever is already there rather than overwrite.
+  const byId = new Map();
+  for (const line of lines) {
+    let e;
+    try { e = JSON.parse(line); } catch { continue; }
+    if (!e?.id) continue;
+    const prev = byId.get(e.id);
+    if (e.ev === DELIVERY_STATUS.PENDING) {
+      byId.set(e.id, {
+        ...e,
+        created_at: e.ts,
+        status: prev?.status || DELIVERY_STATUS.PENDING,
+        ...(prev?.updated_at ? { updated_at: prev.updated_at } : {}),
+        ...(prev?.channel ? { channel: prev.channel } : {}),
+      });
+    } else {
+      byId.set(e.id, {
+        ...(prev || { id: e.id }),
+        id: e.id,
+        status: e.ev,
+        updated_at: e.ts,
+        ...(e.channel ? { channel: e.channel } : {}),
+        ...(e.reason ? { reason: e.reason } : {}),
+      });
+    }
+  }
+  let out = [...byId.values()];
+  if (status) out = out.filter((d) => d.status === status);
+  out.sort((a, b) => String(b.created_at || b.ts || "").localeCompare(String(a.created_at || a.ts || "")));
+  return out.slice(0, limit);
+}
+
+/**
+ * Repoint a queue after an agent's slug changed. The queue keys work by slug —
+ * `answerDeliveries` closes by slug, and the sweep's notify links into that
+ * agent's chat — so a rename that skipped this would leave an open delivery
+ * nobody can ever cross off, still pointing at an agent that no longer exists.
+ *
+ * Append-only like every other write here: a re-stated `pending` event carries
+ * the new slug, and the fold keeps whatever status the delivery had reached
+ * (`status: prev?.status`), so a notified one stays notified and a later
+ * `answered` event still lands on it. `ts` repeats the original `created_at` so
+ * the queue's ordering and age do not jump.
+ *
+ * @returns {number} deliveries repointed
+ */
+export function renameDeliveryAgent(storagePath, oldSlug, newSlug) {
+  if (!storagePath || !oldSlug || !newSlug || oldSlug === newSlug) return 0;
+  const mine = listDeliveries(storagePath, { limit: Infinity }).filter((d) => d.agent === oldSlug);
+  for (const d of mine) {
+    appendEvent(storagePath, {
+      ...d,
+      ev: DELIVERY_STATUS.PENDING,
+      ts: d.created_at || d.ts || nowIso(),
+      agent: newSlug,
+      agent_name: d.agent_name || newSlug,
+    });
+  }
+  return mine.length;
+}