@ai-ecoverse/slicc-agent 3.36.1 → 3.37.0

package/dist/spectrum/monitor-port.js

@@ -0,0 +1,621 @@
+import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
+import { Emitter } from "./emitter.js";
+const RENEW_MS = 1e4;
+const ROWS_MAX = 200;
+const HISTORY = 24;
+const BUDGET_WARN = 80;
+const BUDGET_ERROR = 95;
+const RETRY_WARN = 3;
+const RETRY_ERROR = 6;
+const RETRY_SLOW_MS = 12e4;
+const FAILED_MS = 6e5;
+const ABORT_MS = 3e4;
+const ABORT_ROW_MS = 1e4;
+const GROUP_MAX = 32;
+const TOTAL_MAX = 128;
+const ZOMBIE_MS = 6e4;
+const STALE_MS = 18e5;
+const CONTEXT_WARN = 0.8;
+const DEPTH_MAX = 6;
+const MINUTE = 6e4;
+const HOUR = 36e5;
+const KINDS = {
+  "pi.generation": "generating",
+  "pi.compaction": "compacting",
+  "slicc.cron": "cron",
+  "slicc.scoop-wait": "agent wait",
+  "slicc.scoop-reporter": "report",
+  "slicc.memory-extract": "memory extraction"
+};
+function age(ms) {
+  const seconds = Math.max(0, Math.round(ms / 1e3));
+  if (seconds < 60) return `${seconds}s`;
+  if (seconds < 3600) return `${Math.floor(seconds / 60)}m`;
+  return `${Math.floor(seconds / 3600)}h`;
+}
+function money(value) {
+  return `$${value.toFixed(2)}`;
+}
+function count(value) {
+  if (value < 1e3) return String(value);
+  if (value < 1e6) return `${(value / 1e3).toFixed(1)}k`;
+  return `${(value / 1e6).toFixed(1)}M`;
+}
+function bytes(value) {
+  if (value < 1024) return `${value} B`;
+  const units = ["KiB", "MiB", "GiB", "TiB"];
+  let size = value / 1024;
+  let unit = 0;
+  while (size >= 1024 && unit < units.length - 1) {
+    size /= 1024;
+    unit++;
+  }
+  return `${size.toFixed(size < 10 ? 1 : 0)} ${units[unit]}`;
+}
+function memoryVital(processes) {
+  const measured = processes.filter((row) => typeof row.rss === "number");
+  if (!measured.length) return [];
+  const total = measured.reduce((sum, row) => sum + row.rss, 0);
+  const agents = measured.filter((row) => row.agentId).reduce((sum, row) => sum + row.rss, 0);
+  return [
+    {
+      id: "memory",
+      label: "Process memory",
+      value: bytes(total),
+      unit: `${bytes(agents)} by agents`
+    }
+  ];
+}
+function taskLabel(task) {
+  if (task.kind === "pi.tool") return task.detail ? `tool ${task.detail}` : "tool";
+  if (task.kind === "slicc.scoop-anchor") return task.detail ? `scoop ${task.detail}` : "scoop";
+  return KINDS[task.kind] ?? task.kind;
+}
+function coneOf(agents, id) {
+  const visited = /* @__PURE__ */ new Set();
+  let current = id === null ? void 0 : agents.get(id);
+  while (current?.parentId && !visited.has(current.id)) {
+    visited.add(current.id);
+    const parent = agents.get(current.parentId);
+    if (!parent) break;
+    current = parent;
+  }
+  return current ? current.id : null;
+}
+function ordered(feed) {
+  const byId = new Map(feed.agents.map((agent) => [agent.id, agent]));
+  const out = [];
+  for (const cone of feed.agents.filter((agent) => agent.kind === "cone")) {
+    out.push(cone);
+    for (const agent of feed.agents)
+      if (agent.kind === "scoop" && coneOf(byId, agent.id) === cone.id) out.push(agent);
+  }
+  for (const agent of feed.agents) if (!out.includes(agent)) out.push(agent);
+  return out;
+}
+function taskTree(feed) {
+  const agents = new Map(feed.agents.map((agent) => [agent.id, agent]));
+  const order = ordered(feed).map((agent) => agent.id);
+  const rank = (task) => {
+    const at = task.agentId === null ? -1 : order.indexOf(task.agentId);
+    return at < 0 ? order.length : at;
+  };
+  const groupOf = (task) => coneOf(agents, task.agentId) ?? "";
+  const byId = new Map(feed.tasks.map((task) => [task.id, task]));
+  const children = /* @__PURE__ */ new Map();
+  const roots = [];
+  for (const task of feed.tasks) {
+    const owner = task.owner === null ? void 0 : byId.get(task.owner);
+    if (owner && groupOf(owner) === groupOf(task)) {
+      const list = children.get(owner.id) ?? [];
+      list.push(task);
+      children.set(owner.id, list);
+    } else roots.push(task);
+  }
+  const sort = (list) => list.sort((a, b) => rank(a) - rank(b) || Number(a.id) - Number(b.id));
+  const out = [];
+  const visit = (task, depth) => {
+    out.push({ task, depth });
+    for (const child of sort(children.get(task.id) ?? [])) visit(child, depth + 1);
+  };
+  for (const root of sort(roots)) visit(root, 0);
+  return out;
+}
+function capped(rows) {
+  if (rows.length <= ROWS_MAX) return rows;
+  const shown = rows.slice(0, ROWS_MAX - 1);
+  return [
+    ...shown,
+    { id: "more", name: `+${rows.length - shown.length} more`, meta: "", status: "idle" }
+  ];
+}
+function contextRatio(agent, models) {
+  const window = models.find((model) => model.id === agent.model)?.contextWindow;
+  if (!agent.context || !window) return null;
+  return Math.min(1, agent.context / window);
+}
+function rate(history, pick) {
+  const first = history[0];
+  const last = history.at(-1);
+  if (!first || !last || last.at - first.at < MINUTE) return null;
+  return (pick(last) - pick(first)) / (last.at - first.at) * HOUR;
+}
+function budgetVital(budget) {
+  const resets = new Date(budget.resets);
+  return {
+    id: "budget",
+    label: `Budget (${budget.window})`,
+    value: `${budget.percent}%`,
+    ...Number.isNaN(resets.getTime()) ? {} : { unit: `resets ${resets.toISOString().slice(0, 16).replace("T", " ")} UTC` },
+    ratio: Math.min(1, Math.max(0, budget.percent / 100))
+  };
+}
+function vitals(input, feed) {
+  const out = [];
+  if (feed.usage) {
+    const free = feed.usage.free;
+    const pick = (sample) => free ? sample.tokens : sample.cost;
+    const perHour = rate(input.history, pick);
+    const series = input.history.map(pick);
+    out.push({
+      id: "spend",
+      label: free ? "Tokens" : "Spend",
+      value: free ? count(feed.usage.tokens) : money(feed.usage.cost),
+      unit: free ? "tokens" : `${count(feed.usage.tokens)} tokens`,
+      ...perHour === null ? {} : { delta: `${free ? count(perHour) : money(perHour)}/h` },
+      ...series.length > 1 ? { series } : {}
+    });
+  }
+  const active = feed.agents.filter((agent) => agent.running).length;
+  const actives = input.history.map((sample) => sample.active);
+  out.push({
+    id: "agents",
+    label: "Active agents",
+    value: String(active),
+    unit: `of ${feed.agents.length}`,
+    ...actives.length > 1 ? { series: actives } : {}
+  });
+  const tools = feed.tasks.filter(
+    (task) => task.kind === "pi.tool" && task.status === "running"
+  ).length;
+  out.push({
+    id: "tasks",
+    label: "Running",
+    value: String(tools),
+    unit: `${tools === 1 ? "tool" : "tools"} \xB7 ${feed.tasks.length} ${feed.tasks.length === 1 ? "task" : "tasks"}`
+  });
+  out.push(
+    feed.processes ? {
+      id: "processes",
+      label: "Processes",
+      value: String(feed.processes.length),
+      unit: `${feed.processes.filter((row) => row.agentId).length} by agents`
+    } : { id: "processes", label: "Processes", value: "\u2014", unit: "kernel not reachable" }
+  );
+  out.push(...memoryVital(feed.processes ?? []));
+  const ratios = feed.agents.map((agent) => contextRatio(agent, input.models)).filter((ratio) => ratio !== null);
+  if (ratios.length) {
+    const fullest = Math.max(...ratios);
+    out.push({
+      id: "context",
+      label: "Fullest context",
+      value: `${Math.round(fullest * 100)}%`,
+      ratio: fullest
+    });
+  }
+  return out;
+}
+function stale(process, agents, now) {
+  if (!process.agentId || process.state === "Z" || !process.started) return false;
+  if (agents.get(process.agentId)?.tool === "bash") return false;
+  return now - process.started > STALE_MS;
+}
+function groups(processes) {
+  const out = /* @__PURE__ */ new Map();
+  for (const process of processes) {
+    if (!process.agentId) continue;
+    const list = out.get(process.pgid) ?? [];
+    list.push(process);
+    out.set(process.pgid, list);
+  }
+  return out;
+}
+function resetLine(budget) {
+  const resets = new Date(budget.resets);
+  return Number.isNaN(resets.getTime()) ? `The ${budget.window} budget is nearly used up.` : `Resets ${resets.toISOString().slice(0, 16).replace("T", " ")} UTC.`;
+}
+function budgetAlerts(budget) {
+  if (!budget || budget.percent < BUDGET_WARN) return [];
+  return [
+    {
+      id: "budget",
+      title: `Budget ${budget.percent}% used`,
+      detail: resetLine(budget),
+      severity: budget.percent >= BUDGET_ERROR ? "error" : "warn"
+    }
+  ];
+}
+function agentAlerts(input, feed) {
+  const out = [];
+  for (const agent of feed.agents) {
+    const retry = agent.retry;
+    if (retry && (retry.attempt >= RETRY_WARN || input.now - retry.since > RETRY_SLOW_MS))
+      out.push({
+        id: `retry-${agent.id}`,
+        title: `${agent.name} is retrying (attempt ${retry.attempt})`,
+        detail: retry.error,
+        severity: retry.attempt >= RETRY_ERROR ? "error" : "warn"
+      });
+    if (agent.failed && !agent.running)
+      out.push({
+        id: `turn-${agent.id}`,
+        title: `${agent.name}'s last turn failed`,
+        detail: agent.failed,
+        severity: "error"
+      });
+  }
+  return out;
+}
+function taskAlerts(input, feed, nameOf) {
+  const out = [];
+  for (const problem of feed.problems)
+    if (problem.kind === "task" && input.now - problem.at < FAILED_MS)
+      out.push({
+        id: `failed-${problem.id}`,
+        title: `A task of ${nameOf(problem.agentId)} failed`,
+        detail: problem.message,
+        severity: "error"
+      });
+  for (const task of feed.tasks)
+    if (task.aborting !== null && input.now - task.aborting > ABORT_MS)
+      out.push({
+        id: `abort-${task.id}`,
+        title: `Stop is still waiting for ${taskLabel(task)}`,
+        detail: `${nameOf(task.agentId)} asked it to stop ${age(input.now - task.aborting)} ago.`,
+        severity: "warn"
+      });
+  return out;
+}
+function crowdAlerts(processes, nameOf) {
+  const out = [];
+  for (const [pgid, members] of groups(processes))
+    if (members.length > GROUP_MAX)
+      out.push({
+        id: `runaway-${pgid}`,
+        title: `${nameOf(members[0].agentId)} has ${members.length} processes`,
+        detail: `Kill them with kill -- -${pgid}.`,
+        severity: members.length > GROUP_MAX * 2 ? "error" : "warn"
+      });
+  if (processes.length > TOTAL_MAX)
+    out.push({
+      id: "runaway-total",
+      title: `The kernel has ${processes.length} processes`,
+      detail: "Something may be starting processes in a loop.",
+      severity: processes.length > TOTAL_MAX * 2 ? "error" : "warn"
+    });
+  return out;
+}
+function processAlerts(input, feed, nameOf) {
+  const processes = feed.processes ?? [];
+  const agents = new Map(feed.agents.map((agent) => [agent.id, agent]));
+  const out = crowdAlerts(processes, nameOf);
+  for (const process of processes) {
+    if (process.zombie !== null && input.now - process.zombie > ZOMBIE_MS)
+      out.push({
+        id: `zombie-${process.pid}`,
+        title: `Process ${process.pid} is a zombie`,
+        detail: `Its parent ${process.ppid} hasn't reaped it.`,
+        severity: "warn"
+      });
+    if (stale(process, agents, input.now))
+      out.push({
+        id: `stale-${process.pid}`,
+        title: `${process.command || `Process ${process.pid}`} has run for ${age(input.now - process.started)}`,
+        detail: `${nameOf(process.agentId)} started it in the background. Kill it with kill ${process.pid}.`,
+        severity: input.now - process.started > STALE_MS * 2 ? "error" : "warn"
+      });
+  }
+  return out;
+}
+function contextAlerts(input, feed) {
+  const out = [];
+  for (const agent of feed.agents) {
+    const ratio = contextRatio(agent, input.models);
+    if (ratio !== null && ratio >= CONTEXT_WARN)
+      out.push({
+        id: `context-${agent.id}`,
+        title: `${agent.name} is ${Math.round(ratio * 100)}% full`,
+        detail: "Compact it soon to keep replies fast.",
+        severity: "warn"
+      });
+  }
+  if (feed.paused)
+    out.push({
+      id: "scheduler",
+      title: "The agent isn't scheduling work",
+      detail: "Its scheduler is paused.",
+      severity: "warn"
+    });
+  return out;
+}
+function alerts(input, feed) {
+  const agents = new Map(feed.agents.map((agent) => [agent.id, agent]));
+  const nameOf = (id) => id ? agents.get(id)?.name ?? id : "the session";
+  return [
+    ...budgetAlerts(input.budget),
+    ...agentAlerts(input, feed),
+    ...taskAlerts(input, feed, nameOf),
+    ...processAlerts(input, feed, nameOf),
+    ...contextAlerts(input, feed)
+  ];
+}
+function agentRow(agent, depth, models) {
+  const ratio = contextRatio(agent, models);
+  const word = agent.retry ? "retrying" : agent.running ? "running" : agent.failed ? "failed" : "idle";
+  const status = agent.retry ? "warn" : agent.running ? "active" : agent.failed ? "error" : "idle";
+  const meta = [word, agent.model, ratio === null ? null : `${Math.round(ratio * 100)}% context`].filter(Boolean).join(" \xB7 ");
+  const badges = [
+    ...agent.queued ? [`${agent.queued} queued`] : [],
+    ...agent.tool ? [agent.tool] : [],
+    ...agent.retry ? [`retry ${agent.retry.attempt}`] : []
+  ];
+  return {
+    id: agent.id,
+    name: agent.name,
+    meta,
+    status,
+    depth,
+    ...badges.length ? { badges } : {}
+  };
+}
+function taskRow(task, depth, nameOf, now) {
+  const aborting = task.aborting !== null && now - task.aborting > ABORT_ROW_MS;
+  const status = aborting ? "warn" : task.status === "running" || task.status === "completing" ? "active" : "idle";
+  const badges = [
+    ...task.background ? ["background"] : [],
+    ...task.aborting !== null ? ["aborting"] : [],
+    ...task.waits ? [`waits on ${task.waits}`] : []
+  ];
+  return {
+    id: task.id,
+    name: taskLabel(task),
+    depth: Math.min(depth, DEPTH_MAX),
+    meta: [nameOf(task.agentId), task.phase, age(now - task.since)].filter(Boolean).join(" \xB7 "),
+    status,
+    ...badges.length ? { badges } : {}
+  };
+}
+function processRows(input, feed) {
+  const agents = new Map(feed.agents.map((agent) => [agent.id, agent]));
+  const crowded = /* @__PURE__ */ new Set();
+  for (const [pgid, members] of groups(feed.processes ?? []))
+    if (members.length > GROUP_MAX) crowded.add(pgid);
+  return (feed.processes ?? []).map((process) => {
+    const runaway = stale(process, agents, input.now) || crowded.has(process.pgid);
+    const badges = [
+      ...process.agentId ? [agents.get(process.agentId)?.name ?? process.agentId] : [],
+      ...process.tty ? ["terminal"] : []
+    ];
+    return {
+      id: String(process.pid),
+      name: process.command || `[${process.pid}]`,
+      meta: [
+        `pid ${process.pid}`,
+        process.user,
+        typeof process.rss === "number" ? bytes(process.rss) : null,
+        process.started ? age(input.now - process.started) : null
+      ].filter(Boolean).join(" \xB7 "),
+      status: runaway ? "error" : process.state === "Z" ? "warn" : "idle",
+      ...badges.length ? { badges } : {}
+    };
+  });
+}
+const PROBLEM_KINDS = { task: "task failed", retry: "retry", turn: "turn failed" };
+function depthOf(agents, agent) {
+  const visited = /* @__PURE__ */ new Set();
+  let depth = 0;
+  let current = agent;
+  while (current.parentId && !visited.has(current.id)) {
+    visited.add(current.id);
+    const parent = agents.get(current.parentId);
+    if (!parent) break;
+    depth++;
+    current = parent;
+  }
+  return depth;
+}
+function sections(input, feed) {
+  const agents = new Map(feed.agents.map((agent) => [agent.id, agent]));
+  const nameOf = (id) => id ? agents.get(id)?.name ?? id : "session";
+  return [
+    {
+      id: "agents",
+      label: "Cones and scoops",
+      rows: ordered(feed).map(
+        (agent) => agentRow(agent, Math.min(depthOf(agents, agent), DEPTH_MAX), input.models)
+      )
+    },
+    {
+      id: "tasks",
+      label: "Tasks",
+      rows: capped(
+        taskTree(feed).map(({ task, depth }) => taskRow(task, depth, nameOf, input.now))
+      )
+    },
+    { id: "processes", label: "Processes", rows: capped(processRows(input, feed)) },
+    {
+      id: "problems",
+      label: "Recent problems",
+      rows: feed.problems.map((problem) => ({
+        id: problem.id,
+        name: `${nameOf(problem.agentId)} \xB7 ${PROBLEM_KINDS[problem.kind]}`,
+        meta: problem.message,
+        status: problem.kind === "retry" ? "warn" : "error"
+      }))
+    }
+  ];
+}
+function pageSections(input) {
+  return [
+    {
+      id: "changes",
+      label: "Pending changes",
+      rows: input.changes.map((change) => ({
+        id: change.path,
+        name: change.path,
+        meta: change.status,
+        status: "warn"
+      }))
+    },
+    {
+      id: "sprinkles",
+      label: "Sprinkles",
+      rows: input.sprinkles.map((sprinkle) => ({
+        name: sprinkle.title,
+        meta: sprinkle.name,
+        status: "idle"
+      }))
+    },
+    ...input.extra
+  ];
+}
+function monitorSnapshot(input) {
+  const budget = input.budget ? [budgetVital(input.budget)] : [];
+  const feed = input.feed;
+  if (!feed?.at)
+    return {
+      vitals: budget,
+      alerts: [],
+      sections: pageSections(input),
+      updatedAt: input.now
+    };
+  return {
+    vitals: [...budget, ...vitals(input, feed)],
+    alerts: alerts(input, feed),
+    sections: [...sections(input, feed), ...pageSections(input)],
+    updatedAt: feed.at
+  };
+}
+const globalIntervals = {
+  every: (run, ms) => setInterval(run, ms),
+  stop: (handle) => clearInterval(handle)
+};
+class MonitorAdapter {
+  #connection;
+  #emitter = new Emitter();
+  #extra;
+  #renewMs;
+  #intervals;
+  #now;
+  #listeners = 0;
+  #open;
+  #history = [];
+  constructor(connection, options = {}) {
+    this.#connection = connection;
+    this.#extra = options.extra ?? (() => []);
+    this.#renewMs = options.renewMs ?? RENEW_MS;
+    this.#intervals = options.intervals ?? globalIntervals;
+    this.#now = options.now ?? Date.now;
+    connection.monitor?.subscribe(() => this.#update());
+    connection.settings.state.subscribe(() => this.#update());
+    connection.changes?.subscribe(() => this.#update());
+    connection.sprinkles?.subscribe(() => this.#update());
+  }
+  on(type, listener) {
+    const off = this.#emitter.on(type, listener);
+    this.#listeners++;
+    if (this.#listeners === 1) this.#start();
+    let done = false;
+    return () => {
+      if (done) return;
+      done = true;
+      off();
+      this.#listeners--;
+      if (!this.#listeners) this.#stop();
+    };
+  }
+  snapshot() {
+    const settings = this.#connection.settings.state.value;
+    return monitorSnapshot({
+      feed: this.#connection.monitor?.value,
+      budget: settings?.budget,
+      models: settings?.models ?? [],
+      changes: this.#connection.changes?.value?.changes ?? [],
+      sprinkles: this.#connection.sprinkles?.value ?? [],
+      extra: this.#extra(),
+      history: this.#history,
+      now: this.#now()
+    });
+  }
+  resync() {
+    const open = this.#open;
+    const control = this.#connection.control;
+    if (open)
+      void open.lease.then((lease) => lease ? control.monitorOpen(lease, true, BACKGROUND_CONTEXT) : void 0).catch(() => void 0);
+    this.#emitter.emit("snapshot", this.snapshot());
+  }
+  #start() {
+    const control = this.#connection.control;
+    this.#history = [];
+    void control.changesOpen(BACKGROUND_CONTEXT).catch(() => void 0);
+    const lease = control.monitorOpen(null, true, BACKGROUND_CONTEXT).then(
+      (opened) => opened.lease || null,
+      () => null
+    );
+    const renew = () => void lease.then((held) => held ? control.monitorOpen(held, false, BACKGROUND_CONTEXT) : void 0).catch(() => void 0);
+    this.#open = { lease, timer: this.#intervals.every(renew, this.#renewMs) };
+  }
+  #stop() {
+    const open = this.#open;
+    if (!open) return;
+    this.#open = void 0;
+    this.#intervals.stop(open.timer);
+    const control = this.#connection.control;
+    void open.lease.then((held) => held ? control.monitorClose(held, BACKGROUND_CONTEXT) : void 0).catch(() => void 0);
+  }
+  #sample() {
+    const feed = this.#connection.monitor?.value;
+    if (!feed?.open || !feed.usage || this.#history.at(-1)?.at === feed.at) return;
+    this.#history.push({
+      at: feed.at,
+      cost: feed.usage.cost,
+      tokens: feed.usage.tokens,
+      active: feed.agents.filter((agent) => agent.running).length
+    });
+    if (this.#history.length > HISTORY) this.#history.shift();
+  }
+  #update() {
+    if (!this.#listeners) return;
+    this.#sample();
+    this.#emitter.emit("snapshot", this.snapshot());
+  }
+}
+export {
+  ABORT_MS,
+  ABORT_ROW_MS,
+  BUDGET_ERROR,
+  BUDGET_WARN,
+  CONTEXT_WARN,
+  DEPTH_MAX,
+  FAILED_MS,
+  GROUP_MAX,
+  HISTORY,
+  MonitorAdapter,
+  RENEW_MS,
+  RETRY_ERROR,
+  RETRY_SLOW_MS,
+  RETRY_WARN,
+  ROWS_MAX,
+  STALE_MS,
+  TOTAL_MAX,
+  ZOMBIE_MS,
+  age,
+  bytes,
+  count,
+  globalIntervals,
+  money,
+  monitorSnapshot,
+  taskLabel,
+  taskTree
+};
+//# sourceMappingURL=monitor-port.js.map