@chalky/chalky 0.0.0-stage → 0.1.4

package/app/extensions/mcp/browser.js

@@ -0,0 +1,64 @@
+// Browser half of the mcp extension: "/mcp" in the chat input opens a picker of the servers in app/mcp.json. A
+// server's box turns it on or off; open it to tick single tools (greyed while the server is off). Each change is
+// saved at once (mcp_set). A server's tools show once the agents have connected to it.
+const STYLE = `
+  .mcp { width: min(560px, calc(100vw - 32px)); max-height: 75vh; padding: 16px 18px; box-sizing: border-box;
+         border-radius: var(--radius, 16px); border: 1px solid var(--edge, #eee); color: var(--text, #1d1d1f);
+         background: var(--glass-strong, #fff); backdrop-filter: var(--blur); -webkit-backdrop-filter: var(--blur);
+         box-shadow: var(--shadow, 0 8px 30px rgba(0,0,0,.1)); font-size: 13px; }
+  .mcp::backdrop { background: rgba(0, 0, 0, .08); }
+  .mcp header { display: flex; justify-content: space-between; margin-bottom: 10px; font-weight: 600; }
+  .mcp header button { font: inherit; color: var(--muted, #888); background: none; border: 0; cursor: pointer; }
+  .mcp p { color: var(--muted, #888); margin: 0 0 10px; } .mcp p.error { color: #c62828; }
+  .mcp details { border-top: 1px solid rgba(128, 128, 128, .2); padding: 8px 0; }
+  .mcp summary { display: flex; gap: 8px; align-items: center; cursor: pointer; list-style: none; }
+  .mcp summary span { margin-left: auto; color: var(--muted, #888); }
+  .mcp label { display: flex; gap: 8px; align-items: baseline; padding: 4px 0 4px 24px; cursor: pointer; }
+  .mcp label span { color: var(--muted, #888); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+`;
+
+export default function (page) {
+  const style = document.createElement("style");
+  style.textContent = STYLE;
+  document.head.append(style);
+  const box = document.createElement("dialog");
+  box.className = "mcp";
+  document.body.append(box);
+  box.onclick = (e) => { if (e.target === box) box.close(); };  // a click on the backdrop closes it
+  let state = null;  // the "mcp" event: {file, servers, error}
+  const open = new Set();  // servers whose tools are shown, kept across redraws
+  const el = (tag, props, ...kids) => { const e = Object.assign(document.createElement(tag), props); e.append(...kids); return e; };
+  const tick = (checked, onchange, more = {}) => el("input", { type: "checkbox", checked, onchange, ...more });
+
+  function render() {
+    const close = el("button", { title: "Close", onclick: () => box.close() }, "✕");
+    box.replaceChildren(el("header", {}, "MCP tools", close));
+    if (!state) return box.append("Asking…");
+    box.append(el("p", {}, `Tick what the agents may use. The servers are in ${state.file}.`));
+    if (state.error) box.append(el("p", { className: "error" }, state.error));
+    const servers = Object.entries(state.servers);
+    if (!servers.length && !state.error) box.append(el("p", {}, "No servers yet: add them to that file."));
+    for (const [server, s] of servers) {
+      const on = s.tools.filter((t) => t.on).length;
+      const all = tick(s.enabled, () => page.send({ type: "mcp_set", server, on: all.checked }),
+        { onclick: (e) => e.stopPropagation() });  // ticking the server doesn't open or close it
+      const count = s.tools.length ? `${on}/${s.tools.length}` : s.enabled ? "no tools yet" : "off";
+      const d = el("details", { open: open.has(server) },
+        el("summary", {}, all, el("b", { title: s.description }, server), el("span", {}, count)));
+      d.ontoggle = () => d.open ? open.add(server) : open.delete(server);
+      for (const t of s.tools) {
+        const one = tick(t.on, () => page.send({ type: "mcp_set", server, tool: t.name, on: one.checked }), { disabled: !s.enabled });
+        d.append(el("label", { title: t.description }, one, t.name, el("span", {}, t.description)));
+      }
+      box.append(d);
+    }
+  }
+
+  page.command("mcp", () => {
+    state = null;  // ask again each time: mcp.json or the connections may have changed
+    render();
+    box.showModal();
+    page.send({ type: "mcp_list" });
+  }, "Choose the MCP tools the agents may use");
+  page.on("mcp", (ev) => { state = ev; if (box.open) render(); });
+}

package/app/extensions/mcp/guide.md

@@ -0,0 +1,5 @@
+# MCP servers
+Servers are in {code}/mcp.json (pi's built-in MCP: pi's docs/mcp.md); their tools are named mcp__<server>__<tool>.
+The user ticks which servers and tools are on with /mcp on the page.
+- To add a server when asked: add its entry to that file (secrets as ${ENV_VAR} in headers or env, never written
+  out), then call reload.

package/app/extensions/mcp/index.ts

@@ -0,0 +1,64 @@
+// MCP picker: "/mcp" on the page ticks which servers in app/mcp.json, and which of their tools, the agents may use.
+// pi's built-in MCP does the rest (server.ts loads it; pi's docs/mcp.md). A tick writes the fields pi's own /mcp
+// screen writes, in that file: "enabled": false for a server, toolExposure "<tool>": "hidden" (or back to the
+// server's exposure) for a tool. Then every agent reloads, one change at a time (a busy agent after its turn).
+// Tools: agent1's, from pi (getAllTools), once its connection to the server is up.
+//   page message  mcp_list                                 -> event "mcp" {file, servers: {name: {enabled, description,
+//                 mcp_set {server, on} | {server, tool, on}       tools: [{name, description, on}]}}, error}
+import fs from "node:fs";
+import path from "node:path";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+const host = (globalThis as any).chalky;
+const FILE = path.join(host.app, "mcp.json");
+
+function state() {
+  const servers = JSON.parse(fs.readFileSync(FILE, "utf8")).mcpServers ?? {};
+  const tools = host.agents.get("agent1")?.session.getAllTools() ?? [];
+  const out: Record<string, any> = {};
+  for (const [name, c] of Object.entries<any>(servers)) {
+    const enabled = c.enabled !== false;
+    out[name] = { enabled, description: c.description ?? c.url ?? c.command ?? "",
+      tools: tools.filter((t: any) => t.namespace?.aliases?.[0] === name).map((t: any) => {
+        const tool = t.name.slice(t.namespace.name.length + 2);  // mcp__<server>__<tool>
+        return { name: tool, description: t.description, on: enabled && (c.toolExposure?.[tool] ?? t.exposure) !== "hidden" };
+      }) };
+  }
+  return { file: FILE, servers: out, error: "" };
+}
+
+function set(msg: any) {
+  const json = JSON.parse(fs.readFileSync(FILE, "utf8")), c = json.mcpServers?.[msg.server];
+  if (!c) return;
+  if (msg.tool === undefined) { if (msg.on) delete c.enabled; else c.enabled = false; }
+  else {
+    const own = c.exposure ?? "codemode";  // pi's default
+    (c.toolExposure ??= {})[msg.tool] = !msg.on ? "hidden" : own === "hidden" ? "direct" : own;
+  }
+  fs.writeFileSync(FILE, JSON.stringify(json, null, 2) + "\n");
+}
+
+async function reloadAll() {
+  for (const a of host.agents.values()) {
+    if (a.session.isStreaming) { if (!host.reloadAfterTurn.has(a.name)) host.reloadAfterTurn.set(a.name, true); }
+    else await host.reloadAgent(a.name, true);  // quiet: ticking isn't news for the conversation
+  }
+}
+
+export default function (pi: ExtensionAPI) {
+  const guide = fs.readFileSync(path.join(import.meta.dirname, "guide.md"), "utf8");
+  pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.mcp = guide; });
+
+  host.pageHandlers.set("mcp", async (msg: any) => {
+    if (msg.type !== "mcp_list" && msg.type !== "mcp_set") return;
+    try {
+      if (msg.type === "mcp_set") set(msg);
+      host.broadcast("mcp", state());  // before the reloads: agent1's tools are still listed, the file has the ticks
+    } catch (e: any) {
+      return host.broadcast("mcp", { file: FILE, servers: {}, error: `${FILE}: ${e.message}` });
+    }
+    // reloads one after another, also across this extension's own reloads (host.shared outlives them)
+    if (msg.type === "mcp_set") host.shared.set("mcp-reloads", (host.shared.get("mcp-reloads") ?? Promise.resolve())
+      .then(reloadAll).catch((e: any) => console.error("mcp: reload failed:", e)));
+  });
+}

package/app/extensions/model/index.ts

@@ -0,0 +1,20 @@
+// The model: one OpenAI-compatible endpoint (in Chalky: keyproxy -> LiteLLM), as the Python agent's llm plugin.
+// MODEL_BASE_URL (up to /v1), MODEL_API_KEY, MODEL_NAME (a LiteLLM route, "default"). pi does the calls,
+// streaming, retries, thinking and compaction; this only tells it where. The host selects provider "chalky".
+// MODEL_CONTEXT_WINDOW: the model's context size; pi compacts when a request would leave less than 16384 tokens of it.
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+export default function (pi: ExtensionAPI) {
+  if (!process.env.MODEL_BASE_URL) return;  // pi then says no model is configured on each message
+  pi.registerProvider("chalky", {
+    baseUrl: process.env.MODEL_BASE_URL,
+    apiKey: "$MODEL_API_KEY",  // read from the environment by pi, not stored in the session
+    api: "openai-completions",
+    compat: { supportsStore: false, supportsDeveloperRole: false },  // LiteLLM may not take "store" (pi-ai docs)
+    models: [{
+      id: process.env.MODEL_NAME || "default", name: "Chalky default route", reasoning: true,
+      input: ["text", "image"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+      contextWindow: Number(process.env.MODEL_CONTEXT_WINDOW || 200000), maxTokens: 16000,
+    }],
+  });
+}

package/app/extensions/page/index.ts

@@ -0,0 +1,96 @@
+// The page the user sees (Python: chat_ui.py), as an extension like the others: agents can change it, and the
+// versions extension keeps and rolls back its history.
+//   GET /                                  page.html, read fresh on every load (edits show on reload)
+//   GET /page/scripts                      the browser parts the page imports: one per extension folder with a browser.js
+//   GET /extensions/<name>/browser.js      that file (only browser.js: an extension's other files aren't served)
+// An extension's browser.js is its part of the page: its default export gets the page's API (see page.html's header).
+// Found by looking in each folder, as pi finds index.ts, so an extension has nothing to register.
+// The conversation on the page: pi's messages in the page's format (OpenAI chat, as the Python harness stored them).
+//   event "message"  {agent, role, content, ...} as an agent's pi session ends each message (host.eventHandlers)
+//   events "run_start", "run_end", "compacted"  an agent's run started, ended (settled), its context was compacted
+//   on opening, a page gets every agent's record, in time order (host.connectHandlers): all its session files
+//   (compaction keeps the original entries, so the page still shows everything)
+// "joined" (an agent's first system message) and reply_to (who a turn's answer goes to: the host's "chalky.turn"
+// entry just before the user message that starts it) are read from the session, so nothing is kept here.
+import fs from "node:fs";
+import path from "node:path";
+import { SessionManager, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+const host = (globalThis as any).chalky;
+const EXTENSIONS = path.dirname(import.meta.dirname);
+const browserFile = (name: string) => path.join(EXTENSIONS, name, "browser.js");
+
+export default function (_pi: ExtensionAPI) {
+  host.routes.set("/", (_req: any, res: any) =>
+    res.writeHead(200, { "content-type": "text/html", "cache-control": "no-cache" }).end(fs.readFileSync(path.join(import.meta.dirname, "page.html"))));
+  host.routes.set("/page/scripts", (_req: any, res: any) => {
+    const names = fs.readdirSync(EXTENSIONS).filter((n) => fs.existsSync(browserFile(n))).sort();
+    res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(names.map((n) => `/extensions/${n}/browser.js`)));
+  });
+  host.routes.set("/extensions/", (req: any, res: any) => {
+    const [, , name, file] = new URL(req.url, "http://x").pathname.split("/");
+    if (file !== "browser.js" || !/^[\w-]+$/.test(name ?? "") || !fs.existsSync(browserFile(name))) return res.writeHead(404).end();
+    res.writeHead(200, { "content-type": "text/javascript", "cache-control": "no-cache" }).end(fs.readFileSync(browserFile(name)));
+  });
+}
+
+const text = (content: any) => typeof content === "string" ? content
+  : content.filter((p: any) => p.type === "text").map((p: any) => p.text).join("");
+
+// pi message -> the page's message. first: the agent's first system message, shown as "<agent> joined".
+// replyTo: on the message that starts a turn, who its answer goes to (not on items slipped into a running turn).
+function toPage(m: any, first = false, replyTo?: string) {
+  const ts = (m.timestamp ?? Date.now()) / 1000;
+  if (m.role === "system") return first ? { role: "system", content: "", ts } : null;
+  if (m.role === "user" || (m.role === "custom" && m.customType === "inbox")) {  // inbox: swarm items filed without a turn
+    return { role: "user", content: text(m.content), ...(replyTo && { reply_to: replyTo }), ts };
+  }
+  if (m.role === "assistant") {
+    const calls = m.content.filter((p: any) => p.type === "toolCall")
+      .map((p: any) => ({ id: p.id, type: "function", function: { name: p.name, arguments: JSON.stringify(p.arguments) } }));
+    const reasoning = m.content.filter((p: any) => p.type === "thinking").map((p: any) => p.thinking).join("\n");
+    return { role: "assistant", content: text(m.content), ...(reasoning && { reasoning }), ...(calls.length && { tool_calls: calls }),
+             ...(m.errorMessage && { content: `error: ${m.errorMessage}` }), ts };
+  }
+  if (m.role === "toolResult")
+    return { role: "tool", tool_call_id: m.toolCallId, content: (m.isError ? "error: " : "") + text(m.content), ts };
+  return null;  // pi's own kinds (custom, compaction summary, ...): not shown
+}
+
+// Every message of an agent, from all its session files (older rounds first), in the page's format
+function record(name: string) {
+  const sm = host.agents.get(name).session.sessionManager, current = sm.getSessionFile(), dir = path.dirname(current);
+  return fs.readdirSync(dir).filter((f) => f.endsWith(".jsonl")).sort().flatMap((f) => {
+    const p = path.join(dir, f);
+    const entries = p === current ? sm.getEntries() : SessionManager.open(p, dir).getEntries();
+    let joined = false, replyTo: string | undefined;
+    return entries.flatMap((e: any) => {
+      if (e.type === "custom" && e.customType === "chalky.turn") replyTo = e.data.reply_to;
+      const msg = e.type === "message" ? e.message : e.type === "custom_message" ? { ...e, role: "custom", timestamp: Date.parse(e.timestamp) } : null;
+      if (!msg) return [];
+      const m = toPage(msg, !joined && msg.role === "system", msg.role === "user" ? replyTo : undefined);
+      if (msg.role === "system") joined = true;
+      if (msg.role === "user") replyTo = undefined;
+      return m ? [{ agent: name, ...m }] : [];
+    });
+  });
+}
+
+// pi saves a message after its message_end, so the session holds what came before it
+host.eventHandlers.set("page", (name: string, ev: any) => {
+  if (ev.type === "message_end") {
+    const entries = host.agents.get(name)?.session.sessionManager.getEntries() ?? [];
+    const first = ev.message.role === "system" && !entries.some((e: any) => e.type === "message" && e.message.role === "system");
+    const last = entries.findLast((e: any) => !(e.type === "message" && e.message.role === "system"));  // pi may put a prompt update between
+    const replyTo = ev.message.role === "user" && last?.type === "custom" && last.customType === "chalky.turn" ? last.data.reply_to : undefined;
+    const m = toPage(ev.message, first, replyTo);
+    if (m) host.broadcast("message", { agent: name, ...m });
+  } else if (ev.type === "agent_start") host.broadcast("run_start", { agent: name });
+  else if (ev.type === "agent_settled") host.broadcast("run_end", { agent: name });
+  else if (ev.type === "compaction_end" && ev.result)
+    host.broadcast("compacted", { agent: name, tokens_before: ev.result.tokensBefore, tokens_after: ev.result.estimatedTokensAfter ?? "?" });
+});
+
+host.connectHandlers.set("page", (send: (type: string, data: object) => void) => {
+  for (const m of [...host.agents.keys()].flatMap(record).sort((x: any, y: any) => x.ts - y.ts)) send("message", m);
+});