@theprototype/agent 1.48.0 → 1.50.0

package/src/mcpServer.js

@@ -4,7 +4,9 @@
 // is up (never a hardcoded catalog: it is the live app's set, physics and mesh gen included
 // when the host offers them) and re-read on a poll and after every call, so a host that turns
 // a provider on, or a role change, reaches the client as `notifications/tools/list_changed`.
-// Beside them sit the bridge's own four: connect, status, screenshot, wait_for_edit.
+// Beside them sit the bridge's own four: connect, status, screenshot, wait_for_edit — and (roadmap 56,
+// browser engine) the chat: wait_for_message, reply, ask, get_thread (./chat.js), so people can type to the
+// agent in the app's AI assistant window while the terminal stays a second window onto the same thread.
 //
 // The server answers IMMEDIATELY and connects in the background: approval is a human's, so no
 // tool registration ever waits on it — scene tools refuse with "not in the session yet" until
@@ -14,6 +16,7 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import { bridgeTools, mapHostTools, toolSignature, textResult, resultIsError } from './mcpTools.js';
+import { chatTools, waitMs, ImageSpool, messageContent, withImportPath, execBridged } from './chat.js';
 import { parseLink } from './link.js';
 
 export const PKG_VERSION = (() => {
@@ -27,7 +30,10 @@ export const PKG_VERSION = (() => {
 const INSTRUCTIONS = [
 	'theprototype.app session bridge: you are a peer in a live, shared 3D scene. Everything you change replicates to everyone in the session, with undo.',
 	'Start with status (or connect with the session link the host gave you). The scene tools are the host app\'s own and appear once you are in;',
-	'call list_scene first so you use real uuids, batch many objects into one call, and use screenshot to look at the result. wait_for_edit waits for the humans.'
+	'call list_scene first so you use real uuids, batch many objects into one call, and use screenshot to look at the result. wait_for_edit waits for the humans.',
+	'People can also TYPE TO YOU in the app\'s AI assistant window: when the user asks you to listen (e.g. "listen to my theprototype session"), loop',
+	'wait_for_message → act → reply({text, final: true}) → wait_for_message (null = nobody wrote yet: wait again). Respect each message\'s role (viewers may not edit).',
+	'When the user talks to you in this terminal while you are in a session, echo it: reply({prompt: <their words>, text: <your answer>, final: true}), so the app\'s thread is complete.'
 ].join(' ');
 
 const CONNECT_WAIT_MS = 45000;
@@ -44,11 +50,13 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
  * @param {any[]} [opts.staticTools] host tools known before connecting (the wire engine's catalog)
  * @param {number} [opts.pollMs] how often to re-read the host's tool list (default 5000)
  * @param {number} [opts.connectWaitMs]
+ * @param {ImageSpool | null} [opts.spool] where chat images are written (null = no temp files)
  * @param {(m: string) => void} [opts.log]
  */
 export function createBridgeServer(opts) {
 	const log = opts.log || (() => {});
-	const builtins = bridgeTools(opts.kind);
+	const builtins = [...bridgeTools(opts.kind), ...(opts.kind === 'browser' ? chatTools() : [])];
+	const spool = opts.spool === undefined ? new ImageSpool() : opts.spool;
 	const reserved = builtins.map((t) => t.name);
 	/** @type {any} */
 	let engine = null;
@@ -66,7 +74,7 @@ export function createBridgeServer(opts) {
 
 	if (opts.staticTools) {
 		const m = mapHostTools(opts.staticTools, reserved);
-		hostTools = m.tools;
+		hostTools = m.tools.map(withImportPath);
 		dropped = m.dropped;
 		signature = toolSignature(hostTools);
 	}
@@ -94,10 +102,11 @@ export function createBridgeServer(opts) {
 			return;
 		}
 		const m = mapHostTools(raw, reserved);
-		const sig = toolSignature(m.tools);
+		const mapped = m.tools.map(withImportPath);
+		const sig = toolSignature(mapped);
 		if (sig !== signature) {
 			signature = sig;
-			hostTools = m.tools;
+			hostTools = mapped;
 			dropped = m.dropped;
 			log('host tool list: ' + hostTools.length + ' tools');
 			notifyChanged();
@@ -131,7 +140,11 @@ export function createBridgeServer(opts) {
 		const mine = engine;
 		starting = engine
 			.start()
-			.then(() => refreshTools())
+			.then(() => {
+				// 56: the window labels the agent "via MCP" (an older page has no chat: fine)
+				if (engine === mine && mine.kind === 'browser') mine.chat?.('chatStatus', null, { mode: 'mcp' }).catch(() => {});
+				return refreshTools();
+			})
 			.catch((/** @type {any} */ e) => log('connect failed: ' + e.message));
 		if (!poll) {
 			poll = setInterval(() => refreshTools().catch(() => {}), opts.pollMs ?? 5000);
@@ -178,7 +191,7 @@ export function createBridgeServer(opts) {
 
 	server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [...builtins, ...hostTools] }));
 
-	server.setRequestHandler(CallToolRequestSchema, async (req) => {
+	server.setRequestHandler(CallToolRequestSchema, async (req, extra) => {
 		const name = req.params.name;
 		const a = /** @type {any} */ (req.params.arguments || {});
 		try {
@@ -210,6 +223,10 @@ export function createBridgeServer(opts) {
 				const r = await engine.waitForEdit(ms);
 				return textResult(r, resultIsError(r));
 			}
+			if (name === 'wait_for_message' || name === 'reply' || name === 'ask' || name === 'get_thread') {
+				if (!engineReady()) return textResult({ error: 'the agent page is not up yet (' + engine.state + ')' }, true);
+				return await chatCall(name, a, extra?.signal);
+			}
 			if (!hostTools.some((t) => t.name === name)) {
 				return textResult({ error: 'unknown tool: ' + name, note: engineReady() ? 'call status for the tools this session offers' : 'the scene tools appear once connected' }, true);
 			}
@@ -223,7 +240,7 @@ export function createBridgeServer(opts) {
 					true
 				);
 			}
-			const r = await engine.exec(name, a);
+			const r = await execBridged(engine, name, a);
 			refreshTools().catch(() => {});
 			if (r && r.__missing) return textResult({ error: 'this app build has no exec()' }, true);
 			return textResult(r, resultIsError(r));
@@ -232,9 +249,45 @@ export function createBridgeServer(opts) {
 		}
 	});
 
+	/**
+	 * 56: the chat tools. wait_for_message long-polls the page's inbox; if the client gives up on the call
+	 * (cancel / timeout) the page's wait is released, and a message that arrived in that instant goes back.
+	 * @param {string} name @param {any} a @param {AbortSignal | undefined} signal
+	 */
+	async function chatCall(name, a, signal) {
+		const e = engine;
+		if (name === 'wait_for_message') {
+			const token = 'w' + Date.now().toString(36) + Math.random().toString(36).slice(2, 7);
+			const release = () => e.chat('abortWait', token).catch(() => {});
+			signal?.addEventListener('abort', release, { once: true });
+			try {
+				const msg = await e.chat('nextMessage', waitMs(a.timeoutS ?? a.timeout_s), token);
+				if (!msg) return textResult(null);
+				if (signal?.aborted) {
+					await e.chat('requeue', msg).catch(() => {});
+					return textResult({ error: 'cancelled' }, true);
+				}
+				return { content: messageContent(msg, spool) };
+			} finally {
+				signal?.removeEventListener('abort', release);
+			}
+		}
+		if (name === 'reply') {
+			const r = await e.chat('reply', { text: a.text, final: a.final, to: a.to, chatId: a.chatId ?? a.chat_id, prompt: a.prompt, replyTo: a.replyTo });
+			return textResult(r, !r?.ok);
+		}
+		if (name === 'ask') {
+			const r = await e.chat('ask', { question: a.question, options: a.options, to: a.to, chatId: a.chatId ?? a.chat_id });
+			return textResult(r, !r?.ok);
+		}
+		const r = await e.chat('thread', { chatId: a.chatId ?? a.chat_id, before: a.before, limit: a.limit });
+		return textResult(r);
+	}
+
 	async function shutdown() {
 		if (closed) return;
 		closed = true;
+		spool?.cleanup();
 		if (poll) clearInterval(poll);
 		const e = engine;
 		engine = null;

package/src/repl.js

@@ -1,11 +1,226 @@
-// REPL mode (roadmap 10 B4 → 55 A3): a readline loop that drives the scene through an
-// OpenAI-compatible endpoint's tool calls. Either engine: the browser engine hands the model
-// the host app's own tools and system prompt; the wire engine its fixed object catalog.
-// Slash commands: /status /tools /raw <tool> <json> /quit.
+// REPL mode (roadmap 10 B4 → 55 A3 → 56 C3/C4): your own OpenAI-compatible model drives the scene. Either
+// engine: the browser engine hands the model the host app's own tools and system prompt; the wire engine its
+// fixed object catalog. Slash commands: /status /tools /raw <tool> <json> /quit.
+//
+// 56: the bridge owns the model loop here, so a message typed in the app's AI assistant window goes STRAIGHT
+// IN as a user turn — no terminal needed after launch. Terminal lines and app messages share ONE transcript,
+// run one at a time; the reply streams into the window (and the terminal), terminal turns show "via
+// terminal", and Stop in the window ends the turn (the page answers `cancelled` to the next tool call).
 import readline from 'node:readline';
 import { runChat } from './llmClient.js';
 import { mapHostTools } from './mcpTools.js';
 import { redactLink } from './link.js';
+import { execBridged } from './chat.js';
+
+/** how often a streamed reply is pushed to the window (each push carries the full text so far) */
+const FLUSH_MS = 250;
+/** how long one app wait lasts before it re-arms */
+const APP_WAIT_MS = 25000;
+
+/**
+ * One REPL conversation over an engine. Exported for the tests (a fake engine + a stub endpoint).
+ * @param {Object} o
+ * @param {any} o.engine
+ * @param {{apiUrl: string, apiKey: string, model: string}} o.cfg
+ * @param {(s: string) => void} [o.out] terminal output
+ * @param {(s: string) => void} [o.log] stderr notes
+ * @param {number} [o.flushMs]
+ * @param {number} [o.appWaitMs]
+ */
+export function createReplSession(o) {
+	const { engine, cfg } = o;
+	const out = o.out ?? ((s) => process.stdout.write(s));
+	const log = o.log ?? ((s) => console.error('[agent] ' + s));
+	const flushMs = o.flushMs ?? FLUSH_MS;
+	/** does this page take app messages? (browser engine on tpAgent 1.1.0) */
+	let chatOn = engine.kind === 'browser' && typeof engine.chat === 'function';
+	/** @type {any[]} */
+	const history = [];
+	/** @type {Promise<any>} */
+	let busy = Promise.resolve();
+	let pendingTerminal = 0;
+	let closed = false;
+	/** @type {string | null} */
+	let waitToken = null;
+
+	const toolsFor = async () => mapHostTools(await engine.tools(), []).tools.map((t) => ({ name: t.name, description: t.description, parameters: t.inputSchema }));
+
+	/** @param {string} method @param {...any} args */
+	const chat = (method, ...args) => (chatOn ? engine.chat(method, ...args) : Promise.resolve(null));
+
+	/** @param {() => Promise<any>} fn */
+	function enqueue(fn) {
+		const job = busy.then(fn);
+		busy = job.catch(() => {});
+		return job;
+	}
+
+	/**
+	 * A streamed reply into the window: deltas batched every flushMs, the last push final.
+	 * @param {any} ids replyTo/to/chatId for an app request, or {via:'cli'} for a terminal turn
+	 */
+	function streamer(ids) {
+		let pending = '';
+		let any = false;
+		let lastRound = 0;
+		/** @type {ReturnType<typeof setTimeout> | null} */
+		let timer = null;
+		let sending = Promise.resolve();
+		const push = (/** @type {boolean} */ final) => {
+			const delta = pending;
+			pending = '';
+			sending = sending.then(() => chat('reply', { ...ids, delta, final }).catch(() => null));
+			return sending;
+		};
+		return {
+			/** @param {string} t @param {number} round */
+			delta(t, round) {
+				if (any && round !== lastRound) {
+					pending += '\n\n';
+					out('\n\n');
+				}
+				lastRound = round;
+				any = true;
+				pending += t;
+				out(t);
+				if (!timer) timer = setTimeout(() => ((timer = null), push(false)), flushMs);
+			},
+			/** @param {string} [tail] */
+			async end(tail) {
+				if (timer) clearTimeout(timer);
+				timer = null;
+				if (tail) {
+					pending += (any ? '\n\n' : '') + tail;
+					out((any ? '\n\n' : '') + tail);
+				}
+				out('\n');
+				return push(true);
+			}
+		};
+	}
+
+	/**
+	 * Run one turn of the model with streaming. Returns 'done' | 'stopped'.
+	 * @param {any} user the user message (text or content parts) @param {any} ids
+	 */
+	async function turn(user, ids) {
+		if (!history.length) history.push({ role: 'system', content: await engine.systemPrompt() });
+		history.push({ role: 'user', content: user });
+		const s = streamer(ids);
+		let stopped = false;
+		try {
+			await runChat({
+				cfg,
+				messages: history,
+				tools: await toolsFor(),
+				executeTool: (name, a) => execBridged(engine, name, a), // local `path`s for import_model / import_file
+				onTool: (name, a) => out('\n  · ' + name + ' ' + summarizeArgs(a) + '\n'),
+				onDelta: (t, round) => s.delta(t, round),
+				stopOn: (r) => {
+					if (r && r.cancelled) stopped = true;
+					return stopped;
+				}
+			});
+		} catch (e) {
+			const msg = String(/** @type {any} */ (e).message || e).split('\n')[0];
+			// an endpoint without vision: say so once and go on in text
+			if (Array.isArray(user) && /image|vision|content/i.test(msg) && !/^API 401/.test(msg)) {
+				history.pop();
+				const text = user.filter((p) => p.type === 'text').map((p) => p.text).join('\n');
+				log('this model refused the images (' + msg.slice(0, 80) + '); retrying without them');
+				return turn(text + '\n(The person attached images, but this model cannot see images.)', ids);
+			}
+			await s.end('Error: ' + msg);
+			throw e;
+		}
+		await s.end(stopped ? '(stopped)' : undefined);
+		if (stopped) history.push({ role: 'system', content: 'The person pressed Stop: that request was cancelled. Do not continue it.' });
+		return stopped ? 'stopped' : 'done';
+	}
+
+	/** A line typed in the terminal: echoed into the app's thread as "via terminal", then answered. @param {string} text */
+	function terminal(text) {
+		pendingTerminal++;
+		if (waitToken) chat('abortWait', waitToken).catch(() => {});
+		return enqueue(async () => {
+			try {
+				await chat('reply', { prompt: text, via: 'cli' }).catch(() => null);
+				return await turn(text, { via: 'cli' });
+			} finally {
+				pendingTerminal--;
+			}
+		});
+	}
+
+	/** A message typed in the app's window, as a user turn. @param {any} msg */
+	function appTurn(msg) {
+		return enqueue(async () => {
+			const who = (msg.name || String(msg.from || '').toUpperCase() || 'someone') + ' (' + (msg.role || 'editor') + ')';
+			out('\n[app] ' + who + ': ' + (msg.text || '') + (msg.images?.length ? ' [' + msg.images.length + ' image' + (msg.images.length > 1 ? 's' : '') + ']' : '') + '\n');
+			if (msg.history?.length) {
+				const lines = msg.history.map((/** @type {any} */ h) => (h.name || h.from) + (h.via === 'cli' ? ' (terminal)' : '') + ': ' + String(h.text || '').slice(0, 400));
+				history.push({ role: 'system', content: 'Earlier in this chat (' + msg.chatId + '). ' + (msg.summary || '') + '\n' + lines.join('\n') });
+			}
+			let text = '[' + who + ' in the app' + (msg.answers ? ', answering your question' : '') + ']: ' + (msg.text || '');
+			if (msg.role === 'viewer') text += '\n(This person is a VIEWER: answer, but do not change the scene for them — edits are refused.)';
+			const user = msg.images?.length
+				? [{ type: 'text', text }, ...msg.images.map((/** @type {any} */ im) => ({ type: 'image_url', image_url: { url: 'data:' + im.mimeType + ';base64,' + im.data } }))]
+				: text;
+			return turn(user, { replyTo: msg.id, to: msg.from, chatId: msg.chatId }).catch((e) => log('turn failed: ' + e.message));
+		});
+	}
+
+	/** Pick up app messages until stopped (each one waits for the turn before it). */
+	async function appLoop() {
+		if (!chatOn) return;
+		await chat('chatStatus', null, { mode: 'repl', model: cfg.model }).catch(() => {});
+		while (!closed) {
+			await busy;
+			if (closed) break;
+			const token = 'r' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
+			waitToken = token;
+			let msg;
+			try {
+				msg = await engine.chat('nextMessage', o.appWaitMs ?? APP_WAIT_MS, token);
+			} catch (e) {
+				waitToken = null;
+				if (/no agent chat/.test(String(/** @type {any} */ (e).message))) {
+					log('this app build has no agent chat: only the terminal talks to the model');
+					chatOn = false;
+					return;
+				}
+				await new Promise((r) => setTimeout(r, 1000));
+				continue;
+			}
+			waitToken = null;
+			if (!msg) continue;
+			if (pendingTerminal > 0) {
+				// a terminal line got in first: this one goes back and waits its turn
+				await chat('requeue', msg).catch(() => {});
+				continue;
+			}
+			appTurn(msg);
+		}
+	}
+
+	return {
+		history,
+		terminal,
+		appTurn,
+		appLoop,
+		toolsFor,
+		get busy() {
+			return busy;
+		},
+		get chatOn() {
+			return chatOn;
+		},
+		stop() {
+			closed = true;
+			if (waitToken) chat('abortWait', waitToken).catch(() => {});
+		}
+	};
+}
 
 /**
  * @param {Record<string, any>} args
@@ -21,11 +236,15 @@ export async function runRepl(args, link, makeEngine) {
 	if (!cfg.apiUrl || !cfg.model) throw new Error('repl needs --api-url and --model (or env AGENT_API_URL / AGENT_MODEL)');
 
 	const engine = makeEngine(link);
+	/** @type {ReturnType<typeof createReplSession> | null} */
+	let session = null;
 	const quit = async (/** @type {number} */ code) => {
+		session?.stop();
 		await engine.close().catch(() => {});
 		process.exit(code);
 	};
 	process.on('SIGINT', () => quit(0));
+	process.on('SIGTERM', () => quit(0));
 	console.error('[agent] joining ' + redactLink(link) + ' (' + engine.kind + ' engine)…');
 	await engine.start();
 	console.error('[agent] waiting for the host to let us in (Approve, or an invite link)…');
@@ -37,39 +256,36 @@ export async function runRepl(args, link, makeEngine) {
 		}
 		await new Promise((r) => setTimeout(r, 500));
 	}
-	console.error('[agent] in the session. Type a request, or /help. Ctrl+C to quit.\n');
+	session = createReplSession({ engine, cfg });
+	const s = session;
+	console.error(
+		'[agent] in the session. Type a request, or /help. Ctrl+C to quit.' +
+			(s.chatOn ? ' People can also type to ' + cfg.model + ' in the app\'s AI assistant window.' : '') +
+			'\n'
+	);
 
-	const toolsFor = async () => mapHostTools(await engine.tools(), []).tools.map((t) => ({ name: t.name, description: t.description, parameters: t.inputSchema }));
-	/** @type {any[]} */
-	const history = [{ role: 'system', content: await engine.systemPrompt() }];
-
-	const rl = readline.createInterface({ input: process.stdin, output: process.stdout, prompt: 'scene> ' });
+	const interactive = !!process.stdin.isTTY;
+	const rl = readline.createInterface({ input: process.stdin, output: process.stdout, prompt: 'scene> ', terminal: interactive });
+	s.appLoop().catch((e) => console.error('[agent] app messages stopped: ' + e.message));
 	rl.prompt();
 
 	rl.on('line', async (line) => {
 		const text = line.trim();
 		if (!text) return rl.prompt();
-		rl.pause();
 		try {
-			if (text.startsWith('/')) await handleSlash(text, engine, rl, toolsFor);
-			else {
-				history.push({ role: 'user', content: text });
-				const reply = await runChat({
-					cfg,
-					messages: history,
-					tools: await toolsFor(),
-					executeTool: (name, a) => engine.exec(name, a),
-					onTool: (name, a) => console.log('  · ' + name + ' ' + summarizeArgs(a))
-				});
-				if (reply) console.log('\n' + reply + '\n');
+			if (text.startsWith('/')) {
+				if ((await handleSlash(text, engine, s.toolsFor)) === 'quit') return quit(0);
 			}
+			else await s.terminal(text);
 		} catch (e) {
 			console.error('[agent] error: ' + /** @type {any} */ (e).message + '\n');
 		}
-		rl.resume();
 		rl.prompt();
 	});
-	rl.on('close', () => quit(0));
+	// a closed stdin (a script, `--repl < /dev/null`) keeps the agent answering the app; Ctrl+C quits
+	rl.on('close', () => {
+		if (interactive || !s.chatOn) quit(0);
+	});
 }
 
 /** @param {any} a */
@@ -78,10 +294,10 @@ function summarizeArgs(a) {
 	return '';
 }
 
-/** @param {string} text @param {any} engine @param {any} rl @param {() => Promise<any[]>} toolsFor */
-async function handleSlash(text, engine, rl, toolsFor) {
+/** @param {string} text @param {any} engine @param {() => Promise<any[]>} toolsFor @returns {Promise<'quit' | void>} */
+async function handleSlash(text, engine, toolsFor) {
 	const [cmd, ...rest] = text.slice(1).split(/\s+/);
-	if (cmd === 'quit' || cmd === 'q' || cmd === 'exit') return rl.close();
+	if (cmd === 'quit' || cmd === 'q' || cmd === 'exit') return 'quit';
 	if (cmd === 'status') console.log(JSON.stringify({ ...engine.describe(), session: await engine.status() }, null, 2));
 	else if (cmd === 'tools') for (const t of await toolsFor()) console.log('  ' + t.name);
 	else if (cmd === 'raw') {