@theoremjs/studio 0.0.0-stage → 0.3.2

package/server/serve.js

@@ -0,0 +1,413 @@
+/**
+ * Starts the studio's local server for one project.
+ *
+ *   deno run -A studio/server/serve.ts <setup-module> [--port 4983] [--page http://localhost:5174]
+ *     [--deno-config <deno.json>] [--editor <command>]
+ *
+ * The setup module is the project's: its default export registers the project's
+ * tools, profiles, and providers; an optional `host` export gives tool handlers
+ * their application context (the test user), and an optional `provider` export
+ * says where the providers find their keys. Only this machine can reach the server.
+ *
+ * The project loads in a process of its own (`project.ts`), and this passes the
+ * page's requests on to it, and starts it again when the page asks for a project
+ * whose files the builder's editor changed. Save is here: it writes the builder's
+ * edits into the project's files, checks them, and starts the project again from
+ * disk. It writes only files the project's setup imports, inside the folder the
+ * command ran in, and it never runs git. Open starts the builder's editor on a line of one of
+ * those files: the editor `--editor` names, else `$VISUAL` or `$EDITOR` when it
+ * opens in a window, else VS Code.
+ *
+ * @module
+ */
+import { dirname, relative, resolve } from 'node:path';
+import { stripVTControlCharacters } from 'node:util';
+import { newConversationKey } from "../conversation-key.js";
+import { RUNNING_PATH } from '../ports.mjs';
+import { READ_STEPS } from "../read-steps.js";
+import { createEditedSession } from "./edited-session.js";
+import { corsHeaders, isForeign, json, STUDIO_BASE } from "./handler.js";
+import { answerOpen, chosenEditor, defaultEditor } from "./open-editor.js";
+import { PROJECT_READY } from "./project.js";
+import { isInside } from "./project-source.js";
+import { answerSave, createSaveSession, isSaveRequest } from "./save-session.js";
+import { createWatchFeed, watchFiles } from "./watch.js";
+function flag(name, fallback) {
+    const at = Deno.args.indexOf(`--${name}`);
+    return at >= 0 ? (Deno.args[at + 1] ?? fallback) : fallback;
+}
+const setupPath = Deno.args.find((arg, i) => !arg.startsWith('--') && !Deno.args[i - 1]?.startsWith('--'));
+if (!setupPath) {
+    console.error('usage: studio/server/serve.ts <setup-module> [--port 4983] [--page http://localhost:5174] [--deno-config deno.json] [--editor code] [--no-watch]');
+    Deno.exit(2);
+}
+const port = Number(flag('port', '4983'));
+/** The site's dev server, under either name a browser gives this machine. */
+const pageOrigins = flag('page', 'http://localhost:5174,http://127.0.0.1:5174').split(',');
+const gate = { listenHost: `127.0.0.1:${port}`, pageOrigins };
+/** The folder Save may write in: where the command ran. */
+const root = Deno.realPathSync(Deno.cwd());
+const setupFile = Deno.realPathSync(resolve(setupPath));
+/** The project's Deno config, when the command names one: the project loads and type-checks under it. */
+const denoConfig = flag('deno-config', '');
+const configArgs = denoConfig ? ['--config', denoConfig] : [];
+/** A port nothing listens on. */
+function freePort() {
+    const listener = Deno.listen({ hostname: '127.0.0.1', port: 0 });
+    const { port: free } = listener.addr;
+    listener.close();
+    return free;
+}
+/** Reads a stream to its end, a decoded chunk at a time. */
+async function drain(stream, each) {
+    const decoder = new TextDecoder();
+    for await (const chunk of stream)
+        each(decoder.decode(chunk, { stream: true }));
+}
+/** Seals this run's chats. Every load gets it, so a chat outlives a Save; quitting ends it. */
+const conversationKey = newConversationKey();
+/**
+ * Loads the project from disk in a new process, with the builder's unsaved `edits` laid over it
+ * when there are some. Throws what it printed when it does not start.
+ */
+async function loadProject(edits) {
+    const childPort = freePort();
+    const process = new Deno.Command(Deno.execPath(), {
+        args: [
+            'run',
+            ...configArgs,
+            '-A',
+            // why: The installed package runs this file built to JavaScript, beside a built `project.js`.
+            import.meta.resolve(import.meta.url.endsWith('.js') ? './project.js' : './project.ts'),
+            setupFile,
+            '--port',
+            String(childPort),
+            '--page',
+            pageOrigins.join(','),
+        ],
+        cwd: root,
+        stdin: 'piped',
+        stdout: 'piped',
+        stderr: 'piped',
+    }).spawn();
+    const load = { conversationKey, ...(edits ? { edits } : {}) };
+    const input = process.stdin.getWriter();
+    await input.write(new TextEncoder().encode(JSON.stringify(load)));
+    await input.close();
+    let printed = '';
+    const errors = drain(process.stderr, (text) => {
+        printed = (printed + text).slice(-4000);
+        Deno.stderr.writeSync(new TextEncoder().encode(text));
+    });
+    let ready = () => { };
+    const listening = new Promise((settle) => {
+        let out = '';
+        ready = () => settle(true);
+        drain(process.stdout, (text) => {
+            out = (out + text).slice(-200);
+            if (out.includes(PROJECT_READY))
+                ready();
+        }).then(() => settle(false));
+    });
+    if (!(await listening)) {
+        await Promise.all([process.status, errors]);
+        // The page shows this, so the terminal's colours come out.
+        throw new Error(stripVTControlCharacters(printed).trim() || 'The project did not start.');
+    }
+    const origin = `http://127.0.0.1:${childPort}`;
+    const description = await (await fetch(origin + STUDIO_BASE)).json();
+    return { process, origin, description };
+}
+async function stop(loaded) {
+    try {
+        loaded.process.kill();
+    }
+    catch {
+        // It had already ended.
+    }
+    await loaded.process.status;
+}
+/** Whether the project type-checks as it is on disk, and what the checker printed. */
+async function typeChecks() {
+    const { success, stderr, stdout } = await new Deno.Command(Deno.execPath(), {
+        args: ['check', ...configArgs, setupFile],
+        cwd: root,
+        env: { NO_COLOR: '1' },
+        stdin: 'null',
+    }).output();
+    const decoder = new TextDecoder();
+    return {
+        ok: success,
+        output: (decoder.decode(stderr) + decoder.decode(stdout)).trim().slice(-4000),
+    };
+}
+/** A project file's text, or undefined when it is missing or leads outside the project. */
+function readInside(path) {
+    try {
+        return isInside(root, Deno.realPathSync(path)) ? Deno.readTextFileSync(path) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Whether a write to a path lands inside the project. A link where the file would go, or in a
+ * folder on the way to it, can lead anywhere on the machine; a project from someone else can hold one.
+ */
+function landsInside(path) {
+    try {
+        if (Deno.lstatSync(path).isSymlink)
+            return false;
+    }
+    catch {
+        // No file there yet.
+    }
+    for (let folder = dirname(path);; folder = dirname(folder)) {
+        try {
+            return isInside(root, Deno.realPathSync(folder));
+        }
+        catch {
+            // That folder is not made yet: the one above it decides.
+            if (folder === dirname(folder))
+                return false;
+        }
+    }
+}
+let first;
+try {
+    first = await loadProject();
+}
+catch (error) {
+    console.error(error instanceof Error ? error.message : error);
+    Deno.exit(1);
+}
+/** The load that runs the builder's unsaved edits, while the page compares it with the files. */
+const edited = createEditedSession({
+    load: loadProject,
+    stop,
+    opened: (loaded) => loaded.description.workspace,
+});
+const session = createSaveSession({
+    root,
+    setupFile,
+    read: readInside,
+    landsInside,
+    write: (path, text) => {
+        Deno.mkdirSync(dirname(path), { recursive: true });
+        Deno.writeTextFileSync(path, text);
+    },
+    remove: (path) => Deno.removeSync(path),
+    typeChecks,
+    // The files changed: the edited load lies over files that are gone.
+    load: async () => {
+        await edited.close();
+        return loadProject();
+    },
+    stop,
+    opened: (loaded) => loaded.description.workspace,
+}, first);
+/** Passes a request on to a load of the project, and its answer back as it streams. */
+async function forward(request, loaded = session.project(), path) {
+    const url = new URL(request.url);
+    if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') {
+        return forwardSocket(request, `${loaded.origin}${path ?? url.pathname}${url.search}`);
+    }
+    const headers = new Headers(request.headers);
+    headers.delete('host');
+    const body = request.method === 'GET' || request.method === 'HEAD' ? undefined : await request.arrayBuffer();
+    try {
+        return await fetch(`${loaded.origin}${path ?? url.pathname}${url.search}`, {
+            method: request.method,
+            headers,
+            body,
+        });
+    }
+    catch {
+        return json(502, {}, corsHeaders(request, pageOrigins));
+    }
+}
+/**
+ * Joins the page's socket to the same path of the project: a voice call. What the page sends
+ * before the project's side opens is held, because a call opens with the page's first message.
+ */
+function forwardSocket(request, to) {
+    const { socket: page, response } = Deno.upgradeWebSocket(request);
+    const project = new WebSocket(to.replace(/^http/, 'ws'));
+    page.binaryType = 'arraybuffer';
+    project.binaryType = 'arraybuffer';
+    const held = [];
+    const shut = (other) => () => {
+        if (other.readyState === WebSocket.OPEN || other.readyState === WebSocket.CONNECTING)
+            other.close();
+    };
+    page.addEventListener('message', (event) => {
+        if (project.readyState === WebSocket.OPEN)
+            project.send(event.data);
+        else
+            held.push(event.data);
+    });
+    project.addEventListener('open', () => {
+        for (const data of held.splice(0))
+            project.send(data);
+    });
+    project.addEventListener('message', (event) => {
+        if (page.readyState === WebSocket.OPEN)
+            page.send(event.data);
+    });
+    page.addEventListener('close', shut(project));
+    project.addEventListener('close', shut(page));
+    project.addEventListener('error', shut(page));
+    return response;
+}
+const SAVE = `${STUDIO_BASE}/save`;
+const WATCH = `${STUDIO_BASE}/watch`;
+const OPEN = `${STUDIO_BASE}/open`;
+const EDITED = `${STUDIO_BASE}/edited`;
+/**
+ * The edited load: a POST of the workspace at `EDITED` starts it, and every path under it is the
+ * same path of the project, answered by that load. Undefined when the request is not for it.
+ */
+async function answerEdited(request) {
+    const { pathname } = new URL(request.url);
+    const cors = corsHeaders(request, pageOrigins);
+    if (pathname === EDITED && request.method === 'POST') {
+        const body = await request.json().catch(() => null);
+        if (!isSaveRequest(body))
+            return json(400, {}, cors);
+        return json(200, await edited.open(body.workspace, session.project()), cors);
+    }
+    if (!pathname.startsWith(`${EDITED}/`))
+        return undefined;
+    const loaded = edited.running();
+    // No load holds the edits: the page asks for one again. A preflight is still answered, so the
+    // page reads that and not a blocked request.
+    if (!loaded)
+        return request.method === 'OPTIONS'
+            ? new Response(null, { status: 204, headers: cors })
+            : json(409, {}, cors);
+    return forward(request, loaded, STUDIO_BASE + pathname.slice(EDITED.length));
+}
+/** Starts a command on its own, without a shell. False when the machine has no such command. */
+function start(command, args) {
+    try {
+        new Deno.Command(command, { args, cwd: root, stdin: 'null', stdout: 'null', stderr: 'null' })
+            .spawn()
+            .unref();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+const editor = {
+    editor: chosenEditor(flag('editor', ''), [Deno.env.get('VISUAL'), Deno.env.get('EDITOR')]),
+    place: (file) => session.place(file),
+    start,
+    os: Deno.build.os,
+    home: Deno.env.get('HOME'),
+};
+/**
+ * The project as the page opens it: what it registered, the settings its files say its profiles
+ * share, and the ones they set in code. Files the builder's editor changed are loaded first.
+ */
+async function described(said) {
+    const files = await session.read(said);
+    return {
+        ...session.project().description,
+        ...files,
+        editor: defaultEditor(editor),
+        setup: relative(root, setupFile),
+    };
+}
+/** Whether the request asks for the project as the page opens it, and whether for the steps of reading it first. */
+function opens(request) {
+    const url = new URL(request.url);
+    if (request.method !== 'GET' || url.pathname !== STUDIO_BASE)
+        return undefined;
+    return { steps: url.searchParams.has(READ_STEPS) };
+}
+async function opened(request) {
+    return opens(request)?.steps === false ? await described() : undefined;
+}
+/**
+ * The project as the page opens it, after each step of reading it: a line of JSON for each
+ * step as it happens, then one holding the project.
+ */
+function openedSaying(request) {
+    if (!opens(request)?.steps)
+        return undefined;
+    const encoder = new TextEncoder();
+    const body = new ReadableStream({
+        start(controller) {
+            const line = (said) => controller.enqueue(encoder.encode(`${JSON.stringify(said)}\n`));
+            described(line).then((project) => {
+                line({ opened: project });
+                controller.close();
+            }, (error) => controller.error(error));
+        },
+    });
+    return new Response(body, {
+        headers: {
+            'content-type': 'application/x-ndjson',
+            'cache-control': 'no-store',
+            ...corsHeaders(request, pageOrigins),
+        },
+    });
+}
+/** Tells the page when the builder's editor writes a file the setup reads, unless `--no-watch` says not to. */
+const feed = Deno.args.includes('--no-watch') ? undefined : createWatchFeed();
+if (feed) {
+    watchFiles({
+        files: () => session.watched(),
+        changed: () => session.changed(),
+        tell: feed.tell,
+        watch: (folders) => Deno.watchFs(folders, { recursive: false }),
+    });
+}
+/** The page's subscription to file changes. No content when the studio was started without watching. */
+function watching(request) {
+    if (request.method !== 'GET' || new URL(request.url).pathname !== WATCH)
+        return undefined;
+    const cors = corsHeaders(request, pageOrigins);
+    return feed ? feed.subscribe(cors) : new Response(null, { status: 204, headers: cors });
+}
+/** The setup this studio is open on, for a second studio command to find it already running. */
+function running(request) {
+    if (request.method !== 'GET' || new URL(request.url).pathname !== RUNNING_PATH)
+        return undefined;
+    return json(200, { setup: setupFile }, corsHeaders(request, pageOrigins));
+}
+/** What the studio answers itself: the project as it opens, a save, or the builder's editor on a line. */
+async function answered(request) {
+    const answer = (await opened(request)) ??
+        (await answerSave(session, SAVE, request)) ??
+        (await answerOpen(editor, OPEN, request));
+    if (!answer)
+        return undefined;
+    const cors = corsHeaders(request, pageOrigins);
+    return 'workspace' in answer ? json(200, answer, cors) : json(answer.status, answer.body, cors);
+}
+Deno.serve({ hostname: '127.0.0.1', port }, async (request) => {
+    if (isForeign(request, gate))
+        return json(403, {}, {});
+    return (running(request) ??
+        watching(request) ??
+        openedSaying(request) ??
+        (await answered(request)) ??
+        (await answerEdited(request)) ??
+        forward(request));
+});
+for (const signal of ['SIGINT', 'SIGTERM']) {
+    Deno.addSignalListener(signal, () => {
+        for (const loaded of [session.project(), edited.running()]) {
+            try {
+                loaded?.process.kill();
+            }
+            catch {
+                // It had already ended.
+            }
+        }
+        Deno.exit(0);
+    });
+}
+console.log(`Theorem Studio: open ${pageOrigins[0]}/studio (server on http://127.0.0.1:${port})`);

package/server/test-connection.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Asks a draft HTTP or MCP tool's endpoint one request, through the kernel's network guard, so the
+ * builder sees whether it answers before a model calls it.
+ *
+ * @module
+ */
+import { type ResolveHost } from '@theoremjs/agents';
+/**
+ * `POST <base>/test-connection`: one request to a draft HTTP or MCP tool's endpoint. It reaches
+ * public hosts only, unless `draftNetwork` is set: then it keeps to the network guardrail the
+ * draft sends (`network`), as the draft's runs do. A name is checked against private addresses
+ * only through `resolveHost`; without it, only the address as written is.
+ */
+export declare function testConnection(request: Request, options?: {
+    resolveHost?: ResolveHost;
+    draftNetwork?: boolean;
+}): Promise<Response>;