@takodotid/circuit 0.43.1 → 1.0.0-rc.2

package/src/studio/server.ts

@@ -1,48 +1,41 @@
-// `circuit studio`: the network in the browser, on this machine or on a server for a team. The page is the files in public/, served as they are. What it shows comes from /data: gathered by `circuit studio --json` in a new process, so the config is always read fresh, and gathered again each time a file beside the config changes.
+// `circuit studio`: the project in the browser, on this machine, or on a server for a team. The page is built by Vite from `app/` and carried inside Circuit (`assets.generated.ts`). What it shows comes from /api/data: gathered by Circuit in a new process, so the project is always read fresh, and gathered again each time a file of the project changes.
 
 import { execFile, spawn } from "node:child_process";
-import { existsSync, readFileSync, watch } from "node:fs";
-import { createServer, type ServerResponse } from "node:http";
-import { extname, join } from "node:path";
-import { StudioModel, type StudioData } from "./data";
+import { watch } from "node:fs";
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+import { SecretsFile } from "../core/secrets-file";
+import { Secrets } from "../core/secrets";
+import { Platforms } from "../platforms";
+import { Releases } from "../cli/releases";
+import { STUDIO_ASSETS } from "./assets.generated";
 import { Git } from "./git";
+import type { LiveRead, SecretEntry, SecretsFileListing, SecretsListing, StudioData } from "./types";
 
 export type StudioOptions = {
-    /** The config file. */
-    configPath: string;
-    /** The directory watched for changes: the config's own. */
+    /** The project's folder. */
     root: string;
     port: number;
-    /** Open the browser once the server listens. */
-    open: boolean;
     /** The address to listen on: 127.0.0.1 on a machine of one's own, or a server's address. */
     host: string;
     /** Names besides this machine's own that requests may be addressed to, such as `studio.example.com` behind a proxy. */
     allowHosts: string[];
+    /** How the project is read: by default, Circuit itself in a new process. Tests give their own. */
+    gather?: (folder: string) => Promise<string>;
 };
 
 export class StudioServer {
-    /** Directories whose changes never change what Studio shows. */
-    private static readonly IGNORED = /(^|[\\/])(node_modules|\.git|\.circuit[\\/]cache)([\\/]|$)/;
+    /** Folders whose changes never change what Studio shows. */
+    private static readonly IGNORED = /(^|[\\/])(node_modules|\.git|\.circuit)([\\/]|$)/;
 
-    private static readonly CONTENT_TYPES: Record<string, string> = {
-        ".html": "text/html; charset=utf-8",
-        ".css": "text/css; charset=utf-8",
-        ".js": "text/javascript; charset=utf-8",
-        ".svg": "image/svg+xml",
-        ".woff2": "font/woff2",
-    };
+    /** The header the page sends with a request that reaches a device, which a page on another site cannot send without asking first. */
+    static readonly REQUEST_HEADER = "x-circuit-studio";
 
     /** Whether an address to listen on is this machine's own, which nothing else on the network can reach. */
     static isLoopback(host: string): boolean {
         return host === "localhost" || host === "::1" || /^127\.\d+\.\d+\.\d+$/.test(host);
     }
 
-    /**
-     * Whether a request's Host header names Studio, so a web page elsewhere cannot read the network through a name it points here.
-     *
-     * Studio answers to this machine's own names at its port, and to each name it was told to, with or without a port.
-     */
+    /** Whether a request's Host header names Studio, so a web page elsewhere cannot read the project through a name it points here. */
     static hostAllowed(header: string | undefined, port: number, allowHosts: readonly string[]): boolean {
         if (!header) return false;
         if (header === `127.0.0.1:${port}` || header === `localhost:${port}` || header === `[::1]:${port}`) return true;
@@ -54,23 +47,32 @@ export class StudioServer {
         });
     }
 
-    /** What Studio shows, as JSON, from `circuit studio --json` run as a new process, or what stopped it. */
-    private static gather(configPath: string): Promise<string> {
-        const cli = process.argv[1]!;
-        const args = [...process.execArgv, cli, "studio", "--json", `--config=${configPath}`];
+    /** Circuit itself, to run again: the binary, or Bun with Circuit's sources. */
+    private static self(): { command: string; args: string[] } {
+        return Releases.binary()
+            ? { command: process.execPath, args: [] }
+            : { command: process.execPath, args: [...process.execArgv, process.argv[1]!] };
+    }
 
-        return new Promise((done) => {
-            execFile(process.execPath, args, { maxBuffer: 256 * 1024 * 1024 }, (error, stdout, stderr) => {
-                if (!error) return done(stdout);
+    /** What Studio shows, as JSON, from `circuit studio-data` run as a new process, or what stopped it. A new process reads every file again: one process imports a file once. */
+    static gather(folder: string): Promise<string> {
+        const { command, args } = StudioServer.self();
 
-                const message = (stderr || error.message).trim();
-                done(JSON.stringify({ error: message }));
-            });
+        return new Promise((done) => {
+            execFile(
+                command,
+                [...args, "studio-data", folder],
+                { maxBuffer: 256 * 1024 * 1024, env: { ...process.env, CIRCUIT_LOCAL: "1" } },
+                (error, stdout, stderr) => {
+                    if (!error) return done(stdout);
+                    done(JSON.stringify({ error: (stderr || error.message).trim() }));
+                }
+            );
         });
     }
 
     /** Open the address in the browser. A machine without one only misses the convenience. */
-    private static openBrowser(url: string): void {
+    static openBrowser(url: string): void {
         const [command, args] =
             process.platform === "win32"
                 ? ["cmd", ["/c", "start", "", url]]
@@ -83,20 +85,97 @@ export class StudioServer {
         child.unref();
     }
 
-    /** Serve Studio until the process stops, and gather what it shows again whenever a file beside the config changes. */
-    static async start(options: StudioOptions): Promise<void> {
-        const { configPath, root, port } = options;
+    /** Each secrets file the networks use, as the page may see it: names, how each is kept, ciphertexts. Never a value in plain text, and never the password. */
+    static secretsListing(root: string, data: StudioData): SecretsListing {
+        const files = [...new Set(data.networks.map((network) => network.secretsFile))];
+        return {
+            files: files.map((file) =>
+                StudioServer.fileListing(
+                    root,
+                    file,
+                    data.networks.filter((network) => network.secretsFile === file)
+                )
+            ),
+        };
+    }
 
-        // The page's files ship in the package's sources, which Bun runs as they are and Node's build leaves in place.
-        const installed = StudioModel.packageRoot();
-        if (!installed) throw new Error("cannot find where Circuit is installed, so Studio has no page to serve");
-        const pages = join(installed, "src", "studio", "public");
+    private static fileListing(root: string, name: string, networks: StudioData["networks"]): SecretsFileListing {
+        const file = SecretsFile.at(root, name);
+        const values = file.values();
+        const needed = [...new Set(networks.flatMap((network) => network.secrets))];
+        const names = [...new Set([...file.names(), ...needed])].sort();
+
+        const entries = names.map((entry): SecretEntry => {
+            const value = values[entry];
+            const isNeeded = needed.includes(entry);
+            if (value === undefined) return { name: entry, kind: "missing", needed: isNeeded };
+
+            const kind = SecretsFile.kindOf(value);
+            if (kind === "encrypted") return { name: entry, kind, ciphertext: value, needed: isNeeded };
+            if (kind === "1password") {
+                const [vault = "", item = "", ...field] = value.slice("op://".length).split("/");
+                return { name: entry, kind, onePassword: { vault, item, field: field.join("/") }, needed: isNeeded };
+            }
 
-        let data = await StudioServer.gather(configPath);
+            return { name: entry, kind, needed: isNeeded };
+        });
 
-        // Another commit's network is read once: a commit does not change. Each is kept as it is being read, so two asks share one read.
+        const passwordLocation = file.passwordLocation();
+        return {
+            file: name,
+            networks: networks.map((network) => network.name),
+            ...(passwordLocation ? { passwordLocation } : {}),
+            entries,
+        };
+    }
+
+    /** Read a device now and plan against what it runs, as `circuit diff` does. Nothing is kept. */
+    static async readDevice(root: string, data: StudioData, name: string): Promise<LiveRead> {
+        const device = data.devices.find((candidate) => candidate.name === name);
+        if (!device) throw new Error(`no device ${name}`);
+
+        const adapter = Platforms.find(device.platform);
+        if (!adapter) throw new Error(`no platform ${device.platform}`);
+
+        Secrets.useRoot(root);
+        Secrets.useFile(data.networks.find((network) => network.devices.includes(name))?.secretsFile);
+        const locked = Secrets.lockedFor(device);
+        if (locked)
+            throw new Error(
+                `${locked.name} is encrypted, and Studio has no password for it. Studio reads a device when the password comes from 1Password or ${SecretsFile.PASSWORD}; or run circuit diff ${name}, which asks for it.`
+            );
+        const options = { secrets: false, rollback: 10, ...(device.connection.host ? {} : { console: true }) };
+        const running = await adapter.read(device, data.devices, options);
+        const rendered = adapter.render(device);
+        const plan = adapter.plan(rendered, running, options);
+
+        return {
+            device: name,
+            running,
+            rendered,
+            steps: plan.steps.map((step) => ({ title: step.title, show: step.show })),
+            problems: plan.problems,
+            readAt: new Date().toISOString(),
+        };
+    }
+
+    /** Serve Studio until the process stops, and gather what it shows again whenever a file of the project changes. Resolves with its address once it listens. */
+    static async start(options: StudioOptions): Promise<string> {
+        const { root, port } = options;
+        const gather = options.gather ?? StudioServer.gather;
+        const local = StudioServer.isLoopback(options.host);
+
+        let data = await gather(root);
+
+        // Another commit's project is read once: a commit does not change. Each is kept as it is being read, so two asks share one read.
         const past = new Map<string, Promise<string>>();
 
+        const parsed = (): StudioData => {
+            const value = JSON.parse(data) as StudioData | { error: string };
+            if ("error" in value) throw new Error("the project has errors, so Studio cannot use it");
+            return value;
+        };
+
         /** Answer with JSON, or with an error the page shows. */
         const answer = async (response: ServerResponse, work: () => Promise<unknown>) => {
             let body: string;
@@ -108,13 +187,14 @@ export class StudioServer {
             }
             response.writeHead(200, { "content-type": "application/json", "cache-control": "no-store" }).end(body);
         };
+
         const listeners = new Set<ServerResponse>();
         const tell = (message: string) => {
             for (const listener of listeners) listener.write(`data: ${message}\n\n`);
         };
 
-        const server = createServer((request, response) => {
-            // Studio shows one team's network: every answer asks search engines to leave it out, and robots.txt says so for the whole site.
+        const handle = (request: IncomingMessage, response: ServerResponse) => {
+            // Studio shows one team's network: every answer asks search engines to leave it out.
             response.setHeader("x-robots-tag", "noindex, nofollow, noarchive");
 
             if (!StudioServer.hostAllowed(request.headers.host, port, options.allowHosts)) {
@@ -125,109 +205,118 @@ export class StudioServer {
             const url = new URL(request.url ?? "/", "http://studio");
             const path = url.pathname;
 
-            // The commits that changed a device: its file and its snapshot.
-            if (path === "/history") {
+            if (path === "/api/data") {
+                response.writeHead(200, { "content-type": "application/json", "cache-control": "no-store" }).end(data);
+                return;
+            }
+
+            // Changes are told as they happen; a proxy such as nginx is asked not to hold them back.
+            if (path === "/api/events") {
+                response.writeHead(200, {
+                    "content-type": "text/event-stream",
+                    "cache-control": "no-store",
+                    connection: "keep-alive",
+                    "x-accel-buffering": "no",
+                });
+                response.write(": connected\n\n");
+                listeners.add(response);
+                request.on("close", () => listeners.delete(response));
+                return;
+            }
+
+            if (path === "/api/secrets") {
+                void answer(response, async () => StudioServer.secretsListing(root, parsed()));
+                return;
+            }
+
+            // The commits that changed a device's file.
+            if (path === "/api/history") {
                 void answer(response, async () => {
                     const name = url.searchParams.get("device") ?? "";
-                    const files = (JSON.parse(data) as StudioData).files?.[name];
-                    if (!files) throw new Error(`no device ${name}`);
-                    return Git.history(
-                        root,
-                        [files.source, files.snapshot].filter((file): file is string => Boolean(file))
-                    );
+                    const file = parsed().files[name];
+                    if (!file) throw new Error(`no device ${name}`);
+                    return Git.history(root, [file]);
                 });
                 return;
             }
 
             // What Studio can compare with: branches, tags and the latest commits.
-            if (path === "/refs") {
+            if (path === "/api/refs") {
                 void answer(response, () => Git.references(root));
                 return;
             }
 
-            // The network as it was at a branch, tag or commit.
-            if (path === "/compare") {
+            // The project as it was at a branch, tag or commit.
+            if (path === "/api/compare") {
                 void answer(response, async () => {
                     const asked = url.searchParams.get("ref") ?? "";
                     const commit = await Git.commitOf(root, asked);
                     if (!commit) throw new Error(`${asked} names no commit here`);
 
-                    if (!past.has(commit.commit))
-                        past.set(commit.commit, Git.networkAt(root, configPath, commit.commit, StudioServer.gather));
-                    const network = await past.get(commit.commit)!;
-                    if (network.startsWith('{"error"')) past.delete(commit.commit);
+                    if (!past.has(commit.commit)) past.set(commit.commit, Git.projectAt(root, commit.commit, gather));
+                    const project = await past.get(commit.commit)!;
+                    if (project.startsWith('{"error"')) past.delete(commit.commit);
 
-                    return `{"commit":${JSON.stringify(commit)},"network":${network}}`;
+                    return `{"commit":${JSON.stringify(commit)},"network":${project}}`;
                 });
                 return;
             }
 
-            if (path === "/robots.txt") {
-                response.writeHead(200, { "content-type": "text/plain; charset=utf-8" }).end("User-agent: *\nDisallow: /\n");
-                return;
-            }
-
-            if (path === "/data") {
-                response.writeHead(200, { "content-type": "application/json", "cache-control": "no-store" }).end(data);
+            // Reading a device reaches the device, so it takes a POST with Studio's own header, which a page on another site cannot send unasked.
+            if (path === "/api/read") {
+                if (request.method !== "POST" || request.headers[StudioServer.REQUEST_HEADER] !== "1") {
+                    response.writeHead(403).end();
+                    return;
+                }
+                void answer(response, async () => {
+                    if (!local)
+                        throw new Error("Studio on a server does not read devices. Run circuit diff, or Studio on your own machine.");
+                    return StudioServer.readDevice(root, parsed(), url.searchParams.get("device") ?? "");
+                });
                 return;
             }
 
-            // Changes are told as they happen; a proxy such as nginx is asked not to hold them back.
-            if (path === "/events") {
-                response.writeHead(200, {
-                    "content-type": "text/event-stream",
-                    "cache-control": "no-store",
-                    connection: "keep-alive",
-                    "x-accel-buffering": "no",
-                });
-                response.write(": connected\n\n");
-
-                listeners.add(response);
-                request.on("close", () => listeners.delete(response));
+            if (path === "/robots.txt") {
+                response.writeHead(200, { "content-type": "text/plain; charset=utf-8" }).end("User-agent: *\nDisallow: /\n");
                 return;
             }
 
-            // Only a file directly in public/, by a plain name, is served: nothing above it can be reached.
-            const name = path === "/" ? "index.html" : path.slice(1);
-            const type = StudioServer.CONTENT_TYPES[extname(name)];
-            const file = join(pages, name);
-
-            if (!/^[a-z-]+\.[a-z0-9]+$/.test(name) || !type || !existsSync(file)) {
+            const asset = STUDIO_ASSETS[path === "/" ? "index.html" : path.slice(1)];
+            if (!asset) {
                 response.writeHead(404).end();
                 return;
             }
 
-            response.writeHead(200, { "content-type": type, "cache-control": "no-store" }).end(readFileSync(file));
-        });
+            const cache = path.startsWith("/assets/") ? "public, max-age=31536000, immutable" : "no-store";
+            response.writeHead(200, { "content-type": asset.type, "cache-control": cache }).end(Buffer.from(asset.body, "base64"));
+        };
 
-        // Several files often change at once, such as on a save or a checkout: the config is read again once they settle.
-        let timer: NodeJS.Timeout | undefined;
+        const server = createServer(handle);
 
-        watch(root, { recursive: true }, (_event, filename) => {
+        // Several files often change at once, such as on a save or a checkout: the project is read again once they settle.
+        let timer: NodeJS.Timeout | undefined;
+        const watcher = watch(root, { recursive: true }, (_event, filename) => {
             if (filename && StudioServer.IGNORED.test(filename)) return;
 
             clearTimeout(timer);
             timer = setTimeout(async () => {
-                data = await StudioServer.gather(configPath);
+                data = await gather(root);
                 tell("changed");
-                console.log(`${new Date().toLocaleTimeString()}  ${filename ?? "a file"} changed, read again`);
             }, 300);
         });
+        server.on("close", () => watcher.close());
 
-        // A change to the page's own files, while working on Circuit, loads the page again.
-        watch(pages, () => tell("page"));
-
-        server.on("error", (error: NodeJS.ErrnoException) => {
-            const message = error.code === "EADDRINUSE" ? `port ${port} is in use; choose another with --port=N` : error.message;
-            console.error(message);
-            process.exit(1);
+        await new Promise<void>((done, fail) => {
+            server.once("error", (error: NodeJS.ErrnoException) =>
+                fail(new Error(error.code === "EADDRINUSE" ? `port ${port} is in use; choose another with --port` : error.message))
+            );
+            server.listen(port, options.host, () => done());
         });
 
-        server.listen(port, options.host, () => {
-            const local = StudioServer.isLoopback(options.host);
-            const url = local ? `http://127.0.0.1:${port}` : `http://${options.allowHosts[0] ?? options.host}`;
-            console.log(`Circuit Studio at ${url}, listening on ${options.host}:${port}, watching ${root}. Stop it with Ctrl+C.`);
-            if (options.open && local) StudioServer.openBrowser(url);
-        });
+        StudioServer.servers.add(server);
+        return local ? `http://127.0.0.1:${port}` : `http://${options.allowHosts[0] ?? options.host}:${port}`;
     }
+
+    /** Every server started, so a test can stop them. */
+    static readonly servers = new Set<ReturnType<typeof createServer>>();
 }

package/src/studio/types.ts

@@ -0,0 +1,141 @@
+// What Studio's server sends its page. Types only, so the page shares them without taking any of Circuit's code.
+
+import type { Graph } from "../core/graph";
+import type { Device } from "../model";
+import type { Finding } from "../platforms/finding";
+
+/** One network of the project, without its devices. */
+export type StudioNetwork = {
+    name: string;
+    asn?: number;
+    /** By name. */
+    sites: Record<string, { description?: string; number?: number }>;
+    devices: string[];
+    /** Where its secrets are, relative to the project. */
+    secretsFile: string;
+    /** The secrets its devices and integrations name with `secret("NAME")`. */
+    secrets: string[];
+};
+
+/** One address a device holds. */
+export type AddressRow = {
+    device: string;
+    /** The port or interface. */
+    holder: string;
+    /** With its length. */
+    address: string;
+    /** The prefix it is in. */
+    prefix: string;
+    family: "ipv4" | "ipv6";
+    /** The far end of a point-to-point address. */
+    peer?: string;
+    vrf?: string;
+};
+
+/** One BGP session, with what it takes from its neighbor group, and what is at its far end. */
+export type Session = {
+    device: string;
+    name: string;
+    address: string;
+    group?: string;
+    remoteAs?: number;
+    role?: string;
+    import?: string;
+    export?: string;
+    description?: string;
+    /** One of our devices, by name, or another network, as `AS64501`. */
+    far?: string;
+    /** The far device has no session back. */
+    oneSided?: true;
+};
+
+/** One cable, from the end that names it first. */
+export type Cable = {
+    device: string;
+    port: string;
+    farDevice: string;
+    farPort: string;
+    /** The LAG each end's port is in. */
+    lag?: string;
+    farLag?: string;
+    description?: string;
+    /** What is between the two ends, such as a carrier's local loop. */
+    through?: string;
+    /** VLANs carried, as `name (id)`, from the port or its LAG. */
+    vlans: string[];
+    /** The far end does not name this cable back. */
+    oneSided?: true;
+};
+
+/** Everything Studio shows of the project, read fresh each time a file changes. */
+export type StudioData = {
+    /** The project's folder name. */
+    project: string;
+    networks: StudioNetwork[];
+    devices: Device[];
+    /** Each device's file, from the project's folder. */
+    files: Record<string, string>;
+    /** Every port each device's model has, in order, so Studio can show the ones the project leaves shut. */
+    ports: Record<string, string[]>;
+    /** What each device would run, as `circuit apply` would send it whole, or why it cannot be rendered. */
+    rendered: Record<string, string>;
+    /** What `circuit validate` finds, and what reading the project warns of. */
+    findings: Finding[];
+    graph: Graph;
+    addresses: AddressRow[];
+    sessions: Session[];
+    cables: Cable[];
+    /** Every secret the project names with `secret("NAME")`, by name. */
+    secrets: string[];
+};
+
+/** What reading the project stopped on, instead of the data. */
+export type StudioError = { error: string };
+
+/** One commit, as Studio lists it. */
+export type Commit = {
+    commit: string;
+    short: string;
+    author: string;
+    /** ISO 8601, with the author's time zone. */
+    date: string;
+    subject: string;
+};
+
+/** How one name in a secrets file is kept. The value is sent only when it is encrypted: a value in plain text never leaves the server. */
+export type SecretEntry = {
+    name: string;
+    kind: "encrypted" | "1password" | "empty" | "not encrypted" | "missing";
+    /** The `encrypted:...` value, for the page to decrypt with the password a person types. */
+    ciphertext?: string;
+    /** Where 1Password keeps it. */
+    onePassword?: { vault: string; item: string; field: string };
+    /** The config names it with `secret("NAME")`. */
+    needed: boolean;
+};
+
+/** One secrets file as the page may see it: names, how each is kept, ciphertexts. Never a value in plain text, and never the password. */
+export type SecretsFileListing = {
+    file: string;
+    /** The networks that keep their secrets here. */
+    networks: string[];
+    /** Where the password comes from, without its value. Absent when a person types it. */
+    passwordLocation?: string;
+    entries: SecretEntry[];
+};
+
+export type SecretsListing = { files: SecretsFileListing[] };
+
+/** What `Read from the device now` found: the device's own config, and the plan from it to the project's. */
+export type LiveRead = {
+    device: string;
+    /** What the device runs now, secrets removed. */
+    running: string;
+    /** What the project says it should run. */
+    rendered: string;
+    /** The plan `circuit diff` would show. */
+    steps: { title: string; show: string[] }[];
+    problems: string[];
+    /** When it was read, ISO 8601. */
+    readAt: string;
+};

package/src/transport/console.ts

@@ -5,7 +5,7 @@ import { EventEmitter } from "node:events";
 import { Duplex } from "node:stream";
 import { Client, type ClientChannel } from "ssh2";
 import { Secrets } from "../core/secrets";
-import type { Console, Device, Jump } from "../schema";
+import type { Console, Device, Jump } from "../model";
 import { Ssh } from "./ssh";
 
 /** The relay, in Python's standard library only: the serial port in raw mode, locked against a second session, joined to standard input and output. */
@@ -440,7 +440,7 @@ export class SerialConsole {
 
 /** A CLI on a console or in an SSH shell that runs one command at a time and returns what it printed, as an SSH exec would: no echo, no prompt. */
 export class ConsoleShell {
-    /** The quiet to wait for after the prompt when output can hold a line that looks like it, as `circuit show` prints whatever the device has. */
+    /** The quiet to wait for after the prompt when output can hold a line that looks like it, as `circuit exec` prints whatever the device has. */
     static readonly SETTLE = 1_000;
 
     constructor(
@@ -476,7 +476,7 @@ export class ConsoleShell {
     }
 }
 
-/** One line for `circuit console`: what to send, the quiet that ends its answer, and the text the last answer must end with for the line to be sent. */
+/** One line for `circuit access-console`: what to send, the quiet that ends its answer, and the text the last answer must end with for the line to be sent. */
 export type ScriptLine = { text: string; quiet: number; onlyAfter?: string };
 
 /**

package/src/transport/ssh.ts

@@ -1,7 +1,7 @@
 // SSH sessions to devices. Adapters choose how to talk to their platform; this holds what they share.
 
 import { Client, type ClientChannel } from "ssh2";
-import type { Device } from "../schema";
+import type { Device } from "../model";
 
 export type Login = {
     host: string;