@takodotid/circuit 0.18.5 → 0.20.0

package/src/core/schema-docs.ts

@@ -0,0 +1,166 @@
+// The schema's own comments, read as data: each type, what it is, and each field with its type and description. The schema reference page and Studio both show them.
+
+/** The schema files in reading order, with the title each gets. */
+export const SCHEMA_FILES = [
+    ["device.ts", "Device"],
+    ["interfaces.ts", "Ports and interfaces"],
+    ["routing.ts", "Routing"],
+    ["policy.ts", "Route policies"],
+    ["firewall.ts", "Firewall and ACLs"],
+    ["system.ts", "System and management"],
+    ["common.ts", "Common values"],
+] as const;
+
+export type Field = { path: string; type: string; description: string };
+export type TypeDoc = { name: string; description: string; definition?: string; includes?: string[]; fields: Field[] };
+
+/** The text of a JSDoc comment, without its markers, as one paragraph. */
+function commentText(lines: string[]): string {
+    return lines
+        .map((line) => line.replace(/^\s*\/\*\*\s?|\s*\*\/\s*$|^\s*\*\s?/g, "").trim())
+        .filter(Boolean)
+        .join(" ");
+}
+
+/** What a type parameter stands for, where it is not one of the device's names. */
+const PARAMETER_MEANING: Record<string, string> = { Pl: "platform", M: "model" };
+
+/** A type as a reader needs it: "name" for each of the device's names, and no parameter list on the schema's own types. */
+function withoutGenerics(type: string, generics: Set<string>): string {
+    let result = type;
+    while (/NoInfer<([^<>]*)>/.test(result)) {
+        result = result.replace(/NoInfer<([^<>]*)>/g, (_, inner: string) => (inner.includes("|") ? `(${inner})` : inner));
+    }
+
+    for (const name of generics) {
+        result = result.replace(new RegExp(`\\b${name}<[^<>]*(?:<[^<>]*>[^<>]*)*>`, "g"), name);
+    }
+
+    return result
+        .replace(/\b(Pl|M|N|V|I|P|S|G|A|AS|U|F|C|L|K)\b/g, (parameter) => PARAMETER_MEANING[parameter] ?? "name")
+        .replace(/\bname(?: \| name)+\b/g, "name")
+        .replace(/\(name\)/g, "name");
+}
+
+function parse(source: string, generics: Set<string>): TypeDoc[] {
+    const lines = source.split("\n");
+    const types: TypeDoc[] = [];
+
+    let comment: string[] = [];
+    let inComment = false;
+    let current: TypeDoc | undefined;
+    let depth = 0;
+    const parents: string[] = [];
+    let header: string | undefined;
+
+    /** Reads a whole declaration: a type it aliases, the types whose fields it also has, and whether a body opens. */
+    function finishHeader(): void {
+        const text = header!;
+        header = undefined;
+
+        // The part after `=`, with the generic parameter list before it left out.
+        const body = text.slice(text.indexOf("> =") >= 0 ? text.indexOf("> =") + 3 : text.indexOf("=") + 1).trim();
+
+        if (!body.includes("{")) {
+            current!.definition = body.replace(/;$/, "").replace(/^\|\s*/, "");
+        } else {
+            const before = body.slice(0, body.indexOf("{"));
+            const included = before
+                .split("&")
+                .map((part) => part.trim())
+                .filter((part) => /^\w/.test(part));
+            if (included.length) current!.includes = included;
+        }
+
+        depth = (text.match(/{/g) ?? []).length - (text.match(/}/g) ?? []).length;
+    }
+
+    for (const line of lines) {
+        const trimmed = line.trim();
+
+        if (trimmed.startsWith("/**")) {
+            comment = [trimmed];
+            inComment = !trimmed.endsWith("*/");
+            continue;
+        }
+
+        if (inComment) {
+            comment.push(trimmed);
+            if (trimmed.endsWith("*/")) inComment = false;
+            continue;
+        }
+
+        // A declaration can span lines before its body opens or it ends.
+        if (header !== undefined) {
+            header += ` ${trimmed}`;
+            if (trimmed.endsWith("{") || trimmed.endsWith(";")) finishHeader();
+            continue;
+        }
+
+        // A type the schema does not export is documented too, since exported types take its fields.
+        const declaration = /^(?:export )?type (\w+)(<)?/.exec(trimmed);
+        if (declaration && depth === 0) {
+            if (declaration[2]) generics.add(declaration[1]!);
+
+            current = { name: declaration[1]!, description: commentText(comment), fields: [] };
+            types.push(current);
+            comment = [];
+
+            header = trimmed;
+            if (trimmed.endsWith("{") || trimmed.endsWith(";")) finishHeader();
+            continue;
+        }
+
+        if (!current) continue;
+
+        const field = /^(?:readonly\s+)?("?[\w-]+"?)(\?)?:\s*(.*)$/.exec(trimmed);
+        if (field && depth >= 1) {
+            const [, name, optional, rest] = field;
+            const path = [...parents, name!.replace(/"/g, "")].join(".");
+            const opensObject = rest!.trim().endsWith("{");
+            const type = opensObject ? "object" : rest!.replace(/;$/, "").trim();
+
+            current.fields.push({ path: `${path}${optional ? "?" : ""}`, type, description: commentText(comment) });
+            comment = [];
+
+            if (opensObject) {
+                parents.push(name!.replace(/"/g, ""));
+                depth++;
+            }
+            continue;
+        }
+
+        const opens = (trimmed.match(/{/g) ?? []).length;
+        const closes = (trimmed.match(/}/g) ?? []).length;
+
+        for (let index = 0; index < closes - opens; index++) {
+            if (parents.length >= depth - 1 && parents.length > 0) parents.pop();
+        }
+
+        depth += opens - closes;
+        if (depth <= 0) {
+            depth = 0;
+            parents.length = 0;
+        }
+
+        comment = [];
+    }
+
+    return types;
+}
+
+/** Every type in each schema file, in the order given. Every file is read first, so a type is known to take parameters wherever it is used. */
+export function schemaTypes(sources: readonly string[]): TypeDoc[][] {
+    const generics = new Set<string>();
+    const parsed = sources.map((source) => parse(source, generics));
+
+    for (const types of parsed) {
+        for (const type of types) {
+            if (type.definition) type.definition = withoutGenerics(type.definition, generics);
+            if (type.includes) type.includes = type.includes.map((name) => withoutGenerics(name, generics));
+            for (const field of type.fields) field.type = withoutGenerics(field.type, generics);
+        }
+    }
+
+    return parsed;
+}

package/src/core/validate.ts

@@ -4,6 +4,7 @@ import { X509Certificate } from "node:crypto";
 import { adapters } from "../adapters/devices";
 import { portsOf } from "../adapters/devices/catalog";
 import type { Device } from "../schema";
+import { checkAcross } from "./across";
 import { contains, familyOf, isAddress, isCidr } from "./addr";
 import type { Finding, Network } from "./define";
 
@@ -28,7 +29,7 @@ export function validate(network: Network): Finding[] {
         else for (const field of adapter.unsupported(device)) report(`${device.platform} cannot express ${field}`);
     }
 
-    findings.push(...checkLinks(network.devices));
+    findings.push(...checkAcross(network.devices));
     for (const check of network.checks ?? []) findings.push(...check(network.devices));
     return findings;
 }
@@ -327,54 +328,3 @@ function checkCertificates(device: Device, names: Names, report: Report): void {
         }
     }
 }
-
-/** The VLAN ids a port carries, by how it carries them, so two ends can be compared across devices. */
-function carried(device: Device, port: NonNullable<NonNullable<Device["ports"]>[string]>) {
-    const id = (vlan: string | undefined) => (vlan ? device.vlans?.[vlan]?.id : undefined);
-    return {
-        access: id(port.access_vlan),
-        native: id(port.native_vlan),
-        trunk: (port.trunk_vlans ?? []).map(id).sort((a, b) => (a ?? 0) - (b ?? 0)),
-    };
-}
-
-/** A link names a device and port that exist, the far end links back, and both ends carry the same VLANs the same way. */
-function checkLinks(devices: readonly Device[]): Finding[] {
-    const findings: Finding[] = [];
-    const byName = new Map(devices.map((device) => [device.name, device]));
-
-    for (const device of devices) {
-        for (const [portName, port] of Object.entries(device.ports ?? {})) {
-            const link = port?.link;
-            if (!port || !link) continue;
-
-            const report = (message: string) =>
-                findings.push({
-                    level: "error",
-                    device: device.name,
-                    message: `port ${portName} link: ${message}`,
-                });
-
-            const far = byName.get(link.device);
-            if (!far) {
-                report(`device ${link.device} is not in the network`);
-                continue;
-            }
-
-            const farPort = far.ports?.[link.port];
-            if (!farPort) {
-                report(`${link.device} does not declare port ${link.port}`);
-                continue;
-            }
-
-            const linksBack = farPort.link?.device === device.name && farPort.link.port === portName;
-            if (!linksBack) report(`${link.device} ${link.port} does not link back to it`);
-
-            const here = JSON.stringify(carried(device, port));
-            const there = JSON.stringify(carried(far, farPort));
-            if (here !== there) report(`carries ${here}, and ${link.device} ${link.port} carries ${there}`);
-        }
-    }
-
-    return findings;
-}

package/src/studio/data.ts

@@ -0,0 +1,255 @@
+// What Studio shows, gathered from the config in one pass: the graph with its findings and drift, each device as the config holds it, what each would run, the address plan, every BGP session and cable, and the schema's descriptions of each field.
+
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { portsOf } from "../adapters/devices/catalog";
+import { familyOf, host } from "../core/addr";
+import { type AddressReport, addressesOf } from "../core/addresses";
+import { type Annotated, type Holder, holdersOf, keyOf } from "../core/graph";
+import { SCHEMA_FILES, schemaTypes } from "../core/schema-docs";
+import type { Device } from "../schema";
+
+/** One BGP session, with its group's settings filled in, and what is at its far end. */
+export type Session = {
+    device: string;
+    name: string;
+    address: string;
+    group?: string;
+    remoteAs?: number;
+    role?: string;
+    import?: string;
+    export?: string;
+    description?: string;
+    /** One of our devices, by name, or another network, as `AS64501`. */
+    far?: string;
+    /** The far device has no session back. */
+    oneSided?: true;
+};
+
+/** One cable, from the end that names it first. */
+export type Cable = {
+    device: string;
+    port: string;
+    farDevice: string;
+    farPort: string;
+    /** The LAG each end's port is in. */
+    lag?: string;
+    farLag?: string;
+    description?: string;
+    /** VLANs carried, as `name (id)`, from the port or its LAG. */
+    vlans: string[];
+    /** The far end does not name this cable back. */
+    oneSided?: true;
+};
+
+/** A schema type's description and its fields' descriptions, by field path. */
+export type TypeDocs = {
+    description: string;
+    includes: string[];
+    fields: Record<string, { type: string; description: string }>;
+};
+
+export type StudioData = {
+    graph: Annotated;
+    devices: readonly Device[];
+    /** Every port each device's model has, in order, so Studio can show the ones the config leaves shut. */
+    ports: Record<string, string[]>;
+    /** What each device would run, as `circuit build` prints it, or why it cannot be rendered. */
+    rendered: Record<string, string>;
+    /** What each device ran when Circuit last read it, secrets removed, for the devices with a snapshot. */
+    snapshots: Record<string, string>;
+    /** Each device's file and snapshot, relative to the config, for its history. */
+    files: Record<string, DeviceFiles>;
+    addresses: AddressReport;
+    sessions: Session[];
+    cables: Cable[];
+    /** By type name. */
+    docs: Record<string, TypeDocs>;
+};
+
+function sessionsOf(devices: readonly Device[], owners: Map<string, Holder>): Session[] {
+    const all = devices.flatMap((device) => {
+        const bgp = device.routing?.bgp;
+
+        return Object.entries(bgp?.neighbors ?? {}).map(([name, neighbor]): Session & { deviceObject: Device } => {
+            const group = neighbor.group ? bgp?.groups?.[neighbor.group] : undefined;
+            const settings = { ...group, ...neighbor };
+
+            return {
+                deviceObject: device,
+                device: device.name,
+                name,
+                address: neighbor.address,
+                group: neighbor.group,
+                remoteAs: settings.remote_as,
+                role: settings.role,
+                import: settings.import,
+                export: settings.export,
+                description: settings.description,
+            };
+        });
+    });
+
+    const ownerOf = (address: string) => owners.get(keyOf(address) ?? "")?.device;
+
+    return all.map(({ deviceObject, ...session }) => {
+        const far = ownerOf(session.address);
+
+        if (far && far !== deviceObject) {
+            // The session back is one of the same family: a v4 session back does not answer a v6 session.
+            const back = all.some(
+                (other) =>
+                    other.deviceObject === far &&
+                    ownerOf(other.address) === deviceObject &&
+                    familyOf(other.address) === familyOf(session.address)
+            );
+            return { ...session, far: far.name, ...(back ? {} : { oneSided: true as const }) };
+        }
+
+        return { ...session, ...(session.remoteAs ? { far: `AS${session.remoteAs}` } : {}) };
+    });
+}
+
+function cablesOf(devices: readonly Device[]): Cable[] {
+    const byName = new Map(devices.map((device) => [device.name, device]));
+    const drawn = new Set<string>();
+    const cables: Cable[] = [];
+
+    const vlansOf = (device: Device, portName: string) => {
+        const port = device.ports?.[portName];
+        const lag = port?.lag ? device.interfaces?.[port.lag] : undefined;
+        const carrier = lag?.type === "lag" ? lag : port;
+        const names = [carrier?.access_vlan, carrier?.native_vlan, ...(carrier?.trunk_vlans ?? [])].filter((name) => name !== undefined);
+
+        return [...new Set(names)].map((name) => `${name} (${device.vlans?.[name]?.id ?? "?"})`);
+    };
+
+    for (const device of devices) {
+        for (const [portName, port] of Object.entries(device.ports ?? {})) {
+            const link = port?.link;
+            if (!port || !link) continue;
+
+            const here = `${device.name} ${portName}`;
+            const there = `${link.device} ${link.port}`;
+            if (drawn.has(`${there}|${here}`)) continue;
+            drawn.add(`${here}|${there}`);
+
+            const farPort = byName.get(link.device)?.ports?.[link.port];
+            const linksBack = farPort?.link?.device === device.name && farPort.link.port === portName;
+
+            cables.push({
+                device: device.name,
+                port: portName,
+                farDevice: link.device,
+                farPort: link.port,
+                ...(port.lag ? { lag: port.lag } : {}),
+                ...(farPort?.lag ? { farLag: farPort.lag } : {}),
+                ...(port.description ? { description: port.description } : {}),
+                vlans: vlansOf(device, portName),
+                ...(linksBack ? {} : { oneSided: true as const }),
+            });
+        }
+    }
+
+    return cables;
+}
+
+/** The directory Circuit is installed in: the nearest one up with its package.json. Its sources ship with it, in src: the schema, and the files of Studio's page. */
+export function packageRoot(): string | undefined {
+    let directory = dirname(fileURLToPath(import.meta.url));
+
+    while (true) {
+        const manifest = join(directory, "package.json");
+        if (existsSync(manifest) && JSON.parse(readFileSync(manifest, "utf8")).name === "@takodotid/circuit") return directory;
+
+        const parent = dirname(directory);
+        if (parent === directory) return undefined;
+        directory = parent;
+    }
+}
+
+/** The schema's descriptions, or none when its sources cannot be found. */
+function schemaDocs(): Record<string, TypeDocs> {
+    const root = packageRoot();
+    if (!root) return {};
+
+    const sources = SCHEMA_FILES.map(([file]) => readFileSync(join(root, "src", "schema", file), "utf8"));
+    const docs: Record<string, TypeDocs> = {};
+
+    for (const type of schemaTypes(sources).flat()) {
+        docs[type.name] = {
+            description: type.description,
+            includes: (type.includes ?? []).map((name) => name.replace(/<.*$/, "")),
+            fields: Object.fromEntries(
+                type.fields.map((field) => [field.path.replace(/\?$/, ""), { type: field.type, description: field.description }])
+            ),
+        };
+    }
+
+    return docs;
+}
+
+/** Where a device is kept, relative to the config: the file that declares it, and its snapshot. Either is absent when it cannot be found. */
+export type DeviceFiles = { source?: string; snapshot?: string };
+
+/** How Studio learns what a device would run and what it ran, from the caller, which knows the adapters and the state directory. */
+export type DeviceTexts = {
+    /** What the device would run, as `circuit build` prints it. */
+    render: (device: Device) => string;
+    /** Its last snapshot, or undefined when it has none. */
+    snapshot: (device: Device) => string | undefined;
+    /** Its file and its snapshot's file. */
+    files: (device: Device) => DeviceFiles;
+};
+
+/** Everything Studio shows. `graph` comes ready from the caller, with each device compared with its snapshot. */
+export function studioData(devices: readonly Device[], graph: Annotated, texts: DeviceTexts): StudioData {
+    const owners = new Map<string, Holder>();
+    for (const holder of holdersOf(devices)) {
+        const key = keyOf(host(holder.address))!;
+        if (!owners.has(key)) owners.set(key, holder);
+    }
+
+    const attempt = <T>(work: () => T, otherwise: (error: Error) => T) => {
+        try {
+            return work();
+        } catch (error) {
+            return otherwise(error as Error);
+        }
+    };
+
+    return {
+        graph,
+        devices,
+        ports: Object.fromEntries(
+            devices.map((device) => [
+                device.name,
+                attempt(
+                    () => portsOf(device.platform, device.model),
+                    () => []
+                ),
+            ])
+        ),
+        rendered: Object.fromEntries(
+            devices.map((device) => [
+                device.name,
+                attempt(
+                    () => texts.render(device),
+                    (error) => `Cannot be rendered: ${error.message}`
+                ),
+            ])
+        ),
+        snapshots: Object.fromEntries(
+            devices.flatMap((device) => {
+                const text = texts.snapshot(device);
+                return text === undefined ? [] : [[device.name, text]];
+            })
+        ),
+        files: Object.fromEntries(devices.map((device) => [device.name, texts.files(device)])),
+        addresses: addressesOf(devices),
+        sessions: sessionsOf(devices, owners),
+        cables: cablesOf(devices),
+        docs: schemaDocs(),
+    };
+}

package/src/studio/git.ts

@@ -0,0 +1,119 @@
+// Git, for Studio's history and comparisons: the commits that changed a device, and the network as it was at another commit or branch.
+
+import { execFile } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { basename, join } from "node:path";
+
+/** One commit, as Studio lists it. */
+export type Commit = {
+    commit: string;
+    short: string;
+    author: string;
+    /** ISO 8601, with the author's time zone. */
+    date: string;
+    subject: string;
+};
+
+/** The environment without git's own variables, such as GIT_DIR in a hook, which would point git at another repository than the directory's. */
+const environment = () => Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("GIT_")));
+
+/** Run git in a directory, and give what it printed, or fail with what it said. */
+function git(cwd: string, args: string[]): Promise<string> {
+    return new Promise((done, fail) => {
+        execFile("git", args, { cwd, env: environment(), maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
+            if (error) fail(new Error((stderr || error.message).trim()));
+            else done(stdout);
+        });
+    });
+}
+
+const FORMAT = "--format=%H%x1f%h%x1f%an%x1f%aI%x1f%s";
+
+function commitsOf(text: string): Commit[] {
+    return text
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => {
+            const [commit = "", short = "", author = "", date = "", subject = ""] = line.split("\x1f");
+            return { commit, short, author, date, subject };
+        });
+}
+
+/** The top of the git repository the directory is in, or undefined when it is in none, or git is missing. */
+export async function gitTop(directory: string): Promise<string | undefined> {
+    try {
+        return (await git(directory, ["rev-parse", "--show-toplevel"])).trim();
+    } catch {
+        return undefined;
+    }
+}
+
+/** The latest commits that changed any of these files, relative to the directory, newest first. */
+export async function history(directory: string, files: string[], limit = 50): Promise<Commit[]> {
+    if (!files.length) return [];
+    return commitsOf(await git(directory, ["log", `-n${limit}`, FORMAT, "--", ...files]));
+}
+
+/** Branches and tags to compare with, and the latest commits. */
+export async function references(directory: string): Promise<{ names: string[]; commits: Commit[] }> {
+    const names = (
+        await git(directory, [
+            "for-each-ref",
+            "--count=100",
+            "--sort=-committerdate",
+            "--format=%(refname:short)",
+            "refs/heads",
+            "refs/remotes",
+            "refs/tags",
+        ])
+    )
+        .split("\n")
+        .filter((name) => name && !name.endsWith("/HEAD"));
+
+    return { names, commits: commitsOf(await git(directory, ["log", "-n30", FORMAT])) };
+}
+
+/** A branch, tag or commit, as the commit it names, or undefined when it names none. Only the characters a git name can hold are accepted. */
+export async function commitOf(directory: string, name: string): Promise<Commit | undefined> {
+    if (!/^[A-Za-z0-9][\w./@^~-]*$/.test(name)) return undefined;
+
+    try {
+        const [commit] = commitsOf(await git(directory, ["log", "-n1", FORMAT, `${name}^{commit}`, "--"]));
+        return commit;
+    } catch {
+        return undefined;
+    }
+}
+
+/**
+ * Read the network as it was at a commit: check that commit out beside the project, in `.circuit/cache/`, which git ignores, gather it there with `gather`, then remove the checkout. The config is the file of that name in the same directory, `configDirectory`.
+ *
+ * The checkout sits inside the project, so its imports of Circuit resolve to the version installed now.
+ */
+export async function networkAt(
+    configDirectory: string,
+    configPath: string,
+    commit: string,
+    gather: (configPath: string) => Promise<string>
+): Promise<string> {
+    const top = await gitTop(configDirectory);
+    if (!top) throw new Error("the config is not in a git repository");
+
+    const cache = join(configDirectory, ".circuit", "cache");
+    mkdirSync(cache, { recursive: true });
+    writeFileSync(join(cache, ".gitignore"), "# Written by Circuit Studio: checkouts of other commits, removed once read.\n*\n");
+
+    // A checkout left behind by a run that stopped is forgotten first.
+    await git(top, ["worktree", "prune"]);
+
+    const checkout = join(cache, commit.slice(0, 12));
+    await git(top, ["worktree", "add", "--detach", "--force", checkout, commit]);
+
+    try {
+        // Where the config sits inside the repository, as git says: comparing paths would trip on two spellings of one directory, such as Windows' short names.
+        const prefix = (await git(configDirectory, ["rev-parse", "--show-prefix"])).trim();
+        return await gather(join(checkout, prefix, basename(configPath)));
+    } finally {
+        await git(top, ["worktree", "remove", "--force", checkout]).catch(() => {});
+    }
+}