@takodotid/circuit 0.43.1 → 1.0.0-rc.2

package/src/studio/data.ts

@@ -1,112 +1,65 @@
-// What Studio shows, gathered from the config in one pass: the graph with its findings and drift, each device as the config holds it, what each would run, the address plan, every BGP session and cable, and the schema's descriptions of each field.
+// What Studio shows, gathered from a project in one pass: each device as the model holds it, what each would run, what `validate` finds, the graph, the addresses, every BGP session and every cable.
 
-import { existsSync, readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-import { Catalog } from "../platforms/catalog";
+import { basename } from "node:path";
+import { Validation } from "../checks";
 import { Address } from "../core/address";
-import { AddressSpace, type AddressReport } from "../core/address-space";
-import { Topology, type Annotated, type Holder } from "../core/topology";
-import { SchemaDocs } from "../core/schema-docs";
-import type { Device } from "../schema";
-
-/** One BGP session, with its group's settings filled in, and what is at its far end. */
-export type Session = {
-    device: string;
-    name: string;
-    address: string;
-    group?: string;
-    remoteAs?: number;
-    role?: string;
-    import?: string;
-    export?: string;
-    description?: string;
-    /** One of our devices, by name, or another network, as `AS64501`. */
-    far?: string;
-    /** The far device has no session back. */
-    oneSided?: true;
-};
-
-/** One cable, from the end that names it first. */
-export type Cable = {
-    device: string;
-    port: string;
-    farDevice: string;
-    farPort: string;
-    /** The LAG each end's port is in. */
-    lag?: string;
-    farLag?: string;
-    description?: string;
-    /** What is between the two ends, such as a carrier's local loop. */
-    through?: string;
-    /** VLANs carried, as `name (id)`, from the port or its LAG. */
-    vlans: string[];
-    /** The far end does not name this cable back. */
-    oneSided?: true;
-};
-
-/** A schema type's description and its fields' descriptions, by field path. */
-export type TypeDocs = {
-    description: string;
-    includes: string[];
-    fields: Record<string, { type: string; description: string }>;
-};
-
-export type StudioData = {
-    graph: Annotated;
-    devices: readonly Device[];
-    /** Every port each device's model has, in order, so Studio can show the ones the config leaves shut. */
-    ports: Record<string, string[]>;
-    /** What each device would run, as `circuit build` prints it, or why it cannot be rendered. */
-    rendered: Record<string, string>;
-    /** What each device ran when Circuit last read it, secrets removed, for the devices with a snapshot. */
-    snapshots: Record<string, string>;
-    /** Each device's file and snapshot, relative to the config, for its history. */
-    files: Record<string, DeviceFiles>;
-    addresses: AddressReport;
-    sessions: Session[];
-    cables: Cable[];
-    /** By type name. */
-    docs: Record<string, TypeDocs>;
-};
+import { Secrets } from "../core/secrets";
+import { Topology } from "../core/topology";
+import type { Device } from "../model";
+import { Platforms } from "../platforms";
+import { Catalog } from "../platforms/catalog";
+import { Derive } from "../platforms/derive";
+import type { Finding } from "../platforms/finding";
+import { ProjectError, type ProjectRead, readProject, secretsFileOf } from "../project/project";
+import type { AddressRow, Cable, Session, StudioData, StudioError } from "./types";
 
-/** Where a device is kept, relative to the config: the file that declares it, and its snapshot. Either is absent when it cannot be found. */
-export type DeviceFiles = { source?: string; snapshot?: string };
+export class StudioModel {
+    /** Read the project and gather what Studio shows, or what stopped the read. */
+    static async gather(root: string): Promise<StudioData | StudioError> {
+        let read: ProjectRead;
+        try {
+            read = await readProject(root);
+        } catch (error) {
+            if (error instanceof ProjectError) return { error: error.lines.join("\n") };
+            return { error: (error as Error).message };
+        }
 
-/** How Studio learns what a device would run and what it ran, from the caller, which knows the adapters and the state directory. */
-export type DeviceTexts = {
-    /** What the device would run, as `circuit build` prints it. */
-    render: (device: Device) => string;
-    /** Its last snapshot, or undefined when it has none. */
-    snapshot: (device: Device) => string | undefined;
-    /** Its file and its snapshot's file. */
-    files: (device: Device) => DeviceFiles;
-};
+        return StudioModel.build(basename(root), read);
+    }
 
-export class StudioModel {
-    /** Everything Studio shows. `graph` comes ready from the caller, with each device compared with its snapshot. */
-    static build(devices: readonly Device[], graph: Annotated, texts: DeviceTexts): StudioData {
-        const owners = new Map<string, Holder>();
-        for (const holder of Topology.holders(devices)) {
-            const key = Topology.keyOf(Address.host(holder.address))!;
-            if (!owners.has(key)) owners.set(key, holder);
+    private static attempt<T>(work: () => T, otherwise: (error: Error) => T): T {
+        try {
+            return work();
+        } catch (error) {
+            return otherwise(error as Error);
         }
+    }
 
-        const attempt = <T>(work: () => T, otherwise: (error: Error) => T) => {
-            try {
-                return work();
-            } catch (error) {
-                return otherwise(error as Error);
-            }
-        };
+    static build(project: string, read: ProjectRead): StudioData {
+        const devices = read.networks.flatMap((network) => Object.values(network.devices));
+        const findings: Finding[] = [
+            ...read.warnings.map((message) => ({ level: "warning" as const, message })),
+            ...read.networks.flatMap((network) => Validation.run(Object.values(network.devices), network.settings.checks)),
+        ];
 
         return {
-            graph,
+            project,
+            networks: read.networks.map((network) => ({
+                name: network.name,
+                ...(network.asn !== undefined ? { asn: network.asn } : {}),
+                sites: network.settings.sites ?? {},
+                devices: Object.keys(network.devices),
+                secretsFile: secretsFileOf(network),
+                secrets: [...Secrets.referencesIn([network.devices, network.settings.integrations])]
+                    .filter((name) => !name.startsWith("file:"))
+                    .sort(),
+            })),
             devices,
+            files: Object.fromEntries(read.networks.flatMap((network) => Object.entries(network.files))),
             ports: Object.fromEntries(
                 devices.map((device) => [
                     device.name,
-                    attempt(
+                    StudioModel.attempt(
                         () => Catalog.ports(device.platform, device.model),
                         () => []
                     ),
@@ -115,80 +68,66 @@ export class StudioModel {
             rendered: Object.fromEntries(
                 devices.map((device) => [
                     device.name,
-                    attempt(
-                        () => texts.render(device),
+                    StudioModel.attempt(
+                        () => Platforms.find(device.platform)?.render(device) ?? `Cannot be rendered: no platform ${device.platform}`,
                         (error) => `Cannot be rendered: ${error.message}`
                     ),
                 ])
             ),
-            snapshots: Object.fromEntries(
-                devices.flatMap((device) => {
-                    const text = texts.snapshot(device);
-                    return text === undefined ? [] : [[device.name, text]];
-                })
-            ),
-            files: Object.fromEntries(devices.map((device) => [device.name, texts.files(device)])),
-            addresses: AddressSpace.report(devices),
-            sessions: StudioModel.sessionsOf(devices, owners),
+            findings,
+            graph: Topology.of(devices),
+            addresses: StudioModel.addressesOf(devices),
+            sessions: StudioModel.sessionsOf(devices),
             cables: StudioModel.cablesOf(devices),
-            docs: StudioModel.schemaDocs(),
+            secrets: [...Secrets.referencesIn(read.networks.map((network) => [network.devices, network.settings.integrations]))]
+                .filter((name) => !name.startsWith("file:"))
+                .sort(),
         };
     }
 
-    /** The directory Circuit is installed in: the nearest one up with its package.json. Its sources ship with it, in src: the schema, and the files of Studio's page. */
-    static packageRoot(): string | undefined {
-        let directory = dirname(fileURLToPath(import.meta.url));
-
-        while (true) {
-            const manifest = join(directory, "package.json");
-            if (existsSync(manifest) && JSON.parse(readFileSync(manifest, "utf8")).name === "@takodotid/circuit") return directory;
-
-            const parent = dirname(directory);
-            if (parent === directory) return undefined;
-            directory = parent;
-        }
+    private static addressesOf(devices: readonly Device[]): AddressRow[] {
+        return Topology.holders(devices).map((holder) => ({
+            device: holder.device.name,
+            holder: holder.name,
+            address: holder.address,
+            prefix: Address.prefixOf(holder.address),
+            family: Address.family(holder.address),
+            ...(holder.peer ? { peer: holder.peer } : {}),
+            ...(holder.vrf ? { vrf: holder.vrf } : {}),
+        }));
     }
 
-    private static sessionsOf(devices: readonly Device[], owners: Map<string, Holder>): Session[] {
-        const all = devices.flatMap((device) => {
-            const bgp = device.routing?.bgp;
-
-            return Object.entries(bgp?.neighbors ?? {}).map(([name, neighbor]): Session & { deviceObject: Device } => {
-                const group = neighbor.group ? bgp?.groups?.[neighbor.group] : undefined;
-                const settings = { ...group, ...neighbor };
-
-                return {
-                    deviceObject: device,
-                    device: device.name,
-                    name,
-                    address: neighbor.address,
-                    group: neighbor.group,
-                    remoteAs: settings.remote_as,
-                    role: settings.role,
-                    import: settings.import,
-                    export: settings.export,
-                    description: settings.description,
-                };
-            });
-        });
-
+    private static sessionsOf(devices: readonly Device[]): Session[] {
+        const owners = Topology.owners(Topology.holders(devices));
+        const all = Topology.sessionsOf(devices);
         const ownerOf = (address: string) => owners.get(Topology.keyOf(address) ?? "")?.device;
 
-        return all.map(({ deviceObject, ...session }) => {
+        return all.map(({ device, ownAs: _ownAs, ...session }) => {
+            const base: Session = {
+                device: device.name,
+                name: session.name,
+                address: session.address,
+                ...(session.group ? { group: session.group } : {}),
+                ...(session.remoteAs !== undefined ? { remoteAs: session.remoteAs } : {}),
+                ...(session.role ? { role: session.role } : {}),
+                ...(session.import ? { import: session.import } : {}),
+                ...(session.export ? { export: session.export } : {}),
+                ...(session.description ? { description: session.description } : {}),
+            };
             const far = ownerOf(session.address);
 
-            if (far && far !== deviceObject) {
+            if (far && far !== device) {
                 // The session back is one of the same family: a v4 session back does not answer a v6 session.
                 const back = all.some(
                     (other) =>
-                        other.deviceObject === far &&
-                        ownerOf(other.address) === deviceObject &&
+                        other.device === far &&
+                        ownerOf(other.address) === device &&
                         Address.family(other.address) === Address.family(session.address)
                 );
-                return { ...session, far: far.name, ...(back ? {} : { oneSided: true as const }) };
+                return { ...base, far: far.name, ...(back ? {} : { oneSided: true as const }) };
             }
 
-            return { ...session, ...(session.remoteAs ? { far: `AS${session.remoteAs}` } : {}) };
+            return { ...base, ...(session.remoteAs ? { far: `AS${session.remoteAs}` } : {}) };
         });
     }
 
@@ -197,40 +136,30 @@ export class StudioModel {
         const drawn = new Set<string>();
         const cables: Cable[] = [];
 
-        const vlansOf = (device: Device, portName: string) => {
-            const port = device.ports?.[portName];
-            const lag = port?.lag ? device.interfaces?.[port.lag] : undefined;
-            const carrier = lag?.type === "lag" ? lag : port;
-            const names = [carrier?.access_vlan, carrier?.native_vlan, ...(carrier?.trunk_vlans ?? [])].filter(
-                (name) => name !== undefined
-            );
-
-            return [...new Set(names)].map((name) => `${name} (${device.vlans?.[name]?.id ?? "?"})`);
-        };
-
         for (const device of devices) {
-            for (const [portName, port] of Object.entries(device.ports ?? {})) {
-                const link = port?.link;
-                if (!port || !link) continue;
+            for (const [portName, port] of Derive.ports(device)) {
+                const link = port.link;
+                if (!link) continue;
 
                 const here = `${device.name} ${portName}`;
                 const there = `${link.device} ${link.port}`;
                 if (drawn.has(`${there}|${here}`)) continue;
                 drawn.add(`${here}|${there}`);
 
-                const farPort = byName.get(link.device)?.ports?.[link.port];
+                const farPort = byName.get(link.device)?.interfaces?.ethernet?.[link.port];
                 const linksBack = farPort?.link?.device === device.name && farPort.link.port === portName;
+                const vlans = Topology.carriedBy(device, Topology.carrierOf(device, port)).map((vlan) => `${vlan.name} (${vlan.id})`);
 
                 cables.push({
                     device: device.name,
                     port: portName,
                     farDevice: link.device,
                     farPort: link.port,
-                    ...(port.lag ? { lag: port.lag } : {}),
-                    ...(farPort?.lag ? { farLag: farPort.lag } : {}),
+                    ...(port.in_lag ? { lag: port.in_lag } : {}),
+                    ...(farPort?.in_lag ? { farLag: farPort.in_lag } : {}),
                     ...(port.description ? { description: port.description } : {}),
                     ...(link.through ? { through: link.through } : {}),
-                    vlans: vlansOf(device, portName),
+                    vlans,
                     ...(linksBack ? {} : { oneSided: true as const }),
                 });
             }
@@ -238,25 +167,4 @@ export class StudioModel {
 
         return cables;
     }
-
-    /** The schema's descriptions, or none when its sources cannot be found. */
-    private static schemaDocs(): Record<string, TypeDocs> {
-        const root = StudioModel.packageRoot();
-        if (!root) return {};
-
-        const sources = SchemaDocs.FILES.map(([file]) => readFileSync(join(root, "src", "schema", file), "utf8"));
-        const docs: Record<string, TypeDocs> = {};
-
-        for (const type of SchemaDocs.read(sources).flat()) {
-            docs[type.name] = {
-                description: type.description,
-                includes: (type.includes ?? []).map((name) => name.replace(/<.*$/, "")),
-                fields: Object.fromEntries(
-                    type.fields.map((field) => [field.path.replace(/\?$/, ""), { type: field.type, description: field.description }])
-                ),
-            };
-        }
-
-        return docs;
-    }
 }

package/src/studio/git.ts

@@ -1,28 +1,19 @@
-// Git, for Studio's history and comparisons: the commits that changed a device, and the network as it was at another commit or branch.
+// Git, for Studio's history and comparisons: the commits that changed a device's file, and the project as it was at another commit or branch.
 
 import { execFile } from "node:child_process";
 import { mkdirSync, writeFileSync } from "node:fs";
-import { basename, join } from "node:path";
-
-/** One commit, as Studio lists it. */
-export type Commit = {
-    commit: string;
-    short: string;
-    author: string;
-    /** ISO 8601, with the author's time zone. */
-    date: string;
-    subject: string;
-};
+import { join } from "node:path";
+import type { Commit } from "./types";
 
 export class Git {
     private static readonly FORMAT = "--format=%H%x1f%h%x1f%an%x1f%aI%x1f%s";
 
-    /** The environment without git's own variables, such as GIT_DIR in a hook, which would point git at another repository than the directory's. */
+    /** The environment without git's own variables, such as GIT_DIR in a hook, which would point git at another repository than the folder's. */
     private static environment(): Record<string, string | undefined> {
         return Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith("GIT_")));
     }
 
-    /** Run git in a directory, and give what it printed, or fail with what it said. */
+    /** Run git in a folder, and give what it printed, or fail with what it said. */
     private static run(cwd: string, args: string[]): Promise<string> {
         return new Promise((done, fail) => {
             execFile("git", args, { cwd, env: Git.environment(), maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
@@ -42,25 +33,25 @@ export class Git {
             });
     }
 
-    /** The top of the git repository the directory is in, or undefined when it is in none, or git is missing. */
-    static async top(directory: string): Promise<string | undefined> {
+    /** The top of the git repository the folder is in, or undefined when it is in none, or git is missing. */
+    static async top(folder: string): Promise<string | undefined> {
         try {
-            return (await Git.run(directory, ["rev-parse", "--show-toplevel"])).trim();
+            return (await Git.run(folder, ["rev-parse", "--show-toplevel"])).trim();
         } catch {
             return undefined;
         }
     }
 
-    /** The latest commits that changed any of these files, relative to the directory, newest first. */
-    static async history(directory: string, files: string[], limit = 50): Promise<Commit[]> {
+    /** The latest commits that changed any of these files, from the folder, newest first. */
+    static async history(folder: string, files: string[], limit = 50): Promise<Commit[]> {
         if (!files.length) return [];
-        return Git.commitsOf(await Git.run(directory, ["log", `-n${limit}`, Git.FORMAT, "--", ...files]));
+        return Git.commitsOf(await Git.run(folder, ["log", `-n${limit}`, Git.FORMAT, "--", ...files]));
     }
 
     /** Branches and tags to compare with, and the latest commits. */
-    static async references(directory: string): Promise<{ names: string[]; commits: Commit[] }> {
+    static async references(folder: string): Promise<{ names: string[]; commits: Commit[] }> {
         const names = (
-            await Git.run(directory, [
+            await Git.run(folder, [
                 "for-each-ref",
                 "--count=100",
                 "--sort=-committerdate",
@@ -73,15 +64,15 @@ export class Git {
             .split("\n")
             .filter((name) => name && !name.endsWith("/HEAD"));
 
-        return { names, commits: Git.commitsOf(await Git.run(directory, ["log", "-n30", Git.FORMAT])) };
+        return { names, commits: Git.commitsOf(await Git.run(folder, ["log", "-n30", Git.FORMAT])) };
     }
 
-    /** A branch, tag or commit, as the commit it names, or undefined when it names none. Only the characters a git name can hold are accepted. */
-    static async commitOf(directory: string, name: string): Promise<Commit | undefined> {
+    /** A branch, tag or commit, as the commit it names, or undefined when it names none. Only the characters a git name can hold are taken. */
+    static async commitOf(folder: string, name: string): Promise<Commit | undefined> {
         if (!/^[A-Za-z0-9][\w./@^~-]*$/.test(name)) return undefined;
 
         try {
-            const [commit] = Git.commitsOf(await Git.run(directory, ["log", "-n1", Git.FORMAT, `${name}^{commit}`, "--"]));
+            const [commit] = Git.commitsOf(await Git.run(folder, ["log", "-n1", Git.FORMAT, `${name}^{commit}`, "--"]));
             return commit;
         } catch {
             return undefined;
@@ -89,20 +80,13 @@ export class Git {
     }
 
     /**
-     * Read the network as it was at a commit: check that commit out beside the project, in `.circuit/cache/`, which git ignores, gather it there with `gather`, then remove the checkout. The config is the file of that name in the same directory, `configDirectory`.
-     *
-     * The checkout sits inside the project, so its imports of Circuit resolve to the version installed now.
+     * Read the project as it was at a commit: check that commit out beside the project, in `.circuit/cache/`, which git ignores, gather it there with `gather`, then remove the checkout. It is read by the Circuit running now, so a comparison shows changes to the project, not to Circuit.
      */
-    static async networkAt(
-        configDirectory: string,
-        configPath: string,
-        commit: string,
-        gather: (configPath: string) => Promise<string>
-    ): Promise<string> {
-        const top = await Git.top(configDirectory);
-        if (!top) throw new Error("the config is not in a git repository");
-
-        const cache = join(configDirectory, ".circuit", "cache");
+    static async projectAt(root: string, commit: string, gather: (folder: string) => Promise<string>): Promise<string> {
+        const top = await Git.top(root);
+        if (!top) throw new Error("the project is not in a git repository");
+
+        const cache = join(root, ".circuit", "cache");
         mkdirSync(cache, { recursive: true });
         writeFileSync(join(cache, ".gitignore"), "# Written by Circuit Studio: checkouts of other commits, removed once read.\n*\n");
 
@@ -110,13 +94,13 @@ export class Git {
         await Git.run(top, ["worktree", "prune"]);
 
         const checkout = join(cache, commit.slice(0, 12));
-        // The checkout sits deep inside the project, so a long path in the repository passes the 260 characters Windows allows unless git is told otherwise.
+        // The checkout sits deep inside the project, so a long path passes the 260 characters Windows allows unless git is told otherwise.
         await Git.run(top, ["-c", "core.longpaths=true", "worktree", "add", "--detach", "--force", checkout, commit]);
 
         try {
-            // Where the config sits inside the repository, as git says: comparing paths would trip on two spellings of one directory, such as Windows' short names.
-            const prefix = (await Git.run(configDirectory, ["rev-parse", "--show-prefix"])).trim();
-            return await gather(join(checkout, prefix, basename(configPath)));
+            // Where the project sits inside the repository, as git says: comparing paths would trip on two spellings of one folder, such as Windows' short names.
+            const prefix = (await Git.run(root, ["rev-parse", "--show-prefix"])).trim();
+            return await gather(join(checkout, prefix));
         } finally {
             await Git.run(top, ["-c", "core.longpaths=true", "worktree", "remove", "--force", checkout]).catch(() => {});
         }