@takodotid/circuit 0.43.0 → 1.0.0-rc.1

package/src/core/secrets.ts

@@ -1,11 +1,12 @@
-// Secrets are references in the config and values only at the moment a command is sent. A reference names a variable, set in the environment or in `.env` beside the config, or a file kept out of the repository. A variable's value may itself be a 1Password reference, `op://vault/item/field`, read with the 1Password CLI.
+// Secrets are references in the config and values only at the moment a command is sent. A reference names a variable, set in the environment or in the network's secrets file (see `SecretsFile`), or a file kept out of the repository. A value in the secrets file is encrypted, or a 1Password reference, `op://vault/item/field`, read with the 1Password CLI.
 
-import { spawnSync } from "node:child_process";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import { parseEnv } from "node:util";
-import type { Network } from "./define";
-import type { Console, Device, IP, Secret } from "../schema";
+import type { Console, Device, IP, Secret } from "../model";
+import { OnePassword, SecretsFile } from "./secrets-file";
+
+/** Anything that holds secret references, such as a network's devices. */
+type Network = object;
 
 /** How a device is logged in to: its management address and SSH port, its connection user and that user's password, and its console. */
 export type DeviceLogin = {
@@ -19,68 +20,59 @@ export type DeviceLogin = {
 };
 
 export class Secrets {
-    /** The directory of the config file. `.env` and secret files are found from here. */
+    /** The directory of the config file. The secrets file and secret files are found from here. */
     private static root = process.cwd();
+    /** The secrets file of the network being worked on, relative to `root`. */
+    private static fileName = SecretsFile.DEFAULT;
 
-    private static envFile: Record<string, string> | undefined;
-    private static readonly fromOnePassword = new Map<string, string>();
+    private static values: Record<string, string> | undefined;
     /** Every value read in this run, so `scrub` can hide it. */
     private static readonly revealed = new Set<string>();
 
     /** What `scrub` puts where a secret was. */
     static readonly REDACTED = "<redacted>";
 
-    /** A value in `.env` shorter than this is not hidden: `.env` can hold settings that are not secrets, and hiding a short one would garble the text. */
-    private static readonly SHORTEST_DOTENV_VALUE = 6;
+    /** A value in plain text shorter than this is not hidden: hiding a short one would garble the text. */
+    private static readonly SHORTEST_PLAIN_VALUE = 6;
 
     /** A marker anywhere in a command. Its group is the reference. */
     static readonly MARKER = /<secret:([A-Za-z0-9_./:-]+)>/g;
 
-    /** Find `.env` and secret files from this directory, the config file's, instead of the working directory. */
+    /** Find the secrets file and secret files from this directory, the config file's, instead of the working directory. */
     static useRoot(directory: string): void {
         Secrets.root = directory;
-        Secrets.envFile = undefined;
+        Secrets.useFile(SecretsFile.DEFAULT);
     }
 
-    /** `.env` beside the config, read once. Node and Bun read it the same way they load `.env` themselves: a `#` starts a comment unless the value is quoted. */
-    private static dotenv(): Record<string, string> {
-        if (Secrets.envFile) return Secrets.envFile;
-
-        const path = join(Secrets.root, ".env");
-        const text = existsSync(path) ? readFileSync(path, "utf8") : "";
-
-        Secrets.refuseCutValues(text);
-        Secrets.envFile = parseEnv(text) as Record<string, string>;
-        return Secrets.envFile;
+    /** Read secrets from this file, relative to the project: the one a network's `secrets.file` names, or `.circuit/secrets`. */
+    static useFile(name: string = SecretsFile.DEFAULT): void {
+        Secrets.fileName = name;
+        Secrets.values = undefined;
     }
 
-    /** An unquoted value is cut at a `#`. A `#` after a space is a comment, but one inside the value, as in `abc#def`, would cut a password short and send that to a device, so it stops the run and asks for quotes. */
-    private static refuseCutValues(text: string): void {
-        for (const line of text.split(/\r?\n/)) {
-            const [, name, value] = /^\s*(?:export\s+)?([\w.-]+)\s*=\s*(.*)$/.exec(line) ?? [];
-            if (!name || !value || /^['"`]/.test(value) || !/\S#/.test(value)) continue;
-
-            throw new Error(`.env: the value of ${name} has a # in it, which starts a comment there; put the value in quotes`);
-        }
+    /** The secrets file in use. */
+    static file(): SecretsFile {
+        return SecretsFile.at(Secrets.root, Secrets.fileName);
     }
 
-    /** Read a 1Password reference with the 1Password CLI, `op`, once per run. */
-    private static readOnePassword(name: string, reference: string): string {
-        const cached = Secrets.fromOnePassword.get(reference);
-        if (cached !== undefined) return cached;
+    /** The secrets file's values as written, read once. A `#` starts a comment unless the value is quoted. */
+    private static stored(): Record<string, string> {
+        if (Secrets.values) return Secrets.values;
 
-        const result = spawnSync("op", ["read", "--no-newline", reference], { encoding: "utf8" });
+        const file = Secrets.file();
+        Secrets.refuseCutValues(file.exists() ? readFileSync(file.path, "utf8") : "", file.name);
+        Secrets.values = file.values();
+        return Secrets.values;
+    }
 
-        if (result.error) throw new Error(`secret ${name} is a 1Password reference, and the 1Password CLI, op, is not installed`);
+    /** An unquoted value is cut at a `#`. A `#` after a space is a comment, but one inside the value, as in `abc#def`, would cut a value short and send that to a device, so it stops the run and asks for quotes. */
+    private static refuseCutValues(text: string, name: string): void {
+        for (const line of text.split(/\r?\n/)) {
+            const [, key, value] = /^\s*(?:export\s+)?([\w.-]+)\s*=\s*(.*)$/.exec(line) ?? [];
+            if (!key || !value || /^['"`]/.test(value) || !/\S#/.test(value)) continue;
 
-        if (result.status !== 0) {
-            const reason = result.stderr.trim() || `op exited with ${result.status}`;
-            throw new Error(`secret ${name}: 1Password could not read ${reference}: ${reason}`);
+            throw new Error(`${name}: the value of ${key} has a # in it, which starts a comment there; put the value in quotes`);
         }
-
-        const value = result.stdout;
-        Secrets.fromOnePassword.set(reference, value);
-        return value;
     }
 
     static isSecret(value: unknown): value is Secret {
@@ -92,18 +84,16 @@ export class Secrets {
         return "secret" in secret ? secret.secret : `file:${secret.secret_file}`;
     }
 
-    /** Where a reference's value comes from, such as `.env` or `1Password, through .env`, without reading it. Undefined when it is set nowhere. */
+    /** Where a reference's value comes from, such as `.circuit/secrets, encrypted` or `1Password, through the environment`, without reading it. Undefined when it is set nowhere. */
     static sourceOf(reference: string): string | undefined {
         if (reference.startsWith("file:")) return existsSync(join(Secrets.root, reference.slice("file:".length))) ? "a file" : undefined;
 
-        const fromFile = Secrets.dotenv()[reference];
-        const value = process.env[reference] || fromFile;
+        const where = process.env[reference] ? "the environment" : Secrets.file().name;
+        const value = process.env[reference] || Secrets.stored()[reference];
         if (!value) return undefined;
 
-        // Bun loads `.env` into the environment when it starts in the same directory, so the same value in both came from `.env`.
-        const where = process.env[reference] && process.env[reference] !== fromFile ? "the environment" : ".env";
-
-        return value.startsWith("op://") ? `1Password, through ${where}` : where;
+        if (value.startsWith("op://")) return `1Password, through ${where}`;
+        return SecretsFile.isEncrypted(value) ? `${where}, encrypted` : where;
     }
 
     /** The value behind a reference, as `referenceOf` writes it: one marker's reference, for a platform that takes a secret interactively. */
@@ -120,17 +110,26 @@ export class Secrets {
             return value;
         }
 
-        // `.env` is read first even when the environment has the value, so a value cut at a `#` is caught under Bun too, which loads `.env` itself.
-        const fromFile = Secrets.dotenv()[reference];
-        const value = process.env[reference] || fromFile;
-        if (!value) throw new Error(`secret ${reference} is not set in the environment or in .env`);
+        // The file is read first even when the environment has the value, so a value cut at a `#` is caught anyway.
+        const fromFile = Secrets.stored()[reference];
+        const fromEnvironment = process.env[reference];
+        if (!fromEnvironment && fromFile === undefined)
+            throw new Error(
+                `secret ${reference} is not set in ${Secrets.file().name}: set it with ${SecretsFile.command(Secrets.file().name, `set ${reference}`)}`
+            );
+
+        const revealed = fromEnvironment
+            ? fromEnvironment.startsWith("op://")
+                ? OnePassword.read(fromEnvironment, `secret ${reference}`)
+                : fromEnvironment
+            : Secrets.file().decrypt(reference, fromFile!);
+        if (!revealed) throw new Error(`secret ${reference} is empty in ${Secrets.file().name}`);
 
-        const revealed = value.startsWith("op://") ? Secrets.readOnePassword(reference, value) : value;
         Secrets.revealed.add(revealed);
         return revealed;
     }
 
-    /** The value of a secret, or of a variable named directly: from the environment, `.env`, 1Password or a file. */
+    /** The value of a secret, or of a variable named directly: from the environment, the secrets file, 1Password or a file. */
     static reveal(secret: Secret | string): string {
         return Secrets.valueOf(typeof secret === "string" ? secret : Secrets.referenceOf(secret));
     }
@@ -144,16 +143,35 @@ export class Secrets {
         return found;
     }
 
+    /**
+     * The secrets file when a secret of `value`, such as a device, is encrypted in it and no password is at hand: the person has to type
+     * it before the work starts. Undefined when every secret can be read. A password in the environment or 1Password opens the file here.
+     */
+    static lockedFor(value: unknown): SecretsFile | undefined {
+        const file = Secrets.file();
+        const needed = [...Secrets.referencesIn(value)].some(
+            (reference) =>
+                !reference.startsWith("file:") && !process.env[reference] && SecretsFile.isEncrypted(Secrets.stored()[reference] ?? "")
+        );
+        return needed && !file.tryUnlock() ? file : undefined;
+    }
+
     /**
      * Text with every secret value Circuit knows replaced by `<redacted>`, before it is printed. Devices echo what they are sent, passwords too.
      *
-     * Hidden are: each value in `.env` of at least six characters, each value read in this run (1Password reads and secret files among them), each secret of `network` that the environment, `.env` or a file holds, and `extra`. A 1Password reference not read in this run is not read for this.
+     * Hidden are: each value in plain text in the secrets file of at least six characters, each value read in this run (1Password reads and secret files among them), each secret of `network` that the environment, the secrets file or a file holds, and `extra`. A 1Password reference not read in this run is not read for this.
      */
     static scrub(text: string, options: { network?: Network; extra?: readonly string[] } = {}): string {
         const values = new Set<string>(options.extra ?? []);
 
-        for (const value of Object.values(Secrets.dotenv())) {
-            if (value.length >= Secrets.SHORTEST_DOTENV_VALUE && !value.startsWith("op://")) values.add(value);
+        // An encrypted value or a reference is not the secret itself.
+        for (const [name, value] of Object.entries(Secrets.stored())) {
+            if (
+                name !== SecretsFile.PASSWORD &&
+                value.length >= Secrets.SHORTEST_PLAIN_VALUE &&
+                SecretsFile.kindOf(value) === "not encrypted"
+            )
+                values.add(value);
         }
 
         for (const value of Secrets.revealed) values.add(value);
@@ -168,22 +186,23 @@ export class Secrets {
         return ordered.reduce((out, value) => out.split(value).join(Secrets.REDACTED), text);
     }
 
-    /** The value behind a reference when it can be had without asking anyone: a 1Password reference only once read in this run. Undefined otherwise. */
+    /** The value behind a reference when it can be had without asking anyone: a 1Password reference only once read in this run, an encrypted value only with the file open. Undefined otherwise. */
     private static storedValueOf(reference: string): string | undefined {
         if (reference.startsWith("file:")) {
             const path = join(Secrets.root, reference.slice("file:".length));
             return existsSync(path) ? readFileSync(path, "utf8").trimEnd() : undefined;
         }
 
-        const value = process.env[reference] || Secrets.dotenv()[reference];
-        if (!value?.startsWith("op://")) return value;
+        const fromEnvironment = process.env[reference];
+        if (fromEnvironment) return fromEnvironment.startsWith("op://") ? OnePassword.cached(fromEnvironment) : fromEnvironment;
 
-        return Secrets.fromOnePassword.get(value);
+        const value = Secrets.stored()[reference];
+        return value ? Secrets.file().revealIfOpen(reference, value) : undefined;
     }
 
     /** How Circuit logs in to a device: its address, port, connection user, that user's password revealed, and its console. */
     static loginOf(device: Device): DeviceLogin {
-        const user = device.users?.[device.connection.user];
+        const user = device.system?.users?.[device.connection.user];
         if (!user) throw new Error(`${device.name}: connection user ${device.connection.user} is not one of its users`);
 
         return {