@substrio/cli 0.0.0-stage → 0.1.0

package/src/documents.mjs

@@ -0,0 +1,139 @@
+/**
+ * Substrio CLI: the documents in a file or text, for `substrio data import`.
+ *
+ * The form is decided once: by `--format`, else the file's extension (.ndjson
+ * and .jsonl are NDJSON, .json is a JSON array or {"documents": [...]}, .csv
+ * is CSV), else (stdin, inline text, an odd extension) the whole input is
+ * parsed as one JSON value first and, when that fails, as NDJSON. NDJSON,
+ * JSON and CSV are read a chunk at a time (src/readers.mjs): a file of any
+ * size is never loaded whole.
+ *
+ * Every document comes out as {position, raw, endOffset} (`raw`, a Buffer,
+ * is that document's own JSON text) or {position, error, endOffset,
+ * violations?}: nothing is skipped unseen. `endOffset` is the byte offset in
+ * the input just past that document, for a resumed import (--resume). The
+ * AUTO fallback (stdin, inline text or an odd extension with no --format)
+ * parses the whole input as one JSON value first, already holding every
+ * element as a JS value; it is the one path that re-serializes to produce
+ * `raw`, since nothing here has JSON text for it to begin with.
+ */
+
+import { readFileSync } from 'node:fs';
+import { extname } from 'node:path';
+
+import { CliError, ErrorCode } from './cli.mjs';
+import {
+    byteChunksOf, csvDocuments, documentAt, documentOfLine, jsonArrayDocuments, ndjsonDocuments, unreadable,
+} from './readers.mjs';
+
+export { documentAt, documentOfLine };
+
+/** How the documents of an input are laid out. */
+export const DocumentForm = Object.freeze({
+    NDJSON: 'ndjson', // one JSON object per line
+    JSON: 'json',     // one JSON value: an array, or {"documents": [...]}
+    CSV: 'csv',       // RFC 4180, a header row required
+    AUTO: 'auto',     // decided by parsing the input
+});
+
+const FORM_BY_EXTENSION = new Map([
+    ['.ndjson', DocumentForm.NDJSON], ['.jsonl', DocumentForm.NDJSON], ['.json', DocumentForm.JSON], ['.csv', DocumentForm.CSV],
+]);
+
+/** The form a file name decides (pure): by its extension, else AUTO. */
+export function formOfFile(path) {
+    return FORM_BY_EXTENSION.get(extname(path).toLowerCase()) ?? DocumentForm.AUTO;
+}
+
+// The elements of a parsed JSON value that is the whole input, or null when it is a single object that is not a wrapper.
+function elementsOf(value, form) {
+    const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+    if (Array.isArray(value)) return value;
+    if (isObj(value) && Object.hasOwn(value, 'documents')) {
+        if (!Array.isArray(value.documents)) throw new CliError('"documents" must be an array of objects', { code: ErrorCode.INVALID_ARGUMENT });
+        return value.documents;
+    }
+    if (form === DocumentForm.JSON) {
+        throw new CliError('a .json file holds an array of objects or {"documents": [...]}; for one object per line use .ndjson',
+            { code: ErrorCode.INVALID_ARGUMENT });
+    }
+    return null;
+}
+
+// `found` ({position, document, size} or {position, error, violations?}) as the uniform raw-bytes shape:
+// the AUTO path already holds a JS value, so this is the one place that re-serializes it to JSON text.
+function toRawItem(found, endOffset) {
+    if (found.document === undefined) return { position: found.position, error: found.error, endOffset, ...(found.violations ? { violations: found.violations } : {}) };
+    return { position: found.position, raw: Buffer.from(JSON.stringify(found.document), 'utf8'), endOffset };
+}
+
+// The documents of `text` in `form` (AUTO's legacy whole-parse path), as an iterable; endOffset is the byte length.
+function* documentsOfText(text, form) {
+    const body = text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text;
+    const bytes = Buffer.byteLength(text, 'utf8');
+    if (form !== DocumentForm.NDJSON) {
+        let parsed;
+        try {
+            parsed = JSON.parse(body);
+        } catch (err) {
+            // Not one JSON value: NDJSON, unless the form said JSON.
+            if (form === DocumentForm.JSON) throw new CliError(`not valid JSON (${err.message})`, { code: ErrorCode.INVALID_ARGUMENT });
+        }
+        const elements = parsed === undefined ? null : elementsOf(parsed, form);
+        if (elements !== null) {
+            for (let i = 0; i < elements.length; i++) yield toRawItem(documentAt(i + 1, elements[i]), bytes);
+            return;
+        }
+    }
+    const lines = body.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+        const found = documentOfLine(i + 1, lines[i]);
+        if (found !== null) yield toRawItem(found, bytes);
+    }
+}
+
+/**
+ * The documents of a source, in order, one at a time. `source` is {path}
+ * ("-" is stdin) or {text}. `options.form` overrides the extension (one of
+ * DocumentForm); `options.types` (a Map) and `options.emptyAs` are CSV's;
+ * `options.resumeOffset` continues a read from that byte offset (--resume;
+ * refused for stdin). Throws a CliError when the input as a whole cannot be
+ * read as documents (a missing file, a .json that is not JSON, a truncated
+ * JSON array); a single bad line, element or row comes out as
+ * {position, error, endOffset, violations?}.
+ */
+export async function* readDocuments(source, options = {}) {
+    const { form: requestedForm, types = null, emptyAs = undefined, resumeOffset = 0 } = options;
+    if (source.text !== undefined) {
+        if (resumeOffset > 0) throw new CliError('--resume needs a file, not inline text', { code: ErrorCode.INVALID_ARGUMENT });
+        const form = requestedForm ?? DocumentForm.AUTO;
+        if (form === DocumentForm.CSV) { yield* csvDocuments(byteChunksOf(source), { types, emptyAs }); return; }
+        if (form === DocumentForm.JSON || form === DocumentForm.NDJSON) {
+            if (form === DocumentForm.JSON) { yield* jsonArrayDocuments(byteChunksOf(source)); return; }
+            yield* ndjsonDocuments(byteChunksOf(source));
+            return;
+        }
+        yield* documentsOfText(source.text, DocumentForm.AUTO);
+        return;
+    }
+    const form = requestedForm ?? (source.path === '-' ? DocumentForm.AUTO : formOfFile(source.path));
+    if (form === DocumentForm.NDJSON) {
+        yield* ndjsonDocuments(byteChunksOf(source, resumeOffset), { baseOffset: resumeOffset });
+        return;
+    }
+    if (form === DocumentForm.JSON) {
+        yield* jsonArrayDocuments(byteChunksOf(source, resumeOffset), { baseOffset: resumeOffset, resuming: resumeOffset > 0 });
+        return;
+    }
+    if (form === DocumentForm.CSV) {
+        yield* csvDocuments(byteChunksOf(source), { types, emptyAs, resumeOffset });
+        return;
+    }
+    let text;
+    try {
+        text = readFileSync(source.path === '-' ? 0 : source.path, 'utf8');
+    } catch (err) {
+        throw unreadable(source.path, err);
+    }
+    yield* documentsOfText(text, form);
+}

package/src/function-build.mjs

@@ -0,0 +1,273 @@
+/**
+ * Substrio CLI: the one owner of turning a function's source on disk into the
+ * bytes the server is sent, for every path that reaches a tenant (substrio
+ * deploy, substrio push, substrio dev's first apply, its version-wide apply and its
+ * per-save redeploy: manifest.mjs's collectApp calls buildFunction for every
+ * function it collects, and nothing else reads a function file for upload).
+ *
+ * The server's loader has no resolver (a function is one file, able to
+ * import only the runtime's builtin modules). Whether a function needs
+ * bundling is never decided by reading its source here (no hand-rolled
+ * scanner of JavaScript: esbuild already parses the file to bundle it, and
+ * the server's function.check/function.deploy already parse it, without
+ * running it, to refuse what they do not accept): esbuild, when it resolves
+ * from the project or the CLI's own install, is always asked to build the
+ * entry with bundling on; its own metafile says whether anything beyond the
+ * entry was pulled in. When nothing was (every import was a runtime builtin,
+ * left external, or there was no import at all), the bytes sent are the
+ * entry file's own, unchanged. When esbuild does not resolve, the entry
+ * file's own bytes are sent regardless: a function with no import beyond the
+ * builtins still deploys with no bundler installed, and a function that does
+ * need one is refused by the server, which the caller turns into one hint
+ * naming the fix (deploy.mjs, local/reload.mjs).
+ */
+
+import { spawnSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { mkdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { CliError, ErrorCode } from './cli.mjs';
+import { configDir } from './config.mjs';
+
+/**
+ * The runtime's builtin modules (docs/runtime/index.md, "Builtin modules"),
+ * exactly as that table's specifier column lists them; a function may import
+ * or require only these, and the bare Node name (without "node:") works too,
+ * except substrio:env, which has no bare form. The test function-build.test.mjs
+ * fails when this drifts from the docs table.
+ */
+export const BUILTIN_MODULES = Object.freeze([
+    'substrio:env',
+    'node:buffer',
+    'node:crypto',
+    'node:events',
+    'node:util',
+    'node:url',
+    'node:path',
+    'node:path/posix',
+    'node:querystring',
+    'node:string_decoder',
+    'node:assert',
+    'node:assert/strict',
+]);
+
+const BARE_BUILTINS = new Set(
+    BUILTIN_MODULES.filter((spec) => spec.startsWith('node:')).map((spec) => spec.slice('node:'.length))
+);
+const BUILTIN_SET = new Set(BUILTIN_MODULES);
+
+/** Whether `specifier` is one the runtime provides without bundling, in either form it accepts. */
+export function isBuiltinSpecifier(specifier) {
+    return BUILTIN_SET.has(specifier) || BARE_BUILTINS.has(specifier);
+}
+
+// esbuild's own `external` glob needs every spelling the runtime accepts, so a builtin import
+// is never pulled into the bundle.
+const EXTERNAL_BUILTINS = Object.freeze([...BUILTIN_MODULES, ...BARE_BUILTINS]);
+
+// The bundler the CLI declares (package.json `dependencies.esbuild`): an install that left it out
+// (--omit, a copied CLI, a failed platform binary) gets it here, once a process and only when no
+// resolution finds it, at the declared range, into the CLI's own tools folder under the profiles
+// directory (always the user's to write, unlike a global install). SUBSTRIO_NO_INSTALL=1 turns
+// it off (an offline machine); the deploy then sends the entry file alone, as before.
+const DECLARED_ESBUILD = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8')).dependencies.esbuild;
+export const toolsDir = () => join(configDir(), 'tools');
+let installTried = false;
+
+function resolveFrom(base) {
+    try {
+        return createRequire(base).resolve('esbuild');
+    } catch {
+        return null;
+    }
+}
+
+// The tools folder's own esbuild only, <tools>/node_modules/esbuild: 1 read of its package.json
+// and 1 stat of its entry, no ancestor walk. The folder sits under the profiles directory, and
+// Node's walk up from it would take any esbuild above it (~/node_modules) for the CLI's own.
+function resolveFromTools() {
+    const at = join(toolsDir(), 'node_modules', 'esbuild');
+    try {
+        const entry = join(at, JSON.parse(readFileSync(join(at, 'package.json'), 'utf-8')).main ?? 'index.js');
+        return statSync(entry).isFile() ? entry : null;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Installs the declared esbuild into toolsDir() with npm (its output on stderr, so --json stays
+ * whole); true when npm says it did. `run` is spawnSync's seam for a test.
+ */
+export function installEsbuild({ dir = toolsDir(), run = spawnSync, log = (line) => process.stderr.write(line) } = {}) {
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    log(`installing esbuild@${DECLARED_ESBUILD} (the CLI's bundler) into ${dir}\n`);
+    const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+    const r = run(npm, ['install', '--prefix', dir, '--no-audit', '--no-fund', '--no-package-lock', '--loglevel=error', `esbuild@${DECLARED_ESBUILD}`],
+        { stdio: ['ignore', 2, 2] });
+    return r.status === 0;
+}
+
+// esbuild's path, resolved from the project first (Node's walk: a workspace's hoisted
+// node_modules counts), then the CLI's own install, then the CLI's tools folder (its own
+// node_modules only), installing it there when none has it: the real resolution, used whenever a call
+// site states none of its own. Exported so a test whose subject is bundling with a real,
+// project-resolved esbuild can ask for it by name, regardless of the suite's own mode.
+// `own` and `install` are seams for a test; the process-wide once applies to the real installer.
+export function resolveEsbuildReal(projectDir, { own = import.meta.url, install = installEsbuild } = {}) {
+    const found = resolveFrom(join(projectDir, 'package.json')) ?? resolveFrom(own) ?? resolveFromTools();
+    if (found || process.env.SUBSTRIO_NO_INSTALL === '1') return found;
+    if (install === installEsbuild) {
+        if (installTried) return null;
+        installTried = true;
+    }
+    return install() ? resolveFromTools() : null;
+}
+
+let defaultResolveEsbuild = resolveEsbuildReal;
+
+/**
+ * Test-only seam: every buildFunction call that states no `resolveEsbuild` of its own uses this
+ * one for the rest of the process. Nothing under src/ calls it; a test either passes
+ * `resolveEsbuild` to one call, or sets this once (test/helpers/esbuild-mode.mjs, loaded with
+ * `--import` before any production code runs) for every call that does not.
+ */
+export function setDefaultEsbuildResolver(resolveEsbuild) {
+    defaultResolveEsbuild = resolveEsbuild ?? resolveEsbuildReal;
+}
+
+// esbuild's own JS API, at the path `resolveEsbuild(projectDir)` names, or not loaded at all when
+// it names none. Dynamically imported, so a project with nothing to bundle never loads it.
+async function loadEsbuild(projectDir, resolveEsbuild) {
+    const path = resolveEsbuild(projectDir);
+    if (!path) return null;
+    const mod = await import(pathToFileURL(path).href);
+    return mod.default ?? mod;
+}
+
+function place(d) {
+    return d.file ? `${d.file}:${d.line}:${d.column}` : '';
+}
+
+// esbuild's own diagnostics, in function.check's shape (severity, message, file, line, column):
+// the one shape a local build failure and the server's refusal of a source both report through.
+function buildFailedError(name, err) {
+    const diagnostics = (err.errors ?? [{ text: err.message }]).map((m) => ({
+        severity: 'error',
+        message: m.text,
+        file: m.location?.file ?? null,
+        line: m.location?.line ?? null,
+        column: m.location?.column ?? null,
+    }));
+    const message = `function ${name}: ${diagnostics.map((d) => `${place(d)} ${d.message}`.trim()).join('; ')}`;
+    const error = new CliError(message, { code: ErrorCode.INVALID_ARGUMENT });
+    error.diagnostics = diagnostics;
+    return error;
+}
+
+// Each of `inputs` (relative to `cwd`), as its mtime and size, joined: changes whenever a file's
+// content or presence changes, without reading a byte of it. `buildFunction`'s cache key. `newest` is
+// the latest mtime among them.
+function inputsSignature(cwd, inputs) {
+    let newest = 0;
+    const text = inputs.map((rel) => {
+        try {
+            const s = statSync(join(cwd, rel));
+            if (s.mtimeMs > newest) newest = s.mtimeMs;
+            return `${s.mtimeMs}:${s.size}`;
+        } catch {
+            return 'gone';
+        }
+    }).join('|');
+    return { text, newest };
+}
+
+// A file changed this close to the signature can change again with the same size inside one timestamp
+// tick (up to 10 ms on Linux, 1 s or 2 s on older filesystems), so its mtime cannot prove it unchanged:
+// such an entry is never trusted and the next build bundles again (git's racily clean rule).
+const RACY_MS = 2000;
+
+/**
+ * Builds one function's source into what the server runs: the one owner every path that reaches a
+ * tenant calls (module docstring above).
+ *
+ * @param {object} args
+ * @param {string} args.entry the function's source file, absolute
+ * @param {string} args.projectDir the project's root, absolute (where esbuild, and the packages a
+ *   function imports, are resolved from)
+ * @param {Map<string, {signature: string, result: object}>} [args.cache] keyed by `entry`: a build
+ *   whose `inputs` still match their signature on disk is returned as before, nothing read or run
+ *   again (substrio dev's Reloader keeps one across applies; substrio deploy and substrio push pass none, so
+ *   each of their builds is current).
+ * @returns {Promise<{code: string, inputs: string[], bytes: number, warnings: object[], esbuildAvailable: boolean}>}
+ *   `inputs` are relative to `projectDir`. `code` is the entry file's own bytes, unchanged, when
+ *   esbuild does not resolve, or resolves but finds nothing beyond the entry to bundle; otherwise
+ *   the bundle esbuild wrote. `esbuildAvailable` is false only when esbuild does not resolve at
+ *   all, the one local fact a caller has for turning the server's refusal of an unbundled import
+ *   into a hint (deploy.mjs, local/reload.mjs): the server has no field naming that case.
+ * @param {(projectDir: string) => string | null} [args.resolveEsbuild] where esbuild's own path is,
+ *   or null to use none; the real resolution (project first, then the CLI's own install) unless a
+ *   test states its own, directly or through setDefaultEsbuildResolver.
+ */
+export async function buildFunction({ entry, projectDir, cache, resolveEsbuild = defaultResolveEsbuild }) {
+    const name = basenameNoExt(entry);
+    const entryRelative = relative(projectDir, entry);
+
+    const cached = cache?.get(entry);
+    if (cached && cached.trusted && inputsSignature(projectDir, cached.result.inputs).text === cached.signature) {
+        return cached.result;
+    }
+
+    const esbuild = await loadEsbuild(projectDir, resolveEsbuild);
+    let result;
+    if (!esbuild) {
+        const source = readFileSync(entry, 'utf-8');
+        result = { code: source, inputs: [entryRelative], bytes: Buffer.byteLength(source, 'utf-8'), warnings: [], esbuildAvailable: false };
+    } else {
+        let built;
+        try {
+            built = await esbuild.build({
+                entryPoints: [entry],
+                bundle: true,
+                write: false,
+                platform: 'neutral',
+                target: 'es2022',
+                mainFields: ['module', 'main'],
+                conditions: ['import', 'default'],
+                external: EXTERNAL_BUILTINS,
+                metafile: true,
+                legalComments: 'none',
+                minify: false,
+                sourcemap: false,
+                absWorkingDir: projectDir,
+                logLevel: 'silent',
+            });
+        } catch (err) {
+            throw buildFailedError(name, err);
+        }
+        const inputs = Object.keys(built.metafile.inputs).sort();
+        if (inputs.length === 1 && inputs[0] === entryRelative) {
+            // Nothing beyond the entry was pulled in (every import was a builtin, left external, or
+            // there was none): the bytes sent are the entry file's own, not esbuild's rewrite of them.
+            const source = readFileSync(entry, 'utf-8');
+            result = { code: source, inputs, bytes: Buffer.byteLength(source, 'utf-8'), warnings: [], esbuildAvailable: true };
+        } else {
+            const [output] = built.outputFiles;
+            const warnings = built.warnings.map((w) => ({ text: w.text, location: w.location && place(w.location) }));
+            result = { code: output.text, inputs, bytes: Buffer.byteLength(output.text, 'utf-8'), warnings, esbuildAvailable: true };
+        }
+    }
+
+    if (cache) {
+        const signature = inputsSignature(projectDir, result.inputs);
+        cache.set(entry, { signature: signature.text, trusted: signature.newest < Date.now() - RACY_MS, result });
+    }
+    return result;
+}
+
+function basenameNoExt(file) {
+    const base = file.slice(file.lastIndexOf('/') + 1);
+    return base.endsWith('.js') ? base.slice(0, -3) : base;
+}

package/src/identity.mjs

@@ -0,0 +1,248 @@
+/**
+ * Substrio CLI: the roles and sign-in providers an app declares in substrio.toml.
+ *
+ *   [roles.staff]                        role.create / role.update, by name
+ *   description = "Venue staff"
+ *   inherits = ["editor"]                one role (the server's roles inherit one); a name or a one-item array
+ *   grants = ["data.read"]               optional, as role.create takes them
+ *   denies = ["data.delete"]
+ *
+ *   [auth.providers.google]              auth.provider.update, by provider name
+ *   client_id = "1234.apps.googleusercontent.com"
+ *   client_secret_env = "GOOGLE_CLIENT_SECRET"   the NAME of an environment variable; the secret itself
+ *                                                is never written in the file (client_secret is refused)
+ *   redirect_uri, scopes, provider_url, enabled  optional, as auth.provider.update takes them
+ *   client_id_env, redirect_uri_env, scopes_env, provider_url_env
+ *                                        a value that differs between environments (the callback of a
+ *                                        development site and of the live one): the NAME of the variable
+ *                                        that holds it, in place of the value (never both)
+ *   optional = true                      a variable that is not set leaves its key as it is on the server,
+ *                                        and a provider whose client id is not set is skipped; each is
+ *                                        reported (`unset`), never silent. Without it a variable that is
+ *                                        not set refuses the config
+ *
+ * Applied idempotently by `substrio deploy` and `substrio dev`: a role is created, or
+ * updated when it exists; a provider is set (the server keeps a secret that is
+ * not sent again). A role or provider the file does not name is left alone.
+ * The secret is read from the environment when the file is read and goes to
+ * the server only; it is in no output, no error and no fingerprint (a hash
+ * stands for it). Nothing here talks to the server except applyIdentity,
+ * through the `invoke` it is given.
+ */
+
+import { sha256 } from './blobs.mjs';
+import { CliError, ErrorCode } from './cli.mjs';
+
+/** The HTTP status of "that role exists already" (role.create). */
+const EXISTS = 409;
+
+/** What a declared item is. */
+export const IdentityKind = Object.freeze({
+    ROLE: 'role',
+    PROVIDER: 'provider',
+});
+
+/** The keys of one [roles.NAME] table. */
+export const RoleKey = Object.freeze({
+    DESCRIPTION: 'description',
+    INHERITS: 'inherits',
+    GRANTS: 'grants',
+    DENIES: 'denies',
+});
+const ROLE_KEYS = new Set(Object.values(RoleKey));
+
+/** The keys of one [auth.providers.NAME] table. */
+export const ProviderKey = Object.freeze({
+    CLIENT_ID: 'client_id',
+    CLIENT_SECRET_ENV: 'client_secret_env',
+    REDIRECT_URI: 'redirect_uri',
+    SCOPES: 'scopes',
+    PROVIDER_URL: 'provider_url',
+    ENABLED: 'enabled',
+    OPTIONAL: 'optional',
+});
+/** A value key read from the environment: `<key>_env` names the variable that holds it. */
+const FROM_ENV = '_env';
+/** The value keys of a provider, in the order they are sent; the secret is only ever read from the environment. */
+const VALUE_KEYS = Object.freeze([ProviderKey.CLIENT_ID, 'client_secret', ProviderKey.REDIRECT_URI, ProviderKey.SCOPES, ProviderKey.PROVIDER_URL]);
+const PROVIDER_KEYS = new Set([...Object.values(ProviderKey), ...VALUE_KEYS.map((key) => key + FROM_ENV)]);
+
+/** The key a provider's secret must not be written under; the reference replaces it. */
+export const INLINE_SECRET_KEY = 'client_secret';
+/** The [auth] key that holds the provider tables (settings.mjs leaves it out of the auth settings). */
+export const PROVIDERS_KEY = 'providers';
+
+const isTable = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
+const invalid = (message) => new CliError(`substrio.toml: ${message}`, { code: ErrorCode.INVALID_ARGUMENT });
+
+// A table's keys must all be known; names the first that is not and what the table takes.
+function checkKeys(where, table, known) {
+    const unknown = Object.keys(table).find((key) => !known.has(key));
+    if (unknown !== undefined) throw invalid(`${where} has unknown key "${unknown}"; it takes ${[...known].join(', ')}`);
+}
+
+function checkText(where, key, value) {
+    if (typeof value !== 'string' || value === '') throw invalid(`${where}: ${key} must be a non-empty string`);
+    return value;
+}
+
+/**
+ * The value `key` of `table` takes: written in the file, or read from the environment variable its `<key>_env` names
+ * (`env`). undefined when neither is written, null when the variable is not set. Both written, or a variable name
+ * that is not a non-empty string, throws `fail(message)`. The one reading of a `<key>_env` key: the provider tables
+ * here and the settings tables (settings.mjs).
+ */
+export function valueOrEnv(where, table, key, env, fail = invalid) {
+    const named = key + FROM_ENV;
+    if (table[named] === undefined) return table[key];
+    if (table[key] !== undefined) throw fail(`${where}: ${key} and ${named} are both written; write one`);
+    const variable = table[named];
+    if (typeof variable !== 'string' || variable === '') throw fail(`${where}: ${named} must be a non-empty string`);
+    const value = env[variable];
+    return typeof value === 'string' && value !== '' ? value : null;
+}
+
+/** The refusal of a `<key>_env` whose variable is not set; it names the variable, never a value. */
+export const unsetMessage = (where, table, key) => `${where}: ${key}${FROM_ENV} names ${table[key + FROM_ENV]}, which is not set in the environment`;
+
+function checkList(where, key, value) {
+    if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || item === '')) {
+        throw invalid(`${where}: ${key} must be an array of non-empty strings`);
+    }
+    return value;
+}
+
+// The one role a role inherits: a name, or an array of one name (the server's roles inherit one role).
+function inheritedRole(where, value) {
+    if (typeof value === 'string') return checkText(where, RoleKey.INHERITS, value);
+    const names = checkList(where, RoleKey.INHERITS, value);
+    if (names.length > 1) {
+        throw invalid(`${where}: inherits names ${names.length} roles (${names.join(', ')}); a role inherits one role, and the server has no more`);
+    }
+    return names[0] ?? null;
+}
+
+/**
+ * The roles substrio.toml declares, in file order (pure): [{name, description?, inherits?, grants?, denies?}],
+ * only the keys written. Throws a CliError naming the table at fault.
+ */
+export function declaredRoles(config) {
+    const roles = config.roles;
+    if (roles === undefined || roles === null) return [];
+    if (!isTable(roles)) throw invalid('[roles] must be a table of [roles.NAME] tables');
+    return Object.entries(roles).map(([name, table]) => {
+        const where = `[roles.${name}]`;
+        if (!isTable(table)) throw invalid(`${where} must be a table`);
+        checkKeys(where, table, ROLE_KEYS);
+        const role = { name };
+        if (table[RoleKey.DESCRIPTION] !== undefined) {
+            if (typeof table[RoleKey.DESCRIPTION] !== 'string') throw invalid(`${where}: description must be a string`);
+            role.description = table[RoleKey.DESCRIPTION];
+        }
+        if (table[RoleKey.INHERITS] !== undefined) {
+            const inherits = inheritedRole(where, table[RoleKey.INHERITS]);
+            if (inherits !== null) role.inherits = inherits;
+        }
+        if (table[RoleKey.GRANTS] !== undefined) role.grants = checkList(where, RoleKey.GRANTS, table[RoleKey.GRANTS]);
+        if (table[RoleKey.DENIES] !== undefined) role.denies = checkList(where, RoleKey.DENIES, table[RoleKey.DENIES]);
+        return role;
+    });
+}
+
+/**
+ * The providers substrio.toml's [auth.providers.NAME] tables declare, in file order:
+ * [{provider, client_id, client_secret?, redirect_uri?, scopes?, provider_url?, enabled?}], each value written in the
+ * file or read from the environment variable its `<key>_env` names (`env`, process.env by default; the secret only
+ * so). A variable that is not set refuses the config, or, in an `optional` table, leaves its key out and is pushed
+ * onto `unset` as {provider, variables, skipped} (skipped: the client id was one of them, so the provider is not
+ * sent). Throws a CliError naming the table at fault; the secret itself is never in a message.
+ */
+export function declaredProviders(config, env = process.env, unset = []) {
+    const providers = config.auth?.[PROVIDERS_KEY];
+    if (providers === undefined || providers === null) return [];
+    if (!isTable(providers)) throw invalid(`[auth.${PROVIDERS_KEY}] must be a table of [auth.${PROVIDERS_KEY}.NAME] tables`);
+    return Object.entries(providers).flatMap(([name, table]) => {
+        const where = `[auth.${PROVIDERS_KEY}.${name}]`;
+        if (!isTable(table)) throw invalid(`${where} must be a table`);
+        if (Object.hasOwn(table, INLINE_SECRET_KEY)) {
+            throw invalid(`${where}: ${INLINE_SECRET_KEY} must not be written in the file; name the environment variable that holds it with ${ProviderKey.CLIENT_SECRET_ENV}`);
+        }
+        checkKeys(where, table, PROVIDER_KEYS);
+        const optional = table[ProviderKey.OPTIONAL] ?? false;
+        if (typeof optional !== 'boolean') throw invalid(`${where}: optional must be true or false`);
+        const provider = { provider: name };
+        const missing = [];
+        for (const key of VALUE_KEYS) {
+            const value = valueOrEnv(where, table, key, env);
+            if (value === undefined) continue;
+            if (value !== null) provider[key] = checkText(where, key, value);
+            else if (optional) missing.push(table[key + FROM_ENV]);
+            else throw invalid(unsetMessage(where, table, key));
+        }
+        const skipped = provider.client_id === undefined;
+        if (missing.length > 0) unset.push({ provider: name, variables: missing, skipped });
+        if (skipped) {
+            if (missing.length > 0) return [];
+            throw invalid(`${where}: client_id must be a non-empty string`);
+        }
+        if (table[ProviderKey.ENABLED] !== undefined) {
+            if (typeof table[ProviderKey.ENABLED] !== 'boolean') throw invalid(`${where}: enabled must be true or false`);
+            provider.enabled = table[ProviderKey.ENABLED];
+        }
+        return [provider];
+    });
+}
+
+/**
+ * Everything the project declares of roles and providers: {roles, providers, unset}; each empty when none
+ * (`unset`: the variables an optional provider named that were not set, declaredProviders).
+ */
+export function declaredIdentity(config, env = process.env) {
+    const unset = [];
+    return { roles: declaredRoles(config), providers: declaredProviders(config, env, unset), unset };
+}
+
+/** The line that reports one entry of `unset`; it names variables, never a value. */
+export const unsetLine = ({ provider, variables, skipped }) =>
+    `auth provider ${provider}: ${variables.join(', ')} not set${skipped ? ', so it is not set this time' : ', so the server keeps what it has for it'}`;
+
+/** Whether `identity` declares anything. */
+export const hasIdentity = (identity) => identity.roles.length > 0 || identity.providers.length > 0;
+
+/**
+ * What stands for `identity` when it is compared with an earlier one (pure): the declaration with each
+ * secret replaced by its hash, so a changed secret is a change and no secret is held anywhere else.
+ */
+export function identityFingerprint(identity) {
+    return JSON.stringify({
+        roles: identity.roles,
+        providers: identity.providers.map(({ client_secret: secret, ...rest }) => ({ ...rest, secret: secret === undefined ? null : sha256(secret) })),
+    });
+}
+
+/**
+ * Applies `identity` with `invoke(name, input)` (throws a CliError on a failed call): each role created, or
+ * updated when it exists, then each provider set. Idempotent. `done` receives what was applied as it goes, so a
+ * failure keeps its progress. Returns {applied: [{kind, name, created}]}.
+ */
+export async function applyIdentity(invoke, identity, done = []) {
+    for (const role of identity.roles) {
+        let created = true;
+        try {
+            await invoke('role.create', role);
+        } catch (err) {
+            if (!(err instanceof CliError && err.status === EXISTS)) throw err;
+            created = false;
+            await invoke('role.update', role);
+        }
+        done.push({ kind: IdentityKind.ROLE, name: role.name, created });
+    }
+    for (const provider of identity.providers) {
+        await invoke('auth.provider.update', provider);
+        done.push({ kind: IdentityKind.PROVIDER, name: provider.provider, created: false });
+    }
+    return { applied: done };
+}
+
+/** The names of one kind among `applied`, for the line that reports them. */
+export const appliedNames = (applied, kind) => applied.filter((a) => a.kind === kind).map((a) => a.name);