@substrio/cli 0.0.0-stage → 0.1.1

package/src/local/server.mjs

@@ -0,0 +1,291 @@
+/**
+ * Substrio CLI: the local substrio_server `substrio dev` and `substrio test` run on, or
+ * attach to.
+ *
+ * SPAWN starts a substrio_server of its own on an OS-picked port with a throwaway
+ * data directory (removed on stop, unless the caller keeps it) or one the
+ * caller names (made, owner-only, when it is missing; its database and keys
+ * are reused when they are there), a random master key in a 0600 file, and
+ * learns the port from the server itself: the
+ * server writes it as one integer and a newline to the descriptor named by
+ * `--ready-fd` once every listener is up. The CLI reads that pipe; nothing is
+ * polled. ATTACH uses a server somebody else runs: nothing is started, kept
+ * or removed.
+ */
+
+import { spawn } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
+import { accessSync, constants, createWriteStream, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { delimiter, join } from 'node:path';
+
+import { SubstrioAPI } from '../api.mjs';
+import { serverInvoker } from '../apply.mjs';
+import { CliError, ErrorCode } from '../cli.mjs';
+import { writePrivateFile } from '../config.mjs';
+import { loadOperations } from '../commands/op.mjs';
+
+/** How a local server comes to be. */
+export const ServerTarget = Object.freeze({
+    SPAWN: 'spawn',
+    ATTACH: 'attach',
+});
+
+/** The descriptor of the child the server writes its ready port to (`--ready-fd`). */
+const READY_FD = 3;
+/** The longest a stopping server may take before it is killed. */
+const STOP_DEADLINE_MS = 10_000;
+/** The file name of the server's binary. */
+const BINARY_NAME = 'substrio_server';
+
+/** `build/bin/substrio_server` of the checkout this CLI sits in, the last place looked. */
+const CHECKOUT_BINARY = new URL(`../../../../build/bin/${BINARY_NAME}`, import.meta.url).pathname;
+
+// The key a 0600 file holds, or null when there is no such file or it is empty.
+function readKeyFile(path) {
+    try {
+        return readFileSync(path, 'utf8').trim() || null;
+    } catch (err) {
+        if (err.code === 'ENOENT') return null;
+        throw err;
+    }
+}
+
+const isExecutable = (path) => {
+    try {
+        accessSync(path, constants.X_OK);
+        return true;
+    } catch {
+        return false;
+    }
+};
+
+/**
+ * The substrio_server to run: `explicit` (--server), else SUBSTRIO_SERVER_BIN, else
+ * one on PATH, else the checkout's build. Throws a typed CliError naming every
+ * place looked when there is none.
+ */
+export function resolveServerBinary(explicit, env = process.env) {
+    const named = explicit ?? env.SUBSTRIO_SERVER_BIN;
+    if (named) {
+        if (!isExecutable(named)) {
+            throw new CliError(`The substrio_server ${named} does not exist or is not executable`, { code: ErrorCode.FILE_NOT_FOUND });
+        }
+        return named;
+    }
+    const onPath = (env.PATH ?? '').split(delimiter).filter(Boolean).map((dir) => join(dir, BINARY_NAME));
+    for (const candidate of [...onPath, CHECKOUT_BINARY]) {
+        if (isExecutable(candidate)) return candidate;
+    }
+    throw new CliError(`No substrio_server found: it is not on PATH and not at ${CHECKOUT_BINARY}`, {
+        code: ErrorCode.FILE_NOT_FOUND,
+        hint: 'Build it (cmake --build build --target substrio_server), or give --server PATH or set SUBSTRIO_SERVER_BIN',
+    });
+}
+
+/** The port a server wrote to its ready descriptor: one whole number, once, or a CliError. */
+export function parseReadyPort(text) {
+    // Digits only: Number() would also take a sign, spaces, a hex or an exponent.
+    const digits = text.length > 0 && [...text].every((c) => c >= '0' && c <= '9');
+    const port = digits ? Number(text) : Number.NaN;
+    if (!Number.isInteger(port) || port < 1 || port > 65535) {
+        throw new CliError(`The server's ready line was not a port: ${JSON.stringify(text)}`, { code: ErrorCode.UNKNOWN });
+    }
+    return port;
+}
+
+/**
+ * A running local server: `url`, the master key (`masterKey`, never printed) and
+ * `stop()`.
+ */
+export class LocalServer {
+    #child = null;
+    #dataDir = null;
+    #removeDataDir = false;
+    #stopping = null;
+    #exited;
+    #startupLog = '';
+    #deadlineMs;
+
+    constructor({ target, url, masterKey, child = null, dataDir = null, removeDataDir = false, stopDeadlineMs = STOP_DEADLINE_MS }) {
+        this.#deadlineMs = stopDeadlineMs;
+        this.target = target;
+        this.url = url;
+        this.masterKey = masterKey;
+        this.#child = child;
+        this.#dataDir = dataDir;
+        this.#removeDataDir = removeDataDir;
+        this.#exited = child === null ? Promise.resolve(null) : new Promise((resolve) => {
+            child.once('exit', (code, signal) => resolve({ code, signal }));
+            // A binary that could not be started never exits: its 'error' is its end.
+            child.once('error', () => { if (child.pid === undefined) resolve(null); });
+        });
+    }
+
+    get dataDir() { return this.#dataDir; }
+    get pid() { return this.#child?.pid ?? null; }
+    /** Resolves with {code, signal} when the server process ends (null at once for an attached server). */
+    get exited() { return this.#exited; }
+    get port() { return Number(new URL(this.url).port); }
+
+    /**
+     * Starts a substrio_server: `binary` (resolveServerBinary), on `port` (0: the OS
+     * picks), with data in `dataDir` (made, and removed on stop, when none is
+     * given, and removed on stop unless `keep`; a given one is made (0700) when it is missing, and left as it is on
+     * stop: its master.key, when there, is the server's key again), and its own output appended to `server.log` in it.
+     */
+    static async spawn({ binary, port = 0, dataDir = null, keep = false, extraArgs = [], env = process.env, stopDeadlineMs = STOP_DEADLINE_MS }) {
+        const owned = dataDir === null;
+        const dir = owned ? mkdtempSync(join(tmpdir(), 'substrio-dev-')) : dataDir;
+        if (!owned) mkdirSync(dir, { recursive: true, mode: 0o700 });
+        const keyFile = join(dir, 'master.key');
+        const masterKey = readKeyFile(keyFile) ?? randomBytes(24).toString('base64url');
+        writePrivateFile(keyFile, `${masterKey}\n`);
+        const logFile = join(dir, 'server.log');
+        // Opened now, not by the stream later: a server stopped at once removes the directory before a lazy open would run.
+        const log = createWriteStream(logFile, { fd: openSync(logFile, 'a') });
+        log.on('error', () => {});
+
+        const child = spawn(binary, [
+            '--data', dir, '--port', String(port), '--bind', '127.0.0.1', '--no-metrics',
+            '--master-key-file', keyFile, '--ready-fd', String(READY_FD), '--exit-on-stdin-close', ...extraArgs,
+        ], { stdio: ['pipe', 'pipe', 'pipe', 'pipe'], env });
+        const server = new LocalServer({
+            target: ServerTarget.SPAWN, url: null, masterKey, child, dataDir: dir, removeDataDir: owned && !keep, stopDeadlineMs,
+        });
+        // Last resort when this process dies without stop(): the server must not outlive it.
+        const onExit = () => {
+            if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
+            if (owned && !keep) rmSync(dir, { recursive: true, force: true });
+        };
+        process.once('exit', onExit);
+        child.once('exit', () => process.off('exit', onExit));
+        for (const stream of [child.stdout, child.stderr]) {
+            stream.on('data', (chunk) => {
+                log.write(chunk);
+                if (server.url === null) server.#startupLog += chunk;
+            });
+        }
+        child.once('exit', () => log.end());
+
+        try {
+            const bound = await new Promise((resolve, reject) => {
+                let line = '';
+                child.once('error', (err) => reject(new CliError(`Could not start ${binary}: ${err.message}`, { code: ErrorCode.UNKNOWN })));
+                child.stdio[READY_FD].setEncoding('utf8');
+                child.stdio[READY_FD].on('error', () => {});
+                child.stdio[READY_FD].on('data', (chunk) => {
+                    line += chunk;
+                    if (!line.endsWith('\n')) return;
+                    try {
+                        resolve(parseReadyPort(line.slice(0, -1)));
+                    } catch (err) {
+                        reject(err);
+                    }
+                });
+                // The pipe ends without a port: the server died, or failed, before it was ready.
+                child.stdio[READY_FD].once('end', () => reject(new CliError('The server ended before it was ready', { code: ErrorCode.UNKNOWN })));
+            });
+            server.url = `http://127.0.0.1:${bound}`;
+            server.#startupLog = '';
+            return server;
+        } catch (err) {
+            const output = server.#startupLog.trim();
+            await server.stop();
+            if (output) err.hint = `Server output:\n${output}`;
+            throw err;
+        }
+    }
+
+    /** An attached server: somebody else's, never started, kept or removed here. */
+    static attach({ url, masterKey }) {
+        let base = url;
+        while (base.endsWith('/')) base = base.slice(0, -1);
+        return new LocalServer({ target: ServerTarget.ATTACH, url: base, masterKey });
+    }
+
+    /**
+     * Stops a spawned server (SIGTERM, then SIGKILL if it has not ended by the
+     * deadline) and removes the data directory it owns. Safe to call again.
+     * Resolves with how the process ended ({code, signal}), null when attached.
+     */
+    stop() {
+        this.#stopping ??= this.#stop();
+        return this.#stopping;
+    }
+
+    async #stop() {
+        const child = this.#child;
+        let ended = null;
+        if (child !== null) {
+            if (child.pid === undefined) {
+                ended = await this.#exited;
+            } else if (child.exitCode === null && child.signalCode === null) {
+                const deadline = setTimeout(() => child.kill('SIGKILL'), this.#deadlineMs);
+                child.kill('SIGTERM');
+                ended = await this.#exited;
+                clearTimeout(deadline);
+            } else {
+                ended = await this.#exited;
+            }
+        }
+        if (this.#removeDataDir && this.#dataDir !== null && existsSync(this.#dataDir)) {
+            rmSync(this.#dataDir, { recursive: true, force: true });
+        }
+        return ended;
+    }
+
+    /** An API client on this server as `tenant`, with that tenant's key (or the master key). */
+    api(tenant, key = this.masterKey) {
+        return new SubstrioAPI({ url: this.url, tenant, master_key: key });
+    }
+
+    /** The master-key API, with no tenant: what makes and removes tenants. */
+    admin() {
+        return this.api(null, this.masterKey);
+    }
+
+    /**
+     * Makes tenant `slug` (a test tenant when `test`) with the master key and
+     * returns {slug, key, api, ops, invoke}: the tenant's own client, the
+     * server's operation catalog and a throwing invoke(name, input).
+     */
+    async createTenant(slug, { test = false } = {}) {
+        const admin = this.admin();
+        const adminOps = await loadOperations(admin);
+        const made = await serverInvoker(admin, adminOps)('tenant.create', { slug, ...(test ? { test: true } : {}) });
+        return this.tenant(slug, made.key);
+    }
+
+    /**
+     * The tenant `slug` of a data directory that outlives the server: its key is read from `keyFile` (0600) when
+     * the file is there and the server still takes it, else the tenant is made and its key saved there.
+     * A tenant that exists whose key is gone is the server's own 409.
+     */
+    async ensureTenant(slug, keyFile) {
+        const saved = readKeyFile(keyFile);
+        if (saved !== null) {
+            try {
+                return await this.tenant(slug, saved);
+            } catch (err) {
+                if (!(err instanceof CliError) || ![401, 403, 404].includes(err.status)) throw err;
+            }
+        }
+        const tenant = await this.createTenant(slug);
+        writePrivateFile(keyFile, `${tenant.key}\n`);
+        return tenant;
+    }
+
+    /** The handle of an existing tenant, as the tenant's own key sees it. */
+    async tenant(slug, key) {
+        const api = this.api(slug, key);
+        const ops = await loadOperations(api);
+        return { slug, key, api, ops, invoke: serverInvoker(api, ops) };
+    }
+
+    /** Deletes tenant `slug` with all its data (master key). */
+    async deleteTenant(slug) {
+        const admin = this.admin();
+        await serverInvoker(admin, await loadOperations(admin))('tenant.delete', { slug });
+    }
+}

package/src/local/watch.mjs

@@ -0,0 +1,175 @@
+/**
+ * Substrio CLI: watching the app for `substrio dev`. The OS tells us what changed
+ * (fs.watch, recursive: FSEvents on macOS, inotify on Linux), so nothing is
+ * polled. Each event is classified once, by comparing its path with the
+ * app's own paths (WatchPlan), into a typed Change.
+ */
+
+import { watch } from 'node:fs';
+import { basename, dirname, extname, isAbsolute, relative, resolve, sep } from 'node:path';
+
+import { collectionsDirOf } from '../collections.mjs';
+
+/** What a changed path is to the app. */
+export const ChangeKind = Object.freeze({
+    FUNCTION: 'function',       // a .js file of functions_dir
+    COLLECTION: 'collection',   // a .json file of collections_dir
+    CONFIG: 'config',           // substrio.toml
+    RULES: 'rules',             // rules_file
+    MCP: 'mcp',                 // mcp_file
+    PLANS: 'plans',             // plans_file
+    SITE: 'site',               // a file under a [[site.parts]] from
+    BUILD_INPUT: 'build-input', // a file under a build step's dir (not its outputs)
+    RESYNC: 'resync',           // the OS could not say which file: look at everything
+});
+
+/**
+ * One classified event: its kind, absolute path, and for FUNCTION the function name (`name`, the
+ * first one; `names`, every function this path is an input of: usually one, more when a shared
+ * file is bundled into several; `viaInput` when the path is one of those bundled inputs rather than
+ * the function's own entry file, which the fingerprint the Reloader keeps does not see), for
+ * BUILD_INPUT the step index.
+ */
+export class Change {
+    constructor(kind, path, { name = null, names = null, step = null, viaInput = false } = {}) {
+        this.kind = kind;
+        this.path = path;
+        this.names = names ?? (name !== null ? [name] : null);
+        this.name = this.names?.[0] ?? null;
+        this.step = step;
+        this.viaInput = viaInput;
+    }
+}
+
+/** Directory names never watched for changes: what tools write, not what the app is. */
+const IGNORED_DIRECTORIES = new Set(['node_modules', '.git']);
+/** Suffixes and prefixes of the files editors write beside the one saved. */
+const EDITOR_SUFFIXES = ['~', '.swp', '.swx', '.tmp'];
+const EDITOR_PREFIXES = ['.#', '#'];
+const EDITOR_FILES = new Set(['4913', '.DS_Store']);
+
+/** Whether `name` is an editor's scratch file. */
+export function isEditorTemp(name) {
+    return EDITOR_FILES.has(name) || EDITOR_SUFFIXES.some((s) => name.endsWith(s)) || EDITOR_PREFIXES.some((p) => name.startsWith(p));
+}
+
+/** The path segments of `to` below `from`, or null when it is not below it. */
+function below(from, to) {
+    const rel = relative(from, to);
+    if (rel === '' || isAbsolute(rel)) return null;
+    const segments = rel.split(sep);
+    return segments[0] === '..' ? null : segments;
+}
+
+/**
+ * What the app's paths are, from its config and manifest: classifies a path
+ * once into a Change, or null when it is not the app's.
+ */
+export class WatchPlan {
+    /**
+     * @param {string} root the app's directory
+     * @param {object} config loadConfig's result
+     * @param {object} manifest manifestOf's result
+     * @param {string[]} ignoredDirs absolute directories to leave out (a data directory inside the app)
+     */
+    constructor(root, config, manifest, ignoredDirs = []) {
+        this.root = root;
+        this.config = resolve(root, 'substrio.toml');
+        this.functionsDir = resolve(root, config.functions_dir || 'functions');
+        this.collectionsDir = resolve(root, collectionsDirOf(config));
+        this.single = new Map();
+        if (config.rules_file) this.single.set(resolve(root, config.rules_file), ChangeKind.RULES);
+        if (config.mcp_file) this.single.set(resolve(root, config.mcp_file), ChangeKind.MCP);
+        if (config.plans_file) this.single.set(resolve(root, config.plans_file), ChangeKind.PLANS);
+        this.siteDirs = (manifest.site?.parts ?? []).map((part) => resolve(root, part.from));
+        this.buildDirs = manifest.build.map((step, index) => ({ index, dir: resolve(root, step.dir) }));
+        this.ignoredDirs = ignoredDirs.map((d) => resolve(d));
+        this.inputsByFunction = new Map(); // function name -> Set<absolute path>, its bundle's non-entry inputs
+        this.functionsByInput = new Map(); // absolute path -> Set<function name>, the reverse of the above
+    }
+
+    /**
+     * What `name`'s bundle reads besides its entry file (buildFunction's `inputs`, absolute): a
+     * save to any of them now reclassifies as that function changing, same as saving the entry
+     * file itself. Replaces whatever was recorded for `name` before.
+     */
+    setFunctionInputs(name, absPaths) {
+        const before = this.inputsByFunction.get(name);
+        if (before) {
+            for (const path of before) {
+                const names = this.functionsByInput.get(path);
+                if (names) { names.delete(name); if (names.size === 0) this.functionsByInput.delete(path); }
+            }
+        }
+        const after = new Set(absPaths.filter((p) => p !== resolve(this.functionsDir, `${name}.js`)));
+        this.inputsByFunction.set(name, after);
+        for (const path of after) {
+            let names = this.functionsByInput.get(path);
+            if (!names) this.functionsByInput.set(path, (names = new Set()));
+            names.add(name);
+        }
+    }
+
+    /** The Change an absolute `path` is, or null. */
+    classify(path) {
+        const segments = below(this.root, path);
+        if (segments === null) return null;
+        // A bundle's own inputs, node_modules included, are watched before the usual tool-directory
+        // exclusion: that is exactly where a real npm dependency lives.
+        const inputOf = this.functionsByInput.get(path);
+        if (inputOf !== undefined) return new Change(ChangeKind.FUNCTION, path, { names: [...inputOf], viaInput: true });
+        if (segments.some((s) => IGNORED_DIRECTORIES.has(s))) return null;
+        if (this.ignoredDirs.some((dir) => below(dir, path) !== null)) return null;
+        if (isEditorTemp(basename(path))) return null;
+        if (path === this.config) return new Change(ChangeKind.CONFIG, path);
+        const single = this.single.get(path);
+        if (single !== undefined) return new Change(single, path);
+        // The app's outputs first: a build step's dir often holds them (functions/dist inside functions/).
+        if (dirname(path) === this.functionsDir && path.endsWith('.js')) {
+            return new Change(ChangeKind.FUNCTION, path, { name: basename(path).slice(0, -'.js'.length) });
+        }
+        if (dirname(path) === this.collectionsDir && extname(path) === '.json') {
+            return new Change(ChangeKind.COLLECTION, path, { name: basename(path, '.json') });
+        }
+        if (this.siteDirs.some((dir) => below(dir, path) !== null)) return new Change(ChangeKind.SITE, path);
+        if (below(this.functionsDir, path) !== null) return null;
+        const step = this.buildDirs.find(({ dir }) => below(dir, path) !== null);
+        return step === undefined ? null : new Change(ChangeKind.BUILD_INPUT, path, { step: step.index });
+    }
+}
+
+/**
+ * fs.watch over the app's directory: every event is classified and handed to
+ * `onChange(change)`. close() ends it for good: a start() after it (a session
+ * stopped while it was still starting) watches nothing, so no watcher is left
+ * holding the process open.
+ */
+export class AppWatcher {
+    #watcher = null;
+    #closed = false;
+
+    constructor(plan, onChange) {
+        this.plan = plan;
+        this.onChange = onChange;
+    }
+
+    start() {
+        if (this.#closed) return this;
+        this.#watcher = watch(this.plan.root, { recursive: true, persistent: true }, (eventType, filename) => {
+            // An event with no file name: the OS could not say which, so everything is looked at.
+            if (filename === null || filename === undefined) {
+                this.onChange(new Change(ChangeKind.RESYNC, this.plan.root));
+                return;
+            }
+            const change = this.plan.classify(resolve(this.plan.root, String(filename)));
+            if (change !== null) this.onChange(change);
+        });
+        return this;
+    }
+
+    close() {
+        this.#closed = true;
+        this.#watcher?.close();
+        this.#watcher = null;
+    }
+}