@substrio/cli 0.0.0-stage → 0.1.0

package/src/local/reload.mjs

@@ -0,0 +1,408 @@
+/**
+ * Substrio CLI: hot reload for `substrio dev`. The Reloader keeps what the server
+ * runs (a fingerprint per unit of the app) and, when the watcher says
+ * something changed, brings the server to what is on disk, doing only what
+ * changed:
+ *
+ *   a function's file changed     -> that one function: function.check, then function.deploy
+ *   a function's file removed     -> function.delete
+ *   rules, MCP tools, plans, cron,
+ *   triggers, site, [auth], or a
+ *   function's declaration in
+ *   substrio.toml changed             -> one new version, made live on the app (no backup)
+ *   a collection's file changed   -> that collection made (when missing) and its schema set,
+ *                                    before any version; its rules go with the version's rules
+ *                                    (a file that is removed leaves the collection and its data)
+ *   any apply                     -> every declared collection the server no longer has is made again
+ *                                    (a collection deleted on the server, then a save of anything);
+ *                                    a server declaration (index, expiry, aggregate) the files no
+ *                                    longer name is reported (drift) and left in place, or dropped
+ *                                    with --prune (never a field's undeclared adaptive key)
+ *   [roles.NAME], [auth.providers.NAME]
+ *                                 -> role.create/update and auth.provider.update, before the version
+ *   a build step's input changed  -> that step runs; its outputs come back as changes
+ *   substrio.toml does not parse      -> refused, typed; the server keeps what it runs
+ *
+ * A change the server refuses (a source that does not compile, a version that
+ * fails its check) leaves the server on its last good state, is reported once,
+ * and is tried again only when the file changes again. Saves arriving together
+ * share one apply (a trailing-edge deadline timer, no polling), and one apply
+ * runs at a time: a save during an apply is applied after it, from what is on
+ * disk then. Data is never touched.
+ */
+
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+import { deployFunction, loadApp, makeLive, publishVersion, readApp, runBuild } from '../apply.mjs';
+import { sha256 } from '../blobs.mjs';
+import { CliError, ErrorCode } from '../cli.mjs';
+import { applyCollections, collectionDrift, collectionsMember, missingCollections, pruneDrift } from '../collections.mjs';
+import { applyIdentity, declaredIdentity, identityFingerprint } from '../identity.mjs';
+import { declaredFunction } from '../manifest.mjs';
+import { ChangeKind } from './watch.mjs';
+
+/** What happened, as the Reloader reports it. */
+export const ReloadKind = Object.freeze({
+    FUNCTION_DEPLOYED: 'function-deployed',
+    FUNCTION_REMOVED: 'function-removed',
+    FUNCTION_REFUSED: 'function-refused',   // the source did not check: diagnostics, nothing changed
+    VERSION_APPLIED: 'version-applied',
+    VERSION_REFUSED: 'version-refused',     // the server refused the version: error, nothing changed
+    COLLECTIONS_APPLIED: 'collections-applied', // the declared collections were made and their schemas set
+    COLLECTIONS_REFUSED: 'collections-refused', // the server refused one: error; it is tried again when a file changes
+    COLLECTIONS_DRIFT: 'collections-drift', // the server has an index, expiry or aggregate the files no longer name; left in place
+    COLLECTIONS_PRUNED: 'collections-pruned', // --prune dropped a declaration the files no longer name
+    IDENTITY_APPLIED: 'identity-applied',   // the declared roles and sign-in providers were made or updated
+    IDENTITY_REFUSED: 'identity-refused',   // the server refused one: error; it is tried again with the next apply
+    BUILD_RAN: 'build-ran',
+    BUILD_FAILED: 'build-failed',
+    CONFIG_REFUSED: 'config-refused',       // substrio.toml (or a file it names) is not valid: nothing changed
+    UNCHANGED: 'unchanged',                 // a save that changed nothing the server runs
+    FAILED: 'failed',                       // anything else that went wrong: the error; the reloader goes on
+});
+
+/** The parts of an app that go with a version, not with one function. */
+const UnitKind = Object.freeze({
+    RULES: 'rules',
+    MCP: 'mcp',
+    PLANS: 'plans',
+    SITE: 'site',
+    CRON: 'cron',
+    TRIGGERS: 'triggers',
+    QUEUES: 'queues',
+    SETTINGS: 'settings',
+    COLLECTIONS: 'collections',
+    IDENTITY: 'identity',
+});
+
+const json = (value) => JSON.stringify(value ?? null);
+
+/** What an app runs, as fingerprints: one per function, one per version-wide unit. */
+export function fingerprint(loaded, manifest) {
+    const functions = new Map(loaded.app.functions.map((f) => [f.name, `${f.hash}|${json(declaredFunction(f.name, f.declared))}`]));
+    const units = new Map([
+        [UnitKind.RULES, loaded.app.rules?.hash ?? ''],
+        [UnitKind.MCP, loaded.app.mcp?.hash ?? ''],
+        [UnitKind.PLANS, json(loaded.plans)],
+        [UnitKind.SITE, loaded.app.site ? sha256(json(loaded.app.site.files.map((f) => [f.path, f.hash]))) : ''],
+        [UnitKind.CRON, json(manifest.cron)],
+        [UnitKind.TRIGGERS, json(manifest.triggers)],
+        [UnitKind.QUEUES, json(manifest.queues)],
+        [UnitKind.SETTINGS, json(loaded.settings)],
+        [UnitKind.COLLECTIONS, json(loaded.collections)],
+        [UnitKind.IDENTITY, loaded.identity ? identityFingerprint(loaded.identity) : ''],
+    ]);
+    return { functions, units };
+}
+
+/** The units whose fingerprint differs between two fingerprints. */
+export function changedUnits(before, after) {
+    return [...after.units].filter(([unit, value]) => before.units.get(unit) !== value).map(([unit]) => unit);
+}
+
+/** The functions in `after` that are new or differ from `before`, and the ones only `before` has. */
+export function changedFunctions(before, after) {
+    const changed = [...after.functions].filter(([name, value]) => before.functions.get(name) !== value).map(([name]) => name);
+    const removed = [...before.functions.keys()].filter((name) => !after.functions.has(name));
+    return { changed, removed };
+}
+
+const EMPTY = Object.freeze({ functions: new Map(), units: new Map() });
+
+export class Reloader {
+    #applied = EMPTY;            // what the server runs
+    #refused = new Map();        // function name -> the fingerprint the server refused last (reported once)
+    #collections = new Map();    // collection name -> the fingerprint of what the server was given (its schema and indexes)
+    #dirty = null;               // what changed since the last apply, or null
+    #timer = null;
+    #running = null;             // the apply in flight
+    #closed = false;
+    #idle = [];
+    #app = null;                 // {config, manifest}, the last substrio.toml that was valid
+    #functionCache = new Map();  // entry path -> buildFunction's cache, across every apply
+    #siteDir = null;             // the app's site folder (public_dir), where a function's site_vars are read
+
+    /**
+     * @param {object} deps
+     * @param {string} deps.cwd the app's directory
+     * @param {object} deps.api the tenant's API client (blob uploads)
+     * @param {object} deps.ops the server's operation catalog
+     * @param {(name: string, input: object) => Promise<any>} deps.invoke throws a CliError on a failed call
+     * @param {(event: object) => void} deps.report receives every {kind, ...} event (ReloadKind)
+     * @param {number} [deps.debounceMs] how long saves are gathered
+     * @param {boolean} [deps.build] run build steps (--no-build turns it off)
+     * @param {boolean} [deps.prune] drop a server declaration the files no longer name (--prune)
+     * @param {(steps: object[], cwd: string) => void} [deps.runBuild] the build runner (a test replaces it)
+     * @param {(name: string, absPaths: string[]) => void} [deps.onFunctionInputs] told a function's
+     *   bundled inputs after each build, so the watcher can watch them too
+     * @param {(projectDir: string) => string | null} [deps.resolveEsbuild] forwarded to loadApp, then
+     *   to every buildFunction call (its own default is the real resolution); a test states its own.
+     */
+    constructor({ cwd, api, ops, invoke, report, debounceMs = 50, build = true, prune = false, runBuild: run = runBuild, onFunctionInputs, resolveEsbuild }) {
+        Object.assign(this, { cwd, api, ops, invoke, report, debounceMs, build, prune, onFunctionInputs, resolveEsbuild });
+        this.runBuild = run;
+    }
+
+    /** The first apply: everything on disk, as one version. */
+    async start() {
+        this.#dirty = { config: true, steps: new Set(), all: true };
+        await this.#kick();
+    }
+
+    /** A change from the watcher. */
+    note(change) {
+        if (this.#closed) return;
+        this.#dirty ??= { config: false, steps: new Set(), all: false };
+        if (change.kind === ChangeKind.CONFIG || change.kind === ChangeKind.RESYNC) this.#dirty.config = true;
+        if (change.kind === ChangeKind.RESYNC) this.#dirty.all = true;
+        if (change.kind === ChangeKind.BUILD_INPUT) this.#dirty.steps.add(change.step);
+        if (change.kind === ChangeKind.FUNCTION) {
+            // A function's own change, or one of its bundled inputs', must be tried again even if it
+            // was refused before; its code hash (built from every input) carries the rest.
+            for (const name of change.names ?? [change.name]) this.#refused.delete(name);
+        }
+        clearTimeout(this.#timer);
+        this.#timer = setTimeout(() => { this.#timer = null; void this.#kick(); }, this.debounceMs);
+    }
+
+    /** Resolves once nothing is pending or running. */
+    idle() {
+        if (this.#timer === null && this.#running === null && this.#dirty === null) return Promise.resolve();
+        return new Promise((resolve) => this.#idle.push(resolve));
+    }
+
+    async close() {
+        this.#closed = true;
+        clearTimeout(this.#timer);
+        this.#timer = null;
+        this.#dirty = null;
+        await this.#running;
+        this.#release();
+    }
+
+    #release() {
+        if (this.#timer !== null || this.#running !== null || this.#dirty !== null) return;
+        for (const resolve of this.#idle.splice(0)) resolve();
+    }
+
+    // Starts an apply unless one runs; the one running picks up what arrived meanwhile when it ends. A timer
+    // that fires after the running apply already took its change finds nothing dirty: no apply is started,
+    // and nothing is left marked as running.
+    #kick() {
+        if (this.#running !== null) return this.#running;
+        if (this.#dirty === null || this.#closed) {
+            this.#release();
+            return Promise.resolve();
+        }
+        const run = this.#drain();
+        this.#running = run;
+        // Cleared in a continuation, which always runs after the assignment above: a marker cleared inside
+        // the run itself could be cleared first and then overwritten by a finished promise, and every later
+        // save would wait on it forever.
+        const ended = () => {
+            this.#running = null;
+            this.#release();
+        };
+        run.then(ended, ended);
+        return run;
+    }
+
+    async #drain() {
+        for (; this.#dirty !== null && !this.#closed;) {
+            const dirty = this.#dirty;
+            this.#dirty = null;
+            try {
+                await this.#apply(dirty);
+            } catch (err) {
+                this.report({ kind: ReloadKind.FAILED, error: err });
+            }
+        }
+    }
+
+    async #apply(dirty) {
+        const started = performance.now();
+        const ms = () => Math.round(performance.now() - started);
+
+        // substrio.toml: read strictly; an invalid one keeps the last good app.
+        if (dirty.config || this.#app === null) {
+            try {
+                this.#app = readApp(this.cwd);
+            } catch (err) {
+                this.report({ kind: ReloadKind.CONFIG_REFUSED, error: err, ms: ms() });
+                return;
+            }
+        }
+        const { config, manifest } = this.#app;
+
+        // Build steps whose inputs changed (all of them at the first apply).
+        if (this.build && manifest.build.length > 0) {
+            const steps = dirty.all ? manifest.build.map((_, i) => i) : [...dirty.steps].sort((a, b) => a - b);
+            for (const index of steps) {
+                const step = manifest.build[index];
+                if (step === undefined) continue;
+                const began = performance.now();
+                try {
+                    this.runBuild([step], this.cwd);
+                    this.report({ kind: ReloadKind.BUILD_RAN, step, ms: Math.round(performance.now() - began) });
+                } catch (err) {
+                    this.report({ kind: ReloadKind.BUILD_FAILED, step, error: err, ms: Math.round(performance.now() - began) });
+                    return;
+                }
+            }
+        }
+
+        // Development ships no precompressed sidecars: brotli at its best quality on every save would be slow.
+        const devManifest = manifest.site ? { ...manifest, site: { ...manifest.site, precompress: null } } : manifest;
+        let loaded;
+        try {
+            loaded = { ...await loadApp(config, devManifest, this.cwd, undefined, this.#functionCache, this.resolveEsbuild), identity: declaredIdentity(config) };
+            this.#siteDir = join(this.cwd, config.public_dir || 'public');
+        } catch (err) {
+            this.report({ kind: ReloadKind.CONFIG_REFUSED, error: err, ms: ms() });
+            return;
+        }
+        const now = fingerprint(loaded, devManifest);
+        const units = changedUnits(this.#applied, now);
+        // A declared collection the server no longer has (deleted there) is made again by this apply, whether or
+        // not its file changed.
+        const gone = await missingCollections(this.invoke, loaded.collections);
+        for (const name of gone) this.#collections.delete(name);
+        if (gone.length > 0 && !units.includes(UnitKind.COLLECTIONS)) units.push(UnitKind.COLLECTIONS);
+        const { changed, removed } = changedFunctions(this.#applied, now);
+
+        // A function the server already refused in exactly this form is not tried (or reported) again.
+        const toDeploy = changed.filter((name) => this.#refused.get(name) !== now.functions.get(name));
+        if (units.length === 0 && toDeploy.length === 0 && removed.length === 0) {
+            if (changed.length === 0) this.report({ kind: ReloadKind.UNCHANGED, ms: ms() });
+            return;
+        }
+
+        // A version carries every function; when the change was only collections or identity, applied directly,
+        // the functions that changed are still deployed below.
+        if (units.length > 0 && await this.#applyVersion(loaded, now, units, ms)) return;
+        for (const name of toDeploy) await this.#deploy(loaded, now, name);
+        for (const name of removed) await this.#remove(name);
+    }
+
+    async #deploy(loaded, now, name) {
+        const began = performance.now();
+        const fn = loaded.app.functions.find((f) => f.name === name);
+        this.onFunctionInputs?.(name, fn.inputs.map((p) => join(this.cwd, p)));
+        if (fn.buildError) {
+            // It does not even parse, or needs a package esbuild could not resolve: nothing to send
+            // function.check, so none is sent; the last good deploy (if any) stands, same as a
+            // source function.check itself refuses.
+            this.#refused.set(name, now.functions.get(name));
+            this.report({ kind: ReloadKind.FUNCTION_REFUSED, name, ms: Math.round(performance.now() - began), diagnostics: fn.buildError.diagnostics ?? [] });
+            return;
+        }
+        try {
+            const result = await deployFunction(this.invoke, { name, script: fn.data.toString('utf-8'), declared: fn.declared, siteDir: this.#siteDir });
+            const ms = Math.round(performance.now() - began);
+            if (result.ok) {
+                this.#applied = { functions: new Map(this.#applied.functions).set(name, now.functions.get(name)), units: this.#applied.units };
+                this.#refused.delete(name);
+                this.report({ kind: ReloadKind.FUNCTION_DEPLOYED, name, ms, diagnostics: result.diagnostics });
+            } else {
+                this.#refused.set(name, now.functions.get(name));
+                this.report({ kind: ReloadKind.FUNCTION_REFUSED, name, ms, diagnostics: result.diagnostics });
+            }
+        } catch (err) {
+            this.#refused.set(name, now.functions.get(name));
+            // The server has no field naming "this import needed bundling"; esbuildAvailable (never
+            // asked to bundle this function) plus the generic code it shares with every other way a
+            // source fails to become a handler is the honest best signal, without reading it again.
+            const esbuildHint = fn.esbuildAvailable === false && err instanceof CliError && err.code === ErrorCode.INVALID_ARGUMENT;
+            this.report({ kind: ReloadKind.FUNCTION_REFUSED, name, ms: Math.round(performance.now() - began), error: err, diagnostics: [], esbuildHint });
+        }
+    }
+
+    async #remove(name) {
+        const began = performance.now();
+        try {
+            await this.invoke('function.delete', { name });
+            const functions = new Map(this.#applied.functions);
+            functions.delete(name);
+            this.#applied = { functions, units: this.#applied.units };
+            this.#refused.delete(name);
+            this.report({ kind: ReloadKind.FUNCTION_REMOVED, name, ms: Math.round(performance.now() - began) });
+        } catch (err) {
+            this.report({ kind: ReloadKind.VERSION_REFUSED, error: err, ms: Math.round(performance.now() - began) });
+        }
+    }
+
+    // True when the apply is over (a version was made, or refused, or a collection was refused); false when only
+    // collections and identity were applied and no version was needed.
+    async #applyVersion(loaded, now, changed, ms) {
+        // The collections first: the rules of the version are about them. They are applied to the server directly
+        // (a development server has no sandbox to try them in), and only when some are declared.
+        const collections = changed.includes(UnitKind.COLLECTIONS);
+        const identity = changed.includes(UnitKind.IDENTITY);
+        const units = changed.filter((unit) => unit !== UnitKind.COLLECTIONS && unit !== UnitKind.IDENTITY);
+        if (collections) {
+            // Only the collections the server was not given in this form.
+            const fingerprintOf = (c) => json(collectionsMember([c]));
+            const pending = loaded.collections.filter((c) => this.#collections.get(c.name) !== fingerprintOf(c));
+            const done = [];
+            try {
+                if (pending.length > 0) {
+                    const result = await applyCollections(this.invoke, pending, done);
+                    this.report({ kind: ReloadKind.COLLECTIONS_APPLIED, collections: result.applied, warnings: result.warnings, ms: ms() });
+                }
+                const drift = await collectionDrift(this.invoke, loaded.collections);
+                if (drift.length > 0) {
+                    if (this.prune) {
+                        await pruneDrift(this.invoke, drift);
+                        this.report({ kind: ReloadKind.COLLECTIONS_PRUNED, drift, ms: ms() });
+                    } else {
+                        this.report({ kind: ReloadKind.COLLECTIONS_DRIFT, drift, ms: ms() });
+                    }
+                }
+                this.#applied = {
+                    functions: this.#applied.functions,
+                    units: new Map(this.#applied.units).set(UnitKind.COLLECTIONS, now.units.get(UnitKind.COLLECTIONS)),
+                };
+            } catch (err) {
+                if (!(err instanceof CliError)) throw err;
+                this.report({ kind: ReloadKind.COLLECTIONS_REFUSED, error: err, ms: ms() });
+                return true;
+            } finally {
+                // What was applied stands even when a later collection was refused.
+                for (const { collection } of done) this.#collections.set(collection, fingerprintOf(pending.find((c) => c.name === collection)));
+            }
+        }
+        // The roles and providers: independent of the version, so a refusal is reported and the version goes on.
+        let identityRefused = false;
+        if (identity) {
+            const done = [];
+            try {
+                const result = await applyIdentity(this.invoke, loaded.identity, done);
+                if (result.applied.length > 0) this.report({ kind: ReloadKind.IDENTITY_APPLIED, applied: result.applied, ms: ms() });
+                this.#applied = {
+                    functions: this.#applied.functions,
+                    units: new Map(this.#applied.units).set(UnitKind.IDENTITY, now.units.get(UnitKind.IDENTITY)),
+                };
+            } catch (err) {
+                if (!(err instanceof CliError)) throw err;
+                identityRefused = true;
+                this.report({ kind: ReloadKind.IDENTITY_REFUSED, error: err, applied: done, ms: ms() });
+            }
+        }
+        if (units.length === 0) return false;
+        try {
+            const published = await publishVersion(this.api, this.ops, loaded, this.invoke);
+            await makeLive(this.invoke, published.number, { backup: false });
+            // A refused identity stays unapplied, so the next apply tries it again.
+            this.#applied = identityRefused
+                ? { functions: now.functions, units: new Map(now.units).set(UnitKind.IDENTITY, this.#applied.units.get(UnitKind.IDENTITY)) }
+                : now;
+            this.#refused.clear();
+            this.report({ kind: ReloadKind.VERSION_APPLIED, version: published.number, units, functions: loaded.app.functions.length, ms: ms() });
+        } catch (err) {
+            if (!(err instanceof CliError)) throw err;
+            this.report({ kind: ReloadKind.VERSION_REFUSED, units, error: err, ms: ms() });
+        }
+        return true;
+    }
+}