@substrio/cli 0.0.0-stage → 0.1.0

package/src/apply.mjs

@@ -0,0 +1,204 @@
+/**
+ * Substrio CLI: applying the app. The one owner of what `substrio push`, `substrio dev`
+ * and `substrio test` share: read substrio.toml's manifest, run the build, collect and
+ * hash the app's files, make a version of them and make it live, or deploy one
+ * function on its own.
+ *
+ * Nothing here prints or exits. A failure is a typed CliError, so the push
+ * reports it and exits, and a long-running `substrio dev` reports it and goes on.
+ * The server calls go through an `invoke(name, input)` the caller passes in:
+ * `substrio push` hands in the one that ends the run on a failed call, `substrio dev`
+ * the one that throws (serverInvoker).
+ */
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+import { errorText } from './api.mjs';
+import { DEFAULT_CONCURRENCY, uploadBlobs } from './blobs.mjs';
+import { CliError, ErrorCode } from './cli.mjs';
+import { OpError, callOperation } from './commands/op.mjs';
+import { loadConfig, parseTOML } from './config.mjs';
+import { COLLECTIONS_MEMBER, checkVersionCarriesCollections, versionCollectionSteps } from './collections.mjs';
+import { blobsByHash, collectApp, declaredFunction, declaredLedger, declaredPermissions, manifestOf, versionBody, withSiteVars } from './manifest.mjs';
+import { carrySettings, checkSettings, declaredPlans, declaredSettings } from './settings.mjs';
+
+/** What the server says of a source it checked (function.check): the severity of a diagnostic. */
+export const Severity = Object.freeze({
+    ERROR: 'error',
+    WARNING: 'warning',
+    INFO: 'info',
+});
+
+/**
+ * An `invoke(name, input)` that throws: the server's data on success, else a
+ * CliError holding the server's typed error (its code and HTTP status), and an
+ * OpError of a name or input that does not fit as a CliError of INVALID_ARGUMENT.
+ */
+export function serverInvoker(api, ops) {
+    return async (name, input) => {
+        let result;
+        try {
+            result = await callOperation(api, ops, name, input);
+        } catch (err) {
+            if (err instanceof OpError) throw new CliError(err.message, { code: ErrorCode.INVALID_ARGUMENT });
+            throw err;
+        }
+        if (!result.ok) {
+            throw CliError.fromResponse(`${result.request.method} ${result.request.path}: ${errorText(result.data, result.status)}`,
+                result.data, result.status);
+        }
+        return result.data;
+    };
+}
+
+/** substrio.toml's push tables, checked; throws a CliError naming the table at fault. */
+export function readManifest(config) {
+    try {
+        return manifestOf(config);
+    } catch (err) {
+        throw new CliError(`substrio.toml: ${err.message}`);
+    }
+}
+
+/**
+ * Runs each build step through the shell in its directory, its output on this
+ * process's stdout and stderr. `announce(step)` is called before each. A step
+ * that fails is the project's failure (ErrorCode.UNKNOWN), not a usage error.
+ */
+export function runBuild(steps, cwd, announce = () => {}) {
+    for (const step of steps) {
+        announce(step);
+        const result = spawnSync(step.run, { cwd: resolve(cwd, step.dir), shell: true, stdio: 'inherit' });
+        if (result.error) {
+            throw new CliError(`Build step in ${step.dir} could not start: ${result.error.message}`, { code: ErrorCode.UNKNOWN });
+        }
+        if (result.status !== 0) {
+            throw new CliError(`Build step in ${step.dir} failed (${result.signal || `exit ${result.status}`}): ${step.run}`,
+                { code: ErrorCode.UNKNOWN });
+        }
+    }
+}
+
+/**
+ * The app on disk as one version would hold it: the files collected and
+ * hashed, the settings and plans substrio.toml declares, and the POST
+ * /v1/_versions body that carries them. The declared collections are in
+ * `collections` and not in the body: publishVersion puts them there when the
+ * version is to carry them.
+ * `functionCache` is forwarded to collectApp (substrio dev's Reloader keeps one
+ * across applies; substrio push passes none, so every function is built fresh).
+ * `resolveEsbuild` is forwarded to collectApp, then to every buildFunction call it makes.
+ * Returns {app, settings, plans, collections, body, byHash}; throws a CliError.
+ */
+export async function loadApp(config, manifest, cwd, message, functionCache, resolveEsbuild) {
+    let app;
+    try {
+        app = await collectApp(config, manifest, cwd, functionCache, resolveEsbuild);
+    } catch (err) {
+        throw err instanceof CliError ? err : new CliError(err.message);
+    }
+    let settings;
+    let plans;
+    let permissions;
+    let ledger;
+    try {
+        settings = declaredSettings(config);
+        plans = declaredPlans(config, cwd);
+        permissions = declaredPermissions(config);
+        ledger = declaredLedger(config);
+    } catch (err) {
+        throw new CliError(`substrio.toml: ${err.message}`);
+    }
+    const body = carrySettings(versionBody(app, manifest, message), settings, plans);
+    // [permissions] and [ledger] ride the version: the target declares them first when it
+    // goes live (append only), so a promote switches them with the code that uses them.
+    if (permissions !== null) body.permissions = permissions;
+    if (ledger !== null) body.ledger = ledger;
+    return { app, settings, plans, collections: app.collections, body, byHash: blobsByHash(app) };
+}
+
+/**
+ * Makes a version of a loaded app on the server: the settings' keys checked
+ * against its schemas, version.create, and only the blobs it lacks uploaded.
+ * With `carryCollections` the declared collections go in the version too (its
+ * "collections" member), which a server whose version.create does not take
+ * refuses before anything is sent; without it they are the caller's to apply
+ * (applyCollections).
+ * Returns {result (version.create's answer), number, missing, uploaded}.
+ */
+export async function publishVersion(api, ops, loaded, invoke, concurrency = DEFAULT_CONCURRENCY, { carryCollections = false } = {}) {
+    if (loaded.settings.length > 0) {
+        try {
+            checkSettings(loaded.settings, ops);
+        } catch (err) {
+            throw new CliError(`substrio.toml: ${err.message}`);
+        }
+    }
+    let body = loaded.body;
+    if (carryCollections && loaded.collections.length > 0) {
+        checkVersionCarriesCollections(ops);
+        body = { ...body, [COLLECTIONS_MEMBER]: versionCollectionSteps(ops, loaded.collections) };
+    }
+    const result = await invoke('version.create', body);
+    const missing = result.missing || [];
+    const uploaded = await uploadBlobs(api, ops, missing, loaded.byHash, concurrency);
+    return { result, number: result.version.number, missing, uploaded };
+}
+
+/**
+ * Runs version `number` where it should: in sandbox `sandbox`, or, with none,
+ * on the app itself (`backup` false skips the data backup a promote takes, which
+ * a development server has no use for). `direct` is "replace" or "keep", over
+ * the server's default "refuse", for a function deployed directly (outside a
+ * version) that this would overwrite or delete. Returns version.live.set's answer.
+ */
+export function makeLive(invoke, number, { sandbox = null, backup = true, direct = undefined } = {}) {
+    return invoke('version.live.set', {
+        version: number,
+        ...(sandbox === null ? { backup } : { sandbox }),
+        ...(direct ? { direct } : {}),
+    });
+}
+
+/**
+ * The request function.check and function.deploy take for one function of the app, outside a version:
+ * its site_vars filled from `siteDir` as the server fills them from a version's site (withSiteVars).
+ */
+export function functionRequest({ name, script, declared, siteDir }) {
+    const own = declaredFunction(name, declared);
+    if (own.site_vars && !siteDir) throw new Error(`functions.${name}.site_vars: no site folder to read them from`);
+    return { name, script, ...withSiteVars(own, siteDir) };
+}
+
+/**
+ * Deploys one function on its own: its source is checked first, so a source that
+ * does not compile is refused with the server's diagnostics (severity, message,
+ * file, line, column) and nothing changes. Returns {ok: true, deployed} or
+ * {ok: false, diagnostics}.
+ */
+export async function deployFunction(invoke, fn) {
+    const request = functionRequest(fn);
+    const checked = await invoke('function.check', request);
+    if (!checked.ok) return { ok: false, diagnostics: checked.diagnostics };
+    return { ok: true, deployed: await invoke('function.deploy', request), diagnostics: checked.diagnostics };
+}
+
+/**
+ * substrio.toml and the config it makes (loadConfig), read strictly: a substrio.toml that
+ * does not parse is a typed error here, where loadConfig would warn and carry on
+ * with defaults. Returns {config, manifest}.
+ */
+export function readApp(cwd) {
+    const file = join(cwd, 'substrio.toml');
+    if (existsSync(file)) {
+        try {
+            parseTOML(readFileSync(file, 'utf-8'));
+        } catch (err) {
+            throw new CliError(`substrio.toml: ${err.message}`);
+        }
+    }
+    const config = loadConfig(cwd);
+    return { config, manifest: readManifest(config) };
+}

package/src/blobs.mjs

@@ -0,0 +1,102 @@
+/**
+ * Substrio CLI: content-addressed files, shared by `substrio site deploy` and
+ * `substrio push`.
+ *
+ * A file is named by the SHA-256 of its bytes. The server answers a manifest
+ * with the hashes it lacks, and only those are uploaded, a few at a time.
+ */
+
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+
+import { errorText } from './api.mjs';
+import { CliError, ErrorCode } from './cli.mjs';
+import { callOperation } from './commands/op.mjs';
+
+// Files never published: OS metadata. substrio.toml replaces the list with
+// [site] ignore (an array of file names) or site_ignore (a comma-separated
+// string).
+export const DEFAULT_IGNORE = ['.DS_Store', 'Thumbs.db'];
+export const DEFAULT_CONCURRENCY = 8;
+
+export function formatSize(bytes) {
+    if (bytes < 1024) return `${bytes} B`;
+    if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+export function sha256(data) {
+    return createHash('sha256').update(data).digest('hex');
+}
+
+// The file names a site leaves out, from substrio.toml.
+export function ignoredNames(config) {
+    const listed = config.site?.ignore;
+    if (Array.isArray(listed)) return new Set(listed.map(String));
+    const configured = config.site_ignore;
+    if (typeof configured === 'string') {
+        return new Set(configured.split(',').map((s) => s.trim()).filter(Boolean));
+    }
+    return new Set(DEFAULT_IGNORE);
+}
+
+// One file on disk as { abs, size, hash }.
+export function hashFile(abs) {
+    const data = readFileSync(abs);
+    return { abs, size: data.length, hash: sha256(data) };
+}
+
+// Every file under root, as { path (posix, relative), abs, size, hash }.
+export function collect(root, ignore) {
+    const files = [];
+    const walk = (dir) => {
+        for (const entry of readdirSync(dir, { withFileTypes: true })) {
+            if (ignore.has(entry.name)) continue;
+            const abs = join(dir, entry.name);
+            if (entry.isDirectory()) walk(abs);
+            else if (entry.isFile()) {
+                files.push({ path: relative(root, abs).split(sep).join('/'), ...hashFile(abs) });
+            }
+        }
+    };
+    walk(root);
+    return files;
+}
+
+/**
+ * Uploads the blobs the server lacks, `concurrency` at a time, each by the
+ * registry's site.blob.upload (its bytes the body). `ops` is the server's
+ * catalog (loadOperations).
+ * @param {import('./api.mjs').SubstrioAPI} api
+ * @param {object[]} ops
+ * @param {Iterable<string>} missing  hashes the server asked for
+ * @param {Map<string, {abs?: string, data?: Buffer, size: number, path?: string}>} byHash
+ * @returns {Promise<{count: number, bytes: number}>}
+ * Throws on the first failed upload (a CliError with the server's error) or
+ * a hash the manifest did not send.
+ */
+export async function uploadBlobs(api, ops, missing, byHash, concurrency = DEFAULT_CONCURRENCY) {
+    const queue = [...missing];
+    let bytes = 0;
+    let count = 0;
+    const worker = async () => {
+        for (let hash = queue.shift(); hash; hash = queue.shift()) {
+            const file = byHash.get(hash);
+            if (!file) throw new CliError(`The server asked for ${hash}, which this upload does not hold`, { code: ErrorCode.UNKNOWN });
+            // A file made in memory (a precompressed sidecar) carries its bytes.
+            const result = await callOperation(api, ops, 'site.blob.upload', { hash }, 'PUT', {
+                body: file.data ?? readFileSync(file.abs),
+                headers: { 'Content-Type': 'application/octet-stream' },
+            });
+            if (!result.ok) {
+                throw CliError.fromResponse(`Upload of ${file.path || file.abs} failed: ${errorText(result.data, result.status)}`,
+                    result.data, result.status);
+            }
+            bytes += file.size;
+            count++;
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), queue.length) }, worker));
+    return { count, bytes };
+}