@arc-e-tect/api-only-publisher 0.0.0

package/src/pipeline.js

@@ -0,0 +1,193 @@
+"use strict";
+
+// The build pipeline: stage, substitute, bundle, stamp, lint, distribute.
+
+const fs = require("fs");
+const path = require("path");
+const { execFileSync } = require("child_process");
+
+const { substituteFile } = require("./placeholders");
+const { stampFile } = require("./version");
+const { generateAsyncApi, isAggregate } = require("./aggregate");
+
+class BuildError extends Error {}
+
+function run(command, args, { quiet }) {
+    try {
+        const out = execFileSync(command, args, { encoding: "utf8", stdio: quiet ? "pipe" : "inherit" });
+        return out;
+    } catch (error) {
+        throw new BuildError(
+            `${command} ${args.join(" ")} failed` + (error.stdout ? `\n${error.stdout}` : "") +
+            (error.stderr ? `\n${error.stderr}` : "")
+        );
+    }
+}
+
+/**
+ * Copy the whole source root to the staging directory for one specification
+ * type.
+ *
+ * The *whole* root, not just that type's subtree: the trees $ref each other --
+ * the AsyncAPI event schemas reuse the OpenAPI common schemas -- so a partial
+ * copy breaks those references. Its own staging root per type, so that building
+ * one type never invalidates the other's staged tree.
+ */
+function stage(config, kind, log) {
+    const from = config.sourceRoot();
+    const to = config.stagingRoot(kind);
+    log(`-- Staging ${path.relative(config.root, from)} -> ${path.relative(config.root, to)}`);
+    fs.rmSync(to, { recursive: true, force: true });
+    fs.mkdirSync(to, { recursive: true });
+    fs.cpSync(from, to, { recursive: true });
+    return to;
+}
+
+/**
+ * Substitute placeholders across the staged tree, in place.
+ *
+ * Every YAML file is visited rather than only the bundle roots and info.yaml.
+ * The tool this replaces read only the one file it was handed, which is why the
+ * shared info block had to be preprocessed as a separate up-front step and why
+ * bundles had to $ref a generated merged_info.yaml instead of the file they
+ * meant. Visiting the staged tree removes that special case: a placeholder works
+ * wherever it is written.
+ */
+function substituteTree(config, kind, log) {
+    const stagingRoot = config.stagingRoot(kind);
+    const placeholders = config.defaults.placeholders || {};
+    const strict = placeholders.strict !== false;
+
+    let files = 0;
+    let tokens = 0;
+    const walk = (dir) => {
+        for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) {
+            const full = path.join(dir, entry.name);
+            if (entry.isDirectory()) walk(full);
+            else if (entry.isFile() && /\.ya?ml$/.test(entry.name)) {
+                const result = substituteFile(full, { searchRoot: stagingRoot, strict });
+                if (result.resolved.length > 0) {
+                    files += 1;
+                    tokens += result.resolved.length;
+                }
+            }
+        }
+    };
+    walk(stagingRoot);
+    log(`-- Substituted ${tokens} placeholder(s) across ${files} file(s)`);
+}
+
+function bundle(config, target, kind, outFile, log) {
+    const source = config.bundlePath(target, kind);
+    if (!fs.existsSync(source)) {
+        throw new BuildError(`target '${target}': bundle root not found at ${source}`);
+    }
+    fs.mkdirSync(path.dirname(outFile), { recursive: true });
+    log(`-- Bundling ${path.basename(source)}`);
+    const tool = kind === "openapi" ? config.tool("redocly") : config.tool("asyncapi");
+    run("npx", ["--yes", tool, "bundle", source, "--output", outFile], { quiet: true });
+    if (!fs.existsSync(outFile)) {
+        throw new BuildError(`target '${target}': ${tool} produced no output at ${outFile}`);
+    }
+}
+
+function lint(config, kind, file, log) {
+    const tool = kind === "openapi" ? config.tool("redocly") : config.tool("asyncapi");
+    const args = kind === "openapi"
+        ? ["--yes", tool, "lint"].concat(config.lintConfig("openapi") ? ["--config", config.lintConfig("openapi")] : []).concat([file])
+        : ["--yes", tool, "validate", file];
+    log(`-- Validating ${path.basename(file)}`);
+    run("npx", args, { quiet: true });
+}
+
+/**
+ * Copy a built document to the project that implements the target.
+ *
+ * Transitional. A producer has no business knowing its consumers' directory
+ * layouts; this exists only so that the build stays verifiable against the
+ * recorded fixtures while the Subscriber is not yet in place.
+ */
+function distribute(config, target, kind, builtFile, log) {
+    const destDir = config.destinationDir(target);
+    if (!destDir) return null;
+    if (!fs.existsSync(destDir)) {
+        throw new BuildError(`target '${target}': destination directory does not exist: ${destDir}`);
+    }
+    const destFile = path.join(destDir, config.outputName(kind));
+    fs.copyFileSync(builtFile, destFile);
+    log(`-- Distributed to ${path.relative(config.root, destFile)}`);
+    return destFile;
+}
+
+/**
+ * Stage, substitute and generate, without bundling.
+ *
+ * Everything that reads the fragment graph -- closures, `changed`, `split` --
+ * needs the tree in the state the bundler will see it: substituted, and with
+ * generated aggregate bundle roots present. Doing it in one place is what stops
+ * those commands from quietly disagreeing with `build` about what the library
+ * contains.
+ */
+function prepare(config, { kinds = ["openapi", "asyncapi"], log = () => {} } = {}) {
+    for (const kind of kinds) {
+        const targets = config.targetsFor(kind);
+        if (targets.length === 0) continue;
+        stage(config, kind, log);
+        substituteTree(config, kind, log);
+        for (const target of targets) {
+            if (isAggregate(config, target, kind) && kind === "asyncapi") {
+                generateAsyncApi(config, target, { log });
+            }
+        }
+    }
+}
+
+/**
+ * Build every requested target.
+ *
+ * @returns {Array<{target, kind, file, distributed}>}
+ */
+function build(config, { targets, version, kinds = ["openapi", "asyncapi"], log = () => {} } = {}) {
+    const results = [];
+    for (const kind of kinds) {
+        const all = config.targetsFor(kind).filter((t) => !targets || targets.includes(t));
+        if (all.length === 0) continue;
+
+        log(`=== ${kind} ===`);
+        stage(config, kind, log);
+        substituteTree(config, kind, log);
+
+        for (const target of all) {
+            log(`\n=== ${target} (${kind}) ===`);
+            // An aggregate has no hand-written bundle root; it is synthesised from
+            // its members into the staged tree, so it can never fall behind them.
+            if (isAggregate(config, target, kind)) {
+                if (kind !== "asyncapi") {
+                    throw new BuildError(
+                        `target '${target}': aggregate is only supported for asyncapi; ` +
+                        `an OpenAPI portfolio is a hand-written table of contents of $refs`
+                    );
+                }
+                generateAsyncApi(config, target, { log });
+            }
+            const outFile = path.join(config.distDir(target), config.outputName(kind));
+            bundle(config, target, kind, outFile, log);
+            if (version) {
+                log(`-- Stamping version '${version}'`);
+                stampFile(outFile, version);
+            }
+            lint(config, kind, outFile, log);
+
+            let distributed = null;
+            if (config.isPublished(target)) {
+                distributed = distribute(config, target, kind, outFile, log);
+            } else {
+                log("-- Not distributed (publish: false)");
+            }
+            results.push({ target, kind, file: outFile, distributed });
+        }
+    }
+    return results;
+}
+
+module.exports = { build, prepare, stage, substituteTree, bundle, lint, distribute, BuildError };

package/src/placeholders.js

@@ -0,0 +1,117 @@
+"use strict";
+
+// Placeholder substitution.
+//
+// Absorbed from sedr_utils/openapi/prep_openapi's preprocess_openapi.js, with
+// three defects fixed rather than inherited:
+//
+//   1. Search scope. The original searched for <placeholder>.md recursively
+//      downward from the *input file's own directory*, never from the -d
+//      argument the caller passed. The workaround was to move the Markdown
+//      files next to whatever referenced them, and the gotcha had to be
+//      documented. The search root is now a parameter, defaulting to the source
+//      root the caller actually named.
+//   2. Failure mode. A missing Markdown file produced a warning and the literal
+//      string *MISSING CONTENT* in the output, so a broken document shipped from
+//      a green build. Unresolved placeholders are now an error by default.
+//   3. Token grammar. The pattern \{\{(\w+)\}\} silently excluded '-' and '.'
+//      from placeholder names, so {{status-codes}} was left in the output rather
+//      than reported. The grammar now admits them.
+
+const fs = require("fs");
+const path = require("path");
+
+// {{name}} where name may contain letters, digits, underscore, dash or dot.
+// Deliberately no whitespace tolerance inside the braces: that would make a
+// stray "{{ " in prose look like a placeholder.
+const TOKEN = /\{\{([A-Za-z0-9_.-]+)\}\}/g;
+
+class PlaceholderError extends Error {}
+
+// Depth-first search for `fileName` under `startDir`. Directory entries are
+// sorted so the result cannot depend on filesystem ordering, which would make
+// builds non-reproducible across machines.
+function findFile(startDir, fileName) {
+    let entries;
+    try {
+        entries = fs.readdirSync(startDir, { withFileTypes: true });
+    } catch {
+        return null;
+    }
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+
+    for (const entry of entries) {
+        if (entry.isFile() && entry.name === fileName) return path.join(startDir, entry.name);
+    }
+    for (const entry of entries) {
+        if (entry.isDirectory()) {
+            const found = findFile(path.join(startDir, entry.name), fileName);
+            if (found) return found;
+        }
+    }
+    return null;
+}
+
+// Keep the YAML indentation the placeholder sits at: the first line continues
+// the indentation already present before the token, and every subsequent
+// non-blank line gets that same indent prefixed, so multi-line Markdown stays
+// valid inside an indented YAML scalar.
+function indentReplacement(content, fullString, offset) {
+    const lineStart = fullString.lastIndexOf("\n", offset - 1) + 1;
+    const indent = fullString.slice(lineStart, offset).match(/^\s*/)[0];
+    return content
+        .replace(/\n$/, "")
+        .split("\n")
+        .map((line, i) => (i === 0 || line.length === 0 ? line : indent + line))
+        .join("\n");
+}
+
+/**
+ * Substitute {{token}} placeholders in `text`.
+ *
+ * @param {string} text          the content to substitute into
+ * @param {object} options
+ * @param {string} options.searchRoot  directory to search for <token>.md
+ * @param {boolean} [options.strict]   throw on an unresolved token (default true)
+ * @param {string} [options.describeAs] label used in error messages
+ * @returns {{content: string, resolved: string[], unresolved: string[]}}
+ */
+function substitute(text, { searchRoot, strict = true, describeAs = "input" } = {}) {
+    if (!searchRoot) throw new PlaceholderError("substitute() requires a searchRoot");
+
+    const resolved = [];
+    const unresolved = [];
+
+    const content = text.replace(TOKEN, (match, token, offset, fullString) => {
+        const mdFile = findFile(searchRoot, `${token}.md`);
+        if (!mdFile) {
+            unresolved.push(token);
+            return match; // leave it visible; strict mode turns it into an error
+        }
+        resolved.push(token);
+        return indentReplacement(fs.readFileSync(mdFile, "utf8"), fullString, offset);
+    });
+
+    if (strict && unresolved.length > 0) {
+        throw new PlaceholderError(
+            `${describeAs}: no Markdown file found for ${unresolved.map((t) => `{{${t}}}`).join(", ")}. ` +
+            `Searched for ${unresolved.map((t) => `${t}.md`).join(", ")} under ${searchRoot}.`
+        );
+    }
+
+    return { content, resolved, unresolved };
+}
+
+/**
+ * Substitute a file in place. Unlike the tool this replaces, nothing named
+ * merged_* is left behind: the file is rewritten where it stands, which is safe
+ * because it is always a staged copy.
+ */
+function substituteFile(file, options) {
+    const before = fs.readFileSync(file, "utf8");
+    const result = substitute(before, { ...options, describeAs: file });
+    if (result.content !== before) fs.writeFileSync(file, result.content);
+    return result;
+}
+
+module.exports = { substitute, substituteFile, findFile, PlaceholderError, TOKEN };

package/src/split.js

@@ -0,0 +1,176 @@
+"use strict";
+
+// `split` -- make a fragment library safe to break into separate repositories.
+//
+// The problem it solves is real and easy to miss. A fragment library is one $ref
+// graph, and that graph does not respect the directory boundaries a repository
+// split would follow. In this library the AsyncAPI event schemas reuse
+// `openapi/components/common/schemas/UsernameV1.yaml`, so a username means the
+// same thing over Kafka as over HTTP. Move the two trees into separate
+// repositories as they stand and every one of those references dangles.
+//
+// `split` resolves each part's dependency closure and materialises it whole:
+// each part gets its own subtree *plus* a copy of every foreign file it reaches,
+// so that after the split each side has a complete set of definitions and builds
+// on its own.
+//
+// The copies keep their original path relative to the shared source root, which
+// is what makes this safe: every relative $ref keeps resolving exactly as it did,
+// so nothing has to be rewritten and no reference can be broken by a rewrite
+// getting the depth wrong.
+//
+// What this cannot do is keep the copies in step afterwards. Duplicating a schema
+// is the price of independent repositories, not a way of avoiding it -- see the
+// note written into each part.
+
+const fs = require("fs");
+const path = require("path");
+
+const { forTargets, resolve } = require("./closure");
+
+class SplitError extends Error {}
+
+const NOTE = `= Imported shared fragments
+
+The files under this directory are **copies**, taken from another part of the
+specification library this repository was split out of.
+
+They are here because this repository's own fragments \`$ref\` them: at the point
+of the split, the library was a single \`$ref\` graph that did not follow the
+directory boundary the split was made along. Copying them is what makes this
+repository self-contained and buildable on its own.
+
+They keep the path they had before the split, relative to the old shared source
+root, so every existing relative \`$ref\` still resolves and nothing had to be
+rewritten.
+
+== What this costs
+
+These copies can now drift from the originals, and nothing here will notice.
+A shared schema that two repositories both define is two schemas that happen to
+agree today.
+
+If that matters for a given fragment -- and for something like a username or a
+problem-response shape it usually does -- promote it to a contract of its own,
+published and consumed like any other, rather than copied. That is what the
+API-Only Subscriber is for.
+
+== Imported files
+
+`;
+
+/**
+ * Group targets into parts.
+ *
+ * `kind` splits along the specification types, which is the boundary a
+ * repository split usually follows. `target` gives every target its own
+ * self-contained tree, which is what you want before splitting into
+ * per-service repositories.
+ */
+function partition(config, by) {
+    const parts = new Map();
+
+    if (by === "kind") {
+        for (const kind of ["openapi", "asyncapi"]) {
+            const targets = config.targetsFor(kind);
+            if (targets.length > 0) parts.set(kind, targets.map((t) => ({ target: t, kind })));
+        }
+        return parts;
+    }
+
+    if (by === "target") {
+        for (const target of Object.keys(config.targets)) {
+            const kinds = ["openapi", "asyncapi"].filter((k) => config.targets[target][k]);
+            parts.set(target, kinds.map((k) => ({ target, kind: k })));
+        }
+        return parts;
+    }
+
+    throw new SplitError(`unknown --by '${by}'; expected 'kind' or 'target'`);
+}
+
+/**
+ * Which subtree of the shared source root a file naturally belongs to.
+ *
+ * The first path segment under the root: `openapi/...` or `asyncapi/...`.
+ */
+function homeSubtree(relPath) {
+    return relPath.split("/")[0];
+}
+
+/**
+ * @returns {Array<{part, dir, own: string[], imported: string[]}>}
+ */
+function split(config, { by = "kind", outDir, log = () => {} } = {}) {
+    if (!outDir) throw new SplitError("split requires an output directory");
+
+    const parts = partition(config, by);
+    const results = [];
+
+    for (const [partName, members] of parts) {
+        // Union of every member's closure.
+        const files = new Set();
+        for (const { target, kind } of members) {
+            const entry = config.bundlePath(target, kind);
+            if (!fs.existsSync(entry)) {
+                throw new SplitError(
+                    `target '${target}': ${kind} bundle root not found at ${entry}; build first so the tree is staged`
+                );
+            }
+            for (const file of resolve(entry).files) files.add(file);
+        }
+
+        const partDir = path.join(outDir, partName);
+        fs.rmSync(partDir, { recursive: true, force: true });
+
+        const own = [];
+        const imported = [];
+
+        for (const file of [...files].sort()) {
+            // Every closure file lives under one of the staging roots; map it back
+            // to its path relative to the shared source root.
+            let rel = null;
+            for (const kind of ["openapi", "asyncapi"]) {
+                const candidate = path.relative(config.stagingRoot(kind), file);
+                if (!candidate.startsWith("..") && !path.isAbsolute(candidate)) {
+                    rel = candidate.split(path.sep).join("/");
+                    break;
+                }
+            }
+            if (rel === null) {
+                throw new SplitError(`${file} is outside every staging root; cannot place it in a part`);
+            }
+
+            // Preserving the relative path is the whole trick: the copy sits where
+            // the $refs already expect to find it.
+            const dest = path.join(partDir, rel);
+            fs.mkdirSync(path.dirname(dest), { recursive: true });
+            fs.copyFileSync(file, dest);
+
+            // A file is "imported" when it comes from a subtree this part does not
+            // own. With --by target every part owns both subtrees, so nothing is
+            // imported and the split is clean by construction.
+            const foreign = by === "kind" && homeSubtree(rel) !== config.sources[partName];
+            (foreign ? imported : own).push(rel);
+        }
+
+        if (imported.length > 0) {
+            fs.writeFileSync(
+                path.join(partDir, "IMPORTED.adoc"),
+                NOTE + imported.map((r) => `* \`${r}\`\n`).join("")
+            );
+        }
+
+        log(
+            `${partName.padEnd(14)} ${String(own.length).padStart(3)} own` +
+            (imported.length > 0 ? `, ${imported.length} imported` : "")
+        );
+        for (const rel of imported) log(`               imported: ${rel}`);
+
+        results.push({ part: partName, dir: partDir, own, imported });
+    }
+
+    return results;
+}
+
+module.exports = { split, partition, SplitError };

package/src/version-policy.js

@@ -0,0 +1,67 @@
+"use strict";
+
+// What counts as a pre-release, and what that permits.
+//
+// API-Only design means implementation starts against a contract that is not
+// finished. If the only way to obtain a bundle were a final release, teams would
+// work around the tool by cloning the specification repository -- which is
+// exactly the broad read access that publishing artifacts exists to avoid, with
+// none of the guarantees.
+//
+// So pre-releases are first-class. The hard rule is the other half: a
+// pre-release must never quietly satisfy a production build.
+
+// SemVer, with the pre-release part being what follows a '-' before any '+'.
+const SEMVER = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+([0-9A-Za-z.-]+))?$/;
+
+class VersionError extends Error {}
+
+function parse(version) {
+    const match = SEMVER.exec(String(version));
+    if (!match) {
+        throw new VersionError(
+            `'${version}' is not a semantic version. Contracts are versioned with semver so that ` +
+            `consumers can reason about what a bump means.`
+        );
+    }
+    return {
+        major: Number(match[1]),
+        minor: Number(match[2]),
+        patch: Number(match[3]),
+        prerelease: match[4] || null,
+        build: match[5] || null,
+    };
+}
+
+function isPrerelease(version) {
+    // -SNAPSHOT is not semver-legal as Maven spells it, but it is what Maven
+    // consumers expect, so it is recognised rather than rejected.
+    if (/-SNAPSHOT$/.test(String(version))) return true;
+    return parse(version).prerelease !== null;
+}
+
+/**
+ * The npm dist-tag a version should be published under.
+ *
+ * A pre-release goes to `next`, never `latest`, so that `npm install <pkg>`
+ * cannot pick one up by accident.
+ */
+function npmDistTag(version) {
+    return isPrerelease(version) ? "next" : "latest";
+}
+
+/**
+ * Whether a pre-release identifier is one we recognise, so that a typo in a
+ * release job surfaces immediately instead of publishing something odd.
+ */
+function describe(version) {
+    if (/-SNAPSHOT$/.test(String(version))) {
+        return { prerelease: true, kind: "snapshot" };
+    }
+    const parsed = parse(version);
+    if (!parsed.prerelease) return { prerelease: false, kind: "release" };
+    if (/^rc\.\d+$/.test(parsed.prerelease)) return { prerelease: true, kind: "rc" };
+    return { prerelease: true, kind: "other" };
+}
+
+module.exports = { parse, isPrerelease, npmDistTag, describe, VersionError, SEMVER };

package/src/version.js

@@ -0,0 +1,69 @@
+"use strict";
+
+// Setting info.version on a bundled document.
+//
+// This replaces two different sed expressions, which between them differed per
+// specification type (OpenAPI quoted the version, AsyncAPI did not), did nothing
+// at all when the pattern missed, and could match a `version:` field elsewhere in
+// the document.
+//
+// The field is located structurally and only that one scalar is spliced. It is
+// deliberately not a re-serialisation: re-emitting the document reformats
+// everything around the edit, because the AsyncAPI CLI wraps long descriptions at
+// a width no YAML emitter reproduces. Splicing keeps every other byte exactly as
+// the bundler wrote it.
+
+const fs = require("fs");
+const YAML = require("yaml");
+
+class VersionError extends Error {}
+
+/**
+ * @returns {string} the document with info.version set to `version`
+ */
+function stamp(source, version, describeAs = "document") {
+    let doc;
+    try {
+        doc = YAML.parseDocument(source);
+    } catch (error) {
+        throw new VersionError(`${describeAs} is not valid YAML: ${error.message}`);
+    }
+    if (doc.errors && doc.errors.length > 0) {
+        throw new VersionError(`${describeAs} is not valid YAML: ${doc.errors[0].message}`);
+    }
+
+    if (!doc.get("info", true)) {
+        throw new VersionError(`${describeAs} has no top-level 'info' block to stamp a version into`);
+    }
+
+    const node = doc.getIn(["info", "version"], true);
+    if (!node || !Array.isArray(node.range)) {
+        throw new VersionError(`${describeAs} has an 'info' block with no 'version' key`);
+    }
+
+    const [start, valueEnd] = node.range;
+    const original = source.slice(start, valueEnd);
+    // Keep whatever quoting the bundler chose, so stamping changes the version
+    // and nothing else about the line.
+    const quote = /^['"]/.test(original) ? original[0] : "";
+    return source.slice(0, start) + quote + version + quote + source.slice(valueEnd);
+}
+
+function stampFile(file, version) {
+    const before = fs.readFileSync(file, "utf8");
+    const after = stamp(before, version, file);
+
+    // Prove the edit landed and left the document parseable, rather than
+    // trusting a substitution to have done what it looked like it did.
+    const check = YAML.parse(after);
+    if (!check || !check.info || String(check.info.version) !== String(version)) {
+        throw new VersionError(
+            `${file} still reports info.version ` +
+            `'${check && check.info ? check.info.version : "<none>"}' after stamping '${version}'`
+        );
+    }
+    fs.writeFileSync(file, after);
+    return after;
+}
+
+module.exports = { stamp, stampFile, VersionError };