@shortlink-org/portolan 0.1.0

package/scripts/schema.mjs

@@ -0,0 +1,357 @@
+// Composes the JSON Schema for portolan.json out of what the plugins say about
+// themselves.
+//
+//   node scripts/schema.mjs            write schema/portolan.schema.json
+//   node scripts/schema.mjs --check    fail if the schema on disk is not what
+//                                      the plugins describe now
+//
+// The manifest tells each plugin the things its source cannot carry - which
+// bounded context a Go module belongs to, whose proto package a client calls.
+// Only the plugin knows what it can be told, so it is asked, and the answers
+// are composed into one document keyed by the name the manifest declares each
+// plugin under. An editor reads it while the file is being written; `gen` reads
+// it before running anything.
+//
+// The result is committed, which is what keeps `gen` from having to ask four
+// plugins the same question on every run. `--check` in CI is what keeps the
+// committed copy honest.
+
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+
+import { describePlugin } from "./plugin-host.mjs";
+
+const OUT = "schema/portolan.schema.json";
+
+const check = process.argv.includes("--check");
+const manifest = JSON.parse(readFileSync("portolan.json", "utf8"));
+
+const described = new Map();
+for (const plugin of manifest.plugins ?? []) {
+  let descriptor;
+  try {
+    descriptor = await describePlugin(plugin);
+  } catch (cause) {
+    fail(`plugin ${plugin.name} could not be asked what it is: ${cause.message}`);
+  }
+
+  if (!descriptor) {
+    // A plugin from before this part of the protocol existed. Naming it in the
+    // manifest is not an error; it just means its options are the one thing
+    // here nothing can check.
+    console.warn(
+      `  ${plugin.name} does not describe itself, so its options are not checked`,
+    );
+  }
+
+  described.set(plugin.name, descriptor);
+}
+
+const composed = `${JSON.stringify(compose(), null, 2)}\n`;
+
+let current = null;
+try {
+  current = readFileSync(OUT, "utf8");
+} catch {
+  // Absent, which is the same as out of date.
+}
+
+if (current === composed) {
+  console.log(`${OUT}: up to date`);
+} else if (check) {
+  console.error(
+    `${OUT} is ${current === null ? "missing" : "out of date"}. Run \`npm run schema\`.`,
+  );
+  process.exit(1);
+} else {
+  mkdirSync("schema", { recursive: true });
+  writeFileSync(OUT, composed);
+  console.log(`${OUT}: written from ${described.size} plugin${described.size === 1 ? "" : "s"}`);
+}
+
+function compose() {
+  const defs = {};
+  for (const [name, descriptor] of described) {
+    if (descriptor?.options) defs[optionsKey(name)] = descriptor.options;
+  }
+
+  return {
+    $schema: "https://json-schema.org/draft/2020-12/schema",
+    $id: "https://github.com/shortlink-org/portolan/schema/portolan.schema.json",
+    title: "portolan manifest",
+    description:
+      "Where the facts come from, what produces them, and what turns them into something to read. Generated by scripts/schema.mjs from the plugins this manifest declares; run `npm run schema` after changing them.",
+    type: "object",
+    additionalProperties: false,
+    required: ["sources"],
+    properties: {
+      $schema: { type: "string" },
+      $comment: { type: "string" },
+      sources: {
+        type: "array",
+        minItems: 1,
+        items: { type: "string" },
+        description:
+          "Globs of the catalog fragments that make up the estate. They are merged, then validated as a union, so a fragment naming a peer it does not own is normal rather than broken.",
+      },
+      defaultCatalog: {
+        type: "string",
+        minLength: 1,
+        pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+        description: "Catalog profile selected when the URL does not name one.",
+      },
+      catalogs: {
+        type: "array",
+        minItems: 1,
+        items: { $ref: "#/$defs/catalogProfile" },
+        description:
+          "Independent presentations of the available fragments. Each profile is merged, enriched and validated on its own.",
+      },
+      projects: {
+        type: "array",
+        items: { $ref: "#/$defs/project" },
+        description:
+          "The source projects that make up the estate. A project gives repeated pipeline inputs one name for the generated site's Settings page; estate-wide inputs such as flows need no project.",
+      },
+      plugins: {
+        type: "array",
+        items: { $ref: "#/$defs/plugin" },
+        description: "How to run each plugin. A step names one by the name declared here.",
+      },
+      extract: {
+        type: "array",
+        items: step("extract"),
+        description: "Steps that read source and write catalog fragments. They run first, before there is a catalog.",
+      },
+      verify: {
+        type: "array",
+        items: step("verify"),
+        description:
+          "Steps that read something observed - traces, a test's record - and answer with a fragment about the catalog somebody else declared.",
+      },
+      generate: {
+        type: "array",
+        items: step("generate"),
+        description: "Steps that read the merged catalog and write something to read.",
+      },
+    },
+    $defs: {
+      catalogProfile: {
+        type: "object",
+        additionalProperties: false,
+        required: ["id", "title", "sources", "contexts", "projects"],
+        properties: {
+          id: {
+            type: "string",
+            minLength: 1,
+            pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+            description: "Stable profile id used in URLs and generator steps.",
+          },
+          title: { type: "string", minLength: 1 },
+          sources: {
+            type: "array",
+            minItems: 1,
+            items: { type: "string" },
+            description: "Catalog fragment globs merged for this profile.",
+          },
+          contexts: {
+            type: "array",
+            minItems: 1,
+            items: { type: "string", minLength: 1 },
+            description: "Top-level catalog groups visible in this profile.",
+          },
+          projects: {
+            type: "array",
+            items: { type: "string", minLength: 1 },
+            description: "Project ids belonging to this profile.",
+          },
+        },
+      },
+      project: {
+        type: "object",
+        additionalProperties: false,
+        required: ["id", "name", "root"],
+        properties: {
+          id: {
+            type: "string",
+            minLength: 1,
+            pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+            description: "Stable id used to associate pipeline inputs with this project.",
+          },
+          name: {
+            type: "string",
+            minLength: 1,
+            description: "Human-readable project name shown in the generated site.",
+          },
+          root: {
+            type: "string",
+            minLength: 1,
+            description: "Repository-relative root. A pipeline input at or below this path belongs to the project.",
+          },
+          context: {
+            type: "string",
+            minLength: 1,
+            description: "Legacy alias for group. Kept so existing manifests remain valid.",
+          },
+          service: {
+            type: "string",
+            minLength: 1,
+            description: "Legacy alias for component. Kept so existing manifests remain valid.",
+          },
+          group: {
+            type: "string",
+            minLength: 1,
+            description: "Neutral top-level group this project contributes to.",
+          },
+          component: {
+            type: "string",
+            minLength: 1,
+            description: "Component slug inside the group.",
+          },
+          groupKind: {
+            enum: ["bounded-context", "system", "product", "team", "namespace"],
+            description: "Semantic role of the project's top-level group.",
+          },
+          componentKind: {
+            enum: ["service", "application", "webapp", "worker", "job", "function", "cli", "library", "data-pipeline"],
+            description: "Runtime or code role of the project component.",
+          },
+          repository: {
+            type: "string",
+            minLength: 1,
+            description: "Repository page or clone URL, when the project comes from another repository.",
+          },
+        },
+      },
+      plugin: {
+        type: "object",
+        additionalProperties: false,
+        required: ["name"],
+        oneOf: [{ required: ["wasm"] }, { required: ["process"] }],
+        properties: {
+          name: {
+            type: "string",
+            description: "What the steps below call this plugin. It need not be the plugin's own name.",
+          },
+          wasm: {
+            type: "object",
+            additionalProperties: false,
+            required: ["url"],
+            description:
+              "The default. The module gets no filesystem, no network and no environment, so a plugin from somebody else's repository can be run over your source tree without reading it.",
+            properties: {
+              url: {
+                type: "string",
+                pattern: "^(https://|file://)",
+                description:
+                  "https:// to download, or file:// for a path beside the manifest, as in file://plugins/gen-markdown.wasm.",
+              },
+              sha256: {
+                type: "string",
+                pattern: "^[0-9a-f]{64}$",
+                description:
+                  "Checksum of the module. Required for a download; optional for a local file, which it does not protect.",
+              },
+            },
+          },
+          process: {
+            type: "object",
+            additionalProperties: false,
+            required: ["command"],
+            description:
+              "The escape hatch for a plugin that needs a toolchain. Same protocol, none of the sandbox, which is the trade being made.",
+            properties: {
+              command: {
+                type: "string",
+                minLength: 1,
+                description: "The executable to run. It is passed directly to the OS, never through a shell.",
+              },
+              args: {
+                type: "array",
+                items: { type: "string" },
+                description: "Arguments passed to the executable without shell parsing.",
+              },
+            },
+          },
+        },
+      },
+      ...defs,
+    },
+  };
+}
+
+/** The schema for one step of a phase, with the options of each plugin that belongs in it. */
+function step(phase) {
+  const names = [...described]
+    .filter(([, descriptor]) => !descriptor || (descriptor.phases ?? []).includes(phase))
+    .map(([name]) => name);
+
+  const reads = phase !== "generate";
+
+  return {
+    type: "object",
+    additionalProperties: false,
+    required: reads ? ["plugin", "in", "out"] : ["plugin", "out"],
+    properties: {
+      plugin: {
+        // The enum is the declared names, not the plugins' own, and it leaves
+        // out the ones that do not belong in this phase: an extractor offered
+        // where a renderer goes is a step that would fail with nothing to read.
+        enum: names,
+        description: "A plugin declared above, one that belongs in this phase.",
+      },
+      ...(!reads
+        ? {
+            catalog: {
+              type: "string",
+              minLength: 1,
+              description: "Catalog profile supplied to this generator.",
+            },
+          }
+        : {}),
+      ...(reads
+        ? {
+            in: {
+              type: "string",
+              description: "The directory to read, relative to the repository.",
+            },
+          }
+        : {}),
+      out: {
+        type: "string",
+        description:
+          "Where what comes back is written. A plugin names files; nothing it names is allowed to land outside this directory.",
+      },
+      options: {
+        type: "object",
+        description: "Passed to the plugin unread. What it accepts is described below.",
+      },
+    },
+    // One branch per plugin, so the options offered are the ones that plugin
+    // takes. A plugin that does not describe itself has no branch, and its
+    // options go unchecked rather than being refused.
+    allOf: names
+      .filter((name) => described.get(name)?.options)
+      .map((name) => ({
+        if: { required: ["plugin"], properties: { plugin: { const: name } } },
+        then: {
+          properties: {
+            options: { $ref: `#/$defs/${pointer(optionsKey(name))}` },
+          },
+        },
+      })),
+  };
+}
+
+function optionsKey(name) {
+  return `options.${name}`;
+}
+
+/** Escapes a $defs key for the JSON pointer that refers to it. */
+function pointer(key) {
+  return encodeURIComponent(key.replaceAll("~", "~0").replaceAll("/", "~1"));
+}
+
+function fail(message) {
+  console.error(`portolan schema: ${message}`);
+  process.exit(1);
+}

package/scripts/site-docs.mjs

@@ -0,0 +1,295 @@
+// Puts the generated documentation into the built site.
+//
+//   node scripts/site-docs.mjs            after `vite build`
+//
+// The markdown generator writes docs/ with relative links and knows nothing of
+// where the site is served from, which is right for a directory that is read
+// on a forge or in an editor. llmstxt.org, though, asks for /llms.txt at the
+// root of the site, and a link from there has to reach the page it names. So
+// the built site gets three things:
+//
+//   dist/docs/**          the generated directory, as it is
+//   dist/llms.txt         the index, its relative links pointed into docs/
+//   dist/llms-full.txt    every page in one file, copied as it is: its links
+//                         are relative to each page's own directory already,
+//                         which the file itself says
+//
+// The docs are mounted under /docs/ and not at the root because the root is
+// the app's: /flows, /adrs and every context id are routes, and a markdown
+// tree beside them would shadow whichever it shares a name with.
+//
+// The dev server places nothing, so siteDocsPlugin answers the same three
+// paths from the generated directory as it is on disk, and a page edited or
+// regenerated shows on the next request.
+
+import { cpSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { extname, isAbsolute, join, normalize, posix, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { loadManifest } from "./manifest.mjs";
+
+const LLMS = "llms.txt";
+const LLMS_FULL = "llms-full.txt";
+
+/** Where the generated directory lives inside the site, with its slash. */
+export const DOCS_MOUNT = "docs/";
+
+/** The file list the generator keeps for itself; not a page. */
+const GENERATOR_MANIFEST = ".portolan-manifest";
+
+/**
+ * Prefix every relative markdown link so the text still resolves once the
+ * file is moved up out of its directory. Absolute URLs, root-relative paths
+ * and same-page anchors are left as they are: none of them was relative to
+ * the file's directory to begin with.
+ *
+ * @param {string} text
+ * @param {string} prefix
+ */
+export function mountLinks(text, prefix) {
+  return text.replace(
+    /\]\(([^)\s]+)((?:\s+"[^"]*")?)\)/g,
+    (whole, target, title) =>
+      isRelative(target) ? `](${prefix}${target}${title})` : whole,
+  );
+}
+
+function isRelative(target) {
+  if (target.startsWith("#") || target.startsWith("/")) return false;
+  // A scheme: http:, https:, mailto:. A Windows drive letter is not a link a
+  // generated page would carry, so one letter is treated as a scheme too.
+  return !/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(target);
+}
+
+/**
+ * The generated directory named by the manifest's markdown step, or null when
+ * the manifest declares no such step.
+ *
+ * @param {{generate?: Array<{plugin: string, out: string}>}} manifest
+ */
+export function docsDirectory(manifest) {
+  const step = (manifest.generate ?? []).find((item) => item.plugin === "markdown");
+  return step?.out ?? null;
+}
+
+/**
+ * Copy the docs and place the llms files. Returns the paths written, relative
+ * to dist, so the caller can say what happened.
+ *
+ * @param {{manifest: object, dist: string}} input
+ * @returns {string[]}
+ */
+export function siteDocs({ manifest, dist }) {
+  const docs = docsDirectory(manifest);
+  const written = [];
+
+  if (docs !== null && existsSync(docs)) {
+    const mount = DOCS_MOUNT.replace(/\/$/, "");
+    cpSync(docs, join(dist, mount), {
+      recursive: true,
+      filter: (source) => !source.endsWith(`/${GENERATOR_MANIFEST}`),
+    });
+    written.push(`${mount}/`);
+
+    const index = join(docs, LLMS);
+    if (existsSync(index)) {
+      writeFileSync(join(dist, LLMS), mountLinks(readFileSync(index, "utf8"), DOCS_MOUNT));
+      written.push(LLMS);
+    }
+
+    const full = join(docs, LLMS_FULL);
+    if (existsSync(full)) {
+      cpSync(full, join(dist, LLMS_FULL));
+      written.push(LLMS_FULL);
+    }
+  }
+
+  for (const step of manifest.generate ?? []) {
+    const mount = generatedMount(step);
+    if (mount === null || !existsSync(mount)) continue;
+    cpSync(mount, join(dist, mount), {
+      recursive: true,
+      filter: (source) => !source.endsWith(`/${GENERATOR_MANIFEST}`),
+    });
+    written.push(`${mount}/`);
+  }
+
+  return written;
+}
+
+/** A generated output that can safely be mounted below the built site's root. */
+export function generatedMount(step) {
+  if (
+    step.plugin === "markdown" ||
+    typeof step.out !== "string" ||
+    step.out === "" ||
+    isAbsolute(step.out) ||
+    step.out.includes("\\")
+  ) return null;
+  const mount = posix.normalize(step.out).replace(/^\.\//, "").replace(/\/$/, "");
+  if (mount === "." || mount === ".." || mount.startsWith("../")) return null;
+  return mount;
+}
+
+/**
+ * What one request under the site's base asks for, or null when it is not
+ * one of the files the build would place. Pure, so it can be tested without a
+ * server: the answer says which file to read and how, not what is in it.
+ *
+ * @param {string} pathname  the request path, base included
+ * @param {string} base      the site's base, "/" or "/portolan/"
+ * @returns {{kind: "index"} | {kind: "full"} | {kind: "redirect", to: string}
+ *   | {kind: "page", path: string} | null}
+ */
+export function docsRequest(pathname, base = "/") {
+  const root = base.endsWith("/") ? base : `${base}/`;
+  if (!pathname.startsWith(root)) return null;
+  const rest = pathname.slice(root.length);
+
+  if (rest === LLMS) return { kind: "index" };
+  if (rest === LLMS_FULL) return { kind: "full" };
+  if (rest !== DOCS_MOUNT.slice(0, -1) && !rest.startsWith(DOCS_MOUNT)) return null;
+
+  // "docs" without its slash: send the browser to the slash so the page's own
+  // relative links resolve against the directory, as they do on the forge.
+  if (rest === DOCS_MOUNT.slice(0, -1)) return { kind: "redirect", to: `${root}${DOCS_MOUNT}` };
+
+  let inside;
+  try {
+    inside = decodeURIComponent(rest.slice(DOCS_MOUNT.length));
+  } catch {
+    return null;
+  }
+  const segments = inside.split("/");
+  if (segments.some((segment) => segment === ".." || segment.startsWith(".") && segment !== "")) return null;
+  if (inside.includes("\\")) return null;
+  if (inside === "" || inside.endsWith("/")) inside += "README.md";
+
+  return { kind: "page", path: posix.normalize(inside) };
+}
+
+const TYPES = {
+  ".md": "text/markdown; charset=utf-8",
+  ".txt": "text/plain; charset=utf-8",
+  ".json": "application/json; charset=utf-8",
+  ".yaml": "application/yaml; charset=utf-8",
+  ".yml": "application/yaml; charset=utf-8",
+  ".mmd": "text/plain; charset=utf-8",
+  ".svg": "image/svg+xml",
+  ".png": "image/png",
+};
+
+/**
+ * Read one answer of docsRequest from the generated directory, or null when
+ * the file is not there. Returned with its content type so the server needs
+ * to know nothing about markdown.
+ *
+ * @param {string} docs  the generated directory
+ * @param {ReturnType<typeof docsRequest>} request
+ * @returns {{type: string, body: Buffer | string} | null}
+ */
+export function readDocsRequest(docs, request) {
+  if (request === null || request.kind === "redirect") return null;
+  const root = resolve(docs);
+
+  if (request.kind === "index") {
+    const file = join(root, LLMS);
+    if (!isFile(file)) return null;
+    return { type: TYPES[".txt"], body: mountLinks(readFileSync(file, "utf8"), DOCS_MOUNT) };
+  }
+  if (request.kind === "full") {
+    const file = join(root, LLMS_FULL);
+    if (!isFile(file)) return null;
+    return { type: TYPES[".txt"], body: readFileSync(file) };
+  }
+
+  const file = resolve(root, normalize(request.path));
+  if (!file.startsWith(root + sep) || !isFile(file)) return null;
+  let target = file;
+  // "docs/auth" resolving to a directory is answered by its index too, since a
+  // link written as a bare directory is a link somebody expects to work.
+  if (statSync(file).isDirectory()) {
+    target = join(file, "README.md");
+    if (!isFile(target)) return null;
+  }
+  return { type: TYPES[extname(target)] ?? "application/octet-stream", body: readFileSync(target) };
+}
+
+function isFile(path) {
+  return existsSync(path) && !path.endsWith(`${sep}${GENERATOR_MANIFEST}`);
+}
+
+/** A safe request for a non-markdown generator output in development. */
+export function generatedRequest(pathname, manifest, base = "/") {
+  const root = base.endsWith("/") ? base : `${base}/`;
+  if (!pathname.startsWith(root)) return null;
+  const rest = pathname.slice(root.length);
+  for (const step of manifest.generate ?? []) {
+    const mount = generatedMount(step);
+    if (mount === null) continue;
+    if (rest !== mount && !rest.startsWith(`${mount}/`)) continue;
+    if (rest === mount) return { kind: "redirect", to: `${root}${mount}/` };
+    let inside;
+    try { inside = decodeURIComponent(rest.slice(mount.length + 1)); } catch { return null; }
+    if (inside.includes("\\") || inside.split("/").some((part) => part === ".." || part.startsWith("."))) return null;
+    if (inside === "" || inside.endsWith("/")) inside += "README.md";
+    return { kind: "artifact", path: posix.join(mount, inside) };
+  }
+  return null;
+}
+
+export function readGeneratedRequest(workspace, request) {
+  if (request === null || request.kind !== "artifact") return null;
+  const root = resolve(workspace);
+  const file = resolve(root, normalize(request.path));
+  if (!file.startsWith(root + sep) || !isFile(file)) return null;
+  return { type: TYPES[extname(file)] ?? "application/octet-stream", body: readFileSync(file) };
+}
+
+/**
+ * The dev server's side of the build step: llms.txt, llms-full.txt and docs/
+ * under the site's base, answered from the workspace as it is now. Vite's
+ * own fallback would otherwise hand every one of them the app's index.html.
+ */
+export function siteDocsPlugin(workspace = process.cwd()) {
+  return {
+    name: "portolan-site-docs",
+    apply: "serve",
+    configureServer(server) {
+      server.middlewares.use((req, res, next) => {
+        if (req.method !== "GET" && req.method !== "HEAD") return next();
+        const url = new URL(req.url ?? "/", "http://localhost");
+        let manifest;
+        try {
+          manifest = JSON.parse(readFileSync(join(workspace, "portolan.json"), "utf8"));
+        } catch {
+          return next();
+        }
+        const request = docsRequest(url.pathname, server.config.base) ?? generatedRequest(url.pathname, manifest, server.config.base);
+        if (request === null) return next();
+        if (request.kind === "redirect") {
+          res.writeHead(301, { Location: request.to });
+          return res.end();
+        }
+
+        const docs = docsDirectory(manifest);
+        const answer = request.kind === "artifact"
+          ? readGeneratedRequest(workspace, request)
+          : docs === null ? null : readDocsRequest(join(workspace, docs), request);
+        if (answer === null) return next();
+        res.writeHead(200, { "Content-Type": answer.type, "Cache-Control": "no-cache" });
+        res.end(req.method === "HEAD" ? undefined : answer.body);
+      });
+    },
+  };
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  const { manifest } = loadManifest();
+  const written = siteDocs({ manifest, dist: "dist" });
+  if (written.length === 0) {
+    console.log("site docs: nothing to place (no markdown step, or its output is not generated yet)");
+  } else {
+    console.log(`site docs: ${written.join(", ")} → dist/`);
+  }
+}