mothwing 0.0.0-stage → 0.0.1

package/src/render.js

@@ -0,0 +1,162 @@
+import crypto from "crypto"
+import fs from "fs"
+import path from "path"
+import { posix } from "./log.js"
+import { seoTags } from "./seo.js"
+import { includeHtml, parseAttrs, parseTemplate, renderNodes } from "./template.js"
+
+/*
+  Tags, usable in any page or partial. Paths start at src, which the build mirrors:
+  - <x-html src="partials/nav.html"/>      pastes in the partial, partials can include partials
+  - <x-each>, <x-if>                       loops and conditions, see template.js
+  - <x-js src="/js/index.js"/>             <script src="/js/index.js?v=hash">, add `module` for one of the config's
+                                           modules, which also gets a modulepreload for each chunk it imports up front,
+                                           and preload="/js/data/x.json" adds one for a file it import()s later
+  - <x-css src="/css/index.css"/>          <link href="/css/index.css?v=hash" rel="stylesheet">
+  - <x-seo title="..." description="..."/>  see seo.js
+  - the config's own tags
+*/
+
+/**
+ * What one build has worked out so far, shared by its pages.
+ * @typedef {object} BuildState
+ * @property {import("./data.js").SiteData} data The data folder
+ * @property {Map<string, string>} assetUrls "/js/index.js" -> "/js/index.js?v=hash"
+ * @property {Map<string, { nodes: import("./template.js").Node[], partials: Set<string> }>} templates A page's file ->
+ *   its HTML with the partials pasted in, and which those were
+ * @property {Map<string, Set<string>>} used A page's file -> the names its pages filled in, for what they depend on
+ * @property {Map<string, string[]>} chunkImports "/js/gathering.js" -> the chunks it imports up front
+ * @property {Map<string, string>} entryChunks "/js/data/x.json" -> "/js/chunks/x-hash.js"
+ */
+
+/** @returns {BuildState} */
+export const buildState = () => ({
+  data: { values: {}, files: {} },
+  assetUrls: new Map(),
+  templates: new Map(),
+  used: new Map(),
+  chunkImports: new Map(),
+  entryChunks: new Map()
+})
+
+/**
+ * @param {string} file "js/index.js" or "/js/index.js"
+ * @returns {string} "/js/index.js"
+ */
+const sitePath = (file) => file.startsWith("/") ? file : `/${file}`
+
+/**
+ * @param {import("./config.js").Site} site
+ * @param {BuildState} state
+ * @param {string} from The page, for errors
+ * @param {boolean} hashed Production builds add ?v=hash, so browsers and CDNs fetch it fresh after a deploy
+ * @returns {Record<string, import("./config.js").TagFunction>}
+ */
+function coreTags(site, state, from, hashed) {
+  /**
+   * @param {string} tag
+   * @param {Record<string, string | true>} attrs
+   * @param {string} name
+   */
+  const need = (tag, attrs, name) => {
+    const value = attrs[name]
+    if (typeof value !== "string") throw new Error(`${from}: <${tag}> needs a ${name}="..."`)
+    return value
+  }
+
+  /**
+   * @param {string} tag
+   * @param {string} file "/js/index.js"
+   */
+  const assetUrl = (tag, file) => {
+    if (!hashed) return file
+    let url = state.assetUrls.get(file)
+    if (!url) {
+      const built = path.join(site.out, file)
+      if (!fs.existsSync(built)) throw new Error(`${from}: <${tag}> can't find ${file} in the build`)
+      url = `${file}?v=${crypto.createHash("md5").update(fs.readFileSync(built)).digest("hex").slice(0, 8)}`
+      state.assetUrls.set(file, url)
+    }
+    return url
+  }
+
+  return {
+    "x-seo": ({ attrs, route }) => seoTags(site, route, attrs),
+    "x-js": ({ attrs }) => {
+      const file = sitePath(need("x-js", attrs, "src"))
+      const preload = attrs.preload ? sitePath(need("x-js", attrs, "preload")) : null
+      // Only known when the JS was built in this process, so not on dev renders
+      if (preload && state.entryChunks.size && !state.entryChunks.has(preload)) {
+        throw new Error(`${from}: <x-js preload="${preload}"> isn't imported by any of the modules`)
+      }
+      const chunks = [
+        ...(attrs.module ? state.chunkImports.get(file) ?? [] : []),
+        ...preload ? [state.entryChunks.get(preload) ?? []].flat() : []
+      ]
+      return [
+        ...chunks.map(chunk => `<link rel="modulepreload" href="${chunk}">`),
+        `<script${attrs.module ? " type=\"module\"" : ""} src="${assetUrl("x-js", file)}"></script>`
+      ]
+    },
+    "x-css": ({ attrs }) => `<link href="${assetUrl("x-css", sitePath(need("x-css", attrs, "src")))}" type="text/css" rel="stylesheet">`
+  }
+}
+
+/**
+ * @param {import("./config.js").ActiveLink} decide The config's own say
+ * @param {import("./routes.js").Route} route
+ * @param {string} href
+ * @returns {"page" | "section" | false}
+ */
+function activeLink(decide, route, href) {
+  const own = decide(href, route)
+  if (own !== undefined) return own
+  if (href === route.url) return "page"
+  if (href !== "/" && href.endsWith("/") && route.url.startsWith(href)) return "section"
+  return false
+}
+
+/**
+ * @param {import("./config.js").ActiveLink} decide
+ * @param {import("./routes.js").Route} route
+ * @param {string} html
+ * @returns {string}
+ */
+const markActiveLinks = (decide, route, html) => html.replace(/<a\b([^>]*\bhref="([^"]+)"[^>]*)>/g, (tag, attrs, href) => {
+  const active = activeLink(decide, route, href)
+  return active ? `<a${attrs} aria-current="${active === "page" ? "page" : "true"}">` : tag
+})
+
+/**
+ * @param {import("./config.js").Site} site
+ * @param {BuildState} state
+ * @param {import("./routes.js").Route} route
+ * @param {boolean} hashed Add ?v=hash to the assets
+ * @returns {string} The finished HTML, before minifying
+ */
+export function renderPage(site, state, route, hashed) {
+  const from = posix(path.relative(site.src, path.join(site.pages, route.source)))
+  let template = state.templates.get(route.source)
+  if (template === undefined) {
+    const partials = new Set()
+    const html = includeHtml(site.src, fs.readFileSync(path.join(site.pages, route.source), "utf8"), from, partials)
+    template = { nodes: parseTemplate(html, from), partials }
+    state.templates.set(route.source, template)
+  }
+
+  const used = state.used.get(route.source) ?? new Set()
+  state.used.set(route.source, used)
+  const filled = renderNodes(template.nodes, { ...state.data.values, ...route.props }, { from, used })
+
+  const tags = { ...site.tags, ...coreTags(site, state, from, hashed) }
+  const transformed = site.transforms.reduce((html, transform) => transform(html, route, site), filled)
+
+  // Tags on a line of their own are indented like it, every line of what they turn into
+  const html = transformed.replace(/([ \t]*)<(x-[a-z][a-z-]*)((?:\s+[\w-]+(?:="[^"]*")?)*)\s*\/?>/g, (tag, space, name, attrs, offset, page) => {
+    if (!tags[name]) throw new Error(`${from}: unknown tag <${name}>`)
+    const lineStart = offset === 0 || page[offset - 1] === "\n"
+    const output = [tags[name]({ attrs: parseAttrs(attrs), route, site })].flat()
+    return lineStart ? output.map(line => space + line).join("\n") : space + output.join("\n")
+  })
+  return site.activeLinks ? markActiveLinks(site.activeLinks, route, html) : html
+}

package/src/routes.js

@@ -0,0 +1,125 @@
+import fs from "fs"
+import path from "path"
+import { pathToFileURL } from "url"
+import { slug } from "./html.js"
+import { posix } from "./log.js"
+import { lookup } from "./template.js"
+
+/**
+ * A page to write. Most come one to a file in the pages folder, a [name].html file gives one per item.
+ * Its props can have `files`, more files to write, by their path in the build.
+ * @typedef {object} Route
+ * @property {string} source "gathering/[item].html", relative to the pages folder
+ * @property {string} output Where it goes in the build: "gathering/cedar-log/index.html"
+ * @property {string} url "/gathering/cedar-log/"
+ * @property {Record<string, any>} props Its own values, the data folder's are added when it's rendered
+ * @property {string[]} sources Files its content comes from, relative to src, for the sitemap's lastmod
+ */
+
+/**
+ * @param {string} dir
+ * @param {string} pages The pages folder
+ * @returns {string[]} Every .html file in it and its subfolders, relative to the pages folder
+ */
+const htmlFiles = (dir, pages = dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
+  const file = path.join(dir, entry.name)
+  if (entry.isDirectory()) return htmlFiles(file, pages)
+  return entry.name.endsWith(".html") ? [posix(path.relative(pages, file))] : []
+})
+
+/**
+ * @param {import("./config.js").Site} site
+ * @param {string} file "gathering/index.html", "404.html"
+ * @returns {string} Where it goes in the build. Other names get their own folder, "about.html" -> "about/index.html"
+ */
+const pageOutput = (site, file) => file === site.notFound || file.endsWith("index.html") ? file : file.replace(/\.html$/, "/index.html")
+
+/**
+ * @param {string} output "gathering/index.html"
+ * @returns {string} "/gathering/", "/" for index.html
+ */
+export const outputUrl = (output) => "/" + output.replace(/(^|\/)index\.html$/, "$1")
+
+/**
+ * Every page, each folder's index.html first, so the front page leads the sitemap and a list comes before its items.
+ * - about.html is one page. Its .data.js, when it has one, default exports `() => values`
+ * - [projects.name].html is a page per item of the data folder's projects, by its name, the item is {{ page }}
+ * - [name].html is a page per item its .data.js returns, each `{ slug, ...values }`
+ * A .data.js can return a `sources` list (relative to src) of the files it's made from, for the sitemap's dates.
+ * @param {import("./config.js").Site} site
+ * @param {import("./data.js").SiteData} siteData
+ * @param {Set<string>} [only] Just these pages, relative to the pages folder
+ * @returns {Promise<Route[]>}
+ */
+export async function listRoutes(site, siteData, only) {
+  const pagesDir = posix(path.relative(site.src, site.pages))
+  /** @param {string} file */
+  const sortKey = (file) => file.replace(/index\.html$/, "")
+  const files = htmlFiles(site.pages)
+    .filter(file => !only || only.has(file))
+    .sort((a, b) => sortKey(a).localeCompare(sortKey(b)))
+  const routes = []
+
+  for (const source of files) {
+    const dataFile = source.replace(/\.html$/, ".data.js")
+    const hasData = fs.existsSync(path.join(site.pages, dataFile))
+    const own = [`${pagesDir}/${source}`, ...hasData ? [`${pagesDir}/${dataFile}`] : []]
+
+    const fromData = source.match(/\[([\w.]+)\.(\w+)\]\.html$/)
+    if (fromData) {
+      if (hasData) throw new Error(`${pagesDir}/${source}: gets its pages from ${fromData[1]}, so it can't have a .data.js too`)
+      routes.push(...pagesFromData(site, siteData, source, fromData[1], fromData[2]))
+      continue
+    }
+
+    const data = hasData ? await (await import(pathToFileURL(path.join(site.pages, dataFile)).href)).default() : {}
+
+    if (!/\[\w+\]\.html$/.test(source)) {
+      const output = pageOutput(site, source)
+      routes.push({ source, output, url: outputUrl(output), props: data, sources: [...own, ...data.sources ?? []] })
+      continue
+    }
+
+    if (!hasData) {
+      const name = source.match(/\[(\w+)\]\.html$/)?.[1]
+      throw new Error(`${pagesDir}/${source}: needs to say what it makes pages of, like [projects.${name}].html for one per item of ${site.data}/projects, or have a .data.js that returns them`)
+    }
+    if (!Array.isArray(data)) throw new Error(`${pagesDir}/${dataFile}: a [name].html page needs a list of pages`)
+    const seen = new Set()
+    for (const { slug, sources = [], ...props } of data) {
+      if (seen.has(slug)) throw new Error(`${pagesDir}/${dataFile}: two pages are both "${slug}"`)
+      seen.add(slug)
+      const output = source.replace(/\[\w+\]\.html$/, `${slug}/index.html`)
+      // Dated by its data alone, a template tweak would otherwise date thousands of pages at once
+      routes.push({ source, output, url: outputUrl(output), props, sources: sources.length ? sources : own })
+    }
+  }
+  return routes
+}
+
+/**
+ * A page per item of a list in the data folder.
+ * @param {import("./config.js").Site} site
+ * @param {import("./data.js").SiteData} siteData
+ * @param {string} source "projects/[projects.name].html"
+ * @param {string} list "projects"
+ * @param {string} key "name", what each item's slug is made from
+ * @returns {Route[]}
+ */
+function pagesFromData(site, siteData, source, list, key) {
+  const from = posix(path.relative(site.src, path.join(site.pages, source)))
+  const items = lookup(siteData.values, list)
+  if (!Array.isArray(items)) throw new Error(`${from}: ${site.data} has no list called ${list}`)
+  const dataFile = siteData.files[list.split(".")[0]]
+
+  const seen = new Set()
+  return items.map((item, i) => {
+    if (item?.[key] === undefined) throw new Error(`${from}: item ${i} of ${list} has no ${key}`)
+    const pageSlug = slug(String(item[key]))
+    if (seen.has(pageSlug)) throw new Error(`${from}: two of ${list} are both "${pageSlug}"`)
+    seen.add(pageSlug)
+    const output = source.replace(/\[[\w.]+\]\.html$/, `${pageSlug}/index.html`)
+    // Dated by its data alone, a template tweak would otherwise date every page at once
+    return { source, output, url: outputUrl(output), props: { page: item }, sources: [dataFile] }
+  })
+}

package/src/seo.js

@@ -0,0 +1,105 @@
+import { escapeHtml, unescapeHtml } from "./html.js"
+
+/**
+ * @param {string} description HTML, as the <x-seo> attribute has it
+ * @param {number} max
+ * @returns {string} At most `max` characters of text, cut at a word so it ends on "...", still HTML
+ */
+export function fitDescription(description, max) {
+  const text = unescapeHtml(description)
+  if (text.length <= max) return description
+  const cut = text.slice(0, max - 3)
+  return escapeHtml(`${cut.slice(0, cut.lastIndexOf(" ")).replace(/[\s,.;:-]+$/, "")}...`)
+}
+
+/**
+ * The <title>, description, canonical URL, link preview tags and breadcrumbs for <x-seo title="..." description="..."/>.
+ * Optional: image="..." image-alt="..." image-size="WxH" for its own preview image, and section="..." name="..." for its
+ * breadcrumb in search results.
+ * @param {import("./config.js").Site} site
+ * @param {import("./routes.js").Route} route
+ * @param {Record<string, string | true>} attrs
+ * @returns {string[]} The tags, one per line
+ */
+export function seoTags(site, route, attrs) {
+  /**
+   * @param {string} name
+   * @returns {string | undefined}
+   */
+  const attr = (name) => {
+    const value = attrs[name]
+    return typeof value === "string" ? value : undefined
+  }
+  const { siteName, locale, maxTitle, maxDescription } = site.seo
+  const pageTitle = attr("title")
+  const pageDescription = attr("description")
+  if (!pageTitle || !pageDescription) throw new Error(`${route.source}: <x-seo> needs a title and a description`)
+  const description = fitDescription(pageDescription, maxDescription)
+  // The front page keeps it, for people searching for the site by its name
+  const named = `${pageTitle} - ${siteName}`
+  const title = route.url === "/" || unescapeHtml(named).length <= maxTitle ? named : pageTitle
+
+  const url = `${site.url}${route.url}`
+  /**
+   * @param {string} key
+   * @param {string} name
+   * @param {string | number} content
+   */
+  const meta = (key, name, content) => `<meta ${key}="${name}" content="${content}">`
+  // The attributes are HTML, the JSON is in a <script>, where only "</script" could end it early
+  /** @param {object} data */
+  const jsonLd = (data) => `<script type="application/ld+json">${JSON.stringify(data).replaceAll("<", "\\u003c")}</script>`
+
+  // Lets Google show the site's name in results, it only reads this from the front page
+  const siteNameLd = route.url === "/"
+    ? [jsonLd({ "@context": "https://schema.org", "@type": "WebSite", name: siteName, url })]
+    : []
+
+  // section="Gathering" name="Cedar Log" shows as site › Gathering › Cedar Log in results
+  const section = attr("section")
+  const name = attr("name")
+  const crumbs = [
+    [siteName, `${site.url}/`],
+    ...section ? [[section, `${site.url}${route.url.replace(/[^/]+\/$/, "")}`]] : [],
+    [name ?? "", url]
+  ]
+  const breadcrumbs = name
+    ? [jsonLd({
+      "@context": "https://schema.org",
+      "@type": "BreadcrumbList",
+      itemListElement: crumbs.map(([name, item], i) => ({ "@type": "ListItem", position: i + 1, name: unescapeHtml(name), item }))
+    })]
+    : []
+
+  const [width, height] = (attr("image-size") ?? "").split("x").map(Number)
+  const image = attr("image")
+    ? { src: `${site.url}${attr("image")}`, width, height, alt: attr("image-alt") ?? "" }
+    : site.seo.image && { ...site.seo.image, src: `${site.url}${site.seo.image.src}` }
+  // A wide image shows across the whole preview, a square one as a thumbnail next to the text
+  const card = image && image.width > image.height ? "summary_large_image" : "summary"
+
+  return [
+    `<title>${title}</title>`,
+    meta("name", "description", description),
+    `<link rel="canonical" href="${url}">`,
+    // Lets Google show the preview image big, in results and Discover, it otherwise picks a size itself
+    meta("name", "robots", "max-image-preview:large"),
+    meta("property", "og:locale", locale),
+    meta("property", "og:type", "website"),
+    meta("property", "og:site_name", siteName),
+    meta("property", "og:title", title),
+    meta("property", "og:description", description),
+    meta("property", "og:url", url),
+    ...image
+      ? [
+        meta("property", "og:image", image.src),
+        meta("property", "og:image:width", image.width),
+        meta("property", "og:image:height", image.height),
+        meta("property", "og:image:alt", image.alt)
+      ]
+      : [],
+    meta("name", "twitter:card", card),
+    ...siteNameLd,
+    ...breadcrumbs
+  ]
+}

package/src/site.js

@@ -0,0 +1,96 @@
+import { execFileSync } from "child_process"
+import fs from "fs"
+import path from "path"
+import { log, posix } from "./log.js"
+
+/**
+ * @param {import("./config.js").Site} site
+ * @param {Map<string, string | null>} cache Sorted file list -> its lastmod, many pages are made from the same files
+ * @param {string[]} files Relative to src
+ * @returns {string | null} YYYY-MM-DD of the last commit that changed one of them, null without git
+ */
+function lastModified(site, cache, files) {
+  const key = [...new Set(files)].sort().join("|")
+  if (!cache.has(key)) {
+    try {
+      const date = execFileSync("git", ["log", "-1", "--format=%cs", "--", ...key.split("|").map(f => posix(path.relative(site.root, path.join(site.src, f))))], {
+        cwd: site.root,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"]
+      }).trim()
+      // Not committed yet, so changed today
+      cache.set(key, date || new Date().toISOString().slice(0, 10))
+    } catch {
+      cache.set(key, null)
+    }
+  }
+  return cache.get(key) ?? null
+}
+
+/**
+ * Writes CNAME, the sitemaps and robots.txt.
+ * @param {import("./config.js").Site} site
+ * @param {import("./routes.js").Route[]} routes
+ * @param {boolean} [dated] Add each page's lastmod, from git, which takes a moment
+ */
+export async function writeSiteFiles(site, routes, dated = true) {
+  log("SITE", "Writing CNAME, the sitemaps and robots.txt")
+
+  // Every GitHub Pages deploy wipes the branch, so the custom domain has to be written each time
+  if (site.cname) fs.writeFileSync(path.join(site.out, "CNAME"), site.domain + "\n")
+
+  /** @type {Map<string, { url: string, lastmod: string | null | false }[]>} */
+  const sitemaps = new Map()
+  if (site.sitemap) {
+    const cache = new Map()
+    // A sitemap per section behind a sitemap index, so Search Console shows how each one is indexed on its own
+    for (const route of routes.filter(route => route.source !== site.notFound)) {
+      const section = route.url.split("/")[1]
+      const name = site.sitemap.sections.includes(section) ? section : "pages"
+      const pages = sitemaps.get(name) ?? []
+      sitemaps.set(name, pages)
+      pages.push({ url: route.url, lastmod: dated && site.sitemap.dates && lastModified(site, cache, route.sources) })
+    }
+
+    /**
+     * @param {string} tag
+     * @param {string[]} entries
+     */
+    const xml = (tag, entries) => [
+      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
+      `<${tag} xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+      ...entries,
+      `</${tag}>`,
+      ""
+    ].join("\n")
+    /**
+     * @param {string} tag
+     * @param {string} loc
+     * @param {string | null | false | undefined} lastmod
+     */
+    const entry = (tag, loc, lastmod) => `  <${tag}><loc>${loc}</loc>${lastmod ? `<lastmod>${lastmod}</lastmod>` : ""}</${tag}>`
+
+    const index = []
+    for (const [name, pages] of sitemaps) {
+      const file = `sitemap-${name}.xml`
+      fs.writeFileSync(path.join(site.out, file), xml("urlset", pages.map(p => entry("url", `${site.url}${p.url}`, p.lastmod))))
+      // YYYY-MM-DD sorts as text, so the newest is the last
+      const newest = pages.map(p => p.lastmod).filter(Boolean).sort().at(-1)
+      index.push(entry("sitemap", `${site.url}/${file}`, newest))
+    }
+    fs.writeFileSync(path.join(site.out, "sitemap.xml"), xml("sitemapindex", index))
+  }
+
+  if (site.robots) {
+    fs.writeFileSync(path.join(site.out, "robots.txt"), [
+      ...site.robots.block.flatMap(bot => [`User-agent: ${bot}`, "Disallow: /", ""]),
+      "User-agent: *",
+      "Allow: /",
+      ...site.robots.disallow.map(dir => `Disallow: ${dir}`),
+      "",
+      ...site.sitemap ? [`Sitemap: ${site.url}/sitemap.xml`, ""] : []
+    ].join("\n"))
+  }
+
+  log("SITE", `Done, ${[...sitemaps.values()].flat().length} pages in the sitemap`)
+}