@transclude/core 0.11.0 → 0.11.2

package/bin/build.js

@@ -18,6 +18,7 @@ import transclude from '../src/plugin.js';
 import { loadProject } from '../src/project.js';
 import { renderRoute, urlFor } from '../src/document.js';
 import { prerenderContext, refusePrerender } from '../src/prerender.js';
+import { isGated, readGated } from '../src/gate.js';
 import { feed, feedPath } from '../src/feed.js';
 import { includeContext } from '../src/include.js';
 import { nodeLookup } from '../src/lookup.js';
@@ -121,7 +122,20 @@ fs.writeFileSync(entry, `// @ts-nocheck\n${fs.readFileSync(entry, 'utf8')}`);
 
 // ---- prerender ------------------------------------------------------------
 
-const { pages } = await import(pathToFileURL(entry).href);
+const { pages, gated: declared } = await import(pathToFileURL(entry).href);
+
+/**
+ * Paths `app/server.js` says are not public, and the routes they cover.
+ *
+ * Middleware does not run during a build, so nothing here can tell a payment
+ * gate or an auth check from an open page. Without the declaration a gated page
+ * is written to `dist/static` and served by any static host that finds it, and
+ * the build reports it as a page it prerendered.
+ *
+ * An entry matching nothing is a typo, and a typo here fails open, so it is an
+ * error rather than a shrug.
+ */
+const gated = readGated(declared);
 
 // ---- drafts ---------------------------------------------------------------
 
@@ -158,16 +172,19 @@ manifest.routes = manifest.routes.filter((route) => !isDraft(route));
  */
 async function urlsFor(route) {
   if (pages[route.id]?.prerender === false) return [];
-  if (!route.params.length) return [{ url: route.pattern, params: {} }];
+  if (!route.params.length) {
+    return isGated(route.pattern, gated) ? [] : [{ url: route.pattern, params: {} }];
+  }
 
   const paths = pages[route.id]?.paths;
   if (typeof paths !== 'function') return [];
 
   const listed = (await paths()) ?? [];
-  return listed.map((params) => ({
-    url: urlFor(route, params),
-    params,
-  }));
+  return listed
+    .map((params) => ({ url: urlFor(route, params), params }))
+    // Matched per URL, not per route: `/notes/[id]` can be open while
+    // `/notes/secret` is not, and the pattern is the same for both.
+    .filter(({ url }) => !isGated(url, gated));
 }
 
 /**
@@ -284,7 +301,7 @@ const prerendered = outcomes.filter((outcome) => outcome.ok).map((outcome) => ou
 // site with no sitemap there would be missing one only on the host that needs it
 // written down most.
 if (config.sitemap) {
-  write('sitemap.xml', await sitemap({ routes: manifest.routes }, pages, config.sitemap));
+  write('sitemap.xml', await sitemap({ routes: manifest.routes, gated }, pages, config.sitemap));
   prerendered.push('/sitemap.xml');
 }
 
@@ -306,6 +323,10 @@ fs.writeFileSync(
   path.join(dist, 'routes.json'),
   JSON.stringify(
     {
+      // Carried so `/sitemap.xml` at runtime leaves out what the build left out.
+      // The gate itself is `app/server.js` middleware, which runs at runtime and
+      // needs no help from here.
+      gated,
       dynamic: dynamic.map((route) => ({
         id: route.id,
         pattern: route.pattern,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transclude/core",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "description": "An HTML-first server framework. A page is an .html file, the directory tree is the route table, and any fragment of a page is a URL of its own. Runs on Node, Bun, Deno and workerd, and ships no client JavaScript by default.",
   "keywords": [
     "html",
@@ -57,7 +57,7 @@
   ],
   "scripts": {
     "test": "node --test \"test/**/*.test.js\"",
-    "test:examples": "npm test --prefix examples/showcase && npm test --prefix examples/todomvc && npm test --prefix examples/blog && npm test --prefix examples/search && npm test --prefix examples/htmx && npm test --prefix examples/includes && npm test --prefix examples/auth && npm test --prefix examples/live && npm test --prefix examples/elements && npm test --prefix examples/markdown",
+    "test:examples": "npm test --prefix examples/showcase && npm test --prefix examples/todomvc && npm test --prefix examples/blog && npm test --prefix examples/search && npm test --prefix examples/htmx && npm test --prefix examples/includes && npm test --prefix examples/auth && npm test --prefix examples/live && npm test --prefix examples/elements && npm test --prefix examples/markdown && npm test --prefix examples/atlas",
     "test:www": "npm test --prefix www",
     "showcase": "npm run dev --prefix examples/showcase",
     "todomvc": "npm run dev --prefix examples/todomvc",
@@ -69,6 +69,7 @@
     "live": "npm run dev --prefix examples/live",
     "elements": "npm run dev --prefix examples/elements",
     "markdown": "npm run dev --prefix examples/markdown",
+    "atlas": "npm run dev --prefix examples/atlas",
     "www": "npm run dev --prefix www",
     "check:src": "tsc -p tsconfig.src.json",
     "release": "node bin/release.js"

package/skills/transclude/SKILL.md

@@ -277,6 +277,27 @@ element name. `card.html` is not, and the file is dropped.
 **`<transclude>` has no self-closing form.** `<transclude src="#a" />` is read
 as an open tag and the rest of the page becomes its fallback content.
 
+**`app.get()` in `app/server.js` is not how a route is made.** The `app` there
+is a real Hono instance and accepts one, but the directory tree is the route
+table, and a handler registered by hand is invisible to the build, the sitemap
+and `npm run check`. Middleware is what belongs in that file. Most of Hono's
+API has an answer here already, and
+[references/server.md](references/server.md) has the line between the two next
+to Hono's own documentation for agents.
+
+**Middleware does not run during the build.** A page gated only by
+`app/server.js` is prerendered to a file and served by any static host. Declare
+the paths so the build knows:
+
+```js
+// app/server.js
+export const gated = ['/premium', '/api/*'];
+```
+
+No file is written for them and the sitemap leaves them out. They are still
+routes. A layout guard needs no declaration: the build runs layout loaders, and
+a guard reads a cookie.
+
 **Reading a cookie makes a page personal.** It is then not cached and not
 prerendered. Writing one does not do this; reading one does.
 
@@ -292,6 +313,6 @@ directly. It is the object the whole chain holds.
 - [references/fragments.md](references/fragments.md) — fragments, includes and
   transclusion
 - [references/server.md](references/server.md) — cookies, middleware, security,
-  config, deployment
+  config, caching, deployment, and where Hono's own docs are
 
 Full documentation: https://transclude.dev/docs

package/skills/transclude/references/server.md

@@ -16,6 +16,57 @@ export default (app) => {
 It runs before anything that serves bytes, so a guard there covers prerendered
 pages and public files.
 
+## Hono's documentation
+
+Hono publishes its documentation in a form an agent can read. Fetch it rather
+than recall it. The dependency is `hono@^4`, and it moves.
+
+| URL | Size | What it holds |
+| --- | --- | --- |
+| `https://hono.dev/llms.txt` | 6 KB | An index. Every page, one line each. |
+| `https://hono.dev/llms-small.txt` | 190 KB | The core: routing, the `Context`, middleware, the helpers. |
+| `https://hono.dev/llms-full.txt` | 360 KB | All of it, a page per built-in middleware. |
+
+Read `llms.txt` and follow the one link the question needs. The other two are
+whole manuals, and a question about `cors` does not need one.
+
+### What this framework already answers
+
+Most of Hono's surface has an answer here, and reaching past it is the common
+mistake. A route registered by hand answers requests. The build, the sitemap
+and `npm run check` never see it.
+
+| In Hono | Here |
+| --- | --- |
+| `app.get('/notes', …)` | `app/routes/notes.html` |
+| `c.req.param('id')` | `params.id` in the loader |
+| `c.req.query('q')` | `new URL(url).searchParams` |
+| `c.req.formData()` | `request.formData()` in a verb export |
+| `getCookie(c, 'theme')` | `ctx.cookies` |
+| `csrf()` | on by default, `csrf` in the config |
+| `secureHeaders()` for a policy | `csp` in the config |
+| `trimTrailingSlash()` | `trailingSlash` in the config |
+| `compress()`, `etag()` | both are built in, per response and at rest |
+| `cache()` | `export const revalidate` |
+| `serveStatic()` | `app/public/` |
+| `c.executionCtx.waitUntil(p)` | `ctx.after(p)` |
+| `c.html()`, `hono/html`, `hono/jsx` | a page is an `.html` file |
+| `hono/ssg` | `npm run build` |
+
+Hono's `Context` reaches an app in one place, the middleware in `app/server.js`.
+A loader, an action and an endpoint are handed `ctx`, which belongs to this
+framework and carries no `c`. Reading a form should not cost an author a
+router's API.
+
+### What is still Hono's
+
+`app` in `app/server.js` is a real Hono instance, so its built-in middleware
+works unchanged: `cors`, `basicAuth`, `bearerAuth`, `logger`, `bodyLimit`,
+`ipRestriction`, `requestId`, `timeout`, `timing`. So do the helpers `accepts`,
+`conninfo` and `streaming`.
+
+`app.request()` in a test is Hono's too. See [Testing](#testing).
+
 ## Cookies
 
 `ctx.cookies` reads and writes.
@@ -104,6 +155,7 @@ Source is JavaScript with JSDoc. Do not convert it to TypeScript.
 | `sitemap` | `false` | `{ hostname }` mounts `/sitemap.xml`. |
 | `feed` | `false` | `{ hostname, title, items }` mounts a feed. |
 | `proxy` | `false` | `{ allow: [...] }` for cross-site includes. |
+| `cache` | — | Where a page held by `revalidate` is kept. A bounded map in this process by default. |
 | `precache` | `false` | `true` writes `/precache.json`. |
 | `onError` | `null` | `(error, { request, url, method })` per failed request. |
 
@@ -149,6 +201,81 @@ skipped. Deployed, the URL is a 404. Publishing is deleting the line.
 `prerender` is read off the page, never off its layouts. A layout that reads a
 cookie makes every page under it request-dependent, and nothing says so.
 
+## Holding a render
+
+Between a file written once and a render on every request, there is a page held
+for a while.
+
+```js
+// app/routes/notes.html
+export const prerender = false;
+export const revalidate = 3600;
+```
+
+The number is seconds. Inside that window a request is answered from the store
+and the loader does not run. Past it the held page goes out immediately and a
+fresh one renders behind the response, so nobody waits for a re-render. One
+render happens at a time per URL, however many requests arrive together.
+
+The key is the path and the query, because a page reading `?q=` renders
+differently for each one. A rebuild that throws leaves the held page where it is,
+and the error goes to `onError` with the request that started it.
+
+Three things hold nothing:
+
+- **`npm run dev`.** The dev server renders every request and keeps nothing
+  between them. A window has no effect there at all.
+- **A page the build wrote to a file.** The static handler answers before the
+  route handler that holds anything, so the window never runs. Adding
+  `prerender = false` puts the page back on the path that has one, which is why
+  the example above carries both lines.
+- **A fragment.** `?fragment=list` is rendered on demand, for every route,
+  prerendered or not.
+
+**What is never held:** a page that read a cookie, set a header, answered with a
+`Response`, or has a status outside 2xx. A shared store holding any of those
+hands one visitor's page, or one visitor's `Set-Cookie`, to the next. It is the
+same rule the build uses to decide a route can be a file.
+
+### Tags
+
+Seconds say when a page may be out of date. A tag says when it is.
+
+```js
+// app/routes/notes.html
+export const prerender = false;
+export const revalidate = { seconds: 3600, tags: ['notes'] };
+
+export default async () => ({ notes: await notes.all() });
+```
+
+```js
+// app/routes/api/notes.js
+export const POST = async ({ request, revalidateTag }) => {
+  await notes.add(await request.json());
+
+  // Every held page carrying this tag is dropped. The next request for one
+  // renders it again.
+  revalidateTag('notes');
+
+  return new Response(null, { status: 204 });
+};
+```
+
+`ctx.revalidateTag` belongs where the change was written, which is an action or
+an endpoint: whatever made the change is what says it happened. A prerendered
+page's loader calling it stops the build, since a build holds nothing yet to
+drop.
+
+`revalidate` takes a number of seconds, or `{ seconds, tags }`. Anything else
+throws when the app starts. `revalidate: '1h'` would otherwise hold forever or
+not at all, and say nothing either way.
+
+The default store is a bounded map in this process. That is right for one server
+and wrong for several: each holds its own copy, and `revalidateTag` reaches one
+of them. `cache` in the config takes anything with the same `get`, `set`,
+`delete` and `deleteByTag`.
+
 ## Runtimes
 
 The same app runs on Node, Bun, Deno and workerd. `bin/serve.js`,

package/src/gate.js

@@ -0,0 +1,70 @@
+// Paths an app says are not public, and whether a URL is one of them.
+//
+// Middleware does not run during a build, so nothing in `bin/build.js` can tell
+// a payment gate or an auth check from an open page. `export const gated` in
+// `app/server.js` is the only thing a build can read about a gate, and without
+// it a gated page is written to `dist/static` and handed out by any static host,
+// with the build reporting it as a page it prerendered.
+//
+// A layout guard is caught already, for a reason worth knowing: the build runs
+// layout loaders, and a guard reads a cookie. Nothing runs `app/server.js` here.
+//
+// Pure. No `node:` imports: the sitemap reads this at runtime, on every runtime.
+/**
+ * Whether a URL is one the app declared not public.
+ *
+ * `export const gated` in `app/server.js` is the only thing a build can read
+ * about a gate, because middleware does not run during one. A layout guard is
+ * caught already: the build runs layout loaders and a guard reads a cookie. A
+ * gate in `app/server.js` is run by nobody here, so a paid or signed-in page
+ * with an ordinary loader is written to `dist/static` and handed out by any
+ * static host, with the build reporting it as a success.
+ *
+ * `/premium` matches that path only. `/premium/*` matches it and everything
+ * under it. Nothing else is a pattern: this decides whether to write a file, and
+ * a rule nobody can read at a glance is the wrong shape for that.
+ *
+ * @param {string} url the path the build is about to write
+ * @param {string[]} patterns
+ * @returns {boolean}
+ */
+export function isGated(url, patterns = []) {
+  return patterns.some((pattern) => {
+    if (!pattern.endsWith('/*')) return url === pattern;
+    const base = pattern.slice(0, -2);
+    return url === base || url.startsWith(`${base}/`);
+  });
+}
+
+/**
+ * The declaration, or a refusal naming what is wrong with it.
+ *
+ * Checked rather than trusted, because every mistake here fails open. A typo
+ * matches nothing, the page is written, and the build says it prerendered a page
+ * that was supposed to need paying for.
+ *
+ * @param {unknown} gated whatever `app/server.js` exported
+ * @returns {string[]}
+ * @throws when it is not a list of paths
+ */
+export function readGated(gated) {
+  if (gated === undefined || gated === null) return [];
+
+  if (!Array.isArray(gated)) {
+    throw new Error(
+      `[transclude] app/server.js exports "gated" as ${typeof gated}. It is a list of paths, ` +
+        `like ['/premium', '/api/*'].`,
+    );
+  }
+
+  for (const entry of gated) {
+    if (typeof entry !== 'string' || !entry.startsWith('/')) {
+      throw new Error(
+        `[transclude] "gated" in app/server.js has ${JSON.stringify(entry)}. ` +
+          `Every entry is a path beginning with "/", and "/*" at the end covers what is under it.`,
+      );
+    }
+  }
+
+  return gated;
+}

package/src/plugin.js

@@ -289,7 +289,7 @@ export default function transclude({
         return `
 ${ids.map((pageId, i) => `import * as __P${i} from ${JSON.stringify(`${P_PAGE}${pageId}`)};`).join('\n')}
 ${apiIds.map((apiId, i) => `import * as __E${i} from ${apiSpec(endpoints.get(apiId))};`).join('\n')}
-${hasMiddleware ? `import __middleware from ${JSON.stringify(specifier)};` : ''}
+${hasMiddleware ? `import * as __server from ${JSON.stringify(specifier)};` : ''}
 
 export const pages = {
 ${ids.map((pageId, i) => `  ${JSON.stringify(pageId)}: __P${i},`).join('\n')}
@@ -299,7 +299,12 @@ export const endpoints = {
 ${apiIds.map((apiId, i) => `  ${JSON.stringify(apiId)}: __E${i},`).join('\n')}
 };
 
-export const middleware = ${hasMiddleware ? '__middleware ?? null' : 'null'};
+export const middleware = ${hasMiddleware ? '__server.default ?? null' : 'null'};
+
+// Paths the app says are not public. The build reads this and writes no file for
+// them, because middleware does not run during a build and a file it was meant
+// to gate would be served by any static host.
+export const gated = ${hasMiddleware ? '__server.gated ?? []' : '[]'};
 `;
       }
 

package/src/sitemap.js

@@ -7,6 +7,7 @@
 // crawler can reach by guessing, so it is left out.
 
 import { urlFor } from './document.js';
+import { isGated } from './gate.js';
 
 /** The protocol's cap for one file. Past it the response is an index of files. */
 const LIMIT = 50000;
@@ -65,9 +66,16 @@ export async function sitemapEntries(manifest, pages, { entries = [], exclude =
   const extra = typeof entries === 'function' ? ((await entries()) ?? []) : entries;
   const all = [...found, ...extra];
 
+  // `manifest.gated` rather than a second config key. A path the app declared
+  // not public should not be advertised, and reading it here covers the file the
+  // build writes and the `/sitemap.xml` route together. Two lists is two answers
+  // about which URLs a crawler is invited to.
+  const gated = manifest.gated ?? [];
+
   const seen = new Set();
   return all.filter((entry) => {
     if (seen.has(entry.path) || excluded(entry.path, exclude)) return false;
+    if (isGated(entry.path, gated)) return false;
     seen.add(entry.path);
     return true;
   });