@transclude/core 0.10.2 → 0.11.1

package/README.md

@@ -86,6 +86,9 @@ so a swap cannot drift from the page it replaces part of.
   Light DOM by default: no boundary, page CSS reaches it, `<label for>` works,
   and it ships no JavaScript. `export const shadow = true` opts into a shadow
   root and a re-render on an attribute change.
+- **Markdown pages.** A `.md` file under `app/routes/` is a page, with the same
+  loader block, elements and type checking an `.html` one has. Set `markdown` in
+  the config to a function from source to HTML; this package ships no parser.
 - **Types without writing TypeScript.** `npm run check` catches a misspelled
   field, an unknown prop and a wrong-typed one, from the shapes your loaders
   return. Annotations are optional.
@@ -129,6 +132,7 @@ npm run includes      # transclusion on http://localhost:1966
 npm run auth          # a guarded section on http://localhost:1967
 npm run live          # server-sent events on http://localhost:1968
 npm run elements      # light and shadow elements on http://localhost:1969
+npm run markdown      # Markdown pages on http://localhost:1970
 npm run check:src     # type-check the framework itself
 ```
 

package/bin/build.js

@@ -18,6 +18,7 @@ import transclude from '../src/plugin.js';
 import { loadProject } from '../src/project.js';
 import { renderRoute, urlFor } from '../src/document.js';
 import { prerenderContext, refusePrerender } from '../src/prerender.js';
+import { isGated, readGated } from '../src/gate.js';
 import { feed, feedPath } from '../src/feed.js';
 import { includeContext } from '../src/include.js';
 import { nodeLookup } from '../src/lookup.js';
@@ -121,7 +122,20 @@ fs.writeFileSync(entry, `// @ts-nocheck\n${fs.readFileSync(entry, 'utf8')}`);
 
 // ---- prerender ------------------------------------------------------------
 
-const { pages } = await import(pathToFileURL(entry).href);
+const { pages, gated: declared } = await import(pathToFileURL(entry).href);
+
+/**
+ * Paths `app/server.js` says are not public, and the routes they cover.
+ *
+ * Middleware does not run during a build, so nothing here can tell a payment
+ * gate or an auth check from an open page. Without the declaration a gated page
+ * is written to `dist/static` and served by any static host that finds it, and
+ * the build reports it as a page it prerendered.
+ *
+ * An entry matching nothing is a typo, and a typo here fails open, so it is an
+ * error rather than a shrug.
+ */
+const gated = readGated(declared);
 
 // ---- drafts ---------------------------------------------------------------
 
@@ -158,16 +172,19 @@ manifest.routes = manifest.routes.filter((route) => !isDraft(route));
  */
 async function urlsFor(route) {
   if (pages[route.id]?.prerender === false) return [];
-  if (!route.params.length) return [{ url: route.pattern, params: {} }];
+  if (!route.params.length) {
+    return isGated(route.pattern, gated) ? [] : [{ url: route.pattern, params: {} }];
+  }
 
   const paths = pages[route.id]?.paths;
   if (typeof paths !== 'function') return [];
 
   const listed = (await paths()) ?? [];
-  return listed.map((params) => ({
-    url: urlFor(route, params),
-    params,
-  }));
+  return listed
+    .map((params) => ({ url: urlFor(route, params), params }))
+    // Matched per URL, not per route: `/notes/[id]` can be open while
+    // `/notes/secret` is not, and the pattern is the same for both.
+    .filter(({ url }) => !isGated(url, gated));
 }
 
 /**
@@ -284,7 +301,7 @@ const prerendered = outcomes.filter((outcome) => outcome.ok).map((outcome) => ou
 // site with no sitemap there would be missing one only on the host that needs it
 // written down most.
 if (config.sitemap) {
-  write('sitemap.xml', await sitemap({ routes: manifest.routes }, pages, config.sitemap));
+  write('sitemap.xml', await sitemap({ routes: manifest.routes, gated }, pages, config.sitemap));
   prerendered.push('/sitemap.xml');
 }
 
@@ -306,6 +323,10 @@ fs.writeFileSync(
   path.join(dist, 'routes.json'),
   JSON.stringify(
     {
+      // Carried so `/sitemap.xml` at runtime leaves out what the build left out.
+      // The gate itself is `app/server.js` middleware, which runs at runtime and
+      // needs no help from here.
+      gated,
       dynamic: dynamic.map((route) => ({
         id: route.id,
         pattern: route.pattern,

package/bin/check.js

@@ -7,6 +7,7 @@ import ts from 'typescript';
 import { createChecker, positionAt } from '../src/typecheck.js';
 import { emitTypes } from '../src/compiler/types.js';
 import { loadProject } from '../src/project.js';
+import { isMarkdown } from '../src/markdown.js';
 
 const { root, config } = await loadProject();
 const checker = createChecker({ root, ...config });
@@ -63,16 +64,23 @@ for (const file of files) {
   const diagnostics = checker.check(file);
   if (!diagnostics.length) continue;
 
-  const source = fs.readFileSync(file, 'utf8');
+  // What the checker measured, not what is on disk. They are the same file for
+  // an `.html` page. For a Markdown page they are not, and reading disk here put
+  // a caret under an unrelated word several lines from the mistake.
+  const source = checker.sourceFor(file);
   const lines = source.split('\n');
   const relative = path.relative(root, file);
+  const converted = isMarkdown(file);
 
   for (const diagnostic of diagnostics) {
     const { line, column } = positionAt(source, diagnostic.offset);
     if (diagnostic.severity === 'error') errors++;
     else warnings++;
 
-    console.log(`\n${relative}:${line}:${column + 1}  ${diagnostic.severity}  TS${diagnostic.code}`);
+    // Said plainly rather than left to be worked out. The line and column are
+    // real, and they are not positions in the file the author opens.
+    const where = converted ? `${relative}  (converted HTML, line ${line})` : `${relative}:${line}:${column + 1}`;
+    console.log(`\n${where}  ${diagnostic.severity}  TS${diagnostic.code}`);
     console.log(`  ${diagnostic.message}`);
 
     const text = lines[line - 1] ?? '';

package/bin/dev.js

@@ -23,6 +23,7 @@ import {
 } from '../src/document.js';
 import transclude, { clientEntryUrl, pageModuleId } from '../src/plugin.js';
 import { resolveRoutesDir, scanRoutes } from '../src/routes.js';
+import { MARKDOWN_EXT } from '../src/markdown.js';
 import { baseApp, endpointMethods, runEndpoint, SERVER_FILE } from '../src/server.js';
 import { randomBytes } from 'node:crypto';
 import { cookiesOf } from '../src/cookies.js';
@@ -447,11 +448,12 @@ let app = await buildApp();
 // Adding or removing a page changes the route table, not just a module.
 vite.watcher.on('all', async (event, file) => {
   // `.js` as well as `.html`: an endpoint is a route too, and watching only for
-  // pages meant adding one needed a restart, with a 404 as the only hint.
+  // pages meant adding one needed a restart, with a 404 as the only hint. `.md`
+  // for the same reason: a Markdown page is a page.
   const extension = path.extname(file);
   const routing =
     file.startsWith(routesDir) &&
-    (extension === '.html' || extension === '.js') &&
+    (extension === '.html' || extension === MARKDOWN_EXT || extension === '.js') &&
     event !== 'change';
   // Middleware is registered once when the app is built, so a change to it needs
   // the app rebuilt, unlike a page, which is loaded per request.

package/bin/release.js

@@ -111,6 +111,85 @@ function setVersion(version) {
     manifest.version = version;
     fs.writeFileSync(file, `${JSON.stringify(manifest, null, 2)}\n`);
   }
+  setLockVersions();
+}
+
+/**
+ * Every lockfile that names this package, brought to what `package.json` says.
+ *
+ * This used to write the two manifests and stop. `npm install` does not run in a
+ * release, so nothing else ever corrected them: the root lockfile said 0.2.0
+ * against a package.json of 0.10.2, and the apps carried linked entries from
+ * 0.1.0, 0.1.1, 0.2.0 and 0.8.2. Eight releases of drift, and no test looked.
+ *
+ * Done here rather than by `npm install --package-lock-only`, which also
+ * rewrites the dependency tree from the registry. A release is not the place to
+ * find out that a transitive dependency moved.
+ */
+function setLockVersions() {
+  const core = read('package.json');
+
+  // npm writes bin paths without the leading `./`.
+  const bin = Object.fromEntries(
+    Object.entries(core.bin).map(([name, file]) => [name, file.replace(/^\.\//, '')]),
+  );
+
+  // The fields npm copies into a link entry, in the order it writes them.
+  const linked = {
+    name: core.name,
+    version: core.version,
+    license: core.license,
+    dependencies: core.dependencies,
+    bin,
+    devDependencies: core.devDependencies,
+    engines: core.engines,
+    peerDependencies: core.peerDependencies,
+  };
+
+  for (const [rel, key] of lockfiles()) {
+    const file = path.join(root, rel);
+    const text = fs.readFileSync(file, 'utf8');
+    const lock = JSON.parse(text);
+
+    // A file npm wrote differently is left alone rather than reformatted. The
+    // alternative is a release whose diff is every lockfile in the repository.
+    if (`${JSON.stringify(lock, null, 2)}\n` !== text) {
+      process.stdout.write(`  ${rel}: npm formats this differently, left alone\n`);
+      continue;
+    }
+
+    if (key === '') {
+      lock.name = core.name;
+      lock.version = core.version;
+      lock.packages[''].name = core.name;
+      lock.packages[''].version = core.version;
+    } else if (lock.packages[key]) {
+      lock.packages[key] = linked;
+    } else {
+      continue;
+    }
+
+    fs.writeFileSync(file, `${JSON.stringify(lock, null, 2)}\n`);
+  }
+}
+
+/** `[relative path, the key naming this package]` for every lockfile here. */
+function lockfiles() {
+  const found = [['package-lock.json', '']];
+
+  if (fs.existsSync(path.join(root, 'www', 'package-lock.json'))) {
+    found.push(['www/package-lock.json', '..']);
+  }
+
+  const examples = path.join(root, 'examples');
+  if (fs.existsSync(examples)) {
+    for (const entry of fs.readdirSync(examples).sort()) {
+      const rel = `examples/${entry}/package-lock.json`;
+      if (fs.existsSync(path.join(root, rel))) found.push([rel, '../..']);
+    }
+  }
+
+  return found;
 }
 
 /** Everything, in the order that fails cheapest first. */
@@ -204,7 +283,7 @@ function main() {
     return;
   }
 
-  run('git', ['add', ...MANIFESTS]);
+  run('git', ['add', ...MANIFESTS, ...lockfiles().map(([rel]) => rel)]);
 
   // A first release at the version the manifests already carry changes nothing,
   // and an empty commit is not worth making. The tag is the release either way.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transclude/core",
-  "version": "0.10.2",
+  "version": "0.11.1",
   "description": "An HTML-first server framework. A page is an .html file, the directory tree is the route table, and any fragment of a page is a URL of its own. Runs on Node, Bun, Deno and workerd, and ships no client JavaScript by default.",
   "keywords": [
     "html",
@@ -57,7 +57,7 @@
   ],
   "scripts": {
     "test": "node --test \"test/**/*.test.js\"",
-    "test:examples": "npm test --prefix examples/showcase && npm test --prefix examples/todomvc && npm test --prefix examples/blog && npm test --prefix examples/search && npm test --prefix examples/htmx && npm test --prefix examples/includes && npm test --prefix examples/auth && npm test --prefix examples/live && npm test --prefix examples/elements",
+    "test:examples": "npm test --prefix examples/showcase && npm test --prefix examples/todomvc && npm test --prefix examples/blog && npm test --prefix examples/search && npm test --prefix examples/htmx && npm test --prefix examples/includes && npm test --prefix examples/auth && npm test --prefix examples/live && npm test --prefix examples/elements && npm test --prefix examples/markdown",
     "test:www": "npm test --prefix www",
     "showcase": "npm run dev --prefix examples/showcase",
     "todomvc": "npm run dev --prefix examples/todomvc",
@@ -68,6 +68,7 @@
     "auth": "npm run dev --prefix examples/auth",
     "live": "npm run dev --prefix examples/live",
     "elements": "npm run dev --prefix examples/elements",
+    "markdown": "npm run dev --prefix examples/markdown",
     "www": "npm run dev --prefix www",
     "check:src": "tsc -p tsconfig.src.json",
     "release": "node bin/release.js"

package/skills/transclude/SKILL.md

@@ -98,6 +98,34 @@ Globals available in an expression: `html`, `json`, `Math`, `JSON`, `String`,
 `absolute()`. Returning a `Response` from a loader answers the request and skips
 the render, which is how a layout does a login redirect.
 
+## Markdown pages
+
+A `.md` file under `routes/` is a page. Set `markdown` in the config to a
+function from source to HTML; this package ships no parser.
+
+```js
+// transclude.config.js
+import { Marked } from 'marked';
+const marked = new Marked();
+
+export default { markdown: (source, file) => marked.parse(source) };
+```
+
+The loader is the same `<script server>` block, at the top of the `.md` file.
+CommonMark passes a script block through raw, so there is no frontmatter. After
+the conversion it is an ordinary page: interpolation, directives, elements,
+fragments and type checking all work.
+
+Two rules belong to Markdown rather than to this framework:
+
+- **An HTML block ends at the next blank line, and Markdown inside one is not
+  parsed.** Put blank lines inside a block-level custom element or its content
+  arrives as literal asterisks.
+- **`${` in a code fence would interpolate.** Escape `${` in code tokens in your
+  converter, or write `\${` by hand. `\${` is the escape and works in any page.
+
+A diagnostic in a `.md` page reports a line in the converted HTML and says so.
+
 ## Forms and actions
 
 A page responds to GET with its loader. Other verbs are named exports on the same
@@ -215,12 +243,17 @@ there reaches the page as written, so a value would land in code. `json(value)`
 is the one way through, and only as the entire text of the script. For a style,
 pass the value through a custom property, which is an attribute and is escaped.
 
-**A literal `${` cannot be written in a template.** There is no escape. Pass any
-text containing it in from the loader as data.
+**A literal `${` is written `\${`.** Passing the text in from the loader as data
+also works, and is easier to read when a whole sample is full of them.
 
 **Directive values are expressions, not interpolations.** Write
 `each="note of notes"`, never `each="${notes}"`.
 
+**A page's `<title>`, `<meta>`, `<link>` and `<base>` move into `<head>`.** Write
+them next to the markup that needs them. The last three take `if` and `each`, and
+the tag still lands in `<head>`. `<title>` takes neither directive: which level's
+title wins is settled when the page compiles, not per request.
+
 **A `fragment` element cannot carry `if`, `else` or `each`.** A fragment is one
 element with one id, so it cannot be conditional or repeated. Put the condition
 on something inside it.
@@ -244,6 +277,19 @@ element name. `card.html` is not, and the file is dropped.
 **`<transclude>` has no self-closing form.** `<transclude src="#a" />` is read
 as an open tag and the rest of the page becomes its fallback content.
 
+**Middleware does not run during the build.** A page gated only by
+`app/server.js` is prerendered to a file and served by any static host. Declare
+the paths so the build knows:
+
+```js
+// app/server.js
+export const gated = ['/premium', '/api/*'];
+```
+
+No file is written for them and the sitemap leaves them out. They are still
+routes. A layout guard needs no declaration: the build runs layout loaders, and
+a guard reads a cookie.
+
 **Reading a cookie makes a page personal.** It is then not cached and not
 prerendered. Writing one does not do this; reading one does.
 

package/src/compiler/codegen.js

@@ -903,7 +903,12 @@ class Codegen {
       const dynamic = parts.some((p) => p.type === 'expr');
 
       if (!dynamic) {
-        this.s(out, attr.value === '' ? ` ${attr.name}` : ` ${attr.name}="${escapeAttr(attr.value)}"`);
+        // The parts, not `attr.value`. They are the same string except where a
+        // `\${` was escaped, and reading the raw value there put the backslash
+        // in the output: `title="\${name}"` in the source, `title="\${name}"` in
+        // the page. Text got this right and attributes did not.
+        const literal = parts.map((part) => part.value).join('');
+        this.s(out, literal === '' ? ` ${attr.name}` : ` ${attr.name}="${escapeAttr(literal)}"`);
         continue;
       }
       this.c(

package/src/compiler/interp.js

@@ -8,8 +8,9 @@
 /**
  * Text and `${expr}` in source order.
  *
- * There is no escape for a literal `${`, so anything documenting the syntax has
- * to pass its examples in as data rather than write them in a template.
+ * `\${` is a literal `${`. It matters most to a page that documents this syntax,
+ * and to a Markdown page, whose code fences are full of shell and JavaScript
+ * that means `${` literally.
  *
  * @param {string} str raw text or an attribute value
  * @returns {Part[]} empty only for an empty string

package/src/defaults.js

@@ -34,6 +34,11 @@ export const DEFAULTS = {
   csrf: true,
   csp: false,
   speculate: false,
+  // `(source, file) => html`, and a `.md` page under `routes/` without one is an
+  // error naming the file. This package ships no Markdown parser: which flavor
+  // and which extensions are the app's to pick, the same way `cache` is a store
+  // the app supplies rather than a database this one depends on.
+  markdown: null,
 };
 
 /**

package/src/gate.js

@@ -0,0 +1,70 @@
+// Paths an app says are not public, and whether a URL is one of them.
+//
+// Middleware does not run during a build, so nothing in `bin/build.js` can tell
+// a payment gate or an auth check from an open page. `export const gated` in
+// `app/server.js` is the only thing a build can read about a gate, and without
+// it a gated page is written to `dist/static` and handed out by any static host,
+// with the build reporting it as a page it prerendered.
+//
+// A layout guard is caught already, for a reason worth knowing: the build runs
+// layout loaders, and a guard reads a cookie. Nothing runs `app/server.js` here.
+//
+// Pure. No `node:` imports: the sitemap reads this at runtime, on every runtime.
+/**
+ * Whether a URL is one the app declared not public.
+ *
+ * `export const gated` in `app/server.js` is the only thing a build can read
+ * about a gate, because middleware does not run during one. A layout guard is
+ * caught already: the build runs layout loaders and a guard reads a cookie. A
+ * gate in `app/server.js` is run by nobody here, so a paid or signed-in page
+ * with an ordinary loader is written to `dist/static` and handed out by any
+ * static host, with the build reporting it as a success.
+ *
+ * `/premium` matches that path only. `/premium/*` matches it and everything
+ * under it. Nothing else is a pattern: this decides whether to write a file, and
+ * a rule nobody can read at a glance is the wrong shape for that.
+ *
+ * @param {string} url the path the build is about to write
+ * @param {string[]} patterns
+ * @returns {boolean}
+ */
+export function isGated(url, patterns = []) {
+  return patterns.some((pattern) => {
+    if (!pattern.endsWith('/*')) return url === pattern;
+    const base = pattern.slice(0, -2);
+    return url === base || url.startsWith(`${base}/`);
+  });
+}
+
+/**
+ * The declaration, or a refusal naming what is wrong with it.
+ *
+ * Checked rather than trusted, because every mistake here fails open. A typo
+ * matches nothing, the page is written, and the build says it prerendered a page
+ * that was supposed to need paying for.
+ *
+ * @param {unknown} gated whatever `app/server.js` exported
+ * @returns {string[]}
+ * @throws when it is not a list of paths
+ */
+export function readGated(gated) {
+  if (gated === undefined || gated === null) return [];
+
+  if (!Array.isArray(gated)) {
+    throw new Error(
+      `[transclude] app/server.js exports "gated" as ${typeof gated}. It is a list of paths, ` +
+        `like ['/premium', '/api/*'].`,
+    );
+  }
+
+  for (const entry of gated) {
+    if (typeof entry !== 'string' || !entry.startsWith('/')) {
+      throw new Error(
+        `[transclude] "gated" in app/server.js has ${JSON.stringify(entry)}. ` +
+          `Every entry is a path beginning with "/", and "/*" at the end covers what is under it.`,
+      );
+    }
+  }
+
+  return gated;
+}

package/src/markdown.js

@@ -0,0 +1,52 @@
+// Markdown pages. A `.md` file under `routes/` is converted to HTML, and every
+// step after that is the one an `.html` page takes.
+//
+// The framework ships no Markdown parser. `markdown` in the config is a function
+// from source to HTML, the same way `cache` is a store with four methods: which
+// flavor, which extensions and which highlighter are the app's to pick, and
+// none of them becomes a dependency of this package.
+//
+// Three readers exist and they must not disagree: the plugin compiles a page,
+// the type checker reads the same page to collect its names, and `npm run check`
+// reads it again to place a diagnostic. All three come through here.
+
+export const MARKDOWN_EXT = '.md';
+
+/** @param {string} file @returns {boolean} */
+export const isMarkdown = (file) => file.endsWith(MARKDOWN_EXT);
+
+/**
+ * The source a compiler should see: HTML as written, Markdown converted.
+ *
+ * A `<script server>` block needs no new syntax to survive this. CommonMark
+ * starts an HTML block at `<script` and ends it at `</script>`, with everything
+ * between passed through raw, so the loader arrives at `splitBlocks` exactly as
+ * it would from an `.html` file.
+ *
+ * @param {string} file    the path, which decides whether anything happens
+ * @param {string} source  what is on disk
+ * @param {((source: string, file: string) => string)|null} markdown from the config
+ * @returns {string} HTML
+ * @throws when a `.md` page exists and the config has no converter
+ */
+export function sourceOf(file, source, markdown) {
+  if (!isMarkdown(file)) return source;
+
+  if (typeof markdown !== 'function') {
+    throw new Error(
+      `[transclude] ${file} is Markdown, and transclude.config.js sets no "markdown". ` +
+        'It takes (source, file) and returns HTML. This package ships no parser, ' +
+        'so the flavor is yours to choose.',
+    );
+  }
+
+  const html = markdown(source, file);
+  if (typeof html !== 'string') {
+    throw new Error(
+      `[transclude] markdown() returned ${html === null ? 'null' : typeof html} for ${file}. ` +
+        'It has to return a string of HTML.',
+    );
+  }
+
+  return html;
+}

package/src/plugin.js

@@ -17,6 +17,7 @@ import {
   readFlags,
 } from './compiler/index.js';
 import { resolveRoutesDir, scanRoutes } from './routes.js';
+import { MARKDOWN_EXT, sourceOf } from './markdown.js';
 import { SERVER_FILE } from './server.js';
 
 const P_COMPONENT = 'virtual:transclude-component/';
@@ -34,11 +35,17 @@ export default function transclude({
   routesDir = 'routes',
   fragmentParam = 'fragment',
   watchElements = false,
+  markdown = null,
 } = {}) {
   // Off unless asked for. It puts a script on every page, and it only earns that
   // when swapped-in markup names an element the page did not already render.
   // A page that renders its own elements defines them without this.
   const watching = watchElements === true;
+
+  // Every compile in this file goes through here, so a `.md` page reaches the
+  // compiler as HTML and nothing downstream learns a second format.
+  const read = (file) => sourceOf(file, readRaw(file), markdown);
+
   let root;
   let app;
   let runtime;
@@ -282,7 +289,7 @@ export default function transclude({
         return `
 ${ids.map((pageId, i) => `import * as __P${i} from ${JSON.stringify(`${P_PAGE}${pageId}`)};`).join('\n')}
 ${apiIds.map((apiId, i) => `import * as __E${i} from ${apiSpec(endpoints.get(apiId))};`).join('\n')}
-${hasMiddleware ? `import __middleware from ${JSON.stringify(specifier)};` : ''}
+${hasMiddleware ? `import * as __server from ${JSON.stringify(specifier)};` : ''}
 
 export const pages = {
 ${ids.map((pageId, i) => `  ${JSON.stringify(pageId)}: __P${i},`).join('\n')}
@@ -292,7 +299,12 @@ export const endpoints = {
 ${apiIds.map((apiId, i) => `  ${JSON.stringify(apiId)}: __E${i},`).join('\n')}
 };
 
-export const middleware = ${hasMiddleware ? '__middleware ?? null' : 'null'};
+export const middleware = ${hasMiddleware ? '__server.default ?? null' : 'null'};
+
+// Paths the app says are not public. The build reads this and writes no file for
+// them, because middleware does not run during a build and a file it was meant
+// to gate would be served by any static host.
+export const gated = ${hasMiddleware ? '__server.gated ?? []' : '[]'};
 `;
       }
 
@@ -370,7 +382,7 @@ export const middleware = ${hasMiddleware ? '__middleware ?? null' : 'null'};
       if (duplicate) return;
 
       server.watcher.on('all', (_event, file) => {
-        if (!file.endsWith('.html')) return;
+        if (!file.endsWith('.html') && !file.endsWith(MARKDOWN_EXT)) return;
         if (!file.startsWith(app)) return;
 
         scan();
@@ -429,7 +441,7 @@ function readDir(dir) {
   return map;
 }
 
-function read(file) {
+function readRaw(file) {
   return fs.readFileSync(file, 'utf8');
 }
 

package/src/routes.js

@@ -4,6 +4,7 @@
 //
 //   routes/index.html            ->  /
 //   routes/about.html            ->  /about
+//   routes/notes.md              ->  /notes, Markdown converted before it compiles
 //   routes/blog/index.html       ->  /blog
 //   routes/blog/[slug].html      ->  /blog/:slug
 //   routes/docs/[...path].html   ->  /docs/:path{.+}
@@ -15,7 +16,16 @@
 import fs from 'node:fs';
 import path from 'node:path';
 
+import { MARKDOWN_EXT } from './markdown.js';
+
 const EXT = '.html';
+/**
+ * A page can also be Markdown, converted to HTML before anything compiles it.
+ * The route table does not care which it was: `about.md` and `about.html` both
+ * answer `/about`, and a directory holding one of each is the collision below
+ * rather than a rule of its own.
+ */
+const PAGE_EXTS = [EXT, MARKDOWN_EXT];
 /**
  * A `.js` file in the routes tree is an endpoint: a route with no template, no
  * layout and no regions, which answers with a `Response` of its own. Same
@@ -81,7 +91,7 @@ export function scanRoutes(dir) {
  */
 export function toRoute(rel, file) {
   const kind = rel.endsWith(ENDPOINT_EXT) ? 'endpoint' : 'page';
-  const ext = kind === 'endpoint' ? ENDPOINT_EXT : EXT;
+  const ext = kind === 'endpoint' ? ENDPOINT_EXT : PAGE_EXTS.find((e) => rel.endsWith(e)) ?? EXT;
   const parts = rel.slice(0, -ext.length).split(path.sep);
 
   // `blog/index.html` and `blog.html` both mean /blog; the trailing `index`
@@ -163,7 +173,7 @@ function walk(dir, base = dir, out = []) {
 
     const full = path.join(dir, entry.name);
     if (entry.isDirectory()) walk(full, base, out);
-    else if (entry.name.endsWith(EXT) || entry.name.endsWith(ENDPOINT_EXT)) {
+    else if (entry.name.endsWith(ENDPOINT_EXT) || PAGE_EXTS.some((e) => entry.name.endsWith(e))) {
       out.push(path.relative(base, full));
     }
   }

package/src/sitemap.js

@@ -7,6 +7,7 @@
 // crawler can reach by guessing, so it is left out.
 
 import { urlFor } from './document.js';
+import { isGated } from './gate.js';
 
 /** The protocol's cap for one file. Past it the response is an index of files. */
 const LIMIT = 50000;
@@ -65,9 +66,16 @@ export async function sitemapEntries(manifest, pages, { entries = [], exclude =
   const extra = typeof entries === 'function' ? ((await entries()) ?? []) : entries;
   const all = [...found, ...extra];
 
+  // `manifest.gated` rather than a second config key. A path the app declared
+  // not public should not be advertised, and reading it here covers the file the
+  // build writes and the `/sitemap.xml` route together. Two lists is two answers
+  // about which URLs a crawler is invited to.
+  const gated = manifest.gated ?? [];
+
   const seen = new Set();
   return all.filter((entry) => {
     if (seen.has(entry.path) || excluded(entry.path, exclude)) return false;
+    if (isGated(entry.path, gated)) return false;
     seen.add(entry.path);
     return true;
   });

package/src/typecheck.js

@@ -19,6 +19,8 @@ import { AMBIENT_NAMES } from './compiler/ambient.js';
 import { buildEndpointShim, buildShim, originalOffset } from './compiler/shim.js';
 import { splitBlocks, readFlags } from './compiler/index.js';
 import { resolveRoutesDir, scanRoutes } from './routes.js';
+// Aliased: this file has its own `sourceOf`, which is the one that reads disk.
+import { MARKDOWN_EXT, sourceOf as htmlFrom } from './markdown.js';
 
 /**
  * Annotations are optional, so `noImplicitAny` is off: an unannotated parameter
@@ -79,9 +81,9 @@ const LAYOUT_FILE = '_layout.html';
  * on the layouts above it, a page on its whole chain.
  *
  * @param {{ root: string, appDir: string, routesDir: string, elementsDir: string,
- *   strict?: boolean }} options
- * @returns {{ files: Function, update: Function, rebuild: Function,
- *   check: Function, quickInfo: Function, describe: Function }}
+ *   strict?: boolean, markdown?: ((source: string, file: string) => string)|null }} options
+ * @returns {{ files: Function, sourceFor: Function, update: Function,
+ *   rebuild: Function, check: Function, quickInfo: Function, describe: Function }}
  */
 export function createChecker({
   root,
@@ -89,6 +91,7 @@ export function createChecker({
   elementsDir = 'elements',
   routesDir = 'routes',
   strict = false,
+  markdown = null,
 }) {
   const app = path.resolve(root, appDir);
   const options = compilerOptions(Boolean(strict));
@@ -126,7 +129,10 @@ export function createChecker({
     return built;
   };
 
-  const sourceOf = (file) => overlays.get(file) ?? fs.readFileSync(file, 'utf8');
+  // An overlay is already HTML: the editor hands over what it is checking, and a
+  // Markdown page is checked as the page it compiles to. Off disk, it is not.
+  const sourceOf = (file) =>
+    overlays.get(file) ?? htmlFrom(file, fs.readFileSync(file, 'utf8'), markdown);
 
   /** The type of one of a shim's marker exports. What tsc made of the file. */
   const exportTypeOf = (file, name) => {
@@ -421,11 +427,23 @@ export function createChecker({
     files() {
       const found = componentFiles();
       const dir = resolveRoutesDir(app, routesDir);
-      walkHtml(dir, found);
+      walkPages(dir, found);
       for (const route of scanRoutes(dir).endpoints) found.push(route.file);
       return found;
     },
 
+    /**
+     * The source the diagnostics were measured against.
+     *
+     * For an `.html` page that is the file on disk. For a Markdown page it is
+     * the HTML it converted to, and the difference is why this exists: an offset
+     * from a shim built out of the converted HTML, printed against the Markdown,
+     * points at the wrong column and looks authoritative doing it.
+     */
+    sourceFor(file) {
+      return sourceOf(file);
+    },
+
     /** Replaces a file's contents without touching disk, for an editor buffer. */
     update(file, source) {
       overlays.set(file, source);
@@ -577,12 +595,12 @@ function readDirSafe(dir) {
   return fs.existsSync(dir) ? fs.readdirSync(dir) : [];
 }
 
-function walkHtml(dir, out) {
+function walkPages(dir, out) {
   if (!fs.existsSync(dir)) return out;
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     const full = path.join(dir, entry.name);
-    if (entry.isDirectory()) walkHtml(full, out);
-    else if (entry.name.endsWith('.html')) out.push(full);
+    if (entry.isDirectory()) walkPages(full, out);
+    else if (entry.name.endsWith('.html') || entry.name.endsWith(MARKDOWN_EXT)) out.push(full);
   }
   return out;
 }