@jxsuite/compiler 1.4.0 → 2.0.0

package/src/site/pages-discovery.ts

@@ -17,6 +17,8 @@ import { extname, join, relative, resolve } from "node:path";
 import type { ExtensionRegistry } from "@jxsuite/schema/extension-registry";
 import type { FormatRegistry } from "@jxsuite/schema/format-registry";
 import type { JxDocument, JxPathsDef, ProjectConfig } from "@jxsuite/schema/types";
+import { localeOfRoute } from "./i18n.ts";
+import type { ResolvedI18n } from "./i18n.ts";
 
 interface Route {
   urlPattern: string; // URL pattern (e.g. "/blog/:slug")
@@ -27,6 +29,13 @@ interface Route {
   params: string[]; // Parameter names (e.g. ["slug"])
   $layout: string | null; // Layout override from page frontmatter, if any
   _pathParams?: Record<string, string>; // Resolved path parameters
+  /**
+   * The timestamp of the thing this concrete route was generated FROM, when that is not the
+   * template file. A collection route's source is an entry; `sourcePath` still points at the
+   * `[slug].json` that rendered it, so without this every post in an archive reports the template's
+   * mtime and the whole collection looks edited whenever the template is.
+   */
+  sourceMtime?: string;
 }
 
 /**
@@ -57,6 +66,104 @@ export async function readPageDocument(
   return (await entry.call("parse", source)) as JxDocument;
 }
 
+/**
+ * The `$translationKey` each route's document declares, for the routes that declare one.
+ *
+ * A **pre-pass**, because a page's alternates depend on the whole route table (§13.5): the set has
+ * to be complete before the first page is compiled, so a document cannot tell the build what its
+ * key is while it is being compiled.
+ *
+ * The file is read and the **parse is skipped** unless the text mentions the key. Most pages of a
+ * multilingual site never declare one — their paths are parallel and the derivation is right — so
+ * paying a second full parse for every page to find a key that is usually absent would be a cost
+ * with nothing behind it. Documents sharing a source (a `$paths` template's expansions) are read
+ * once.
+ *
+ * A document that fails to parse is skipped rather than thrown from here. It will fail again a
+ * moment later while being compiled, where the error names the page and the rest of the site still
+ * builds; failing in a pre-pass would turn one bad page into no site at all.
+ *
+ * **A key may name its route's parameters**, as `${slug}`, and that is what makes a collection's
+ * localized URLs work: one `[slug]` template expands to one route per entry, so a key that could
+ * not vary per entry would claim a single identity for the whole collection and the build would
+ * report it as a duplicate. `pages/fr-ca/expositions/[slug].json` declaring `"exhibitions/${slug}"`
+ * pairs each French post with the English one it translates, because two translations of an entry
+ * share an id (§13.3) and the id is what the parameter carries.
+ *
+ * @param {readonly {
+ *   sourcePath: string;
+ *   urlPattern: string;
+ *   _pathParams?: Record<string, string>;
+ * }[]} routes
+ *   - Concrete routes
+ * @param {FormatRegistry} [registry]
+ * @returns {Promise<Map<string, string>>} Keyed by `urlPattern`
+ */
+export async function readTranslationKeys(
+  routes: readonly {
+    sourcePath: string;
+    urlPattern: string;
+    _pathParams?: Record<string, string> | undefined;
+  }[],
+  registry?: FormatRegistry,
+): Promise<Map<string, string>> {
+  const out = new Map<string, string>();
+  const byFile = new Map<string, string | null>();
+  for (const route of routes) {
+    if (!byFile.has(route.sourcePath)) {
+      byFile.set(route.sourcePath, await declaredTranslationKey(route.sourcePath, registry));
+    }
+    const key = byFile.get(route.sourcePath);
+    if (key !== null && key !== undefined) {
+      out.set(route.urlPattern, withRouteParams(key, route._pathParams));
+    }
+  }
+  return out;
+}
+
+/**
+ * Substitute a route's own parameters into a declared key.
+ *
+ * A parameter with no value is left as it was written rather than blanked: a key that quietly
+ * became `exhibitions/` would pair every entry in the collection with every other, which is a
+ * duplicate the build reports — and the report naming `${slug}` says what is actually wrong.
+ *
+ * @param {string} key
+ * @param {Record<string, string> | undefined} params
+ * @returns {string}
+ */
+function withRouteParams(key: string, params: Record<string, string> | undefined): string {
+  if (!key.includes("${")) {
+    return key;
+  }
+  return key.replaceAll(/\$\{(\w+)\}/g, (whole, name: string) => params?.[name] ?? whole);
+}
+
+/**
+ * One document's `$translationKey`, or null when it declares none or cannot be read.
+ *
+ * @param {string} sourcePath
+ * @param {FormatRegistry} [registry]
+ * @returns {Promise<string | null>}
+ */
+async function declaredTranslationKey(
+  sourcePath: string,
+  registry?: FormatRegistry,
+): Promise<string | null> {
+  if (!readFileSync(sourcePath, "utf8").includes("$translationKey")) {
+    return null;
+  }
+  let doc: JxDocument;
+  try {
+    doc = await readPageDocument(sourcePath, registry);
+  } catch {
+    return null;
+  }
+  return typeof doc.$translationKey === "string" && doc.$translationKey !== ""
+    ? doc.$translationKey
+    : null;
+}
+
 /**
  * Discover all routable pages in a pages/ directory.
  *
@@ -204,6 +311,35 @@ async function fileToRoute(relativePath: string, absolutePath: string, registry?
   };
 }
 
+/**
+ * The reserved key a `resolvePaths` result may carry alongside its route parameters.
+ *
+ * A route parameter comes from a `[bracket]` segment in a filename, so this name cannot collide
+ * with one by accident, and it matches what the content loader already calls the same data on an
+ * entry (`parser.md` §9.3) — so a fact about an entry keeps one name from the file it was read out
+ * of all the way to the sitemap.
+ */
+const PATH_ENTRY_META = "_meta";
+
+/**
+ * The entry timestamp carried on a `$paths` result, or null when there is none.
+ *
+ * Null is ordinary: the core `$paths` shapes (`values`, `$ref`, a literal array) describe route
+ * parameters and nothing else, and a route with no entry of its own correctly falls back to its
+ * template's own modification time.
+ *
+ * @param {Record<string, unknown>} pathEntry
+ * @returns {string | null}
+ */
+function entryMtime(pathEntry: Record<string, unknown>): string | null {
+  const meta = pathEntry[PATH_ENTRY_META];
+  if (meta === null || typeof meta !== "object") {
+    return null;
+  }
+  const { mtime } = meta as { mtime?: unknown };
+  return typeof mtime === "string" && mtime !== "" ? mtime : null;
+}
+
 /**
  * Expand dynamic routes by resolving $paths from each dynamic page.
  *
@@ -226,6 +362,7 @@ export async function expandDynamicRoutes(
   sections: Record<string, unknown> = {},
   registry?: ExtensionRegistry,
   projectConfig?: ProjectConfig,
+  i18n?: ResolvedI18n | null,
 ) {
   const expanded: Route[] = [];
 
@@ -256,23 +393,33 @@ export async function expandDynamicRoutes(
       sections,
       registry,
       projectConfig,
+      // The template's OWN prefix, read before expansion: `/fr/blog/:slug` is a French route
+      // Whatever its entries turn out to be called, and that is what scopes a localized collection.
+      localeOfRoute(route.urlPattern, i18n ?? null),
     );
 
     for (const pathEntry of pathEntries) {
       let concreteUrl = route.urlPattern;
       const params: Record<string, string> = {};
       for (const [param, value] of Object.entries(pathEntry)) {
+        // `_meta` is the reserved carrier for facts about the source ENTRY (extensions.md §8).
+        // It is not a route parameter and must never reach substitution.
+        if (param === PATH_ENTRY_META) {
+          continue;
+        }
         params[param] = String(value);
         concreteUrl = concreteUrl.replace(`:${param}`, params[param]);
         concreteUrl = concreteUrl.replace("*", params[param]);
       }
 
+      const mtime = entryMtime(pathEntry);
       expanded.push({
         ...route,
         _pathParams: params,
         isCatchAll: false,
         isDynamic: false,
         params: [],
+        ...(mtime === null ? {} : { sourceMtime: mtime }),
         urlPattern: concreteUrl,
       });
     }
@@ -301,6 +448,7 @@ async function resolvePathEntries(
   sections: Record<string, unknown>,
   registry?: ExtensionRegistry,
   projectConfig?: ProjectConfig,
+  locale?: string | null,
 ): Promise<Record<string, unknown>[]> {
   // Legacy: array of param objects
   if (Array.isArray($paths)) {
@@ -345,6 +493,7 @@ async function resolvePathEntries(
     }
     return (await entry.call("resolvePaths", $paths, {
       data: sections[entry.project.key],
+      locale,
       projectConfig,
       root: projectRoot,
     })) as Record<string, unknown>[];

package/src/site/preview-server.ts

@@ -12,7 +12,15 @@ import { createServer } from "node:http";
 import type { Server } from "node:http";
 import { existsSync, readFileSync, statSync } from "node:fs";
 import { extname, join, normalize, resolve, sep } from "node:path";
+import { MEDIA_TYPE_BY_EXTENSION } from "@jxsuite/schema/media-type";
 
+/*
+ * The extensions this server can be asked for. `MEDIA_TYPE_BY_EXTENSION` is spread in **last** so
+ * the registered spellings win: `.md` carries the `variant` that says which markdown it is, and
+ * `.yaml`/`.yml` are `application/yaml` rather than an `application/octet-stream` download prompt.
+ * Everything else is this table's own, because a shared table for `image/png` would be a second
+ * source of truth for something no standard is ambiguous about.
+ */
 const MIME_TYPES: Record<string, string> = {
   ".avif": "image/avif",
   ".css": "text/css; charset=utf-8",
@@ -23,13 +31,13 @@ const MIME_TYPES: Record<string, string> = {
   ".jpg": "image/jpeg",
   ".js": "text/javascript; charset=utf-8",
   ".json": "application/json; charset=utf-8",
-  ".md": "text/markdown; charset=utf-8",
   ".png": "image/png",
   ".svg": "image/svg+xml",
   ".txt": "text/plain; charset=utf-8",
   ".webp": "image/webp",
   ".woff2": "font/woff2",
   ".xml": "application/xml; charset=utf-8",
+  ...MEDIA_TYPE_BY_EXTENSION,
 };
 
 /**

package/src/site/schema-command.ts

@@ -3,12 +3,16 @@
  *
  * Composes the per-project entry documents (project.schema.json and document.schema.json) from the
  * core schemas and each enabled extension's shipped fragments, then writes them into the project
- * root as committed, SELF-CONTAINED bundles: every fragment ref is resolved through
- * {@link bundleSchema}, embedding the core + fragment resources under `$defs` keyed by their
- * canonical `$id`s. Relative `./node_modules/...` refs only resolve through Node module resolution,
- * which editors do not perform (hoisted workspaces leave projects without their own node_modules) —
- * the bundled form resolves offline for ajv `jx validate`, VS Code's JSON language service, and the
- * studio's Monaco alike, with one resolution behavior for all consumers.
+ * root as committed, SELF-CONTAINED, SINGLE-RESOURCE schemas: every fragment ref is resolved
+ * through {@link bundleSchema} (embedding the core + fragment resources under `$defs`) and then
+ * collapsed by {@link flattenSchema}, which drops the embedded `$id`s and rewrites every ref to a
+ * root-relative JSON Pointer. Both passes are needed for one resolution behavior across all
+ * consumers: relative `./node_modules/...` refs only resolve through Node module resolution, which
+ * editors do not perform (hoisted workspaces leave projects without their own node_modules), while
+ * `$id`-scoped refs inside a compound document resolve only in a fully compliant validator — VS
+ * Code's JSON language service and Monaco resolve `#/...` against the document root and fetch
+ * anything with a URI before the `#` over the network. Root pointers are what ajv, the language
+ * service, and Monaco all agree on.
  */
 
 import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
@@ -16,8 +20,8 @@ import { createRequire } from "node:module";
 import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
 import {
   bundleSchema,
-  emitDocumentSchema,
-  emitProjectSchema,
+  composeProjectSchemas,
+  flattenSchema,
 } from "@jxsuite/schema/project-schemas";
 import { buildProjectExtensionRegistry } from "./format-host.ts";
 import { loadProjectConfig } from "./site-loader.ts";
@@ -33,49 +37,38 @@ const CORE_DOCUMENT_SPECIFIER = "@jxsuite/schema/schema.json";
  * @param {string} projectRoot - Absolute path to the project directory
  * @returns {Promise<{ projectSchemaPath: string; documentSchemaPath: string }>} Written paths
  */
-export async function writeProjectSchemas(projectRoot: string) {
+export async function composeEntryDocuments(projectRoot: string) {
   const { config } = loadProjectConfig(projectRoot);
   const registry = await buildProjectExtensionRegistry(projectRoot, config);
 
-  const corePath = coreSchemaRef(projectRoot, CORE_PROJECT_SPECIFIER);
-  const fragments: string[] = [];
-  const pathsValueRefs: string[] = [];
-  const fieldSchemaRefs: string[] = [];
-  for (const ext of registry.extensions) {
-    const projectFragment = ext.schemas.project;
-    if (projectFragment) {
-      fragments.push(fragmentRef(projectRoot, ext, projectFragment));
-    }
-    const documentFragment = ext.schemas.document;
-    if (documentFragment) {
-      pathsValueRefs.push(...fragmentDefsRefs(documentFragment));
-    }
-    // The `schemas.fields` manifest convention: a fragment whose $defs members are extension
-    // Field extras, unioned into the per-project field resource (specs/extensions.md §5.3).
-    const fieldsFragment = ext.schemas.fields;
-    if (fieldsFragment) {
-      fieldSchemaRefs.push(...fragmentDefsRefs(fieldsFragment));
-      fragments.push(fragmentRef(projectRoot, ext, fieldsFragment));
-    }
-  }
-
-  // Bundle before committing: the emitted relative refs are an intermediate form only.
+  /* Composition itself is host-agnostic (@jxsuite/schema `composeProjectSchemas`) — the filesystem
+     appears only in what this host contributes: project-relative refs for the fragments the
+     registry resolved, and a loader restricted to the project root. The cloud session composes the
+     same two documents from bundled artifacts through the same function. */
   const root = resolve(projectRoot);
-  const loadJson = restrictedSchemaLoader(root);
-  const projectSchema = await bundleSchema(
-    emitProjectSchema({ corePath, fieldSchemaRefs, fragments }),
-    loadJson,
-    root,
-  );
-  const documentSchema = await bundleSchema(
-    emitDocumentSchema({
-      corePath: coreSchemaRef(projectRoot, CORE_DOCUMENT_SPECIFIER),
-      pathsValueRefs,
-    }),
-    loadJson,
-    root,
-  );
+  const { document: documentSchema, project: projectSchema } = await composeProjectSchemas({
+    baseDir: root,
+    coreDocumentRef: coreSchemaRef(projectRoot, CORE_DOCUMENT_SPECIFIER),
+    coreProjectRef: coreSchemaRef(projectRoot, CORE_PROJECT_SPECIFIER),
+    extensions: registry.extensions.map((ext) => ({
+      document: ext.schemas.document && fragmentRef(projectRoot, ext, ext.schemas.document),
+      fields: ext.schemas.fields && fragmentRef(projectRoot, ext, ext.schemas.fields),
+      project: ext.schemas.project && fragmentRef(projectRoot, ext, ext.schemas.project),
+    })),
+    loadJson: restrictedSchemaLoader(root),
+  });
+
+  return { documentSchema, projectSchema };
+}
 
+/**
+ * Compose the entry documents and WRITE them to the project root.
+ *
+ * @param {string} projectRoot - Absolute path to the project directory
+ * @returns {Promise<{ projectSchemaPath: string; documentSchemaPath: string }>} Written paths
+ */
+export async function writeProjectSchemas(projectRoot: string) {
+  const { documentSchema, projectSchema } = await composeEntryDocuments(projectRoot);
   const projectSchemaPath = resolve(projectRoot, "project.schema.json");
   const documentSchemaPath = resolve(projectRoot, "document.schema.json");
   writeFileSync(projectSchemaPath, `${JSON.stringify(projectSchema, null, 2)}\n`, "utf8");
@@ -94,7 +87,11 @@ export async function writeProjectSchemas(projectRoot: string) {
  * @param {string} projectRoot - Absolute path to the project directory
  * @returns {Promise<{ project: Record<string, unknown>; document: Record<string, unknown> }>}
  */
-export async function readBundledProjectSchemas(projectRoot: string) {
+export async function readBundledProjectSchemas(
+  projectRoot: string,
+  options: { write?: boolean } = {},
+) {
+  const { write = true } = options;
   const root = resolve(projectRoot);
   const projectJsonPath = resolve(root, "project.json");
   const projectSchemaPath = resolve(root, "project.schema.json");
@@ -110,49 +107,125 @@ export async function readBundledProjectSchemas(projectRoot: string) {
       return false;
     }
   };
-  if (stale(projectSchemaPath) || stale(documentSchemaPath)) {
-    await writeProjectSchemas(root);
+  const isStale = stale(projectSchemaPath) || stale(documentSchemaPath);
+  // `write: false` is what a VALIDATOR passes. A stale entry document is regenerated in memory and
+  // The committed file is left exactly as it is: `jx validate` reports on the repository, and a
+  // Checker that edits what it is checking is not a checker. This mattered — running
+  // `schema:validate-all` on a machine with a shadowed core silently rewrote the committed
+  // Artifact with a narrower one, and it was green either way, because it validated against
+  // Whatever it had just written.
+  const composed = isStale ? await composeEntryDocuments(root) : null;
+  if (composed && write) {
+    writeFileSync(
+      projectSchemaPath,
+      `${JSON.stringify(composed.projectSchema, null, 2)}\n`,
+      "utf8",
+    );
+    writeFileSync(
+      documentSchemaPath,
+      `${JSON.stringify(composed.documentSchema, null, 2)}\n`,
+      "utf8",
+    );
   }
 
   const loadJson = restrictedSchemaLoader(root);
-  const project = await bundleSchema(
-    JSON.parse(readFileSync(projectSchemaPath, "utf8")) as Record<string, unknown>,
-    loadJson,
-    root,
-  );
-  const document = await bundleSchema(
-    JSON.parse(readFileSync(documentSchemaPath, "utf8")) as Record<string, unknown>,
-    loadJson,
-    root,
-  );
+  const projectFile =
+    composed?.projectSchema ??
+    (JSON.parse(readFileSync(projectSchemaPath, "utf8")) as Record<string, unknown>);
+  const documentFile =
+    composed?.documentSchema ??
+    (JSON.parse(readFileSync(documentSchemaPath, "utf8")) as Record<string, unknown>);
+  const project = flattenSchema(await bundleSchema(projectFile, loadJson, root));
+  const document = flattenSchema(await bundleSchema(documentFile, loadJson, root));
   return { document, project };
 }
 
+/** Packages whose schemas define the Jx document contract itself, never a project's own content. */
+const FIRST_PARTY_SCHEMA_SCOPE = "@jxsuite/";
+
+/**
+ * Is this specifier one of the shipped core schemas, as opposed to a project-local fragment?
+ *
+ * @param {string} specifier - Bare specifier, e.g. `@jxsuite/schema/schema.json`
+ * @returns {boolean}
+ */
+export function isFirstPartySchema(specifier: string): boolean {
+  return specifier.startsWith(FIRST_PARTY_SCHEMA_SCOPE) && specifier.endsWith(".json");
+}
+
+/**
+ * Resolve a first-party schema from the HOST — the workspace running the generator.
+ *
+ * Throws rather than falling back to the project, because a fallback is exactly the behaviour that
+ * let a stale published core answer for the workspace one. A generator that cannot find its own
+ * schema must stop, not improvise.
+ *
+ * @param {string} specifier
+ * @param {string} path - The original ref, for the error message
+ * @returns {string} Absolute path to the resolved schema file
+ */
+function hostResolve(specifier: string, path: string): string {
+  try {
+    return createRequire(import.meta.url).resolve(specifier);
+  } catch (error) {
+    throw new Error(
+      `Schema ref "${path}" names the first-party schema "${specifier}", which must resolve from ` +
+        `the host workspace and does not. The project's own copy is deliberately not consulted, so ` +
+        `the host install is missing "${specifier.split("/").slice(0, 2).join("/")}" — check that ` +
+        `the package ships with this build`,
+      { cause: error },
+    );
+  }
+}
+
 /**
  * Bundler loader restricted to the project root, with the validate-project host fallback: a missing
  * `<root>/node_modules/<specifier>` file resolves the specifier project-first, then through the
  * host (workspace symlinks resolve transparently).
  *
+ * Exported for test: both refusals below are reachable only through a project whose `node_modules`
+ * has been arranged to provoke them, and a fixture that arranges an extension registry as well says
+ * less about the rule than a call that states the path directly.
+ *
  * @param {string} root - Absolute project root
  * @returns {SchemaJsonLoader}
  */
-function restrictedSchemaLoader(root: string): SchemaJsonLoader {
+export function restrictedSchemaLoader(root: string): SchemaJsonLoader {
   return async (path) => {
     const abs = resolve(path);
     const rel = relative(root, abs);
     if (rel.startsWith("..") || isAbsolute(rel)) {
       throw new Error(`Schema ref "${path}" escapes the project root ${root}`);
     }
-    if (existsSync(abs)) {
-      return JSON.parse(readFileSync(abs, "utf8")) as Record<string, unknown>;
-    }
     const posixRel = rel.split(sep).join("/");
     const marker = "node_modules/";
     const markerIndex = posixRel.lastIndexOf(marker);
-    if (markerIndex === -1) {
+    const specifier = markerIndex === -1 ? null : posixRel.slice(markerIndex + marker.length);
+    // A FIRST-PARTY core comes from the host, always — before the project-local file is even
+    // Looked at. A project may legitimately carry its own fragments, which is what the rest of this
+    // Loader is for; what it may not do is answer for `@jxsuite/schema` itself.
+    //
+    // This is not hypothetical. A stray `bun install` in packages/starters/sites/shop left a
+    // Gitignored node_modules holding @jxsuite/schema@0.35.0 beside a 1.5.0 workspace, and BOTH the
+    // `existsSync` shortcut below and the project-first `createRequire` resolved to it. The
+    // Generator then emitted a shop document schema missing Statement, StatementList, PathsValue
+    // And the three admission blocks, with ClassMethodDef.role down from 16 members to 6 and
+    // ChildrenValue.oneOf down from 3 to 2 — dropping the computed-children branch, which made
+    // Shop's own pages/products/[sku].json invalid against its own committed schema. Nothing
+    // Noticed for six weeks, because the artifact was only ever regenerated on the machine that
+    // Had the shadow.
+    if (specifier !== null && isFirstPartySchema(specifier)) {
+      return JSON.parse(readFileSync(hostResolve(specifier, path), "utf8")) as Record<
+        string,
+        unknown
+      >;
+    }
+    if (existsSync(abs)) {
+      return JSON.parse(readFileSync(abs, "utf8")) as Record<string, unknown>;
+    }
+    if (specifier === null) {
       throw new Error(`Schema ref "${path}" does not exist`);
     }
-    const specifier = posixRel.slice(markerIndex + marker.length);
     let resolved: string;
     try {
       const projRequire = createRequire(resolve(root, "package.json"));
@@ -231,25 +304,3 @@ function projectRelativeRef(projectRoot: string, absPath: string, conventional:
   }
   return rel.startsWith("./") ? rel : `./${rel}`;
 }
-
-/**
- * Canonical "uri#/pointer" refs into a fragment's `$defs` shapes: one per `$defs` entry, addressed
- * by the fragment's own `$id` (refs inside the entry document's embeds resolve against canonical
- * URIs, never file locations). Serves both the document paths union and the fields-extras union.
- *
- * @param {string} fragmentPath - Resolved absolute fragment path
- * @returns {string[]}
- */
-function fragmentDefsRefs(fragmentPath: string): string[] {
-  let fragment: { $id?: unknown; $defs?: Record<string, unknown> };
-  try {
-    fragment = JSON.parse(readFileSync(fragmentPath, "utf8")) as typeof fragment;
-  } catch {
-    return [];
-  }
-  const { $id } = fragment;
-  if (typeof $id !== "string" || !fragment.$defs) {
-    return [];
-  }
-  return Object.keys(fragment.$defs).map((name) => `${$id}#/$defs/${name}`);
-}