@jant/core 0.3.42 → 0.3.44

package/src/routes/feed/__tests__/sitemap.test.ts

@@ -4,15 +4,18 @@ import type { Bindings } from "../../../types.js";
 import type { AppVariables } from "../../../types/app-context.js";
 import { DEFAULT_APP_PORT } from "../../../lib/env.js";
 import { resolveConfig } from "../../../lib/resolve-config.js";
+import { createTestApp } from "../../../__tests__/helpers/app.js";
 import { sitemapRoutes } from "../sitemap.js";
+import { SITEMAP_SHARD_SIZE } from "../../../lib/feed.js";
 
 type Env = { Bindings: Bindings; Variables: AppVariables };
 const TEST_SITE_ORIGIN = `http://localhost:${DEFAULT_APP_PORT}`;
 
-function createSitemapTestApp(
+function createRobotsTestApp(
   allSettings: Record<string, string> = {},
   envOverrides: Partial<Bindings> = {},
 ) {
+  // robots.txt doesn't need service wiring; keep a minimal harness for it.
   const app = new Hono<Env>();
 
   app.use("*", async (c, next) => {
@@ -27,14 +30,19 @@ function createSitemapTestApp(
   });
 
   app.route("/", sitemapRoutes);
-
   return app;
 }
 
+function createSitemapTestApp() {
+  const testApp = createTestApp();
+  testApp.app.route("/", sitemapRoutes);
+  return testApp;
+}
+
 describe("Sitemap Routes", () => {
   describe("/robots.txt", () => {
     it("disallows internal utility routes while allowing the public site", async () => {
-      const app = createSitemapTestApp();
+      const app = createRobotsTestApp();
 
       const res = await app.request("/robots.txt");
 
@@ -49,7 +57,7 @@ describe("Sitemap Routes", () => {
     });
 
     it("disallows the entire site when global noindex is enabled", async () => {
-      const app = createSitemapTestApp({ NOINDEX: "true" });
+      const app = createRobotsTestApp({ NOINDEX: "true" });
 
       const res = await app.request("/robots.txt");
 
@@ -61,4 +69,312 @@ describe("Sitemap Routes", () => {
       expect(robots).not.toContain("Disallow: /_/");
     });
   });
+
+  describe("/sitemap.xml (index)", () => {
+    it("lists pages shard + one post shard when there are a few posts", async () => {
+      const { app, services } = createSitemapTestApp();
+      for (let i = 0; i < 3; i++) {
+        await services.posts.create({
+          format: "note",
+          bodyMarkdown: `post ${i}`,
+          status: "published",
+        });
+      }
+
+      const res = await app.request("/sitemap.xml");
+      expect(res.status).toBe(200);
+      expect(res.headers.get("Content-Type")).toContain("application/xml");
+
+      const xml = await res.text();
+      expect(xml).toContain("<sitemapindex");
+      expect(xml).toContain("/sitemap-pages.xml");
+      expect(xml).toContain("/sitemap-posts-1.xml");
+      expect(xml).not.toContain("/sitemap-posts-2.xml");
+      // No collections → no collections shard entry
+      expect(xml).not.toContain("/sitemap-collections.xml");
+    });
+
+    it("lists multiple post shards when post count exceeds shard size", async () => {
+      const { app, services } = createSitemapTestApp();
+      // Create enough posts to span 2 shards. Use shard_size + 1 to minimize
+      // work while still forcing a second shard.
+      for (let i = 0; i < SITEMAP_SHARD_SIZE + 1; i++) {
+        await services.posts.create({
+          format: "note",
+          bodyMarkdown: `p${i}`,
+          status: "published",
+        });
+      }
+
+      const res = await app.request("/sitemap.xml");
+      const xml = await res.text();
+      expect(xml).toContain("/sitemap-posts-1.xml");
+      expect(xml).toContain("/sitemap-posts-2.xml");
+      expect(xml).not.toContain("/sitemap-posts-3.xml");
+    });
+
+    it("includes the collections shard when collections exist", async () => {
+      const { app, services } = createSitemapTestApp();
+      await services.collections.create({
+        slug: "reading",
+        title: "Reading",
+      });
+
+      const res = await app.request("/sitemap.xml");
+      const xml = await res.text();
+      expect(xml).toContain("/sitemap-collections.xml");
+    });
+  });
+
+  describe("/sitemap-posts-N.xml", () => {
+    it("emits <url> entries for page 1 in ascending id order", async () => {
+      const { app, services } = createSitemapTestApp();
+      const created: string[] = [];
+      for (let i = 0; i < 3; i++) {
+        const post = await services.posts.create({
+          format: "note",
+          bodyMarkdown: `post ${i}`,
+          title: `Post ${i}`,
+          status: "published",
+        });
+        created.push(post.slug);
+      }
+
+      const res = await app.request("/sitemap-posts-1.xml");
+      expect(res.status).toBe(200);
+
+      const xml = await res.text();
+      expect(xml).toContain("<urlset");
+      for (const slug of created) {
+        expect(xml).toContain(`/${slug}`);
+      }
+
+      // Slugs appear in id-ascending order (== creation order for TypeIDs).
+      const indices = created.map((slug) => xml.indexOf(`/${slug}`));
+      expect(indices).toEqual([...indices].sort((a, b) => a - b));
+    });
+
+    it("excludes private posts, replies, and drafts", async () => {
+      const { app, services } = createSitemapTestApp();
+      const root = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "root",
+        status: "published",
+      });
+      const reply = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "reply",
+        replyToId: root.id,
+        status: "published",
+      });
+      const priv = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "private",
+        visibility: "private",
+        status: "published",
+      });
+      const draft = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "draft",
+        status: "draft",
+      });
+
+      const xml = await (await app.request("/sitemap-posts-1.xml")).text();
+      expect(xml).toContain(`/${root.slug}`);
+      expect(xml).not.toContain(`/${reply.slug}`);
+      expect(xml).not.toContain(`/${priv.slug}`);
+      expect(xml).not.toContain(`/${draft.slug}`);
+    });
+
+    it("includes latest_hidden posts (they are public URLs)", async () => {
+      const { app, services } = createSitemapTestApp();
+      const hidden = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "hidden",
+        visibility: "latest_hidden",
+        status: "published",
+      });
+
+      const xml = await (await app.request("/sitemap-posts-1.xml")).text();
+      expect(xml).toContain(`/${hidden.slug}`);
+    });
+
+    it("splits content across shards at SITEMAP_SHARD_SIZE boundaries", async () => {
+      const { app, services } = createSitemapTestApp();
+      const created: string[] = [];
+      for (let i = 0; i < SITEMAP_SHARD_SIZE + 2; i++) {
+        const post = await services.posts.create({
+          format: "note",
+          bodyMarkdown: `p${i}`,
+          status: "published",
+        });
+        created.push(post.slug);
+      }
+
+      const page1 = await (await app.request("/sitemap-posts-1.xml")).text();
+      const page2 = await (await app.request("/sitemap-posts-2.xml")).text();
+
+      // First 500 slugs in page 1
+      expect(page1).toContain(`/${created[0]}`);
+      expect(page1).toContain(`/${created[SITEMAP_SHARD_SIZE - 1]}`);
+      expect(page1).not.toContain(`/${created[SITEMAP_SHARD_SIZE]}`);
+
+      // Remainder in page 2
+      expect(page2).toContain(`/${created[SITEMAP_SHARD_SIZE]}`);
+      expect(page2).toContain(`/${created[SITEMAP_SHARD_SIZE + 1]}`);
+      expect(page2).not.toContain(`/${created[0]}`);
+    });
+
+    it("returns 404 for a shard beyond the last page", async () => {
+      const { app, services } = createSitemapTestApp();
+      await services.posts.create({
+        format: "note",
+        bodyMarkdown: "only",
+        status: "published",
+      });
+
+      const res = await app.request("/sitemap-posts-2.xml");
+      expect(res.status).toBe(404);
+    });
+
+    it("uses long cache for a full shard and short cache for a partial shard", async () => {
+      const { app, services } = createSitemapTestApp();
+      // Exactly one full shard followed by one post in the next shard.
+      for (let i = 0; i < SITEMAP_SHARD_SIZE + 1; i++) {
+        await services.posts.create({
+          format: "note",
+          bodyMarkdown: `p${i}`,
+          status: "published",
+        });
+      }
+
+      const full = await app.request("/sitemap-posts-1.xml");
+      expect(full.headers.get("Cache-Control")).toContain("86400");
+
+      const partial = await app.request("/sitemap-posts-2.xml");
+      expect(partial.headers.get("Cache-Control")).toContain("max-age=180");
+    });
+
+    it("uses alias in <loc> when a post has one", async () => {
+      const { app, services } = createSitemapTestApp();
+      const post = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "body",
+        status: "published",
+      });
+      await services.customUrls.create({
+        path: "my-alias",
+        targetType: "post",
+        targetId: post.id,
+      });
+
+      const xml = await (await app.request("/sitemap-posts-1.xml")).text();
+      expect(xml).toContain(`<loc>${TEST_SITE_ORIGIN}/my-alias</loc>`);
+      expect(xml).not.toContain(`/${post.slug}<`);
+    });
+
+    it("emits absolute URLs anchored to the site origin for nested aliases", async () => {
+      // Regression: `getPostAliases` returns aliases with a leading "/", so
+      // blindly prepending another "/" produces "//blog/foo", which
+      // `new URL()` resolves as protocol-relative and hijacks the hostname
+      // (e.g. `https://blog/foo`).
+      const { app, services } = createSitemapTestApp();
+      const post = await services.posts.create({
+        format: "note",
+        bodyMarkdown: "body",
+        status: "published",
+      });
+      await services.customUrls.create({
+        path: "blog/about-notes",
+        targetType: "post",
+        targetId: post.id,
+      });
+
+      const xml = await (await app.request("/sitemap-posts-1.xml")).text();
+      expect(xml).toContain(`<loc>${TEST_SITE_ORIGIN}/blog/about-notes</loc>`);
+      expect(xml).not.toMatch(/<loc>https?:\/\/blog\//);
+    });
+  });
+
+  describe("/sitemap-collections.xml", () => {
+    it("lists public collections", async () => {
+      const { app, services } = createSitemapTestApp();
+      await services.collections.create({ slug: "reading", title: "Reading" });
+      await services.collections.create({ slug: "movies", title: "Movies" });
+
+      const xml = await (await app.request("/sitemap-collections.xml")).text();
+      expect(xml).toContain("<urlset");
+      expect(xml).toContain("/reading");
+      expect(xml).toContain("/movies");
+    });
+
+    it("does not include the /collections directory page (lives in pages shard)", async () => {
+      const { app, services } = createSitemapTestApp();
+      await services.collections.create({ slug: "reading", title: "Reading" });
+
+      const xml = await (await app.request("/sitemap-collections.xml")).text();
+      // Only per-collection URLs should appear here; the directory landing
+      // is emitted by `/sitemap-pages.xml`.
+      expect(xml).not.toContain("<loc>http://localhost:8787/collections</loc>");
+    });
+
+    it("returns an empty urlset when there are no collections", async () => {
+      const { app } = createSitemapTestApp();
+      const res = await app.request("/sitemap-collections.xml");
+      expect(res.status).toBe(200);
+      const xml = await res.text();
+      expect(xml).toContain("<urlset");
+      expect(xml).not.toContain("<url>");
+    });
+  });
+
+  describe("/sitemap-pages.xml", () => {
+    it("lists the homepage with priority 1.0", async () => {
+      const { app } = createSitemapTestApp();
+      const res = await app.request("/sitemap-pages.xml");
+      expect(res.status).toBe(200);
+      const xml = await res.text();
+      expect(xml).toContain("<urlset");
+      expect(xml).toContain("<priority>1.0</priority>");
+      expect(xml).toContain("<changefreq>daily</changefreq>");
+    });
+
+    it("includes the archive aggregate page", async () => {
+      const { app } = createSitemapTestApp();
+      const xml = await (await app.request("/sitemap-pages.xml")).text();
+      expect(xml).toContain(`${TEST_SITE_ORIGIN}/archive`);
+    });
+
+    it("includes /latest when the homepage default is 'featured'", async () => {
+      const { app, services } = createSitemapTestApp();
+      await services.settings.set("HOME_DEFAULT_VIEW", "featured");
+
+      const xml = await (await app.request("/sitemap-pages.xml")).text();
+      expect(xml).toContain(`${TEST_SITE_ORIGIN}/latest`);
+      expect(xml).not.toContain(`${TEST_SITE_ORIGIN}/featured`);
+    });
+
+    it("includes /featured when the homepage default is 'latest'", async () => {
+      const { app, services } = createSitemapTestApp();
+      await services.settings.set("HOME_DEFAULT_VIEW", "latest");
+
+      const xml = await (await app.request("/sitemap-pages.xml")).text();
+      expect(xml).toContain(`${TEST_SITE_ORIGIN}/featured`);
+      expect(xml).not.toContain(`${TEST_SITE_ORIGIN}/latest`);
+    });
+
+    it("includes /collections only when collections exist", async () => {
+      const { app, services } = createSitemapTestApp();
+
+      const emptyXml = await (await app.request("/sitemap-pages.xml")).text();
+      expect(emptyXml).not.toContain(`${TEST_SITE_ORIGIN}/collections`);
+
+      await services.collections.create({ slug: "reading", title: "Reading" });
+
+      const populatedXml = await (
+        await app.request("/sitemap-pages.xml")
+      ).text();
+      expect(populatedXml).toContain(`${TEST_SITE_ORIGIN}/collections`);
+    });
+  });
 });

package/src/routes/feed/sitemap.ts

@@ -1,60 +1,235 @@
 /**
  * Sitemap Routes
+ *
+ * Sitemap is sharded to keep each shard small, cache-friendly, and stable:
+ *
+ *   /sitemap.xml            → sitemap index listing all shards
+ *   /sitemap-posts-N.xml    → one shard of published non-reply posts
+ *   /sitemap-collections.xml → public collection pages
+ *   /sitemap-pages.xml       → homepage + static aggregate pages
+ *
+ * Post shards are keyset-paginated by post `id` (TypeIDs embed a
+ * creation-ordered UUIDv7 timestamp), so once a shard fills up its membership
+ * never changes: new posts always land in the last shard, never rewriting an
+ * older one. This lets old shards be cached at the edge for a long time.
  */
 
 import { Hono } from "hono";
 import type { Bindings } from "../../types.js";
 import type { AppVariables } from "../../types/app-context.js";
-import { defaultSitemapRenderer } from "../../lib/feed.js";
-import { createMediaContext, toPostViewsFromPosts } from "../../lib/view.js";
+import {
+  renderSitemapIndex,
+  renderSitemapUrlSet,
+  SITEMAP_SHARD_SIZE,
+  type SitemapIndexEntry,
+  type SitemapUrlEntry,
+} from "../../lib/feed.js";
 import { toAbsoluteSiteUrl } from "../../lib/url.js";
 
 type Env = { Bindings: Bindings; Variables: AppVariables };
 
 export const sitemapRoutes = new Hono<Env>();
 
-// XML Sitemap
+const CACHE_SHORT = "public, max-age=180";
+const CACHE_FULL_SHARD = "public, max-age=86400, s-maxage=86400";
+
+function xmlResponse(xml: string, cacheControl: string): Response {
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": cacheControl,
+    },
+  });
+}
+
+/**
+ * Build a public URL entry (absolute URL) from an internal path.
+ */
+function absoluteUrl(
+  internalPath: string,
+  siteUrl: string,
+  sitePathPrefix: string,
+): string {
+  return toAbsoluteSiteUrl(internalPath, siteUrl, sitePathPrefix);
+}
+
+/** Convert a unix-seconds timestamp into a `YYYY-MM-DD` string. */
+function toIsoDate(unixSeconds: number): string {
+  return new Date(unixSeconds * 1000).toISOString().slice(0, 10);
+}
+
+// =============================================================================
+// Sitemap Index
+// =============================================================================
+
 sitemapRoutes.get("/sitemap.xml", async (c) => {
   const { appConfig } = c.var;
-  const siteUrl = appConfig.siteUrl;
+  const { siteUrl, sitePathPrefix } = appConfig;
 
-  const posts = await c.var.services.posts.list({
-    status: "published",
-    excludeReplies: true,
-    excludePrivate: true,
-    limit: 1000,
-  });
+  const postCount = await c.var.services.posts.countForSitemap();
+  const postShardCount = Math.max(1, Math.ceil(postCount / SITEMAP_SHARD_SIZE));
 
-  // Transform to View Models
-  const mediaCtx = createMediaContext(appConfig);
-  const aliasesMap = await c.var.services.paths.getPostAliases(
-    posts.map((p) => p.id),
-  );
-  const aliasMap = new Map<string, string>();
-  for (const [id, aliases] of aliasesMap) {
-    if (aliases[0]) aliasMap.set(id, aliases[0]);
+  const entries: SitemapIndexEntry[] = [
+    { loc: absoluteUrl("/sitemap-pages.xml", siteUrl, sitePathPrefix) },
+  ];
+
+  // Only include post shards if there's at least one post. When postCount is
+  // 0 we still list `sitemap-posts-1.xml` so the site always has a posts
+  // shard — the renderer emits an empty <urlset>, which is valid.
+  for (let page = 1; page <= postShardCount; page++) {
+    entries.push({
+      loc: absoluteUrl(`/sitemap-posts-${page}.xml`, siteUrl, sitePathPrefix),
+    });
   }
-  const postViews = toPostViewsFromPosts(posts, mediaCtx, undefined, aliasMap);
-
-  const xml = defaultSitemapRenderer({
-    siteUrl,
-    sitemapUrl: toAbsoluteSiteUrl(
-      "/sitemap.xml",
-      siteUrl,
-      appConfig.sitePathPrefix,
-    ),
-    posts: postViews,
+
+  const collections = await c.var.services.collections.list();
+  if (collections.length > 0) {
+    entries.push({
+      loc: absoluteUrl("/sitemap-collections.xml", siteUrl, sitePathPrefix),
+    });
+  }
+
+  return xmlResponse(renderSitemapIndex(entries), CACHE_SHORT);
+});
+
+// =============================================================================
+// Post Shards
+// =============================================================================
+
+// Hono's path parser does not allow a param alongside a literal prefix in
+// the same segment (e.g. `/sitemap-posts-:page.xml` does not match). The
+// param must own the whole segment, so we match the full filename with a
+// regex and parse the page number out inside the handler.
+sitemapRoutes.get("/:file{sitemap-posts-[0-9]+\\.xml}", async (c) => {
+  const { appConfig } = c.var;
+  const { siteUrl, sitePathPrefix } = appConfig;
+  const file = c.req.param("file");
+  const match = /^sitemap-posts-([0-9]+)\.xml$/.exec(file);
+  if (!match) return c.notFound();
+  const page = Number(match[1]);
+  if (!Number.isFinite(page) || page < 1) return c.notFound();
+
+  // Keyset cursor: for page N (>1) we want the id just before the shard's
+  // first row, so `listForSitemap({ afterId })` returns the shard. For page
+  // 1 there is no cursor.
+  let afterId: string | undefined;
+  if (page > 1) {
+    const cursorOffset = (page - 1) * SITEMAP_SHARD_SIZE - 1;
+    const cursor = await c.var.services.posts.getSitemapIdAt(cursorOffset);
+    if (cursor === null) return c.notFound();
+    afterId = cursor;
+  }
+
+  const shardEntries = await c.var.services.posts.listForSitemap({
+    afterId,
+    limit: SITEMAP_SHARD_SIZE,
   });
 
-  return new Response(xml, {
-    headers: {
-      "Content-Type": "application/xml; charset=utf-8",
-      "Cache-Control": "public, max-age=180",
+  const urls: SitemapUrlEntry[] = shardEntries.map((entry) => {
+    // `entry.alias` already includes a leading "/" (see
+    // `paths.getPostAliases`); slugs are stored raw. Prepending "/" to an
+    // alias would create "//path" which `new URL()` interprets as
+    // protocol-relative and hijacks the hostname.
+    const path = entry.alias ?? `/${entry.slug}`;
+    return {
+      loc: absoluteUrl(path, siteUrl, sitePathPrefix),
+      lastmod: toIsoDate(entry.updatedAt),
+      priority: entry.featuredAt ? "0.8" : "0.6",
+    };
+  });
+
+  // The last (not-yet-filled) shard needs short caching because new posts
+  // will append to it. Full shards are immutable in membership and can be
+  // cached aggressively — only a post edit inside them moves `<lastmod>`,
+  // which is acceptable sitemap staleness.
+  const isFullShard = shardEntries.length === SITEMAP_SHARD_SIZE;
+  const cacheControl = isFullShard ? CACHE_FULL_SHARD : CACHE_SHORT;
+
+  return xmlResponse(renderSitemapUrlSet(urls), cacheControl);
+});
+
+// =============================================================================
+// Collections Shard
+// =============================================================================
+
+sitemapRoutes.get("/sitemap-collections.xml", async (c) => {
+  const { appConfig } = c.var;
+  const { siteUrl, sitePathPrefix } = appConfig;
+
+  const collections = await c.var.services.collections.list();
+
+  // Resolve each collection's canonical URL (alias if one exists, else slug).
+  // The `/collections` directory itself lives in `/sitemap-pages.xml`, since
+  // it's a static aggregate page rather than per-collection content.
+  const urls: SitemapUrlEntry[] = await Promise.all(
+    collections.map(async (collection) => {
+      const alias = await c.var.services.customUrls.getByTarget(
+        "collection",
+        collection.id,
+      );
+      const path = alias ? `/${alias.path}` : `/${collection.slug}`;
+      return {
+        loc: absoluteUrl(path, siteUrl, sitePathPrefix),
+        lastmod: toIsoDate(collection.updatedAt),
+        priority: "0.7",
+      };
+    }),
+  );
+
+  return xmlResponse(renderSitemapUrlSet(urls), CACHE_SHORT);
+});
+
+// =============================================================================
+// Static Pages Shard (homepage)
+// =============================================================================
+
+sitemapRoutes.get("/sitemap-pages.xml", async (c) => {
+  const { appConfig } = c.var;
+  const { siteUrl, sitePathPrefix, homeDefaultView } = appConfig;
+
+  const urls: SitemapUrlEntry[] = [
+    {
+      loc: absoluteUrl("/", siteUrl, sitePathPrefix),
+      priority: "1.0",
+      changefreq: "daily",
     },
+    {
+      loc: absoluteUrl("/archive", siteUrl, sitePathPrefix),
+      priority: "0.5",
+      changefreq: "weekly",
+    },
+  ];
+
+  // Whichever of /latest and /featured is NOT the homepage default is a
+  // standalone URL worth indexing; the other 302-redirects to `/`.
+  const secondaryAggregate =
+    homeDefaultView === "featured" ? "/latest" : "/featured";
+  urls.push({
+    loc: absoluteUrl(secondaryAggregate, siteUrl, sitePathPrefix),
+    priority: "0.6",
+    changefreq: "daily",
   });
+
+  // Include the collections directory landing page when at least one
+  // collection exists. When there are no collections, `/collections` still
+  // renders (as an empty directory), but indexing an empty aggregate page
+  // adds no value.
+  const collections = await c.var.services.collections.list();
+  if (collections.length > 0) {
+    urls.push({
+      loc: absoluteUrl("/collections", siteUrl, sitePathPrefix),
+      priority: "0.5",
+      changefreq: "weekly",
+    });
+  }
+
+  return xmlResponse(renderSitemapUrlSet(urls), CACHE_SHORT);
 });
 
+// =============================================================================
 // robots.txt
+// =============================================================================
+
 sitemapRoutes.get("/robots.txt", async (c) => {
   const { appConfig } = c.var;
   const siteUrl = appConfig.siteUrl;
@@ -74,7 +249,7 @@ sitemapRoutes.get("/robots.txt", async (c) => {
   return new Response(robots, {
     headers: {
       "Content-Type": "text/plain; charset=utf-8",
-      "Cache-Control": "public, max-age=180",
+      "Cache-Control": CACHE_SHORT,
     },
   });
 });