@jant/core 0.3.42 → 0.3.43

package/src/routes/feed/sitemap.ts

@@ -1,60 +1,235 @@
 /**
  * Sitemap Routes
+ *
+ * Sitemap is sharded to keep each shard small, cache-friendly, and stable:
+ *
+ *   /sitemap.xml            → sitemap index listing all shards
+ *   /sitemap-posts-N.xml    → one shard of published non-reply posts
+ *   /sitemap-collections.xml → public collection pages
+ *   /sitemap-pages.xml       → homepage + static aggregate pages
+ *
+ * Post shards are keyset-paginated by post `id` (TypeIDs embed a
+ * creation-ordered UUIDv7 timestamp), so once a shard fills up its membership
+ * never changes: new posts always land in the last shard, never rewriting an
+ * older one. This lets old shards be cached at the edge for a long time.
  */
 
 import { Hono } from "hono";
 import type { Bindings } from "../../types.js";
 import type { AppVariables } from "../../types/app-context.js";
-import { defaultSitemapRenderer } from "../../lib/feed.js";
-import { createMediaContext, toPostViewsFromPosts } from "../../lib/view.js";
+import {
+  renderSitemapIndex,
+  renderSitemapUrlSet,
+  SITEMAP_SHARD_SIZE,
+  type SitemapIndexEntry,
+  type SitemapUrlEntry,
+} from "../../lib/feed.js";
 import { toAbsoluteSiteUrl } from "../../lib/url.js";
 
 type Env = { Bindings: Bindings; Variables: AppVariables };
 
 export const sitemapRoutes = new Hono<Env>();
 
-// XML Sitemap
+const CACHE_SHORT = "public, max-age=180";
+const CACHE_FULL_SHARD = "public, max-age=86400, s-maxage=86400";
+
+function xmlResponse(xml: string, cacheControl: string): Response {
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": cacheControl,
+    },
+  });
+}
+
+/**
+ * Build a public URL entry (absolute URL) from an internal path.
+ */
+function absoluteUrl(
+  internalPath: string,
+  siteUrl: string,
+  sitePathPrefix: string,
+): string {
+  return toAbsoluteSiteUrl(internalPath, siteUrl, sitePathPrefix);
+}
+
+/** Convert a unix-seconds timestamp into a `YYYY-MM-DD` string. */
+function toIsoDate(unixSeconds: number): string {
+  return new Date(unixSeconds * 1000).toISOString().slice(0, 10);
+}
+
+// =============================================================================
+// Sitemap Index
+// =============================================================================
+
 sitemapRoutes.get("/sitemap.xml", async (c) => {
   const { appConfig } = c.var;
-  const siteUrl = appConfig.siteUrl;
+  const { siteUrl, sitePathPrefix } = appConfig;
 
-  const posts = await c.var.services.posts.list({
-    status: "published",
-    excludeReplies: true,
-    excludePrivate: true,
-    limit: 1000,
-  });
+  const postCount = await c.var.services.posts.countForSitemap();
+  const postShardCount = Math.max(1, Math.ceil(postCount / SITEMAP_SHARD_SIZE));
 
-  // Transform to View Models
-  const mediaCtx = createMediaContext(appConfig);
-  const aliasesMap = await c.var.services.paths.getPostAliases(
-    posts.map((p) => p.id),
-  );
-  const aliasMap = new Map<string, string>();
-  for (const [id, aliases] of aliasesMap) {
-    if (aliases[0]) aliasMap.set(id, aliases[0]);
+  const entries: SitemapIndexEntry[] = [
+    { loc: absoluteUrl("/sitemap-pages.xml", siteUrl, sitePathPrefix) },
+  ];
+
+  // Only include post shards if there's at least one post. When postCount is
+  // 0 we still list `sitemap-posts-1.xml` so the site always has a posts
+  // shard — the renderer emits an empty <urlset>, which is valid.
+  for (let page = 1; page <= postShardCount; page++) {
+    entries.push({
+      loc: absoluteUrl(`/sitemap-posts-${page}.xml`, siteUrl, sitePathPrefix),
+    });
   }
-  const postViews = toPostViewsFromPosts(posts, mediaCtx, undefined, aliasMap);
-
-  const xml = defaultSitemapRenderer({
-    siteUrl,
-    sitemapUrl: toAbsoluteSiteUrl(
-      "/sitemap.xml",
-      siteUrl,
-      appConfig.sitePathPrefix,
-    ),
-    posts: postViews,
+
+  const collections = await c.var.services.collections.list();
+  if (collections.length > 0) {
+    entries.push({
+      loc: absoluteUrl("/sitemap-collections.xml", siteUrl, sitePathPrefix),
+    });
+  }
+
+  return xmlResponse(renderSitemapIndex(entries), CACHE_SHORT);
+});
+
+// =============================================================================
+// Post Shards
+// =============================================================================
+
+// Hono's path parser does not allow a param alongside a literal prefix in
+// the same segment (e.g. `/sitemap-posts-:page.xml` does not match). The
+// param must own the whole segment, so we match the full filename with a
+// regex and parse the page number out inside the handler.
+sitemapRoutes.get("/:file{sitemap-posts-[0-9]+\\.xml}", async (c) => {
+  const { appConfig } = c.var;
+  const { siteUrl, sitePathPrefix } = appConfig;
+  const file = c.req.param("file");
+  const match = /^sitemap-posts-([0-9]+)\.xml$/.exec(file);
+  if (!match) return c.notFound();
+  const page = Number(match[1]);
+  if (!Number.isFinite(page) || page < 1) return c.notFound();
+
+  // Keyset cursor: for page N (>1) we want the id just before the shard's
+  // first row, so `listForSitemap({ afterId })` returns the shard. For page
+  // 1 there is no cursor.
+  let afterId: string | undefined;
+  if (page > 1) {
+    const cursorOffset = (page - 1) * SITEMAP_SHARD_SIZE - 1;
+    const cursor = await c.var.services.posts.getSitemapIdAt(cursorOffset);
+    if (cursor === null) return c.notFound();
+    afterId = cursor;
+  }
+
+  const shardEntries = await c.var.services.posts.listForSitemap({
+    afterId,
+    limit: SITEMAP_SHARD_SIZE,
   });
 
-  return new Response(xml, {
-    headers: {
-      "Content-Type": "application/xml; charset=utf-8",
-      "Cache-Control": "public, max-age=180",
+  const urls: SitemapUrlEntry[] = shardEntries.map((entry) => {
+    // `entry.alias` already includes a leading "/" (see
+    // `paths.getPostAliases`); slugs are stored raw. Prepending "/" to an
+    // alias would create "//path" which `new URL()` interprets as
+    // protocol-relative and hijacks the hostname.
+    const path = entry.alias ?? `/${entry.slug}`;
+    return {
+      loc: absoluteUrl(path, siteUrl, sitePathPrefix),
+      lastmod: toIsoDate(entry.updatedAt),
+      priority: entry.featuredAt ? "0.8" : "0.6",
+    };
+  });
+
+  // The last (not-yet-filled) shard needs short caching because new posts
+  // will append to it. Full shards are immutable in membership and can be
+  // cached aggressively — only a post edit inside them moves `<lastmod>`,
+  // which is acceptable sitemap staleness.
+  const isFullShard = shardEntries.length === SITEMAP_SHARD_SIZE;
+  const cacheControl = isFullShard ? CACHE_FULL_SHARD : CACHE_SHORT;
+
+  return xmlResponse(renderSitemapUrlSet(urls), cacheControl);
+});
+
+// =============================================================================
+// Collections Shard
+// =============================================================================
+
+sitemapRoutes.get("/sitemap-collections.xml", async (c) => {
+  const { appConfig } = c.var;
+  const { siteUrl, sitePathPrefix } = appConfig;
+
+  const collections = await c.var.services.collections.list();
+
+  // Resolve each collection's canonical URL (alias if one exists, else slug).
+  // The `/collections` directory itself lives in `/sitemap-pages.xml`, since
+  // it's a static aggregate page rather than per-collection content.
+  const urls: SitemapUrlEntry[] = await Promise.all(
+    collections.map(async (collection) => {
+      const alias = await c.var.services.customUrls.getByTarget(
+        "collection",
+        collection.id,
+      );
+      const path = alias ? `/${alias.path}` : `/${collection.slug}`;
+      return {
+        loc: absoluteUrl(path, siteUrl, sitePathPrefix),
+        lastmod: toIsoDate(collection.updatedAt),
+        priority: "0.7",
+      };
+    }),
+  );
+
+  return xmlResponse(renderSitemapUrlSet(urls), CACHE_SHORT);
+});
+
+// =============================================================================
+// Static Pages Shard (homepage)
+// =============================================================================
+
+sitemapRoutes.get("/sitemap-pages.xml", async (c) => {
+  const { appConfig } = c.var;
+  const { siteUrl, sitePathPrefix, homeDefaultView } = appConfig;
+
+  const urls: SitemapUrlEntry[] = [
+    {
+      loc: absoluteUrl("/", siteUrl, sitePathPrefix),
+      priority: "1.0",
+      changefreq: "daily",
     },
+    {
+      loc: absoluteUrl("/archive", siteUrl, sitePathPrefix),
+      priority: "0.5",
+      changefreq: "weekly",
+    },
+  ];
+
+  // Whichever of /latest and /featured is NOT the homepage default is a
+  // standalone URL worth indexing; the other 302-redirects to `/`.
+  const secondaryAggregate =
+    homeDefaultView === "featured" ? "/latest" : "/featured";
+  urls.push({
+    loc: absoluteUrl(secondaryAggregate, siteUrl, sitePathPrefix),
+    priority: "0.6",
+    changefreq: "daily",
   });
+
+  // Include the collections directory landing page when at least one
+  // collection exists. When there are no collections, `/collections` still
+  // renders (as an empty directory), but indexing an empty aggregate page
+  // adds no value.
+  const collections = await c.var.services.collections.list();
+  if (collections.length > 0) {
+    urls.push({
+      loc: absoluteUrl("/collections", siteUrl, sitePathPrefix),
+      priority: "0.5",
+      changefreq: "weekly",
+    });
+  }
+
+  return xmlResponse(renderSitemapUrlSet(urls), CACHE_SHORT);
 });
 
+// =============================================================================
 // robots.txt
+// =============================================================================
+
 sitemapRoutes.get("/robots.txt", async (c) => {
   const { appConfig } = c.var;
   const siteUrl = appConfig.siteUrl;
@@ -74,7 +249,7 @@ sitemapRoutes.get("/robots.txt", async (c) => {
   return new Response(robots, {
     headers: {
       "Content-Type": "text/plain; charset=utf-8",
-      "Cache-Control": "public, max-age=180",
+      "Cache-Control": CACHE_SHORT,
     },
   });
 });

package/src/routes/pages/__tests__/page-canonical.test.ts

@@ -0,0 +1,101 @@
+/**
+ * Tests for the `<link rel="canonical">` tag on post pages.
+ *
+ * Reply URLs render the full thread, so each reply URL would otherwise look
+ * like duplicate content to crawlers. The canonical tag points every page in
+ * the thread back to the thread root.
+ */
+
+import { describe, expect, it } from "vitest";
+import { createTestApp } from "../../../__tests__/helpers/app.js";
+import { pageRoutes } from "../page.js";
+
+function createPageTestApp() {
+  const testApp = createTestApp();
+  const { app } = testApp;
+
+  app.use("*", async (c, next) => {
+    c.set("publicPath", c.req.path);
+    c.set("publicRequestUrl", c.req.url);
+    await next();
+  });
+
+  app.route("/", pageRoutes);
+
+  return testApp;
+}
+
+function extractCanonicalHref(html: string): string | null {
+  const match = html.match(/<link\s+rel="canonical"\s+href="([^"]+)"\s*\/?>/i);
+  return match?.[1] ?? null;
+}
+
+describe("Post page canonical link", () => {
+  it("root post canonical points at its own permalink", async () => {
+    const { app, services } = createPageTestApp();
+
+    const root = await services.posts.create({
+      format: "note",
+      title: "Root post",
+      bodyMarkdown: "Root body",
+      status: "published",
+    });
+
+    const res = await app.request(`/${root.slug}`);
+    expect(res.status).toBe(200);
+
+    const html = await res.text();
+    const canonical = extractCanonicalHref(html);
+    expect(canonical).not.toBeNull();
+    expect(canonical).toMatch(new RegExp(`/${root.slug}$`));
+  });
+
+  it("reply canonical points back to the thread root", async () => {
+    const { app, services } = createPageTestApp();
+
+    const root = await services.posts.create({
+      format: "note",
+      title: "Thread root",
+      bodyMarkdown: "Root body",
+      status: "published",
+    });
+    const reply = await services.posts.create({
+      format: "note",
+      bodyMarkdown: "Reply body",
+      replyToId: root.id,
+      status: "published",
+    });
+
+    // Visiting the reply URL should canonicalize to the root URL.
+    const replyRes = await app.request(`/${reply.slug}`);
+    expect(replyRes.status).toBe(200);
+
+    const replyHtml = await replyRes.text();
+    const replyCanonical = extractCanonicalHref(replyHtml);
+    expect(replyCanonical).not.toBeNull();
+    expect(replyCanonical).toMatch(new RegExp(`/${root.slug}$`));
+    expect(replyCanonical).not.toMatch(new RegExp(`/${reply.slug}$`));
+
+    // And the root URL should canonicalize to itself.
+    const rootRes = await app.request(`/${root.slug}`);
+    const rootCanonical = extractCanonicalHref(await rootRes.text());
+    expect(rootCanonical).toMatch(new RegExp(`/${root.slug}$`));
+  });
+
+  it("canonical is absolute when siteUrl is configured", async () => {
+    const { app, services } = createPageTestApp();
+
+    const post = await services.posts.create({
+      format: "note",
+      title: "Absolute test",
+      bodyMarkdown: "Body",
+      status: "published",
+    });
+
+    const res = await app.request(`/${post.slug}`);
+    const canonical = extractCanonicalHref(await res.text());
+    // SITE_ORIGIN in the test harness is http://localhost:<port>
+    expect(canonical).toMatch(/^https?:\/\//);
+    expect(canonical).toContain(`/${post.slug}`);
+  });
+});

package/src/routes/pages/home.tsx

@@ -12,7 +12,10 @@ import { Hono } from "hono";
 import { msg } from "@lingui/core/macro";
 import type { Bindings } from "../../types.js";
 import type { AppVariables } from "../../types/app-context.js";
-import { getNavigationData } from "../../lib/navigation.js";
+import {
+  getHomeDefaultViewFromNavItems,
+  getNavigationData,
+} from "../../lib/navigation.js";
 import { getI18n } from "../../i18n/index.js";
 import { formatPageLabel, parsePageNumber } from "../../lib/pagination.js";
 import { buildPageTitle } from "../../lib/page-title.js";
@@ -30,25 +33,36 @@ type Env = { Bindings: Bindings; Variables: AppVariables };
 export const homeRoutes = new Hono<Env>();
 
 homeRoutes.get("/", async (c) => {
-  const navData = await getNavigationData(c);
   const i18n = getI18n(c);
   const page = parsePageNumber(c.req.query("page"));
   const paginatedPageTitle = formatPageLabel(page);
+  const isAuthenticated = c.var.isAuthenticated;
+
+  // Fetch nav items once — we need `homeDefaultView` to decide which timeline
+  // to assemble, but `getNavigationData` also consumes them. Passing them
+  // through avoids a duplicate DB query and unlocks the Promise.all below.
+  const navItems = await c.var.services.navItems.list();
+  const homeDefaultView = getHomeDefaultViewFromNavItems(navItems);
+
+  const timelinePromise =
+    homeDefaultView === "featured"
+      ? assembleFeaturedTimeline(c, { page, isAuthenticated })
+      : assembleTimeline(c, { page, isAuthenticated });
+
+  const [navData, timeline] = await Promise.all([
+    getNavigationData(c, { preloadedItems: navItems }),
+    timelinePromise,
+  ]);
 
-  if (navData.homeDefaultView === "featured") {
+  const { items, currentPage, totalPages } = timeline;
+
+  if (homeDefaultView === "featured") {
     const featuredTitle = i18n._(
       msg({
         message: "Featured",
         comment: "@context: Browser page title for the featured feed",
       }),
     );
-    const { items, currentPage, totalPages } = await assembleFeaturedTimeline(
-      c,
-      {
-        page,
-        isAuthenticated: navData.isAuthenticated,
-      },
-    );
 
     return renderPublicPage(c, {
       title:
@@ -76,11 +90,6 @@ homeRoutes.get("/", async (c) => {
     }),
   );
 
-  const { items, currentPage, totalPages } = await assembleTimeline(c, {
-    page,
-    isAuthenticated: navData.isAuthenticated,
-  });
-
   return renderPublicPage(c, {
     title:
       page > 1

package/src/routes/pages/page.tsx

@@ -33,6 +33,28 @@ interface TextPreviewAutoOpen {
   mediaId: string;
 }
 
+/**
+ * Build the canonical absolute URL for a post page.
+ *
+ * Reply URLs render the full thread, so search engines see overlapping
+ * content at every reply URL. Point the canonical to the thread root so
+ * crawlers consolidate ranking on one URL.
+ *
+ * The root post is always at index 0 of `threadPostViews` (getThread orders
+ * by createdAt ASC, and the DB check constraint guarantees root has the
+ * smallest createdAt in its thread). When `threadPostViews` is undefined the
+ * post is not part of a multi-post thread, so the post itself is the root.
+ */
+function buildPostCanonicalHref(
+  postView: { permalink: string },
+  threadPostViews: Array<{ permalink: string }> | undefined,
+  siteUrl: string,
+): string {
+  const rootPermalink = threadPostViews?.[0]?.permalink ?? postView.permalink;
+  if (!siteUrl) return rootPermalink;
+  return new URL(rootPermalink, siteUrl).toString();
+}
+
 async function renderPostWithTextPreview(
   c: Context<Env>,
   post: Post,
@@ -48,6 +70,11 @@ async function renderPostWithTextPreview(
 
   const navData = await navDataPromise;
   const meta = buildPostMeta(post, navData.siteName);
+  const canonicalHref = buildPostCanonicalHref(
+    display.postView,
+    display.threadPostViews,
+    c.var.appConfig.siteUrl,
+  );
 
   // Use the attachment summary as the page title (for OG/link previews),
   // and pass the post title in the payload so the client can restore it
@@ -68,6 +95,7 @@ async function renderPostWithTextPreview(
   return renderPublicPage(c, {
     title: pageTitle,
     description: meta.description,
+    canonicalHref,
     navData,
     content: (
       <>
@@ -127,10 +155,16 @@ async function renderPost(c: Context<Env>, post: Post) {
 
   const navData = await navDataPromise;
   const meta = buildPostMeta(post, navData.siteName);
+  const canonicalHref = buildPostCanonicalHref(
+    display.postView,
+    display.threadPostViews,
+    c.var.appConfig.siteUrl,
+  );
 
   return renderPublicPage(c, {
     title: meta.title,
     description: meta.description,
+    canonicalHref,
     navData,
     content: (
       <PostPage post={display.postView} threadPosts={display.threadPostViews} />

package/src/routes/pages/partials.tsx

@@ -1,4 +1,4 @@
-import { Hono, type Context } from "hono";
+import { Hono } from "hono";
 import { I18nProvider } from "../../i18n/index.js";
 import { parseIdParam } from "../../lib/errors.js";
 import { ID_PREFIX } from "../../lib/ids.js";
@@ -18,17 +18,6 @@ type Env = { Bindings: Bindings; Variables: AppVariables };
 
 export const partialPageRoutes = new Hono<Env>();
 
-async function getIsAuthenticated(c: Context<Env>): Promise<boolean> {
-  try {
-    const session = await c.var.auth.api.getSession({
-      headers: c.req.raw.headers,
-    });
-    return !!session?.user;
-  } catch {
-    return false;
-  }
-}
-
 partialPageRoutes.get("/_/version", (c) => {
   return c.json({ version: CORE_VERSION });
 });
@@ -39,7 +28,7 @@ partialPageRoutes.get("/_/timeline-item/:threadRootId", async (c) => {
     ID_PREFIX.post,
   );
   const item = await assembleTimelineItem(c, threadRootId, {
-    isAuthenticated: await getIsAuthenticated(c),
+    isAuthenticated: c.var.isAuthenticated,
   });
 
   if (!item) {
@@ -56,7 +45,7 @@ partialPageRoutes.get("/_/timeline-item/:threadRootId", async (c) => {
 partialPageRoutes.get("/_/post-card/:postId", async (c) => {
   const postId = parseIdParam(c.req.param("postId"), ID_PREFIX.post);
   const postView = await assemblePostCardView(c, postId, {
-    isAuthenticated: await getIsAuthenticated(c),
+    isAuthenticated: c.var.isAuthenticated,
   });
 
   if (!postView) {
@@ -73,7 +62,7 @@ partialPageRoutes.get("/_/post-card/:postId", async (c) => {
 partialPageRoutes.get("/_/post-view/:postId", async (c) => {
   const postId = parseIdParam(c.req.param("postId"), ID_PREFIX.post);
   const display = await assemblePostPageDisplay(c, postId, {
-    isAuthenticated: await getIsAuthenticated(c),
+    isAuthenticated: c.var.isAuthenticated,
   });
 
   if (!display) {

package/src/runtime/cloudflare.ts

@@ -10,6 +10,8 @@ import {
   shouldUseSecureCookies,
 } from "../lib/env.js";
 import { createHostedControlPlaneClient } from "../lib/hosted-control-plane.js";
+import { createD1RateLimiter } from "../lib/rate-limit-d1.js";
+import type { RateLimiter } from "../lib/rate-limit.js";
 import { createStorageDriver, type StorageDriver } from "../lib/storage.js";
 import {
   createHostedHandoffService,
@@ -30,6 +32,7 @@ export interface CloudflareRequestRuntime {
   currentSiteDomain: SiteDomain | null;
   db: Database;
   hostedHandoff: HostedHandoffService;
+  rateLimiter: RateLimiter;
   services: Services;
   storage: StorageDriver | null;
 }
@@ -88,6 +91,7 @@ export async function createCloudflareRequestRuntime(
       schema: sqliteSchemaBundle,
       secret: hostedControlPlaneSsoSecret,
     }),
+    rateLimiter: createD1RateLimiter(db, sqliteSchemaBundle),
     services: createServices(db, session, siteLookup.site.id, {
       databaseDialect: "sqlite",
       bootstrapSite: getSingleSiteBootstrapOptions(env),

package/src/runtime/node.ts

@@ -12,6 +12,8 @@ import {
   shouldUseSecureCookies,
 } from "../lib/env.js";
 import { createHostedControlPlaneClient } from "../lib/hosted-control-plane.js";
+import { createMemoryRateLimiter } from "../lib/rate-limit-memory.js";
+import type { RateLimiter } from "../lib/rate-limit.js";
 import { createStorageDriver, type StorageDriver } from "../lib/storage.js";
 import {
   createHostedHandoffService,
@@ -33,10 +35,23 @@ export interface NodeRequestRuntime {
   currentSiteDomain: SiteDomain | null;
   db: Database;
   hostedHandoff: HostedHandoffService;
+  rateLimiter: RateLimiter;
   services: Services;
   storage: StorageDriver | null;
 }
 
+/**
+ * Single process-wide rate limiter for the Node runtime. Node serves all
+ * requests out of one persistent process, so in-memory counters are
+ * reliable and avoid per-request D1 round-trips. Constructed lazily on
+ * first use so tests that never build a request runtime don't pay for it.
+ */
+let sharedNodeRateLimiter: RateLimiter | null = null;
+function getNodeRateLimiter(): RateLimiter {
+  sharedNodeRateLimiter ??= createMemoryRateLimiter();
+  return sharedNodeRateLimiter;
+}
+
 export interface NodeCliRuntime {
   currentSite: Site;
   currentSiteDomain: SiteDomain | null;
@@ -131,6 +146,7 @@ export async function createNodeRequestRuntime(
       schema: databaseSchema,
       secret: hostedControlPlaneSsoSecret,
     }),
+    rateLimiter: getNodeRateLimiter(),
     services: createServices(db, rawQuery, siteLookup.site.id, {
       databaseDialect,
       bootstrapSite: getSingleSiteBootstrapOptions(env),