@jant/core 0.3.42 → 0.3.43

package/src/db/migrations/pg/meta/_journal.json

@@ -113,6 +113,13 @@
       "when": 1776405301273,
       "tag": "0015_daffy_mikhail_rasputin",
       "breakpoints": true
+    },
+    {
+      "idx": 16,
+      "version": "7",
+      "when": 1776685653392,
+      "tag": "0016_familiar_lionheart",
+      "breakpoints": true
     }
   ]
-}
+}

package/src/db/pg/schema.ts

@@ -855,3 +855,21 @@ export const githubAppInstallation = pgTable(
     ),
   ],
 );
+
+// ---------------------------------------------------------------------------
+// Rate Limit
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-key sliding-window rate-limit counters. Mirrors the SQLite `rate_limit`
+ * table — kept in lockstep because both dialects are production targets.
+ */
+export const rateLimit = pgTable(
+  "rate_limit",
+  {
+    key: text("key").notNull(),
+    windowStart: integer("window_start").notNull(),
+    count: integer("count").notNull().default(0),
+  },
+  (table) => [primaryKey({ columns: [table.key, table.windowStart] })],
+);

package/src/db/schema.ts

@@ -813,3 +813,26 @@ export const githubAppInstallation = sqliteTable(
     ),
   ],
 );
+
+// =============================================================================
+// Rate Limit
+// =============================================================================
+
+/**
+ * Per-key sliding-window rate-limit counters. Used by the Cloudflare Workers
+ * runtime; Node deployments keep the state in memory instead.
+ *
+ * `key` is typically a scope prefix plus client IP (e.g. "search:1.2.3.4").
+ * `window_start` is the aligned start of the window in Unix seconds — we
+ * store two adjacent windows per key to support the sliding-window-counter
+ * algorithm.
+ */
+export const rateLimit = sqliteTable(
+  "rate_limit",
+  {
+    key: text("key").notNull(),
+    windowStart: integer("window_start").notNull(),
+    count: integer("count").notNull().default(0),
+  },
+  (table) => [primaryKey({ columns: [table.key, table.windowStart] })],
+);

package/src/index.ts

@@ -44,7 +44,6 @@ export type {
   ArchiveFilters,
   // Feed types
   FeedData,
-  SitemapData,
   // Search
   SearchResult,
 } from "./types.js";
@@ -82,7 +81,7 @@ export {
 export type { MediaContext } from "./lib/view.js";
 
 // Default feed renderers (for custom feed implementations)
-export { defaultFeedRenderer, defaultSitemapRenderer } from "./lib/feed.js";
+export { defaultFeedRenderer } from "./lib/feed.js";
 
 // GitHub Sync queue handler (for Cloudflare Workers queue consumer)
 export { handleQueueBatch as handleGitHubSyncQueueBatch } from "./lib/github-sync-queue-handler.js";

package/src/lib/__tests__/hosted-signin.test.ts

@@ -26,6 +26,36 @@ describe("getHostedControlPlaneSigninUrl", () => {
     );
   });
 
+  it("forwards a safe post-signin redirect through the handoff URL", () => {
+    const url = getHostedControlPlaneSigninUrl(
+      {
+        HOSTED_CONTROL_PLANE_BASE_URL: "https://cloud-jant.localtest.me",
+        SITE_RESOLUTION_MODE: "host-based",
+      },
+      "https://site7.localtest.me/signin",
+      "/settings/general",
+    );
+
+    expect(url).toBe(
+      "https://cloud-jant.localtest.me/auth/handoff/start?host=site7.localtest.me&redirect=%2Fsettings%2Fgeneral",
+    );
+  });
+
+  it("falls back to / when the supplied redirect is unsafe", () => {
+    const url = getHostedControlPlaneSigninUrl(
+      {
+        HOSTED_CONTROL_PLANE_BASE_URL: "https://cloud-jant.localtest.me",
+        SITE_RESOLUTION_MODE: "host-based",
+      },
+      "https://site7.localtest.me/signin",
+      "//evil.example/steal",
+    );
+
+    expect(url).toBe(
+      "https://cloud-jant.localtest.me/auth/handoff/start?host=site7.localtest.me&redirect=%2F",
+    );
+  });
+
   it("returns null outside host-based mode", () => {
     const url = getHostedControlPlaneSigninUrl(
       {

package/src/lib/__tests__/navigation.test.ts

@@ -56,16 +56,8 @@ describe("getNavigationData", () => {
             listByRecentActivity: async () => [],
           },
         },
-        auth: {
-          api: {
-            getSession: async () => null,
-          },
-        },
-      },
-      req: {
-        raw: {
-          headers: new Headers(),
-        },
+        isAuthenticated: false,
+        session: null,
       },
     } as unknown as Context;
 
@@ -126,16 +118,8 @@ describe("getNavigationData", () => {
             listByRecentActivity: async () => [],
           },
         },
-        auth: {
-          api: {
-            getSession: async () => null,
-          },
-        },
-      },
-      req: {
-        raw: {
-          headers: new Headers(),
-        },
+        isAuthenticated: false,
+        session: null,
       },
     } as unknown as Context;
 

package/src/lib/__tests__/rate-limit-d1.test.ts

@@ -0,0 +1,82 @@
+import { describe, it, expect } from "vitest";
+import { createTestDatabase } from "../../__tests__/helpers/db.js";
+import { sqliteSchemaBundle } from "../../db/schema-bundle.js";
+import type { Database } from "../../db/index.js";
+import { createD1RateLimiter } from "../rate-limit-d1.js";
+
+function createLimiter(now: () => number) {
+  const testDb = createTestDatabase();
+  const db = testDb.db as unknown as Database;
+  return {
+    limiter: createD1RateLimiter(db, sqliteSchemaBundle, now),
+    db,
+    sqlite: testDb.sqlite,
+  };
+}
+
+describe("createD1RateLimiter", () => {
+  it("allows requests under the limit", async () => {
+    const { limiter } = createLimiter(() => 1_000);
+    for (let i = 0; i < 3; i++) {
+      const result = await limiter.check("k", { limit: 3, windowSec: 60 });
+      expect(result.ok).toBe(true);
+    }
+  });
+
+  it("rejects requests at or over the limit", async () => {
+    const { limiter } = createLimiter(() => 1_000);
+    await limiter.check("k", { limit: 2, windowSec: 60 });
+    await limiter.check("k", { limit: 2, windowSec: 60 });
+    const blocked = await limiter.check("k", { limit: 2, windowSec: 60 });
+    expect(blocked.ok).toBe(false);
+    expect(blocked.retryAfterSec).toBeGreaterThan(0);
+  });
+
+  it("does not persist a row when a request is rejected", async () => {
+    const { limiter, sqlite } = createLimiter(() => 1_000);
+    await limiter.check("k", { limit: 1, windowSec: 60 });
+    await limiter.check("k", { limit: 1, windowSec: 60 }); // rejected
+
+    // Only one write should have happened — the first (allowed) one.
+    const row = sqlite
+      .prepare("SELECT count FROM rate_limit WHERE key = 'k'")
+      .get() as { count: number };
+    expect(row.count).toBe(1);
+  });
+
+  it("keeps keys independent", async () => {
+    const { limiter } = createLimiter(() => 1_000);
+    await limiter.check("a", { limit: 1, windowSec: 60 });
+    const b = await limiter.check("b", { limit: 1, windowSec: 60 });
+    expect(b.ok).toBe(true);
+  });
+
+  it("releases capacity after two full windows", async () => {
+    let now = 960;
+    const { limiter } = createLimiter(() => now);
+    await limiter.check("k", { limit: 1, windowSec: 60 });
+    const blocked = await limiter.check("k", { limit: 1, windowSec: 60 });
+    expect(blocked.ok).toBe(false);
+
+    now += 60 * 2;
+    const released = await limiter.check("k", { limit: 1, windowSec: 60 });
+    expect(released.ok).toBe(true);
+  });
+
+  it("applies previous-window weighting across a boundary", async () => {
+    let now = 960;
+    const { limiter } = createLimiter(() => now);
+    for (let i = 0; i < 10; i++) {
+      const r = await limiter.check("k", { limit: 10, windowSec: 60 });
+      expect(r.ok).toBe(true);
+    }
+
+    now = 960 + 60; // t=0 into next window, prev weight = 1.0
+    const justAfter = await limiter.check("k", { limit: 10, windowSec: 60 });
+    expect(justAfter.ok).toBe(false);
+
+    now = 960 + 60 + 59; // prev weight ≈ 1/60, estimate ≈ 0.167
+    const released = await limiter.check("k", { limit: 10, windowSec: 60 });
+    expect(released.ok).toBe(true);
+  });
+});

package/src/lib/__tests__/rate-limit-memory.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from "vitest";
+import { createMemoryRateLimiter } from "../rate-limit-memory.js";
+
+describe("createMemoryRateLimiter", () => {
+  it("allows requests under the limit", async () => {
+    const limiter = createMemoryRateLimiter(() => 1_000);
+    for (let i = 0; i < 3; i++) {
+      const result = await limiter.check("k", { limit: 3, windowSec: 60 });
+      expect(result.ok).toBe(true);
+    }
+  });
+
+  it("rejects requests at or over the limit", async () => {
+    const limiter = createMemoryRateLimiter(() => 1_000);
+    await limiter.check("k", { limit: 2, windowSec: 60 });
+    await limiter.check("k", { limit: 2, windowSec: 60 });
+    const blocked = await limiter.check("k", { limit: 2, windowSec: 60 });
+    expect(blocked.ok).toBe(false);
+    expect(blocked.retryAfterSec).toBeGreaterThan(0);
+  });
+
+  it("keeps keys independent", async () => {
+    const limiter = createMemoryRateLimiter(() => 1_000);
+    await limiter.check("a", { limit: 1, windowSec: 60 });
+    // "a" is now at limit, but "b" should still pass.
+    const b = await limiter.check("b", { limit: 1, windowSec: 60 });
+    expect(b.ok).toBe(true);
+  });
+
+  it("releases capacity after the window rolls over", async () => {
+    let now = 1_000;
+    const limiter = createMemoryRateLimiter(() => now);
+
+    // Fill the current window.
+    await limiter.check("k", { limit: 1, windowSec: 60 });
+    const blocked = await limiter.check("k", { limit: 1, windowSec: 60 });
+    expect(blocked.ok).toBe(false);
+
+    // Jump two full windows into the future — previous window carries no
+    // weight, so capacity fully resets.
+    now += 60 * 2;
+    const released = await limiter.check("k", { limit: 1, windowSec: 60 });
+    expect(released.ok).toBe(true);
+  });
+
+  it("applies previous-window weighting across a boundary", async () => {
+    // Start aligned to a window boundary so window math is exact.
+    // limit=10, windowSec=60. Fill the first window to the limit, then
+    // step to the start of the next window. At t=0 into the new window,
+    // the prev-window weight is 1.0, so the sliding estimate equals the
+    // prev count (10) and the next request should still be rejected.
+    let now = 960;
+    const limiter = createMemoryRateLimiter(() => now);
+    for (let i = 0; i < 10; i++) {
+      const r = await limiter.check("k", { limit: 10, windowSec: 60 });
+      expect(r.ok).toBe(true);
+    }
+
+    now = 960 + 60; // exact start of the next window
+    const justAfter = await limiter.check("k", { limit: 10, windowSec: 60 });
+    expect(justAfter.ok).toBe(false);
+
+    // Well into the next window the prev-window weight decays; capacity
+    // should become available again.
+    now = 960 + 60 + 59; // 59s into the new window, weight ≈ 1/60
+    const released = await limiter.check("k", { limit: 10, windowSec: 60 });
+    expect(released.ok).toBe(true);
+  });
+});

package/src/lib/__tests__/summary.test.ts

@@ -246,6 +246,115 @@ describe("extractBodyText", () => {
     expect(result).toContain("Below rule");
   });
 
+  it("includes link mark hrefs when includeLinkHrefs is true", () => {
+    const doc = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [
+            { type: "text", text: "See " },
+            {
+              type: "text",
+              text: "this page",
+              marks: [
+                {
+                  type: "link",
+                  attrs: { href: "https://example.com/foo" },
+                },
+              ],
+            },
+            { type: "text", text: " for details." },
+          ],
+        },
+      ],
+    });
+
+    const result = extractBodyText(doc, { includeLinkHrefs: true });
+    expect(result).toContain("this page");
+    expect(result).toContain("https://example.com/foo");
+    expect(result).toContain("See");
+    expect(result).toContain("for details.");
+  });
+
+  it("omits link mark hrefs by default (plain-text contract)", () => {
+    const doc = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [
+            { type: "text", text: "See " },
+            {
+              type: "text",
+              text: "this page",
+              marks: [
+                {
+                  type: "link",
+                  attrs: { href: "https://example.com/foo" },
+                },
+              ],
+            },
+            { type: "text", text: "." },
+          ],
+        },
+      ],
+    });
+
+    const result = extractBodyText(doc);
+    expect(result).toContain("this page");
+    expect(result).not.toContain("example.com");
+    expect(result).not.toContain("https://");
+  });
+
+  it("ignores non-link marks (bold, italic, code, etc.)", () => {
+    const doc = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [
+            {
+              type: "text",
+              text: "emphasized",
+              marks: [{ type: "bold" }, { type: "italic" }],
+            },
+          ],
+        },
+      ],
+    });
+
+    expect(extractBodyText(doc, { includeLinkHrefs: true })).toBe("emphasized");
+  });
+
+  it("skips link marks with empty or whitespace-only hrefs", () => {
+    const doc = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [
+            {
+              type: "text",
+              text: "broken",
+              marks: [{ type: "link", attrs: { href: "   " } }],
+            },
+            {
+              type: "text",
+              text: "also-broken",
+              marks: [{ type: "link", attrs: {} }],
+            },
+          ],
+        },
+      ],
+    });
+
+    const result = extractBodyText(doc, { includeLinkHrefs: true });
+    expect(result).toContain("broken");
+    expect(result).toContain("also-broken");
+    expect(result).not.toMatch(/https?:\/\//);
+  });
+
   it("returns null for invalid JSON", () => {
     expect(extractBodyText("not json")).toBeNull();
     expect(extractBodyText("{invalid")).toBeNull();
@@ -454,6 +563,37 @@ describe("extractSummary", () => {
     expect(extractSummary(doc, 5, 500)).toBe("After decorations");
   });
 
+  it("does not leak link mark URLs into the plaintext summary", () => {
+    // Regression guard: extractBodyText (for search) includes link hrefs,
+    // but extractSummary (for feeds/meta descriptions) must not.
+    const doc = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [
+            { type: "text", text: "See " },
+            {
+              type: "text",
+              text: "this link",
+              marks: [
+                {
+                  type: "link",
+                  attrs: { href: "https://example.com/foo" },
+                },
+              ],
+            },
+            { type: "text", text: "." },
+          ],
+        },
+      ],
+    });
+
+    const result = extractSummary(doc, 5, 500);
+    expect(result).toBe("See this link.");
+    expect(result).not.toContain("example.com");
+  });
+
   it("returns null for invalid JSON", () => {
     expect(extractSummary("not json", 5, 500)).toBeNull();
   });

package/src/lib/__tests__/view.test.ts

@@ -206,6 +206,72 @@ describe("toPostView", () => {
     expect(view.summaryHasMore).toBe(true);
   });
 
+  it("injects #continue anchor at the block boundary when body has a leading <hr>", () => {
+    // Regression: structural nodes like `horizontalRule` appear in bodyHtml
+    // but are excluded from the summary. Slicing bodyHtml by summary.length
+    // landed mid-tag (e.g. inside </h2>), producing corrupted markup like
+    // `<h2>...&lt;<span id="continue"></span>/h2&gt;`.
+    const textA = "A".repeat(300);
+    const textB = "B".repeat(300);
+    const body = JSON.stringify({
+      type: "doc",
+      content: [
+        { type: "horizontalRule" },
+        {
+          type: "paragraph",
+          content: [{ type: "text", text: textA }],
+        },
+        {
+          type: "paragraph",
+          content: [{ type: "text", text: textB }],
+        },
+      ],
+    });
+    const p1 = `<p>${textA}</p>`;
+    const p2 = `<p>${textB}</p>`;
+    const bodyHtml = `<hr>${p1}${p2}`;
+    const view = toPostView(
+      makePostWithMedia({ title: "Article", body, bodyHtml }),
+      EMPTY_CTX,
+    );
+    expect(view.summaryHasMore).toBe(true);
+    expect(view.bodyHtml).toBe(`<hr>${p1}<span id="continue"></span>${p2}`);
+    // Must not corrupt tags by splitting them mid-character.
+    expect(view.bodyHtml).not.toContain("&lt;");
+    expect(view.bodyHtml).not.toContain("&gt;");
+  });
+
+  it("places the #continue anchor at the moreBreak boundary", () => {
+    const body = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [{ type: "text", text: "Lead" }],
+        },
+        { type: "moreBreak" },
+        {
+          type: "paragraph",
+          content: [{ type: "text", text: "Rest" }],
+        },
+      ],
+    });
+    const view = toPostView(
+      makePostWithMedia({
+        title: "Article",
+        body,
+        bodyHtml: "<p>Lead</p><!--more--><p>Rest</p>",
+      }),
+      EMPTY_CTX,
+    );
+    expect(view.summaryHasMore).toBe(true);
+    // The anchor sits at the summary boundary. The `<!--more-->` marker is an
+    // inert HTML comment, so it's fine to keep in the post-anchor body.
+    expect(view.bodyHtml).toBe(
+      '<p>Lead</p><span id="continue"></span><!--more--><p>Rest</p>',
+    );
+  });
+
   it("does not compute summaryHtml for posts without title", () => {
     const view = toPostView(
       makePostWithMedia({

package/src/lib/feed.ts

@@ -10,12 +10,7 @@
  * ```
  */
 
-import type {
-  FeedData,
-  FeedPostView,
-  PostView,
-  SitemapData,
-} from "../types.js";
+import type { FeedData, FeedPostView, PostView } from "../types.js";
 import { extractDisplayDomain } from "./url.js";
 
 /**
@@ -242,40 +237,81 @@ export function defaultFeedRenderer(data: FeedData): string {
 }
 
 /**
- * Default Sitemap renderer.
- *
- * @param data - Sitemap data with PostView[]
- * @returns Sitemap XML string
+ * Maximum URLs per sitemap shard. The sitemap.xml spec allows up to 50,000
+ * per file; 500 keeps individual shards cheap to generate on D1 and makes old
+ * (already-filled) shards small enough to cache aggressively at the edge.
  */
-export function defaultSitemapRenderer(data: SitemapData): string {
-  const { siteUrl, sitemapUrl, posts } = data;
+export const SITEMAP_SHARD_SIZE = 500;
 
-  const postUrls = posts
-    .map((post) => {
-      const loc = escapeXml(new URL(post.permalink, siteUrl).toString());
-      const lastmod = post.updatedAt.split("T")[0];
-      const priority = post.featured ? "0.8" : "0.6";
+/** One `<url>` entry inside a sitemap `<urlset>`. */
+export interface SitemapUrlEntry {
+  loc: string;
+  /** ISO date (YYYY-MM-DD) or full ISO datetime */
+  lastmod?: string;
+  changefreq?:
+    | "always"
+    | "hourly"
+    | "daily"
+    | "weekly"
+    | "monthly"
+    | "yearly"
+    | "never";
+  /** "0.0" – "1.0" */
+  priority?: string;
+}
 
-      return `
-  <url>
-    <loc>${loc}</loc>
-    <lastmod>${lastmod}</lastmod>
-    <priority>${priority}</priority>
-  </url>`;
-    })
-    .join("");
+/** One `<sitemap>` entry inside a `<sitemapindex>`. */
+export interface SitemapIndexEntry {
+  loc: string;
+  lastmod?: string;
+}
 
-  const homepageUrl = `
-  <url>
-    <loc>${escapeXml(siteUrl)}</loc>
-    <priority>1.0</priority>
-    <changefreq>daily</changefreq>
-  </url>`;
+/**
+ * Render a sitemap `<urlset>` XML document from a list of URL entries.
+ *
+ * Used by the sharded sitemap endpoints in `routes/feed/sitemap.ts`.
+ */
+export function renderSitemapUrlSet(entries: SitemapUrlEntry[]): string {
+  const urls = entries
+    .map((entry) => {
+      const parts = [`    <loc>${escapeXml(entry.loc)}</loc>`];
+      if (entry.lastmod) {
+        parts.push(`    <lastmod>${escapeXml(entry.lastmod)}</lastmod>`);
+      }
+      if (entry.changefreq) {
+        parts.push(
+          `    <changefreq>${escapeXml(entry.changefreq)}</changefreq>`,
+        );
+      }
+      if (entry.priority) {
+        parts.push(`    <priority>${escapeXml(entry.priority)}</priority>`);
+      }
+      return `  <url>\n${parts.join("\n")}\n  </url>`;
+    })
+    .join("\n");
 
   return `<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-  <!-- Generated from ${escapeXml(sitemapUrl)} -->
-  ${homepageUrl}
-  ${postUrls}
+${urls}
 </urlset>`;
 }
+
+/**
+ * Render a `<sitemapindex>` XML document listing shard sitemap URLs.
+ */
+export function renderSitemapIndex(entries: SitemapIndexEntry[]): string {
+  const items = entries
+    .map((entry) => {
+      const parts = [`    <loc>${escapeXml(entry.loc)}</loc>`];
+      if (entry.lastmod) {
+        parts.push(`    <lastmod>${escapeXml(entry.lastmod)}</lastmod>`);
+      }
+      return `  <sitemap>\n${parts.join("\n")}\n  </sitemap>`;
+    })
+    .join("\n");
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${items}
+</sitemapindex>`;
+}