@jant/core 0.3.42 → 0.3.43

package/src/services/__tests__/post.test.ts

@@ -2073,4 +2073,209 @@ describe("PostService", () => {
       expect(updatedRoot?.lastActivityAt).toBe(1000);
     });
   });
+
+  describe("reindexBodyText", () => {
+    function bodyWithLink(text: string, href: string): string {
+      return JSON.stringify({
+        type: "doc",
+        content: [
+          {
+            type: "paragraph",
+            content: [
+              {
+                type: "text",
+                text,
+                marks: [{ type: "link", attrs: { href } }],
+              },
+            ],
+          },
+        ],
+      });
+    }
+
+    it("recomputes body_text and updates only rows that differ", async () => {
+      const post = await postService.create({
+        format: "note",
+        body: bodyWithLink("docs", "https://rebuild.example/page"),
+      });
+
+      // Simulate the pre-fix state by stripping URLs from body_text directly.
+      await db
+        .update(posts)
+        .set({ bodyText: "docs" })
+        .where(eq(posts.id, post.id));
+
+      const firstPass = await postService.reindexBodyText();
+      expect(firstPass.processed).toBe(1);
+      expect(firstPass.updated).toBe(1);
+      expect(firstPass.skipped).toBe(0);
+      expect(firstPass.done).toBe(true);
+      expect(firstPass.nextCursor).toBeNull();
+
+      const reindexed = await postService.getById(post.id);
+      expect(reindexed?.bodyText).toContain("rebuild.example");
+
+      // Idempotent: re-running immediately should be a no-op.
+      const secondPass = await postService.reindexBodyText();
+      expect(secondPass.updated).toBe(0);
+      expect(secondPass.skipped).toBe(1);
+      expect(secondPass.done).toBe(true);
+    });
+
+    it("skips soft-deleted posts", async () => {
+      const live = await postService.create({
+        format: "note",
+        body: bodyWithLink("a", "https://live.example"),
+      });
+      const gone = await postService.create({
+        format: "note",
+        body: bodyWithLink("b", "https://gone.example"),
+      });
+
+      // Strip body_text on both to force an update on the next pass.
+      await db
+        .update(posts)
+        .set({ bodyText: "a" })
+        .where(eq(posts.id, live.id));
+      await db
+        .update(posts)
+        .set({ bodyText: "b" })
+        .where(eq(posts.id, gone.id));
+      await postService.delete(gone.id);
+
+      const result = await postService.reindexBodyText();
+      expect(result.processed).toBe(1);
+      expect(result.updated).toBe(1);
+      expect(result.done).toBe(true);
+    });
+
+    it("paginates with cursor when more posts remain", async () => {
+      for (let i = 0; i < 3; i++) {
+        await postService.create({
+          format: "note",
+          body: bodyWithLink(`p${i}`, `https://p${i}.example`),
+        });
+      }
+
+      const first = await postService.reindexBodyText({ limit: 2 });
+      expect(first.processed).toBe(2);
+      expect(first.done).toBe(false);
+      expect(first.nextCursor).not.toBeNull();
+
+      const second = await postService.reindexBodyText({
+        limit: 2,
+        cursor: first.nextCursor ?? undefined,
+      });
+      expect(second.processed).toBe(1);
+      expect(second.done).toBe(true);
+      expect(second.nextCursor).toBeNull();
+    });
+  });
+
+  describe("listForSitemap", () => {
+    it("returns published non-reply non-private non-deleted posts", async () => {
+      const root = await postService.create({
+        format: "note",
+        bodyMarkdown: "root",
+        status: "published",
+      });
+      await postService.create({
+        format: "note",
+        bodyMarkdown: "reply",
+        replyToId: root.id,
+        status: "published",
+      });
+      await postService.create({
+        format: "note",
+        bodyMarkdown: "private",
+        visibility: "private",
+        status: "published",
+      });
+      await postService.create({
+        format: "note",
+        bodyMarkdown: "draft",
+        status: "draft",
+      });
+      await postService.create({
+        format: "note",
+        bodyMarkdown: "latest hidden",
+        visibility: "latest_hidden",
+        status: "published",
+      });
+
+      const entries = await postService.listForSitemap({ limit: 100 });
+      const ids = entries.map((e) => e.id);
+      // Root post and latest_hidden post should be included; reply/private/draft excluded.
+      expect(ids).toHaveLength(2);
+      expect(ids).toContain(root.id);
+    });
+
+    it("returns entries in ascending id order", async () => {
+      const created: string[] = [];
+      for (let i = 0; i < 5; i++) {
+        const post = await postService.create({
+          format: "note",
+          bodyMarkdown: `post ${i}`,
+          status: "published",
+        });
+        created.push(post.id);
+      }
+
+      const entries = await postService.listForSitemap({ limit: 100 });
+      const ids = entries.map((e) => e.id);
+      // TypeIDs embed a UUIDv7 timestamp, so creation order == ascending id.
+      expect(ids).toEqual([...ids].sort());
+      expect(ids).toEqual(created);
+    });
+
+    it("respects afterId as an exclusive cursor", async () => {
+      const posts = [];
+      for (let i = 0; i < 5; i++) {
+        posts.push(
+          await postService.create({
+            format: "note",
+            bodyMarkdown: `post ${i}`,
+            status: "published",
+          }),
+        );
+      }
+
+      const firstPage = await postService.listForSitemap({ limit: 2 });
+      expect(firstPage).toHaveLength(2);
+
+      const cursor = firstPage[firstPage.length - 1]?.id;
+      const secondPage = await postService.listForSitemap({
+        afterId: cursor,
+        limit: 2,
+      });
+      expect(secondPage.map((e) => e.id)).toEqual([posts[2]?.id, posts[3]?.id]);
+
+      const thirdPage = await postService.listForSitemap({
+        afterId: secondPage[secondPage.length - 1]?.id,
+        limit: 2,
+      });
+      expect(thirdPage.map((e) => e.id)).toEqual([posts[4]?.id]);
+    });
+
+    it("countForSitemap matches listForSitemap without a cursor", async () => {
+      for (let i = 0; i < 3; i++) {
+        await postService.create({
+          format: "note",
+          bodyMarkdown: `p${i}`,
+          status: "published",
+        });
+      }
+      await postService.create({
+        format: "note",
+        bodyMarkdown: "private",
+        visibility: "private",
+        status: "published",
+      });
+
+      const count = await postService.countForSitemap();
+      const entries = await postService.listForSitemap({ limit: 100 });
+      expect(count).toBe(entries.length);
+      expect(count).toBe(3);
+    });
+  });
 });

package/src/services/__tests__/search.test.ts

@@ -204,6 +204,50 @@ describe("SearchService", () => {
     expect(results[0]?.post.url).toContain("example.com");
   });
 
+  it("finds posts by URL embedded in inline markdown links", async () => {
+    // TipTap stores markdown links as marks on text nodes. Their href
+    // must reach body_text so users can search for the URL, not just
+    // the visible link text.
+    const bodyWithLink = JSON.stringify({
+      type: "doc",
+      content: [
+        {
+          type: "paragraph",
+          content: [
+            { type: "text", text: "See " },
+            {
+              type: "text",
+              text: "this page",
+              marks: [
+                {
+                  type: "link",
+                  attrs: { href: "https://inline-link.example/article" },
+                },
+              ],
+            },
+            { type: "text", text: " for details." },
+          ],
+        },
+      ],
+    });
+
+    await postService.create({
+      format: "note",
+      body: bodyWithLink,
+    });
+
+    const d1 = createMockD1(sqlite);
+    const searchService = createSearchService(d1, DEFAULT_TEST_SITE_ID);
+
+    // Searching by the link's URL host should match.
+    const byUrl = await searchService.search("inline-link.example");
+    expect(byUrl.length).toBeGreaterThanOrEqual(1);
+
+    // Regression guard: the visible link text still matches too.
+    const byText = await searchService.search("this page");
+    expect(byText.length).toBeGreaterThanOrEqual(1);
+  });
+
   it("finds posts with short queries (< 3 chars) via LIKE fallback", async () => {
     await postService.create({
       format: "note",

package/src/services/export.ts

@@ -28,6 +28,7 @@ import {
   getDefaultJantFaviconIcoBytes,
 } from "../lib/jant-branding.js";
 import { tiptapJsonToMarkdown } from "../lib/tiptap-to-markdown.js";
+import { extractBodyText } from "../lib/summary.js";
 import { getMediaUrl, getPublicUrlForProvider } from "../lib/image.js";
 import { render as renderMarkdown } from "../lib/markdown.js";
 import { formatRelativeAge, toISOString } from "../lib/time.js";
@@ -1074,10 +1075,16 @@ function getArchiveSummaryText(post: Post): string | null {
   //   serialized as `link_url` and rendered as a domain badge, so using
   //   it as `summary_text` would duplicate that information.
   // - Note: body only. If the body is empty, there's nothing to describe.
+  //
+  // Note: we re-derive the body text from `post.body` (TipTap JSON) rather
+  // than reusing `post.bodyText`, because `bodyText` is written with
+  // `includeLinkHrefs: true` for FTS search indexing — that pollutes the
+  // stored text with trailing link URLs. Here we need clean prose.
+  const cleanBodyText = post.body ? extractBodyText(post.body) : null;
   const candidates =
     post.format === "quote"
-      ? [post.summary, post.bodyText, post.quoteText]
-      : [post.summary, post.bodyText];
+      ? [post.summary, cleanBodyText, post.quoteText]
+      : [post.summary, cleanBodyText];
 
   for (const candidate of candidates) {
     const normalized = normalizeArchiveText(candidate);

package/src/services/post.ts

@@ -19,6 +19,7 @@ import {
   isNotNull,
   asc,
   lte,
+  gt,
 } from "drizzle-orm";
 import {
   type Database,
@@ -156,6 +157,17 @@ export interface PostBodyContent {
   chars: number;
 }
 
+/** Minimal projection used by the sitemap renderer. */
+export interface SitemapPostEntry {
+  id: string;
+  /** Canonical slug from `path_registry` */
+  slug: string;
+  /** Primary alias, if the post has one; used in preference to `slug` for URLs */
+  alias: string | null;
+  updatedAt: number;
+  featuredAt: number | null;
+}
+
 export interface PostService {
   getById(id: string): Promise<Post | null>;
   getBodyContent(id: string): Promise<PostBodyContent | null>;
@@ -167,6 +179,35 @@ export interface PostService {
   }): Promise<string>;
   checkSlugAvailability(slug: string, excludePostId?: string): Promise<boolean>;
   list(filters?: PostFilters): Promise<Post[]>;
+  /**
+   * List minimal fields needed to render sitemap entries, paginated by `id`
+   * (ascending). Excludes replies, private posts, deleted posts, and drafts.
+   *
+   * Uses keyset pagination on the primary key so old sitemap shards are cheap
+   * to serve and stable across shard boundaries: a newly created post always
+   * gets a larger TypeID than any previously-committed post, so it lands in
+   * the last shard and never rewrites older ones.
+   *
+   * @param options.afterId  Exclusive lower bound on `id`. Omit for the first
+   *                         shard.
+   * @param options.limit    Maximum rows to return.
+   */
+  listForSitemap(options: {
+    afterId?: string;
+    limit: number;
+  }): Promise<SitemapPostEntry[]>;
+  /** Count posts that qualify for the sitemap (same filters as `listForSitemap`) */
+  countForSitemap(): Promise<number>;
+  /**
+   * Return the id at the given 0-based offset in the sitemap ordering.
+   * Used to compute keyset cursors for sharded sitemap endpoints.
+   *
+   * Returns `null` when the offset is beyond the available rows.
+   *
+   * Walks the primary-key index with `ORDER BY id ASC LIMIT 1 OFFSET ?` —
+   * SQLite/D1 scan only the index for this, not the row data.
+   */
+  getSitemapIdAt(offset: number): Promise<string | null>;
   /** Count posts matching filters (ignores cursor, offset, limit) */
   count(filters?: PostFilters): Promise<number>;
   /** Count posts matching filters up to a fixed limit (ignores cursor, offset, limit) */
@@ -292,6 +333,28 @@ export interface PostService {
   getDistinctYears(filters?: PostFilters): Promise<number[]>;
   /** For each thread ID, return the ID of the last published, non-deleted post */
   getLastPostIdsByThread(threadIds: string[]): Promise<Map<string, string>>;
+  /**
+   * Rebuild `post.body_text` for a batch of non-deleted posts, cursor-paginated
+   * by post id. For each row, recomputes the plain-text extraction via
+   * `extractBodyText(body)` and writes it back only when it differs from the
+   * stored value. FTS indexes (SQLite trigger / Postgres generated column)
+   * refresh automatically on the UPDATE.
+   *
+   * Idempotent: re-running after a no-op pass returns `updated: 0`.
+   *
+   * @param options.limit  Batch size (1..500, default 50)
+   * @param options.cursor Exclusive lower bound on post id; pass the previous
+   *                       response's `nextCursor` to continue
+   * @returns processed/updated/skipped counts, the next cursor, and a `done`
+   *          flag the caller uses to terminate the loop
+   */
+  reindexBodyText(options?: { limit?: number; cursor?: string }): Promise<{
+    processed: number;
+    updated: number;
+    skipped: number;
+    nextCursor: string | null;
+    done: boolean;
+  }>;
 }
 
 const SLUG_RE = /^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$/;
@@ -1300,6 +1363,83 @@ export function createPostService(
       return hydratePosts(rows);
     },
 
+    async listForSitemap({ afterId, limit }) {
+      // Share the filter conditions with `list()` so visibility/reply/deleted
+      // semantics stay consistent if they ever change.
+      const conditions = buildFilterConditions({
+        status: "published",
+        excludePrivate: true,
+        excludeReplies: true,
+      });
+      if (afterId !== undefined) {
+        conditions.push(sql`${posts.id} > ${afterId}`);
+      }
+
+      const rows = await db
+        .select({
+          id: posts.id,
+          updatedAt: posts.updatedAt,
+          featuredAt: posts.featuredAt,
+        })
+        .from(posts)
+        .where(conditions.length > 0 ? and(...conditions) : undefined)
+        .orderBy(asc(posts.id))
+        .limit(limit);
+
+      if (rows.length === 0) return [];
+
+      const ids = rows.map((row) => row.id);
+      const [slugMap, aliasesMap] = await Promise.all([
+        resolvedPaths.getPostSlugMap(ids),
+        resolvedPaths.getPostAliases(ids),
+      ]);
+
+      return rows
+        .map((row): SitemapPostEntry | null => {
+          const slug = slugMap.get(row.id);
+          if (!slug) return null;
+          const alias = aliasesMap.get(row.id)?.[0] ?? null;
+          return {
+            id: row.id,
+            slug,
+            alias,
+            updatedAt: row.updatedAt,
+            featuredAt: row.featuredAt,
+          };
+        })
+        .filter((entry): entry is SitemapPostEntry => entry !== null);
+    },
+
+    async countForSitemap() {
+      const conditions = buildFilterConditions({
+        status: "published",
+        excludePrivate: true,
+        excludeReplies: true,
+      });
+      const result = await db
+        .select({ count: sql<number>`CAST(count(*) AS INTEGER)`.as("count") })
+        .from(posts)
+        .where(conditions.length > 0 ? and(...conditions) : undefined);
+      return result[0]?.count ?? 0;
+    },
+
+    async getSitemapIdAt(offset) {
+      if (offset < 0) return null;
+      const conditions = buildFilterConditions({
+        status: "published",
+        excludePrivate: true,
+        excludeReplies: true,
+      });
+      const rows = await db
+        .select({ id: posts.id })
+        .from(posts)
+        .where(conditions.length > 0 ? and(...conditions) : undefined)
+        .orderBy(asc(posts.id))
+        .limit(1)
+        .offset(offset);
+      return rows[0]?.id ?? null;
+    },
+
     async count(filters = {}) {
       const conditions = buildFilterConditions(filters);
 
@@ -1377,7 +1517,9 @@ export function createPostService(
       });
 
       const bodyHtml = body ? renderTiptapJson(body) : null;
-      const bodyText = body ? extractBodyText(body) : null;
+      const bodyText = body
+        ? extractBodyText(body, { includeLinkHrefs: true })
+        : null;
 
       // Generate summary for titled notes with body content
       let summary: string | null = null;
@@ -1879,7 +2021,7 @@ export function createPostService(
           ? renderTiptapJson(normalizedBody)
           : null;
         updates.bodyText = normalizedBody
-          ? extractBodyText(normalizedBody)
+          ? extractBodyText(normalizedBody, { includeLinkHrefs: true })
           : null;
       }
 
@@ -2967,5 +3109,61 @@ export function createPostService(
 
       return rows.map((r) => parseInt(r.year, 10));
     },
+
+    async reindexBodyText(options = {}) {
+      const requested = options.limit ?? 50;
+      const limit = Math.min(Math.max(Math.trunc(requested), 1), 500);
+      const cursor = options.cursor;
+
+      const whereConditions = [
+        eq(posts.siteId, siteId),
+        isNull(posts.deletedAt),
+      ];
+      if (cursor) whereConditions.push(gt(posts.id, cursor));
+
+      // Fetch one extra row to detect end-of-data without a separate COUNT.
+      const rows = await db
+        .select({
+          id: posts.id,
+          body: posts.body,
+          bodyText: posts.bodyText,
+        })
+        .from(posts)
+        .where(and(...whereConditions))
+        .orderBy(asc(posts.id))
+        .limit(limit + 1);
+
+      const hasMore = rows.length > limit;
+      const batch = hasMore ? rows.slice(0, limit) : rows;
+
+      let updated = 0;
+      let skipped = 0;
+
+      for (const row of batch) {
+        const nextBodyText = row.body
+          ? extractBodyText(row.body, { includeLinkHrefs: true })
+          : null;
+        if (nextBodyText === row.bodyText) {
+          skipped++;
+          continue;
+        }
+        await db
+          .update(posts)
+          .set({ bodyText: nextBodyText })
+          .where(and(eq(posts.siteId, siteId), eq(posts.id, row.id)));
+        updated++;
+      }
+
+      const lastRow = batch.at(-1);
+      const lastId = lastRow ? lastRow.id : null;
+
+      return {
+        processed: batch.length,
+        updated,
+        skipped,
+        nextCursor: hasMore ? lastId : null,
+        done: !hasMore,
+      };
+    },
   };
 }

package/src/types/app-context.ts

@@ -11,10 +11,17 @@ import type { Services } from "../services/index.js";
 import type { HostedHandoffService } from "../services/hosted-handoff.js";
 import type { Auth } from "../auth.js";
 import type { AppConfig } from "./config.js";
+import type { RateLimiter } from "../lib/rate-limit.js";
 import type { StorageDriver } from "../lib/storage.js";
 import type { Bindings } from "./bindings.js";
 import type { Site, SiteDomain } from "./entities.js";
 
+/**
+ * Session payload as returned by better-auth's `getSession`.
+ * Populated once per request by the `attachSession` middleware.
+ */
+export type AppSession = Awaited<ReturnType<Auth["api"]["getSession"]>>;
+
 export interface AppVariables {
   services: Services;
   hostedHandoff: HostedHandoffService;
@@ -27,6 +34,19 @@ export interface AppVariables {
   storage: StorageDriver | null;
   publicRequestUrl: string;
   publicPath: string;
+  /**
+   * Cached session for the current request. `null` when unauthenticated or
+   * when the session lookup errored. Populated by `attachSession` middleware.
+   */
+  session: AppSession;
+  /** True when `session?.user` is set. Shortcut for the common read. */
+  isAuthenticated: boolean;
+  /**
+   * Runtime-appropriate rate limiter. Populated per-request from the
+   * runtime (D1 on Workers, in-memory on Node). Middleware calls
+   * `c.var.rateLimiter.check(...)` instead of caring which impl is used.
+   */
+  rateLimiter: RateLimiter;
 }
 
 export type App = Hono<{ Bindings: Bindings; Variables: AppVariables }>;

package/src/types/config.ts

@@ -476,6 +476,14 @@ export interface AppConfig {
   siteAvatarUrl: string;
   faviconVersion: string;
 
+  // Rate limiting (ENV only)
+  rateLimit: {
+    /** When true, all rate-limit middleware becomes a no-op. */
+    disabled: boolean;
+    /** Per-IP cap for `/api/search` requests per 60-second window. */
+    searchPerMinute: number;
+  };
+
   // Settings form placeholders (ENV > Default, without DB)
   fallbacks: {
     siteName: string;

package/src/types/props.ts

@@ -138,13 +138,6 @@ export interface FeedData {
   posts: FeedPostView[];
 }
 
-/** Data passed to sitemap renderers */
-export interface SitemapData {
-  siteUrl: string;
-  sitemapUrl: string;
-  posts: PostView[];
-}
-
 // =============================================================================
 // Timeline Types
 // =============================================================================

package/src/ui/layouts/BaseLayout.tsx

@@ -49,6 +49,13 @@ export interface BaseLayoutProps {
   faviconUrl?: string;
   faviconVersion?: string;
   socialImageUrl?: string;
+  /**
+   * Absolute canonical URL for the current page. Rendered as
+   * `<link rel="canonical">` when set. Use on pages whose primary content is
+   * also reachable via another URL (e.g. reply posts, which render the full
+   * thread at both the reply URL and the thread-root URL).
+   */
+  canonicalHref?: string;
   noindex?: boolean;
   isAuthenticated?: boolean;
   clientBundle?: "public" | "full";
@@ -65,6 +72,7 @@ export const BaseLayout: FC<PropsWithChildren<BaseLayoutProps>> = ({
   faviconUrl,
   faviconVersion,
   socialImageUrl,
+  canonicalHref,
   noindex,
   isAuthenticated = false,
   clientBundle,
@@ -265,6 +273,7 @@ export const BaseLayout: FC<PropsWithChildren<BaseLayoutProps>> = ({
           {resolvedNoindex && (
             <meta name="robots" content="noindex, nofollow" />
           )}
+          {canonicalHref && <link rel="canonical" href={canonicalHref} />}
           <link rel="icon" href={resolvedFaviconHref} sizes="16x16 32x32" />
           <link rel="apple-touch-icon" href={resolvedAppleTouchHref} />
           <link

package/dist/app-DzCB4yOp.js

@@ -1,5 +0,0 @@
-import "./url-FvvgARU9.js";
-import { t as createApp } from "./app-Cu3lveYI.js";
-import "./github-sync-zohnA9qv.js";
-import "./env-wCpMcNXs.js";
-export { createApp };