@fullstackdatasolutions/articles 1.2.3 → 1.3.1

package/src/seoUtils.ts

@@ -8,20 +8,24 @@ import {
   getArticlesByCategory,
   getArticlesBySeries,
   getAuthorBySlug,
+  getArticleMarkdown,
   getArticleMarkdownUrl,
   getAvailableArticleSlugs,
+  buildMarkdownTwinHeader,
   categoryToSlug,
 } from './server-articles'
 import {
   breadcrumbsAreEnabled,
+  formatPageTitle,
   getBreadcrumbsConfig,
+  DEFAULT_PAGE_SIZE,
   type ArticlesConfig,
   type ArticleBreadcrumbEntry,
   type AuthorBreadcrumbEntry,
   type CategoryBreadcrumbEntry,
   type CustomBreadcrumbItem,
 } from './articlesConfig'
-import { buildPaginationLinks } from './pagination'
+import { buildPageUrl, buildPaginationLinks, getTotalPages } from './pagination'
 import type { Article, AuthorProfile, BreadcrumbItem } from './articleTypes'
 
 function escapeXml(str: string): string {
@@ -33,9 +37,22 @@ function escapeXml(str: string): string {
     .replaceAll("'", ''')
 }
 
-export function generateRssFeed(articles: Article[], config: ArticlesConfig): string {
+/**
+ * `options.fullContent` adds `<content:encoded>` with each article's rendered
+ * HTML, for feeds meant to be ingested rather than previewed - an
+ * excerpt-only feed gives a consumer nothing to work with. Off by default:
+ * it requires articles loaded with `htmlContent` (i.e. via
+ * `getArticleMetadata`, not `getAllArticles`' summaries), and articles
+ * without it are simply emitted without the element.
+ */
+export function generateRssFeed(
+  articles: Article[],
+  config: ArticlesConfig,
+  options?: Readonly<{ fullContent?: boolean }>
+): string {
   const siteUrl = config.siteUrl.replace(/\/$/, '')
   const showAuthor = config.showAuthor !== false
+  const fullContent = options?.fullContent === true
 
   const items = articles
     .map((article) => {
@@ -50,6 +67,9 @@ export function generateRssFeed(articles: Article[], config: ArticlesConfig): st
         `      <guid isPermaLink="true">${url}</guid>`,
         pubDate ? `      <pubDate>${pubDate}</pubDate>` : '',
         article.excerpt ? `      <description><![CDATA[${article.excerpt}]]></description>` : '',
+        fullContent && article.htmlContent
+          ? `      <content:encoded><![CDATA[${article.htmlContent}]]></content:encoded>`
+          : '',
         showAuthor && article.author ? `      <author>${escapeXml(article.author)}</author>` : '',
         article.category ? `      <category><![CDATA[${article.category}]]></category>` : '',
         imageUrl
@@ -65,18 +85,116 @@ export function generateRssFeed(articles: Article[], config: ArticlesConfig): st
   const description = config.description ?? `${config.siteName} articles`
 
   return `<?xml version="1.0" encoding="UTF-8" ?>
-<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/" xmlns:content="http://purl.org/rss/1.0/modules/content/">
   <channel>
     <title><![CDATA[${config.siteName}]]></title>
     <link>${siteUrl}/articles</link>
     <description><![CDATA[${description}]]></description>
-    <language>en</language>
+    <language>${config.language ?? 'en'}</language>
     <atom:link href="${siteUrl}/articles/feed.xml" rel="self" type="application/rss+xml" />
 ${items}
   </channel>
 </rss>`
 }
 
+/** Shared `# name` / `> summary` preamble for both llms.txt variants. */
+function buildLlmsHeader(config: ArticlesConfig): string[] {
+  const summary = config.description ?? `${config.siteName} articles`
+  return [`# ${config.siteName}`, '', `> ${summary}`, '']
+}
+
+/**
+ * `llms.txt` index - the emerging convention for pointing an LLM at a site's
+ * canonical, markdown-native content (https://llmstxt.org). Lists only
+ * articles opted in via `aiCrawl` (see `ArticlesConfig.aiCrawlDefault`),
+ * grouped by category, linking to each article's `.md` twin rather than its
+ * HTML page.
+ *
+ * Wire it up in the consuming app as `app/llms.txt/route.ts`:
+ *   export async function GET() {
+ *     return new Response(generateLlmsTxt(await getAllArticles(siteConfig), siteConfig), {
+ *       headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+ *     })
+ *   }
+ */
+export function generateLlmsTxt(articles: Article[], config: ArticlesConfig): string {
+  const siteUrl = config.siteUrl.replace(/\/$/, '')
+  const crawlable = articles.filter((article) => article.aiCrawl === true)
+
+  const byCategory = new Map<string, Article[]>()
+  for (const article of crawlable) {
+    const category = article.category || 'Articles'
+    const existing = byCategory.get(category)
+    if (existing) existing.push(article)
+    else byCategory.set(category, [article])
+  }
+
+  const sections = [...byCategory.entries()].map(([category, categoryArticles]) => {
+    const lines = categoryArticles.map((article) => {
+      const url = `${siteUrl}/articles/${article.slug}.md`
+      const summary = article.excerpt ? `: ${article.excerpt}` : ''
+      return `- [${article.title}](${url})${summary}`
+    })
+    return [`## ${category}`, '', ...lines].join('\n')
+  })
+
+  return [
+    ...buildLlmsHeader(config),
+    ...(sections.length > 0 ? sections : ['## Articles', '', '_No articles available._']),
+    ...buildLlmsListingSection(crawlable, config),
+    '',
+  ].join('\n')
+}
+
+/**
+ * Links the category, author, and series twins alongside the article list.
+ * Without them a model gets a flat bag of pages; these are the surfaces that
+ * answer "what does this site cover, and who writes it".
+ */
+function buildLlmsListingSection(articles: Article[], config: ArticlesConfig): string[] {
+  const siteUrl = config.siteUrl.replace(/\/$/, '')
+  const categories = [...new Set(articles.flatMap((a) => a.categories ?? []))].filter(Boolean)
+  const series = [...new Set(articles.map((a) => a.seriesSlug).filter(Boolean))] as string[]
+  const authors = Object.values(config.authors ?? {})
+
+  const lines = [
+    ...categories.map(
+      (name) => `- [${name}](${siteUrl}/articles/category/${categoryToSlug(name)}.md)`
+    ),
+    ...(config.showAuthorPage === false
+      ? []
+      : authors.map((a) => `- [${a.name}](${siteUrl}/articles/authors/${a.slug}.md)`)),
+    ...series.map((slug) => `- [${slug}](${siteUrl}/articles/series/${slug}.md)`),
+  ]
+  if (lines.length === 0) return []
+  return ['## Collections', '', ...lines]
+}
+
+/**
+ * `llms-full.txt` - every opted-in article's full markdown twin, headers
+ * included, concatenated into one document. Larger and slower to build than
+ * `generateLlmsTxt`; generate it in a route handler or at build time, not on
+ * every request.
+ */
+export async function generateLlmsFullTxt(
+  articles: Article[],
+  config: ArticlesConfig
+): Promise<string> {
+  const crawlable = articles.filter((article) => article.aiCrawl === true)
+  const documents = await Promise.all(
+    crawlable.map(async (article) => {
+      const body = await getArticleMarkdown(article.slug, config)
+      if (body === null) return null
+      return `${buildMarkdownTwinHeader(article, config, body)}${body.trimStart()}`
+    })
+  )
+
+  return [
+    ...buildLlmsHeader(config),
+    ...documents.filter((doc): doc is string => doc !== null),
+  ].join('\n')
+}
+
 export function generateArticleStaticParams(): { slug: string }[] {
   return getAvailableArticleSlugs().map((slug) => ({ slug }))
 }
@@ -169,7 +287,7 @@ export async function generateArticleMetadata(
   const authorNames = getArticleAuthors(article, config).map((author) => author.name)
 
   return {
-    title: `${search.title} | ${config.siteName}`,
+    title: formatPageTitle(search.title, config),
     description,
     keywords: [...(article.tags ?? []).map((tag) => tag.toLowerCase())].join(', '),
     openGraph: {
@@ -228,7 +346,7 @@ export async function generateArticleMetadata(
 export function generateArticlesIndexMetadata(config: ArticlesConfig): Metadata {
   const siteUrl = config.siteUrl.replace(/\/$/, '')
   const indexUrl = `${siteUrl}/articles`
-  const title = `Articles | ${config.siteName}`
+  const title = formatPageTitle('Articles', config)
   const description =
     config.hero?.description ?? `Expert analysis and insights from ${config.siteName}.`
   return {
@@ -282,7 +400,7 @@ export async function generateCategoryMetadata(
   const fallback = `Browse ${articles.length} article${articles.length === 1 ? '' : 's'} in the ${categoryName} category.`
   const description = typeof raw === 'string' ? raw : (raw?.short ?? fallback)
 
-  const title = `${categoryName} Articles | ${config.siteName}`
+  const title = formatPageTitle(`${categoryName} Articles`, config)
   return {
     title,
     description,
@@ -330,7 +448,7 @@ export async function generateSeriesMetadata(
   const siteUrl = config.siteUrl.replace(/\/$/, '')
   const seriesUrl = `${siteUrl}/articles/series/${seriesSlug}`
   const description = `Follow the ${seriesName} series - ${articles.length} article${articles.length === 1 ? '' : 's'} on ${config.siteName}.`
-  const title = `${seriesName} Series | ${config.siteName}`
+  const title = formatPageTitle(`${seriesName} Series`, config)
 
   return {
     title,
@@ -376,7 +494,7 @@ export async function generateAuthorMetadata(
 
   const siteUrl = config.siteUrl.replace(/\/$/, '')
   const authorUrl = author.url ?? `${siteUrl}/articles/authors/${author.slug}`
-  const title = `${author.name} Articles | ${config.siteName}`
+  const title = formatPageTitle(`${author.name} Articles`, config)
 
   return {
     title,
@@ -646,18 +764,53 @@ export function resolveAuthorAvatar(author: AuthorProfile, config: ArticlesConfi
   return `${siteUrl}/articles/authors/${author.slug}/${author.avatar.replace(/^\/+/, '')}`
 }
 
+/** Newest `lastmod`/`date` across a set of articles, or `undefined` if none carry one. */
+function newestArticleDate(articles: readonly Article[]): Date | undefined {
+  let newest: Date | undefined
+  for (const article of articles) {
+    const stamp = article.lastmod ?? article.date
+    if (!stamp) continue
+    const parsed = new Date(stamp)
+    if (Number.isNaN(parsed.getTime())) continue
+    if (!newest || parsed > newest) newest = parsed
+  }
+  return newest
+}
+
+/**
+ * Paginated listing routes, emitted only in `listingPagination: 'pages'` mode.
+ * Page 1 is the listing URL itself, already emitted by the caller, so this
+ * starts at page 2. Without these, the paginated routes exist and carry
+ * correct canonical/prev/next metadata but appear in no sitemap.
+ */
+function paginationEntries(
+  basePath: string,
+  itemCount: number,
+  pageSize: number,
+  lastModified: Date | undefined,
+  priority: number
+): MetadataRoute.Sitemap {
+  const totalPages = getTotalPages(itemCount, pageSize)
+  const entries: MetadataRoute.Sitemap = []
+  for (let page = 2; page <= totalPages; page++) {
+    entries.push({
+      url: buildPageUrl(basePath, page),
+      lastModified,
+      changeFrequency: 'weekly' as const,
+      priority,
+    })
+  }
+  return entries
+}
+
 export async function getArticleSitemapEntries(
   baseUrlOrConfig: string | ArticlesConfig
 ): Promise<MetadataRoute.Sitemap> {
-  const baseUrl = (
-    typeof baseUrlOrConfig === 'string' ? baseUrlOrConfig : baseUrlOrConfig.siteUrl
-  ).replace(/\/$/, '')
+  const config = typeof baseUrlOrConfig === 'string' ? undefined : baseUrlOrConfig
+  const baseUrl = (config?.siteUrl ?? (baseUrlOrConfig as string)).replace(/\/$/, '')
 
   try {
-    const [articles, categories] = await Promise.all([
-      getAllArticles(typeof baseUrlOrConfig === 'string' ? undefined : baseUrlOrConfig),
-      getAllCategories(),
-    ])
+    const [articles, categories] = await Promise.all([getAllArticles(config), getAllCategories()])
 
     const articleEntries: MetadataRoute.Sitemap = articles.map((article) => {
       const dateStr = article.lastmod ?? article.date
@@ -670,23 +823,91 @@ export async function getArticleSitemapEntries(
       }
     })
 
+    // Derived from the newest article in each category, not `new Date()` -
+    // stamping "now" told every crawl that every category changed today,
+    // which is exactly the freshness signal a sitemap exists to carry.
     const categoryEntries: MetadataRoute.Sitemap = categories.map((cat) => ({
       url: `${baseUrl}/articles/category/${cat.slug}`,
-      lastModified: new Date(),
+      lastModified: newestArticleDate(
+        articles.filter((article) =>
+          (article.categories ?? []).some((name) => categoryToSlug(name) === cat.slug)
+        )
+      ),
       changeFrequency: 'weekly' as const,
       priority: 0.7,
     }))
 
-    const authorEntries: MetadataRoute.Sitemap =
-      typeof baseUrlOrConfig === 'string' || baseUrlOrConfig.showAuthorPage === false
-        ? []
-        : getAllAuthors(baseUrlOrConfig).map((author) => ({
-            url: `${baseUrl}/articles/authors/${author.slug}`,
-            changeFrequency: 'monthly' as const,
-            priority: 0.6,
-          }))
+    const authors = config && config.showAuthorPage !== false ? getAllAuthors(config) : []
+    const authorEntries: MetadataRoute.Sitemap = authors.map((author) => ({
+      url: `${baseUrl}/articles/authors/${author.slug}`,
+      lastModified: newestArticleDate(
+        articles.filter((article) =>
+          getArticleAuthors(article, config!).some((profile) => profile.slug === author.slug)
+        )
+      ),
+      changeFrequency: 'monthly' as const,
+      priority: 0.6,
+    }))
 
-    return [...articleEntries, ...categoryEntries, ...authorEntries]
+    // Series routes (`/articles/series/[series]`) are real - they have static
+    // params, metadata, and a markdown twin - but were absent from the sitemap.
+    const seriesSlugs = [...new Set(articles.map((a) => a.seriesSlug).filter(Boolean))] as string[]
+    const seriesEntries: MetadataRoute.Sitemap = seriesSlugs.map((seriesSlug) => ({
+      url: `${baseUrl}/articles/series/${seriesSlug}`,
+      lastModified: newestArticleDate(articles.filter((a) => a.seriesSlug === seriesSlug)),
+      changeFrequency: 'weekly' as const,
+      priority: 0.6,
+    }))
+
+    const pageEntries: MetadataRoute.Sitemap = []
+    if (config?.listingPagination === 'pages') {
+      const pageSize = config.pageSize ?? DEFAULT_PAGE_SIZE
+      pageEntries.push(
+        ...paginationEntries(
+          `${baseUrl}/articles`,
+          articles.length,
+          pageSize,
+          newestArticleDate(articles),
+          0.5
+        )
+      )
+      for (const cat of categories) {
+        const inCategory = articles.filter((article) =>
+          (article.categories ?? []).some((name) => categoryToSlug(name) === cat.slug)
+        )
+        pageEntries.push(
+          ...paginationEntries(
+            `${baseUrl}/articles/category/${cat.slug}`,
+            inCategory.length,
+            pageSize,
+            newestArticleDate(inCategory),
+            0.4
+          )
+        )
+      }
+      for (const author of authors) {
+        const byAuthor = articles.filter((article) =>
+          getArticleAuthors(article, config).some((profile) => profile.slug === author.slug)
+        )
+        pageEntries.push(
+          ...paginationEntries(
+            `${baseUrl}/articles/authors/${author.slug}`,
+            byAuthor.length,
+            pageSize,
+            newestArticleDate(byAuthor),
+            0.4
+          )
+        )
+      }
+    }
+
+    return [
+      ...articleEntries,
+      ...categoryEntries,
+      ...authorEntries,
+      ...seriesEntries,
+      ...pageEntries,
+    ]
   } catch {
     return []
   }