@fullstackdatasolutions/articles 1.2.2 → 1.3.0

package/src/server-articles.ts

@@ -4,11 +4,13 @@ import fs from 'node:fs'
 import path from 'node:path'
 import readingTime from 'reading-time'
 import { getAuthorAvatar } from './authorUtils'
-import { markdownToHtml, extractToc } from './markdown'
+import { markdownToHtml, extractToc, deriveFaqFromHeadings } from './markdown'
 import type {
   Article,
   AuthorProfile,
   CategoryInfo,
+  CitationReference,
+  EntityReference,
   FaqItem,
   HowToStep,
   PathDefinition,
@@ -171,6 +173,97 @@ function parseOptionalString(raw: unknown): string | undefined {
   return typeof raw === 'string' && raw.trim().length > 0 ? raw.trim() : undefined
 }
 
+// `aiCrawl` is the one frontmatter flag with a config-level default
+// (`ArticlesConfig.aiCrawlDefault`). Explicit `true`/`false` in frontmatter
+// always wins; only an omitted/non-boolean key falls through to the config.
+// Omitting both reproduces the original "blocked unless opted in" behavior.
+// Explicit `faq` frontmatter always wins - derivation only fills the gap
+// for articles that never declared one, and only when the site opted in.
+function deriveFaq(markdownContent: string, config?: ArticlesConfig): FaqItem[] | undefined {
+  if (config?.deriveFaqFromHeadings !== true) return undefined
+  const derived = deriveFaqFromHeadings(markdownContent)
+  return derived.length ? derived : undefined
+}
+
+// Frontmatter `lastmod` always wins. Only `'fileMtime'` produces a value
+// here; `'published'` (the default) and `'none'` differ in what
+// `ArticleSEO` does with an absent `lastmod` - reuse `datePublished`, or
+// omit `dateModified` rather than repeat a stale date.
+function resolveLastmod(
+  rawLastmod: unknown,
+  rawDate: unknown,
+  filePath: string,
+  config?: ArticlesConfig
+): string | undefined {
+  const declared = parseDateField(rawLastmod)
+  if (declared) return declared
+  if (config?.lastmodFallback !== 'fileMtime') return undefined
+  try {
+    return parseDateField(fs.statSync(filePath).mtime)
+  } catch {
+    return parseDateField(rawDate)
+  }
+}
+
+function resolveAiCrawl(raw: unknown, config?: ArticlesConfig): boolean {
+  if (typeof raw === 'boolean') return raw
+  return config?.aiCrawlDefault === true
+}
+
+// A bare string resolves through `config.entities` first, so a corpus can
+// share one canonical name/`sameAs` pair per topic instead of each article
+// spelling it out (and spelling it differently). Unregistered strings still
+// work as plain names - `validateArticles` warns about them rather than
+// dropping the entry.
+function parseEntityReferences(
+  raw: unknown,
+  config?: ArticlesConfig
+): EntityReference[] | undefined {
+  if (!Array.isArray(raw)) return undefined
+  const items = raw
+    .map((item) => {
+      if (typeof item === 'string') {
+        const key = item.trim()
+        if (!key) return null
+        return config?.entities?.[key] ?? { name: key }
+      }
+      if (
+        typeof item === 'object' &&
+        item !== null &&
+        typeof (item as EntityReference).name === 'string'
+      ) {
+        const entity = item as EntityReference
+        const name = entity.name.trim()
+        if (!name) return null
+        return entity.sameAs ? { name, sameAs: entity.sameAs } : { name }
+      }
+      return null
+    })
+    .filter((item): item is EntityReference => item !== null)
+  return items.length ? items : undefined
+}
+
+function parseCitations(raw: unknown): CitationReference[] | undefined {
+  if (!Array.isArray(raw)) return undefined
+  const items = raw
+    .map((item) => {
+      if (typeof item === 'string') return item.trim() ? { name: item.trim() } : null
+      if (
+        typeof item === 'object' &&
+        item !== null &&
+        typeof (item as CitationReference).name === 'string'
+      ) {
+        const citation = item as CitationReference
+        const name = citation.name.trim()
+        if (!name) return null
+        return citation.url ? { name, url: citation.url } : { name }
+      }
+      return null
+    })
+    .filter((item): item is CitationReference => item !== null)
+  return items.length ? items : undefined
+}
+
 function parseSeriesOrder(raw: unknown): number | undefined {
   return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined
 }
@@ -292,7 +385,7 @@ async function getArticleSummary(slug: string, config?: ArticlesConfig): Promise
       title: data.title || slug.replaceAll('-', ' '),
       excerpt: data.excerpt || '',
       date: parseDateField(data.date),
-      lastmod: parseDateField(data.lastmod),
+      lastmod: resolveLastmod(data.lastmod, data.date, found.filePath, config),
       author,
       authors,
       authorSlug: primaryAuthorProfile?.slug,
@@ -305,14 +398,17 @@ async function getArticleSummary(slug: string, config?: ArticlesConfig): Promise
       tags: data.tags || [],
       contentType: found.contentType,
       draft: data.draft === true,
-      faq: parseFaqItems(data.faq),
+      faq: parseFaqItems(data.faq) ?? deriveFaq(markdownContent, config),
       howTo: parseHowToSteps(data.howTo),
+      answer: parseOptionalString(data.answer),
+      about: parseEntityReferences(data.about, config),
+      citation: parseCitations(data.citation),
       canonicalUrl: typeof data.canonicalUrl === 'string' ? data.canonicalUrl : undefined,
       articleType: typeof data.articleType === 'string' ? data.articleType : undefined,
       series: typeof data.series === 'string' ? data.series : undefined,
       seriesSlug: parseOptionalString(data.seriesSlug),
       seriesOrder: parseSeriesOrder(data.seriesOrder),
-      aiCrawl: data.aiCrawl === true,
+      aiCrawl: resolveAiCrawl(data.aiCrawl, config),
       searchTitle: parseOptionalString(data.searchTitle),
       searchDescription: parseOptionalString(data.searchDescription),
       socialTitle: parseOptionalString(data.socialTitle),
@@ -388,9 +484,12 @@ export async function getAdjacentArticles(
   return { previous, next }
 }
 
-export async function getArticleMarkdown(slug: string): Promise<string | null> {
+export async function getArticleMarkdown(
+  slug: string,
+  config?: ArticlesConfig
+): Promise<string | null> {
   try {
-    const summary = await getArticleSummary(slug)
+    const summary = await getArticleSummary(slug, config)
     if (!summary?.aiCrawl) return null
     const found = findArticleFile(slug)
     if (!found) return null
@@ -408,14 +507,275 @@ export async function getArticleMarkdown(slug: string): Promise<string | null> {
   }
 }
 
-export async function getArticleMarkdownResponse(
+/**
+ * Crawlers that read content on behalf of an answer engine. Used both to
+ * write robots.txt rules and to classify markdown-twin fetches for
+ * `ArticlesConfig.onAiCrawl` - one list, so the two can never disagree about
+ * what counts as an AI crawler.
+ */
+export const AI_CRAWLERS = [
+  'GPTBot',
+  'ChatGPT-User',
+  'OAI-SearchBot',
+  'CCBot',
+  'ClaudeBot',
+  'Claude-User',
+  'Claude-SearchBot',
+  'anthropic-ai',
+  'PerplexityBot',
+  'Perplexity-User',
+  'Google-Extended',
+  'Applebot-Extended',
+  'Bytespider',
+  'Amazonbot',
+  'meta-externalagent',
+  'cohere-ai',
+  'DuckAssistBot',
+  'MistralAI-User',
+] as const
+
+/** Matches a `User-Agent` against `AI_CRAWLERS`, returning the crawler name or `null`. */
+export function matchAiCrawler(userAgent: string): string | null {
+  if (!userAgent) return null
+  const normalized = userAgent.toLowerCase()
+  return AI_CRAWLERS.find((crawler) => normalized.includes(crawler.toLowerCase())) ?? null
+}
+
+/**
+ * Attribution header prepended to an article's markdown twin.
+ *
+ * The twin is what an AI crawler actually reads, and `matter()` strips every
+ * frontmatter field before it is served - so without this the model gets an
+ * anonymous body with no title, date, author, or link back to the canonical
+ * page. Skips its own `# {title}` line when the body already opens with the
+ * same H1, so the common "body repeats the title" layout doesn't end up with
+ * two.
+ */
+export function buildMarkdownTwinHeader(
+  article: Article,
+  config: ArticlesConfig,
+  body: string
+): string {
+  const siteUrl = config.siteUrl.replace(/\/$/, '')
+  const firstLine = body.trimStart().split('\n', 1)[0]?.trim() ?? ''
+  const bodyRepeatsTitle = firstLine.toLowerCase() === `# ${article.title}`.toLowerCase()
+
+  const facts = [
+    `Source: ${siteUrl}/articles/${article.slug}`,
+    article.date ? `Published: ${article.date}` : '',
+    article.lastmod ? `Updated: ${article.lastmod}` : '',
+    config.showAuthor !== false && article.author ? `Author: ${article.author}` : '',
+    `Site: ${config.siteName}`,
+  ].filter(Boolean)
+
+  const blocks = [
+    bodyRepeatsTitle ? '' : `# ${article.title}`,
+    article.excerpt ? `> ${article.excerpt}` : '',
+    facts.join('\n'),
+    article.answer ? `**Short answer:** ${article.answer}` : '',
+    '---',
+  ].filter((block) => block !== '')
+
+  return `${blocks.join('\n\n')}\n\n`
+}
+
+/**
+ * Reports a markdown-twin fetch to `config.onAiCrawl`. Never lets a consumer
+ * callback break the response - a telemetry handler throwing must not turn a
+ * served article into a 500.
+ */
+function reportAiCrawl(slug: string, config: ArticlesConfig, headers?: RequestHeaders): void {
+  if (!config.onAiCrawl) return
+  const userAgent = headers?.get('user-agent') ?? ''
+  try {
+    config.onAiCrawl({ slug, crawler: matchAiCrawler(userAgent) ?? 'unknown', userAgent })
+  } catch (error) {
+    reportArticlesError({
+      code: 'ai-crawl-handler-failed',
+      message: 'onAiCrawl handler threw.',
+      error,
+      context: { slug },
+    })
+  }
+}
+
+/** Minimal shape of a request's headers - avoids depending on `next/server` here. */
+export type RequestHeaders = Readonly<{ get(name: string): string | null }>
+
+/**
+ * Markdown twin for a listing surface - a category, author, or series.
+ *
+ * Article twins alone leave a model with a bag of pages and no map: the
+ * listing surfaces are what answer "what does this site cover, and who
+ * writes it". Only articles resolved to `aiCrawl: true` are listed, so a
+ * blocked article stays invisible here too.
+ */
+function buildListingMarkdown(
+  heading: string,
+  intro: string[],
+  articles: readonly Article[],
+  config: ArticlesConfig
+): string {
+  const siteUrl = config.siteUrl.replace(/\/$/, '')
+  const crawlable = articles.filter((article) => article.aiCrawl === true)
+  const entries = crawlable.map((article) => {
+    const summary = article.answer ?? article.excerpt
+    const line = `- [${article.title}](${siteUrl}/articles/${article.slug}.md)`
+    return summary ? `${line}: ${summary}` : line
+  })
+
+  return [
+    `# ${heading}`,
+    '',
+    ...intro.flatMap((line) => [line, '']),
+    `Source: ${siteUrl}`,
+    `Site: ${config.siteName}`,
+    '',
+    '---',
+    '',
+    ...(entries.length > 0 ? entries : ['_No articles available._']),
+    '',
+  ].join('\n')
+}
+
+/** Markdown twin for `/articles/category/[category]`. `null` when the category has no articles. */
+export async function getCategoryMarkdown(
+  categorySlug: string,
+  config: ArticlesConfig
+): Promise<string | null> {
+  const articles = await getArticlesByCategory(categorySlug, config)
+  if (articles.length === 0) return null
+  const name =
+    articles[0].categories.find((c) => categoryToSlug(c) === categorySlug) ?? categorySlug
+  const description = resolveCategoryDescription(categorySlug, config)
+  return buildListingMarkdown(name, description ? [description] : [], articles, config)
+}
+
+function resolveCategoryDescription(
+  categorySlug: string,
+  config: ArticlesConfig
+): string | undefined {
+  const entry = config.categoryDescriptions?.[categorySlug]
+  if (!entry) return undefined
+  return typeof entry === 'string' ? entry : (entry.long ?? entry.short)
+}
+
+/**
+ * Markdown twin for `/articles/authors/[author]`.
+ *
+ * Carries the author's bio, promise, principles, and sourced proof alongside
+ * their article list - the "who is this and why trust them" context that
+ * otherwise exists only inside React components, and the exact question asked
+ * before anything they wrote gets cited. `credentials` are included as the
+ * author's own stated claims; unlike JSON-LD, prose can attribute a claim
+ * without asserting it as a verified fact.
+ */
+export async function getAuthorMarkdown(
+  authorSlug: string,
+  config: ArticlesConfig
+): Promise<string | null> {
+  const author = getAuthorBySlug(authorSlug, config)
+  if (!author) return null
+  const articles = await getArticlesByAuthor(authorSlug, config)
+
+  const intro = [
+    author.promise ?? '',
+    author.bio ?? '',
+    ...(author.servesWho?.length ? [`Writes for: ${author.servesWho.join(', ')}`] : []),
+    ...(author.knowsAbout?.length ? [`Writes about: ${author.knowsAbout.join(', ')}`] : []),
+    ...(author.credentials?.length
+      ? ['## Stated experience', ...author.credentials.map((item) => `- ${item}`)]
+      : []),
+    ...(author.proof?.length
+      ? [
+          '## Proof points',
+          ...author.proof.map((item) =>
+            item.url ? `- [${item.claim}](${item.url})` : `- ${item.claim}`
+          ),
+        ]
+      : []),
+    ...(author.originStory?.length
+      ? [
+          '## Background',
+          ...author.originStory.flatMap((section) => [
+            ...(section.heading ? [`### ${section.heading}`] : []),
+            ...section.paragraphs,
+          ]),
+        ]
+      : []),
+  ].filter((line) => line.trim() !== '')
+
+  return buildListingMarkdown(author.name, intro, articles, config)
+}
+
+/** Markdown twin for `/articles/series/[series]`. `null` when the series has no articles. */
+export async function getSeriesMarkdown(
+  seriesSlug: string,
+  config: ArticlesConfig
+): Promise<string | null> {
+  const articles = await getArticlesBySeries(seriesSlug, config)
+  if (articles.length === 0) return null
+  const name = articles[0].series ?? seriesSlug
+  return buildListingMarkdown(name, [], articles, config)
+}
+
+/**
+ * Serves the markdown twin for any `/articles/...` path - an article, or a
+ * category/author/series listing.
+ *
+ * Dispatching on the slug prefix here rather than adding three more app
+ * routes keeps the existing single rewrite (`/articles/:path*.md`) working
+ * unchanged: without it, `/articles/category/campaigns.md` matches that
+ * rewrite, reaches the article handler, and 404s.
+ */
+export async function getMarkdownTwinResponse(
+  slug: string,
+  config: ArticlesConfig,
+  options?: Readonly<{ headers?: RequestHeaders }>
+): Promise<Response> {
+  const listing = await resolveListingMarkdown(slug, config)
+  if (listing !== undefined) {
+    if (listing === null) return new Response('Not Found', { status: 404 })
+    reportAiCrawl(slug, config, options?.headers)
+    return new Response(listing, { headers: LISTING_MARKDOWN_HEADERS })
+  }
+  return getArticleMarkdownResponse(slug, config, options)
+}
+
+const LISTING_MARKDOWN_HEADERS = {
+  'Content-Type': 'text/markdown; charset=utf-8',
+  'Cache-Control': 'public, max-age=3600, s-maxage=3600',
+}
+
+// `undefined` means "not a listing path" (fall through to the article
+// handler); `null` means "a listing path that resolved to nothing" (404).
+async function resolveListingMarkdown(
   slug: string,
   config: ArticlesConfig
+): Promise<string | null | undefined> {
+  const [prefix, ...rest] = slug.split('/')
+  const key = rest.join('/')
+  if (!key) return undefined
+  if (prefix === 'category') return getCategoryMarkdown(key, config)
+  if (prefix === 'authors') return getAuthorMarkdown(key, config)
+  if (prefix === 'series') return getSeriesMarkdown(key, config)
+  return undefined
+}
+
+export async function getArticleMarkdownResponse(
+  slug: string,
+  config: ArticlesConfig,
+  options?: Readonly<{ headers?: RequestHeaders }>
 ): Promise<Response> {
-  const markdown = await getArticleMarkdown(slug)
+  const markdown = await getArticleMarkdown(slug, config)
   if (markdown === null) return new Response('Not Found', { status: 404 })
-  const article = await getArticleMetadata(slug)
-  return new Response(markdown, {
+  const article = await getArticleMetadata(slug, config)
+  reportAiCrawl(slug, config, options?.headers)
+  const body =
+    article && config.markdownTwinHeader !== false
+      ? `${buildMarkdownTwinHeader(article, config, markdown)}${markdown.trimStart()}`
+      : markdown
+  return new Response(body, {
     headers: {
       'Content-Type': 'text/markdown; charset=utf-8',
       'Cache-Control': 'public, max-age=3600, s-maxage=3600',
@@ -449,27 +809,18 @@ export function getArticleAiHeaders(
   }
 }
 
-export async function getAiRobotsTxtRules(): Promise<string> {
-  const articles = await getAllArticles()
+export async function getAiRobotsTxtRules(config?: ArticlesConfig): Promise<string> {
+  const articles = await getAllArticles(config)
   const blockedArticles = articles.filter((article) => article.aiCrawl !== true)
   if (blockedArticles.length === 0) return ''
 
-  const aiCrawlers = [
-    'GPTBot',
-    'ChatGPT-User',
-    'CCBot',
-    'ClaudeBot',
-    'Claude-User',
-    'PerplexityBot',
-    'Google-Extended',
-  ]
   const disallowRules = blockedArticles
     .map((article) => `Disallow: /articles/${article.slug}`)
     .join('\n')
 
-  return aiCrawlers
-    .map((crawler) => [`User-agent: ${crawler}`, disallowRules].join('\n'))
-    .join('\n\n')
+  return AI_CRAWLERS.map((crawler) => [`User-agent: ${crawler}`, disallowRules].join('\n')).join(
+    '\n\n'
+  )
 }
 
 export async function searchArticles(query: string, config?: ArticlesConfig): Promise<Article[]> {

package/src/server.ts

@@ -2,6 +2,13 @@
 export {
   getAllArticles,
   getAiRobotsTxtRules,
+  buildMarkdownTwinHeader,
+  getMarkdownTwinResponse,
+  getCategoryMarkdown,
+  getAuthorMarkdown,
+  getSeriesMarkdown,
+  matchAiCrawler,
+  AI_CRAWLERS,
   getArticleAiHeaders,
   getArticleMarkdown,
   getArticleMarkdownResponse,
@@ -28,6 +35,8 @@ export {
 
 export {
   generateRssFeed,
+  generateLlmsTxt,
+  generateLlmsFullTxt,
   generateArticleStaticParams,
   generateCategoryStaticParams,
   generateSeriesStaticParams,
@@ -59,11 +68,23 @@ export {
   isPageOutOfRange,
 } from './pagination'
 
-export { markdownToHtml, extractToc, getContentSlotBoundaries } from './markdown'
+export {
+  markdownToHtml,
+  extractToc,
+  getContentSlotBoundaries,
+  deriveFaqFromHeadings,
+} from './markdown'
 export { setArticlesErrorHandler } from './errorReporting'
-export { getBreadcrumbsConfig } from './articlesConfig'
+export {
+  formatPageTitle,
+  getBreadcrumbsConfig,
+  getOrganizationId,
+  getPersonId,
+  getWebSiteId,
+} from './articlesConfig'
 export { ArticleContent } from './ArticleContent'
 export { ArticleTOC } from './ArticleTOC'
+export { ArticleAnswer } from './ArticleAnswer'
 export { validateArticles, validateAllArticles } from './validateArticles'
 export { emitArticleEvent } from './events'
 
@@ -73,14 +94,24 @@ export type {
   AuthorSocial,
   BreadcrumbItem,
   CategoryInfo,
+  CitationReference,
+  EntityReference,
+  FaqItem,
+  HowToStep,
   PathDefinition,
   TocItem,
 } from './articleTypes'
-export type { ArticlesConfig, LinkTargetStrategy, ListingPagination } from './articlesConfig'
+export type {
+  AiCrawlEvent,
+  ArticlesConfig,
+  LinkTargetStrategy,
+  ListingPagination,
+  OrganizationConfig,
+} from './articlesConfig'
 export type { PaginatedArticles, PaginationLinks, ListingPaginationContext } from './pagination'
 export type { ContentSlotBoundaries } from './markdown'
 export type { ArticleSlotContext, ArticleSlotContent } from './ArticleContent'
-export type { RelatedContentResult, RelatedContentSource } from './server-articles'
+export type { RelatedContentResult, RelatedContentSource, RequestHeaders } from './server-articles'
 export type { ValidationIssue, ValidationResult, ValidationSeverity } from './validateArticles'
 export type {
   ArticleEvent,