@fullstackdatasolutions/articles 1.3.1 → 1.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fullstackdatasolutions/articles",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "private": false,
   "license": "MIT",
   "funding": {

package/src/__tests__/server-articles.test.ts

@@ -1038,6 +1038,44 @@ describe('AI markdown helpers', () => {
     expect(response.headers.get('Link')).not.toContain('alternate')
   })
 
+  it('keeps markdown twins out of the search index by default', async () => {
+    setupArticleTreeMock([
+      {
+        slug: 'allowed',
+        frontmatter: 'date: 2025-01-01\naiCrawl: true\n',
+        body: '# Allowed',
+      },
+    ])
+
+    const response = await getArticleMarkdownResponse('allowed', {
+      siteUrl: 'https://example.com',
+      siteName: 'Example',
+    })
+
+    expect(response.headers.get('X-Robots-Tag')).toBe('noindex')
+  })
+
+  it('always noindexes article twins, even with listingTwinNoindex off', async () => {
+    setupArticleTreeMock([
+      {
+        slug: 'allowed',
+        frontmatter: 'date: 2025-01-01\naiCrawl: true\n',
+        body: '# Allowed',
+      },
+    ])
+
+    const response = await getArticleMarkdownResponse('allowed', {
+      siteUrl: 'https://example.com',
+      siteName: 'Example',
+      listingTwinNoindex: false,
+    })
+
+    expect(response.headers.get('X-Robots-Tag')).toBe('noindex')
+    expect(response.headers.get('Link')).toBe(
+      '<https://example.com/articles/allowed>; rel="canonical"'
+    )
+  })
+
   it('generates robots rules for AI crawlers and blocked articles only', async () => {
     setupArticleTreeMock([
       { slug: 'allowed', frontmatter: 'date: 2025-01-01\naiCrawl: true\n' },
@@ -1451,9 +1489,20 @@ describe('getMarkdownTwinResponse', () => {
     const response = await getMarkdownTwinResponse('category/campaigns', config)
     expect(response.status).toBe(200)
     expect(response.headers.get('Content-Type')).toBe('text/markdown; charset=utf-8')
+    expect(response.headers.get('X-Robots-Tag')).toBe('noindex')
     await expect(response.text()).resolves.toContain('# campaigns')
   })
 
+  it('honours listingTwinNoindex: false on listing twins', async () => {
+    setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\n' }])
+    const response = await getMarkdownTwinResponse('category/campaigns', {
+      ...config,
+      listingTwinNoindex: false,
+    })
+    expect(response.status).toBe(200)
+    expect(response.headers.get('X-Robots-Tag')).toBeNull()
+  })
+
   it('404s a listing path that resolves to nothing', async () => {
     setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\n' }])
     const response = await getMarkdownTwinResponse('category/missing', config)

package/src/articlesConfig.ts

@@ -341,6 +341,22 @@ export interface ArticlesConfig {
    * a config is available (i.e. via `getArticleMarkdownResponse`).
    */
   markdownTwinHeader?: boolean
+  /**
+   * Set to `false` to stop serving the `category/`, `authors/`, and `series/`
+   * listing twins with `X-Robots-Tag: noindex`. Default: `true`.
+   *
+   * Only the listing twins are configurable. Article twins are always
+   * `noindex`: an article twin is the same text as exactly one HTML article,
+   * so indexing it can only ever split that article's own signal between two
+   * URLs. A listing twin is a different case - it is a generated index of a
+   * category or author rather than a copy of one page - so a site may have a
+   * reason to let it rank, and this leaves that open.
+   *
+   * Either way AI crawlers can still fetch every twin; `noindex` only removes
+   * them from search results, and never blocks the fetch that is the point of
+   * publishing them.
+   */
+  listingTwinNoindex?: boolean
   /**
    * Vendor-neutral event callback (Phase 27F). Fired by components/hooks at
    * meaningful reader-journey moments (see `ArticleEvent` in `events.ts`).

package/src/server-articles.ts

@@ -737,14 +737,21 @@ export async function getMarkdownTwinResponse(
   if (listing !== undefined) {
     if (listing === null) return new Response('Not Found', { status: 404 })
     reportAiCrawl(slug, config, options?.headers)
-    return new Response(listing, { headers: LISTING_MARKDOWN_HEADERS })
+    return new Response(listing, { headers: listingMarkdownHeaders(config) })
   }
   return getArticleMarkdownResponse(slug, config, options)
 }
 
-const LISTING_MARKDOWN_HEADERS = {
-  'Content-Type': 'text/markdown; charset=utf-8',
-  'Cache-Control': 'public, max-age=3600, s-maxage=3600',
+// A listing twin is a generated index of a category/author/series rather than
+// a copy of a single page, so unlike the article twin below this one is
+// configurable - a site may have a reason to let it rank. Opt out with
+// `listingTwinNoindex: false`.
+function listingMarkdownHeaders(config: ArticlesConfig): Record<string, string> {
+  return {
+    'Content-Type': 'text/markdown; charset=utf-8',
+    'Cache-Control': 'public, max-age=3600, s-maxage=3600',
+    ...(config.listingTwinNoindex !== false && { 'X-Robots-Tag': 'noindex' }),
+  }
 }
 
 // `undefined` means "not a listing path" (fall through to the article
@@ -789,6 +796,17 @@ export async function getArticleMarkdownResponse(
       'Content-Type': 'text/markdown; charset=utf-8',
       'Cache-Control': 'public, max-age=3600, s-maxage=3600',
       Link: `<${canonicalUrl}>; rel="canonical"`,
+      // A twin is a second representation of a page that is already indexed,
+      // so it should not compete with that page in a search index - on a large
+      // corpus it doubles the crawlable URL count with near-duplicate content.
+      // noindex rather than a robots.txt disallow: AI crawlers still need to
+      // fetch these, and a disallow would block the fetch that is the whole
+      // point of publishing them. Set here rather than in the consuming app
+      // because the `.md` path is typically a rewrite, so a host-level header
+      // rule keyed on `.md` never sees the request. Not configurable: an
+      // article twin is the same text as exactly one HTML article, so indexing
+      // it can only ever split that article's own signal across two URLs.
+      'X-Robots-Tag': 'noindex',
     },
   })
 }