@fullstackdatasolutions/articles 1.3.1 → 1.4.0

package/dist/server.d.cts

@@ -599,6 +599,22 @@ interface ArticlesConfig {
      * a config is available (i.e. via `getArticleMarkdownResponse`).
      */
     markdownTwinHeader?: boolean;
+    /**
+     * Set to `false` to stop serving the `category/`, `authors/`, and `series/`
+     * listing twins with `X-Robots-Tag: noindex`. Default: `true`.
+     *
+     * Only the listing twins are configurable. Article twins are always
+     * `noindex`: an article twin is the same text as exactly one HTML article,
+     * so indexing it can only ever split that article's own signal between two
+     * URLs. A listing twin is a different case - it is a generated index of a
+     * category or author rather than a copy of one page - so a site may have a
+     * reason to let it rank, and this leaves that open.
+     *
+     * Either way AI crawlers can still fetch every twin; `noindex` only removes
+     * them from search results, and never blocks the fetch that is the point of
+     * publishing them.
+     */
+    listingTwinNoindex?: boolean;
     /**
      * Vendor-neutral event callback (Phase 27F). Fired by components/hooks at
      * meaningful reader-journey moments (see `ArticleEvent` in `events.ts`).

package/dist/server.d.ts

@@ -599,6 +599,22 @@ interface ArticlesConfig {
      * a config is available (i.e. via `getArticleMarkdownResponse`).
      */
     markdownTwinHeader?: boolean;
+    /**
+     * Set to `false` to stop serving the `category/`, `authors/`, and `series/`
+     * listing twins with `X-Robots-Tag: noindex`. Default: `true`.
+     *
+     * Only the listing twins are configurable. Article twins are always
+     * `noindex`: an article twin is the same text as exactly one HTML article,
+     * so indexing it can only ever split that article's own signal between two
+     * URLs. A listing twin is a different case - it is a generated index of a
+     * category or author rather than a copy of one page - so a site may have a
+     * reason to let it rank, and this leaves that open.
+     *
+     * Either way AI crawlers can still fetch every twin; `noindex` only removes
+     * them from search results, and never blocks the fetch that is the point of
+     * publishing them.
+     */
+    listingTwinNoindex?: boolean;
     /**
      * Vendor-neutral event callback (Phase 27F). Fired by components/hooks at
      * meaningful reader-journey moments (see `ArticleEvent` in `events.ts`).

package/dist/server.js

@@ -1025,15 +1025,17 @@ function getMarkdownTwinResponse(slug, config, options) {
     if (listing !== void 0) {
       if (listing === null) return new Response("Not Found", { status: 404 });
       reportAiCrawl(slug, config, options == null ? void 0 : options.headers);
-      return new Response(listing, { headers: LISTING_MARKDOWN_HEADERS });
+      return new Response(listing, { headers: listingMarkdownHeaders(config) });
     }
     return getArticleMarkdownResponse(slug, config, options);
   });
 }
-var LISTING_MARKDOWN_HEADERS = {
-  "Content-Type": "text/markdown; charset=utf-8",
-  "Cache-Control": "public, max-age=3600, s-maxage=3600"
-};
+function listingMarkdownHeaders(config) {
+  return __spreadValues({
+    "Content-Type": "text/markdown; charset=utf-8",
+    "Cache-Control": "public, max-age=3600, s-maxage=3600"
+  }, config.listingTwinNoindex !== false && { "X-Robots-Tag": "noindex" });
+}
 function resolveListingMarkdown(slug, config) {
   return __async(this, null, function* () {
     const [prefix, ...rest] = slug.split("/");
@@ -1057,7 +1059,18 @@ function getArticleMarkdownResponse(slug, config, options) {
       headers: {
         "Content-Type": "text/markdown; charset=utf-8",
         "Cache-Control": "public, max-age=3600, s-maxage=3600",
-        Link: `<${canonicalUrl}>; rel="canonical"`
+        Link: `<${canonicalUrl}>; rel="canonical"`,
+        // A twin is a second representation of a page that is already indexed,
+        // so it should not compete with that page in a search index - on a large
+        // corpus it doubles the crawlable URL count with near-duplicate content.
+        // noindex rather than a robots.txt disallow: AI crawlers still need to
+        // fetch these, and a disallow would block the fetch that is the whole
+        // point of publishing them. Set here rather than in the consuming app
+        // because the `.md` path is typically a rewrite, so a host-level header
+        // rule keyed on `.md` never sees the request. Not configurable: an
+        // article twin is the same text as exactly one HTML article, so indexing
+        // it can only ever split that article's own signal across two URLs.
+        "X-Robots-Tag": "noindex"
       }
     });
   });