@rsc-kit/core 0.29.7 → 0.29.9

package/dist/crawler.d.ts

@@ -0,0 +1,2 @@
+/** Whether this user agent is answered with the finished document. */
+export declare function isCrawler(userAgent: string | null | undefined): boolean;

package/dist/crawler.js

@@ -0,0 +1,62 @@
+// Who is answered with the finished document.
+//
+// A person is sent the shell the moment it exists and everything else as it
+// streams: the page paints at once, the title and the slow regions arrive
+// when they are ready. The status line leaves with the shell, so a notFound()
+// or a redirect() decided later than that is delivered in the body - which a
+// person's browser acts on and a search engine's indexer does not. It keeps
+// a 200, a soft 404.
+//
+// A crawler is not waiting to see anything. It is answered once the whole
+// render is done: a notFound() anywhere is a real 404, a redirect anywhere a
+// real 3xx, and the generated metadata is in <head> where a link preview's
+// scraper reads it and nowhere else. The cost - the slowest read on the page
+// before the first byte - is paid only by something that is not a person.
+//
+// Matched by name, from the user agents the engines and preview scrapers
+// publish. A crawler missing from the list is served as a person is, which
+// is correct HTML with a noindex tag where the page was missing; one that
+// should be here is a line to add.
+const CRAWLERS = new RegExp([
+    // Search engines.
+    'Googlebot',
+    'Google-InspectionTool',
+    'Storebot-Google',
+    'AdsBot-Google',
+    'Mediapartners-Google',
+    'bingbot',
+    'BingPreview',
+    'Slurp',
+    'DuckDuckBot',
+    'Baiduspider',
+    'YandexBot',
+    'YandexMobileBot',
+    'Sogou',
+    'Applebot',
+    'PetalBot',
+    'SeznamBot',
+    'Yeti',
+    'ia_archiver',
+    // Link previews: a share card is the page's metadata, read once.
+    'facebookexternalhit',
+    'facebookcatalog',
+    'meta-externalagent',
+    'Twitterbot',
+    'LinkedInBot',
+    'Slackbot',
+    'Discordbot',
+    'TelegramBot',
+    'WhatsApp',
+    'redditbot',
+    'Pinterestbot',
+    'SkypeUriPreview',
+    'vkShare',
+    'Embedly',
+    'bitlybot',
+    'Iframely',
+].join('|'), 'i');
+/** Whether this user agent is answered with the finished document. */
+export function isCrawler(userAgent) {
+    return typeof userAgent === 'string' && CRAWLERS.test(userAgent);
+}
+//# sourceMappingURL=crawler.js.map

package/dist/crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../src/crawler.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAC9C,EAAE;AACF,4EAA4E;AAC5E,0EAA0E;AAC1E,8EAA8E;AAC9E,6EAA6E;AAC7E,4EAA4E;AAC5E,qBAAqB;AACrB,EAAE;AACF,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,0EAA0E;AAC1E,EAAE;AACF,yEAAyE;AACzE,2EAA2E;AAC3E,0EAA0E;AAC1E,mCAAmC;AAEnC,MAAM,QAAQ,GAAG,IAAI,MAAM,CACzB;IACE,kBAAkB;IAClB,WAAW;IACX,uBAAuB;IACvB,iBAAiB;IACjB,eAAe;IACf,sBAAsB;IACtB,SAAS;IACT,aAAa;IACb,OAAO;IACP,aAAa;IACb,aAAa;IACb,WAAW;IACX,iBAAiB;IACjB,OAAO;IACP,UAAU;IACV,UAAU;IACV,WAAW;IACX,MAAM;IACN,aAAa;IACb,iEAAiE;IACjE,qBAAqB;IACrB,iBAAiB;IACjB,oBAAoB;IACpB,YAAY;IACZ,aAAa;IACb,UAAU;IACV,YAAY;IACZ,aAAa;IACb,UAAU;IACV,WAAW;IACX,cAAc;IACd,iBAAiB;IACjB,SAAS;IACT,SAAS;IACT,UAAU;IACV,UAAU;CACX,CAAC,IAAI,CAAC,GAAG,CAAC,EACX,GAAG,CACJ,CAAA;AAED,sEAAsE;AACtE,MAAM,UAAU,SAAS,CAAC,SAAoC;IAC5D,OAAO,OAAO,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;AAClE,CAAC","sourcesContent":["// Who is answered with the finished document.\n//\n// A person is sent the shell the moment it exists and everything else as it\n// streams: the page paints at once, the title and the slow regions arrive\n// when they are ready. The status line leaves with the shell, so a notFound()\n// or a redirect() decided later than that is delivered in the body - which a\n// person's browser acts on and a search engine's indexer does not. It keeps\n// a 200, a soft 404.\n//\n// A crawler is not waiting to see anything. It is answered once the whole\n// render is done: a notFound() anywhere is a real 404, a redirect anywhere a\n// real 3xx, and the generated metadata is in <head> where a link preview's\n// scraper reads it and nowhere else. The cost - the slowest read on the page\n// before the first byte - is paid only by something that is not a person.\n//\n// Matched by name, from the user agents the engines and preview scrapers\n// publish. A crawler missing from the list is served as a person is, which\n// is correct HTML with a noindex tag where the page was missing; one that\n// should be here is a line to add.\n\nconst CRAWLERS = new RegExp(\n  [\n    // Search engines.\n    'Googlebot',\n    'Google-InspectionTool',\n    'Storebot-Google',\n    'AdsBot-Google',\n    'Mediapartners-Google',\n    'bingbot',\n    'BingPreview',\n    'Slurp',\n    'DuckDuckBot',\n    'Baiduspider',\n    'YandexBot',\n    'YandexMobileBot',\n    'Sogou',\n    'Applebot',\n    'PetalBot',\n    'SeznamBot',\n    'Yeti',\n    'ia_archiver',\n    // Link previews: a share card is the page's metadata, read once.\n    'facebookexternalhit',\n    'facebookcatalog',\n    'meta-externalagent',\n    'Twitterbot',\n    'LinkedInBot',\n    'Slackbot',\n    'Discordbot',\n    'TelegramBot',\n    'WhatsApp',\n    'redditbot',\n    'Pinterestbot',\n    'SkypeUriPreview',\n    'vkShare',\n    'Embedly',\n    'bitlybot',\n    'Iframely',\n  ].join('|'),\n  'i',\n)\n\n/** Whether this user agent is answered with the finished document. */\nexport function isCrawler(userAgent: string | null | undefined): boolean {\n  return typeof userAgent === 'string' && CRAWLERS.test(userAgent)\n}\n"]}

package/dist/host.d.ts

@@ -83,15 +83,6 @@ export interface RscEngine {
     share?: string): Promise<{
         rscPayload: string;
     }>;
-    /**
-     * The page's metadata for these params, merged with its layouts'. Used to
-     * put the real title into a shell stored for a whole pattern, whose build
-     * could not know the url.
-     */
-    resolveMetadata?(component: string, props?: Record<string, unknown>, layouts?: {
-        component: string;
-        props: Record<string, unknown>;
-    }[]): Promise<Record<string, unknown> | null>;
     /**
      * Run a route's middleware without rendering anything.
      *

package/dist/host.js

@@ -47,12 +47,12 @@ export function pageSaidNotFound(request) {
     return notFoundRequests.has(request);
 }
 export { revalidate } from "./revalidate.js";
+import { isCrawler } from "./crawler.js";
 import { currentNotFound, withRedirect } from "./redirect.js";
 import { compressed } from "./compress.js";
 import { withCache } from "./cache.js";
 import { takeAfterWork, withRequest, withResponseDraft } from "./request.js";
 import { criticalAssetsOf, linkHeader, mergeAssets } from "./earlyHints.js";
-import { withHead } from "./shellHead.js";
 /**
  * @internal For a host adapter that embeds the engine. An app imports this
  * from `@rsc-kit/core/redirect` - the one the guides teach, and the one an
@@ -123,6 +123,16 @@ function redirectResponse(to, isPayloadRequest) {
         headers: { Location: to.location, Vary: VARY_ON_RSC },
     });
 }
+/**
+ * What a page that said notFound() after its status line had gone out ends
+ * with: 200 is spent, so the document asks not to be indexed instead - a
+ * not-found page that a search engine keeps as a real one is a soft 404.
+ * The tag, then a script that moves it into <head>, where a crawler that
+ * runs the page reads it; a crawler that does not run it is answered with
+ * the whole document and a real 404 - see `forCrawler`.
+ */
+const LATE_NOINDEX = '<meta name="robots" content="noindex">' +
+    "<script>document.head.appendChild(document.currentScript.previousSibling)</script>";
 /**
  * Append the redirect a render asked for after its shell had already gone out.
  *
@@ -134,7 +144,7 @@ function redirectResponse(to, isPayloadRequest) {
  * Read on flush, because that is when the render has finished and a redirect
  * from inside a Suspense boundary is finally known.
  */
-function appendLateRedirect(stream, taken) {
+function appendLateRedirect(stream, taken, missing = () => false) {
     // An engine that answered with a finished body rather than a stream has no
     // late window at all: the render was over before this was called, so the
     // caller's own check already saw everything there was to see.
@@ -158,6 +168,9 @@ function appendLateRedirect(stream, taken) {
             if (to) {
                 controller.enqueue(encoder.encode(`<script>location.replace(${inScript(to.location)})</script>`));
             }
+            else if (missing()) {
+                controller.enqueue(encoder.encode(LATE_NOINDEX));
+            }
             controller.close();
         },
         cancel(reason) {
@@ -846,7 +859,7 @@ export function createRscHandler(options) {
         if (request.headers.get(HEADER.rsc) === null) {
             // Scoped to this render, so two requests redirecting at once cannot read
             // each other's destination.
-            return await withRedirect(async (taken) => {
+            return await withRedirect(async (taken, missing) => {
                 // Awaited, and that await is the whole design: React resolves this
                 // when the SHELL is ready, so a redirect thrown above every Suspense
                 // boundary rejects here — before a byte is written, with a status line
@@ -901,7 +914,7 @@ export function createRscHandler(options) {
                 // stays 200 because the status line has gone.
                 if (currentNotFound())
                     return saidNotFound(request);
-                return new Response(appendLateRedirect(htmlStream, taken), {
+                return new Response(appendLateRedirect(htmlStream, taken, missing), {
                     headers: withVersion({
                         "Content-Type": HTML_TYPE,
                         [HEADER.layouts]: chain.join(","),
@@ -1276,6 +1289,11 @@ export function createRscHandler(options) {
                     "whole instead. Rebuild the prerendered output with the server bundle - a cached .output is the usual cause.");
                 return null;
             }
+            // A crawler is answered with the finished page, rendered now: a shell
+            // has gone out with its 200 before its holes know whether the page
+            // exists. See crawler.ts.
+            if (isCrawler(request.headers.get("user-agent")))
+                return null;
             // A shell with no state cannot be finished by anyone. It was frozen with
             // its fallbacks showing and there is no record of what came next, so
             // serving it would be serving a page that stays on its loading state for
@@ -1294,33 +1312,37 @@ export function createRscHandler(options) {
             const forShape = shellKey === key
                 ? route.params
                 : Object.fromEntries(Object.keys(route.params).map((name) => [name, PARAM_PLACEHOLDER]));
-            const { htmlStream, replayed } = await engine.handleRscResume(route.route.component, forShape, 
-            // Empty props, because that is what the build passed. Resuming replays
-            // the tree against the slots the shell left, and React matches those by
-            // key — so an argument that differs from the frozen render at all is a
-            // tree that "doesn't match", and every boundary falls back to the
-            // client instead of being filled here.
-            route.route.layouts.map((component) => ({ component, props: {} })), route.route.loadings, route.route.slots, {}, JSON.parse(state), undefined, 
-            // A shell found under the route's pattern was frozen for no particular
-            // url, so it was rendered with no page key. Handing one over now would
-            // key the tree differently from the one being resumed.
-            shellKey === key ? url.pathname : "", 
-            // But the url itself, for the hooks: the holes are rendered for it.
-            url.pathname, route.params);
-            // A shell stored for the pattern was built without a url, so its title
-            // is the layouts' - the page's generateMetadata reads the params and
-            // was left out. This request has the params.
-            const served = shellKey === key || !engine.resolveMetadata
-                ? shell
-                : withHead(shell, await engine
-                    .resolveMetadata(route.route.component, route.params, route.route.layouts.map((component) => ({ component, props: {} })))
-                    .catch(() => null));
+            // In a render scope, so a notFound() in a hole is recorded for the
+            // stream's end - see LATE_NOINDEX.
+            let missing = () => false;
+            const resume = engine.handleRscResume;
+            const { htmlStream, replayed } = await withRedirect((_, said) => {
+                missing = said;
+                return resume(route.route.component, forShape, 
+                // Empty props, because that is what the build passed. Resuming replays
+                // the tree against the slots the shell left, and React matches those by
+                // key — so an argument that differs from the frozen render at all is a
+                // tree that "doesn't match", and every boundary falls back to the
+                // client instead of being filled here.
+                route.route.layouts.map((component) => ({ component, props: {} })), route.route.loadings, route.route.slots, {}, JSON.parse(state), undefined, 
+                // A shell found under the route's pattern was frozen for no particular
+                // url, so it was rendered with no page key. Handing one over now would
+                // key the tree differently from the one being resumed.
+                shellKey === key ? url.pathname : "", 
+                // But the url itself, for the hooks: the holes are rendered for it.
+                url.pathname, route.params);
+            });
+            // A shell stored for the pattern has no title of the page's: its
+            // generateMetadata streamed under its own boundary, the probe left that
+            // a hole, and the resume fills it with the real one. Nothing is looked
+            // up before the shell leaves.
+            //
             // The shell first, then whatever the resume writes. React's own script
             // travels with the resumed segments and moves them into place, so this is
             // a plain concatenation and the holes land without hydration.
             const body = new ReadableStream({
                 async start(controller) {
-                    controller.enqueue(new TextEncoder().encode(served));
+                    controller.enqueue(new TextEncoder().encode(shell));
                     const reader = htmlStream.getReader();
                     try {
                         while (true) {
@@ -1329,6 +1351,8 @@ export function createRscHandler(options) {
                                 break;
                             controller.enqueue(value);
                         }
+                        if (missing())
+                            controller.enqueue(new TextEncoder().encode(LATE_NOINDEX));
                     }
                     finally {
                         controller.close();