@farm.js/core 0.1.24 → 0.1.26

package/dist/{chunk-7BC6GAL7.mjs → chunk-RAI7NULA.mjs}

@@ -2,7 +2,10 @@ import {
   getFarmMarkdownMirrorPath,
   isFarmMarkdownMirrorExposed,
   matchesFarmMarkdownRoutePattern
-} from "./chunk-OFEY6KHO.mjs";
+} from "./chunk-OOIL3BWR.mjs";
+import {
+  FARM_INTERNAL_READ_HEADER
+} from "./chunk-MVL2RAJW.mjs";
 import {
   applyFarmBasePath
 } from "./chunk-4RB44K6Z.mjs";
@@ -67,6 +70,7 @@ function collectFarmLlmsTxtPages(sources, options) {
       continue;
     }
     if (exclude.some((pattern) => matchesFarmMarkdownRoutePattern(pattern, path))) continue;
+    if (isMetadataNoIndex(source.metadata)) continue;
     const target = isFarmMarkdownMirrorExposed(options.markdown, path) ? getFarmMarkdownMirrorPath(path) : path;
     const description = readMetadataString(source.metadata, "description");
     pages.set(path, {
@@ -79,6 +83,17 @@ function collectFarmLlmsTxtPages(sources, options) {
   return [...pages.values()].sort((left, right) => left.path.localeCompare(right.path));
 }
 __name(collectFarmLlmsTxtPages, "collectFarmLlmsTxtPages");
+function isMetadataNoIndex(metadata) {
+  if (!metadata || typeof metadata !== "object") return false;
+  const robots = metadata.robots;
+  if (typeof robots === "string") {
+    return robots.split(",").map((directive) => directive.trim().toLowerCase()).some((directive) => directive === "noindex" || directive === "none");
+  }
+  return Boolean(
+    robots && typeof robots === "object" && robots.index === false
+  );
+}
+__name(isMetadataNoIndex, "isMetadataNoIndex");
 function createFarmDefaultLlmsTxt(options) {
   const { config, pages } = options;
   const summary = config.summary ?? readMetadataString(options.rootMetadata, "description");
@@ -146,7 +161,9 @@ function createFarmLlmsMarkdownReader(fetchMarkdown) {
   return async (url) => {
     try {
       const response = await fetchMarkdown(
-        new Request(url, { headers: { accept: "text/markdown" } })
+        new Request(url, {
+          headers: { accept: "text/markdown", [FARM_INTERNAL_READ_HEADER]: "llms-full" }
+        })
       );
       const type = response.headers.get("content-type") ?? "";
       if (!response.ok || !type.toLowerCase().startsWith("text/markdown")) {
@@ -160,6 +177,8 @@ function createFarmLlmsMarkdownReader(fetchMarkdown) {
   };
 }
 __name(createFarmLlmsMarkdownReader, "createFarmLlmsMarkdownReader");
+var FARM_LLMS_FULL_PAGE_TIMEOUT_MS = 3e4;
+var LLMS_PAGE_TIMED_OUT = /* @__PURE__ */ Symbol("farm.llms.pageTimedOut");
 async function renderFarmLlmsFullTxt(value, options) {
   const index = serializeFarmLlmsTxt(value);
   if (typeof value === "string") return index;
@@ -173,10 +192,25 @@ async function renderFarmLlmsFullTxt(value, options) {
       links.push(link);
     }
   }
+  const pageTimeout = options.pageTimeout ?? FARM_LLMS_FULL_PAGE_TIMEOUT_MS;
+  const readWithinTimeout = /* @__PURE__ */ __name(async (url) => {
+    let timer;
+    const timedOut = new Promise((resolve) => {
+      timer = setTimeout(resolve, pageTimeout, LLMS_PAGE_TIMED_OUT);
+    });
+    try {
+      const content = await Promise.race([options.readMarkdown(url), timedOut]);
+      if (content !== LLMS_PAGE_TIMED_OUT) return content;
+      options.onIncomplete?.();
+      return null;
+    } finally {
+      clearTimeout(timer);
+    }
+  }, "readWithinTimeout");
   const contents = await mapWithConcurrency(
     links,
     options.concurrency ?? 4,
-    (link) => shouldInlineLlmsLink(link.url, options.origin) ? options.readMarkdown(link.url) : null
+    (link) => shouldInlineLlmsLink(link.url, options.origin) ? readWithinTimeout(link.url) : null
   );
   const blocks = [`# ${toSingleLine(llms.title)}`];
   if (llms.summary?.trim()) blocks.push(`> ${toSingleLine(llms.summary)}`);
@@ -193,6 +227,44 @@ async function renderFarmLlmsFullTxt(value, options) {
 `;
 }
 __name(renderFarmLlmsFullTxt, "renderFarmLlmsFullTxt");
+function createFarmLlmsFullRenderer(options = {}) {
+  const now = options.now ?? Date.now;
+  const ttl = (options.revalidate ?? 0) * 1e3;
+  const maxEntries = options.maxEntries ?? 16;
+  const entries = /* @__PURE__ */ new Map();
+  let queue = Promise.resolve();
+  const settle = /* @__PURE__ */ __name((body, complete, onSettled) => onSettled ? body.then((value) => (onSettled(complete()), value)) : body, "settle");
+  return (origin, render, onSettled) => {
+    const entry = entries.get(origin);
+    if (entry && entry.expires > now()) return settle(entry.body, entry.complete, onSettled);
+    entries.delete(origin);
+    let complete = true;
+    const body = queue.then(
+      () => render({
+        incomplete: /* @__PURE__ */ __name(() => {
+          complete = false;
+        }, "incomplete")
+      })
+    );
+    queue = body.catch(() => {
+    });
+    const pending = { body, expires: Number.POSITIVE_INFINITY, complete: /* @__PURE__ */ __name(() => complete, "complete") };
+    entries.set(origin, pending);
+    if (entries.size > maxEntries) entries.delete(entries.keys().next().value);
+    body.then(
+      () => {
+        if (entries.get(origin) !== pending) return;
+        if (ttl > 0 && complete) pending.expires = now() + ttl;
+        else entries.delete(origin);
+      },
+      () => {
+        if (entries.get(origin) === pending) entries.delete(origin);
+      }
+    );
+    return settle(body, pending.complete, onSettled);
+  };
+}
+__name(createFarmLlmsFullRenderer, "createFarmLlmsFullRenderer");
 function stripFrontmatter(markdown) {
   const match = /^\uFEFF?---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/.exec(markdown);
   return match ? markdown.slice(match[0].length) : markdown;
@@ -1336,6 +1408,7 @@ export {
   serializeFarmLlmsTxt,
   createFarmLlmsMarkdownReader,
   renderFarmLlmsFullTxt,
+  createFarmLlmsFullRenderer,
   resolveFarmAgentCrawlers,
   createFarmAgentRobots,
   resolveFarmSecurityConfig,

package/dist/{chunk-AH7HOAQ3.cjs → chunk-RCYINC4A.cjs}

@@ -1,6 +1,7 @@
 "use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
 
-var _chunkBBVSJVBZcjs = require('./chunk-BBVSJVBZ.cjs');
+
+var _chunkQIH2HD6Hcjs = require('./chunk-QIH2HD6H.cjs');
 
 
 var _chunkS3JO3N4Hcjs = require('./chunk-S3JO3N4H.cjs');
@@ -115,7 +116,7 @@ async function createFarmMarkdownSourceResponse(options) {
   }
   const url = new URL(options.request.url);
   const hasMarkdownExtension = url.pathname.toLowerCase().endsWith(".md");
-  if (!hasMarkdownExtension && !_chunkBBVSJVBZcjs.requestAcceptsMarkdown.call(void 0, options.request.headers.get("accept"))) {
+  if (!hasMarkdownExtension && !_chunkQIH2HD6Hcjs.requestAcceptsMarkdown.call(void 0, options.request.headers.get("accept"))) {
     return null;
   }
   const targetPathname = normalizeFarmMarkdownRoutePath(url.pathname);
@@ -133,9 +134,13 @@ async function createFarmMarkdownSourceResponse(options) {
   if (!hasMarkdownExtension) {
     headers.set("Vary", "Accept");
   }
-  return new Response(options.request.method === "HEAD" ? null : source.source, {
+  return _chunkQIH2HD6Hcjs.createFarmMarkdownViewResponse.call(void 0, {
+    request: options.request,
+    requestUrl: url,
+    markdown: source.source,
     status: 200,
-    headers
+    headers,
+    markdownPath: headers.get("Content-Location")
   });
 }
 _chunkS3JO3N4Hcjs.__name.call(void 0, createFarmMarkdownSourceResponse, "createFarmMarkdownSourceResponse");