@farm.js/core 0.1.34 → 0.1.36

package/dist/{chunk-AGC75XAE.mjs → chunk-4WCZJQUO.mjs}

@@ -1,7 +1,7 @@
 import {
   createFarmMarkdownViewResponse,
   requestAcceptsMarkdown
-} from "./chunk-ZUSJDQ2Q.mjs";
+} from "./chunk-WWIKMBRN.mjs";
 import {
   __name
 } from "./chunk-LPEX4YW6.mjs";
@@ -112,6 +112,55 @@ function isFarmMarkdownPublicSourceResponse(response) {
   return publicSourceResponses.has(response);
 }
 __name(isFarmMarkdownPublicSourceResponse, "isFarmMarkdownPublicSourceResponse");
+var ANONYMOUS_REQUEST_HEADERS = /* @__PURE__ */ new Set([
+  "accept",
+  "accept-encoding",
+  "accept-language",
+  "cache-control",
+  "connection",
+  "dnt",
+  "forwarded",
+  "host",
+  "if-match",
+  "if-modified-since",
+  "if-none-match",
+  "if-range",
+  "if-unmodified-since",
+  "keep-alive",
+  "origin",
+  "pragma",
+  "priority",
+  "purpose",
+  "range",
+  "referer",
+  "save-data",
+  "sec-gpc",
+  "sec-purpose",
+  "te",
+  "traceparent",
+  "tracestate",
+  "upgrade-insecure-requests",
+  "user-agent",
+  "via",
+  "x-forwarded-for",
+  "x-forwarded-host",
+  "x-forwarded-port",
+  "x-forwarded-proto",
+  "x-real-ip",
+  "x-request-id"
+]);
+function isAnonymousRequestHeader(name) {
+  const lower = name.toLowerCase();
+  return ANONYMOUS_REQUEST_HEADERS.has(lower) || lower.startsWith("sec-fetch-") || lower.startsWith("sec-ch-");
+}
+__name(isAnonymousRequestHeader, "isAnonymousRequestHeader");
+function hasOnlyAnonymousHeaders(headers) {
+  for (const name of headers.keys()) {
+    if (!isAnonymousRequestHeader(name)) return false;
+  }
+  return true;
+}
+__name(hasOnlyAnonymousHeaders, "hasOnlyAnonymousHeaders");
 async function createFarmMarkdownSourceResponse(options) {
   if (options.request.method !== "GET" && options.request.method !== "HEAD") {
     return null;
@@ -129,13 +178,14 @@ async function createFarmMarkdownSourceResponse(options) {
   if (!source) {
     return null;
   }
-  const anonymous = !options.request.headers.has("cookie") && !options.request.headers.has("authorization");
+  const anonymous = hasOnlyAnonymousHeaders(options.request.headers);
   if (options.renderPage && (hasMarkdownExtension || !anonymous)) {
     const publicUrl = new URL(url);
     publicUrl.pathname = targetPathname;
-    const publicHeaders = new Headers(options.request.headers);
-    publicHeaders.delete("cookie");
-    publicHeaders.delete("authorization");
+    const publicHeaders = new Headers();
+    for (const [name, value] of options.request.headers) {
+      if (isAnonymousRequestHeader(name)) publicHeaders.append(name, value);
+    }
     publicHeaders.set("accept", "text/markdown");
     const publicResponse = await options.renderPage(
       new Request(publicUrl, {

package/dist/chunk-4WQ6PGSJ.cjs

@@ -0,0 +1,367 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
+
+
+
+var _chunkEUV4YR3Vcjs = require('./chunk-EUV4YR3V.cjs');
+
+
+var _chunkJ63ZH5OQcjs = require('./chunk-J63ZH5OQ.cjs');
+
+
+var _chunkI57LW5VZcjs = require('./chunk-I57LW5VZ.cjs');
+
+
+var _chunkS3JO3N4Hcjs = require('./chunk-S3JO3N4H.cjs');
+
+// src/llms-txt.ts
+function resolveFarmLlmsTxtConfig(input) {
+  if (!input) return { enabled: false, include: [], exclude: [], full: false };
+  if (input === true) return { enabled: true, include: [], exclude: [], full: true };
+  if (typeof input !== "object") {
+    throw new TypeError("agent.llmsTxt must be true, false, or an options object.");
+  }
+  if (input.enabled === false) return { enabled: false, include: [], exclude: [], full: false };
+  for (const key of ["title", "summary", "details"]) {
+    if (input[key] !== void 0 && typeof input[key] !== "string") {
+      throw new TypeError(`agent.llmsTxt.${key} must be a string.`);
+    }
+  }
+  if (input.full !== void 0 && typeof input.full !== "boolean") {
+    throw new TypeError("agent.llmsTxt.full must be a boolean.");
+  }
+  if (input.revalidate !== void 0 && (typeof input.revalidate !== "number" || !Number.isFinite(input.revalidate) || input.revalidate < 0)) {
+    throw new TypeError("agent.llmsTxt.revalidate must be a number of seconds.");
+  }
+  return {
+    enabled: true,
+    ...input.title ? { title: input.title } : {},
+    ...input.summary ? { summary: input.summary } : {},
+    ...input.details ? { details: input.details } : {},
+    include: resolveRoutePatterns(input.include, "include"),
+    exclude: resolveRoutePatterns(input.exclude, "exclude"),
+    full: input.full !== false,
+    ...input.revalidate !== void 0 ? { revalidate: input.revalidate } : {}
+  };
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, resolveFarmLlmsTxtConfig, "resolveFarmLlmsTxtConfig");
+function resolveRoutePatterns(value, key) {
+  if (value === void 0) return [];
+  if (!Array.isArray(value)) {
+    throw new TypeError(`agent.llmsTxt.${key} must be an array of route patterns.`);
+  }
+  return value.map((pattern, index) => {
+    if (typeof pattern !== "string" || !pattern.startsWith("/")) {
+      throw new TypeError(
+        `agent.llmsTxt.${key}[${index}] must be a route pattern starting with "/", such as "/docs/[...slug]".`
+      );
+    }
+    return pattern;
+  });
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, resolveRoutePatterns, "resolveRoutePatterns");
+function collectFarmLlmsTxtPages(sources, options) {
+  const include = _nullishCoalesce(options.include, () => ( []));
+  const exclude = _nullishCoalesce(options.exclude, () => ( []));
+  const pages = /* @__PURE__ */ new Map();
+  for (const source of sources) {
+    const path = source.pattern;
+    if (path.includes("[") || pages.has(path)) continue;
+    if (include.length && !include.some((pattern) => _chunkEUV4YR3Vcjs.matchesFarmMarkdownRoutePattern.call(void 0, pattern, path))) {
+      continue;
+    }
+    if (exclude.some((pattern) => _chunkEUV4YR3Vcjs.matchesFarmMarkdownRoutePattern.call(void 0, pattern, path))) continue;
+    if (isFarmMetadataNoIndex(source.metadata)) continue;
+    const target = _chunkEUV4YR3Vcjs.isFarmMarkdownMirrorExposed.call(void 0, options.markdown, path) ? _chunkEUV4YR3Vcjs.getFarmMarkdownMirrorPath.call(void 0, path) : path;
+    const description = readMetadataString(source.metadata, "description");
+    pages.set(path, {
+      path,
+      title: _nullishCoalesce(readMetadataTitle(source.metadata), () => ( fallbackPageTitle(path))),
+      url: `${options.origin}${_chunkI57LW5VZcjs.applyFarmBasePath.call(void 0, target, options.basePath)}`,
+      ...description ? { description } : {}
+    });
+  }
+  return [...pages.values()].sort((left, right) => left.path.localeCompare(right.path));
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, collectFarmLlmsTxtPages, "collectFarmLlmsTxtPages");
+function isFarmMetadataNoIndex(metadata) {
+  if (!metadata || typeof metadata !== "object") return false;
+  const robots = metadata.robots;
+  if (typeof robots === "string") {
+    return robots.split(",").map((directive) => directive.trim().toLowerCase()).some((directive) => directive === "noindex" || directive === "none");
+  }
+  return Boolean(
+    robots && typeof robots === "object" && robots.index === false
+  );
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, isFarmMetadataNoIndex, "isFarmMetadataNoIndex");
+function createFarmDefaultLlmsTxt(options) {
+  const { config, pages } = options;
+  const mcp = _nullishCoalesce(options.mcp, () => ( []));
+  const summary = _nullishCoalesce(config.summary, () => ( readMetadataString(options.rootMetadata, "description")));
+  return {
+    title: _nullishCoalesce(_nullishCoalesce(_nullishCoalesce(config.title, () => ( readMetadataTitle(options.rootMetadata))), () => ( _optionalChain([pages, 'access', _ => _.find, 'call', _2 => _2((page) => page.path === "/"), 'optionalAccess', _3 => _3.title]))), () => ( new URL(options.origin).host)),
+    ...summary ? { summary } : {},
+    ...config.details ? { details: config.details } : {},
+    sections: [
+      ...pages.length ? [
+        {
+          title: "Pages",
+          links: pages.map(({ title, url, description }) => ({
+            title,
+            url,
+            ...description ? { description } : {}
+          }))
+        }
+      ] : [],
+      ...mcp.length ? [{ title: "MCP", links: mcp }] : []
+    ]
+  };
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, createFarmDefaultLlmsTxt, "createFarmDefaultLlmsTxt");
+function serializeFarmLlmsTxt(value) {
+  if (typeof value === "string") {
+    if (!value.trim()) throw new TypeError("llms.ts returned an empty llms.txt");
+    return value.endsWith("\n") ? value : `${value}
+`;
+  }
+  if (!isRecord(value)) {
+    throw new TypeError("llms.ts must return llms.txt text, an llms.txt object, or a Response");
+  }
+  const title = requireText(value.title, "llms.ts must return a non-empty title");
+  const summary = optionalText(value.summary, "llms.ts summary must be a string");
+  const details = optionalText(value.details, "llms.ts details must be a string");
+  if (value.sections !== void 0 && !Array.isArray(value.sections)) {
+    throw new TypeError("llms.ts sections must be an array");
+  }
+  const blocks = [`# ${toSingleLine(title)}`];
+  if (_optionalChain([summary, 'optionalAccess', _4 => _4.trim, 'call', _5 => _5()])) blocks.push(`> ${toSingleLine(summary)}`);
+  if (_optionalChain([details, 'optionalAccess', _6 => _6.trim, 'call', _7 => _7()])) blocks.push(details.trim());
+  for (const [sectionIndex, section] of (_nullishCoalesce(value.sections, () => ( []))).entries()) {
+    if (!isRecord(section)) {
+      throw new TypeError(`llms.ts section ${sectionIndex} must be an object`);
+    }
+    const sectionTitle = requireText(
+      section.title,
+      `llms.ts section ${sectionIndex} must include a non-empty title`
+    );
+    if (!Array.isArray(section.links)) {
+      throw new TypeError(`llms.ts section "${sectionTitle}" must include a links array`);
+    }
+    const links = section.links.map((link, linkIndex) => {
+      const where = `llms.ts section "${sectionTitle}" link ${linkIndex}`;
+      if (!isRecord(link)) throw new TypeError(`${where} must be an object`);
+      const linkTitle = requireText(link.title, `${where} must include a non-empty title`);
+      const url = requireText(link.url, `${where} must include a non-empty url`);
+      const description = optionalText(link.description, `${where} description must be a string`);
+      return `- [${escapeLinkText(linkTitle)}](${escapeLinkUrl(url)})${_optionalChain([description, 'optionalAccess', _8 => _8.trim, 'call', _9 => _9()]) ? `: ${toSingleLine(description)}` : ""}`;
+    });
+    blocks.push(`## ${toSingleLine(sectionTitle)}`, links.join("\n"));
+  }
+  return `${blocks.filter(Boolean).join("\n\n")}
+`;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, serializeFarmLlmsTxt, "serializeFarmLlmsTxt");
+function createFarmLlmsMarkdownReader(fetchMarkdown) {
+  return async (url) => {
+    try {
+      const response = await fetchMarkdown(
+        new Request(url, {
+          headers: { accept: "text/markdown", [_chunkJ63ZH5OQcjs.FARM_INTERNAL_READ_HEADER]: "llms-full" }
+        })
+      );
+      const type = _nullishCoalesce(response.headers.get("content-type"), () => ( ""));
+      if (!response.ok || !type.toLowerCase().startsWith("text/markdown")) {
+        await _optionalChain([response, 'access', _10 => _10.body, 'optionalAccess', _11 => _11.cancel, 'call', _12 => _12()]);
+        return null;
+      }
+      return await response.text();
+    } catch (e) {
+      return null;
+    }
+  };
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, createFarmLlmsMarkdownReader, "createFarmLlmsMarkdownReader");
+var FARM_LLMS_FULL_PAGE_TIMEOUT_MS = 3e4;
+var LLMS_PAGE_TIMED_OUT = /* @__PURE__ */ Symbol("farm.llms.pageTimedOut");
+async function renderFarmLlmsFullTxt(value, options) {
+  const index = serializeFarmLlmsTxt(value);
+  if (typeof value === "string") return index;
+  const llms = value;
+  const links = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const section of _nullishCoalesce(llms.sections, () => ( []))) {
+    for (const link of section.links) {
+      if (seen.has(link.url)) continue;
+      seen.add(link.url);
+      links.push(link);
+    }
+  }
+  const pageTimeout = _nullishCoalesce(options.pageTimeout, () => ( FARM_LLMS_FULL_PAGE_TIMEOUT_MS));
+  const readWithinTimeout = /* @__PURE__ */ _chunkS3JO3N4Hcjs.__name.call(void 0, async (url) => {
+    let timer;
+    const timedOut = new Promise((resolve) => {
+      timer = setTimeout(resolve, pageTimeout, LLMS_PAGE_TIMED_OUT);
+    });
+    try {
+      const content = await Promise.race([options.readMarkdown(url), timedOut]);
+      if (content !== LLMS_PAGE_TIMED_OUT) return content;
+      _optionalChain([options, 'access', _13 => _13.onIncomplete, 'optionalCall', _14 => _14()]);
+      return null;
+    } finally {
+      clearTimeout(timer);
+    }
+  }, "readWithinTimeout");
+  const contents = await mapWithConcurrency(
+    links,
+    _nullishCoalesce(options.concurrency, () => ( 4)),
+    (link) => shouldInlineLlmsLink(link.url, options.origin) ? readWithinTimeout(link.url) : null
+  );
+  const blocks = [`# ${toSingleLine(llms.title)}`];
+  if (_optionalChain([llms, 'access', _15 => _15.summary, 'optionalAccess', _16 => _16.trim, 'call', _17 => _17()])) blocks.push(`> ${toSingleLine(llms.summary)}`);
+  if (_optionalChain([llms, 'access', _18 => _18.details, 'optionalAccess', _19 => _19.trim, 'call', _20 => _20()])) blocks.push(llms.details.trim());
+  const pages = links.map((link, index2) => {
+    const parts = [`## ${toSingleLine(link.title)}`, `URL: ${escapeLinkUrl(link.url)}`];
+    if (_optionalChain([link, 'access', _21 => _21.description, 'optionalAccess', _22 => _22.trim, 'call', _23 => _23()])) parts.push(toSingleLine(link.description));
+    const content = stripFrontmatter(_nullishCoalesce(contents[index2], () => ( ""))).trim();
+    if (content) parts.push(content);
+    return parts.join("\n\n");
+  });
+  if (pages.length) blocks.push(pages.join("\n\n---\n\n"));
+  return `${blocks.join("\n\n")}
+`;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, renderFarmLlmsFullTxt, "renderFarmLlmsFullTxt");
+function createFarmLlmsFullRenderer(options = {}) {
+  const now = _nullishCoalesce(options.now, () => ( Date.now));
+  const ttl = (_nullishCoalesce(options.revalidate, () => ( 0))) * 1e3;
+  const maxEntries = _nullishCoalesce(options.maxEntries, () => ( 16));
+  const entries = /* @__PURE__ */ new Map();
+  let queue = Promise.resolve();
+  const settle = /* @__PURE__ */ _chunkS3JO3N4Hcjs.__name.call(void 0, (body, complete, onSettled) => onSettled ? body.then((value) => (onSettled(complete()), value)) : body, "settle");
+  return (origin, render, onSettled) => {
+    const entry = entries.get(origin);
+    if (entry && entry.expires > now()) return settle(entry.body, entry.complete, onSettled);
+    entries.delete(origin);
+    let complete = true;
+    const body = queue.then(
+      () => render({
+        incomplete: /* @__PURE__ */ _chunkS3JO3N4Hcjs.__name.call(void 0, () => {
+          complete = false;
+        }, "incomplete")
+      })
+    );
+    queue = body.catch(() => {
+    });
+    const pending = { body, expires: Number.POSITIVE_INFINITY, complete: /* @__PURE__ */ _chunkS3JO3N4Hcjs.__name.call(void 0, () => complete, "complete") };
+    entries.set(origin, pending);
+    if (entries.size > maxEntries) entries.delete(entries.keys().next().value);
+    body.then(
+      () => {
+        if (entries.get(origin) !== pending) return;
+        if (ttl > 0 && complete) pending.expires = now() + ttl;
+        else entries.delete(origin);
+      },
+      () => {
+        if (entries.get(origin) === pending) entries.delete(origin);
+      }
+    );
+    return settle(body, pending.complete, onSettled);
+  };
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, createFarmLlmsFullRenderer, "createFarmLlmsFullRenderer");
+function stripFrontmatter(markdown) {
+  const match = /^\uFEFF?---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/.exec(markdown);
+  return match ? markdown.slice(match[0].length) : markdown;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, stripFrontmatter, "stripFrontmatter");
+function shouldInlineLlmsLink(url, origin) {
+  let target;
+  try {
+    target = new URL(url, origin);
+  } catch (e2) {
+    return false;
+  }
+  return target.origin === origin && !/\/llms(-full)?\.txt$/.test(target.pathname);
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, shouldInlineLlmsLink, "shouldInlineLlmsLink");
+async function mapWithConcurrency(items, limit, run) {
+  const results = [];
+  let next = 0;
+  const worker = /* @__PURE__ */ _chunkS3JO3N4Hcjs.__name.call(void 0, async () => {
+    while (next < items.length) {
+      const index = next++;
+      results[index] = await run(items[index]);
+    }
+  }, "worker");
+  await Promise.all(Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, worker));
+  return results;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, mapWithConcurrency, "mapWithConcurrency");
+function readMetadataTitle(metadata) {
+  if (!isRecord(metadata)) return void 0;
+  const { title } = metadata;
+  if (typeof title === "string") return title.trim() || void 0;
+  if (isRecord(title)) {
+    for (const key of ["absolute", "default"]) {
+      const value = title[key];
+      if (typeof value === "string" && value.trim()) return value.trim();
+    }
+  }
+  return void 0;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, readMetadataTitle, "readMetadataTitle");
+function readMetadataString(metadata, key) {
+  if (!isRecord(metadata)) return void 0;
+  const value = metadata[key];
+  return typeof value === "string" && value.trim() ? value.trim() : void 0;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, readMetadataString, "readMetadataString");
+function fallbackPageTitle(path) {
+  if (path === "/") return "Home";
+  const segments = path.split("/").filter(Boolean);
+  const segment = _nullishCoalesce(segments[segments.length - 1], () => ( path));
+  const words = segment.replace(/[-_]+/g, " ").trim();
+  return words ? words[0].toUpperCase() + words.slice(1) : path;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, fallbackPageTitle, "fallbackPageTitle");
+function toSingleLine(value) {
+  return value.replace(/\s+/g, " ").trim();
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, toSingleLine, "toSingleLine");
+function escapeLinkText(value) {
+  return toSingleLine(value).replace(/[\\[\]]/g, "\\$&");
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, escapeLinkText, "escapeLinkText");
+function escapeLinkUrl(value) {
+  return value.trim().replace(
+    /[\s()<>]/g,
+    (character) => character === "(" ? "%28" : character === ")" ? "%29" : encodeURIComponent(character)
+  );
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, escapeLinkUrl, "escapeLinkUrl");
+function requireText(value, message) {
+  if (typeof value !== "string" || !value.trim()) throw new TypeError(message);
+  return value;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, requireText, "requireText");
+function optionalText(value, message) {
+  if (value === void 0) return void 0;
+  if (typeof value !== "string") throw new TypeError(message);
+  return value;
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, optionalText, "optionalText");
+function isRecord(value) {
+  return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+_chunkS3JO3N4Hcjs.__name.call(void 0, isRecord, "isRecord");
+
+
+
+
+
+
+
+
+
+
+exports.resolveFarmLlmsTxtConfig = resolveFarmLlmsTxtConfig; exports.collectFarmLlmsTxtPages = collectFarmLlmsTxtPages; exports.isFarmMetadataNoIndex = isFarmMetadataNoIndex; exports.createFarmDefaultLlmsTxt = createFarmDefaultLlmsTxt; exports.serializeFarmLlmsTxt = serializeFarmLlmsTxt; exports.createFarmLlmsMarkdownReader = createFarmLlmsMarkdownReader; exports.renderFarmLlmsFullTxt = renderFarmLlmsFullTxt; exports.createFarmLlmsFullRenderer = createFarmLlmsFullRenderer;

package/dist/{chunk-QCFFHV5H.mjs → chunk-4WVJLURN.mjs}

@@ -6,7 +6,7 @@ import {
 } from "./chunk-FP2INE7H.mjs";
 import {
   parseBodySizeLimit
-} from "./chunk-XF7Q24PF.mjs";
+} from "./chunk-ZMMBIVQM.mjs";
 import {
   subscribeFarmCacheInvalidation,
   subscribeFarmCacheTask