@farming-labs/docs 0.1.83 → 0.1.84

package/dist/{agent-D97eZuWH.mjs → agent-BqsweF-z.mjs}

@@ -1,6 +1,6 @@
 import "./reading-time-pKUeloSI.mjs";
 import { _ as parseGeneratedAgentDocument, h as hashGeneratedAgentContent, m as GENERATED_AGENT_PROVENANCE_VERSION, v as serializeGeneratedAgentDocument } from "./search-B5ze-heM.mjs";
-import { D as findDocsMarkdownPage, I as renderDocsMarkdownDocument } from "./robots-DCR-ZFLO.mjs";
+import { D as findDocsMarkdownPage, I as renderDocsMarkdownDocument } from "./robots-WLElbTr1.mjs";
 import "./index.mjs";
 import "./sitemap-server-C8Ppk29g.mjs";
 import { createFilesystemDocsMcpSource } from "./mcp.mjs";

package/dist/cli/index.mjs

@@ -86,7 +86,7 @@ async function main() {
 		const { runMcp } = await import("../mcp-OVgCyHrR.mjs");
 		await runMcp(mcpOptions);
 	} else if (parsedCommand.command === "agent" && subcommand === "compact") {
-		const { compactAgentDocs, parseAgentCompactArgs, printAgentCompactHelp } = await import("../agent-D97eZuWH.mjs");
+		const { compactAgentDocs, parseAgentCompactArgs, printAgentCompactHelp } = await import("../agent-BqsweF-z.mjs");
 		const agentCompactOptions = parseAgentCompactArgs(args.slice(2));
 		if (agentCompactOptions.help) {
 			printAgentCompactHelp();
@@ -96,11 +96,11 @@ async function main() {
 	} else if (parsedCommand.command === "agent") {
 		console.error(pc.red(`Unknown agent subcommand: ${subcommand ?? "(missing)"}`));
 		console.error();
-		const { printAgentCompactHelp } = await import("../agent-D97eZuWH.mjs");
+		const { printAgentCompactHelp } = await import("../agent-BqsweF-z.mjs");
 		printAgentCompactHelp();
 		process.exit(1);
 	} else if (parsedCommand.command === "doctor") {
-		const { parseDoctorArgs, printDoctorHelp, runDoctor } = await import("../doctor-DHUZK-iX.mjs");
+		const { parseDoctorArgs, printDoctorHelp, runDoctor } = await import("../doctor-DG6MQTbq.mjs");
 		const doctorOptions = parseDoctorArgs(args.slice(1));
 		if (doctorOptions.help) {
 			printDoctorHelp();
@@ -130,7 +130,7 @@ async function main() {
 		printSitemapGenerateHelp();
 		process.exit(1);
 	} else if (parsedCommand.command === "robots" && subcommand === "generate") {
-		const { generateRobots, parseRobotsGenerateArgs, printRobotsGenerateHelp } = await import("../robots-Div3kkxI.mjs");
+		const { generateRobots, parseRobotsGenerateArgs, printRobotsGenerateHelp } = await import("../robots-8FR5Gbm_.mjs");
 		const robotsOptions = parseRobotsGenerateArgs(args.slice(2));
 		if (robotsOptions.help) {
 			printRobotsGenerateHelp();
@@ -140,7 +140,7 @@ async function main() {
 	} else if (parsedCommand.command === "robots") {
 		console.error(pc.red(`Unknown robots subcommand: ${subcommand ?? "(missing)"}`));
 		console.error();
-		const { printRobotsGenerateHelp } = await import("../robots-Div3kkxI.mjs");
+		const { printRobotsGenerateHelp } = await import("../robots-8FR5Gbm_.mjs");
 		printRobotsGenerateHelp();
 		process.exit(1);
 	} else if (parsedCommand.command === "upgrade") {

package/dist/{doctor-DHUZK-iX.mjs → doctor-DG6MQTbq.mjs}

@@ -1,12 +1,12 @@
 import "./reading-time-pKUeloSI.mjs";
 import "./search-B5ze-heM.mjs";
 import { i as DEFAULT_SITEMAP_XML_ROUTE, n as DEFAULT_SITEMAP_MD_ROUTE, r as DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE, u as resolveDocsSitemapConfig } from "./sitemap-Ccfh6GXO.mjs";
-import { C as DEFAULT_SKILL_MD_ROUTE, S as DEFAULT_MCP_WELL_KNOWN_ROUTE, a as analyzeDocsRobotsTxt, b as DEFAULT_MCP_PUBLIC_ROUTE, d as DEFAULT_AGENT_FEEDBACK_ROUTE, g as DEFAULT_LLMS_FULL_TXT_ROUTE, l as resolveDocsRobotsConfig, m as DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, n as DEFAULT_ROBOTS_TXT_ROUTE, p as DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, v as DEFAULT_LLMS_TXT_ROUTE, w as DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE } from "./robots-DCR-ZFLO.mjs";
+import { C as DEFAULT_SKILL_MD_ROUTE, S as DEFAULT_MCP_WELL_KNOWN_ROUTE, a as analyzeDocsRobotsTxt, b as DEFAULT_MCP_PUBLIC_ROUTE, d as DEFAULT_AGENT_FEEDBACK_ROUTE, g as DEFAULT_LLMS_FULL_TXT_ROUTE, l as resolveDocsRobotsConfig, m as DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, n as DEFAULT_ROBOTS_TXT_ROUTE, p as DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, v as DEFAULT_LLMS_TXT_ROUTE, w as DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE } from "./robots-WLElbTr1.mjs";
 import "./sitemap-server-C8Ppk29g.mjs";
 import { createFilesystemDocsMcpSource, resolveDocsMcpConfig } from "./mcp.mjs";
 import "./server.mjs";
 import { a as loadProjectEnv, c as readNavTitle, d as readTopLevelStringProperty, f as resolveDocsConfigPath, i as loadDocsConfigModule, o as readBooleanProperty, p as resolveDocsContentDir, r as extractTopLevelConfigObject, t as extractNestedObjectLiteral } from "./config-BR6CcCfr.mjs";
-import { inspectAgentCompactionState, scanDocsPageTargets } from "./agent-D97eZuWH.mjs";
+import { inspectAgentCompactionState, scanDocsPageTargets } from "./agent-BqsweF-z.mjs";
 import { t as detectFramework } from "./utils-AmYxHDoz.mjs";
 import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs";
 import path from "node:path";
@@ -739,8 +739,7 @@ async function probeTextRoute(baseUrl, route) {
 		};
 	}
 }
-async function probeRobotsRoute(baseUrl) {
-	const route = DEFAULT_ROBOTS_TXT_ROUTE;
+async function probeRobotsRoute(baseUrl, route = DEFAULT_ROBOTS_TXT_ROUTE) {
 	const url = joinDoctorUrl(baseUrl, route);
 	try {
 		const response = await fetchWithTimeout(url, { headers: { Accept: "text/plain, */*" } });
@@ -915,6 +914,13 @@ function hostedSitemapRoutes(discoveryBody) {
 		routes: Array.from(new Set(routes))
 	};
 }
+function hostedRobotsRoute(discoveryBody) {
+	const robots = asRecord(asRecord(discoveryBody)?.robots);
+	return {
+		enabled: robots?.enabled === false ? false : true,
+		route: readDiscoveryRoute(robots?.route) ?? DEFAULT_ROBOTS_TXT_ROUTE
+	};
+}
 async function buildHostedAgentChecks(url, pages) {
 	let baseUrl;
 	try {
@@ -935,11 +941,14 @@ async function buildHostedAgentChecks(url, pages) {
 		checks.push(makeCheck("hosted-sitemap", "Hosted sitemap", sitemapPassed === sitemap.length ? "pass" : sitemapPassed > 0 ? "warn" : "fail", sitemapPassed === sitemap.length ? 5 : sitemapPassed > 0 ? 3 : 0, 5, sitemap.map((result) => result.detail).join(" "), sitemapPassed === sitemap.length ? void 0 : `Verify deployed sitemap routes return non-empty text: ${sitemapRoutes.routes.join(", ")}.`));
 	} else if (sitemapRoutes.enabled) checks.push(makeCheck("hosted-sitemap", "Hosted sitemap", "warn", 0, 5, "The hosted discovery spec reports sitemap support but did not expose sitemap routes.", "Check sitemap.xml and sitemap.markdown config so at least one sitemap route is enabled."));
 	else checks.push(makeCheck("hosted-sitemap", "Hosted sitemap", "warn", 0, 5, "The hosted discovery spec reports sitemap routes as disabled.", "Enable sitemap in docs.config when agents and crawlers should discover canonical URLs and freshness metadata."));
-	const robots = await probeRobotsRoute(baseUrl);
-	const robotsAnalysis = robots.body ? analyzeDocsRobotsTxt(robots.body) : void 0;
-	const robotsBlocked = robotsAnalysis?.blocksAgentRoutes || robotsAnalysis?.blocksAiAgents;
-	const robotsComplete = robotsAnalysis?.hasAgentRoutes && robotsAnalysis?.hasAiPolicy;
-	checks.push(makeCheck("hosted-robots", "Hosted robots.txt", robots.ok && !robotsBlocked && robotsComplete ? "pass" : robots.ok && !robotsBlocked ? "warn" : "fail", robots.ok && !robotsBlocked && robotsComplete ? 5 : robots.ok && !robotsBlocked ? 3 : 0, 5, robots.ok ? robotsBlocked ? `${DEFAULT_ROBOTS_TXT_ROUTE} is reachable but blocks ${robotsAnalysis?.blocksAiAgents ? "common AI crawlers" : "agent-readable docs routes"}.` : robotsComplete ? `${robots.detail} It advertises agent-readable routes and common AI crawler policy.` : `${robots.detail} It is missing ${robotsAnalysis?.missingRoutes.length ? `agent routes (${robotsAnalysis.missingRoutes.join(", ")})` : "common AI crawler policy"}.` : robots.detail, robots.ok && !robotsBlocked && robotsComplete ? void 0 : "Publish an agent-friendly robots.txt with `docs robots generate`, or append the generated block to the existing file."));
+	const robotsRoute = hostedRobotsRoute(discovery.body);
+	if (robotsRoute.enabled) {
+		const robots = await probeRobotsRoute(baseUrl, robotsRoute.route);
+		const robotsAnalysis = robots.body ? analyzeDocsRobotsTxt(robots.body) : void 0;
+		const robotsBlocked = robotsAnalysis?.blocksAgentRoutes || robotsAnalysis?.blocksAiAgents;
+		const robotsComplete = robotsAnalysis?.hasAgentRoutes && robotsAnalysis?.hasAiPolicy;
+		checks.push(makeCheck("hosted-robots", "Hosted robots.txt", robots.ok && !robotsBlocked && robotsComplete ? "pass" : robots.ok && !robotsBlocked ? "warn" : "fail", robots.ok && !robotsBlocked && robotsComplete ? 5 : robots.ok && !robotsBlocked ? 3 : 0, 5, robots.ok ? robotsBlocked ? `${robotsRoute.route} is reachable but blocks ${robotsAnalysis?.blocksAiAgents ? "common AI crawlers" : "agent-readable docs routes"}.` : robotsComplete ? `${robots.detail} It advertises agent-readable routes and common AI crawler policy.` : `${robots.detail} It is missing ${robotsAnalysis?.missingRoutes.length ? `agent routes (${robotsAnalysis.missingRoutes.join(", ")})` : "common AI crawler policy"}.` : robots.detail, robots.ok && !robotsBlocked && robotsComplete ? void 0 : "Publish an agent-friendly robots.txt with `docs robots generate`, or append the generated block to the existing file."));
+	} else checks.push(makeCheck("hosted-robots", "Hosted robots.txt", "warn", 0, 5, "The hosted discovery spec reports robots.txt as disabled.", "Enable robots and publish an agent-friendly robots.txt with `docs robots generate`."));
 	const skill = await Promise.all([probeTextRoute(baseUrl, DEFAULT_SKILL_MD_ROUTE), probeTextRoute(baseUrl, DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE)]);
 	const skillPassed = skill.filter((result) => result.ok).length;
 	checks.push(makeCheck("hosted-skill", "Hosted skill.md", skillPassed === skill.length ? "pass" : skillPassed > 0 ? "warn" : "fail", skillPassed === skill.length ? 5 : skillPassed > 0 ? 3 : 0, 5, skill.map((result) => result.detail).join(" "), skillPassed === skill.length ? void 0 : "Verify deployed /skill.md and /.well-known/skill.md routes return non-empty markdown."));

package/dist/index.d.mts

@@ -204,6 +204,7 @@ interface DocsAgentDiscoverySpecOptions {
   feedback?: DocsAgentFeedbackDiscoveryConfig;
   llms?: DocsLlmsDiscoveryConfig;
   sitemap?: boolean | DocsSitemapConfig;
+  robots?: boolean | DocsRobotsConfig;
   markdown?: {
     acceptHeader?: boolean;
     signatureAgentHeader?: boolean;
@@ -217,6 +218,7 @@ interface DocsSkillDocumentOptions {
   feedback?: DocsAgentFeedbackDiscoveryConfig;
   llms?: DocsLlmsDiscoveryConfig;
   sitemap?: boolean | DocsSitemapConfig;
+  robots?: boolean | DocsRobotsConfig;
   markdown?: {
     acceptHeader?: boolean;
     signatureAgentHeader?: boolean;
@@ -274,6 +276,7 @@ declare function renderDocsSkillDocument({
   feedback,
   llms,
   sitemap,
+  robots,
   markdown
 }: DocsSkillDocumentOptions): string;
 declare function resolveDocsAgentMdxContent(content: string, audience: "human" | "agent"): string;
@@ -286,6 +289,7 @@ declare function buildDocsAgentDiscoverySpec({
   feedback,
   llms,
   sitemap,
+  robots,
   markdown
 }: DocsAgentDiscoverySpecOptions): {
   version: string;
@@ -313,6 +317,7 @@ declare function buildDocsAgentDiscoverySpec({
     mcp: boolean;
     search: boolean;
     sitemap: boolean;
+    robots: boolean;
     agentFeedback: boolean;
     locales: boolean;
   };
@@ -362,6 +367,11 @@ declare function buildDocsAgentDiscoverySpec({
       defaultWellKnownRoute: string;
     };
   };
+  robots: {
+    enabled: boolean;
+    route: string;
+    defaultRoute: string;
+  };
   search: {
     enabled: boolean;
     endpoint: string;

package/dist/index.mjs

@@ -2,6 +2,6 @@ import { _ as defineDocs, a as resolveReadingTimeOptions, c as resolveOGImage, d
 import { A as resolveDocsAnalyticsConfig, C as resolveSidebarFolderIndexBehaviorForPath, D as emitDocsAgentTraceEvent, E as createDocsAgentTraceId, M as createDocsCloudAnalytics, O as emitDocsAnalyticsEvent, S as resolveSidebarFolderIndexBehavior, T as createDocsAgentTraceContext, _ as parseGeneratedAgentDocument, a as createMcpSearchAdapter, b as applySidebarFolderIndexBehavior, c as formatDocsAskAIPackageHints, d as resolveAskAISearchRequestConfig, f as resolveSearchRequestConfig, g as normalizeGeneratedAgentContent, h as hashGeneratedAgentContent, i as createCustomSearchAdapter, j as resolveDocsObservabilityConfig, k as emitDocsObservabilityEvent, l as inferDocsAskAIPackageHints, m as GENERATED_AGENT_PROVENANCE_VERSION, n as buildDocsSearchDocuments, o as createSimpleSearchAdapter, p as GENERATED_AGENT_PROVENANCE_MARKER, r as createAlgoliaSearchAdapter, s as createTypesenseSearchAdapter, t as buildDocsAskAIContext, u as performDocsSearch, v as serializeGeneratedAgentDocument, w as DOCS_AGENT_TRACE_EVENT_TYPES, x as resolvePageSidebarFolderIndexBehavior, y as stripGeneratedAgentProvenance } from "./search-B5ze-heM.mjs";
 import { n as renderDocsRelatedMarkdownLines, t as normalizeDocsRelated } from "./related-BNj_NdHq.mjs";
 import { a as buildDocsSitemapManifest, c as renderDocsSitemapMarkdown, d as resolveDocsSitemapRequest, f as toDocsSitemapMarkdownUrl, i as DEFAULT_SITEMAP_XML_ROUTE, l as renderDocsSitemapXml, n as DEFAULT_SITEMAP_MD_ROUTE, o as createDocsSitemapResponse, r as DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE, s as readDocsSitemapManifestFromContentMap, t as DEFAULT_SITEMAP_MANIFEST_PATH, u as resolveDocsSitemapConfig } from "./sitemap-Ccfh6GXO.mjs";
-import { A as isDocsAgentDiscoveryRequest, B as resolveDocsLlmsTxtFormat, C as DEFAULT_SKILL_MD_ROUTE, D as findDocsMarkdownPage, E as buildDocsAgentDiscoverySpec, F as normalizeDocsUrlPath, H as resolveDocsSkillFormat, I as renderDocsMarkdownDocument, L as renderDocsMarkdownNotFound, M as isDocsPublicGetRequest, N as isDocsSkillRequest, O as getDocsMarkdownVaryHeader, P as normalizeDocsPathSegment, R as renderDocsSkillDocument, S as DEFAULT_MCP_WELL_KNOWN_ROUTE, T as DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER, U as toDocsMarkdownUrl, V as resolveDocsMarkdownRequest, _ as DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE, a as analyzeDocsRobotsTxt, b as DEFAULT_MCP_PUBLIC_ROUTE, c as renderDocsRobotsTxt, d as DEFAULT_AGENT_FEEDBACK_ROUTE, f as DEFAULT_AGENT_SPEC_ROUTE, g as DEFAULT_LLMS_FULL_TXT_ROUTE, h as DEFAULT_DOCS_API_ROUTE, i as DOCS_ROBOTS_GENERATED_BLOCK_START, j as isDocsMcpRequest, k as hasDocsMarkdownSignatureAgent, l as resolveDocsRobotsConfig, m as DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, n as DEFAULT_ROBOTS_TXT_ROUTE, o as getDocsRobotsAllowRoutes, p as DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, r as DOCS_ROBOTS_GENERATED_BLOCK_END, s as renderDocsRobotsGeneratedBlock, t as DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS, u as upsertDocsRobotsGeneratedBlock, v as DEFAULT_LLMS_TXT_ROUTE, w as DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE, x as DEFAULT_MCP_ROUTE, y as DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE, z as resolveDocsAgentMdxContent } from "./robots-DCR-ZFLO.mjs";
+import { A as isDocsAgentDiscoveryRequest, B as resolveDocsLlmsTxtFormat, C as DEFAULT_SKILL_MD_ROUTE, D as findDocsMarkdownPage, E as buildDocsAgentDiscoverySpec, F as normalizeDocsUrlPath, H as resolveDocsSkillFormat, I as renderDocsMarkdownDocument, L as renderDocsMarkdownNotFound, M as isDocsPublicGetRequest, N as isDocsSkillRequest, O as getDocsMarkdownVaryHeader, P as normalizeDocsPathSegment, R as renderDocsSkillDocument, S as DEFAULT_MCP_WELL_KNOWN_ROUTE, T as DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER, U as toDocsMarkdownUrl, V as resolveDocsMarkdownRequest, _ as DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE, a as analyzeDocsRobotsTxt, b as DEFAULT_MCP_PUBLIC_ROUTE, c as renderDocsRobotsTxt, d as DEFAULT_AGENT_FEEDBACK_ROUTE, f as DEFAULT_AGENT_SPEC_ROUTE, g as DEFAULT_LLMS_FULL_TXT_ROUTE, h as DEFAULT_DOCS_API_ROUTE, i as DOCS_ROBOTS_GENERATED_BLOCK_START, j as isDocsMcpRequest, k as hasDocsMarkdownSignatureAgent, l as resolveDocsRobotsConfig, m as DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, n as DEFAULT_ROBOTS_TXT_ROUTE, o as getDocsRobotsAllowRoutes, p as DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, r as DOCS_ROBOTS_GENERATED_BLOCK_END, s as renderDocsRobotsGeneratedBlock, t as DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS, u as upsertDocsRobotsGeneratedBlock, v as DEFAULT_LLMS_TXT_ROUTE, w as DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE, x as DEFAULT_MCP_ROUTE, y as DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE, z as resolveDocsAgentMdxContent } from "./robots-WLElbTr1.mjs";
 
 export { DEFAULT_AGENT_FEEDBACK_ROUTE, DEFAULT_AGENT_SPEC_ROUTE, DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS, DEFAULT_DOCS_API_ROUTE, DEFAULT_LLMS_FULL_TXT_ROUTE, DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE, DEFAULT_LLMS_TXT_ROUTE, DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE, DEFAULT_MCP_PUBLIC_ROUTE, DEFAULT_MCP_ROUTE, DEFAULT_MCP_WELL_KNOWN_ROUTE, DEFAULT_ROBOTS_TXT_ROUTE, DEFAULT_SITEMAP_MANIFEST_PATH, DEFAULT_SITEMAP_MD_ROUTE, DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE, DEFAULT_SITEMAP_XML_ROUTE, DEFAULT_SKILL_MD_ROUTE, DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE, DOCS_AGENT_TRACE_EVENT_TYPES, DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER, DOCS_ROBOTS_GENERATED_BLOCK_END, DOCS_ROBOTS_GENERATED_BLOCK_START, GENERATED_AGENT_PROVENANCE_MARKER, GENERATED_AGENT_PROVENANCE_VERSION, analyzeDocsRobotsTxt, applySidebarFolderIndexBehavior, buildDocsAgentDiscoverySpec, buildDocsAskAIContext, buildDocsSearchDocuments, buildDocsSitemapManifest, buildPageOpenGraph, buildPageTwitter, createAlgoliaSearchAdapter, createCustomSearchAdapter, createDocsAgentTraceContext, createDocsAgentTraceId, createDocsCloudAnalytics, createDocsSitemapResponse, createMcpSearchAdapter, createSimpleSearchAdapter, createTheme, createTypesenseSearchAdapter, deepMerge, defineDocs, emitDocsAgentTraceEvent, emitDocsAnalyticsEvent, emitDocsObservabilityEvent, estimateReadingTimeMinutes, extendTheme, findDocsMarkdownPage, formatDocsAskAIPackageHints, getDocsMarkdownVaryHeader, getDocsRobotsAllowRoutes, hasDocsMarkdownSignatureAgent, hashGeneratedAgentContent, inferDocsAskAIPackageHints, isDocsAgentDiscoveryRequest, isDocsMcpRequest, isDocsPublicGetRequest, isDocsSkillRequest, normalizeDocsPathSegment, normalizeDocsRelated, normalizeDocsUrlPath, normalizeGeneratedAgentContent, parseGeneratedAgentDocument, performDocsSearch, readDocsSitemapManifestFromContentMap, renderDocsMarkdownDocument, renderDocsMarkdownNotFound, renderDocsRelatedMarkdownLines, renderDocsRobotsGeneratedBlock, renderDocsRobotsTxt, renderDocsSitemapMarkdown, renderDocsSitemapXml, renderDocsSkillDocument, resolveAskAISearchRequestConfig, resolveChangelogConfig, resolveDocsAgentMdxContent, resolveDocsAnalyticsConfig, resolveDocsI18n, resolveDocsLlmsTxtFormat, resolveDocsLocale, resolveDocsMarkdownRequest, resolveDocsObservabilityConfig, resolveDocsPath, resolveDocsRobotsConfig, resolveDocsSitemapConfig, resolveDocsSitemapRequest, resolveDocsSkillFormat, resolveOGImage, resolvePageReadingTime, resolvePageSidebarFolderIndexBehavior, resolveReadingTimeFromContent, resolveReadingTimeFromSource, resolveReadingTimeOptions, resolveSearchRequestConfig, resolveSidebarFolderIndexBehavior, resolveSidebarFolderIndexBehaviorForPath, resolveTitle, serializeGeneratedAgentDocument, stripGeneratedAgentProvenance, toDocsMarkdownUrl, toDocsSitemapMarkdownUrl, upsertDocsRobotsGeneratedBlock };

package/dist/{robots-Div3kkxI.mjs → robots-8FR5Gbm_.mjs}

@@ -1,4 +1,4 @@
-import { i as DOCS_ROBOTS_GENERATED_BLOCK_START, l as resolveDocsRobotsConfig, r as DOCS_ROBOTS_GENERATED_BLOCK_END, s as renderDocsRobotsGeneratedBlock, u as upsertDocsRobotsGeneratedBlock } from "./robots-DCR-ZFLO.mjs";
+import { i as DOCS_ROBOTS_GENERATED_BLOCK_START, l as resolveDocsRobotsConfig, r as DOCS_ROBOTS_GENERATED_BLOCK_END, s as renderDocsRobotsGeneratedBlock, u as upsertDocsRobotsGeneratedBlock } from "./robots-WLElbTr1.mjs";
 import { d as readTopLevelStringProperty, f as resolveDocsConfigPath, i as loadDocsConfigModule, o as readBooleanProperty, t as extractNestedObjectLiteral, u as readStringProperty } from "./config-BR6CcCfr.mjs";
 import { t as detectFramework } from "./utils-AmYxHDoz.mjs";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";

package/dist/{robots-DCR-ZFLO.mjs → robots-WLElbTr1.mjs}

@@ -15,6 +15,7 @@ const DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE = "/.well-known/llms.txt";
 const DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE = "/.well-known/llms-full.txt";
 const DEFAULT_SKILL_MD_ROUTE = "/skill.md";
 const DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE = "/.well-known/skill.md";
+const DEFAULT_AGENT_DISCOVERY_ROBOTS_TXT_ROUTE = "/robots.txt";
 const DEFAULT_AGENT_FEEDBACK_ROUTE = "/api/docs/agent/feedback";
 const DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER = "Signature-Agent";
 function normalizeDocsPathSegment(value) {
@@ -115,7 +116,7 @@ function renderDocsMarkdownNotFound({ entry = "docs", requestedPath, sitemap })
 		}
 		if (sitemapConfig.xml.enabled) lines.push(`- XML sitemap: \`${sitemapConfig.xml.route}\``);
 	} else lines.push(`- Sitemap discovery, if enabled: \`${DEFAULT_SITEMAP_MD_ROUTE}\`, \`${DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE}\`, or \`${DEFAULT_SITEMAP_XML_ROUTE}\``);
-	lines.push("", "The agent discovery spec is the safest first step because it lists the active markdown, sitemap, search, MCP, and feedback routes for this deployment.");
+	lines.push("", "The agent discovery spec is the safest first step because it lists the active markdown, sitemap, robots, search, MCP, and feedback routes for this deployment.");
 	return lines.join("\n");
 }
 function findDocsMarkdownPage(entry, pages, requestedPath) {
@@ -134,7 +135,7 @@ function renderDocsMarkdownDocument(page) {
 	lines.push("", page.agentFallbackRawContent ?? page.rawContent ?? page.content);
 	return lines.join("\n");
 }
-function renderDocsSkillDocument({ origin, entry = "docs", search, mcp, feedback, llms, sitemap, markdown }) {
+function renderDocsSkillDocument({ origin, entry = "docs", search, mcp, feedback, llms, sitemap, robots, markdown }) {
 	const normalizedEntry = normalizeDocsPathSegment(entry) || "docs";
 	const siteTitle = compactSkillText(llms?.siteTitle ?? "Documentation");
 	const siteDescription = llms?.siteDescription ? compactSkillText(llms.siteDescription) : void 0;
@@ -142,9 +143,10 @@ function renderDocsSkillDocument({ origin, entry = "docs", search, mcp, feedback
 	const searchEnabled = isSearchEnabled(search);
 	const feedbackEnabled = feedback?.enabled ?? false;
 	const sitemapConfig = resolveDocsSitemapConfig(sitemap);
+	const robotsEnabled = isRobotsDiscoveryEnabled(robots);
 	const feedbackRoute = feedback?.route ?? DEFAULT_AGENT_FEEDBACK_ROUTE;
 	const feedbackSchemaRoute = feedback?.schemaRoute ?? `${feedbackRoute}/schema`;
-	const description = truncateSkillDescription(`Use ${siteTitle} through markdown routes, llms.txt, agent discovery, search, and MCP when available.`);
+	const description = truncateSkillDescription(`Use ${siteTitle} through markdown routes, llms.txt, robots.txt, agent discovery, search, and MCP when available.`);
 	const markdownAcceptHeader = markdown?.acceptHeader === false ? null : "text/markdown";
 	const markdownSignatureAgentHeader = markdown?.signatureAgentHeader === false ? null : DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER;
 	const lines = [
@@ -167,9 +169,11 @@ function renderDocsSkillDocument({ origin, entry = "docs", search, mcp, feedback
 		if (sitemapConfig.xml.enabled) lines.push(`- Use ${sitemapConfig.xml.route} to check canonical page freshness.`);
 		if (sitemapConfig.markdown.enabled) lines.push(`- Use ${sitemapConfig.markdown.route} for a semantic docs map.`);
 	}
+	if (robotsEnabled) lines.push(`- Check ${DEFAULT_AGENT_DISCOVERY_ROBOTS_TXT_ROUTE} for crawler and AI-agent access policy.`);
 	if (mcp.enabled) lines.push(`- Use ${DEFAULT_MCP_WELL_KNOWN_ROUTE} or ${DEFAULT_MCP_PUBLIC_ROUTE} for MCP tools when your environment supports MCP.`);
 	if (feedbackEnabled) lines.push(`- Read ${feedbackSchemaRoute} before posting agent feedback to ${feedbackRoute}.`);
 	lines.push("", "## Routes", `- Skill document: ${DEFAULT_SKILL_MD_ROUTE}`, `- Skill well-known alias: ${DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE}`, `- Skill API format: ${DEFAULT_DOCS_API_ROUTE}?format=skill`, `- Agent discovery: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE}`, `- Agent discovery fallback: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE}`, `- Markdown root: /${normalizedEntry}.md`, `- Markdown pages: /${normalizedEntry}/{slug}.md`);
+	if (robotsEnabled) lines.push(`- Robots policy: ${DEFAULT_AGENT_DISCOVERY_ROBOTS_TXT_ROUTE}`);
 	if (llmsEnabled) lines.push(`- llms.txt: ${DEFAULT_LLMS_TXT_ROUTE}`, `- llms-full.txt: ${DEFAULT_LLMS_FULL_TXT_ROUTE}`, `- llms well-known aliases: ${DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE}, ${DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE}`);
 	if (sitemapConfig.enabled) {
 		if (sitemapConfig.xml.enabled) lines.push(`- Sitemap XML: ${sitemapConfig.xml.route}`);
@@ -226,7 +230,7 @@ function resolveDocsAgentMdxContent(content, audience) {
 	}
 	return output.join("\n").replace(/\n{3,}/g, "\n\n").trim();
 }
-function buildDocsAgentDiscoverySpec({ origin, entry = "docs", i18n = null, search, mcp, feedback, llms, sitemap, markdown }) {
+function buildDocsAgentDiscoverySpec({ origin, entry = "docs", i18n = null, search, mcp, feedback, llms, sitemap, robots, markdown }) {
 	const normalizedEntry = normalizeDocsPathSegment(entry) || "docs";
 	const localesEnabled = i18n !== null;
 	const searchEnabled = isSearchEnabled(search);
@@ -234,6 +238,7 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", i18n = null, sear
 	const feedbackSchemaRoute = feedback?.schemaRoute ?? `${feedbackRoute}/schema`;
 	const llmsEnabled = llms?.enabled ?? true;
 	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: llms?.baseUrl });
+	const robotsEnabled = isRobotsDiscoveryEnabled(robots);
 	return {
 		version: "1",
 		name: "@farming-labs/docs",
@@ -260,6 +265,7 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", i18n = null, sear
 			mcp: mcp.enabled,
 			search: searchEnabled,
 			sitemap: sitemapConfig.enabled,
+			robots: robotsEnabled,
 			agentFeedback: feedback?.enabled ?? false,
 			locales: localesEnabled
 		},
@@ -313,6 +319,11 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", i18n = null, sear
 				defaultWellKnownRoute: DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE
 			}
 		},
+		robots: {
+			enabled: robotsEnabled,
+			route: DEFAULT_AGENT_DISCOVERY_ROBOTS_TXT_ROUTE,
+			defaultRoute: DEFAULT_AGENT_DISCOVERY_ROBOTS_TXT_ROUTE
+		},
 		search: {
 			enabled: searchEnabled,
 			endpoint: `${DEFAULT_DOCS_API_ROUTE}?query={query}`,
@@ -388,6 +399,11 @@ function isSearchEnabled(search) {
 	if (search && typeof search === "object" && search.enabled === false) return false;
 	return true;
 }
+function isRobotsDiscoveryEnabled(robots) {
+	if (robots === false) return false;
+	if (robots && typeof robots === "object" && robots.enabled === false) return false;
+	return true;
+}
 function compactSkillText(value) {
 	return value.replace(/\s+/g, " ").trim();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@farming-labs/docs",
-  "version": "0.1.83",
+  "version": "0.1.84",
   "description": "Modern, flexible MDX-based docs framework — core types, config, and CLI",
   "keywords": [
     "docs",