@farming-labs/theme 0.1.83 → 0.1.84

package/dist/docs-api.d.mts

@@ -1,4 +1,4 @@
-import { ChangelogConfig, DocsAnalyticsConfig, DocsAskAIMcpConfig, DocsI18nConfig, DocsMcpConfig, DocsObservabilityConfig, DocsSearchConfig, DocsSitemapConfig, FeedbackConfig, OrderingItem } from "@farming-labs/docs";
+import { ChangelogConfig, DocsAnalyticsConfig, DocsAskAIMcpConfig, DocsI18nConfig, DocsMcpConfig, DocsObservabilityConfig, DocsRobotsConfig, DocsSearchConfig, DocsSitemapConfig, FeedbackConfig, OrderingItem } from "@farming-labs/docs";
 
 //#region src/docs-api.d.ts
 interface AIProviderConfig {
@@ -56,6 +56,8 @@ interface DocsAPIOptions {
   mcp?: boolean | DocsMcpConfig;
   /** Sitemap configuration used for sitemap.xml and sitemap.md. */
   sitemap?: boolean | DocsSitemapConfig;
+  /** Robots.txt generation policy used for the agent discovery spec. */
+  robots?: boolean | DocsRobotsConfig;
 }
 interface DocsMCPAPIOptions {
   rootDir?: string;

package/dist/docs-api.mjs

@@ -46,6 +46,7 @@ const DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE = "/.well-known/llms.txt";
 const DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE = "/.well-known/llms-full.txt";
 const DEFAULT_SKILL_MD_ROUTE = "/skill.md";
 const DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE = "/.well-known/skill.md";
+const DEFAULT_ROBOTS_TXT_ROUTE = "/robots.txt";
 const DEFAULT_AGENT_FEEDBACK_ROUTE = "/api/docs/agent/feedback";
 const DEFAULT_AGENT_FEEDBACK_PAYLOAD_SCHEMA = {
 	type: "object",
@@ -172,11 +173,17 @@ function isSearchEnabled(search) {
 	if (search && typeof search === "object" && search.enabled === false) return false;
 	return true;
 }
-function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, sitemap }) {
+function isRobotsDiscoveryEnabled(robots) {
+	if (robots === false) return false;
+	if (robots && typeof robots === "object" && robots.enabled === false) return false;
+	return true;
+}
+function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, sitemap, robots }) {
 	const normalizedEntry = normalizePathSegment(entry) || "docs";
 	const localesEnabled = i18n !== null;
 	const searchEnabled = isSearchEnabled(search);
 	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: llms.baseUrl });
+	const robotsEnabled = isRobotsDiscoveryEnabled(robots);
 	return {
 		version: "1",
 		name: "@farming-labs/docs",
@@ -203,6 +210,7 @@ function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, site
 			mcp: mcp.enabled,
 			search: searchEnabled,
 			sitemap: sitemapConfig.enabled,
+			robots: robotsEnabled,
 			agentFeedback: feedback.enabled,
 			locales: localesEnabled
 		},
@@ -252,6 +260,11 @@ function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, site
 				api: `${DEFAULT_DOCS_API_ROUTE}?format=sitemap-md`
 			}
 		},
+		robots: {
+			enabled: robotsEnabled,
+			route: DEFAULT_ROBOTS_TXT_ROUTE,
+			defaultRoute: DEFAULT_ROBOTS_TXT_ROUTE
+		},
 		search: {
 			enabled: searchEnabled,
 			endpoint: `${DEFAULT_DOCS_API_ROUTE}?query={query}`,
@@ -915,16 +928,17 @@ function renderMarkdownDocument(page) {
 	lines.push("", page.agentFallbackRawContent ?? page.rawContent ?? page.content);
 	return lines.join("\n");
 }
-function renderSkillDocument({ origin, entry, search, mcp, feedback, llms, sitemap }) {
+function renderSkillDocument({ origin, entry, search, mcp, feedback, llms, sitemap, robots }) {
 	const normalizedEntry = normalizePathSegment(entry) || "docs";
 	const siteTitle = compactSkillText(llms.siteTitle ?? "Documentation");
 	const siteDescription = llms.siteDescription ? compactSkillText(llms.siteDescription) : void 0;
 	const searchEnabled = isSearchEnabled(search);
 	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: llms.baseUrl });
+	const robotsEnabled = isRobotsDiscoveryEnabled(robots);
 	const lines = [
 		"---",
 		"name: docs",
-		`description: ${toYamlString(truncateSkillDescription(`Use ${siteTitle} through markdown routes, llms.txt, agent discovery, search, and MCP when available.`))}`,
+		`description: ${toYamlString(truncateSkillDescription(`Use ${siteTitle} through markdown routes, llms.txt, robots.txt, agent discovery, search, and MCP when available.`))}`,
 		"---",
 		"",
 		`# ${siteTitle} Skill`,
@@ -939,9 +953,11 @@ function renderSkillDocument({ origin, entry, search, mcp, feedback, llms, sitem
 		if (sitemapConfig.xml.enabled) lines.push(`- Use ${sitemapConfig.xml.route} to check canonical page freshness.`);
 		if (sitemapConfig.markdown.enabled) lines.push(`- Use ${sitemapConfig.markdown.route} for a semantic docs map.`);
 	}
+	if (robotsEnabled) lines.push(`- Check ${DEFAULT_ROBOTS_TXT_ROUTE} for crawler and AI-agent access policy.`);
 	if (mcp.enabled) lines.push(`- Use ${DEFAULT_MCP_WELL_KNOWN_ROUTE} or ${DEFAULT_MCP_PUBLIC_ROUTE} for MCP tools when your environment supports MCP.`);
 	if (feedback.enabled) lines.push(`- Read ${feedback.schemaRoute} before posting agent feedback to ${feedback.route}.`);
 	lines.push("", "## Routes", `- Skill document: ${DEFAULT_SKILL_MD_ROUTE}`, `- Skill well-known alias: ${DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE}`, `- Skill API format: ${DEFAULT_DOCS_API_ROUTE}?format=skill`, `- Agent discovery: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE}`, `- Agent discovery fallback: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE}`, `- Markdown root: /${normalizedEntry}.md`, `- Markdown pages: /${normalizedEntry}/{slug}.md`);
+	if (robotsEnabled) lines.push(`- Robots policy: ${DEFAULT_ROBOTS_TXT_ROUTE}`);
 	if (llms.enabled) lines.push(`- llms.txt: ${DEFAULT_LLMS_TXT_ROUTE}`, `- llms-full.txt: ${DEFAULT_LLMS_FULL_TXT_ROUTE}`, `- llms well-known aliases: ${DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE}, ${DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE}`);
 	if (sitemapConfig.enabled) {
 		if (sitemapConfig.xml.enabled) lines.push(`- Sitemap XML: ${sitemapConfig.xml.route}`);
@@ -1507,6 +1523,31 @@ function readSitemapConfig(root) {
 		}
 	}
 }
+function readRobotsConfig(root) {
+	for (const ext of FILE_EXTS) {
+		const configPath = path.join(root, `docs.config.${ext}`);
+		if (!fs.existsSync(configPath)) continue;
+		try {
+			const content = fs.readFileSync(configPath, "utf-8");
+			if (!content.includes("robots")) return void 0;
+			if (/robots\s*:\s*false/.test(content)) return false;
+			if (/robots\s*:\s*true/.test(content)) return true;
+			const robotsBlock = content.match(/robots\s*:\s*\{([\s\S]*?)\n\s*\}/)?.[1] ?? "";
+			const enabledMatch = robotsBlock.match(/enabled\s*:\s*(true|false)/);
+			const pathMatch = robotsBlock.match(/path\s*:\s*["']([^"']+)["']/);
+			const baseUrlMatch = robotsBlock.match(/baseUrl\s*:\s*["']([^"']+)["']/);
+			const aiMatch = robotsBlock.match(/ai\s*:\s*["'](allow|disallow)["']/);
+			return {
+				enabled: enabledMatch ? enabledMatch[1] === "true" : true,
+				path: pathMatch?.[1],
+				baseUrl: baseUrlMatch?.[1],
+				ai: aiMatch?.[1]
+			};
+		} catch {
+			return;
+		}
+	}
+}
 function generateLlmsTxt(indexes, options) {
 	const { siteTitle = "Documentation", siteDescription, baseUrl = "" } = options;
 	let llmsTxt = `# ${siteTitle}\n\n`;
@@ -1568,6 +1609,7 @@ function createDocsAPI(options) {
 	const searchConfig = options?.search;
 	const llmsConfig = readLlmsTxtConfig(root);
 	const sitemapConfig = options?.sitemap ?? readSitemapConfig(root);
+	const robotsConfig = options?.robots ?? readRobotsConfig(root);
 	const mcpConfig = resolveDocsMcpConfig(options?.mcp ?? readMcpConfig(root), { defaultName: llmsConfig.siteTitle ?? "Documentation" });
 	function resolveDocsDirCandidates(locale) {
 		const relativeCandidates = /* @__PURE__ */ new Set();
@@ -1708,7 +1750,8 @@ function createDocsAPI(options) {
 					mcp: mcpConfig,
 					feedback: agentFeedbackConfig,
 					llms: llmsConfig,
-					sitemap: sitemapConfig
+					sitemap: sitemapConfig,
+					robots: robotsConfig
 				}), { headers: {
 					"Cache-Control": "public, max-age=0, s-maxage=3600",
 					"X-Robots-Tag": "noindex"
@@ -1751,7 +1794,8 @@ function createDocsAPI(options) {
 					mcp: mcpConfig,
 					feedback: agentFeedbackConfig,
 					llms: llmsConfig,
-					sitemap: sitemapConfig
+					sitemap: sitemapConfig,
+					robots: robotsConfig
 				}), { headers: {
 					"Content-Type": "text/markdown; charset=utf-8",
 					"Cache-Control": "public, max-age=0, s-maxage=3600",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@farming-labs/theme",
-  "version": "0.1.83",
+  "version": "0.1.84",
   "description": "Theme package for @farming-labs/docs — layout, provider, MDX components, and styles",
   "keywords": [
     "docs",
@@ -139,7 +139,7 @@
     "tsdown": "^0.20.3",
     "typescript": "^5.9.3",
     "vitest": "^3.2.4",
-    "@farming-labs/docs": "0.1.83"
+    "@farming-labs/docs": "0.1.84"
   },
   "peerDependencies": {
     "@farming-labs/docs": ">=0.0.1",