@farming-labs/theme 0.1.83 → 0.1.85

package/dist/docs-api.d.mts

@@ -1,4 +1,4 @@
-import { ChangelogConfig, DocsAnalyticsConfig, DocsAskAIMcpConfig, DocsI18nConfig, DocsMcpConfig, DocsObservabilityConfig, DocsSearchConfig, DocsSitemapConfig, FeedbackConfig, OrderingItem } from "@farming-labs/docs";
+import { ChangelogConfig, DocsAnalyticsConfig, DocsAskAIMcpConfig, DocsI18nConfig, DocsMcpConfig, DocsObservabilityConfig, DocsRobotsConfig, DocsSearchConfig, DocsSitemapConfig, FeedbackConfig, OrderingItem } from "@farming-labs/docs";
 
 //#region src/docs-api.d.ts
 interface AIProviderConfig {
@@ -56,6 +56,8 @@ interface DocsAPIOptions {
   mcp?: boolean | DocsMcpConfig;
   /** Sitemap configuration used for sitemap.xml and sitemap.md. */
   sitemap?: boolean | DocsSitemapConfig;
+  /** Robots.txt generation policy used for the agent discovery spec. */
+  robots?: boolean | DocsRobotsConfig;
 }
 interface DocsMCPAPIOptions {
   rootDir?: string;

package/dist/docs-api.mjs

@@ -46,6 +46,7 @@ const DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE = "/.well-known/llms.txt";
 const DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE = "/.well-known/llms-full.txt";
 const DEFAULT_SKILL_MD_ROUTE = "/skill.md";
 const DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE = "/.well-known/skill.md";
+const DEFAULT_ROBOTS_TXT_ROUTE = "/robots.txt";
 const DEFAULT_AGENT_FEEDBACK_ROUTE = "/api/docs/agent/feedback";
 const DEFAULT_AGENT_FEEDBACK_PAYLOAD_SCHEMA = {
 	type: "object",
@@ -172,11 +173,17 @@ function isSearchEnabled(search) {
 	if (search && typeof search === "object" && search.enabled === false) return false;
 	return true;
 }
-function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, sitemap }) {
+function isRobotsDiscoveryEnabled(robots) {
+	if (robots === false) return false;
+	if (robots && typeof robots === "object" && robots.enabled === false) return false;
+	return true;
+}
+function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, sitemap, robots }) {
 	const normalizedEntry = normalizePathSegment(entry) || "docs";
 	const localesEnabled = i18n !== null;
 	const searchEnabled = isSearchEnabled(search);
 	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: llms.baseUrl });
+	const robotsEnabled = isRobotsDiscoveryEnabled(robots);
 	return {
 		version: "1",
 		name: "@farming-labs/docs",
@@ -203,6 +210,8 @@ function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, site
 			mcp: mcp.enabled,
 			search: searchEnabled,
 			sitemap: sitemapConfig.enabled,
+			robots: robotsEnabled,
+			structuredData: true,
 			agentFeedback: feedback.enabled,
 			locales: localesEnabled
 		},
@@ -252,6 +261,25 @@ function buildAgentSpec({ origin, entry, i18n, search, mcp, feedback, llms, site
 				api: `${DEFAULT_DOCS_API_ROUTE}?format=sitemap-md`
 			}
 		},
+		robots: {
+			enabled: robotsEnabled,
+			route: DEFAULT_ROBOTS_TXT_ROUTE,
+			defaultRoute: DEFAULT_ROBOTS_TXT_ROUTE
+		},
+		structuredData: {
+			enabled: true,
+			format: "application/ld+json",
+			schema: "https://schema.org/TechArticle",
+			fields: [
+				"headline",
+				"description",
+				"url",
+				"dateModified",
+				"breadcrumb"
+			],
+			canonicalUrlField: "url",
+			breadcrumbType: "BreadcrumbList"
+		},
 		search: {
 			enabled: searchEnabled,
 			endpoint: `${DEFAULT_DOCS_API_ROUTE}?query={query}`,
@@ -915,16 +943,17 @@ function renderMarkdownDocument(page) {
 	lines.push("", page.agentFallbackRawContent ?? page.rawContent ?? page.content);
 	return lines.join("\n");
 }
-function renderSkillDocument({ origin, entry, search, mcp, feedback, llms, sitemap }) {
+function renderSkillDocument({ origin, entry, search, mcp, feedback, llms, sitemap, robots }) {
 	const normalizedEntry = normalizePathSegment(entry) || "docs";
 	const siteTitle = compactSkillText(llms.siteTitle ?? "Documentation");
 	const siteDescription = llms.siteDescription ? compactSkillText(llms.siteDescription) : void 0;
 	const searchEnabled = isSearchEnabled(search);
 	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: llms.baseUrl });
+	const robotsEnabled = isRobotsDiscoveryEnabled(robots);
 	const lines = [
 		"---",
 		"name: docs",
-		`description: ${toYamlString(truncateSkillDescription(`Use ${siteTitle} through markdown routes, llms.txt, agent discovery, search, and MCP when available.`))}`,
+		`description: ${toYamlString(truncateSkillDescription(`Use ${siteTitle} through markdown routes, llms.txt, robots.txt, agent discovery, search, and MCP when available.`))}`,
 		"---",
 		"",
 		`# ${siteTitle} Skill`,
@@ -939,9 +968,11 @@ function renderSkillDocument({ origin, entry, search, mcp, feedback, llms, sitem
 		if (sitemapConfig.xml.enabled) lines.push(`- Use ${sitemapConfig.xml.route} to check canonical page freshness.`);
 		if (sitemapConfig.markdown.enabled) lines.push(`- Use ${sitemapConfig.markdown.route} for a semantic docs map.`);
 	}
+	if (robotsEnabled) lines.push(`- Check ${DEFAULT_ROBOTS_TXT_ROUTE} for crawler and AI-agent access policy.`);
 	if (mcp.enabled) lines.push(`- Use ${DEFAULT_MCP_WELL_KNOWN_ROUTE} or ${DEFAULT_MCP_PUBLIC_ROUTE} for MCP tools when your environment supports MCP.`);
 	if (feedback.enabled) lines.push(`- Read ${feedback.schemaRoute} before posting agent feedback to ${feedback.route}.`);
 	lines.push("", "## Routes", `- Skill document: ${DEFAULT_SKILL_MD_ROUTE}`, `- Skill well-known alias: ${DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE}`, `- Skill API format: ${DEFAULT_DOCS_API_ROUTE}?format=skill`, `- Agent discovery: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE}`, `- Agent discovery fallback: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE}`, `- Markdown root: /${normalizedEntry}.md`, `- Markdown pages: /${normalizedEntry}/{slug}.md`);
+	if (robotsEnabled) lines.push(`- Robots policy: ${DEFAULT_ROBOTS_TXT_ROUTE}`);
 	if (llms.enabled) lines.push(`- llms.txt: ${DEFAULT_LLMS_TXT_ROUTE}`, `- llms-full.txt: ${DEFAULT_LLMS_FULL_TXT_ROUTE}`, `- llms well-known aliases: ${DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE}, ${DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE}`);
 	if (sitemapConfig.enabled) {
 		if (sitemapConfig.xml.enabled) lines.push(`- Sitemap XML: ${sitemapConfig.xml.route}`);
@@ -1507,6 +1538,31 @@ function readSitemapConfig(root) {
 		}
 	}
 }
+function readRobotsConfig(root) {
+	for (const ext of FILE_EXTS) {
+		const configPath = path.join(root, `docs.config.${ext}`);
+		if (!fs.existsSync(configPath)) continue;
+		try {
+			const content = fs.readFileSync(configPath, "utf-8");
+			if (!content.includes("robots")) return void 0;
+			if (/robots\s*:\s*false/.test(content)) return false;
+			if (/robots\s*:\s*true/.test(content)) return true;
+			const robotsBlock = content.match(/robots\s*:\s*\{([\s\S]*?)\n\s*\}/)?.[1] ?? "";
+			const enabledMatch = robotsBlock.match(/enabled\s*:\s*(true|false)/);
+			const pathMatch = robotsBlock.match(/path\s*:\s*["']([^"']+)["']/);
+			const baseUrlMatch = robotsBlock.match(/baseUrl\s*:\s*["']([^"']+)["']/);
+			const aiMatch = robotsBlock.match(/ai\s*:\s*["'](allow|disallow)["']/);
+			return {
+				enabled: enabledMatch ? enabledMatch[1] === "true" : true,
+				path: pathMatch?.[1],
+				baseUrl: baseUrlMatch?.[1],
+				ai: aiMatch?.[1]
+			};
+		} catch {
+			return;
+		}
+	}
+}
 function generateLlmsTxt(indexes, options) {
 	const { siteTitle = "Documentation", siteDescription, baseUrl = "" } = options;
 	let llmsTxt = `# ${siteTitle}\n\n`;
@@ -1568,6 +1624,7 @@ function createDocsAPI(options) {
 	const searchConfig = options?.search;
 	const llmsConfig = readLlmsTxtConfig(root);
 	const sitemapConfig = options?.sitemap ?? readSitemapConfig(root);
+	const robotsConfig = options?.robots ?? readRobotsConfig(root);
 	const mcpConfig = resolveDocsMcpConfig(options?.mcp ?? readMcpConfig(root), { defaultName: llmsConfig.siteTitle ?? "Documentation" });
 	function resolveDocsDirCandidates(locale) {
 		const relativeCandidates = /* @__PURE__ */ new Set();
@@ -1708,7 +1765,8 @@ function createDocsAPI(options) {
 					mcp: mcpConfig,
 					feedback: agentFeedbackConfig,
 					llms: llmsConfig,
-					sitemap: sitemapConfig
+					sitemap: sitemapConfig,
+					robots: robotsConfig
 				}), { headers: {
 					"Cache-Control": "public, max-age=0, s-maxage=3600",
 					"X-Robots-Tag": "noindex"
@@ -1751,7 +1809,8 @@ function createDocsAPI(options) {
 					mcp: mcpConfig,
 					feedback: agentFeedbackConfig,
 					llms: llmsConfig,
-					sitemap: sitemapConfig
+					sitemap: sitemapConfig,
+					robots: robotsConfig
 				}), { headers: {
 					"Content-Type": "text/markdown; charset=utf-8",
 					"Cache-Control": "public, max-age=0, s-maxage=3600",

package/dist/docs-layout.mjs

@@ -11,7 +11,7 @@ import path from "node:path";
 import matter from "gray-matter";
 import { DocsLayout } from "fumadocs-ui/layouts/docs";
 import { Suspense } from "react";
-import { applySidebarFolderIndexBehavior, buildPageOpenGraph, buildPageTwitter, resolveChangelogConfig, resolveDocsAgentMdxContent, resolveDocsAnalyticsConfig, resolvePageSidebarFolderIndexBehavior, toDocsMarkdownUrl } from "@farming-labs/docs";
+import { applySidebarFolderIndexBehavior, buildPageOpenGraph, buildPageTwitter, renderDocsPageStructuredDataJson, resolveChangelogConfig, resolveDocsAgentMdxContent, resolveDocsAnalyticsConfig, resolveDocsMetadataBaseUrl, resolvePageSidebarFolderIndexBehavior, toDocsMarkdownUrl } from "@farming-labs/docs";
 import { jsx, jsxs } from "react/jsx-runtime";
 
 //#region src/docs-layout.tsx
@@ -380,6 +380,41 @@ function buildReadingTimeMap(config, ctx, options) {
 	scan(docsDir, []);
 	return map;
 }
+function findDocsPageFile(dir) {
+	return ["page.mdx", "page.md"].map((fileName) => path.join(dir, fileName)).find(fs.existsSync);
+}
+function buildStructuredDataMap(config, ctx) {
+	const docsDir = ctx.docsDir;
+	const map = {};
+	const excludedDirs = getExcludedDocsDirs(config, ctx);
+	const baseUrl = resolveDocsMetadataBaseUrl(config);
+	function scan(dir, slugParts) {
+		if (!fs.existsSync(dir)) return;
+		if (isExcludedDir(dir, excludedDirs)) return;
+		const pagePath = findDocsPageFile(dir);
+		if (pagePath) {
+			const { data } = matter(fs.readFileSync(pagePath, "utf-8"));
+			const route = slugParts.length === 0 ? `/${ctx.entryPath}` : `/${ctx.entryPath}/${slugParts.join("/")}`;
+			const title = typeof data.title === "string" ? data.title : slugParts.at(-1)?.replace(/-/g, " ") || "Documentation";
+			const description = typeof data.description === "string" ? data.description : void 0;
+			const stat = fs.statSync(pagePath);
+			map[route] = renderDocsPageStructuredDataJson({
+				title,
+				description,
+				url: withLangInUrl(route, ctx.locale),
+				baseUrl,
+				entry: ctx.entryPath,
+				dateModified: stat.mtime.toISOString()
+			});
+		}
+		for (const name of fs.readdirSync(dir)) {
+			const full = path.join(dir, name);
+			if (fs.statSync(full).isDirectory()) scan(full, [...slugParts, name]);
+		}
+	}
+	scan(docsDir, []);
+	return map;
+}
 /**
 * Build a Next.js Metadata object from the docs config.
 *
@@ -638,6 +673,7 @@ function createDocsLayout(config, options) {
 		enabledByDefault: readingTimeEnabledByDefault,
 		wordsPerMinute: readingTimeWordsPerMinute
 	});
+	const structuredDataMap = buildStructuredDataMap(config, localeContext);
 	const readingTimeEnabled = readingTimeEnabledByDefault || Object.keys(readingTimeMap).length > 0;
 	return function DocsLayoutWrapper({ children }) {
 		const tree = applySidebarFolderIndexBehavior(buildTree(config, localeContext, !!sidebarFlat), {
@@ -736,6 +772,7 @@ function createDocsLayout(config, options) {
 						lastUpdatedPosition,
 						readingTimeEnabled,
 						readingTimeMap,
+						structuredDataMap,
 						llmsTxtEnabled,
 						descriptionMap,
 						feedbackEnabled: feedbackConfig.enabled,

package/dist/docs-page-client.d.mts

@@ -43,6 +43,10 @@ interface DocsPageClientProps {
   readingTimeMap?: Record<string, number>;
   /** Direct reading-time override for the current page. */
   readingTime?: number | null;
+  /** Map of pathname → serialized Schema.org JSON-LD. */
+  structuredDataMap?: Record<string, string>;
+  /** Direct serialized Schema.org JSON-LD override for the current page. */
+  structuredData?: string;
   /**
    * Whether path-based reading time values should render by default.
    * Explicit `readingTime` overrides can still render when this is false.
@@ -90,6 +94,8 @@ declare function DocsPageClient({
   lastModified: lastModifiedProp,
   readingTimeMap,
   readingTime: readingTimeProp,
+  structuredDataMap,
+  structuredData: structuredDataProp,
   readingTimeEnabled,
   lastUpdatedEnabled,
   lastUpdatedPosition,

package/dist/docs-page-client.mjs

@@ -5,6 +5,7 @@ import { PageActions } from "./page-actions.mjs";
 import { useWindowSearchParams } from "./client-location.mjs";
 import { DocsFeedback } from "./docs-feedback.mjs";
 import { resolveClientLocale, withLangInUrl } from "./i18n.mjs";
+import { escapeJsonLdForScript } from "./json-ld.mjs";
 import { Children, Fragment, cloneElement, isValidElement, useEffect, useState } from "react";
 import { DocsBody, DocsPage, EditOnGitHub } from "fumadocs-ui/layouts/docs/page";
 import { createPortal } from "react-dom";
@@ -163,7 +164,7 @@ function TitleDecorations({ description, belowTitle }) {
 	if (!description && !belowTitle) return null;
 	return /* @__PURE__ */ jsx(Fragment$1, { children: Children.toArray([description, belowTitle].filter(Boolean)) });
 }
-function DocsPageClient({ tocEnabled, tocStyle = "default", breadcrumbEnabled = true, changelogBasePath, entry = "docs", locale, copyMarkdown = false, openDocs = false, openDocsProviders, pageActionsPosition = "below-title", pageActionsAlignment = "left", githubUrl, contentDir, githubBranch = "main", githubDirectory, editOnGithubUrl, lastModifiedMap, lastModified: lastModifiedProp, readingTimeMap, readingTime: readingTimeProp, readingTimeEnabled = false, lastUpdatedEnabled = true, lastUpdatedPosition = "footer", llmsTxtEnabled = false, descriptionMap, description, feedbackEnabled = false, feedbackQuestion, feedbackPlaceholder, feedbackPositiveLabel, feedbackNegativeLabel, feedbackSubmitLabel, feedbackOnFeedback, analytics = false, children }) {
+function DocsPageClient({ tocEnabled, tocStyle = "default", breadcrumbEnabled = true, changelogBasePath, entry = "docs", locale, copyMarkdown = false, openDocs = false, openDocsProviders, pageActionsPosition = "below-title", pageActionsAlignment = "left", githubUrl, contentDir, githubBranch = "main", githubDirectory, editOnGithubUrl, lastModifiedMap, lastModified: lastModifiedProp, readingTimeMap, readingTime: readingTimeProp, structuredDataMap, structuredData: structuredDataProp, readingTimeEnabled = false, lastUpdatedEnabled = true, lastUpdatedPosition = "footer", llmsTxtEnabled = false, descriptionMap, description, feedbackEnabled = false, feedbackQuestion, feedbackPlaceholder, feedbackPositiveLabel, feedbackNegativeLabel, feedbackSubmitLabel, feedbackOnFeedback, analytics = false, children }) {
 	const fdTocStyle = tocStyle === "directional" ? "clerk" : void 0;
 	const [toc, setToc] = useState([]);
 	const [titlePortalHost, setTitlePortalHost] = useState(null);
@@ -175,6 +176,7 @@ function DocsPageClient({ tocEnabled, tocStyle = "default", breadcrumbEnabled =
 	const normalizedPath = (browserPath ?? pathname).replace(/\/$/, "") || "/";
 	const isChangelogRoute = !!(changelogBasePath && (normalizedPath === changelogBasePath || normalizedPath.startsWith(`${changelogBasePath}/`)));
 	const matchedReadingTime = readingTimeMap?.[normalizedPath];
+	const structuredDataJson = !isChangelogRoute ? structuredDataProp ?? structuredDataMap?.[normalizedPath] : void 0;
 	useEffect(() => {
 		if (!analytics) return;
 		emitClientAnalyticsEvent({
@@ -341,7 +343,10 @@ function DocsPageClient({ tocEnabled, tocStyle = "default", breadcrumbEnabled =
 		belowTitle: belowTitleBlock
 	}), titlePortalHost) : null;
 	const renderedChildren = Children.toArray(decoratedChildren);
-	return /* @__PURE__ */ jsxs(DocsPage, {
+	return /* @__PURE__ */ jsxs(Fragment$1, { children: [structuredDataJson && /* @__PURE__ */ jsx("script", {
+		type: "application/ld+json",
+		dangerouslySetInnerHTML: { __html: escapeJsonLdForScript(structuredDataJson) }
+	}), /* @__PURE__ */ jsxs(DocsPage, {
 		full: false,
 		toc,
 		tableOfContent: {
@@ -428,7 +433,7 @@ function DocsPageClient({ tocEnabled, tocStyle = "default", breadcrumbEnabled =
 				]
 			})
 		]
-	});
+	})] });
 }
 
 //#endregion

package/dist/json-ld.mjs

@@ -0,0 +1,7 @@
+//#region src/json-ld.ts
+function escapeJsonLdForScript(json) {
+	return json.replace(/</g, "\\u003c");
+}
+
+//#endregion
+export { escapeJsonLdForScript };

package/dist/tanstack-layout.d.mts

@@ -31,6 +31,7 @@ interface TanstackDocsLayoutProps {
   description?: string;
   readingTime?: number | null;
   lastModified?: string;
+  structuredData?: string;
   editOnGithubUrl?: string;
   children: ReactNode;
 }
@@ -41,6 +42,7 @@ declare function TanstackDocsLayout({
   description,
   readingTime,
   lastModified,
+  structuredData,
   editOnGithubUrl,
   children
 }: TanstackDocsLayoutProps): react_jsx_runtime0.JSX.Element;

package/dist/tanstack-layout.mjs

@@ -1,4 +1,5 @@
 import { withLangInUrl } from "./i18n.mjs";
+import { escapeJsonLdForScript } from "./json-ld.mjs";
 import { DocsPageClient } from "./docs-page-client.mjs";
 import { DocsAIFeatures } from "./docs-ai-features.mjs";
 import { DocsCommandSearch } from "./docs-command-search.mjs";
@@ -8,7 +9,7 @@ import { LocaleThemeControl } from "./locale-theme-control.mjs";
 import { DocsLayout } from "fumadocs-ui/layouts/docs";
 import { Suspense } from "react";
 import { applySidebarFolderIndexBehavior, resolveDocsAnalyticsConfig } from "@farming-labs/docs";
-import { jsx, jsxs } from "react/jsx-runtime";
+import { Fragment as Fragment$1, jsx, jsxs } from "react/jsx-runtime";
 
 //#region src/tanstack-layout.tsx
 function resolveTreeIcon(icon, registry) {
@@ -207,7 +208,7 @@ function resolveFeedbackConfig(feedback) {
 function ForcedThemeScript({ theme }) {
 	return /* @__PURE__ */ jsx("script", { dangerouslySetInnerHTML: { __html: `document.documentElement.classList.remove('light','dark');document.documentElement.classList.add('${theme === "light" || theme === "dark" ? theme : "light"}');` } });
 }
-function TanstackDocsLayout({ config, tree, locale, description, readingTime, lastModified, editOnGithubUrl, children }) {
+function TanstackDocsLayout({ config, tree, locale, description, readingTime, lastModified, structuredData, editOnGithubUrl, children }) {
 	const tocConfig = config.theme?.ui?.layout?.toc;
 	const tocEnabled = tocConfig?.enabled !== false;
 	const tocStyle = tocConfig?.style;
@@ -290,7 +291,7 @@ function TanstackDocsLayout({ config, tree, locale, description, readingTime, la
 		collapsible: sidebarProps.collapsible !== false,
 		flat: !!sidebarFlat
 	});
-	return /* @__PURE__ */ jsxs(DocsLayout, {
+	const layout = /* @__PURE__ */ jsxs(DocsLayout, {
 		tree: resolvedTree,
 		nav: {
 			title: navTitle,
@@ -364,6 +365,11 @@ function TanstackDocsLayout({ config, tree, locale, description, readingTime, la
 			})
 		]
 	});
+	if (!structuredData) return layout;
+	return /* @__PURE__ */ jsxs(Fragment$1, { children: [/* @__PURE__ */ jsx("script", {
+		type: "application/ld+json",
+		dangerouslySetInnerHTML: { __html: escapeJsonLdForScript(structuredData) }
+	}), layout] });
 }
 
 //#endregion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@farming-labs/theme",
-  "version": "0.1.83",
+  "version": "0.1.85",
   "description": "Theme package for @farming-labs/docs — layout, provider, MDX components, and styles",
   "keywords": [
     "docs",
@@ -139,7 +139,7 @@
     "tsdown": "^0.20.3",
     "typescript": "^5.9.3",
     "vitest": "^3.2.4",
-    "@farming-labs/docs": "0.1.83"
+    "@farming-labs/docs": "0.1.85"
   },
   "peerDependencies": {
     "@farming-labs/docs": ">=0.0.1",