@farming-labs/docs 0.2.62 → 0.2.64

package/dist/robots-BIpC4j4P.mjs

@@ -0,0 +1,201 @@
+import { Ft as resolveDocsSitemapConfig, S as DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE, X as normalizeDocsPathSegment, a as DEFAULT_AGENT_MD_ROUTE, c as DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, en as DEFAULT_MCP_PUBLIC_ROUTE, g as DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE, h as DEFAULT_LLMS_FULL_TXT_ROUTE, l as DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, n as DEFAULT_AGENTS_MD_WELL_KNOWN_ROUTE, nn as DEFAULT_MCP_WELL_KNOWN_ROUTE, o as DEFAULT_AGENT_MD_WELL_KNOWN_ROUTE, t as DEFAULT_AGENTS_MD_ROUTE, v as DEFAULT_LLMS_TXT_ROUTE, x as DEFAULT_SKILL_MD_ROUTE, y as DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE } from "./agent-DKKptIgy.mjs";
+import { a as DEFAULT_A2A_AGENT_CARD_ROUTE, c as DEFAULT_AGENT_SKILLS_INDEX_ROUTE, g as DEFAULT_API_CATALOG_ROUTE, u as DEFAULT_AGENT_SKILLS_ROUTE_PREFIX, v as DEFAULT_LEGACY_SKILLS_INDEX_ROUTE, y as DEFAULT_LEGACY_SKILLS_ROUTE_PREFIX } from "./standards-discovery-jkykaXq1.mjs";
+
+//#region src/robots.ts
+const DEFAULT_ROBOTS_TXT_ROUTE = "/robots.txt";
+const DOCS_ROBOTS_GENERATED_BLOCK_START = "# BEGIN @farming-labs/docs robots";
+const DOCS_ROBOTS_GENERATED_BLOCK_END = "# END @farming-labs/docs robots";
+const DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS = [
+	"GPTBot",
+	"ChatGPT-User",
+	"OAI-SearchBot",
+	"ClaudeBot",
+	"Claude-User",
+	"anthropic-ai",
+	"CCBot",
+	"Google-Extended"
+];
+function normalizeRoute(value) {
+	const trimmed = value.trim();
+	if (!trimmed) return "/";
+	return trimmed.startsWith("/") ? trimmed.replace(/\/{2,}/g, "/") : `/${trimmed}`;
+}
+function normalizeBaseUrl(value) {
+	if (!value) return void 0;
+	try {
+		const url = new URL(value);
+		url.pathname = url.pathname.replace(/\/+$/, "");
+		url.search = "";
+		url.hash = "";
+		return url.toString().replace(/\/+$/, "");
+	} catch {
+		return;
+	}
+}
+function normalizeStringArray(value) {
+	if (Array.isArray(value)) return value;
+	return value ? [value] : [];
+}
+function normalizeAiPolicy(value) {
+	return value === false || value === "disallow" ? "disallow" : "allow";
+}
+function unique(values) {
+	return [...new Set(values.filter(Boolean))];
+}
+function resolveDocsRobotsConfig(input, options = {}) {
+	if (input === false) return {
+		enabled: false,
+		ai: "allow",
+		userAgents: [...DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS],
+		extraRules: []
+	};
+	const config = typeof input === "object" ? input : {};
+	return {
+		enabled: config.enabled ?? true,
+		path: config.path,
+		baseUrl: normalizeBaseUrl(config.baseUrl) ?? normalizeBaseUrl(options.baseUrl),
+		ai: normalizeAiPolicy(config.ai),
+		userAgents: unique([...DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS, ...normalizeStringArray(config.aiUserAgents)]),
+		extraRules: config.extraRules ?? []
+	};
+}
+function getDocsRobotsAllowRoutes(options = {}) {
+	const normalizedEntry = normalizeDocsPathSegment(options.entry ?? "docs") || "docs";
+	const sitemapConfig = resolveDocsSitemapConfig(options.sitemap);
+	const routes = [
+		"/",
+		`/${normalizedEntry}`,
+		`/${normalizedEntry}/`,
+		`/${normalizedEntry}.md`,
+		`/${normalizedEntry}/*.md`,
+		DEFAULT_LLMS_TXT_ROUTE,
+		DEFAULT_LLMS_FULL_TXT_ROUTE,
+		DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE,
+		DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE,
+		DEFAULT_SKILL_MD_ROUTE,
+		DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE,
+		DEFAULT_AGENTS_MD_ROUTE,
+		DEFAULT_AGENTS_MD_WELL_KNOWN_ROUTE,
+		DEFAULT_AGENT_MD_ROUTE,
+		DEFAULT_AGENT_MD_WELL_KNOWN_ROUTE,
+		DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE,
+		DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE,
+		...options.apiCatalog === false ? [] : [DEFAULT_API_CATALOG_ROUTE],
+		DEFAULT_AGENT_SKILLS_INDEX_ROUTE,
+		`${DEFAULT_AGENT_SKILLS_ROUTE_PREFIX}/*`,
+		DEFAULT_LEGACY_SKILLS_INDEX_ROUTE,
+		`${DEFAULT_LEGACY_SKILLS_ROUTE_PREFIX}/*`,
+		...options.agentCard ? [DEFAULT_A2A_AGENT_CARD_ROUTE] : [],
+		DEFAULT_MCP_PUBLIC_ROUTE,
+		DEFAULT_MCP_WELL_KNOWN_ROUTE
+	];
+	if (sitemapConfig.enabled) {
+		if (sitemapConfig.xml.enabled) routes.push(sitemapConfig.xml.route);
+		if (sitemapConfig.markdown.enabled) {
+			routes.push(sitemapConfig.markdown.route, sitemapConfig.markdown.wellKnownRoute);
+			if (sitemapConfig.markdown.docsRoute) routes.push(sitemapConfig.markdown.docsRoute);
+		}
+	}
+	return unique(routes.map(normalizeRoute));
+}
+function sitemapUrls(robots, sitemap) {
+	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: robots.baseUrl });
+	if (!robots.baseUrl || !sitemapConfig.enabled || !sitemapConfig.xml.enabled) return [];
+	try {
+		return [new URL(sitemapConfig.xml.route, `${robots.baseUrl}/`).toString()];
+	} catch {
+		return [];
+	}
+}
+function appendRule(lines, rule) {
+	const userAgents = normalizeStringArray(rule.userAgent);
+	if (userAgents.length === 0) return;
+	lines.push("");
+	for (const userAgent of userAgents) lines.push(`User-agent: ${userAgent}`);
+	for (const route of normalizeStringArray(rule.allow)) lines.push(`Allow: ${normalizeRoute(route)}`);
+	for (const route of normalizeStringArray(rule.disallow)) lines.push(`Disallow: ${normalizeRoute(route)}`);
+	if (typeof rule.crawlDelay === "number" && Number.isFinite(rule.crawlDelay)) lines.push(`Crawl-delay: ${rule.crawlDelay}`);
+}
+function renderDocsRobotsTxt(options = {}) {
+	const robots = resolveDocsRobotsConfig(options.robots, { baseUrl: options.baseUrl });
+	if (!robots.enabled) return "";
+	const lines = [
+		"# Generated by @farming-labs/docs.",
+		"# Edit docs.config or rerun `docs robots generate --append` / `--force`.",
+		"",
+		"User-agent: *"
+	];
+	for (const route of getDocsRobotsAllowRoutes(options)) lines.push(`Allow: ${route}`);
+	for (const userAgent of robots.userAgents) {
+		lines.push("", `User-agent: ${userAgent}`);
+		lines.push(robots.ai === "allow" ? "Allow: /" : "Disallow: /");
+	}
+	for (const rule of robots.extraRules) appendRule(lines, rule);
+	const sitemap = sitemapUrls(robots, options.sitemap);
+	if (sitemap.length > 0) {
+		lines.push("");
+		for (const url of sitemap) lines.push(`Sitemap: ${url}`);
+	}
+	return `${lines.join("\n")}\n`;
+}
+function resolveDocsRobotsRequest(url, robots) {
+	const pathname = normalizeRoute(url.pathname);
+	const format = url.searchParams.get("format")?.trim();
+	if (pathname === "/api/docs" && format === "robots") return "robots";
+	if (!resolveDocsRobotsConfig(robots).enabled) return null;
+	return pathname === DEFAULT_ROBOTS_TXT_ROUTE ? "robots" : null;
+}
+function createDocsRobotsResponse({ request, ...options }) {
+	const url = new URL(request.url);
+	if (!resolveDocsRobotsRequest(url, options.robots)) return null;
+	const content = renderDocsRobotsTxt({
+		...options,
+		baseUrl: options.baseUrl ?? url.origin
+	});
+	if (!content) return null;
+	return new Response(content, { headers: {
+		"Content-Type": "text/plain; charset=utf-8",
+		"Cache-Control": "public, max-age=0, s-maxage=3600"
+	} });
+}
+function renderDocsRobotsGeneratedBlock(options = {}) {
+	return `${DOCS_ROBOTS_GENERATED_BLOCK_START}\n${renderDocsRobotsTxt(options).trimEnd()}\n${DOCS_ROBOTS_GENERATED_BLOCK_END}\n`;
+}
+function upsertDocsRobotsGeneratedBlock(existing, block) {
+	const escapedStart = DOCS_ROBOTS_GENERATED_BLOCK_START.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	const escapedEnd = DOCS_ROBOTS_GENERATED_BLOCK_END.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	const pattern = new RegExp(`${escapedStart}[\\s\\S]*?${escapedEnd}\\n?`);
+	if (pattern.test(existing)) return existing.replace(pattern, block);
+	const prefix = existing.trimEnd();
+	return `${prefix}${prefix ? "\n\n" : ""}${block}`;
+}
+function blockForUserAgent(content, userAgent) {
+	const escaped = userAgent.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return content.match(new RegExp(`(?:^|\\n)\\s*User-agent:\\s*${escaped}\\s*(?:\\n[ \\t]*(?!User-agent:).*)*`, "i"))?.[0] ?? "";
+}
+function disallowsRoute(content, route) {
+	const escapedRoute = route.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return new RegExp(`^\\s*Disallow:\\s*${escapedRoute}(?:\\s|$)`, "im").test(content);
+}
+function analyzeDocsRobotsTxt(content, options = {}) {
+	const expectedRoutes = getDocsRobotsAllowRoutes(options).filter((route) => [
+		DEFAULT_LLMS_TXT_ROUTE,
+		DEFAULT_LLMS_FULL_TXT_ROUTE,
+		DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE,
+		DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE,
+		DEFAULT_AGENTS_MD_ROUTE,
+		DEFAULT_SKILL_MD_ROUTE,
+		DEFAULT_MCP_PUBLIC_ROUTE
+	].includes(route) || route.includes("sitemap"));
+	const missingRoutes = expectedRoutes.filter((route) => !content.includes(route));
+	return {
+		blocksAgentRoutes: /Disallow:\s*\/(?:\s|$)/i.test(blockForUserAgent(content, "*")) || expectedRoutes.some((route) => disallowsRoute(content, route)),
+		blocksAiAgents: DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS.some((userAgent) => /Disallow:\s*\/(?:\s|$)/i.test(blockForUserAgent(content, userAgent))),
+		hasAgentRoutes: missingRoutes.length === 0,
+		hasAiPolicy: DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS.some((userAgent) => content.includes(userAgent)),
+		missingRoutes
+	};
+}
+
+//#endregion
+export { analyzeDocsRobotsTxt as a, renderDocsRobotsGeneratedBlock as c, resolveDocsRobotsRequest as d, upsertDocsRobotsGeneratedBlock as f, DOCS_ROBOTS_GENERATED_BLOCK_START as i, renderDocsRobotsTxt as l, DEFAULT_ROBOTS_TXT_ROUTE as n, createDocsRobotsResponse as o, DOCS_ROBOTS_GENERATED_BLOCK_END as r, getDocsRobotsAllowRoutes as s, DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS as t, resolveDocsRobotsConfig as u };

package/dist/robots-CUTahhoY.mjs

@@ -0,0 +1,179 @@
+import "./agent-DKKptIgy.mjs";
+import "./standards-discovery-jkykaXq1.mjs";
+import { c as renderDocsRobotsGeneratedBlock, f as upsertDocsRobotsGeneratedBlock, i as DOCS_ROBOTS_GENERATED_BLOCK_START, r as DOCS_ROBOTS_GENERATED_BLOCK_END, u as resolveDocsRobotsConfig } from "./robots-BIpC4j4P.mjs";
+import { a as loadDocsConfigModule, g as resolveDocsConfigPath, h as readTopLevelStringProperty, l as readBooleanProperty, p as readStringProperty, t as extractNestedObjectLiteral } from "./config-Wcdj-D0a.mjs";
+import { t as detectFramework } from "./utils-DpiIioYb.mjs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import pc from "picocolors";
+
+//#region src/cli/robots.ts
+function parseInlineFlag(arg) {
+	const [rawKey, value] = arg.slice(2).split("=", 2);
+	return {
+		key: rawKey.trim(),
+		value
+	};
+}
+function parseRobotsGenerateArgs(argv) {
+	const parsed = {};
+	for (let index = 0; index < argv.length; index += 1) {
+		const arg = argv[index];
+		if (arg === "--help" || arg === "-h") {
+			parsed.help = true;
+			continue;
+		}
+		if (arg === "--append") {
+			parsed.append = true;
+			continue;
+		}
+		if (arg === "--force") {
+			parsed.force = true;
+			continue;
+		}
+		if (arg === "--check") {
+			parsed.check = true;
+			continue;
+		}
+		if (arg.startsWith("--config=")) {
+			const value = parseInlineFlag(arg).value;
+			if (!value) throw new Error("Missing value for --config.");
+			parsed.configPath = value;
+			continue;
+		}
+		if (arg === "--config") {
+			const value = argv[index + 1];
+			if (!value || value.startsWith("--")) throw new Error("Missing value for --config.");
+			parsed.configPath = value;
+			index += 1;
+			continue;
+		}
+		if (arg.startsWith("--path=")) {
+			const value = parseInlineFlag(arg).value;
+			if (!value) throw new Error("Missing value for --path.");
+			parsed.path = value;
+			continue;
+		}
+		if (arg === "--path") {
+			const value = argv[index + 1];
+			if (!value || value.startsWith("--")) throw new Error("Missing value for --path.");
+			parsed.path = value;
+			index += 1;
+			continue;
+		}
+		if (!arg.startsWith("--") && !parsed.path) {
+			parsed.path = arg;
+			continue;
+		}
+		throw new Error(`Unknown robots generate flag: ${arg}.`);
+	}
+	if (parsed.append && parsed.force) throw new Error("Use either --append or --force, not both.");
+	return parsed;
+}
+function readTopLevelBooleanProperty(content, key) {
+	const match = content.match(new RegExp(`\\b${key}\\b\\s*:\\s*(true|false)`));
+	return match ? match[1] === "true" : void 0;
+}
+function readLlmsBaseUrlFromConfig(content, config) {
+	if (config?.llmsTxt && typeof config.llmsTxt === "object") return config.llmsTxt.baseUrl;
+	const block = extractNestedObjectLiteral(content, ["llmsTxt"]);
+	return block ? readStringProperty(block, "baseUrl") : void 0;
+}
+function readSitemapConfigFromStatic(content) {
+	const topLevelBoolean = readTopLevelBooleanProperty(content, "sitemap");
+	if (typeof topLevelBoolean === "boolean") return topLevelBoolean;
+	const block = extractNestedObjectLiteral(content, ["sitemap"]);
+	if (!block) return void 0;
+	return {
+		enabled: readBooleanProperty(block, "enabled") ?? true,
+		routePrefix: readStringProperty(block, "routePrefix"),
+		baseUrl: readStringProperty(block, "baseUrl"),
+		manifestPath: readStringProperty(block, "manifestPath")
+	};
+}
+function readRobotsConfigFromStatic(content) {
+	const topLevelBoolean = readTopLevelBooleanProperty(content, "robots");
+	if (typeof topLevelBoolean === "boolean") return topLevelBoolean;
+	const block = extractNestedObjectLiteral(content, ["robots"]);
+	if (!block) return void 0;
+	const aiString = readStringProperty(block, "ai");
+	const aiBoolean = readBooleanProperty(block, "ai");
+	return {
+		enabled: readBooleanProperty(block, "enabled") ?? true,
+		path: readStringProperty(block, "path"),
+		baseUrl: readStringProperty(block, "baseUrl"),
+		ai: aiString === "allow" || aiString === "disallow" ? aiString : typeof aiBoolean === "boolean" ? aiBoolean : void 0
+	};
+}
+function resolveConfiguredRobots(content, config) {
+	if (config?.robots !== void 0) return config.robots;
+	return readRobotsConfigFromStatic(content) ?? true;
+}
+function resolvePublicDir(rootDir) {
+	if (detectFramework(rootDir) === "sveltekit") return path.join(rootDir, "static");
+	return path.join(rootDir, "public");
+}
+function resolveRobotsPath(rootDir, options, robots) {
+	const configuredPath = options.path ?? robots?.path;
+	if (configuredPath) return path.isAbsolute(configuredPath) ? configuredPath : path.resolve(rootDir, configuredPath);
+	return path.join(resolvePublicDir(rootDir), "robots.txt");
+}
+function writeIfChanged(filePath, content, check) {
+	if ((existsSync(filePath) ? readFileSync(filePath, "utf-8") : void 0) === content) return false;
+	if (check) return true;
+	mkdirSync(path.dirname(filePath), { recursive: true });
+	writeFileSync(filePath, content, "utf-8");
+	return true;
+}
+async function generateRobots(options = {}) {
+	const rootDir = process.cwd();
+	const loadedConfigModule = await loadDocsConfigModule(rootDir, options.configPath);
+	const configContent = readFileSync(loadedConfigModule?.path ?? resolveDocsConfigPath(rootDir, options.configPath), "utf-8");
+	const config = loadedConfigModule?.config;
+	const entry = config?.entry ?? readTopLevelStringProperty(configContent, "entry") ?? "docs";
+	const sitemap = config?.sitemap ?? readSitemapConfigFromStatic(configContent) ?? true;
+	const sitemapBaseUrl = typeof sitemap === "object" ? sitemap.baseUrl : void 0;
+	const llmsBaseUrl = readLlmsBaseUrlFromConfig(configContent, config);
+	const configuredRobots = resolveConfiguredRobots(configContent, config);
+	if (configuredRobots === false) throw new Error("Robots generation is disabled by `robots: false`.");
+	const robotsInput = typeof configuredRobots === "object" ? configuredRobots : {};
+	const robots = resolveDocsRobotsConfig(robotsInput, { baseUrl: robotsInput.baseUrl ?? sitemapBaseUrl ?? llmsBaseUrl });
+	if (!robots.enabled) throw new Error("Robots generation is disabled by `robots.enabled: false`.");
+	const robotsPath = resolveRobotsPath(rootDir, options, robotsInput);
+	const relativeRobotsPath = path.relative(rootDir, robotsPath).replace(/\\/g, "/");
+	const generatedBlock = renderDocsRobotsGeneratedBlock({
+		entry,
+		sitemap,
+		baseUrl: robots.baseUrl,
+		robots
+	});
+	const existing = existsSync(robotsPath) ? readFileSync(robotsPath, "utf-8") : void 0;
+	const hasGeneratedBlock = existing?.includes(DOCS_ROBOTS_GENERATED_BLOCK_START) === true && existing.includes(DOCS_ROBOTS_GENERATED_BLOCK_END);
+	if (existing !== void 0 && !options.force && !options.append && !hasGeneratedBlock) {
+		console.log(pc.yellow(`Found existing robots.txt at ${relativeRobotsPath}.`));
+		console.log(pc.dim("Keeping the user-owned file. Use --append to add/update the generated block, or --force to replace it."));
+		return;
+	}
+	const changed = writeIfChanged(robotsPath, existing !== void 0 && (options.append || hasGeneratedBlock) ? upsertDocsRobotsGeneratedBlock(existing, generatedBlock) : generatedBlock, options.check === true);
+	if (options.check && changed) throw new Error(`Robots output is stale at ${relativeRobotsPath}. Run \`docs robots generate${options.append ? " --append" : options.force ? " --force" : ""}${options.path ? ` --path ${options.path}` : ""}\` to update it.`);
+	console.log(changed ? pc.green(`Generated robots policy at ${relativeRobotsPath}.`) : pc.green(`Robots policy is current at ${relativeRobotsPath}.`));
+}
+function printRobotsGenerateHelp() {
+	console.log(`
+${pc.bold("docs robots generate")} — Generate a static robots.txt agent access policy.
+
+${pc.dim("Usage:")}
+  pnpm exec docs ${pc.cyan("robots generate")} ${pc.dim("[path]")}
+
+${pc.dim("Options:")}
+  ${pc.cyan("--path <path>")}       Write to a specific robots.txt path; defaults to the framework public directory
+  ${pc.cyan("--append")}            Add or update a generated block inside an existing robots.txt
+  ${pc.cyan("--force")}             Replace the target robots.txt with the generated policy
+  ${pc.cyan("--check")}             Fail if the generated output would change
+  ${pc.cyan("--config <path>")}     Use a custom docs config path instead of ${pc.dim("docs.config.ts[x]")}
+  ${pc.cyan("-h, --help")}          Show this help message
+`);
+}
+
+//#endregion
+export { generateRobots, parseRobotsGenerateArgs, printRobotsGenerateHelp };