@farming-labs/docs 0.2.60 → 0.2.62

package/dist/robots-DskPvGPw.mjs

@@ -1,178 +0,0 @@
-import "./agent-BFqyqEnC.mjs";
-import { c as renderDocsRobotsGeneratedBlock, f as upsertDocsRobotsGeneratedBlock, i as DOCS_ROBOTS_GENERATED_BLOCK_START, r as DOCS_ROBOTS_GENERATED_BLOCK_END, u as resolveDocsRobotsConfig } from "./robots-ltltiLJF.mjs";
-import { d as readStringProperty, i as loadDocsConfigModule, m as resolveDocsConfigPath, p as readTopLevelStringProperty, s as readBooleanProperty, t as extractNestedObjectLiteral } from "./config-DASewQ0x.mjs";
-import { t as detectFramework } from "./utils-6UCLxv4B.mjs";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import path from "node:path";
-import pc from "picocolors";
-
-//#region src/cli/robots.ts
-function parseInlineFlag(arg) {
-	const [rawKey, value] = arg.slice(2).split("=", 2);
-	return {
-		key: rawKey.trim(),
-		value
-	};
-}
-function parseRobotsGenerateArgs(argv) {
-	const parsed = {};
-	for (let index = 0; index < argv.length; index += 1) {
-		const arg = argv[index];
-		if (arg === "--help" || arg === "-h") {
-			parsed.help = true;
-			continue;
-		}
-		if (arg === "--append") {
-			parsed.append = true;
-			continue;
-		}
-		if (arg === "--force") {
-			parsed.force = true;
-			continue;
-		}
-		if (arg === "--check") {
-			parsed.check = true;
-			continue;
-		}
-		if (arg.startsWith("--config=")) {
-			const value = parseInlineFlag(arg).value;
-			if (!value) throw new Error("Missing value for --config.");
-			parsed.configPath = value;
-			continue;
-		}
-		if (arg === "--config") {
-			const value = argv[index + 1];
-			if (!value || value.startsWith("--")) throw new Error("Missing value for --config.");
-			parsed.configPath = value;
-			index += 1;
-			continue;
-		}
-		if (arg.startsWith("--path=")) {
-			const value = parseInlineFlag(arg).value;
-			if (!value) throw new Error("Missing value for --path.");
-			parsed.path = value;
-			continue;
-		}
-		if (arg === "--path") {
-			const value = argv[index + 1];
-			if (!value || value.startsWith("--")) throw new Error("Missing value for --path.");
-			parsed.path = value;
-			index += 1;
-			continue;
-		}
-		if (!arg.startsWith("--") && !parsed.path) {
-			parsed.path = arg;
-			continue;
-		}
-		throw new Error(`Unknown robots generate flag: ${arg}.`);
-	}
-	if (parsed.append && parsed.force) throw new Error("Use either --append or --force, not both.");
-	return parsed;
-}
-function readTopLevelBooleanProperty(content, key) {
-	const match = content.match(new RegExp(`\\b${key}\\b\\s*:\\s*(true|false)`));
-	return match ? match[1] === "true" : void 0;
-}
-function readLlmsBaseUrlFromConfig(content, config) {
-	if (config?.llmsTxt && typeof config.llmsTxt === "object") return config.llmsTxt.baseUrl;
-	const block = extractNestedObjectLiteral(content, ["llmsTxt"]);
-	return block ? readStringProperty(block, "baseUrl") : void 0;
-}
-function readSitemapConfigFromStatic(content) {
-	const topLevelBoolean = readTopLevelBooleanProperty(content, "sitemap");
-	if (typeof topLevelBoolean === "boolean") return topLevelBoolean;
-	const block = extractNestedObjectLiteral(content, ["sitemap"]);
-	if (!block) return void 0;
-	return {
-		enabled: readBooleanProperty(block, "enabled") ?? true,
-		routePrefix: readStringProperty(block, "routePrefix"),
-		baseUrl: readStringProperty(block, "baseUrl"),
-		manifestPath: readStringProperty(block, "manifestPath")
-	};
-}
-function readRobotsConfigFromStatic(content) {
-	const topLevelBoolean = readTopLevelBooleanProperty(content, "robots");
-	if (typeof topLevelBoolean === "boolean") return topLevelBoolean;
-	const block = extractNestedObjectLiteral(content, ["robots"]);
-	if (!block) return void 0;
-	const aiString = readStringProperty(block, "ai");
-	const aiBoolean = readBooleanProperty(block, "ai");
-	return {
-		enabled: readBooleanProperty(block, "enabled") ?? true,
-		path: readStringProperty(block, "path"),
-		baseUrl: readStringProperty(block, "baseUrl"),
-		ai: aiString === "allow" || aiString === "disallow" ? aiString : typeof aiBoolean === "boolean" ? aiBoolean : void 0
-	};
-}
-function resolveConfiguredRobots(content, config) {
-	if (config?.robots !== void 0) return config.robots;
-	return readRobotsConfigFromStatic(content) ?? true;
-}
-function resolvePublicDir(rootDir) {
-	if (detectFramework(rootDir) === "sveltekit") return path.join(rootDir, "static");
-	return path.join(rootDir, "public");
-}
-function resolveRobotsPath(rootDir, options, robots) {
-	const configuredPath = options.path ?? robots?.path;
-	if (configuredPath) return path.isAbsolute(configuredPath) ? configuredPath : path.resolve(rootDir, configuredPath);
-	return path.join(resolvePublicDir(rootDir), "robots.txt");
-}
-function writeIfChanged(filePath, content, check) {
-	if ((existsSync(filePath) ? readFileSync(filePath, "utf-8") : void 0) === content) return false;
-	if (check) return true;
-	mkdirSync(path.dirname(filePath), { recursive: true });
-	writeFileSync(filePath, content, "utf-8");
-	return true;
-}
-async function generateRobots(options = {}) {
-	const rootDir = process.cwd();
-	const loadedConfigModule = await loadDocsConfigModule(rootDir, options.configPath);
-	const configContent = readFileSync(loadedConfigModule?.path ?? resolveDocsConfigPath(rootDir, options.configPath), "utf-8");
-	const config = loadedConfigModule?.config;
-	const entry = config?.entry ?? readTopLevelStringProperty(configContent, "entry") ?? "docs";
-	const sitemap = config?.sitemap ?? readSitemapConfigFromStatic(configContent) ?? true;
-	const sitemapBaseUrl = typeof sitemap === "object" ? sitemap.baseUrl : void 0;
-	const llmsBaseUrl = readLlmsBaseUrlFromConfig(configContent, config);
-	const configuredRobots = resolveConfiguredRobots(configContent, config);
-	if (configuredRobots === false) throw new Error("Robots generation is disabled by `robots: false`.");
-	const robotsInput = typeof configuredRobots === "object" ? configuredRobots : {};
-	const robots = resolveDocsRobotsConfig(robotsInput, { baseUrl: robotsInput.baseUrl ?? sitemapBaseUrl ?? llmsBaseUrl });
-	if (!robots.enabled) throw new Error("Robots generation is disabled by `robots.enabled: false`.");
-	const robotsPath = resolveRobotsPath(rootDir, options, robotsInput);
-	const relativeRobotsPath = path.relative(rootDir, robotsPath).replace(/\\/g, "/");
-	const generatedBlock = renderDocsRobotsGeneratedBlock({
-		entry,
-		sitemap,
-		baseUrl: robots.baseUrl,
-		robots
-	});
-	const existing = existsSync(robotsPath) ? readFileSync(robotsPath, "utf-8") : void 0;
-	const hasGeneratedBlock = existing?.includes(DOCS_ROBOTS_GENERATED_BLOCK_START) === true && existing.includes(DOCS_ROBOTS_GENERATED_BLOCK_END);
-	if (existing !== void 0 && !options.force && !options.append && !hasGeneratedBlock) {
-		console.log(pc.yellow(`Found existing robots.txt at ${relativeRobotsPath}.`));
-		console.log(pc.dim("Keeping the user-owned file. Use --append to add/update the generated block, or --force to replace it."));
-		return;
-	}
-	const changed = writeIfChanged(robotsPath, existing !== void 0 && (options.append || hasGeneratedBlock) ? upsertDocsRobotsGeneratedBlock(existing, generatedBlock) : generatedBlock, options.check === true);
-	if (options.check && changed) throw new Error(`Robots output is stale at ${relativeRobotsPath}. Run \`docs robots generate${options.append ? " --append" : options.force ? " --force" : ""}${options.path ? ` --path ${options.path}` : ""}\` to update it.`);
-	console.log(changed ? pc.green(`Generated robots policy at ${relativeRobotsPath}.`) : pc.green(`Robots policy is current at ${relativeRobotsPath}.`));
-}
-function printRobotsGenerateHelp() {
-	console.log(`
-${pc.bold("docs robots generate")} — Generate a static robots.txt agent access policy.
-
-${pc.dim("Usage:")}
-  pnpm exec docs ${pc.cyan("robots generate")} ${pc.dim("[path]")}
-
-${pc.dim("Options:")}
-  ${pc.cyan("--path <path>")}       Write to a specific robots.txt path; defaults to the framework public directory
-  ${pc.cyan("--append")}            Add or update a generated block inside an existing robots.txt
-  ${pc.cyan("--force")}             Replace the target robots.txt with the generated policy
-  ${pc.cyan("--check")}             Fail if the generated output would change
-  ${pc.cyan("--config <path>")}     Use a custom docs config path instead of ${pc.dim("docs.config.ts[x]")}
-  ${pc.cyan("-h, --help")}          Show this help message
-`);
-}
-
-//#endregion
-export { generateRobots, parseRobotsGenerateArgs, printRobotsGenerateHelp };

package/dist/robots-ltltiLJF.mjs

@@ -1,197 +0,0 @@
-import { Qt as DEFAULT_API_CATALOG_ROUTE, S as DEFAULT_MCP_WELL_KNOWN_ROUTE, T as DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE, Yt as DEFAULT_AGENT_SKILLS_ROUTE_PREFIX, a as DEFAULT_AGENT_MD_ROUTE, b as DEFAULT_MCP_PUBLIC_ROUTE, c as DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE, et as normalizeDocsPathSegment, g as DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE, h as DEFAULT_LLMS_FULL_TXT_ROUTE, l as DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE, n as DEFAULT_AGENTS_MD_WELL_KNOWN_ROUTE, o as DEFAULT_AGENT_MD_WELL_KNOWN_ROUTE, qt as DEFAULT_AGENT_SKILLS_INDEX_ROUTE, t as DEFAULT_AGENTS_MD_ROUTE, v as DEFAULT_LLMS_TXT_ROUTE, w as DEFAULT_SKILL_MD_ROUTE, y as DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE, zt as resolveDocsSitemapConfig } from "./agent-BFqyqEnC.mjs";
-
-//#region src/robots.ts
-const DEFAULT_ROBOTS_TXT_ROUTE = "/robots.txt";
-const DOCS_ROBOTS_GENERATED_BLOCK_START = "# BEGIN @farming-labs/docs robots";
-const DOCS_ROBOTS_GENERATED_BLOCK_END = "# END @farming-labs/docs robots";
-const DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS = [
-	"GPTBot",
-	"ChatGPT-User",
-	"OAI-SearchBot",
-	"ClaudeBot",
-	"Claude-User",
-	"anthropic-ai",
-	"CCBot",
-	"Google-Extended"
-];
-function normalizeRoute(value) {
-	const trimmed = value.trim();
-	if (!trimmed) return "/";
-	return trimmed.startsWith("/") ? trimmed.replace(/\/{2,}/g, "/") : `/${trimmed}`;
-}
-function normalizeBaseUrl(value) {
-	if (!value) return void 0;
-	try {
-		const url = new URL(value);
-		url.pathname = url.pathname.replace(/\/+$/, "");
-		url.search = "";
-		url.hash = "";
-		return url.toString().replace(/\/+$/, "");
-	} catch {
-		return;
-	}
-}
-function normalizeStringArray(value) {
-	if (Array.isArray(value)) return value;
-	return value ? [value] : [];
-}
-function normalizeAiPolicy(value) {
-	return value === false || value === "disallow" ? "disallow" : "allow";
-}
-function unique(values) {
-	return [...new Set(values.filter(Boolean))];
-}
-function resolveDocsRobotsConfig(input, options = {}) {
-	if (input === false) return {
-		enabled: false,
-		ai: "allow",
-		userAgents: [...DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS],
-		extraRules: []
-	};
-	const config = typeof input === "object" ? input : {};
-	return {
-		enabled: config.enabled ?? true,
-		path: config.path,
-		baseUrl: normalizeBaseUrl(config.baseUrl) ?? normalizeBaseUrl(options.baseUrl),
-		ai: normalizeAiPolicy(config.ai),
-		userAgents: unique([...DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS, ...normalizeStringArray(config.aiUserAgents)]),
-		extraRules: config.extraRules ?? []
-	};
-}
-function getDocsRobotsAllowRoutes(options = {}) {
-	const normalizedEntry = normalizeDocsPathSegment(options.entry ?? "docs") || "docs";
-	const sitemapConfig = resolveDocsSitemapConfig(options.sitemap);
-	const routes = [
-		"/",
-		`/${normalizedEntry}`,
-		`/${normalizedEntry}/`,
-		`/${normalizedEntry}.md`,
-		`/${normalizedEntry}/*.md`,
-		DEFAULT_LLMS_TXT_ROUTE,
-		DEFAULT_LLMS_FULL_TXT_ROUTE,
-		DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE,
-		DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE,
-		DEFAULT_SKILL_MD_ROUTE,
-		DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE,
-		DEFAULT_AGENTS_MD_ROUTE,
-		DEFAULT_AGENTS_MD_WELL_KNOWN_ROUTE,
-		DEFAULT_AGENT_MD_ROUTE,
-		DEFAULT_AGENT_MD_WELL_KNOWN_ROUTE,
-		DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE,
-		DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE,
-		...options.apiCatalog === false ? [] : [DEFAULT_API_CATALOG_ROUTE],
-		DEFAULT_AGENT_SKILLS_INDEX_ROUTE,
-		`${DEFAULT_AGENT_SKILLS_ROUTE_PREFIX}/*/SKILL.md`,
-		DEFAULT_MCP_PUBLIC_ROUTE,
-		DEFAULT_MCP_WELL_KNOWN_ROUTE
-	];
-	if (sitemapConfig.enabled) {
-		if (sitemapConfig.xml.enabled) routes.push(sitemapConfig.xml.route);
-		if (sitemapConfig.markdown.enabled) {
-			routes.push(sitemapConfig.markdown.route, sitemapConfig.markdown.wellKnownRoute);
-			if (sitemapConfig.markdown.docsRoute) routes.push(sitemapConfig.markdown.docsRoute);
-		}
-	}
-	return unique(routes.map(normalizeRoute));
-}
-function sitemapUrls(robots, sitemap) {
-	const sitemapConfig = resolveDocsSitemapConfig(sitemap, { baseUrl: robots.baseUrl });
-	if (!robots.baseUrl || !sitemapConfig.enabled || !sitemapConfig.xml.enabled) return [];
-	try {
-		return [new URL(sitemapConfig.xml.route, `${robots.baseUrl}/`).toString()];
-	} catch {
-		return [];
-	}
-}
-function appendRule(lines, rule) {
-	const userAgents = normalizeStringArray(rule.userAgent);
-	if (userAgents.length === 0) return;
-	lines.push("");
-	for (const userAgent of userAgents) lines.push(`User-agent: ${userAgent}`);
-	for (const route of normalizeStringArray(rule.allow)) lines.push(`Allow: ${normalizeRoute(route)}`);
-	for (const route of normalizeStringArray(rule.disallow)) lines.push(`Disallow: ${normalizeRoute(route)}`);
-	if (typeof rule.crawlDelay === "number" && Number.isFinite(rule.crawlDelay)) lines.push(`Crawl-delay: ${rule.crawlDelay}`);
-}
-function renderDocsRobotsTxt(options = {}) {
-	const robots = resolveDocsRobotsConfig(options.robots, { baseUrl: options.baseUrl });
-	if (!robots.enabled) return "";
-	const lines = [
-		"# Generated by @farming-labs/docs.",
-		"# Edit docs.config or rerun `docs robots generate --append` / `--force`.",
-		"",
-		"User-agent: *"
-	];
-	for (const route of getDocsRobotsAllowRoutes(options)) lines.push(`Allow: ${route}`);
-	for (const userAgent of robots.userAgents) {
-		lines.push("", `User-agent: ${userAgent}`);
-		lines.push(robots.ai === "allow" ? "Allow: /" : "Disallow: /");
-	}
-	for (const rule of robots.extraRules) appendRule(lines, rule);
-	const sitemap = sitemapUrls(robots, options.sitemap);
-	if (sitemap.length > 0) {
-		lines.push("");
-		for (const url of sitemap) lines.push(`Sitemap: ${url}`);
-	}
-	return `${lines.join("\n")}\n`;
-}
-function resolveDocsRobotsRequest(url, robots) {
-	const pathname = normalizeRoute(url.pathname);
-	const format = url.searchParams.get("format")?.trim();
-	if (pathname === "/api/docs" && format === "robots") return "robots";
-	if (!resolveDocsRobotsConfig(robots).enabled) return null;
-	return pathname === DEFAULT_ROBOTS_TXT_ROUTE ? "robots" : null;
-}
-function createDocsRobotsResponse({ request, ...options }) {
-	const url = new URL(request.url);
-	if (!resolveDocsRobotsRequest(url, options.robots)) return null;
-	const content = renderDocsRobotsTxt({
-		...options,
-		baseUrl: options.baseUrl ?? url.origin
-	});
-	if (!content) return null;
-	return new Response(content, { headers: {
-		"Content-Type": "text/plain; charset=utf-8",
-		"Cache-Control": "public, max-age=0, s-maxage=3600"
-	} });
-}
-function renderDocsRobotsGeneratedBlock(options = {}) {
-	return `${DOCS_ROBOTS_GENERATED_BLOCK_START}\n${renderDocsRobotsTxt(options).trimEnd()}\n${DOCS_ROBOTS_GENERATED_BLOCK_END}\n`;
-}
-function upsertDocsRobotsGeneratedBlock(existing, block) {
-	const escapedStart = DOCS_ROBOTS_GENERATED_BLOCK_START.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-	const escapedEnd = DOCS_ROBOTS_GENERATED_BLOCK_END.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-	const pattern = new RegExp(`${escapedStart}[\\s\\S]*?${escapedEnd}\\n?`);
-	if (pattern.test(existing)) return existing.replace(pattern, block);
-	const prefix = existing.trimEnd();
-	return `${prefix}${prefix ? "\n\n" : ""}${block}`;
-}
-function blockForUserAgent(content, userAgent) {
-	const escaped = userAgent.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-	return content.match(new RegExp(`(?:^|\\n)\\s*User-agent:\\s*${escaped}\\s*(?:\\n[ \\t]*(?!User-agent:).*)*`, "i"))?.[0] ?? "";
-}
-function disallowsRoute(content, route) {
-	const escapedRoute = route.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-	return new RegExp(`^\\s*Disallow:\\s*${escapedRoute}(?:\\s|$)`, "im").test(content);
-}
-function analyzeDocsRobotsTxt(content, options = {}) {
-	const expectedRoutes = getDocsRobotsAllowRoutes(options).filter((route) => [
-		DEFAULT_LLMS_TXT_ROUTE,
-		DEFAULT_LLMS_FULL_TXT_ROUTE,
-		DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE,
-		DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE,
-		DEFAULT_AGENTS_MD_ROUTE,
-		DEFAULT_SKILL_MD_ROUTE,
-		DEFAULT_MCP_PUBLIC_ROUTE
-	].includes(route) || route.includes("sitemap"));
-	const missingRoutes = expectedRoutes.filter((route) => !content.includes(route));
-	return {
-		blocksAgentRoutes: /Disallow:\s*\/(?:\s|$)/i.test(blockForUserAgent(content, "*")) || expectedRoutes.some((route) => disallowsRoute(content, route)),
-		blocksAiAgents: DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS.some((userAgent) => /Disallow:\s*\/(?:\s|$)/i.test(blockForUserAgent(content, userAgent))),
-		hasAgentRoutes: missingRoutes.length === 0,
-		hasAiPolicy: DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS.some((userAgent) => content.includes(userAgent)),
-		missingRoutes
-	};
-}
-
-//#endregion
-export { analyzeDocsRobotsTxt as a, renderDocsRobotsGeneratedBlock as c, resolveDocsRobotsRequest as d, upsertDocsRobotsGeneratedBlock as f, DOCS_ROBOTS_GENERATED_BLOCK_START as i, renderDocsRobotsTxt as l, DEFAULT_ROBOTS_TXT_ROUTE as n, createDocsRobotsResponse as o, DOCS_ROBOTS_GENERATED_BLOCK_END as r, getDocsRobotsAllowRoutes as s, DEFAULT_DOCS_AI_ROBOTS_USER_AGENTS as t, resolveDocsRobotsConfig as u };