docs-to-agent 0.1.0

package/LICENSE

@@ -0,0 +1,19 @@
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,149 @@
+# docs-to-agent
+
+<!-- automd:badges license color=yellow -->
+
+[![npm version](https://img.shields.io/npm/v/docs-to-agent?color=yellow)](https://npmjs.com/package/docs-to-agent)
+[![npm downloads](https://img.shields.io/npm/dm/docs-to-agent?color=yellow)](https://npm.chart.dev/docs-to-agent)
+[![license](https://img.shields.io/github/license/angelorc/docs-to-agent?color=yellow)](https://github.com/angelorc/docs-to-agent/blob/main/LICENSE)
+
+<!-- /automd -->
+
+Download docs from any GitHub repo and generate a compact index for AI coding agents.
+
+Inspired by the Next.js team's `agents-md` approach ([PR #88961](https://github.com/vercel/next.js/pull/88961)) which scored 100% on agent evals vs 79% for skill-based approaches.
+
+- **PR**: [vercel/next.js#88961](https://github.com/vercel/next.js/pull/88961) - the original implementation
+- **Article**: [`AGENTS.md` outperforms skills in our agent evals](https://vercel.com/blog/agents-md-outperforms-skills-in-our-agent-evals) - explains the eval methodology and why retrieval beats pre-training
+
+**This tool extracts that approach into a standalone CLI that works with any GitHub repo's docs** - not just Next.js.
+
+## Usage
+
+<!-- automd:pm-x args="https://github.com/nuxt/nuxt/tree/main/docs" -->
+
+```sh
+# npm
+npx docs-to-agent https://github.com/nuxt/nuxt/tree/main/docs
+
+# pnpm
+pnpm dlx docs-to-agent https://github.com/nuxt/nuxt/tree/main/docs
+
+# bun
+bunx docs-to-agent https://github.com/nuxt/nuxt/tree/main/docs
+
+# deno
+deno run -A npm:docs-to-agent https://github.com/nuxt/nuxt/tree/main/docs
+```
+
+<!-- /automd -->
+
+This will:
+
+1. Sparse-checkout only the docs folder
+2. Store it under `.docs-to-agent/nuxt-nuxt/`
+3. Generate a compact index in `AGENTS.md`
+4. Add `.docs-to-agent/` to `.gitignore`
+
+### Multiple repos
+
+Run it multiple times — each repo gets its own namespaced folder and keyed block in `AGENTS.md`:
+
+```bash
+npx docs-to-agent https://github.com/nuxt/nuxt/tree/main/docs
+npx docs-to-agent https://github.com/drizzle-team/drizzle-orm/tree/main/docs
+npx docs-to-agent https://github.com/shadcn-ui/ui/tree/main/apps/v4/content/docs
+```
+
+```
+.docs-to-agent/
+├── nuxt-nuxt/
+├── drizzle-team-drizzle-orm/
+└── shadcn-ui-ui/
+```
+
+### Options
+
+```
+-o, --output <file>   Target file (default: AGENTS.md)
+--name <name>         Project name override (default: repo name)
+```
+
+## Install
+
+<!-- automd:pm-install -->
+
+```sh
+# ✨ Auto-detect
+npx nypm install docs-to-agent
+
+# npm
+npm install docs-to-agent
+
+# yarn
+yarn add docs-to-agent
+
+# pnpm
+pnpm add docs-to-agent
+
+# bun
+bun install docs-to-agent
+
+# deno
+deno install npm:docs-to-agent
+```
+
+<!-- /automd -->
+
+## Programmatic Usage
+
+<!-- automd:jsimport src="./src/index.ts" -->
+
+**ESM** (Node.js, Bun, Deno)
+
+```js
+import {
+  DOCS_BASE_DIR,
+  cloneDocsFolder,
+  collectDocFiles,
+  ensureGitignoreEntry,
+  parseGitHubUrl,
+  pullDocs,
+  repoKey,
+  buildDocTree,
+  generateIndex,
+  injectIntoFile,
+} from "docs-to-agent";
+```
+
+<!-- /automd -->
+
+## Development
+
+```bash
+pnpm install
+pnpm run build       # obuild
+pnpm run lint        # oxlint + oxfmt --check
+pnpm run test        # vitest
+pnpm run typecheck   # tsc --noEmit
+```
+
+## License
+
+<!-- automd:contributors license=MIT author="angelorc" -->
+
+Published under the [MIT](https://github.com/angelorc/docs-to-agent/blob/main/LICENSE) license.
+Made by [@angelorc](https://github.com/angelorc) and [community](https://github.com/angelorc/docs-to-agent/graphs/contributors) 💛
+<br><br>
+<a href="https://github.com/angelorc/docs-to-agent/graphs/contributors">
+<img src="https://contrib.rocks/image?repo=angelorc/docs-to-agent" />
+</a>
+
+<!-- /automd -->
+
+<!-- automd:with-automd -->
+
+---
+
+_🤖 auto updated with [automd](https://automd.unjs.io)_
+
+<!-- /automd -->

package/dist/_chunks/content.mjs

@@ -0,0 +1,181 @@
+import { execSync } from "node:child_process";
+import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+const DOCS_BASE_DIR = ".docs-to-agent";
+function repoKey(owner, repo) {
+	return `${owner}-${repo}`;
+}
+function parseGitHubUrl(url) {
+	const treeMatch = url.match(/^https?:\/\/github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/(.+?)\/?\s*$/);
+	if (treeMatch) return {
+		owner: treeMatch[1],
+		repo: treeMatch[2],
+		branch: treeMatch[3],
+		docsPath: treeMatch[4]
+	};
+	const repoMatch = url.match(/^https?:\/\/github\.com\/([^/]+)\/([^/]+?)(\.git)?\/?$/);
+	if (repoMatch) throw new Error(`URL must include a docs path. Example: https://github.com/${repoMatch[1]}/${repoMatch[2]}/tree/main/docs`);
+	throw new Error("Invalid GitHub URL. Expected format: https://github.com/owner/repo/tree/branch/docs-path");
+}
+function copyDirRecursive(src, dest) {
+	mkdirSync(dest, { recursive: true });
+	const entries = readdirSync(src, { withFileTypes: true });
+	for (const entry of entries) {
+		const srcPath = join(src, entry.name);
+		const destPath = join(dest, entry.name);
+		if (entry.isDirectory()) copyDirRecursive(srcPath, destPath);
+		else copyFileSync(srcPath, destPath);
+	}
+}
+function cloneDocsFolder(repoUrl, branch, docsPath, destDir) {
+	const tmpDir = destDir + "-tmp";
+	if (existsSync(tmpDir)) rmSync(tmpDir, {
+		recursive: true,
+		force: true
+	});
+	mkdirSync(tmpDir, { recursive: true });
+	try {
+		execSync(`git clone --depth 1 --filter=blob:none --sparse --branch ${branch} ${repoUrl} .`, {
+			cwd: tmpDir,
+			stdio: "pipe"
+		});
+		execSync(`git sparse-checkout set ${docsPath}`, {
+			cwd: tmpDir,
+			stdio: "pipe"
+		});
+		const srcDir = join(tmpDir, docsPath);
+		if (!existsSync(srcDir)) throw new Error(`Docs path "${docsPath}" not found in repo. Check the URL path.`);
+		if (existsSync(destDir)) rmSync(destDir, {
+			recursive: true,
+			force: true
+		});
+		copyDirRecursive(srcDir, destDir);
+	} finally {
+		rmSync(tmpDir, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+function pullDocs(options) {
+	const repoUrl = `https://github.com/${options.owner}/${options.repo}.git`;
+	const key = repoKey(options.owner, options.repo);
+	const localDocsDir = join(options.cwd, DOCS_BASE_DIR, key);
+	cloneDocsFolder(repoUrl, options.branch, options.docsPath, localDocsDir);
+	const files = collectDocFiles(localDocsDir);
+	return {
+		localDocsDir: `${DOCS_BASE_DIR}/${key}`,
+		fileCount: files.length
+	};
+}
+function collectDocFiles(dir) {
+	const files = [];
+	function walk(currentDir) {
+		if (!existsSync(currentDir)) return;
+		const entries = readdirSync(currentDir, { withFileTypes: true });
+		for (const entry of entries) {
+			const fullPath = join(currentDir, entry.name);
+			if (entry.isDirectory()) walk(fullPath);
+			else if (/\.(md|mdx)$/i.test(entry.name)) {
+				if (/^index\.(md|mdx)$/i.test(entry.name)) continue;
+				files.push({ relativePath: relative(dir, fullPath) });
+			}
+		}
+	}
+	walk(dir);
+	return files.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+}
+function ensureGitignoreEntry(cwd, dirName) {
+	const gitignorePath = join(cwd, ".gitignore");
+	const entry = `${dirName}/`;
+	if (existsSync(gitignorePath)) {
+		const content = readFileSync(gitignorePath, "utf-8");
+		if (content.split("\n").some((line) => line.trim() === entry || line.trim() === dirName)) return {
+			path: gitignorePath,
+			updated: false,
+			alreadyPresent: true
+		};
+		appendFileSync(gitignorePath, `${content.endsWith("\n") ? "" : "\n"}${entry}\n`);
+		return {
+			path: gitignorePath,
+			updated: true,
+			alreadyPresent: false
+		};
+	}
+	writeFileSync(gitignorePath, `${entry}\n`);
+	return {
+		path: gitignorePath,
+		updated: true,
+		alreadyPresent: false
+	};
+}
+function buildDocTree(files) {
+	const sectionMap = /* @__PURE__ */ new Map();
+	for (const file of files) {
+		const parts = file.relativePath.split("/");
+		if (parts.length < 2) continue;
+		const topDir = parts[0];
+		if (!sectionMap.has(topDir)) sectionMap.set(topDir, {
+			files: [],
+			children: /* @__PURE__ */ new Map()
+		});
+		const section = sectionMap.get(topDir);
+		if (parts.length === 2) section.files.push(parts[1]);
+		else {
+			const subDir = parts.slice(1, -1).join("/");
+			if (!section.children.has(subDir)) section.children.set(subDir, []);
+			section.children.get(subDir).push(file);
+		}
+	}
+	const sections = [];
+	for (const [name, data] of sectionMap) {
+		const subsections = [];
+		for (const [subName, subFiles] of data.children) subsections.push({
+			name: subName,
+			files: subFiles.map((f) => {
+				const parts = f.relativePath.split("/");
+				return parts[parts.length - 1];
+			}),
+			subsections: []
+		});
+		subsections.sort((a, b) => a.name.localeCompare(b.name));
+		sections.push({
+			name,
+			files: [...data.files].sort(),
+			subsections
+		});
+	}
+	sections.sort((a, b) => a.name.localeCompare(b.name));
+	return sections;
+}
+function generateIndex(data) {
+	const parts = [];
+	parts.push(`[${data.name} Docs Index]|root: ./${data.docsDir}|IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning for any ${data.name} tasks.`);
+	for (const section of data.sections) {
+		const sectionPrefix = section.name;
+		if (section.files.length > 0) parts.push(`${sectionPrefix}:{${section.files.join(",")}}`);
+		for (const sub of section.subsections) {
+			const subPrefix = `${sectionPrefix}/${sub.name}`;
+			if (sub.files.length > 0) parts.push(`${subPrefix}:{${sub.files.join(",")}}`);
+		}
+	}
+	return parts.join("|");
+}
+function markerStart(key) {
+	return `<!-- DOCS-TO-AGENT:${key}-START -->`;
+}
+function markerEnd(key) {
+	return `<!-- DOCS-TO-AGENT:${key}-END -->`;
+}
+function injectIntoFile(content, indexContent, key) {
+	const start = markerStart(key);
+	const end = markerEnd(key);
+	const block = `${start}\n${indexContent}\n${end}`;
+	const startIdx = content.indexOf(start);
+	const endIdx = content.indexOf(end);
+	if (startIdx !== -1 && endIdx !== -1) return content.slice(0, startIdx) + block + content.slice(endIdx + end.length);
+	if (content.length > 0 && !content.endsWith("\n")) return content + "\n\n" + block + "\n";
+	if (content.length > 0) return content + "\n" + block + "\n";
+	return block + "\n";
+}
+export { cloneDocsFolder as a, parseGitHubUrl as c, DOCS_BASE_DIR as i, pullDocs as l, generateIndex as n, collectDocFiles as o, injectIntoFile as r, ensureGitignoreEntry as s, buildDocTree as t, repoKey as u };

package/dist/bin/cli.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/bin/cli.mjs

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+import { c as parseGitHubUrl, i as DOCS_BASE_DIR, l as pullDocs, n as generateIndex, o as collectDocFiles, r as injectIntoFile, s as ensureGitignoreEntry, t as buildDocTree, u as repoKey } from "../_chunks/content.mjs";
+import "../index.mjs";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { Command } from "commander";
+import pc from "picocolors";
+const program = new Command();
+program.name("docs-to-agent").description("Download docs from a GitHub repo and generate a compact index for AI coding agents").argument("<github-url>", "GitHub URL with docs path (e.g. https://github.com/nuxt/nuxt/tree/main/docs)").option("-o, --output <file>", "Target file", "AGENTS.md").option("--name <name>", "Project name override (defaults to repo name)").action(async (url, opts) => {
+	try {
+		console.log(pc.cyan("Parsing GitHub URL..."));
+		const parsed = parseGitHubUrl(url);
+		const projectName = opts.name || parsed.repo;
+		const key = repoKey(parsed.owner, parsed.repo);
+		console.log(pc.dim(`  repo: ${parsed.owner}/${parsed.repo}, branch: ${parsed.branch}, path: ${parsed.docsPath}`));
+		const cwd = process.cwd();
+		console.log(pc.cyan("Downloading documentation..."));
+		const result = pullDocs({
+			owner: parsed.owner,
+			repo: parsed.repo,
+			branch: parsed.branch,
+			docsPath: parsed.docsPath,
+			cwd
+		});
+		console.log(pc.green(`  Downloaded ${result.fileCount} doc files → ${result.localDocsDir}/`));
+		const files = collectDocFiles(join(cwd, result.localDocsDir));
+		if (files.length === 0) {
+			console.log(pc.yellow("No .md/.mdx files found in docs folder."));
+			process.exit(1);
+		}
+		const sections = buildDocTree(files);
+		const indexContent = generateIndex({
+			name: projectName,
+			docsDir: result.localDocsDir,
+			sections
+		});
+		const outputPath = resolve(cwd, opts.output);
+		let existingContent = "";
+		if (existsSync(outputPath)) existingContent = readFileSync(outputPath, "utf-8");
+		writeFileSync(outputPath, injectIntoFile(existingContent, indexContent, key));
+		console.log(pc.green(`  Updated ${opts.output}`));
+		if (ensureGitignoreEntry(cwd, DOCS_BASE_DIR).updated) console.log(pc.dim(`  Added ${DOCS_BASE_DIR}/ to .gitignore`));
+		console.log();
+		console.log(pc.green(pc.bold("Done!")));
+		console.log(pc.dim(`  ${files.length} docs indexed → ${opts.output} [${key}]`));
+	} catch (err) {
+		const msg = err instanceof Error ? err.message : String(err);
+		console.error(pc.red(`Error: ${msg}`));
+		process.exit(1);
+	}
+});
+program.parse();
+export {};

package/dist/index.d.mts

@@ -0,0 +1,51 @@
+//#region src/types.d.ts
+interface GitHubUrl {
+  owner: string;
+  repo: string;
+  branch: string;
+  docsPath: string;
+}
+interface DocFile {
+  relativePath: string;
+}
+interface DocSection {
+  name: string;
+  files: string[];
+  subsections: DocSection[];
+}
+interface PullDocsOptions {
+  owner: string;
+  repo: string;
+  branch: string;
+  docsPath: string;
+  cwd: string;
+}
+interface PullDocsResult {
+  localDocsDir: string;
+  fileCount: number;
+}
+interface IndexData {
+  name: string;
+  docsDir: string;
+  sections: DocSection[];
+}
+//#endregion
+//#region src/utils.d.ts
+declare const DOCS_BASE_DIR = ".docs-to-agent";
+declare function repoKey(owner: string, repo: string): string;
+declare function parseGitHubUrl(url: string): GitHubUrl;
+declare function cloneDocsFolder(repoUrl: string, branch: string, docsPath: string, destDir: string): void;
+declare function pullDocs(options: PullDocsOptions): PullDocsResult;
+declare function collectDocFiles(dir: string): DocFile[];
+declare function ensureGitignoreEntry(cwd: string, dirName: string): {
+  path: string;
+  updated: boolean;
+  alreadyPresent: boolean;
+};
+//#endregion
+//#region src/content.d.ts
+declare function buildDocTree(files: DocFile[]): DocSection[];
+declare function generateIndex(data: IndexData): string;
+declare function injectIntoFile(content: string, indexContent: string, key: string): string;
+//#endregion
+export { DOCS_BASE_DIR, type DocFile, type DocSection, type GitHubUrl, type IndexData, type PullDocsOptions, type PullDocsResult, buildDocTree, cloneDocsFolder, collectDocFiles, ensureGitignoreEntry, generateIndex, injectIntoFile, parseGitHubUrl, pullDocs, repoKey };

package/dist/index.mjs

@@ -0,0 +1,2 @@
+import { a as cloneDocsFolder, c as parseGitHubUrl, i as DOCS_BASE_DIR, l as pullDocs, n as generateIndex, o as collectDocFiles, r as injectIntoFile, s as ensureGitignoreEntry, t as buildDocTree, u as repoKey } from "./_chunks/content.mjs";
+export { DOCS_BASE_DIR, buildDocTree, cloneDocsFolder, collectDocFiles, ensureGitignoreEntry, generateIndex, injectIntoFile, parseGitHubUrl, pullDocs, repoKey };

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "docs-to-agent",
+  "version": "0.1.0",
+  "description": "Download docs from any GitHub repo and generate a compact index for AI coding agents",
+  "homepage": "https://github.com/angelorc/docs-to-agent",
+  "bugs": "https://github.com/angelorc/docs-to-agent/issues",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/angelorc/docs-to-agent.git"
+  },
+  "bin": {
+    "docs-to-agent": "./dist/bin/cli.mjs"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    }
+  },
+  "dependencies": {
+    "commander": "^14.0.3",
+    "picocolors": "^1.1.1"
+  },
+  "devDependencies": {
+    "@changesets/cli": "^2.29.4",
+    "@types/node": "^25.2.3",
+    "automd": "^0.4.3",
+    "obuild": "^0.4.27",
+    "oxfmt": "^0.31.0",
+    "oxlint": "^1.46.0",
+    "typescript": "^5.7.3",
+    "vitest": "^4.0.18"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "obuild",
+    "dev": "obuild --stub",
+    "lint": "oxlint . && oxfmt --check .",
+    "lint:fix": "automd && oxlint . --fix && oxfmt .",
+    "fmt": "oxfmt .",
+    "fmt:check": "oxfmt --check .",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit",
+    "changeset": "changeset",
+    "version": "changeset version",
+    "release": "changeset version && git add . && git commit -m 'chore(release): version packages' && changeset publish && git push --follow-tags"
+  }
+}