for-humanity 0.5.0 → 0.6.0

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+## 0.6.0 — 2026-10-04
+
+- Generate `robots.txt` and `sitemap.xml` in dev and static builds when an optional public root `url` is configured; include home and encoded document URLs, and leave projects without a URL unchanged.
+- Validate public HTTP(S) origins and reject crawler-file document path collisions before processing; document Cloudflare crawler policy and Markdown negotiation separately.
+
 ## 0.5.0 — 2026-10-04
 
 - Place document loading, Markdown processing and server setup under the CLI owner; keep document tree construction under navigation and share a standalone shell-controls widget through a lifecycle-managed Hono browser root.

package/README.md

@@ -33,7 +33,7 @@ Write Markdown. Keep the context. Share a static site.
 Node.js 22 or later and pnpm. Install the CLI in your document project:
 
 ```sh
-pnpm add -D --save-exact for-humanity@0.5.0
+pnpm add -D --save-exact for-humanity@0.6.0
 mkdir docs
 ```
 
@@ -52,8 +52,9 @@ pnpm exec for-humanity preview docs
 
 If your Markdown files are in the project root, use `.` instead of `docs`. The root `README.md` becomes the home page; root `AGENTS.md` and `CLAUDE.md` stay out of the site.
 
-Version `0.5.0` supports `dev`, `build` and `preview`. `init` is planned; see the [Roadmap](https://github.com/minu-ha/for-humanity/blob/main/templates/document/roadmap.md) and [Changelog](https://github.com/minu-ha/for-humanity/blob/main/CHANGELOG.md).
+Version `0.6.0` supports `dev`, `build` and `preview`. `init` is planned; see the [Roadmap](https://github.com/minu-ha/for-humanity/blob/main/templates/document/roadmap.md) and [Changelog](https://github.com/minu-ha/for-humanity/blob/main/CHANGELOG.md).
 Cloudflare Pages can host the generated site. See [Deployment](https://github.com/minu-ha/for-humanity/blob/main/templates/document/deployment.md) for the build settings.
+Set an optional public root `url` in the site configuration to generate `robots.txt` and `sitemap.xml`; projects without a URL emit neither file. See [Crawling](https://github.com/minu-ha/for-humanity/blob/main/templates/document/crawling.md).
 
 ### Develop the kit
 
@@ -74,7 +75,7 @@ pnpm preview    # Preview the static site
 pnpm build:pages # Add Cloudflare Pages headers and a 404 page
 ```
 
-Changes pushed to `main` update the documentation site. A stable version tag such as `v0.5.0` runs checks, publishes the package to npm, then creates a GitHub Release with the same package file.
+Changes pushed to `main` update the documentation site. A stable version tag such as `v0.6.0` runs checks, publishes the package to npm, then creates a GitHub Release with the same package file.
 The npm README updates with each package release. See [Maintenance](https://github.com/minu-ha/for-humanity/blob/main/templates/document/maintenance.md#packaging-and-release) for the release steps.
 
 ## Make it yours

package/dist/cli.js

@@ -9,7 +9,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { serve } from "@hono/node-server";
 import { serveStatic } from "@hono/node-server/serve-static";
 import { debounce } from "es-toolkit";
-import { Hono as Hono3 } from "hono";
+import { Hono as Hono4 } from "hono";
 import { toSSG } from "hono/ssg";
 
 // src/constant/asset.ts
@@ -51,6 +51,7 @@ var copy_error_prefix = "for-humanity";
 var copy_error_font_preload = "Missing bundled font for preload";
 var copy_error_unknown_command = "\uBAA8\uB974\uB294 \uBA85\uB839\uC774\uB2E4";
 var copy_error_config = "\uC124\uC815\uC774 \uD2C0\uB838\uB2E4";
+var copy_error_site_url = "url\uC740 \uC0AC\uC6A9\uC790 \uC815\uBCF4\xB7\uD558\uC704 \uACBD\uB85C\xB7query\xB7hash\uAC00 \uC5C6\uB294 HTTP(S) \uC0AC\uC774\uD2B8 \uB8E8\uD2B8\uC5EC\uC57C \uD55C\uB2E4";
 var copy_error_frontmatter = "frontmatter\uAC00 \uC62C\uBC14\uB974\uC9C0 \uC54A\uB2E4";
 var copy_error_navigation_clash = "navigation\uC758 \uBB36\uC74C\uC774 \uACB9\uCE5C\uB2E4";
 var copy_error_doc_id = "\uBB38\uC11C id\uC5D0 \uC0AC\uC6A9\uD560 \uC218 \uC5C6\uB294 \uACBD\uB85C";
@@ -85,6 +86,11 @@ var cli_config_file_name = "for-humanity.config.mjs";
 var cli_dev_port = 4321;
 var cli_reload_delay_ms = 150;
 
+// src/entry/cli/_constant/crawling.ts
+var crawling_robots_path = "/robots.txt";
+var crawling_sitemap_path = "/sitemap.xml";
+var crawling_reserved_roots = [crawling_robots_path.slice(1), crawling_sitemap_path.slice(1)];
+
 // src/entry/cli/_function/create-app.tsx
 import { Hono } from "hono";
 import { html } from "hono/html";
@@ -1227,6 +1233,9 @@ var createDevApp = (options) => {
   );
   app.get(`/${asset_dir}/*`, handleAsset);
   app.get(asset_favicon_path, handleAsset);
+  if (options.crawling !== void 0) {
+    app.on("GET", [crawling_robots_path, crawling_sitemap_path], (c) => options.crawling?.fetch(c.req.raw, c.env) ?? c.notFound());
+  }
   app.all("/*", (c) => options.pages.fetch(c.req.raw, c.env));
   return app;
 };
@@ -1995,7 +2004,7 @@ var readDocs = async (options) => {
     paths.map(async (path) => {
       const id = path.replace(/\.mdx?$/i, "").split(sep2).join("/").toLowerCase();
       const segments = id.split("/");
-      if (segments.some((segment) => segment === "" || segment === "." || segment === ".." || segment === "index.html") || /[#?%*:\\]/.test(id) || /[\p{Cc}]/u.test(id) || segments[0] === asset_dir || segments[0] === asset_favicon_path.slice(1)) {
+      if (segments.some((segment) => segment === "" || segment === "." || segment === ".." || segment === "index.html") || /[#?%*:\\]/.test(id) || /[\p{Cc}]/u.test(id) || options.reserved?.includes(segments[0]) === true || segments[0] === asset_dir || segments[0] === asset_favicon_path.slice(1)) {
         throw new Error(`${copy_error_doc_id}: ${path}`);
       }
       const source = await readFile2(join(options.root, path), "utf8");
@@ -2067,6 +2076,30 @@ var readHome = async (options) => {
   return { html: String(file), outline: { headings: fh.headings, sections: fh.sections } };
 };
 
+// src/entry/cli/_function/to-crawling-app.ts
+import { Hono as Hono3 } from "hono";
+import { html as html2 } from "hono/html";
+var toCrawlingApp = (params) => {
+  const app = new Hono3();
+  app.get(crawling_robots_path, (c) => c.text(`User-agent: *
+Allow: /
+
+Sitemap: ${params.url}${crawling_sitemap_path}
+`));
+  app.get(
+    crawling_sitemap_path,
+    (_c) => new Response(
+      html2`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<url><loc>${params.url}/</loc></url>
+${params.store.docs.map((doc) => html2`<url><loc>${params.url}/${doc.id.split("/").map(encodeURIComponent).join("/")}/</loc></url>`)}
+</urlset>`.toString(),
+      { headers: { "Content-Type": "application/xml; charset=utf-8" } }
+    )
+  );
+  return app;
+};
+
 // src/type/site-config.ts
 import { z as z3 } from "zod";
 
@@ -2090,6 +2123,10 @@ var siteConfigSchema = z3.object({
    * HTML 설명 메타데이터 · 선택
    */
   description: z3.string().optional(),
+  /**
+   * 공개 사이트 루트 URL · 지정할 때만 robots.txt와 sitemap.xml 생성
+   */
+  url: z3.url({ protocol: /^https?$/ }).transform((value) => new URL(value)).refine((value) => value.username === "" && value.password === "" && value.pathname === "/" && value.search === "" && value.hash === "", copy_error_site_url).transform((value) => value.origin).optional(),
   /**
    * 탐색 묶음 경로의 읽는 순서 · 미지정 형제 묶음은 뒤에서 이름순
    */
@@ -2179,7 +2216,7 @@ var main = async () => {
   const docsRoot = resolve3(docsDir);
   const outDir = join4(docsRoot, "dist");
   if (command === "preview") {
-    const app2 = new Hono3();
+    const app2 = new Hono4();
     app2.use(
       "/*",
       serveStatic({
@@ -2219,8 +2256,12 @@ var main = async () => {
   ]);
   const files = new Map(kitFiles);
   const processor = createProcessor({ site: siteConfig, root: docsRoot, files });
-  const [docs, home] = await Promise.all([readDocs({ root: docsRoot, processor }), readHome({ root: docsRoot, processor })]);
+  const [docs, home] = await Promise.all([
+    readDocs({ root: docsRoot, processor, reserved: siteConfig.url === void 0 ? void 0 : crawling_reserved_roots }),
+    readHome({ root: docsRoot, processor })
+  ]);
   const store = { docs, home };
+  const crawling = siteConfig.url === void 0 ? void 0 : toCrawlingApp({ url: siteConfig.url, store });
   const app = createApp({
     site: siteConfig,
     store,
@@ -2232,23 +2273,32 @@ var main = async () => {
     if (!result.success) {
       throw result.error;
     }
+    const crawlerResult = crawling === void 0 ? void 0 : await toSSG(crawling, { writeFile, mkdir }, { dir: outDir });
+    if (crawlerResult?.success === false) {
+      throw crawlerResult.error;
+    }
     for (const [url, file] of files) {
       await mkdir(dirname4(join4(outDir, url)), { recursive: true });
       await copyFile(file, join4(outDir, url));
     }
-    console.log(`${result.files.length} pages, ${files.size} assets \u2192 ${relative4(process.cwd(), outDir)}`);
+    console.log(
+      `${crawlerResult === void 0 ? result.files.length : result.files.length + crawlerResult.files.length} files, ${files.size} assets \u2192 ${relative4(process.cwd(), outDir)}`
+    );
     return;
   }
   const reload = new EventEmitter();
   let generation = 0;
-  serve({ fetch: createDevApp({ pages: app, files, reload }).fetch, port: cli_dev_port }, (info) => console.log(`http://localhost:${info.port}/`));
+  serve({ fetch: createDevApp({ pages: app, crawling, files, reload }).fetch, port: cli_dev_port }, (info) => console.log(`http://localhost:${info.port}/`));
   const reloadDocs = debounce(
     toChangeQueue({
       run: async () => {
         const currentGeneration = generation;
         const nextFiles = new Map(kitFiles);
         const nextProcessor = createProcessor({ site: siteConfig, root: docsRoot, files: nextFiles });
-        const [docs2, home2] = await Promise.all([readDocs({ root: docsRoot, processor: nextProcessor }), readHome({ root: docsRoot, processor: nextProcessor })]);
+        const [docs2, home2] = await Promise.all([
+          readDocs({ root: docsRoot, processor: nextProcessor, reserved: siteConfig.url === void 0 ? void 0 : crawling_reserved_roots }),
+          readHome({ root: docsRoot, processor: nextProcessor })
+        ]);
         if (currentGeneration !== generation) {
           return;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "for-humanity",
-    "version": "0.5.0",
+    "version": "0.6.0",
     "description": "Documents written for humans to read: a docs kit with a sidebar, parts, sections and flowcharts, built on Hono JSX",
     "keywords": [
         "docs",

package/src/constant/copy.ts

@@ -62,6 +62,11 @@ export const copy_error_unknown_command = "모르는 명령이다";
  */
 export const copy_error_config = "설정이 틀렸다";
 
+/**
+ * 공개 사이트 URL의 미지원 경로·자격 증명·검색·앵커 오류
+ */
+export const copy_error_site_url = "url은 사용자 정보·하위 경로·query·hash가 없는 HTTP(S) 사이트 루트여야 한다";
+
 /**
  * 문서 frontmatter 오류 · 파일과 항목 추가
  */

package/src/entry/cli/_constant/crawling.ts

@@ -0,0 +1,14 @@
+/**
+ * 크롤러 안내 파일의 사이트 루트 경로
+ */
+export const crawling_robots_path = "/robots.txt";
+
+/**
+ * 공개 페이지 목록의 사이트 루트 경로
+ */
+export const crawling_sitemap_path = "/sitemap.xml";
+
+/**
+ * 공개 크롤러 파일과 정적 출력이 충돌하는 문서의 첫 경로 조각
+ */
+export const crawling_reserved_roots = [crawling_robots_path.slice(1), crawling_sitemap_path.slice(1)];

package/src/entry/cli/_function/create-dev-app.ts

@@ -5,6 +5,7 @@ import {type Context, Hono} from "hono";
 import {streamSSE} from "hono/streaming";
 import {asset_content_type_default, asset_content_types, asset_dir, asset_favicon_path, asset_reload_path} from "@/constant/asset";
 import {font_cache_control} from "@/constant/font";
+import {crawling_robots_path, crawling_sitemap_path} from "@/entry/cli/_constant/crawling";
 
 /**
  * 개발 서버 입력 계약
@@ -14,6 +15,10 @@ export interface DevOptions {
      * 정적 자원 외 요청을 처리할 페이지 앱
      */
     pages: Hono;
+    /**
+     * 공개 URL을 설정한 프로젝트의 크롤러 파일 앱
+     */
+    crawling?: Hono;
     /**
      * 자원 URL → 파일 경로 · 정적 빌드와 동일 목록
      */
@@ -25,7 +30,7 @@ export interface DevOptions {
 }
 
 /**
- * 개발 전용 자원·SSE 라우트 · 나머지 요청은 페이지 앱에 위임
+ * 개발 전용 자원·SSE와 선택적 크롤러 응답 · 나머지 요청은 페이지 앱에 위임
  * 정적 빌드의 페이지 라우트와 분리
  */
 export const createDevApp = (options: DevOptions) => {
@@ -73,6 +78,9 @@ export const createDevApp = (options: DevOptions) => {
     );
     app.get(`/${asset_dir}/*`, handleAsset);
     app.get(asset_favicon_path, handleAsset);
+    if (options.crawling !== undefined) {
+        app.on("GET", [crawling_robots_path, crawling_sitemap_path], (c) => options.crawling?.fetch(c.req.raw, c.env) ?? c.notFound());
+    }
     app.all("/*", (c) => options.pages.fetch(c.req.raw, c.env));
 
     return app;

package/src/entry/cli/_function/read-docs.ts

@@ -16,12 +16,30 @@ import type {Doc} from "@/type/doc";
 import {docDataSchema} from "@/type/doc-data";
 import type {DocFileData} from "@/type/doc-file-data";
 
+/**
+ * 문서 폴더를 읽고 명령별 출력 경로와 함께 검증하는 입력
+ */
+export interface ReadDocsParams {
+    /**
+     * CLI에 넘긴 문서 폴더의 절대 경로
+     */
+    root: string;
+    /**
+     * 설정·자원 목록을 연결한 Markdown 처리기
+     */
+    processor: Processor;
+    /**
+     * 명령이 생성할 파일과 충돌하는 첫 경로 조각 · 생략 시 추가 예약 없음
+     */
+    reserved?: readonly string[];
+}
+
 /**
  * Markdown 수집과 렌더링 · 루트 README는 홈, AGENTS·CLAUDE는 작성 지침으로 분리
  * 문서 목록에서 루트 안내 파일·node_modules·dist 제외
  * 머리말·문서 식별자 중복·플러그인 오류 시 실패
  */
-export const readDocs = async (options: {root: string; processor: Processor}): Promise<Doc[]> => {
+export const readDocs = async (options: ReadDocsParams): Promise<Doc[]> => {
     const paths = (await readdir(options.root, {recursive: true})).filter(
         (path) => /\.mdx?$/i.test(path) && !/^(readme|agents|claude)\.md$/i.test(path) && !path.split(sep).includes("node_modules") && !path.startsWith(`dist${sep}`),
     );
@@ -39,6 +57,7 @@ export const readDocs = async (options: {root: string; processor: Processor}): P
                 segments.some((segment) => segment === "" || segment === "." || segment === ".." || segment === "index.html") ||
                 /[#?%*:\\]/.test(id) ||
                 /[\p{Cc}]/u.test(id) ||
+                options.reserved?.includes(segments[0]) === true ||
                 segments[0] === asset_dir ||
                 segments[0] === asset_favicon_path.slice(1)
             ) {

package/src/entry/cli/_function/to-crawling-app.ts

@@ -0,0 +1,42 @@
+import {Hono} from "hono";
+import {html} from "hono/html";
+import {crawling_robots_path, crawling_sitemap_path} from "@/entry/cli/_constant/crawling";
+import type {AppOptions} from "@/entry/cli/_function/create-app";
+import type {SiteConfig} from "@/type/site-config";
+
+/**
+ * URL을 명시한 공개 프로젝트의 크롤러 응답 입력
+ */
+export interface CrawlingParams {
+    /**
+     * 설정 검증에서 루트 origin으로 정규화한 공개 HTTP(S) URL
+     */
+    url: NonNullable<SiteConfig["url"]>;
+    /**
+     * dev 파일 변경 시 교체되는 현재 문서 목록 · 본문은 사이트맵에 포함하지 않음
+     */
+    store: AppOptions["store"];
+}
+
+/**
+ * dev와 정적 빌드에서 같은 크롤러 파일 생성 · 정책·수정 시각은 추정하지 않음
+ */
+export const toCrawlingApp = (params: CrawlingParams) => {
+    const app = new Hono();
+
+    app.get(crawling_robots_path, (c) => c.text(`User-agent: *\nAllow: /\n\nSitemap: ${params.url}${crawling_sitemap_path}\n`));
+    app.get(
+        crawling_sitemap_path,
+        (_c) =>
+            new Response(
+                html`<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<url><loc>${params.url}/</loc></url>
+${params.store.docs.map((doc) => html`<url><loc>${params.url}/${doc.id.split("/").map(encodeURIComponent).join("/")}/</loc></url>`)}
+</urlset>`.toString(),
+                {headers: {"Content-Type": "application/xml; charset=utf-8"}},
+            ),
+    );
+
+    return app;
+};

package/src/entry/cli/cli.test.ts

@@ -1,8 +1,10 @@
 import assert from "node:assert/strict";
-import {mkdir, mkdtemp, rm, symlink, writeFile} from "node:fs/promises";
+import {EventEmitter} from "node:events";
+import {mkdir, mkdtemp, readFile, rm, symlink, writeFile} from "node:fs/promises";
 import {tmpdir} from "node:os";
 import {join} from "node:path";
 import {test} from "node:test";
+import {toSSG} from "hono/ssg";
 import {navigationDataSchema} from "@/component/widget/navigation/_constant/navigation-data-schema";
 import {toDocGroups} from "@/component/widget/navigation/_function/to-doc-groups/to-doc-groups";
 import {toNavigationData} from "@/component/widget/navigation/_function/to-navigation-data/to-navigation-data";
@@ -10,9 +12,11 @@ import {toNavigationJson} from "@/component/widget/navigation/_function/to-navig
 import {WgNavigation} from "@/component/widget/navigation/wg-navigation";
 import {asset_style_path} from "@/constant/asset";
 import {createApp} from "@/entry/cli/_function/create-app";
+import {createDevApp} from "@/entry/cli/_function/create-dev-app";
 import {createProcessor} from "@/entry/cli/_function/create-processor/create-processor";
 import {readDocs} from "@/entry/cli/_function/read-docs";
 import {readHome} from "@/entry/cli/_function/read-home";
+import {toCrawlingApp} from "@/entry/cli/_function/to-crawling-app";
 import {docDataSchema} from "@/type/doc-data";
 import {siteConfigSchema} from "@/type/site-config";
 
@@ -54,6 +58,107 @@ test("page resource URLs use the supplied content fingerprints instead of fixed
     assert.doesNotMatch(html, /(?:href|src)="\/_fh\/(?:style\.css|client\.js)"/);
 });
 
+test("public site URL is opt-in and normalizes an HTTP(S) root URL", () => {
+    assert.deepEqual(siteConfigSchema.parse({url: "https://docs.example.org/"}), {...siteConfigSchema.parse({}), url: "https://docs.example.org"});
+    assert.deepEqual(siteConfigSchema.parse({url: "http://localhost:4321"}), {...siteConfigSchema.parse({}), url: "http://localhost:4321"});
+    assert.equal("url" in siteConfigSchema.parse({}), false);
+});
+
+test("public site URL rejects unsupported protocols, credentials and non-root URLs", () => {
+    for (const url of [
+        "",
+        "/",
+        "file:///docs",
+        "ftp://docs.example.org",
+        "https://user:password@docs.example.org",
+        "https://docs.example.org/docs/",
+        "https://docs.example.org/?q=guide",
+        "https://docs.example.org/#guide",
+    ]) {
+        assert.equal(siteConfigSchema.safeParse({url}).success, false, url);
+    }
+});
+
+test("crawler files are absent when the public site URL is omitted", async () => {
+    const site = siteConfigSchema.parse({});
+    const assets = {fontCss: "", preload: [], reload: false, browserScript: "", stylePath: asset_style_path};
+    const app = createDevApp({pages: createApp({site, store: {docs: []}, assets}), files: new Map(), reload: new EventEmitter()});
+
+    assert.equal((await app.request("/robots.txt")).status, 404);
+    assert.equal((await app.request("/sitemap.xml")).status, 404);
+});
+
+test("crawler files list home and encoded nested document URLs with the current document store", async (t) => {
+    const root = await mkdtemp(join(tmpdir(), "for-humanity-crawling-"));
+    t.after(() => rm(root, {recursive: true, force: true}));
+    await mkdir(join(root, "guide"));
+    await writeFile(join(root, "guide.md"), "---\nname: Start\nlabel: Start\ngroup: Guide\n---\n\nStart here.\n");
+    await writeFile(join(root, "guide/한국어 & tools.md"), "---\nname: Guide\nlabel: Guide\ngroup: Guide\n---\n\n## Read\n\nText.\n");
+    const site = siteConfigSchema.parse({url: "https://docs.example.org/"});
+    const docs = await readDocs({root, processor: createProcessor({site, root})});
+    const store = {docs};
+    const assets = {fontCss: "", preload: [], reload: false, browserScript: "", stylePath: asset_style_path};
+    assert.ok(site.url);
+    const app = createDevApp({pages: createApp({site, store, assets}), crawling: toCrawlingApp({url: site.url, store}), files: new Map(), reload: new EventEmitter()});
+    const robots = await app.request("/robots.txt");
+    const sitemap = await app.request("/sitemap.xml");
+
+    assert.equal((await app.request("/guide/")).status, 200, "crawler routes must preserve ordinary document routes");
+    assert.equal((await app.request(`/guide/${encodeURIComponent("한국어 & tools")}/`)).status, 200, "crawler routes must preserve nested document routes");
+    assert.equal(robots.status, 200);
+    assert.match(String(robots.headers.get("content-type")), /^text\/plain/);
+    assert.equal(await robots.text(), "User-agent: *\nAllow: /\n\nSitemap: https://docs.example.org/sitemap.xml\n");
+    assert.equal(sitemap.status, 200);
+    assert.match(String(sitemap.headers.get("content-type")), /^application\/xml/);
+    const xml = await sitemap.text();
+    assert.match(xml, /^<\?xml version="1\.0" encoding="UTF-8"\?>/);
+    assert.match(xml, /<urlset xmlns="http:\/\/www\.sitemaps\.org\/schemas\/sitemap\/0\.9">/);
+    assert.match(xml, /<loc>https:\/\/docs\.example\.org\/<\/loc>/);
+    assert.ok(xml.includes(`<loc>https://docs.example.org/guide/${encodeURIComponent("한국어 & tools")}/</loc>`));
+    assert.equal([...xml.matchAll(/<loc>/g)].length, 3);
+    assert.doesNotMatch(xml, /<lastmod>|<changefreq>|<priority>|robots\.txt|sitemap\.xml|_fh|#read/);
+
+    store.docs = [];
+    assert.equal([...(await (await app.request("/sitemap.xml")).text()).matchAll(/<loc>/g)].length, 1);
+});
+
+test("SSG writes crawler files and preserves ordinary and nested document pages", async (t) => {
+    const root = await mkdtemp(join(tmpdir(), "for-humanity-crawling-build-"));
+    t.after(() => rm(root, {recursive: true, force: true}));
+    await mkdir(join(root, "guide"));
+    await writeFile(join(root, "guide.md"), "---\nname: Start\nlabel: Start\ngroup: Guide\n---\n\nStart here.\n");
+    await writeFile(join(root, "guide/details.md"), "---\nname: Details\nlabel: Details\ngroup: Guide\n---\n\nNested details.\n");
+    const site = siteConfigSchema.parse({url: "https://docs.example.org"});
+    const docs = await readDocs({root, processor: createProcessor({site, root})});
+    const assets = {fontCss: "", preload: [], reload: false, browserScript: "", stylePath: asset_style_path};
+    const app = createApp({site, store: {docs}, assets});
+    const result = await toSSG(app, {writeFile, mkdir}, {dir: root});
+    assert.ok(site.url);
+    const crawlerResult = await toSSG(toCrawlingApp({url: site.url, store: {docs}}), {writeFile, mkdir}, {dir: root});
+
+    assert.equal(result.success, true);
+    assert.equal(crawlerResult.success, true);
+    assert.equal(result.files.length + crawlerResult.files.length, 5, "home, both documents and both crawler files must be emitted");
+    assert.match(await readFile(join(root, "guide/index.html"), "utf8"), /Start here\./);
+    assert.match(await readFile(join(root, "guide/details/index.html"), "utf8"), /Nested details\./);
+    assert.match(await readFile(join(root, "robots.txt"), "utf8"), /Sitemap: https:\/\/docs\.example\.org\/sitemap\.xml/);
+    assert.match(await readFile(join(root, "sitemap.xml"), "utf8"), /<loc>https:\/\/docs\.example\.org\/<\/loc>/);
+});
+
+test("crawler filenames reject conflicting document roots only when reserved", async (t) => {
+    const root = await mkdtemp(join(tmpdir(), "for-humanity-crawling-paths-"));
+    t.after(() => rm(root, {recursive: true, force: true}));
+    await mkdir(join(root, "sitemap.xml"));
+    await writeFile(join(root, "Robots.txt.md"), "---\nname: Robots\nlabel: Robots\ngroup: Guide\n---\n\nText.\n");
+    await writeFile(join(root, "sitemap.xml/guide.md"), "---\nname: Sitemap\nlabel: Sitemap\ngroup: Guide\n---\n\nText.\n");
+    const site = siteConfigSchema.parse({});
+    const processor = createProcessor({site, root});
+    const params = {root, processor, reserved: ["robots.txt", "sitemap.xml"]};
+
+    await assert.rejects(readDocs(params), /문서 id에 사용할 수 없는 경로/);
+    assert.equal((await readDocs({root, processor})).length, 2);
+});
+
 test("navigation initialization runs after its DOM and before the document body", async () => {
     const site = siteConfigSchema.parse({});
     const browserScript = "window.navigationFixture = true;";

package/src/entry/cli/cli.ts

@@ -19,11 +19,13 @@ import {copy_error_config, copy_error_font_preload, copy_error_prefix, copy_erro
 import {font_brand_css, font_cache_control, font_mono_css, font_sans_css, font_sans_preload_file} from "@/constant/font";
 import {site_config_absent} from "@/constant/site";
 import {cli_config_file_name, cli_default_command, cli_default_docs_dir, cli_dev_port, cli_reload_delay_ms} from "@/entry/cli/_constant/cli";
+import {crawling_reserved_roots} from "@/entry/cli/_constant/crawling";
 import {createApp} from "@/entry/cli/_function/create-app";
 import {createDevApp} from "@/entry/cli/_function/create-dev-app";
 import {createProcessor} from "@/entry/cli/_function/create-processor/create-processor";
 import {readDocs} from "@/entry/cli/_function/read-docs";
 import {readHome} from "@/entry/cli/_function/read-home";
+import {toCrawlingApp} from "@/entry/cli/_function/to-crawling-app";
 import {siteConfigSchema} from "@/type/site-config";
 import {toChangeQueue} from "@/util/async/to-change-queue";
 import {toErrorMessage} from "@/util/error/to-error-message";
@@ -96,8 +98,12 @@ const main = async () => {
     ]);
     const files = new Map(kitFiles);
     const processor = createProcessor({site: siteConfig, root: docsRoot, files});
-    const [docs, home] = await Promise.all([readDocs({root: docsRoot, processor}), readHome({root: docsRoot, processor})]);
+    const [docs, home] = await Promise.all([
+        readDocs({root: docsRoot, processor, reserved: siteConfig.url === undefined ? undefined : crawling_reserved_roots}),
+        readHome({root: docsRoot, processor}),
+    ]);
     const store = {docs, home};
+    const crawling = siteConfig.url === undefined ? undefined : toCrawlingApp({url: siteConfig.url, store});
     const app = createApp({
         site: siteConfig,
         store,
@@ -114,12 +120,20 @@ const main = async () => {
             throw result.error;
         }
 
+        const crawlerResult = crawling === undefined ? undefined : await toSSG(crawling, {writeFile, mkdir}, {dir: outDir});
+
+        if (crawlerResult?.success === false) {
+            throw crawlerResult.error;
+        }
+
         for (const [url, file] of files) {
             await mkdir(dirname(join(outDir, url)), {recursive: true});
             await copyFile(file, join(outDir, url));
         }
 
-        console.log(`${result.files.length} pages, ${files.size} assets → ${relative(process.cwd(), outDir)}`);
+        console.log(
+            `${crawlerResult === undefined ? result.files.length : result.files.length + crawlerResult.files.length} files, ${files.size} assets → ${relative(process.cwd(), outDir)}`,
+        );
 
         return;
     }
@@ -127,7 +141,7 @@ const main = async () => {
     const reload = new EventEmitter();
     let generation = 0;
 
-    serve({fetch: createDevApp({pages: app, files, reload}).fetch, port: cli_dev_port}, (info) => console.log(`http://localhost:${info.port}/`));
+    serve({fetch: createDevApp({pages: app, crawling, files, reload}).fetch, port: cli_dev_port}, (info) => console.log(`http://localhost:${info.port}/`));
 
     /**
      * 갱신은 하나씩 실행 · 연속 저장 합치기 · 최신 변경만 적용
@@ -139,7 +153,10 @@ const main = async () => {
                 const currentGeneration = generation;
                 const nextFiles = new Map(kitFiles);
                 const nextProcessor = createProcessor({site: siteConfig, root: docsRoot, files: nextFiles});
-                const [docs, home] = await Promise.all([readDocs({root: docsRoot, processor: nextProcessor}), readHome({root: docsRoot, processor: nextProcessor})]);
+                const [docs, home] = await Promise.all([
+                    readDocs({root: docsRoot, processor: nextProcessor, reserved: siteConfig.url === undefined ? undefined : crawling_reserved_roots}),
+                    readHome({root: docsRoot, processor: nextProcessor}),
+                ]);
 
                 if (currentGeneration !== generation) {
                     return;

package/src/type/site-config.ts

@@ -1,5 +1,5 @@
 import {z} from "zod";
-import {copy_error_navigation_clash, copy_site_title_default} from "@/constant/copy";
+import {copy_error_navigation_clash, copy_error_site_url, copy_site_title_default} from "@/constant/copy";
 import {site_navigation_default} from "@/constant/site";
 import {status_default_phrases, status_kind} from "@/constant/status";
 import {groupPathSchema} from "@/type/group-path";
@@ -17,6 +17,15 @@ export const siteConfigSchema = z.object({
      * HTML 설명 메타데이터 · 선택
      */
     description: z.string().optional(),
+    /**
+     * 공개 사이트 루트 URL · 지정할 때만 robots.txt와 sitemap.xml 생성
+     */
+    url: z
+        .url({protocol: /^https?$/})
+        .transform((value) => new URL(value))
+        .refine((value) => value.username === "" && value.password === "" && value.pathname === "/" && value.search === "" && value.hash === "", copy_error_site_url)
+        .transform((value) => value.origin)
+        .optional(),
     /**
      * 탐색 묶음 경로의 읽는 순서 · 미지정 형제 묶음은 뒤에서 이름순
      */