@anubhav-choubey/web-for-agents 0.1.0 → 0.3.0

package/dist/{chunk-IF32TPGR.js → chunk-357ATBMS.js}

@@ -1,16 +1,23 @@
 import {
-  AI_BOTS
-} from "./chunk-CKXHTGGT.js";
+  AI_BOTS,
+  scanHtml
+} from "./chunk-CGIIB6AF.js";
 import {
-  AGENT_MAP_VERSION
-} from "./chunk-LCVI2PRP.js";
+  DESCRIPTION_MAX_CHARS,
+  DESCRIPTION_MIN_CHARS,
+  TITLE_MAX_CHARS
+} from "./chunk-L3AYWCSX.js";
 import {
-  jsonLdTypes
-} from "./chunk-LCLDQ3A3.js";
+  MAX_JSON_LD_NODES,
+  jsonLdNodes,
+  jsonLdTypes,
+  nodeTypes
+} from "./chunk-MIVGE5TM.js";
 import {
+  AGENT_MAP_VERSION,
   collapseWhitespace,
   stripBom
-} from "./chunk-FEAKVNMG.js";
+} from "./chunk-4STTPCBR.js";
 import {
   AGENT_MAP_PATH,
   LLMS_TXT_PATH,
@@ -22,12 +29,177 @@ import {
   markdownPathFor,
   trimTrailingSlash,
   tryResolveUrl
-} from "./chunk-AZG5MLNG.js";
+} from "./chunk-2Y3LLYEC.js";
 import {
   errorMessage,
   withPrefix
 } from "./chunk-JJN6FNJD.js";
 
+// src/shared/schemaRules.ts
+var DOCS = "https://developers.google.com/search/docs/appearance/structured-data/";
+var req = (...anyOf) => ({ anyOf });
+var ARTICLE = {
+  docs: `${DOCS}article`,
+  required: [],
+  recommended: ["author", "author.name", "author.url", "dateModified", "datePublished", "headline", "image"]
+};
+var SOFTWARE_APPLICATION = {
+  docs: `${DOCS}software-app`,
+  required: [req("name"), req("offers"), req("aggregateRating", "review")],
+  recommended: ["applicationCategory", "operatingSystem"],
+  nested: [{ prop: "offers", whenType: ["Offer"], orUntyped: true, required: [req("price")] }]
+};
+var SCHEMA_RULES = {
+  Article: ARTICLE,
+  NewsArticle: ARTICLE,
+  BlogPosting: ARTICLE,
+  BreadcrumbList: {
+    docs: `${DOCS}breadcrumb`,
+    required: [req("itemListElement")],
+    recommended: [],
+    nested: [
+      {
+        prop: "itemListElement",
+        required: [req("position"), req("name", "item.name"), { anyOf: ["item"], exceptLast: true }]
+      }
+    ]
+  },
+  Event: {
+    docs: `${DOCS}event`,
+    required: [req("name"), req("startDate"), req("location")],
+    recommended: ["description", "endDate", "eventStatus", "image", "location.name", "offers", "organizer", "performer"],
+    nested: [{ prop: "location", whenType: ["Place"], required: [req("address")] }]
+  },
+  FAQPage: {
+    docs: `${DOCS}faqpage`,
+    required: [req("mainEntity")],
+    recommended: [],
+    nested: [
+      {
+        prop: "mainEntity",
+        required: [req("name"), req("acceptedAnswer")],
+        nested: [{ prop: "acceptedAnswer", required: [req("text")] }]
+      }
+    ],
+    note: "Google no longer shows FAQ rich results; answer engines may still use it",
+    retired: true
+  },
+  Organization: {
+    docs: `${DOCS}organization`,
+    required: [],
+    recommended: ["name", "url", "logo", "description", "sameAs", "address", "contactPoint", "email", "telephone"]
+  },
+  Person: {
+    docs: `${DOCS}profile-page`,
+    required: [],
+    recommended: ["name", "alternateName", "description", "identifier", "image", "sameAs"]
+  },
+  ProfilePage: {
+    docs: `${DOCS}profile-page`,
+    required: [req("mainEntity")],
+    recommended: ["dateCreated", "dateModified"],
+    nested: [{ prop: "mainEntity", required: [req("name", "alternateName")] }]
+  },
+  Product: {
+    docs: `${DOCS}product-snippet`,
+    required: [req("name"), req("offers", "aggregateRating", "review")],
+    recommended: ["aggregateRating", "offers", "review", "offers.priceCurrency", "offers.availability"],
+    nested: [
+      { prop: "offers", whenType: ["Offer"], orUntyped: true, required: [req("price", "priceSpecification.price")] },
+      { prop: "offers", whenType: ["AggregateOffer"], required: [req("lowPrice"), req("priceCurrency")] }
+    ]
+  },
+  Recipe: {
+    docs: `${DOCS}recipe`,
+    required: [req("name"), req("image")],
+    recommended: ["author", "datePublished", "description", "recipeIngredient", "recipeInstructions", "totalTime"]
+  },
+  SoftwareApplication: SOFTWARE_APPLICATION,
+  MobileApplication: SOFTWARE_APPLICATION,
+  WebApplication: SOFTWARE_APPLICATION,
+  WebSite: {
+    docs: "https://developers.google.com/search/docs/appearance/site-names",
+    required: [req("name"), req("url")],
+    recommended: ["alternateName"]
+  }
+};
+function present(v) {
+  if (v === void 0 || v === null) return false;
+  if (typeof v === "string") return v.trim() !== "";
+  if (Array.isArray(v)) return v.length > 0;
+  return true;
+}
+var isObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
+function hasPath(node, path2) {
+  let level = [node];
+  for (const key of path2.split(".")) {
+    const next = [];
+    for (const v of level) {
+      for (const o of Array.isArray(v) ? v : [v]) if (isObject(o) && present(o[key])) next.push(o[key]);
+    }
+    if (next.length === 0) return false;
+    level = next;
+  }
+  return true;
+}
+function resolveRef(o, byId) {
+  const keys = Object.keys(o);
+  if (keys.length !== 1 || keys[0] !== "@id") return o;
+  return typeof o["@id"] === "string" ? byId.get(o["@id"]) ?? null : null;
+}
+var label = (prefix, anyOf) => prefix + anyOf[0] + (anyOf.length > 1 ? ` (or ${anyOf.slice(1).join(", ")})` : "");
+function checkSchemaNodes(nodes, limit = Infinity) {
+  const byId = /* @__PURE__ */ new Map();
+  for (const n of nodes) if (typeof n["@id"] === "string" && !byId.has(n["@id"])) byId.set(n["@id"], n);
+  const scoredIssues = [];
+  const retiredIssues = [];
+  let total = 0;
+  let scored = 0;
+  let budget = MAX_JSON_LD_NODES;
+  const report = (type, path2) => {
+    total++;
+    const retired = SCHEMA_RULES[type].retired === true;
+    if (!retired) scored++;
+    const list = retired ? retiredIssues : scoredIssues;
+    if (list.length < limit) list.push({ type, path: path2 });
+  };
+  const apply = (type, node, prefix, required, nested, isLast) => {
+    for (const r of required) {
+      if (r.exceptLast && isLast) continue;
+      if (!r.anyOf.some((p) => hasPath(node, p))) report(type, label(prefix, r.anyOf));
+    }
+    for (const n of nested ?? []) {
+      const v = node[n.prop];
+      const items = Array.isArray(v) ? v : [v];
+      for (let i = 0; i < items.length; i++) {
+        if (--budget < 0) return;
+        const raw = items[i];
+        if (!isObject(raw)) continue;
+        const child = resolveRef(raw, byId);
+        if (child === null) continue;
+        if (n.whenType !== void 0) {
+          const types = nodeTypes(child);
+          const matches2 = types.length === 0 ? n.orUntyped === true : types.some((t) => n.whenType.includes(t));
+          if (!matches2) continue;
+        }
+        const at = `${prefix}${n.prop}${Array.isArray(v) ? `[${i}]` : ""}.`;
+        apply(type, child, at, n.required, n.nested, i === items.length - 1);
+      }
+    }
+  };
+  for (const node of nodes) {
+    if (--budget < 0) break;
+    const seen = /* @__PURE__ */ new Set();
+    for (const type of nodeTypes(node)) {
+      const rule = Object.hasOwn(SCHEMA_RULES, type) ? SCHEMA_RULES[type] : void 0;
+      if (rule === void 0 || seen.has(rule)) continue;
+      seen.add(rule);
+      apply(type, node, "", rule.required, rule.nested, true);
+    }
+  }
+  return { issues: [...scoredIssues, ...retiredIssues].slice(0, limit), total, scored };
+}
+
 // src/cli/privateAddress.ts
 import { lookup } from "dns/promises";
 import { BlockList, isIP } from "net";
@@ -79,10 +251,10 @@ function abortable(p, signal) {
   });
 }
 function decode(bytes, contentType) {
-  const label = /charset\s*=\s*"?([^;"\s]+)/i.exec(contentType)?.[1] ?? "utf-8";
+  const label2 = /charset\s*=\s*"?([^;"\s]+)/i.exec(contentType)?.[1] ?? "utf-8";
   let decoder;
   try {
-    decoder = new TextDecoder(label);
+    decoder = new TextDecoder(label2);
   } catch (e) {
     if (!(e instanceof RangeError)) throw e;
     decoder = new TextDecoder("utf-8");
@@ -266,16 +438,6 @@ function decode2(s) {
     return String.fromCodePoint(cp);
   });
 }
-var isSpace = (c) => c === 32 || c === 9 || c === 10 || c === 12 || c === 13;
-var isAlpha = (c) => c >= 65 && c <= 90 || c >= 97 && c <= 122;
-function findRawEnd(html, name, from) {
-  let i = html.indexOf("</", from);
-  while (i !== -1) {
-    if (html.slice(i + 2, i + 2 + name.length).toLowerCase() === name) return i;
-    i = html.indexOf("</", i + 2);
-  }
-  return -1;
-}
 function extractHtmlFacts(html) {
   const metas = [];
   const links = [];
@@ -286,88 +448,29 @@ function extractHtmlFacts(html) {
   let prerendered = false;
   const depth = {};
   let excluded = 0;
-  const n = html.length;
-  let i = 0;
-  while (i < n) {
-    const lt = html.indexOf("<", i);
-    if (lt === -1) {
-      if (excluded === 0) chunks.push(decode2(html.slice(i)));
-      break;
-    }
-    if (lt > i && excluded === 0) chunks.push(decode2(html.slice(i, lt)));
-    const c1 = lt + 1 < n ? html.charCodeAt(lt + 1) : 0;
-    if (c1 === 33 || c1 === 63) {
-      if (html.startsWith("<!--", lt)) {
-        const end = html.indexOf("-->", lt + 4);
-        i = end === -1 ? n : end + 3;
-      } else {
-        const end = html.indexOf(">", lt + 2);
-        i = end === -1 ? n : end + 1;
+  let pos = 0;
+  let lastAttrs = {};
+  for (const t of scanHtml(html, RAW_TEXT)) {
+    if (t.start > pos && excluded === 0) chunks.push(decode2(html.slice(pos, t.start)));
+    pos = t.end;
+    if (t.type === "comment") continue;
+    if (t.type === "raw") {
+      const raw = html.slice(t.start, t.end);
+      if (t.name === "title") {
+        if (title === null && (depth.svg ?? 0) === 0) {
+          title = decode2(raw).replace(/\s+/g, " ").trim();
+        }
+      } else if (t.name === "script") {
+        const type = (lastAttrs.type ?? "").split(";")[0].trim().toLowerCase();
+        if (type === "application/ld+json") jsonLd.push(raw);
       }
       continue;
     }
-    const closing = c1 === 47;
-    const nameStart = lt + (closing ? 2 : 1);
-    if (!isAlpha(nameStart < n ? html.charCodeAt(nameStart) : 0)) {
-      if (excluded === 0) chunks.push("<");
-      i = lt + 1;
-      continue;
-    }
-    let p = nameStart;
-    while (p < n) {
-      const c = html.charCodeAt(p);
-      if (isSpace(c) || c === 62 || c === 47) break;
-      p++;
-    }
-    const name = html.slice(nameStart, p).toLowerCase();
+    const { name, closing, selfClosing } = t;
     const attrs = {};
-    let selfClosing = false;
-    while (p < n) {
-      const c = html.charCodeAt(p);
-      if (isSpace(c)) {
-        p++;
-        continue;
-      }
-      if (c === 62) {
-        p++;
-        break;
-      }
-      if (c === 47) {
-        selfClosing = p + 1 < n && html.charCodeAt(p + 1) === 62;
-        p++;
-        continue;
-      }
-      const as = p;
-      while (p < n) {
-        const d = html.charCodeAt(p);
-        if (isSpace(d) || d === 62 || d === 47 || d === 61) break;
-        p++;
-      }
-      if (p === as) {
-        p++;
-        continue;
-      }
-      const aname = html.slice(as, p).toLowerCase();
-      while (p < n && isSpace(html.charCodeAt(p))) p++;
-      let value = "";
-      if (p < n && html.charCodeAt(p) === 61) {
-        p++;
-        while (p < n && isSpace(html.charCodeAt(p))) p++;
-        const q = p < n ? html[p] : "";
-        if (q === '"' || q === "'") {
-          const end = html.indexOf(q, p + 1);
-          const stop = end === -1 ? n : end;
-          value = html.slice(p + 1, stop);
-          p = end === -1 ? n : end + 1;
-        } else {
-          const vs = p;
-          while (p < n && !isSpace(html.charCodeAt(p)) && html.charCodeAt(p) !== 62) p++;
-          value = html.slice(vs, p);
-        }
-      }
-      if (!(aname in attrs) && !closing) attrs[aname] = decode2(value);
+    if (!closing) {
+      for (const a of t.attrs) if (!(a.name in attrs)) attrs[a.name] = decode2(a.value);
     }
-    i = p;
     if (closing) {
       if (EXCLUDED.has(name) && (depth[name] ?? 0) > 0) {
         depth[name]--;
@@ -376,6 +479,7 @@ function extractHtmlFacts(html) {
       if (excluded === 0 && !INLINE.has(name)) chunks.push(" ");
       continue;
     }
+    lastAttrs = attrs;
     if ((depth.head ?? 0) > 0 && excluded === depth.head && !HEAD_OK.has(name)) {
       excluded -= depth.head;
       depth.head = 0;
@@ -387,27 +491,14 @@ function extractHtmlFacts(html) {
     } else if (name === "link") {
       links.push(attrs);
     }
-    if (RAW_TEXT.has(name) && !selfClosing) {
-      const end = findRawEnd(html, name, i);
-      const raw = html.slice(i, end === -1 ? n : end);
-      i = end === -1 ? n : end;
-      if (name === "title") {
-        if (title === null && (depth.svg ?? 0) === 0) {
-          title = decode2(raw).replace(/\s+/g, " ").trim();
-        }
-      } else if (name === "script") {
-        const type = (attrs.type ?? "").split(";")[0].trim().toLowerCase();
-        if (type === "application/ld+json") jsonLd.push(raw);
-      }
-      if (EXCLUDED.has(name) && end === -1) break;
-      continue;
-    }
+    if (RAW_TEXT.has(name) && !selfClosing) continue;
     if (EXCLUDED.has(name) && !selfClosing) {
       depth[name] = (depth[name] ?? 0) + 1;
       excluded++;
     }
     if (excluded === 0 && !INLINE.has(name)) chunks.push(" ");
   }
+  if (pos < html.length && excluded === 0) chunks.push(decode2(html.slice(pos)));
   return {
     title,
     metas,
@@ -515,13 +606,13 @@ function matches(pattern, text) {
   return last === "" || text.indexOf(last, at) >= 0;
 }
 var matchCache = /* @__PURE__ */ new WeakMap();
-function cachedMatch(robots, pattern, path) {
+function cachedMatch(robots, pattern, path2) {
   let byPath = matchCache.get(robots);
   if (!byPath) matchCache.set(robots, byPath = /* @__PURE__ */ new Map());
-  let byPattern = byPath.get(path);
-  if (!byPattern) byPath.set(path, byPattern = /* @__PURE__ */ new Map());
+  let byPattern = byPath.get(path2);
+  if (!byPattern) byPath.set(path2, byPattern = /* @__PURE__ */ new Map());
   let hit = byPattern.get(pattern);
-  if (hit === void 0) byPattern.set(pattern, hit = matches(pattern, path));
+  if (hit === void 0) byPattern.set(pattern, hit = matches(pattern, path2));
   return hit;
 }
 function isAllowed(robots, userAgent, pathAndQuery2) {
@@ -556,9 +647,7 @@ var CHECK_WEIGHTS = {
 };
 var CONTENT_MIN_CHARS = 200;
 var CONTENT_WARN_CHARS = 50;
-var TITLE_MAX_CHARS = 70;
-var DESCRIPTION_MIN_CHARS = 50;
-var DESCRIPTION_MAX_CHARS = 160;
+var SCHEMA_ISSUES_SHOWN = 3;
 var MAX_BODY_LABEL = `${MAX_BODY_BYTES / (1024 * 1024)} MiB`;
 var TITLES = {
   content: "Content without JavaScript",
@@ -681,7 +770,32 @@ function checkJsonLd({ html }) {
   if (!parsed.some((p) => jsonLdTypes(p).length > 0)) {
     return warn("JSON-LD parses but has no @type", "Add an @type (for example Organization) to your JSON-LD.");
   }
-  return pass(`${parsed.length} JSON-LD block${parsed.length === 1 ? " parses" : "s parse"} with an @type`);
+  const { issues, total, scored } = checkSchemaNodes(parsed.flatMap(jsonLdNodes), SCHEMA_ISSUES_SHOWN);
+  const base = `${parsed.length} JSON-LD block${parsed.length === 1 ? " parses" : "s parse"} with an @type`;
+  if (total === 0) return pass(base);
+  const isRetired = (i) => SCHEMA_RULES[i.type].retired === true;
+  const shownScored = issues.filter((i) => !isRetired(i));
+  const shownRetired = issues.filter(isRetired);
+  const retiredNote = notScored(shownRetired, total - scored - shownRetired.length);
+  if (scored === 0) return pass(`${base}; ${retiredNote}`);
+  const moreScored = scored - shownScored.length;
+  const rules = [...new Set(shownScored.map((i) => SCHEMA_RULES[i.type]))];
+  return warn(
+    `Missing required structured data fields: ${shownScored.map((i) => `${i.type} ${i.path}`).join("; ")}` + (moreScored > 0 ? `; +${moreScored} more` : "") + (total > scored ? `. ${retiredNote}` : ""),
+    `Add the missing properties; Google requires them for rich results (${rules.map((r) => r.docs).join(", ")}).`
+  );
+}
+var RETIRED_RULES = [...new Set(Object.values(SCHEMA_RULES))].filter((r) => r.retired === true);
+var retiredNotes = (rules) => rules.flatMap((r) => r.note !== void 0 ? [r.note] : []).join("; ");
+function notScored(shown, more) {
+  if (shown.length === 0) {
+    return `+${more} more not scored (${retiredNotes(RETIRED_RULES)})`;
+  }
+  const byType = /* @__PURE__ */ new Map();
+  for (const i of shown) byType.set(i.type, [...byType.get(i.type) ?? [], i.path]);
+  const listed = [...byType].map(([type, paths]) => `${type}: ${paths.join(", ")}`).join("; ");
+  const notes = retiredNotes([...new Set(shown.map((i) => SCHEMA_RULES[i.type]))]);
+  return `${listed}${more > 0 ? `, +${more} more` : ""} missing${notes ? ` (${notes})` : ""}`;
 }
 function checkRobots({ page, robots }) {
   const missingFix = `Add ${ROBOTS_PATH}; webForAgents() generates one from robots.ai.training / robots.ai.search.`;
@@ -695,21 +809,21 @@ function checkRobots({ page, robots }) {
   const out = robotsVerdict(parsed, pathAndQuery(page.finalUrl));
   return parsed.truncated ? { ...out, message: `${out.message} (${parsed.truncated})` } : out;
 }
-function robotsVerdict(parsed, path) {
-  if (!isAllowed(parsed, "*", path)) {
+function robotsVerdict(parsed, path2) {
+  if (!isAllowed(parsed, "*", path2)) {
     return fail(
-      `${ROBOTS_PATH} blocks ${path} for all crawlers`,
+      `${ROBOTS_PATH} blocks ${path2} for all crawlers`,
       "Remove the Disallow rule matching this path under User-agent: *."
     );
   }
-  const searchBlocked = AI_BOTS.search.filter((b) => !isAllowed(parsed, b, path));
+  const searchBlocked = AI_BOTS.search.filter((b) => !isAllowed(parsed, b, path2));
   if (searchBlocked.length > 0) {
     return warn(
       `AI search bots blocked: ${searchBlocked.join(", ")}`,
       'Allow AI search bots (robots.ai.search: "allow") so answer engines can cite this page.'
     );
   }
-  const trainingBlocked = AI_BOTS.training.filter((b) => !isAllowed(parsed, b, path));
+  const trainingBlocked = AI_BOTS.training.filter((b) => !isAllowed(parsed, b, path2));
   if (trainingBlocked.length > 0) {
     return pass(`Training crawlers blocked (${trainingBlocked.join(", ")}); search bots allowed`);
   }
@@ -875,7 +989,7 @@ async function auditWithHooks(url, opts = {}, hooks = {}) {
       return report;
     }
     const html = extractHtmlFacts(page.body);
-    const at = (path) => new URL(path, page.finalUrl).href;
+    const at = (path2) => new URL(path2, page.finalUrl).href;
     const getSameOrigin = (u, accept) => sameOrigin(u, page.finalUrl) ? get(u, accept) : Promise.resolve(notFetched(u));
     const alt = markdownAlternate(html);
     const altUrl = alt === null ? null : tryResolveUrl(alt, page.finalUrl);
@@ -945,8 +1059,9 @@ function normalizeInputUrl(input) {
   return u.href;
 }
 var USAGE = `Usage: web-for-agents audit <url> [options]
+       web-for-agents generate <outDir> --site-url <url> [options]
 
-Audit how readable a page is for AI agents and crawlers.
+audit: check how readable a page is for AI agents and crawlers.
 
 Options:
   --json               print the report as JSON (page data is not sanitized; control
@@ -959,6 +1074,22 @@ Options:
                        link-local address (default: follow them and print a warning)
   -h, --help           show this help
 
+generate: write llms.txt, llms-full.txt, .md pages, robots.txt, sitemap.xml and
+agents.json for HTML you prerendered yourself.
+
+Options:
+  --site-url <url>     absolute URL of the deployed site (or siteUrl in --config)
+  --ai-training <allow|block>
+  --ai-search <allow|block>
+                       robots.txt policy for AI crawlers; both are required here or
+                       as robots.ai in --config
+  --config <file>      .json, .js or .mjs file whose default export holds options
+                       (exclude, markdown, llms, agentMap, sitemap, robots, ...)
+  --no-link-markdown   do not add the markdown <link> to each page's <head>
+  --overwrite          replace an existing robots.txt, sitemap.xml, llms.txt,
+                       llms-full.txt or agents.json that differs
+  --json               print the result as JSON
+
 Exit codes: 0 ok, 1 score below --min-score, 2 usage error or fatal error
 `;
 function parseCliArgs(argv) {
@@ -973,6 +1104,12 @@ function parseCliArgs(argv) {
         timeout: { type: "string" },
         "agent-map": { type: "string" },
         "no-private-redirects": { type: "boolean" },
+        "site-url": { type: "string" },
+        "ai-training": { type: "string" },
+        "ai-search": { type: "string" },
+        config: { type: "string" },
+        "no-link-markdown": { type: "boolean" },
+        overwrite: { type: "boolean" },
         help: { type: "boolean", short: "h" }
       }
     });
@@ -982,7 +1119,34 @@ function parseCliArgs(argv) {
   const { values, positionals } = parsed;
   if (values.help || positionals.length === 0) return { command: "help", json: false, timeoutMs: DEFAULT_TIMEOUT_MS };
   const [command, input, ...rest] = positionals;
+  const GENERATE_FLAGS = ["site-url", "ai-training", "ai-search", "config", "no-link-markdown", "overwrite"];
+  const AUDIT_FLAGS = ["min-score", "timeout", "agent-map", "no-private-redirects"];
+  if (command === "generate") {
+    if (input === void 0) return fail2("generate requires an output directory");
+    if (rest.length > 0) return fail2(`unexpected argument "${rest[0]}"`);
+    const wrong2 = AUDIT_FLAGS.find((f) => values[f] !== void 0);
+    if (wrong2) return fail2(`--${wrong2} is not an option of generate`);
+    const gen = {
+      outDir: input,
+      linkMarkdown: values["no-link-markdown"] !== true,
+      overwrite: values.overwrite === true
+    };
+    for (const [flag, key] of [["ai-training", "aiTraining"], ["ai-search", "aiSearch"]]) {
+      const v = values[flag];
+      if (v === void 0) continue;
+      if (v !== "allow" && v !== "block") return fail2(`--${flag} must be "allow" or "block", got "${v}"`);
+      gen[key] = v;
+    }
+    if (values["site-url"] !== void 0) gen.siteUrl = values["site-url"];
+    if (values.config !== void 0) {
+      if (values.config.trim() === "") return fail2("--config must be a file path");
+      gen.config = values.config;
+    }
+    return { command: "generate", generate: gen, json: values.json === true, timeoutMs: DEFAULT_TIMEOUT_MS };
+  }
   if (command !== "audit") return fail2(`unknown command "${command}"`);
+  const wrong = GENERATE_FLAGS.find((f) => values[f] !== void 0);
+  if (wrong) return fail2(`--${wrong} is an option of generate, not audit`);
   if (input === void 0) return fail2("audit requires a URL");
   if (rest.length > 0) return fail2(`unexpected argument "${rest[0]}"`);
   const args = {
@@ -1020,15 +1184,20 @@ function parseCliArgs(argv) {
   return args;
 }
 
+// src/cli/generate.ts
+import { existsSync, readFileSync } from "fs";
+import path from "path";
+import { pathToFileURL } from "url";
+
 // src/cli/terminal.ts
 var MAX_TEXT_CHARS = 500;
 var CONTROL = /[\u0000-\u001f\u007f-\u009f]/g;
 var BIDI_CHARS = "\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069";
 var BIDI = new RegExp(`[${BIDI_CHARS}]`, "g");
 function safeText(s, max = MAX_TEXT_CHARS) {
-  const clean = collapseWhitespace(s.replace(CONTROL, " ").replace(BIDI, ""));
-  const chars = Array.from(clean);
-  return chars.length > max ? chars.slice(0, max - 1).join("") + "\u2026" : clean;
+  const clean2 = collapseWhitespace(s.replace(CONTROL, " ").replace(BIDI, ""));
+  const chars = Array.from(clean2);
+  return chars.length > max ? chars.slice(0, max - 1).join("") + "\u2026" : clean2;
 }
 var JSON_UNSAFE = new RegExp(`[\\u007f-\\u009f\\u2028\\u2029${BIDI_CHARS}]`, "g");
 function safeJson(value, space) {
@@ -1038,6 +1207,75 @@ function safeJson(value, space) {
   );
 }
 
+// src/cli/generate.ts
+var VITE_ENTRY = "@anubhav-choubey/web-for-agents/vite";
+var clean = (s) => s.split("\n").map((l) => safeText(l, Infinity)).join("\n");
+var isObject2 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+async function loadConfig(file) {
+  const abs = path.resolve(file);
+  if (!existsSync(abs)) throw new Error(withPrefix(`config file not found: ${abs}`));
+  let config;
+  try {
+    if (abs.endsWith(".json")) config = JSON.parse(readFileSync(abs, "utf8"));
+    else if (/\.m?js$/.test(abs)) config = (await import(pathToFileURL(abs).href)).default;
+    else throw new Error("expected a .json, .js or .mjs file");
+  } catch (e) {
+    throw new Error(withPrefix(`could not load config ${abs}: ${errorMessage(e)}`), { cause: e });
+  }
+  if (!isObject2(config)) throw new Error(withPrefix(`config ${abs} must export an options object`));
+  return config;
+}
+async function generateOptions(args) {
+  const config = args.config ? await loadConfig(args.config) : {};
+  const robots = isObject2(config.robots) ? config.robots : {};
+  const ai = { ...isObject2(robots.ai) ? robots.ai : {} };
+  if (args.aiTraining) ai.training = args.aiTraining;
+  if (args.aiSearch) ai.search = args.aiSearch;
+  if (ai.training === void 0 || ai.search === void 0) {
+    throw new Error(withPrefix("--ai-training and --ai-search are required (or robots.ai in --config)"));
+  }
+  const siteUrl = args.siteUrl ?? config.siteUrl;
+  if (siteUrl === void 0) throw new Error(withPrefix("--site-url is required (or siteUrl in --config)"));
+  return {
+    ...config,
+    outDir: args.outDir,
+    siteUrl,
+    robots: { ...robots, ai },
+    ...args.linkMarkdown ? {} : { linkMarkdown: false },
+    ...args.overwrite ? { overwrite: true } : {}
+  };
+}
+async function runGenerate(args, json, io) {
+  let result;
+  try {
+    const options = await generateOptions(args);
+    const { generateAgentFiles } = await import(VITE_ENTRY);
+    result = await generateAgentFiles({
+      ...options,
+      logger: {
+        warn: (m) => io.stderr(clean(m) + "\n"),
+        info: (m) => (json ? io.stderr : io.stdout)(clean(m) + "\n")
+      }
+    });
+  } catch (e) {
+    io.stderr(withPrefix(`generate failed: ${clean(errorMessage(e).replace(/^\[web-for-agents\] /, ""))}
+`));
+    return 2;
+  }
+  if (json) {
+    io.stdout(safeJson(result, 2) + "\n");
+  } else {
+    const lines = [
+      `Layout: ${result.fileLayout}`,
+      `Pages: ${result.pages.map((p) => p.route + (p.listed ? "" : " (noindex)")).join(", ")}`,
+      `Wrote ${result.written.length} files: ${result.written.join(", ")}`
+    ];
+    if (result.kept.length > 0) lines.push(`Kept existing: ${result.kept.join(", ")} (use --overwrite to replace)`);
+    io.stdout(clean(lines.join("\n")) + "\n");
+  }
+  return 0;
+}
+
 // src/cli/report.ts
 function formatText(report) {
   const lines = [
@@ -1077,6 +1315,7 @@ ${USAGE}`);
     io.stdout(USAGE);
     return 0;
   }
+  if (args.command === "generate") return runGenerate(args.generate, args.json, io);
   const report = await audit(args.url, {
     timeoutMs: args.timeoutMs,
     ...args.agentMap === void 0 ? {} : { agentMapPath: args.agentMap },