@anubhav-choubey/web-for-agents 0.1.0 → 0.2.0

package/dist/{chunk-IF32TPGR.js → chunk-P6JSPAA2.js}

@@ -1,9 +1,13 @@
 import {
-  AI_BOTS
-} from "./chunk-CKXHTGGT.js";
+  AI_BOTS,
+  scanHtml
+} from "./chunk-CGIIB6AF.js";
 import {
-  AGENT_MAP_VERSION
-} from "./chunk-LCVI2PRP.js";
+  AGENT_MAP_VERSION,
+  DESCRIPTION_MAX_CHARS,
+  DESCRIPTION_MIN_CHARS,
+  TITLE_MAX_CHARS
+} from "./chunk-PHTV5NAY.js";
 import {
   jsonLdTypes
 } from "./chunk-LCLDQ3A3.js";
@@ -22,7 +26,7 @@ import {
   markdownPathFor,
   trimTrailingSlash,
   tryResolveUrl
-} from "./chunk-AZG5MLNG.js";
+} from "./chunk-F7PN5R25.js";
 import {
   errorMessage,
   withPrefix
@@ -266,16 +270,6 @@ function decode2(s) {
     return String.fromCodePoint(cp);
   });
 }
-var isSpace = (c) => c === 32 || c === 9 || c === 10 || c === 12 || c === 13;
-var isAlpha = (c) => c >= 65 && c <= 90 || c >= 97 && c <= 122;
-function findRawEnd(html, name, from) {
-  let i = html.indexOf("</", from);
-  while (i !== -1) {
-    if (html.slice(i + 2, i + 2 + name.length).toLowerCase() === name) return i;
-    i = html.indexOf("</", i + 2);
-  }
-  return -1;
-}
 function extractHtmlFacts(html) {
   const metas = [];
   const links = [];
@@ -286,88 +280,29 @@ function extractHtmlFacts(html) {
   let prerendered = false;
   const depth = {};
   let excluded = 0;
-  const n = html.length;
-  let i = 0;
-  while (i < n) {
-    const lt = html.indexOf("<", i);
-    if (lt === -1) {
-      if (excluded === 0) chunks.push(decode2(html.slice(i)));
-      break;
-    }
-    if (lt > i && excluded === 0) chunks.push(decode2(html.slice(i, lt)));
-    const c1 = lt + 1 < n ? html.charCodeAt(lt + 1) : 0;
-    if (c1 === 33 || c1 === 63) {
-      if (html.startsWith("<!--", lt)) {
-        const end = html.indexOf("-->", lt + 4);
-        i = end === -1 ? n : end + 3;
-      } else {
-        const end = html.indexOf(">", lt + 2);
-        i = end === -1 ? n : end + 1;
+  let pos = 0;
+  let lastAttrs = {};
+  for (const t of scanHtml(html, RAW_TEXT)) {
+    if (t.start > pos && excluded === 0) chunks.push(decode2(html.slice(pos, t.start)));
+    pos = t.end;
+    if (t.type === "comment") continue;
+    if (t.type === "raw") {
+      const raw = html.slice(t.start, t.end);
+      if (t.name === "title") {
+        if (title === null && (depth.svg ?? 0) === 0) {
+          title = decode2(raw).replace(/\s+/g, " ").trim();
+        }
+      } else if (t.name === "script") {
+        const type = (lastAttrs.type ?? "").split(";")[0].trim().toLowerCase();
+        if (type === "application/ld+json") jsonLd.push(raw);
       }
       continue;
     }
-    const closing = c1 === 47;
-    const nameStart = lt + (closing ? 2 : 1);
-    if (!isAlpha(nameStart < n ? html.charCodeAt(nameStart) : 0)) {
-      if (excluded === 0) chunks.push("<");
-      i = lt + 1;
-      continue;
-    }
-    let p = nameStart;
-    while (p < n) {
-      const c = html.charCodeAt(p);
-      if (isSpace(c) || c === 62 || c === 47) break;
-      p++;
-    }
-    const name = html.slice(nameStart, p).toLowerCase();
+    const { name, closing, selfClosing } = t;
     const attrs = {};
-    let selfClosing = false;
-    while (p < n) {
-      const c = html.charCodeAt(p);
-      if (isSpace(c)) {
-        p++;
-        continue;
-      }
-      if (c === 62) {
-        p++;
-        break;
-      }
-      if (c === 47) {
-        selfClosing = p + 1 < n && html.charCodeAt(p + 1) === 62;
-        p++;
-        continue;
-      }
-      const as = p;
-      while (p < n) {
-        const d = html.charCodeAt(p);
-        if (isSpace(d) || d === 62 || d === 47 || d === 61) break;
-        p++;
-      }
-      if (p === as) {
-        p++;
-        continue;
-      }
-      const aname = html.slice(as, p).toLowerCase();
-      while (p < n && isSpace(html.charCodeAt(p))) p++;
-      let value = "";
-      if (p < n && html.charCodeAt(p) === 61) {
-        p++;
-        while (p < n && isSpace(html.charCodeAt(p))) p++;
-        const q = p < n ? html[p] : "";
-        if (q === '"' || q === "'") {
-          const end = html.indexOf(q, p + 1);
-          const stop = end === -1 ? n : end;
-          value = html.slice(p + 1, stop);
-          p = end === -1 ? n : end + 1;
-        } else {
-          const vs = p;
-          while (p < n && !isSpace(html.charCodeAt(p)) && html.charCodeAt(p) !== 62) p++;
-          value = html.slice(vs, p);
-        }
-      }
-      if (!(aname in attrs) && !closing) attrs[aname] = decode2(value);
+    if (!closing) {
+      for (const a of t.attrs) if (!(a.name in attrs)) attrs[a.name] = decode2(a.value);
     }
-    i = p;
     if (closing) {
       if (EXCLUDED.has(name) && (depth[name] ?? 0) > 0) {
         depth[name]--;
@@ -376,6 +311,7 @@ function extractHtmlFacts(html) {
       if (excluded === 0 && !INLINE.has(name)) chunks.push(" ");
       continue;
     }
+    lastAttrs = attrs;
     if ((depth.head ?? 0) > 0 && excluded === depth.head && !HEAD_OK.has(name)) {
       excluded -= depth.head;
       depth.head = 0;
@@ -387,27 +323,14 @@ function extractHtmlFacts(html) {
     } else if (name === "link") {
       links.push(attrs);
     }
-    if (RAW_TEXT.has(name) && !selfClosing) {
-      const end = findRawEnd(html, name, i);
-      const raw = html.slice(i, end === -1 ? n : end);
-      i = end === -1 ? n : end;
-      if (name === "title") {
-        if (title === null && (depth.svg ?? 0) === 0) {
-          title = decode2(raw).replace(/\s+/g, " ").trim();
-        }
-      } else if (name === "script") {
-        const type = (attrs.type ?? "").split(";")[0].trim().toLowerCase();
-        if (type === "application/ld+json") jsonLd.push(raw);
-      }
-      if (EXCLUDED.has(name) && end === -1) break;
-      continue;
-    }
+    if (RAW_TEXT.has(name) && !selfClosing) continue;
     if (EXCLUDED.has(name) && !selfClosing) {
       depth[name] = (depth[name] ?? 0) + 1;
       excluded++;
     }
     if (excluded === 0 && !INLINE.has(name)) chunks.push(" ");
   }
+  if (pos < html.length && excluded === 0) chunks.push(decode2(html.slice(pos)));
   return {
     title,
     metas,
@@ -515,13 +438,13 @@ function matches(pattern, text) {
   return last === "" || text.indexOf(last, at) >= 0;
 }
 var matchCache = /* @__PURE__ */ new WeakMap();
-function cachedMatch(robots, pattern, path) {
+function cachedMatch(robots, pattern, path2) {
   let byPath = matchCache.get(robots);
   if (!byPath) matchCache.set(robots, byPath = /* @__PURE__ */ new Map());
-  let byPattern = byPath.get(path);
-  if (!byPattern) byPath.set(path, byPattern = /* @__PURE__ */ new Map());
+  let byPattern = byPath.get(path2);
+  if (!byPattern) byPath.set(path2, byPattern = /* @__PURE__ */ new Map());
   let hit = byPattern.get(pattern);
-  if (hit === void 0) byPattern.set(pattern, hit = matches(pattern, path));
+  if (hit === void 0) byPattern.set(pattern, hit = matches(pattern, path2));
   return hit;
 }
 function isAllowed(robots, userAgent, pathAndQuery2) {
@@ -556,9 +479,6 @@ var CHECK_WEIGHTS = {
 };
 var CONTENT_MIN_CHARS = 200;
 var CONTENT_WARN_CHARS = 50;
-var TITLE_MAX_CHARS = 70;
-var DESCRIPTION_MIN_CHARS = 50;
-var DESCRIPTION_MAX_CHARS = 160;
 var MAX_BODY_LABEL = `${MAX_BODY_BYTES / (1024 * 1024)} MiB`;
 var TITLES = {
   content: "Content without JavaScript",
@@ -695,21 +615,21 @@ function checkRobots({ page, robots }) {
   const out = robotsVerdict(parsed, pathAndQuery(page.finalUrl));
   return parsed.truncated ? { ...out, message: `${out.message} (${parsed.truncated})` } : out;
 }
-function robotsVerdict(parsed, path) {
-  if (!isAllowed(parsed, "*", path)) {
+function robotsVerdict(parsed, path2) {
+  if (!isAllowed(parsed, "*", path2)) {
     return fail(
-      `${ROBOTS_PATH} blocks ${path} for all crawlers`,
+      `${ROBOTS_PATH} blocks ${path2} for all crawlers`,
       "Remove the Disallow rule matching this path under User-agent: *."
     );
   }
-  const searchBlocked = AI_BOTS.search.filter((b) => !isAllowed(parsed, b, path));
+  const searchBlocked = AI_BOTS.search.filter((b) => !isAllowed(parsed, b, path2));
   if (searchBlocked.length > 0) {
     return warn(
       `AI search bots blocked: ${searchBlocked.join(", ")}`,
       'Allow AI search bots (robots.ai.search: "allow") so answer engines can cite this page.'
     );
   }
-  const trainingBlocked = AI_BOTS.training.filter((b) => !isAllowed(parsed, b, path));
+  const trainingBlocked = AI_BOTS.training.filter((b) => !isAllowed(parsed, b, path2));
   if (trainingBlocked.length > 0) {
     return pass(`Training crawlers blocked (${trainingBlocked.join(", ")}); search bots allowed`);
   }
@@ -875,7 +795,7 @@ async function auditWithHooks(url, opts = {}, hooks = {}) {
       return report;
     }
     const html = extractHtmlFacts(page.body);
-    const at = (path) => new URL(path, page.finalUrl).href;
+    const at = (path2) => new URL(path2, page.finalUrl).href;
     const getSameOrigin = (u, accept) => sameOrigin(u, page.finalUrl) ? get(u, accept) : Promise.resolve(notFetched(u));
     const alt = markdownAlternate(html);
     const altUrl = alt === null ? null : tryResolveUrl(alt, page.finalUrl);
@@ -945,8 +865,9 @@ function normalizeInputUrl(input) {
   return u.href;
 }
 var USAGE = `Usage: web-for-agents audit <url> [options]
+       web-for-agents generate <outDir> --site-url <url> [options]
 
-Audit how readable a page is for AI agents and crawlers.
+audit: check how readable a page is for AI agents and crawlers.
 
 Options:
   --json               print the report as JSON (page data is not sanitized; control
@@ -959,6 +880,22 @@ Options:
                        link-local address (default: follow them and print a warning)
   -h, --help           show this help
 
+generate: write llms.txt, llms-full.txt, .md pages, robots.txt, sitemap.xml and
+agents.json for HTML you prerendered yourself.
+
+Options:
+  --site-url <url>     absolute URL of the deployed site (or siteUrl in --config)
+  --ai-training <allow|block>
+  --ai-search <allow|block>
+                       robots.txt policy for AI crawlers; both are required here or
+                       as robots.ai in --config
+  --config <file>      .json, .js or .mjs file whose default export holds options
+                       (exclude, markdown, llms, agentMap, sitemap, robots, ...)
+  --no-link-markdown   do not add the markdown <link> to each page's <head>
+  --overwrite          replace an existing robots.txt, sitemap.xml, llms.txt,
+                       llms-full.txt or agents.json that differs
+  --json               print the result as JSON
+
 Exit codes: 0 ok, 1 score below --min-score, 2 usage error or fatal error
 `;
 function parseCliArgs(argv) {
@@ -973,6 +910,12 @@ function parseCliArgs(argv) {
         timeout: { type: "string" },
         "agent-map": { type: "string" },
         "no-private-redirects": { type: "boolean" },
+        "site-url": { type: "string" },
+        "ai-training": { type: "string" },
+        "ai-search": { type: "string" },
+        config: { type: "string" },
+        "no-link-markdown": { type: "boolean" },
+        overwrite: { type: "boolean" },
         help: { type: "boolean", short: "h" }
       }
     });
@@ -982,7 +925,34 @@ function parseCliArgs(argv) {
   const { values, positionals } = parsed;
   if (values.help || positionals.length === 0) return { command: "help", json: false, timeoutMs: DEFAULT_TIMEOUT_MS };
   const [command, input, ...rest] = positionals;
+  const GENERATE_FLAGS = ["site-url", "ai-training", "ai-search", "config", "no-link-markdown", "overwrite"];
+  const AUDIT_FLAGS = ["min-score", "timeout", "agent-map", "no-private-redirects"];
+  if (command === "generate") {
+    if (input === void 0) return fail2("generate requires an output directory");
+    if (rest.length > 0) return fail2(`unexpected argument "${rest[0]}"`);
+    const wrong2 = AUDIT_FLAGS.find((f) => values[f] !== void 0);
+    if (wrong2) return fail2(`--${wrong2} is not an option of generate`);
+    const gen = {
+      outDir: input,
+      linkMarkdown: values["no-link-markdown"] !== true,
+      overwrite: values.overwrite === true
+    };
+    for (const [flag, key] of [["ai-training", "aiTraining"], ["ai-search", "aiSearch"]]) {
+      const v = values[flag];
+      if (v === void 0) continue;
+      if (v !== "allow" && v !== "block") return fail2(`--${flag} must be "allow" or "block", got "${v}"`);
+      gen[key] = v;
+    }
+    if (values["site-url"] !== void 0) gen.siteUrl = values["site-url"];
+    if (values.config !== void 0) {
+      if (values.config.trim() === "") return fail2("--config must be a file path");
+      gen.config = values.config;
+    }
+    return { command: "generate", generate: gen, json: values.json === true, timeoutMs: DEFAULT_TIMEOUT_MS };
+  }
   if (command !== "audit") return fail2(`unknown command "${command}"`);
+  const wrong = GENERATE_FLAGS.find((f) => values[f] !== void 0);
+  if (wrong) return fail2(`--${wrong} is an option of generate, not audit`);
   if (input === void 0) return fail2("audit requires a URL");
   if (rest.length > 0) return fail2(`unexpected argument "${rest[0]}"`);
   const args = {
@@ -1020,15 +990,20 @@ function parseCliArgs(argv) {
   return args;
 }
 
+// src/cli/generate.ts
+import { existsSync, readFileSync } from "fs";
+import path from "path";
+import { pathToFileURL } from "url";
+
 // src/cli/terminal.ts
 var MAX_TEXT_CHARS = 500;
 var CONTROL = /[\u0000-\u001f\u007f-\u009f]/g;
 var BIDI_CHARS = "\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069";
 var BIDI = new RegExp(`[${BIDI_CHARS}]`, "g");
 function safeText(s, max = MAX_TEXT_CHARS) {
-  const clean = collapseWhitespace(s.replace(CONTROL, " ").replace(BIDI, ""));
-  const chars = Array.from(clean);
-  return chars.length > max ? chars.slice(0, max - 1).join("") + "\u2026" : clean;
+  const clean2 = collapseWhitespace(s.replace(CONTROL, " ").replace(BIDI, ""));
+  const chars = Array.from(clean2);
+  return chars.length > max ? chars.slice(0, max - 1).join("") + "\u2026" : clean2;
 }
 var JSON_UNSAFE = new RegExp(`[\\u007f-\\u009f\\u2028\\u2029${BIDI_CHARS}]`, "g");
 function safeJson(value, space) {
@@ -1038,6 +1013,75 @@ function safeJson(value, space) {
   );
 }
 
+// src/cli/generate.ts
+var VITE_ENTRY = "@anubhav-choubey/web-for-agents/vite";
+var clean = (s) => s.split("\n").map((l) => safeText(l, Infinity)).join("\n");
+var isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+async function loadConfig(file) {
+  const abs = path.resolve(file);
+  if (!existsSync(abs)) throw new Error(withPrefix(`config file not found: ${abs}`));
+  let config;
+  try {
+    if (abs.endsWith(".json")) config = JSON.parse(readFileSync(abs, "utf8"));
+    else if (/\.m?js$/.test(abs)) config = (await import(pathToFileURL(abs).href)).default;
+    else throw new Error("expected a .json, .js or .mjs file");
+  } catch (e) {
+    throw new Error(withPrefix(`could not load config ${abs}: ${errorMessage(e)}`), { cause: e });
+  }
+  if (!isObject(config)) throw new Error(withPrefix(`config ${abs} must export an options object`));
+  return config;
+}
+async function generateOptions(args) {
+  const config = args.config ? await loadConfig(args.config) : {};
+  const robots = isObject(config.robots) ? config.robots : {};
+  const ai = { ...isObject(robots.ai) ? robots.ai : {} };
+  if (args.aiTraining) ai.training = args.aiTraining;
+  if (args.aiSearch) ai.search = args.aiSearch;
+  if (ai.training === void 0 || ai.search === void 0) {
+    throw new Error(withPrefix("--ai-training and --ai-search are required (or robots.ai in --config)"));
+  }
+  const siteUrl = args.siteUrl ?? config.siteUrl;
+  if (siteUrl === void 0) throw new Error(withPrefix("--site-url is required (or siteUrl in --config)"));
+  return {
+    ...config,
+    outDir: args.outDir,
+    siteUrl,
+    robots: { ...robots, ai },
+    ...args.linkMarkdown ? {} : { linkMarkdown: false },
+    ...args.overwrite ? { overwrite: true } : {}
+  };
+}
+async function runGenerate(args, json, io) {
+  let result;
+  try {
+    const options = await generateOptions(args);
+    const { generateAgentFiles } = await import(VITE_ENTRY);
+    result = await generateAgentFiles({
+      ...options,
+      logger: {
+        warn: (m) => io.stderr(clean(m) + "\n"),
+        info: (m) => (json ? io.stderr : io.stdout)(clean(m) + "\n")
+      }
+    });
+  } catch (e) {
+    io.stderr(withPrefix(`generate failed: ${clean(errorMessage(e).replace(/^\[web-for-agents\] /, ""))}
+`));
+    return 2;
+  }
+  if (json) {
+    io.stdout(safeJson(result, 2) + "\n");
+  } else {
+    const lines = [
+      `Layout: ${result.fileLayout}`,
+      `Pages: ${result.pages.map((p) => p.route + (p.listed ? "" : " (noindex)")).join(", ")}`,
+      `Wrote ${result.written.length} files: ${result.written.join(", ")}`
+    ];
+    if (result.kept.length > 0) lines.push(`Kept existing: ${result.kept.join(", ")} (use --overwrite to replace)`);
+    io.stdout(clean(lines.join("\n")) + "\n");
+  }
+  return 0;
+}
+
 // src/cli/report.ts
 function formatText(report) {
   const lines = [
@@ -1077,6 +1121,7 @@ ${USAGE}`);
     io.stdout(USAGE);
     return 0;
   }
+  if (args.command === "generate") return runGenerate(args.generate, args.json, io);
   const report = await audit(args.url, {
     timeoutMs: args.timeoutMs,
     ...args.agentMap === void 0 ? {} : { agentMapPath: args.agentMap },

package/dist/{chunk-5TNG4QMP.js → chunk-PGZH33B5.js}

@@ -52,15 +52,29 @@ function withoutBlockedLoads(errors, blocked) {
 function readyTimeoutMessage(timeoutMs) {
   return `did not become ready within ${timeoutMs}ms`;
 }
+var TimeoutError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "TimeoutError";
+  }
+};
+var SNAPSHOT_GRACE_MS = 5e3;
+function snapshotWarning(timeoutMs) {
+  return `${readyTimeoutMessage(timeoutMs)}; saved the page as it was at the deadline (prerender.onTimeout: "snapshot"). If it is still loading or animating, set prerender.readySelector to an element that marks it ready`;
+}
 var POLL_MS = 50;
 var SETTLED_POLL_MS = 20;
+async function assertNoPageErrors(readErrors) {
+  const errors = await readErrors();
+  if (errors.length) throw new Error(errors[0]);
+}
 async function waitForReady(check, timeoutMs, pollMs = POLL_MS) {
   const deadline = Date.now() + timeoutMs;
   for (; ; ) {
     const { ready, errors } = await check();
     if (errors.length) throw new Error(errors[0]);
     if (ready) return;
-    if (Date.now() >= deadline) throw new Error(readyTimeoutMessage(timeoutMs));
+    if (Date.now() >= deadline) throw new TimeoutError(readyTimeoutMessage(timeoutMs));
     await new Promise((r) => setTimeout(r, pollMs));
   }
 }
@@ -72,7 +86,7 @@ function createDeadline(timeoutMs) {
     race(work, message) {
       let timer;
       const expired = new Promise((_, reject) => {
-        timer = setTimeout(() => reject(new Error(message)), remaining());
+        timer = setTimeout(() => reject(new TimeoutError(message)), remaining());
       });
       work.catch(() => {
       });
@@ -103,8 +117,12 @@ export {
   blockedHint,
   withoutBlockedLoads,
   readyTimeoutMessage,
+  TimeoutError,
+  SNAPSHOT_GRACE_MS,
+  snapshotWarning,
   POLL_MS,
   SETTLED_POLL_MS,
+  assertNoPageErrors,
   waitForReady,
   createDeadline,
   runInPage

package/dist/chunk-PHTV5NAY.js

@@ -0,0 +1,44 @@
+// src/shared/metaLength.ts
+var TITLE_MAX_CHARS = 60;
+var DESCRIPTION_MIN_CHARS = 50;
+var DESCRIPTION_MAX_CHARS = 160;
+function metaLengthWarnings(pages) {
+  const longTitle = [];
+  const shortDesc = [];
+  const longDesc = [];
+  for (const { route, title, description } of pages) {
+    if (title && title.length > TITLE_MAX_CHARS) longTitle.push(`${route} (${title.length})`);
+    if (!description) continue;
+    if (description.length < DESCRIPTION_MIN_CHARS) shortDesc.push(`${route} (${description.length})`);
+    else if (description.length > DESCRIPTION_MAX_CHARS) longDesc.push(`${route} (${description.length})`);
+  }
+  const out = [];
+  if (longTitle.length > 0) {
+    out.push(`[web-for-agents] Title longer than ${TITLE_MAX_CHARS} characters on ${longTitle.join(", ")}`);
+  }
+  if (shortDesc.length > 0) {
+    out.push(`[web-for-agents] Meta description shorter than ${DESCRIPTION_MIN_CHARS} characters on ${shortDesc.join(", ")}`);
+  }
+  if (longDesc.length > 0) {
+    out.push(`[web-for-agents] Meta description longer than ${DESCRIPTION_MAX_CHARS} characters on ${longDesc.join(", ")}`);
+  }
+  return out;
+}
+
+// src/shared/agent.ts
+var ACTION_NAME_RE = /^[A-Za-z0-9_.-]{1,128}$/;
+function isValidActionName(name) {
+  return typeof name === "string" && ACTION_NAME_RE.test(name);
+}
+var AGENT_GLOBAL_VERSION = 1;
+var AGENT_MAP_VERSION = 1;
+
+export {
+  TITLE_MAX_CHARS,
+  DESCRIPTION_MIN_CHARS,
+  DESCRIPTION_MAX_CHARS,
+  metaLengthWarnings,
+  isValidActionName,
+  AGENT_GLOBAL_VERSION,
+  AGENT_MAP_VERSION
+};