@anubhav-choubey/web-for-agents 0.0.0-stage → 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Anubhav Choubey
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,169 @@
-# Temporary Holding Version
+# @anubhav-choubey/web-for-agents
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[![CI](https://github.com/LazyyVenom/Web-For-Agents/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/LazyyVenom/Web-For-Agents/actions/workflows/ci.yml)
+
+SEO and LLM-readiness helpers for React + Vite. At runtime it manages `<head>` metadata (title, description, canonical, Open Graph, Twitter) and emits typed JSON-LD. At build time a Vite plugin prerenders every route to static HTML and writes a markdown version of each page, `llms.txt`, `llms-full.txt`, an `agents.json` map of the actions agents can take, a `sitemap.xml`, and a `robots.txt` with an explicit AI-crawler policy, so you decide which bots may train on your content and which may use it for search.
+
+## Install
+
+```sh
+npm i @anubhav-choubey/web-for-agents
+```
+
+Requires Node 20.19 or later; Node 22.13 or later is recommended for builds, because prerendering is better isolated there (see [Security](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/security.md)). The runtime needs `react >=18` and `react-dom >=18`, and the Vite plugin needs `vite >=5` (tested on Vite 5 to 8). All three are optional peers, so the CLI works without them.
+
+## Quick start
+
+```ts
+// vite.config.ts
+import { defineConfig } from "vite";
+import react from "@vitejs/plugin-react";
+import { webForAgents } from "@anubhav-choubey/web-for-agents/vite";
+
+export default defineConfig({
+  plugins: [
+    react(),
+    webForAgents({
+      siteUrl: "https://example.com",
+      routes: ["/about", "/blog/hello"],
+      robots: { ai: { training: "block", search: "allow" } },
+    }),
+  ],
+});
+```
+
+```tsx
+import { SeoProvider, Seo, JsonLd, schema } from "@anubhav-choubey/web-for-agents";
+
+export function Root({ children }: { children: React.ReactNode }) {
+  return (
+    <SeoProvider defaults={{ siteName: "My Site", siteUrl: "https://example.com", titleTemplate: "%s | My Site" }}>
+      {children}
+    </SeoProvider>
+  );
+}
+
+export function Post() {
+  return (
+    <>
+      <Seo title="Hello world" description="My first post" canonical="/blog/hello" type="article" />
+      <JsonLd data={schema.article({ headline: "Hello world", datePublished: "2026-10-28" })} />
+    </>
+  );
+}
+```
+
+`robots.ai` is required: the plugin throws if you do not state a policy for both `training` and `search`. Use one `SeoProvider` per document. More in [Head tags and JSON-LD](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/seo.md).
+
+## What the build produces
+
+After `vite build`, the plugin crawls from `/` plus `routes` and writes:
+
+| File | Contents |
+| --- | --- |
+| `dist/<route>/index.html` | Prerendered HTML with the full `<head>`; your client bundle still boots. |
+| `dist/<route>.md` | Markdown version of the page (`dist/index.md` for `/`). |
+| `dist/200.html` | The untouched SPA shell, for host fallbacks. |
+| `llms.txt`, `llms-full.txt` | Page index linking to the `.md` files, and all pages' markdown in one file. |
+| `agents.json` | Every page and the agent actions on it. |
+| `sitemap.xml`, `robots.txt` | Sitemap of discovered routes, and your AI-crawler policy. |
+
+Already have your own prerender script? Skip the plugin's prerender and run `npx web-for-agents generate dist --site-url https://example.com --ai-training block --ai-search allow` after it. It adds the same `.md`, `llms.txt`, `robots.txt`, `sitemap.xml` and `agents.json` files to your HTML. See [Using your own prerender](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md#using-your-own-prerender).
+
+Files you place in `public/` are never overwritten. Content regions, `holdPrerender`, `hydrateApp`, host fallback configs and memory limits: see [Prerendering](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md).
+
+## Agent actions
+
+Mark what an agent can do on a page. Each action is listed in `agents.json` and, in browsers that support it, registered as a [WebMCP](https://webmachinelearning.github.io/webmcp/) tool.
+
+```tsx
+import { useAgentAction } from "@anubhav-choubey/web-for-agents";
+
+function DeleteAccount() {
+  const { ref, hintProps } = useAgentAction({
+    name: "delete_account",
+    description: "Delete the account",
+    destructive: true,
+    handler: async () => deleteAccount(),
+  });
+  return (
+    <button ref={ref} {...hintProps} onClick={() => deleteAccount()}>
+      Delete account
+    </button>
+  );
+}
+```
+
+`AgentForm` turns a form into an action; its schema comes from the fields, and the agent submits through your own `onSubmit`:
+
+```tsx
+import { AgentForm, agentParam } from "@anubhav-choubey/web-for-agents";
+
+<AgentForm agent={{ name: "contact", description: "Send us a message" }} onSubmit={send}>
+  <input type="email" name="email" required {...agentParam("Your email address")} />
+  <button type="submit">Send</button>
+</AgentForm>
+```
+
+Agents can never reach hidden, password, file, disabled or `readonly` fields, or fields hidden by markup. Full API, confirm dialog and field rules: [Agent actions](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/agent-actions.md).
+
+## CLI
+
+Audit any URL for AI-agent readiness, with nothing installed in the target site:
+
+```sh
+npx @anubhav-choubey/web-for-agents audit https://example.com --min-score 80
+```
+
+It scores 0 to 100 across content without JavaScript, JSON-LD, robots.txt, title, meta description, sitemap, Open Graph, llms.txt, the markdown copy, canonical URL and the agent map, and prints a `Fix:` line for each miss. Flags, weights and JSON output: [CLI](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/cli.md).
+
+## Dev overlay
+
+While `vite` (the dev server) runs, a "WFA" button on every page (or `Alt+Shift+A`) opens a panel showing what an agent sees: head tags, JSON-LD, the generated markdown and agent actions. It is never included in `vite build`. See [Dev overlay](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/overlay.md).
+
+## Security
+
+- Prerendering runs your app's code at build time. happy-dom is **not a sandbox**; it renders in a separate, locked-down Node.js process with a scrubbed environment.
+- During the build, pages can only load your build's own files unless you allow other origins with `prerender.allowOrigins`.
+- Build on Node.js 22.13+ for the permission model, or use the Chrome renderer for untrusted scripts.
+- Agent input is validated strictly against the action's schema, and destructive actions ask the user first.
+- The CLI strips terminal control and bidi characters from site data in its text report.
+
+Details: [Security](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/security.md).
+
+## Key options
+
+| Option | Description |
+| --- | --- |
+| `siteUrl` | Required. Absolute http(s) URL of the deployed site. |
+| `robots.ai.training` / `robots.ai.search` | Required. `"allow"` or `"block"` for AI training and AI search crawlers. |
+| `routes` | Extra routes to prerender, for pages no link reaches. |
+| `exclude` | Routes to leave out (exact path, `/*` prefix or `RegExp`). |
+| `prerender` | Renderer, timeouts, concurrency, `allowOrigins`; `false` turns off prerender, markdown, llms and the agent map. |
+| `markdown`, `llms`, `agentMap`, `sitemap`, `overlay` | Turn each output off with `false`, or configure it. |
+
+All options and the AI crawler list: [Plugin options](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/options.md).
+
+## Documentation
+
+- [Plugin options](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/options.md)
+- [Head tags and JSON-LD](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/seo.md)
+- [Prerendering](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md)
+- [Agent actions](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/agent-actions.md)
+- [CLI](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/cli.md)
+- [Dev overlay](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/overlay.md)
+- [Security](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/security.md)
+- [Advanced exports](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/advanced.md)
+- [Changelog](https://github.com/LazyyVenom/Web-For-Agents/blob/master/CHANGELOG.md)
+
+## Development
+
+```sh
+npm ci && npm test && npm run build
+```
+
+Scripts, CI and testing against other Vite majors: [Development](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/development.md).
+
+## License
+
+MIT

package/dist/bin.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import {
+  runCli
+} from "./chunk-P6JSPAA2.js";
+import "./chunk-CGIIB6AF.js";
+import "./chunk-PHTV5NAY.js";
+import "./chunk-LCLDQ3A3.js";
+import "./chunk-FEAKVNMG.js";
+import "./chunk-F7PN5R25.js";
+import "./chunk-JJN6FNJD.js";
+
+// src/cli/bin.ts
+process.exitCode = await runCli(process.argv.slice(2));

package/dist/chunk-AAIYCJD3.js

@@ -0,0 +1,8 @@
+// src/shared/sort.ts
+function compareStrings(a, b) {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
+export {
+  compareStrings
+};

package/dist/chunk-CGIIB6AF.js

@@ -0,0 +1,127 @@
+// src/shared/bots.ts
+var AI_BOTS = {
+  training: [
+    "GPTBot",
+    "ClaudeBot",
+    "CCBot",
+    "Google-Extended",
+    "Applebot-Extended",
+    "Meta-ExternalAgent",
+    "Bytespider",
+    "cohere-training-data-crawler"
+  ],
+  search: [
+    "OAI-SearchBot",
+    "ChatGPT-User",
+    "Claude-SearchBot",
+    "Claude-User",
+    "PerplexityBot",
+    "Perplexity-User"
+  ]
+};
+
+// src/shared/htmlScan.ts
+var isSpace = (c) => c === 32 || c === 9 || c === 10 || c === 12 || c === 13;
+var isAlpha = (c) => c >= 65 && c <= 90 || c >= 97 && c <= 122;
+function findRawEnd(html, name, from) {
+  let i = html.indexOf("</", from);
+  while (i !== -1) {
+    if (html.slice(i + 2, i + 2 + name.length).toLowerCase() === name) return i;
+    i = html.indexOf("</", i + 2);
+  }
+  return -1;
+}
+function* scanHtml(html, rawText) {
+  const n = html.length;
+  let i = 0;
+  while (i < n) {
+    const lt = html.indexOf("<", i);
+    if (lt === -1) return;
+    const c1 = lt + 1 < n ? html.charCodeAt(lt + 1) : 0;
+    if (c1 === 33 || c1 === 63) {
+      let end;
+      if (html.startsWith("<!--", lt)) {
+        const close = html.indexOf("-->", lt + 4);
+        end = close === -1 ? n : close + 3;
+      } else {
+        const close = html.indexOf(">", lt + 2);
+        end = close === -1 ? n : close + 1;
+      }
+      yield { type: "comment", start: lt, end };
+      i = end;
+      continue;
+    }
+    const closing = c1 === 47;
+    const nameStart = lt + (closing ? 2 : 1);
+    if (!isAlpha(nameStart < n ? html.charCodeAt(nameStart) : 0)) {
+      i = lt + 1;
+      continue;
+    }
+    let p = nameStart;
+    while (p < n) {
+      const c = html.charCodeAt(p);
+      if (isSpace(c) || c === 62 || c === 47) break;
+      p++;
+    }
+    const name = html.slice(nameStart, p).toLowerCase();
+    const attrs = [];
+    let selfClosing = false;
+    while (p < n) {
+      const c = html.charCodeAt(p);
+      if (isSpace(c)) {
+        p++;
+        continue;
+      }
+      if (c === 62) {
+        p++;
+        break;
+      }
+      if (c === 47) {
+        selfClosing = p + 1 < n && html.charCodeAt(p + 1) === 62;
+        p++;
+        continue;
+      }
+      const as = p;
+      while (p < n) {
+        const d = html.charCodeAt(p);
+        if (isSpace(d) || d === 62 || d === 47 || d === 61) break;
+        p++;
+      }
+      if (p === as) {
+        p++;
+        continue;
+      }
+      const attr = { name: html.slice(as, p).toLowerCase(), value: "", valueStart: -1, valueEnd: -1, quote: "" };
+      while (p < n && isSpace(html.charCodeAt(p))) p++;
+      if (p < n && html.charCodeAt(p) === 61) {
+        p++;
+        while (p < n && isSpace(html.charCodeAt(p))) p++;
+        const q = p < n ? html[p] : "";
+        if (q === '"' || q === "'") {
+          const close = html.indexOf(q, p + 1);
+          const stop = close === -1 ? n : close;
+          Object.assign(attr, { value: html.slice(p + 1, stop), valueStart: p + 1, valueEnd: stop, quote: q });
+          p = close === -1 ? n : close + 1;
+        } else {
+          const vs = p;
+          while (p < n && !isSpace(html.charCodeAt(p)) && html.charCodeAt(p) !== 62) p++;
+          Object.assign(attr, { value: html.slice(vs, p), valueStart: vs, valueEnd: p });
+        }
+      }
+      attrs.push(attr);
+    }
+    yield { type: "tag", closing, name, attrs, selfClosing, start: lt, end: p };
+    i = p;
+    if (!closing && !selfClosing && rawText.has(name)) {
+      const end = findRawEnd(html, name, i);
+      const stop = end === -1 ? n : end;
+      yield { type: "raw", name, start: i, end: stop };
+      i = stop;
+    }
+  }
+}
+
+export {
+  AI_BOTS,
+  scanHtml
+};

package/dist/chunk-F7PN5R25.js

@@ -0,0 +1,95 @@
+// src/shared/url.ts
+function normalizeRoute(path) {
+  let p = path.split("#")[0].split("?")[0];
+  if (!p.startsWith("/")) p = "/" + p;
+  if (p.length > 1) p = p.replace(/\/+$/, "");
+  return p === "" ? "/" : p;
+}
+function trimTrailingSlash(s) {
+  return s.replace(/\/+$/, "");
+}
+function hasScheme(s) {
+  return /^[a-z][a-z0-9+.-]*:/i.test(s);
+}
+function tryResolveUrl(value, base) {
+  try {
+    return new URL(value, base).href;
+  } catch {
+    return null;
+  }
+}
+function resolveUrl(value, base) {
+  if (!base || !value) return value;
+  return tryResolveUrl(value, base) ?? value;
+}
+function mountPath(base) {
+  return base.startsWith("/") && !base.startsWith("//") ? trimTrailingSlash(base) : "";
+}
+function routeOf(pathname, base = "/") {
+  const b = mountPath(base);
+  if (b !== "" && (pathname === b || pathname.startsWith(b + "/"))) pathname = pathname.slice(b.length);
+  return normalizeRoute(pathname);
+}
+function publicUrl(siteUrl, route) {
+  return trimTrailingSlash(siteUrl) + normalizeRoute(route);
+}
+
+// src/shared/conventions.ts
+var ROBOTS_PATH = "/robots.txt";
+var SITEMAP_PATH = "/sitemap.xml";
+var LLMS_TXT_PATH = "/llms.txt";
+var LLMS_FULL_TXT_PATH = "/llms-full.txt";
+var AGENT_MAP_PATH = "/agents.json";
+var MARKDOWN_MIME = "text/markdown";
+var CONTENT_ATTR = "data-wfa-content";
+var EXCLUDE_ATTR = "data-wfa-exclude";
+var PRERENDERED_ATTR = "data-wfa-prerendered";
+var READY_GLOBAL = "__WFA_READY__";
+var HIDDEN_SELECTOR = '[hidden], [aria-hidden="true"]';
+var HEAD_OWNED_ATTR = "data-wfa";
+var MARKDOWN_LINK_KEY = "markdown";
+function markdownPathFor(pathname) {
+  const p = normalizeRoute(pathname);
+  return p === "/" ? "/index.md" : p + ".md";
+}
+var ACTION_ATTR = "data-agent-action";
+var ACTION_DESCRIPTION_ATTR = "data-agent-description";
+var ACTION_DESTRUCTIVE_ATTR = "data-agent-destructive";
+var ACTION_PARAM_ATTR = "data-agent-param";
+var WEBMCP_TOOLNAME_ATTR = "toolname";
+var WEBMCP_TOOLDESCRIPTION_ATTR = "tooldescription";
+var WEBMCP_TOOLPARAMDESCRIPTION_ATTR = "toolparamdescription";
+var AGENT_GLOBAL = "__WEB_FOR_AGENTS__";
+
+export {
+  normalizeRoute,
+  trimTrailingSlash,
+  hasScheme,
+  tryResolveUrl,
+  resolveUrl,
+  mountPath,
+  routeOf,
+  publicUrl,
+  ROBOTS_PATH,
+  SITEMAP_PATH,
+  LLMS_TXT_PATH,
+  LLMS_FULL_TXT_PATH,
+  AGENT_MAP_PATH,
+  MARKDOWN_MIME,
+  CONTENT_ATTR,
+  EXCLUDE_ATTR,
+  PRERENDERED_ATTR,
+  READY_GLOBAL,
+  HIDDEN_SELECTOR,
+  HEAD_OWNED_ATTR,
+  MARKDOWN_LINK_KEY,
+  markdownPathFor,
+  ACTION_ATTR,
+  ACTION_DESCRIPTION_ATTR,
+  ACTION_DESTRUCTIVE_ATTR,
+  ACTION_PARAM_ATTR,
+  WEBMCP_TOOLNAME_ATTR,
+  WEBMCP_TOOLDESCRIPTION_ATTR,
+  WEBMCP_TOOLPARAMDESCRIPTION_ATTR,
+  AGENT_GLOBAL
+};

package/dist/chunk-FEAKVNMG.js

@@ -0,0 +1,12 @@
+// src/shared/text.ts
+function collapseWhitespace(s) {
+  return (s ?? "").replace(/\s+/g, " ").trim();
+}
+function stripBom(s) {
+  return s.charCodeAt(0) === 65279 ? s.slice(1) : s;
+}
+
+export {
+  collapseWhitespace,
+  stripBom
+};

package/dist/chunk-JJN6FNJD.js

@@ -0,0 +1,20 @@
+// src/shared/errors.ts
+var LOG_PREFIX = "[web-for-agents]";
+function withPrefix(msg) {
+  return msg.startsWith(LOG_PREFIX) ? msg : `${LOG_PREFIX} ${msg}`;
+}
+function errorMessage(e) {
+  if (e instanceof Error) return e.message;
+  if (typeof e === "string") return e;
+  try {
+    return String(e);
+  } catch {
+    return "unknown error";
+  }
+}
+
+export {
+  LOG_PREFIX,
+  withPrefix,
+  errorMessage
+};

package/dist/chunk-LCLDQ3A3.js

@@ -0,0 +1,26 @@
+// src/shared/jsonLd.ts
+var MAX_NODES = 1e5;
+function jsonLdTypes(value) {
+  const out = [];
+  const stack = [value];
+  for (let visited = 0; stack.length > 0 && visited < MAX_NODES; visited++) {
+    const v = stack.pop();
+    if (Array.isArray(v)) {
+      for (let i = v.length - 1; i >= 0; i--) stack.push(v[i]);
+      continue;
+    }
+    if (v === null || typeof v !== "object") continue;
+    const o = v;
+    const t = o["@type"];
+    if (typeof t === "string") out.push(t);
+    else if (Array.isArray(t)) {
+      for (const x of t) if (typeof x === "string") out.push(x);
+    }
+    if ("@graph" in o) stack.push(o["@graph"]);
+  }
+  return out;
+}
+
+export {
+  jsonLdTypes
+};