@anubhav-choubey/web-for-agents 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/bin.js +5 -5
- package/dist/{chunk-F7PN5R25.js → chunk-2Y3LLYEC.js} +3 -1
- package/dist/{chunk-P6JSPAA2.js → chunk-357ATBMS.js} +207 -13
- package/dist/chunk-4STTPCBR.js +25 -0
- package/dist/{chunk-SMFRAZAA.js → chunk-K3B53RIO.js} +1 -1
- package/dist/{chunk-PHTV5NAY.js → chunk-L3AYWCSX.js} +1 -12
- package/dist/chunk-MIVGE5TM.js +34 -0
- package/dist/{chunk-LCVPXUFN.js → chunk-OKCYZE6C.js} +2 -2
- package/dist/{chunk-PSNLS4GL.js → chunk-T42S3WRV.js} +99 -36
- package/dist/cli.js +5 -5
- package/dist/{happyDom-WTYQ5UAC.js → happyDom-ZEFNMD7B.js} +2 -2
- package/dist/index.d.ts +89 -11
- package/dist/index.js +80 -19
- package/dist/overlay.js +22 -17
- package/dist/{renderChild-H6TR4SGV.js → renderChild-PSQLAIUW.js} +1 -1
- package/dist/vite.d.ts +30 -3
- package/dist/vite.js +265 -72
- package/package.json +1 -1
- package/dist/chunk-FEAKVNMG.js +0 -12
- package/dist/chunk-LCLDQ3A3.js +0 -26
package/README.md
CHANGED
|
@@ -53,6 +53,8 @@ export function Post() {
|
|
|
53
53
|
}
|
|
54
54
|
```
|
|
55
55
|
|
|
56
|
+
Builders also cover `person`, `softwareApplication`, `profilePage`, `creativeWork`, `webPage` and `graph` (several nodes in one `@graph`); see [Schema builders](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/seo.md#schema-builders).
|
|
57
|
+
|
|
56
58
|
`robots.ai` is required: the plugin throws if you do not state a policy for both `training` and `search`. Use one `SeoProvider` per document. More in [Head tags and JSON-LD](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/seo.md).
|
|
57
59
|
|
|
58
60
|
## What the build produces
|
|
@@ -70,6 +72,8 @@ After `vite build`, the plugin crawls from `/` plus `routes` and writes:
|
|
|
70
72
|
|
|
71
73
|
Already have your own prerender script? Skip the plugin's prerender and run `npx web-for-agents generate dist --site-url https://example.com --ai-training block --ai-search allow` after it. It adds the same `.md`, `llms.txt`, `robots.txt`, `sitemap.xml` and `agents.json` files to your HTML. See [Using your own prerender](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md#using-your-own-prerender).
|
|
72
74
|
|
|
75
|
+
Put an "About me" summary, key links or featured projects at the top of `llms.txt` with `llms.intro` and `llms.sections`, and add `<lastmod>` dates to the sitemap with `sitemap: { lastmod: "file" }`. See [Custom sections](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md#custom-sections) and [Sitemap lastmod](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md#sitemap-lastmod).
|
|
76
|
+
|
|
73
77
|
Files you place in `public/` are never overwritten. Content regions, `holdPrerender`, `hydrateApp`, host fallback configs and memory limits: see [Prerendering](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/prerendering.md).
|
|
74
78
|
|
|
75
79
|
## Agent actions
|
|
@@ -105,6 +109,8 @@ import { AgentForm, agentParam } from "@anubhav-choubey/web-for-agents";
|
|
|
105
109
|
</AgentForm>
|
|
106
110
|
```
|
|
107
111
|
|
|
112
|
+
Plain links work too, without JavaScript: `<a href="/resume.pdf" data-agent-action="open_resume" data-agent-description="Open the resume">` (or `agentLinkProps`) is listed in `agents.json` as a `link` action. See [Link actions](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/agent-actions.md#link-actions-no-javascript).
|
|
113
|
+
|
|
108
114
|
Agents can never reach hidden, password, file, disabled or `readonly` fields, or fields hidden by markup. Full API, confirm dialog and field rules: [Agent actions](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/agent-actions.md).
|
|
109
115
|
|
|
110
116
|
## CLI
|
|
@@ -115,7 +121,7 @@ Audit any URL for AI-agent readiness, with nothing installed in the target site:
|
|
|
115
121
|
npx @anubhav-choubey/web-for-agents audit https://example.com --min-score 80
|
|
116
122
|
```
|
|
117
123
|
|
|
118
|
-
It scores 0 to 100 across content without JavaScript, JSON-LD, robots.txt, title, meta description, sitemap, Open Graph, llms.txt, the markdown copy, canonical URL and the agent map, and prints a `Fix:` line for each miss. Flags, weights and JSON output: [CLI](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/cli.md).
|
|
124
|
+
It scores 0 to 100 across content without JavaScript, JSON-LD, robots.txt, title, meta description, sitemap, Open Graph, llms.txt, the markdown copy, canonical URL and the agent map, and prints a `Fix:` line for each miss. The JSON-LD check names missing required rich-result fields (Product, BreadcrumbList, Event, SoftwareApplication and more). Flags, weights and JSON output: [CLI](https://github.com/LazyyVenom/Web-For-Agents/blob/master/docs/cli.md).
|
|
119
125
|
|
|
120
126
|
## Dev overlay
|
|
121
127
|
|
package/dist/bin.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runCli
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-357ATBMS.js";
|
|
5
5
|
import "./chunk-CGIIB6AF.js";
|
|
6
|
-
import "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
6
|
+
import "./chunk-L3AYWCSX.js";
|
|
7
|
+
import "./chunk-MIVGE5TM.js";
|
|
8
|
+
import "./chunk-4STTPCBR.js";
|
|
9
|
+
import "./chunk-2Y3LLYEC.js";
|
|
10
10
|
import "./chunk-JJN6FNJD.js";
|
|
11
11
|
|
|
12
12
|
// src/cli/bin.ts
|
|
@@ -60,6 +60,7 @@ var WEBMCP_TOOLNAME_ATTR = "toolname";
|
|
|
60
60
|
var WEBMCP_TOOLDESCRIPTION_ATTR = "tooldescription";
|
|
61
61
|
var WEBMCP_TOOLPARAMDESCRIPTION_ATTR = "toolparamdescription";
|
|
62
62
|
var AGENT_GLOBAL = "__WEB_FOR_AGENTS__";
|
|
63
|
+
var LINK_ACTION_SELECTOR = `a[href][${ACTION_ATTR}]`;
|
|
63
64
|
|
|
64
65
|
export {
|
|
65
66
|
normalizeRoute,
|
|
@@ -91,5 +92,6 @@ export {
|
|
|
91
92
|
WEBMCP_TOOLNAME_ATTR,
|
|
92
93
|
WEBMCP_TOOLDESCRIPTION_ATTR,
|
|
93
94
|
WEBMCP_TOOLPARAMDESCRIPTION_ATTR,
|
|
94
|
-
AGENT_GLOBAL
|
|
95
|
+
AGENT_GLOBAL,
|
|
96
|
+
LINK_ACTION_SELECTOR
|
|
95
97
|
};
|
|
@@ -3,18 +3,21 @@ import {
|
|
|
3
3
|
scanHtml
|
|
4
4
|
} from "./chunk-CGIIB6AF.js";
|
|
5
5
|
import {
|
|
6
|
-
AGENT_MAP_VERSION,
|
|
7
6
|
DESCRIPTION_MAX_CHARS,
|
|
8
7
|
DESCRIPTION_MIN_CHARS,
|
|
9
8
|
TITLE_MAX_CHARS
|
|
10
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-L3AYWCSX.js";
|
|
11
10
|
import {
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
MAX_JSON_LD_NODES,
|
|
12
|
+
jsonLdNodes,
|
|
13
|
+
jsonLdTypes,
|
|
14
|
+
nodeTypes
|
|
15
|
+
} from "./chunk-MIVGE5TM.js";
|
|
14
16
|
import {
|
|
17
|
+
AGENT_MAP_VERSION,
|
|
15
18
|
collapseWhitespace,
|
|
16
19
|
stripBom
|
|
17
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-4STTPCBR.js";
|
|
18
21
|
import {
|
|
19
22
|
AGENT_MAP_PATH,
|
|
20
23
|
LLMS_TXT_PATH,
|
|
@@ -26,12 +29,177 @@ import {
|
|
|
26
29
|
markdownPathFor,
|
|
27
30
|
trimTrailingSlash,
|
|
28
31
|
tryResolveUrl
|
|
29
|
-
} from "./chunk-
|
|
32
|
+
} from "./chunk-2Y3LLYEC.js";
|
|
30
33
|
import {
|
|
31
34
|
errorMessage,
|
|
32
35
|
withPrefix
|
|
33
36
|
} from "./chunk-JJN6FNJD.js";
|
|
34
37
|
|
|
38
|
+
// src/shared/schemaRules.ts
|
|
39
|
+
var DOCS = "https://developers.google.com/search/docs/appearance/structured-data/";
|
|
40
|
+
var req = (...anyOf) => ({ anyOf });
|
|
41
|
+
var ARTICLE = {
|
|
42
|
+
docs: `${DOCS}article`,
|
|
43
|
+
required: [],
|
|
44
|
+
recommended: ["author", "author.name", "author.url", "dateModified", "datePublished", "headline", "image"]
|
|
45
|
+
};
|
|
46
|
+
var SOFTWARE_APPLICATION = {
|
|
47
|
+
docs: `${DOCS}software-app`,
|
|
48
|
+
required: [req("name"), req("offers"), req("aggregateRating", "review")],
|
|
49
|
+
recommended: ["applicationCategory", "operatingSystem"],
|
|
50
|
+
nested: [{ prop: "offers", whenType: ["Offer"], orUntyped: true, required: [req("price")] }]
|
|
51
|
+
};
|
|
52
|
+
var SCHEMA_RULES = {
|
|
53
|
+
Article: ARTICLE,
|
|
54
|
+
NewsArticle: ARTICLE,
|
|
55
|
+
BlogPosting: ARTICLE,
|
|
56
|
+
BreadcrumbList: {
|
|
57
|
+
docs: `${DOCS}breadcrumb`,
|
|
58
|
+
required: [req("itemListElement")],
|
|
59
|
+
recommended: [],
|
|
60
|
+
nested: [
|
|
61
|
+
{
|
|
62
|
+
prop: "itemListElement",
|
|
63
|
+
required: [req("position"), req("name", "item.name"), { anyOf: ["item"], exceptLast: true }]
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
},
|
|
67
|
+
Event: {
|
|
68
|
+
docs: `${DOCS}event`,
|
|
69
|
+
required: [req("name"), req("startDate"), req("location")],
|
|
70
|
+
recommended: ["description", "endDate", "eventStatus", "image", "location.name", "offers", "organizer", "performer"],
|
|
71
|
+
nested: [{ prop: "location", whenType: ["Place"], required: [req("address")] }]
|
|
72
|
+
},
|
|
73
|
+
FAQPage: {
|
|
74
|
+
docs: `${DOCS}faqpage`,
|
|
75
|
+
required: [req("mainEntity")],
|
|
76
|
+
recommended: [],
|
|
77
|
+
nested: [
|
|
78
|
+
{
|
|
79
|
+
prop: "mainEntity",
|
|
80
|
+
required: [req("name"), req("acceptedAnswer")],
|
|
81
|
+
nested: [{ prop: "acceptedAnswer", required: [req("text")] }]
|
|
82
|
+
}
|
|
83
|
+
],
|
|
84
|
+
note: "Google no longer shows FAQ rich results; answer engines may still use it",
|
|
85
|
+
retired: true
|
|
86
|
+
},
|
|
87
|
+
Organization: {
|
|
88
|
+
docs: `${DOCS}organization`,
|
|
89
|
+
required: [],
|
|
90
|
+
recommended: ["name", "url", "logo", "description", "sameAs", "address", "contactPoint", "email", "telephone"]
|
|
91
|
+
},
|
|
92
|
+
Person: {
|
|
93
|
+
docs: `${DOCS}profile-page`,
|
|
94
|
+
required: [],
|
|
95
|
+
recommended: ["name", "alternateName", "description", "identifier", "image", "sameAs"]
|
|
96
|
+
},
|
|
97
|
+
ProfilePage: {
|
|
98
|
+
docs: `${DOCS}profile-page`,
|
|
99
|
+
required: [req("mainEntity")],
|
|
100
|
+
recommended: ["dateCreated", "dateModified"],
|
|
101
|
+
nested: [{ prop: "mainEntity", required: [req("name", "alternateName")] }]
|
|
102
|
+
},
|
|
103
|
+
Product: {
|
|
104
|
+
docs: `${DOCS}product-snippet`,
|
|
105
|
+
required: [req("name"), req("offers", "aggregateRating", "review")],
|
|
106
|
+
recommended: ["aggregateRating", "offers", "review", "offers.priceCurrency", "offers.availability"],
|
|
107
|
+
nested: [
|
|
108
|
+
{ prop: "offers", whenType: ["Offer"], orUntyped: true, required: [req("price", "priceSpecification.price")] },
|
|
109
|
+
{ prop: "offers", whenType: ["AggregateOffer"], required: [req("lowPrice"), req("priceCurrency")] }
|
|
110
|
+
]
|
|
111
|
+
},
|
|
112
|
+
Recipe: {
|
|
113
|
+
docs: `${DOCS}recipe`,
|
|
114
|
+
required: [req("name"), req("image")],
|
|
115
|
+
recommended: ["author", "datePublished", "description", "recipeIngredient", "recipeInstructions", "totalTime"]
|
|
116
|
+
},
|
|
117
|
+
SoftwareApplication: SOFTWARE_APPLICATION,
|
|
118
|
+
MobileApplication: SOFTWARE_APPLICATION,
|
|
119
|
+
WebApplication: SOFTWARE_APPLICATION,
|
|
120
|
+
WebSite: {
|
|
121
|
+
docs: "https://developers.google.com/search/docs/appearance/site-names",
|
|
122
|
+
required: [req("name"), req("url")],
|
|
123
|
+
recommended: ["alternateName"]
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
function present(v) {
|
|
127
|
+
if (v === void 0 || v === null) return false;
|
|
128
|
+
if (typeof v === "string") return v.trim() !== "";
|
|
129
|
+
if (Array.isArray(v)) return v.length > 0;
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
var isObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
133
|
+
function hasPath(node, path2) {
|
|
134
|
+
let level = [node];
|
|
135
|
+
for (const key of path2.split(".")) {
|
|
136
|
+
const next = [];
|
|
137
|
+
for (const v of level) {
|
|
138
|
+
for (const o of Array.isArray(v) ? v : [v]) if (isObject(o) && present(o[key])) next.push(o[key]);
|
|
139
|
+
}
|
|
140
|
+
if (next.length === 0) return false;
|
|
141
|
+
level = next;
|
|
142
|
+
}
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
function resolveRef(o, byId) {
|
|
146
|
+
const keys = Object.keys(o);
|
|
147
|
+
if (keys.length !== 1 || keys[0] !== "@id") return o;
|
|
148
|
+
return typeof o["@id"] === "string" ? byId.get(o["@id"]) ?? null : null;
|
|
149
|
+
}
|
|
150
|
+
var label = (prefix, anyOf) => prefix + anyOf[0] + (anyOf.length > 1 ? ` (or ${anyOf.slice(1).join(", ")})` : "");
|
|
151
|
+
function checkSchemaNodes(nodes, limit = Infinity) {
|
|
152
|
+
const byId = /* @__PURE__ */ new Map();
|
|
153
|
+
for (const n of nodes) if (typeof n["@id"] === "string" && !byId.has(n["@id"])) byId.set(n["@id"], n);
|
|
154
|
+
const scoredIssues = [];
|
|
155
|
+
const retiredIssues = [];
|
|
156
|
+
let total = 0;
|
|
157
|
+
let scored = 0;
|
|
158
|
+
let budget = MAX_JSON_LD_NODES;
|
|
159
|
+
const report = (type, path2) => {
|
|
160
|
+
total++;
|
|
161
|
+
const retired = SCHEMA_RULES[type].retired === true;
|
|
162
|
+
if (!retired) scored++;
|
|
163
|
+
const list = retired ? retiredIssues : scoredIssues;
|
|
164
|
+
if (list.length < limit) list.push({ type, path: path2 });
|
|
165
|
+
};
|
|
166
|
+
const apply = (type, node, prefix, required, nested, isLast) => {
|
|
167
|
+
for (const r of required) {
|
|
168
|
+
if (r.exceptLast && isLast) continue;
|
|
169
|
+
if (!r.anyOf.some((p) => hasPath(node, p))) report(type, label(prefix, r.anyOf));
|
|
170
|
+
}
|
|
171
|
+
for (const n of nested ?? []) {
|
|
172
|
+
const v = node[n.prop];
|
|
173
|
+
const items = Array.isArray(v) ? v : [v];
|
|
174
|
+
for (let i = 0; i < items.length; i++) {
|
|
175
|
+
if (--budget < 0) return;
|
|
176
|
+
const raw = items[i];
|
|
177
|
+
if (!isObject(raw)) continue;
|
|
178
|
+
const child = resolveRef(raw, byId);
|
|
179
|
+
if (child === null) continue;
|
|
180
|
+
if (n.whenType !== void 0) {
|
|
181
|
+
const types = nodeTypes(child);
|
|
182
|
+
const matches2 = types.length === 0 ? n.orUntyped === true : types.some((t) => n.whenType.includes(t));
|
|
183
|
+
if (!matches2) continue;
|
|
184
|
+
}
|
|
185
|
+
const at = `${prefix}${n.prop}${Array.isArray(v) ? `[${i}]` : ""}.`;
|
|
186
|
+
apply(type, child, at, n.required, n.nested, i === items.length - 1);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
for (const node of nodes) {
|
|
191
|
+
if (--budget < 0) break;
|
|
192
|
+
const seen = /* @__PURE__ */ new Set();
|
|
193
|
+
for (const type of nodeTypes(node)) {
|
|
194
|
+
const rule = Object.hasOwn(SCHEMA_RULES, type) ? SCHEMA_RULES[type] : void 0;
|
|
195
|
+
if (rule === void 0 || seen.has(rule)) continue;
|
|
196
|
+
seen.add(rule);
|
|
197
|
+
apply(type, node, "", rule.required, rule.nested, true);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return { issues: [...scoredIssues, ...retiredIssues].slice(0, limit), total, scored };
|
|
201
|
+
}
|
|
202
|
+
|
|
35
203
|
// src/cli/privateAddress.ts
|
|
36
204
|
import { lookup } from "dns/promises";
|
|
37
205
|
import { BlockList, isIP } from "net";
|
|
@@ -83,10 +251,10 @@ function abortable(p, signal) {
|
|
|
83
251
|
});
|
|
84
252
|
}
|
|
85
253
|
function decode(bytes, contentType) {
|
|
86
|
-
const
|
|
254
|
+
const label2 = /charset\s*=\s*"?([^;"\s]+)/i.exec(contentType)?.[1] ?? "utf-8";
|
|
87
255
|
let decoder;
|
|
88
256
|
try {
|
|
89
|
-
decoder = new TextDecoder(
|
|
257
|
+
decoder = new TextDecoder(label2);
|
|
90
258
|
} catch (e) {
|
|
91
259
|
if (!(e instanceof RangeError)) throw e;
|
|
92
260
|
decoder = new TextDecoder("utf-8");
|
|
@@ -479,6 +647,7 @@ var CHECK_WEIGHTS = {
|
|
|
479
647
|
};
|
|
480
648
|
var CONTENT_MIN_CHARS = 200;
|
|
481
649
|
var CONTENT_WARN_CHARS = 50;
|
|
650
|
+
var SCHEMA_ISSUES_SHOWN = 3;
|
|
482
651
|
var MAX_BODY_LABEL = `${MAX_BODY_BYTES / (1024 * 1024)} MiB`;
|
|
483
652
|
var TITLES = {
|
|
484
653
|
content: "Content without JavaScript",
|
|
@@ -601,7 +770,32 @@ function checkJsonLd({ html }) {
|
|
|
601
770
|
if (!parsed.some((p) => jsonLdTypes(p).length > 0)) {
|
|
602
771
|
return warn("JSON-LD parses but has no @type", "Add an @type (for example Organization) to your JSON-LD.");
|
|
603
772
|
}
|
|
604
|
-
|
|
773
|
+
const { issues, total, scored } = checkSchemaNodes(parsed.flatMap(jsonLdNodes), SCHEMA_ISSUES_SHOWN);
|
|
774
|
+
const base = `${parsed.length} JSON-LD block${parsed.length === 1 ? " parses" : "s parse"} with an @type`;
|
|
775
|
+
if (total === 0) return pass(base);
|
|
776
|
+
const isRetired = (i) => SCHEMA_RULES[i.type].retired === true;
|
|
777
|
+
const shownScored = issues.filter((i) => !isRetired(i));
|
|
778
|
+
const shownRetired = issues.filter(isRetired);
|
|
779
|
+
const retiredNote = notScored(shownRetired, total - scored - shownRetired.length);
|
|
780
|
+
if (scored === 0) return pass(`${base}; ${retiredNote}`);
|
|
781
|
+
const moreScored = scored - shownScored.length;
|
|
782
|
+
const rules = [...new Set(shownScored.map((i) => SCHEMA_RULES[i.type]))];
|
|
783
|
+
return warn(
|
|
784
|
+
`Missing required structured data fields: ${shownScored.map((i) => `${i.type} ${i.path}`).join("; ")}` + (moreScored > 0 ? `; +${moreScored} more` : "") + (total > scored ? `. ${retiredNote}` : ""),
|
|
785
|
+
`Add the missing properties; Google requires them for rich results (${rules.map((r) => r.docs).join(", ")}).`
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
var RETIRED_RULES = [...new Set(Object.values(SCHEMA_RULES))].filter((r) => r.retired === true);
|
|
789
|
+
var retiredNotes = (rules) => rules.flatMap((r) => r.note !== void 0 ? [r.note] : []).join("; ");
|
|
790
|
+
function notScored(shown, more) {
|
|
791
|
+
if (shown.length === 0) {
|
|
792
|
+
return `+${more} more not scored (${retiredNotes(RETIRED_RULES)})`;
|
|
793
|
+
}
|
|
794
|
+
const byType = /* @__PURE__ */ new Map();
|
|
795
|
+
for (const i of shown) byType.set(i.type, [...byType.get(i.type) ?? [], i.path]);
|
|
796
|
+
const listed = [...byType].map(([type, paths]) => `${type}: ${paths.join(", ")}`).join("; ");
|
|
797
|
+
const notes = retiredNotes([...new Set(shown.map((i) => SCHEMA_RULES[i.type]))]);
|
|
798
|
+
return `${listed}${more > 0 ? `, +${more} more` : ""} missing${notes ? ` (${notes})` : ""}`;
|
|
605
799
|
}
|
|
606
800
|
function checkRobots({ page, robots }) {
|
|
607
801
|
const missingFix = `Add ${ROBOTS_PATH}; webForAgents() generates one from robots.ai.training / robots.ai.search.`;
|
|
@@ -1016,7 +1210,7 @@ function safeJson(value, space) {
|
|
|
1016
1210
|
// src/cli/generate.ts
|
|
1017
1211
|
var VITE_ENTRY = "@anubhav-choubey/web-for-agents/vite";
|
|
1018
1212
|
var clean = (s) => s.split("\n").map((l) => safeText(l, Infinity)).join("\n");
|
|
1019
|
-
var
|
|
1213
|
+
var isObject2 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
1020
1214
|
async function loadConfig(file) {
|
|
1021
1215
|
const abs = path.resolve(file);
|
|
1022
1216
|
if (!existsSync(abs)) throw new Error(withPrefix(`config file not found: ${abs}`));
|
|
@@ -1028,13 +1222,13 @@ async function loadConfig(file) {
|
|
|
1028
1222
|
} catch (e) {
|
|
1029
1223
|
throw new Error(withPrefix(`could not load config ${abs}: ${errorMessage(e)}`), { cause: e });
|
|
1030
1224
|
}
|
|
1031
|
-
if (!
|
|
1225
|
+
if (!isObject2(config)) throw new Error(withPrefix(`config ${abs} must export an options object`));
|
|
1032
1226
|
return config;
|
|
1033
1227
|
}
|
|
1034
1228
|
async function generateOptions(args) {
|
|
1035
1229
|
const config = args.config ? await loadConfig(args.config) : {};
|
|
1036
|
-
const robots =
|
|
1037
|
-
const ai = { ...
|
|
1230
|
+
const robots = isObject2(config.robots) ? config.robots : {};
|
|
1231
|
+
const ai = { ...isObject2(robots.ai) ? robots.ai : {} };
|
|
1038
1232
|
if (args.aiTraining) ai.training = args.aiTraining;
|
|
1039
1233
|
if (args.aiSearch) ai.search = args.aiSearch;
|
|
1040
1234
|
if (ai.training === void 0 || ai.search === void 0) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// src/shared/agent.ts
|
|
2
|
+
var ACTION_NAME_RE = /^[A-Za-z0-9_.-]{1,128}$/;
|
|
3
|
+
function isValidActionName(name) {
|
|
4
|
+
return typeof name === "string" && ACTION_NAME_RE.test(name);
|
|
5
|
+
}
|
|
6
|
+
var ACTION_NAME_RULE = "use 1-128 chars of A-Z a-z 0-9 _ - .";
|
|
7
|
+
var AGENT_GLOBAL_VERSION = 1;
|
|
8
|
+
var AGENT_MAP_VERSION = 1;
|
|
9
|
+
|
|
10
|
+
// src/shared/text.ts
|
|
11
|
+
function collapseWhitespace(s) {
|
|
12
|
+
return (s ?? "").replace(/\s+/g, " ").trim();
|
|
13
|
+
}
|
|
14
|
+
function stripBom(s) {
|
|
15
|
+
return s.charCodeAt(0) === 65279 ? s.slice(1) : s;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export {
|
|
19
|
+
isValidActionName,
|
|
20
|
+
ACTION_NAME_RULE,
|
|
21
|
+
AGENT_GLOBAL_VERSION,
|
|
22
|
+
AGENT_MAP_VERSION,
|
|
23
|
+
collapseWhitespace,
|
|
24
|
+
stripBom
|
|
25
|
+
};
|
|
@@ -25,20 +25,9 @@ function metaLengthWarnings(pages) {
|
|
|
25
25
|
return out;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
// src/shared/agent.ts
|
|
29
|
-
var ACTION_NAME_RE = /^[A-Za-z0-9_.-]{1,128}$/;
|
|
30
|
-
function isValidActionName(name) {
|
|
31
|
-
return typeof name === "string" && ACTION_NAME_RE.test(name);
|
|
32
|
-
}
|
|
33
|
-
var AGENT_GLOBAL_VERSION = 1;
|
|
34
|
-
var AGENT_MAP_VERSION = 1;
|
|
35
|
-
|
|
36
28
|
export {
|
|
37
29
|
TITLE_MAX_CHARS,
|
|
38
30
|
DESCRIPTION_MIN_CHARS,
|
|
39
31
|
DESCRIPTION_MAX_CHARS,
|
|
40
|
-
metaLengthWarnings
|
|
41
|
-
isValidActionName,
|
|
42
|
-
AGENT_GLOBAL_VERSION,
|
|
43
|
-
AGENT_MAP_VERSION
|
|
32
|
+
metaLengthWarnings
|
|
44
33
|
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// src/shared/jsonLd.ts
|
|
2
|
+
var MAX_JSON_LD_NODES = 1e5;
|
|
3
|
+
function jsonLdNodes(value) {
|
|
4
|
+
const out = [];
|
|
5
|
+
const stack = [value];
|
|
6
|
+
for (let visited = 0; stack.length > 0 && visited < MAX_JSON_LD_NODES; visited++) {
|
|
7
|
+
const v = stack.pop();
|
|
8
|
+
if (Array.isArray(v)) {
|
|
9
|
+
for (let i = v.length - 1; i >= 0; i--) stack.push(v[i]);
|
|
10
|
+
continue;
|
|
11
|
+
}
|
|
12
|
+
if (v === null || typeof v !== "object") continue;
|
|
13
|
+
const o = v;
|
|
14
|
+
out.push(o);
|
|
15
|
+
if ("@graph" in o) stack.push(o["@graph"]);
|
|
16
|
+
}
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
function nodeTypes(node) {
|
|
20
|
+
const t = node["@type"];
|
|
21
|
+
if (typeof t === "string") return [t];
|
|
22
|
+
if (Array.isArray(t)) return t.filter((x) => typeof x === "string");
|
|
23
|
+
return [];
|
|
24
|
+
}
|
|
25
|
+
function jsonLdTypes(value) {
|
|
26
|
+
return jsonLdNodes(value).flatMap(nodeTypes);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export {
|
|
30
|
+
MAX_JSON_LD_NODES,
|
|
31
|
+
jsonLdNodes,
|
|
32
|
+
nodeTypes,
|
|
33
|
+
jsonLdTypes
|
|
34
|
+
};
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
collapseWhitespace
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-4STTPCBR.js";
|
|
4
4
|
import {
|
|
5
5
|
ACTION_PARAM_ATTR,
|
|
6
6
|
HIDDEN_SELECTOR,
|
|
7
7
|
WEBMCP_TOOLPARAMDESCRIPTION_ATTR
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-2Y3LLYEC.js";
|
|
9
9
|
|
|
10
10
|
// src/shared/object.ts
|
|
11
11
|
var hasOwn = (o, k) => Object.prototype.hasOwnProperty.call(o, k);
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
ACTION_NAME_RULE,
|
|
3
|
+
collapseWhitespace,
|
|
4
|
+
isValidActionName
|
|
5
|
+
} from "./chunk-4STTPCBR.js";
|
|
4
6
|
import {
|
|
5
7
|
ACTION_ATTR,
|
|
6
8
|
ACTION_DESCRIPTION_ATTR,
|
|
@@ -8,10 +10,12 @@ import {
|
|
|
8
10
|
CONTENT_ATTR,
|
|
9
11
|
EXCLUDE_ATTR,
|
|
10
12
|
HIDDEN_SELECTOR,
|
|
13
|
+
LINK_ACTION_SELECTOR,
|
|
11
14
|
WEBMCP_TOOLDESCRIPTION_ATTR,
|
|
12
15
|
WEBMCP_TOOLNAME_ATTR,
|
|
13
|
-
resolveUrl
|
|
14
|
-
|
|
16
|
+
resolveUrl,
|
|
17
|
+
tryResolveUrl
|
|
18
|
+
} from "./chunk-2Y3LLYEC.js";
|
|
15
19
|
|
|
16
20
|
// src/shared/shortcut.ts
|
|
17
21
|
var DEFAULT_SHORTCUT = "alt+shift+a";
|
|
@@ -42,6 +46,89 @@ function matchesShortcut(e, s) {
|
|
|
42
46
|
return e.code === s.code && e.altKey === s.alt && e.shiftKey === s.shift && e.ctrlKey === s.ctrl && e.metaKey === s.meta;
|
|
43
47
|
}
|
|
44
48
|
|
|
49
|
+
// src/shared/markupActions.ts
|
|
50
|
+
function markupActions(doc) {
|
|
51
|
+
const out = [];
|
|
52
|
+
doc.querySelectorAll(`[${ACTION_ATTR}]`).forEach((el) => {
|
|
53
|
+
const name = el.getAttribute(ACTION_ATTR)?.trim();
|
|
54
|
+
if (!name) return;
|
|
55
|
+
const link = el.matches(LINK_ACTION_SELECTOR);
|
|
56
|
+
out.push({
|
|
57
|
+
element: el,
|
|
58
|
+
name,
|
|
59
|
+
description: el.getAttribute(ACTION_DESCRIPTION_ATTR) || null,
|
|
60
|
+
kind: el.tagName === "FORM" ? "form" : link ? "link" : "click",
|
|
61
|
+
destructive: el.getAttribute(ACTION_DESTRUCTIVE_ATTR) === "true",
|
|
62
|
+
source: "markup",
|
|
63
|
+
...link ? { href: el.getAttribute("href") ?? "" } : {}
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
doc.querySelectorAll(`form[${WEBMCP_TOOLNAME_ATTR}]`).forEach((el) => {
|
|
67
|
+
const name = el.getAttribute(WEBMCP_TOOLNAME_ATTR)?.trim();
|
|
68
|
+
if (!name) return;
|
|
69
|
+
out.push({
|
|
70
|
+
element: el,
|
|
71
|
+
name,
|
|
72
|
+
description: el.getAttribute(WEBMCP_TOOLDESCRIPTION_ATTR) || null,
|
|
73
|
+
kind: "form",
|
|
74
|
+
destructive: false,
|
|
75
|
+
source: "webmcp-declarative"
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
function markupLinks(doc) {
|
|
81
|
+
return markupActions(doc).flatMap(
|
|
82
|
+
(a) => a.kind === "link" ? [{ name: a.name, description: a.description, destructive: a.destructive, href: a.href ?? "" }] : []
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
var LINK_ACTION_PROTOCOLS = ["http:", "https:", "mailto:", "tel:"];
|
|
86
|
+
function resolveLinkHref(href, pageUrl) {
|
|
87
|
+
const url = tryResolveUrl(href, pageUrl);
|
|
88
|
+
if (url === null) return { ok: false, reason: "its href is not a valid URL" };
|
|
89
|
+
const protocol = new URL(url).protocol;
|
|
90
|
+
if (!LINK_ACTION_PROTOCOLS.includes(protocol)) {
|
|
91
|
+
return { ok: false, reason: `${protocol} links are not allowed (only http(s), mailto: and tel:)` };
|
|
92
|
+
}
|
|
93
|
+
return { ok: true, href: url };
|
|
94
|
+
}
|
|
95
|
+
var INVALID_NAME_REASON = `invalid name (${ACTION_NAME_RULE})`;
|
|
96
|
+
function firstPerName(...groups) {
|
|
97
|
+
const seen = /* @__PURE__ */ new Set();
|
|
98
|
+
return groups.flat().filter((a) => !seen.has(a.name) && !!seen.add(a.name));
|
|
99
|
+
}
|
|
100
|
+
function actionGroups(markup) {
|
|
101
|
+
return [markup.filter((a) => a.kind !== "link"), markup.filter((a) => a.kind === "link")];
|
|
102
|
+
}
|
|
103
|
+
function linkActions(links, opts, warn) {
|
|
104
|
+
const seen = new Set(opts.taken);
|
|
105
|
+
const out = [];
|
|
106
|
+
for (const link of links) {
|
|
107
|
+
if (seen.has(link.name)) continue;
|
|
108
|
+
seen.add(link.name);
|
|
109
|
+
if (!isValidActionName(link.name)) {
|
|
110
|
+
warn(`link action "${link.name}" on ${opts.page} skipped: ${INVALID_NAME_REASON}`);
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
const r = resolveLinkHref(link.href, opts.pageUrl);
|
|
114
|
+
if (!r.ok) {
|
|
115
|
+
warn(`link action "${link.name}" on ${opts.page} skipped: ${r.reason}`);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
out.push({
|
|
119
|
+
name: link.name,
|
|
120
|
+
description: link.description?.trim() || link.name,
|
|
121
|
+
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
|
122
|
+
destructive: link.destructive,
|
|
123
|
+
readOnly: !link.destructive,
|
|
124
|
+
kind: "link",
|
|
125
|
+
href: r.href,
|
|
126
|
+
.../^https?:$/.test(new URL(r.href).protocol) ? { method: "GET" } : {}
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
|
|
45
132
|
// src/shared/content.ts
|
|
46
133
|
var DEFAULT_CONTENT_SELECTOR = "main";
|
|
47
134
|
var STRIP = `script, style, noscript, template, nav, [${EXCLUDE_ATTR}], ${HIDDEN_SELECTOR}`;
|
|
@@ -167,37 +254,14 @@ function buildPageMarkdown(title, contentHtml, pageUrl) {
|
|
|
167
254
|
${body}`.replace(/\n+$/, "\n");
|
|
168
255
|
}
|
|
169
256
|
|
|
170
|
-
// src/shared/markupActions.ts
|
|
171
|
-
function markupActions(doc) {
|
|
172
|
-
const out = [];
|
|
173
|
-
doc.querySelectorAll(`[${ACTION_ATTR}]`).forEach((el) => {
|
|
174
|
-
const name = el.getAttribute(ACTION_ATTR)?.trim();
|
|
175
|
-
if (!name) return;
|
|
176
|
-
out.push({
|
|
177
|
-
element: el,
|
|
178
|
-
name,
|
|
179
|
-
description: el.getAttribute(ACTION_DESCRIPTION_ATTR) || null,
|
|
180
|
-
kind: el.tagName === "FORM" ? "form" : "click",
|
|
181
|
-
destructive: el.getAttribute(ACTION_DESTRUCTIVE_ATTR) === "true",
|
|
182
|
-
source: "markup"
|
|
183
|
-
});
|
|
184
|
-
});
|
|
185
|
-
doc.querySelectorAll(`form[${WEBMCP_TOOLNAME_ATTR}]`).forEach((el) => {
|
|
186
|
-
const name = el.getAttribute(WEBMCP_TOOLNAME_ATTR)?.trim();
|
|
187
|
-
if (!name) return;
|
|
188
|
-
out.push({
|
|
189
|
-
element: el,
|
|
190
|
-
name,
|
|
191
|
-
description: el.getAttribute(WEBMCP_TOOLDESCRIPTION_ATTR) || null,
|
|
192
|
-
kind: "form",
|
|
193
|
-
destructive: false,
|
|
194
|
-
source: "webmcp-declarative"
|
|
195
|
-
});
|
|
196
|
-
});
|
|
197
|
-
return out;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
257
|
export {
|
|
258
|
+
markupActions,
|
|
259
|
+
markupLinks,
|
|
260
|
+
resolveLinkHref,
|
|
261
|
+
INVALID_NAME_REASON,
|
|
262
|
+
firstPerName,
|
|
263
|
+
actionGroups,
|
|
264
|
+
linkActions,
|
|
201
265
|
DEFAULT_CONTENT_SELECTOR,
|
|
202
266
|
selectContentRegion,
|
|
203
267
|
DEFAULT_SHORTCUT,
|
|
@@ -206,6 +270,5 @@ export {
|
|
|
206
270
|
readPageMeta,
|
|
207
271
|
escapeMarkdownInline,
|
|
208
272
|
escapeLinkTarget,
|
|
209
|
-
buildPageMarkdown
|
|
210
|
-
markupActions
|
|
273
|
+
buildPageMarkdown
|
|
211
274
|
};
|
package/dist/cli.js
CHANGED
|
@@ -2,12 +2,12 @@ import {
|
|
|
2
2
|
audit,
|
|
3
3
|
formatText,
|
|
4
4
|
runCli
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-357ATBMS.js";
|
|
6
6
|
import "./chunk-CGIIB6AF.js";
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
10
|
-
import "./chunk-
|
|
7
|
+
import "./chunk-L3AYWCSX.js";
|
|
8
|
+
import "./chunk-MIVGE5TM.js";
|
|
9
|
+
import "./chunk-4STTPCBR.js";
|
|
10
|
+
import "./chunk-2Y3LLYEC.js";
|
|
11
11
|
import "./chunk-JJN6FNJD.js";
|
|
12
12
|
export {
|
|
13
13
|
audit,
|
|
@@ -2,8 +2,8 @@ import {
|
|
|
2
2
|
createHappyDomRenderer,
|
|
3
3
|
syncHint,
|
|
4
4
|
syncRefusalText
|
|
5
|
-
} from "./chunk-
|
|
6
|
-
import "./chunk-
|
|
5
|
+
} from "./chunk-K3B53RIO.js";
|
|
6
|
+
import "./chunk-2Y3LLYEC.js";
|
|
7
7
|
import "./chunk-PGZH33B5.js";
|
|
8
8
|
import "./chunk-JJN6FNJD.js";
|
|
9
9
|
export {
|