@anubhav-choubey/web-for-agents 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/bin.js +5 -5
- package/dist/{chunk-F7PN5R25.js → chunk-2Y3LLYEC.js} +3 -1
- package/dist/{chunk-P6JSPAA2.js → chunk-357ATBMS.js} +207 -13
- package/dist/chunk-4STTPCBR.js +25 -0
- package/dist/{chunk-SMFRAZAA.js → chunk-K3B53RIO.js} +1 -1
- package/dist/{chunk-PHTV5NAY.js → chunk-L3AYWCSX.js} +1 -12
- package/dist/chunk-MIVGE5TM.js +34 -0
- package/dist/{chunk-LCVPXUFN.js → chunk-OKCYZE6C.js} +2 -2
- package/dist/{chunk-PSNLS4GL.js → chunk-T42S3WRV.js} +99 -36
- package/dist/cli.js +5 -5
- package/dist/{happyDom-WTYQ5UAC.js → happyDom-ZEFNMD7B.js} +2 -2
- package/dist/index.d.ts +89 -11
- package/dist/index.js +80 -19
- package/dist/overlay.js +22 -17
- package/dist/{renderChild-H6TR4SGV.js → renderChild-PSQLAIUW.js} +1 -1
- package/dist/vite.d.ts +30 -3
- package/dist/vite.js +265 -72
- package/package.json +1 -1
- package/dist/chunk-FEAKVNMG.js +0 -12
- package/dist/chunk-LCLDQ3A3.js +0 -26
package/dist/vite.js
CHANGED
|
@@ -1,31 +1,35 @@
|
|
|
1
1
|
import {
|
|
2
2
|
EMPTY_OBJECT_SCHEMA,
|
|
3
3
|
schemaFromForm
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-OKCYZE6C.js";
|
|
5
5
|
import {
|
|
6
6
|
AI_BOTS,
|
|
7
7
|
scanHtml
|
|
8
8
|
} from "./chunk-CGIIB6AF.js";
|
|
9
9
|
import {
|
|
10
|
-
AGENT_MAP_VERSION,
|
|
11
|
-
isValidActionName,
|
|
12
10
|
metaLengthWarnings
|
|
13
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-L3AYWCSX.js";
|
|
14
12
|
import {
|
|
15
13
|
DEFAULT_CONTENT_SELECTOR,
|
|
16
14
|
DEFAULT_SHORTCUT,
|
|
17
15
|
buildPageMarkdown,
|
|
18
16
|
escapeLinkTarget,
|
|
19
17
|
escapeMarkdownInline,
|
|
18
|
+
firstPerName,
|
|
19
|
+
linkActions,
|
|
20
20
|
markupActions,
|
|
21
|
+
markupLinks,
|
|
21
22
|
parseShortcut,
|
|
22
23
|
readPageMeta,
|
|
23
24
|
selectContentRegion
|
|
24
|
-
} from "./chunk-
|
|
25
|
+
} from "./chunk-T42S3WRV.js";
|
|
25
26
|
import {
|
|
26
27
|
compareStrings
|
|
27
28
|
} from "./chunk-AAIYCJD3.js";
|
|
28
|
-
import
|
|
29
|
+
import {
|
|
30
|
+
AGENT_MAP_VERSION,
|
|
31
|
+
isValidActionName
|
|
32
|
+
} from "./chunk-4STTPCBR.js";
|
|
29
33
|
import {
|
|
30
34
|
IDLE_MS,
|
|
31
35
|
PROBE_ATTR,
|
|
@@ -33,12 +37,16 @@ import {
|
|
|
33
37
|
TEMP_BASE_ATTR,
|
|
34
38
|
prepareShell,
|
|
35
39
|
readyCheck
|
|
36
|
-
} from "./chunk-
|
|
40
|
+
} from "./chunk-K3B53RIO.js";
|
|
37
41
|
import {
|
|
42
|
+
ACTION_ATTR,
|
|
43
|
+
ACTION_DESCRIPTION_ATTR,
|
|
44
|
+
ACTION_DESTRUCTIVE_ATTR,
|
|
38
45
|
AGENT_GLOBAL,
|
|
39
46
|
AGENT_MAP_PATH,
|
|
40
47
|
CONTENT_ATTR,
|
|
41
48
|
HEAD_OWNED_ATTR,
|
|
49
|
+
LINK_ACTION_SELECTOR,
|
|
42
50
|
LLMS_FULL_TXT_PATH,
|
|
43
51
|
LLMS_TXT_PATH,
|
|
44
52
|
MARKDOWN_LINK_KEY,
|
|
@@ -51,8 +59,9 @@ import {
|
|
|
51
59
|
mountPath,
|
|
52
60
|
normalizeRoute,
|
|
53
61
|
publicUrl,
|
|
54
|
-
trimTrailingSlash
|
|
55
|
-
|
|
62
|
+
trimTrailingSlash,
|
|
63
|
+
tryResolveUrl
|
|
64
|
+
} from "./chunk-2Y3LLYEC.js";
|
|
56
65
|
import {
|
|
57
66
|
POLL_MS,
|
|
58
67
|
RenderError,
|
|
@@ -179,14 +188,32 @@ var WEBMCP_INFO = {
|
|
|
179
188
|
api: "document.modelContext",
|
|
180
189
|
spec: "https://webmachinelearning.github.io/webmcp/"
|
|
181
190
|
};
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
191
|
+
var AGENT_PAGE_IN_PAGE_ARG = {
|
|
192
|
+
globalKey: AGENT_GLOBAL,
|
|
193
|
+
linkSelector: LINK_ACTION_SELECTOR,
|
|
194
|
+
nameAttr: ACTION_ATTR,
|
|
195
|
+
descriptionAttr: ACTION_DESCRIPTION_ATTR,
|
|
196
|
+
destructiveAttr: ACTION_DESTRUCTIVE_ATTR
|
|
197
|
+
};
|
|
198
|
+
function collectAgentPageInPage(arg) {
|
|
199
|
+
const api = window[arg.globalKey];
|
|
200
|
+
const actions = api && typeof api.actions === "function" ? api.actions() : [];
|
|
201
|
+
const links = [];
|
|
202
|
+
document.querySelectorAll(arg.linkSelector).forEach((el) => {
|
|
203
|
+
const name = (el.getAttribute(arg.nameAttr) || "").trim();
|
|
204
|
+
if (!name) return;
|
|
205
|
+
links.push({
|
|
206
|
+
name,
|
|
207
|
+
description: el.getAttribute(arg.descriptionAttr) || null,
|
|
208
|
+
destructive: el.getAttribute(arg.destructiveAttr) === "true",
|
|
209
|
+
href: el.getAttribute("href") || ""
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
return { actions, links };
|
|
185
213
|
}
|
|
186
214
|
var byName = (a, b) => compareStrings(a.name, b.name);
|
|
187
215
|
function toPage(siteUrl, p) {
|
|
188
|
-
const
|
|
189
|
-
const actions = p.actions.filter((a) => !seen.has(a.name) && !!seen.add(a.name)).sort(byName);
|
|
216
|
+
const actions = firstPerName(p.actions).sort(byName);
|
|
190
217
|
return {
|
|
191
218
|
path: p.route,
|
|
192
219
|
url: publicUrl(siteUrl, p.route),
|
|
@@ -223,14 +250,21 @@ function agentMapExtension(opts) {
|
|
|
223
250
|
return {
|
|
224
251
|
name: "agent-map",
|
|
225
252
|
inPage: collectAgentPageInPage,
|
|
226
|
-
inPageArg:
|
|
253
|
+
inPageArg: AGENT_PAGE_IN_PAGE_ARG,
|
|
227
254
|
onRoute(route, page) {
|
|
228
255
|
data.set(route, page);
|
|
229
256
|
},
|
|
230
257
|
llmsSections: () => [agentMapLlmsSection(siteUrl, fileName)],
|
|
231
258
|
finalize(ctx) {
|
|
232
259
|
const pages = ctx.pages.map((p) => {
|
|
233
|
-
const
|
|
260
|
+
const found = data.get(p.route);
|
|
261
|
+
const actions = found?.actions ?? [];
|
|
262
|
+
const links = linkActions(
|
|
263
|
+
found?.links ?? [],
|
|
264
|
+
{ pageUrl: p.url, page: p.route, taken: new Set(actions.map((a) => a.name)) },
|
|
265
|
+
ctx.warn
|
|
266
|
+
);
|
|
267
|
+
const page = { route: p.route, actions: [...actions, ...links] };
|
|
234
268
|
if (p.title) page.title = p.title;
|
|
235
269
|
if (p.description) page.description = p.description;
|
|
236
270
|
if (p.mdUrl) page.markdown = p.mdUrl;
|
|
@@ -371,7 +405,7 @@ function nodeModulesRoot(file) {
|
|
|
371
405
|
}
|
|
372
406
|
var sourceEntry;
|
|
373
407
|
async function childEntryFile() {
|
|
374
|
-
const { CHILD_ENTRY } = await import("./renderChild-
|
|
408
|
+
const { CHILD_ENTRY } = await import("./renderChild-PSQLAIUW.js");
|
|
375
409
|
const file = fileURLToPath(CHILD_ENTRY);
|
|
376
410
|
if (!file.endsWith(".ts")) return file;
|
|
377
411
|
return sourceEntry ??= bundleFromSource(file);
|
|
@@ -795,6 +829,75 @@ function resolveMarkdown(input) {
|
|
|
795
829
|
}
|
|
796
830
|
return out;
|
|
797
831
|
}
|
|
832
|
+
function resolveSitemap(input) {
|
|
833
|
+
if (input === void 0 || typeof input === "boolean") return { sitemap: input ?? true, sitemapLastmod: "none" };
|
|
834
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
835
|
+
throw new Error(withPrefix("sitemap must be a boolean or { lastmod }"));
|
|
836
|
+
}
|
|
837
|
+
const lastmod = input.lastmod ?? "none";
|
|
838
|
+
if (lastmod !== "file" && lastmod !== "none" && typeof lastmod !== "function") {
|
|
839
|
+
throw new Error(withPrefix('sitemap.lastmod must be "file", "none" or a function'));
|
|
840
|
+
}
|
|
841
|
+
return { sitemap: true, sitemapLastmod: lastmod };
|
|
842
|
+
}
|
|
843
|
+
function resolveLlms(input, siteUrl) {
|
|
844
|
+
if (input === void 0) return {};
|
|
845
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
846
|
+
throw new Error(withPrefix("llms must be false or { title, description, intro, sections }"));
|
|
847
|
+
}
|
|
848
|
+
const str = (v, field, nonEmpty = false) => {
|
|
849
|
+
if (v === void 0) return void 0;
|
|
850
|
+
if (typeof v !== "string" || nonEmpty && v.trim() === "") {
|
|
851
|
+
throw new Error(withPrefix(`llms.${field} must be a ${nonEmpty ? "non-empty " : ""}string`));
|
|
852
|
+
}
|
|
853
|
+
return v;
|
|
854
|
+
};
|
|
855
|
+
const out = { ...input };
|
|
856
|
+
str(input.title, "title");
|
|
857
|
+
str(input.description, "description");
|
|
858
|
+
str(input.intro, "intro");
|
|
859
|
+
if (input.sections === void 0) return out;
|
|
860
|
+
if (!Array.isArray(input.sections)) throw new Error(withPrefix("llms.sections must be an array"));
|
|
861
|
+
out.sections = input.sections.map((sec, i) => {
|
|
862
|
+
const at = `sections[${i}]`;
|
|
863
|
+
if (typeof sec !== "object" || sec === null || Array.isArray(sec)) {
|
|
864
|
+
throw new Error(withPrefix(`llms.${at} must be an object such as { title, text, links }`));
|
|
865
|
+
}
|
|
866
|
+
const { title, text, links } = sec;
|
|
867
|
+
if (title === void 0) throw new Error(withPrefix(`llms.${at}.title is required`));
|
|
868
|
+
const s = { title: str(title, `${at}.title`, true) };
|
|
869
|
+
const t = str(text, `${at}.text`);
|
|
870
|
+
if (t !== void 0) s.text = t;
|
|
871
|
+
if (links !== void 0) {
|
|
872
|
+
if (!Array.isArray(links)) throw new Error(withPrefix(`llms.${at}.links must be an array`));
|
|
873
|
+
s.links = links.map((l, j) => {
|
|
874
|
+
const lat = `${at}.links[${j}]`;
|
|
875
|
+
if (typeof l !== "object" || l === null || Array.isArray(l)) {
|
|
876
|
+
throw new Error(withPrefix(`llms.${lat} must be an object such as { title, url, description }`));
|
|
877
|
+
}
|
|
878
|
+
const link = l;
|
|
879
|
+
if (link.title === void 0) throw new Error(withPrefix(`llms.${lat}.title is required`));
|
|
880
|
+
if (link.url === void 0) throw new Error(withPrefix(`llms.${lat}.url is required`));
|
|
881
|
+
const url = typeof link.url === "string" ? tryResolveUrl(link.url.trim(), siteUrl + "/") : null;
|
|
882
|
+
const scheme = url === null ? "" : new URL(url).protocol;
|
|
883
|
+
if (url === null || !["http:", "https:", "mailto:"].includes(scheme)) {
|
|
884
|
+
throw new Error(
|
|
885
|
+
withPrefix(`llms.${lat}.url must be an http(s) or mailto: URL, or a path on the site (got ${JSON.stringify(link.url)})`)
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
const o = { title: str(link.title, `${lat}.title`, true), url };
|
|
889
|
+
const d = str(link.description, `${lat}.description`);
|
|
890
|
+
if (d !== void 0) o.description = d;
|
|
891
|
+
return o;
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
if (!s.text?.trim() && !s.links?.length) {
|
|
895
|
+
throw new Error(withPrefix(`llms.${at} needs text or links`));
|
|
896
|
+
}
|
|
897
|
+
return s;
|
|
898
|
+
});
|
|
899
|
+
return out;
|
|
900
|
+
}
|
|
798
901
|
var AGENT_MAP_FILE = /^[A-Za-z0-9_.-]+\.json$/;
|
|
799
902
|
function resolveAgentMap(input) {
|
|
800
903
|
if (input === false) return false;
|
|
@@ -870,6 +973,7 @@ function resolveOptions(input) {
|
|
|
870
973
|
const prerender = input.prerender === false ? false : resolvePrerender(input.prerender ?? {});
|
|
871
974
|
let markdown = false;
|
|
872
975
|
let llms = false;
|
|
976
|
+
const requestedLlms = input.llms === false ? false : resolveLlms(input.llms, siteUrl);
|
|
873
977
|
const requestedAgentMap = resolveAgentMap(input.agentMap);
|
|
874
978
|
let agentMap = false;
|
|
875
979
|
if (prerender === false) {
|
|
@@ -881,14 +985,14 @@ function resolveOptions(input) {
|
|
|
881
985
|
}
|
|
882
986
|
} else {
|
|
883
987
|
if (input.markdown !== false) markdown = resolveMarkdown(input.markdown);
|
|
884
|
-
|
|
988
|
+
llms = requestedLlms;
|
|
885
989
|
agentMap = requestedAgentMap;
|
|
886
990
|
}
|
|
887
991
|
const resolved = {
|
|
888
992
|
siteUrl,
|
|
889
993
|
routes,
|
|
890
994
|
robots: input.robots,
|
|
891
|
-
|
|
995
|
+
...resolveSitemap(input.sitemap),
|
|
892
996
|
exclude,
|
|
893
997
|
prerender,
|
|
894
998
|
markdown,
|
|
@@ -921,9 +1025,9 @@ function parseHtml(html) {
|
|
|
921
1025
|
handleDisabledFileLoadingAsSuccess: true
|
|
922
1026
|
}
|
|
923
1027
|
});
|
|
924
|
-
const
|
|
1028
|
+
const document2 = new window2.DOMParser().parseFromString(html, "text/html");
|
|
925
1029
|
return {
|
|
926
|
-
document,
|
|
1030
|
+
document: document2,
|
|
927
1031
|
dispose() {
|
|
928
1032
|
window2.happyDOM.close();
|
|
929
1033
|
}
|
|
@@ -936,9 +1040,9 @@ var SHOW_TEXT = 4;
|
|
|
936
1040
|
var FILTER_ACCEPT = 1;
|
|
937
1041
|
var FILTER_REJECT = 2;
|
|
938
1042
|
var NO_TEXT = /* @__PURE__ */ new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE"]);
|
|
939
|
-
function bodyHasText(
|
|
940
|
-
if (!
|
|
941
|
-
const walker =
|
|
1043
|
+
function bodyHasText(document2) {
|
|
1044
|
+
if (!document2.body) return false;
|
|
1045
|
+
const walker = document2.createTreeWalker(document2.body, SHOW_ELEMENT | SHOW_TEXT, {
|
|
942
1046
|
acceptNode: (n) => n.nodeType === 1 && NO_TEXT.has(n.tagName.toUpperCase()) ? FILTER_REJECT : FILTER_ACCEPT
|
|
943
1047
|
});
|
|
944
1048
|
for (let n = walker.nextNode(); n; n = walker.nextNode()) {
|
|
@@ -946,11 +1050,11 @@ function bodyHasText(document) {
|
|
|
946
1050
|
}
|
|
947
1051
|
return false;
|
|
948
1052
|
}
|
|
949
|
-
function analyzeDocument(
|
|
950
|
-
const meta = readPageMeta(
|
|
1053
|
+
function analyzeDocument(document2, opts, mode = {}) {
|
|
1054
|
+
const meta = readPageMeta(document2);
|
|
951
1055
|
const title = meta.title ?? opts.route;
|
|
952
1056
|
const links = [];
|
|
953
|
-
|
|
1057
|
+
document2.querySelectorAll("a[href]").forEach((a) => {
|
|
954
1058
|
if (a.hasAttribute("download")) return;
|
|
955
1059
|
try {
|
|
956
1060
|
const u = new URL(a.getAttribute("href"), opts.pageUrl);
|
|
@@ -958,8 +1062,8 @@ function analyzeDocument(document, opts, mode = {}) {
|
|
|
958
1062
|
} catch {
|
|
959
1063
|
}
|
|
960
1064
|
});
|
|
961
|
-
const hasText = bodyHasText(
|
|
962
|
-
const { element, source } = selectContentRegion(
|
|
1065
|
+
const hasText = bodyHasText(document2);
|
|
1066
|
+
const { element, source } = selectContentRegion(document2, {
|
|
963
1067
|
contentSelector: opts.contentSelector,
|
|
964
1068
|
...mode.inPlace ? { inPlace: true } : {}
|
|
965
1069
|
});
|
|
@@ -989,9 +1093,9 @@ var URL_ATTRS = [
|
|
|
989
1093
|
function mapSrcset(value, fn) {
|
|
990
1094
|
return value.replace(/(^|,)(\s*)([^\s,][^\s]*?)(?=,|\s|$)/g, (_m, sep2, ws, url) => sep2 + ws + fn(url));
|
|
991
1095
|
}
|
|
992
|
-
function rewriteUrls(
|
|
1096
|
+
function rewriteUrls(document2, fn, attrs = URL_ATTRS) {
|
|
993
1097
|
for (const [tag, attr] of attrs) {
|
|
994
|
-
|
|
1098
|
+
document2.querySelectorAll(`${tag}[${attr}]`).forEach((el) => {
|
|
995
1099
|
const v = el.getAttribute(attr);
|
|
996
1100
|
const next = attr === "srcset" ? mapSrcset(v, fn) : fn(v);
|
|
997
1101
|
if (next !== v) el.setAttribute(attr, next);
|
|
@@ -999,18 +1103,18 @@ function rewriteUrls(document, fn, attrs = URL_ATTRS) {
|
|
|
999
1103
|
}
|
|
1000
1104
|
}
|
|
1001
1105
|
function finalizePage(html, opts, analyze) {
|
|
1002
|
-
const { document, dispose } = parseHtml(html);
|
|
1106
|
+
const { document: document2, dispose } = parseHtml(html);
|
|
1003
1107
|
try {
|
|
1004
|
-
const out = finalizeDocument(
|
|
1005
|
-
return { html: out, info: analyzeDocument(
|
|
1108
|
+
const out = finalizeDocument(document2, opts);
|
|
1109
|
+
return { html: out, info: analyzeDocument(document2, analyze, { inPlace: true }) };
|
|
1006
1110
|
} finally {
|
|
1007
1111
|
dispose();
|
|
1008
1112
|
}
|
|
1009
1113
|
}
|
|
1010
|
-
function finalizeDocument(
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1114
|
+
function finalizeDocument(document2, opts) {
|
|
1115
|
+
document2.querySelectorAll(`[${PROBE_ATTR}], [${TEMP_BASE_ATTR}]`).forEach((el) => el.remove());
|
|
1116
|
+
document2.documentElement.setAttribute(PRERENDERED_ATTR, "");
|
|
1117
|
+
document2.querySelectorAll(`script[${REL_SRC_ATTR}]`).forEach((el) => {
|
|
1014
1118
|
const src = el.getAttribute("src");
|
|
1015
1119
|
if (src?.startsWith("/")) el.setAttribute("src", "." + src);
|
|
1016
1120
|
el.removeAttribute(REL_SRC_ATTR);
|
|
@@ -1018,7 +1122,7 @@ function finalizeDocument(document, opts) {
|
|
|
1018
1122
|
if (opts.origin) {
|
|
1019
1123
|
const origin = trimTrailingSlash(opts.origin);
|
|
1020
1124
|
const attrs = opts.fullBase ? [...URL_ATTRS, ["a", "href"]] : URL_ATTRS;
|
|
1021
|
-
rewriteUrls(
|
|
1125
|
+
rewriteUrls(document2, (u) => {
|
|
1022
1126
|
if (!u.startsWith(origin)) return u;
|
|
1023
1127
|
const rest = u.slice(origin.length);
|
|
1024
1128
|
if (rest !== "" && !/^[/?#]/.test(rest)) return u;
|
|
@@ -1031,11 +1135,11 @@ function finalizeDocument(document, opts) {
|
|
|
1031
1135
|
const depth = routeDepth(opts.route);
|
|
1032
1136
|
if (depth > 0) {
|
|
1033
1137
|
const up = "../".repeat(depth);
|
|
1034
|
-
rewriteUrls(
|
|
1138
|
+
rewriteUrls(document2, (u) => u.startsWith("./") ? up + u.slice(2) : u);
|
|
1035
1139
|
}
|
|
1036
1140
|
}
|
|
1037
1141
|
const seen = /* @__PURE__ */ new Set();
|
|
1038
|
-
|
|
1142
|
+
document2.querySelectorAll("link[href]").forEach((el) => {
|
|
1039
1143
|
const rel = el.getAttribute("rel") ?? "";
|
|
1040
1144
|
if (!DEDUPE_RELS.has(rel)) return;
|
|
1041
1145
|
const key = [rel, el.getAttribute("href"), el.getAttribute("as"), el.getAttribute("media"), el.getAttribute("type")].join("\n");
|
|
@@ -1043,16 +1147,16 @@ function finalizeDocument(document, opts) {
|
|
|
1043
1147
|
else seen.add(key);
|
|
1044
1148
|
});
|
|
1045
1149
|
if (opts.markdownUrl) {
|
|
1046
|
-
let link =
|
|
1150
|
+
let link = document2.head.querySelector(`link[rel="alternate"][type="${MARKDOWN_MIME}"]`);
|
|
1047
1151
|
if (!link) {
|
|
1048
|
-
link =
|
|
1152
|
+
link = document2.createElement("link");
|
|
1049
1153
|
link.setAttribute("rel", "alternate");
|
|
1050
1154
|
link.setAttribute("type", MARKDOWN_MIME);
|
|
1051
|
-
|
|
1155
|
+
document2.head.appendChild(link);
|
|
1052
1156
|
}
|
|
1053
1157
|
link.setAttribute("href", opts.markdownUrl);
|
|
1054
1158
|
}
|
|
1055
|
-
return "<!DOCTYPE html>" +
|
|
1159
|
+
return "<!DOCTYPE html>" + document2.documentElement.outerHTML;
|
|
1056
1160
|
}
|
|
1057
1161
|
|
|
1058
1162
|
// src/vite/prerender/crawl.ts
|
|
@@ -1749,10 +1853,10 @@ async function prerenderSite(input) {
|
|
|
1749
1853
|
}
|
|
1750
1854
|
}
|
|
1751
1855
|
function classicScriptSrcs(html) {
|
|
1752
|
-
const { document, dispose } = parseHtml(html);
|
|
1856
|
+
const { document: document2, dispose } = parseHtml(html);
|
|
1753
1857
|
try {
|
|
1754
1858
|
const out = [];
|
|
1755
|
-
for (const el of Array.from(
|
|
1859
|
+
for (const el of Array.from(document2.querySelectorAll("script[src]"))) {
|
|
1756
1860
|
if (el.hasAttribute("async") || el.hasAttribute("defer")) continue;
|
|
1757
1861
|
const type = (el.getAttribute("type") ?? "").trim().toLowerCase();
|
|
1758
1862
|
if (type !== "" && !/^(text|application)\/(x-)?(java|ecma)script$/.test(type)) continue;
|
|
@@ -1850,29 +1954,32 @@ function buildRobots(opts) {
|
|
|
1850
1954
|
return blocks.join("\n\n") + "\n";
|
|
1851
1955
|
}
|
|
1852
1956
|
|
|
1853
|
-
// src/vite/
|
|
1854
|
-
|
|
1855
|
-
|
|
1957
|
+
// src/vite/llms.ts
|
|
1958
|
+
var BLOCK_START = /^(\d*)([#>\-+*=_`~.)])/;
|
|
1959
|
+
function paragraphs(text) {
|
|
1960
|
+
if (text === void 0) return [];
|
|
1961
|
+
return text.split(/\n\s*\n/).map((p) => escapeMarkdownInline(p)).filter(Boolean).map((p) => p.replace(BLOCK_START, "$1\\$2"));
|
|
1856
1962
|
}
|
|
1857
|
-
function
|
|
1858
|
-
const
|
|
1859
|
-
return
|
|
1963
|
+
function linkLine(title, url, description) {
|
|
1964
|
+
const desc = escapeMarkdownInline(description);
|
|
1965
|
+
return `- [${escapeMarkdownInline(title)}](${escapeLinkTarget(url)})${desc ? `: ${desc}` : ""}`;
|
|
1860
1966
|
}
|
|
1861
|
-
|
|
1862
|
-
// src/vite/llms.ts
|
|
1863
1967
|
function header(site) {
|
|
1864
1968
|
const blocks = [`# ${escapeMarkdownInline(site.title)}`];
|
|
1865
1969
|
const desc = escapeMarkdownInline(site.description);
|
|
1866
1970
|
if (desc) blocks.push(`> ${desc}`);
|
|
1971
|
+
blocks.push(...paragraphs(site.intro));
|
|
1972
|
+
for (const sec of site.sections ?? []) {
|
|
1973
|
+
const body = [...paragraphs(sec.text)];
|
|
1974
|
+
const links = (sec.links ?? []).map((l) => linkLine(l.title, l.url, l.description));
|
|
1975
|
+
if (links.length) body.push(links.join("\n"));
|
|
1976
|
+
blocks.push([`## ${escapeMarkdownInline(sec.title)}`, ...body].join("\n\n"));
|
|
1977
|
+
}
|
|
1867
1978
|
return blocks;
|
|
1868
1979
|
}
|
|
1869
1980
|
function buildLlmsTxt(site, pages, sections = []) {
|
|
1870
1981
|
const blocks = header(site);
|
|
1871
|
-
const lines = [...pages].sort(byRoute).map((p) =>
|
|
1872
|
-
const title = escapeMarkdownInline(p.title);
|
|
1873
|
-
const desc = escapeMarkdownInline(p.description);
|
|
1874
|
-
return `- [${title}](${escapeLinkTarget(p.mdUrl ?? p.url)})${desc ? `: ${desc}` : ""}`;
|
|
1875
|
-
});
|
|
1982
|
+
const lines = [...pages].sort(byRoute).map((p) => linkLine(p.title, p.mdUrl ?? p.url, p.description));
|
|
1876
1983
|
blocks.push(["## Pages", ...lines.length ? ["", ...lines] : []].join("\n"));
|
|
1877
1984
|
for (const s of sections) {
|
|
1878
1985
|
blocks.push([`## ${s.heading}`, ...s.lines.length ? ["", ...s.lines] : []].join("\n"));
|
|
@@ -1981,8 +2088,13 @@ async function stageSiteFiles(opts, stager, { pages: allPages, listed, mdByRoute
|
|
|
1981
2088
|
const md = mdByRoute.get(p.route);
|
|
1982
2089
|
return md !== void 0 ? { ...p, markdown: md } : { ...p };
|
|
1983
2090
|
});
|
|
1984
|
-
|
|
1985
|
-
|
|
2091
|
+
const llmsSite = {
|
|
2092
|
+
...site,
|
|
2093
|
+
...llms.intro !== void 0 ? { intro: llms.intro } : {},
|
|
2094
|
+
...llms.sections !== void 0 ? { sections: llms.sections } : {}
|
|
2095
|
+
};
|
|
2096
|
+
stager.stage(fileOf(LLMS_TXT_PATH), buildLlmsTxt(llmsSite, llmsPages, sections));
|
|
2097
|
+
if (markdown) stager.stage(fileOf(LLMS_FULL_TXT_PATH), buildLlmsFullTxt(llmsSite, llmsPages));
|
|
1986
2098
|
}
|
|
1987
2099
|
for (const ext of extensions) {
|
|
1988
2100
|
if (!ext.finalize) continue;
|
|
@@ -2000,6 +2112,60 @@ async function stageSiteFiles(opts, stager, { pages: allPages, listed, mdByRoute
|
|
|
2000
2112
|
}
|
|
2001
2113
|
}
|
|
2002
2114
|
|
|
2115
|
+
// src/vite/sitemap.ts
|
|
2116
|
+
function escapeXml(s) {
|
|
2117
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
2118
|
+
}
|
|
2119
|
+
function buildSitemap(siteUrl, routes, lastmod) {
|
|
2120
|
+
const urls = routes.map((r) => {
|
|
2121
|
+
const date = lastmod?.get(r);
|
|
2122
|
+
const mod = date !== void 0 ? `<lastmod>${escapeXml(date)}</lastmod>` : "";
|
|
2123
|
+
return ` <url><loc>${escapeXml(new URL(siteUrl + r).href)}</loc>${mod}</url>`;
|
|
2124
|
+
}).join("\n");
|
|
2125
|
+
return '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n' + urls + "\n</urlset>\n";
|
|
2126
|
+
}
|
|
2127
|
+
function isoDate(d) {
|
|
2128
|
+
return d.toISOString().slice(0, 10);
|
|
2129
|
+
}
|
|
2130
|
+
var DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
2131
|
+
var DATETIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/;
|
|
2132
|
+
function validDate(s) {
|
|
2133
|
+
const m = DATE.exec(s.slice(0, 10));
|
|
2134
|
+
if (!m) return false;
|
|
2135
|
+
const d = new Date(Date.UTC(+m[1], +m[2] - 1, +m[3]));
|
|
2136
|
+
return isoDate(d) === s.slice(0, 10) && (s.length === 10 || !Number.isNaN(Date.parse(s)));
|
|
2137
|
+
}
|
|
2138
|
+
function lastmodValue(v, route) {
|
|
2139
|
+
if (v === void 0) return void 0;
|
|
2140
|
+
if (v instanceof Date && !Number.isNaN(v.getTime())) return isoDate(v);
|
|
2141
|
+
if (typeof v === "string" && (DATE.test(v) || DATETIME.test(v)) && validDate(v)) return v;
|
|
2142
|
+
const got = v instanceof Date ? "an invalid Date" : JSON.stringify(v) ?? typeof v;
|
|
2143
|
+
throw new Error(
|
|
2144
|
+
withPrefix(
|
|
2145
|
+
`sitemap.lastmod must return "YYYY-MM-DD", a W3C datetime such as "2026-01-31T09:00:00Z", a Date or undefined (got ${got} for ${shortRoute(route)})`
|
|
2146
|
+
)
|
|
2147
|
+
);
|
|
2148
|
+
}
|
|
2149
|
+
async function sitemapLastmods(lastmod, routes, fileDate) {
|
|
2150
|
+
const out = /* @__PURE__ */ new Map();
|
|
2151
|
+
if (lastmod === "none") return out;
|
|
2152
|
+
for (const route of routes) {
|
|
2153
|
+
let v;
|
|
2154
|
+
if (lastmod === "file") {
|
|
2155
|
+
v = fileDate(route);
|
|
2156
|
+
} else {
|
|
2157
|
+
try {
|
|
2158
|
+
v = await lastmod(route);
|
|
2159
|
+
} catch (e) {
|
|
2160
|
+
throw prefixed(e, `sitemap.lastmod failed for ${shortRoute(route)}`);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
const value = lastmodValue(v, route);
|
|
2164
|
+
if (value !== void 0) out.set(route, value);
|
|
2165
|
+
}
|
|
2166
|
+
return out;
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2003
2169
|
// src/vite/buildPlugin.ts
|
|
2004
2170
|
function isClientEnv(env) {
|
|
2005
2171
|
if (!env) return true;
|
|
@@ -2078,7 +2244,11 @@ function createBuildPlugin(resolved, extensions) {
|
|
|
2078
2244
|
)
|
|
2079
2245
|
])
|
|
2080
2246
|
].sort() : resolved.routes;
|
|
2081
|
-
if (resolved.sitemap)
|
|
2247
|
+
if (resolved.sitemap) {
|
|
2248
|
+
const buildDate = /* @__PURE__ */ new Date();
|
|
2249
|
+
const lastmod = await sitemapLastmods(resolved.sitemapLastmod, sitemapRoutes, () => buildDate);
|
|
2250
|
+
stager.stage(fileOf(SITEMAP_PATH), buildSitemap(siteUrl, sitemapRoutes, lastmod));
|
|
2251
|
+
}
|
|
2082
2252
|
stager.stage(fileOf(ROBOTS_PATH), buildRobots(resolved));
|
|
2083
2253
|
if (rendered) {
|
|
2084
2254
|
const { siteUrl: url, markdown: md, llms } = resolved;
|
|
@@ -2168,10 +2338,10 @@ import fs3 from "fs";
|
|
|
2168
2338
|
import path4 from "path";
|
|
2169
2339
|
|
|
2170
2340
|
// src/vite/generateActions.ts
|
|
2171
|
-
function staticActions(
|
|
2341
|
+
function staticActions(document2) {
|
|
2172
2342
|
const out = /* @__PURE__ */ new Map();
|
|
2173
|
-
for (const a of markupActions(
|
|
2174
|
-
if (!isValidActionName(a.name) || out.has(a.name)) continue;
|
|
2343
|
+
for (const a of markupActions(document2)) {
|
|
2344
|
+
if (a.kind === "link" || !isValidActionName(a.name) || out.has(a.name)) continue;
|
|
2175
2345
|
out.set(a.name, {
|
|
2176
2346
|
name: a.name,
|
|
2177
2347
|
description: a.description?.trim() || a.name,
|
|
@@ -2380,6 +2550,7 @@ async function generateAgentFiles(options) {
|
|
|
2380
2550
|
const pages = [];
|
|
2381
2551
|
const result = [];
|
|
2382
2552
|
const mdByRoute = /* @__PURE__ */ new Map();
|
|
2553
|
+
const mtimeByRoute = /* @__PURE__ */ new Map();
|
|
2383
2554
|
const bodyFallback = [];
|
|
2384
2555
|
const utf16 = [];
|
|
2385
2556
|
const ownLinks = [];
|
|
@@ -2390,19 +2561,21 @@ async function generateAgentFiles(options) {
|
|
|
2390
2561
|
(m) => logger.warn(m)
|
|
2391
2562
|
);
|
|
2392
2563
|
for (const { route, file } of routes) {
|
|
2564
|
+
mtimeByRoute.set(route, fs3.statSync(path4.join(outDir, file)).mtime);
|
|
2393
2565
|
const bytes = fs3.readFileSync(path4.join(outDir, file));
|
|
2394
2566
|
const { charset, text: html } = decodeHtml(bytes);
|
|
2395
2567
|
const url = publicUrl(siteUrl, route);
|
|
2396
|
-
const { document, dispose } = parseHtml(html);
|
|
2568
|
+
const { document: document2, dispose } = parseHtml(html);
|
|
2397
2569
|
let page;
|
|
2398
2570
|
try {
|
|
2399
|
-
const actions = resolved.agentMap ? staticActions(
|
|
2400
|
-
const
|
|
2571
|
+
const actions = resolved.agentMap ? staticActions(document2) : [];
|
|
2572
|
+
const links = resolved.agentMap ? markupLinks(document2) : [];
|
|
2573
|
+
const info = analyzeDocument(document2, {
|
|
2401
2574
|
pageUrl: url,
|
|
2402
2575
|
contentSelector: markdown ? markdown.contentSelector : "main",
|
|
2403
2576
|
route
|
|
2404
2577
|
});
|
|
2405
|
-
const inPage = { "agent-map": { actions } };
|
|
2578
|
+
const inPage = { "agent-map": { actions, links } };
|
|
2406
2579
|
page = { route, info, listed: !info.noindex, inPage };
|
|
2407
2580
|
} catch (e) {
|
|
2408
2581
|
throw prefixed(e, `reading ${file} failed`);
|
|
@@ -2428,7 +2601,11 @@ async function generateAgentFiles(options) {
|
|
|
2428
2601
|
}
|
|
2429
2602
|
}
|
|
2430
2603
|
const listed = pages.filter((p) => p.listed);
|
|
2431
|
-
if (resolved.sitemap)
|
|
2604
|
+
if (resolved.sitemap) {
|
|
2605
|
+
const routes2 = listed.map((p) => p.route);
|
|
2606
|
+
const lastmod = await sitemapLastmods(resolved.sitemapLastmod, routes2, (r) => mtimeByRoute.get(r));
|
|
2607
|
+
guarded.stage(fileOf(SITEMAP_PATH), buildSitemap(siteUrl, routes2, lastmod));
|
|
2608
|
+
}
|
|
2432
2609
|
guarded.stage(fileOf(ROBOTS_PATH), buildRobots(resolved));
|
|
2433
2610
|
const extensions = resolved.agentMap ? [agentMapExtension({ siteUrl, fileName: resolved.agentMap.fileName })] : [];
|
|
2434
2611
|
await stageSiteFiles({ siteUrl, markdown, llms: resolved.llms, extensions, logger }, guarded, {
|
|
@@ -2436,7 +2613,23 @@ async function generateAgentFiles(options) {
|
|
|
2436
2613
|
listed,
|
|
2437
2614
|
mdByRoute
|
|
2438
2615
|
});
|
|
2439
|
-
|
|
2616
|
+
let flushed = false;
|
|
2617
|
+
try {
|
|
2618
|
+
stager.flush();
|
|
2619
|
+
flushed = true;
|
|
2620
|
+
} finally {
|
|
2621
|
+
if (!flushed || resolved.sitemapLastmod === "file") {
|
|
2622
|
+
for (const [route, mtime] of mtimeByRoute) {
|
|
2623
|
+
const file = fileByRoute.get(route);
|
|
2624
|
+
if (!written.has(file)) continue;
|
|
2625
|
+
try {
|
|
2626
|
+
fs3.utimesSync(path4.join(outDir, file), mtime, mtime);
|
|
2627
|
+
} catch (e) {
|
|
2628
|
+
if (flushed) throw e;
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2440
2633
|
if (markdown && bodyFallback.length > 0) {
|
|
2441
2634
|
logger.warn(
|
|
2442
2635
|
`${LOG_PREFIX} no [${CONTENT_ATTR}] or "${markdown.contentSelector}" element on ${bodyFallback.join(", ")}; content was taken from <body> minus header, nav, footer, aside`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anubhav-choubey/web-for-agents",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "SEO and AI-agent readiness for React + Vite: head tags, JSON-LD, prerendering, markdown pages, llms.txt, agents.json and WebMCP",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Anubhav Choubey",
|
package/dist/chunk-FEAKVNMG.js
DELETED
package/dist/chunk-LCLDQ3A3.js
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
// src/shared/jsonLd.ts
|
|
2
|
-
var MAX_NODES = 1e5;
|
|
3
|
-
function jsonLdTypes(value) {
|
|
4
|
-
const out = [];
|
|
5
|
-
const stack = [value];
|
|
6
|
-
for (let visited = 0; stack.length > 0 && visited < MAX_NODES; visited++) {
|
|
7
|
-
const v = stack.pop();
|
|
8
|
-
if (Array.isArray(v)) {
|
|
9
|
-
for (let i = v.length - 1; i >= 0; i--) stack.push(v[i]);
|
|
10
|
-
continue;
|
|
11
|
-
}
|
|
12
|
-
if (v === null || typeof v !== "object") continue;
|
|
13
|
-
const o = v;
|
|
14
|
-
const t = o["@type"];
|
|
15
|
-
if (typeof t === "string") out.push(t);
|
|
16
|
-
else if (Array.isArray(t)) {
|
|
17
|
-
for (const x of t) if (typeof x === "string") out.push(x);
|
|
18
|
-
}
|
|
19
|
-
if ("@graph" in o) stack.push(o["@graph"]);
|
|
20
|
-
}
|
|
21
|
-
return out;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export {
|
|
25
|
-
jsonLdTypes
|
|
26
|
-
};
|