@usenotra/agentside 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build.d.ts +12 -0
- package/dist/build.js +49 -5
- package/dist/html.js +78 -20
- package/dist/index.js +1 -1
- package/dist/types.d.ts +2 -0
- package/package.json +1 -1
package/dist/build.d.ts
CHANGED
|
@@ -15,6 +15,7 @@ export declare function llmsTxt(pages: readonly PrerenderedPage[], options: {
|
|
|
15
15
|
readonly title: string;
|
|
16
16
|
readonly description?: string | undefined;
|
|
17
17
|
readonly siteUrl?: string | undefined;
|
|
18
|
+
readonly sections?: Readonly<Record<string, string>> | undefined;
|
|
18
19
|
}): string;
|
|
19
20
|
export declare function llmsFullTxt(pages: readonly PrerenderedPage[]): string;
|
|
20
21
|
export declare function vercelRoutes(paths?: readonly string[]): {
|
|
@@ -26,6 +27,13 @@ export declare function vercelRoutes(paths?: readonly string[]): {
|
|
|
26
27
|
"X-Robots-Tag": string;
|
|
27
28
|
};
|
|
28
29
|
continue: boolean;
|
|
30
|
+
} | {
|
|
31
|
+
src: string;
|
|
32
|
+
headers: {
|
|
33
|
+
Vary: string;
|
|
34
|
+
Link: string;
|
|
35
|
+
};
|
|
36
|
+
continue: boolean;
|
|
29
37
|
} | {
|
|
30
38
|
src: string;
|
|
31
39
|
has: ({
|
|
@@ -45,3 +53,7 @@ export declare function vercelRoutes(paths?: readonly string[]): {
|
|
|
45
53
|
};
|
|
46
54
|
})[];
|
|
47
55
|
};
|
|
56
|
+
export declare function applyVercelBuildOutput(options?: {
|
|
57
|
+
readonly outputDir?: string;
|
|
58
|
+
readonly paths?: readonly string[];
|
|
59
|
+
}): Promise<void>;
|
package/dist/build.js
CHANGED
|
@@ -54,7 +54,8 @@ export async function prerenderHtml(options) {
|
|
|
54
54
|
return;
|
|
55
55
|
try {
|
|
56
56
|
const html = await readFile(join(options.dir, file), "utf8");
|
|
57
|
-
const
|
|
57
|
+
const converted = htmlToMarkdown(html, options);
|
|
58
|
+
const markdown = options.normalize ? normalizeUnicode(converted) : converted;
|
|
58
59
|
const target = join(outDir, toMarkdownPath(path));
|
|
59
60
|
await mkdir(dirname(target), { recursive: true });
|
|
60
61
|
await writeFile(target, markdown);
|
|
@@ -74,15 +75,30 @@ export async function prerenderHtml(options) {
|
|
|
74
75
|
}
|
|
75
76
|
return pages;
|
|
76
77
|
}
|
|
78
|
+
function sectionTitle(segment) {
|
|
79
|
+
return segment.replace(/[-_]+/g, " ").replace(/^\w/, (letter) => letter.toUpperCase());
|
|
80
|
+
}
|
|
77
81
|
export function llmsTxt(pages, options) {
|
|
78
82
|
const url = (path) => `${options.siteUrl ?? ""}${toMarkdownPath(path)}`;
|
|
83
|
+
const prefixes = Object.keys(options.sections ?? {}).sort((left, right) => right.length - left.length);
|
|
84
|
+
const groups = new Map();
|
|
85
|
+
for (const page of pages) {
|
|
86
|
+
const prefix = prefixes.find((candidate) => page.path === candidate || page.path.startsWith(`${candidate.replace(/\/$/, "")}/`));
|
|
87
|
+
const segment = page.path.split("/")[1] ?? "";
|
|
88
|
+
const isLeaf = page.path.split("/").length <= 2 && !pages.some((other) => other.path.startsWith(`${page.path}/`));
|
|
89
|
+
const title = prefix ? (options.sections?.[prefix] ?? "Pages") : isLeaf || !segment ? "Pages" : sectionTitle(segment);
|
|
90
|
+
groups.set(title, [...(groups.get(title) ?? []), page]);
|
|
91
|
+
}
|
|
92
|
+
const ordered = [...groups.entries()].sort(([left], [right]) => (left === "Pages" ? -1 : right === "Pages" ? 1 : 0));
|
|
79
93
|
return [
|
|
80
94
|
`# ${options.title}`,
|
|
81
95
|
...(options.description ? ["", `> ${options.description}`] : []),
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
96
|
+
...ordered.flatMap(([title, items]) => [
|
|
97
|
+
"",
|
|
98
|
+
`## ${title}`,
|
|
99
|
+
"",
|
|
100
|
+
...items.map((page) => `- [${page.title || page.path}](${url(page.path)})${page.description ? `: ${page.description}` : ""}`),
|
|
101
|
+
]),
|
|
86
102
|
"",
|
|
87
103
|
].join("\n");
|
|
88
104
|
}
|
|
@@ -112,6 +128,24 @@ export function vercelRoutes(paths) {
|
|
|
112
128
|
const pagePattern = nested ? `(${nested.join("|")})` : "((?!.*\\.).*[^/])";
|
|
113
129
|
const routes = [
|
|
114
130
|
{ src: "^/(.+)\\.md$", headers: markdownHeaders, continue: true },
|
|
131
|
+
...(!paths || paths.includes("/")
|
|
132
|
+
? [
|
|
133
|
+
{
|
|
134
|
+
src: "^/$",
|
|
135
|
+
headers: { Vary: "Accept, User-Agent", Link: '</index.md>; rel="alternate"; type="text/markdown"' },
|
|
136
|
+
continue: true,
|
|
137
|
+
},
|
|
138
|
+
]
|
|
139
|
+
: []),
|
|
140
|
+
...(!nested || nested.length > 0
|
|
141
|
+
? [
|
|
142
|
+
{
|
|
143
|
+
src: `^/${pagePattern}/?$`,
|
|
144
|
+
headers: { Vary: "Accept, User-Agent", Link: '</$1.md>; rel="alternate"; type="text/markdown"' },
|
|
145
|
+
continue: true,
|
|
146
|
+
},
|
|
147
|
+
]
|
|
148
|
+
: []),
|
|
115
149
|
...conditions.flatMap((condition) => [
|
|
116
150
|
...(!paths || paths.includes("/")
|
|
117
151
|
? [{ src: "^/$", has: [condition], dest: "/index.md", headers: markdownHeaders }]
|
|
@@ -123,3 +157,13 @@ export function vercelRoutes(paths) {
|
|
|
123
157
|
];
|
|
124
158
|
return { routes };
|
|
125
159
|
}
|
|
160
|
+
export async function applyVercelBuildOutput(options = {}) {
|
|
161
|
+
const file = join(options.outputDir ?? ".vercel/output", "config.json");
|
|
162
|
+
const config = JSON.parse(await readFile(file, "utf8"));
|
|
163
|
+
const existing = config.routes ?? [];
|
|
164
|
+
const filesystem = existing.findIndex((route) => route["handle"] === "filesystem");
|
|
165
|
+
const { routes } = vercelRoutes(options.paths);
|
|
166
|
+
const insertAt = filesystem === -1 ? 0 : filesystem;
|
|
167
|
+
config.routes = [...existing.slice(0, insertAt), ...routes, ...existing.slice(insertAt)];
|
|
168
|
+
await writeFile(file, `${JSON.stringify(config, null, 2)}\n`);
|
|
169
|
+
}
|
package/dist/html.js
CHANGED
|
@@ -1,10 +1,52 @@
|
|
|
1
1
|
import { NodeHtmlMarkdown } from "node-html-markdown";
|
|
2
2
|
import { parse } from "node-html-parser";
|
|
3
|
-
const DROP = "script, style, noscript, template, svg, iframe, canvas, form, button, [hidden], [aria-hidden='true']";
|
|
4
|
-
const BLOCK = new Set(["H1", "H2", "H3", "H4", "H5", "H6", "P", "DIV", "UL", "OL", "IMG", "SECTION"]);
|
|
5
|
-
const converter = new NodeHtmlMarkdown({
|
|
3
|
+
const DROP = "script, style, noscript, template, svg, iframe, canvas, form, button, nav, [hidden], [aria-hidden='true'], img[alt=''], img:not([alt])";
|
|
4
|
+
const BLOCK = new Set(["H1", "H2", "H3", "H4", "H5", "H6", "P", "DIV", "UL", "OL", "IMG", "SECTION", "ARTICLE", "HEADER", "FOOTER"]);
|
|
5
|
+
const converter = new NodeHtmlMarkdown({
|
|
6
|
+
bulletMarker: "-",
|
|
7
|
+
codeBlockStyle: "fenced",
|
|
8
|
+
maxConsecutiveNewlines: 2,
|
|
9
|
+
globalEscape: [/`/g, "\\`"],
|
|
10
|
+
lineStartEscape: [/^(\s*)((?:[-+>]\s)|(?:#{1,6}\s)|(?:\d+\.\s))/g, "$1\\$2"],
|
|
11
|
+
});
|
|
6
12
|
function text(element) {
|
|
7
|
-
|
|
13
|
+
const parts = element.childNodes.map((child) => "tagName" in child
|
|
14
|
+
? BLOCK.has(child.tagName)
|
|
15
|
+
? ` ${text(child)} `
|
|
16
|
+
: text(child)
|
|
17
|
+
: child.textContent);
|
|
18
|
+
return parts.join("").replace(/\s+/g, " ").trim();
|
|
19
|
+
}
|
|
20
|
+
function blockTexts(element) {
|
|
21
|
+
const out = [];
|
|
22
|
+
let inline = "";
|
|
23
|
+
for (const child of element.childNodes) {
|
|
24
|
+
if ("tagName" in child && BLOCK.has(child.tagName)) {
|
|
25
|
+
if (inline.trim())
|
|
26
|
+
out.push(inline.trim());
|
|
27
|
+
inline = "";
|
|
28
|
+
out.push(...blockTexts(child));
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
inline += ` ${"tagName" in child ? text(child) : child.textContent}`;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (inline.trim())
|
|
35
|
+
out.push(inline.replace(/\s+/g, " ").trim());
|
|
36
|
+
return out.map((part) => part.replace(/\s+/g, " ").trim()).filter(Boolean);
|
|
37
|
+
}
|
|
38
|
+
function tightenTables(markdown) {
|
|
39
|
+
let fenced = false;
|
|
40
|
+
return markdown
|
|
41
|
+
.split("\n")
|
|
42
|
+
.map((line) => {
|
|
43
|
+
if (line.startsWith("```"))
|
|
44
|
+
fenced = !fenced;
|
|
45
|
+
if (fenced || !line.startsWith("|"))
|
|
46
|
+
return line;
|
|
47
|
+
return line.replace(/ {2,}/g, " ").replace(/-{3,}/g, "---");
|
|
48
|
+
})
|
|
49
|
+
.join("\n");
|
|
8
50
|
}
|
|
9
51
|
function escapeHtml(value) {
|
|
10
52
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
@@ -16,7 +58,7 @@ function absolutize(element, attribute, siteUrl) {
|
|
|
16
58
|
element.setAttribute(attribute, new URL(value, siteUrl).href);
|
|
17
59
|
}
|
|
18
60
|
export function htmlToMarkdown(html, options = {}) {
|
|
19
|
-
const document = parse(html, { comment: false });
|
|
61
|
+
const document = parse(html, { comment: false, blockTextElements: { script: true, noscript: true, style: true } });
|
|
20
62
|
const selectors = options.selector ? [options.selector] : ["main", "article", "body"];
|
|
21
63
|
const root = selectors.map((selector) => document.querySelector(selector)).find(Boolean) ?? document;
|
|
22
64
|
for (const element of root.querySelectorAll("[data-ai-ignore]"))
|
|
@@ -28,29 +70,45 @@ export function htmlToMarkdown(html, options = {}) {
|
|
|
28
70
|
slots.push(element.getAttribute("data-ai-content") ?? "");
|
|
29
71
|
element.replaceWith(`<p>agentsideslot${slots.length - 1}x</p>`);
|
|
30
72
|
}
|
|
31
|
-
for (const
|
|
73
|
+
for (const pre of root.querySelectorAll("pre")) {
|
|
74
|
+
const lines = pre.querySelectorAll(".line, [data-line]");
|
|
75
|
+
const code = pre.querySelector("code");
|
|
76
|
+
const language = pre.getAttribute("data-language") ??
|
|
77
|
+
/(?:^|\s)language-([\w-]+)/.exec(`${code?.getAttribute("class") ?? ""} ${pre.getAttribute("class") ?? ""}`)?.[1] ??
|
|
78
|
+
"";
|
|
79
|
+
if (lines.length < 2 && !pre.getAttribute("data-language"))
|
|
80
|
+
continue;
|
|
81
|
+
const body = lines.length > 1 ? lines.map((line) => line.textContent).join("\n") : (code ?? pre).textContent;
|
|
82
|
+
pre.replaceWith(`<pre><code class="language-${escapeHtml(language)}">${escapeHtml(body)}</code></pre>`);
|
|
83
|
+
}
|
|
84
|
+
for (const list of [...root.querySelectorAll("dl")].reverse()) {
|
|
32
85
|
const items = [];
|
|
33
|
-
for (const
|
|
34
|
-
if (!("tagName" in child))
|
|
35
|
-
continue;
|
|
36
|
-
const element = child;
|
|
86
|
+
for (const element of list.querySelectorAll("dt, dd")) {
|
|
37
87
|
if (element.tagName === "DT")
|
|
38
88
|
items.push(`<li><strong>${escapeHtml(text(element))}</strong>`);
|
|
39
|
-
else if (
|
|
40
|
-
items[items.length - 1] +=
|
|
89
|
+
else if (items.length > 0)
|
|
90
|
+
items[items.length - 1] += `${items[items.length - 1]?.endsWith("</strong>") ? ": " : "; "}${element.innerHTML}`;
|
|
41
91
|
}
|
|
42
92
|
list.replaceWith(`<ul>${items.map((item) => `${item}</li>`).join("")}</ul>`);
|
|
43
93
|
}
|
|
44
94
|
for (const card of root.querySelectorAll("a")) {
|
|
45
|
-
const
|
|
46
|
-
if (
|
|
95
|
+
const inner = card.querySelectorAll("*");
|
|
96
|
+
if (!inner.some((element) => BLOCK.has(element.tagName)) && inner.filter((element) => element.tagName === "SPAN").length < 2) {
|
|
47
97
|
continue;
|
|
48
|
-
|
|
49
|
-
const title = heading ? text(heading) : text(card);
|
|
50
|
-
heading?.remove();
|
|
51
|
-
const rest = text(card);
|
|
98
|
+
}
|
|
52
99
|
const href = escapeHtml(card.getAttribute("href") ?? "");
|
|
53
|
-
card.
|
|
100
|
+
const heading = card.querySelector("h1, h2, h3, h4, h5, h6");
|
|
101
|
+
if (heading) {
|
|
102
|
+
const title = text(heading);
|
|
103
|
+
heading.remove();
|
|
104
|
+
const rest = blockTexts(card).join(" ");
|
|
105
|
+
const level = heading.tagName.toLowerCase();
|
|
106
|
+
card.replaceWith(`<${level}><a href="${href}">${escapeHtml(title)}</a></${level}>${rest ? `<p>${escapeHtml(rest)}</p>` : ""}`);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const [title = "", ...rest] = blockTexts(card).length > 1 ? blockTexts(card) : card.childNodes.map((child) => child.textContent.trim()).filter(Boolean);
|
|
110
|
+
card.replaceWith(`<p><a href="${href}">${escapeHtml(title)}</a>${rest.length ? `: ${escapeHtml(rest.join(" "))}` : ""}</p>`);
|
|
111
|
+
}
|
|
54
112
|
}
|
|
55
113
|
if (options.siteUrl) {
|
|
56
114
|
for (const link of root.querySelectorAll("a[href]"))
|
|
@@ -58,7 +116,7 @@ export function htmlToMarkdown(html, options = {}) {
|
|
|
58
116
|
for (const image of root.querySelectorAll("img[src]"))
|
|
59
117
|
absolutize(image, "src", options.siteUrl);
|
|
60
118
|
}
|
|
61
|
-
let markdown = converter.translate(root.innerHTML);
|
|
119
|
+
let markdown = tightenTables(converter.translate(root.innerHTML));
|
|
62
120
|
markdown = markdown
|
|
63
121
|
.replace(/\)\[(?=[^\]]*\]\()/g, ") [")
|
|
64
122
|
.replace(/agentsideslot(\d+)x/g, (_, index) => slots[Number(index)] ?? "");
|
package/dist/index.js
CHANGED
|
@@ -63,7 +63,7 @@ export function agentside(config) {
|
|
|
63
63
|
return render(request, pathname);
|
|
64
64
|
}
|
|
65
65
|
const accept = request.headers.get("accept");
|
|
66
|
-
const format = negotiateFormat(accept, config.accept);
|
|
66
|
+
const format = negotiateFormat(accept, config.accept ?? "contains");
|
|
67
67
|
const agent = detectAgent(request.headers.get("user-agent"));
|
|
68
68
|
if (format === "markdown" || agent) {
|
|
69
69
|
const response = await render(request, pathname, agent);
|
package/dist/types.d.ts
CHANGED
|
@@ -65,7 +65,9 @@ export interface PrerenderHtmlOptions extends HtmlToMarkdownOptions {
|
|
|
65
65
|
readonly llms?: {
|
|
66
66
|
readonly title: string;
|
|
67
67
|
readonly description?: string;
|
|
68
|
+
readonly sections?: Readonly<Record<string, string>>;
|
|
68
69
|
} | false;
|
|
70
|
+
readonly normalize?: boolean;
|
|
69
71
|
}
|
|
70
72
|
export interface PrerenderedPage {
|
|
71
73
|
readonly path: string;
|