@usenotra/agentside 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/build.d.ts CHANGED
@@ -15,6 +15,7 @@ export declare function llmsTxt(pages: readonly PrerenderedPage[], options: {
15
15
  readonly title: string;
16
16
  readonly description?: string | undefined;
17
17
  readonly siteUrl?: string | undefined;
18
+ readonly sections?: Readonly<Record<string, string>> | undefined;
18
19
  }): string;
19
20
  export declare function llmsFullTxt(pages: readonly PrerenderedPage[]): string;
20
21
  export declare function vercelRoutes(paths?: readonly string[]): {
@@ -30,6 +31,7 @@ export declare function vercelRoutes(paths?: readonly string[]): {
30
31
  src: string;
31
32
  headers: {
32
33
  Vary: string;
34
+ Link: string;
33
35
  };
34
36
  continue: boolean;
35
37
  } | {
@@ -51,3 +53,7 @@ export declare function vercelRoutes(paths?: readonly string[]): {
51
53
  };
52
54
  })[];
53
55
  };
56
+ export declare function applyVercelBuildOutput(options?: {
57
+ readonly outputDir?: string;
58
+ readonly paths?: readonly string[];
59
+ }): Promise<void>;
package/dist/build.js CHANGED
@@ -54,7 +54,8 @@ export async function prerenderHtml(options) {
54
54
  return;
55
55
  try {
56
56
  const html = await readFile(join(options.dir, file), "utf8");
57
- const markdown = normalizeUnicode(htmlToMarkdown(html, options));
57
+ const converted = htmlToMarkdown(html, options);
58
+ const markdown = options.normalize ? normalizeUnicode(converted) : converted;
58
59
  const target = join(outDir, toMarkdownPath(path));
59
60
  await mkdir(dirname(target), { recursive: true });
60
61
  await writeFile(target, markdown);
@@ -74,15 +75,30 @@ export async function prerenderHtml(options) {
74
75
  }
75
76
  return pages;
76
77
  }
78
+ function sectionTitle(segment) {
79
+ return segment.replace(/[-_]+/g, " ").replace(/^\w/, (letter) => letter.toUpperCase());
80
+ }
77
81
  export function llmsTxt(pages, options) {
78
82
  const url = (path) => `${options.siteUrl ?? ""}${toMarkdownPath(path)}`;
83
+ const prefixes = Object.keys(options.sections ?? {}).sort((left, right) => right.length - left.length);
84
+ const groups = new Map();
85
+ for (const page of pages) {
86
+ const prefix = prefixes.find((candidate) => page.path === candidate || page.path.startsWith(`${candidate.replace(/\/$/, "")}/`));
87
+ const segment = page.path.split("/")[1] ?? "";
88
+ const isLeaf = page.path.split("/").length <= 2 && !pages.some((other) => other.path.startsWith(`${page.path}/`));
89
+ const title = prefix ? (options.sections?.[prefix] ?? "Pages") : isLeaf || !segment ? "Pages" : sectionTitle(segment);
90
+ groups.set(title, [...(groups.get(title) ?? []), page]);
91
+ }
92
+ const ordered = [...groups.entries()].sort(([left], [right]) => (left === "Pages" ? -1 : right === "Pages" ? 1 : 0));
79
93
  return [
80
94
  `# ${options.title}`,
81
95
  ...(options.description ? ["", `> ${options.description}`] : []),
82
- "",
83
- "## Pages",
84
- "",
85
- ...pages.map((page) => `- [${page.title || page.path}](${url(page.path)})${page.description ? `: ${page.description}` : ""}`),
96
+ ...ordered.flatMap(([title, items]) => [
97
+ "",
98
+ `## ${title}`,
99
+ "",
100
+ ...items.map((page) => `- [${page.title || page.path}](${url(page.path)})${page.description ? `: ${page.description}` : ""}`),
101
+ ]),
86
102
  "",
87
103
  ].join("\n");
88
104
  }
@@ -112,7 +128,24 @@ export function vercelRoutes(paths) {
112
128
  const pagePattern = nested ? `(${nested.join("|")})` : "((?!.*\\.).*[^/])";
113
129
  const routes = [
114
130
  { src: "^/(.+)\\.md$", headers: markdownHeaders, continue: true },
115
- { src: `^/$|^/${pagePattern}/?$`, headers: { Vary: "Accept, User-Agent" }, continue: true },
131
+ ...(!paths || paths.includes("/")
132
+ ? [
133
+ {
134
+ src: "^/$",
135
+ headers: { Vary: "Accept, User-Agent", Link: '</index.md>; rel="alternate"; type="text/markdown"' },
136
+ continue: true,
137
+ },
138
+ ]
139
+ : []),
140
+ ...(!nested || nested.length > 0
141
+ ? [
142
+ {
143
+ src: `^/${pagePattern}/?$`,
144
+ headers: { Vary: "Accept, User-Agent", Link: '</$1.md>; rel="alternate"; type="text/markdown"' },
145
+ continue: true,
146
+ },
147
+ ]
148
+ : []),
116
149
  ...conditions.flatMap((condition) => [
117
150
  ...(!paths || paths.includes("/")
118
151
  ? [{ src: "^/$", has: [condition], dest: "/index.md", headers: markdownHeaders }]
@@ -124,3 +157,13 @@ export function vercelRoutes(paths) {
124
157
  ];
125
158
  return { routes };
126
159
  }
160
+ export async function applyVercelBuildOutput(options = {}) {
161
+ const file = join(options.outputDir ?? ".vercel/output", "config.json");
162
+ const config = JSON.parse(await readFile(file, "utf8"));
163
+ const existing = config.routes ?? [];
164
+ const filesystem = existing.findIndex((route) => route["handle"] === "filesystem");
165
+ const { routes } = vercelRoutes(options.paths);
166
+ const insertAt = filesystem === -1 ? 0 : filesystem;
167
+ config.routes = [...existing.slice(0, insertAt), ...routes, ...existing.slice(insertAt)];
168
+ await writeFile(file, `${JSON.stringify(config, null, 2)}\n`);
169
+ }
package/dist/html.js CHANGED
@@ -1,10 +1,52 @@
1
1
  import { NodeHtmlMarkdown } from "node-html-markdown";
2
2
  import { parse } from "node-html-parser";
3
- const DROP = "script, style, noscript, template, svg, iframe, canvas, form, button, [hidden], [aria-hidden='true']";
4
- const BLOCK = new Set(["H1", "H2", "H3", "H4", "H5", "H6", "P", "DIV", "UL", "OL", "IMG", "SECTION"]);
5
- const converter = new NodeHtmlMarkdown({ bulletMarker: "-", codeBlockStyle: "fenced", maxConsecutiveNewlines: 2 });
3
+ const DROP = "script, style, noscript, template, svg, iframe, canvas, form, button, nav, [hidden], [aria-hidden='true'], img[alt=''], img:not([alt])";
4
+ const BLOCK = new Set(["H1", "H2", "H3", "H4", "H5", "H6", "P", "DIV", "UL", "OL", "IMG", "SECTION", "ARTICLE", "HEADER", "FOOTER"]);
5
+ const converter = new NodeHtmlMarkdown({
6
+ bulletMarker: "-",
7
+ codeBlockStyle: "fenced",
8
+ maxConsecutiveNewlines: 2,
9
+ globalEscape: [/`/g, "\\`"],
10
+ lineStartEscape: [/^(\s*)((?:[-+>]\s)|(?:#{1,6}\s)|(?:\d+\.\s))/g, "$1\\$2"],
11
+ });
6
12
  function text(element) {
7
- return element.textContent.replace(/\s+/g, " ").trim();
13
+ const parts = element.childNodes.map((child) => "tagName" in child
14
+ ? BLOCK.has(child.tagName)
15
+ ? ` ${text(child)} `
16
+ : text(child)
17
+ : child.textContent);
18
+ return parts.join("").replace(/\s+/g, " ").trim();
19
+ }
20
+ function blockTexts(element) {
21
+ const out = [];
22
+ let inline = "";
23
+ for (const child of element.childNodes) {
24
+ if ("tagName" in child && BLOCK.has(child.tagName)) {
25
+ if (inline.trim())
26
+ out.push(inline.trim());
27
+ inline = "";
28
+ out.push(...blockTexts(child));
29
+ }
30
+ else {
31
+ inline += ` ${"tagName" in child ? text(child) : child.textContent}`;
32
+ }
33
+ }
34
+ if (inline.trim())
35
+ out.push(inline.replace(/\s+/g, " ").trim());
36
+ return out.map((part) => part.replace(/\s+/g, " ").trim()).filter(Boolean);
37
+ }
38
+ function tightenTables(markdown) {
39
+ let fenced = false;
40
+ return markdown
41
+ .split("\n")
42
+ .map((line) => {
43
+ if (line.startsWith("```"))
44
+ fenced = !fenced;
45
+ if (fenced || !line.startsWith("|"))
46
+ return line;
47
+ return line.replace(/ {2,}/g, " ").replace(/-{3,}/g, "---");
48
+ })
49
+ .join("\n");
8
50
  }
9
51
  function escapeHtml(value) {
10
52
  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
@@ -16,7 +58,7 @@ function absolutize(element, attribute, siteUrl) {
16
58
  element.setAttribute(attribute, new URL(value, siteUrl).href);
17
59
  }
18
60
  export function htmlToMarkdown(html, options = {}) {
19
- const document = parse(html, { comment: false });
61
+ const document = parse(html, { comment: false, blockTextElements: { script: true, noscript: true, style: true } });
20
62
  const selectors = options.selector ? [options.selector] : ["main", "article", "body"];
21
63
  const root = selectors.map((selector) => document.querySelector(selector)).find(Boolean) ?? document;
22
64
  for (const element of root.querySelectorAll("[data-ai-ignore]"))
@@ -28,29 +70,45 @@ export function htmlToMarkdown(html, options = {}) {
28
70
  slots.push(element.getAttribute("data-ai-content") ?? "");
29
71
  element.replaceWith(`<p>agentsideslot${slots.length - 1}x</p>`);
30
72
  }
31
- for (const list of root.querySelectorAll("dl")) {
73
+ for (const pre of root.querySelectorAll("pre")) {
74
+ const lines = pre.querySelectorAll(".line, [data-line]");
75
+ const code = pre.querySelector("code");
76
+ const language = pre.getAttribute("data-language") ??
77
+ /(?:^|\s)language-([\w-]+)/.exec(`${code?.getAttribute("class") ?? ""} ${pre.getAttribute("class") ?? ""}`)?.[1] ??
78
+ "";
79
+ if (lines.length < 2 && !pre.getAttribute("data-language"))
80
+ continue;
81
+ const body = lines.length > 1 ? lines.map((line) => line.textContent).join("\n") : (code ?? pre).textContent;
82
+ pre.replaceWith(`<pre><code class="language-${escapeHtml(language)}">${escapeHtml(body)}</code></pre>`);
83
+ }
84
+ for (const list of [...root.querySelectorAll("dl")].reverse()) {
32
85
  const items = [];
33
- for (const child of list.childNodes) {
34
- if (!("tagName" in child))
35
- continue;
36
- const element = child;
86
+ for (const element of list.querySelectorAll("dt, dd")) {
37
87
  if (element.tagName === "DT")
38
88
  items.push(`<li><strong>${escapeHtml(text(element))}</strong>`);
39
- else if (element.tagName === "DD" && items.length > 0)
40
- items[items.length - 1] += `: ${element.innerHTML}`;
89
+ else if (items.length > 0)
90
+ items[items.length - 1] += `${items[items.length - 1]?.endsWith("</strong>") ? ": " : "; "}${element.innerHTML}`;
41
91
  }
42
92
  list.replaceWith(`<ul>${items.map((item) => `${item}</li>`).join("")}</ul>`);
43
93
  }
44
94
  for (const card of root.querySelectorAll("a")) {
45
- const blocks = card.childNodes.filter((child) => "tagName" in child && BLOCK.has(child.tagName));
46
- if (blocks.length === 0)
95
+ const inner = card.querySelectorAll("*");
96
+ if (!inner.some((element) => BLOCK.has(element.tagName)) && inner.filter((element) => element.tagName === "SPAN").length < 2) {
47
97
  continue;
48
- const heading = card.querySelector("h1, h2, h3, h4, h5, h6") ?? blocks[0];
49
- const title = heading ? text(heading) : text(card);
50
- heading?.remove();
51
- const rest = text(card);
98
+ }
52
99
  const href = escapeHtml(card.getAttribute("href") ?? "");
53
- card.replaceWith(`<p><a href="${href}">${escapeHtml(title)}</a>${rest ? `: ${escapeHtml(rest)}` : ""}</p>`);
100
+ const heading = card.querySelector("h1, h2, h3, h4, h5, h6");
101
+ if (heading) {
102
+ const title = text(heading);
103
+ heading.remove();
104
+ const rest = blockTexts(card).join(" ");
105
+ const level = heading.tagName.toLowerCase();
106
+ card.replaceWith(`<${level}><a href="${href}">${escapeHtml(title)}</a></${level}>${rest ? `<p>${escapeHtml(rest)}</p>` : ""}`);
107
+ }
108
+ else {
109
+ const [title = "", ...rest] = blockTexts(card).length > 1 ? blockTexts(card) : card.childNodes.map((child) => child.textContent.trim()).filter(Boolean);
110
+ card.replaceWith(`<p><a href="${href}">${escapeHtml(title)}</a>${rest.length ? `: ${escapeHtml(rest.join(" "))}` : ""}</p>`);
111
+ }
54
112
  }
55
113
  if (options.siteUrl) {
56
114
  for (const link of root.querySelectorAll("a[href]"))
@@ -58,7 +116,7 @@ export function htmlToMarkdown(html, options = {}) {
58
116
  for (const image of root.querySelectorAll("img[src]"))
59
117
  absolutize(image, "src", options.siteUrl);
60
118
  }
61
- let markdown = converter.translate(root.innerHTML);
119
+ let markdown = tightenTables(converter.translate(root.innerHTML));
62
120
  markdown = markdown
63
121
  .replace(/\)\[(?=[^\]]*\]\()/g, ") [")
64
122
  .replace(/agentsideslot(\d+)x/g, (_, index) => slots[Number(index)] ?? "");
package/dist/index.js CHANGED
@@ -63,7 +63,7 @@ export function agentside(config) {
63
63
  return render(request, pathname);
64
64
  }
65
65
  const accept = request.headers.get("accept");
66
- const format = negotiateFormat(accept, config.accept);
66
+ const format = negotiateFormat(accept, config.accept ?? "contains");
67
67
  const agent = detectAgent(request.headers.get("user-agent"));
68
68
  if (format === "markdown" || agent) {
69
69
  const response = await render(request, pathname, agent);
package/dist/types.d.ts CHANGED
@@ -65,7 +65,9 @@ export interface PrerenderHtmlOptions extends HtmlToMarkdownOptions {
65
65
  readonly llms?: {
66
66
  readonly title: string;
67
67
  readonly description?: string;
68
+ readonly sections?: Readonly<Record<string, string>>;
68
69
  } | false;
70
+ readonly normalize?: boolean;
69
71
  }
70
72
  export interface PrerenderedPage {
71
73
  readonly path: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@usenotra/agentside",
3
- "version": "0.2.3",
3
+ "version": "0.3.0",
4
4
  "description": "Serve markdown to AI agents and A/B test what they read. Built on Effect. By Notra.",
5
5
  "keywords": [
6
6
  "ai",