@usenotra/agentside 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/build.d.ts CHANGED
@@ -1,4 +1,5 @@
1
- import type { AgentsideConfig } from "./types.js";
1
+ import type { AgentsideConfig, PrerenderedPage, PrerenderHtmlOptions } from "./types.js";
2
+ export { htmlToMarkdown } from "./html.js";
2
3
  export interface PrerenderOptions {
3
4
  readonly outDir: string;
4
5
  readonly paths?: readonly string[];
@@ -9,3 +10,32 @@ export interface PrerenderResult {
9
10
  readonly skipped: readonly string[];
10
11
  }
11
12
  export declare function prerender(config: AgentsideConfig, options: PrerenderOptions): Promise<PrerenderResult>;
13
+ export declare function prerenderHtml(options: PrerenderHtmlOptions): Promise<PrerenderedPage[]>;
14
+ export declare function llmsTxt(pages: readonly PrerenderedPage[], options: {
15
+ readonly title: string;
16
+ readonly description?: string | undefined;
17
+ readonly siteUrl?: string | undefined;
18
+ }): string;
19
+ export declare function llmsFullTxt(pages: readonly PrerenderedPage[]): string;
20
+ export declare function vercelRoutes(paths?: readonly string[]): {
21
+ rewrites: {
22
+ source: string;
23
+ has: ({
24
+ readonly type: "header";
25
+ readonly key: "accept";
26
+ readonly value: ".*text/markdown.*";
27
+ } | {
28
+ readonly type: "header";
29
+ readonly key: "user-agent";
30
+ readonly value: `.*(${string}).*`;
31
+ })[];
32
+ destination: string;
33
+ }[];
34
+ headers: {
35
+ source: string;
36
+ headers: {
37
+ key: string;
38
+ value: string;
39
+ }[];
40
+ }[];
41
+ };
package/dist/build.js CHANGED
@@ -1,8 +1,11 @@
1
- import { mkdir, writeFile } from "node:fs/promises";
2
- import { dirname, join } from "node:path";
1
+ import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
2
+ import { dirname, join, sep } from "node:path";
3
3
  import { Effect, Exit } from "effect";
4
+ import { KNOWN_AGENTS } from "./agents.js";
4
5
  import { make } from "./core.js";
5
- import { toMarkdownPath } from "./text.js";
6
+ import { htmlToMarkdown, readMeta } from "./html.js";
7
+ import { normalizeUnicode, toMarkdownPath } from "./text.js";
8
+ export { htmlToMarkdown } from "./html.js";
6
9
  export async function prerender(config, options) {
7
10
  const server = Effect.runSync(make({ ...config, cache: false }));
8
11
  const paths = options.paths ?? Object.keys(config.routes).filter((pattern) => !pattern.includes("["));
@@ -30,3 +33,94 @@ export async function prerender(config, options) {
30
33
  }
31
34
  return { written, skipped };
32
35
  }
36
+ function pagePath(file) {
37
+ const path = `/${file.split(sep).join("/")}`.replace(/\/index\.html$/, "").replace(/\.html$/, "");
38
+ return path === "" ? "/" : path;
39
+ }
40
+ function isExcluded(path, exclude) {
41
+ return exclude.some((prefix) => path === prefix || path.startsWith(prefix.endsWith("/") ? prefix : `${prefix}/`));
42
+ }
43
+ export async function prerenderHtml(options) {
44
+ const outDir = options.outDir ?? options.dir;
45
+ const exclude = options.exclude ?? [];
46
+ const files = (await readdir(options.dir, { recursive: true }))
47
+ .filter((file) => file.endsWith(".html"))
48
+ .filter((file) => !/(^|[\\/])(404|500)\.html$/.test(file));
49
+ const pages = [];
50
+ const failures = [];
51
+ await Promise.all(files.map(async (file) => {
52
+ const path = pagePath(file);
53
+ if (isExcluded(path, exclude))
54
+ return;
55
+ try {
56
+ const html = await readFile(join(options.dir, file), "utf8");
57
+ const markdown = normalizeUnicode(htmlToMarkdown(html, options));
58
+ const target = join(outDir, toMarkdownPath(path));
59
+ await mkdir(dirname(target), { recursive: true });
60
+ await writeFile(target, markdown);
61
+ pages.push({ path, ...readMeta(html), markdown });
62
+ }
63
+ catch (error) {
64
+ failures.push(`${path}: ${error instanceof Error ? error.message : String(error)}`);
65
+ }
66
+ }));
67
+ if (failures.length > 0) {
68
+ throw new Error(`agentside: ${failures.length} page(s) failed to convert\n${failures.join("\n")}`);
69
+ }
70
+ pages.sort((left, right) => left.path.localeCompare(right.path));
71
+ if (options.llms) {
72
+ await writeFile(join(outDir, "llms.txt"), llmsTxt(pages, { ...options.llms, siteUrl: options.siteUrl }));
73
+ await writeFile(join(outDir, "llms-full.txt"), llmsFullTxt(pages));
74
+ }
75
+ return pages;
76
+ }
77
+ export function llmsTxt(pages, options) {
78
+ const url = (path) => `${options.siteUrl ?? ""}${toMarkdownPath(path)}`;
79
+ return [
80
+ `# ${options.title}`,
81
+ ...(options.description ? ["", `> ${options.description}`] : []),
82
+ "",
83
+ "## Pages",
84
+ "",
85
+ ...pages.map((page) => `- [${page.title || page.path}](${url(page.path)})${page.description ? `: ${page.description}` : ""}`),
86
+ "",
87
+ ].join("\n");
88
+ }
89
+ export function llmsFullTxt(pages) {
90
+ return `${pages.map((page) => page.markdown.trim()).join("\n\n---\n\n")}\n`;
91
+ }
92
+ function escapeRegex(value) {
93
+ return value.replace(/[.*+?^${}()|[\]\\:]/g, "\\$&");
94
+ }
95
+ export function vercelRoutes(paths) {
96
+ const agentTokens = [
97
+ ...new Set(KNOWN_AGENTS.filter((agent) => agent.name !== "Google-Extended").flatMap((agent) => [
98
+ agent.name,
99
+ agent.name.toLowerCase(),
100
+ ])),
101
+ ];
102
+ const conditions = [
103
+ { type: "header", key: "accept", value: ".*text/markdown.*" },
104
+ { type: "header", key: "user-agent", value: `.*(${agentTokens.map(escapeRegex).join("|")}).*` },
105
+ ];
106
+ const nested = paths?.filter((path) => path !== "/").map((path) => escapeRegex(path.slice(1)));
107
+ const rewrites = conditions.flatMap((condition) => [
108
+ ...(!paths || paths.includes("/") ? [{ source: "/", has: [condition], destination: "/index.md" }] : []),
109
+ ...(!nested
110
+ ? [{ source: "/((?!.*\\.)[^?]*[^/])/?", has: [condition], destination: "/$1.md" }]
111
+ : nested.length > 0
112
+ ? [{ source: `/:path(${nested.join("|")})`, has: [condition], destination: "/:path.md" }]
113
+ : []),
114
+ ]);
115
+ const headers = [
116
+ {
117
+ source: "/(.*)\\.md",
118
+ headers: [
119
+ { key: "Content-Type", value: "text/markdown; charset=utf-8" },
120
+ { key: "Vary", value: "Accept, User-Agent" },
121
+ { key: "X-Robots-Tag", value: "noindex" },
122
+ ],
123
+ },
124
+ ];
125
+ return { rewrites, headers };
126
+ }
package/dist/html.d.ts ADDED
@@ -0,0 +1,6 @@
1
+ import type { HtmlToMarkdownOptions } from "./types.js";
2
+ export declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string;
3
+ export declare function readMeta(html: string): {
4
+ title: string;
5
+ description: string;
6
+ };
package/dist/html.js ADDED
@@ -0,0 +1,78 @@
1
+ import { NodeHtmlMarkdown } from "node-html-markdown";
2
+ import { parse } from "node-html-parser";
3
+ const DROP = "script, style, noscript, template, svg, iframe, canvas, form, button, [hidden], [aria-hidden='true']";
4
+ const BLOCK = new Set(["H1", "H2", "H3", "H4", "H5", "H6", "P", "DIV", "UL", "OL", "IMG", "SECTION"]);
5
+ const converter = new NodeHtmlMarkdown({ bulletMarker: "-", codeBlockStyle: "fenced", maxConsecutiveNewlines: 2 });
6
+ function text(element) {
7
+ return element.textContent.replace(/\s+/g, " ").trim();
8
+ }
9
+ function escapeHtml(value) {
10
+ return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
11
+ }
12
+ function absolutize(element, attribute, siteUrl) {
13
+ const value = element.getAttribute(attribute);
14
+ if (!value || value.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(value))
15
+ return;
16
+ element.setAttribute(attribute, new URL(value, siteUrl).href);
17
+ }
18
+ export function htmlToMarkdown(html, options = {}) {
19
+ const document = parse(html, { comment: false });
20
+ const selectors = options.selector ? [options.selector] : ["main", "article", "body"];
21
+ const root = selectors.map((selector) => document.querySelector(selector)).find(Boolean) ?? document;
22
+ for (const element of root.querySelectorAll("[data-ai-ignore]"))
23
+ element.remove();
24
+ for (const element of root.querySelectorAll(DROP))
25
+ element.remove();
26
+ const slots = [];
27
+ for (const element of root.querySelectorAll("[data-ai-content]")) {
28
+ slots.push(element.getAttribute("data-ai-content") ?? "");
29
+ element.replaceWith(`<p>agentsideslot${slots.length - 1}x</p>`);
30
+ }
31
+ for (const list of root.querySelectorAll("dl")) {
32
+ const items = [];
33
+ for (const child of list.childNodes) {
34
+ if (!("tagName" in child))
35
+ continue;
36
+ const element = child;
37
+ if (element.tagName === "DT")
38
+ items.push(`<li><strong>${escapeHtml(text(element))}</strong>`);
39
+ else if (element.tagName === "DD" && items.length > 0)
40
+ items[items.length - 1] += `: ${element.innerHTML}`;
41
+ }
42
+ list.replaceWith(`<ul>${items.map((item) => `${item}</li>`).join("")}</ul>`);
43
+ }
44
+ for (const card of root.querySelectorAll("a")) {
45
+ const blocks = card.childNodes.filter((child) => "tagName" in child && BLOCK.has(child.tagName));
46
+ if (blocks.length === 0)
47
+ continue;
48
+ const heading = card.querySelector("h1, h2, h3, h4, h5, h6") ?? blocks[0];
49
+ const title = heading ? text(heading) : text(card);
50
+ heading?.remove();
51
+ const rest = text(card);
52
+ const href = escapeHtml(card.getAttribute("href") ?? "");
53
+ card.replaceWith(`<p><a href="${href}">${escapeHtml(title)}</a>${rest ? `: ${escapeHtml(rest)}` : ""}</p>`);
54
+ }
55
+ if (options.siteUrl) {
56
+ for (const link of root.querySelectorAll("a[href]"))
57
+ absolutize(link, "href", options.siteUrl);
58
+ for (const image of root.querySelectorAll("img[src]"))
59
+ absolutize(image, "src", options.siteUrl);
60
+ }
61
+ let markdown = converter.translate(root.innerHTML);
62
+ markdown = markdown
63
+ .replace(/\)\[(?=[^\]]*\]\()/g, ") [")
64
+ .replace(/agentsideslot(\d+)x/g, (_, index) => slots[Number(index)] ?? "");
65
+ if (!root.querySelector("h1")) {
66
+ const title = document.querySelector("h1") ?? document.querySelector("title");
67
+ if (title && text(title))
68
+ markdown = `# ${text(title)}\n\n${markdown}`;
69
+ }
70
+ return `${markdown.trim()}\n`;
71
+ }
72
+ export function readMeta(html) {
73
+ const document = parse(html, { comment: false });
74
+ return {
75
+ title: text(document.querySelector("title") ?? document.querySelector("h1") ?? parse("")),
76
+ description: document.querySelector('meta[name="description"]')?.getAttribute("content")?.trim() ?? "",
77
+ };
78
+ }
package/dist/index.js CHANGED
@@ -63,7 +63,7 @@ export function agentside(config) {
63
63
  return render(request, pathname);
64
64
  }
65
65
  const accept = request.headers.get("accept");
66
- const format = negotiateFormat(accept);
66
+ const format = negotiateFormat(accept, config.accept);
67
67
  const agent = detectAgent(request.headers.get("user-agent"));
68
68
  if (format === "markdown" || agent) {
69
69
  const response = await render(request, pathname, agent);
@@ -1,2 +1,2 @@
1
1
  import type { Format } from "./types.js";
2
- export declare function negotiateFormat(accept: string | null | undefined): Format | null;
2
+ export declare function negotiateFormat(accept: string | null | undefined, mode?: "q-value" | "contains"): Format | null;
package/dist/negotiate.js CHANGED
@@ -1,7 +1,9 @@
1
1
  const HTML_TYPES = new Set(["text/html", "application/xhtml+xml"]);
2
- export function negotiateFormat(accept) {
2
+ export function negotiateFormat(accept, mode = "q-value") {
3
3
  if (!accept)
4
4
  return "html";
5
+ if (mode === "contains" && accept.includes("text/markdown"))
6
+ return "markdown";
5
7
  if (!accept.includes("markdown") && !accept.includes("q=")) {
6
8
  return accept.includes("html") || accept.includes("*/*") || accept.includes("text/*") ? "html" : null;
7
9
  }
package/dist/types.d.ts CHANGED
@@ -50,6 +50,26 @@ export interface AgentsideConfig {
50
50
  readonly noindex?: boolean;
51
51
  readonly linkHeader?: boolean;
52
52
  readonly normalize?: boolean;
53
+ readonly accept?: "q-value" | "contains";
53
54
  readonly onServe?: (event: ServeEvent) => void;
54
55
  readonly onError?: (error: unknown, path: string) => void;
55
56
  }
57
+ export interface HtmlToMarkdownOptions {
58
+ readonly selector?: string;
59
+ readonly siteUrl?: string;
60
+ }
61
+ export interface PrerenderHtmlOptions extends HtmlToMarkdownOptions {
62
+ readonly dir: string;
63
+ readonly outDir?: string;
64
+ readonly exclude?: readonly string[];
65
+ readonly llms?: {
66
+ readonly title: string;
67
+ readonly description?: string;
68
+ } | false;
69
+ }
70
+ export interface PrerenderedPage {
71
+ readonly path: string;
72
+ readonly title: string;
73
+ readonly description: string;
74
+ readonly markdown: string;
75
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@usenotra/agentside",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "Serve markdown to AI agents and A/B test what they read. Built on Effect. By Notra.",
5
5
  "keywords": [
6
6
  "ai",
@@ -53,5 +53,9 @@
53
53
  "publishConfig": {
54
54
  "access": "public",
55
55
  "provenance": true
56
+ },
57
+ "dependencies": {
58
+ "node-html-markdown": "2.0.0",
59
+ "node-html-parser": "6.1.13"
56
60
  }
57
61
  }