@anubhav-choubey/web-for-agents 0.1.0 → 0.3.0

package/dist/vite.d.ts

@@ -19,7 +19,7 @@ interface WebForAgentsOptions {
         disallow?: string[];
     };
     /** Generate sitemap.xml and the `Sitemap:` line in robots.txt. Default true. */
-    sitemap?: boolean;
+    sitemap?: boolean | SitemapOptions;
     /** Prerender routes after the build. `false` also turns off markdown, llms and the agent map. */
     prerender?: PrerenderOptions | false;
     /** Write a `.md` version of each page. `false` also turns off llms-full.txt. */
@@ -80,17 +80,130 @@ interface PrerenderOptions {
     isolation?: {
         codeGeneration?: boolean;
     };
+    /**
+     * Where each route's HTML is written. "directory": "/about" -> about/index.html.
+     * "flat": "/about" -> about.html ("/" stays index.html). Default "directory".
+     */
+    fileLayout?: FileLayout;
+    /**
+     * CSS selector that marks a route as rendered: the page is ready once it matches and no request
+     * is in flight, even if the DOM keeps changing (animations). Default: wait for the DOM to go quiet.
+     */
+    readySelector?: string;
+    /**
+     * What to do when a route is not ready by `timeoutMs`. "fail" stops the build; "snapshot" saves the
+     * page as it is at the deadline and logs a warning. Default "fail".
+     */
+    onTimeout?: "fail" | "snapshot";
+}
+type FileLayout = "directory" | "flat";
+/** What `markdown.transform` receives for each page. */
+interface MarkdownTransformPage {
+    /** Route path, e.g. "/about". */
+    route: string;
+    /** Absolute URL of the page. */
+    url: string;
+    title: string;
+    description?: string;
+    /** The generated markdown. */
+    markdown: string;
+    /** The page's full rendered HTML. */
+    html: string;
 }
+/** Returns the markdown to use instead, or undefined to keep the generated one. */
+type MarkdownTransform = (page: MarkdownTransformPage) => string | undefined | Promise<string | undefined>;
 interface MarkdownOptions {
     /** CSS selector for the page content converted to markdown. Default "main". */
     contentSelector?: string;
+    /** Rewrites each page's markdown (used for the .md file and llms-full.txt). */
+    transform?: MarkdownTransform;
 }
 interface LlmsOptions {
     /** llms.txt heading. Default: the title of "/". */
     title?: string;
     /** llms.txt summary. Default: the description of "/". */
     description?: string;
+    /** Plain text after the summary, in llms.txt and llms-full.txt. Blank lines separate paragraphs. */
+    intro?: string;
+    /** Sections written before "## Pages" in llms.txt, and in the llms-full.txt header. */
+    sections?: LlmsSectionOptions[];
+}
+interface LlmsSectionOptions {
+    /** The section heading. */
+    title: string;
+    /** Plain text under the heading. Blank lines separate paragraphs. */
+    text?: string;
+    /** Links listed under the text. */
+    links?: LlmsLinkOptions[];
+}
+interface LlmsLinkOptions {
+    title: string;
+    /** http(s) or mailto: URL. A relative URL ("/resume.pdf") is resolved against siteUrl. */
+    url: string;
+    description?: string;
+}
+/** A route's last modification date: "YYYY-MM-DD", a W3C datetime, a Date, or undefined for none. */
+type SitemapLastmod = (route: string) => string | Date | undefined | Promise<string | Date | undefined>;
+interface SitemapOptions {
+    /**
+     * `<lastmod>` for each URL. "file": the source HTML file's modification time with `generate`, the
+     * build date with the plugin. A function returns the date per route. Default "none".
+     */
+    lastmod?: "file" | "none" | SitemapLastmod;
+}
+
+interface GenerateOptions {
+    /** Build output directory holding the prerendered HTML, e.g. "dist". */
+    outDir: string;
+    /** Absolute http(s) URL of the deployed site. */
+    siteUrl: string;
+    /** robots.txt policy. `ai.training` and `ai.search` are required. */
+    robots: WebForAgentsOptions["robots"];
+    /** Routes to leave out. A string is an exact path, or a prefix when it ends in "/*". */
+    exclude?: Array<string | RegExp>;
+    /** Write sitemap.xml and the `Sitemap:` line in robots.txt. Default true. */
+    sitemap?: WebForAgentsOptions["sitemap"];
+    /** Write a `.md` version of each page. `false` also turns off llms-full.txt. */
+    markdown?: MarkdownOptions | false;
+    /** Write llms.txt and llms-full.txt. Default on. */
+    llms?: LlmsOptions | false;
+    /** Write agents.json from the actions in each page's markup. Default true. */
+    agentMap?: boolean | AgentMapOptions;
+    /** Add `<link rel="alternate" type="text/markdown">` to each page's head. Default true. */
+    linkMarkdown?: boolean;
+    /** Replace an existing robots.txt, sitemap.xml, llms.txt, llms-full.txt or agent map that differs. Default false. */
+    overwrite?: boolean;
+    /** Where warnings and the summary go. Default console. */
+    logger?: {
+        warn(message: string): void;
+        info(message: string): void;
+    };
+}
+interface GeneratedPage {
+    route: string;
+    /** HTML file, relative to outDir, with "/" separators. */
+    file: string;
+    /** False for noindex pages: they get a .md file but stay out of llms.txt, the sitemap and the agent map. */
+    listed: boolean;
 }
+interface GenerateResult {
+    outDir: string;
+    /** "flat" (about.html), "directory" (about/index.html), or "mixed". Only "/" found counts as "directory". */
+    fileLayout: "directory" | "flat" | "mixed";
+    /** Sorted by route. */
+    pages: GeneratedPage[];
+    /** Files written, relative to outDir, sorted. */
+    written: string[];
+    /** Existing files that were kept because their content differs (see `overwrite`), sorted. */
+    kept: string[];
+    warnings: string[];
+}
+/**
+ * Writes llms.txt, llms-full.txt, a `.md` copy of each page, robots.txt, sitemap.xml and agents.json
+ * for HTML you prerendered yourself, and links each page to its markdown. Reads every `.html`
+ * under outDir except 200.html, 404.html and assets/. Nothing is written if any step fails.
+ */
+declare function generateAgentFiles(options: GenerateOptions): Promise<GenerateResult>;
 
 /** Known AI crawler user agents, grouped by robots.ai policy. */
 declare const AI_BOTS: {
@@ -104,5 +217,5 @@ declare const AI_BOTS: {
  */
 declare function webForAgents(options: WebForAgentsOptions): Plugin[];
 
-export { AI_BOTS, webForAgents };
-export type { AgentMapOptions, LlmsOptions, MarkdownOptions, OverlayPluginOptions, PrerenderOptions, RendererName, Rule, WebForAgentsOptions };
+export { AI_BOTS, generateAgentFiles, webForAgents };
+export type { AgentMapOptions, FileLayout, GenerateOptions, GenerateResult, GeneratedPage, LlmsLinkOptions, LlmsOptions, LlmsSectionOptions, MarkdownOptions, MarkdownTransform, MarkdownTransformPage, OverlayPluginOptions, PrerenderOptions, RendererName, Rule, SitemapLastmod, SitemapOptions, WebForAgentsOptions };