meowpress 0.3.0-alpha.1 → 0.3.0-alpha.2

package/README.md

@@ -50,6 +50,14 @@ export default meowSite({
 
 - `pages` is a provider of complete HTML documents. A path that resolves above the site root is
   refused rather than clamped.
+- A page's optional `markdown` is the same page for agents. It is published next to the route
+  (`components/button.md` for `/components/button/`, `index.md` for the root), and `llms.txt`,
+  `llm.txt` and `agents.md` link it instead of the HTML. `llms.txt` follows the
+  [llms.txt](https://llmstxt.org/) shape: the site summary, then one section of links per page
+  `group` with each page's `description`. `llms-full.txt` holds every Markdown copy in one file.
+- `agentGuide` is Markdown that opens `llms.txt` and `llms-full.txt` after the summary: what an
+  agent must know before following any link, such as how to install and set up the documented
+  packages.
 - `mdxComponents` adds named components to the MDX content pages; the built-in `Callout` stays
   available.
 - `themeStylesheet` names the theme stylesheet the build writes, so a consumer page can link it. It

package/dist/src/build.d.ts

@@ -3,6 +3,7 @@ import { type SiteFont } from "./fonts.js";
 import { type SiteIdentity } from "./identity.js";
 import { type ContentSourceInput, type MdxComponent } from "./content.js";
 import { type SitePageProvider } from "./pages.js";
+export { sitePageMarkdownFile } from "./pages.js";
 export type { SitePage, SitePageContext, SitePageProvider } from "./pages.js";
 export type { ContentItem } from "./content.js";
 export type { SiteFont } from "./fonts.js";
@@ -22,6 +23,11 @@ export interface SiteConfig {
     site?: Partial<SiteIdentity>;
     /** Consumer-supplied pages; the site writes and indexes them next to its own. */
     pages?: SitePageProvider;
+    /**
+     * Markdown that opens `llms.txt` and `llms-full.txt` after the site summary: what an agent must
+     * know before following any link, such as how to install and set up what the site documents.
+     */
+    agentGuide?: string;
     /** Extra components available to MDX content pages. */
     mdxComponents?: Record<string, MdxComponent>;
     /** Stylesheet file name for the theme; defaults to "blog-theme.css". */
@@ -40,6 +46,7 @@ export interface ResolvedSiteConfig {
     projectRootDirectory?: string;
     site: SiteIdentity;
     pages?: SitePageProvider;
+    agentGuide?: string;
     mdxComponents: Record<string, MdxComponent>;
     themeStylesheet: string;
     fonts: readonly SiteFont[];

package/dist/src/build.js

@@ -5,9 +5,10 @@ import { DEFAULT_SITE } from "./identity.js";
 import { mkdir, readFile, rm, writeFile, cp } from "node:fs/promises";
 import { isAbsolute, join, relative, resolve } from "node:path";
 import { collectContentAssets, isDraftAsset, loadSection, normalizeContentSources, } from "./content.js";
-import { aboutPage, agentMarkdown, DEFAULT_THEME_STYLESHEET, homePage, listPage, llmText, postPage, skillPage, skillsIndexPage, thoughtPage, } from "./site.js";
+import { aboutPage, agentMarkdown, DEFAULT_THEME_STYLESHEET, homePage, listPage, llmFullText, llmText, postPage, skillPage, skillsIndexPage, thoughtPage, } from "./site.js";
 import { loadSkills } from "./skills.js";
-import { sitePageFile } from "./pages.js";
+import { sitePageFile, sitePageMarkdownFile } from "./pages.js";
+export { sitePageMarkdownFile } from "./pages.js";
 /** Define a site config in the app package without taking on the build pipeline. */
 export function meowSite(config) {
     return {
@@ -20,6 +21,7 @@ export function meowSite(config) {
             : undefined,
         site: { ...DEFAULT_SITE, ...config.site },
         pages: config.pages,
+        agentGuide: config.agentGuide,
         mdxComponents: config.mdxComponents ?? {},
         themeStylesheet: config.themeStylesheet ?? DEFAULT_THEME_STYLESHEET,
         fonts: config.fonts ?? (config.theme === undefined || config.theme === blogTheme ? blogFonts : []),
@@ -71,8 +73,8 @@ async function buildStaticHtmlFromEntrypoints(entrypoints, generatedDirectory, o
             await writeFile(output.path, removeLegacyIsFallbacks(await readFile(output.path, "utf8")));
 }
 /** Consumer pages are written inside the generated directory and never above it. */
-function consumerPageFile(generatedDirectory, path) {
-    const file = sitePageFile(path);
+function consumerPageFile(generatedDirectory, path, fileFor = sitePageFile) {
+    const file = fileFor(path);
     const target = resolve(generatedDirectory, file);
     const inside = relative(generatedDirectory, target);
     if (!inside || inside.startsWith("..") || isAbsolute(inside)) {
@@ -87,6 +89,18 @@ async function writeConsumerPages(generatedDirectory, pages) {
     }
     return entrypoints;
 }
+/** Files the build writes itself; a page's Markdown copy may not take their place. */
+const RESERVED_MARKDOWN = new Set(["agents.md"]);
+/** Each page's Markdown copy, written straight into the output next to its route. */
+async function writeConsumerMarkdown(outputDirectory, pages) {
+    for (const page of pages) {
+        if (page.markdown === undefined)
+            continue;
+        if (RESERVED_MARKDOWN.has(sitePageMarkdownFile(page.path)))
+            throw new Error(`Site page Markdown would replace the agent index: ${page.path}`);
+        await writePage(outputDirectory, consumerPageFile(outputDirectory, page.path, sitePageMarkdownFile), page.markdown);
+    }
+}
 async function writeContentPages(generatedDirectory, posts, thoughts, theme, site, stylesheet) {
     const entrypoints = [];
     for (const post of posts) {
@@ -148,9 +162,15 @@ export async function buildSite(input) {
     await stageAssets(generatedDirectory, config.content);
     await buildStaticHtmlFromEntrypoints(entrypoints, generatedDirectory, config.outputDirectory);
     await copyPostAssets(config.outputDirectory, config.content);
+    if (pages)
+        await writeConsumerMarkdown(config.outputDirectory, pages);
+    const full = pages ? llmFullText(config.site, pages, config.agentGuide) : undefined;
     await Promise.all([
-        writeFile(join(config.outputDirectory, "llm.txt"), llmText(posts, thoughts, config.site, pages), "utf8"),
-        writeFile(join(config.outputDirectory, "llms.txt"), llmText(posts, thoughts, config.site, pages), "utf8"),
+        writeFile(join(config.outputDirectory, "llm.txt"), llmText(posts, thoughts, config.site, pages, config.agentGuide), "utf8"),
+        writeFile(join(config.outputDirectory, "llms.txt"), llmText(posts, thoughts, config.site, pages, config.agentGuide), "utf8"),
+        ...(full === undefined
+            ? []
+            : [writeFile(join(config.outputDirectory, "llms-full.txt"), full, "utf8")]),
         writeFile(join(config.outputDirectory, "agents.md"), agentMarkdown(posts, thoughts, config.site, pages), "utf8"),
     ]);
     await rm(generatedDirectory, { recursive: true, force: true });

package/dist/src/pages.d.ts

@@ -8,6 +8,11 @@ export interface SitePage {
     title: string;
     /** A complete HTML document for the route. */
     html: string;
+    /**
+     * The same page as Markdown for agents. When present it is published next to the route (see
+     * `sitePageMarkdownFile`), and the agent-facing indexes link it instead of the HTML document.
+     */
+    markdown?: string;
     description?: string;
     /** Grouping used by the site index and the agent-facing indexes. */
     group?: string;
@@ -26,3 +31,10 @@ export type SitePageProvider = (context: SitePageContext) => readonly SitePage[]
 export declare function sitePageUrl(path: string): string;
 /** The output file a page is written to, relative to the site root. */
 export declare function sitePageFile(path: string): string;
+/**
+ * The Markdown copy of a page, relative to the site root: "index.md" for the root and
+ * "components/button.md" for /components/button/, the convention llms.txt readers expect.
+ */
+export declare function sitePageMarkdownFile(path: string): string;
+/** Where an agent reads a page: its Markdown copy when it has one, its HTML route otherwise. */
+export declare function sitePageAgentUrl(page: SitePage): string;

package/dist/src/pages.js

@@ -8,3 +8,17 @@ export function sitePageFile(path) {
     const normalized = path.replace(/^\/+|\/+$/g, "");
     return normalized ? `${normalized}/index.html` : "index.html";
 }
+/**
+ * The Markdown copy of a page, relative to the site root: "index.md" for the root and
+ * "components/button.md" for /components/button/, the convention llms.txt readers expect.
+ */
+export function sitePageMarkdownFile(path) {
+    const normalized = path.replace(/^\/+|\/+$/g, "");
+    return normalized ? `${normalized}.md` : "index.md";
+}
+/** Where an agent reads a page: its Markdown copy when it has one, its HTML route otherwise. */
+export function sitePageAgentUrl(page) {
+    return page.markdown === undefined
+        ? sitePageUrl(page.path)
+        : `/${sitePageMarkdownFile(page.path)}`;
+}

package/dist/src/site.d.ts

@@ -13,4 +13,9 @@ export declare function aboutPage(theme?: ThemeDefinition, site?: SiteIdentity,
 export declare function skillsIndexPage(skills: Skill[], theme?: ThemeDefinition, site?: SiteIdentity, stylesheet?: string): string;
 export declare function skillPage(skill: Skill, theme?: ThemeDefinition, site?: SiteIdentity, stylesheet?: string): string;
 export declare function agentMarkdown(posts: ContentItem[], thoughts: ContentItem[], site?: SiteIdentity, pages?: readonly SitePage[]): string;
-export declare function llmText(posts: ContentItem[], thoughts: ContentItem[], site?: SiteIdentity, pages?: readonly SitePage[]): string;
+export declare function llmText(posts: ContentItem[], thoughts: ContentItem[], site?: SiteIdentity, pages?: readonly SitePage[], agentGuide?: string): string;
+/**
+ * Every page's Markdown in one file, after the same summary and guide as llms.txt. Pages without
+ * a Markdown copy are left out; llms.txt still links their HTML.
+ */
+export declare function llmFullText(site: SiteIdentity, pages: readonly SitePage[], agentGuide?: string): string | undefined;

package/dist/src/site.js

@@ -4,7 +4,7 @@ import { Fragment, h } from "meowjsx/runtime";
 import { renderToHtml } from "meowjsx/web";
 import { withoutLeadingHeading } from "./content.js";
 import { DEFAULT_SITE } from "./identity.js";
-import { sitePageUrl } from "./pages.js";
+import { sitePageAgentUrl } from "./pages.js";
 import { layoutSkillGraph } from "./skill-graph.js";
 /** The name of the theme stylesheet the package writes by default. */
 export const DEFAULT_THEME_STYLESHEET = "blog-theme.css";
@@ -675,7 +675,7 @@ export function skillPage(skill, theme = blogTheme, site = DEFAULT_SITE, stylesh
 }
 export function agentMarkdown(posts, thoughts, site = DEFAULT_SITE, pages) {
     const siteMap = pages
-        ? pages.map((page) => `- [${page.title}](${sitePageUrl(page.path)})`)
+        ? pages.map((page) => `- [${page.title}](${sitePageAgentUrl(page)})`)
         : [
             "- [Home](/)",
             "- [Posts](/posts/)",
@@ -720,38 +720,94 @@ export function agentMarkdown(posts, thoughts, site = DEFAULT_SITE, pages) {
     ];
     return lines.join("\n");
 }
-export function llmText(posts, thoughts, site = DEFAULT_SITE, pages) {
-    const routes = pages
-        ? pages.map((page) => `- ${page.title}: ${sitePageUrl(page.path)}`)
-        : [
-            "- Home: /",
-            "- Posts: /posts/",
-            "- Thoughts: /thoughts/",
-            "- Skills: /skills/",
-            "- About: /about/",
-        ];
+export function llmText(posts, thoughts, site = DEFAULT_SITE, pages, agentGuide) {
+    // A consumer that owns the site map owns the routes of its content too, so its index follows
+    // the llms.txt shape: the summary, the guide, then one section of links per page group.
+    if (pages)
+        return consumerLlmText(site, pages, agentGuide);
     return [
         `# ${site.name}`,
         `> ${site.description}`,
         "",
         "## Pages",
-        ...routes,
+        "- Home: /",
+        "- Posts: /posts/",
+        "- Thoughts: /thoughts/",
+        "- Skills: /skills/",
+        "- About: /about/",
         "- Agent Markdown: /agents.md",
         "",
-        // A consumer that owns the site map owns the routes of its content too;
-        // listing the content under routes the site does not publish would be wrong.
-        ...(pages
-            ? []
-            : [
-                "## Posts",
-                ...posts.map((post) => `- ${post.title}: ${itemUrl(post)}`),
-                "",
-                "## Thoughts",
-                ...thoughts.map((thought) => `- ${thought.title}: ${itemUrl(thought)}`),
-                "",
-            ]),
+        "## Posts",
+        ...posts.map((post) => `- ${post.title}: ${itemUrl(post)}`),
+        "",
+        "## Thoughts",
+        ...thoughts.map((thought) => `- ${thought.title}: ${itemUrl(thought)}`),
+        "",
         "## Rules",
         "- Only the public build output is considered presentable content.",
         "- Drafts whose names start with _ under posts are not published or rendered.",
     ].join("\n");
 }
+/** The pages by group, in the order each group first appears. */
+function pageGroups(pages) {
+    const groups = new Map();
+    for (const page of pages) {
+        const group = page.group ?? "Pages";
+        const members = groups.get(group);
+        if (members)
+            members.push(page);
+        else
+            groups.set(group, [page]);
+    }
+    return [...groups];
+}
+/** One llms.txt link: the page's Markdown when it has one, and its description after a colon. */
+function llmLink(page) {
+    const description = page.description?.replace(/\s+/g, " ").trim();
+    return `- [${page.title}](${sitePageAgentUrl(page)})${description ? `: ${description}` : ""}`;
+}
+function consumerLlmText(site, pages, agentGuide) {
+    const full = pages.some((page) => page.markdown !== undefined);
+    return [
+        `# ${site.name}`,
+        "",
+        `> ${site.description}`,
+        "",
+        ...(agentGuide ? [agentGuide.trim(), ""] : []),
+        ...pageGroups(pages).flatMap(([group, members]) => [
+            `## ${group}`,
+            "",
+            ...members.map(llmLink),
+            "",
+        ]),
+        "## Optional",
+        "",
+        ...(full
+            ? [
+                "- [Full text](/llms-full.txt): every page linked here as Markdown, in one file; large, so fetch single pages first",
+            ]
+            : []),
+        "- [Agent Markdown](/agents.md): the site map",
+        "",
+    ].join("\n");
+}
+/**
+ * Every page's Markdown in one file, after the same summary and guide as llms.txt. Pages without
+ * a Markdown copy are left out; llms.txt still links their HTML.
+ */
+export function llmFullText(site, pages, agentGuide) {
+    const documents = pages.flatMap((page) => page.markdown === undefined
+        ? []
+        : [`<!-- Source: ${sitePageAgentUrl(page)} -->\n\n${page.markdown.trim()}`]);
+    if (!documents.length)
+        return undefined;
+    return [
+        `# ${site.name}`,
+        "",
+        `> ${site.description}`,
+        "",
+        ...(agentGuide ? [agentGuide.trim(), ""] : []),
+        documents.join("\n\n---\n\n"),
+        "",
+    ].join("\n");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "meowpress",
-  "version": "0.3.0-alpha.1",
+  "version": "0.3.0-alpha.2",
   "license": "MIT",
   "type": "module",
   "scripts": {
@@ -41,7 +41,7 @@
     "meowjsx": "^0.3.0-alpha.0"
   },
   "devDependencies": {
-    "meowjsx": "0.3.0-alpha.0",
-    "meowcss": "0.3.0-alpha.0"
+    "meowjsx": "0.3.0-alpha.1",
+    "meowcss": "0.3.0-alpha.2"
   }
 }