livemark 0.26.0 → 0.27.0

package/source/plugins/vite-agents.ts

@@ -0,0 +1,135 @@
+import { existsSync, readFileSync } from "node:fs"
+import { join, resolve } from "node:path"
+import { pathToFileURL } from "node:url"
+import type { Plugin } from "vite"
+import { z } from "zod"
+import {
+  buildSkillFileName,
+  loadAgentSkills,
+  renderSkillsIndex,
+} from "../helpers/agent-skills.ts"
+import {
+  buildTwinFileName,
+  renderLlmsText,
+  renderMarkdownTwin,
+  renderSectionIndex,
+} from "../helpers/llms.ts"
+import { resolveIncludes } from "../helpers/resolve-includes.ts"
+import type { Config } from "../models/config.ts"
+
+export interface AgentsOptions {
+  config: Config
+}
+
+const GeneratedArticles = z.array(
+  z.object({
+    path: z.string(),
+    title: z.string(),
+    description: z.string().optional(),
+    order: z.number().optional(),
+    date: z.string().optional(),
+    file: z.string(),
+    content: z.string(),
+  }),
+)
+
+/** Emit what AI agents look for into the client build: a Markdown twin of
+ *  every article and section listing at `<path>index.md`, `llms.txt`, the Agent Skills index for
+ *  `config.skills`, and a `_headers` file whose `Link` points agents at them. */
+export function agents(opts: AgentsOptions): Plugin {
+  const publicDir = join(opts.config.root, ".livemark", "public")
+  let root = ""
+  return {
+    name: "livemark:agents",
+    applyToEnvironment(env) {
+      return env.name === "client"
+    },
+    configResolved(resolved) {
+      root = resolved.root
+    },
+    async generateBundle() {
+      // Vite copies the publicDir before writeBundle, so emitting a file the
+      // project ships under `.livemark/public/` would silently replace it.
+      const emit = (fileName: string, source: string | Buffer) => {
+        if (existsSync(join(publicDir, fileName))) return
+        this.emitFile({ type: "asset", fileName, source })
+      }
+
+      const articles = await loadGeneratedArticles(root)
+      for (const article of articles) {
+        const content = resolveIncludes(
+          article.content,
+          resolve(opts.config.root, article.file),
+        )
+        emit(buildTwinFileName(article.path), renderMarkdownTwin({ article, content }))
+      }
+
+      const llms = renderLlmsText({ config: opts.config, articles })
+      emit("llms.txt", llms)
+      const hasTwin = (path: string) => articles.some(a => a.path === path)
+      if (!hasTwin("/")) emit("index.md", llms)
+      // A rewrite to a missing twin is a 404, so every listing page needs one too.
+      for (const section of opts.config.sections ?? []) {
+        if (section.type === "custom" || section.prefix === "/") continue
+        if (hasTwin(section.prefix)) continue
+        emit(
+          buildTwinFileName(section.prefix),
+          renderSectionIndex({ config: opts.config, section, articles }),
+        )
+      }
+
+      const patterns = opts.config.skills ?? []
+      const skills = loadAgentSkills({
+        root: opts.config.root,
+        patterns: Array.isArray(patterns) ? patterns : [patterns],
+      })
+      for (const skill of skills) emit(buildSkillFileName(skill.name), skill.source)
+      if (skills.length) {
+        emit(
+          ".well-known/agent-skills/index.json",
+          renderSkillsIndex({ skills, base: opts.config.base }),
+        )
+      }
+
+      const headersFile = join(publicDir, "_headers")
+      this.emitFile({
+        type: "asset",
+        fileName: "_headers",
+        source: renderHeaders({
+          base: opts.config.base,
+          hasSkills: skills.length > 0,
+          existing: existsSync(headersFile) ? readFileSync(headersFile, "utf-8") : "",
+        }),
+      })
+    },
+  }
+}
+
+/** Render `_headers` (Cloudflare, Netlify): a `Link` on the home page naming
+ *  `llms.txt`, its Markdown twin and the skills index, after any rules the
+ *  project ships itself. */
+export function renderHeaders(props: {
+  base?: string
+  hasSkills: boolean
+  existing?: string
+}) {
+  const base = props.base ?? ""
+  const links = [
+    `<${base}/llms.txt>; rel="describedby"; type="text/plain"`,
+    `<${base}/index.md>; rel="alternate"; type="text/markdown"`,
+  ]
+  if (props.hasSkills) {
+    links.push(
+      `<${base}/.well-known/agent-skills/index.json>; rel="describedby"; type="application/json"`,
+    )
+  }
+  const rule = `${base}/\n  Link: ${links.join(", ")}\n`
+  const existing = props.existing?.trimEnd()
+  return existing ? `${existing}\n\n${rule}` : rule
+}
+
+async function loadGeneratedArticles(root: string) {
+  const file = join(root, ".content-collections", "generated", "allArticles.js")
+  const module: { default: unknown } = await import(pathToFileURL(file).href)
+  return GeneratedArticles.parse(module.default)
+}

package/source/plugins/vite-agents.unit.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from "vite-plus/test"
+import { renderHeaders } from "./vite-agents.ts"
+
+describe("renderHeaders", () => {
+  it("links the home page to llms.txt and its markdown twin", () => {
+    expect(renderHeaders({ hasSkills: false })).toBe(
+      '/\n  Link: </llms.txt>; rel="describedby"; type="text/plain", </index.md>; rel="alternate"; type="text/markdown"\n',
+    )
+  })
+
+  it("adds the skills index and mounts everything under base", () => {
+    const headers = renderHeaders({ base: "/repo", hasSkills: true })
+    expect(headers.startsWith("/repo/\n  Link: </repo/llms.txt>")).toBe(true)
+    expect(headers).toContain(
+      '</repo/.well-known/agent-skills/index.json>; rel="describedby"; type="application/json"',
+    )
+  })
+
+  it("keeps the project's own rules ahead of the generated one", () => {
+    const headers = renderHeaders({
+      hasSkills: false,
+      existing: "/*\n  X-Frame-Options: DENY\n\n",
+    })
+    expect(headers.startsWith("/*\n  X-Frame-Options: DENY\n\n/\n  Link: ")).toBe(true)
+  })
+})

package/source/plugins/vite-robots.ts

@@ -1,7 +1,7 @@
 import { existsSync } from "node:fs"
 import { join } from "node:path"
 import type { Plugin } from "vite"
-import type { Config } from "../models/config.ts"
+import type { Config, ContentSignals } from "../models/config.ts"
 
 export interface RobotsOptions {
   config: Config
@@ -35,8 +35,13 @@ export function robots(opts: RobotsOptions): Plugin {
  *  line is appended only when `site` is set, because the directive
  *  requires a fully-qualified URL; `base` is included so the URL matches
  *  where the build (and the `sitemap.xml` inside it) is actually mounted. */
-export function renderRobots(config: Pick<Config, "site" | "base">) {
-  const lines = ["User-agent: *", "Allow: /"]
+export function renderRobots(
+  config: Pick<Config, "site" | "base"> & { contentSignals?: ContentSignals },
+) {
+  const lines = ["User-agent: *"]
+  // Agents read the signal as part of a user-agent group, so it must sit inside one.
+  if (config.contentSignals) lines.push(renderContentSignal(config.contentSignals))
+  lines.push("Allow: /")
   if (config.site) {
     const host = config.site.replace(/\/+$/, "")
     const base = config.base ?? ""
@@ -44,3 +49,13 @@ export function renderRobots(config: Pick<Config, "site" | "base">) {
   }
   return `${lines.join("\n")}\n`
 }
+
+function renderContentSignal(signals: ContentSignals) {
+  const flag = (value: boolean) => (value ? "yes" : "no")
+  const pairs = [
+    `search=${flag(signals.search)}`,
+    `ai-input=${flag(signals.aiInput)}`,
+    `ai-train=${flag(signals.aiTrain)}`,
+  ]
+  return `Content-Signal: ${pairs.join(", ")}`
+}

package/source/plugins/vite-robots.unit.ts

@@ -27,4 +27,11 @@ describe("renderRobots", () => {
   it("ignores base when site is not set", () => {
     expect(renderRobots({ base: "/repo" })).toBe("User-agent: *\nAllow: /\n")
   })
+
+  it("puts the content signals inside the wildcard group", () => {
+    const contentSignals = { search: true, aiInput: true, aiTrain: false }
+    expect(renderRobots({ contentSignals })).toBe(
+      "User-agent: *\nContent-Signal: search=yes, ai-input=yes, ai-train=no\nAllow: /\n",
+    )
+  })
 })

package/source/vite.config.ts

@@ -6,6 +6,7 @@ import react from "@vitejs/plugin-react"
 import { defineConfig } from "vite"
 import svgr from "vite-plugin-svgr"
 import { loadConfig } from "./actions/config/load.ts"
+import { agents } from "./plugins/vite-agents.ts"
 import { livemark } from "./plugins/vite-livemark.ts"
 import { robots } from "./plugins/vite-robots.ts"
 
@@ -28,6 +29,7 @@ export default defineConfig({
   plugins: [
     livemark({ config }),
     robots({ config }),
+    agents({ config }),
     devtools(),
     tailwind(),
     contentCollections({ configPath: "content-collections.ts" }),