@rimelight/seo 0.0.3 → 0.0.5

package/src/llms.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vite-plus/test"
+import { buildLlmsTxt, buildLlmsFullTxt } from "./llms.js"
+
+describe("seo-llms", () => {
+  it("builds standard /llms.txt with sections and pages", () => {
+    const output = buildLlmsTxt({
+      site: "https://rimelight.com",
+      title: "Rimelight Documentation",
+      description: "Docs for developers and AI agents.",
+      pages: [
+        {
+          title: "Introduction",
+          url: "https://rimelight.com/en/docs/overview/introduction",
+          markdownUrl: "https://rimelight.com/en/docs/overview/introduction.md",
+          description: "Getting started guide",
+          section: "Overview"
+        },
+        {
+          title: "Architecture",
+          url: "https://rimelight.com/en/docs/overview/architecture",
+          markdownUrl: "https://rimelight.com/en/docs/overview/architecture.md",
+          section: "Overview"
+        }
+      ]
+    })
+
+    expect(output).toContain("# Rimelight Documentation")
+    expect(output).toContain(
+      "Full corpus (all pages, one document): https://rimelight.com/llms-full.txt"
+    )
+    expect(output).toContain("### Overview")
+    expect(output).toContain(
+      "- [Introduction](https://rimelight.com/en/docs/overview/introduction.md) — Getting started guide"
+    )
+    expect(output).toContain(
+      "- [Architecture](https://rimelight.com/en/docs/overview/architecture.md)"
+    )
+  })
+
+  it("builds full corpus /llms-full.txt", () => {
+    const output = buildLlmsFullTxt({
+      site: "https://rimelight.com",
+      title: "Rimelight Docs",
+      pages: [
+        {
+          title: "Quickstart",
+          url: "https://rimelight.com/en/docs/quickstart",
+          description: "Step-by-step tutorial",
+          version: "v1",
+          markdown: "Run `vp dev` to start the development server."
+        }
+      ]
+    })
+
+    expect(output).toContain("# Rimelight Docs — Full Documentation Corpus")
+    expect(output).toContain("## Quickstart")
+    expect(output).toContain("URL: https://rimelight.com/en/docs/quickstart")
+    expect(output).toContain("Version: v1")
+    expect(output).toContain("Run `vp dev` to start the development server.")
+  })
+})

package/src/llms.ts

@@ -0,0 +1,124 @@
+export interface LlmsPage {
+  title: string
+  url: string
+  description?: string
+  markdownUrl?: string
+  section?: string
+}
+
+export interface LlmsTxtOptions {
+  site: string
+  title: string
+  description?: string
+  fullCorpusUrl?: string
+  pages: LlmsPage[]
+  sections?: Array<{ label: string; url: string }>
+}
+
+export interface LlmsFullTxtOptions {
+  site: string
+  title: string
+  description?: string
+  pages: Array<{
+    title: string
+    url: string
+    description?: string
+    markdown: string
+    version?: string
+  }>
+}
+
+/**
+ * Builds standard /llms.txt document format for AI discovery.
+ */
+export function buildLlmsTxt(options: LlmsTxtOptions): string {
+  const {
+    site,
+    title,
+    description,
+    fullCorpusUrl = `${site.replace(/\/$/, "")}/llms-full.txt`,
+    pages,
+    sections = []
+  } = options
+
+  const lines = [
+    `# ${title}`,
+    "",
+    description || "Documentation index for AI agents.",
+    "",
+    `Full corpus (all pages, one document): ${fullCorpusUrl}`,
+    ""
+  ]
+
+  if (sections.length > 0) {
+    lines.push("## Sections", "")
+    for (const sec of sections) {
+      lines.push(`- [${sec.label}](${sec.url})`)
+    }
+    lines.push("")
+  }
+
+  lines.push("## Pages", "")
+
+  // Group pages by section if present, or flat
+  const grouped = new Map<string, LlmsPage[]>()
+  for (const page of pages) {
+    const sec = page.section || "General"
+    if (!grouped.has(sec)) grouped.set(sec, [])
+    grouped.get(sec)!.push(page)
+  }
+
+  if (grouped.size === 1 && grouped.has("General")) {
+    for (const page of pages) {
+      const pageUrl = page.markdownUrl || page.url
+      const desc = page.description ? ` — ${page.description}` : ""
+      lines.push(`- [${page.title}](${pageUrl})${desc}`)
+    }
+  } else {
+    for (const [secName, secPages] of grouped.entries()) {
+      lines.push(`### ${secName}`, "")
+      for (const page of secPages) {
+        const pageUrl = page.markdownUrl || page.url
+        const desc = page.description ? ` — ${page.description}` : ""
+        lines.push(`- [${page.title}](${pageUrl})${desc}`)
+      }
+      lines.push("")
+    }
+  }
+
+  lines.push("")
+  return lines.join("\n")
+}
+
+/**
+ * Builds concatenated /llms-full.txt single corpus document for AI agents.
+ */
+export function buildLlmsFullTxt(options: LlmsFullTxtOptions): string {
+  const { title, description, pages } = options
+  const parts = [
+    `# ${title} — Full Documentation Corpus`,
+    "",
+    description || "Complete documentation corpus for AI agents.",
+    "",
+    "---",
+    ""
+  ]
+
+  for (const page of pages) {
+    const metaHeader = [
+      `## ${page.title}`,
+      "",
+      `URL: ${page.url}`,
+      ...(page.description ? [`Description: ${page.description}`] : []),
+      ...(page.version ? [`Version: ${page.version}`] : []),
+      "",
+      page.markdown.trim(),
+      "",
+      "---",
+      ""
+    ]
+    parts.push(...metaHeader)
+  }
+
+  return parts.join("\n")
+}

package/src/meta.test.ts

@@ -0,0 +1,134 @@
+import { describe, it, expect } from "vite-plus/test"
+import {
+  buildCanonicalUrl,
+  buildPageTitle,
+  buildSafeDescription,
+  buildRobotsMetaContent,
+  buildWebSiteSchema,
+  deepMerge
+} from "./meta.js"
+
+const BASE = "https://example.com"
+
+describe("buildCanonicalUrl", () => {
+  it("adds trailing slash to clean paths", () => {
+    expect(buildCanonicalUrl("/about", BASE)).toBe("https://example.com/about/")
+  })
+
+  it("strips trailing slash from query-param paths", () => {
+    expect(buildCanonicalUrl("/search?q=hello", BASE)).toBe("https://example.com/search?q=hello")
+  })
+
+  it("handles root path", () => {
+    expect(buildCanonicalUrl("/", BASE)).toBe("https://example.com/")
+  })
+
+  it("resolves relative paths against base", () => {
+    expect(buildCanonicalUrl(new URL("https://example.com/docs"), BASE)).toBe(
+      "https://example.com/docs/"
+    )
+  })
+})
+
+describe("buildPageTitle", () => {
+  const opts = { siteName: "My Site", titleTemplate: "%s | My Site" }
+
+  it("applies title template", () => {
+    expect(buildPageTitle({ ...opts, title: "About" })).toBe("About | My Site")
+  })
+
+  it("returns siteName when no title", () => {
+    expect(buildPageTitle(opts)).toBe("My Site")
+  })
+
+  it("returns 404 label for error pages", () => {
+    expect(buildPageTitle({ ...opts, is404: true })).toBe("404 - Not Found | My Site")
+  })
+
+  it("404 overrides title if both provided", () => {
+    expect(buildPageTitle({ ...opts, title: "Oops", is404: true })).toBe(
+      "404 - Not Found | My Site"
+    )
+  })
+
+  it("bypasses title template when absolute is true", () => {
+    expect(buildPageTitle({ ...opts, title: "Landing Page", absolute: true })).toBe("Landing Page")
+  })
+})
+
+describe("deepMerge", () => {
+  it("deep merges nested objects", () => {
+    const base = {
+      openGraph: { type: "website", siteName: "Base", locale: "en_US" },
+      twitter: { card: "summary", site: "@base" }
+    }
+    const override = {
+      openGraph: { siteName: "Override" },
+      twitter: { card: "summary_large_image" }
+    }
+    const result = deepMerge(base, override)
+    expect(result.openGraph.type).toBe("website")
+    expect(result.openGraph.siteName).toBe("Override")
+    expect(result.openGraph.locale).toBe("en_US")
+    expect(result.twitter.card).toBe("summary_large_image")
+    expect(result.twitter.site).toBe("@base")
+  })
+})
+
+describe("buildSafeDescription", () => {
+  it("returns description unchanged when within limit", () => {
+    expect(buildSafeDescription("Short desc", 160)).toBe("Short desc")
+  })
+
+  it("truncates and appends ellipsis when over limit", () => {
+    const long = "a".repeat(200)
+    const result = buildSafeDescription(long, 160)
+    expect(result).toHaveLength(160)
+    expect(result.endsWith("...")).toBe(true)
+  })
+
+  it("handles empty string", () => {
+    expect(buildSafeDescription("", 160)).toBe("")
+  })
+
+  it("truncates at exact boundary", () => {
+    const text = "a".repeat(160)
+    expect(buildSafeDescription(text, 160)).toBe(text)
+    expect(buildSafeDescription(text + "b", 160)).toHaveLength(160)
+  })
+})
+
+describe("buildRobotsMetaContent", () => {
+  it("returns indexable string by default", () => {
+    expect(buildRobotsMetaContent({})).toBe("index, follow, max-image-preview:large")
+  })
+
+  it("returns noindex for noindex=true", () => {
+    expect(buildRobotsMetaContent({ noindex: true })).toBe("noindex, nofollow")
+  })
+
+  it("returns noindex for is404=true", () => {
+    expect(buildRobotsMetaContent({ is404: true })).toBe("noindex, nofollow")
+  })
+
+  it("respects override", () => {
+    expect(buildRobotsMetaContent({ override: "noarchive" })).toBe("noarchive")
+  })
+})
+
+describe("buildWebSiteSchema", () => {
+  const config = { name: "My Site", url: "https://example.com", description: "A test site" }
+
+  it("builds schema.org WebSite object", () => {
+    const schema = buildWebSiteSchema(config)
+    expect(schema["@context"]).toBe("https://schema.org")
+    expect(schema["@type"]).toBe("WebSite")
+    expect(schema["name"]).toBe("My Site")
+    expect(schema["url"]).toBe("https://example.com")
+  })
+
+  it("prefers site param over config.url", () => {
+    const schema = buildWebSiteSchema(config, "https://override.com")
+    expect(schema["url"]).toBe("https://override.com")
+  })
+})

package/src/meta.ts

@@ -0,0 +1,100 @@
+import type { SiteConfig } from "./types.js"
+
+/**
+ * Build a canonical URL from a page URL and site base. - Strips trailing slash unless the path has
+ * query params (where trailing slash is removed too). - Returns undefined-safe: pass Astro.url and
+ * Astro.site directly.
+ */
+export function buildCanonicalUrl(url: string | URL, base: string | URL): string {
+  const resolved = new URL(url, base)
+  const path = resolved.toString()
+  // Keep query params as-is but strip trailing slash; for clean paths also ensure trailing slash
+  return path.includes("?") ? path.replace(/\/?$/, "") : path.replace(/\/?$/, "/")
+}
+
+/**
+ * Deep merge two metadata or configuration objects.
+ */
+export function deepMerge<T extends Record<string, any>>(
+  base: T,
+  override?: Record<string, any>
+): T {
+  if (!override) return { ...base }
+  const result: Record<string, any> = { ...base }
+  for (const key of Object.keys(override)) {
+    const baseVal = base[key]
+    const overrideVal = override[key]
+    if (
+      baseVal &&
+      overrideVal &&
+      typeof baseVal === "object" &&
+      typeof overrideVal === "object" &&
+      !Array.isArray(baseVal) &&
+      !Array.isArray(overrideVal)
+    ) {
+      result[key] = deepMerge(baseVal, overrideVal)
+    } else if (overrideVal !== undefined) {
+      result[key] = overrideVal
+    }
+  }
+  return result as T
+}
+
+/**
+ * Build a page title using a site's title template. Falls back to siteName for untitled pages; uses
+ * a 404 label for error pages. Supports absolute title bypass.
+ */
+export function buildPageTitle(opts: {
+  title?: string
+  siteName: string
+  titleTemplate: string
+  is404?: boolean
+  absolute?: boolean
+}): string {
+  const { title, siteName, titleTemplate, is404, absolute } = opts
+  if (is404) return `404 - Not Found | ${siteName}`
+  if (!title) return siteName
+  if (absolute) return title
+  return titleTemplate.replace("%s", title)
+}
+
+/**
+ * Truncate a description to a maximum character length, appending "…" if trimmed.
+ */
+export function buildSafeDescription(description: string, maxLength: number): string {
+  if (!description) return description
+  if (description.length <= maxLength) return description
+  return description.slice(0, maxLength - 3) + "..."
+}
+
+/**
+ * Build the value for the <meta name="robots"> tag. Returns the full directive string including
+ * max-image-preview for indexable pages.
+ */
+export function buildRobotsMetaContent(opts: {
+  noindex?: boolean
+  is404?: boolean
+  override?: string
+}): string {
+  if (opts.override) return opts.override
+  return !opts.noindex && !opts.is404
+    ? "index, follow, max-image-preview:large"
+    : "noindex, nofollow"
+}
+
+/**
+ * \n * Build a Schema.org WebSite JSON-LD object.\n * Pass the result to JSON.stringify and emit
+ * via <script type="application/ld+json">.\n
+ */
+export function buildWebSiteSchema(
+  config: Pick<SiteConfig, "name" | "url" | "description">,
+  site?: string
+): Record<string, unknown> {
+  return {
+    "@context": "https://schema.org",
+    "@type": "WebSite",
+    "name": config.name,
+    "url": site ?? config.url,
+    "description": config.description
+  }
+}

package/src/robots.test.ts

@@ -0,0 +1,75 @@
+import { describe, it, expect } from "vite-plus/test"
+import { buildRobotsTxt } from "./robots.js"
+import type { RobotsOptions } from "./types.js"
+
+describe("buildRobotsTxt", () => {
+  it("generates basic robots.txt with sitemap", () => {
+    const options: RobotsOptions = {
+      sitemapUrl: "https://example.com/sitemap.xml"
+    }
+
+    const robots = buildRobotsTxt(options)
+
+    expect(robots).toContain("User-agent: *")
+    expect(robots).toContain("Sitemap: https://example.com/sitemap.xml")
+  })
+
+  it("includes disallow rules", () => {
+    const options: RobotsOptions = {
+      sitemapUrl: "https://example.com/sitemap.xml",
+      disallow: ["/admin/", "/dashboard/"]
+    }
+
+    const robots = buildRobotsTxt(options)
+
+    expect(robots).toContain("Disallow: /admin/")
+    expect(robots).toContain("Disallow: /dashboard/")
+  })
+
+  it("includes allow rules", () => {
+    const options: RobotsOptions = {
+      sitemapUrl: "https://example.com/sitemap.xml",
+      allow: ["/api/public/"]
+    }
+
+    const robots = buildRobotsTxt(options)
+
+    expect(robots).toContain("Allow: /api/public/")
+  })
+
+  it("includes user-agent specific rules", () => {
+    const options: RobotsOptions = {
+      sitemapUrl: "https://example.com/sitemap.xml",
+      userAgentRules: [
+        {
+          userAgent: "GPTBot",
+          disallow: ["/admin/"]
+        }
+      ]
+    }
+
+    const robots = buildRobotsTxt(options)
+
+    expect(robots).toContain("User-agent: GPTBot")
+    expect(robots).toContain("Disallow: /admin/")
+  })
+
+  it("orders user-agent specific rules before wildcard", () => {
+    const options: RobotsOptions = {
+      sitemapUrl: "https://example.com/sitemap.xml",
+      userAgentRules: [
+        {
+          userAgent: "GPTBot",
+          disallow: ["/admin/"]
+        }
+      ]
+    }
+
+    const robots = buildRobotsTxt(options)
+
+    const gptBotIndex = robots.indexOf("User-agent: GPTBot")
+    const wildcardIndex = robots.indexOf("User-agent: *")
+
+    expect(gptBotIndex).toBeLessThan(wildcardIndex)
+  })
+})

package/src/robots.ts

@@ -0,0 +1,59 @@
+import type { RobotsOptions, UserAgentRule } from "./types.js"
+
+/**
+ * Build robots.txt content from options
+ */
+export function buildRobotsTxt(options: RobotsOptions): string {
+  const lines: string[] = []
+
+  // Add user-agent specific rules first
+  if (options.userAgentRules && options.userAgentRules.length > 0) {
+    for (const rule of options.userAgentRules) {
+      lines.push(buildUserAgentSection(rule))
+      lines.push("")
+    }
+  }
+
+  // Add wildcard user-agent (all bots)
+  lines.push("User-agent: *")
+
+  if (options.disallow && options.disallow.length > 0) {
+    for (const path of options.disallow) {
+      lines.push(`Disallow: ${path}`)
+    }
+  }
+
+  if (options.allow && options.allow.length > 0) {
+    for (const path of options.allow) {
+      lines.push(`Allow: ${path}`)
+    }
+  }
+
+  lines.push("")
+  lines.push(`Sitemap: ${options.sitemapUrl}`)
+
+  return lines.join("\n")
+}
+
+/**
+ * Build a user-agent specific section
+ */
+function buildUserAgentSection(rule: UserAgentRule): string {
+  const lines: string[] = []
+
+  lines.push(`User-agent: ${rule.userAgent}`)
+
+  if (rule.disallow && rule.disallow.length > 0) {
+    for (const path of rule.disallow) {
+      lines.push(`Disallow: ${path}`)
+    }
+  }
+
+  if (rule.allow && rule.allow.length > 0) {
+    for (const path of rule.allow) {
+      lines.push(`Allow: ${path}`)
+    }
+  }
+
+  return lines.join("\n")
+}

package/src/routes/llms.txt.ts

@@ -0,0 +1,22 @@
+import type { APIRoute } from "astro"
+import { buildLlmsTxt } from "../llms.js"
+import seoConfig from "virtual:rimelight-seo-config"
+
+export const GET: APIRoute = ({ site }) => {
+  const llmsOpts = typeof seoConfig?.llms === "object" ? seoConfig.llms : {}
+  const siteStr = site ? site.toString().replace(/\/$/, "") : ""
+  const hostname = site ? new URL(site).hostname : "Site"
+  const content = buildLlmsTxt({
+    site: siteStr,
+    title: hostname,
+    pages: [],
+    ...llmsOpts
+  })
+
+  return new Response(content, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/routes/robots.txt.ts

@@ -0,0 +1,20 @@
+import type { APIRoute } from "astro"
+import { buildRobotsTxt } from "../robots.js"
+import seoConfig from "virtual:rimelight-seo-config"
+
+export const GET: APIRoute = ({ site }) => {
+  const sitemapURL = site ? new URL("sitemap.xml", site).toString() : ""
+  const robotsOpts = typeof seoConfig?.robots === "object" ? seoConfig.robots : {}
+  const robots = buildRobotsTxt({
+    sitemapUrl: sitemapURL,
+    allow: ["/"],
+    ...robotsOpts
+  })
+
+  return new Response(robots, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/routes/sitemap.xml.ts

@@ -0,0 +1,16 @@
+import type { APIRoute } from "astro"
+import { buildSitemapXml } from "../sitemap.js"
+import seoConfig from "virtual:rimelight-seo-config"
+
+export const GET: APIRoute = () => {
+  const sitemapOpts = typeof seoConfig?.sitemap === "object" ? seoConfig.sitemap : {}
+  const urls = sitemapOpts.urls || []
+  const xml = buildSitemapXml(urls)
+
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/schema.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vite-plus/test"
+import { buildArticleSchema, buildBreadcrumbSchema } from "./schema.js"
+
+describe("seo-schema", () => {
+  it("builds TechArticle JSON-LD schema", () => {
+    const schema = buildArticleSchema({
+      type: "TechArticle",
+      title: "Building on Cloudflare Workers",
+      description: "A complete guide to edge rendering",
+      url: "https://rimelight.com/blog/cloudflare-workers",
+      datePublished: new Date("2026-08-01T12:00:00Z"),
+      authorName: "Idan",
+      image: "https://rimelight.com/og/article.png"
+    })
+
+    expect(schema["@context"]).toBe("https://schema.org")
+    expect(schema["@type"]).toBe("TechArticle")
+    expect(schema["headline"]).toBe("Building on Cloudflare Workers")
+    expect(schema["author"]).toEqual({ "@type": "Person", "name": "Idan" })
+    expect(schema["datePublished"]).toBe("2026-08-01T12:00:00.000Z")
+  })
+
+  it("builds BreadcrumbList JSON-LD schema", () => {
+    const schema = buildBreadcrumbSchema([
+      { name: "Home", url: "https://rimelight.com" },
+      { name: "Docs", url: "https://rimelight.com/en/docs" },
+      { name: "Installation", url: "https://rimelight.com/en/docs/installation" }
+    ])
+
+    expect(schema["@type"]).toBe("BreadcrumbList")
+    expect(schema["itemListElement"]).toHaveLength(3)
+    expect(schema["itemListElement"][0]).toEqual({
+      "@type": "ListItem",
+      "position": 1,
+      "name": "Home",
+      "item": "https://rimelight.com"
+    })
+    expect(schema["itemListElement"][2]).toEqual({
+      "@type": "ListItem",
+      "position": 3,
+      "name": "Installation",
+      "item": "https://rimelight.com/en/docs/installation"
+    })
+  })
+})