@rimelight/seo 0.0.5 → 0.0.6

package/src/routes/llms-full.txt.ts

@@ -0,0 +1,77 @@
+import type { APIRoute } from "astro"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import * as dbMod from "virtual:rimelight-seo/db"
+import * as corpusMod from "virtual:rimelight-seo/corpus"
+
+export const prerender = false
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const siteUrl = site ? site.toString() : (siteConfig.url || `${url.protocol}//${url.host}`)
+  const title = siteConfig.name
+    ? `${siteConfig.name} Full Documentation Corpus`
+    : "Full Documentation Corpus"
+  const description = "Complete single-corpus documentation for AI agents and LLM ingest."
+
+  // 1. Try corpusPages if available
+  if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
+    const sections = new Map<string, string[]>()
+    for (const p of corpusMod.corpusPages) {
+      const key = (p.section || "GENERAL").toUpperCase()
+      if (!sections.has(key)) sections.set(key, [])
+      const slug = p.slug === "index" ? "" : p.slug
+      const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
+      sections.get(key)!.push(`### ${p.title}\nURL: ${pageUrl}\n${p.description || ""}`)
+    }
+
+    const body = [
+      `# ${title}`,
+      "",
+      description,
+      "",
+      ...Array.from(sections.entries()).flatMap(([section, pages]) => [
+        `## ${section}`,
+        "",
+        ...pages,
+        ""
+      ])
+    ].join("\n")
+
+    return new Response(body, {
+      headers: {
+        "Content-Type": "text/plain; charset=utf-8",
+        "Cache-Control": "public, max-age=3600, s-maxage=86400"
+      }
+    })
+  }
+
+  // 2. Try CMS renderCorpusMarkdown if db exists
+  if (dbMod.db) {
+    try {
+      const { renderCorpusMarkdown } = await import("@rimelight/cms")
+      const body = await renderCorpusMarkdown(dbMod.db, {
+        type: "doc",
+        locale: "en",
+        siteUrl,
+        title,
+        description
+      })
+
+      return new Response(body, {
+        headers: {
+          "Content-Type": "text/plain; charset=utf-8",
+          "Cache-Control": "public, max-age=3600, s-maxage=86400"
+        }
+      })
+    } catch {
+      // Fallback
+    }
+  }
+
+  return new Response(`# ${title}\n\n${description}\n`, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, s-maxage=86400"
+    }
+  })
+}

package/src/routes/llms.txt.ts

@@ -1,22 +1,90 @@
-import type { APIRoute } from "astro"
-import { buildLlmsTxt } from "../llms.js"
-import seoConfig from "virtual:rimelight-seo-config"
-
-export const GET: APIRoute = ({ site }) => {
-  const llmsOpts = typeof seoConfig?.llms === "object" ? seoConfig.llms : {}
-  const siteStr = site ? site.toString().replace(/\/$/, "") : ""
-  const hostname = site ? new URL(site).hostname : "Site"
-  const content = buildLlmsTxt({
-    site: siteStr,
-    title: hostname,
-    pages: [],
-    ...llmsOpts
-  })
-
-  return new Response(content, {
-    headers: {
-      "Content-Type": "text/plain; charset=utf-8",
-      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
-    }
-  })
-}
+import type { APIRoute } from "astro"
+import { buildLlmsTxt, type LlmsPage } from "../llms.js"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import * as dbMod from "virtual:rimelight-seo/db"
+import * as corpusMod from "virtual:rimelight-seo/corpus"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const prerender = false
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const siteUrl = site ? site.toString() : (siteConfig.url || `${url.protocol}//${url.host}`)
+  const llmsOpts = typeof seoOptions?.llms === "object" ? seoOptions.llms : {}
+
+  let pages: LlmsPage[] = []
+
+  if (Array.isArray(llmsOpts.pages) && llmsOpts.pages.length > 0) {
+    pages = llmsOpts.pages
+  } else if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
+    pages = corpusMod.corpusPages.map((p: any) => {
+      const slug = p.slug === "index" ? "" : p.slug
+      const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
+      return {
+        title: p.title,
+        description: p.description,
+        url: pageUrl,
+        section: p.section ? p.section.toUpperCase() : "GENERAL"
+      }
+    })
+  } else if (dbMod.db) {
+    try {
+      const { pages: pagesTable } = await import("#db/schema").catch(() => ({ pages: null }))
+      const { and, isNotNull, isNull } = await import("drizzle-orm")
+      if (pagesTable) {
+        const docRows = await dbMod.db
+          .select()
+          .from(pagesTable)
+          .where(and(isNull(pagesTable.deletedAt), isNotNull(pagesTable.publishedVersionId)))
+          .catch(() => [])
+
+        const resolveLocalized = (val: unknown): string => {
+          if (typeof val === "string") return val
+          if (typeof val === "object" && val !== null) {
+            const rec = val as Record<string, string>
+            return rec["en"] || Object.values(rec)[0] || ""
+          }
+          return typeof val === "number" ? String(val) : ""
+        }
+
+        pages = docRows.map((p: any) => {
+          const title = resolveLocalized(p.title) || p.slug
+          const description = resolveLocalized(p.description)
+          const slug = p.slug === "index" ? "" : p.slug
+          const pathPrefix = p.type === "doc" ? "docs" : p.type || "docs"
+          const pageUrl = `${siteUrl.replace(/\/$/, "")}/en/${pathPrefix}/${slug}`.replace(/\/+$/, "")
+          const markdownUrl = `${pageUrl}.md`
+
+          return {
+            title,
+            description,
+            url: pageUrl,
+            markdownUrl,
+            section: p.type ? p.type.toUpperCase() : "GENERAL"
+          }
+        })
+      }
+    } catch {
+      // Fallback
+    }
+  }
+
+  const title = llmsOpts.title || siteConfig.name || "Documentation"
+  const description =
+    llmsOpts.description || siteConfig.description || "Documentation index for AI agents."
+
+  const body = buildLlmsTxt({
+    site: siteUrl,
+    title,
+    description,
+    pages,
+    ...llmsOpts
+  })
+
+  return new Response(body, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, s-maxage=86400"
+    }
+  })
+}

package/src/routes/robots.txt.ts

@@ -1,20 +1,63 @@
-import type { APIRoute } from "astro"
-import { buildRobotsTxt } from "../robots.js"
-import seoConfig from "virtual:rimelight-seo-config"
-
-export const GET: APIRoute = ({ site }) => {
-  const sitemapURL = site ? new URL("sitemap.xml", site).toString() : ""
-  const robotsOpts = typeof seoConfig?.robots === "object" ? seoConfig.robots : {}
-  const robots = buildRobotsTxt({
-    sitemapUrl: sitemapURL,
-    allow: ["/"],
-    ...robotsOpts
-  })
-
-  return new Response(robots, {
-    headers: {
-      "Content-Type": "text/plain; charset=utf-8",
-      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
-    }
-  })
-}
+import type { APIRoute } from "astro"
+import { buildRobotsTxt } from "../robots.js"
+import * as seoConfig from "virtual:rimelight-seo/config"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const GET: APIRoute = ({ site, url }) => {
+  const siteUrl = site ? site.toString() : (siteConfigMod.siteConfig?.url || url.origin)
+  const sitemapURL = new URL("sitemap.xml", siteUrl).toString()
+
+  const defaultPrivatePrefixes = [
+    "/dashboard",
+    "/admin",
+    "/cms",
+    "/internal",
+    "/api",
+    "/dev",
+    "/og",
+    "/open-graph",
+    "/auth"
+  ]
+
+  const customPrivatePrefixes = Array.isArray(seoConfig.PRIVATE_PATH_PREFIXES)
+    ? seoConfig.PRIVATE_PATH_PREFIXES
+    : []
+
+  const privatePrefixes = Array.from(new Set([...defaultPrivatePrefixes, ...customPrivatePrefixes]))
+
+  const disallowPatterns = [
+    ...privatePrefixes,
+    ...privatePrefixes.map((p) => `/*${p.startsWith("/") ? p : `/${p}`}/`)
+  ]
+
+  const robotsOpts = typeof seoOptions?.robots === "object" ? seoOptions.robots : {}
+
+  const defaultAgentRules = [
+    {
+      userAgent: "GPTBot",
+      disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
+      allow: ["/"]
+    },
+    {
+      userAgent: "ClaudeBot",
+      disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
+      allow: ["/"]
+    }
+  ]
+
+  const robots = buildRobotsTxt({
+    sitemapUrl: sitemapURL,
+    disallow: disallowPatterns,
+    allow: ["/"],
+    userAgentRules: defaultAgentRules,
+    ...robotsOpts
+  })
+
+  return new Response(robots, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/routes/rss.xml.ts

@@ -0,0 +1,87 @@
+import type { APIRoute } from "astro"
+import { buildRssXml } from "../rss.js"
+import * as seoConfig from "virtual:rimelight-seo/config"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import * as dbMod from "virtual:rimelight-seo/db"
+import seoOptions from "virtual:rimelight-seo/options"
+import type { RssItem } from "../types.js"
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const rssOpts = typeof seoOptions?.rss === "object" ? seoOptions.rss : {}
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const siteUrl = site ? site.toString() : (rssOpts.site || siteConfig.url || url.origin)
+
+  let items: RssItem[] = []
+
+  if (Array.isArray(rssOpts.items)) {
+    items = rssOpts.items
+  } else if (typeof (seoConfig as any).rssEntries === "function") {
+    items = await (seoConfig as any).rssEntries()
+  } else if (dbMod.db) {
+    try {
+      const { pages } = await import("#db/schema").catch(() => ({ pages: null }))
+      const { and, isNull, sql } = await import("drizzle-orm")
+      if (pages) {
+        const blogPages = await dbMod.db
+          .select()
+          .from(pages)
+          .where(and(sql`${pages.type} = 'blog'`, isNull(pages.deletedAt)))
+          .catch(() => [])
+
+        items = blogPages.map((p: any) => {
+          const title =
+            typeof p.title === "string"
+              ? p.title
+              : p.title?.en || (p.title && Object.values(p.title)[0]) || p.slug
+          const description =
+            typeof p.description === "string"
+              ? p.description
+              : p.description?.en || (p.description && Object.values(p.description)[0]) || ""
+          return {
+            title,
+            description,
+            link: `/blog/${p.slug}`,
+            pubDate: p.postedAt || p.createdAt || new Date()
+          }
+        })
+      }
+    } catch {
+      // Fallback
+    }
+  }
+
+  if (items.length === 0) {
+    try {
+      const { getCollection } = await import("astro:content")
+      const blog = await getCollection("blog")
+      if (Array.isArray(blog)) {
+        items = blog.map((post: any) => ({
+          title: post.data?.title || post.id,
+          pubDate: post.data?.pubDate || new Date(),
+          description: post.data?.description,
+          link: `/blog/${post.id}/`
+        }))
+      }
+    } catch {
+      // No collections found
+    }
+  }
+
+  const feedTitle = rssOpts.title || siteConfig.name || "RSS Feed"
+  const feedDesc = rssOpts.description || siteConfig.description || ""
+
+  const xml = buildRssXml({
+    title: feedTitle,
+    description: feedDesc,
+    site: siteUrl,
+    language: rssOpts.language || "en-US",
+    items
+  })
+
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/routes/sitemap.xml.ts

@@ -1,16 +1,43 @@
-import type { APIRoute } from "astro"
-import { buildSitemapXml } from "../sitemap.js"
-import seoConfig from "virtual:rimelight-seo-config"
-
-export const GET: APIRoute = () => {
-  const sitemapOpts = typeof seoConfig?.sitemap === "object" ? seoConfig.sitemap : {}
-  const urls = sitemapOpts.urls || []
-  const xml = buildSitemapXml(urls)
-
-  return new Response(xml, {
-    headers: {
-      "Content-Type": "application/xml; charset=utf-8",
-      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
-    }
-  })
-}
+import type { APIRoute } from "astro"
+import { buildSitemapUrls, buildSitemapXml } from "../sitemap.js"
+import * as seoConfig from "virtual:rimelight-seo/config"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const sitemapOpts = typeof seoOptions?.sitemap === "object" ? seoOptions.sitemap : {}
+  const siteUrl = site ? site.toString() : (siteConfigMod.siteConfig?.url || url.origin)
+
+  let entries = []
+  if (typeof sitemapOpts.entries === "function") {
+    entries = await sitemapOpts.entries()
+  } else if (typeof seoConfig.seoEntries === "function") {
+    entries = await seoConfig.seoEntries()
+  }
+
+  const locales =
+    sitemapOpts.locales ||
+    seoConfig.SEO_LOCALES ||
+    (siteUrl
+      ? {
+          site: siteUrl.replace(/\/$/, ""),
+          locales: { en: "en-US" },
+          defaultLocale: "en",
+          trailingSlash: "never" as const
+        }
+      : undefined)
+
+  let urls = sitemapOpts.urls || []
+  if (urls.length === 0 && entries.length > 0 && locales) {
+    urls = buildSitemapUrls(entries, locales)
+  }
+
+  const xml = buildSitemapXml(urls)
+
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/rss.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest"
+import { buildRssXml } from "./rss.js"
+
+describe("buildRssXml", () => {
+  it("should generate valid RSS XML with channel and items", () => {
+    const xml = buildRssXml({
+      title: "My Site",
+      description: "My site description",
+      site: "https://example.com",
+      items: [
+        {
+          title: "Post 1 & More",
+          link: "/blog/post-1",
+          pubDate: new Date("2026-01-01T00:00:00Z"),
+          description: "Hello & welcome"
+        }
+      ]
+    })
+
+    expect(xml).toContain("<rss version=\"2.0\"")
+    expect(xml).toContain("<title>My Site</title>")
+    expect(xml).toContain("<description>My site description</description>")
+    expect(xml).toContain("<link>https://example.com</link>")
+    expect(xml).toContain("<title>Post 1 &amp; More</title>")
+    expect(xml).toContain("<link>https://example.com/blog/post-1</link>")
+    expect(xml).toContain("<description>Hello &amp; welcome</description>")
+  })
+})

package/src/rss.ts

@@ -0,0 +1,63 @@
+import type { RssItem, RssFeedOptions } from "./types.js"
+
+export function escapeXml(str: string): string {
+  return str
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;")
+}
+
+/**
+ * Builds standard RSS 2.0 feed XML document with Atom namespace link.
+ */
+export function buildRssXml(options: RssFeedOptions): string {
+  const siteUrl = (options.site || "").replace(/\/$/, "")
+  const feedUrl = options.feedUrl || `${siteUrl}/rss.xml`
+  const language = options.language || "en-US"
+  const lastBuildDate = new Date().toUTCString()
+
+  const itemXmls = (options.items || []).map((item) => {
+    const itemLink =
+      item.link.startsWith("http://") || item.link.startsWith("https://")
+        ? item.link
+        : `${siteUrl}${item.link.startsWith("/") ? "" : "/"}${item.link}`
+    const pubDate = new Date(item.pubDate).toUTCString()
+    const guid = item.guid || itemLink
+
+    const lines: string[] = [
+      "    <item>",
+      `      <title>${escapeXml(item.title)}</title>`,
+      `      <link>${escapeXml(itemLink)}</link>`,
+      `      <guid isPermaLink="${guid.startsWith("http") ? "true" : "false"}">${escapeXml(guid)}</guid>`,
+      `      <pubDate>${pubDate}</pubDate>`
+    ]
+
+    if (item.description) {
+      lines.push(`      <description>${escapeXml(item.description)}</description>`)
+    }
+    if (item.author) {
+      lines.push(`      <author>${escapeXml(item.author)}</author>`)
+    }
+    if (item.content) {
+      lines.push(`      <content:encoded><![CDATA[${item.content}]]></content:encoded>`)
+    }
+
+    lines.push("    </item>")
+    return lines.join("\n")
+  })
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+  <channel>
+    <title>${escapeXml(options.title || "")}</title>
+    <description>${escapeXml(options.description || "")}</description>
+    <link>${escapeXml(siteUrl)}</link>
+    <language>${escapeXml(language)}</language>
+    <lastBuildDate>${lastBuildDate}</lastBuildDate>
+    <atom:link href="${escapeXml(feedUrl)}" rel="self" type="application/rss+xml"/>
+${itemXmls.join("\n")}
+  </channel>
+</rss>`
+}