@rimelight/seo 0.0.5 → 0.0.7

package/src/routes/llms.txt.ts

@@ -1,22 +1,93 @@
 import type { APIRoute } from "astro"
-import { buildLlmsTxt } from "../llms.js"
-import seoConfig from "virtual:rimelight-seo-config"
-
-export const GET: APIRoute = ({ site }) => {
-  const llmsOpts = typeof seoConfig?.llms === "object" ? seoConfig.llms : {}
-  const siteStr = site ? site.toString().replace(/\/$/, "") : ""
-  const hostname = site ? new URL(site).hostname : "Site"
-  const content = buildLlmsTxt({
-    site: siteStr,
-    title: hostname,
-    pages: [],
+import { buildLlmsTxt, type LlmsPage } from "../llms.js"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import * as dbMod from "virtual:rimelight-seo/db"
+import * as corpusMod from "virtual:rimelight-seo/corpus"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const prerender = false
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const siteUrl = site ? site.toString() : siteConfig.url || `${url.protocol}//${url.host}`
+  const llmsOpts = typeof seoOptions?.llms === "object" ? seoOptions.llms : {}
+
+  let pages: LlmsPage[] = []
+
+  if (Array.isArray(llmsOpts.pages) && llmsOpts.pages.length > 0) {
+    pages = llmsOpts.pages
+  } else if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
+    pages = corpusMod.corpusPages.map((p: any) => {
+      const slug = p.slug === "index" ? "" : p.slug
+      const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
+      return {
+        title: p.title,
+        description: p.description,
+        url: pageUrl,
+        section: p.section ? p.section.toUpperCase() : "GENERAL"
+      }
+    })
+  } else if (dbMod.db) {
+    try {
+      const { pages: pagesTable } = await import("#db/schema").catch(() => ({ pages: null }))
+      const { and, isNotNull, isNull } = await import("drizzle-orm")
+      if (pagesTable) {
+        const docRows = await dbMod.db
+          .select()
+          .from(pagesTable)
+          .where(and(isNull(pagesTable.deletedAt), isNotNull(pagesTable.publishedVersionId)))
+          .catch(() => [])
+
+        const resolveLocalized = (val: unknown): string => {
+          if (typeof val === "string") return val
+          if (typeof val === "object" && val !== null) {
+            const rec = val as Record<string, string>
+            return rec["en"] || Object.values(rec)[0] || ""
+          }
+          return typeof val === "number" ? String(val) : ""
+        }
+
+        pages = docRows.map((p: any) => {
+          const title = resolveLocalized(p.title) || p.slug
+          const description = resolveLocalized(p.description)
+          const slug = p.slug === "index" ? "" : p.slug
+          const pathPrefix = p.type === "doc" ? "docs" : p.type || "docs"
+          const pageUrl = `${siteUrl.replace(/\/$/, "")}/en/${pathPrefix}/${slug}`.replace(
+            /\/+$/,
+            ""
+          )
+          const markdownUrl = `${pageUrl}.md`
+
+          return {
+            title,
+            description,
+            url: pageUrl,
+            markdownUrl,
+            section: p.type ? p.type.toUpperCase() : "GENERAL"
+          }
+        })
+      }
+    } catch {
+      // Fallback
+    }
+  }
+
+  const title = llmsOpts.title || siteConfig.name || "Documentation"
+  const description =
+    llmsOpts.description || siteConfig.description || "Documentation index for AI agents."
+
+  const body = buildLlmsTxt({
+    site: siteUrl,
+    title,
+    description,
+    pages,
     ...llmsOpts
   })
 
-  return new Response(content, {
+  return new Response(body, {
     headers: {
       "Content-Type": "text/plain; charset=utf-8",
-      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+      "Cache-Control": "public, max-age=3600, s-maxage=86400"
     }
   })
 }

package/src/routes/robots.txt.ts

@@ -1,13 +1,56 @@
 import type { APIRoute } from "astro"
 import { buildRobotsTxt } from "../robots.js"
-import seoConfig from "virtual:rimelight-seo-config"
+import * as seoConfig from "virtual:rimelight-seo/config"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const GET: APIRoute = ({ site, url }) => {
+  const siteUrl = site ? site.toString() : siteConfigMod.siteConfig?.url || url.origin
+  const sitemapURL = new URL("sitemap.xml", siteUrl).toString()
+
+  const defaultPrivatePrefixes = [
+    "/dashboard",
+    "/admin",
+    "/cms",
+    "/internal",
+    "/api",
+    "/dev",
+    "/og",
+    "/open-graph",
+    "/auth"
+  ]
+
+  const customPrivatePrefixes = Array.isArray(seoConfig.PRIVATE_PATH_PREFIXES)
+    ? seoConfig.PRIVATE_PATH_PREFIXES
+    : []
+
+  const privatePrefixes = Array.from(new Set([...defaultPrivatePrefixes, ...customPrivatePrefixes]))
+
+  const disallowPatterns = [
+    ...privatePrefixes,
+    ...privatePrefixes.map((p) => `/*${p.startsWith("/") ? p : `/${p}`}/`)
+  ]
+
+  const robotsOpts = typeof seoOptions?.robots === "object" ? seoOptions.robots : {}
+
+  const defaultAgentRules = [
+    {
+      userAgent: "GPTBot",
+      disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
+      allow: ["/"]
+    },
+    {
+      userAgent: "ClaudeBot",
+      disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
+      allow: ["/"]
+    }
+  ]
 
-export const GET: APIRoute = ({ site }) => {
-  const sitemapURL = site ? new URL("sitemap.xml", site).toString() : ""
-  const robotsOpts = typeof seoConfig?.robots === "object" ? seoConfig.robots : {}
   const robots = buildRobotsTxt({
     sitemapUrl: sitemapURL,
+    disallow: disallowPatterns,
     allow: ["/"],
+    userAgentRules: defaultAgentRules,
     ...robotsOpts
   })
 

package/src/routes/rss.xml.ts

@@ -0,0 +1,129 @@
+import type { APIRoute } from "astro"
+import { buildRssXml } from "../rss.js"
+import * as seoConfig from "virtual:rimelight-seo/config"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import * as dbMod from "virtual:rimelight-seo/db"
+import seoOptions from "virtual:rimelight-seo/options"
+import type { RssItem, RssCollectionConfig } from "../types.js"
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const rssOpts = typeof seoOptions?.rss === "object" ? seoOptions.rss : {}
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const siteUrl = site ? site.toString() : rssOpts.site || siteConfig.url || url.origin
+
+  let items: RssItem[] = []
+
+  if (Array.isArray(rssOpts.items)) {
+    items = rssOpts.items
+  } else if (typeof rssOpts.items === "function") {
+    items = await rssOpts.items()
+  } else if (typeof (seoConfig as any).rssEntries === "function") {
+    items = await (seoConfig as any).rssEntries()
+  } else {
+    const defaultCollections: (string | RssCollectionConfig)[] = [
+      "blog",
+      "posts",
+      "news",
+      "articles"
+    ]
+    const rawCollections =
+      rssOpts.collections && rssOpts.collections.length > 0
+        ? rssOpts.collections
+        : defaultCollections
+
+    const normalizedCollections: Array<{ name: string; pathPrefix: string }> = rawCollections.map(
+      (c: string | RssCollectionConfig) => {
+        if (typeof c === "string") {
+          return { name: c, pathPrefix: `/${c.replace(/^\/+|\/+$/g, "")}` }
+        }
+        return {
+          name: c.name,
+          pathPrefix: c.pathPrefix
+            ? `/${c.pathPrefix.replace(/^\/+|\/+$/g, "")}`
+            : `/${c.name.replace(/^\/+|\/+$/g, "")}`
+        }
+      }
+    )
+
+    if (dbMod.db) {
+      try {
+        const { pages } = await import("#db/schema").catch(() => ({ pages: null }))
+        const { and, isNull } = await import("drizzle-orm")
+        if (pages) {
+          const collectionNames = normalizedCollections.map((c) => c.name)
+          const prefixMap = new Map(normalizedCollections.map((c) => [c.name, c.pathPrefix]))
+
+          const dbPages = await dbMod.db
+            .select()
+            .from(pages)
+            .where(and(isNull(pages.deletedAt)))
+            .catch(() => [])
+
+          const matchingPages = dbPages.filter((p: any) => collectionNames.includes(p.type))
+          for (const p of matchingPages) {
+            const prefix = prefixMap.get(p.type) || `/${p.type}`
+            const title =
+              typeof p.title === "string"
+                ? p.title
+                : p.title?.en || (p.title && Object.values(p.title)[0]) || p.slug
+            const description =
+              typeof p.description === "string"
+                ? p.description
+                : p.description?.en || (p.description && Object.values(p.description)[0]) || ""
+            items.push({
+              title,
+              description,
+              link: `${prefix}/${p.slug}`,
+              pubDate: p.postedAt || p.createdAt || new Date()
+            })
+          }
+        }
+      } catch {
+        // DB lookup fallback
+      }
+    }
+
+    if (items.length === 0) {
+      try {
+        const { getCollection } = await import("astro:content")
+        for (const coll of normalizedCollections) {
+          try {
+            const collEntries = await (getCollection as any)(coll.name)
+            if (Array.isArray(collEntries)) {
+              for (const post of collEntries) {
+                items.push({
+                  title: post.data?.title || post.id,
+                  pubDate: post.data?.pubDate || post.data?.date || new Date(),
+                  description: post.data?.description || post.data?.summary,
+                  link: `${coll.pathPrefix}/${post.id}/`
+                })
+              }
+            }
+          } catch {
+            // Collection not defined in content config, continue
+          }
+        }
+      } catch {
+        // No astro:content available
+      }
+    }
+  }
+
+  const feedTitle = rssOpts.title || siteConfig.name || "RSS Feed"
+  const feedDesc = rssOpts.description || siteConfig.description || ""
+
+  const xml = buildRssXml({
+    title: feedTitle,
+    description: feedDesc,
+    site: siteUrl,
+    language: rssOpts.language || "en-US",
+    items
+  })
+
+  return new Response(xml, {
+    headers: {
+      "Content-Type": "application/xml; charset=utf-8",
+      "Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
+    }
+  })
+}

package/src/routes/security.txt.ts

@@ -0,0 +1,30 @@
+import type { APIRoute } from "astro"
+import { buildSecurityTxt } from "../security.js"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const prerender = false
+
+export const GET: APIRoute = ({ site, url }) => {
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const siteUrl = site ? site.toString() : siteConfig.url || url.origin
+  const secOpts = typeof seoOptions?.security === "object" ? seoOptions.security : {}
+
+  const defaultContact = siteConfig.email ? `mailto:${siteConfig.email}` : undefined
+  const contacts =
+    secOpts.contact || defaultContact || `mailto:security@${new URL(siteUrl).hostname}`
+  const canonical = secOpts.canonical || new URL("/.well-known/security.txt", siteUrl).toString()
+
+  const content = buildSecurityTxt({
+    contact: contacts,
+    canonical,
+    ...secOpts
+  })
+
+  return new Response(content, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public, max-age=86400, stale-while-revalidate=604800"
+    }
+  })
+}

package/src/routes/site.webmanifest.ts

@@ -0,0 +1,28 @@
+import type { APIRoute } from "astro"
+import { buildWebManifest } from "../manifest.js"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const prerender = false
+
+export const GET: APIRoute = () => {
+  const siteConfig = siteConfigMod.siteConfig || {}
+  const manifestOpts = typeof seoOptions?.manifest === "object" ? seoOptions.manifest : {}
+
+  const manifest = buildWebManifest({
+    name: manifestOpts.name || siteConfig.name || "Site",
+    shortName: manifestOpts.shortName || siteConfig.name || "Site",
+    description: manifestOpts.description || siteConfig.description || "",
+    backgroundColor:
+      manifestOpts.backgroundColor || siteConfig.branding?.colors?.backgroundColor || "#ffffff",
+    themeColor: manifestOpts.themeColor || siteConfig.branding?.colors?.themeColor || "#ffffff",
+    ...manifestOpts
+  })
+
+  return new Response(JSON.stringify(manifest, null, 2), {
+    headers: {
+      "Content-Type": "application/manifest+json; charset=utf-8",
+      "Cache-Control": "public, max-age=86400, stale-while-revalidate=604800"
+    }
+  })
+}

package/src/routes/sitemap.xml.ts

@@ -1,10 +1,37 @@
 import type { APIRoute } from "astro"
-import { buildSitemapXml } from "../sitemap.js"
-import seoConfig from "virtual:rimelight-seo-config"
+import { buildSitemapUrls, buildSitemapXml } from "../sitemap.js"
+import * as seoConfig from "virtual:rimelight-seo/config"
+import * as siteConfigMod from "virtual:rimelight-seo/site-config"
+import seoOptions from "virtual:rimelight-seo/options"
+
+export const GET: APIRoute = async ({ site, url }) => {
+  const sitemapOpts = typeof seoOptions?.sitemap === "object" ? seoOptions.sitemap : {}
+  const siteUrl = site ? site.toString() : siteConfigMod.siteConfig?.url || url.origin
+
+  let entries: any[] = []
+  if (typeof sitemapOpts.entries === "function") {
+    entries = await sitemapOpts.entries()
+  } else if (typeof seoConfig.seoEntries === "function") {
+    entries = await seoConfig.seoEntries()
+  }
+
+  const locales =
+    sitemapOpts.locales ||
+    seoConfig.SEO_LOCALES ||
+    (siteUrl
+      ? {
+          site: siteUrl.replace(/\/$/, ""),
+          locales: { en: "en-US" },
+          defaultLocale: "en",
+          trailingSlash: "never" as const
+        }
+      : undefined)
+
+  let urls = sitemapOpts.urls || []
+  if (urls.length === 0 && entries.length > 0 && locales) {
+    urls = buildSitemapUrls(entries, locales)
+  }
 
-export const GET: APIRoute = () => {
-  const sitemapOpts = typeof seoConfig?.sitemap === "object" ? seoConfig.sitemap : {}
-  const urls = sitemapOpts.urls || []
   const xml = buildSitemapXml(urls)
 
   return new Response(xml, {

package/src/rss.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vite-plus/test"
+import { buildRssXml } from "./rss.js"
+
+describe("buildRssXml", () => {
+  it("should generate valid RSS XML with channel and items", () => {
+    const xml = buildRssXml({
+      title: "My Site",
+      description: "My site description",
+      site: "https://example.com",
+      items: [
+        {
+          title: "Post 1 & More",
+          link: "/blog/post-1",
+          pubDate: new Date("2026-01-01T00:00:00Z"),
+          description: "Hello & welcome"
+        }
+      ]
+    })
+
+    expect(xml).toContain('<rss version="2.0"')
+    expect(xml).toContain("<title>My Site</title>")
+    expect(xml).toContain("<description>My site description</description>")
+    expect(xml).toContain("<link>https://example.com</link>")
+    expect(xml).toContain("<title>Post 1 &amp; More</title>")
+    expect(xml).toContain("<link>https://example.com/blog/post-1</link>")
+    expect(xml).toContain("<description>Hello &amp; welcome</description>")
+  })
+})

package/src/rss.ts

@@ -0,0 +1,63 @@
+import type { RssFeedOptions } from "./types.js"
+
+export function escapeXml(str: string): string {
+  return str
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;")
+}
+
+/**
+ * Builds standard RSS 2.0 feed XML document with Atom namespace link.
+ */
+export function buildRssXml(options: RssFeedOptions): string {
+  const siteUrl = (options.site || "").replace(/\/$/, "")
+  const feedUrl = options.feedUrl || `${siteUrl}/rss.xml`
+  const language = options.language || "en-US"
+  const lastBuildDate = new Date().toUTCString()
+
+  const itemXmls = (options.items || []).map((item) => {
+    const itemLink =
+      item.link.startsWith("http://") || item.link.startsWith("https://")
+        ? item.link
+        : `${siteUrl}${item.link.startsWith("/") ? "" : "/"}${item.link}`
+    const pubDate = new Date(item.pubDate).toUTCString()
+    const guid = item.guid || itemLink
+
+    const lines: string[] = [
+      "    <item>",
+      `      <title>${escapeXml(item.title)}</title>`,
+      `      <link>${escapeXml(itemLink)}</link>`,
+      `      <guid isPermaLink="${guid.startsWith("http") ? "true" : "false"}">${escapeXml(guid)}</guid>`,
+      `      <pubDate>${pubDate}</pubDate>`
+    ]
+
+    if (item.description) {
+      lines.push(`      <description>${escapeXml(item.description)}</description>`)
+    }
+    if (item.author) {
+      lines.push(`      <author>${escapeXml(item.author)}</author>`)
+    }
+    if (item.content) {
+      lines.push(`      <content:encoded><![CDATA[${item.content}]]></content:encoded>`)
+    }
+
+    lines.push("    </item>")
+    return lines.join("\n")
+  })
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+  <channel>
+    <title>${escapeXml(options.title || "")}</title>
+    <description>${escapeXml(options.description || "")}</description>
+    <link>${escapeXml(siteUrl)}</link>
+    <language>${escapeXml(language)}</language>
+    <lastBuildDate>${lastBuildDate}</lastBuildDate>
+    <atom:link href="${escapeXml(feedUrl)}" rel="self" type="application/rss+xml"/>
+${itemXmls.join("\n")}
+  </channel>
+</rss>`
+}

package/src/security.test.ts

@@ -0,0 +1,46 @@
+import { describe, it, expect } from "vite-plus/test"
+import { buildSecurityTxt } from "./security.js"
+
+describe("buildSecurityTxt", () => {
+  it("generates RFC 9116 compliant security.txt with single contact and default expires", () => {
+    const output = buildSecurityTxt({
+      contact: "mailto:security@example.com"
+    })
+
+    expect(output).toContain("Contact: mailto:security@example.com")
+    expect(output).toMatch(/Expires: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/)
+  })
+
+  it("handles multiple contacts and explicit date expires", () => {
+    const expiry = new Date("2027-12-31T23:59:59.000Z")
+    const output = buildSecurityTxt({
+      contact: ["mailto:security@example.com", "https://example.com/security/report"],
+      expires: expiry,
+      preferredLanguages: "en, pt",
+      canonical: "https://example.com/.well-known/security.txt",
+      policy: "https://example.com/legal/security",
+      hiring: "https://example.com/careers",
+      acknowledgments: "https://example.com/security/acknowledgments"
+    })
+
+    expect(output).toContain("Contact: mailto:security@example.com")
+    expect(output).toContain("Contact: https://example.com/security/report")
+    expect(output).toContain("Expires: 2027-12-31T23:59:59.000Z")
+    expect(output).toContain("Preferred-Languages: en, pt")
+    expect(output).toContain("Canonical: https://example.com/.well-known/security.txt")
+    expect(output).toContain("Policy: https://example.com/legal/security")
+    expect(output).toContain("Hiring: https://example.com/careers")
+    expect(output).toContain("Acknowledgments: https://example.com/security/acknowledgments")
+  })
+
+  it("formats encryption and csaf directives", () => {
+    const output = buildSecurityTxt({
+      contact: "mailto:security@example.com",
+      encryption: "https://example.com/pgp-key.txt",
+      csaf: "https://example.com/.well-known/csaf/provider-metadata.json"
+    })
+
+    expect(output).toContain("Encryption: https://example.com/pgp-key.txt")
+    expect(output).toContain("CSAF: https://example.com/.well-known/csaf/provider-metadata.json")
+  })
+})

package/src/security.ts

@@ -0,0 +1,78 @@
+import type { SecurityTxtOptions } from "./types.js"
+
+/**
+ * Build RFC 9116 compliant security.txt content. https://www.rfc-editor.org/rfc/rfc9116
+ */
+export function buildSecurityTxt(options: SecurityTxtOptions): string {
+  const lines: string[] = []
+
+  // Contact (REQUIRED in RFC 9116: at least one)
+  const contacts = Array.isArray(options.contact)
+    ? options.contact
+    : options.contact
+      ? [options.contact]
+      : []
+
+  for (const contact of contacts) {
+    lines.push(`Contact: ${contact}`)
+  }
+
+  // Expires (REQUIRED in RFC 9116)
+  let expiresStr: string
+  if (options.expires instanceof Date) {
+    expiresStr = options.expires.toISOString()
+  } else if (typeof options.expires === "string" && options.expires.trim().length > 0) {
+    expiresStr = options.expires
+  } else {
+    // Default to 1 year in the future from generation
+    const oneYearFromNow = new Date()
+    oneYearFromNow.setFullYear(oneYearFromNow.getFullYear() + 1)
+    expiresStr = oneYearFromNow.toISOString()
+  }
+  lines.push(`Expires: ${expiresStr}`)
+
+  // Encryption (OPTIONAL)
+  if (options.encryption) {
+    const encList = Array.isArray(options.encryption) ? options.encryption : [options.encryption]
+    for (const enc of encList) {
+      lines.push(`Encryption: ${enc}`)
+    }
+  }
+
+  // Acknowledgments (OPTIONAL)
+  if (options.acknowledgments) {
+    const acks = Array.isArray(options.acknowledgments)
+      ? options.acknowledgments
+      : [options.acknowledgments]
+    for (const ack of acks) {
+      lines.push(`Acknowledgments: ${ack}`)
+    }
+  }
+
+  // Preferred-Languages (OPTIONAL)
+  if (options.preferredLanguages) {
+    lines.push(`Preferred-Languages: ${options.preferredLanguages}`)
+  }
+
+  // Canonical (OPTIONAL)
+  if (options.canonical) {
+    lines.push(`Canonical: ${options.canonical}`)
+  }
+
+  // Policy (OPTIONAL)
+  if (options.policy) {
+    lines.push(`Policy: ${options.policy}`)
+  }
+
+  // Hiring (OPTIONAL)
+  if (options.hiring) {
+    lines.push(`Hiring: ${options.hiring}`)
+  }
+
+  // CSAF (OPTIONAL)
+  if (options.csaf) {
+    lines.push(`CSAF: ${options.csaf}`)
+  }
+
+  return lines.join("\n") + "\n"
+}