@rimelight/seo 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets.d.mts +39 -0
- package/dist/assets.mjs +96 -0
- package/dist/catalog.d.mts +7 -0
- package/dist/catalog.mjs +10 -0
- package/dist/index.d.mts +6 -2
- package/dist/index.mjs +5 -1
- package/dist/integration.mjs +182 -0
- package/dist/manifest.d.mts +7 -0
- package/dist/manifest.mjs +44 -0
- package/dist/og.mjs +1 -1
- package/dist/routes/api-catalog.d.mts +5 -0
- package/dist/routes/api-catalog.mjs +38 -0
- package/dist/routes/apple-touch-icon.png.d.mts +5 -0
- package/dist/routes/apple-touch-icon.png.mjs +17 -0
- package/dist/routes/favicon.ico.d.mts +5 -0
- package/dist/routes/favicon.ico.mjs +14 -0
- package/dist/routes/favicon.svg.d.mts +5 -0
- package/dist/routes/favicon.svg.mjs +12 -0
- package/dist/routes/icon-192.png.d.mts +5 -0
- package/dist/routes/icon-192.png.mjs +17 -0
- package/dist/routes/icon-512.png.d.mts +5 -0
- package/dist/routes/icon-512.png.mjs +17 -0
- package/dist/routes/icon-maskable-512.png.d.mts +5 -0
- package/dist/routes/icon-maskable-512.png.mjs +18 -0
- package/dist/routes/llms.txt.mjs +1 -1
- package/dist/routes/robots.txt.mjs +1 -1
- package/dist/routes/rss.xml.mjs +48 -20
- package/dist/routes/security.txt.d.mts +5 -0
- package/dist/routes/security.txt.mjs +24 -0
- package/dist/routes/site.webmanifest.d.mts +5 -0
- package/dist/routes/site.webmanifest.mjs +23 -0
- package/dist/routes/sitemap.xml.mjs +1 -1
- package/dist/security.d.mts +7 -0
- package/dist/security.mjs +34 -0
- package/dist/types.d.mts +191 -8
- package/package.json +23 -2
- package/src/assets.test.ts +70 -0
- package/src/assets.ts +137 -0
- package/src/catalog.test.ts +26 -0
- package/src/catalog.ts +11 -0
- package/src/components/SEOHead.astro +1 -1
- package/src/env.d.ts +56 -33
- package/src/index.ts +18 -1
- package/src/integration.ts +353 -172
- package/src/manifest.test.ts +41 -0
- package/src/manifest.ts +59 -0
- package/src/og.ts +1 -9
- package/src/routes/api-catalog.ts +55 -0
- package/src/routes/apple-touch-icon.png.ts +19 -0
- package/src/routes/favicon.ico.ts +19 -0
- package/src/routes/favicon.svg.ts +17 -0
- package/src/routes/icon-192.png.ts +19 -0
- package/src/routes/icon-512.png.ts +19 -0
- package/src/routes/icon-maskable-512.png.ts +24 -0
- package/src/routes/llms-full.txt.ts +77 -77
- package/src/routes/llms.txt.ts +93 -90
- package/src/routes/robots.txt.ts +63 -63
- package/src/routes/rss.xml.ts +129 -87
- package/src/routes/security.txt.ts +30 -0
- package/src/routes/site.webmanifest.ts +28 -0
- package/src/routes/sitemap.xml.ts +43 -43
- package/src/rss.test.ts +28 -28
- package/src/rss.ts +63 -63
- package/src/security.test.ts +46 -0
- package/src/security.ts +78 -0
- package/src/types.ts +564 -367
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { buildApiCatalog } from "../catalog.js"
|
|
3
|
+
import type { ApiCatalogItem } from "../types.js"
|
|
4
|
+
import seoOptions from "virtual:rimelight-seo/options"
|
|
5
|
+
|
|
6
|
+
export const prerender = false
|
|
7
|
+
|
|
8
|
+
export const GET: APIRoute = () => {
|
|
9
|
+
const catalogOpts = typeof seoOptions?.apiCatalog === "object" ? seoOptions.apiCatalog : {}
|
|
10
|
+
|
|
11
|
+
const defaultItems: ApiCatalogItem[] = [
|
|
12
|
+
{
|
|
13
|
+
rel: "self",
|
|
14
|
+
href: "/.well-known/api-catalog",
|
|
15
|
+
type: "application/linkset+json"
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
if (seoOptions?.sitemap !== false) {
|
|
20
|
+
defaultItems.push({
|
|
21
|
+
rel: "alternate",
|
|
22
|
+
href: "/sitemap.xml",
|
|
23
|
+
type: "application/xml",
|
|
24
|
+
title: "Sitemap"
|
|
25
|
+
})
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (seoOptions?.rss !== false) {
|
|
29
|
+
defaultItems.push({
|
|
30
|
+
rel: "alternate",
|
|
31
|
+
href: "/rss.xml",
|
|
32
|
+
type: "application/rss+xml",
|
|
33
|
+
title: "RSS Feed"
|
|
34
|
+
})
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
if (seoOptions?.llms !== false) {
|
|
38
|
+
defaultItems.push({
|
|
39
|
+
rel: "llms",
|
|
40
|
+
href: "/llms.txt",
|
|
41
|
+
type: "text/plain",
|
|
42
|
+
title: "LLMs.txt"
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const items = catalogOpts.items || defaultItems
|
|
47
|
+
const catalog = buildApiCatalog({ items })
|
|
48
|
+
|
|
49
|
+
return new Response(JSON.stringify(catalog, null, 2), {
|
|
50
|
+
headers: {
|
|
51
|
+
"Content-Type": "application/linkset+json; charset=utf-8",
|
|
52
|
+
"Cache-Control": "public, max-age=86400, stale-while-revalidate=604800"
|
|
53
|
+
}
|
|
54
|
+
})
|
|
55
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { generatePng } from "../assets.js"
|
|
3
|
+
import { masterIconBuffer } from "virtual:rimelight-seo/assets"
|
|
4
|
+
|
|
5
|
+
export const prerender = true
|
|
6
|
+
|
|
7
|
+
export const GET: APIRoute = async () => {
|
|
8
|
+
if (!masterIconBuffer) {
|
|
9
|
+
return new Response(null, { status: 404 })
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const png = await generatePng(masterIconBuffer, { width: 180, height: 180 })
|
|
13
|
+
return new Response(new Uint8Array(png), {
|
|
14
|
+
headers: {
|
|
15
|
+
"Content-Type": "image/png",
|
|
16
|
+
"Cache-Control": "public, max-age=31536000, immutable"
|
|
17
|
+
}
|
|
18
|
+
})
|
|
19
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { generateIco } from "../assets.js"
|
|
3
|
+
import { masterIconBuffer } from "virtual:rimelight-seo/assets"
|
|
4
|
+
|
|
5
|
+
export const prerender = true
|
|
6
|
+
|
|
7
|
+
export const GET: APIRoute = async () => {
|
|
8
|
+
if (!masterIconBuffer) {
|
|
9
|
+
return new Response(null, { status: 404 })
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const ico = await generateIco(masterIconBuffer, 32)
|
|
13
|
+
return new Response(new Uint8Array(ico), {
|
|
14
|
+
headers: {
|
|
15
|
+
"Content-Type": "image/x-icon",
|
|
16
|
+
"Cache-Control": "public, max-age=31536000, immutable"
|
|
17
|
+
}
|
|
18
|
+
})
|
|
19
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { masterIconBuffer, isSvg } from "virtual:rimelight-seo/assets"
|
|
3
|
+
|
|
4
|
+
export const prerender = true
|
|
5
|
+
|
|
6
|
+
export const GET: APIRoute = async () => {
|
|
7
|
+
if (!masterIconBuffer || !isSvg) {
|
|
8
|
+
return new Response(null, { status: 404 })
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
return new Response(new Uint8Array(masterIconBuffer), {
|
|
12
|
+
headers: {
|
|
13
|
+
"Content-Type": "image/svg+xml",
|
|
14
|
+
"Cache-Control": "public, max-age=31536000, immutable"
|
|
15
|
+
}
|
|
16
|
+
})
|
|
17
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { generatePng } from "../assets.js"
|
|
3
|
+
import { masterIconBuffer } from "virtual:rimelight-seo/assets"
|
|
4
|
+
|
|
5
|
+
export const prerender = true
|
|
6
|
+
|
|
7
|
+
export const GET: APIRoute = async () => {
|
|
8
|
+
if (!masterIconBuffer) {
|
|
9
|
+
return new Response(null, { status: 404 })
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const png = await generatePng(masterIconBuffer, { width: 192, height: 192 })
|
|
13
|
+
return new Response(new Uint8Array(png), {
|
|
14
|
+
headers: {
|
|
15
|
+
"Content-Type": "image/png",
|
|
16
|
+
"Cache-Control": "public, max-age=31536000, immutable"
|
|
17
|
+
}
|
|
18
|
+
})
|
|
19
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { generatePng } from "../assets.js"
|
|
3
|
+
import { masterIconBuffer } from "virtual:rimelight-seo/assets"
|
|
4
|
+
|
|
5
|
+
export const prerender = true
|
|
6
|
+
|
|
7
|
+
export const GET: APIRoute = async () => {
|
|
8
|
+
if (!masterIconBuffer) {
|
|
9
|
+
return new Response(null, { status: 404 })
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const png = await generatePng(masterIconBuffer, { width: 512, height: 512 })
|
|
13
|
+
return new Response(new Uint8Array(png), {
|
|
14
|
+
headers: {
|
|
15
|
+
"Content-Type": "image/png",
|
|
16
|
+
"Cache-Control": "public, max-age=31536000, immutable"
|
|
17
|
+
}
|
|
18
|
+
})
|
|
19
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { generateMaskablePng } from "../assets.js"
|
|
3
|
+
import { masterIconBuffer, backgroundColor } from "virtual:rimelight-seo/assets"
|
|
4
|
+
|
|
5
|
+
export const prerender = true
|
|
6
|
+
|
|
7
|
+
export const GET: APIRoute = async () => {
|
|
8
|
+
if (!masterIconBuffer) {
|
|
9
|
+
return new Response(null, { status: 404 })
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const png = await generateMaskablePng(masterIconBuffer, {
|
|
13
|
+
size: 512,
|
|
14
|
+
background: backgroundColor || "#ffffff",
|
|
15
|
+
paddingRatio: 0.15
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
return new Response(new Uint8Array(png), {
|
|
19
|
+
headers: {
|
|
20
|
+
"Content-Type": "image/png",
|
|
21
|
+
"Cache-Control": "public, max-age=31536000, immutable"
|
|
22
|
+
}
|
|
23
|
+
})
|
|
24
|
+
}
|
|
@@ -1,77 +1,77 @@
|
|
|
1
|
-
import type { APIRoute } from "astro"
|
|
2
|
-
import * as siteConfigMod from "virtual:rimelight-seo/site-config"
|
|
3
|
-
import * as dbMod from "virtual:rimelight-seo/db"
|
|
4
|
-
import * as corpusMod from "virtual:rimelight-seo/corpus"
|
|
5
|
-
|
|
6
|
-
export const prerender = false
|
|
7
|
-
|
|
8
|
-
export const GET: APIRoute = async ({ site, url }) => {
|
|
9
|
-
const siteConfig = siteConfigMod.siteConfig || {}
|
|
10
|
-
const siteUrl = site ? site.toString() :
|
|
11
|
-
const title = siteConfig.name
|
|
12
|
-
? `${siteConfig.name} Full Documentation Corpus`
|
|
13
|
-
: "Full Documentation Corpus"
|
|
14
|
-
const description = "Complete single-corpus documentation for AI agents and LLM ingest."
|
|
15
|
-
|
|
16
|
-
// 1. Try corpusPages if available
|
|
17
|
-
if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
|
|
18
|
-
const sections = new Map<string, string[]>()
|
|
19
|
-
for (const p of corpusMod.corpusPages) {
|
|
20
|
-
const key = (p.section || "GENERAL").toUpperCase()
|
|
21
|
-
if (!sections.has(key)) sections.set(key, [])
|
|
22
|
-
const slug = p.slug === "index" ? "" : p.slug
|
|
23
|
-
const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
|
|
24
|
-
sections.get(key)!.push(`### ${p.title}\nURL: ${pageUrl}\n${p.description || ""}`)
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const body = [
|
|
28
|
-
`# ${title}`,
|
|
29
|
-
"",
|
|
30
|
-
description,
|
|
31
|
-
"",
|
|
32
|
-
...Array.from(sections.entries()).flatMap(([section, pages]) => [
|
|
33
|
-
`## ${section}`,
|
|
34
|
-
"",
|
|
35
|
-
...pages,
|
|
36
|
-
""
|
|
37
|
-
])
|
|
38
|
-
].join("\n")
|
|
39
|
-
|
|
40
|
-
return new Response(body, {
|
|
41
|
-
headers: {
|
|
42
|
-
"Content-Type": "text/plain; charset=utf-8",
|
|
43
|
-
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
44
|
-
}
|
|
45
|
-
})
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
// 2. Try CMS renderCorpusMarkdown if db exists
|
|
49
|
-
if (dbMod.db) {
|
|
50
|
-
try {
|
|
51
|
-
const { renderCorpusMarkdown } = await import("@rimelight/cms")
|
|
52
|
-
const body = await renderCorpusMarkdown(dbMod.db, {
|
|
53
|
-
type: "doc",
|
|
54
|
-
locale: "en",
|
|
55
|
-
siteUrl,
|
|
56
|
-
title,
|
|
57
|
-
description
|
|
58
|
-
})
|
|
59
|
-
|
|
60
|
-
return new Response(body, {
|
|
61
|
-
headers: {
|
|
62
|
-
"Content-Type": "text/plain; charset=utf-8",
|
|
63
|
-
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
64
|
-
}
|
|
65
|
-
})
|
|
66
|
-
} catch {
|
|
67
|
-
// Fallback
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
return new Response(`# ${title}\n\n${description}\n`, {
|
|
72
|
-
headers: {
|
|
73
|
-
"Content-Type": "text/plain; charset=utf-8",
|
|
74
|
-
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
75
|
-
}
|
|
76
|
-
})
|
|
77
|
-
}
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import * as siteConfigMod from "virtual:rimelight-seo/site-config"
|
|
3
|
+
import * as dbMod from "virtual:rimelight-seo/db"
|
|
4
|
+
import * as corpusMod from "virtual:rimelight-seo/corpus"
|
|
5
|
+
|
|
6
|
+
export const prerender = false
|
|
7
|
+
|
|
8
|
+
export const GET: APIRoute = async ({ site, url }) => {
|
|
9
|
+
const siteConfig = siteConfigMod.siteConfig || {}
|
|
10
|
+
const siteUrl = site ? site.toString() : siteConfig.url || `${url.protocol}//${url.host}`
|
|
11
|
+
const title = siteConfig.name
|
|
12
|
+
? `${siteConfig.name} Full Documentation Corpus`
|
|
13
|
+
: "Full Documentation Corpus"
|
|
14
|
+
const description = "Complete single-corpus documentation for AI agents and LLM ingest."
|
|
15
|
+
|
|
16
|
+
// 1. Try corpusPages if available
|
|
17
|
+
if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
|
|
18
|
+
const sections = new Map<string, string[]>()
|
|
19
|
+
for (const p of corpusMod.corpusPages) {
|
|
20
|
+
const key = (p.section || "GENERAL").toUpperCase()
|
|
21
|
+
if (!sections.has(key)) sections.set(key, [])
|
|
22
|
+
const slug = p.slug === "index" ? "" : p.slug
|
|
23
|
+
const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
|
|
24
|
+
sections.get(key)!.push(`### ${p.title}\nURL: ${pageUrl}\n${p.description || ""}`)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const body = [
|
|
28
|
+
`# ${title}`,
|
|
29
|
+
"",
|
|
30
|
+
description,
|
|
31
|
+
"",
|
|
32
|
+
...Array.from(sections.entries()).flatMap(([section, pages]) => [
|
|
33
|
+
`## ${section}`,
|
|
34
|
+
"",
|
|
35
|
+
...pages,
|
|
36
|
+
""
|
|
37
|
+
])
|
|
38
|
+
].join("\n")
|
|
39
|
+
|
|
40
|
+
return new Response(body, {
|
|
41
|
+
headers: {
|
|
42
|
+
"Content-Type": "text/plain; charset=utf-8",
|
|
43
|
+
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
44
|
+
}
|
|
45
|
+
})
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// 2. Try CMS renderCorpusMarkdown if db exists
|
|
49
|
+
if (dbMod.db) {
|
|
50
|
+
try {
|
|
51
|
+
const { renderCorpusMarkdown } = await import("@rimelight/cms")
|
|
52
|
+
const body = await renderCorpusMarkdown(dbMod.db, {
|
|
53
|
+
type: "doc",
|
|
54
|
+
locale: "en",
|
|
55
|
+
siteUrl,
|
|
56
|
+
title,
|
|
57
|
+
description
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
return new Response(body, {
|
|
61
|
+
headers: {
|
|
62
|
+
"Content-Type": "text/plain; charset=utf-8",
|
|
63
|
+
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
64
|
+
}
|
|
65
|
+
})
|
|
66
|
+
} catch {
|
|
67
|
+
// Fallback
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return new Response(`# ${title}\n\n${description}\n`, {
|
|
72
|
+
headers: {
|
|
73
|
+
"Content-Type": "text/plain; charset=utf-8",
|
|
74
|
+
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
75
|
+
}
|
|
76
|
+
})
|
|
77
|
+
}
|
package/src/routes/llms.txt.ts
CHANGED
|
@@ -1,90 +1,93 @@
|
|
|
1
|
-
import type { APIRoute } from "astro"
|
|
2
|
-
import { buildLlmsTxt, type LlmsPage } from "../llms.js"
|
|
3
|
-
import * as siteConfigMod from "virtual:rimelight-seo/site-config"
|
|
4
|
-
import * as dbMod from "virtual:rimelight-seo/db"
|
|
5
|
-
import * as corpusMod from "virtual:rimelight-seo/corpus"
|
|
6
|
-
import seoOptions from "virtual:rimelight-seo/options"
|
|
7
|
-
|
|
8
|
-
export const prerender = false
|
|
9
|
-
|
|
10
|
-
export const GET: APIRoute = async ({ site, url }) => {
|
|
11
|
-
const siteConfig = siteConfigMod.siteConfig || {}
|
|
12
|
-
const siteUrl = site ? site.toString() :
|
|
13
|
-
const llmsOpts = typeof seoOptions?.llms === "object" ? seoOptions.llms : {}
|
|
14
|
-
|
|
15
|
-
let pages: LlmsPage[] = []
|
|
16
|
-
|
|
17
|
-
if (Array.isArray(llmsOpts.pages) && llmsOpts.pages.length > 0) {
|
|
18
|
-
pages = llmsOpts.pages
|
|
19
|
-
} else if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
|
|
20
|
-
pages = corpusMod.corpusPages.map((p: any) => {
|
|
21
|
-
const slug = p.slug === "index" ? "" : p.slug
|
|
22
|
-
const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
|
|
23
|
-
return {
|
|
24
|
-
title: p.title,
|
|
25
|
-
description: p.description,
|
|
26
|
-
url: pageUrl,
|
|
27
|
-
section: p.section ? p.section.toUpperCase() : "GENERAL"
|
|
28
|
-
}
|
|
29
|
-
})
|
|
30
|
-
} else if (dbMod.db) {
|
|
31
|
-
try {
|
|
32
|
-
const { pages: pagesTable } = await import("#db/schema").catch(() => ({ pages: null }))
|
|
33
|
-
const { and, isNotNull, isNull } = await import("drizzle-orm")
|
|
34
|
-
if (pagesTable) {
|
|
35
|
-
const docRows = await dbMod.db
|
|
36
|
-
.select()
|
|
37
|
-
.from(pagesTable)
|
|
38
|
-
.where(and(isNull(pagesTable.deletedAt), isNotNull(pagesTable.publishedVersionId)))
|
|
39
|
-
.catch(() => [])
|
|
40
|
-
|
|
41
|
-
const resolveLocalized = (val: unknown): string => {
|
|
42
|
-
if (typeof val === "string") return val
|
|
43
|
-
if (typeof val === "object" && val !== null) {
|
|
44
|
-
const rec = val as Record<string, string>
|
|
45
|
-
return rec["en"] || Object.values(rec)[0] || ""
|
|
46
|
-
}
|
|
47
|
-
return typeof val === "number" ? String(val) : ""
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
pages = docRows.map((p: any) => {
|
|
51
|
-
const title = resolveLocalized(p.title) || p.slug
|
|
52
|
-
const description = resolveLocalized(p.description)
|
|
53
|
-
const slug = p.slug === "index" ? "" : p.slug
|
|
54
|
-
const pathPrefix = p.type === "doc" ? "docs" : p.type || "docs"
|
|
55
|
-
const pageUrl = `${siteUrl.replace(/\/$/, "")}/en/${pathPrefix}/${slug}`.replace(
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { buildLlmsTxt, type LlmsPage } from "../llms.js"
|
|
3
|
+
import * as siteConfigMod from "virtual:rimelight-seo/site-config"
|
|
4
|
+
import * as dbMod from "virtual:rimelight-seo/db"
|
|
5
|
+
import * as corpusMod from "virtual:rimelight-seo/corpus"
|
|
6
|
+
import seoOptions from "virtual:rimelight-seo/options"
|
|
7
|
+
|
|
8
|
+
export const prerender = false
|
|
9
|
+
|
|
10
|
+
export const GET: APIRoute = async ({ site, url }) => {
|
|
11
|
+
const siteConfig = siteConfigMod.siteConfig || {}
|
|
12
|
+
const siteUrl = site ? site.toString() : siteConfig.url || `${url.protocol}//${url.host}`
|
|
13
|
+
const llmsOpts = typeof seoOptions?.llms === "object" ? seoOptions.llms : {}
|
|
14
|
+
|
|
15
|
+
let pages: LlmsPage[] = []
|
|
16
|
+
|
|
17
|
+
if (Array.isArray(llmsOpts.pages) && llmsOpts.pages.length > 0) {
|
|
18
|
+
pages = llmsOpts.pages
|
|
19
|
+
} else if (Array.isArray(corpusMod.corpusPages) && corpusMod.corpusPages.length > 0) {
|
|
20
|
+
pages = corpusMod.corpusPages.map((p: any) => {
|
|
21
|
+
const slug = p.slug === "index" ? "" : p.slug
|
|
22
|
+
const pageUrl = `${siteUrl.replace(/\/$/, "")}/${slug}`.replace(/\/+$/, "")
|
|
23
|
+
return {
|
|
24
|
+
title: p.title,
|
|
25
|
+
description: p.description,
|
|
26
|
+
url: pageUrl,
|
|
27
|
+
section: p.section ? p.section.toUpperCase() : "GENERAL"
|
|
28
|
+
}
|
|
29
|
+
})
|
|
30
|
+
} else if (dbMod.db) {
|
|
31
|
+
try {
|
|
32
|
+
const { pages: pagesTable } = await import("#db/schema").catch(() => ({ pages: null }))
|
|
33
|
+
const { and, isNotNull, isNull } = await import("drizzle-orm")
|
|
34
|
+
if (pagesTable) {
|
|
35
|
+
const docRows = await dbMod.db
|
|
36
|
+
.select()
|
|
37
|
+
.from(pagesTable)
|
|
38
|
+
.where(and(isNull(pagesTable.deletedAt), isNotNull(pagesTable.publishedVersionId)))
|
|
39
|
+
.catch(() => [])
|
|
40
|
+
|
|
41
|
+
const resolveLocalized = (val: unknown): string => {
|
|
42
|
+
if (typeof val === "string") return val
|
|
43
|
+
if (typeof val === "object" && val !== null) {
|
|
44
|
+
const rec = val as Record<string, string>
|
|
45
|
+
return rec["en"] || Object.values(rec)[0] || ""
|
|
46
|
+
}
|
|
47
|
+
return typeof val === "number" ? String(val) : ""
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
pages = docRows.map((p: any) => {
|
|
51
|
+
const title = resolveLocalized(p.title) || p.slug
|
|
52
|
+
const description = resolveLocalized(p.description)
|
|
53
|
+
const slug = p.slug === "index" ? "" : p.slug
|
|
54
|
+
const pathPrefix = p.type === "doc" ? "docs" : p.type || "docs"
|
|
55
|
+
const pageUrl = `${siteUrl.replace(/\/$/, "")}/en/${pathPrefix}/${slug}`.replace(
|
|
56
|
+
/\/+$/,
|
|
57
|
+
""
|
|
58
|
+
)
|
|
59
|
+
const markdownUrl = `${pageUrl}.md`
|
|
60
|
+
|
|
61
|
+
return {
|
|
62
|
+
title,
|
|
63
|
+
description,
|
|
64
|
+
url: pageUrl,
|
|
65
|
+
markdownUrl,
|
|
66
|
+
section: p.type ? p.type.toUpperCase() : "GENERAL"
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
}
|
|
70
|
+
} catch {
|
|
71
|
+
// Fallback
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const title = llmsOpts.title || siteConfig.name || "Documentation"
|
|
76
|
+
const description =
|
|
77
|
+
llmsOpts.description || siteConfig.description || "Documentation index for AI agents."
|
|
78
|
+
|
|
79
|
+
const body = buildLlmsTxt({
|
|
80
|
+
site: siteUrl,
|
|
81
|
+
title,
|
|
82
|
+
description,
|
|
83
|
+
pages,
|
|
84
|
+
...llmsOpts
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
return new Response(body, {
|
|
88
|
+
headers: {
|
|
89
|
+
"Content-Type": "text/plain; charset=utf-8",
|
|
90
|
+
"Cache-Control": "public, max-age=3600, s-maxage=86400"
|
|
91
|
+
}
|
|
92
|
+
})
|
|
93
|
+
}
|
package/src/routes/robots.txt.ts
CHANGED
|
@@ -1,63 +1,63 @@
|
|
|
1
|
-
import type { APIRoute } from "astro"
|
|
2
|
-
import { buildRobotsTxt } from "../robots.js"
|
|
3
|
-
import * as seoConfig from "virtual:rimelight-seo/config"
|
|
4
|
-
import * as siteConfigMod from "virtual:rimelight-seo/site-config"
|
|
5
|
-
import seoOptions from "virtual:rimelight-seo/options"
|
|
6
|
-
|
|
7
|
-
export const GET: APIRoute = ({ site, url }) => {
|
|
8
|
-
const siteUrl = site ? site.toString() :
|
|
9
|
-
const sitemapURL = new URL("sitemap.xml", siteUrl).toString()
|
|
10
|
-
|
|
11
|
-
const defaultPrivatePrefixes = [
|
|
12
|
-
"/dashboard",
|
|
13
|
-
"/admin",
|
|
14
|
-
"/cms",
|
|
15
|
-
"/internal",
|
|
16
|
-
"/api",
|
|
17
|
-
"/dev",
|
|
18
|
-
"/og",
|
|
19
|
-
"/open-graph",
|
|
20
|
-
"/auth"
|
|
21
|
-
]
|
|
22
|
-
|
|
23
|
-
const customPrivatePrefixes = Array.isArray(seoConfig.PRIVATE_PATH_PREFIXES)
|
|
24
|
-
? seoConfig.PRIVATE_PATH_PREFIXES
|
|
25
|
-
: []
|
|
26
|
-
|
|
27
|
-
const privatePrefixes = Array.from(new Set([...defaultPrivatePrefixes, ...customPrivatePrefixes]))
|
|
28
|
-
|
|
29
|
-
const disallowPatterns = [
|
|
30
|
-
...privatePrefixes,
|
|
31
|
-
...privatePrefixes.map((p) => `/*${p.startsWith("/") ? p : `/${p}`}/`)
|
|
32
|
-
]
|
|
33
|
-
|
|
34
|
-
const robotsOpts = typeof seoOptions?.robots === "object" ? seoOptions.robots : {}
|
|
35
|
-
|
|
36
|
-
const defaultAgentRules = [
|
|
37
|
-
{
|
|
38
|
-
userAgent: "GPTBot",
|
|
39
|
-
disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
|
|
40
|
-
allow: ["/"]
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
userAgent: "ClaudeBot",
|
|
44
|
-
disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
|
|
45
|
-
allow: ["/"]
|
|
46
|
-
}
|
|
47
|
-
]
|
|
48
|
-
|
|
49
|
-
const robots = buildRobotsTxt({
|
|
50
|
-
sitemapUrl: sitemapURL,
|
|
51
|
-
disallow: disallowPatterns,
|
|
52
|
-
allow: ["/"],
|
|
53
|
-
userAgentRules: defaultAgentRules,
|
|
54
|
-
...robotsOpts
|
|
55
|
-
})
|
|
56
|
-
|
|
57
|
-
return new Response(robots, {
|
|
58
|
-
headers: {
|
|
59
|
-
"Content-Type": "text/plain; charset=utf-8",
|
|
60
|
-
"Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
|
|
61
|
-
}
|
|
62
|
-
})
|
|
63
|
-
}
|
|
1
|
+
import type { APIRoute } from "astro"
|
|
2
|
+
import { buildRobotsTxt } from "../robots.js"
|
|
3
|
+
import * as seoConfig from "virtual:rimelight-seo/config"
|
|
4
|
+
import * as siteConfigMod from "virtual:rimelight-seo/site-config"
|
|
5
|
+
import seoOptions from "virtual:rimelight-seo/options"
|
|
6
|
+
|
|
7
|
+
export const GET: APIRoute = ({ site, url }) => {
|
|
8
|
+
const siteUrl = site ? site.toString() : siteConfigMod.siteConfig?.url || url.origin
|
|
9
|
+
const sitemapURL = new URL("sitemap.xml", siteUrl).toString()
|
|
10
|
+
|
|
11
|
+
const defaultPrivatePrefixes = [
|
|
12
|
+
"/dashboard",
|
|
13
|
+
"/admin",
|
|
14
|
+
"/cms",
|
|
15
|
+
"/internal",
|
|
16
|
+
"/api",
|
|
17
|
+
"/dev",
|
|
18
|
+
"/og",
|
|
19
|
+
"/open-graph",
|
|
20
|
+
"/auth"
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
const customPrivatePrefixes = Array.isArray(seoConfig.PRIVATE_PATH_PREFIXES)
|
|
24
|
+
? seoConfig.PRIVATE_PATH_PREFIXES
|
|
25
|
+
: []
|
|
26
|
+
|
|
27
|
+
const privatePrefixes = Array.from(new Set([...defaultPrivatePrefixes, ...customPrivatePrefixes]))
|
|
28
|
+
|
|
29
|
+
const disallowPatterns = [
|
|
30
|
+
...privatePrefixes,
|
|
31
|
+
...privatePrefixes.map((p) => `/*${p.startsWith("/") ? p : `/${p}`}/`)
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
const robotsOpts = typeof seoOptions?.robots === "object" ? seoOptions.robots : {}
|
|
35
|
+
|
|
36
|
+
const defaultAgentRules = [
|
|
37
|
+
{
|
|
38
|
+
userAgent: "GPTBot",
|
|
39
|
+
disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
|
|
40
|
+
allow: ["/"]
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
userAgent: "ClaudeBot",
|
|
44
|
+
disallow: privatePrefixes.map((p) => `${p.startsWith("/") ? p : `/${p}`}/`),
|
|
45
|
+
allow: ["/"]
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
const robots = buildRobotsTxt({
|
|
50
|
+
sitemapUrl: sitemapURL,
|
|
51
|
+
disallow: disallowPatterns,
|
|
52
|
+
allow: ["/"],
|
|
53
|
+
userAgentRules: defaultAgentRules,
|
|
54
|
+
...robotsOpts
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
return new Response(robots, {
|
|
58
|
+
headers: {
|
|
59
|
+
"Content-Type": "text/plain; charset=utf-8",
|
|
60
|
+
"Cache-Control": "public, max-age=3600, stale-while-revalidate=86400"
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
}
|