lightnet 4.6.3 → 4.7.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # lightnet
 
+## 4.7.0
+
+### Minor Changes
+
+- [#465](https://github.com/LightNetDev/LightNet/pull/465) [`24f5b61`](https://github.com/LightNetDev/LightNet/commit/24f5b614c50ef355f595644b9b0763e00a2210cd) - Generate XML sitemaps automatically using Astro's sitemap integration and LightNet's site languages. Respect an explicitly configured `@astrojs/sitemap` integration for custom sitemap behavior.
+
+- [#465](https://github.com/LightNetDev/LightNet/pull/465) [`24f5b61`](https://github.com/LightNetDev/LightNet/commit/24f5b614c50ef355f595644b9b0763e00a2210cd) - Automatically generate robots.txt allowing crawling and linking to the sitemap index using Astro's site and base configuration. Preserve user-provided robots.txt files in the public directory and skip generation when search indexing is disabled.
+
 ## 4.6.3
 
 ### Patch Changes

package/package.json

@@ -3,7 +3,7 @@
   "description": "LightNet makes it easy to run your own digital media library.",
   "type": "module",
   "license": "MIT",
-  "version": "4.6.3",
+  "version": "4.7.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/LightNetDev/lightnet",
@@ -42,7 +42,8 @@
     "./pages/SearchPageRoute.astro": "./src/pages/search-page/SearchPageRoute.astro",
     "./pages/DetailsPageRoute.astro": "./src/pages/details-page/DetailsPageRoute.astro",
     "./api/internal/search.ts": "./src/pages/search-page/api/search.ts",
-    "./api/versions.ts": "./src/api/versions.ts"
+    "./api/versions.ts": "./src/api/versions.ts",
+    "./api/robots.ts": "./src/api/robots.ts"
   },
   "peerDependencies": {
     "astro": "^6.0.0",
@@ -52,6 +53,7 @@
   },
   "dependencies": {
     "@astrojs/react": "^5.0.7",
+    "@astrojs/sitemap": "^3.7.4",
     "@iconify-json/lucide": "^1.2.145",
     "@iconify-json/mdi": "^1.2.3",
     "@iconify/tailwind": "^1.2.0",

package/src/api/robots.ts

@@ -0,0 +1,11 @@
+import type { APIRoute } from "astro"
+
+export const GET: APIRoute = ({ site }) => {
+  const base = new URL(import.meta.env.BASE_URL, site)
+  if (!base.pathname.endsWith("/")) base.pathname += "/"
+  const sitemap = new URL("sitemap-index.xml", base)
+
+  return new Response(`User-agent: *\nAllow: /\n\nSitemap: ${sitemap.href}\n`, {
+    headers: { "Content-Type": "text/plain; charset=utf-8" },
+  })
+}

package/src/astro-integration/integration.ts

@@ -1,10 +1,13 @@
 /// <reference path="../i18n/locals.d.ts" />
+import { existsSync } from "node:fs"
+
 import react from "@astrojs/react"
 import type { AstroIntegration } from "astro"
 import { AstroError } from "astro/errors"
 
 import { verifySchema } from "../utils/verify-schema"
 import { extendedConfigSchema, type LightnetConfig } from "./config"
+import { sitemap } from "./sitemap"
 import tailwind from "./tailwind"
 import { vitePluginLightnetConfig } from "./vite-plugin-lightnet-config"
 
@@ -33,6 +36,17 @@ export function lightnet(lightnetConfig: LightnetConfig): AstroIntegration {
           "Fix these errors on the LightNet configuration:",
         )
 
+        if (
+          !config.disallowSearchIndexing &&
+          !existsSync(new URL("robots.txt", astroConfig.publicDir))
+        ) {
+          injectRoute({
+            pattern: "/robots.txt",
+            entrypoint: "lightnet/api/robots.ts",
+            prerender: true,
+          })
+        }
+
         injectRoute({
           pattern: "404",
           entrypoint: "lightnet/pages/404Route.astro",
@@ -72,7 +86,7 @@ export function lightnet(lightnetConfig: LightnetConfig): AstroIntegration {
         addMiddleware({ entrypoint: "lightnet/locals", order: "pre" })
 
         updateConfig({
-          integrations: [tailwind(), react()],
+          integrations: [tailwind(), react(), ...sitemap(config, astroConfig)],
           vite: {
             plugins: [vitePluginLightnetConfig(config, astroConfig, logger)],
           },

package/src/astro-integration/sitemap.ts

@@ -0,0 +1,50 @@
+import astroSitemap from "@astrojs/sitemap"
+import type { AstroConfig, AstroIntegration } from "astro"
+
+import type { ExtendedLightnetConfig } from "./config"
+
+export const sitemap = (
+  lightnetConfig: ExtendedLightnetConfig,
+  astroConfig: AstroConfig,
+): AstroIntegration[] => {
+  if (
+    astroConfig.integrations.some(({ name }) => name === "@astrojs/sitemap")
+  ) {
+    return []
+  }
+
+  if (lightnetConfig.disallowSearchIndexing) {
+    return []
+  }
+
+  let adminRoutePatterns: RegExp[] = []
+
+  return [
+    {
+      name: "lightnet/sitemap-exclusions",
+      hooks: {
+        "astro:routes:resolved": ({ routes }) => {
+          // Astro resolves the package entrypoint to its source file, including
+          // workspace paths when the admin package is linked locally.
+          adminRoutePatterns = routes
+            .filter(({ entrypoint }) =>
+              /(?:^|\/)sveltia-admin\/src\/Admin\.astro$/.test(entrypoint),
+            )
+            .map(({ patternRegex }) => patternRegex)
+        },
+      },
+    },
+    astroSitemap({
+      filter: (page) =>
+        !adminRoutePatterns.some((pattern) =>
+          pattern.test(new URL(page).pathname),
+        ),
+      i18n: {
+        defaultLocale: lightnetConfig.defaultLocale,
+        locales: Object.fromEntries(
+          lightnetConfig.locales.map((locale) => [locale, locale]),
+        ),
+      },
+    }),
+  ]
+}