lightnet 4.6.3 → 4.7.1

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # lightnet
 
+## 4.7.1
+
+### Patch Changes
+
+- [#469](https://github.com/LightNetDev/LightNet/pull/469) [`a0961c4`](https://github.com/LightNetDev/LightNet/commit/a0961c40fdc30a93d3d5c814343f9c12ec26b2ef) - Warn about user-defined ln.* translation overrides that no longer match a built-in English key. Include source filenames and check unused keys without failing check-translations for stale overrides alone.
+
+- [#469](https://github.com/LightNetDev/LightNet/pull/469) [`a0961c4`](https://github.com/LightNetDev/LightNet/commit/a0961c40fdc30a93d3d5c814343f9c12ec26b2ef) - Only record translation keys requested during the build, so check-translations no longer reports missing translations for unused built-in or user keys.
+
+## 4.7.0
+
+### Minor Changes
+
+- [#465](https://github.com/LightNetDev/LightNet/pull/465) [`24f5b61`](https://github.com/LightNetDev/LightNet/commit/24f5b614c50ef355f595644b9b0763e00a2210cd) - Generate XML sitemaps automatically using Astro's sitemap integration and LightNet's site languages. Respect an explicitly configured `@astrojs/sitemap` integration for custom sitemap behavior.
+
+- [#465](https://github.com/LightNetDev/LightNet/pull/465) [`24f5b61`](https://github.com/LightNetDev/LightNet/commit/24f5b614c50ef355f595644b9b0763e00a2210cd) - Automatically generate robots.txt allowing crawling and linking to the sitemap index using Astro's site and base configuration. Preserve user-provided robots.txt files in the public directory and skip generation when search indexing is disabled.
+
 ## 4.6.3
 
 ### Patch Changes

package/package.json

@@ -3,7 +3,7 @@
   "description": "LightNet makes it easy to run your own digital media library.",
   "type": "module",
   "license": "MIT",
-  "version": "4.6.3",
+  "version": "4.7.1",
   "repository": {
     "type": "git",
     "url": "https://github.com/LightNetDev/lightnet",
@@ -42,7 +42,8 @@
     "./pages/SearchPageRoute.astro": "./src/pages/search-page/SearchPageRoute.astro",
     "./pages/DetailsPageRoute.astro": "./src/pages/details-page/DetailsPageRoute.astro",
     "./api/internal/search.ts": "./src/pages/search-page/api/search.ts",
-    "./api/versions.ts": "./src/api/versions.ts"
+    "./api/versions.ts": "./src/api/versions.ts",
+    "./api/robots.ts": "./src/api/robots.ts"
   },
   "peerDependencies": {
     "astro": "^6.0.0",
@@ -52,6 +53,7 @@
   },
   "dependencies": {
     "@astrojs/react": "^5.0.7",
+    "@astrojs/sitemap": "^3.7.4",
     "@iconify-json/lucide": "^1.2.145",
     "@iconify-json/mdi": "^1.2.3",
     "@iconify/tailwind": "^1.2.0",

package/src/api/robots.ts

@@ -0,0 +1,11 @@
+import type { APIRoute } from "astro"
+
+export const GET: APIRoute = ({ site }) => {
+  const base = new URL(import.meta.env.BASE_URL, site)
+  if (!base.pathname.endsWith("/")) base.pathname += "/"
+  const sitemap = new URL("sitemap-index.xml", base)
+
+  return new Response(`User-agent: *\nAllow: /\n\nSitemap: ${sitemap.href}\n`, {
+    headers: { "Content-Type": "text/plain; charset=utf-8" },
+  })
+}

package/src/astro-integration/integration.ts

@@ -1,10 +1,13 @@
 /// <reference path="../i18n/locals.d.ts" />
+import { existsSync } from "node:fs"
+
 import react from "@astrojs/react"
 import type { AstroIntegration } from "astro"
 import { AstroError } from "astro/errors"
 
 import { verifySchema } from "../utils/verify-schema"
 import { extendedConfigSchema, type LightnetConfig } from "./config"
+import { sitemap } from "./sitemap"
 import tailwind from "./tailwind"
 import { vitePluginLightnetConfig } from "./vite-plugin-lightnet-config"
 
@@ -33,6 +36,17 @@ export function lightnet(lightnetConfig: LightnetConfig): AstroIntegration {
           "Fix these errors on the LightNet configuration:",
         )
 
+        if (
+          !config.disallowSearchIndexing &&
+          !existsSync(new URL("robots.txt", astroConfig.publicDir))
+        ) {
+          injectRoute({
+            pattern: "/robots.txt",
+            entrypoint: "lightnet/api/robots.ts",
+            prerender: true,
+          })
+        }
+
         injectRoute({
           pattern: "404",
           entrypoint: "lightnet/pages/404Route.astro",
@@ -72,7 +86,7 @@ export function lightnet(lightnetConfig: LightnetConfig): AstroIntegration {
         addMiddleware({ entrypoint: "lightnet/locals", order: "pre" })
 
         updateConfig({
-          integrations: [tailwind(), react()],
+          integrations: [tailwind(), react(), ...sitemap(config, astroConfig)],
           vite: {
             plugins: [vitePluginLightnetConfig(config, astroConfig, logger)],
           },

package/src/astro-integration/sitemap.ts

@@ -0,0 +1,50 @@
+import astroSitemap from "@astrojs/sitemap"
+import type { AstroConfig, AstroIntegration } from "astro"
+
+import type { ExtendedLightnetConfig } from "./config"
+
+export const sitemap = (
+  lightnetConfig: ExtendedLightnetConfig,
+  astroConfig: AstroConfig,
+): AstroIntegration[] => {
+  if (
+    astroConfig.integrations.some(({ name }) => name === "@astrojs/sitemap")
+  ) {
+    return []
+  }
+
+  if (lightnetConfig.disallowSearchIndexing) {
+    return []
+  }
+
+  let adminRoutePatterns: RegExp[] = []
+
+  return [
+    {
+      name: "lightnet/sitemap-exclusions",
+      hooks: {
+        "astro:routes:resolved": ({ routes }) => {
+          // Astro resolves the package entrypoint to its source file, including
+          // workspace paths when the admin package is linked locally.
+          adminRoutePatterns = routes
+            .filter(({ entrypoint }) =>
+              /(?:^|\/)sveltia-admin\/src\/Admin\.astro$/.test(entrypoint),
+            )
+            .map(({ patternRegex }) => patternRegex)
+        },
+      },
+    },
+    astroSitemap({
+      filter: (page) =>
+        !adminRoutePatterns.some((pattern) =>
+          pattern.test(new URL(page).pathname),
+        ),
+      i18n: {
+        defaultLocale: lightnetConfig.defaultLocale,
+        locales: Object.fromEntries(
+          lightnetConfig.locales.map((locale) => [locale, locale]),
+        ),
+      },
+    }),
+  ]
+}

package/src/i18n/record-translation.ts

@@ -25,6 +25,20 @@ const recordedTranslations = new Set<string>()
 
 const translationStore = lazy(() => createTranslationStore())
 
+export async function writeStaleTranslationOverrides(
+  overrides: { key: string; path: string }[],
+) {
+  if (!import.meta.env.PROD) {
+    return
+  }
+  await mkdir(lightnetCachePath, { recursive: true })
+  await writeFile(
+    resolve(lightnetCachePath, "stale-translation-overrides.json"),
+    JSON.stringify(overrides),
+    "utf-8",
+  )
+}
+
 const writeLanguagesManifest = async () => {
   const manifestPath = resolve(lightnetCachePath, "languages.json")
   const { defaultLocale, locales } = config

package/src/i18n/translate.ts

@@ -68,7 +68,6 @@ export async function useTranslate(
   const resolvedLocale = bcp47 ?? config.defaultLocale
   const translations = await i18nextTranslations.get()
   const availableTranslationKeys = new Set(await translationKeys.get())
-  recordAllTranslations()
 
   const i18n = i18next.createInstance()
 
@@ -87,6 +86,19 @@ export async function useTranslate(
     "en",
   ]
   return (input, options) => {
+    if (import.meta.env.PROD) {
+      recordTranslation({
+        key: input,
+        type: input.startsWith("ln.") ? "lightnet" : "user",
+        values: Object.fromEntries(
+          config.locales.map((locale) => [
+            locale,
+            translations[locale]?.translation[input],
+          ]),
+        ),
+      })
+    }
+
     const t = i18n.getFixedT(resolvedLocale) as (
       key: TranslationKey,
       options?: TOptions,
@@ -101,39 +113,3 @@ export async function useTranslate(
     return value
   }
 }
-
-async function recordAllTranslations() {
-  if (!import.meta.env.PROD) {
-    // only record translations during build time
-    return
-  }
-
-  const { locales } = config
-  const localesIncludingEnglish = [...locales.filter((l) => l !== "en"), "en"]
-  const keys = await translationKeys.get()
-  const translations = await i18nextTranslations.get()
-
-  const collectValues = (key: string, locales: string[]) => {
-    const values = {} as Record<string, string | undefined>
-    for (const locale of locales) {
-      values[locale] = translations[locale]?.translation[key]
-    }
-    return values
-  }
-
-  for (const key of keys) {
-    if (key.startsWith("ln.")) {
-      recordTranslation({
-        key,
-        values: collectValues(key, localesIncludingEnglish),
-        type: "lightnet",
-      })
-    } else {
-      recordTranslation({
-        key,
-        values: collectValues(key, locales),
-        type: "user",
-      })
-    }
-  }
-}

package/src/i18n/translations.ts

@@ -1,6 +1,9 @@
 import { z } from "astro/zod"
 import YAML from "yaml"
 
+import { lazy } from "../utils/lazy"
+import { writeStaleTranslationOverrides } from "./record-translation"
+
 const builtInTranslations = {
   ar: () => import("./translations/ar.yml?raw"),
   bn: () => import("./translations/bn.yml?raw"),
@@ -22,24 +25,52 @@ type BuiltInLanguage = keyof typeof builtInTranslations
 
 const translationFileSchema = z.record(z.string(), z.string())
 
+const userTranslationFiles = import.meta.glob<string>(
+  ["/src/translations/*.(yml|yaml)"],
+  { query: "?raw", import: "default" },
+)
+
 const userTranslations = Object.fromEntries(
-  Object.entries(
-    import.meta.glob(["/src/translations/*.(yml|yaml)"], {
-      query: "?raw",
-      import: "default",
-    }),
-  ).map(([path, translationImport]) => {
+  Object.entries(userTranslationFiles).map(([path, translationImport]) => {
     const [fileName] = path.split("/").slice(-1)
     const lang = fileName.replace(/\.ya?ml/, "")
     return [lang, translationImport]
   }),
 )
 
-export const loadTranslations = async (bcp47: string) => ({
-  ...(await loadBuiltInTranslations(builtInTranslations, bcp47)),
-  ...(await loadUserTranslations(bcp47)),
+const staleOverrides = lazy(async () => {
+  const english = await loadBuiltInTranslations(builtInTranslations, "en")
+  await writeStaleTranslationOverrides(
+    await findStaleTranslationOverrides(english, userTranslationFiles),
+  )
 })
 
+export async function findStaleTranslationOverrides(
+  english: Record<string, string>,
+  files: Record<string, () => Promise<string>>,
+) {
+  const overrides: { key: string; path: string }[] = []
+  for (const [path, load] of Object.entries(files)) {
+    const translations = translationFileSchema.parse(YAML.parse(await load()))
+    for (const key of Object.keys(translations)) {
+      if (key.startsWith("ln.") && !Object.hasOwn(english, key)) {
+        overrides.push({ key, path: path.replace(/^\//, "") })
+      }
+    }
+  }
+  return overrides
+}
+
+export const loadTranslations = async (bcp47: string) => {
+  if (import.meta.env.PROD) {
+    await staleOverrides.get()
+  }
+  return {
+    ...(await loadBuiltInTranslations(builtInTranslations, bcp47)),
+    ...(await loadUserTranslations(bcp47)),
+  }
+}
+
 function hasTranslations(
   translationMap: Record<string, unknown>,
   bcp47: string,