@djangocfg/nextjs 2.1.411 → 2.1.413

package/src/sitemap/README.md

@@ -0,0 +1,315 @@
+# @djangocfg/nextjs/sitemap
+
+Frontend half of the djangocfg ↔ Next.js sitemap integration.
+
+The Django-side backend (`django_cfg.modules.django_sitemap`) exposes a
+paginated JSON feed. This subpackage turns that feed into a Google-compliant
+XML sitemap-index via Next.js's native `generateSitemaps()` API.
+
+## Why this exists
+
+Two naive approaches fail at scale:
+
+1. **Django serves XML directly** — wrong host (api.example.com vs
+   www.example.com), can't include frontend-only routes, hard to add
+   per-locale alternates, awkward sitemap-index dance.
+2. **Frontend queries the DB** — frontends don't have DB access; falling
+   back to public REST endpoints means N round trips and no streaming.
+
+This package solves both. Django returns path-only JSON; the frontend
+joins the host and emits sitemap-index XML. Same backend can serve
+multiple frontends (preview, staging, EU domain) — they each bring their
+own host.
+
+## The big picture
+
+```
+┌──────────────────────────────────────────────────────────────────────┐
+│ Django app (e.g. apps/catalog)                                       │
+│   sitemap_sources.py                                                 │
+│     register(SitemapSource(                                          │
+│       name="vehicles",                                               │
+│       queryset_factory=lambda: Vehicle.objects.sitemap_eligible(),   │
+│       url_template="/catalog/{id}",                                  │
+│       lastmod_field="last_seen_at",                                  │
+│       cursor_fields=("last_seen_at", "id"),                          │
+│       order="-last_seen_at,-id",                                     │
+│       page_size=50_000,                                              │
+│     ))                                                               │
+└──────────────────────────┬───────────────────────────────────────────┘
+                           │ AppConfig.ready() → in-process registry
+                           ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│ django_cfg.modules.django_sitemap (HTTP layer)                       │
+│                                                                      │
+│   GET /cfg/sitemap/index/                                            │
+│     → list of chunks per source, with opaque keyset cursors          │
+│                                                                      │
+│   GET /cfg/sitemap/feed/?source=<name>&cursor=<opaque>               │
+│     → 50 000 entries: { loc, lastmod }                               │
+│                                                                      │
+│   Cache-Control: public, s-maxage=3600, stale-while-revalidate=86400 │
+└──────────────────────────┬───────────────────────────────────────────┘
+                           │ HTTP (server-side fetch from Next)
+                           ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│ Next.js app — apps/<app>/app/                                        │
+│                                                                      │
+│   sitemap.ts                                                         │
+│     createDjangoSitemap({ host, apiUrl, staticRoutes })              │
+│     ├ generateSitemaps()  — reads /cfg/sitemap/index/, returns ids   │
+│     └ default sitemap()   — fetches one /cfg/sitemap/feed/ per id    │
+│                                                                      │
+│   robots.ts                                                          │
+│     createRobots({ host, disallow })                                 │
+└──────────────────────────────────────────────────────────────────────┘
+```
+
+Next emits:
+
+- `/sitemap.xml` — auto-generated `<sitemapindex>` listing every chunk.
+- `/sitemap/<id>.xml` — one `<urlset>` per chunk (50k URLs max, per
+  Google's cap).
+- `/robots.txt` — points crawlers at `/sitemap.xml`.
+
+## Quick start
+
+### 1. Register sources in Django
+
+```python
+# apps/<your-app>/sitemap_sources.py
+from django_cfg.modules.django_sitemap import SitemapSource, register
+from apps.catalog.models import Vehicle, Brand
+
+register(SitemapSource(
+    name="vehicles",
+    url_template="/catalog/{id}",
+    queryset_factory=lambda: Vehicle.objects.get_queryset().sitemap_eligible(),
+    fields={"id": "id"},
+    lastmod_field="last_seen_at",
+    cursor_fields=("last_seen_at", "id"),
+    order="-last_seen_at,-id",
+    page_size=50_000,
+))
+
+register(SitemapSource(
+    name="brands",
+    url_template="/catalog?brand={slug}",
+    queryset_factory=lambda: Brand.objects.filter(is_active=True),
+    fields={"slug": "slug"},
+    lastmod_field="updated_at",
+    cursor_fields=("slug",),
+    order="slug",
+))
+```
+
+`SitemapAppConfig.ready()` autoloads every installed app's
+`sitemap_sources.py` — no host-side wiring needed. Verify with
+`python manage.py sitemap_inspect`.
+
+### 2. Wire up the frontend
+
+```tsx
+// apps/<your-app>/app/sitemap.ts
+import { createDjangoSitemap } from '@djangocfg/nextjs/sitemap';
+
+const { generateSitemaps, sitemap } = createDjangoSitemap({
+  host: process.env.NEXT_PUBLIC_SITE_URL!,
+  apiUrl: process.env.NEXT_PUBLIC_API_URL!,
+  staticRoutes: [
+    { path: '/', changeFrequency: 'daily', priority: 1.0 },
+    { path: '/catalog', changeFrequency: 'daily', priority: 0.9 },
+    { path: '/ai-search', changeFrequency: 'weekly', priority: 0.7 },
+  ],
+});
+
+export { generateSitemaps, sitemap as default };
+export const revalidate = 3600;
+```
+
+```tsx
+// apps/<your-app>/app/robots.ts
+import { createRobots } from '@djangocfg/nextjs/sitemap';
+
+export default createRobots({
+  host: process.env.NEXT_PUBLIC_SITE_URL!,
+  disallow: ['/account/', '/auth', '/api/', '/apix/'],
+});
+```
+
+That's the whole app-side integration.
+
+## Backend without backend
+
+Marketing sites and documentation portals usually don't have a Django
+catalog to paginate. Omit `apiUrl` — the frontend then emits a
+single-chunk sitemap from `staticRoutes` only.
+
+```tsx
+const { generateSitemaps, sitemap } = createDjangoSitemap({
+  host: 'https://example.com',
+  staticRoutes: [
+    { path: '/', priority: 1.0 },
+    { path: '/about', priority: 0.8 },
+    { path: '/pricing', priority: 0.9 },
+  ],
+});
+```
+
+## What gets emitted
+
+For a 1 200 000-vehicle catalog + 6 static pages:
+
+- `/sitemap.xml` — index with ~25 child entries (24 vehicle chunks +
+  brands + models + static).
+- `/sitemap/static.xml` — 6 static URLs with `<changefreq>`/`<priority>`.
+- `/sitemap/vehicles--<cursor>.xml` × 24 — 50 000 vehicle URLs each
+  (sorted by `last_seen_at DESC` so the freshest listings live in chunk 1
+  and Googlebot re-crawls them first).
+- `/sitemap/brands--.xml` — one chunk.
+- `/sitemap/models--.xml` — one chunk.
+
+**Only `loc` and `lastmod`** are emitted from backend entries: Google
+ignores `<changefreq>` and `<priority>` on dynamic URLs, and omitting
+them keeps the payload small. Static routes still emit them — non-Google
+crawlers (Bing, Yandex) read those fields, and they're free.
+
+## Cache layers
+
+Three layers cooperate; each TTL aligned with how often the data
+*actually* changes.
+
+| Layer | Default TTL | Tuned via |
+|---|---|---|
+| Django Redis (sitemap-index JSON) | 5 min | `SitemapConfig.cache_index_seconds` |
+| Django Redis (per-chunk JSON) | 1 h | `SitemapConfig.cache_feed_seconds` |
+| Next.js ISR (XML) | 1 h | `createDjangoSitemap({ feedRevalidate })` + `export const revalidate = ...` |
+| HTTP `Cache-Control` | `s-maxage=3600, swr=86400` | Django sets, CDN respects |
+
+Result: Googlebot fetching `/sitemap.xml` rarely touches Django — most
+hits land on the CDN edge or Next ISR. Django only does real work once
+per TTL.
+
+To force a cache bust without `FLUSHDB`, bump `SitemapConfig.cache_version`
+on the Django side; every Redis key includes it.
+
+## Failure mode
+
+`fetchSitemapIndex` and `fetchSitemapFeed` **never throw**. A transient
+Django outage during `next build` is downgraded to:
+
+- empty index → only the static chunk is emitted
+- empty feed → that chunk renders an empty `<urlset/>`
+
+`next build` completes, the bad cache entry expires on the next ISR
+cycle, and the sitemap heals automatically. No build-time coupling to
+Django uptime.
+
+## Deployment
+
+### Env vars
+
+```
+NEXT_PUBLIC_SITE_URL=https://example.com         # canonical host
+NEXT_PUBLIC_API_URL=https://api.example.com      # Django backend
+```
+
+Both are public — sitemap.ts runs on the server but the same `host` is
+also embedded in client-side metadata.
+
+### Docker
+
+Nothing sitemap-specific is required in the container — the package is a
+plain TypeScript dependency, no native modules, no extra ports. Just pass
+the two env vars above via `env_file` or `environment:`.
+
+The Next service's existing healthcheck on `/` is enough; no separate
+sitemap probe needed (a missing sitemap doesn't break the app).
+
+### CDN / Cloudflare
+
+Cache by URL, respect origin `Cache-Control`:
+
+| URL pattern | TTL fresh | TTL stale |
+|---|---|---|
+| `/sitemap.xml` | 1 h | 1 day |
+| `/sitemap/*.xml` | 1 h | 1 day |
+| `/robots.txt` | 24 h | 1 day |
+
+Stale-while-revalidate keeps Googlebot fast even during a Django outage.
+
+### Sitemap submission
+
+Submit `https://<host>/sitemap.xml` once via Google Search Console.
+Google polls it on its own schedule afterwards — no `ping` integration
+needed.
+
+## API reference
+
+### `createDjangoSitemap(options)`
+
+Returns `{ generateSitemaps, sitemap }`. Export them as Next.js
+expects: named `generateSitemaps` + default-exported `sitemap`.
+
+**Options:**
+
+| Field | Type | Default | Notes |
+|---|---|---|---|
+| `host` | `string` | (required) | Canonical host (e.g. `https://example.com`). URLs are joined as `${host}${loc}`. |
+| `apiUrl` | `string` | `undefined` | Django backend base URL. Omit to skip the backend feed entirely. |
+| `staticRoutes` | `StaticRoute[]` | `[]` | Frontend-only routes. Auth/account routes should NOT be listed here. |
+| `indexRevalidate` | `number` | `600` | Seconds — `next: { revalidate }` on the index fetch. |
+| `feedRevalidate` | `number` | `3600` | Seconds — `next: { revalidate }` on each chunk fetch. |
+
+**`StaticRoute`:**
+
+```ts
+interface StaticRoute {
+  path: string;
+  changeFrequency?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+  priority?: number;  // 0.0 – 1.0
+}
+```
+
+### `createRobots(options)`
+
+Returns a Next.js `() => MetadataRoute.Robots` function. Default-export
+it from `app/robots.ts`.
+
+**Options:**
+
+| Field | Type | Default | Notes |
+|---|---|---|---|
+| `host` | `string` | (required) | Canonical host. |
+| `disallow` | `string[]` | `['/account/', '/auth', '/api/']` | Path prefixes to block. |
+| `sitemap` | `string` | `${host}/sitemap.xml` | Override the sitemap pointer. |
+
+### Lower-level utilities
+
+Exposed for advanced use cases (custom Next route handlers, debugging):
+
+```ts
+import {
+  fetchSitemapIndex,
+  fetchSitemapFeed,
+  encodeChunkId,
+  decodeChunkId,
+} from '@djangocfg/nextjs/sitemap';
+```
+
+## When NOT to use this
+
+- Sites under ~1 000 URLs with no backend pagination needs — a flat
+  `staticRoutes`-only configuration is fine, but a hand-written
+  `sitemap.ts` returning a literal array works just as well.
+- You need per-locale `<xhtml:link rel="alternate">` tags — not currently
+  emitted; the wire contract has room for it (`alternates`) but no
+  consumer requires it yet.
+- You need image/video/news sitemap extensions — not implemented.
+
+## See also
+
+- `django_cfg.modules.django_sitemap` — backend module (registry, HTTP
+  views, `sitemap_inspect` management command).
+- [Next.js docs — generateSitemaps](https://nextjs.org/docs/app/api-reference/functions/generate-sitemaps)
+- [Google — large sitemaps](https://developers.google.com/search/docs/crawling-indexing/sitemaps/large-sitemaps)

package/src/sitemap/fetch.ts

@@ -0,0 +1,66 @@
+/**
+ * Sitemap fetch helpers — never throw.
+ *
+ * Failures are converted to empty payloads so a transient Django outage
+ * doesn't break `next build`. The next ISR cycle (`revalidate`) recovers
+ * automatically.
+ */
+
+import type { SitemapFeedPage, SitemapIndex } from './types';
+
+const EMPTY_INDEX: SitemapIndex = {
+  sources: [],
+  generated_at: new Date(0).toISOString(),
+  ttl_seconds: 60,
+};
+
+const emptyFeed = (source: string, cursor: string | null): SitemapFeedPage => ({
+  source,
+  chunk_id: `${source}-empty`,
+  count: 0,
+  has_more: false,
+  next_cursor: cursor,
+  entries: [],
+});
+
+export async function fetchSitemapIndex(
+  apiUrl: string,
+  revalidate: number,
+): Promise<SitemapIndex> {
+  try {
+    const r = await fetch(`${apiUrl}/cfg/sitemap/index/`, {
+      next: { revalidate },
+    });
+    if (!r.ok) {
+      console.warn(`[sitemap] index fetch returned ${r.status}; falling back to empty`);
+      return EMPTY_INDEX;
+    }
+    return (await r.json()) as SitemapIndex;
+  } catch (err) {
+    console.warn('[sitemap] index fetch failed:', err);
+    return EMPTY_INDEX;
+  }
+}
+
+export async function fetchSitemapFeed(
+  apiUrl: string,
+  source: string,
+  cursor: string | null,
+  revalidate: number,
+): Promise<SitemapFeedPage> {
+  const params = new URLSearchParams({ source });
+  if (cursor) params.set('cursor', cursor);
+  try {
+    const r = await fetch(`${apiUrl}/cfg/sitemap/feed/?${params}`, {
+      next: { revalidate },
+    });
+    if (!r.ok) {
+      console.warn(`[sitemap] feed ${source} returned ${r.status}; falling back to empty`);
+      return emptyFeed(source, cursor);
+    }
+    return (await r.json()) as SitemapFeedPage;
+  } catch (err) {
+    console.warn(`[sitemap] feed ${source} fetch failed:`, err);
+    return emptyFeed(source, cursor);
+  }
+}

package/src/sitemap/ids.ts

@@ -0,0 +1,21 @@
+/**
+ * Sitemap chunk id encoding.
+ *
+ * Each Next.js sitemap file is addressed by an opaque string id. We
+ * round-trip a `(source, cursor)` pair through that id so the default
+ * sitemap handler knows which Django chunk to fetch.
+ *
+ * Separator `--` is unambiguous: backend cursors use URL-safe base64 (`-_`).
+ */
+
+export function encodeChunkId(source: string, cursor: string | null): string {
+  return `${source}--${cursor ?? ''}`;
+}
+
+export function decodeChunkId(id: string): { source: string; cursor: string | null } {
+  const sep = id.indexOf('--');
+  if (sep === -1) return { source: id, cursor: null };
+  const source = id.slice(0, sep);
+  const cursorRaw = id.slice(sep + 2);
+  return { source, cursor: cursorRaw === '' ? null : cursorRaw };
+}

package/src/sitemap/index.ts

@@ -1,8 +1,26 @@
 /**
- * Sitemap exports
+ * @djangocfg/nextjs/sitemap
+ *
+ * Sitemap-index + per-chunk Next.js sitemap factory backed by
+ * `django_cfg.modules.django_sitemap`. Handles catalogs up to Google's
+ * per-file URL caps without materialising data in the frontend.
+ *
+ * See `createDjangoSitemap` for the main entry point and `createRobots`
+ * for the matching robots.txt factory.
  */
 
-export { createSitemapHandler } from './route';
-export { generateSitemapXml, normalizeUrl } from './generator';
-export type { SitemapGeneratorOptions, SitemapRoute, SitemapI18nOptions } from './types';
+export { createDjangoSitemap } from './sitemap';
+export { createRobots } from './robots';
+export { encodeChunkId, decodeChunkId } from './ids';
+export { fetchSitemapIndex, fetchSitemapFeed } from './fetch';
 
+export type {
+  SitemapChunkInfo,
+  SitemapEntry,
+  SitemapFeedPage,
+  SitemapIndex,
+  SitemapSourceInfo,
+  CreateDjangoSitemapOptions,
+  CreateRobotsOptions,
+  StaticRoute,
+} from './types';

package/src/sitemap/robots.ts

@@ -0,0 +1,34 @@
+/**
+ * createRobots — factory for Next.js `app/robots.ts`.
+ *
+ * Emits a single `User-Agent: *` block plus a Sitemap pointer. Disallow
+ * defaults cover the common cases (auth, account, API surface) — override
+ * per app if needed.
+ *
+ * Usage:
+ *
+ * ```ts
+ * // apps/<app>/app/robots.ts
+ * import { createRobots } from '@djangocfg/nextjs/sitemap';
+ *
+ * export default createRobots({
+ *   host: process.env.NEXT_PUBLIC_SITE_URL!,
+ *   disallow: ['/account/', '/auth', '/api/', '/apix/'],
+ * });
+ * ```
+ */
+
+import type { MetadataRoute } from 'next';
+
+import type { CreateRobotsOptions } from './types';
+
+const DEFAULT_DISALLOW = ['/account/', '/auth', '/api/'];
+
+export function createRobots(opts: CreateRobotsOptions): () => MetadataRoute.Robots {
+  const { host, disallow = DEFAULT_DISALLOW, sitemap } = opts;
+  return () => ({
+    rules: [{ userAgent: '*', allow: '/', disallow }],
+    sitemap: sitemap ?? `${host}/sitemap.xml`,
+    host,
+  });
+}

package/src/sitemap/sitemap.ts

@@ -0,0 +1,94 @@
+/**
+ * createDjangoSitemap — factory for Next.js `app/sitemap.ts`.
+ *
+ * Backed by `django_cfg.modules.django_sitemap`: the backend exposes a
+ * paginated JSON feed, this factory turns each chunk into a Next.js
+ * sitemap file via `generateSitemaps()`.
+ *
+ * Usage:
+ *
+ * ```ts
+ * // apps/<app>/app/sitemap.ts
+ * import { createDjangoSitemap } from '@djangocfg/nextjs/sitemap';
+ *
+ * const { generateSitemaps, sitemap } = createDjangoSitemap({
+ *   host: process.env.NEXT_PUBLIC_SITE_URL!,
+ *   apiUrl: process.env.NEXT_PUBLIC_API_URL!,
+ *   staticRoutes: [
+ *     { path: '/', changeFrequency: 'daily', priority: 1.0 },
+ *     { path: '/catalog', changeFrequency: 'daily', priority: 0.9 },
+ *   ],
+ * });
+ *
+ * export { generateSitemaps, sitemap as default };
+ * export const revalidate = 3600;
+ * ```
+ *
+ * Without `apiUrl`, the sitemap emits only the static routes — handy for
+ * marketing sites that don't have a backend-driven URL space.
+ */
+
+import type { MetadataRoute } from 'next';
+
+import { fetchSitemapFeed, fetchSitemapIndex } from './fetch';
+import { decodeChunkId, encodeChunkId } from './ids';
+
+import type { CreateDjangoSitemapOptions, StaticRoute } from './types';
+
+const STATIC_ID = 'static';
+const DEFAULT_INDEX_REVALIDATE = 600;
+const DEFAULT_FEED_REVALIDATE = 3600;
+
+interface SitemapApi {
+  generateSitemaps: () => Promise<Array<{ id: string }>>;
+  sitemap: (props: { id: Promise<string> }) => Promise<MetadataRoute.Sitemap>;
+}
+
+export function createDjangoSitemap(opts: CreateDjangoSitemapOptions): SitemapApi {
+  const {
+    host,
+    apiUrl,
+    staticRoutes = [],
+    indexRevalidate = DEFAULT_INDEX_REVALIDATE,
+    feedRevalidate = DEFAULT_FEED_REVALIDATE,
+  } = opts;
+
+  return {
+    async generateSitemaps() {
+      const ids: Array<{ id: string }> = [{ id: STATIC_ID }];
+      if (!apiUrl) return ids;
+      const index = await fetchSitemapIndex(apiUrl, indexRevalidate);
+      for (const s of index.sources) {
+        for (const c of s.chunks) {
+          ids.push({ id: encodeChunkId(s.name, c.cursor_to) });
+        }
+      }
+      return ids;
+    },
+
+    async sitemap({ id: idPromise }) {
+      const id = await idPromise;
+
+      if (id === STATIC_ID) {
+        return renderStatic(host, staticRoutes);
+      }
+
+      if (!apiUrl) return [];
+
+      const { source, cursor } = decodeChunkId(id);
+      const feed = await fetchSitemapFeed(apiUrl, source, cursor, feedRevalidate);
+      return feed.entries.map((e) => ({
+        url: `${host}${e.loc}`,
+        lastModified: e.lastmod ? new Date(e.lastmod) : undefined,
+      }));
+    },
+  };
+}
+
+function renderStatic(host: string, routes: StaticRoute[]): MetadataRoute.Sitemap {
+  return routes.map((r) => ({
+    url: `${host}${r.path}`,
+    changeFrequency: r.changeFrequency,
+    priority: r.priority,
+  }));
+}

package/src/sitemap/types.ts

@@ -1,29 +1,79 @@
 /**
- * Sitemap types
+ * Sitemap module — types
+ *
+ * Wire-contract for `django_cfg.modules.django_sitemap` JSON endpoints
+ * plus the public option types for `createDjangoSitemap` and
+ * `createRobots`.
  */
 
-import type { ChangeFreq, SitemapUrl } from '../types';
+import type { ChangeFreq } from '../types';
 
-export interface SitemapI18nOptions {
-  /** Supported locales (e.g., ['en', 'ru', 'ko']) */
-  locales: string[];
-  /** Default locale for x-default hreflang */
-  defaultLocale: string;
+// ── Wire contract with Django backend ───────────────────────────────────
+
+export interface SitemapChunkInfo {
+  id: string;
+  cursor_to: string | null;
+  count_estimate: number;
+}
+
+export interface SitemapSourceInfo {
+  name: string;
+  chunks: SitemapChunkInfo[];
+  total_estimate: number;
+}
+
+export interface SitemapIndex {
+  sources: SitemapSourceInfo[];
+  generated_at: string;
+  ttl_seconds: number;
+}
+
+export interface SitemapEntry {
+  loc: string;
+  lastmod?: string | null;
 }
 
-export interface SitemapGeneratorOptions {
-  siteUrl: string;
-  staticPages?: SitemapUrl[];
-  dynamicPages?: (() => Promise<SitemapUrl[]>) | SitemapUrl[];
-  cacheControl?: string;
-  /** i18n configuration for hreflang support */
-  i18n?: SitemapI18nOptions;
+export interface SitemapFeedPage {
+  source: string;
+  chunk_id: string;
+  count: number;
+  has_more: boolean;
+  next_cursor: string | null;
+  entries: SitemapEntry[];
 }
 
-export interface SitemapRoute {
+// ── Public option types ─────────────────────────────────────────────────
+
+export interface StaticRoute {
   path: string;
-  lastmod?: string | Date;
-  changefreq?: ChangeFreq;
+  changeFrequency?: ChangeFreq;
   priority?: number;
 }
 
+export interface CreateDjangoSitemapOptions {
+  /** Canonical site host, e.g. `https://vamcar.com`. URLs in the sitemap
+   *  are joined as `${host}${loc}`. */
+  host: string;
+
+  /** Django backend base URL. When omitted, backend chunks are skipped
+   *  and only `staticRoutes` are emitted (single-chunk sitemap). */
+  apiUrl?: string;
+
+  /** Frontend-only routes the backend doesn't know about. Auth/account
+   *  pages should NOT be listed here. */
+  staticRoutes?: StaticRoute[];
+
+  /** Index revalidate window in seconds. Default: 600. */
+  indexRevalidate?: number;
+
+  /** Feed (chunk) revalidate window in seconds. Default: 3600. */
+  feedRevalidate?: number;
+}
+
+export interface CreateRobotsOptions {
+  host: string;
+  /** Disallow patterns. Default: ['/account/', '/auth', '/api/']. */
+  disallow?: string[];
+  /** Override sitemap URL. Default: `${host}/sitemap.xml`. */
+  sitemap?: string;
+}