create-nextblock 0.15.9 → 0.16.1

package/templates/nextblock-template/lib/seo/robots-txt.ts

@@ -0,0 +1,510 @@
+/**
+ * Renders the operator's stored robots settings into the two shapes the app needs:
+ * the `MetadataRoute.Robots` object Next.js serialises for `app/robots.ts`, and the
+ * literal robots.txt text the admin preview shows, because somebody has to be able to
+ * read the file they are about to publish.
+ *
+ * Those two are not siblings, and that is the whole point of this module. The
+ * metadata object is the single source of truth; the text is produced from it by
+ * {@link renderRobotsMetadata}, a line-for-line port of the serialiser Next actually
+ * runs — `resolveRobots` in
+ * `next/dist/build/webpack/loaders/metadata/resolve-route-data`, which the route
+ * module Next generates for a dynamic `robots.ts` calls as
+ * `resolveRouteData(await handler(), 'robots')`. The preview is therefore the file,
+ * byte for byte, and not a second opinion about it.
+ *
+ * It used to be a second opinion, and that was a bug worth recording. `buildRobotsTxt`
+ * appended `settings.customRules` verbatim while `buildRobotsMetadata` could only
+ * re-express the standard directives it managed to parse, so every other custom line
+ * was shown to the operator as live and never reached a crawler. The wrong assumption
+ * behind that split was that `MetadataRoute.Robots` has no escape hatch for arbitrary
+ * lines. It has one: each rule's `other` record, which Next emits verbatim as
+ * `key: value`. Custom lines now genuinely reach the served file, and the two kinds
+ * that still cannot be represented — comments, and lines with no `:` at all, neither
+ * of which any crawler acts on — are reported by {@link listUnservedCustomRuleLines}
+ * so the SEO screen can say so, rather than being displayed as though they were live.
+ *
+ * Nothing here throws or reads the environment. Callers hand in `isSandbox` and the
+ * sitemap URL, which keeps the module pure enough to test and — more usefully —
+ * means the admin preview can render "what production would serve" rather than
+ * "what this dev machine would serve".
+ */
+
+import type { MetadataRoute } from 'next';
+import type { RobotsSettings } from '@nextblock-cms/utils/seo';
+
+export interface RobotsTxtOptions {
+  /**
+   * True on a disposable sandbox deployment. This overrides every stored setting,
+   * so it is passed in rather than read from `process.env` here — the admin preview
+   * must be able to show the production answer, and the sandbox's demo admin login
+   * must not be able to change what the sandbox serves.
+   */
+  isSandbox: boolean;
+  /**
+   * Absolute URL of the sitemap, or null when there is nothing to advertise. Still
+   * subject to `settings.sitemapEnabled`: a null (or blank) URL means "we could not
+   * build one", the setting means "the operator does not want one".
+   */
+  sitemapUrl: string | null;
+}
+
+/**
+ * One entry of `MetadataRoute.Robots['rules']`, derived from Next's own type rather
+ * than restated, so a future change to the metadata contract surfaces here as a
+ * compile error instead of as a silently ignored field.
+ */
+type RobotsRuleList = Extract<MetadataRoute.Robots['rules'], readonly unknown[]>;
+type RobotsRule = RobotsRuleList[number];
+
+/**
+ * The sandbox file, and the reasoning that has to travel with it.
+ *
+ * Sandbox is a copy of production, so it must stay out of the search index. The
+ * counter-intuitive part is that we ALLOW crawling in order to achieve that. Every
+ * sandbox response already carries `noindex` — the `X-Robots-Tag` header from
+ * next.config.js plus the robots meta in app/layout.tsx — and Googlebot can only
+ * obey a noindex it is allowed to fetch. `Disallow: /` would block the fetch, so
+ * Google would never read the noindex and could leave URL-only entries in the index
+ * forever: the exact opposite of what we want. The Sitemap line is omitted because
+ * the sandbox sitemap is empty by design (see app/sitemap.ts).
+ *
+ * This branch ignores the stored settings wholesale, deliberately. A sandbox visitor
+ * signs in with a shared demo admin account, and "the demo admin flipped indexing
+ * off" must not be able to turn the sandbox into a permanently half-indexed ghost.
+ */
+const SANDBOX_USER_AGENT_RULE = { allow: ['/'], disallow: [], userAgent: '*' } as const;
+
+/** The file served when an operator switches indexing off for the whole site. */
+const INDEXING_DISABLED_USER_AGENT_RULE = { allow: [], disallow: ['/'], userAgent: '*' } as const;
+
+/** What a site with no usable configuration gets: fully crawlable. */
+const PERMISSIVE_USER_AGENT_RULE = { allow: ['/'], disallow: [], userAgent: '*' } as const;
+
+interface ResolvedUserAgentRule {
+  allow: string[];
+  disallow: string[];
+  userAgent: string;
+}
+
+/**
+ * The single decision layer both serialisers run through.
+ *
+ * Returning the resolved groups rather than a formatted string is what makes the two
+ * public functions incapable of disagreeing about who may crawl what: they differ
+ * only in how they render this answer, never in what it is.
+ */
+function resolveUserAgentRules(
+  settings: RobotsSettings,
+  options: RobotsTxtOptions,
+): ResolvedUserAgentRule[] {
+  if (options.isSandbox) {
+    return [{ ...SANDBOX_USER_AGENT_RULE, allow: [...SANDBOX_USER_AGENT_RULE.allow], disallow: [] }];
+  }
+
+  if (!settings.isIndexingEnabled) {
+    return [
+      {
+        ...INDEXING_DISABLED_USER_AGENT_RULE,
+        allow: [],
+        disallow: [...INDEXING_DISABLED_USER_AGENT_RULE.disallow],
+      },
+    ];
+  }
+
+  const rules = settings.userAgentRules.map((rule) => ({
+    allow: [...rule.allow],
+    disallow: [...rule.disallow],
+    userAgent: rule.userAgent,
+  }));
+
+  // A robots.txt with no User-agent group is not a stricter file, it is a
+  // meaningless one, and crawlers disagree about how to read one — so a
+  // configuration that reduced to nothing falls back to the permissive wildcard
+  // rather than to an empty file.
+  return rules.length > 0
+    ? rules
+    : [{ ...PERMISSIVE_USER_AGENT_RULE, allow: [...PERMISSIVE_USER_AGENT_RULE.allow], disallow: [] }];
+}
+
+/**
+ * The operator's free-form lines, normalised for parsing.
+ *
+ * CRLF is folded to LF because the textarea these come from submits whatever line
+ * endings the operator's browser produced, and a Windows-authored block would
+ * otherwise put stray carriage returns into a file that is parsed line by line.
+ * Surrounding whitespace is trimmed so a block that is nothing but blank lines
+ * reduces to the empty string.
+ */
+function normalizeCustomRules(customRules: string): string {
+  if (typeof customRules !== 'string') {
+    return '';
+  }
+
+  return customRules.replace(/\r\n/g, '\n').trim();
+}
+
+/**
+ * The sitemap line's URL, or null when there is nothing worth advertising.
+ *
+ * A whitespace-only URL is treated as absent rather than emitted, because
+ * `Sitemap:` with a blank value is a line a crawler will try to fetch and fail on.
+ */
+function resolveSitemapUrl(settings: RobotsSettings, options: RobotsTxtOptions): string | null {
+  if (options.isSandbox || !settings.isIndexingEnabled || !settings.sitemapEnabled) {
+    return null;
+  }
+
+  const trimmed = (options.sitemapUrl ?? '').trim();
+  return trimmed === '' ? null : trimmed;
+}
+
+/** One custom-rules group while it is still being collected, before it becomes a rule. */
+interface CustomRuleGroup {
+  allow: string[];
+  crawlDelay?: number;
+  disallow: string[];
+  /** Directive name (original casing) to the values typed under it, in order. */
+  other: Record<string, string[]>;
+  userAgents: string[];
+}
+
+interface ParsedCustomRules {
+  rules: RobotsRule[];
+  /**
+   * Lines the served file cannot carry, in the order they were typed. Every one of
+   * them is a line crawlers would ignore anyway — see {@link parseCustomRules} — but
+   * they are surfaced rather than swallowed so the operator is never left looking for
+   * a line that quietly disappeared.
+   */
+  unserved: string[];
+}
+
+/**
+ * Turns the free-form custom-rules block into `MetadataRoute.Robots` rules.
+ *
+ * Standard directives become the typed fields Next understands; everything else rides
+ * the per-rule `other` record, which Next's serialiser emits verbatim as
+ * `key: value`, preserving the key's casing. That covers the non-standard directives
+ * operators actually write — `Clean-param`, `Request-rate`, `Host`, an extra
+ * `Sitemap` — none of which had any representation before.
+ *
+ * Two shapes still cannot be represented, and both are reported as unserved rather
+ * than pretended away:
+ *
+ *   - comments, because every line Next emits is a `name: value` pair; and
+ *   - lines with no `:` (or an empty name), which are not robots.txt syntax at all.
+ *
+ * Neither changes what a crawler does, so the served file loses no meaning — but the
+ * preview must not show them, and the operator deserves to be told why they are gone.
+ *
+ * Note that `other` lines render after their group's Allow/Disallow/Crawl-delay
+ * regardless of where the operator typed them. That reordering is harmless (robots.txt
+ * groups are unordered sets of directives) and, more to the point, it is visible: the
+ * preview is generated from these same rules, so what the operator reads is what
+ * crawlers get.
+ */
+function parseCustomRules(customRules: string): ParsedCustomRules {
+  const normalized = normalizeCustomRules(customRules);
+  if (normalized === '') {
+    return { rules: [], unserved: [] };
+  }
+
+  const groups: CustomRuleGroup[] = [];
+  const unserved: string[] = [];
+
+  for (const rawLine of normalized.split('\n')) {
+    const line = rawLine.trim();
+    if (line === '') {
+      continue;
+    }
+
+    if (line.startsWith('#')) {
+      unserved.push(line);
+      continue;
+    }
+
+    const separator = line.indexOf(':');
+    if (separator === -1) {
+      unserved.push(line);
+      continue;
+    }
+
+    const name = line.slice(0, separator).trim();
+    const value = line.slice(separator + 1).trim();
+    if (name === '') {
+      unserved.push(line);
+      continue;
+    }
+
+    const directive = name.toLowerCase();
+    let current = groups[groups.length - 1];
+
+    if (directive === 'user-agent') {
+      // A blank agent name would open a group that applies to nobody, silently
+      // orphaning every directive typed under it, so it is rejected outright.
+      if (value === '') {
+        unserved.push(line);
+        continue;
+      }
+
+      // Consecutive `User-agent:` lines share the rules that follow them, which is
+      // standard robots.txt and something operators genuinely write. Only start a
+      // new group once the previous one has actually collected a directive.
+      if (current && isEmptyGroup(current)) {
+        current.userAgents.push(value);
+      } else {
+        groups.push({ allow: [], disallow: [], other: {}, userAgents: [value] });
+      }
+      continue;
+    }
+
+    // A directive with no preceding `User-agent:` is malformed robots.txt, but the
+    // operator's intent is unambiguous, so it is attributed to the wildcard agent
+    // rather than discarded.
+    if (!current) {
+      current = { allow: [], disallow: [], other: {}, userAgents: ['*'] };
+      groups.push(current);
+    }
+
+    if (directive === 'allow') {
+      // An empty value is kept rather than skipped: `Disallow:` with nothing after it
+      // is the specified way of saying "nothing is disallowed", and dropping it would
+      // turn a meaningful line into a missing one.
+      current.allow.push(value);
+      continue;
+    }
+
+    if (directive === 'disallow') {
+      current.disallow.push(value);
+      continue;
+    }
+
+    if (directive === 'crawl-delay') {
+      const delay = Number.parseFloat(value);
+      // Next emits the typed `crawlDelay` field only when it is truthy, so a zero,
+      // a second crawl-delay in the same group, or something unparseable would be
+      // dropped from the served file. Those go through `other` instead, where they
+      // are emitted verbatim — the operator's line survives either way.
+      if (current.crawlDelay === undefined && Number.isFinite(delay) && delay > 0) {
+        current.crawlDelay = delay;
+      } else {
+        pushOther(current, name, value);
+      }
+      continue;
+    }
+
+    pushOther(current, name, value);
+  }
+
+  return { rules: groups.map(toCustomRule), unserved };
+}
+
+/** True while a group has collected nothing but `User-agent:` lines. */
+function isEmptyGroup(group: CustomRuleGroup): boolean {
+  return (
+    group.allow.length === 0 &&
+    group.disallow.length === 0 &&
+    group.crawlDelay === undefined &&
+    Object.keys(group.other).length === 0
+  );
+}
+
+/**
+ * Records a non-standard directive. Repeats accumulate into an array because Next
+ * emits one line per array entry, which is the only way a group can carry the same
+ * directive name twice.
+ */
+function pushOther(group: CustomRuleGroup, name: string, value: string): void {
+  const existing = group.other[name];
+  if (existing) {
+    existing.push(value);
+  } else {
+    group.other[name] = [value];
+  }
+}
+
+function toCustomRule(group: CustomRuleGroup): RobotsRule {
+  const rule: RobotsRule = {
+    userAgent: group.userAgents.length === 1 ? group.userAgents[0] : group.userAgents,
+  };
+
+  if (group.allow.length > 0) {
+    rule.allow = group.allow;
+  }
+  if (group.disallow.length > 0) {
+    rule.disallow = group.disallow;
+  }
+  if (group.crawlDelay !== undefined) {
+    rule.crawlDelay = group.crawlDelay;
+  }
+
+  const otherKeys = Object.keys(group.other);
+  if (otherKeys.length > 0) {
+    const other: Record<string, string | string[]> = {};
+    for (const key of otherKeys) {
+      const values = group.other[key];
+      other[key] = values.length === 1 ? values[0] : values;
+    }
+    rule.other = other;
+  }
+
+  return rule;
+}
+
+/**
+ * The `MetadataRoute.Robots` object served at /robots.txt by `app/robots.ts`, and the
+ * object {@link buildRobotsTxt} renders for the admin preview.
+ *
+ * Empty `allow` / `disallow` arrays are omitted rather than passed through: Next
+ * renders each entry as its own line, so an empty array contributes nothing while
+ * still occupying a field, and leaving it out keeps the emitted object honest about
+ * what the operator actually configured.
+ */
+export function buildRobotsMetadata(
+  settings: RobotsSettings,
+  options: RobotsTxtOptions,
+): MetadataRoute.Robots {
+  const rules: RobotsRule[] = resolveUserAgentRules(settings, options).map((rule) => {
+    const metadataRule: RobotsRule = { userAgent: rule.userAgent };
+    if (rule.allow.length > 0) {
+      metadataRule.allow = rule.allow;
+    }
+    if (rule.disallow.length > 0) {
+      metadataRule.disallow = rule.disallow;
+    }
+    // A group consisting of nothing but its `User-agent:` header is malformed, and
+    // some parsers discard the groups that follow it. An empty `Disallow` value is
+    // the specified way to say "nothing is disallowed", so that is what an otherwise
+    // empty group carries.
+    if (rule.allow.length === 0 && rule.disallow.length === 0) {
+      metadataRule.disallow = [''];
+    }
+    return metadataRule;
+  });
+
+  // Custom rules survive an indexing-off switch, because they are how an operator
+  // carves out the exceptions that must keep working regardless (a `.well-known`
+  // path, a verification file). They do NOT survive the sandbox, for the reason
+  // recorded on SANDBOX_USER_AGENT_RULE.
+  if (!options.isSandbox) {
+    rules.push(...parseCustomRules(settings.customRules).rules);
+  }
+
+  const result: MetadataRoute.Robots = { rules };
+
+  const sitemapUrl = resolveSitemapUrl(settings, options);
+  if (sitemapUrl) {
+    result.sitemap = sitemapUrl;
+  }
+
+  return result;
+}
+
+/**
+ * The literal robots.txt text for a settings object — what the admin preview shows.
+ *
+ * This is {@link buildRobotsMetadata} put through {@link renderRobotsMetadata}, which
+ * is what makes the preview incapable of disagreeing with the served file: there is
+ * one object, one renderer, and the renderer is Next's own algorithm. The route
+ * itself still returns the metadata object rather than this string, so caching,
+ * revalidation and content type are handled exactly the way they are for the sitemap.
+ */
+export function buildRobotsTxt(settings: RobotsSettings, options: RobotsTxtOptions): string {
+  return renderRobotsMetadata(buildRobotsMetadata(settings, options));
+}
+
+/**
+ * The custom-rules lines that will not appear in the served file, so the SEO screen
+ * can tell the operator instead of leaving them to notice the absence.
+ *
+ * In the sandbox that is the entire block: the sandbox answer ignores stored settings
+ * wholesale (see SANDBOX_USER_AGENT_RULE), so every custom line is inert there.
+ */
+export function listUnservedCustomRuleLines(
+  settings: RobotsSettings,
+  options: RobotsTxtOptions,
+): string[] {
+  const normalized = normalizeCustomRules(settings.customRules);
+
+  if (options.isSandbox) {
+    return normalized === ''
+      ? []
+      : normalized
+          .split('\n')
+          .map((line) => line.trim())
+          .filter((line) => line !== '');
+  }
+
+  return parseCustomRules(normalized).unserved;
+}
+
+/**
+ * Serialises a `MetadataRoute.Robots` object exactly as Next.js does.
+ *
+ * This is a deliberate port of `resolveRobots` in
+ * `next/dist/build/webpack/loaders/metadata/resolve-route-data`, down to the
+ * capitalised `User-Agent:`, the blank line after every group including the last, and
+ * the order of fields within a group. It exists so the admin preview can be the
+ * served bytes rather than an approximation of them; the colocated test asserts the
+ * two agree by importing Next's serialiser directly, so a change to Next's rendering
+ * fails a test here instead of silently turning the preview back into a lie.
+ */
+export function renderRobotsMetadata(metadata: MetadataRoute.Robots): string {
+  let content = '';
+
+  const rules = Array.isArray(metadata.rules) ? metadata.rules : [metadata.rules];
+
+  for (const rule of rules) {
+    for (const agent of toArray(rule.userAgent || ['*'])) {
+      content += `User-Agent: ${agent}\n`;
+    }
+
+    if (rule.allow) {
+      for (const item of toArray(rule.allow)) {
+        content += `Allow: ${item}\n`;
+      }
+    }
+
+    if (rule.disallow) {
+      for (const item of toArray(rule.disallow)) {
+        content += `Disallow: ${item}\n`;
+      }
+    }
+
+    if (rule.crawlDelay) {
+      content += `Crawl-delay: ${rule.crawlDelay}\n`;
+    }
+
+    if (rule.other) {
+      for (const key of Object.keys(rule.other)) {
+        const value = rule.other[key];
+        if (value === null || value === undefined) {
+          continue;
+        }
+        for (const entry of toArray(value)) {
+          content += `${key}: ${entry}\n`;
+        }
+      }
+    }
+
+    content += '\n';
+  }
+
+  if (metadata.host) {
+    content += `Host: ${metadata.host}\n`;
+  }
+
+  if (metadata.sitemap) {
+    for (const item of toArray(metadata.sitemap)) {
+      content += `Sitemap: ${item}\n`;
+    }
+  }
+
+  return content;
+}
+
+/** Next's `resolveArray`: a bare value becomes a one-element list, a list is left alone. */
+function toArray<T>(value: T | T[]): T[] {
+  return Array.isArray(value) ? value : [value];
+}

package/templates/nextblock-template/lib/setup/migrations-bundle.ts

@@ -163,5 +163,15 @@ export const MIGRATIONS_BUNDLE: BundledMigration[] = [
     "version": "00000000000029",
     "name": "00000000000029_form_endpoints_default_empty.sql",
     "sql": "-- A contact form has no address of its own by default.\n--\n-- The starter content ships a contact page addressed to `contact@example.com`, and\n-- migration 27 faithfully carried that placeholder into `form_endpoints`. Faithful was\n-- the wrong call. `example.com` is reserved by RFC 2606 precisely so it can never be\n-- registered, so the address is not merely unhelpful — it is guaranteed undeliverable,\n-- while looking like a real setting to every layer downstream. The visitor is thanked,\n-- the relay accepts the message, and nobody is ever notified. Nothing errors, so an\n-- install can run that way indefinitely.\n--\n-- Since the messaging system arrived, a per-form address is not something an operator\n-- needs to think about at all. Submissions are stored as threads and answered in\n-- CMS → Messages; the notification address is a single site-wide setting, and a form\n-- only carries its own address when someone deliberately wants that form routed\n-- elsewhere (a careers form to HR, say).\n--\n-- So the default becomes NULL, meaning \"use the site contact address\", which resolves\n-- to the address set in CMS → Messages and finally to the first admin's own login. An\n-- install therefore reaches a real human out of the box without configuring anything.\n--\n-- The sandbox is unaffected: `resolveFormRecipient` overrides everything with\n-- SANDBOX_CONTACT_EMAIL when NEXT_PUBLIC_IS_SANDBOX is set, so the hosted demo keeps\n-- routing to the operator's own inbox without storing an address here.\n--\n-- Forward-only and idempotent.\n\n-- Reserved domains from RFC 2606 / RFC 6761. Matching on the domain rather than the\n-- exact seeded string also catches an operator who typed their own placeholder.\nUPDATE public.form_endpoints\n   SET recipient_email = NULL\n WHERE recipient_email IS NOT NULL\n   AND (\n     lower(recipient_email) LIKE '%@example.com'\n     OR lower(recipient_email) LIKE '%@example.org'\n     OR lower(recipient_email) LIKE '%@example.net'\n     OR lower(recipient_email) LIKE '%@example.edu'\n     OR lower(recipient_email) LIKE '%.example'\n     OR lower(recipient_email) LIKE '%.invalid'\n     OR lower(recipient_email) LIKE '%.test'\n     OR lower(recipient_email) LIKE '%.localhost'\n     OR lower(recipient_email) LIKE '%.local'\n   );\n\nCOMMENT ON COLUMN public.form_endpoints.recipient_email IS 'Per-form override. NULL means \"use the site contact address\" (CMS -> Messages, falling back to the first admin), which is the default and the usual case.';\n\n-- Same reasoning for the site-wide rows: a placeholder there is worse than an empty\n-- value, because an empty one falls through to the first admin and actually arrives.\nUPDATE public.site_settings\n   SET value = jsonb_set(value, '{contactEmail}', '\"\"'::jsonb)\n WHERE key IN ('forms_contact', 'store_contact')\n   AND value ? 'contactEmail'\n   AND lower(value->>'contactEmail') LIKE '%@example.%';\n"
+  },
+  {
+    "version": "00000000000030",
+    "name": "00000000000030_seo_redirects_and_robots.sql",
+    "sql": "-- SEO engine: managed 301/302 redirects and operator-configurable robots directives.\n--\n-- Two unrelated-looking things ship in one migration because they are the same\n-- feature from an operator's point of view: the `/cms/settings/seo` screen is where\n-- someone goes to say \"this URL moved\" and \"do not crawl that\". Splitting them\n-- across two migrations would only mean two files that must always be applied\n-- together.\n--\n-- WHY A TABLE AND NOT next.config.js redirects(). Redirects are content, not\n-- configuration. An editor who renames a page's slug needs the old URL to keep\n-- working immediately, without a redeploy and without touching source control.\n-- That rules out the build-time array; it has to be data.\n--\n-- WHY THE PROXY READS THIS WITH THE ANON KEY. apps/nextblock/proxy.ts (Next 16's\n-- renamed middleware) resolves redirects before rendering, and it holds a Supabase\n-- client built from the anon key. So the public SELECT policy below is load-bearing:\n-- without an explicit `TO authenticated, anon` grant the proxy's lookup would return\n-- zero rows for every anonymous visitor -- silently, with no error -- and no redirect\n-- would ever fire. Only is_active rows are exposed, so a half-written rule is never\n-- live.\n--\n-- WHY status_code IS AN integer AND NOT AN ENUM. The same reasoning migration 27\n-- recorded for `source`: a Postgres enum cannot be extended and used inside the same\n-- transaction, which is exactly the scope of one migration file. A CHECK constraint\n-- is replaceable in a single statement. 301 and 302 are the only two values the\n-- admin UI offers; 307/308 are deliberately not exposed, because their\n-- method-preserving semantics surprise operators who just want \"this page moved\".\n--\n-- LOOP SAFETY is enforced in application code (wouldCreateLoop in\n-- @nextblock-cms/utils, called by the admin server actions) rather than by a\n-- constraint, because detecting a cycle requires walking the whole table and a CHECK\n-- constraint can only see one row. The self-redirect case IS cheap to check per row,\n-- so that one is a constraint -- it is the cycle operators actually hit.\n--\n-- SECURITY POSTURE. A redirect can send every visitor of a path to an arbitrary\n-- external origin, which makes this table an open-redirect surface and a phishing\n-- lever. Writes are therefore ADMIN-only -- WRITER is deliberately excluded, matching\n-- site_scripts rather than the content tables. Reads are public but limited to active\n-- rows. The robots settings live in site_settings, whose existing read policy is\n-- already public for non-secret keys; nothing here is secret.\n--\n-- Forward-only and idempotent.\n\nCREATE TABLE IF NOT EXISTS public.cms_redirects (\n    id uuid DEFAULT gen_random_uuid() NOT NULL,\n    -- The incoming site-relative path to match, normalized by the application to a\n    -- leading slash with no trailing slash (except root). Matching is exact: prefix\n    -- and wildcard rules are deliberately not supported, because they are the usual\n    -- way an operator builds an accidental loop.\n    source_path text NOT NULL,\n    -- Where to send the visitor. Either another site-relative path or a fully\n    -- qualified https URL for an off-site move.\n    destination_path text NOT NULL,\n    -- 301 permanent (the SEO-meaningful one: search engines transfer ranking signals\n    -- and browsers cache it aggressively) or 302 temporary.\n    status_code integer DEFAULT 301 NOT NULL,\n    is_active boolean DEFAULT true NOT NULL,\n    created_at timestamp with time zone DEFAULT now() NOT NULL,\n    updated_at timestamp with time zone DEFAULT now() NOT NULL,\n    CONSTRAINT cms_redirects_pkey PRIMARY KEY (id),\n    -- One rule per source. This UNIQUE constraint also provides the index the proxy\n    -- lookup relies on, so no separate plain index on source_path is needed.\n    CONSTRAINT cms_redirects_source_path_key UNIQUE (source_path),\n    CONSTRAINT cms_redirects_status_code_check\n        CHECK ((status_code = ANY (ARRAY[301, 302]))),\n    -- A source is always a path on this site; accepting an absolute URL here would\n    -- silently never match, since the proxy only ever compares pathnames.\n    CONSTRAINT cms_redirects_source_path_check\n        CHECK ((source_path ~ '^/')),\n    -- A destination is either a site-relative path or an https URL. Plain http is\n    -- refused so a redirect can never downgrade a visitor to cleartext.\n    CONSTRAINT cms_redirects_destination_path_check\n        CHECK (((destination_path ~ '^/') OR (destination_path ~ '^https://'))),\n    -- The one cycle a single row can express, and the one operators actually create.\n    CONSTRAINT cms_redirects_no_self_redirect_check\n        CHECK ((source_path <> destination_path))\n);\n\nCOMMENT ON TABLE public.cms_redirects IS\n    'Operator-managed 301/302 redirects resolved by apps/nextblock/proxy.ts before rendering. Only is_active rows are publicly readable; only ADMIN may write, because a redirect rule is an open-redirect surface.';\nCOMMENT ON COLUMN public.cms_redirects.source_path IS\n    'Exact site-relative path to match, normalized to a leading slash and no trailing slash (except root). No wildcards, by design.';\nCOMMENT ON COLUMN public.cms_redirects.destination_path IS\n    'Site-relative path or absolute https URL to send the visitor to.';\nCOMMENT ON COLUMN public.cms_redirects.status_code IS\n    '301 permanent or 302 temporary. 307/308 are not offered by the admin UI.';\nCOMMENT ON COLUMN public.cms_redirects.is_active IS\n    'Only active rows are readable by anon, and only active rows are matched by the proxy.';\n\n-- The proxy loads the whole active set once per cache window rather than querying per\n-- request, so the hot query is \"all active rows\" and this partial index is what serves\n-- it. Carrying source_path in the index keeps that read index-only.\nCREATE INDEX IF NOT EXISTS cms_redirects_active_source_idx\n    ON public.cms_redirects USING btree (source_path) WHERE (is_active);\n\nDROP TRIGGER IF EXISTS set_cms_redirects_updated_at ON public.cms_redirects;\nCREATE TRIGGER set_cms_redirects_updated_at\n    BEFORE UPDATE ON public.cms_redirects\n    FOR EACH ROW EXECUTE FUNCTION public.set_current_timestamp_updated_at();\n\nALTER TABLE public.cms_redirects ENABLE ROW LEVEL SECURITY;\n\nGRANT ALL ON TABLE public.cms_redirects TO anon;\nGRANT ALL ON TABLE public.cms_redirects TO authenticated;\nGRANT ALL ON TABLE public.cms_redirects TO service_role;\n\n-- The proxy runs as anon for a logged-out visitor, which is the overwhelming majority\n-- of traffic and the only traffic redirects really matter for. Without this policy the\n-- lookup returns zero rows and the feature is silently dead.\nDROP POLICY IF EXISTS \"Public read active redirects\" ON public.cms_redirects;\nCREATE POLICY \"Public read active redirects\" ON public.cms_redirects\n    FOR SELECT TO authenticated, anon USING (is_active);\n\nDROP POLICY IF EXISTS \"Admins read all redirects\" ON public.cms_redirects;\nCREATE POLICY \"Admins read all redirects\" ON public.cms_redirects\n    FOR SELECT TO authenticated\n    USING (((SELECT public.get_current_user_role()) = 'ADMIN'::public.user_role));\n\nDROP POLICY IF EXISTS \"Admins insert redirects\" ON public.cms_redirects;\nCREATE POLICY \"Admins insert redirects\" ON public.cms_redirects\n    FOR INSERT TO authenticated\n    WITH CHECK (((SELECT public.get_current_user_role()) = 'ADMIN'::public.user_role));\n\nDROP POLICY IF EXISTS \"Admins update redirects\" ON public.cms_redirects;\nCREATE POLICY \"Admins update redirects\" ON public.cms_redirects\n    FOR UPDATE TO authenticated\n    USING (((SELECT public.get_current_user_role()) = 'ADMIN'::public.user_role))\n    WITH CHECK (((SELECT public.get_current_user_role()) = 'ADMIN'::public.user_role));\n\nDROP POLICY IF EXISTS \"Admins delete redirects\" ON public.cms_redirects;\nCREATE POLICY \"Admins delete redirects\" ON public.cms_redirects\n    FOR DELETE TO authenticated\n    USING (((SELECT public.get_current_user_role()) = 'ADMIN'::public.user_role));\n\n-- Robots directives live in site_settings rather than a table of their own: there is\n-- exactly one robots.txt per install, so a dedicated table would only ever hold one\n-- row, and site_settings already carries the public-read / staff-write policy this\n-- needs. The stored shape matches `RobotsSettings` in @nextblock-cms/utils, and\n-- `normalizeRobotsSettings` tolerates a missing or malformed value -- so this seed is\n-- a convenience for the settings screen, not a correctness requirement for rendering.\n--\n-- ON CONFLICT DO NOTHING because an install that has already configured robots must\n-- not have its rules reset by a replay of this migration.\nINSERT INTO public.site_settings (key, value)\nVALUES (\n    'seo_robots_settings',\n    '{\"customRules\": \"\", \"isIndexingEnabled\": true, \"sitemapEnabled\": true, \"userAgentRules\": [{\"allow\": [\"/\"], \"disallow\": [], \"userAgent\": \"*\"}]}'::jsonb\n)\nON CONFLICT (key) DO NOTHING;\n"
+  },
+  {
+    "version": "00000000000031",
+    "name": "00000000000031_seo_robots_settings_admin_only.sql",
+    "sql": "-- Restrict writes to the robots.txt settings row to ADMIN only.\n--\n-- `site_settings.seo_robots_settings` is the row that decides whether the whole site\n-- is crawlable: app/robots.ts reads it on every /robots.txt hit and serves a blanket\n-- `Disallow: /` when `isIndexingEnabled` is false. The CMS only offers that switch\n-- under /cms/settings/seo, and `saveRobotsSettings` re-checks for ADMIN before it\n-- writes — but RLS is the independent boundary, and it did not agree. The baseline\n-- write policies let ADMIN *or* WRITER write any key outside the sensitive array, so\n-- a WRITER holding a normal session could PATCH this row through PostgREST and take\n-- the entire site out of Google. That failure is quiet (nothing in the CMS shows it),\n-- slow to notice (search traffic decays over weeks) and hard to attribute after the\n-- fact, which is why the database has to refuse it rather than trusting the one\n-- server action that happens to guard it today.\n--\n-- The SELECT policy is deliberately NOT touched. This key must stay anon-READABLE:\n-- app/robots.ts reads it with the anon (SSG) client on every crawl, and adding the key\n-- to the read policy's sensitive array would make robots.txt fall back to its\n-- permissive defaults for every crawler. Only INSERT/UPDATE/DELETE move to ADMIN-only,\n-- matching the UI boundary — exactly the shape migration 00000000000011 used for\n-- language_detection_settings, which is anon-readable for the same reason.\n--\n-- Every key already present in each policy's array is preserved (dropping one would\n-- silently widen write access back to WRITER for that key); `seo_robots_settings` is\n-- appended to the three write policies only.\n\nDROP POLICY IF EXISTS site_settings_insert_policy ON public.site_settings;\nCREATE POLICY site_settings_insert_policy ON public.site_settings FOR INSERT TO authenticated WITH CHECK ((((key <> ALL (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = ANY (ARRAY['ADMIN'::public.user_role, 'WRITER'::public.user_role]))) OR ((key = ANY (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = 'ADMIN'::public.user_role))));\n\nDROP POLICY IF EXISTS site_settings_update_policy ON public.site_settings;\nCREATE POLICY site_settings_update_policy ON public.site_settings FOR UPDATE TO authenticated USING ((((key <> ALL (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = ANY (ARRAY['ADMIN'::public.user_role, 'WRITER'::public.user_role]))) OR ((key = ANY (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = 'ADMIN'::public.user_role)))) WITH CHECK ((((key <> ALL (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = ANY (ARRAY['ADMIN'::public.user_role, 'WRITER'::public.user_role]))) OR ((key = ANY (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = 'ADMIN'::public.user_role))));\n\nDROP POLICY IF EXISTS site_settings_delete_policy ON public.site_settings;\nCREATE POLICY site_settings_delete_policy ON public.site_settings FOR DELETE TO authenticated USING ((((key <> ALL (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = ANY (ARRAY['ADMIN'::public.user_role, 'WRITER'::public.user_role]))) OR ((key = ANY (ARRAY['cortex_ai_openrouter_api_key'::text, 'bot_protection_secret'::text, 'email_secret'::text, 'payment_secret'::text, 'language_detection_settings'::text, 'cortex_ai_pexels_api_key'::text, 'cortex_ai_unsplash_access_key'::text, 'cortex_ai_mcp_settings'::text, 'seo_robots_settings'::text])) AND (( SELECT public.get_current_user_role() AS get_current_user_role) = 'ADMIN'::public.user_role))));\n\n-- Forward-only and idempotent.\n"
   }
 ];

package/templates/nextblock-template/next-env.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
-import "./.next/dev/types/routes.d.ts";
-import "./.next/dev/types/root-params.d.ts";
+import "./.next/types/routes.d.ts";
+import "./.next/types/root-params.d.ts";
 
 // NOTE: This file should not be edited
 // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

package/templates/nextblock-template/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nextblock-cms/template",
-  "version": "0.15.9",
+  "version": "0.16.1",
   "description": "The open-source, full-stack AI-native CMS for Next.js 16, Supabase, and Tailwind CSS.",
   "keywords": [
     "cms",