blume 1.4.2 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +54 -0
  2. package/dist/cli/index.js +694 -579
  3. package/dist/cli/index.js.map +56 -55
  4. package/dist/types/core/base-path.d.ts +8 -0
  5. package/dist/types/core/config-input.d.ts +8 -0
  6. package/dist/types/core/schema.d.ts +4 -0
  7. package/dist/types/core/sources/types.d.ts +9 -1
  8. package/dist/types/openapi/references.d.ts +8 -2
  9. package/docs/configuration/ai.mdx +26 -8
  10. package/docs/content/sources.mdx +1 -1
  11. package/package.json +11 -1
  12. package/src/ai/agent-readability.ts +3 -2
  13. package/src/ai/api-catalog.ts +2 -2
  14. package/src/ai/ask-context.ts +45 -12
  15. package/src/ai/llms.ts +2 -1
  16. package/src/ai/mcp/discovery.ts +25 -6
  17. package/src/ai/mcp/server.ts +108 -98
  18. package/src/ai/tar.ts +29 -70
  19. package/src/astro/examples.ts +7 -3
  20. package/src/astro/generate.ts +59 -34
  21. package/src/astro/islands.ts +7 -3
  22. package/src/astro/templates.ts +34 -9
  23. package/src/audit/agent.ts +14 -29
  24. package/src/audit/crawl.ts +41 -16
  25. package/src/audit/run.ts +10 -3
  26. package/src/audit/snapshot.ts +27 -2
  27. package/src/cli/commands/audit.ts +12 -17
  28. package/src/cli/commands/build.ts +15 -7
  29. package/src/cli/commands/dev.ts +13 -15
  30. package/src/cli/commands/eject.ts +4 -4
  31. package/src/cli/commands/eval.ts +17 -27
  32. package/src/cli/env.ts +13 -30
  33. package/src/cli/init/scaffold.ts +21 -0
  34. package/src/cli/report-format.ts +22 -0
  35. package/src/components/content/AccordionItem.astro +2 -9
  36. package/src/components/content/ColorItem.astro +5 -13
  37. package/src/components/content/Component.astro +12 -8
  38. package/src/components/content/Frame.astro +2 -12
  39. package/src/components/content/Prompt.astro +12 -31
  40. package/src/components/content/Tab.astro +2 -9
  41. package/src/components/content/Tooltip.astro +1 -9
  42. package/src/components/content/Update.astro +2 -9
  43. package/src/components/content/inline-markdown.ts +28 -0
  44. package/src/components/copy-feedback.ts +96 -0
  45. package/src/components/islands/ask-ai.tsx +78 -9
  46. package/src/components/layout/PageActions.astro +20 -32
  47. package/src/components/layout/PageLayout.astro +8 -28
  48. package/src/components/layout/RootLayout.astro +6 -48
  49. package/src/components/layout/Search.astro +56 -9
  50. package/src/components/layout/drawer-inert.ts +31 -0
  51. package/src/components/layout/search/pagefind.ts +6 -5
  52. package/src/components/layout/search/types.ts +32 -0
  53. package/src/components/openapi/panel.ts +11 -8
  54. package/src/components/raf-throttle.ts +21 -0
  55. package/src/components/slug.ts +14 -0
  56. package/src/core/base-path.ts +18 -1
  57. package/src/core/config-input.ts +8 -0
  58. package/src/core/frontmatter.ts +45 -1
  59. package/src/core/probe.ts +7 -19
  60. package/src/core/project-graph.ts +12 -1
  61. package/src/core/schema.ts +6 -0
  62. package/src/core/site-url.ts +27 -0
  63. package/src/core/sources/cache.ts +10 -8
  64. package/src/core/sources/github-releases.ts +21 -1
  65. package/src/core/sources/normalize.ts +26 -2
  66. package/src/core/sources/notion.ts +27 -5
  67. package/src/core/sources/portable-text.ts +16 -1
  68. package/src/core/sources/resolve.ts +1 -0
  69. package/src/core/sources/types.ts +13 -1
  70. package/src/deploy/cloudflare-negotiation.ts +15 -1
  71. package/src/deploy/robots.ts +2 -1
  72. package/src/deploy/rss.ts +2 -1
  73. package/src/deploy/sitemap.ts +56 -7
  74. package/src/eval/agents.ts +13 -10
  75. package/src/eval/report.ts +1 -14
  76. package/src/markdown/package-commands.ts +61 -54
  77. package/src/og/card.ts +24 -26
  78. package/src/openapi/model.ts +9 -9
  79. package/src/openapi/parse.ts +69 -28
  80. package/src/openapi/references.ts +35 -12
  81. package/src/openapi/render-mdx.ts +64 -25
  82. package/src/openapi/scalar.ts +2 -2
  83. package/src/openapi/source.ts +28 -1
  84. package/src/search/documents.ts +78 -34
  85. package/src/search/orama-index.ts +51 -12
  86. package/src/theme/palette.ts +6 -2
  87. package/src/translate/ledger.ts +4 -2
  88. package/src/translate/report.ts +1 -14
  89. package/src/translate/run.ts +20 -35
  90. package/src/cli/coalesce.ts +0 -43
@@ -28,13 +28,57 @@ const withYamlEngine = <O>(options: O): O =>
28
28
  },
29
29
  }) as O;
30
30
 
31
+ /**
32
+ * True when a document's leading `---` line is a CommonMark thematic break,
33
+ * not a front matter fence. Two shapes qualify (mirroring
34
+ * `linesWithoutFrontMatter` in `sources/normalize.ts`):
35
+ * - the next line is blank (or absent) — YAML metadata starts on the very
36
+ * next line, so a gap means the body *opens* with a divider (e.g. a
37
+ * Notion page whose first block is one);
38
+ * - no closing `---` line follows — gray-matter would swallow the whole
39
+ * document as one unclosed YAML block and hand it to js-yaml, which
40
+ * crashes on ordinary Markdown (`> quote` → "a line break is expected").
41
+ */
42
+ const opensWithThematicBreak = (input: string): boolean => {
43
+ const [first = "", second] = input.split(/\r?\n/u, 2);
44
+ if (!/^-{3}\s*$/u.test(first)) {
45
+ return false;
46
+ }
47
+ if (second === undefined || second.trim() === "") {
48
+ return true;
49
+ }
50
+ // gray-matter closes the block at the next line-leading `---`; matching its
51
+ // search exactly keeps this guard from firing on any document it parses.
52
+ return !input.includes("\n---", 1);
53
+ };
54
+
55
+ /**
56
+ * The parse result for a document with no front matter: the input passes
57
+ * through as content, untouched. Shaped like gray-matter's own no-matter
58
+ * result (every Blume call site reads only `content` and `data`).
59
+ */
60
+ const passthrough = (input: string): ReturnType<typeof baseMatter> =>
61
+ ({
62
+ content: input,
63
+ data: {},
64
+ excerpt: "",
65
+ isEmpty: false,
66
+ language: "",
67
+ matter: "",
68
+ orig: input,
69
+ // Recomposing a file with no matter and empty data is the content itself.
70
+ stringify: (): string => input,
71
+ }) as unknown as ReturnType<typeof baseMatter>;
72
+
31
73
  // Every helper that parses or emits YAML (`read`, `stringify`) must be
32
74
  // re-wrapped here — Object.assign copies gray-matter's own helpers, which use
33
75
  // its default `safeLoad` engine and would reintroduce the crash. `test` only
34
76
  // checks for a delimiter, so the copied original is safe.
35
77
  const matter = Object.assign(
36
78
  (input: MatterInput, options?: MatterOptions) =>
37
- baseMatter(input, withYamlEngine(options)),
79
+ typeof input === "string" && opensWithThematicBreak(input)
80
+ ? passthrough(input)
81
+ : baseMatter(input, withYamlEngine(options)),
38
82
  baseMatter,
39
83
  {
40
84
  read: (filepath: ReadArgs[0], options?: ReadArgs[1]) =>
package/src/core/probe.ts CHANGED
@@ -1,3 +1,5 @@
1
+ import pMap from "p-map";
2
+
1
3
  import type { DiagnosticSeverity } from "./types.ts";
2
4
 
3
5
  export const PROBE_CONCURRENCY = 8;
@@ -113,24 +115,10 @@ export const probeAll = async (
113
115
  options: { concurrency?: number; timeoutMs?: number } = {}
114
116
  ): Promise<Map<string, ProbeResult>> => {
115
117
  const unique = [...new Set(urls)];
116
- const results = new Map<string, ProbeResult>();
117
- const limit = Math.min(
118
- options.concurrency ?? PROBE_CONCURRENCY,
119
- unique.length
118
+ const entries = await pMap(
119
+ unique,
120
+ async (url) => [url, await probe(url, options)] as const,
121
+ { concurrency: Math.max(1, options.concurrency ?? PROBE_CONCURRENCY) }
120
122
  );
121
-
122
- let cursor = 0;
123
- const worker = async (): Promise<void> => {
124
- while (cursor < unique.length) {
125
- const url = unique[cursor];
126
- cursor += 1;
127
- if (url !== undefined) {
128
- // oxlint-disable-next-line no-await-in-loop -- bounded-concurrency pool
129
- results.set(url, await probe(url, options));
130
- }
131
- }
132
- };
133
- await Promise.all(Array.from({ length: limit }, worker));
134
-
135
- return results;
123
+ return new Map(entries);
136
124
  };
@@ -262,6 +262,17 @@ export const scanProject = async (
262
262
  discoverFolderMeta(metaSources, { localeDirs }),
263
263
  ]);
264
264
 
265
+ // Folder meta contributed by the sources themselves (the OpenAPI source
266
+ // labels each tag directory with the spec's own tag name). It applies to
267
+ // every locale, so it merges into the shared map — beneath user-authored
268
+ // entries, which are spread last and win.
269
+ const sharedFolderMeta = new Map([
270
+ ...loaded.flatMap(({ folderMeta: sourceMeta }) =>
271
+ Object.entries(sourceMeta ?? {})
272
+ ),
273
+ ...folderMeta.shared,
274
+ ]);
275
+
265
276
  const {
266
277
  diagnostics: contentDiagnostics,
267
278
  droppedPages,
@@ -301,7 +312,7 @@ export const scanProject = async (
301
312
  folderMeta: folderMeta.meta,
302
313
  i18n: config.i18n,
303
314
  navigation: config.navigation,
304
- sharedFolderMeta: folderMeta.shared,
315
+ sharedFolderMeta,
305
316
  });
306
317
  const manifest = buildManifest({ config, context, graph });
307
318
 
@@ -304,6 +304,8 @@ const sanitySourceSchema = z.object({
304
304
 
305
305
  /** A Notion database; pages become entries, blocks become MDX. */
306
306
  const notionSourceSchema = z.object({
307
+ /** Max concurrent Notion API requests; default 3 (Notion's per-integration pace). */
308
+ concurrency: z.number().positive().optional(),
307
309
  database: z.string(),
308
310
  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
309
311
  pollInterval: z.number().positive().optional(),
@@ -780,6 +782,10 @@ const aiConfigSchema = z.strictObject({
780
782
  // and host Ask AI in an existing backend. Absolute URLs and root-relative
781
783
  // paths are both valid; the built-in request/stream contract is unchanged.
782
784
  endpoint: askEndpointSchema.optional(),
785
+ // Extra system-prompt text (identity, language, tone) appended to the
786
+ // built-in instructions, so the grounding contract — answer from the
787
+ // retrieved excerpts, cite pages as Markdown links — stays intact.
788
+ instructions: z.string().trim().min(1).optional(),
783
789
  model: z.string().default("openai/gpt-5.5"),
784
790
  provider: z.enum(askAiProviders).default("gateway"),
785
791
  // Empty-state prompts shown before the first question. Each renders as a
@@ -0,0 +1,27 @@
1
+ import { joinURL } from "ufo";
2
+
3
+ import { trimEnd } from "./trim.ts";
4
+
5
+ /**
6
+ * Absolute-URL building for the configured `deployment.site`, shared by every
7
+ * emitter that prints site URLs (sitemap, RSS, robots, llms.txt, the MCP and
8
+ * agent-discovery documents). One implementation replaces eight per-file
9
+ * copies that had drifted across three different trailing-slash treatments.
10
+ *
11
+ * Deliberately not `new URL(path, site)`: a root-absolute path would drop the
12
+ * base path of a subpath deployment (`acme.com/docs`). ufo's `joinURL` joins
13
+ * without that footgun; the site is first trimmed with the ReDoS-safe
14
+ * `trimEnd` loop so even a malformed `site` with piled-up trailing slashes
15
+ * joins cleanly.
16
+ */
17
+
18
+ /** The configured site with any trailing slashes dropped. */
19
+ export const siteRoot = (site: string): string => trimEnd(site, "/");
20
+
21
+ /** `site` + root-absolute `path` (already carrying any deployment base). */
22
+ export const absoluteUrl = (site: string, path: string): string => {
23
+ const root = siteRoot(site);
24
+ // joinURL folds a lone "/" away entirely; the homepage keeps its slash
25
+ // (`https://example.com/`), matching what every emitter always printed.
26
+ return path === "/" ? `${root}/` : joinURL(root, path);
27
+ };
@@ -1,3 +1,4 @@
1
+ import { createHash } from "node:crypto";
1
2
  import { mkdir, readFile, writeFile } from "node:fs/promises";
2
3
 
3
4
  import { join } from "pathe";
@@ -6,14 +7,15 @@ import { BlumeError } from "../diagnostics.ts";
6
7
  import type { Diagnostic } from "../types.ts";
7
8
  import type { SourceEntry, SourceLoadResult } from "./types.ts";
8
9
 
9
- /** Small, stable content hash for cache/HMR bookkeeping (non-bitwise). */
10
- export const hashText = (text: string): string => {
11
- let hash = 5381;
12
- for (let i = 0; i < text.length; i += 1) {
13
- hash = (hash * 33 + (text.codePointAt(i) ?? 0)) % 2_147_483_647;
14
- }
15
- return hash.toString(36);
16
- };
10
+ /**
11
+ * Small, stable content hash for cache/HMR bookkeeping — and for staged asset
12
+ * *filenames* (see sources/assets.ts and content-assets.ts), where a collision
13
+ * silently serves the wrong file. 64 bits of SHA-256 keeps those names
14
+ * collision-safe at any realistic asset count; the old 31-bit DJB2 hash had a
15
+ * ~46k-item birthday bound.
16
+ */
17
+ export const hashText = (text: string): string =>
18
+ createHash("sha256").update(text).digest("hex").slice(0, 16);
17
19
 
18
20
  /** A stable digest of a source's entries, for change detection while polling. */
19
21
  export const entriesDigest = (entries: SourceEntry[]): string =>
@@ -85,6 +85,26 @@ const NON_PROSE = new Set(["code", "heading", "html", "thematicBreak"]);
85
85
  * content kept — then cut at a word boundary to fit the search snippet cap.
86
86
  * Undefined when the notes have no prose at all.
87
87
  */
88
+ /**
89
+ * The longest prefix of `text` that fits `max` UTF-16 units without cutting
90
+ * inside a grapheme cluster. A bare `String#slice` counts code units, so it
91
+ * can split a surrogate pair (emitting a lone surrogate — invalid Unicode —
92
+ * into a meta description) or halve an emoji sequence. Grapheme segmentation
93
+ * is rule-based (UAX #29), so unlike word segmentation it does not drift
94
+ * across ICU builds.
95
+ */
96
+ const graphemePrefix = (text: string, max: number): string => {
97
+ let end = 0;
98
+ const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
99
+ for (const { index, segment } of graphemes.segment(text)) {
100
+ if (index + segment.length > max) {
101
+ break;
102
+ }
103
+ end = index + segment.length;
104
+ }
105
+ return text.slice(0, end);
106
+ };
107
+
88
108
  const releaseDescription = (body: string): string | undefined => {
89
109
  const tree = fromMarkdown(body.replaceAll(CHANGESET_HASH, "$<mark>"), {
90
110
  extensions: [gfm()],
@@ -108,7 +128,7 @@ const releaseDescription = (body: string): string | undefined => {
108
128
  }
109
129
  // Cut before the cap at a word boundary (kept only when it doesn't drop the
110
130
  // summary under the minimum), shed any dangling punctuation, and mark the cut.
111
- const slice = text.slice(0, DESCRIPTION_MAX - 1);
131
+ const slice = graphemePrefix(text, DESCRIPTION_MAX - 1);
112
132
  const boundary = slice.lastIndexOf(" ");
113
133
  const head = (
114
134
  boundary >= DESCRIPTION_MIN ? slice.slice(0, boundary) : slice
@@ -28,12 +28,19 @@ const groupLabel = (segment: string): string | null =>
28
28
  * anchor ids are *not* slugged here — they use a `github-slugger` in
29
29
  * {@link extractHeadings}, matching the renderer (see `markdown/heading-anchors`)
30
30
  * so `blume validate` checks anchors against the exact rendered heading ids.
31
+ *
32
+ * The keep-class is Unicode letters/marks/numbers, not `\w`: ASCII slugs are
33
+ * unchanged, but a CJK/Cyrillic/accented slug keeps its characters instead of
34
+ * collapsing to `""` (which forced Sanity/Notion routes onto their opaque
35
+ * document-id fallbacks) or dropping accents (`café` → `caf`). NFC first so a
36
+ * macOS-NFD `é` (e + combining mark) slugs identically to the composed form.
31
37
  */
32
38
  export const slugify = (text: string): string =>
33
39
  text
40
+ .normalize("NFC")
34
41
  .toLowerCase()
35
42
  .trim()
36
- .replaceAll(/[^\w\s-]/gu, "")
43
+ .replaceAll(/[^\p{L}\p{M}\p{N}\s_-]/gu, "")
37
44
  .replaceAll(/[\s_]+/gu, "-")
38
45
  .replaceAll(/-+/gu, "-")
39
46
  .replaceAll(/^-|-$/gu, "");
@@ -55,6 +62,18 @@ const titleCase = (value: string): string =>
55
62
  .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
56
63
  .join(" ");
57
64
 
65
+ /**
66
+ * Strip characters that cannot survive the route → URL → output-file round
67
+ * trip. A `:` ahead of the first `/` makes `new URL()` read the segment as a
68
+ * scheme (`Guide: Architecture.md` → `guide:`), which crashes Astro's
69
+ * prerender write with "The URL must be of scheme file"; control characters
70
+ * (an embedded newline in a filename) are silently dropped by the URL parser,
71
+ * desyncing the route from its output path. Both are legal in macOS/Linux
72
+ * filenames, so they are removed here rather than rejected.
73
+ */
74
+ const sanitizeSegment = (segment: string): string =>
75
+ segment.replaceAll(/[:\p{Cc}]/gu, "");
76
+
58
77
  /** Fold one raw path part into the accumulating route segments/groups. */
59
78
  const addRouteSegment = (
60
79
  part: string,
@@ -75,7 +94,12 @@ const addRouteSegment = (
75
94
  if (clean === "index") {
76
95
  return;
77
96
  }
78
- segments.push(clean);
97
+ const safe = sanitizeSegment(clean);
98
+ // A part that was nothing but stripped characters cannot name a segment.
99
+ if (safe === "") {
100
+ return;
101
+ }
102
+ segments.push(safe);
79
103
  };
80
104
 
81
105
  /** Convert a content-root-relative path into URL + nav metadata. */
@@ -1,5 +1,6 @@
1
1
  import { setTimeout as sleep } from "node:timers/promises";
2
2
 
3
+ import pLimit from "p-limit";
3
4
  import { join } from "pathe";
4
5
 
5
6
  import { BlumeError } from "../diagnostics.ts";
@@ -96,6 +97,12 @@ export interface NotionSourceOptions {
96
97
  prefix?: string;
97
98
  /** The Notion database id. */
98
99
  database: string;
100
+ /**
101
+ * Maximum concurrent Notion API requests. Notion allows an average of 3
102
+ * requests per second per integration, so a large database must pace its
103
+ * block-tree fan-out or every request 429s. Default 3.
104
+ */
105
+ concurrency?: number;
99
106
  /** Integration token; defaults to `NOTION_TOKEN`. */
100
107
  token?: string;
101
108
  properties?: NotionPropertyMap;
@@ -141,12 +148,15 @@ const RATE_LIMITED = 429;
141
148
  const MAX_RETRIES = 4;
142
149
  const BASE_DELAY_MS = 500;
143
150
  const SECOND_MS = 1000;
151
+ const DEFAULT_CONCURRENCY = 3;
144
152
 
145
153
  /**
146
154
  * Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
147
155
  * header and otherwise backing off exponentially. A large workspace fans out
148
156
  * many concurrent block-children requests, so without this a single 429 would
149
- * reject the batch and abort the whole import.
157
+ * reject the batch and abort the whole import. The exponential wait is
158
+ * jittered so calls rate-limited together don't retry in lockstep and trip
159
+ * the limit again as a herd.
150
160
  */
151
161
  const withNotionRetry = async <T>(
152
162
  call: () => Promise<T>,
@@ -163,7 +173,9 @@ const withNotionRetry = async <T>(
163
173
  (error as { headers?: Record<string, string> }).headers?.["retry-after"]
164
174
  );
165
175
  const wait =
166
- retryAfter > 0 ? retryAfter * SECOND_MS : BASE_DELAY_MS * 2 ** attempt;
176
+ retryAfter > 0
177
+ ? retryAfter * SECOND_MS
178
+ : BASE_DELAY_MS * 2 ** attempt * (1 + Math.random());
167
179
  await sleep(wait);
168
180
  return withNotionRetry(call, attempt + 1);
169
181
  }
@@ -252,6 +264,16 @@ export const notionSource = (
252
264
  ctx?: SourceContext
253
265
  ): ContentSource => {
254
266
  const props = options.properties ?? {};
267
+ // A FIFO semaphore: at most N calls run at once, the rest queue. Notion's
268
+ // rate limit is per-integration (an average of 3 req/s), and a large
269
+ // database fans out one block-children request per page plus one per nested
270
+ // container — an unbounded burst guarantees 429s that even the retry loop
271
+ // can't recover from, so every API call funnels through this limiter.
272
+ const limit = pLimit(Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY));
273
+ // Every Notion API call goes through the limiter, inside the retry — so a
274
+ // call sleeping through a backoff doesn't hold a slot while it waits.
275
+ const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
276
+ withNotionRetry(() => limit(call));
255
277
  const cache = snapshotCache(
256
278
  ctx?.cacheDir ?? join(".blume", "cache", options.name)
257
279
  );
@@ -284,7 +306,7 @@ export const notionSource = (
284
306
  blockId: string
285
307
  ): Promise<NotionBlock[]> =>
286
308
  collectAll((cursor) =>
287
- withNotionRetry(() =>
309
+ notionCall(() =>
288
310
  client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
289
311
  )
290
312
  );
@@ -458,12 +480,12 @@ export const notionSource = (
458
480
  };
459
481
 
460
482
  // Hoisted out of `load` so the retry closure doesn't nest past the linter's
461
- // 4-level limit (source factory → queryDatabase → withNotionRetry callback).
483
+ // 4-level limit (source factory → queryDatabase → notionCall callback).
462
484
  const queryDatabase = (
463
485
  client: NotionClientLike,
464
486
  cursor?: string
465
487
  ): Promise<NotionList<NotionPage>> =>
466
- withNotionRetry(() =>
488
+ notionCall(() =>
467
489
  client.databases.query({
468
490
  database_id: options.database,
469
491
  start_cursor: cursor,
@@ -45,12 +45,27 @@ const HEADING_STYLES: Record<string, string> = {
45
45
  h6: "###### ",
46
46
  };
47
47
 
48
+ // Markdown/raw-HTML structure characters. Portable Text spans are *plain
49
+ // text* — formatting arrives as marks, never as syntax in the text — so a
50
+ // literal `*`, `_`, `[`, backtick, `~`, or `<` typed in the CMS must render
51
+ // as itself. Unescaped, it opened emphasis or a code span mid-paragraph, and
52
+ // `<` let CMS prose inject raw HTML into the rendered page. CommonMark
53
+ // backslash-escapes every ASCII punctuation character, so `\*` is always the
54
+ // literal asterisk.
55
+ const MARKDOWN_SPECIALS = /[\\`*_[\]~<]/gu;
56
+
57
+ const escapeText = (text: string): string =>
58
+ text.replaceAll(MARKDOWN_SPECIALS, String.raw`\$&`);
59
+
48
60
  /** Wrap a span's text in Markdown for its marks (decorators + link defs). */
49
61
  const renderSpan = (
50
62
  span: PortableTextSpan,
51
63
  defs: Map<string, PortableTextMarkDef>
52
64
  ): string => {
53
- let text = span.text ?? "";
65
+ // Code spans stay verbatim: their text is literal inside the backticks,
66
+ // and backslash escapes would render as backslashes.
67
+ const isCode = span.marks?.includes("code") ?? false;
68
+ let text = isCode ? (span.text ?? "") : escapeText(span.text ?? "");
54
69
  if (!span.marks || span.marks.length === 0) {
55
70
  return text;
56
71
  }
@@ -86,6 +86,7 @@ const buildSource = (
86
86
  if (def.type === "notion") {
87
87
  return notionSource(
88
88
  {
89
+ concurrency: def.concurrency,
89
90
  database: def.database,
90
91
  name,
91
92
  pollInterval: def.pollInterval,
@@ -1,4 +1,8 @@
1
- import type { FrontmatterExtend, ResolvedI18nConfig } from "../schema.ts";
1
+ import type {
2
+ FolderMeta,
3
+ FrontmatterExtend,
4
+ ResolvedI18nConfig,
5
+ } from "../schema.ts";
2
6
  import type { Diagnostic } from "../types.ts";
3
7
 
4
8
  /**
@@ -40,6 +44,14 @@ export interface SourceLoadResult {
40
44
  entries: SourceEntry[];
41
45
  /** Source-level diagnostics (e.g. an offline cache fallback warning). */
42
46
  diagnostics: Diagnostic[];
47
+ /**
48
+ * Folder meta the source derives for the sidebar groups its entries create,
49
+ * keyed by locale-stripped group path (the `meta.ts` key space). The OpenAPI
50
+ * source labels each tag directory with the spec's own tag name, so the
51
+ * sidebar shows `OAuth2`/`Größe` instead of a re-humanized slug. Merged
52
+ * beneath user-authored meta files, which always win.
53
+ */
54
+ folderMeta?: Record<string, FolderMeta>;
43
55
  }
44
56
 
45
57
  /**
@@ -276,6 +276,20 @@ const redirectFor = (pathname) => {
276
276
  return Object.hasOwn(REDIRECTS, path) ? REDIRECTS[path] : null;
277
277
  };
278
278
 
279
+ // \`_redirects\` semantics, which the static layer applies to these same
280
+ // paths: the request's query string is forwarded unless the destination
281
+ // carries its own, and a destination fragment stays after the query.
282
+ const redirectLocation = (destination, search) => {
283
+ const hashIndex = destination.indexOf("#");
284
+ const bare = hashIndex === -1 ? destination : destination.slice(0, hashIndex);
285
+ if (!search || bare.includes("?")) {
286
+ return destination;
287
+ }
288
+ return hashIndex === -1
289
+ ? bare + search
290
+ : bare + search + destination.slice(hashIndex);
291
+ };
292
+
279
293
  const parseAccept = (accept) =>
280
294
  accept.split(",").map((part) => {
281
295
  const segments = part.trim().split(";");
@@ -355,7 +369,7 @@ export default {
355
369
  const redirect = redirectFor(url.pathname);
356
370
  if (redirect !== null) {
357
371
  return new Response(null, {
358
- headers: { location: redirect[0] },
372
+ headers: { location: redirectLocation(redirect[0], url.search) },
359
373
  status: redirect[1],
360
374
  });
361
375
  }
@@ -1,6 +1,7 @@
1
1
  import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
2
2
  import type { BlumeProject } from "../core/project-graph.ts";
3
3
  import type { ContentSignalPolicy, ContentSignals } from "../core/schema.ts";
4
+ import { absoluteUrl } from "../core/site-url.ts";
4
5
 
5
6
  /**
6
7
  * Ordered mapping from config field to its `Content-Signal` token. The order
@@ -52,7 +53,7 @@ export const buildRobots = (project: BlumeProject): string | null => {
52
53
  normalizeBasePath(config.deployment.base),
53
54
  "/sitemap.xml"
54
55
  );
55
- lines.push("", `Sitemap: ${site.replace(/\/$/u, "")}${sitemapPath}`);
56
+ lines.push("", `Sitemap: ${absoluteUrl(site, sitemapPath)}`);
56
57
  }
57
58
  return `${lines.join("\n")}\n`;
58
59
  };
package/src/deploy/rss.ts CHANGED
@@ -4,6 +4,7 @@ import { escape as escapeXml } from "html-escaper";
4
4
 
5
5
  import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
6
6
  import type { BlumeProject } from "../core/project-graph.ts";
7
+ import { siteRoot } from "../core/site-url.ts";
7
8
  import type { PageRecord } from "../core/types.ts";
8
9
 
9
10
  /** A single feed entry derived from a content page. */
@@ -55,7 +56,7 @@ export const buildRssFeeds = (project: BlumeProject): RssFeed[] => {
55
56
  if (!(rss.enabled && site)) {
56
57
  return [];
57
58
  }
58
- const base = site.replace(/\/$/u, "");
59
+ const base = siteRoot(site);
59
60
  // Routes carry `basePath`; a `deployment.base` subdirectory is layered on top.
60
61
  // The feed's own `link`/self URL points at the docs root under that base, while
61
62
  // `path` stays base-less (it's also the on-disk output location).
@@ -8,6 +8,7 @@ import {
8
8
  } from "../astro/pages.ts";
9
9
  import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
10
10
  import type { BlumeProject } from "../core/project-graph.ts";
11
+ import { siteRoot } from "../core/site-url.ts";
11
12
 
12
13
  /**
13
14
  * Astro's reserved error routes. A user-authored override (`pages/404.astro`,
@@ -29,20 +30,48 @@ const lastmodTag = (value: string | undefined): string => {
29
30
  : `<lastmod>${date.toISOString().slice(0, 10)}</lastmod>`;
30
31
  };
31
32
 
33
+ /** One emitted sitemap artifact: its dist-root filename and XML body. */
34
+ export interface SitemapFile {
35
+ name: string;
36
+ xml: string;
37
+ }
38
+
32
39
  /**
33
- * Build a sitemap.xml from the route manifest plus the routes the manifest
40
+ * The sitemaps.org cap on `<url>` entries in a single file. Beyond it,
41
+ * `sitemap.xml` becomes a sitemap index pointing at numbered chunk files —
42
+ * search engines reject an oversized urlset outright.
43
+ */
44
+ const URLS_PER_FILE = 50_000;
45
+
46
+ const renderUrlset = (
47
+ urls: string[]
48
+ ): string => `<?xml version="1.0" encoding="UTF-8"?>
49
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
50
+ ${urls.join("\n")}
51
+ </urlset>
52
+ `;
53
+
54
+ /**
55
+ * Build the sitemap files from the route manifest plus the routes the manifest
34
56
  * can't see: custom `.astro` pages (most importantly a custom landing `/`) and
35
57
  * the generated `/changelog` index. Returns null when the sitemap is disabled
36
58
  * or no `site` is configured (absolute URLs are required for a valid sitemap).
37
59
  * Drafts, hidden, and `noindex` pages are excluded.
60
+ *
61
+ * Sites within the per-file URL cap get the single classic `sitemap.xml`;
62
+ * larger sites get `sitemap.xml` as a sitemap index over numbered
63
+ * `sitemap-N.xml` chunks, all served from the same directory robots.txt
64
+ * already points at.
38
65
  */
39
- export const buildSitemap = (project: BlumeProject): string | null => {
66
+ export const buildSitemapFiles = (
67
+ project: BlumeProject
68
+ ): SitemapFile[] | null => {
40
69
  const { site } = project.config.deployment;
41
70
  if (!(site && project.config.seo.sitemap)) {
42
71
  return null;
43
72
  }
44
73
 
45
- const base = site.replace(/\/$/u, "");
74
+ const base = siteRoot(site);
46
75
  // Routes carry `basePath`; a `deployment.base` subdirectory is layered on top.
47
76
  const deployBase = normalizeBasePath(project.config.deployment.base);
48
77
  const seen = new Set<string>();
@@ -87,9 +116,29 @@ export const buildSitemap = (project: BlumeProject): string | null => {
87
116
  }
88
117
  urls.sort();
89
118
 
90
- return `<?xml version="1.0" encoding="UTF-8"?>
91
- <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
92
- ${urls.join("\n")}
93
- </urlset>
119
+ if (urls.length <= URLS_PER_FILE) {
120
+ return [{ name: "sitemap.xml", xml: renderUrlset(urls) }];
121
+ }
122
+
123
+ const chunks: SitemapFile[] = [];
124
+ const references: string[] = [];
125
+ for (let start = 0; start < urls.length; start += URLS_PER_FILE) {
126
+ const name = `sitemap-${chunks.length + 1}.xml`;
127
+ chunks.push({
128
+ name,
129
+ xml: renderUrlset(urls.slice(start, start + URLS_PER_FILE)),
130
+ });
131
+ // Chunks sit next to sitemap.xml, so their URLs layer the same deployment
132
+ // base robots.txt uses for the index.
133
+ const loc = escapeXml(
134
+ encodeURI(`${base}${withBasePath(deployBase, `/${name}`)}`)
135
+ );
136
+ references.push(` <sitemap><loc>${loc}</loc></sitemap>`);
137
+ }
138
+ const index = `<?xml version="1.0" encoding="UTF-8"?>
139
+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
140
+ ${references.join("\n")}
141
+ </sitemapindex>
94
142
  `;
143
+ return [{ name: "sitemap.xml", xml: index }, ...chunks];
95
144
  };