blume 1.4.1 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/dist/cli/index.js +700 -579
  3. package/dist/cli/index.js.map +57 -56
  4. package/dist/types/core/base-path.d.ts +8 -0
  5. package/dist/types/core/config-input.d.ts +8 -0
  6. package/dist/types/core/data.d.ts +10 -0
  7. package/dist/types/core/schema.d.ts +4 -0
  8. package/dist/types/core/sources/types.d.ts +9 -1
  9. package/dist/types/openapi/references.d.ts +8 -2
  10. package/docs/configuration/ai.mdx +41 -9
  11. package/docs/content/sources.mdx +1 -1
  12. package/package.json +17 -7
  13. package/src/ai/agent-readability.ts +3 -2
  14. package/src/ai/api-catalog.ts +2 -2
  15. package/src/ai/ask-context.ts +45 -12
  16. package/src/ai/link-headers.ts +7 -2
  17. package/src/ai/llms.ts +2 -1
  18. package/src/ai/mcp/discovery.ts +25 -6
  19. package/src/ai/mcp/server.ts +108 -98
  20. package/src/ai/tar.ts +29 -70
  21. package/src/astro/examples.ts +7 -3
  22. package/src/astro/generate.ts +63 -34
  23. package/src/astro/islands.ts +7 -3
  24. package/src/astro/templates.ts +36 -9
  25. package/src/audit/agent.ts +14 -29
  26. package/src/audit/crawl.ts +41 -16
  27. package/src/audit/run.ts +10 -3
  28. package/src/audit/snapshot.ts +27 -2
  29. package/src/cli/commands/audit.ts +12 -17
  30. package/src/cli/commands/build.ts +15 -7
  31. package/src/cli/commands/dev.ts +13 -15
  32. package/src/cli/commands/eject.ts +4 -4
  33. package/src/cli/commands/eval.ts +17 -27
  34. package/src/cli/env.ts +13 -30
  35. package/src/cli/init/scaffold.ts +21 -0
  36. package/src/cli/report-format.ts +22 -0
  37. package/src/components/content/AccordionItem.astro +2 -9
  38. package/src/components/content/ColorItem.astro +5 -13
  39. package/src/components/content/Component.astro +12 -8
  40. package/src/components/content/Frame.astro +2 -12
  41. package/src/components/content/Prompt.astro +12 -31
  42. package/src/components/content/Tab.astro +2 -9
  43. package/src/components/content/Tooltip.astro +1 -9
  44. package/src/components/content/Update.astro +2 -9
  45. package/src/components/content/inline-markdown.ts +28 -0
  46. package/src/components/copy-feedback.ts +96 -0
  47. package/src/components/islands/ask-ai.tsx +78 -9
  48. package/src/components/layout/PageActions.astro +20 -32
  49. package/src/components/layout/PageLayout.astro +8 -28
  50. package/src/components/layout/RootLayout.astro +47 -48
  51. package/src/components/layout/Search.astro +56 -9
  52. package/src/components/layout/drawer-inert.ts +31 -0
  53. package/src/components/layout/search/pagefind.ts +6 -5
  54. package/src/components/layout/search/types.ts +32 -0
  55. package/src/components/openapi/panel.ts +11 -8
  56. package/src/components/raf-throttle.ts +21 -0
  57. package/src/components/slug.ts +14 -0
  58. package/src/core/base-path.ts +18 -1
  59. package/src/core/config-input.ts +8 -0
  60. package/src/core/data.ts +7 -0
  61. package/src/core/frontmatter.ts +45 -1
  62. package/src/core/probe.ts +7 -19
  63. package/src/core/project-graph.ts +12 -1
  64. package/src/core/schema.ts +6 -0
  65. package/src/core/site-url.ts +27 -0
  66. package/src/core/sources/cache.ts +10 -8
  67. package/src/core/sources/github-releases.ts +21 -1
  68. package/src/core/sources/normalize.ts +26 -2
  69. package/src/core/sources/notion.ts +27 -5
  70. package/src/core/sources/portable-text.ts +16 -1
  71. package/src/core/sources/resolve.ts +1 -0
  72. package/src/core/sources/types.ts +13 -1
  73. package/src/deploy/cloudflare-negotiation.ts +15 -1
  74. package/src/deploy/robots.ts +2 -1
  75. package/src/deploy/rss.ts +2 -1
  76. package/src/deploy/sitemap.ts +56 -7
  77. package/src/eval/agents.ts +13 -10
  78. package/src/eval/report.ts +1 -14
  79. package/src/markdown/package-commands.ts +61 -54
  80. package/src/og/card.ts +24 -26
  81. package/src/openapi/model.ts +9 -9
  82. package/src/openapi/parse.ts +69 -28
  83. package/src/openapi/references.ts +35 -12
  84. package/src/openapi/render-mdx.ts +64 -25
  85. package/src/openapi/scalar.ts +2 -2
  86. package/src/openapi/source.ts +28 -1
  87. package/src/search/documents.ts +78 -34
  88. package/src/search/orama-index.ts +51 -12
  89. package/src/theme/palette.ts +6 -2
  90. package/src/translate/ledger.ts +4 -2
  91. package/src/translate/report.ts +1 -14
  92. package/src/translate/run.ts +20 -35
  93. package/src/cli/coalesce.ts +0 -43
@@ -84,6 +84,38 @@ export const highlight = (text: string, query: string): string => {
84
84
  .join("");
85
85
  };
86
86
 
87
+ // Either a tag-shaped run — an opening `<` with a letter or `/` after it,
88
+ // through the closing `>` (or end of string for an unterminated tag) — or a
89
+ // lone `<`. A run can't span a later `<` (`[^<>]`), so between the two
90
+ // alternatives every `<` in the input lands inside a captured run.
91
+ const ANGLE_RUN = /(?<run><\/?[a-z][^<>]*>?|<)/iu;
92
+ const BARE_MARK = /^<\/?mark>$/iu;
93
+
94
+ /**
95
+ * Reduce provider-supplied excerpt markup to the `<mark>` highlighting the
96
+ * dialog expects. Remote excerpts (Pagefind's index, hosted engines) are
97
+ * rendered via `innerHTML`, so the output alphabet is pinned: bare
98
+ * `<mark>`/`</mark>` tags (attributes make even a mark untrusted), text, and
99
+ * entities. Tag-shaped runs are dropped; every other `<` is escaped, which
100
+ * renders identically but can't be parsed as markup (`<!--` would otherwise
101
+ * open a comment in `innerHTML` and swallow the rest of the excerpt). Split on
102
+ * runs covering every `<` rather than deleting tags in place: a deletion can
103
+ * splice the text around it into a fresh tag (`<<b>script>` → `<script>`),
104
+ * while here no `<` survives outside a run, so the only ones emitted are the
105
+ * bare mark tags. String-level on purpose: this also runs under DOM-less
106
+ * tests, where DOMPurify/DOMParser don't exist.
107
+ */
108
+ export const sanitizeExcerpt = (html: string): string =>
109
+ html
110
+ .split(ANGLE_RUN)
111
+ .map((part, index) => {
112
+ if (index % 2 === 0 || BARE_MARK.test(part)) {
113
+ return part;
114
+ }
115
+ return part === "<" ? "&lt;" : "";
116
+ })
117
+ .join("");
118
+
87
119
  /** First index in `text` where any query token matches (case-insensitive). */
88
120
  const matchIndex = (text: string, query: string): number => {
89
121
  const tokens = queryTokens(query);
@@ -6,6 +6,8 @@
6
6
  * theme.
7
7
  */
8
8
 
9
+ import { copyText, createCopyFlash } from "../copy-feedback.ts";
10
+
9
11
  class BlumePanelTabs extends HTMLElement {
10
12
  connectedCallback() {
11
13
  const tabs = [
@@ -36,18 +38,19 @@ class BlumePanelTabs extends HTMLElement {
36
38
  }
37
39
 
38
40
  if (copy) {
41
+ const flash = createCopyFlash((copied) => {
42
+ if (copied) {
43
+ copy.dataset.copied = "true";
44
+ } else {
45
+ delete copy.dataset.copied;
46
+ }
47
+ }, "Copied");
39
48
  copy.addEventListener("click", async () => {
40
49
  const active = panels.find(
41
50
  (panel) => !panel.classList.contains("hidden")
42
51
  );
43
- try {
44
- await navigator.clipboard.writeText(active?.textContent ?? "");
45
- copy.dataset.copied = "true";
46
- setTimeout(() => {
47
- delete copy.dataset.copied;
48
- }, 1500);
49
- } catch {
50
- // Clipboard unavailable (insecure context); silently ignore.
52
+ if (await copyText(active?.textContent ?? "")) {
53
+ flash();
51
54
  }
52
55
  });
53
56
  }
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Coalesce a high-frequency event handler (resize, scroll) into at most one
3
+ * call per animation frame — the toc-element scroll pattern, shared. Calls
4
+ * landing while a frame is pending are dropped; the handler runs once on the
5
+ * next frame with the latest state. Layout reads inside `fn` then happen once
6
+ * per frame instead of once per event, without the settle lag a debounce
7
+ * would add to position-tracking handlers.
8
+ */
9
+ export const rafThrottle = (fn: () => void): (() => void) => {
10
+ let ticking = false;
11
+ return () => {
12
+ if (ticking) {
13
+ return;
14
+ }
15
+ ticking = true;
16
+ requestAnimationFrame(() => {
17
+ ticking = false;
18
+ fn();
19
+ });
20
+ };
21
+ };
@@ -0,0 +1,14 @@
1
+ import { slug } from "github-slugger";
2
+
3
+ /**
4
+ * Slug a component's title into a DOM id (accordions, tabs, changelog
5
+ * updates), replacing three identical per-component copies. github-slugger —
6
+ * already what heading anchors use, both here and in Satteri's own
7
+ * heading-ids — so a component id slugs exactly like a heading with the same
8
+ * text (unicode letters kept, `user_id` keeps its underscore, `C#` keeps
9
+ * nothing extra dropped). Stateless on purpose: components render across many
10
+ * pages in one build process, so a stateful slugger would leak duplicate
11
+ * suffixes between pages — same-page duplicates are de-duplicated client-side
12
+ * by each component's own script.
13
+ */
14
+ export const componentSlug = (value: string): string => slug(value);
@@ -12,6 +12,8 @@
12
12
  * `components/islands/base-path.ts` and serves `deployment.base` via `BASE_URL`.
13
13
  */
14
14
 
15
+ import { trimEnd } from "./trim.ts";
16
+
15
17
  /**
16
18
  * Canonicalize a configured base path to either `""` (none) or `/seg[/seg…]`
17
19
  * (leading slash, no trailing slash, collapsed inner slashes). A blank value or
@@ -33,10 +35,25 @@ export const normalizeBasePath = (input?: string): string => {
33
35
  * `/docs` and `/docs/` as the same page) and collapse an empty path to `/`.
34
36
  */
35
37
  export const normalizePath = (path: string): string => {
36
- const trimmed = path.replace(/\/+$/u, "");
38
+ const trimmed = trimEnd(path, "/");
37
39
  return trimmed === "" ? "/" : trimmed;
38
40
  };
39
41
 
42
+ /**
43
+ * Canonicalize a route-ish string (a configured route, a page path, an agent-
44
+ * supplied route) to `/` or `/seg[/seg…]`: trimmed, exactly one leading slash,
45
+ * no trailing slash. The shared spelling of what openapi/references,
46
+ * ai/ask-context, and ai/mcp/server each hand-rolled with slightly different
47
+ * regexes.
48
+ */
49
+ export const normalizeRoute = (input: string): string => {
50
+ const trimmed = trimEnd(input.trim(), "/");
51
+ if (trimmed === "") {
52
+ return "/";
53
+ }
54
+ return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
55
+ };
56
+
40
57
  /**
41
58
  * Whether a link target is a root-relative internal path (`/x`) — the only
42
59
  * shape a base path applies to. Protocol-relative (`//host`), absolute URLs,
@@ -205,6 +205,8 @@ export interface SanitySource {
205
205
  /** A Notion database; pages become entries, blocks become MDX. */
206
206
  export interface NotionSource {
207
207
  type: "notion";
208
+ /** Max concurrent Notion API requests; default 3 (Notion's per-integration pace). */
209
+ concurrency?: number;
208
210
  /** Notion database id. */
209
211
  database: string;
210
212
  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
@@ -653,6 +655,12 @@ export interface AskConfig {
653
655
  * limiting, and streaming. Accepts an absolute URL or root-relative path.
654
656
  */
655
657
  endpoint?: string;
658
+ /**
659
+ * Extra system-prompt text appended to the built-in instructions — use it
660
+ * for identity, language, or tone. The built-in grounding behavior (answer
661
+ * from the retrieved excerpts, cite pages as Markdown links) is preserved.
662
+ */
663
+ instructions?: string;
656
664
  /** Model id to use. Defaults to `openai/gpt-5.5`. */
657
665
  model?: string;
658
666
  /** Which backend routes the request. Defaults to `gateway`. */
package/src/core/data.ts CHANGED
@@ -107,6 +107,13 @@ export interface BlumeDataConfig {
107
107
  /** `dateFormat`: `Intl.DateTimeFormat` options for the date stamps. */
108
108
  dateFormat: ResolvedConfig["dateFormat"];
109
109
  description: string | undefined;
110
+ /**
111
+ * Which agent-discovery resources exist for the layout to advertise in every
112
+ * page's `<head>` (`seo.agentReadability`, `ai.llmsTxt.enabled`) — the HTML
113
+ * counterpart of the homepage-only HTTP `Link` header, for agents that enter
114
+ * on a deep page (see `ai/link-headers.ts`).
115
+ */
116
+ discovery: { agentReadability: boolean; llmsTxt: boolean };
110
117
  favicon: BlumeFavicon;
111
118
  feedback: boolean;
112
119
  i18n: BlumeDataI18n | null;
@@ -28,13 +28,57 @@ const withYamlEngine = <O>(options: O): O =>
28
28
  },
29
29
  }) as O;
30
30
 
31
+ /**
32
+ * True when a document's leading `---` line is a CommonMark thematic break,
33
+ * not a front matter fence. Two shapes qualify (mirroring
34
+ * `linesWithoutFrontMatter` in `sources/normalize.ts`):
35
+ * - the next line is blank (or absent) — YAML metadata starts on the very
36
+ * next line, so a gap means the body *opens* with a divider (e.g. a
37
+ * Notion page whose first block is one);
38
+ * - no closing `---` line follows — gray-matter would swallow the whole
39
+ * document as one unclosed YAML block and hand it to js-yaml, which
40
+ * crashes on ordinary Markdown (`> quote` → "a line break is expected").
41
+ */
42
+ const opensWithThematicBreak = (input: string): boolean => {
43
+ const [first = "", second] = input.split(/\r?\n/u, 2);
44
+ if (!/^-{3}\s*$/u.test(first)) {
45
+ return false;
46
+ }
47
+ if (second === undefined || second.trim() === "") {
48
+ return true;
49
+ }
50
+ // gray-matter closes the block at the next line-leading `---`; matching its
51
+ // search exactly keeps this guard from firing on any document it parses.
52
+ return !input.includes("\n---", 1);
53
+ };
54
+
55
+ /**
56
+ * The parse result for a document with no front matter: the input passes
57
+ * through as content, untouched. Shaped like gray-matter's own no-matter
58
+ * result (every Blume call site reads only `content` and `data`).
59
+ */
60
+ const passthrough = (input: string): ReturnType<typeof baseMatter> =>
61
+ ({
62
+ content: input,
63
+ data: {},
64
+ excerpt: "",
65
+ isEmpty: false,
66
+ language: "",
67
+ matter: "",
68
+ orig: input,
69
+ // Recomposing a file with no matter and empty data is the content itself.
70
+ stringify: (): string => input,
71
+ }) as unknown as ReturnType<typeof baseMatter>;
72
+
31
73
  // Every helper that parses or emits YAML (`read`, `stringify`) must be
32
74
  // re-wrapped here — Object.assign copies gray-matter's own helpers, which use
33
75
  // its default `safeLoad` engine and would reintroduce the crash. `test` only
34
76
  // checks for a delimiter, so the copied original is safe.
35
77
  const matter = Object.assign(
36
78
  (input: MatterInput, options?: MatterOptions) =>
37
- baseMatter(input, withYamlEngine(options)),
79
+ typeof input === "string" && opensWithThematicBreak(input)
80
+ ? passthrough(input)
81
+ : baseMatter(input, withYamlEngine(options)),
38
82
  baseMatter,
39
83
  {
40
84
  read: (filepath: ReadArgs[0], options?: ReadArgs[1]) =>
package/src/core/probe.ts CHANGED
@@ -1,3 +1,5 @@
1
+ import pMap from "p-map";
2
+
1
3
  import type { DiagnosticSeverity } from "./types.ts";
2
4
 
3
5
  export const PROBE_CONCURRENCY = 8;
@@ -113,24 +115,10 @@ export const probeAll = async (
113
115
  options: { concurrency?: number; timeoutMs?: number } = {}
114
116
  ): Promise<Map<string, ProbeResult>> => {
115
117
  const unique = [...new Set(urls)];
116
- const results = new Map<string, ProbeResult>();
117
- const limit = Math.min(
118
- options.concurrency ?? PROBE_CONCURRENCY,
119
- unique.length
118
+ const entries = await pMap(
119
+ unique,
120
+ async (url) => [url, await probe(url, options)] as const,
121
+ { concurrency: Math.max(1, options.concurrency ?? PROBE_CONCURRENCY) }
120
122
  );
121
-
122
- let cursor = 0;
123
- const worker = async (): Promise<void> => {
124
- while (cursor < unique.length) {
125
- const url = unique[cursor];
126
- cursor += 1;
127
- if (url !== undefined) {
128
- // oxlint-disable-next-line no-await-in-loop -- bounded-concurrency pool
129
- results.set(url, await probe(url, options));
130
- }
131
- }
132
- };
133
- await Promise.all(Array.from({ length: limit }, worker));
134
-
135
- return results;
123
+ return new Map(entries);
136
124
  };
@@ -262,6 +262,17 @@ export const scanProject = async (
262
262
  discoverFolderMeta(metaSources, { localeDirs }),
263
263
  ]);
264
264
 
265
+ // Folder meta contributed by the sources themselves (the OpenAPI source
266
+ // labels each tag directory with the spec's own tag name). It applies to
267
+ // every locale, so it merges into the shared map — beneath user-authored
268
+ // entries, which are spread last and win.
269
+ const sharedFolderMeta = new Map([
270
+ ...loaded.flatMap(({ folderMeta: sourceMeta }) =>
271
+ Object.entries(sourceMeta ?? {})
272
+ ),
273
+ ...folderMeta.shared,
274
+ ]);
275
+
265
276
  const {
266
277
  diagnostics: contentDiagnostics,
267
278
  droppedPages,
@@ -301,7 +312,7 @@ export const scanProject = async (
301
312
  folderMeta: folderMeta.meta,
302
313
  i18n: config.i18n,
303
314
  navigation: config.navigation,
304
- sharedFolderMeta: folderMeta.shared,
315
+ sharedFolderMeta,
305
316
  });
306
317
  const manifest = buildManifest({ config, context, graph });
307
318
 
@@ -304,6 +304,8 @@ const sanitySourceSchema = z.object({
304
304
 
305
305
  /** A Notion database; pages become entries, blocks become MDX. */
306
306
  const notionSourceSchema = z.object({
307
+ /** Max concurrent Notion API requests; default 3 (Notion's per-integration pace). */
308
+ concurrency: z.number().positive().optional(),
307
309
  database: z.string(),
308
310
  /** Opt-in dev polling interval (seconds); omit to freeze for the session. */
309
311
  pollInterval: z.number().positive().optional(),
@@ -780,6 +782,10 @@ const aiConfigSchema = z.strictObject({
780
782
  // and host Ask AI in an existing backend. Absolute URLs and root-relative
781
783
  // paths are both valid; the built-in request/stream contract is unchanged.
782
784
  endpoint: askEndpointSchema.optional(),
785
+ // Extra system-prompt text (identity, language, tone) appended to the
786
+ // built-in instructions, so the grounding contract — answer from the
787
+ // retrieved excerpts, cite pages as Markdown links — stays intact.
788
+ instructions: z.string().trim().min(1).optional(),
783
789
  model: z.string().default("openai/gpt-5.5"),
784
790
  provider: z.enum(askAiProviders).default("gateway"),
785
791
  // Empty-state prompts shown before the first question. Each renders as a
@@ -0,0 +1,27 @@
1
+ import { joinURL } from "ufo";
2
+
3
+ import { trimEnd } from "./trim.ts";
4
+
5
+ /**
6
+ * Absolute-URL building for the configured `deployment.site`, shared by every
7
+ * emitter that prints site URLs (sitemap, RSS, robots, llms.txt, the MCP and
8
+ * agent-discovery documents). One implementation replaces eight per-file
9
+ * copies that had drifted across three different trailing-slash treatments.
10
+ *
11
+ * Deliberately not `new URL(path, site)`: a root-absolute path would drop the
12
+ * base path of a subpath deployment (`acme.com/docs`). ufo's `joinURL` joins
13
+ * without that footgun; the site is first trimmed with the ReDoS-safe
14
+ * `trimEnd` loop so even a malformed `site` with piled-up trailing slashes
15
+ * joins cleanly.
16
+ */
17
+
18
+ /** The configured site with any trailing slashes dropped. */
19
+ export const siteRoot = (site: string): string => trimEnd(site, "/");
20
+
21
+ /** `site` + root-absolute `path` (already carrying any deployment base). */
22
+ export const absoluteUrl = (site: string, path: string): string => {
23
+ const root = siteRoot(site);
24
+ // joinURL folds a lone "/" away entirely; the homepage keeps its slash
25
+ // (`https://example.com/`), matching what every emitter always printed.
26
+ return path === "/" ? `${root}/` : joinURL(root, path);
27
+ };
@@ -1,3 +1,4 @@
1
+ import { createHash } from "node:crypto";
1
2
  import { mkdir, readFile, writeFile } from "node:fs/promises";
2
3
 
3
4
  import { join } from "pathe";
@@ -6,14 +7,15 @@ import { BlumeError } from "../diagnostics.ts";
6
7
  import type { Diagnostic } from "../types.ts";
7
8
  import type { SourceEntry, SourceLoadResult } from "./types.ts";
8
9
 
9
- /** Small, stable content hash for cache/HMR bookkeeping (non-bitwise). */
10
- export const hashText = (text: string): string => {
11
- let hash = 5381;
12
- for (let i = 0; i < text.length; i += 1) {
13
- hash = (hash * 33 + (text.codePointAt(i) ?? 0)) % 2_147_483_647;
14
- }
15
- return hash.toString(36);
16
- };
10
+ /**
11
+ * Small, stable content hash for cache/HMR bookkeeping — and for staged asset
12
+ * *filenames* (see sources/assets.ts and content-assets.ts), where a collision
13
+ * silently serves the wrong file. 64 bits of SHA-256 keeps those names
14
+ * collision-safe at any realistic asset count; the old 31-bit DJB2 hash had a
15
+ * ~46k-item birthday bound.
16
+ */
17
+ export const hashText = (text: string): string =>
18
+ createHash("sha256").update(text).digest("hex").slice(0, 16);
17
19
 
18
20
  /** A stable digest of a source's entries, for change detection while polling. */
19
21
  export const entriesDigest = (entries: SourceEntry[]): string =>
@@ -85,6 +85,26 @@ const NON_PROSE = new Set(["code", "heading", "html", "thematicBreak"]);
85
85
  * content kept — then cut at a word boundary to fit the search snippet cap.
86
86
  * Undefined when the notes have no prose at all.
87
87
  */
88
+ /**
89
+ * The longest prefix of `text` that fits `max` UTF-16 units without cutting
90
+ * inside a grapheme cluster. A bare `String#slice` counts code units, so it
91
+ * can split a surrogate pair (emitting a lone surrogate — invalid Unicode —
92
+ * into a meta description) or halve an emoji sequence. Grapheme segmentation
93
+ * is rule-based (UAX #29), so unlike word segmentation it does not drift
94
+ * across ICU builds.
95
+ */
96
+ const graphemePrefix = (text: string, max: number): string => {
97
+ let end = 0;
98
+ const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
99
+ for (const { index, segment } of graphemes.segment(text)) {
100
+ if (index + segment.length > max) {
101
+ break;
102
+ }
103
+ end = index + segment.length;
104
+ }
105
+ return text.slice(0, end);
106
+ };
107
+
88
108
  const releaseDescription = (body: string): string | undefined => {
89
109
  const tree = fromMarkdown(body.replaceAll(CHANGESET_HASH, "$<mark>"), {
90
110
  extensions: [gfm()],
@@ -108,7 +128,7 @@ const releaseDescription = (body: string): string | undefined => {
108
128
  }
109
129
  // Cut before the cap at a word boundary (kept only when it doesn't drop the
110
130
  // summary under the minimum), shed any dangling punctuation, and mark the cut.
111
- const slice = text.slice(0, DESCRIPTION_MAX - 1);
131
+ const slice = graphemePrefix(text, DESCRIPTION_MAX - 1);
112
132
  const boundary = slice.lastIndexOf(" ");
113
133
  const head = (
114
134
  boundary >= DESCRIPTION_MIN ? slice.slice(0, boundary) : slice
@@ -28,12 +28,19 @@ const groupLabel = (segment: string): string | null =>
28
28
  * anchor ids are *not* slugged here — they use a `github-slugger` in
29
29
  * {@link extractHeadings}, matching the renderer (see `markdown/heading-anchors`)
30
30
  * so `blume validate` checks anchors against the exact rendered heading ids.
31
+ *
32
+ * The keep-class is Unicode letters/marks/numbers, not `\w`: ASCII slugs are
33
+ * unchanged, but a CJK/Cyrillic/accented slug keeps its characters instead of
34
+ * collapsing to `""` (which forced Sanity/Notion routes onto their opaque
35
+ * document-id fallbacks) or dropping accents (`café` → `caf`). NFC first so a
36
+ * macOS-NFD `é` (e + combining mark) slugs identically to the composed form.
31
37
  */
32
38
  export const slugify = (text: string): string =>
33
39
  text
40
+ .normalize("NFC")
34
41
  .toLowerCase()
35
42
  .trim()
36
- .replaceAll(/[^\w\s-]/gu, "")
43
+ .replaceAll(/[^\p{L}\p{M}\p{N}\s_-]/gu, "")
37
44
  .replaceAll(/[\s_]+/gu, "-")
38
45
  .replaceAll(/-+/gu, "-")
39
46
  .replaceAll(/^-|-$/gu, "");
@@ -55,6 +62,18 @@ const titleCase = (value: string): string =>
55
62
  .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
56
63
  .join(" ");
57
64
 
65
+ /**
66
+ * Strip characters that cannot survive the route → URL → output-file round
67
+ * trip. A `:` ahead of the first `/` makes `new URL()` read the segment as a
68
+ * scheme (`Guide: Architecture.md` → `guide:`), which crashes Astro's
69
+ * prerender write with "The URL must be of scheme file"; control characters
70
+ * (an embedded newline in a filename) are silently dropped by the URL parser,
71
+ * desyncing the route from its output path. Both are legal in macOS/Linux
72
+ * filenames, so they are removed here rather than rejected.
73
+ */
74
+ const sanitizeSegment = (segment: string): string =>
75
+ segment.replaceAll(/[:\p{Cc}]/gu, "");
76
+
58
77
  /** Fold one raw path part into the accumulating route segments/groups. */
59
78
  const addRouteSegment = (
60
79
  part: string,
@@ -75,7 +94,12 @@ const addRouteSegment = (
75
94
  if (clean === "index") {
76
95
  return;
77
96
  }
78
- segments.push(clean);
97
+ const safe = sanitizeSegment(clean);
98
+ // A part that was nothing but stripped characters cannot name a segment.
99
+ if (safe === "") {
100
+ return;
101
+ }
102
+ segments.push(safe);
79
103
  };
80
104
 
81
105
  /** Convert a content-root-relative path into URL + nav metadata. */
@@ -1,5 +1,6 @@
1
1
  import { setTimeout as sleep } from "node:timers/promises";
2
2
 
3
+ import pLimit from "p-limit";
3
4
  import { join } from "pathe";
4
5
 
5
6
  import { BlumeError } from "../diagnostics.ts";
@@ -96,6 +97,12 @@ export interface NotionSourceOptions {
96
97
  prefix?: string;
97
98
  /** The Notion database id. */
98
99
  database: string;
100
+ /**
101
+ * Maximum concurrent Notion API requests. Notion allows an average of 3
102
+ * requests per second per integration, so a large database must pace its
103
+ * block-tree fan-out or every request 429s. Default 3.
104
+ */
105
+ concurrency?: number;
99
106
  /** Integration token; defaults to `NOTION_TOKEN`. */
100
107
  token?: string;
101
108
  properties?: NotionPropertyMap;
@@ -141,12 +148,15 @@ const RATE_LIMITED = 429;
141
148
  const MAX_RETRIES = 4;
142
149
  const BASE_DELAY_MS = 500;
143
150
  const SECOND_MS = 1000;
151
+ const DEFAULT_CONCURRENCY = 3;
144
152
 
145
153
  /**
146
154
  * Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
147
155
  * header and otherwise backing off exponentially. A large workspace fans out
148
156
  * many concurrent block-children requests, so without this a single 429 would
149
- * reject the batch and abort the whole import.
157
+ * reject the batch and abort the whole import. The exponential wait is
158
+ * jittered so calls rate-limited together don't retry in lockstep and trip
159
+ * the limit again as a herd.
150
160
  */
151
161
  const withNotionRetry = async <T>(
152
162
  call: () => Promise<T>,
@@ -163,7 +173,9 @@ const withNotionRetry = async <T>(
163
173
  (error as { headers?: Record<string, string> }).headers?.["retry-after"]
164
174
  );
165
175
  const wait =
166
- retryAfter > 0 ? retryAfter * SECOND_MS : BASE_DELAY_MS * 2 ** attempt;
176
+ retryAfter > 0
177
+ ? retryAfter * SECOND_MS
178
+ : BASE_DELAY_MS * 2 ** attempt * (1 + Math.random());
167
179
  await sleep(wait);
168
180
  return withNotionRetry(call, attempt + 1);
169
181
  }
@@ -252,6 +264,16 @@ export const notionSource = (
252
264
  ctx?: SourceContext
253
265
  ): ContentSource => {
254
266
  const props = options.properties ?? {};
267
+ // A FIFO semaphore: at most N calls run at once, the rest queue. Notion's
268
+ // rate limit is per-integration (an average of 3 req/s), and a large
269
+ // database fans out one block-children request per page plus one per nested
270
+ // container — an unbounded burst guarantees 429s that even the retry loop
271
+ // can't recover from, so every API call funnels through this limiter.
272
+ const limit = pLimit(Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY));
273
+ // Every Notion API call goes through the limiter, inside the retry — so a
274
+ // call sleeping through a backoff doesn't hold a slot while it waits.
275
+ const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
276
+ withNotionRetry(() => limit(call));
255
277
  const cache = snapshotCache(
256
278
  ctx?.cacheDir ?? join(".blume", "cache", options.name)
257
279
  );
@@ -284,7 +306,7 @@ export const notionSource = (
284
306
  blockId: string
285
307
  ): Promise<NotionBlock[]> =>
286
308
  collectAll((cursor) =>
287
- withNotionRetry(() =>
309
+ notionCall(() =>
288
310
  client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
289
311
  )
290
312
  );
@@ -458,12 +480,12 @@ export const notionSource = (
458
480
  };
459
481
 
460
482
  // Hoisted out of `load` so the retry closure doesn't nest past the linter's
461
- // 4-level limit (source factory → queryDatabase → withNotionRetry callback).
483
+ // 4-level limit (source factory → queryDatabase → notionCall callback).
462
484
  const queryDatabase = (
463
485
  client: NotionClientLike,
464
486
  cursor?: string
465
487
  ): Promise<NotionList<NotionPage>> =>
466
- withNotionRetry(() =>
488
+ notionCall(() =>
467
489
  client.databases.query({
468
490
  database_id: options.database,
469
491
  start_cursor: cursor,
@@ -45,12 +45,27 @@ const HEADING_STYLES: Record<string, string> = {
45
45
  h6: "###### ",
46
46
  };
47
47
 
48
+ // Markdown/raw-HTML structure characters. Portable Text spans are *plain
49
+ // text* — formatting arrives as marks, never as syntax in the text — so a
50
+ // literal `*`, `_`, `[`, backtick, `~`, or `<` typed in the CMS must render
51
+ // as itself. Unescaped, it opened emphasis or a code span mid-paragraph, and
52
+ // `<` let CMS prose inject raw HTML into the rendered page. CommonMark
53
+ // backslash-escapes every ASCII punctuation character, so `\*` is always the
54
+ // literal asterisk.
55
+ const MARKDOWN_SPECIALS = /[\\`*_[\]~<]/gu;
56
+
57
+ const escapeText = (text: string): string =>
58
+ text.replaceAll(MARKDOWN_SPECIALS, String.raw`\$&`);
59
+
48
60
  /** Wrap a span's text in Markdown for its marks (decorators + link defs). */
49
61
  const renderSpan = (
50
62
  span: PortableTextSpan,
51
63
  defs: Map<string, PortableTextMarkDef>
52
64
  ): string => {
53
- let text = span.text ?? "";
65
+ // Code spans stay verbatim: their text is literal inside the backticks,
66
+ // and backslash escapes would render as backslashes.
67
+ const isCode = span.marks?.includes("code") ?? false;
68
+ let text = isCode ? (span.text ?? "") : escapeText(span.text ?? "");
54
69
  if (!span.marks || span.marks.length === 0) {
55
70
  return text;
56
71
  }
@@ -86,6 +86,7 @@ const buildSource = (
86
86
  if (def.type === "notion") {
87
87
  return notionSource(
88
88
  {
89
+ concurrency: def.concurrency,
89
90
  database: def.database,
90
91
  name,
91
92
  pollInterval: def.pollInterval,
@@ -1,4 +1,8 @@
1
- import type { FrontmatterExtend, ResolvedI18nConfig } from "../schema.ts";
1
+ import type {
2
+ FolderMeta,
3
+ FrontmatterExtend,
4
+ ResolvedI18nConfig,
5
+ } from "../schema.ts";
2
6
  import type { Diagnostic } from "../types.ts";
3
7
 
4
8
  /**
@@ -40,6 +44,14 @@ export interface SourceLoadResult {
40
44
  entries: SourceEntry[];
41
45
  /** Source-level diagnostics (e.g. an offline cache fallback warning). */
42
46
  diagnostics: Diagnostic[];
47
+ /**
48
+ * Folder meta the source derives for the sidebar groups its entries create,
49
+ * keyed by locale-stripped group path (the `meta.ts` key space). The OpenAPI
50
+ * source labels each tag directory with the spec's own tag name, so the
51
+ * sidebar shows `OAuth2`/`Größe` instead of a re-humanized slug. Merged
52
+ * beneath user-authored meta files, which always win.
53
+ */
54
+ folderMeta?: Record<string, FolderMeta>;
43
55
  }
44
56
 
45
57
  /**
@@ -276,6 +276,20 @@ const redirectFor = (pathname) => {
276
276
  return Object.hasOwn(REDIRECTS, path) ? REDIRECTS[path] : null;
277
277
  };
278
278
 
279
+ // \`_redirects\` semantics, which the static layer applies to these same
280
+ // paths: the request's query string is forwarded unless the destination
281
+ // carries its own, and a destination fragment stays after the query.
282
+ const redirectLocation = (destination, search) => {
283
+ const hashIndex = destination.indexOf("#");
284
+ const bare = hashIndex === -1 ? destination : destination.slice(0, hashIndex);
285
+ if (!search || bare.includes("?")) {
286
+ return destination;
287
+ }
288
+ return hashIndex === -1
289
+ ? bare + search
290
+ : bare + search + destination.slice(hashIndex);
291
+ };
292
+
279
293
  const parseAccept = (accept) =>
280
294
  accept.split(",").map((part) => {
281
295
  const segments = part.trim().split(";");
@@ -355,7 +369,7 @@ export default {
355
369
  const redirect = redirectFor(url.pathname);
356
370
  if (redirect !== null) {
357
371
  return new Response(null, {
358
- headers: { location: redirect[0] },
372
+ headers: { location: redirectLocation(redirect[0], url.search) },
359
373
  status: redirect[1],
360
374
  });
361
375
  }