blume 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/cli/index.js +747 -471
  2. package/dist/cli/index.js.map +45 -38
  3. package/dist/types/core/schema.d.ts +289 -278
  4. package/dist/types/migrate/mintlify/assets.d.ts +8 -0
  5. package/docs/01-quickstart.mdx +5 -16
  6. package/docs/02-deployment.mdx +21 -54
  7. package/docs/advanced/api-reference.mdx +10 -37
  8. package/docs/advanced/blog.mdx +9 -25
  9. package/docs/advanced/changelog.mdx +10 -33
  10. package/docs/advanced/custom-pages.mdx +21 -78
  11. package/docs/configuration/ai.mdx +42 -103
  12. package/docs/configuration/analytics.mdx +20 -38
  13. package/docs/configuration/customization.mdx +40 -73
  14. package/docs/configuration/export.mdx +9 -34
  15. package/docs/configuration/index.mdx +67 -87
  16. package/docs/configuration/search.mdx +17 -54
  17. package/docs/configuration/seo.mdx +17 -48
  18. package/docs/configuration/theming.mdx +20 -42
  19. package/docs/content/components.mdx +42 -101
  20. package/docs/content/i18n.mdx +21 -72
  21. package/docs/content/index.mdx +18 -48
  22. package/docs/content/islands.mdx +25 -52
  23. package/docs/content/meta.mdx +23 -50
  24. package/docs/content/navigation.mdx +23 -62
  25. package/docs/content/sources.mdx +20 -83
  26. package/docs/content/syntax.mdx +37 -105
  27. package/docs/index.mdx +11 -40
  28. package/docs/reference/cli.mdx +18 -29
  29. package/docs/reference/frontmatter.mdx +2 -5
  30. package/package.json +1 -1
  31. package/src/astro/integration.ts +26 -3
  32. package/src/astro/islands.ts +6 -2
  33. package/src/astro/markdown-negotiation.ts +17 -3
  34. package/src/astro/pages.ts +6 -1
  35. package/src/astro/static-assets.ts +117 -0
  36. package/src/astro/templates.ts +48 -26
  37. package/src/cli/args.ts +23 -0
  38. package/src/cli/commands/build.ts +23 -0
  39. package/src/cli/commands/dev.ts +11 -2
  40. package/src/cli/commands/doctor.ts +10 -1
  41. package/src/cli/commands/eject.ts +3 -1
  42. package/src/cli/commands/init.ts +21 -1
  43. package/src/cli/commands/preview.ts +2 -1
  44. package/src/cli/commands/validate.ts +12 -1
  45. package/src/cli/dev-lock.ts +84 -0
  46. package/src/cli/log.ts +11 -0
  47. package/src/components/BlumePage.astro +2 -0
  48. package/src/components/content/YouTube.astro +35 -0
  49. package/src/components/content/youtube.ts +46 -0
  50. package/src/components/islands/ask-ai.tsx +14 -14
  51. package/src/components/props.ts +3 -0
  52. package/src/core/assets.ts +31 -0
  53. package/src/core/bridge.ts +10 -0
  54. package/src/core/builtin-tags.ts +1 -0
  55. package/src/core/diagnostics.ts +6 -1
  56. package/src/core/gitignore.ts +30 -0
  57. package/src/core/links.ts +60 -19
  58. package/src/core/schema.ts +7 -0
  59. package/src/core/sources/mdx-remote.ts +54 -8
  60. package/src/core/sources/normalize.ts +6 -1
  61. package/src/core/sources/notion.ts +49 -5
  62. package/src/core/sources/sanity.ts +5 -1
  63. package/src/deploy/rss.ts +1 -8
  64. package/src/deploy/sitemap.ts +20 -1
  65. package/src/deploy/xml.ts +8 -0
  66. package/src/markdown/directives.ts +15 -7
  67. package/src/markdown/package-commands.ts +26 -4
  68. package/src/migrate/fumadocs/content.ts +14 -1
  69. package/src/migrate/fumadocs/groups.ts +7 -0
  70. package/src/migrate/fumadocs/index.ts +5 -2
  71. package/src/migrate/mintlify/assets.ts +46 -0
  72. package/src/migrate/mintlify/index.ts +53 -45
  73. package/src/migrate/shared.ts +12 -27
  74. package/src/og/card.ts +14 -2
  75. package/src/registry/eject.ts +13 -3
  76. package/src/registry/registry.ts +6 -0
  77. package/src/registry/rewrite-imports.ts +31 -19
  78. package/src/search/documents.ts +23 -5
  79. package/src/search/sync/algolia.ts +5 -1
  80. package/src/search/sync/typesense.ts +24 -16
  81. package/src/theme/palette.ts +26 -7
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Helpers for the `<YouTube>` content component. Kept in a sibling `.ts` (like
3
+ * `diff.ts`/`github-info.ts`) so the id parsing and embed-URL building are pure,
4
+ * unit-testable functions — the `.astro` file stays a thin presentational shell.
5
+ */
6
+
7
+ // A YouTube video id is 11 characters of [A-Za-z0-9_-].
8
+ const BARE_ID = /^[\w-]{11}$/u;
9
+
10
+ // Pull the id out of any common YouTube URL: youtu.be/<id>, watch?v=<id>,
11
+ // /embed/<id>, /shorts/<id>, /live/<id>.
12
+ const URL_ID =
13
+ /(?:youtu\.be\/|\/embed\/|\/shorts\/|\/live\/|[?&]v=)(?<id>[\w-]{11})/u;
14
+
15
+ /**
16
+ * Resolve a YouTube video id from either a bare id or a full URL. Returns `null`
17
+ * when nothing that looks like an id can be found, so the component can render
18
+ * nothing rather than a broken embed.
19
+ */
20
+ export const parseYouTubeId = (input: string): string | null => {
21
+ const value = input.trim();
22
+ if (!value) {
23
+ return null;
24
+ }
25
+ if (BARE_ID.test(value)) {
26
+ return value;
27
+ }
28
+ return URL_ID.exec(value)?.groups?.id ?? null;
29
+ };
30
+
31
+ /**
32
+ * Build a privacy-enhanced (`youtube-nocookie.com`) embed URL, optionally
33
+ * starting at `start` seconds.
34
+ */
35
+ export const youtubeEmbedUrl = (
36
+ id: string,
37
+ options: { start?: number } = {}
38
+ ): string => {
39
+ const base = `https://www.youtube-nocookie.com/embed/${id}`;
40
+ const { start } = options;
41
+ if (start && start > 0) {
42
+ const params = new URLSearchParams({ start: String(Math.floor(start)) });
43
+ return `${base}?${params.toString()}`;
44
+ }
45
+ return base;
46
+ };
@@ -93,21 +93,21 @@ const AskAI = ({ strings }: { strings?: UIStrings["ask"] }) => {
93
93
  headers: { "content-type": "application/json" },
94
94
  method: "POST",
95
95
  });
96
- const reader = response.body?.getReader();
96
+ // A 4xx/5xx still has a body; without this guard its error text would be
97
+ // decoded and shown as the assistant's answer instead of the error notice.
98
+ if (!(response.ok && response.body)) {
99
+ throw new Error(`Ask AI request failed (${response.status}).`);
100
+ }
101
+ const reader = response.body.getReader();
97
102
  const decoder = new TextDecoder();
98
- if (reader) {
99
- let done = false;
100
- while (!done) {
101
- // oxlint-disable-next-line no-await-in-loop -- sequential stream reads
102
- const chunk = await reader.read();
103
- ({ done } = chunk);
104
- if (chunk.value) {
105
- assistant.content += decoder.decode(chunk.value);
106
- setMessages((current) => [
107
- ...current.slice(0, -1),
108
- { ...assistant },
109
- ]);
110
- }
103
+ let done = false;
104
+ while (!done) {
105
+ // oxlint-disable-next-line no-await-in-loop -- sequential stream reads
106
+ const chunk = await reader.read();
107
+ ({ done } = chunk);
108
+ if (chunk.value) {
109
+ assistant.content += decoder.decode(chunk.value);
110
+ setMessages((current) => [...current.slice(0, -1), { ...assistant }]);
111
111
  }
112
112
  }
113
113
  } catch {
@@ -65,4 +65,7 @@ export type TileProps = ComponentProps<
65
65
  export type TooltipProps = ComponentProps<
66
66
  typeof import("./content/Tooltip.astro").default
67
67
  >;
68
+ export type YouTubeProps = ComponentProps<
69
+ typeof import("./content/YouTube.astro").default
70
+ >;
68
71
  export type IconProps = ComponentProps<typeof import("./Icon.astro").default>;
@@ -0,0 +1,31 @@
1
+ import { join } from "pathe";
2
+
3
+ /** A static directory served at a URL prefix, in addition to `public/`. */
4
+ export interface AssetMount {
5
+ /** Absolute filesystem path to the source directory (or file). */
6
+ dir: string;
7
+ /** URL path prefix the source is served at, e.g. `/images`. */
8
+ url: string;
9
+ }
10
+
11
+ /**
12
+ * Resolve `content.assets` entries (top-level dirs served at the site root,
13
+ * alongside `public/`) to `{ dir, url }` mounts. Shared by the generated Astro
14
+ * runtime (dev middleware + build copy) and by link validation, so all three
15
+ * agree on where a `/images/foo.png` reference resolves on disk.
16
+ *
17
+ * Each entry is normalized to a leading-slash URL and joined to the project
18
+ * root; leading `./` or `/` and any `..` segments are stripped so a mount can't
19
+ * escape the root or collide with the site's own routing prefix.
20
+ */
21
+ export const resolveAssetMounts = (
22
+ root: string,
23
+ assets: string[]
24
+ ): AssetMount[] =>
25
+ assets.map((entry) => {
26
+ const rel = entry
27
+ .replace(/^[./]+/u, "")
28
+ .replaceAll(/\.\.\/?/gu, "")
29
+ .replace(/\/+$/u, "");
30
+ return { dir: join(root, rel), url: `/${rel}` };
31
+ });
@@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises";
3
3
 
4
4
  import { join } from "pathe";
5
5
 
6
+ import { assetSegments } from "../migrate/mintlify/assets.ts";
6
7
  import { loadMintlifyConfig } from "../migrate/mintlify/config.ts";
7
8
  import { mintlifyI18n } from "../migrate/mintlify/i18n.ts";
8
9
  import type { BlumeConfig } from "./schema.ts";
@@ -64,11 +65,20 @@ export const detectMintlifyBridge = async (
64
65
  const root_ = config.content?.root ?? ".";
65
66
  const exclude = config.content?.exclude ?? [];
66
67
 
68
+ // Mintlify serves assets from the project root; the bridge never moves files,
69
+ // so referenced root-level asset folders (e.g. `images/`) are served in place
70
+ // via `content.assets` instead. This is the read-only twin of the migrator's
71
+ // relocation — same referenced segments, just no `public/` move.
72
+ const assets = assetSegments(config).filter(
73
+ (segment) => segment !== "public" && existsSync(join(root, segment))
74
+ );
75
+
67
76
  return {
68
77
  configFile,
69
78
  raw: {
70
79
  ...config,
71
80
  content: {
81
+ assets,
72
82
  // Mirror the excludes onto `content.exclude` too: the generated Astro
73
83
  // `docs` collection globs `content.root` (here the project root) and
74
84
  // must skip node_modules/snippets just like the source does.
@@ -36,4 +36,5 @@ export const BUILTIN_MDX_TAGS = new Set<string>([
36
36
  "Tree",
37
37
  "TypeTable",
38
38
  "Visibility",
39
+ "YouTube",
39
40
  ]);
@@ -84,7 +84,12 @@ const locatePath = (
84
84
  if (typeof segment !== "string") {
85
85
  continue;
86
86
  }
87
- const matcher = new RegExp(`${escapeRegExp(segment)}\\s*[:=]`, "gu");
87
+ // The negative lookbehind keeps a segment like `title` from matching the
88
+ // tail of an unrelated key such as `subtitle:`.
89
+ const matcher = new RegExp(
90
+ `(?<![\\w$])${escapeRegExp(segment)}\\s*[:=]`,
91
+ "gu"
92
+ );
88
93
  matcher.lastIndex = cursor;
89
94
  const match = matcher.exec(source);
90
95
  if (!match) {
@@ -0,0 +1,30 @@
1
+ import { existsSync } from "node:fs";
2
+ import { readFile, writeFile } from "node:fs/promises";
3
+
4
+ import { join } from "pathe";
5
+
6
+ /** A `.gitignore` line, normalized for comparison (trailing slashes dropped). */
7
+ const gitignoreKey = (line: string): string => line.trim().replace(/\/+$/u, "");
8
+
9
+ /**
10
+ * Ensure `.gitignore` ignores each of `entries`, appending any that are missing
11
+ * (creating the file when absent). Trailing-slash differences (`dist` vs
12
+ * `dist/`) count as already present. Returns the entries actually added.
13
+ */
14
+ export const ensureGitignore = async (
15
+ root: string,
16
+ entries: string[]
17
+ ): Promise<string[]> => {
18
+ const path = join(root, ".gitignore");
19
+ const existing = existsSync(path) ? await readFile(path, "utf-8") : "";
20
+ const present = new Set(
21
+ existing.split("\n").map(gitignoreKey).filter(Boolean)
22
+ );
23
+ const added = entries.filter((entry) => !present.has(gitignoreKey(entry)));
24
+ if (added.length === 0) {
25
+ return [];
26
+ }
27
+ const gap = existing.length > 0 && !existing.endsWith("\n") ? "\n" : "";
28
+ await writeFile(path, `${existing}${gap}${added.join("\n")}\n`, "utf-8");
29
+ return added;
30
+ };
package/src/core/links.ts CHANGED
@@ -1,7 +1,8 @@
1
1
  import { existsSync } from "node:fs";
2
2
 
3
- import { join } from "pathe";
3
+ import { basename, join } from "pathe";
4
4
 
5
+ import type { AssetMount } from "./assets.ts";
5
6
  import type {
6
7
  ContentGraph,
7
8
  Diagnostic,
@@ -37,20 +38,52 @@ interface ExternalRef extends LinkSite {
37
38
  /** Lookups derived once from the content graph. */
38
39
  interface LinkContext {
39
40
  anchors: Map<string, Set<string>>;
41
+ /** `content.assets` mounts served alongside `public/` (checked in place). */
42
+ assetMounts: AssetMount[];
40
43
  publicDir: string | null;
41
44
  /** Normalized `redirect.from` paths — valid targets that resolve at runtime. */
42
45
  redirects: Set<string>;
43
46
  routes: Set<string>;
44
47
  }
45
48
 
49
+ /** Whether a resolved asset path exists under `public/` or an asset mount. */
50
+ const assetIsPresent = (resolved: string, ctx: LinkContext): boolean => {
51
+ if (ctx.publicDir && existsSync(join(ctx.publicDir, resolved))) {
52
+ return true;
53
+ }
54
+ return ctx.assetMounts.some(
55
+ (mount) =>
56
+ (resolved === mount.url || resolved.startsWith(`${mount.url}/`)) &&
57
+ existsSync(join(mount.dir, resolved.slice(mount.url.length)))
58
+ );
59
+ };
60
+
46
61
  /** Outcome of classifying one link target. */
47
62
  type LinkResult = Diagnostic | "asset-unchecked" | null;
48
63
 
64
+ /**
65
+ * Whether a page is a directory index (`…/index.md(x)`). Its route already *is*
66
+ * its directory, so a relative link must resolve against the route itself, not
67
+ * its parent — otherwise `./sibling` from `guides/index.mdx` (route `/guides`)
68
+ * would resolve to `/sibling` and be falsely flagged as broken.
69
+ */
70
+ const isIndexPage = (page: PageRecord): boolean => {
71
+ const ref = page.source?.ref ?? page.sourcePath ?? "";
72
+ return /^index\.(?:md|mdx)$/iu.test(basename(ref));
73
+ };
74
+
49
75
  /** Resolve a relative link target against the directory of a page route. */
50
- const resolveRelative = (pageRoute: string, target: string): string => {
76
+ const resolveRelative = (
77
+ pageRoute: string,
78
+ target: string,
79
+ isIndex: boolean
80
+ ): string => {
51
81
  const segments = pageRoute.split("/").filter(Boolean);
52
- // Drop the page's own segment so links resolve against its parent directory.
53
- segments.pop();
82
+ // Drop a leaf page's own segment so links resolve against its parent
83
+ // directory. An index page's route already is its directory, so keep it.
84
+ if (!isIndex) {
85
+ segments.pop();
86
+ }
54
87
  for (const part of target.split("/")) {
55
88
  if (part === "" || part === ".") {
56
89
  continue;
@@ -114,13 +147,28 @@ const checkPathLink = (
114
147
  site: LinkSite,
115
148
  ctx: LinkContext
116
149
  ): LinkResult => {
150
+ // A real route always wins over the asset-extension heuristic, so a path
151
+ // whose last segment merely contains a dot (e.g. a page at `/releases/v1.0`)
152
+ // isn't misread as a missing asset.
153
+ const route = toRoute(resolved);
154
+ if (ctx.routes.has(route)) {
155
+ return fragment ? checkAnchor(route, fragment, site, ctx) : null;
156
+ }
157
+ // A configured `redirect.from` resolves at runtime, so it's a valid target.
158
+ // Its destination (and any anchor there) is validated on its own page, so we
159
+ // don't follow the redirect to check the fragment here.
160
+ if (ctx.redirects.has(route)) {
161
+ return null;
162
+ }
163
+
117
164
  if (FILE_EXT.test(resolved) && !DOC_EXT.test(resolved)) {
118
- if (ctx.publicDir === null) {
119
- return "asset-unchecked";
120
- }
121
- if (existsSync(join(ctx.publicDir, resolved))) {
165
+ if (assetIsPresent(resolved, ctx)) {
122
166
  return null;
123
167
  }
168
+ // Nowhere to look: no `public/` and no asset mounts configured.
169
+ if (ctx.publicDir === null && ctx.assetMounts.length === 0) {
170
+ return "asset-unchecked";
171
+ }
124
172
  return {
125
173
  ...site,
126
174
  code: "BLUME_BROKEN_ASSET",
@@ -130,16 +178,6 @@ const checkPathLink = (
130
178
  };
131
179
  }
132
180
 
133
- const route = toRoute(resolved);
134
- if (ctx.routes.has(route)) {
135
- return fragment ? checkAnchor(route, fragment, site, ctx) : null;
136
- }
137
- // A configured `redirect.from` resolves at runtime, so it's a valid target.
138
- // Its destination (and any anchor there) is validated on its own page, so we
139
- // don't follow the redirect to check the fragment here.
140
- if (ctx.redirects.has(route)) {
141
- return null;
142
- }
143
181
  return {
144
182
  ...site,
145
183
  code: "BLUME_BROKEN_LINK",
@@ -295,7 +333,7 @@ const classifyLink = (
295
333
 
296
334
  const resolved = rawPath.startsWith("/")
297
335
  ? rawPath
298
- : resolveRelative(page.route, rawPath);
336
+ : resolveRelative(page.route, rawPath, isIndexPage(page));
299
337
  return checkPathLink(resolved, fragment, target, site, ctx);
300
338
  };
301
339
 
@@ -309,12 +347,15 @@ export const validateLinks = async (
309
347
  options: {
310
348
  publicDir: string | null;
311
349
  checkExternal?: boolean;
350
+ /** `content.assets` mounts served alongside `public/`. */
351
+ assetMounts?: AssetMount[];
312
352
  /** Configured redirects; their `from` paths count as valid link targets. */
313
353
  redirects?: { from: string }[];
314
354
  }
315
355
  ): Promise<Diagnostic[]> => {
316
356
  const ctx: LinkContext = {
317
357
  anchors: buildAnchorIndex(graph.pages),
358
+ assetMounts: options.assetMounts ?? [],
318
359
  publicDir: options.publicDir,
319
360
  redirects: new Set(
320
361
  (options.redirects ?? []).map((redirect) => toRoute(redirect.from))
@@ -365,6 +365,13 @@ export type ContentSourceConfig = z.infer<typeof contentSourceSchema>;
365
365
 
366
366
  const contentConfigSchema = z
367
367
  .object({
368
+ /**
369
+ * Extra top-level directories (relative to the project root) served as
370
+ * static assets at the site root, alongside `public/`. Lets projects keep
371
+ * root-served asset folders in place — e.g. a Mintlify migration keeps
372
+ * `images/` where it is instead of relocating it under `public/`.
373
+ */
374
+ assets: z.array(z.string()).default([]),
368
375
  defaultType: z.string().default("doc"),
369
376
  exclude: z.array(z.string()).default(["**/_*", "**/.*"]),
370
377
  include: z.array(z.string()).default(["**/*.{md,mdx}"]),
@@ -1,5 +1,6 @@
1
1
  import { BlumeError } from "../diagnostics.ts";
2
2
  import matter from "../frontmatter.ts";
3
+ import type { Diagnostic } from "../types.ts";
3
4
  import {
4
5
  hashText,
5
6
  loadWithCache,
@@ -105,7 +106,7 @@ const enumerateGithub = async (
105
106
  github: { owner: string; repo: string; ref: string; path: string },
106
107
  include: string[],
107
108
  doFetch: typeof fetch
108
- ): Promise<RemoteRef[]> => {
109
+ ): Promise<{ refs: RemoteRef[]; truncated: boolean }> => {
109
110
  const { owner, repo, ref } = github;
110
111
  const base = github.path.replaceAll(/^\/|\/$/gu, "");
111
112
  const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`;
@@ -113,9 +114,12 @@ const enumerateGithub = async (
113
114
  if (!res.ok) {
114
115
  throw new Error(`${treeUrl} -> ${res.status}`);
115
116
  }
116
- const body = (await res.json()) as { tree?: GithubTreeEntry[] };
117
+ const body = (await res.json()) as {
118
+ tree?: GithubTreeEntry[];
119
+ truncated?: boolean;
120
+ };
117
121
  const prefix = base ? `${base}/` : "";
118
- return (body.tree ?? [])
122
+ const refs = (body.tree ?? [])
119
123
  .filter((node) => node.type === "blob" && node.path.startsWith(prefix))
120
124
  .map((node) => node.path.slice(prefix.length))
121
125
  .filter((rel) => matchesInclude(rel, include))
@@ -124,6 +128,9 @@ const enumerateGithub = async (
124
128
  fetchUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${prefix}${rel}`,
125
129
  ref: rel,
126
130
  }));
131
+ // GitHub caps the recursive tree response (~100k entries / 7MB) and flags it
132
+ // with `truncated`; ignoring it would silently import only part of the repo.
133
+ return { refs, truncated: body.truncated === true };
127
134
  };
128
135
 
129
136
  /**
@@ -139,19 +146,23 @@ export const mdxRemoteSource = (
139
146
  const cache = snapshotCache(ctx.cacheDir);
140
147
  let snapshot = new Map<string, SourceEntry>();
141
148
 
142
- const enumerate = async (): Promise<RemoteRef[]> => {
149
+ const enumerate = async (): Promise<{
150
+ refs: RemoteRef[];
151
+ truncated: boolean;
152
+ }> => {
143
153
  if (options.github) {
144
154
  return await enumerateGithub(options.github, options.include, doFetch);
145
155
  }
146
156
  if (options.files && options.url) {
147
157
  const base = options.url.replace(/\/$/u, "");
148
- return options.files
158
+ const refs = options.files
149
159
  .filter((ref) => matchesInclude(ref, options.include))
150
160
  .map((ref) => ({
151
161
  editUrl: `${base}/${ref}`,
152
162
  fetchUrl: `${base}/${ref}`,
153
163
  ref,
154
164
  }));
165
+ return { refs, truncated: false };
155
166
  }
156
167
  throw new BlumeError({
157
168
  code: "BLUME_SOURCE_MISCONFIGURED",
@@ -179,17 +190,52 @@ export const mdxRemoteSource = (
179
190
  };
180
191
 
181
192
  const load = async (): Promise<SourceLoadResult> => {
193
+ const skipped: Diagnostic[] = [];
182
194
  const result = await loadWithCache(
183
195
  options.name,
184
196
  cache,
185
197
  async () => {
186
- const refs = await enumerate();
187
- return await Promise.all(refs.map(fetchEntry));
198
+ const { refs, truncated } = await enumerate();
199
+ if (truncated) {
200
+ skipped.push({
201
+ code: "BLUME_SOURCE_TRUNCATED",
202
+ message: `Source "${options.name}" hit GitHub's tree listing limit; some files were not enumerated. Narrow the source path or split the repo.`,
203
+ severity: "warning",
204
+ });
205
+ }
206
+ const settled = await Promise.all(
207
+ refs.map(async (ref) => {
208
+ try {
209
+ return await fetchEntry(ref);
210
+ } catch (error) {
211
+ skipped.push({
212
+ code: "BLUME_SOURCE_FETCH_FAILED",
213
+ message: `Source "${options.name}" skipped "${ref.ref}" (${(error as Error).message}); the rest were imported.`,
214
+ severity: "warning",
215
+ });
216
+ return null;
217
+ }
218
+ })
219
+ );
220
+ const entries = settled.filter(
221
+ (entry): entry is SourceEntry => entry !== null
222
+ );
223
+ // Only a total wipeout is a hard failure — let loadWithCache fall back
224
+ // to cache or fail loudly rather than silently importing nothing. A
225
+ // partial failure keeps the healthy pages and warns about the rest.
226
+ if (refs.length > 0 && entries.length === 0) {
227
+ skipped.length = 0;
228
+ throw new Error(`all ${refs.length} remote file(s) failed to fetch`);
229
+ }
230
+ return entries;
188
231
  },
189
232
  ctx.refresh ?? true
190
233
  );
191
234
  snapshot = new Map(result.entries.map((entry) => [entry.ref, entry]));
192
- return result;
235
+ return {
236
+ ...result,
237
+ diagnostics: [...result.diagnostics, ...skipped],
238
+ };
193
239
  };
194
240
 
195
241
  const read = async (ref: string): Promise<string> => {
@@ -137,8 +137,13 @@ export const extractLinks = (body: string): PageLink[] => {
137
137
  if (target === undefined || match.index === undefined) {
138
138
  continue;
139
139
  }
140
+ // Locate the target from the `](` boundary rather than searching for the
141
+ // target text from the match start — otherwise a label that contains the
142
+ // same text (e.g. `[/a/b](/a/b)`) reports the column inside the label. The
143
+ // label can't contain `]`, so `](` is unambiguous.
144
+ const targetOffset = match.index + match[0].indexOf("](") + "](".length;
140
145
  links.push({
141
- column: line.indexOf(target, match.index) + 1,
146
+ column: targetOffset + 1,
142
147
  line: lineNumber,
143
148
  target,
144
149
  });
@@ -1,3 +1,5 @@
1
+ import { setTimeout as sleep } from "node:timers/promises";
2
+
1
3
  import { join } from "pathe";
2
4
 
3
5
  import { BlumeError } from "../diagnostics.ts";
@@ -136,6 +138,44 @@ const blockField = (block: NotionBlock): NotionRichText[] =>
136
138
  ((block[block.type] as { rich_text?: NotionRichText[] })?.rich_text ??
137
139
  []) as NotionRichText[];
138
140
 
141
+ const RATE_LIMITED = 429;
142
+ const MAX_RETRIES = 4;
143
+ const BASE_DELAY_MS = 500;
144
+ const SECOND_MS = 1000;
145
+
146
+ /**
147
+ * Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
148
+ * header and otherwise backing off exponentially. A large workspace fans out
149
+ * many concurrent block-children requests, so without this a single 429 would
150
+ * reject the batch and abort the whole import.
151
+ */
152
+ const withNotionRetry = async <T>(call: () => Promise<T>): Promise<T> => {
153
+ let lastError: unknown;
154
+ for (let attempt = 0; attempt <= MAX_RETRIES; attempt += 1) {
155
+ try {
156
+ // oxlint-disable-next-line no-await-in-loop -- sequential retry attempts
157
+ return await call();
158
+ } catch (error) {
159
+ lastError = error;
160
+ const { status } = error as { status?: number };
161
+ if (status !== RATE_LIMITED || attempt === MAX_RETRIES) {
162
+ throw error;
163
+ }
164
+ const retryAfter = Number(
165
+ (error as { headers?: Record<string, string> }).headers?.["retry-after"]
166
+ );
167
+ const wait =
168
+ retryAfter > 0 ? retryAfter * SECOND_MS : BASE_DELAY_MS * 2 ** attempt;
169
+ // oxlint-disable-next-line no-await-in-loop -- back off before retrying
170
+ await sleep(wait);
171
+ }
172
+ }
173
+ // Unreachable — the loop always returns or rethrows — but keeps types honest.
174
+ throw lastError instanceof Error
175
+ ? lastError
176
+ : new Error("Notion request failed after retries.");
177
+ };
178
+
139
179
  /** Paginate a Notion list endpoint via recursion (no await-in-loop). */
140
180
  const collectAll = async <T>(
141
181
  page: (cursor?: string) => Promise<NotionList<T>>,
@@ -251,7 +291,9 @@ export const notionSource = (
251
291
  blockId: string
252
292
  ): Promise<NotionBlock[]> =>
253
293
  collectAll((cursor) =>
254
- client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
294
+ withNotionRetry(() =>
295
+ client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
296
+ )
255
297
  );
256
298
 
257
299
  // `render` is injected (rather than referenced) so this stays a forward-free
@@ -396,10 +438,12 @@ export const notionSource = (
396
438
  async () => {
397
439
  const client = await resolveClient();
398
440
  const pages = await collectAll((cursor) =>
399
- client.databases.query({
400
- database_id: options.database,
401
- start_cursor: cursor,
402
- })
441
+ withNotionRetry(() =>
442
+ client.databases.query({
443
+ database_id: options.database,
444
+ start_cursor: cursor,
445
+ })
446
+ )
403
447
  );
404
448
  const built = await Promise.all(
405
449
  pages.map((page) => toEntry(client, page))
@@ -140,7 +140,11 @@ export const sanitySource = (
140
140
  asString(getPath(doc, fields.slug ?? "slug.current")) ??
141
141
  asString(doc._id) ??
142
142
  "untitled";
143
- const slug = slugify(slugValue) || "untitled";
143
+ // Fall back to the unique `_id` when a slug (e.g. a non-ASCII `slug.current`)
144
+ // slugifies to empty, so distinct documents don't all collapse to the same
145
+ // `untitled.md` ref and silently overwrite each other.
146
+ const slug =
147
+ slugify(slugValue) || slugify(asString(doc._id) ?? "") || "untitled";
144
148
 
145
149
  const data: Record<string, unknown> = {};
146
150
  const title = asString(getPath(doc, fields.title ?? "title"));
package/src/deploy/rss.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import type { BlumeProject } from "../core/project-graph.ts";
2
2
  import type { PageRecord } from "../core/types.ts";
3
+ import { escapeXml } from "./xml.ts";
3
4
 
4
5
  /** A single feed entry derived from a content page. */
5
6
  export interface RssItem {
@@ -85,14 +86,6 @@ export const buildRssFeeds = (project: BlumeProject): RssFeed[] => {
85
86
  return feeds;
86
87
  };
87
88
 
88
- const escapeXml = (value: string): string =>
89
- value
90
- .replaceAll("&", "&amp;")
91
- .replaceAll("<", "&lt;")
92
- .replaceAll(">", "&gt;")
93
- .replaceAll('"', "&quot;")
94
- .replaceAll("'", "&apos;");
95
-
96
89
  const renderItem = (item: RssItem): string => {
97
90
  const parts = [
98
91
  ` <title>${escapeXml(item.title)}</title>`,
@@ -1,4 +1,16 @@
1
1
  import type { BlumeProject } from "../core/project-graph.ts";
2
+ import { escapeXml } from "./xml.ts";
3
+
4
+ /** A `<lastmod>` element (W3C date) when the page has a valid modified date. */
5
+ const lastmodTag = (value: string | undefined): string => {
6
+ if (!value) {
7
+ return "";
8
+ }
9
+ const date = new Date(value);
10
+ return Number.isNaN(date.getTime())
11
+ ? ""
12
+ : `<lastmod>${date.toISOString().slice(0, 10)}</lastmod>`;
13
+ };
2
14
 
3
15
  /**
4
16
  * Build a sitemap.xml from the route manifest. Returns null when the sitemap is
@@ -17,7 +29,14 @@ export const buildSitemap = (project: BlumeProject): string | null => {
17
29
  (page) =>
18
30
  !(page.meta.draft || page.meta.sidebar.hidden || page.meta.seo.noindex)
19
31
  )
20
- .map((page) => ` <url><loc>${base}${page.route}</loc></url>`)
32
+ // `<loc>` must be a well-formed, XML-escaped URL: percent-encode the path,
33
+ // then escape XML metacharacters (notably `&`) so a route like
34
+ // `/Tips & Tricks` doesn't produce invalid XML that gets the whole sitemap
35
+ // rejected.
36
+ .map(
37
+ (page) =>
38
+ ` <url><loc>${escapeXml(encodeURI(`${base}${page.route}`))}</loc>${lastmodTag(page.lastModified)}</url>`
39
+ )
21
40
  .toSorted();
22
41
 
23
42
  return `<?xml version="1.0" encoding="UTF-8"?>
@@ -0,0 +1,8 @@
1
+ /** Escape a string for safe inclusion in XML text or attribute content. */
2
+ export const escapeXml = (value: string): string =>
3
+ value
4
+ .replaceAll("&", "&amp;")
5
+ .replaceAll("<", "&lt;")
6
+ .replaceAll(">", "&gt;")
7
+ .replaceAll('"', "&quot;")
8
+ .replaceAll("'", "&apos;");