blume 1.4.2 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/dist/cli/index.js +694 -579
- package/dist/cli/index.js.map +56 -55
- package/dist/types/core/base-path.d.ts +8 -0
- package/dist/types/core/config-input.d.ts +8 -0
- package/dist/types/core/schema.d.ts +4 -0
- package/dist/types/core/sources/types.d.ts +9 -1
- package/dist/types/openapi/references.d.ts +8 -2
- package/docs/configuration/ai.mdx +26 -8
- package/docs/content/sources.mdx +1 -1
- package/package.json +11 -1
- package/src/ai/agent-readability.ts +3 -2
- package/src/ai/api-catalog.ts +2 -2
- package/src/ai/ask-context.ts +45 -12
- package/src/ai/llms.ts +2 -1
- package/src/ai/mcp/discovery.ts +25 -6
- package/src/ai/mcp/server.ts +108 -98
- package/src/ai/tar.ts +29 -70
- package/src/astro/examples.ts +7 -3
- package/src/astro/generate.ts +59 -34
- package/src/astro/islands.ts +7 -3
- package/src/astro/templates.ts +34 -9
- package/src/audit/agent.ts +14 -29
- package/src/audit/crawl.ts +41 -16
- package/src/audit/run.ts +10 -3
- package/src/audit/snapshot.ts +27 -2
- package/src/cli/commands/audit.ts +12 -17
- package/src/cli/commands/build.ts +15 -7
- package/src/cli/commands/dev.ts +13 -15
- package/src/cli/commands/eject.ts +4 -4
- package/src/cli/commands/eval.ts +17 -27
- package/src/cli/env.ts +13 -30
- package/src/cli/init/scaffold.ts +21 -0
- package/src/cli/report-format.ts +22 -0
- package/src/components/content/AccordionItem.astro +2 -9
- package/src/components/content/ColorItem.astro +5 -13
- package/src/components/content/Component.astro +12 -8
- package/src/components/content/Frame.astro +2 -12
- package/src/components/content/Prompt.astro +12 -31
- package/src/components/content/Tab.astro +2 -9
- package/src/components/content/Tooltip.astro +1 -9
- package/src/components/content/Update.astro +2 -9
- package/src/components/content/inline-markdown.ts +28 -0
- package/src/components/copy-feedback.ts +96 -0
- package/src/components/islands/ask-ai.tsx +78 -9
- package/src/components/layout/PageActions.astro +20 -32
- package/src/components/layout/PageLayout.astro +8 -28
- package/src/components/layout/RootLayout.astro +6 -48
- package/src/components/layout/Search.astro +56 -9
- package/src/components/layout/drawer-inert.ts +31 -0
- package/src/components/layout/search/pagefind.ts +6 -5
- package/src/components/layout/search/types.ts +32 -0
- package/src/components/openapi/panel.ts +11 -8
- package/src/components/raf-throttle.ts +21 -0
- package/src/components/slug.ts +14 -0
- package/src/core/base-path.ts +18 -1
- package/src/core/config-input.ts +8 -0
- package/src/core/frontmatter.ts +45 -1
- package/src/core/probe.ts +7 -19
- package/src/core/project-graph.ts +12 -1
- package/src/core/schema.ts +6 -0
- package/src/core/site-url.ts +27 -0
- package/src/core/sources/cache.ts +10 -8
- package/src/core/sources/github-releases.ts +21 -1
- package/src/core/sources/normalize.ts +26 -2
- package/src/core/sources/notion.ts +27 -5
- package/src/core/sources/portable-text.ts +16 -1
- package/src/core/sources/resolve.ts +1 -0
- package/src/core/sources/types.ts +13 -1
- package/src/deploy/cloudflare-negotiation.ts +15 -1
- package/src/deploy/robots.ts +2 -1
- package/src/deploy/rss.ts +2 -1
- package/src/deploy/sitemap.ts +56 -7
- package/src/eval/agents.ts +13 -10
- package/src/eval/report.ts +1 -14
- package/src/markdown/package-commands.ts +61 -54
- package/src/og/card.ts +24 -26
- package/src/openapi/model.ts +9 -9
- package/src/openapi/parse.ts +69 -28
- package/src/openapi/references.ts +35 -12
- package/src/openapi/render-mdx.ts +64 -25
- package/src/openapi/scalar.ts +2 -2
- package/src/openapi/source.ts +28 -1
- package/src/search/documents.ts +78 -34
- package/src/search/orama-index.ts +51 -12
- package/src/theme/palette.ts +6 -2
- package/src/translate/ledger.ts +4 -2
- package/src/translate/report.ts +1 -14
- package/src/translate/run.ts +20 -35
- package/src/cli/coalesce.ts +0 -43
package/src/core/frontmatter.ts
CHANGED
|
@@ -28,13 +28,57 @@ const withYamlEngine = <O>(options: O): O =>
|
|
|
28
28
|
},
|
|
29
29
|
}) as O;
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* True when a document's leading `---` line is a CommonMark thematic break,
|
|
33
|
+
* not a front matter fence. Two shapes qualify (mirroring
|
|
34
|
+
* `linesWithoutFrontMatter` in `sources/normalize.ts`):
|
|
35
|
+
* - the next line is blank (or absent) — YAML metadata starts on the very
|
|
36
|
+
* next line, so a gap means the body *opens* with a divider (e.g. a
|
|
37
|
+
* Notion page whose first block is one);
|
|
38
|
+
* - no closing `---` line follows — gray-matter would swallow the whole
|
|
39
|
+
* document as one unclosed YAML block and hand it to js-yaml, which
|
|
40
|
+
* crashes on ordinary Markdown (`> quote` → "a line break is expected").
|
|
41
|
+
*/
|
|
42
|
+
const opensWithThematicBreak = (input: string): boolean => {
|
|
43
|
+
const [first = "", second] = input.split(/\r?\n/u, 2);
|
|
44
|
+
if (!/^-{3}\s*$/u.test(first)) {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
if (second === undefined || second.trim() === "") {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
// gray-matter closes the block at the next line-leading `---`; matching its
|
|
51
|
+
// search exactly keeps this guard from firing on any document it parses.
|
|
52
|
+
return !input.includes("\n---", 1);
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The parse result for a document with no front matter: the input passes
|
|
57
|
+
* through as content, untouched. Shaped like gray-matter's own no-matter
|
|
58
|
+
* result (every Blume call site reads only `content` and `data`).
|
|
59
|
+
*/
|
|
60
|
+
const passthrough = (input: string): ReturnType<typeof baseMatter> =>
|
|
61
|
+
({
|
|
62
|
+
content: input,
|
|
63
|
+
data: {},
|
|
64
|
+
excerpt: "",
|
|
65
|
+
isEmpty: false,
|
|
66
|
+
language: "",
|
|
67
|
+
matter: "",
|
|
68
|
+
orig: input,
|
|
69
|
+
// Recomposing a file with no matter and empty data is the content itself.
|
|
70
|
+
stringify: (): string => input,
|
|
71
|
+
}) as unknown as ReturnType<typeof baseMatter>;
|
|
72
|
+
|
|
31
73
|
// Every helper that parses or emits YAML (`read`, `stringify`) must be
|
|
32
74
|
// re-wrapped here — Object.assign copies gray-matter's own helpers, which use
|
|
33
75
|
// its default `safeLoad` engine and would reintroduce the crash. `test` only
|
|
34
76
|
// checks for a delimiter, so the copied original is safe.
|
|
35
77
|
const matter = Object.assign(
|
|
36
78
|
(input: MatterInput, options?: MatterOptions) =>
|
|
37
|
-
|
|
79
|
+
typeof input === "string" && opensWithThematicBreak(input)
|
|
80
|
+
? passthrough(input)
|
|
81
|
+
: baseMatter(input, withYamlEngine(options)),
|
|
38
82
|
baseMatter,
|
|
39
83
|
{
|
|
40
84
|
read: (filepath: ReadArgs[0], options?: ReadArgs[1]) =>
|
package/src/core/probe.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import pMap from "p-map";
|
|
2
|
+
|
|
1
3
|
import type { DiagnosticSeverity } from "./types.ts";
|
|
2
4
|
|
|
3
5
|
export const PROBE_CONCURRENCY = 8;
|
|
@@ -113,24 +115,10 @@ export const probeAll = async (
|
|
|
113
115
|
options: { concurrency?: number; timeoutMs?: number } = {}
|
|
114
116
|
): Promise<Map<string, ProbeResult>> => {
|
|
115
117
|
const unique = [...new Set(urls)];
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
options
|
|
119
|
-
|
|
118
|
+
const entries = await pMap(
|
|
119
|
+
unique,
|
|
120
|
+
async (url) => [url, await probe(url, options)] as const,
|
|
121
|
+
{ concurrency: Math.max(1, options.concurrency ?? PROBE_CONCURRENCY) }
|
|
120
122
|
);
|
|
121
|
-
|
|
122
|
-
let cursor = 0;
|
|
123
|
-
const worker = async (): Promise<void> => {
|
|
124
|
-
while (cursor < unique.length) {
|
|
125
|
-
const url = unique[cursor];
|
|
126
|
-
cursor += 1;
|
|
127
|
-
if (url !== undefined) {
|
|
128
|
-
// oxlint-disable-next-line no-await-in-loop -- bounded-concurrency pool
|
|
129
|
-
results.set(url, await probe(url, options));
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
};
|
|
133
|
-
await Promise.all(Array.from({ length: limit }, worker));
|
|
134
|
-
|
|
135
|
-
return results;
|
|
123
|
+
return new Map(entries);
|
|
136
124
|
};
|
|
@@ -262,6 +262,17 @@ export const scanProject = async (
|
|
|
262
262
|
discoverFolderMeta(metaSources, { localeDirs }),
|
|
263
263
|
]);
|
|
264
264
|
|
|
265
|
+
// Folder meta contributed by the sources themselves (the OpenAPI source
|
|
266
|
+
// labels each tag directory with the spec's own tag name). It applies to
|
|
267
|
+
// every locale, so it merges into the shared map — beneath user-authored
|
|
268
|
+
// entries, which are spread last and win.
|
|
269
|
+
const sharedFolderMeta = new Map([
|
|
270
|
+
...loaded.flatMap(({ folderMeta: sourceMeta }) =>
|
|
271
|
+
Object.entries(sourceMeta ?? {})
|
|
272
|
+
),
|
|
273
|
+
...folderMeta.shared,
|
|
274
|
+
]);
|
|
275
|
+
|
|
265
276
|
const {
|
|
266
277
|
diagnostics: contentDiagnostics,
|
|
267
278
|
droppedPages,
|
|
@@ -301,7 +312,7 @@ export const scanProject = async (
|
|
|
301
312
|
folderMeta: folderMeta.meta,
|
|
302
313
|
i18n: config.i18n,
|
|
303
314
|
navigation: config.navigation,
|
|
304
|
-
sharedFolderMeta
|
|
315
|
+
sharedFolderMeta,
|
|
305
316
|
});
|
|
306
317
|
const manifest = buildManifest({ config, context, graph });
|
|
307
318
|
|
package/src/core/schema.ts
CHANGED
|
@@ -304,6 +304,8 @@ const sanitySourceSchema = z.object({
|
|
|
304
304
|
|
|
305
305
|
/** A Notion database; pages become entries, blocks become MDX. */
|
|
306
306
|
const notionSourceSchema = z.object({
|
|
307
|
+
/** Max concurrent Notion API requests; default 3 (Notion's per-integration pace). */
|
|
308
|
+
concurrency: z.number().positive().optional(),
|
|
307
309
|
database: z.string(),
|
|
308
310
|
/** Opt-in dev polling interval (seconds); omit to freeze for the session. */
|
|
309
311
|
pollInterval: z.number().positive().optional(),
|
|
@@ -780,6 +782,10 @@ const aiConfigSchema = z.strictObject({
|
|
|
780
782
|
// and host Ask AI in an existing backend. Absolute URLs and root-relative
|
|
781
783
|
// paths are both valid; the built-in request/stream contract is unchanged.
|
|
782
784
|
endpoint: askEndpointSchema.optional(),
|
|
785
|
+
// Extra system-prompt text (identity, language, tone) appended to the
|
|
786
|
+
// built-in instructions, so the grounding contract — answer from the
|
|
787
|
+
// retrieved excerpts, cite pages as Markdown links — stays intact.
|
|
788
|
+
instructions: z.string().trim().min(1).optional(),
|
|
783
789
|
model: z.string().default("openai/gpt-5.5"),
|
|
784
790
|
provider: z.enum(askAiProviders).default("gateway"),
|
|
785
791
|
// Empty-state prompts shown before the first question. Each renders as a
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { joinURL } from "ufo";
|
|
2
|
+
|
|
3
|
+
import { trimEnd } from "./trim.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Absolute-URL building for the configured `deployment.site`, shared by every
|
|
7
|
+
* emitter that prints site URLs (sitemap, RSS, robots, llms.txt, the MCP and
|
|
8
|
+
* agent-discovery documents). One implementation replaces eight per-file
|
|
9
|
+
* copies that had drifted across three different trailing-slash treatments.
|
|
10
|
+
*
|
|
11
|
+
* Deliberately not `new URL(path, site)`: a root-absolute path would drop the
|
|
12
|
+
* base path of a subpath deployment (`acme.com/docs`). ufo's `joinURL` joins
|
|
13
|
+
* without that footgun; the site is first trimmed with the ReDoS-safe
|
|
14
|
+
* `trimEnd` loop so even a malformed `site` with piled-up trailing slashes
|
|
15
|
+
* joins cleanly.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The configured site with any trailing slashes dropped. */
|
|
19
|
+
export const siteRoot = (site: string): string => trimEnd(site, "/");
|
|
20
|
+
|
|
21
|
+
/** `site` + root-absolute `path` (already carrying any deployment base). */
|
|
22
|
+
export const absoluteUrl = (site: string, path: string): string => {
|
|
23
|
+
const root = siteRoot(site);
|
|
24
|
+
// joinURL folds a lone "/" away entirely; the homepage keeps its slash
|
|
25
|
+
// (`https://example.com/`), matching what every emitter always printed.
|
|
26
|
+
return path === "/" ? `${root}/` : joinURL(root, path);
|
|
27
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
3
|
|
|
3
4
|
import { join } from "pathe";
|
|
@@ -6,14 +7,15 @@ import { BlumeError } from "../diagnostics.ts";
|
|
|
6
7
|
import type { Diagnostic } from "../types.ts";
|
|
7
8
|
import type { SourceEntry, SourceLoadResult } from "./types.ts";
|
|
8
9
|
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Small, stable content hash for cache/HMR bookkeeping — and for staged asset
|
|
12
|
+
* *filenames* (see sources/assets.ts and content-assets.ts), where a collision
|
|
13
|
+
* silently serves the wrong file. 64 bits of SHA-256 keeps those names
|
|
14
|
+
* collision-safe at any realistic asset count; the old 31-bit DJB2 hash had a
|
|
15
|
+
* ~46k-item birthday bound.
|
|
16
|
+
*/
|
|
17
|
+
export const hashText = (text: string): string =>
|
|
18
|
+
createHash("sha256").update(text).digest("hex").slice(0, 16);
|
|
17
19
|
|
|
18
20
|
/** A stable digest of a source's entries, for change detection while polling. */
|
|
19
21
|
export const entriesDigest = (entries: SourceEntry[]): string =>
|
|
@@ -85,6 +85,26 @@ const NON_PROSE = new Set(["code", "heading", "html", "thematicBreak"]);
|
|
|
85
85
|
* content kept — then cut at a word boundary to fit the search snippet cap.
|
|
86
86
|
* Undefined when the notes have no prose at all.
|
|
87
87
|
*/
|
|
88
|
+
/**
|
|
89
|
+
* The longest prefix of `text` that fits `max` UTF-16 units without cutting
|
|
90
|
+
* inside a grapheme cluster. A bare `String#slice` counts code units, so it
|
|
91
|
+
* can split a surrogate pair (emitting a lone surrogate — invalid Unicode —
|
|
92
|
+
* into a meta description) or halve an emoji sequence. Grapheme segmentation
|
|
93
|
+
* is rule-based (UAX #29), so unlike word segmentation it does not drift
|
|
94
|
+
* across ICU builds.
|
|
95
|
+
*/
|
|
96
|
+
const graphemePrefix = (text: string, max: number): string => {
|
|
97
|
+
let end = 0;
|
|
98
|
+
const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
|
|
99
|
+
for (const { index, segment } of graphemes.segment(text)) {
|
|
100
|
+
if (index + segment.length > max) {
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
end = index + segment.length;
|
|
104
|
+
}
|
|
105
|
+
return text.slice(0, end);
|
|
106
|
+
};
|
|
107
|
+
|
|
88
108
|
const releaseDescription = (body: string): string | undefined => {
|
|
89
109
|
const tree = fromMarkdown(body.replaceAll(CHANGESET_HASH, "$<mark>"), {
|
|
90
110
|
extensions: [gfm()],
|
|
@@ -108,7 +128,7 @@ const releaseDescription = (body: string): string | undefined => {
|
|
|
108
128
|
}
|
|
109
129
|
// Cut before the cap at a word boundary (kept only when it doesn't drop the
|
|
110
130
|
// summary under the minimum), shed any dangling punctuation, and mark the cut.
|
|
111
|
-
const slice = text
|
|
131
|
+
const slice = graphemePrefix(text, DESCRIPTION_MAX - 1);
|
|
112
132
|
const boundary = slice.lastIndexOf(" ");
|
|
113
133
|
const head = (
|
|
114
134
|
boundary >= DESCRIPTION_MIN ? slice.slice(0, boundary) : slice
|
|
@@ -28,12 +28,19 @@ const groupLabel = (segment: string): string | null =>
|
|
|
28
28
|
* anchor ids are *not* slugged here — they use a `github-slugger` in
|
|
29
29
|
* {@link extractHeadings}, matching the renderer (see `markdown/heading-anchors`)
|
|
30
30
|
* so `blume validate` checks anchors against the exact rendered heading ids.
|
|
31
|
+
*
|
|
32
|
+
* The keep-class is Unicode letters/marks/numbers, not `\w`: ASCII slugs are
|
|
33
|
+
* unchanged, but a CJK/Cyrillic/accented slug keeps its characters instead of
|
|
34
|
+
* collapsing to `""` (which forced Sanity/Notion routes onto their opaque
|
|
35
|
+
* document-id fallbacks) or dropping accents (`café` → `caf`). NFC first so a
|
|
36
|
+
* macOS-NFD `é` (e + combining mark) slugs identically to the composed form.
|
|
31
37
|
*/
|
|
32
38
|
export const slugify = (text: string): string =>
|
|
33
39
|
text
|
|
40
|
+
.normalize("NFC")
|
|
34
41
|
.toLowerCase()
|
|
35
42
|
.trim()
|
|
36
|
-
.replaceAll(/[^\
|
|
43
|
+
.replaceAll(/[^\p{L}\p{M}\p{N}\s_-]/gu, "")
|
|
37
44
|
.replaceAll(/[\s_]+/gu, "-")
|
|
38
45
|
.replaceAll(/-+/gu, "-")
|
|
39
46
|
.replaceAll(/^-|-$/gu, "");
|
|
@@ -55,6 +62,18 @@ const titleCase = (value: string): string =>
|
|
|
55
62
|
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
|
56
63
|
.join(" ");
|
|
57
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Strip characters that cannot survive the route → URL → output-file round
|
|
67
|
+
* trip. A `:` ahead of the first `/` makes `new URL()` read the segment as a
|
|
68
|
+
* scheme (`Guide: Architecture.md` → `guide:`), which crashes Astro's
|
|
69
|
+
* prerender write with "The URL must be of scheme file"; control characters
|
|
70
|
+
* (an embedded newline in a filename) are silently dropped by the URL parser,
|
|
71
|
+
* desyncing the route from its output path. Both are legal in macOS/Linux
|
|
72
|
+
* filenames, so they are removed here rather than rejected.
|
|
73
|
+
*/
|
|
74
|
+
const sanitizeSegment = (segment: string): string =>
|
|
75
|
+
segment.replaceAll(/[:\p{Cc}]/gu, "");
|
|
76
|
+
|
|
58
77
|
/** Fold one raw path part into the accumulating route segments/groups. */
|
|
59
78
|
const addRouteSegment = (
|
|
60
79
|
part: string,
|
|
@@ -75,7 +94,12 @@ const addRouteSegment = (
|
|
|
75
94
|
if (clean === "index") {
|
|
76
95
|
return;
|
|
77
96
|
}
|
|
78
|
-
|
|
97
|
+
const safe = sanitizeSegment(clean);
|
|
98
|
+
// A part that was nothing but stripped characters cannot name a segment.
|
|
99
|
+
if (safe === "") {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
segments.push(safe);
|
|
79
103
|
};
|
|
80
104
|
|
|
81
105
|
/** Convert a content-root-relative path into URL + nav metadata. */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
2
2
|
|
|
3
|
+
import pLimit from "p-limit";
|
|
3
4
|
import { join } from "pathe";
|
|
4
5
|
|
|
5
6
|
import { BlumeError } from "../diagnostics.ts";
|
|
@@ -96,6 +97,12 @@ export interface NotionSourceOptions {
|
|
|
96
97
|
prefix?: string;
|
|
97
98
|
/** The Notion database id. */
|
|
98
99
|
database: string;
|
|
100
|
+
/**
|
|
101
|
+
* Maximum concurrent Notion API requests. Notion allows an average of 3
|
|
102
|
+
* requests per second per integration, so a large database must pace its
|
|
103
|
+
* block-tree fan-out or every request 429s. Default 3.
|
|
104
|
+
*/
|
|
105
|
+
concurrency?: number;
|
|
99
106
|
/** Integration token; defaults to `NOTION_TOKEN`. */
|
|
100
107
|
token?: string;
|
|
101
108
|
properties?: NotionPropertyMap;
|
|
@@ -141,12 +148,15 @@ const RATE_LIMITED = 429;
|
|
|
141
148
|
const MAX_RETRIES = 4;
|
|
142
149
|
const BASE_DELAY_MS = 500;
|
|
143
150
|
const SECOND_MS = 1000;
|
|
151
|
+
const DEFAULT_CONCURRENCY = 3;
|
|
144
152
|
|
|
145
153
|
/**
|
|
146
154
|
* Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
|
|
147
155
|
* header and otherwise backing off exponentially. A large workspace fans out
|
|
148
156
|
* many concurrent block-children requests, so without this a single 429 would
|
|
149
|
-
* reject the batch and abort the whole import.
|
|
157
|
+
* reject the batch and abort the whole import. The exponential wait is
|
|
158
|
+
* jittered so calls rate-limited together don't retry in lockstep and trip
|
|
159
|
+
* the limit again as a herd.
|
|
150
160
|
*/
|
|
151
161
|
const withNotionRetry = async <T>(
|
|
152
162
|
call: () => Promise<T>,
|
|
@@ -163,7 +173,9 @@ const withNotionRetry = async <T>(
|
|
|
163
173
|
(error as { headers?: Record<string, string> }).headers?.["retry-after"]
|
|
164
174
|
);
|
|
165
175
|
const wait =
|
|
166
|
-
retryAfter > 0
|
|
176
|
+
retryAfter > 0
|
|
177
|
+
? retryAfter * SECOND_MS
|
|
178
|
+
: BASE_DELAY_MS * 2 ** attempt * (1 + Math.random());
|
|
167
179
|
await sleep(wait);
|
|
168
180
|
return withNotionRetry(call, attempt + 1);
|
|
169
181
|
}
|
|
@@ -252,6 +264,16 @@ export const notionSource = (
|
|
|
252
264
|
ctx?: SourceContext
|
|
253
265
|
): ContentSource => {
|
|
254
266
|
const props = options.properties ?? {};
|
|
267
|
+
// A FIFO semaphore: at most N calls run at once, the rest queue. Notion's
|
|
268
|
+
// rate limit is per-integration (an average of 3 req/s), and a large
|
|
269
|
+
// database fans out one block-children request per page plus one per nested
|
|
270
|
+
// container — an unbounded burst guarantees 429s that even the retry loop
|
|
271
|
+
// can't recover from, so every API call funnels through this limiter.
|
|
272
|
+
const limit = pLimit(Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY));
|
|
273
|
+
// Every Notion API call goes through the limiter, inside the retry — so a
|
|
274
|
+
// call sleeping through a backoff doesn't hold a slot while it waits.
|
|
275
|
+
const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
|
|
276
|
+
withNotionRetry(() => limit(call));
|
|
255
277
|
const cache = snapshotCache(
|
|
256
278
|
ctx?.cacheDir ?? join(".blume", "cache", options.name)
|
|
257
279
|
);
|
|
@@ -284,7 +306,7 @@ export const notionSource = (
|
|
|
284
306
|
blockId: string
|
|
285
307
|
): Promise<NotionBlock[]> =>
|
|
286
308
|
collectAll((cursor) =>
|
|
287
|
-
|
|
309
|
+
notionCall(() =>
|
|
288
310
|
client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
|
|
289
311
|
)
|
|
290
312
|
);
|
|
@@ -458,12 +480,12 @@ export const notionSource = (
|
|
|
458
480
|
};
|
|
459
481
|
|
|
460
482
|
// Hoisted out of `load` so the retry closure doesn't nest past the linter's
|
|
461
|
-
// 4-level limit (source factory → queryDatabase →
|
|
483
|
+
// 4-level limit (source factory → queryDatabase → notionCall callback).
|
|
462
484
|
const queryDatabase = (
|
|
463
485
|
client: NotionClientLike,
|
|
464
486
|
cursor?: string
|
|
465
487
|
): Promise<NotionList<NotionPage>> =>
|
|
466
|
-
|
|
488
|
+
notionCall(() =>
|
|
467
489
|
client.databases.query({
|
|
468
490
|
database_id: options.database,
|
|
469
491
|
start_cursor: cursor,
|
|
@@ -45,12 +45,27 @@ const HEADING_STYLES: Record<string, string> = {
|
|
|
45
45
|
h6: "###### ",
|
|
46
46
|
};
|
|
47
47
|
|
|
48
|
+
// Markdown/raw-HTML structure characters. Portable Text spans are *plain
|
|
49
|
+
// text* — formatting arrives as marks, never as syntax in the text — so a
|
|
50
|
+
// literal `*`, `_`, `[`, backtick, `~`, or `<` typed in the CMS must render
|
|
51
|
+
// as itself. Unescaped, it opened emphasis or a code span mid-paragraph, and
|
|
52
|
+
// `<` let CMS prose inject raw HTML into the rendered page. CommonMark
|
|
53
|
+
// backslash-escapes every ASCII punctuation character, so `\*` is always the
|
|
54
|
+
// literal asterisk.
|
|
55
|
+
const MARKDOWN_SPECIALS = /[\\`*_[\]~<]/gu;
|
|
56
|
+
|
|
57
|
+
const escapeText = (text: string): string =>
|
|
58
|
+
text.replaceAll(MARKDOWN_SPECIALS, String.raw`\$&`);
|
|
59
|
+
|
|
48
60
|
/** Wrap a span's text in Markdown for its marks (decorators + link defs). */
|
|
49
61
|
const renderSpan = (
|
|
50
62
|
span: PortableTextSpan,
|
|
51
63
|
defs: Map<string, PortableTextMarkDef>
|
|
52
64
|
): string => {
|
|
53
|
-
|
|
65
|
+
// Code spans stay verbatim: their text is literal inside the backticks,
|
|
66
|
+
// and backslash escapes would render as backslashes.
|
|
67
|
+
const isCode = span.marks?.includes("code") ?? false;
|
|
68
|
+
let text = isCode ? (span.text ?? "") : escapeText(span.text ?? "");
|
|
54
69
|
if (!span.marks || span.marks.length === 0) {
|
|
55
70
|
return text;
|
|
56
71
|
}
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
FolderMeta,
|
|
3
|
+
FrontmatterExtend,
|
|
4
|
+
ResolvedI18nConfig,
|
|
5
|
+
} from "../schema.ts";
|
|
2
6
|
import type { Diagnostic } from "../types.ts";
|
|
3
7
|
|
|
4
8
|
/**
|
|
@@ -40,6 +44,14 @@ export interface SourceLoadResult {
|
|
|
40
44
|
entries: SourceEntry[];
|
|
41
45
|
/** Source-level diagnostics (e.g. an offline cache fallback warning). */
|
|
42
46
|
diagnostics: Diagnostic[];
|
|
47
|
+
/**
|
|
48
|
+
* Folder meta the source derives for the sidebar groups its entries create,
|
|
49
|
+
* keyed by locale-stripped group path (the `meta.ts` key space). The OpenAPI
|
|
50
|
+
* source labels each tag directory with the spec's own tag name, so the
|
|
51
|
+
* sidebar shows `OAuth2`/`Größe` instead of a re-humanized slug. Merged
|
|
52
|
+
* beneath user-authored meta files, which always win.
|
|
53
|
+
*/
|
|
54
|
+
folderMeta?: Record<string, FolderMeta>;
|
|
43
55
|
}
|
|
44
56
|
|
|
45
57
|
/**
|
|
@@ -276,6 +276,20 @@ const redirectFor = (pathname) => {
|
|
|
276
276
|
return Object.hasOwn(REDIRECTS, path) ? REDIRECTS[path] : null;
|
|
277
277
|
};
|
|
278
278
|
|
|
279
|
+
// \`_redirects\` semantics, which the static layer applies to these same
|
|
280
|
+
// paths: the request's query string is forwarded unless the destination
|
|
281
|
+
// carries its own, and a destination fragment stays after the query.
|
|
282
|
+
const redirectLocation = (destination, search) => {
|
|
283
|
+
const hashIndex = destination.indexOf("#");
|
|
284
|
+
const bare = hashIndex === -1 ? destination : destination.slice(0, hashIndex);
|
|
285
|
+
if (!search || bare.includes("?")) {
|
|
286
|
+
return destination;
|
|
287
|
+
}
|
|
288
|
+
return hashIndex === -1
|
|
289
|
+
? bare + search
|
|
290
|
+
: bare + search + destination.slice(hashIndex);
|
|
291
|
+
};
|
|
292
|
+
|
|
279
293
|
const parseAccept = (accept) =>
|
|
280
294
|
accept.split(",").map((part) => {
|
|
281
295
|
const segments = part.trim().split(";");
|
|
@@ -355,7 +369,7 @@ export default {
|
|
|
355
369
|
const redirect = redirectFor(url.pathname);
|
|
356
370
|
if (redirect !== null) {
|
|
357
371
|
return new Response(null, {
|
|
358
|
-
headers: { location: redirect[0] },
|
|
372
|
+
headers: { location: redirectLocation(redirect[0], url.search) },
|
|
359
373
|
status: redirect[1],
|
|
360
374
|
});
|
|
361
375
|
}
|
package/src/deploy/robots.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
|
|
2
2
|
import type { BlumeProject } from "../core/project-graph.ts";
|
|
3
3
|
import type { ContentSignalPolicy, ContentSignals } from "../core/schema.ts";
|
|
4
|
+
import { absoluteUrl } from "../core/site-url.ts";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Ordered mapping from config field to its `Content-Signal` token. The order
|
|
@@ -52,7 +53,7 @@ export const buildRobots = (project: BlumeProject): string | null => {
|
|
|
52
53
|
normalizeBasePath(config.deployment.base),
|
|
53
54
|
"/sitemap.xml"
|
|
54
55
|
);
|
|
55
|
-
lines.push("", `Sitemap: ${site
|
|
56
|
+
lines.push("", `Sitemap: ${absoluteUrl(site, sitemapPath)}`);
|
|
56
57
|
}
|
|
57
58
|
return `${lines.join("\n")}\n`;
|
|
58
59
|
};
|
package/src/deploy/rss.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { escape as escapeXml } from "html-escaper";
|
|
|
4
4
|
|
|
5
5
|
import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
|
|
6
6
|
import type { BlumeProject } from "../core/project-graph.ts";
|
|
7
|
+
import { siteRoot } from "../core/site-url.ts";
|
|
7
8
|
import type { PageRecord } from "../core/types.ts";
|
|
8
9
|
|
|
9
10
|
/** A single feed entry derived from a content page. */
|
|
@@ -55,7 +56,7 @@ export const buildRssFeeds = (project: BlumeProject): RssFeed[] => {
|
|
|
55
56
|
if (!(rss.enabled && site)) {
|
|
56
57
|
return [];
|
|
57
58
|
}
|
|
58
|
-
const base = site
|
|
59
|
+
const base = siteRoot(site);
|
|
59
60
|
// Routes carry `basePath`; a `deployment.base` subdirectory is layered on top.
|
|
60
61
|
// The feed's own `link`/self URL points at the docs root under that base, while
|
|
61
62
|
// `path` stays base-less (it's also the on-disk output location).
|
package/src/deploy/sitemap.ts
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
} from "../astro/pages.ts";
|
|
9
9
|
import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
|
|
10
10
|
import type { BlumeProject } from "../core/project-graph.ts";
|
|
11
|
+
import { siteRoot } from "../core/site-url.ts";
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* Astro's reserved error routes. A user-authored override (`pages/404.astro`,
|
|
@@ -29,20 +30,48 @@ const lastmodTag = (value: string | undefined): string => {
|
|
|
29
30
|
: `<lastmod>${date.toISOString().slice(0, 10)}</lastmod>`;
|
|
30
31
|
};
|
|
31
32
|
|
|
33
|
+
/** One emitted sitemap artifact: its dist-root filename and XML body. */
|
|
34
|
+
export interface SitemapFile {
|
|
35
|
+
name: string;
|
|
36
|
+
xml: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
32
39
|
/**
|
|
33
|
-
*
|
|
40
|
+
* The sitemaps.org cap on `<url>` entries in a single file. Beyond it,
|
|
41
|
+
* `sitemap.xml` becomes a sitemap index pointing at numbered chunk files —
|
|
42
|
+
* search engines reject an oversized urlset outright.
|
|
43
|
+
*/
|
|
44
|
+
const URLS_PER_FILE = 50_000;
|
|
45
|
+
|
|
46
|
+
const renderUrlset = (
|
|
47
|
+
urls: string[]
|
|
48
|
+
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
|
49
|
+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
|
50
|
+
${urls.join("\n")}
|
|
51
|
+
</urlset>
|
|
52
|
+
`;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Build the sitemap files from the route manifest plus the routes the manifest
|
|
34
56
|
* can't see: custom `.astro` pages (most importantly a custom landing `/`) and
|
|
35
57
|
* the generated `/changelog` index. Returns null when the sitemap is disabled
|
|
36
58
|
* or no `site` is configured (absolute URLs are required for a valid sitemap).
|
|
37
59
|
* Drafts, hidden, and `noindex` pages are excluded.
|
|
60
|
+
*
|
|
61
|
+
* Sites within the per-file URL cap get the single classic `sitemap.xml`;
|
|
62
|
+
* larger sites get `sitemap.xml` as a sitemap index over numbered
|
|
63
|
+
* `sitemap-N.xml` chunks, all served from the same directory robots.txt
|
|
64
|
+
* already points at.
|
|
38
65
|
*/
|
|
39
|
-
export const
|
|
66
|
+
export const buildSitemapFiles = (
|
|
67
|
+
project: BlumeProject
|
|
68
|
+
): SitemapFile[] | null => {
|
|
40
69
|
const { site } = project.config.deployment;
|
|
41
70
|
if (!(site && project.config.seo.sitemap)) {
|
|
42
71
|
return null;
|
|
43
72
|
}
|
|
44
73
|
|
|
45
|
-
const base = site
|
|
74
|
+
const base = siteRoot(site);
|
|
46
75
|
// Routes carry `basePath`; a `deployment.base` subdirectory is layered on top.
|
|
47
76
|
const deployBase = normalizeBasePath(project.config.deployment.base);
|
|
48
77
|
const seen = new Set<string>();
|
|
@@ -87,9 +116,29 @@ export const buildSitemap = (project: BlumeProject): string | null => {
|
|
|
87
116
|
}
|
|
88
117
|
urls.sort();
|
|
89
118
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
119
|
+
if (urls.length <= URLS_PER_FILE) {
|
|
120
|
+
return [{ name: "sitemap.xml", xml: renderUrlset(urls) }];
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const chunks: SitemapFile[] = [];
|
|
124
|
+
const references: string[] = [];
|
|
125
|
+
for (let start = 0; start < urls.length; start += URLS_PER_FILE) {
|
|
126
|
+
const name = `sitemap-${chunks.length + 1}.xml`;
|
|
127
|
+
chunks.push({
|
|
128
|
+
name,
|
|
129
|
+
xml: renderUrlset(urls.slice(start, start + URLS_PER_FILE)),
|
|
130
|
+
});
|
|
131
|
+
// Chunks sit next to sitemap.xml, so their URLs layer the same deployment
|
|
132
|
+
// base robots.txt uses for the index.
|
|
133
|
+
const loc = escapeXml(
|
|
134
|
+
encodeURI(`${base}${withBasePath(deployBase, `/${name}`)}`)
|
|
135
|
+
);
|
|
136
|
+
references.push(` <sitemap><loc>${loc}</loc></sitemap>`);
|
|
137
|
+
}
|
|
138
|
+
const index = `<?xml version="1.0" encoding="UTF-8"?>
|
|
139
|
+
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
|
140
|
+
${references.join("\n")}
|
|
141
|
+
</sitemapindex>
|
|
94
142
|
`;
|
|
143
|
+
return [{ name: "sitemap.xml", xml: index }, ...chunks];
|
|
95
144
|
};
|