blume 1.4.1 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/cli/index.js +700 -579
- package/dist/cli/index.js.map +57 -56
- package/dist/types/core/base-path.d.ts +8 -0
- package/dist/types/core/config-input.d.ts +8 -0
- package/dist/types/core/data.d.ts +10 -0
- package/dist/types/core/schema.d.ts +4 -0
- package/dist/types/core/sources/types.d.ts +9 -1
- package/dist/types/openapi/references.d.ts +8 -2
- package/docs/configuration/ai.mdx +41 -9
- package/docs/content/sources.mdx +1 -1
- package/package.json +17 -7
- package/src/ai/agent-readability.ts +3 -2
- package/src/ai/api-catalog.ts +2 -2
- package/src/ai/ask-context.ts +45 -12
- package/src/ai/link-headers.ts +7 -2
- package/src/ai/llms.ts +2 -1
- package/src/ai/mcp/discovery.ts +25 -6
- package/src/ai/mcp/server.ts +108 -98
- package/src/ai/tar.ts +29 -70
- package/src/astro/examples.ts +7 -3
- package/src/astro/generate.ts +63 -34
- package/src/astro/islands.ts +7 -3
- package/src/astro/templates.ts +36 -9
- package/src/audit/agent.ts +14 -29
- package/src/audit/crawl.ts +41 -16
- package/src/audit/run.ts +10 -3
- package/src/audit/snapshot.ts +27 -2
- package/src/cli/commands/audit.ts +12 -17
- package/src/cli/commands/build.ts +15 -7
- package/src/cli/commands/dev.ts +13 -15
- package/src/cli/commands/eject.ts +4 -4
- package/src/cli/commands/eval.ts +17 -27
- package/src/cli/env.ts +13 -30
- package/src/cli/init/scaffold.ts +21 -0
- package/src/cli/report-format.ts +22 -0
- package/src/components/content/AccordionItem.astro +2 -9
- package/src/components/content/ColorItem.astro +5 -13
- package/src/components/content/Component.astro +12 -8
- package/src/components/content/Frame.astro +2 -12
- package/src/components/content/Prompt.astro +12 -31
- package/src/components/content/Tab.astro +2 -9
- package/src/components/content/Tooltip.astro +1 -9
- package/src/components/content/Update.astro +2 -9
- package/src/components/content/inline-markdown.ts +28 -0
- package/src/components/copy-feedback.ts +96 -0
- package/src/components/islands/ask-ai.tsx +78 -9
- package/src/components/layout/PageActions.astro +20 -32
- package/src/components/layout/PageLayout.astro +8 -28
- package/src/components/layout/RootLayout.astro +47 -48
- package/src/components/layout/Search.astro +56 -9
- package/src/components/layout/drawer-inert.ts +31 -0
- package/src/components/layout/search/pagefind.ts +6 -5
- package/src/components/layout/search/types.ts +32 -0
- package/src/components/openapi/panel.ts +11 -8
- package/src/components/raf-throttle.ts +21 -0
- package/src/components/slug.ts +14 -0
- package/src/core/base-path.ts +18 -1
- package/src/core/config-input.ts +8 -0
- package/src/core/data.ts +7 -0
- package/src/core/frontmatter.ts +45 -1
- package/src/core/probe.ts +7 -19
- package/src/core/project-graph.ts +12 -1
- package/src/core/schema.ts +6 -0
- package/src/core/site-url.ts +27 -0
- package/src/core/sources/cache.ts +10 -8
- package/src/core/sources/github-releases.ts +21 -1
- package/src/core/sources/normalize.ts +26 -2
- package/src/core/sources/notion.ts +27 -5
- package/src/core/sources/portable-text.ts +16 -1
- package/src/core/sources/resolve.ts +1 -0
- package/src/core/sources/types.ts +13 -1
- package/src/deploy/cloudflare-negotiation.ts +15 -1
- package/src/deploy/robots.ts +2 -1
- package/src/deploy/rss.ts +2 -1
- package/src/deploy/sitemap.ts +56 -7
- package/src/eval/agents.ts +13 -10
- package/src/eval/report.ts +1 -14
- package/src/markdown/package-commands.ts +61 -54
- package/src/og/card.ts +24 -26
- package/src/openapi/model.ts +9 -9
- package/src/openapi/parse.ts +69 -28
- package/src/openapi/references.ts +35 -12
- package/src/openapi/render-mdx.ts +64 -25
- package/src/openapi/scalar.ts +2 -2
- package/src/openapi/source.ts +28 -1
- package/src/search/documents.ts +78 -34
- package/src/search/orama-index.ts +51 -12
- package/src/theme/palette.ts +6 -2
- package/src/translate/ledger.ts +4 -2
- package/src/translate/report.ts +1 -14
- package/src/translate/run.ts +20 -35
- package/src/cli/coalesce.ts +0 -43
|
@@ -84,6 +84,38 @@ export const highlight = (text: string, query: string): string => {
|
|
|
84
84
|
.join("");
|
|
85
85
|
};
|
|
86
86
|
|
|
87
|
+
// Either a tag-shaped run — an opening `<` with a letter or `/` after it,
|
|
88
|
+
// through the closing `>` (or end of string for an unterminated tag) — or a
|
|
89
|
+
// lone `<`. A run can't span a later `<` (`[^<>]`), so between the two
|
|
90
|
+
// alternatives every `<` in the input lands inside a captured run.
|
|
91
|
+
const ANGLE_RUN = /(?<run><\/?[a-z][^<>]*>?|<)/iu;
|
|
92
|
+
const BARE_MARK = /^<\/?mark>$/iu;
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Reduce provider-supplied excerpt markup to the `<mark>` highlighting the
|
|
96
|
+
* dialog expects. Remote excerpts (Pagefind's index, hosted engines) are
|
|
97
|
+
* rendered via `innerHTML`, so the output alphabet is pinned: bare
|
|
98
|
+
* `<mark>`/`</mark>` tags (attributes make even a mark untrusted), text, and
|
|
99
|
+
* entities. Tag-shaped runs are dropped; every other `<` is escaped, which
|
|
100
|
+
* renders identically but can't be parsed as markup (`<!--` would otherwise
|
|
101
|
+
* open a comment in `innerHTML` and swallow the rest of the excerpt). Split on
|
|
102
|
+
* runs covering every `<` rather than deleting tags in place: a deletion can
|
|
103
|
+
* splice the text around it into a fresh tag (`<<b>script>` → `<script>`),
|
|
104
|
+
* while here no `<` survives outside a run, so the only ones emitted are the
|
|
105
|
+
* bare mark tags. String-level on purpose: this also runs under DOM-less
|
|
106
|
+
* tests, where DOMPurify/DOMParser don't exist.
|
|
107
|
+
*/
|
|
108
|
+
export const sanitizeExcerpt = (html: string): string =>
|
|
109
|
+
html
|
|
110
|
+
.split(ANGLE_RUN)
|
|
111
|
+
.map((part, index) => {
|
|
112
|
+
if (index % 2 === 0 || BARE_MARK.test(part)) {
|
|
113
|
+
return part;
|
|
114
|
+
}
|
|
115
|
+
return part === "<" ? "<" : "";
|
|
116
|
+
})
|
|
117
|
+
.join("");
|
|
118
|
+
|
|
87
119
|
/** First index in `text` where any query token matches (case-insensitive). */
|
|
88
120
|
const matchIndex = (text: string, query: string): number => {
|
|
89
121
|
const tokens = queryTokens(query);
|
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
* theme.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { copyText, createCopyFlash } from "../copy-feedback.ts";
|
|
10
|
+
|
|
9
11
|
class BlumePanelTabs extends HTMLElement {
|
|
10
12
|
connectedCallback() {
|
|
11
13
|
const tabs = [
|
|
@@ -36,18 +38,19 @@ class BlumePanelTabs extends HTMLElement {
|
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
if (copy) {
|
|
41
|
+
const flash = createCopyFlash((copied) => {
|
|
42
|
+
if (copied) {
|
|
43
|
+
copy.dataset.copied = "true";
|
|
44
|
+
} else {
|
|
45
|
+
delete copy.dataset.copied;
|
|
46
|
+
}
|
|
47
|
+
}, "Copied");
|
|
39
48
|
copy.addEventListener("click", async () => {
|
|
40
49
|
const active = panels.find(
|
|
41
50
|
(panel) => !panel.classList.contains("hidden")
|
|
42
51
|
);
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
copy.dataset.copied = "true";
|
|
46
|
-
setTimeout(() => {
|
|
47
|
-
delete copy.dataset.copied;
|
|
48
|
-
}, 1500);
|
|
49
|
-
} catch {
|
|
50
|
-
// Clipboard unavailable (insecure context); silently ignore.
|
|
52
|
+
if (await copyText(active?.textContent ?? "")) {
|
|
53
|
+
flash();
|
|
51
54
|
}
|
|
52
55
|
});
|
|
53
56
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Coalesce a high-frequency event handler (resize, scroll) into at most one
|
|
3
|
+
* call per animation frame — the toc-element scroll pattern, shared. Calls
|
|
4
|
+
* landing while a frame is pending are dropped; the handler runs once on the
|
|
5
|
+
* next frame with the latest state. Layout reads inside `fn` then happen once
|
|
6
|
+
* per frame instead of once per event, without the settle lag a debounce
|
|
7
|
+
* would add to position-tracking handlers.
|
|
8
|
+
*/
|
|
9
|
+
export const rafThrottle = (fn: () => void): (() => void) => {
|
|
10
|
+
let ticking = false;
|
|
11
|
+
return () => {
|
|
12
|
+
if (ticking) {
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
ticking = true;
|
|
16
|
+
requestAnimationFrame(() => {
|
|
17
|
+
ticking = false;
|
|
18
|
+
fn();
|
|
19
|
+
});
|
|
20
|
+
};
|
|
21
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { slug } from "github-slugger";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Slug a component's title into a DOM id (accordions, tabs, changelog
|
|
5
|
+
* updates), replacing three identical per-component copies. github-slugger —
|
|
6
|
+
* already what heading anchors use, both here and in Satteri's own
|
|
7
|
+
* heading-ids — so a component id slugs exactly like a heading with the same
|
|
8
|
+
* text (unicode letters kept, `user_id` keeps its underscore, `C#` keeps
|
|
9
|
+
* nothing extra dropped). Stateless on purpose: components render across many
|
|
10
|
+
* pages in one build process, so a stateful slugger would leak duplicate
|
|
11
|
+
* suffixes between pages — same-page duplicates are de-duplicated client-side
|
|
12
|
+
* by each component's own script.
|
|
13
|
+
*/
|
|
14
|
+
export const componentSlug = (value: string): string => slug(value);
|
package/src/core/base-path.ts
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* `components/islands/base-path.ts` and serves `deployment.base` via `BASE_URL`.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
+
import { trimEnd } from "./trim.ts";
|
|
16
|
+
|
|
15
17
|
/**
|
|
16
18
|
* Canonicalize a configured base path to either `""` (none) or `/seg[/seg…]`
|
|
17
19
|
* (leading slash, no trailing slash, collapsed inner slashes). A blank value or
|
|
@@ -33,10 +35,25 @@ export const normalizeBasePath = (input?: string): string => {
|
|
|
33
35
|
* `/docs` and `/docs/` as the same page) and collapse an empty path to `/`.
|
|
34
36
|
*/
|
|
35
37
|
export const normalizePath = (path: string): string => {
|
|
36
|
-
const trimmed = path
|
|
38
|
+
const trimmed = trimEnd(path, "/");
|
|
37
39
|
return trimmed === "" ? "/" : trimmed;
|
|
38
40
|
};
|
|
39
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Canonicalize a route-ish string (a configured route, a page path, an agent-
|
|
44
|
+
* supplied route) to `/` or `/seg[/seg…]`: trimmed, exactly one leading slash,
|
|
45
|
+
* no trailing slash. The shared spelling of what openapi/references,
|
|
46
|
+
* ai/ask-context, and ai/mcp/server each hand-rolled with slightly different
|
|
47
|
+
* regexes.
|
|
48
|
+
*/
|
|
49
|
+
export const normalizeRoute = (input: string): string => {
|
|
50
|
+
const trimmed = trimEnd(input.trim(), "/");
|
|
51
|
+
if (trimmed === "") {
|
|
52
|
+
return "/";
|
|
53
|
+
}
|
|
54
|
+
return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
|
|
55
|
+
};
|
|
56
|
+
|
|
40
57
|
/**
|
|
41
58
|
* Whether a link target is a root-relative internal path (`/x`) — the only
|
|
42
59
|
* shape a base path applies to. Protocol-relative (`//host`), absolute URLs,
|
package/src/core/config-input.ts
CHANGED
|
@@ -205,6 +205,8 @@ export interface SanitySource {
|
|
|
205
205
|
/** A Notion database; pages become entries, blocks become MDX. */
|
|
206
206
|
export interface NotionSource {
|
|
207
207
|
type: "notion";
|
|
208
|
+
/** Max concurrent Notion API requests; default 3 (Notion's per-integration pace). */
|
|
209
|
+
concurrency?: number;
|
|
208
210
|
/** Notion database id. */
|
|
209
211
|
database: string;
|
|
210
212
|
/** Opt-in dev polling interval (seconds); omit to freeze for the session. */
|
|
@@ -653,6 +655,12 @@ export interface AskConfig {
|
|
|
653
655
|
* limiting, and streaming. Accepts an absolute URL or root-relative path.
|
|
654
656
|
*/
|
|
655
657
|
endpoint?: string;
|
|
658
|
+
/**
|
|
659
|
+
* Extra system-prompt text appended to the built-in instructions — use it
|
|
660
|
+
* for identity, language, or tone. The built-in grounding behavior (answer
|
|
661
|
+
* from the retrieved excerpts, cite pages as Markdown links) is preserved.
|
|
662
|
+
*/
|
|
663
|
+
instructions?: string;
|
|
656
664
|
/** Model id to use. Defaults to `openai/gpt-5.5`. */
|
|
657
665
|
model?: string;
|
|
658
666
|
/** Which backend routes the request. Defaults to `gateway`. */
|
package/src/core/data.ts
CHANGED
|
@@ -107,6 +107,13 @@ export interface BlumeDataConfig {
|
|
|
107
107
|
/** `dateFormat`: `Intl.DateTimeFormat` options for the date stamps. */
|
|
108
108
|
dateFormat: ResolvedConfig["dateFormat"];
|
|
109
109
|
description: string | undefined;
|
|
110
|
+
/**
|
|
111
|
+
* Which agent-discovery resources exist for the layout to advertise in every
|
|
112
|
+
* page's `<head>` (`seo.agentReadability`, `ai.llmsTxt.enabled`) — the HTML
|
|
113
|
+
* counterpart of the homepage-only HTTP `Link` header, for agents that enter
|
|
114
|
+
* on a deep page (see `ai/link-headers.ts`).
|
|
115
|
+
*/
|
|
116
|
+
discovery: { agentReadability: boolean; llmsTxt: boolean };
|
|
110
117
|
favicon: BlumeFavicon;
|
|
111
118
|
feedback: boolean;
|
|
112
119
|
i18n: BlumeDataI18n | null;
|
package/src/core/frontmatter.ts
CHANGED
|
@@ -28,13 +28,57 @@ const withYamlEngine = <O>(options: O): O =>
|
|
|
28
28
|
},
|
|
29
29
|
}) as O;
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* True when a document's leading `---` line is a CommonMark thematic break,
|
|
33
|
+
* not a front matter fence. Two shapes qualify (mirroring
|
|
34
|
+
* `linesWithoutFrontMatter` in `sources/normalize.ts`):
|
|
35
|
+
* - the next line is blank (or absent) — YAML metadata starts on the very
|
|
36
|
+
* next line, so a gap means the body *opens* with a divider (e.g. a
|
|
37
|
+
* Notion page whose first block is one);
|
|
38
|
+
* - no closing `---` line follows — gray-matter would swallow the whole
|
|
39
|
+
* document as one unclosed YAML block and hand it to js-yaml, which
|
|
40
|
+
* crashes on ordinary Markdown (`> quote` → "a line break is expected").
|
|
41
|
+
*/
|
|
42
|
+
const opensWithThematicBreak = (input: string): boolean => {
|
|
43
|
+
const [first = "", second] = input.split(/\r?\n/u, 2);
|
|
44
|
+
if (!/^-{3}\s*$/u.test(first)) {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
if (second === undefined || second.trim() === "") {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
// gray-matter closes the block at the next line-leading `---`; matching its
|
|
51
|
+
// search exactly keeps this guard from firing on any document it parses.
|
|
52
|
+
return !input.includes("\n---", 1);
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The parse result for a document with no front matter: the input passes
|
|
57
|
+
* through as content, untouched. Shaped like gray-matter's own no-matter
|
|
58
|
+
* result (every Blume call site reads only `content` and `data`).
|
|
59
|
+
*/
|
|
60
|
+
const passthrough = (input: string): ReturnType<typeof baseMatter> =>
|
|
61
|
+
({
|
|
62
|
+
content: input,
|
|
63
|
+
data: {},
|
|
64
|
+
excerpt: "",
|
|
65
|
+
isEmpty: false,
|
|
66
|
+
language: "",
|
|
67
|
+
matter: "",
|
|
68
|
+
orig: input,
|
|
69
|
+
// Recomposing a file with no matter and empty data is the content itself.
|
|
70
|
+
stringify: (): string => input,
|
|
71
|
+
}) as unknown as ReturnType<typeof baseMatter>;
|
|
72
|
+
|
|
31
73
|
// Every helper that parses or emits YAML (`read`, `stringify`) must be
|
|
32
74
|
// re-wrapped here — Object.assign copies gray-matter's own helpers, which use
|
|
33
75
|
// its default `safeLoad` engine and would reintroduce the crash. `test` only
|
|
34
76
|
// checks for a delimiter, so the copied original is safe.
|
|
35
77
|
const matter = Object.assign(
|
|
36
78
|
(input: MatterInput, options?: MatterOptions) =>
|
|
37
|
-
|
|
79
|
+
typeof input === "string" && opensWithThematicBreak(input)
|
|
80
|
+
? passthrough(input)
|
|
81
|
+
: baseMatter(input, withYamlEngine(options)),
|
|
38
82
|
baseMatter,
|
|
39
83
|
{
|
|
40
84
|
read: (filepath: ReadArgs[0], options?: ReadArgs[1]) =>
|
package/src/core/probe.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import pMap from "p-map";
|
|
2
|
+
|
|
1
3
|
import type { DiagnosticSeverity } from "./types.ts";
|
|
2
4
|
|
|
3
5
|
export const PROBE_CONCURRENCY = 8;
|
|
@@ -113,24 +115,10 @@ export const probeAll = async (
|
|
|
113
115
|
options: { concurrency?: number; timeoutMs?: number } = {}
|
|
114
116
|
): Promise<Map<string, ProbeResult>> => {
|
|
115
117
|
const unique = [...new Set(urls)];
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
options
|
|
119
|
-
|
|
118
|
+
const entries = await pMap(
|
|
119
|
+
unique,
|
|
120
|
+
async (url) => [url, await probe(url, options)] as const,
|
|
121
|
+
{ concurrency: Math.max(1, options.concurrency ?? PROBE_CONCURRENCY) }
|
|
120
122
|
);
|
|
121
|
-
|
|
122
|
-
let cursor = 0;
|
|
123
|
-
const worker = async (): Promise<void> => {
|
|
124
|
-
while (cursor < unique.length) {
|
|
125
|
-
const url = unique[cursor];
|
|
126
|
-
cursor += 1;
|
|
127
|
-
if (url !== undefined) {
|
|
128
|
-
// oxlint-disable-next-line no-await-in-loop -- bounded-concurrency pool
|
|
129
|
-
results.set(url, await probe(url, options));
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
};
|
|
133
|
-
await Promise.all(Array.from({ length: limit }, worker));
|
|
134
|
-
|
|
135
|
-
return results;
|
|
123
|
+
return new Map(entries);
|
|
136
124
|
};
|
|
@@ -262,6 +262,17 @@ export const scanProject = async (
|
|
|
262
262
|
discoverFolderMeta(metaSources, { localeDirs }),
|
|
263
263
|
]);
|
|
264
264
|
|
|
265
|
+
// Folder meta contributed by the sources themselves (the OpenAPI source
|
|
266
|
+
// labels each tag directory with the spec's own tag name). It applies to
|
|
267
|
+
// every locale, so it merges into the shared map — beneath user-authored
|
|
268
|
+
// entries, which are spread last and win.
|
|
269
|
+
const sharedFolderMeta = new Map([
|
|
270
|
+
...loaded.flatMap(({ folderMeta: sourceMeta }) =>
|
|
271
|
+
Object.entries(sourceMeta ?? {})
|
|
272
|
+
),
|
|
273
|
+
...folderMeta.shared,
|
|
274
|
+
]);
|
|
275
|
+
|
|
265
276
|
const {
|
|
266
277
|
diagnostics: contentDiagnostics,
|
|
267
278
|
droppedPages,
|
|
@@ -301,7 +312,7 @@ export const scanProject = async (
|
|
|
301
312
|
folderMeta: folderMeta.meta,
|
|
302
313
|
i18n: config.i18n,
|
|
303
314
|
navigation: config.navigation,
|
|
304
|
-
sharedFolderMeta
|
|
315
|
+
sharedFolderMeta,
|
|
305
316
|
});
|
|
306
317
|
const manifest = buildManifest({ config, context, graph });
|
|
307
318
|
|
package/src/core/schema.ts
CHANGED
|
@@ -304,6 +304,8 @@ const sanitySourceSchema = z.object({
|
|
|
304
304
|
|
|
305
305
|
/** A Notion database; pages become entries, blocks become MDX. */
|
|
306
306
|
const notionSourceSchema = z.object({
|
|
307
|
+
/** Max concurrent Notion API requests; default 3 (Notion's per-integration pace). */
|
|
308
|
+
concurrency: z.number().positive().optional(),
|
|
307
309
|
database: z.string(),
|
|
308
310
|
/** Opt-in dev polling interval (seconds); omit to freeze for the session. */
|
|
309
311
|
pollInterval: z.number().positive().optional(),
|
|
@@ -780,6 +782,10 @@ const aiConfigSchema = z.strictObject({
|
|
|
780
782
|
// and host Ask AI in an existing backend. Absolute URLs and root-relative
|
|
781
783
|
// paths are both valid; the built-in request/stream contract is unchanged.
|
|
782
784
|
endpoint: askEndpointSchema.optional(),
|
|
785
|
+
// Extra system-prompt text (identity, language, tone) appended to the
|
|
786
|
+
// built-in instructions, so the grounding contract — answer from the
|
|
787
|
+
// retrieved excerpts, cite pages as Markdown links — stays intact.
|
|
788
|
+
instructions: z.string().trim().min(1).optional(),
|
|
783
789
|
model: z.string().default("openai/gpt-5.5"),
|
|
784
790
|
provider: z.enum(askAiProviders).default("gateway"),
|
|
785
791
|
// Empty-state prompts shown before the first question. Each renders as a
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { joinURL } from "ufo";
|
|
2
|
+
|
|
3
|
+
import { trimEnd } from "./trim.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Absolute-URL building for the configured `deployment.site`, shared by every
|
|
7
|
+
* emitter that prints site URLs (sitemap, RSS, robots, llms.txt, the MCP and
|
|
8
|
+
* agent-discovery documents). One implementation replaces eight per-file
|
|
9
|
+
* copies that had drifted across three different trailing-slash treatments.
|
|
10
|
+
*
|
|
11
|
+
* Deliberately not `new URL(path, site)`: a root-absolute path would drop the
|
|
12
|
+
* base path of a subpath deployment (`acme.com/docs`). ufo's `joinURL` joins
|
|
13
|
+
* without that footgun; the site is first trimmed with the ReDoS-safe
|
|
14
|
+
* `trimEnd` loop so even a malformed `site` with piled-up trailing slashes
|
|
15
|
+
* joins cleanly.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The configured site with any trailing slashes dropped. */
|
|
19
|
+
export const siteRoot = (site: string): string => trimEnd(site, "/");
|
|
20
|
+
|
|
21
|
+
/** `site` + root-absolute `path` (already carrying any deployment base). */
|
|
22
|
+
export const absoluteUrl = (site: string, path: string): string => {
|
|
23
|
+
const root = siteRoot(site);
|
|
24
|
+
// joinURL folds a lone "/" away entirely; the homepage keeps its slash
|
|
25
|
+
// (`https://example.com/`), matching what every emitter always printed.
|
|
26
|
+
return path === "/" ? `${root}/` : joinURL(root, path);
|
|
27
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
3
|
|
|
3
4
|
import { join } from "pathe";
|
|
@@ -6,14 +7,15 @@ import { BlumeError } from "../diagnostics.ts";
|
|
|
6
7
|
import type { Diagnostic } from "../types.ts";
|
|
7
8
|
import type { SourceEntry, SourceLoadResult } from "./types.ts";
|
|
8
9
|
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Small, stable content hash for cache/HMR bookkeeping — and for staged asset
|
|
12
|
+
* *filenames* (see sources/assets.ts and content-assets.ts), where a collision
|
|
13
|
+
* silently serves the wrong file. 64 bits of SHA-256 keeps those names
|
|
14
|
+
* collision-safe at any realistic asset count; the old 31-bit DJB2 hash had a
|
|
15
|
+
* ~46k-item birthday bound.
|
|
16
|
+
*/
|
|
17
|
+
export const hashText = (text: string): string =>
|
|
18
|
+
createHash("sha256").update(text).digest("hex").slice(0, 16);
|
|
17
19
|
|
|
18
20
|
/** A stable digest of a source's entries, for change detection while polling. */
|
|
19
21
|
export const entriesDigest = (entries: SourceEntry[]): string =>
|
|
@@ -85,6 +85,26 @@ const NON_PROSE = new Set(["code", "heading", "html", "thematicBreak"]);
|
|
|
85
85
|
* content kept — then cut at a word boundary to fit the search snippet cap.
|
|
86
86
|
* Undefined when the notes have no prose at all.
|
|
87
87
|
*/
|
|
88
|
+
/**
|
|
89
|
+
* The longest prefix of `text` that fits `max` UTF-16 units without cutting
|
|
90
|
+
* inside a grapheme cluster. A bare `String#slice` counts code units, so it
|
|
91
|
+
* can split a surrogate pair (emitting a lone surrogate — invalid Unicode —
|
|
92
|
+
* into a meta description) or halve an emoji sequence. Grapheme segmentation
|
|
93
|
+
* is rule-based (UAX #29), so unlike word segmentation it does not drift
|
|
94
|
+
* across ICU builds.
|
|
95
|
+
*/
|
|
96
|
+
const graphemePrefix = (text: string, max: number): string => {
|
|
97
|
+
let end = 0;
|
|
98
|
+
const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
|
|
99
|
+
for (const { index, segment } of graphemes.segment(text)) {
|
|
100
|
+
if (index + segment.length > max) {
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
end = index + segment.length;
|
|
104
|
+
}
|
|
105
|
+
return text.slice(0, end);
|
|
106
|
+
};
|
|
107
|
+
|
|
88
108
|
const releaseDescription = (body: string): string | undefined => {
|
|
89
109
|
const tree = fromMarkdown(body.replaceAll(CHANGESET_HASH, "$<mark>"), {
|
|
90
110
|
extensions: [gfm()],
|
|
@@ -108,7 +128,7 @@ const releaseDescription = (body: string): string | undefined => {
|
|
|
108
128
|
}
|
|
109
129
|
// Cut before the cap at a word boundary (kept only when it doesn't drop the
|
|
110
130
|
// summary under the minimum), shed any dangling punctuation, and mark the cut.
|
|
111
|
-
const slice = text
|
|
131
|
+
const slice = graphemePrefix(text, DESCRIPTION_MAX - 1);
|
|
112
132
|
const boundary = slice.lastIndexOf(" ");
|
|
113
133
|
const head = (
|
|
114
134
|
boundary >= DESCRIPTION_MIN ? slice.slice(0, boundary) : slice
|
|
@@ -28,12 +28,19 @@ const groupLabel = (segment: string): string | null =>
|
|
|
28
28
|
* anchor ids are *not* slugged here — they use a `github-slugger` in
|
|
29
29
|
* {@link extractHeadings}, matching the renderer (see `markdown/heading-anchors`)
|
|
30
30
|
* so `blume validate` checks anchors against the exact rendered heading ids.
|
|
31
|
+
*
|
|
32
|
+
* The keep-class is Unicode letters/marks/numbers, not `\w`: ASCII slugs are
|
|
33
|
+
* unchanged, but a CJK/Cyrillic/accented slug keeps its characters instead of
|
|
34
|
+
* collapsing to `""` (which forced Sanity/Notion routes onto their opaque
|
|
35
|
+
* document-id fallbacks) or dropping accents (`café` → `caf`). NFC first so a
|
|
36
|
+
* macOS-NFD `é` (e + combining mark) slugs identically to the composed form.
|
|
31
37
|
*/
|
|
32
38
|
export const slugify = (text: string): string =>
|
|
33
39
|
text
|
|
40
|
+
.normalize("NFC")
|
|
34
41
|
.toLowerCase()
|
|
35
42
|
.trim()
|
|
36
|
-
.replaceAll(/[^\
|
|
43
|
+
.replaceAll(/[^\p{L}\p{M}\p{N}\s_-]/gu, "")
|
|
37
44
|
.replaceAll(/[\s_]+/gu, "-")
|
|
38
45
|
.replaceAll(/-+/gu, "-")
|
|
39
46
|
.replaceAll(/^-|-$/gu, "");
|
|
@@ -55,6 +62,18 @@ const titleCase = (value: string): string =>
|
|
|
55
62
|
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
|
56
63
|
.join(" ");
|
|
57
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Strip characters that cannot survive the route → URL → output-file round
|
|
67
|
+
* trip. A `:` ahead of the first `/` makes `new URL()` read the segment as a
|
|
68
|
+
* scheme (`Guide: Architecture.md` → `guide:`), which crashes Astro's
|
|
69
|
+
* prerender write with "The URL must be of scheme file"; control characters
|
|
70
|
+
* (an embedded newline in a filename) are silently dropped by the URL parser,
|
|
71
|
+
* desyncing the route from its output path. Both are legal in macOS/Linux
|
|
72
|
+
* filenames, so they are removed here rather than rejected.
|
|
73
|
+
*/
|
|
74
|
+
const sanitizeSegment = (segment: string): string =>
|
|
75
|
+
segment.replaceAll(/[:\p{Cc}]/gu, "");
|
|
76
|
+
|
|
58
77
|
/** Fold one raw path part into the accumulating route segments/groups. */
|
|
59
78
|
const addRouteSegment = (
|
|
60
79
|
part: string,
|
|
@@ -75,7 +94,12 @@ const addRouteSegment = (
|
|
|
75
94
|
if (clean === "index") {
|
|
76
95
|
return;
|
|
77
96
|
}
|
|
78
|
-
|
|
97
|
+
const safe = sanitizeSegment(clean);
|
|
98
|
+
// A part that was nothing but stripped characters cannot name a segment.
|
|
99
|
+
if (safe === "") {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
segments.push(safe);
|
|
79
103
|
};
|
|
80
104
|
|
|
81
105
|
/** Convert a content-root-relative path into URL + nav metadata. */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
2
2
|
|
|
3
|
+
import pLimit from "p-limit";
|
|
3
4
|
import { join } from "pathe";
|
|
4
5
|
|
|
5
6
|
import { BlumeError } from "../diagnostics.ts";
|
|
@@ -96,6 +97,12 @@ export interface NotionSourceOptions {
|
|
|
96
97
|
prefix?: string;
|
|
97
98
|
/** The Notion database id. */
|
|
98
99
|
database: string;
|
|
100
|
+
/**
|
|
101
|
+
* Maximum concurrent Notion API requests. Notion allows an average of 3
|
|
102
|
+
* requests per second per integration, so a large database must pace its
|
|
103
|
+
* block-tree fan-out or every request 429s. Default 3.
|
|
104
|
+
*/
|
|
105
|
+
concurrency?: number;
|
|
99
106
|
/** Integration token; defaults to `NOTION_TOKEN`. */
|
|
100
107
|
token?: string;
|
|
101
108
|
properties?: NotionPropertyMap;
|
|
@@ -141,12 +148,15 @@ const RATE_LIMITED = 429;
|
|
|
141
148
|
const MAX_RETRIES = 4;
|
|
142
149
|
const BASE_DELAY_MS = 500;
|
|
143
150
|
const SECOND_MS = 1000;
|
|
151
|
+
const DEFAULT_CONCURRENCY = 3;
|
|
144
152
|
|
|
145
153
|
/**
|
|
146
154
|
* Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
|
|
147
155
|
* header and otherwise backing off exponentially. A large workspace fans out
|
|
148
156
|
* many concurrent block-children requests, so without this a single 429 would
|
|
149
|
-
* reject the batch and abort the whole import.
|
|
157
|
+
* reject the batch and abort the whole import. The exponential wait is
|
|
158
|
+
* jittered so calls rate-limited together don't retry in lockstep and trip
|
|
159
|
+
* the limit again as a herd.
|
|
150
160
|
*/
|
|
151
161
|
const withNotionRetry = async <T>(
|
|
152
162
|
call: () => Promise<T>,
|
|
@@ -163,7 +173,9 @@ const withNotionRetry = async <T>(
|
|
|
163
173
|
(error as { headers?: Record<string, string> }).headers?.["retry-after"]
|
|
164
174
|
);
|
|
165
175
|
const wait =
|
|
166
|
-
retryAfter > 0
|
|
176
|
+
retryAfter > 0
|
|
177
|
+
? retryAfter * SECOND_MS
|
|
178
|
+
: BASE_DELAY_MS * 2 ** attempt * (1 + Math.random());
|
|
167
179
|
await sleep(wait);
|
|
168
180
|
return withNotionRetry(call, attempt + 1);
|
|
169
181
|
}
|
|
@@ -252,6 +264,16 @@ export const notionSource = (
|
|
|
252
264
|
ctx?: SourceContext
|
|
253
265
|
): ContentSource => {
|
|
254
266
|
const props = options.properties ?? {};
|
|
267
|
+
// A FIFO semaphore: at most N calls run at once, the rest queue. Notion's
|
|
268
|
+
// rate limit is per-integration (an average of 3 req/s), and a large
|
|
269
|
+
// database fans out one block-children request per page plus one per nested
|
|
270
|
+
// container — an unbounded burst guarantees 429s that even the retry loop
|
|
271
|
+
// can't recover from, so every API call funnels through this limiter.
|
|
272
|
+
const limit = pLimit(Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY));
|
|
273
|
+
// Every Notion API call goes through the limiter, inside the retry — so a
|
|
274
|
+
// call sleeping through a backoff doesn't hold a slot while it waits.
|
|
275
|
+
const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
|
|
276
|
+
withNotionRetry(() => limit(call));
|
|
255
277
|
const cache = snapshotCache(
|
|
256
278
|
ctx?.cacheDir ?? join(".blume", "cache", options.name)
|
|
257
279
|
);
|
|
@@ -284,7 +306,7 @@ export const notionSource = (
|
|
|
284
306
|
blockId: string
|
|
285
307
|
): Promise<NotionBlock[]> =>
|
|
286
308
|
collectAll((cursor) =>
|
|
287
|
-
|
|
309
|
+
notionCall(() =>
|
|
288
310
|
client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
|
|
289
311
|
)
|
|
290
312
|
);
|
|
@@ -458,12 +480,12 @@ export const notionSource = (
|
|
|
458
480
|
};
|
|
459
481
|
|
|
460
482
|
// Hoisted out of `load` so the retry closure doesn't nest past the linter's
|
|
461
|
-
// 4-level limit (source factory → queryDatabase →
|
|
483
|
+
// 4-level limit (source factory → queryDatabase → notionCall callback).
|
|
462
484
|
const queryDatabase = (
|
|
463
485
|
client: NotionClientLike,
|
|
464
486
|
cursor?: string
|
|
465
487
|
): Promise<NotionList<NotionPage>> =>
|
|
466
|
-
|
|
488
|
+
notionCall(() =>
|
|
467
489
|
client.databases.query({
|
|
468
490
|
database_id: options.database,
|
|
469
491
|
start_cursor: cursor,
|
|
@@ -45,12 +45,27 @@ const HEADING_STYLES: Record<string, string> = {
|
|
|
45
45
|
h6: "###### ",
|
|
46
46
|
};
|
|
47
47
|
|
|
48
|
+
// Markdown/raw-HTML structure characters. Portable Text spans are *plain
|
|
49
|
+
// text* — formatting arrives as marks, never as syntax in the text — so a
|
|
50
|
+
// literal `*`, `_`, `[`, backtick, `~`, or `<` typed in the CMS must render
|
|
51
|
+
// as itself. Unescaped, it opened emphasis or a code span mid-paragraph, and
|
|
52
|
+
// `<` let CMS prose inject raw HTML into the rendered page. CommonMark
|
|
53
|
+
// backslash-escapes every ASCII punctuation character, so `\*` is always the
|
|
54
|
+
// literal asterisk.
|
|
55
|
+
const MARKDOWN_SPECIALS = /[\\`*_[\]~<]/gu;
|
|
56
|
+
|
|
57
|
+
const escapeText = (text: string): string =>
|
|
58
|
+
text.replaceAll(MARKDOWN_SPECIALS, String.raw`\$&`);
|
|
59
|
+
|
|
48
60
|
/** Wrap a span's text in Markdown for its marks (decorators + link defs). */
|
|
49
61
|
const renderSpan = (
|
|
50
62
|
span: PortableTextSpan,
|
|
51
63
|
defs: Map<string, PortableTextMarkDef>
|
|
52
64
|
): string => {
|
|
53
|
-
|
|
65
|
+
// Code spans stay verbatim: their text is literal inside the backticks,
|
|
66
|
+
// and backslash escapes would render as backslashes.
|
|
67
|
+
const isCode = span.marks?.includes("code") ?? false;
|
|
68
|
+
let text = isCode ? (span.text ?? "") : escapeText(span.text ?? "");
|
|
54
69
|
if (!span.marks || span.marks.length === 0) {
|
|
55
70
|
return text;
|
|
56
71
|
}
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
FolderMeta,
|
|
3
|
+
FrontmatterExtend,
|
|
4
|
+
ResolvedI18nConfig,
|
|
5
|
+
} from "../schema.ts";
|
|
2
6
|
import type { Diagnostic } from "../types.ts";
|
|
3
7
|
|
|
4
8
|
/**
|
|
@@ -40,6 +44,14 @@ export interface SourceLoadResult {
|
|
|
40
44
|
entries: SourceEntry[];
|
|
41
45
|
/** Source-level diagnostics (e.g. an offline cache fallback warning). */
|
|
42
46
|
diagnostics: Diagnostic[];
|
|
47
|
+
/**
|
|
48
|
+
* Folder meta the source derives for the sidebar groups its entries create,
|
|
49
|
+
* keyed by locale-stripped group path (the `meta.ts` key space). The OpenAPI
|
|
50
|
+
* source labels each tag directory with the spec's own tag name, so the
|
|
51
|
+
* sidebar shows `OAuth2`/`Größe` instead of a re-humanized slug. Merged
|
|
52
|
+
* beneath user-authored meta files, which always win.
|
|
53
|
+
*/
|
|
54
|
+
folderMeta?: Record<string, FolderMeta>;
|
|
43
55
|
}
|
|
44
56
|
|
|
45
57
|
/**
|
|
@@ -276,6 +276,20 @@ const redirectFor = (pathname) => {
|
|
|
276
276
|
return Object.hasOwn(REDIRECTS, path) ? REDIRECTS[path] : null;
|
|
277
277
|
};
|
|
278
278
|
|
|
279
|
+
// \`_redirects\` semantics, which the static layer applies to these same
|
|
280
|
+
// paths: the request's query string is forwarded unless the destination
|
|
281
|
+
// carries its own, and a destination fragment stays after the query.
|
|
282
|
+
const redirectLocation = (destination, search) => {
|
|
283
|
+
const hashIndex = destination.indexOf("#");
|
|
284
|
+
const bare = hashIndex === -1 ? destination : destination.slice(0, hashIndex);
|
|
285
|
+
if (!search || bare.includes("?")) {
|
|
286
|
+
return destination;
|
|
287
|
+
}
|
|
288
|
+
return hashIndex === -1
|
|
289
|
+
? bare + search
|
|
290
|
+
: bare + search + destination.slice(hashIndex);
|
|
291
|
+
};
|
|
292
|
+
|
|
279
293
|
const parseAccept = (accept) =>
|
|
280
294
|
accept.split(",").map((part) => {
|
|
281
295
|
const segments = part.trim().split(";");
|
|
@@ -355,7 +369,7 @@ export default {
|
|
|
355
369
|
const redirect = redirectFor(url.pathname);
|
|
356
370
|
if (redirect !== null) {
|
|
357
371
|
return new Response(null, {
|
|
358
|
-
headers: { location: redirect[0] },
|
|
372
|
+
headers: { location: redirectLocation(redirect[0], url.search) },
|
|
359
373
|
status: redirect[1],
|
|
360
374
|
});
|
|
361
375
|
}
|