blume 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +1 -1
- package/dist/cli/index.js +1286 -63
- package/dist/cli/index.js.map +32 -21
- package/dist/types/core/config-input.d.ts +18 -0
- package/dist/types/core/config.d.ts +4 -0
- package/dist/types/core/data.d.ts +1 -0
- package/dist/types/core/schema.d.ts +132 -17
- package/dist/types/core/types.d.ts +5 -3
- package/dist/types/openapi/references.d.ts +6 -0
- package/docs/advanced/api-reference.mdx +27 -0
- package/docs/advanced/changelog.mdx +10 -0
- package/docs/configuration/ai.mdx +38 -2
- package/docs/configuration/customization.mdx +27 -0
- package/docs/configuration/index.mdx +5 -0
- package/docs/content/navigation.mdx +12 -0
- package/docs/reference/cli.mdx +17 -13
- package/docs/reference/eval.mdx +106 -0
- package/docs/reference/meta.ts +1 -1
- package/package.json +1 -1
- package/src/ai/agent-readability.ts +19 -1
- package/src/ai/llms.ts +9 -4
- package/src/ai/mcp/server.ts +19 -8
- package/src/ai/mcp/stdio.ts +35 -0
- package/src/astro/generate.ts +25 -2
- package/src/astro/templates.ts +114 -22
- package/src/cli/commands/eval.ts +291 -0
- package/src/cli/commands/init.ts +9 -4
- package/src/cli/commands/mcp-stdio.ts +36 -0
- package/src/cli/index.ts +4 -0
- package/src/cli/required-secrets.ts +1 -1
- package/src/components/content/AccordionItem.astro +2 -2
- package/src/components/content/TreeFolder.astro +1 -2
- package/src/components/islands/AskAI.astro +9 -2
- package/src/components/islands/ask-ai.tsx +4 -2
- package/src/components/islands/hooks.ts +10 -4
- package/src/components/layout/NavTree.astro +37 -19
- package/src/components/layout/ReferenceLayout.astro +4 -0
- package/src/components/layout/RootLayout.astro +1 -1
- package/src/components/openapi/SchemaProperty.astro +3 -3
- package/src/core/config-input.ts +18 -0
- package/src/core/config.ts +4 -0
- package/src/core/data.ts +1 -0
- package/src/core/graph.ts +1 -0
- package/src/core/navigation.ts +9 -2
- package/src/core/schema.ts +51 -4
- package/src/core/server-features.ts +1 -1
- package/src/core/types.ts +5 -3
- package/src/eval/agents.ts +340 -0
- package/src/eval/findings.ts +103 -0
- package/src/eval/prompts.ts +78 -0
- package/src/eval/report.ts +214 -0
- package/src/eval/run.ts +290 -0
- package/src/eval/schema.ts +124 -0
- package/src/openapi/references.ts +23 -2
- package/src/openapi/render-mdx.ts +27 -4
- package/src/openapi/scalar.ts +1 -0
- package/src/openapi/source.ts +11 -4
- package/src/registry/eject.ts +23 -1
- package/src/search/build.ts +4 -3
|
@@ -31,6 +31,12 @@ export interface ReferenceSource {
|
|
|
31
31
|
*/
|
|
32
32
|
basePath: string;
|
|
33
33
|
label: string;
|
|
34
|
+
/** Whether generated pages are included in llms.txt/llms-full.txt. */
|
|
35
|
+
includeInLlms: boolean;
|
|
36
|
+
/** Whether generated pages are included in site search. */
|
|
37
|
+
includeInSearch: boolean;
|
|
38
|
+
/** Whether generated pages emit noindex metadata and stay out of the sitemap. */
|
|
39
|
+
noindex: boolean;
|
|
34
40
|
/** Local path or `http(s)` URL, verbatim from config. */
|
|
35
41
|
spec: string;
|
|
36
42
|
/** Per-block Scalar theme name override, if any (Scalar renderer only). */
|
|
@@ -78,6 +78,33 @@ openapi: {
|
|
|
78
78
|
|
|
79
79
|
`spec` is shorthand for a single-entry `sources`, so you only reach for `sources` when you have more than one.
|
|
80
80
|
|
|
81
|
+
### Per-source indexing
|
|
82
|
+
|
|
83
|
+
Generated pages participate in search, `llms.txt`, and crawler indexing by default. A secondary or overlapping spec can opt out of any surface without hiding its pages or removing it from navigation:
|
|
84
|
+
|
|
85
|
+
```ts blume.config.ts lineNumbers
|
|
86
|
+
openapi: {
|
|
87
|
+
enabled: true,
|
|
88
|
+
sources: [
|
|
89
|
+
{ label: "Public API", route: "/api", spec: "./public.json" },
|
|
90
|
+
{
|
|
91
|
+
label: "Platform API",
|
|
92
|
+
route: "/platform",
|
|
93
|
+
spec: "./platform.json",
|
|
94
|
+
includeInSearch: false,
|
|
95
|
+
includeInLlms: false,
|
|
96
|
+
noindex: true,
|
|
97
|
+
},
|
|
98
|
+
],
|
|
99
|
+
}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
- `includeInSearch: false` keeps the source's overview and operations out of site search.
|
|
103
|
+
- `includeInLlms: false` keeps them out of both `llms.txt` files.
|
|
104
|
+
- `noindex: true` adds crawler noindex metadata and removes the pages from the sitemap.
|
|
105
|
+
|
|
106
|
+
With the [Scalar renderer](#the-scalar-renderer), only `noindex` applies — a Scalar-rendered reference already sits outside Blume's search and `llms.txt`, so the two `include*` settings have nothing to act on there.
|
|
107
|
+
|
|
81
108
|
## Authorization
|
|
82
109
|
|
|
83
110
|
Operations that declare [security requirements](https://spec.openapis.org/oas/v3.1.0#security-requirement-object) render an **Authorization** section above their parameters, and the generated code samples send a placeholder credential (`Authorization: Bearer YOUR_TOKEN`, an API-key header, or a query key — whatever the scheme calls for). There's nothing to configure: Blume reads `security` from the spec, so the reference always matches what the API actually enforces.
|
|
@@ -61,6 +61,16 @@ Once you have at least one `type: changelog` entry, Blume generates a **`/change
|
|
|
61
61
|
|
|
62
62
|
The page appears only when nothing already occupies the `/changelog` route. To replace it with your own design, add a [custom page](/docs/advanced/custom-pages) at `pages/changelog.astro` — it takes over and Blume stops generating the default timeline.
|
|
63
63
|
|
|
64
|
+
Because this page is generated rather than authored, it isn't part of the content tree — so a header [tab](/docs/content/navigation#tabs) pointing at `/changelog` resolves to the newest entry instead. Give the tab an `href` to land on the index itself:
|
|
65
|
+
|
|
66
|
+
```ts blume.config.ts
|
|
67
|
+
navigation: {
|
|
68
|
+
tabs: [
|
|
69
|
+
{ label: "Changelog", path: "/changelog", href: "/changelog" },
|
|
70
|
+
],
|
|
71
|
+
}
|
|
72
|
+
```
|
|
73
|
+
|
|
64
74
|
### Grouped by major version
|
|
65
75
|
|
|
66
76
|
When your versions follow [semver](https://semver.org) and span more than one major, Blume paginates the timeline by major version. Only the newest major line is shown, with a **Show N.x releases** button at the bottom that reveals the next-oldest major one click at a time:
|
|
@@ -33,6 +33,18 @@ ai: {
|
|
|
33
33
|
}
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
+
To keep an individual page out of both files, set `ai.exclude` in its frontmatter:
|
|
37
|
+
|
|
38
|
+
```mdx
|
|
39
|
+
---
|
|
40
|
+
title: Internal notes
|
|
41
|
+
ai:
|
|
42
|
+
exclude: true
|
|
43
|
+
---
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The page still renders, stays in search, and keeps its place in the sitemap — only the `llms.txt` files skip it.
|
|
47
|
+
|
|
36
48
|
To take full control of either file, add your own `llms.txt` or `llms-full.txt` to your `public/` folder. Like a custom favicon, it's picked up automatically and ships in place of the generated file — override one and Blume still generates the other.
|
|
37
49
|
|
|
38
50
|
## Raw Markdown
|
|
@@ -128,9 +140,33 @@ The page the reader is currently on is added to the context first and used to sc
|
|
|
128
140
|
|
|
129
141
|
Grounding is on for every backend except **[Inkeep](#backends)**, which runs its own retrieval over the content you've indexed in its dashboard.
|
|
130
142
|
|
|
143
|
+
### External endpoint
|
|
144
|
+
|
|
145
|
+
Already have an API backend for AI? Point the panel at it and keep the docs build static:
|
|
146
|
+
|
|
147
|
+
```ts blume.config.ts lineNumbers
|
|
148
|
+
ai: {
|
|
149
|
+
ask: {
|
|
150
|
+
enabled: true,
|
|
151
|
+
endpoint: "https://api.example.com/v1/docs/ask",
|
|
152
|
+
},
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Blume sends the same `POST` body as its built-in route:
|
|
157
|
+
|
|
158
|
+
```json
|
|
159
|
+
{
|
|
160
|
+
"messages": [{ "role": "user", "content": "How do I deploy?" }],
|
|
161
|
+
"page": { "path": "/deployment" }
|
|
162
|
+
}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Return a successful response whose body is a plain UTF-8 text stream. If the endpoint is on another origin, allow the docs origin with CORS: accept `OPTIONS` and `POST`, permit the `content-type` request header, and return the CORS headers on both the preflight and streamed response. With `endpoint` set, Blume generates the chat UI but no server route, grounding snapshot, provider dependency, or provider-secret warning; your backend owns retrieval, authentication, rate limiting, model access, and citations.
|
|
166
|
+
|
|
131
167
|
### Server output required
|
|
132
168
|
|
|
133
|
-
Ask AI is a server route (`POST /api/ask`), so it can't run on a static build. Switch to server output and pick an adapter:
|
|
169
|
+
Blume's built-in Ask AI backend is a server route (`POST /api/ask`), so it can't run on a static build. Switch to server output and pick an adapter:
|
|
134
170
|
|
|
135
171
|
```ts blume.config.ts lineNumbers
|
|
136
172
|
deployment: {
|
|
@@ -139,7 +175,7 @@ deployment: {
|
|
|
139
175
|
}
|
|
140
176
|
```
|
|
141
177
|
|
|
142
|
-
A static build with Ask AI enabled fails fast with a message telling you to set `deployment.output` to `server`. See [Deployment](/docs/deployment) for the adapters.
|
|
178
|
+
A static build with Ask AI enabled and no external `endpoint` fails fast with a message telling you to set `deployment.output` to `server`. See [Deployment](/docs/deployment) for the adapters.
|
|
143
179
|
|
|
144
180
|
### Backends
|
|
145
181
|
|
|
@@ -145,6 +145,33 @@ blume add pagination
|
|
|
145
145
|
|
|
146
146
|
The copy imports the rest of the framework from `blume/*`, so it renders exactly like the built-in until you change it. `blume add` prints the `defineComponents` snippet to register it — content components under `mdx`, layout pieces under `layout`.
|
|
147
147
|
|
|
148
|
+
## Astro integrations
|
|
149
|
+
|
|
150
|
+
Add any Astro integration from the top-level `integrations` array in `blume.config.ts`. Install the integration in your site first; Blume does not add it to the generated runtime's dependencies or manage its Astro compatibility.
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
npm install @astrojs/sitemap
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
```ts blume.config.ts lineNumbers
|
|
157
|
+
import sitemap from "@astrojs/sitemap";
|
|
158
|
+
import { defineConfig } from "blume";
|
|
159
|
+
|
|
160
|
+
export default defineConfig({
|
|
161
|
+
integrations: [
|
|
162
|
+
sitemap({
|
|
163
|
+
filter: (page) => !page.includes("/drafts/"),
|
|
164
|
+
}),
|
|
165
|
+
],
|
|
166
|
+
});
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Blume keeps its built-in integrations in their existing order, then appends your entries in declaration order. It does not sort or deduplicate them, so two integrations with the same `name` both run. Blume validates that `integrations` is an array, while Astro validates each entry and reports invalid integrations.
|
|
170
|
+
|
|
171
|
+
Because Blume loads your integrations by re-importing `blume.config.ts` from the generated Astro config rather than copying the instances, the config module evaluates twice per run — once when Blume reads your config and once when Astro loads it. Keep integration factories side-effect free (return the integration; don't write files or open connections at construction) so the second evaluation is harmless.
|
|
172
|
+
|
|
173
|
+
The same integrations run in `blume dev` and `blume build`. Editing `blume.config.ts` during `blume dev` regenerates the hidden Astro config and triggers a config restart; if you don't see an edited integration take effect, restart `blume dev`. Blume can't tell which config edits affect integrations, so once `integrations` is non-empty, every edit to `blume.config.ts` — even to an unrelated field — restarts the dev server rather than hot-applying. Blume only tracks the contents of `blume.config.ts`, so editing a separate file it imports won't trigger that regeneration on its own — restart `blume dev` after such edits. If you eject, the owned `astro.config.mjs` keeps a relative bridge to `blume.config.ts`, so the configured integrations continue to run; you can later move them directly into the Astro config as part of taking full ownership.
|
|
174
|
+
|
|
148
175
|
## Eject
|
|
149
176
|
|
|
150
177
|
When you want full control, eject the generated runtime into a standalone Astro project:
|
|
@@ -21,6 +21,7 @@ export default defineConfig({
|
|
|
21
21
|
A broader example touching the most common options (see each feature's guide for the rest):
|
|
22
22
|
|
|
23
23
|
```ts blume.config.ts lineNumbers
|
|
24
|
+
import sitemap from "@astrojs/sitemap";
|
|
24
25
|
import { defineConfig } from "blume";
|
|
25
26
|
|
|
26
27
|
export default defineConfig({
|
|
@@ -29,6 +30,9 @@ export default defineConfig({
|
|
|
29
30
|
description: "Documentation for my project.",
|
|
30
31
|
logo: "/logo.svg",
|
|
31
32
|
|
|
33
|
+
// Astro integrations — installed and versioned by this site
|
|
34
|
+
integrations: [sitemap()],
|
|
35
|
+
|
|
32
36
|
// Content
|
|
33
37
|
content: {
|
|
34
38
|
root: "docs",
|
|
@@ -338,6 +342,7 @@ Each of these has its own guide. The config field is the entry point:
|
|
|
338
342
|
| `seo` | Metadata, OG images, feeds, structured data | [SEO](/docs/configuration/seo) |
|
|
339
343
|
| `deployment` | Output mode, adapter, and site URL | [Deployment](/docs/deployment) |
|
|
340
344
|
| `redirects` | Permanent and temporary redirects | [Deployment](/docs/deployment#redirects) |
|
|
345
|
+
| `integrations` | Astro integrations appended after Blume's built-ins | [Customization](/docs/configuration/customization#astro-integrations) |
|
|
341
346
|
|
|
342
347
|
## Precedence
|
|
343
348
|
|
|
@@ -123,6 +123,18 @@ navigation: {
|
|
|
123
123
|
}
|
|
124
124
|
```
|
|
125
125
|
|
|
126
|
+
A tab's `path` is its section prefix, and it doubles as the link target. A section whose `path` isn't a page of its own — a folder with no `index.mdx` — would link to a 404, so the tab falls back to the first page in the section instead. Set `href` when you want it to land somewhere else:
|
|
127
|
+
|
|
128
|
+
```ts blume.config.ts
|
|
129
|
+
navigation: {
|
|
130
|
+
tabs: [
|
|
131
|
+
{ label: "Changelog", path: "/changelog", href: "/changelog" },
|
|
132
|
+
],
|
|
133
|
+
}
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
This matters for routes that aren't part of the content tree, since the fallback can't see them: the generated [changelog](/docs/advanced/changelog) index, or a [custom page](/docs/advanced/custom-pages) you added under `pages/`. Without `href`, a `/changelog` tab lands on the newest entry rather than the index. Tabs that don't set `href` are unaffected.
|
|
137
|
+
|
|
126
138
|
Tabs also **scope the sidebar**: when the current route falls under a tab's `path`, the sidebar shows only that section's pages — so `/adapters/*` lists the adapters and nothing else. The folder at a tab's `path` becomes the section, so this needs no extra config beyond the tabs themselves; structure your content into a folder per tab and point each tab at it.
|
|
127
139
|
|
|
128
140
|
On a route under no tab (or a tab whose `path` is `/`), the sidebar shows the pages that _don't_ belong to a tab — each tab's folder is hidden from it, since that section already has its own tab in the header. So a root landing page lists your loose top-level pages while the sectioned content stays behind its tab, mirroring Fumadocs' root folders. If a route has no pages of its own to show this way, the full tree is shown instead, so the sidebar is never left blank.
|
package/docs/reference/cli.mdx
CHANGED
|
@@ -9,19 +9,20 @@ blume <command> [options]
|
|
|
9
9
|
|
|
10
10
|
## Commands
|
|
11
11
|
|
|
12
|
-
| Command
|
|
13
|
-
|
|
|
14
|
-
| `blume init [dir]` | Scaffold a project (interactive by default).
|
|
15
|
-
| `blume dev`
|
|
16
|
-
| `blume build`
|
|
17
|
-
| `blume preview`
|
|
18
|
-
| `blume add <item>` | Install a source component from the registry.
|
|
19
|
-
| `blume sync`
|
|
20
|
-
| `blume eject`
|
|
21
|
-
| `blume check`
|
|
22
|
-
| `blume doctor`
|
|
23
|
-
| `blume validate`
|
|
24
|
-
| `blume audit`
|
|
12
|
+
| Command | Description |
|
|
13
|
+
| --- | --- |
|
|
14
|
+
| `blume init [dir]` | Scaffold a project (interactive by default). |
|
|
15
|
+
| `blume dev` | Start the dev server with hot reload. |
|
|
16
|
+
| `blume build` | Build the static (or server) site. |
|
|
17
|
+
| `blume preview` | Preview the last build. |
|
|
18
|
+
| `blume add <item>` | Install a source component from the registry. |
|
|
19
|
+
| `blume sync` | Re-fetch remote content sources and regenerate. |
|
|
20
|
+
| `blume eject` | Promote the runtime into a standalone Astro app. |
|
|
21
|
+
| `blume check` | Type-check the site with `astro check`. |
|
|
22
|
+
| `blume doctor` | Diagnose config and content problems. |
|
|
23
|
+
| `blume validate` | Validate links across your content. |
|
|
24
|
+
| `blume audit` | Audit the built site for SEO and health issues. |
|
|
25
|
+
| `blume eval` | Test the docs: an agent answers your questions using only the documentation. |
|
|
25
26
|
|
|
26
27
|
## Common flags
|
|
27
28
|
|
|
@@ -58,6 +59,9 @@ blume <command> [options]
|
|
|
58
59
|
- `blume audit --verbose` — list every affected page instead of the first few.
|
|
59
60
|
- `blume audit --json` — emit the report as JSON on stdout.
|
|
60
61
|
- `blume audit --claude` / `--codex` — hand the findings to Claude Code or Codex to fix interactively.
|
|
62
|
+
- `blume eval` — run the questions in `evals.yaml` through an agent that reads only your docs; see [Evals](/docs/reference/eval).
|
|
63
|
+
- `blume eval init` — have the agent draft a starter `evals.yaml` from your docs.
|
|
64
|
+
- `blume eval --agent claude|codex --threshold <0..1> --timeout <seconds> --json --fix --verbose` — see [Evals](/docs/reference/eval) for each flag.
|
|
61
65
|
|
|
62
66
|
## Verifying while the dev server runs
|
|
63
67
|
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Evals
|
|
3
|
+
description: blume eval gives your docs a test suite — an AI agent answers your users' questions using only the documentation, a judge grades the answers, and CI fails when the docs can't answer.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
`blume audit` tells you whether crawlers can find your docs. `blume eval` tells you whether anyone can actually _use_ them: an AI agent reads your documentation the way a stranger would and tries to answer real user questions from it. When the docs don't state the answer, the run fails and names the page that should.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
blume eval
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
blume eval 3 question(s) · Claude Code
|
|
14
|
+
|
|
15
|
+
✔ install-node-version pass 1.00 14.2s $0.14
|
|
16
|
+
✖ deploy-vercel fail 0.40 38.9s $0.31
|
|
17
|
+
missing: the adapter is auto-detected
|
|
18
|
+
⊘ search-providers skipped
|
|
19
|
+
|
|
20
|
+
fix: content/docs/deployment.mdx Docs could not answer: "How do I deploy to Vercel?" — missing: the adapter is auto-detected
|
|
21
|
+
|
|
22
|
+
2 passed · 1 failed · 1 skipped · 1m 42s · $0.45
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## How it works
|
|
26
|
+
|
|
27
|
+
Each question runs through two agent sessions, using an agent CLI you already have installed — [Claude Code](https://claude.com/claude-code) by default, or [Codex](https://developers.openai.com/codex/cli) with `--agent codex`. Blume holds no API keys and calls no model itself.
|
|
28
|
+
|
|
29
|
+
1. **The reader** answers the question using _only_ your documentation. It runs in an empty directory with its file, shell, and web tools disabled, connected to a private [MCP server](/docs/configuration/ai) that serves your docs — the same `search_docs`/`get_page` tools a real agent uses against your deployed site. It cannot read your repo, so it experiences the docs exactly like a fresh user: what isn't written doesn't exist.
|
|
30
|
+
2. **The judge** grades the answer against the facts you listed, with no tools at all. Paraphrase passes; a missing or contradicted fact fails — and so does "the documentation doesn't say."
|
|
31
|
+
|
|
32
|
+
The MCP snapshot is built from your content sources directly, so there is no need to run `blume build` first, and nothing is deployed or uploaded anywhere.
|
|
33
|
+
|
|
34
|
+
An answer the docs _can't_ support fails even when the agent's prior knowledge happens to be right — that's the point. Your docs are the only source that ships.
|
|
35
|
+
|
|
36
|
+
## Writing evals
|
|
37
|
+
|
|
38
|
+
Questions live in `evals.yaml` at the project root. To have an agent draft a starter file from your existing docs:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
blume eval init
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Or write it by hand:
|
|
45
|
+
|
|
46
|
+
```yaml
|
|
47
|
+
questions:
|
|
48
|
+
- id: install-node-version
|
|
49
|
+
question: What is the minimum Node.js version required?
|
|
50
|
+
expected:
|
|
51
|
+
- Node 22.12 or newer
|
|
52
|
+
routes: /docs/quickstart
|
|
53
|
+
- id: deploy-vercel
|
|
54
|
+
question: How do I deploy to Vercel?
|
|
55
|
+
expected:
|
|
56
|
+
- run blume build
|
|
57
|
+
- the output directory is dist
|
|
58
|
+
routes:
|
|
59
|
+
- /docs/deployment
|
|
60
|
+
- id: search-providers
|
|
61
|
+
question: Which search providers are supported?
|
|
62
|
+
expected:
|
|
63
|
+
- pagefind is the default
|
|
64
|
+
severity: warning # a miss warns instead of failing CI
|
|
65
|
+
skip: true # temporarily excluded, reported as skipped
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
- `expected` lists the facts a correct answer must state, in substance — the judge accepts paraphrase and rejects contradiction.
|
|
69
|
+
- `routes` names the page(s) that should answer the question. A failure is then anchored to that page's source file in the report; a hint that no longer matches a page is warned about rather than silently dropped.
|
|
70
|
+
- `severity: warning` keeps a question in the report without failing CI; `skip: true` sits a question out entirely.
|
|
71
|
+
|
|
72
|
+
Write questions your users actually ask — the ones from support threads, GitHub issues, and onboarding calls. The best evals encode a promise your docs make ("zero-config deploys") as a question that breaks when a PR breaks the promise.
|
|
73
|
+
|
|
74
|
+
## Failing CI
|
|
75
|
+
|
|
76
|
+
The exit code is the contract: any failed question exits non-zero. `--threshold` relaxes the gate to a passing fraction when you're digging out of a backlog:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
blume eval # every question must pass
|
|
80
|
+
blume eval --threshold 0.8 # at least 80% must pass
|
|
81
|
+
blume eval --json # machine-readable report on stdout
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The JSON report carries the same `diagnostics` + `summary` shape as `blume validate --json` and `blume audit --json`, with the per-question results (answer, score, missing facts, cost) alongside.
|
|
85
|
+
|
|
86
|
+
Because each question is two model sessions, an eval run costs real money and minutes — the per-question spend is printed as it runs. A sensible CI setup runs `blume eval` on docs changes rather than every push.
|
|
87
|
+
|
|
88
|
+
## Fixing the findings
|
|
89
|
+
|
|
90
|
+
Each failure names the missing facts and the page that should state them. To hand the whole report to the agent instead:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
blume eval --fix
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
This writes the full JSON report to a file and opens the agent interactively with a prompt that walks it through each failed question: read the named page, add the missing facts in the page's voice, and rerun `blume eval` until everything passes. The session is interactive by design — you review the edits through the agent's own permission flow — and the agent is told never to delete questions or weaken expected facts to get to green.
|
|
97
|
+
|
|
98
|
+
## Flags
|
|
99
|
+
|
|
100
|
+
- `--agent claude|codex` — which agent CLI runs the reader and judge. Defaults to `claude`.
|
|
101
|
+
- `--file <path>` — the evals file. Defaults to `evals.yaml`.
|
|
102
|
+
- `--threshold <0..1>` — minimum passing fraction before the run exits non-zero. Defaults to `1`.
|
|
103
|
+
- `--timeout <seconds>` — reader time limit per question. Defaults to `180`.
|
|
104
|
+
- `--json` — emit the report as JSON on stdout.
|
|
105
|
+
- `--fix` — after a failing run, hand the report to the agent to fix the docs interactively.
|
|
106
|
+
- `--verbose` — include the reader's full answer under each failure.
|
package/docs/reference/meta.ts
CHANGED
package/package.json
CHANGED
|
@@ -27,6 +27,24 @@ const usagePolicy = (
|
|
|
27
27
|
);
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* The advertised Ask AI URL. An external endpoint is not served under
|
|
32
|
+
* `deployment.base`, so a root-relative one absolutizes against the site
|
|
33
|
+
* origin alone; the built-in route gets site and base via `abs`.
|
|
34
|
+
*/
|
|
35
|
+
const askApiUrl = (
|
|
36
|
+
endpoint: string | undefined,
|
|
37
|
+
site: string | null,
|
|
38
|
+
abs: (path: string) => string
|
|
39
|
+
): string => {
|
|
40
|
+
if (!endpoint) {
|
|
41
|
+
return abs("/api/ask");
|
|
42
|
+
}
|
|
43
|
+
return site && endpoint.startsWith("/")
|
|
44
|
+
? `${site.replace(/\/+$/u, "")}${endpoint}`
|
|
45
|
+
: endpoint;
|
|
46
|
+
};
|
|
47
|
+
|
|
30
48
|
/**
|
|
31
49
|
* Build `agent-readability.json`: a root manifest that indexes the project's
|
|
32
50
|
* agent-facing surface — llms.txt, the raw-Markdown mirrors, the MCP server,
|
|
@@ -69,7 +87,7 @@ export const buildAgentReadability = (
|
|
|
69
87
|
};
|
|
70
88
|
}
|
|
71
89
|
if (config.ai.ask?.enabled) {
|
|
72
|
-
artifacts.askApi =
|
|
90
|
+
artifacts.askApi = askApiUrl(config.ai.ask.endpoint, site, abs);
|
|
73
91
|
}
|
|
74
92
|
if (site && config.seo.sitemap) {
|
|
75
93
|
artifacts.sitemap = abs("/sitemap.xml");
|
package/src/ai/llms.ts
CHANGED
|
@@ -19,13 +19,18 @@ const pageUrl = (route: string, site?: string, base = ""): string => {
|
|
|
19
19
|
return encodeURI(site ? `${site.replace(/\/$/u, "")}${path}` : path);
|
|
20
20
|
};
|
|
21
21
|
|
|
22
|
-
// Drafts, hidden, and `noindex` pages are excluded
|
|
23
|
-
//
|
|
24
|
-
// (
|
|
22
|
+
// Drafts, hidden, and ordinary `noindex` pages are excluded. Generated API
|
|
23
|
+
// references keep crawler visibility (`noindex`) separate from LLM visibility
|
|
24
|
+
// (`ai.exclude`), and are excluded wholesale when `ai.llmsTxt.openapi` is off.
|
|
25
25
|
const eligiblePages = (project: BlumeProject): PageRecord[] =>
|
|
26
26
|
project.graph.pages.filter(
|
|
27
27
|
(page) =>
|
|
28
|
-
!(
|
|
28
|
+
!(
|
|
29
|
+
page.meta.ai.exclude ||
|
|
30
|
+
page.meta.draft ||
|
|
31
|
+
page.meta.sidebar.hidden ||
|
|
32
|
+
(page.meta.seo.noindex && page.source.name !== "openapi")
|
|
33
|
+
) &&
|
|
29
34
|
(project.config.ai.llmsTxt.openapi || page.source.name !== "openapi")
|
|
30
35
|
);
|
|
31
36
|
|
package/src/ai/mcp/server.ts
CHANGED
|
@@ -134,10 +134,26 @@ const text = (value: string, isError = false) => ({
|
|
|
134
134
|
...(isError ? { isError: true } : {}),
|
|
135
135
|
});
|
|
136
136
|
|
|
137
|
+
/** Lazily builds the Orama index over a snapshot's documents, once. */
|
|
138
|
+
export type OramaIndexProvider = () => Promise<
|
|
139
|
+
Awaited<ReturnType<typeof buildOramaIndex>>
|
|
140
|
+
>;
|
|
141
|
+
|
|
142
|
+
/** Memoize the search index so every server built from a snapshot shares it. */
|
|
143
|
+
export const createIndexProvider = (
|
|
144
|
+
documents: OramaDoc[]
|
|
145
|
+
): OramaIndexProvider => {
|
|
146
|
+
let dbPromise: ReturnType<OramaIndexProvider> | null = null;
|
|
147
|
+
return () => {
|
|
148
|
+
dbPromise ??= buildOramaIndex(documents);
|
|
149
|
+
return dbPromise;
|
|
150
|
+
};
|
|
151
|
+
};
|
|
152
|
+
|
|
137
153
|
/** Construct a fresh MCP server with Blume's read-only docs tools registered. */
|
|
138
|
-
const buildServer = (
|
|
154
|
+
export const buildServer = (
|
|
139
155
|
data: McpData,
|
|
140
|
-
index:
|
|
156
|
+
index: OramaIndexProvider
|
|
141
157
|
): Server => {
|
|
142
158
|
const server = new Server(
|
|
143
159
|
{ name: data.name, version: data.version },
|
|
@@ -224,12 +240,7 @@ const buildServer = (
|
|
|
224
240
|
export const createMcpFetchHandler = (
|
|
225
241
|
data: McpData
|
|
226
242
|
): ((request: Request) => Promise<Response>) => {
|
|
227
|
-
|
|
228
|
-
null;
|
|
229
|
-
const index = () => {
|
|
230
|
-
dbPromise ??= buildOramaIndex(data.documents);
|
|
231
|
-
return dbPromise;
|
|
232
|
-
};
|
|
243
|
+
const index = createIndexProvider(data.documents);
|
|
233
244
|
|
|
234
245
|
return async (request: Request): Promise<Response> => {
|
|
235
246
|
if (request.method === "OPTIONS") {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { once } from "node:events";
|
|
2
|
+
import type { Readable, Writable } from "node:stream";
|
|
3
|
+
|
|
4
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
5
|
+
|
|
6
|
+
import type { McpData } from "./data.ts";
|
|
7
|
+
import { buildServer, createIndexProvider } from "./server.ts";
|
|
8
|
+
|
|
9
|
+
/** Streams for `serveMcpStdio`, injectable so tests can use in-memory pipes. */
|
|
10
|
+
export interface McpStdioStreams {
|
|
11
|
+
stdin?: Readable;
|
|
12
|
+
stdout?: Writable;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Serve a precomputed MCP data snapshot over a stdio transport until the
|
|
17
|
+
* client hangs up.
|
|
18
|
+
*
|
|
19
|
+
* The SDK transport reads its input stream but never watches for its end, so
|
|
20
|
+
* a client that simply closes the pipe — which is how every MCP host shuts a
|
|
21
|
+
* stdio server down — would leave the process running forever. EOF on the
|
|
22
|
+
* input stream therefore ends the serve loop explicitly.
|
|
23
|
+
*/
|
|
24
|
+
export const serveMcpStdio = async (
|
|
25
|
+
data: McpData,
|
|
26
|
+
streams: McpStdioStreams = {}
|
|
27
|
+
): Promise<void> => {
|
|
28
|
+
const stdin = streams.stdin ?? process.stdin;
|
|
29
|
+
const stdout = streams.stdout ?? process.stdout;
|
|
30
|
+
const transport = new StdioServerTransport(stdin, stdout);
|
|
31
|
+
const server = buildServer(data, createIndexProvider(data.documents));
|
|
32
|
+
await server.connect(transport);
|
|
33
|
+
await once(stdin, "end");
|
|
34
|
+
await transport.close();
|
|
35
|
+
};
|
package/src/astro/generate.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
2
3
|
import {
|
|
3
4
|
lstat,
|
|
@@ -723,6 +724,22 @@ export const detectUsesMath = async (
|
|
|
723
724
|
return [...contents, ...staged].some(containsMath);
|
|
724
725
|
};
|
|
725
726
|
|
|
727
|
+
const hashConfigSource = (source: string): string =>
|
|
728
|
+
createHash("sha256").update(source).digest("hex");
|
|
729
|
+
|
|
730
|
+
const loadIntegrationBridge = async (
|
|
731
|
+
config: ResolvedConfig,
|
|
732
|
+
context: BlumeProject["context"]
|
|
733
|
+
): Promise<Parameters<typeof astroConfigTemplate>[0]["integrationBridge"]> => {
|
|
734
|
+
if (config.integrations.length === 0 || !context.configFile) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
return {
|
|
738
|
+
configFile: relative(context.outDir, context.configFile),
|
|
739
|
+
sourceHash: hashConfigSource(await readOptional(context.configFile)),
|
|
740
|
+
};
|
|
741
|
+
};
|
|
742
|
+
|
|
726
743
|
const writeIfChanged = async (
|
|
727
744
|
path: string,
|
|
728
745
|
content: string
|
|
@@ -1085,7 +1102,10 @@ export const buildRuntimeData = (project: BlumeProject): string => {
|
|
|
1085
1102
|
analytics: config.analytics ?? null,
|
|
1086
1103
|
appleIcon: resolveAppleIcon(project),
|
|
1087
1104
|
ask: config.ai.ask?.enabled
|
|
1088
|
-
? {
|
|
1105
|
+
? {
|
|
1106
|
+
endpoint: config.ai.ask.endpoint ?? null,
|
|
1107
|
+
suggestions: config.ai.ask.suggestions,
|
|
1108
|
+
}
|
|
1089
1109
|
: null,
|
|
1090
1110
|
banner: resolveBanner(config),
|
|
1091
1111
|
basePath: config.basePath,
|
|
@@ -1279,7 +1299,7 @@ const writeAskFiles = async (
|
|
|
1279
1299
|
write: (path: string, content: string) => Promise<boolean>
|
|
1280
1300
|
): Promise<void> => {
|
|
1281
1301
|
const { ask } = project.config.ai;
|
|
1282
|
-
if (!ask?.enabled) {
|
|
1302
|
+
if (!(ask?.enabled && !ask.endpoint)) {
|
|
1283
1303
|
return;
|
|
1284
1304
|
}
|
|
1285
1305
|
const grounded = ask.provider !== "inkeep";
|
|
@@ -1418,6 +1438,7 @@ export const generateRuntime = async (
|
|
|
1418
1438
|
usesMath,
|
|
1419
1439
|
userTheme,
|
|
1420
1440
|
userExamplesCss,
|
|
1441
|
+
integrationBridge,
|
|
1421
1442
|
islandDiscovery,
|
|
1422
1443
|
exampleDiscovery,
|
|
1423
1444
|
componentSlots,
|
|
@@ -1427,6 +1448,7 @@ export const generateRuntime = async (
|
|
|
1427
1448
|
detectUsesMath(context.root, staged.values()),
|
|
1428
1449
|
readOptional(context.themeFile),
|
|
1429
1450
|
readOptional(examplesCssFile(context.root, config)),
|
|
1451
|
+
loadIntegrationBridge(config, context),
|
|
1430
1452
|
discoverIslands(context.root),
|
|
1431
1453
|
discoverExamples(context.root, config.examples.source),
|
|
1432
1454
|
buildComponentSlots(context.componentsFile),
|
|
@@ -1500,6 +1522,7 @@ export const generateRuntime = async (
|
|
|
1500
1522
|
dataPath,
|
|
1501
1523
|
examplesPath,
|
|
1502
1524
|
examplesThemePath,
|
|
1525
|
+
integrationBridge,
|
|
1503
1526
|
needsReact,
|
|
1504
1527
|
needsSvelte,
|
|
1505
1528
|
needsVue,
|