blume 1.4.2 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/dist/cli/index.js +694 -579
- package/dist/cli/index.js.map +56 -55
- package/dist/types/core/base-path.d.ts +8 -0
- package/dist/types/core/config-input.d.ts +8 -0
- package/dist/types/core/schema.d.ts +4 -0
- package/dist/types/core/sources/types.d.ts +9 -1
- package/dist/types/openapi/references.d.ts +8 -2
- package/docs/configuration/ai.mdx +26 -8
- package/docs/content/sources.mdx +1 -1
- package/package.json +11 -1
- package/src/ai/agent-readability.ts +3 -2
- package/src/ai/api-catalog.ts +2 -2
- package/src/ai/ask-context.ts +45 -12
- package/src/ai/llms.ts +2 -1
- package/src/ai/mcp/discovery.ts +25 -6
- package/src/ai/mcp/server.ts +108 -98
- package/src/ai/tar.ts +29 -70
- package/src/astro/examples.ts +7 -3
- package/src/astro/generate.ts +59 -34
- package/src/astro/islands.ts +7 -3
- package/src/astro/templates.ts +34 -9
- package/src/audit/agent.ts +14 -29
- package/src/audit/crawl.ts +41 -16
- package/src/audit/run.ts +10 -3
- package/src/audit/snapshot.ts +27 -2
- package/src/cli/commands/audit.ts +12 -17
- package/src/cli/commands/build.ts +15 -7
- package/src/cli/commands/dev.ts +13 -15
- package/src/cli/commands/eject.ts +4 -4
- package/src/cli/commands/eval.ts +17 -27
- package/src/cli/env.ts +13 -30
- package/src/cli/init/scaffold.ts +21 -0
- package/src/cli/report-format.ts +22 -0
- package/src/components/content/AccordionItem.astro +2 -9
- package/src/components/content/ColorItem.astro +5 -13
- package/src/components/content/Component.astro +12 -8
- package/src/components/content/Frame.astro +2 -12
- package/src/components/content/Prompt.astro +12 -31
- package/src/components/content/Tab.astro +2 -9
- package/src/components/content/Tooltip.astro +1 -9
- package/src/components/content/Update.astro +2 -9
- package/src/components/content/inline-markdown.ts +28 -0
- package/src/components/copy-feedback.ts +96 -0
- package/src/components/islands/ask-ai.tsx +78 -9
- package/src/components/layout/PageActions.astro +20 -32
- package/src/components/layout/PageLayout.astro +8 -28
- package/src/components/layout/RootLayout.astro +6 -48
- package/src/components/layout/Search.astro +56 -9
- package/src/components/layout/drawer-inert.ts +31 -0
- package/src/components/layout/search/pagefind.ts +6 -5
- package/src/components/layout/search/types.ts +32 -0
- package/src/components/openapi/panel.ts +11 -8
- package/src/components/raf-throttle.ts +21 -0
- package/src/components/slug.ts +14 -0
- package/src/core/base-path.ts +18 -1
- package/src/core/config-input.ts +8 -0
- package/src/core/frontmatter.ts +45 -1
- package/src/core/probe.ts +7 -19
- package/src/core/project-graph.ts +12 -1
- package/src/core/schema.ts +6 -0
- package/src/core/site-url.ts +27 -0
- package/src/core/sources/cache.ts +10 -8
- package/src/core/sources/github-releases.ts +21 -1
- package/src/core/sources/normalize.ts +26 -2
- package/src/core/sources/notion.ts +27 -5
- package/src/core/sources/portable-text.ts +16 -1
- package/src/core/sources/resolve.ts +1 -0
- package/src/core/sources/types.ts +13 -1
- package/src/deploy/cloudflare-negotiation.ts +15 -1
- package/src/deploy/robots.ts +2 -1
- package/src/deploy/rss.ts +2 -1
- package/src/deploy/sitemap.ts +56 -7
- package/src/eval/agents.ts +13 -10
- package/src/eval/report.ts +1 -14
- package/src/markdown/package-commands.ts +61 -54
- package/src/og/card.ts +24 -26
- package/src/openapi/model.ts +9 -9
- package/src/openapi/parse.ts +69 -28
- package/src/openapi/references.ts +35 -12
- package/src/openapi/render-mdx.ts +64 -25
- package/src/openapi/scalar.ts +2 -2
- package/src/openapi/source.ts +28 -1
- package/src/search/documents.ts +78 -34
- package/src/search/orama-index.ts +51 -12
- package/src/theme/palette.ts +6 -2
- package/src/translate/ledger.ts +4 -2
- package/src/translate/report.ts +1 -14
- package/src/translate/run.ts +20 -35
- package/src/cli/coalesce.ts +0 -43
package/src/astro/templates.ts
CHANGED
|
@@ -868,11 +868,24 @@ export const collections = { docs${options.staged ? ", staged" : ""} };
|
|
|
868
868
|
};
|
|
869
869
|
|
|
870
870
|
/** Generate `.blume/src/pages/[...slug].astro`, the docs catch-all route. */
|
|
871
|
-
/**
|
|
871
|
+
/** The plain prompt used when there is no grounding context to inject. */
|
|
872
|
+
const ASK_FALLBACK_PROMPT =
|
|
873
|
+
"You are a helpful documentation assistant. Answer using the project's documentation.";
|
|
874
|
+
|
|
875
|
+
/**
|
|
876
|
+
* Generate the Ask AI server endpoint (`.blume/src/pages/api/ask.ts`).
|
|
877
|
+
* `instructions` (the `ai.ask.instructions` config) is appended to the
|
|
878
|
+
* built-in prompt on every path: the grounded prompt via `createAskContext`,
|
|
879
|
+
* and the plain fallback here.
|
|
880
|
+
*/
|
|
872
881
|
export const askEndpointTemplate = (
|
|
873
882
|
backend: AskBackend,
|
|
874
|
-
grounded: boolean
|
|
883
|
+
grounded: boolean,
|
|
884
|
+
instructions?: string
|
|
875
885
|
): string => {
|
|
886
|
+
const fallbackPrompt = instructions
|
|
887
|
+
? `${ASK_FALLBACK_PROMPT}\n\n${instructions}`
|
|
888
|
+
: ASK_FALLBACK_PROMPT;
|
|
876
889
|
const imports = [
|
|
877
890
|
'import type { APIRoute } from "astro";',
|
|
878
891
|
'import { streamText } from "ai";',
|
|
@@ -905,7 +918,10 @@ export const askEndpointTemplate = (
|
|
|
905
918
|
'import { createAskContext } from "blume/ai/ask-context.ts";',
|
|
906
919
|
'import askData from "../../generated/ask-data.json";'
|
|
907
920
|
);
|
|
908
|
-
|
|
921
|
+
const groundOptions = instructions
|
|
922
|
+
? `, { instructions: ${JSON.stringify(instructions)} }`
|
|
923
|
+
: "";
|
|
924
|
+
setup += `\nconst ground = createAskContext(askData${groundOptions});\n`;
|
|
909
925
|
}
|
|
910
926
|
// Validate the client-supplied body and cap its size. The endpoint is
|
|
911
927
|
// unauthenticated, so bounding message count/length limits how much a caller
|
|
@@ -966,7 +982,7 @@ export const askEndpointTemplate = (
|
|
|
966
982
|
const stream = grounded
|
|
967
983
|
? ` const instructions =
|
|
968
984
|
(await ground(messages, body.page)) ??
|
|
969
|
-
|
|
985
|
+
${JSON.stringify(fallbackPrompt)};
|
|
970
986
|
const result = streamText({
|
|
971
987
|
model: ${modelExpr},
|
|
972
988
|
instructions,
|
|
@@ -976,7 +992,7 @@ ${onError}
|
|
|
976
992
|
: ` const result = streamText({
|
|
977
993
|
model: ${modelExpr},
|
|
978
994
|
instructions:
|
|
979
|
-
|
|
995
|
+
${JSON.stringify(fallbackPrompt)},
|
|
980
996
|
messages,
|
|
981
997
|
${onError}
|
|
982
998
|
});`;
|
|
@@ -1261,7 +1277,7 @@ export const contentAssetsEndpointTemplate = (
|
|
|
1261
1277
|
`// Generated by Blume. Do not edit.
|
|
1262
1278
|
import { existsSync } from "node:fs";
|
|
1263
1279
|
import { readdir, readFile } from "node:fs/promises";
|
|
1264
|
-
import { join, relative, resolve } from "node:path";
|
|
1280
|
+
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
1265
1281
|
import type { APIRoute } from "astro";
|
|
1266
1282
|
import assets from "../../generated/content-assets.json";
|
|
1267
1283
|
|
|
@@ -1321,7 +1337,13 @@ const resolveAsset = (asset: string): string | null => {
|
|
|
1321
1337
|
const abs = resolve(STAGED_DIR, asset);
|
|
1322
1338
|
// Traversal guard: the dev server renders on demand, so the param is
|
|
1323
1339
|
// attacker-controlled there — never step outside the staged directory.
|
|
1324
|
-
|
|
1340
|
+
// path.relative rather than a string-prefix test: STAGED_DIR is baked in
|
|
1341
|
+
// with forward slashes while resolve() answers in the platform's
|
|
1342
|
+
// separators, so on Windows the prefix test 404'd every legitimate staged
|
|
1343
|
+
// asset — and a bare prefix also admits a sibling directory that merely
|
|
1344
|
+
// shares the name.
|
|
1345
|
+
const rel = relative(STAGED_DIR, abs);
|
|
1346
|
+
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
|
|
1325
1347
|
return null;
|
|
1326
1348
|
}
|
|
1327
1349
|
return existsSync(abs) ? abs : null;
|
|
@@ -1680,8 +1702,11 @@ const frontmatter = entry.data ?? {};
|
|
|
1680
1702
|
const seo = frontmatter.seo ?? {};
|
|
1681
1703
|
const base = data.config.site ? data.config.site.replace(/\\/$/, "") : null;
|
|
1682
1704
|
|
|
1705
|
+
// Percent-encode the route-derived path (the sitemap convention): a Unicode
|
|
1706
|
+
// slug (\`/api/größe\`) is not legal in a raw URI, and crawlers compare
|
|
1707
|
+
// canonical against the sitemap's encoded <loc> byte-for-byte.
|
|
1683
1708
|
const ogPath = data.config.og.enabled
|
|
1684
|
-
? \`/og/\${route === "/" ? "index" : route.slice(1)}.png\`
|
|
1709
|
+
? encodeURI(\`/og/\${route === "/" ? "index" : route.slice(1)}.png\`)
|
|
1685
1710
|
: null;
|
|
1686
1711
|
const ogRel = seo.image ?? ogPath;
|
|
1687
1712
|
// Absolute URLs also carry the deployment base (the page is served under it):
|
|
@@ -1700,7 +1725,7 @@ const x = { ...data.config.x, ...(seo.x?.creator ? { creator: seo.x.creator } :
|
|
|
1700
1725
|
const basedRoute = withBase(route);
|
|
1701
1726
|
const canonical =
|
|
1702
1727
|
seo.canonical ??
|
|
1703
|
-
(base ? \`\${base}\${basedRoute === "/" ? "" : basedRoute}\` : null);
|
|
1728
|
+
(base ? \`\${base}\${basedRoute === "/" ? "" : encodeURI(basedRoute)}\` : null);
|
|
1704
1729
|
|
|
1705
1730
|
// Locale resolution. With i18n on, pick the active locale's nav + dictionary,
|
|
1706
1731
|
// build hreflang alternates, and derive the language-switcher targets.
|
package/src/audit/agent.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
2
1
|
import { mkdtemp, writeFile } from "node:fs/promises";
|
|
3
2
|
import { tmpdir } from "node:os";
|
|
4
3
|
|
|
4
|
+
import spawn from "cross-spawn";
|
|
5
5
|
import { join } from "pathe";
|
|
6
6
|
|
|
7
7
|
import { reportJson } from "./report.ts";
|
|
@@ -61,37 +61,26 @@ Work through every finding:
|
|
|
61
61
|
|
|
62
62
|
When you are done, run \`blume build\` and then \`blume audit\` to verify, and repeat until the audit reports no issues.`;
|
|
63
63
|
|
|
64
|
-
const spawnAgent = (
|
|
65
|
-
command: string,
|
|
66
|
-
args: string[],
|
|
67
|
-
shell: boolean
|
|
68
|
-
): Promise<number> =>
|
|
64
|
+
const spawnAgent = (bin: string, args: string[]): Promise<number> =>
|
|
69
65
|
// oxlint-disable-next-line promise/avoid-new -- adapt spawn's event callbacks
|
|
70
66
|
new Promise((resolve, reject) => {
|
|
71
|
-
const child = spawn(
|
|
67
|
+
const child = spawn(bin, args, { stdio: "inherit" });
|
|
72
68
|
child.once("error", reject);
|
|
73
69
|
child.once("close", (code) => resolve(code ?? 1));
|
|
74
70
|
});
|
|
75
71
|
|
|
76
|
-
/**
|
|
77
|
-
* cmd.exe reports a missing executable through this exit code instead of a
|
|
78
|
-
* spawn error, so a shell launch can't rely on the `error` event for the
|
|
79
|
-
* "not installed" diagnosis.
|
|
80
|
-
*/
|
|
81
|
-
export const WINDOWS_COMMAND_NOT_FOUND = 9009;
|
|
82
|
-
|
|
83
72
|
/**
|
|
84
73
|
* Run the agent CLI interactively with the handoff prompt, inheriting the
|
|
85
74
|
* terminal so the user watches and steers the fixes rather than granting a
|
|
86
75
|
* headless process blanket write access. Resolves with the agent's exit code;
|
|
87
|
-
* rejects when the executable isn't on PATH
|
|
76
|
+
* rejects when the executable isn't on PATH — on every platform, since
|
|
77
|
+
* cross-spawn resolves npm's `.cmd` shims itself and escapes their arguments
|
|
78
|
+
* (the old `shell: true` launch left quoting to a hand-built command line and
|
|
79
|
+
* reported a missing executable as cmd.exe's exit code 9009).
|
|
88
80
|
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* file next to the report and handed over via a one-line pointer that survives
|
|
93
|
-
* cmd.exe quoting; a missing executable surfaces as
|
|
94
|
-
* {@link WINDOWS_COMMAND_NOT_FOUND} rather than a rejection.
|
|
81
|
+
* cmd.exe still cannot carry the multi-line prompt as an argument (a newline
|
|
82
|
+
* ends the command), so on Windows the prompt is written to a file next to
|
|
83
|
+
* the report and handed over via a one-line pointer.
|
|
95
84
|
*/
|
|
96
85
|
export const launchAgent = async (
|
|
97
86
|
bin: string,
|
|
@@ -99,16 +88,12 @@ export const launchAgent = async (
|
|
|
99
88
|
platform: NodeJS.Platform = process.platform
|
|
100
89
|
): Promise<number> => {
|
|
101
90
|
if (platform !== "win32") {
|
|
102
|
-
return await spawnAgent(bin, [prompt]
|
|
91
|
+
return await spawnAgent(bin, [prompt]);
|
|
103
92
|
}
|
|
104
93
|
const dir = await mkdtemp(join(tmpdir(), "blume-audit-"));
|
|
105
94
|
const promptPath = join(dir, "prompt.md");
|
|
106
95
|
await writeFile(promptPath, prompt);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
`"${bin}" "Read ${promptPath} and follow its instructions exactly."`,
|
|
111
|
-
[],
|
|
112
|
-
true
|
|
113
|
-
);
|
|
96
|
+
return await spawnAgent(bin, [
|
|
97
|
+
`Read ${promptPath} and follow its instructions exactly.`,
|
|
98
|
+
]);
|
|
114
99
|
};
|
package/src/audit/crawl.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { readFile, stat } from "node:fs/promises";
|
|
2
2
|
|
|
3
3
|
import { XMLParser } from "fast-xml-parser";
|
|
4
|
+
import type { Nodes } from "mdast";
|
|
5
|
+
import { fromMarkdown } from "mdast-util-from-markdown";
|
|
6
|
+
import pMap from "p-map";
|
|
4
7
|
import { join, relative } from "pathe";
|
|
5
8
|
import { glob } from "tinyglobby";
|
|
6
9
|
|
|
@@ -18,6 +21,13 @@ import type { LlmsDoc, PageSnapshot, RobotsDoc, SitemapDoc } from "./types.ts";
|
|
|
18
21
|
*/
|
|
19
22
|
const EXAMPLES_PREFIX = `${examplesRouteBase("")}/`;
|
|
20
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Ceiling on concurrent file reads/stats while crawling. Unbounded fan-out
|
|
26
|
+
* over a large `dist` risks EMFILE and holds every page's HTML in memory at
|
|
27
|
+
* once.
|
|
28
|
+
*/
|
|
29
|
+
const CRAWL_CONCURRENCY = 16;
|
|
30
|
+
|
|
21
31
|
/** Everything read off disk in one pass over the built site. */
|
|
22
32
|
export interface CrawlResult {
|
|
23
33
|
pages: PageSnapshot[];
|
|
@@ -45,12 +55,14 @@ export const fileToUrl = (staticDir: string, file: string): string => {
|
|
|
45
55
|
*/
|
|
46
56
|
const indexFiles = async (staticDir: string): Promise<Map<string, number>> => {
|
|
47
57
|
const found = await glob("**/*", { cwd: staticDir, dot: true });
|
|
48
|
-
const sized = await
|
|
49
|
-
found
|
|
58
|
+
const sized = await pMap(
|
|
59
|
+
found,
|
|
60
|
+
async (file) => {
|
|
50
61
|
const path = `/${file.replaceAll("\\", "/")}`;
|
|
51
62
|
const info = await stat(join(staticDir, file));
|
|
52
63
|
return [path, info.size] as const;
|
|
53
|
-
}
|
|
64
|
+
},
|
|
65
|
+
{ concurrency: CRAWL_CONCURRENCY }
|
|
54
66
|
);
|
|
55
67
|
return new Map(sized);
|
|
56
68
|
};
|
|
@@ -135,21 +147,32 @@ export const parseSitemap = (
|
|
|
135
147
|
};
|
|
136
148
|
|
|
137
149
|
/**
|
|
138
|
-
* Parse the `llms.txt` index into its Markdown link targets
|
|
139
|
-
*
|
|
140
|
-
*
|
|
150
|
+
* Parse the `llms.txt` index into its Markdown link targets, with the line
|
|
151
|
+
* each target sits on so findings can point at it. A real parse rather than a
|
|
152
|
+
* `](url)` regex: angle-bracket destinations, link titles, reference-style
|
|
153
|
+
* links (the definition line carries the URL), and autolinks all resolve, and
|
|
154
|
+
* a link-shaped string inside a fenced code block is no longer reported as a
|
|
155
|
+
* claim. Blume's own llms.txt only emits inline links, but the file is also
|
|
156
|
+
* hand-edited.
|
|
141
157
|
*/
|
|
142
158
|
export const parseLlms = (file: string, text: string): LlmsDoc => {
|
|
143
159
|
const entries: LlmsDoc["entries"] = [];
|
|
144
|
-
const
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
160
|
+
const collect = (node: Nodes): void => {
|
|
161
|
+
if (
|
|
162
|
+
(node.type === "link" ||
|
|
163
|
+
node.type === "image" ||
|
|
164
|
+
node.type === "definition") &&
|
|
165
|
+
node.url
|
|
166
|
+
) {
|
|
167
|
+
entries.push({ line: node.position?.start.line ?? 1, url: node.url });
|
|
168
|
+
}
|
|
169
|
+
if ("children" in node) {
|
|
170
|
+
for (const child of node.children) {
|
|
171
|
+
collect(child);
|
|
150
172
|
}
|
|
151
173
|
}
|
|
152
|
-
}
|
|
174
|
+
};
|
|
175
|
+
collect(fromMarkdown(text));
|
|
153
176
|
return { entries, file };
|
|
154
177
|
};
|
|
155
178
|
|
|
@@ -214,8 +237,9 @@ export const crawlStaticDir = async (options: {
|
|
|
214
237
|
const routes = routeIndex(manifest, basePath);
|
|
215
238
|
|
|
216
239
|
const htmlFiles = await glob("**/*.html", { absolute: true, cwd: staticDir });
|
|
217
|
-
const snapshots = await
|
|
218
|
-
htmlFiles.toSorted()
|
|
240
|
+
const snapshots = await pMap(
|
|
241
|
+
htmlFiles.toSorted(),
|
|
242
|
+
async (file) => {
|
|
219
243
|
const url = fileToUrl(staticDir, file);
|
|
220
244
|
if (stripBasePath(basePath, url).startsWith(EXAMPLES_PREFIX)) {
|
|
221
245
|
return null;
|
|
@@ -230,7 +254,8 @@ export const crawlStaticDir = async (options: {
|
|
|
230
254
|
route: routes.get(url) ?? routes.get(stripBasePath(basePath, url)),
|
|
231
255
|
url,
|
|
232
256
|
});
|
|
233
|
-
}
|
|
257
|
+
},
|
|
258
|
+
{ concurrency: CRAWL_CONCURRENCY }
|
|
234
259
|
);
|
|
235
260
|
const pages = snapshots.filter((page) => page !== null);
|
|
236
261
|
|
package/src/audit/run.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
|
|
3
|
+
import pMap from "p-map";
|
|
4
|
+
|
|
3
5
|
import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
|
|
4
6
|
import type { BlumeProject } from "../core/project-graph.ts";
|
|
5
7
|
import type { Diagnostic } from "../core/types.ts";
|
|
@@ -88,6 +90,9 @@ export class NoBuildError extends Error {
|
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
|
|
93
|
+
/** Ceiling on concurrent source reads; unbounded fan-out risks EMFILE. */
|
|
94
|
+
const READ_CONCURRENCY = 16;
|
|
95
|
+
|
|
91
96
|
/** Read every page's source file once, so findings can cite front matter lines. */
|
|
92
97
|
const readSources = async (
|
|
93
98
|
pages: PageSnapshot[]
|
|
@@ -95,8 +100,9 @@ const readSources = async (
|
|
|
95
100
|
const paths = [
|
|
96
101
|
...new Set(pages.flatMap((page) => (page.source ? [page.source] : []))),
|
|
97
102
|
];
|
|
98
|
-
const entries = await
|
|
99
|
-
paths
|
|
103
|
+
const entries = await pMap(
|
|
104
|
+
paths,
|
|
105
|
+
async (path) => {
|
|
100
106
|
try {
|
|
101
107
|
return [path, await readFile(path, "utf-8")] as const;
|
|
102
108
|
} catch {
|
|
@@ -104,7 +110,8 @@ const readSources = async (
|
|
|
104
110
|
// still names the URL; it just can't cite a line.
|
|
105
111
|
return null;
|
|
106
112
|
}
|
|
107
|
-
}
|
|
113
|
+
},
|
|
114
|
+
{ concurrency: READ_CONCURRENCY }
|
|
108
115
|
);
|
|
109
116
|
return new Map(entries.filter((entry) => entry !== null));
|
|
110
117
|
};
|
package/src/audit/snapshot.ts
CHANGED
|
@@ -8,6 +8,30 @@ import type { PageSnapshot, SnapshotAsset, SnapshotLink } from "./types.ts";
|
|
|
8
8
|
/** Site chrome: links here are navigation, not editorial. */
|
|
9
9
|
const CHROME = "nav, aside, header, footer";
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Count prose words with `Intl.Segmenter`, seeded with the page's declared
|
|
13
|
+
* language. The whitespace split this replaces counted a fully written
|
|
14
|
+
* Japanese or Chinese page — scripts with no interword spaces — as a handful
|
|
15
|
+
* of "words", tripping BLUME_AUDIT_LOW_WORD_COUNT on every page of a CJK
|
|
16
|
+
* site. An unparseable `lang` attribute falls back to the default locale
|
|
17
|
+
* rather than failing the crawl.
|
|
18
|
+
*/
|
|
19
|
+
const countWords = (prose: string, lang: string | null): number => {
|
|
20
|
+
let segmenter: Intl.Segmenter;
|
|
21
|
+
try {
|
|
22
|
+
segmenter = new Intl.Segmenter(lang ?? undefined, { granularity: "word" });
|
|
23
|
+
} catch {
|
|
24
|
+
segmenter = new Intl.Segmenter(undefined, { granularity: "word" });
|
|
25
|
+
}
|
|
26
|
+
let count = 0;
|
|
27
|
+
for (const segment of segmenter.segment(prose)) {
|
|
28
|
+
if (segment.isWordLike) {
|
|
29
|
+
count += 1;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return count;
|
|
33
|
+
};
|
|
34
|
+
|
|
11
35
|
/** Where a page's prose lives, in preference order. */
|
|
12
36
|
const CONTENT_ROOTS = ["main", "article", "body"];
|
|
13
37
|
|
|
@@ -123,6 +147,7 @@ export const buildSnapshot = (options: {
|
|
|
123
147
|
const root = contentRoot(document);
|
|
124
148
|
const prose = root ? visibleText(root) : "";
|
|
125
149
|
const robots = document.querySelector('meta[name="robots"]');
|
|
150
|
+
const lang = attr(document.querySelector("html") ?? document, "lang");
|
|
126
151
|
const { jsonld, jsonldErrors } = collectJsonLd(document);
|
|
127
152
|
|
|
128
153
|
return {
|
|
@@ -160,7 +185,7 @@ export const buildSnapshot = (options: {
|
|
|
160
185
|
indexable: !robots?.getAttribute("content")?.includes("noindex"),
|
|
161
186
|
jsonld,
|
|
162
187
|
jsonldErrors,
|
|
163
|
-
lang
|
|
188
|
+
lang,
|
|
164
189
|
links,
|
|
165
190
|
metaRefresh:
|
|
166
191
|
document
|
|
@@ -184,6 +209,6 @@ export const buildSnapshot = (options: {
|
|
|
184
209
|
.querySelector('meta[name="viewport"]')
|
|
185
210
|
?.getAttribute("content")
|
|
186
211
|
?.trim() ?? null,
|
|
187
|
-
wordCount: prose
|
|
212
|
+
wordCount: countWords(prose, lang),
|
|
188
213
|
};
|
|
189
214
|
};
|
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
AGENTS,
|
|
5
5
|
fixPrompt,
|
|
6
6
|
launchAgent,
|
|
7
|
-
WINDOWS_COMMAND_NOT_FOUND,
|
|
8
7
|
writeAgentReport,
|
|
9
8
|
} from "../../audit/agent.ts";
|
|
10
9
|
import type { AgentKind } from "../../audit/agent.ts";
|
|
@@ -41,22 +40,26 @@ export const shouldFail = (
|
|
|
41
40
|
};
|
|
42
41
|
|
|
43
42
|
/**
|
|
44
|
-
* Launch the agent CLI,
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
43
|
+
* Launch the agent CLI, turning a missing executable into the install hint.
|
|
44
|
+
* Only `ENOENT` means "not installed" — any other spawn failure (`EACCES`,
|
|
45
|
+
* `EMFILE`, …) must surface as itself, not be masked by an irrelevant
|
|
46
|
+
* install hint.
|
|
48
47
|
*/
|
|
49
48
|
const launchAgentCode = async (
|
|
50
|
-
|
|
49
|
+
agent: AgentKind,
|
|
51
50
|
prompt: string
|
|
52
51
|
): Promise<number> => {
|
|
52
|
+
const cli = AGENTS[agent];
|
|
53
53
|
try {
|
|
54
|
-
return await launchAgent(bin, prompt);
|
|
54
|
+
return await launchAgent(cli.bin, prompt);
|
|
55
55
|
} catch (error) {
|
|
56
56
|
if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
|
57
57
|
throw error;
|
|
58
58
|
}
|
|
59
|
-
|
|
59
|
+
logger.error(
|
|
60
|
+
`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
|
|
61
|
+
);
|
|
62
|
+
return process.exit(1);
|
|
60
63
|
}
|
|
61
64
|
};
|
|
62
65
|
|
|
@@ -183,15 +186,7 @@ export const auditCommand = defineCommand({
|
|
|
183
186
|
process.stderr.write(
|
|
184
187
|
` Handing ${count} finding${count === 1 ? "" : "s"} to ${cli.name}…\n\n`
|
|
185
188
|
);
|
|
186
|
-
const code = await launchAgentCode(
|
|
187
|
-
// A POSIX spawn rejects on a missing executable; the Windows shell
|
|
188
|
-
// launch reports it through cmd.exe's 9009 instead. Same diagnosis.
|
|
189
|
-
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
190
|
-
logger.error(
|
|
191
|
-
`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
|
|
192
|
-
);
|
|
193
|
-
process.exit(1);
|
|
194
|
-
}
|
|
189
|
+
const code = await launchAgentCode(agent, fixPrompt(report));
|
|
195
190
|
if (code !== 0) {
|
|
196
191
|
process.exit(code);
|
|
197
192
|
}
|
|
@@ -54,7 +54,7 @@ import {
|
|
|
54
54
|
platformRedirects,
|
|
55
55
|
} from "../../deploy/redirects.ts";
|
|
56
56
|
import { buildRobots } from "../../deploy/robots.ts";
|
|
57
|
-
import {
|
|
57
|
+
import { buildSitemapFiles } from "../../deploy/sitemap.ts";
|
|
58
58
|
import { injectNegotiationRoutes } from "../../deploy/vercel-negotiation.ts";
|
|
59
59
|
import { buildSearchIndex } from "../../search/build.ts";
|
|
60
60
|
import { syncSearchProvider } from "../../search/sync/index.ts";
|
|
@@ -616,10 +616,18 @@ const publishBuildArtifacts = async (
|
|
|
616
616
|
}
|
|
617
617
|
|
|
618
618
|
// A user's own public/ file (copied into dist by Astro) always wins.
|
|
619
|
-
const
|
|
620
|
-
if (
|
|
621
|
-
await
|
|
622
|
-
|
|
619
|
+
const sitemapFiles = buildSitemapFiles(project);
|
|
620
|
+
if (sitemapFiles && !existsSync(join(distDir, "sitemap.xml"))) {
|
|
621
|
+
await Promise.all(
|
|
622
|
+
sitemapFiles.map((file) =>
|
|
623
|
+
writeFile(join(distDir, file.name), file.xml, "utf-8")
|
|
624
|
+
)
|
|
625
|
+
);
|
|
626
|
+
logger.success(
|
|
627
|
+
sitemapFiles.length === 1
|
|
628
|
+
? "Generated sitemap.xml"
|
|
629
|
+
: `Generated sitemap.xml (index of ${sitemapFiles.length - 1} sitemap files)`
|
|
630
|
+
);
|
|
623
631
|
}
|
|
624
632
|
|
|
625
633
|
const robots = buildRobots(project);
|
|
@@ -649,7 +657,7 @@ const publishBuildArtifacts = async (
|
|
|
649
657
|
|
|
650
658
|
const { config } = project;
|
|
651
659
|
const features = serverFeatures(config);
|
|
652
|
-
// `
|
|
660
|
+
// `buildSitemapFiles` returns null both when the sitemap is disabled and when no
|
|
653
661
|
// `site` is configured — only the latter deserves the remediation hint.
|
|
654
662
|
const sitemapNote = config.seo.sitemap
|
|
655
663
|
? "no (set deployment.site)"
|
|
@@ -661,7 +669,7 @@ const publishBuildArtifacts = async (
|
|
|
661
669
|
`Site ${config.deployment.site ?? "not set"}`,
|
|
662
670
|
`Search ${config.search.provider}`,
|
|
663
671
|
`Redirects ${config.redirects.length}`,
|
|
664
|
-
`Sitemap ${
|
|
672
|
+
`Sitemap ${sitemapFiles ? "yes" : sitemapNote}`,
|
|
665
673
|
`Robots ${robots ? "yes" : "no"}`,
|
|
666
674
|
`Agent JSON ${agentReadability ? "yes" : "no"}`,
|
|
667
675
|
`LLM files ${config.ai.llmsTxt.enabled ? "yes" : "no"}`,
|
package/src/cli/commands/dev.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { dev } from "astro";
|
|
2
2
|
import { watch } from "chokidar";
|
|
3
3
|
import { defineCommand } from "citty";
|
|
4
|
+
import { debounce } from "perfect-debounce";
|
|
4
5
|
|
|
5
6
|
import { generateRuntime } from "../../astro/generate.ts";
|
|
6
7
|
import { showBlumeErrorOverlay } from "../../astro/integration.ts";
|
|
7
8
|
import { scanProject } from "../../core/project-graph.ts";
|
|
8
9
|
import { resolveRuntimeDir } from "../../core/project.ts";
|
|
9
10
|
import { parsePort } from "../args.ts";
|
|
10
|
-
import { coalescedRunner } from "../coalesce.ts";
|
|
11
11
|
import {
|
|
12
12
|
acquireDevLock,
|
|
13
13
|
describeDevLock,
|
|
@@ -146,10 +146,16 @@ export const devCommand = defineCommand({
|
|
|
146
146
|
// watcher misses directory renames, so a renamed page 404s (`getEntry` reads
|
|
147
147
|
// a stale in-memory store) until the server is restarted. We restart it
|
|
148
148
|
// ourselves — stop, regenerate while down (no watcher races), then bring up
|
|
149
|
-
// a fresh container whose cold sync re-globs everything.
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
|
|
149
|
+
// a fresh container whose cold sync re-globs everything. perfect-debounce
|
|
150
|
+
// both debounces the watch burst (80ms) and single-flights the scan: a
|
|
151
|
+
// trigger during a run never starts a second run, only marks one trailing
|
|
152
|
+
// rerun after the current settles. Both halves are load-bearing — a plain
|
|
153
|
+
// debounce once let bursts stack overlapping scans until the heap was
|
|
154
|
+
// exhausted (observed as an OOM after minutes of looping). The contract is
|
|
155
|
+
// pinned by test/dev-debounce.test.ts. The task must not reject (the
|
|
156
|
+
// library re-invokes it from an unhandled .finally), so the body catches
|
|
157
|
+
// its own errors and always resolves.
|
|
158
|
+
const regenerate = debounce(async () => {
|
|
153
159
|
try {
|
|
154
160
|
const next = await scanProject(root, {
|
|
155
161
|
devServerUrl,
|
|
@@ -181,21 +187,13 @@ export const devCommand = defineCommand({
|
|
|
181
187
|
} catch (error) {
|
|
182
188
|
logger.error(`Regeneration failed: ${(error as Error).message}`);
|
|
183
189
|
}
|
|
184
|
-
});
|
|
185
|
-
|
|
186
|
-
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
187
|
-
const regenerate = () => {
|
|
188
|
-
if (timer) {
|
|
189
|
-
clearTimeout(timer);
|
|
190
|
-
}
|
|
191
|
-
timer = setTimeout(runRegenerate, 80);
|
|
192
|
-
};
|
|
190
|
+
}, 80);
|
|
193
191
|
|
|
194
192
|
// The runtime prepared above baked the *requested* port into the site
|
|
195
193
|
// fallback; if Vite bumped it, regenerate so OG images, canonicals, and
|
|
196
194
|
// other site-gated URLs point at the port actually serving.
|
|
197
195
|
if (boundPort !== port) {
|
|
198
|
-
void
|
|
196
|
+
void regenerate();
|
|
199
197
|
}
|
|
200
198
|
|
|
201
199
|
// Content is watched per source (filesystem uses fs.watch; remote sources
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
droppedArtifactNotices,
|
|
9
9
|
updatePackageScripts,
|
|
10
10
|
} from "../eject-scripts.ts";
|
|
11
|
-
import { commandsFor,
|
|
11
|
+
import { commandsFor, detectProjectPackageManager } from "../init/scaffold.ts";
|
|
12
12
|
import { logger } from "../log.ts";
|
|
13
13
|
|
|
14
14
|
/**
|
|
@@ -76,9 +76,9 @@ export const ejectCommand = defineCommand({
|
|
|
76
76
|
process.stdout.write(` ${relative(root, file)}\n`);
|
|
77
77
|
}
|
|
78
78
|
reportDroppedArtifacts(notices);
|
|
79
|
-
// Print run commands matching the
|
|
80
|
-
//
|
|
81
|
-
const pm =
|
|
79
|
+
// Print run commands matching the project's package manager (lockfile
|
|
80
|
+
// detection, since eject runs inside an existing project).
|
|
81
|
+
const pm = await detectProjectPackageManager(root);
|
|
82
82
|
const { build, dev } = commandsFor(pm);
|
|
83
83
|
logger.box(
|
|
84
84
|
`Your project is now a standalone Astro app.\n\n ${dev}\n ${build}\n\nThe blume package remains importable.`
|
package/src/cli/commands/eval.ts
CHANGED
|
@@ -3,11 +3,7 @@ import { existsSync } from "node:fs";
|
|
|
3
3
|
import { defineCommand } from "citty";
|
|
4
4
|
import { join } from "pathe";
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
AGENTS,
|
|
8
|
-
launchAgent,
|
|
9
|
-
WINDOWS_COMMAND_NOT_FOUND,
|
|
10
|
-
} from "../../audit/agent.ts";
|
|
6
|
+
import { AGENTS, launchAgent } from "../../audit/agent.ts";
|
|
11
7
|
import type { AgentKind } from "../../audit/agent.ts";
|
|
12
8
|
import { BlumeError } from "../../core/diagnostics.ts";
|
|
13
9
|
import { scanProject } from "../../core/project-graph.ts";
|
|
@@ -37,32 +33,32 @@ const DEFAULT_TIMEOUT_S = 180;
|
|
|
37
33
|
const isAgentKind = (value: string): value is AgentKind => value in AGENTS;
|
|
38
34
|
|
|
39
35
|
/**
|
|
40
|
-
* Launch the interactive agent CLI,
|
|
41
|
-
*
|
|
42
|
-
*
|
|
36
|
+
* Launch the interactive agent CLI, turning a missing executable into the
|
|
37
|
+
* install hint. Only `ENOENT` means "not installed" — any other spawn failure
|
|
38
|
+
* (`EACCES`, `EMFILE`, …) must surface as itself.
|
|
43
39
|
*/
|
|
40
|
+
const notInstalled = (agent: AgentKind): never => {
|
|
41
|
+
const cli = AGENTS[agent];
|
|
42
|
+
logger.error(
|
|
43
|
+
`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
|
|
44
|
+
);
|
|
45
|
+
return process.exit(1);
|
|
46
|
+
};
|
|
47
|
+
|
|
44
48
|
const launchAgentCode = async (
|
|
45
|
-
|
|
49
|
+
agent: AgentKind,
|
|
46
50
|
prompt: string
|
|
47
51
|
): Promise<number> => {
|
|
48
52
|
try {
|
|
49
|
-
return await launchAgent(bin, prompt);
|
|
53
|
+
return await launchAgent(AGENTS[agent].bin, prompt);
|
|
50
54
|
} catch (error) {
|
|
51
55
|
if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
|
52
56
|
throw error;
|
|
53
57
|
}
|
|
54
|
-
return
|
|
58
|
+
return notInstalled(agent);
|
|
55
59
|
}
|
|
56
60
|
};
|
|
57
61
|
|
|
58
|
-
const notInstalled = (agent: AgentKind): never => {
|
|
59
|
-
const cli = AGENTS[agent];
|
|
60
|
-
logger.error(
|
|
61
|
-
`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
|
|
62
|
-
);
|
|
63
|
-
return process.exit(1);
|
|
64
|
-
};
|
|
65
|
-
|
|
66
62
|
/** The fraction of run (non-skipped) questions that passed. */
|
|
67
63
|
export const passFraction = (result: EvalResult): number => {
|
|
68
64
|
const ran = result.results.length - result.counts.skip;
|
|
@@ -125,10 +121,7 @@ const runFixHandoff = async (
|
|
|
125
121
|
process.stderr.write(
|
|
126
122
|
` Handing ${count} failed question${count === 1 ? "" : "s"} to ${cli.name}…\n\n`
|
|
127
123
|
);
|
|
128
|
-
const code = await launchAgentCode(
|
|
129
|
-
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
130
|
-
notInstalled(agent);
|
|
131
|
-
}
|
|
124
|
+
const code = await launchAgentCode(agent, evalFixPrompt(report));
|
|
132
125
|
if (code !== 0) {
|
|
133
126
|
process.exit(code);
|
|
134
127
|
}
|
|
@@ -143,10 +136,7 @@ const runInit = async (agent: AgentKind, file: string): Promise<void> => {
|
|
|
143
136
|
);
|
|
144
137
|
process.exit(1);
|
|
145
138
|
}
|
|
146
|
-
const code = await launchAgentCode(
|
|
147
|
-
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
148
|
-
notInstalled(agent);
|
|
149
|
-
}
|
|
139
|
+
const code = await launchAgentCode(agent, initPrompt(file));
|
|
150
140
|
if (code !== 0) {
|
|
151
141
|
process.exit(code);
|
|
152
142
|
}
|