blume 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +1 -1
- package/dist/cli/index.js +1286 -63
- package/dist/cli/index.js.map +32 -21
- package/dist/types/core/config-input.d.ts +18 -0
- package/dist/types/core/config.d.ts +4 -0
- package/dist/types/core/data.d.ts +1 -0
- package/dist/types/core/schema.d.ts +132 -17
- package/dist/types/core/types.d.ts +5 -3
- package/dist/types/openapi/references.d.ts +6 -0
- package/docs/advanced/api-reference.mdx +27 -0
- package/docs/advanced/changelog.mdx +10 -0
- package/docs/configuration/ai.mdx +38 -2
- package/docs/configuration/customization.mdx +27 -0
- package/docs/configuration/index.mdx +5 -0
- package/docs/content/navigation.mdx +12 -0
- package/docs/reference/cli.mdx +17 -13
- package/docs/reference/eval.mdx +106 -0
- package/docs/reference/meta.ts +1 -1
- package/package.json +1 -1
- package/src/ai/agent-readability.ts +19 -1
- package/src/ai/llms.ts +9 -4
- package/src/ai/mcp/server.ts +19 -8
- package/src/ai/mcp/stdio.ts +35 -0
- package/src/astro/generate.ts +25 -2
- package/src/astro/templates.ts +114 -22
- package/src/cli/commands/eval.ts +291 -0
- package/src/cli/commands/init.ts +9 -4
- package/src/cli/commands/mcp-stdio.ts +36 -0
- package/src/cli/index.ts +4 -0
- package/src/cli/required-secrets.ts +1 -1
- package/src/components/content/AccordionItem.astro +2 -2
- package/src/components/content/TreeFolder.astro +1 -2
- package/src/components/islands/AskAI.astro +9 -2
- package/src/components/islands/ask-ai.tsx +4 -2
- package/src/components/islands/hooks.ts +10 -4
- package/src/components/layout/NavTree.astro +37 -19
- package/src/components/layout/ReferenceLayout.astro +4 -0
- package/src/components/layout/RootLayout.astro +1 -1
- package/src/components/openapi/SchemaProperty.astro +3 -3
- package/src/core/config-input.ts +18 -0
- package/src/core/config.ts +4 -0
- package/src/core/data.ts +1 -0
- package/src/core/graph.ts +1 -0
- package/src/core/navigation.ts +9 -2
- package/src/core/schema.ts +51 -4
- package/src/core/server-features.ts +1 -1
- package/src/core/types.ts +5 -3
- package/src/eval/agents.ts +340 -0
- package/src/eval/findings.ts +103 -0
- package/src/eval/prompts.ts +78 -0
- package/src/eval/report.ts +214 -0
- package/src/eval/run.ts +290 -0
- package/src/eval/schema.ts +124 -0
- package/src/openapi/references.ts +23 -2
- package/src/openapi/render-mdx.ts +27 -4
- package/src/openapi/scalar.ts +1 -0
- package/src/openapi/source.ts +11 -4
- package/src/registry/eject.ts +23 -1
- package/src/search/build.ts +4 -3
package/src/astro/templates.ts
CHANGED
|
@@ -87,7 +87,14 @@ const WRANGLER_CONFIG_FILES = [
|
|
|
87
87
|
];
|
|
88
88
|
|
|
89
89
|
const resolveCloudflareAdapterArgs = (context: ProjectContext): string => {
|
|
90
|
-
|
|
90
|
+
// Every Blume HTML route prerenders (the only server routes are API
|
|
91
|
+
// endpoints), so images are optimized at build time with sharp. The
|
|
92
|
+
// adapter's default (`cloudflare-binding`) would instead declare a runtime
|
|
93
|
+
// `IMAGES` binding in the generated wrangler config that nothing uses.
|
|
94
|
+
const args: string[] = [
|
|
95
|
+
'prerenderEnvironment: "node"',
|
|
96
|
+
'imageService: "compile"',
|
|
97
|
+
];
|
|
91
98
|
const wranglerPath = WRANGLER_CONFIG_FILES.map((file) =>
|
|
92
99
|
join(context.root, file)
|
|
93
100
|
).find((file) => existsSync(file));
|
|
@@ -104,6 +111,37 @@ const resolveCloudflareAdapterArgs = (context: ProjectContext): string => {
|
|
|
104
111
|
return `{ ${args.join(", ")} }`;
|
|
105
112
|
};
|
|
106
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Without a configured driver, `@astrojs/cloudflare` force-enables KV-backed
|
|
116
|
+
* sessions and declares a `SESSION` kv_namespaces entry in the generated
|
|
117
|
+
* wrangler config — which `wrangler deploy` then requires a real KV namespace
|
|
118
|
+
* for, even though Blume never reads `Astro.session`. An explicit in-memory
|
|
119
|
+
* driver keeps the binding out. Swap for Astro's session opt-out once
|
|
120
|
+
* withastro/astro#16871 ships in the supported range.
|
|
121
|
+
*/
|
|
122
|
+
const resolveSessionOption = (deployment: {
|
|
123
|
+
adapter: string | null;
|
|
124
|
+
output: string;
|
|
125
|
+
}): string =>
|
|
126
|
+
deployment.output === "server" && deployment.adapter === "cloudflare"
|
|
127
|
+
? "\n session: { driver: sessionDrivers.memory() },"
|
|
128
|
+
: "";
|
|
129
|
+
|
|
130
|
+
/** The named imports the generated config pulls from `astro/config`. */
|
|
131
|
+
const astroConfigImportLine = (options: {
|
|
132
|
+
hasFonts: boolean;
|
|
133
|
+
hasSession: boolean;
|
|
134
|
+
}): string => {
|
|
135
|
+
const names = ["defineConfig"];
|
|
136
|
+
if (options.hasFonts) {
|
|
137
|
+
names.push("fontProviders");
|
|
138
|
+
}
|
|
139
|
+
if (options.hasSession) {
|
|
140
|
+
names.push("sessionDrivers");
|
|
141
|
+
}
|
|
142
|
+
return `import { ${names.join(", ")} } from "astro/config";`;
|
|
143
|
+
};
|
|
144
|
+
|
|
107
145
|
/**
|
|
108
146
|
* Integration packages the generated runtime imports. Declaring them in
|
|
109
147
|
* `.blume/package.json` lets Astro's framework-package crawl discover and bundle
|
|
@@ -138,7 +176,7 @@ export const runtimeDependencies = (options: {
|
|
|
138
176
|
// (and the user installs) exactly the backend it uses — nothing more.
|
|
139
177
|
deps.push(...searchProviderMeta(config.search.provider).runtimeDeps);
|
|
140
178
|
// Ask AI's provider SDK, when its backend needs one (gateway uses core `ai`).
|
|
141
|
-
if (config.ai.ask?.enabled) {
|
|
179
|
+
if (config.ai.ask?.enabled && !config.ai.ask.endpoint) {
|
|
142
180
|
const askDep = askBackendRuntimeDep(config.ai.ask);
|
|
143
181
|
if (askDep) {
|
|
144
182
|
deps.push(askDep);
|
|
@@ -290,6 +328,36 @@ const devWatchOption = (
|
|
|
290
328
|
},`
|
|
291
329
|
: "";
|
|
292
330
|
|
|
331
|
+
interface IntegrationBridgeOptions {
|
|
332
|
+
/** Config path relative to the generated Astro config. */
|
|
333
|
+
configFile: string;
|
|
334
|
+
/** SHA-256 used to invalidate Astro's generated config. */
|
|
335
|
+
sourceHash?: string;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
const renderIntegrationBridge = (
|
|
339
|
+
bridge: IntegrationBridgeOptions | undefined
|
|
340
|
+
) => {
|
|
341
|
+
if (!bridge) {
|
|
342
|
+
return {
|
|
343
|
+
configSourceMarker: "",
|
|
344
|
+
userConfigImports: "",
|
|
345
|
+
userConfigSetup: "",
|
|
346
|
+
userIntegrationSpread: "",
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
return {
|
|
350
|
+
configSourceMarker: bridge.sourceHash
|
|
351
|
+
? `// Blume config source SHA-256: ${bridge.sourceHash}\n`
|
|
352
|
+
: "",
|
|
353
|
+
userConfigImports: `import { dirname, resolve } from "node:path";\nimport { fileURLToPath } from "node:url";\nimport { createModuleLoader } from "blume/core/load-module.ts";\n`,
|
|
354
|
+
userConfigSetup: `const loadBlumeConfig = createModuleLoader();\nconst blumeConfig = await loadBlumeConfig(resolve(dirname(fileURLToPath(import.meta.url)), ${JSON.stringify(
|
|
355
|
+
bridge.configFile
|
|
356
|
+
)}));\n\n`,
|
|
357
|
+
userIntegrationSpread: ", ...(blumeConfig?.integrations ?? [])",
|
|
358
|
+
};
|
|
359
|
+
};
|
|
360
|
+
|
|
293
361
|
export const astroConfigTemplate = (options: {
|
|
294
362
|
context: ProjectContext;
|
|
295
363
|
config: ResolvedConfig;
|
|
@@ -322,6 +390,8 @@ export const astroConfigTemplate = (options: {
|
|
|
322
390
|
* See {@link devWatchOption} for why this must stay scoped.
|
|
323
391
|
*/
|
|
324
392
|
contentWatchesRuntimeDir?: boolean;
|
|
393
|
+
/** Bridge used to load configured integrations without serializing them. */
|
|
394
|
+
integrationBridge?: IntegrationBridgeOptions;
|
|
325
395
|
}): string => {
|
|
326
396
|
const { context, config, needsReact, pages, dataPath, themePath } = options;
|
|
327
397
|
const {
|
|
@@ -367,6 +437,8 @@ export const astroConfigTemplate = (options: {
|
|
|
367
437
|
const adapterOption =
|
|
368
438
|
server && deployment.adapter ? `\n adapter: ${adapterExpr},` : "";
|
|
369
439
|
|
|
440
|
+
const sessionOption = resolveSessionOption(deployment);
|
|
441
|
+
|
|
370
442
|
const siteOption = deployment.site
|
|
371
443
|
? `\n site: ${JSON.stringify(deployment.site)},`
|
|
372
444
|
: "";
|
|
@@ -424,9 +496,10 @@ export const astroConfigTemplate = (options: {
|
|
|
424
496
|
)
|
|
425
497
|
.join(", ")}],`
|
|
426
498
|
: "";
|
|
427
|
-
const defineConfigImport =
|
|
428
|
-
|
|
429
|
-
:
|
|
499
|
+
const defineConfigImport = astroConfigImportLine({
|
|
500
|
+
hasFonts: fontEntries.length > 0,
|
|
501
|
+
hasSession: sessionOption.length > 0,
|
|
502
|
+
});
|
|
430
503
|
|
|
431
504
|
// Framework renderers are only wired in when an island (or Ask AI, for React)
|
|
432
505
|
// needs them. The core theme is Astro-first and ships no client JS.
|
|
@@ -486,20 +559,26 @@ export const astroConfigTemplate = (options: {
|
|
|
486
559
|
context.outDir,
|
|
487
560
|
options.contentWatchesRuntimeDir
|
|
488
561
|
);
|
|
562
|
+
const {
|
|
563
|
+
configSourceMarker,
|
|
564
|
+
userConfigImports,
|
|
565
|
+
userConfigSetup,
|
|
566
|
+
userIntegrationSpread,
|
|
567
|
+
} = renderIntegrationBridge(options.integrationBridge);
|
|
489
568
|
|
|
490
569
|
return `// Generated by Blume. Do not edit; this file is recreated on each run.
|
|
491
|
-
${defineConfigImport}
|
|
570
|
+
${configSourceMarker}${userConfigImports}${defineConfigImport}
|
|
492
571
|
import mdx from "@astrojs/mdx";
|
|
493
572
|
import tailwindcss from "@tailwindcss/vite";
|
|
494
573
|
import { blumeMarkdownProcessor, blumeMdxProcessor, blumeShikiTransformers, blumeTwoslashTransformer } from "blume/markdown";
|
|
495
574
|
${reactImport}${vueImport}${svelteImport}${blumeImport}${adapterImport}
|
|
496
|
-
export default defineConfig({
|
|
575
|
+
${userConfigSetup}export default defineConfig({
|
|
497
576
|
root: ${JSON.stringify(context.outDir)},
|
|
498
577
|
srcDir: ${JSON.stringify(`${context.outDir}/src`)},
|
|
499
578
|
outDir: ${JSON.stringify(astroOutDir(context))},
|
|
500
579
|
publicDir: ${JSON.stringify(`${context.root}/public`)},
|
|
501
|
-
output: ${JSON.stringify(deployment.output)},${adapterOption}${siteOption}${baseOption}${redirectsOption}${i18nOption}${fontsOption}
|
|
502
|
-
integrations: [${integrations.join(", ")}],
|
|
580
|
+
output: ${JSON.stringify(deployment.output)},${adapterOption}${sessionOption}${siteOption}${baseOption}${redirectsOption}${i18nOption}${fontsOption}
|
|
581
|
+
integrations: [${integrations.join(", ")}${userIntegrationSpread}],
|
|
503
582
|
markdown: {
|
|
504
583
|
processor: blumeMarkdownProcessor(${JSON.stringify({
|
|
505
584
|
basePath: config.basePath,
|
|
@@ -521,18 +600,25 @@ export default defineConfig({
|
|
|
521
600
|
devToolbar: { enabled: false },
|
|
522
601
|
vite: {
|
|
523
602
|
plugins: [tailwindcss(), prerenderDepsPlugin(), serverAppResolvePlugin()],
|
|
524
|
-
//
|
|
525
|
-
// CJS (\`dayjs/dayjs.min.js\`)
|
|
526
|
-
//
|
|
527
|
-
//
|
|
528
|
-
//
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
//
|
|
532
|
-
//
|
|
533
|
-
//
|
|
534
|
-
//
|
|
535
|
-
|
|
603
|
+
// The lazy client-side imports both land on CJS/UMD files: mermaid (for
|
|
604
|
+
// diagrams) statically imports dayjs as CJS (\`dayjs/dayjs.min.js\`), and
|
|
605
|
+
// epub-gen-memory's browser bundle is a browserified UMD. In dev, an
|
|
606
|
+
// un-pre-bundled dependency is served as raw ESM, where such a file
|
|
607
|
+
// exposes no \`default\` export — mermaid throws on load and diagrams
|
|
608
|
+
// render blank, and the EPUB export throws \`epub is not a function\`
|
|
609
|
+
// (the UMD finds no \`exports\`/\`define\` and strands its callable on
|
|
610
|
+
// \`window.epubGen\` instead). Forcing them through the dep optimizer
|
|
611
|
+
// restores the CJS interop. In a standalone install these dynamic imports
|
|
612
|
+
// live inside \`node_modules/blume\`, which Vite's optimizer scan doesn't
|
|
613
|
+
// crawl, so neither is discovered on its own — hence the explicit
|
|
614
|
+
// includes. They resolve through the \`blume\` package (they aren't direct
|
|
615
|
+
// deps of the generated project), so the nested \`blume > x\` form is
|
|
616
|
+
// required, and epub-gen-memory must name the \`/bundle\` subpath that is
|
|
617
|
+
// actually imported: optimizing the package root leaves that entry out.
|
|
618
|
+
// Production (Rollup) already handles the interop, so this only affects dev.
|
|
619
|
+
optimizeDeps: {
|
|
620
|
+
include: ["blume > mermaid", "blume > epub-gen-memory/bundle"],
|
|
621
|
+
},
|
|
536
622
|
// Blume's render-time deps are forced external on both build environments so
|
|
537
623
|
// native bindings resolve at runtime and isolated linkers don't bundle
|
|
538
624
|
// symlinked store copies (which would surface their children as unresolvable
|
|
@@ -850,7 +936,11 @@ import data from "blume:data";
|
|
|
850
936
|
const { strings } = Astro.props;
|
|
851
937
|
---
|
|
852
938
|
|
|
853
|
-
<AskAI
|
|
939
|
+
<AskAI
|
|
940
|
+
endpoint={data.config.ask?.endpoint ?? undefined}
|
|
941
|
+
strings={strings ?? data.ui.ask}
|
|
942
|
+
suggestions={data.config.ask?.suggestions ?? []}
|
|
943
|
+
/>
|
|
854
944
|
`
|
|
855
945
|
: `---
|
|
856
946
|
// Generated by Blume. Do not edit.
|
|
@@ -1195,6 +1285,7 @@ export async function GET({ props }: { props: { title: string } }) {
|
|
|
1195
1285
|
export const scalarReferenceTemplate = (options: {
|
|
1196
1286
|
configuration: Record<string, unknown>;
|
|
1197
1287
|
dataImport: string;
|
|
1288
|
+
noindex?: boolean;
|
|
1198
1289
|
route: string;
|
|
1199
1290
|
title: string;
|
|
1200
1291
|
}): string =>
|
|
@@ -1229,6 +1320,7 @@ const htmlLang = i18n ? i18n.defaultLocale : "en";
|
|
|
1229
1320
|
favicon={data.config.favicon}
|
|
1230
1321
|
appleIcon={data.config.appleIcon}
|
|
1231
1322
|
navigation={data.navigation}
|
|
1323
|
+
noindex={${options.noindex === true}}
|
|
1232
1324
|
pageTitle={${JSON.stringify(options.title)}}
|
|
1233
1325
|
route={${JSON.stringify(options.route)}}
|
|
1234
1326
|
searchEnabled={data.config.search.enabled}
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
import { defineCommand } from "citty";
|
|
4
|
+
import { join } from "pathe";
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
AGENTS,
|
|
8
|
+
launchAgent,
|
|
9
|
+
WINDOWS_COMMAND_NOT_FOUND,
|
|
10
|
+
} from "../../audit/agent.ts";
|
|
11
|
+
import type { AgentKind } from "../../audit/agent.ts";
|
|
12
|
+
import { BlumeError } from "../../core/diagnostics.ts";
|
|
13
|
+
import { scanProject } from "../../core/project-graph.ts";
|
|
14
|
+
import { evalFixPrompt, initPrompt } from "../../eval/prompts.ts";
|
|
15
|
+
import {
|
|
16
|
+
evalReportJson,
|
|
17
|
+
fixLines,
|
|
18
|
+
headerLine,
|
|
19
|
+
questionDetails,
|
|
20
|
+
questionLine,
|
|
21
|
+
startLine,
|
|
22
|
+
summaryLine,
|
|
23
|
+
warningLines,
|
|
24
|
+
writeEvalReport,
|
|
25
|
+
} from "../../eval/report.ts";
|
|
26
|
+
import { runEval } from "../../eval/run.ts";
|
|
27
|
+
import type { EvalResult } from "../../eval/run.ts";
|
|
28
|
+
import { EvalsFileError, loadEvalsFile } from "../../eval/schema.ts";
|
|
29
|
+
import { reportInternalError } from "../internal-error.ts";
|
|
30
|
+
import { flushStdout, logger } from "../log.ts";
|
|
31
|
+
|
|
32
|
+
const DEFAULT_FILE = "evals.yaml";
|
|
33
|
+
|
|
34
|
+
/** Reader wall-clock ceiling per question, in seconds. */
|
|
35
|
+
const DEFAULT_TIMEOUT_S = 180;
|
|
36
|
+
|
|
37
|
+
const isAgentKind = (value: string): value is AgentKind => value in AGENTS;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Launch the interactive agent CLI, translating a missing executable into the
|
|
41
|
+
* Windows not-found sentinel. Only `ENOENT` means "not installed" — any other
|
|
42
|
+
* spawn failure (`EACCES`, `EMFILE`, …) must surface as itself.
|
|
43
|
+
*/
|
|
44
|
+
const launchAgentCode = async (
|
|
45
|
+
bin: string,
|
|
46
|
+
prompt: string
|
|
47
|
+
): Promise<number> => {
|
|
48
|
+
try {
|
|
49
|
+
return await launchAgent(bin, prompt);
|
|
50
|
+
} catch (error) {
|
|
51
|
+
if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
|
52
|
+
throw error;
|
|
53
|
+
}
|
|
54
|
+
return WINDOWS_COMMAND_NOT_FOUND;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const notInstalled = (agent: AgentKind): never => {
|
|
59
|
+
const cli = AGENTS[agent];
|
|
60
|
+
logger.error(
|
|
61
|
+
`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
|
|
62
|
+
);
|
|
63
|
+
return process.exit(1);
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** The fraction of run (non-skipped) questions that passed. */
|
|
67
|
+
export const passFraction = (result: EvalResult): number => {
|
|
68
|
+
const ran = result.results.length - result.counts.skip;
|
|
69
|
+
return ran === 0 ? 1 : result.counts.pass / ran;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
interface EvalFlags {
|
|
73
|
+
action?: string;
|
|
74
|
+
agent: string;
|
|
75
|
+
file: string;
|
|
76
|
+
fix?: boolean;
|
|
77
|
+
json?: boolean;
|
|
78
|
+
threshold?: string;
|
|
79
|
+
timeout?: string;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Validate the flag surface, exiting with a message on the first offense. */
|
|
83
|
+
const parseFlags = (
|
|
84
|
+
args: EvalFlags
|
|
85
|
+
): { agent: AgentKind; threshold: number; timeoutS: number } => {
|
|
86
|
+
if (!isAgentKind(args.agent)) {
|
|
87
|
+
logger.error(`Invalid --agent "${args.agent}" (use claude | codex).`);
|
|
88
|
+
process.exit(1);
|
|
89
|
+
}
|
|
90
|
+
if (args.action !== undefined && args.action !== "init") {
|
|
91
|
+
logger.error(`Unknown action "${args.action}" (did you mean "init"?).`);
|
|
92
|
+
process.exit(1);
|
|
93
|
+
}
|
|
94
|
+
if (args.json && args.fix) {
|
|
95
|
+
logger.error("--json and --fix are mutually exclusive.");
|
|
96
|
+
process.exit(1);
|
|
97
|
+
}
|
|
98
|
+
const threshold = args.threshold === undefined ? 1 : Number(args.threshold);
|
|
99
|
+
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
|
|
100
|
+
logger.error(`Invalid --threshold "${args.threshold}" (use 0..1).`);
|
|
101
|
+
process.exit(1);
|
|
102
|
+
}
|
|
103
|
+
const timeoutS =
|
|
104
|
+
args.timeout === undefined ? DEFAULT_TIMEOUT_S : Number(args.timeout);
|
|
105
|
+
if (!Number.isInteger(timeoutS) || timeoutS <= 0) {
|
|
106
|
+
logger.error(`Invalid --timeout "${args.timeout}" (whole seconds).`);
|
|
107
|
+
process.exit(1);
|
|
108
|
+
}
|
|
109
|
+
return { agent: args.agent, threshold, timeoutS };
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/** `blume eval --fix`: hand the failing report to the interactive agent. */
|
|
113
|
+
const runFixHandoff = async (
|
|
114
|
+
agent: AgentKind,
|
|
115
|
+
result: EvalResult,
|
|
116
|
+
root: string,
|
|
117
|
+
threshold: number
|
|
118
|
+
): Promise<void> => {
|
|
119
|
+
const count = result.counts.fail + result.counts.error;
|
|
120
|
+
if (count === 0) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const cli = AGENTS[agent];
|
|
124
|
+
const report = await writeEvalReport(result, root, threshold);
|
|
125
|
+
process.stderr.write(
|
|
126
|
+
` Handing ${count} failed question${count === 1 ? "" : "s"} to ${cli.name}…\n\n`
|
|
127
|
+
);
|
|
128
|
+
const code = await launchAgentCode(cli.bin, evalFixPrompt(report));
|
|
129
|
+
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
130
|
+
notInstalled(agent);
|
|
131
|
+
}
|
|
132
|
+
if (code !== 0) {
|
|
133
|
+
process.exit(code);
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
/** `blume eval init`: draft a starter evals file via the interactive agent. */
|
|
138
|
+
const runInit = async (agent: AgentKind, file: string): Promise<void> => {
|
|
139
|
+
const path = join(process.cwd(), file);
|
|
140
|
+
if (existsSync(path)) {
|
|
141
|
+
logger.error(
|
|
142
|
+
`${file} already exists — edit it directly, or pass --file to draft elsewhere.`
|
|
143
|
+
);
|
|
144
|
+
process.exit(1);
|
|
145
|
+
}
|
|
146
|
+
const code = await launchAgentCode(AGENTS[agent].bin, initPrompt(file));
|
|
147
|
+
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
148
|
+
notInstalled(agent);
|
|
149
|
+
}
|
|
150
|
+
if (code !== 0) {
|
|
151
|
+
process.exit(code);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
export const evalCommand = defineCommand({
|
|
156
|
+
args: {
|
|
157
|
+
action: {
|
|
158
|
+
description: 'Optional action: "init" drafts a starter evals file.',
|
|
159
|
+
required: false,
|
|
160
|
+
type: "positional",
|
|
161
|
+
},
|
|
162
|
+
agent: {
|
|
163
|
+
default: "claude",
|
|
164
|
+
description: "Agent CLI that reads and grades the docs: claude | codex.",
|
|
165
|
+
type: "string",
|
|
166
|
+
},
|
|
167
|
+
file: {
|
|
168
|
+
default: DEFAULT_FILE,
|
|
169
|
+
description: "The evals file to run.",
|
|
170
|
+
type: "string",
|
|
171
|
+
},
|
|
172
|
+
fix: {
|
|
173
|
+
description:
|
|
174
|
+
"After a failing run, hand the report to the agent to fix the docs interactively.",
|
|
175
|
+
type: "boolean",
|
|
176
|
+
},
|
|
177
|
+
json: {
|
|
178
|
+
description: "Emit the report as JSON on stdout (for CI/editors).",
|
|
179
|
+
type: "boolean",
|
|
180
|
+
},
|
|
181
|
+
threshold: {
|
|
182
|
+
description:
|
|
183
|
+
"Minimum passing fraction (0..1) before the run exits non-zero. Defaults to 1.",
|
|
184
|
+
type: "string",
|
|
185
|
+
},
|
|
186
|
+
timeout: {
|
|
187
|
+
description: `Reader time limit per question, in seconds. Defaults to ${DEFAULT_TIMEOUT_S}.`,
|
|
188
|
+
type: "string",
|
|
189
|
+
},
|
|
190
|
+
verbose: {
|
|
191
|
+
description: "Include the reader's full answer under each failure.",
|
|
192
|
+
type: "boolean",
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
meta: {
|
|
196
|
+
description:
|
|
197
|
+
"Test the docs: an agent answers your questions using only the documentation.",
|
|
198
|
+
name: "eval",
|
|
199
|
+
},
|
|
200
|
+
async run({ args }) {
|
|
201
|
+
const root = process.cwd();
|
|
202
|
+
const { agent, threshold, timeoutS } = parseFlags(args);
|
|
203
|
+
if (args.action === "init") {
|
|
204
|
+
await runInit(agent, args.file);
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
let result: EvalResult;
|
|
209
|
+
try {
|
|
210
|
+
// `scanProject`, not `prepareProject`: the eval reads the content tree
|
|
211
|
+
// and never regenerates the runtime, so it doesn't contend with a
|
|
212
|
+
// running dev server. Same reasoning as `blume audit`.
|
|
213
|
+
const project = await scanProject(root, { mode: "build" });
|
|
214
|
+
const evalsPath = join(root, args.file);
|
|
215
|
+
const { evals, raw } = await loadEvalsFile(evalsPath);
|
|
216
|
+
|
|
217
|
+
process.stderr.write(`${headerLine(evals.questions.length, agent)}\n\n`);
|
|
218
|
+
result = await runEval({
|
|
219
|
+
agent,
|
|
220
|
+
evals,
|
|
221
|
+
evalsPath,
|
|
222
|
+
onProgress: (event) => {
|
|
223
|
+
// Straight to stderr, not `logger.info` — consola drops info-level
|
|
224
|
+
// lines in test and CI environments.
|
|
225
|
+
if (event.kind === "question-start") {
|
|
226
|
+
process.stderr.write(
|
|
227
|
+
`${startLine(event.id, event.index, event.total)}\n`
|
|
228
|
+
);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
const lines = [
|
|
232
|
+
questionLine(event.result),
|
|
233
|
+
...questionDetails(event.result, Boolean(args.verbose)),
|
|
234
|
+
];
|
|
235
|
+
process.stderr.write(`${lines.join("\n")}\n`);
|
|
236
|
+
},
|
|
237
|
+
project,
|
|
238
|
+
rawEvals: raw,
|
|
239
|
+
readerTimeoutMs: timeoutS * 1000,
|
|
240
|
+
});
|
|
241
|
+
} catch (error) {
|
|
242
|
+
if (error instanceof EvalsFileError) {
|
|
243
|
+
logger.error(error.message);
|
|
244
|
+
process.exit(1);
|
|
245
|
+
}
|
|
246
|
+
if (error instanceof BlumeError) {
|
|
247
|
+
logger.error(error.diagnostic.message);
|
|
248
|
+
process.exit(1);
|
|
249
|
+
}
|
|
250
|
+
if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
|
|
251
|
+
notInstalled(agent);
|
|
252
|
+
}
|
|
253
|
+
reportInternalError(error);
|
|
254
|
+
process.exit(1);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const tail = [
|
|
258
|
+
"",
|
|
259
|
+
...warningLines(result, root),
|
|
260
|
+
...fixLines(result, root),
|
|
261
|
+
"",
|
|
262
|
+
` ${summaryLine(result)}`,
|
|
263
|
+
"",
|
|
264
|
+
];
|
|
265
|
+
process.stderr.write(tail.join("\n"));
|
|
266
|
+
|
|
267
|
+
const failed = passFraction(result) < threshold;
|
|
268
|
+
|
|
269
|
+
if (args.fix) {
|
|
270
|
+
// The gate is a CI concern; a handoff run succeeds when the agent
|
|
271
|
+
// session does, not when the docs already passed.
|
|
272
|
+
await runFixHandoff(agent, result, root, threshold);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (args.json) {
|
|
277
|
+
process.stdout.write(evalReportJson(result, root, threshold));
|
|
278
|
+
if (failed) {
|
|
279
|
+
// `process.exit` doesn't flush a piped stdout — without this the JSON
|
|
280
|
+
// is truncated mid-write in exactly the CI setups that consume it.
|
|
281
|
+
await flushStdout();
|
|
282
|
+
process.exit(1);
|
|
283
|
+
}
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (failed) {
|
|
288
|
+
process.exit(1);
|
|
289
|
+
}
|
|
290
|
+
},
|
|
291
|
+
});
|
package/src/cli/commands/init.ts
CHANGED
|
@@ -157,10 +157,15 @@ export const initCommand = defineCommand({
|
|
|
157
157
|
const sink = interactive ? clack.log : logger;
|
|
158
158
|
const { createdPackage } = await applyPlan(buildPlan(root, answers), sink);
|
|
159
159
|
|
|
160
|
-
// Keep Blume's generated runtime (`.blume/`) and
|
|
161
|
-
// of version control. Idempotent: creates
|
|
162
|
-
// entries already present
|
|
163
|
-
|
|
160
|
+
// Keep installed dependencies, Blume's generated runtime (`.blume/`), and
|
|
161
|
+
// build output (`dist/`) out of version control. Idempotent: creates
|
|
162
|
+
// `.gitignore` when absent and skips entries already present
|
|
163
|
+
// (trailing-slash agnostic).
|
|
164
|
+
const ignored = await ensureGitignore(root, [
|
|
165
|
+
"node_modules/",
|
|
166
|
+
".blume/",
|
|
167
|
+
"dist/",
|
|
168
|
+
]);
|
|
164
169
|
if (ignored.length > 0) {
|
|
165
170
|
sink.success(`Added ${ignored.join(", ")} to .gitignore`);
|
|
166
171
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
|
|
3
|
+
import { defineCommand } from "citty";
|
|
4
|
+
|
|
5
|
+
import type { McpData } from "../../ai/mcp/data.ts";
|
|
6
|
+
import { serveMcpStdio } from "../../ai/mcp/stdio.ts";
|
|
7
|
+
|
|
8
|
+
export const mcpStdioCommand = defineCommand({
|
|
9
|
+
args: {
|
|
10
|
+
data: {
|
|
11
|
+
description: "Path to a serialized MCP data snapshot (JSON).",
|
|
12
|
+
required: true,
|
|
13
|
+
type: "string",
|
|
14
|
+
},
|
|
15
|
+
},
|
|
16
|
+
meta: {
|
|
17
|
+
description:
|
|
18
|
+
"Serve an MCP data snapshot over stdio (internal, used by `blume eval`).",
|
|
19
|
+
name: "mcp-stdio",
|
|
20
|
+
},
|
|
21
|
+
async run({ args }) {
|
|
22
|
+
// stdout belongs to the JSON-RPC transport from here on; every diagnostic
|
|
23
|
+
// must go to stderr or the MCP client chokes on the stray line.
|
|
24
|
+
let data: McpData;
|
|
25
|
+
try {
|
|
26
|
+
data = JSON.parse(await readFile(args.data, "utf-8")) as McpData;
|
|
27
|
+
} catch (error) {
|
|
28
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
29
|
+
process.stderr.write(
|
|
30
|
+
`blume mcp-stdio: cannot load the snapshot at ${args.data}: ${detail}\n`
|
|
31
|
+
);
|
|
32
|
+
process.exit(1);
|
|
33
|
+
}
|
|
34
|
+
await serveMcpStdio(data);
|
|
35
|
+
},
|
|
36
|
+
});
|
package/src/cli/index.ts
CHANGED
|
@@ -8,7 +8,9 @@ import { checkCommand } from "./commands/check.ts";
|
|
|
8
8
|
import { devCommand } from "./commands/dev.ts";
|
|
9
9
|
import { doctorCommand } from "./commands/doctor.ts";
|
|
10
10
|
import { ejectCommand } from "./commands/eject.ts";
|
|
11
|
+
import { evalCommand } from "./commands/eval.ts";
|
|
11
12
|
import { initCommand } from "./commands/init.ts";
|
|
13
|
+
import { mcpStdioCommand } from "./commands/mcp-stdio.ts";
|
|
12
14
|
import { previewCommand } from "./commands/preview.ts";
|
|
13
15
|
import { syncCommand } from "./commands/sync.ts";
|
|
14
16
|
import { validateCommand } from "./commands/validate.ts";
|
|
@@ -29,7 +31,9 @@ const main = defineCommand({
|
|
|
29
31
|
dev: devCommand,
|
|
30
32
|
doctor: doctorCommand,
|
|
31
33
|
eject: ejectCommand,
|
|
34
|
+
eval: evalCommand,
|
|
32
35
|
init: initCommand,
|
|
36
|
+
"mcp-stdio": mcpStdioCommand,
|
|
33
37
|
preview: previewCommand,
|
|
34
38
|
sync: syncCommand,
|
|
35
39
|
validate: validateCommand,
|
|
@@ -24,7 +24,7 @@ export const checkRequiredSecrets = (config: ResolvedConfig): Diagnostic[] => {
|
|
|
24
24
|
});
|
|
25
25
|
};
|
|
26
26
|
|
|
27
|
-
if (config.ai.ask?.enabled) {
|
|
27
|
+
if (config.ai.ask?.enabled && !config.ai.ask.endpoint) {
|
|
28
28
|
const backend = resolveAskBackend(config.ai.ask);
|
|
29
29
|
if (backend.kind === "gateway") {
|
|
30
30
|
requireSecret(
|
|
@@ -22,7 +22,7 @@ const accordionId = id ?? slugify(title);
|
|
|
22
22
|
---
|
|
23
23
|
|
|
24
24
|
<details
|
|
25
|
-
class="not-prose
|
|
25
|
+
class="not-prose"
|
|
26
26
|
data-blume-accordion
|
|
27
27
|
id={accordionId}
|
|
28
28
|
open={defaultOpen}
|
|
@@ -52,7 +52,7 @@ const accordionId = id ?? slugify(title);
|
|
|
52
52
|
</span>
|
|
53
53
|
</span>
|
|
54
54
|
<Icon
|
|
55
|
-
class="text-muted-foreground transition-transform
|
|
55
|
+
class="text-muted-foreground transition-transform [details[open]>summary_&]:rotate-180"
|
|
56
56
|
name="chevron-down"
|
|
57
57
|
size={16}
|
|
58
58
|
/>
|
|
@@ -7,7 +7,6 @@ const { defaultOpen = false, name, openable = true } = Astro.props;
|
|
|
7
7
|
{
|
|
8
8
|
openable ? (
|
|
9
9
|
<details
|
|
10
|
-
class="group"
|
|
11
10
|
data-blume-tree-folder
|
|
12
11
|
data-blume-tree-openable="true"
|
|
13
12
|
open={defaultOpen}
|
|
@@ -20,7 +19,7 @@ const { defaultOpen = false, name, openable = true } = Astro.props;
|
|
|
20
19
|
role="treeitem"
|
|
21
20
|
>
|
|
22
21
|
<Icon
|
|
23
|
-
class="shrink-0 transition-transform
|
|
22
|
+
class="shrink-0 transition-transform [details[open]>summary_&]:rotate-90"
|
|
24
23
|
name="chevron-right"
|
|
25
24
|
size={15}
|
|
26
25
|
/>
|
|
@@ -9,11 +9,12 @@ interface Suggestion {
|
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
interface Props {
|
|
12
|
+
endpoint?: string;
|
|
12
13
|
strings?: UIStrings["ask"];
|
|
13
14
|
suggestions?: Suggestion[];
|
|
14
15
|
}
|
|
15
16
|
|
|
16
|
-
const { strings, suggestions = [] } = Astro.props;
|
|
17
|
+
const { endpoint, strings, suggestions = [] } = Astro.props;
|
|
17
18
|
|
|
18
19
|
// Icons resolve to inline SVG here (server-only module), so the client island
|
|
19
20
|
// gets ready-to-render markup rather than a name it can't resolve. Bare Lucide
|
|
@@ -43,7 +44,13 @@ const icons = {
|
|
|
43
44
|
};
|
|
44
45
|
---
|
|
45
46
|
|
|
46
|
-
<AskAI
|
|
47
|
+
<AskAI
|
|
48
|
+
client:load
|
|
49
|
+
endpoint={endpoint}
|
|
50
|
+
icons={icons}
|
|
51
|
+
strings={strings}
|
|
52
|
+
suggestions={items}
|
|
53
|
+
/>
|
|
47
54
|
|
|
48
55
|
<style is:global>
|
|
49
56
|
:root {
|