blume 1.1.4 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +1 -1
- package/dist/cli/index.js +1319 -66
- package/dist/cli/index.js.map +34 -23
- package/dist/types/core/config-input.d.ts +18 -0
- package/dist/types/core/config.d.ts +4 -0
- package/dist/types/core/data.d.ts +1 -0
- package/dist/types/core/schema.d.ts +132 -17
- package/dist/types/core/types.d.ts +5 -3
- package/dist/types/openapi/references.d.ts +6 -0
- package/docs/advanced/api-reference.mdx +27 -0
- package/docs/advanced/changelog.mdx +10 -0
- package/docs/configuration/ai.mdx +38 -2
- package/docs/configuration/customization.mdx +27 -0
- package/docs/configuration/index.mdx +5 -0
- package/docs/configuration/search.mdx +15 -0
- package/docs/content/navigation.mdx +12 -0
- package/docs/reference/cli.mdx +17 -13
- package/docs/reference/eval.mdx +106 -0
- package/docs/reference/meta.ts +1 -1
- package/package.json +2 -2
- package/src/ai/agent-readability.ts +19 -1
- package/src/ai/ask-context.ts +7 -1
- package/src/ai/ask-data.ts +1 -0
- package/src/ai/llms.ts +9 -4
- package/src/ai/mcp/data.ts +7 -0
- package/src/ai/mcp/server.ts +24 -8
- package/src/ai/mcp/stdio.ts +38 -0
- package/src/astro/generate.ts +25 -2
- package/src/astro/templates.ts +129 -26
- package/src/cli/commands/eval.ts +291 -0
- package/src/cli/commands/init.ts +9 -4
- package/src/cli/commands/mcp-stdio.ts +36 -0
- package/src/cli/index.ts +4 -0
- package/src/cli/required-secrets.ts +1 -1
- package/src/components/content/AccordionItem.astro +2 -2
- package/src/components/content/TreeFolder.astro +1 -2
- package/src/components/islands/AskAI.astro +9 -2
- package/src/components/islands/ask-ai.tsx +4 -2
- package/src/components/islands/hooks.ts +10 -4
- package/src/components/layout/NavTree.astro +38 -20
- package/src/components/layout/ReferenceLayout.astro +4 -0
- package/src/components/layout/RootLayout.astro +2 -2
- package/src/components/layout/search/orama.ts +5 -2
- package/src/components/openapi/SchemaProperty.astro +3 -3
- package/src/core/config-input.ts +18 -0
- package/src/core/config.ts +4 -0
- package/src/core/data.ts +1 -0
- package/src/core/graph.ts +1 -0
- package/src/core/navigation.ts +9 -2
- package/src/core/schema.ts +51 -4
- package/src/core/server-features.ts +1 -1
- package/src/core/types.ts +5 -3
- package/src/eval/agents.ts +340 -0
- package/src/eval/findings.ts +103 -0
- package/src/eval/prompts.ts +78 -0
- package/src/eval/report.ts +214 -0
- package/src/eval/run.ts +290 -0
- package/src/eval/schema.ts +124 -0
- package/src/openapi/references.ts +23 -2
- package/src/openapi/render-mdx.ts +27 -4
- package/src/openapi/scalar.ts +1 -0
- package/src/openapi/source.ts +11 -4
- package/src/registry/eject.ts +23 -1
- package/src/search/build.ts +4 -3
- package/src/search/orama-index.ts +55 -5
package/src/astro/generate.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
2
3
|
import {
|
|
3
4
|
lstat,
|
|
@@ -723,6 +724,22 @@ export const detectUsesMath = async (
|
|
|
723
724
|
return [...contents, ...staged].some(containsMath);
|
|
724
725
|
};
|
|
725
726
|
|
|
727
|
+
const hashConfigSource = (source: string): string =>
|
|
728
|
+
createHash("sha256").update(source).digest("hex");
|
|
729
|
+
|
|
730
|
+
const loadIntegrationBridge = async (
|
|
731
|
+
config: ResolvedConfig,
|
|
732
|
+
context: BlumeProject["context"]
|
|
733
|
+
): Promise<Parameters<typeof astroConfigTemplate>[0]["integrationBridge"]> => {
|
|
734
|
+
if (config.integrations.length === 0 || !context.configFile) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
return {
|
|
738
|
+
configFile: relative(context.outDir, context.configFile),
|
|
739
|
+
sourceHash: hashConfigSource(await readOptional(context.configFile)),
|
|
740
|
+
};
|
|
741
|
+
};
|
|
742
|
+
|
|
726
743
|
const writeIfChanged = async (
|
|
727
744
|
path: string,
|
|
728
745
|
content: string
|
|
@@ -1085,7 +1102,10 @@ export const buildRuntimeData = (project: BlumeProject): string => {
|
|
|
1085
1102
|
analytics: config.analytics ?? null,
|
|
1086
1103
|
appleIcon: resolveAppleIcon(project),
|
|
1087
1104
|
ask: config.ai.ask?.enabled
|
|
1088
|
-
? {
|
|
1105
|
+
? {
|
|
1106
|
+
endpoint: config.ai.ask.endpoint ?? null,
|
|
1107
|
+
suggestions: config.ai.ask.suggestions,
|
|
1108
|
+
}
|
|
1089
1109
|
: null,
|
|
1090
1110
|
banner: resolveBanner(config),
|
|
1091
1111
|
basePath: config.basePath,
|
|
@@ -1279,7 +1299,7 @@ const writeAskFiles = async (
|
|
|
1279
1299
|
write: (path: string, content: string) => Promise<boolean>
|
|
1280
1300
|
): Promise<void> => {
|
|
1281
1301
|
const { ask } = project.config.ai;
|
|
1282
|
-
if (!ask?.enabled) {
|
|
1302
|
+
if (!(ask?.enabled && !ask.endpoint)) {
|
|
1283
1303
|
return;
|
|
1284
1304
|
}
|
|
1285
1305
|
const grounded = ask.provider !== "inkeep";
|
|
@@ -1418,6 +1438,7 @@ export const generateRuntime = async (
|
|
|
1418
1438
|
usesMath,
|
|
1419
1439
|
userTheme,
|
|
1420
1440
|
userExamplesCss,
|
|
1441
|
+
integrationBridge,
|
|
1421
1442
|
islandDiscovery,
|
|
1422
1443
|
exampleDiscovery,
|
|
1423
1444
|
componentSlots,
|
|
@@ -1427,6 +1448,7 @@ export const generateRuntime = async (
|
|
|
1427
1448
|
detectUsesMath(context.root, staged.values()),
|
|
1428
1449
|
readOptional(context.themeFile),
|
|
1429
1450
|
readOptional(examplesCssFile(context.root, config)),
|
|
1451
|
+
loadIntegrationBridge(config, context),
|
|
1430
1452
|
discoverIslands(context.root),
|
|
1431
1453
|
discoverExamples(context.root, config.examples.source),
|
|
1432
1454
|
buildComponentSlots(context.componentsFile),
|
|
@@ -1500,6 +1522,7 @@ export const generateRuntime = async (
|
|
|
1500
1522
|
dataPath,
|
|
1501
1523
|
examplesPath,
|
|
1502
1524
|
examplesThemePath,
|
|
1525
|
+
integrationBridge,
|
|
1503
1526
|
needsReact,
|
|
1504
1527
|
needsSvelte,
|
|
1505
1528
|
needsVue,
|
package/src/astro/templates.ts
CHANGED
|
@@ -87,7 +87,14 @@ const WRANGLER_CONFIG_FILES = [
|
|
|
87
87
|
];
|
|
88
88
|
|
|
89
89
|
const resolveCloudflareAdapterArgs = (context: ProjectContext): string => {
|
|
90
|
-
|
|
90
|
+
// Every Blume HTML route prerenders (the only server routes are API
|
|
91
|
+
// endpoints), so images are optimized at build time with sharp. The
|
|
92
|
+
// adapter's default (`cloudflare-binding`) would instead declare a runtime
|
|
93
|
+
// `IMAGES` binding in the generated wrangler config that nothing uses.
|
|
94
|
+
const args: string[] = [
|
|
95
|
+
'prerenderEnvironment: "node"',
|
|
96
|
+
'imageService: "compile"',
|
|
97
|
+
];
|
|
91
98
|
const wranglerPath = WRANGLER_CONFIG_FILES.map((file) =>
|
|
92
99
|
join(context.root, file)
|
|
93
100
|
).find((file) => existsSync(file));
|
|
@@ -104,6 +111,37 @@ const resolveCloudflareAdapterArgs = (context: ProjectContext): string => {
|
|
|
104
111
|
return `{ ${args.join(", ")} }`;
|
|
105
112
|
};
|
|
106
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Without a configured driver, `@astrojs/cloudflare` force-enables KV-backed
|
|
116
|
+
* sessions and declares a `SESSION` kv_namespaces entry in the generated
|
|
117
|
+
* wrangler config — which `wrangler deploy` then requires a real KV namespace
|
|
118
|
+
* for, even though Blume never reads `Astro.session`. An explicit in-memory
|
|
119
|
+
* driver keeps the binding out. Swap for Astro's session opt-out once
|
|
120
|
+
* withastro/astro#16871 ships in the supported range.
|
|
121
|
+
*/
|
|
122
|
+
const resolveSessionOption = (deployment: {
|
|
123
|
+
adapter: string | null;
|
|
124
|
+
output: string;
|
|
125
|
+
}): string =>
|
|
126
|
+
deployment.output === "server" && deployment.adapter === "cloudflare"
|
|
127
|
+
? "\n session: { driver: sessionDrivers.memory() },"
|
|
128
|
+
: "";
|
|
129
|
+
|
|
130
|
+
/** The named imports the generated config pulls from `astro/config`. */
|
|
131
|
+
const astroConfigImportLine = (options: {
|
|
132
|
+
hasFonts: boolean;
|
|
133
|
+
hasSession: boolean;
|
|
134
|
+
}): string => {
|
|
135
|
+
const names = ["defineConfig"];
|
|
136
|
+
if (options.hasFonts) {
|
|
137
|
+
names.push("fontProviders");
|
|
138
|
+
}
|
|
139
|
+
if (options.hasSession) {
|
|
140
|
+
names.push("sessionDrivers");
|
|
141
|
+
}
|
|
142
|
+
return `import { ${names.join(", ")} } from "astro/config";`;
|
|
143
|
+
};
|
|
144
|
+
|
|
107
145
|
/**
|
|
108
146
|
* Integration packages the generated runtime imports. Declaring them in
|
|
109
147
|
* `.blume/package.json` lets Astro's framework-package crawl discover and bundle
|
|
@@ -138,7 +176,7 @@ export const runtimeDependencies = (options: {
|
|
|
138
176
|
// (and the user installs) exactly the backend it uses — nothing more.
|
|
139
177
|
deps.push(...searchProviderMeta(config.search.provider).runtimeDeps);
|
|
140
178
|
// Ask AI's provider SDK, when its backend needs one (gateway uses core `ai`).
|
|
141
|
-
if (config.ai.ask?.enabled) {
|
|
179
|
+
if (config.ai.ask?.enabled && !config.ai.ask.endpoint) {
|
|
142
180
|
const askDep = askBackendRuntimeDep(config.ai.ask);
|
|
143
181
|
if (askDep) {
|
|
144
182
|
deps.push(askDep);
|
|
@@ -290,6 +328,36 @@ const devWatchOption = (
|
|
|
290
328
|
},`
|
|
291
329
|
: "";
|
|
292
330
|
|
|
331
|
+
interface IntegrationBridgeOptions {
|
|
332
|
+
/** Config path relative to the generated Astro config. */
|
|
333
|
+
configFile: string;
|
|
334
|
+
/** SHA-256 used to invalidate Astro's generated config. */
|
|
335
|
+
sourceHash?: string;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
const renderIntegrationBridge = (
|
|
339
|
+
bridge: IntegrationBridgeOptions | undefined
|
|
340
|
+
) => {
|
|
341
|
+
if (!bridge) {
|
|
342
|
+
return {
|
|
343
|
+
configSourceMarker: "",
|
|
344
|
+
userConfigImports: "",
|
|
345
|
+
userConfigSetup: "",
|
|
346
|
+
userIntegrationSpread: "",
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
return {
|
|
350
|
+
configSourceMarker: bridge.sourceHash
|
|
351
|
+
? `// Blume config source SHA-256: ${bridge.sourceHash}\n`
|
|
352
|
+
: "",
|
|
353
|
+
userConfigImports: `import { dirname, resolve } from "node:path";\nimport { fileURLToPath } from "node:url";\nimport { createModuleLoader } from "blume/core/load-module.ts";\n`,
|
|
354
|
+
userConfigSetup: `const loadBlumeConfig = createModuleLoader();\nconst blumeConfig = await loadBlumeConfig(resolve(dirname(fileURLToPath(import.meta.url)), ${JSON.stringify(
|
|
355
|
+
bridge.configFile
|
|
356
|
+
)}));\n\n`,
|
|
357
|
+
userIntegrationSpread: ", ...(blumeConfig?.integrations ?? [])",
|
|
358
|
+
};
|
|
359
|
+
};
|
|
360
|
+
|
|
293
361
|
export const astroConfigTemplate = (options: {
|
|
294
362
|
context: ProjectContext;
|
|
295
363
|
config: ResolvedConfig;
|
|
@@ -322,6 +390,8 @@ export const astroConfigTemplate = (options: {
|
|
|
322
390
|
* See {@link devWatchOption} for why this must stay scoped.
|
|
323
391
|
*/
|
|
324
392
|
contentWatchesRuntimeDir?: boolean;
|
|
393
|
+
/** Bridge used to load configured integrations without serializing them. */
|
|
394
|
+
integrationBridge?: IntegrationBridgeOptions;
|
|
325
395
|
}): string => {
|
|
326
396
|
const { context, config, needsReact, pages, dataPath, themePath } = options;
|
|
327
397
|
const {
|
|
@@ -367,6 +437,8 @@ export const astroConfigTemplate = (options: {
|
|
|
367
437
|
const adapterOption =
|
|
368
438
|
server && deployment.adapter ? `\n adapter: ${adapterExpr},` : "";
|
|
369
439
|
|
|
440
|
+
const sessionOption = resolveSessionOption(deployment);
|
|
441
|
+
|
|
370
442
|
const siteOption = deployment.site
|
|
371
443
|
? `\n site: ${JSON.stringify(deployment.site)},`
|
|
372
444
|
: "";
|
|
@@ -424,9 +496,10 @@ export const astroConfigTemplate = (options: {
|
|
|
424
496
|
)
|
|
425
497
|
.join(", ")}],`
|
|
426
498
|
: "";
|
|
427
|
-
const defineConfigImport =
|
|
428
|
-
|
|
429
|
-
:
|
|
499
|
+
const defineConfigImport = astroConfigImportLine({
|
|
500
|
+
hasFonts: fontEntries.length > 0,
|
|
501
|
+
hasSession: sessionOption.length > 0,
|
|
502
|
+
});
|
|
430
503
|
|
|
431
504
|
// Framework renderers are only wired in when an island (or Ask AI, for React)
|
|
432
505
|
// needs them. The core theme is Astro-first and ships no client JS.
|
|
@@ -486,20 +559,26 @@ export const astroConfigTemplate = (options: {
|
|
|
486
559
|
context.outDir,
|
|
487
560
|
options.contentWatchesRuntimeDir
|
|
488
561
|
);
|
|
562
|
+
const {
|
|
563
|
+
configSourceMarker,
|
|
564
|
+
userConfigImports,
|
|
565
|
+
userConfigSetup,
|
|
566
|
+
userIntegrationSpread,
|
|
567
|
+
} = renderIntegrationBridge(options.integrationBridge);
|
|
489
568
|
|
|
490
569
|
return `// Generated by Blume. Do not edit; this file is recreated on each run.
|
|
491
|
-
${defineConfigImport}
|
|
570
|
+
${configSourceMarker}${userConfigImports}${defineConfigImport}
|
|
492
571
|
import mdx from "@astrojs/mdx";
|
|
493
572
|
import tailwindcss from "@tailwindcss/vite";
|
|
494
573
|
import { blumeMarkdownProcessor, blumeMdxProcessor, blumeShikiTransformers, blumeTwoslashTransformer } from "blume/markdown";
|
|
495
574
|
${reactImport}${vueImport}${svelteImport}${blumeImport}${adapterImport}
|
|
496
|
-
export default defineConfig({
|
|
575
|
+
${userConfigSetup}export default defineConfig({
|
|
497
576
|
root: ${JSON.stringify(context.outDir)},
|
|
498
577
|
srcDir: ${JSON.stringify(`${context.outDir}/src`)},
|
|
499
578
|
outDir: ${JSON.stringify(astroOutDir(context))},
|
|
500
579
|
publicDir: ${JSON.stringify(`${context.root}/public`)},
|
|
501
|
-
output: ${JSON.stringify(deployment.output)},${adapterOption}${siteOption}${baseOption}${redirectsOption}${i18nOption}${fontsOption}
|
|
502
|
-
integrations: [${integrations.join(", ")}],
|
|
580
|
+
output: ${JSON.stringify(deployment.output)},${adapterOption}${sessionOption}${siteOption}${baseOption}${redirectsOption}${i18nOption}${fontsOption}
|
|
581
|
+
integrations: [${integrations.join(", ")}${userIntegrationSpread}],
|
|
503
582
|
markdown: {
|
|
504
583
|
processor: blumeMarkdownProcessor(${JSON.stringify({
|
|
505
584
|
basePath: config.basePath,
|
|
@@ -521,18 +600,25 @@ export default defineConfig({
|
|
|
521
600
|
devToolbar: { enabled: false },
|
|
522
601
|
vite: {
|
|
523
602
|
plugins: [tailwindcss(), prerenderDepsPlugin(), serverAppResolvePlugin()],
|
|
524
|
-
//
|
|
525
|
-
// CJS (\`dayjs/dayjs.min.js\`)
|
|
526
|
-
//
|
|
527
|
-
//
|
|
528
|
-
//
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
//
|
|
532
|
-
//
|
|
533
|
-
//
|
|
534
|
-
//
|
|
535
|
-
|
|
603
|
+
// The lazy client-side imports both land on CJS/UMD files: mermaid (for
|
|
604
|
+
// diagrams) statically imports dayjs as CJS (\`dayjs/dayjs.min.js\`), and
|
|
605
|
+
// epub-gen-memory's browser bundle is a browserified UMD. In dev, an
|
|
606
|
+
// un-pre-bundled dependency is served as raw ESM, where such a file
|
|
607
|
+
// exposes no \`default\` export — mermaid throws on load and diagrams
|
|
608
|
+
// render blank, and the EPUB export throws \`epub is not a function\`
|
|
609
|
+
// (the UMD finds no \`exports\`/\`define\` and strands its callable on
|
|
610
|
+
// \`window.epubGen\` instead). Forcing them through the dep optimizer
|
|
611
|
+
// restores the CJS interop. In a standalone install these dynamic imports
|
|
612
|
+
// live inside \`node_modules/blume\`, which Vite's optimizer scan doesn't
|
|
613
|
+
// crawl, so neither is discovered on its own — hence the explicit
|
|
614
|
+
// includes. They resolve through the \`blume\` package (they aren't direct
|
|
615
|
+
// deps of the generated project), so the nested \`blume > x\` form is
|
|
616
|
+
// required, and epub-gen-memory must name the \`/bundle\` subpath that is
|
|
617
|
+
// actually imported: optimizing the package root leaves that entry out.
|
|
618
|
+
// Production (Rollup) already handles the interop, so this only affects dev.
|
|
619
|
+
optimizeDeps: {
|
|
620
|
+
include: ["blume > mermaid", "blume > epub-gen-memory/bundle"],
|
|
621
|
+
},
|
|
536
622
|
// Blume's render-time deps are forced external on both build environments so
|
|
537
623
|
// native bindings resolve at runtime and isolated linkers don't bundle
|
|
538
624
|
// symlinked store copies (which would surface their children as unresolvable
|
|
@@ -850,7 +936,11 @@ import data from "blume:data";
|
|
|
850
936
|
const { strings } = Astro.props;
|
|
851
937
|
---
|
|
852
938
|
|
|
853
|
-
<AskAI
|
|
939
|
+
<AskAI
|
|
940
|
+
endpoint={data.config.ask?.endpoint ?? undefined}
|
|
941
|
+
strings={strings ?? data.ui.ask}
|
|
942
|
+
suggestions={data.config.ask?.suggestions ?? []}
|
|
943
|
+
/>
|
|
854
944
|
`
|
|
855
945
|
: `---
|
|
856
946
|
// Generated by Blume. Do not edit.
|
|
@@ -886,12 +976,18 @@ const searchClientImport = (module: string): string =>
|
|
|
886
976
|
const SEARCH_BASE_IMPORT =
|
|
887
977
|
'import { joinBase } from "blume/components/islands/base-path.ts";\n';
|
|
888
978
|
|
|
889
|
-
/**
|
|
890
|
-
|
|
979
|
+
/**
|
|
980
|
+
* A client that loads a static `blume-search.json` index (Orama, FlexSearch).
|
|
981
|
+
* `locale` (Orama only) is the site's `i18n.defaultLocale`, which selects a
|
|
982
|
+
* word-segmenting tokenizer for languages written without spaces.
|
|
983
|
+
*/
|
|
984
|
+
const staticSearchClient = (module: string, locale?: string): string =>
|
|
891
985
|
`${SEARCH_CLIENT_HEADER}${searchClientImport(module)}${SEARCH_BASE_IMPORT}
|
|
892
986
|
const indexUrl = joinBase(import.meta.env.BASE_URL, "blume-search.json");
|
|
893
987
|
|
|
894
|
-
export const createSearch = () => create({ indexUrl
|
|
988
|
+
export const createSearch = () => create({ indexUrl${
|
|
989
|
+
locale ? `, locale: ${JSON.stringify(locale)}` : ""
|
|
990
|
+
} });
|
|
895
991
|
`;
|
|
896
992
|
|
|
897
993
|
/** A client that passes public credentials straight to the provider SDK. */
|
|
@@ -940,7 +1036,12 @@ export const searchClientTemplate = (config: ResolvedConfig): string => {
|
|
|
940
1036
|
const { search } = config;
|
|
941
1037
|
|
|
942
1038
|
if (search.provider === "orama" || search.provider === "flexsearch") {
|
|
943
|
-
|
|
1039
|
+
// Only Orama derives a tokenizer from the locale; FlexSearch has no
|
|
1040
|
+
// equivalent hook, so its client keeps the bare index URL.
|
|
1041
|
+
return staticSearchClient(
|
|
1042
|
+
search.provider,
|
|
1043
|
+
search.provider === "orama" ? config.i18n?.defaultLocale : undefined
|
|
1044
|
+
);
|
|
944
1045
|
}
|
|
945
1046
|
|
|
946
1047
|
const hosted = hostedSearchOptions(search);
|
|
@@ -1195,6 +1296,7 @@ export async function GET({ props }: { props: { title: string } }) {
|
|
|
1195
1296
|
export const scalarReferenceTemplate = (options: {
|
|
1196
1297
|
configuration: Record<string, unknown>;
|
|
1197
1298
|
dataImport: string;
|
|
1299
|
+
noindex?: boolean;
|
|
1198
1300
|
route: string;
|
|
1199
1301
|
title: string;
|
|
1200
1302
|
}): string =>
|
|
@@ -1229,6 +1331,7 @@ const htmlLang = i18n ? i18n.defaultLocale : "en";
|
|
|
1229
1331
|
favicon={data.config.favicon}
|
|
1230
1332
|
appleIcon={data.config.appleIcon}
|
|
1231
1333
|
navigation={data.navigation}
|
|
1334
|
+
noindex={${options.noindex === true}}
|
|
1232
1335
|
pageTitle={${JSON.stringify(options.title)}}
|
|
1233
1336
|
route={${JSON.stringify(options.route)}}
|
|
1234
1337
|
searchEnabled={data.config.search.enabled}
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
import { defineCommand } from "citty";
|
|
4
|
+
import { join } from "pathe";
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
AGENTS,
|
|
8
|
+
launchAgent,
|
|
9
|
+
WINDOWS_COMMAND_NOT_FOUND,
|
|
10
|
+
} from "../../audit/agent.ts";
|
|
11
|
+
import type { AgentKind } from "../../audit/agent.ts";
|
|
12
|
+
import { BlumeError } from "../../core/diagnostics.ts";
|
|
13
|
+
import { scanProject } from "../../core/project-graph.ts";
|
|
14
|
+
import { evalFixPrompt, initPrompt } from "../../eval/prompts.ts";
|
|
15
|
+
import {
|
|
16
|
+
evalReportJson,
|
|
17
|
+
fixLines,
|
|
18
|
+
headerLine,
|
|
19
|
+
questionDetails,
|
|
20
|
+
questionLine,
|
|
21
|
+
startLine,
|
|
22
|
+
summaryLine,
|
|
23
|
+
warningLines,
|
|
24
|
+
writeEvalReport,
|
|
25
|
+
} from "../../eval/report.ts";
|
|
26
|
+
import { runEval } from "../../eval/run.ts";
|
|
27
|
+
import type { EvalResult } from "../../eval/run.ts";
|
|
28
|
+
import { EvalsFileError, loadEvalsFile } from "../../eval/schema.ts";
|
|
29
|
+
import { reportInternalError } from "../internal-error.ts";
|
|
30
|
+
import { flushStdout, logger } from "../log.ts";
|
|
31
|
+
|
|
32
|
+
const DEFAULT_FILE = "evals.yaml";
|
|
33
|
+
|
|
34
|
+
/** Reader wall-clock ceiling per question, in seconds. */
|
|
35
|
+
const DEFAULT_TIMEOUT_S = 180;
|
|
36
|
+
|
|
37
|
+
const isAgentKind = (value: string): value is AgentKind => value in AGENTS;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Launch the interactive agent CLI, translating a missing executable into the
|
|
41
|
+
* Windows not-found sentinel. Only `ENOENT` means "not installed" — any other
|
|
42
|
+
* spawn failure (`EACCES`, `EMFILE`, …) must surface as itself.
|
|
43
|
+
*/
|
|
44
|
+
const launchAgentCode = async (
|
|
45
|
+
bin: string,
|
|
46
|
+
prompt: string
|
|
47
|
+
): Promise<number> => {
|
|
48
|
+
try {
|
|
49
|
+
return await launchAgent(bin, prompt);
|
|
50
|
+
} catch (error) {
|
|
51
|
+
if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
|
52
|
+
throw error;
|
|
53
|
+
}
|
|
54
|
+
return WINDOWS_COMMAND_NOT_FOUND;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const notInstalled = (agent: AgentKind): never => {
|
|
59
|
+
const cli = AGENTS[agent];
|
|
60
|
+
logger.error(
|
|
61
|
+
`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
|
|
62
|
+
);
|
|
63
|
+
return process.exit(1);
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** The fraction of run (non-skipped) questions that passed. */
|
|
67
|
+
export const passFraction = (result: EvalResult): number => {
|
|
68
|
+
const ran = result.results.length - result.counts.skip;
|
|
69
|
+
return ran === 0 ? 1 : result.counts.pass / ran;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
interface EvalFlags {
|
|
73
|
+
action?: string;
|
|
74
|
+
agent: string;
|
|
75
|
+
file: string;
|
|
76
|
+
fix?: boolean;
|
|
77
|
+
json?: boolean;
|
|
78
|
+
threshold?: string;
|
|
79
|
+
timeout?: string;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Validate the flag surface, exiting with a message on the first offense. */
|
|
83
|
+
const parseFlags = (
|
|
84
|
+
args: EvalFlags
|
|
85
|
+
): { agent: AgentKind; threshold: number; timeoutS: number } => {
|
|
86
|
+
if (!isAgentKind(args.agent)) {
|
|
87
|
+
logger.error(`Invalid --agent "${args.agent}" (use claude | codex).`);
|
|
88
|
+
process.exit(1);
|
|
89
|
+
}
|
|
90
|
+
if (args.action !== undefined && args.action !== "init") {
|
|
91
|
+
logger.error(`Unknown action "${args.action}" (did you mean "init"?).`);
|
|
92
|
+
process.exit(1);
|
|
93
|
+
}
|
|
94
|
+
if (args.json && args.fix) {
|
|
95
|
+
logger.error("--json and --fix are mutually exclusive.");
|
|
96
|
+
process.exit(1);
|
|
97
|
+
}
|
|
98
|
+
const threshold = args.threshold === undefined ? 1 : Number(args.threshold);
|
|
99
|
+
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
|
|
100
|
+
logger.error(`Invalid --threshold "${args.threshold}" (use 0..1).`);
|
|
101
|
+
process.exit(1);
|
|
102
|
+
}
|
|
103
|
+
const timeoutS =
|
|
104
|
+
args.timeout === undefined ? DEFAULT_TIMEOUT_S : Number(args.timeout);
|
|
105
|
+
if (!Number.isInteger(timeoutS) || timeoutS <= 0) {
|
|
106
|
+
logger.error(`Invalid --timeout "${args.timeout}" (whole seconds).`);
|
|
107
|
+
process.exit(1);
|
|
108
|
+
}
|
|
109
|
+
return { agent: args.agent, threshold, timeoutS };
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/** `blume eval --fix`: hand the failing report to the interactive agent. */
|
|
113
|
+
const runFixHandoff = async (
|
|
114
|
+
agent: AgentKind,
|
|
115
|
+
result: EvalResult,
|
|
116
|
+
root: string,
|
|
117
|
+
threshold: number
|
|
118
|
+
): Promise<void> => {
|
|
119
|
+
const count = result.counts.fail + result.counts.error;
|
|
120
|
+
if (count === 0) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const cli = AGENTS[agent];
|
|
124
|
+
const report = await writeEvalReport(result, root, threshold);
|
|
125
|
+
process.stderr.write(
|
|
126
|
+
` Handing ${count} failed question${count === 1 ? "" : "s"} to ${cli.name}…\n\n`
|
|
127
|
+
);
|
|
128
|
+
const code = await launchAgentCode(cli.bin, evalFixPrompt(report));
|
|
129
|
+
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
130
|
+
notInstalled(agent);
|
|
131
|
+
}
|
|
132
|
+
if (code !== 0) {
|
|
133
|
+
process.exit(code);
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
/** `blume eval init`: draft a starter evals file via the interactive agent. */
|
|
138
|
+
const runInit = async (agent: AgentKind, file: string): Promise<void> => {
|
|
139
|
+
const path = join(process.cwd(), file);
|
|
140
|
+
if (existsSync(path)) {
|
|
141
|
+
logger.error(
|
|
142
|
+
`${file} already exists — edit it directly, or pass --file to draft elsewhere.`
|
|
143
|
+
);
|
|
144
|
+
process.exit(1);
|
|
145
|
+
}
|
|
146
|
+
const code = await launchAgentCode(AGENTS[agent].bin, initPrompt(file));
|
|
147
|
+
if (code === WINDOWS_COMMAND_NOT_FOUND) {
|
|
148
|
+
notInstalled(agent);
|
|
149
|
+
}
|
|
150
|
+
if (code !== 0) {
|
|
151
|
+
process.exit(code);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
export const evalCommand = defineCommand({
|
|
156
|
+
args: {
|
|
157
|
+
action: {
|
|
158
|
+
description: 'Optional action: "init" drafts a starter evals file.',
|
|
159
|
+
required: false,
|
|
160
|
+
type: "positional",
|
|
161
|
+
},
|
|
162
|
+
agent: {
|
|
163
|
+
default: "claude",
|
|
164
|
+
description: "Agent CLI that reads and grades the docs: claude | codex.",
|
|
165
|
+
type: "string",
|
|
166
|
+
},
|
|
167
|
+
file: {
|
|
168
|
+
default: DEFAULT_FILE,
|
|
169
|
+
description: "The evals file to run.",
|
|
170
|
+
type: "string",
|
|
171
|
+
},
|
|
172
|
+
fix: {
|
|
173
|
+
description:
|
|
174
|
+
"After a failing run, hand the report to the agent to fix the docs interactively.",
|
|
175
|
+
type: "boolean",
|
|
176
|
+
},
|
|
177
|
+
json: {
|
|
178
|
+
description: "Emit the report as JSON on stdout (for CI/editors).",
|
|
179
|
+
type: "boolean",
|
|
180
|
+
},
|
|
181
|
+
threshold: {
|
|
182
|
+
description:
|
|
183
|
+
"Minimum passing fraction (0..1) before the run exits non-zero. Defaults to 1.",
|
|
184
|
+
type: "string",
|
|
185
|
+
},
|
|
186
|
+
timeout: {
|
|
187
|
+
description: `Reader time limit per question, in seconds. Defaults to ${DEFAULT_TIMEOUT_S}.`,
|
|
188
|
+
type: "string",
|
|
189
|
+
},
|
|
190
|
+
verbose: {
|
|
191
|
+
description: "Include the reader's full answer under each failure.",
|
|
192
|
+
type: "boolean",
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
meta: {
|
|
196
|
+
description:
|
|
197
|
+
"Test the docs: an agent answers your questions using only the documentation.",
|
|
198
|
+
name: "eval",
|
|
199
|
+
},
|
|
200
|
+
async run({ args }) {
|
|
201
|
+
const root = process.cwd();
|
|
202
|
+
const { agent, threshold, timeoutS } = parseFlags(args);
|
|
203
|
+
if (args.action === "init") {
|
|
204
|
+
await runInit(agent, args.file);
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
let result: EvalResult;
|
|
209
|
+
try {
|
|
210
|
+
// `scanProject`, not `prepareProject`: the eval reads the content tree
|
|
211
|
+
// and never regenerates the runtime, so it doesn't contend with a
|
|
212
|
+
// running dev server. Same reasoning as `blume audit`.
|
|
213
|
+
const project = await scanProject(root, { mode: "build" });
|
|
214
|
+
const evalsPath = join(root, args.file);
|
|
215
|
+
const { evals, raw } = await loadEvalsFile(evalsPath);
|
|
216
|
+
|
|
217
|
+
process.stderr.write(`${headerLine(evals.questions.length, agent)}\n\n`);
|
|
218
|
+
result = await runEval({
|
|
219
|
+
agent,
|
|
220
|
+
evals,
|
|
221
|
+
evalsPath,
|
|
222
|
+
onProgress: (event) => {
|
|
223
|
+
// Straight to stderr, not `logger.info` — consola drops info-level
|
|
224
|
+
// lines in test and CI environments.
|
|
225
|
+
if (event.kind === "question-start") {
|
|
226
|
+
process.stderr.write(
|
|
227
|
+
`${startLine(event.id, event.index, event.total)}\n`
|
|
228
|
+
);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
const lines = [
|
|
232
|
+
questionLine(event.result),
|
|
233
|
+
...questionDetails(event.result, Boolean(args.verbose)),
|
|
234
|
+
];
|
|
235
|
+
process.stderr.write(`${lines.join("\n")}\n`);
|
|
236
|
+
},
|
|
237
|
+
project,
|
|
238
|
+
rawEvals: raw,
|
|
239
|
+
readerTimeoutMs: timeoutS * 1000,
|
|
240
|
+
});
|
|
241
|
+
} catch (error) {
|
|
242
|
+
if (error instanceof EvalsFileError) {
|
|
243
|
+
logger.error(error.message);
|
|
244
|
+
process.exit(1);
|
|
245
|
+
}
|
|
246
|
+
if (error instanceof BlumeError) {
|
|
247
|
+
logger.error(error.diagnostic.message);
|
|
248
|
+
process.exit(1);
|
|
249
|
+
}
|
|
250
|
+
if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
|
|
251
|
+
notInstalled(agent);
|
|
252
|
+
}
|
|
253
|
+
reportInternalError(error);
|
|
254
|
+
process.exit(1);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const tail = [
|
|
258
|
+
"",
|
|
259
|
+
...warningLines(result, root),
|
|
260
|
+
...fixLines(result, root),
|
|
261
|
+
"",
|
|
262
|
+
` ${summaryLine(result)}`,
|
|
263
|
+
"",
|
|
264
|
+
];
|
|
265
|
+
process.stderr.write(tail.join("\n"));
|
|
266
|
+
|
|
267
|
+
const failed = passFraction(result) < threshold;
|
|
268
|
+
|
|
269
|
+
if (args.fix) {
|
|
270
|
+
// The gate is a CI concern; a handoff run succeeds when the agent
|
|
271
|
+
// session does, not when the docs already passed.
|
|
272
|
+
await runFixHandoff(agent, result, root, threshold);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (args.json) {
|
|
277
|
+
process.stdout.write(evalReportJson(result, root, threshold));
|
|
278
|
+
if (failed) {
|
|
279
|
+
// `process.exit` doesn't flush a piped stdout — without this the JSON
|
|
280
|
+
// is truncated mid-write in exactly the CI setups that consume it.
|
|
281
|
+
await flushStdout();
|
|
282
|
+
process.exit(1);
|
|
283
|
+
}
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (failed) {
|
|
288
|
+
process.exit(1);
|
|
289
|
+
}
|
|
290
|
+
},
|
|
291
|
+
});
|
package/src/cli/commands/init.ts
CHANGED
|
@@ -157,10 +157,15 @@ export const initCommand = defineCommand({
|
|
|
157
157
|
const sink = interactive ? clack.log : logger;
|
|
158
158
|
const { createdPackage } = await applyPlan(buildPlan(root, answers), sink);
|
|
159
159
|
|
|
160
|
-
// Keep Blume's generated runtime (`.blume/`) and
|
|
161
|
-
// of version control. Idempotent: creates
|
|
162
|
-
// entries already present
|
|
163
|
-
|
|
160
|
+
// Keep installed dependencies, Blume's generated runtime (`.blume/`), and
|
|
161
|
+
// build output (`dist/`) out of version control. Idempotent: creates
|
|
162
|
+
// `.gitignore` when absent and skips entries already present
|
|
163
|
+
// (trailing-slash agnostic).
|
|
164
|
+
const ignored = await ensureGitignore(root, [
|
|
165
|
+
"node_modules/",
|
|
166
|
+
".blume/",
|
|
167
|
+
"dist/",
|
|
168
|
+
]);
|
|
164
169
|
if (ignored.length > 0) {
|
|
165
170
|
sink.success(`Added ${ignored.join(", ")} to .gitignore`);
|
|
166
171
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
|
|
3
|
+
import { defineCommand } from "citty";
|
|
4
|
+
|
|
5
|
+
import type { McpData } from "../../ai/mcp/data.ts";
|
|
6
|
+
import { serveMcpStdio } from "../../ai/mcp/stdio.ts";
|
|
7
|
+
|
|
8
|
+
export const mcpStdioCommand = defineCommand({
|
|
9
|
+
args: {
|
|
10
|
+
data: {
|
|
11
|
+
description: "Path to a serialized MCP data snapshot (JSON).",
|
|
12
|
+
required: true,
|
|
13
|
+
type: "string",
|
|
14
|
+
},
|
|
15
|
+
},
|
|
16
|
+
meta: {
|
|
17
|
+
description:
|
|
18
|
+
"Serve an MCP data snapshot over stdio (internal, used by `blume eval`).",
|
|
19
|
+
name: "mcp-stdio",
|
|
20
|
+
},
|
|
21
|
+
async run({ args }) {
|
|
22
|
+
// stdout belongs to the JSON-RPC transport from here on; every diagnostic
|
|
23
|
+
// must go to stderr or the MCP client chokes on the stray line.
|
|
24
|
+
let data: McpData;
|
|
25
|
+
try {
|
|
26
|
+
data = JSON.parse(await readFile(args.data, "utf-8")) as McpData;
|
|
27
|
+
} catch (error) {
|
|
28
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
29
|
+
process.stderr.write(
|
|
30
|
+
`blume mcp-stdio: cannot load the snapshot at ${args.data}: ${detail}\n`
|
|
31
|
+
);
|
|
32
|
+
process.exit(1);
|
|
33
|
+
}
|
|
34
|
+
await serveMcpStdio(data);
|
|
35
|
+
},
|
|
36
|
+
});
|