blume 1.1.4 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +1 -1
  3. package/dist/cli/index.js +1319 -66
  4. package/dist/cli/index.js.map +34 -23
  5. package/dist/types/core/config-input.d.ts +18 -0
  6. package/dist/types/core/config.d.ts +4 -0
  7. package/dist/types/core/data.d.ts +1 -0
  8. package/dist/types/core/schema.d.ts +132 -17
  9. package/dist/types/core/types.d.ts +5 -3
  10. package/dist/types/openapi/references.d.ts +6 -0
  11. package/docs/advanced/api-reference.mdx +27 -0
  12. package/docs/advanced/changelog.mdx +10 -0
  13. package/docs/configuration/ai.mdx +38 -2
  14. package/docs/configuration/customization.mdx +27 -0
  15. package/docs/configuration/index.mdx +5 -0
  16. package/docs/configuration/search.mdx +15 -0
  17. package/docs/content/navigation.mdx +12 -0
  18. package/docs/reference/cli.mdx +17 -13
  19. package/docs/reference/eval.mdx +106 -0
  20. package/docs/reference/meta.ts +1 -1
  21. package/package.json +2 -2
  22. package/src/ai/agent-readability.ts +19 -1
  23. package/src/ai/ask-context.ts +7 -1
  24. package/src/ai/ask-data.ts +1 -0
  25. package/src/ai/llms.ts +9 -4
  26. package/src/ai/mcp/data.ts +7 -0
  27. package/src/ai/mcp/server.ts +24 -8
  28. package/src/ai/mcp/stdio.ts +38 -0
  29. package/src/astro/generate.ts +25 -2
  30. package/src/astro/templates.ts +129 -26
  31. package/src/cli/commands/eval.ts +291 -0
  32. package/src/cli/commands/init.ts +9 -4
  33. package/src/cli/commands/mcp-stdio.ts +36 -0
  34. package/src/cli/index.ts +4 -0
  35. package/src/cli/required-secrets.ts +1 -1
  36. package/src/components/content/AccordionItem.astro +2 -2
  37. package/src/components/content/TreeFolder.astro +1 -2
  38. package/src/components/islands/AskAI.astro +9 -2
  39. package/src/components/islands/ask-ai.tsx +4 -2
  40. package/src/components/islands/hooks.ts +10 -4
  41. package/src/components/layout/NavTree.astro +38 -20
  42. package/src/components/layout/ReferenceLayout.astro +4 -0
  43. package/src/components/layout/RootLayout.astro +2 -2
  44. package/src/components/layout/search/orama.ts +5 -2
  45. package/src/components/openapi/SchemaProperty.astro +3 -3
  46. package/src/core/config-input.ts +18 -0
  47. package/src/core/config.ts +4 -0
  48. package/src/core/data.ts +1 -0
  49. package/src/core/graph.ts +1 -0
  50. package/src/core/navigation.ts +9 -2
  51. package/src/core/schema.ts +51 -4
  52. package/src/core/server-features.ts +1 -1
  53. package/src/core/types.ts +5 -3
  54. package/src/eval/agents.ts +340 -0
  55. package/src/eval/findings.ts +103 -0
  56. package/src/eval/prompts.ts +78 -0
  57. package/src/eval/report.ts +214 -0
  58. package/src/eval/run.ts +290 -0
  59. package/src/eval/schema.ts +124 -0
  60. package/src/openapi/references.ts +23 -2
  61. package/src/openapi/render-mdx.ts +27 -4
  62. package/src/openapi/scalar.ts +1 -0
  63. package/src/openapi/source.ts +11 -4
  64. package/src/registry/eject.ts +23 -1
  65. package/src/search/build.ts +4 -3
  66. package/src/search/orama-index.ts +55 -5
@@ -1,3 +1,4 @@
1
+ import { createHash } from "node:crypto";
1
2
  import { existsSync, readFileSync, realpathSync } from "node:fs";
2
3
  import {
3
4
  lstat,
@@ -723,6 +724,22 @@ export const detectUsesMath = async (
723
724
  return [...contents, ...staged].some(containsMath);
724
725
  };
725
726
 
727
+ const hashConfigSource = (source: string): string =>
728
+ createHash("sha256").update(source).digest("hex");
729
+
730
+ const loadIntegrationBridge = async (
731
+ config: ResolvedConfig,
732
+ context: BlumeProject["context"]
733
+ ): Promise<Parameters<typeof astroConfigTemplate>[0]["integrationBridge"]> => {
734
+ if (config.integrations.length === 0 || !context.configFile) {
735
+ return;
736
+ }
737
+ return {
738
+ configFile: relative(context.outDir, context.configFile),
739
+ sourceHash: hashConfigSource(await readOptional(context.configFile)),
740
+ };
741
+ };
742
+
726
743
  const writeIfChanged = async (
727
744
  path: string,
728
745
  content: string
@@ -1085,7 +1102,10 @@ export const buildRuntimeData = (project: BlumeProject): string => {
1085
1102
  analytics: config.analytics ?? null,
1086
1103
  appleIcon: resolveAppleIcon(project),
1087
1104
  ask: config.ai.ask?.enabled
1088
- ? { suggestions: config.ai.ask.suggestions }
1105
+ ? {
1106
+ endpoint: config.ai.ask.endpoint ?? null,
1107
+ suggestions: config.ai.ask.suggestions,
1108
+ }
1089
1109
  : null,
1090
1110
  banner: resolveBanner(config),
1091
1111
  basePath: config.basePath,
@@ -1279,7 +1299,7 @@ const writeAskFiles = async (
1279
1299
  write: (path: string, content: string) => Promise<boolean>
1280
1300
  ): Promise<void> => {
1281
1301
  const { ask } = project.config.ai;
1282
- if (!ask?.enabled) {
1302
+ if (!(ask?.enabled && !ask.endpoint)) {
1283
1303
  return;
1284
1304
  }
1285
1305
  const grounded = ask.provider !== "inkeep";
@@ -1418,6 +1438,7 @@ export const generateRuntime = async (
1418
1438
  usesMath,
1419
1439
  userTheme,
1420
1440
  userExamplesCss,
1441
+ integrationBridge,
1421
1442
  islandDiscovery,
1422
1443
  exampleDiscovery,
1423
1444
  componentSlots,
@@ -1427,6 +1448,7 @@ export const generateRuntime = async (
1427
1448
  detectUsesMath(context.root, staged.values()),
1428
1449
  readOptional(context.themeFile),
1429
1450
  readOptional(examplesCssFile(context.root, config)),
1451
+ loadIntegrationBridge(config, context),
1430
1452
  discoverIslands(context.root),
1431
1453
  discoverExamples(context.root, config.examples.source),
1432
1454
  buildComponentSlots(context.componentsFile),
@@ -1500,6 +1522,7 @@ export const generateRuntime = async (
1500
1522
  dataPath,
1501
1523
  examplesPath,
1502
1524
  examplesThemePath,
1525
+ integrationBridge,
1503
1526
  needsReact,
1504
1527
  needsSvelte,
1505
1528
  needsVue,
@@ -87,7 +87,14 @@ const WRANGLER_CONFIG_FILES = [
87
87
  ];
88
88
 
89
89
  const resolveCloudflareAdapterArgs = (context: ProjectContext): string => {
90
- const args: string[] = ['prerenderEnvironment: "node"'];
90
+ // Every Blume HTML route prerenders (the only server routes are API
91
+ // endpoints), so images are optimized at build time with sharp. The
92
+ // adapter's default (`cloudflare-binding`) would instead declare a runtime
93
+ // `IMAGES` binding in the generated wrangler config that nothing uses.
94
+ const args: string[] = [
95
+ 'prerenderEnvironment: "node"',
96
+ 'imageService: "compile"',
97
+ ];
91
98
  const wranglerPath = WRANGLER_CONFIG_FILES.map((file) =>
92
99
  join(context.root, file)
93
100
  ).find((file) => existsSync(file));
@@ -104,6 +111,37 @@ const resolveCloudflareAdapterArgs = (context: ProjectContext): string => {
104
111
  return `{ ${args.join(", ")} }`;
105
112
  };
106
113
 
114
+ /**
115
+ * Without a configured driver, `@astrojs/cloudflare` force-enables KV-backed
116
+ * sessions and declares a `SESSION` kv_namespaces entry in the generated
117
+ * wrangler config — which `wrangler deploy` then requires a real KV namespace
118
+ * for, even though Blume never reads `Astro.session`. An explicit in-memory
119
+ * driver keeps the binding out. Swap for Astro's session opt-out once
120
+ * withastro/astro#16871 ships in the supported range.
121
+ */
122
+ const resolveSessionOption = (deployment: {
123
+ adapter: string | null;
124
+ output: string;
125
+ }): string =>
126
+ deployment.output === "server" && deployment.adapter === "cloudflare"
127
+ ? "\n session: { driver: sessionDrivers.memory() },"
128
+ : "";
129
+
130
+ /** The named imports the generated config pulls from `astro/config`. */
131
+ const astroConfigImportLine = (options: {
132
+ hasFonts: boolean;
133
+ hasSession: boolean;
134
+ }): string => {
135
+ const names = ["defineConfig"];
136
+ if (options.hasFonts) {
137
+ names.push("fontProviders");
138
+ }
139
+ if (options.hasSession) {
140
+ names.push("sessionDrivers");
141
+ }
142
+ return `import { ${names.join(", ")} } from "astro/config";`;
143
+ };
144
+
107
145
  /**
108
146
  * Integration packages the generated runtime imports. Declaring them in
109
147
  * `.blume/package.json` lets Astro's framework-package crawl discover and bundle
@@ -138,7 +176,7 @@ export const runtimeDependencies = (options: {
138
176
  // (and the user installs) exactly the backend it uses — nothing more.
139
177
  deps.push(...searchProviderMeta(config.search.provider).runtimeDeps);
140
178
  // Ask AI's provider SDK, when its backend needs one (gateway uses core `ai`).
141
- if (config.ai.ask?.enabled) {
179
+ if (config.ai.ask?.enabled && !config.ai.ask.endpoint) {
142
180
  const askDep = askBackendRuntimeDep(config.ai.ask);
143
181
  if (askDep) {
144
182
  deps.push(askDep);
@@ -290,6 +328,36 @@ const devWatchOption = (
290
328
  },`
291
329
  : "";
292
330
 
331
+ interface IntegrationBridgeOptions {
332
+ /** Config path relative to the generated Astro config. */
333
+ configFile: string;
334
+ /** SHA-256 used to invalidate Astro's generated config. */
335
+ sourceHash?: string;
336
+ }
337
+
338
+ const renderIntegrationBridge = (
339
+ bridge: IntegrationBridgeOptions | undefined
340
+ ) => {
341
+ if (!bridge) {
342
+ return {
343
+ configSourceMarker: "",
344
+ userConfigImports: "",
345
+ userConfigSetup: "",
346
+ userIntegrationSpread: "",
347
+ };
348
+ }
349
+ return {
350
+ configSourceMarker: bridge.sourceHash
351
+ ? `// Blume config source SHA-256: ${bridge.sourceHash}\n`
352
+ : "",
353
+ userConfigImports: `import { dirname, resolve } from "node:path";\nimport { fileURLToPath } from "node:url";\nimport { createModuleLoader } from "blume/core/load-module.ts";\n`,
354
+ userConfigSetup: `const loadBlumeConfig = createModuleLoader();\nconst blumeConfig = await loadBlumeConfig(resolve(dirname(fileURLToPath(import.meta.url)), ${JSON.stringify(
355
+ bridge.configFile
356
+ )}));\n\n`,
357
+ userIntegrationSpread: ", ...(blumeConfig?.integrations ?? [])",
358
+ };
359
+ };
360
+
293
361
  export const astroConfigTemplate = (options: {
294
362
  context: ProjectContext;
295
363
  config: ResolvedConfig;
@@ -322,6 +390,8 @@ export const astroConfigTemplate = (options: {
322
390
  * See {@link devWatchOption} for why this must stay scoped.
323
391
  */
324
392
  contentWatchesRuntimeDir?: boolean;
393
+ /** Bridge used to load configured integrations without serializing them. */
394
+ integrationBridge?: IntegrationBridgeOptions;
325
395
  }): string => {
326
396
  const { context, config, needsReact, pages, dataPath, themePath } = options;
327
397
  const {
@@ -367,6 +437,8 @@ export const astroConfigTemplate = (options: {
367
437
  const adapterOption =
368
438
  server && deployment.adapter ? `\n adapter: ${adapterExpr},` : "";
369
439
 
440
+ const sessionOption = resolveSessionOption(deployment);
441
+
370
442
  const siteOption = deployment.site
371
443
  ? `\n site: ${JSON.stringify(deployment.site)},`
372
444
  : "";
@@ -424,9 +496,10 @@ export const astroConfigTemplate = (options: {
424
496
  )
425
497
  .join(", ")}],`
426
498
  : "";
427
- const defineConfigImport = fontEntries.length
428
- ? `import { defineConfig, fontProviders } from "astro/config";`
429
- : `import { defineConfig } from "astro/config";`;
499
+ const defineConfigImport = astroConfigImportLine({
500
+ hasFonts: fontEntries.length > 0,
501
+ hasSession: sessionOption.length > 0,
502
+ });
430
503
 
431
504
  // Framework renderers are only wired in when an island (or Ask AI, for React)
432
505
  // needs them. The core theme is Astro-first and ships no client JS.
@@ -486,20 +559,26 @@ export const astroConfigTemplate = (options: {
486
559
  context.outDir,
487
560
  options.contentWatchesRuntimeDir
488
561
  );
562
+ const {
563
+ configSourceMarker,
564
+ userConfigImports,
565
+ userConfigSetup,
566
+ userIntegrationSpread,
567
+ } = renderIntegrationBridge(options.integrationBridge);
489
568
 
490
569
  return `// Generated by Blume. Do not edit; this file is recreated on each run.
491
- ${defineConfigImport}
570
+ ${configSourceMarker}${userConfigImports}${defineConfigImport}
492
571
  import mdx from "@astrojs/mdx";
493
572
  import tailwindcss from "@tailwindcss/vite";
494
573
  import { blumeMarkdownProcessor, blumeMdxProcessor, blumeShikiTransformers, blumeTwoslashTransformer } from "blume/markdown";
495
574
  ${reactImport}${vueImport}${svelteImport}${blumeImport}${adapterImport}
496
- export default defineConfig({
575
+ ${userConfigSetup}export default defineConfig({
497
576
  root: ${JSON.stringify(context.outDir)},
498
577
  srcDir: ${JSON.stringify(`${context.outDir}/src`)},
499
578
  outDir: ${JSON.stringify(astroOutDir(context))},
500
579
  publicDir: ${JSON.stringify(`${context.root}/public`)},
501
- output: ${JSON.stringify(deployment.output)},${adapterOption}${siteOption}${baseOption}${redirectsOption}${i18nOption}${fontsOption}
502
- integrations: [${integrations.join(", ")}],
580
+ output: ${JSON.stringify(deployment.output)},${adapterOption}${sessionOption}${siteOption}${baseOption}${redirectsOption}${i18nOption}${fontsOption}
581
+ integrations: [${integrations.join(", ")}${userIntegrationSpread}],
503
582
  markdown: {
504
583
  processor: blumeMarkdownProcessor(${JSON.stringify({
505
584
  basePath: config.basePath,
@@ -521,18 +600,25 @@ export default defineConfig({
521
600
  devToolbar: { enabled: false },
522
601
  vite: {
523
602
  plugins: [tailwindcss(), prerenderDepsPlugin(), serverAppResolvePlugin()],
524
- // Mermaid (lazy-loaded client-side for diagrams) statically imports dayjs as
525
- // CJS (\`dayjs/dayjs.min.js\`). In dev, an un-pre-bundled dependency is served
526
- // as raw ESM, and that UMD file exposes no \`default\` export, so mermaid
527
- // throws on load and diagrams render blank. Forcing mermaid through the dep
528
- // optimizer bundles dayjs with correct CJS interop. In a standalone install
529
- // Blume's dynamic \`import("mermaid")\` lives inside \`node_modules/blume\`,
530
- // which Vite's optimizer scan doesn't crawl, so mermaid is never discovered
531
- // on its own hence the explicit include. mermaid resolves through the
532
- // \`blume\` package (it isn't a direct dep of the generated project), so the
533
- // nested \`blume > mermaid\` form is required. Production (Rollup) already
534
- // handles the interop, so this only affects dev.
535
- optimizeDeps: { include: ["blume > mermaid"] },
603
+ // The lazy client-side imports both land on CJS/UMD files: mermaid (for
604
+ // diagrams) statically imports dayjs as CJS (\`dayjs/dayjs.min.js\`), and
605
+ // epub-gen-memory's browser bundle is a browserified UMD. In dev, an
606
+ // un-pre-bundled dependency is served as raw ESM, where such a file
607
+ // exposes no \`default\` export mermaid throws on load and diagrams
608
+ // render blank, and the EPUB export throws \`epub is not a function\`
609
+ // (the UMD finds no \`exports\`/\`define\` and strands its callable on
610
+ // \`window.epubGen\` instead). Forcing them through the dep optimizer
611
+ // restores the CJS interop. In a standalone install these dynamic imports
612
+ // live inside \`node_modules/blume\`, which Vite's optimizer scan doesn't
613
+ // crawl, so neither is discovered on its own — hence the explicit
614
+ // includes. They resolve through the \`blume\` package (they aren't direct
615
+ // deps of the generated project), so the nested \`blume > x\` form is
616
+ // required, and epub-gen-memory must name the \`/bundle\` subpath that is
617
+ // actually imported: optimizing the package root leaves that entry out.
618
+ // Production (Rollup) already handles the interop, so this only affects dev.
619
+ optimizeDeps: {
620
+ include: ["blume > mermaid", "blume > epub-gen-memory/bundle"],
621
+ },
536
622
  // Blume's render-time deps are forced external on both build environments so
537
623
  // native bindings resolve at runtime and isolated linkers don't bundle
538
624
  // symlinked store copies (which would surface their children as unresolvable
@@ -850,7 +936,11 @@ import data from "blume:data";
850
936
  const { strings } = Astro.props;
851
937
  ---
852
938
 
853
- <AskAI strings={strings ?? data.ui.ask} suggestions={data.config.ask?.suggestions ?? []} />
939
+ <AskAI
940
+ endpoint={data.config.ask?.endpoint ?? undefined}
941
+ strings={strings ?? data.ui.ask}
942
+ suggestions={data.config.ask?.suggestions ?? []}
943
+ />
854
944
  `
855
945
  : `---
856
946
  // Generated by Blume. Do not edit.
@@ -886,12 +976,18 @@ const searchClientImport = (module: string): string =>
886
976
  const SEARCH_BASE_IMPORT =
887
977
  'import { joinBase } from "blume/components/islands/base-path.ts";\n';
888
978
 
889
- /** A client that loads a static `blume-search.json` index (Orama, FlexSearch). */
890
- const staticSearchClient = (module: string): string =>
979
+ /**
980
+ * A client that loads a static `blume-search.json` index (Orama, FlexSearch).
981
+ * `locale` (Orama only) is the site's `i18n.defaultLocale`, which selects a
982
+ * word-segmenting tokenizer for languages written without spaces.
983
+ */
984
+ const staticSearchClient = (module: string, locale?: string): string =>
891
985
  `${SEARCH_CLIENT_HEADER}${searchClientImport(module)}${SEARCH_BASE_IMPORT}
892
986
  const indexUrl = joinBase(import.meta.env.BASE_URL, "blume-search.json");
893
987
 
894
- export const createSearch = () => create({ indexUrl });
988
+ export const createSearch = () => create({ indexUrl${
989
+ locale ? `, locale: ${JSON.stringify(locale)}` : ""
990
+ } });
895
991
  `;
896
992
 
897
993
  /** A client that passes public credentials straight to the provider SDK. */
@@ -940,7 +1036,12 @@ export const searchClientTemplate = (config: ResolvedConfig): string => {
940
1036
  const { search } = config;
941
1037
 
942
1038
  if (search.provider === "orama" || search.provider === "flexsearch") {
943
- return staticSearchClient(search.provider);
1039
+ // Only Orama derives a tokenizer from the locale; FlexSearch has no
1040
+ // equivalent hook, so its client keeps the bare index URL.
1041
+ return staticSearchClient(
1042
+ search.provider,
1043
+ search.provider === "orama" ? config.i18n?.defaultLocale : undefined
1044
+ );
944
1045
  }
945
1046
 
946
1047
  const hosted = hostedSearchOptions(search);
@@ -1195,6 +1296,7 @@ export async function GET({ props }: { props: { title: string } }) {
1195
1296
  export const scalarReferenceTemplate = (options: {
1196
1297
  configuration: Record<string, unknown>;
1197
1298
  dataImport: string;
1299
+ noindex?: boolean;
1198
1300
  route: string;
1199
1301
  title: string;
1200
1302
  }): string =>
@@ -1229,6 +1331,7 @@ const htmlLang = i18n ? i18n.defaultLocale : "en";
1229
1331
  favicon={data.config.favicon}
1230
1332
  appleIcon={data.config.appleIcon}
1231
1333
  navigation={data.navigation}
1334
+ noindex={${options.noindex === true}}
1232
1335
  pageTitle={${JSON.stringify(options.title)}}
1233
1336
  route={${JSON.stringify(options.route)}}
1234
1337
  searchEnabled={data.config.search.enabled}
@@ -0,0 +1,291 @@
1
+ import { existsSync } from "node:fs";
2
+
3
+ import { defineCommand } from "citty";
4
+ import { join } from "pathe";
5
+
6
+ import {
7
+ AGENTS,
8
+ launchAgent,
9
+ WINDOWS_COMMAND_NOT_FOUND,
10
+ } from "../../audit/agent.ts";
11
+ import type { AgentKind } from "../../audit/agent.ts";
12
+ import { BlumeError } from "../../core/diagnostics.ts";
13
+ import { scanProject } from "../../core/project-graph.ts";
14
+ import { evalFixPrompt, initPrompt } from "../../eval/prompts.ts";
15
+ import {
16
+ evalReportJson,
17
+ fixLines,
18
+ headerLine,
19
+ questionDetails,
20
+ questionLine,
21
+ startLine,
22
+ summaryLine,
23
+ warningLines,
24
+ writeEvalReport,
25
+ } from "../../eval/report.ts";
26
+ import { runEval } from "../../eval/run.ts";
27
+ import type { EvalResult } from "../../eval/run.ts";
28
+ import { EvalsFileError, loadEvalsFile } from "../../eval/schema.ts";
29
+ import { reportInternalError } from "../internal-error.ts";
30
+ import { flushStdout, logger } from "../log.ts";
31
+
32
+ const DEFAULT_FILE = "evals.yaml";
33
+
34
+ /** Reader wall-clock ceiling per question, in seconds. */
35
+ const DEFAULT_TIMEOUT_S = 180;
36
+
37
+ const isAgentKind = (value: string): value is AgentKind => value in AGENTS;
38
+
39
+ /**
40
+ * Launch the interactive agent CLI, translating a missing executable into the
41
+ * Windows not-found sentinel. Only `ENOENT` means "not installed" — any other
42
+ * spawn failure (`EACCES`, `EMFILE`, …) must surface as itself.
43
+ */
44
+ const launchAgentCode = async (
45
+ bin: string,
46
+ prompt: string
47
+ ): Promise<number> => {
48
+ try {
49
+ return await launchAgent(bin, prompt);
50
+ } catch (error) {
51
+ if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
52
+ throw error;
53
+ }
54
+ return WINDOWS_COMMAND_NOT_FOUND;
55
+ }
56
+ };
57
+
58
+ const notInstalled = (agent: AgentKind): never => {
59
+ const cli = AGENTS[agent];
60
+ logger.error(
61
+ `${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
62
+ );
63
+ return process.exit(1);
64
+ };
65
+
66
+ /** The fraction of run (non-skipped) questions that passed. */
67
+ export const passFraction = (result: EvalResult): number => {
68
+ const ran = result.results.length - result.counts.skip;
69
+ return ran === 0 ? 1 : result.counts.pass / ran;
70
+ };
71
+
72
+ interface EvalFlags {
73
+ action?: string;
74
+ agent: string;
75
+ file: string;
76
+ fix?: boolean;
77
+ json?: boolean;
78
+ threshold?: string;
79
+ timeout?: string;
80
+ }
81
+
82
+ /** Validate the flag surface, exiting with a message on the first offense. */
83
+ const parseFlags = (
84
+ args: EvalFlags
85
+ ): { agent: AgentKind; threshold: number; timeoutS: number } => {
86
+ if (!isAgentKind(args.agent)) {
87
+ logger.error(`Invalid --agent "${args.agent}" (use claude | codex).`);
88
+ process.exit(1);
89
+ }
90
+ if (args.action !== undefined && args.action !== "init") {
91
+ logger.error(`Unknown action "${args.action}" (did you mean "init"?).`);
92
+ process.exit(1);
93
+ }
94
+ if (args.json && args.fix) {
95
+ logger.error("--json and --fix are mutually exclusive.");
96
+ process.exit(1);
97
+ }
98
+ const threshold = args.threshold === undefined ? 1 : Number(args.threshold);
99
+ if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
100
+ logger.error(`Invalid --threshold "${args.threshold}" (use 0..1).`);
101
+ process.exit(1);
102
+ }
103
+ const timeoutS =
104
+ args.timeout === undefined ? DEFAULT_TIMEOUT_S : Number(args.timeout);
105
+ if (!Number.isInteger(timeoutS) || timeoutS <= 0) {
106
+ logger.error(`Invalid --timeout "${args.timeout}" (whole seconds).`);
107
+ process.exit(1);
108
+ }
109
+ return { agent: args.agent, threshold, timeoutS };
110
+ };
111
+
112
+ /** `blume eval --fix`: hand the failing report to the interactive agent. */
113
+ const runFixHandoff = async (
114
+ agent: AgentKind,
115
+ result: EvalResult,
116
+ root: string,
117
+ threshold: number
118
+ ): Promise<void> => {
119
+ const count = result.counts.fail + result.counts.error;
120
+ if (count === 0) {
121
+ return;
122
+ }
123
+ const cli = AGENTS[agent];
124
+ const report = await writeEvalReport(result, root, threshold);
125
+ process.stderr.write(
126
+ ` Handing ${count} failed question${count === 1 ? "" : "s"} to ${cli.name}…\n\n`
127
+ );
128
+ const code = await launchAgentCode(cli.bin, evalFixPrompt(report));
129
+ if (code === WINDOWS_COMMAND_NOT_FOUND) {
130
+ notInstalled(agent);
131
+ }
132
+ if (code !== 0) {
133
+ process.exit(code);
134
+ }
135
+ };
136
+
137
+ /** `blume eval init`: draft a starter evals file via the interactive agent. */
138
+ const runInit = async (agent: AgentKind, file: string): Promise<void> => {
139
+ const path = join(process.cwd(), file);
140
+ if (existsSync(path)) {
141
+ logger.error(
142
+ `${file} already exists — edit it directly, or pass --file to draft elsewhere.`
143
+ );
144
+ process.exit(1);
145
+ }
146
+ const code = await launchAgentCode(AGENTS[agent].bin, initPrompt(file));
147
+ if (code === WINDOWS_COMMAND_NOT_FOUND) {
148
+ notInstalled(agent);
149
+ }
150
+ if (code !== 0) {
151
+ process.exit(code);
152
+ }
153
+ };
154
+
155
+ export const evalCommand = defineCommand({
156
+ args: {
157
+ action: {
158
+ description: 'Optional action: "init" drafts a starter evals file.',
159
+ required: false,
160
+ type: "positional",
161
+ },
162
+ agent: {
163
+ default: "claude",
164
+ description: "Agent CLI that reads and grades the docs: claude | codex.",
165
+ type: "string",
166
+ },
167
+ file: {
168
+ default: DEFAULT_FILE,
169
+ description: "The evals file to run.",
170
+ type: "string",
171
+ },
172
+ fix: {
173
+ description:
174
+ "After a failing run, hand the report to the agent to fix the docs interactively.",
175
+ type: "boolean",
176
+ },
177
+ json: {
178
+ description: "Emit the report as JSON on stdout (for CI/editors).",
179
+ type: "boolean",
180
+ },
181
+ threshold: {
182
+ description:
183
+ "Minimum passing fraction (0..1) before the run exits non-zero. Defaults to 1.",
184
+ type: "string",
185
+ },
186
+ timeout: {
187
+ description: `Reader time limit per question, in seconds. Defaults to ${DEFAULT_TIMEOUT_S}.`,
188
+ type: "string",
189
+ },
190
+ verbose: {
191
+ description: "Include the reader's full answer under each failure.",
192
+ type: "boolean",
193
+ },
194
+ },
195
+ meta: {
196
+ description:
197
+ "Test the docs: an agent answers your questions using only the documentation.",
198
+ name: "eval",
199
+ },
200
+ async run({ args }) {
201
+ const root = process.cwd();
202
+ const { agent, threshold, timeoutS } = parseFlags(args);
203
+ if (args.action === "init") {
204
+ await runInit(agent, args.file);
205
+ return;
206
+ }
207
+
208
+ let result: EvalResult;
209
+ try {
210
+ // `scanProject`, not `prepareProject`: the eval reads the content tree
211
+ // and never regenerates the runtime, so it doesn't contend with a
212
+ // running dev server. Same reasoning as `blume audit`.
213
+ const project = await scanProject(root, { mode: "build" });
214
+ const evalsPath = join(root, args.file);
215
+ const { evals, raw } = await loadEvalsFile(evalsPath);
216
+
217
+ process.stderr.write(`${headerLine(evals.questions.length, agent)}\n\n`);
218
+ result = await runEval({
219
+ agent,
220
+ evals,
221
+ evalsPath,
222
+ onProgress: (event) => {
223
+ // Straight to stderr, not `logger.info` — consola drops info-level
224
+ // lines in test and CI environments.
225
+ if (event.kind === "question-start") {
226
+ process.stderr.write(
227
+ `${startLine(event.id, event.index, event.total)}\n`
228
+ );
229
+ return;
230
+ }
231
+ const lines = [
232
+ questionLine(event.result),
233
+ ...questionDetails(event.result, Boolean(args.verbose)),
234
+ ];
235
+ process.stderr.write(`${lines.join("\n")}\n`);
236
+ },
237
+ project,
238
+ rawEvals: raw,
239
+ readerTimeoutMs: timeoutS * 1000,
240
+ });
241
+ } catch (error) {
242
+ if (error instanceof EvalsFileError) {
243
+ logger.error(error.message);
244
+ process.exit(1);
245
+ }
246
+ if (error instanceof BlumeError) {
247
+ logger.error(error.diagnostic.message);
248
+ process.exit(1);
249
+ }
250
+ if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
251
+ notInstalled(agent);
252
+ }
253
+ reportInternalError(error);
254
+ process.exit(1);
255
+ }
256
+
257
+ const tail = [
258
+ "",
259
+ ...warningLines(result, root),
260
+ ...fixLines(result, root),
261
+ "",
262
+ ` ${summaryLine(result)}`,
263
+ "",
264
+ ];
265
+ process.stderr.write(tail.join("\n"));
266
+
267
+ const failed = passFraction(result) < threshold;
268
+
269
+ if (args.fix) {
270
+ // The gate is a CI concern; a handoff run succeeds when the agent
271
+ // session does, not when the docs already passed.
272
+ await runFixHandoff(agent, result, root, threshold);
273
+ return;
274
+ }
275
+
276
+ if (args.json) {
277
+ process.stdout.write(evalReportJson(result, root, threshold));
278
+ if (failed) {
279
+ // `process.exit` doesn't flush a piped stdout — without this the JSON
280
+ // is truncated mid-write in exactly the CI setups that consume it.
281
+ await flushStdout();
282
+ process.exit(1);
283
+ }
284
+ return;
285
+ }
286
+
287
+ if (failed) {
288
+ process.exit(1);
289
+ }
290
+ },
291
+ });
@@ -157,10 +157,15 @@ export const initCommand = defineCommand({
157
157
  const sink = interactive ? clack.log : logger;
158
158
  const { createdPackage } = await applyPlan(buildPlan(root, answers), sink);
159
159
 
160
- // Keep Blume's generated runtime (`.blume/`) and build output (`dist/`) out
161
- // of version control. Idempotent: creates `.gitignore` when absent and skips
162
- // entries already present (trailing-slash agnostic).
163
- const ignored = await ensureGitignore(root, [".blume/", "dist/"]);
160
+ // Keep installed dependencies, Blume's generated runtime (`.blume/`), and
161
+ // build output (`dist/`) out of version control. Idempotent: creates
162
+ // `.gitignore` when absent and skips entries already present
163
+ // (trailing-slash agnostic).
164
+ const ignored = await ensureGitignore(root, [
165
+ "node_modules/",
166
+ ".blume/",
167
+ "dist/",
168
+ ]);
164
169
  if (ignored.length > 0) {
165
170
  sink.success(`Added ${ignored.join(", ")} to .gitignore`);
166
171
  }
@@ -0,0 +1,36 @@
1
+ import { readFile } from "node:fs/promises";
2
+
3
+ import { defineCommand } from "citty";
4
+
5
+ import type { McpData } from "../../ai/mcp/data.ts";
6
+ import { serveMcpStdio } from "../../ai/mcp/stdio.ts";
7
+
8
+ export const mcpStdioCommand = defineCommand({
9
+ args: {
10
+ data: {
11
+ description: "Path to a serialized MCP data snapshot (JSON).",
12
+ required: true,
13
+ type: "string",
14
+ },
15
+ },
16
+ meta: {
17
+ description:
18
+ "Serve an MCP data snapshot over stdio (internal, used by `blume eval`).",
19
+ name: "mcp-stdio",
20
+ },
21
+ async run({ args }) {
22
+ // stdout belongs to the JSON-RPC transport from here on; every diagnostic
23
+ // must go to stderr or the MCP client chokes on the stray line.
24
+ let data: McpData;
25
+ try {
26
+ data = JSON.parse(await readFile(args.data, "utf-8")) as McpData;
27
+ } catch (error) {
28
+ const detail = error instanceof Error ? error.message : String(error);
29
+ process.stderr.write(
30
+ `blume mcp-stdio: cannot load the snapshot at ${args.data}: ${detail}\n`
31
+ );
32
+ process.exit(1);
33
+ }
34
+ await serveMcpStdio(data);
35
+ },
36
+ });