blume 2.1.0 → 2.1.1

package/src/eval/prompts.ts

@@ -40,8 +40,11 @@ Reply with ONLY this JSON object on a single line, no markdown fences:
 {"pass": true|false, "score": 0.0-1.0, "missing": ["expected facts absent or contradicted"], "notes": "one sentence"}`;
 };
 
-/** The `--fix` handoff prompt: where the report is and the ground rules. */
-export const evalFixPrompt = (reportPath: string): string =>
+/**
+ * The `--fix` handoff prompt: where the report is, the ground rules, and the
+ * `blume eval` command that verifies the fixes (the run's own flags).
+ */
+export const evalFixPrompt = (reportPath: string, command: string): string =>
   `Fix the documentation gaps found by \`blume eval\` in this project.
 
 The full report is at ${reportPath}. It is JSON: each entry in \`eval.results\` with status "fail" is one question the documentation could not answer. Each carries the \`question\`, the \`expected\` facts, the judge's \`missing\` facts, and the reader agent's \`answer\` (what the docs currently convey). The matching \`diagnostics\` entry names the source \`file\` of the page that should answer it.
@@ -51,7 +54,7 @@ Work through every failed question:
 2. Edit the documentation so it states the missing facts explicitly. Add prose, not filler; keep the page's voice.
 3. Never delete questions from the evals file or weaken expected facts.
 
-When you are done, run \`blume eval\` to verify, and repeat until every question passes.`;
+When you are done, run \`${command}\` to verify, and repeat until every question passes.`;
 
 /** The `eval init` prompt: draft a starter evals file from the docs. */
 export const initPrompt = (evalsPath: string): string =>

package/src/eval/report.ts

@@ -10,27 +10,40 @@ import { duration, money, seconds } from "../cli/report-format.ts";
 import { countBySeverity } from "../core/diagnostics.ts";
 import type { EvalResult, QuestionResult, QuestionStatus } from "./run.ts";
 
+/** How a question reads in the report: its status, or `warn` for a warned miss. */
+type ShownStatus = QuestionStatus | "warn";
+
 const GLYPH = {
   error: "!",
   fail: "✖",
   pass: "✔",
   skip: "⊘",
-} satisfies Record<QuestionStatus, string>;
+  warn: "⚠",
+} satisfies Record<ShownStatus, string>;
 
 const STATUS_COLOR = {
   error: colors.yellow,
   fail: colors.red,
   pass: colors.green,
   skip: colors.dim,
-} satisfies Record<QuestionStatus, ColorFunction>;
+  warn: colors.yellow,
+} satisfies Record<ShownStatus, ColorFunction>;
+
+/**
+ * A failed `severity: warning` question: its miss never counts against the
+ * gate, so the report shows and counts it as a warning, not a failure.
+ */
+const warned = (result: QuestionResult): boolean =>
+  result.status === "fail" && result.severity === "warning";
 
 /** Longest id gets the room; everything shorter aligns to it. */
 const ID_PAD = 28;
 
 /** One question's progress/report line: glyph, id, status, score, time, cost. */
 export const questionLine = (result: QuestionResult): string => {
-  const color = STATUS_COLOR[result.status];
-  const glyph = color(GLYPH[result.status]);
+  const shown: ShownStatus = warned(result) ? "warn" : result.status;
+  const color = STATUS_COLOR[shown];
+  const glyph = color(GLYPH[shown]);
   const id = result.id.padEnd(ID_PAD);
   if (result.status === "skip") {
     return `  ${glyph} ${id} ${colors.dim("skipped")}`;
@@ -38,7 +51,7 @@ export const questionLine = (result: QuestionResult): string => {
   const score = result.score === undefined ? "" : result.score.toFixed(2);
   const cost = money(result.costUsd);
   const cells = [
-    color(result.status),
+    color(shown),
     score,
     colors.dim(seconds(result.durationMs)),
     cost === "" ? "" : colors.dim(cost),
@@ -75,9 +88,12 @@ export const questionDetails = (
 /** The one-line totals: `9 passed · 2 failed · 1 skipped · 1m 42s · $0.71`. */
 export const summaryLine = (result: EvalResult): string => {
   const { counts } = result;
+  const warnings = result.results.filter(warned).length;
+  const failures = counts.fail - warnings;
   const parts = [
     `${counts.pass} passed`,
-    counts.fail > 0 ? `${counts.fail} failed` : "",
+    failures > 0 ? `${failures} failed` : "",
+    warnings > 0 ? `${warnings} warned` : "",
     counts.error > 0 ? `${counts.error} errored` : "",
     counts.skip > 0 ? `${counts.skip} skipped` : "",
     duration(result.durationMs),
@@ -98,10 +114,11 @@ export const startLine = (id: string, index: number, total: number): string =>
  * `fix:` pointers for failed questions, naming the file that resolves each. A
  * question whose agent run errored gets a `run failed:` line pointing at the
  * question instead: the run never graded the docs, so there's no page to fix.
+ * A `severity: warning` question's miss is a warning line instead.
  */
 export const fixLines = (result: EvalResult, root: string): string[] =>
   result.diagnostics
-    .filter((diagnostic) => diagnostic.code !== "BLUME_EVAL_ROUTE_UNKNOWN")
+    .filter((diagnostic) => diagnostic.severity !== "warning")
     .map((finding) => {
       const site = finding.file
         ? `${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}`
@@ -113,10 +130,13 @@ export const fixLines = (result: EvalResult, root: string): string[] =>
       return `  ${label} ${site} ${colors.dim(finding.message)}`;
     });
 
-/** Dim warnings for route hints that no longer match a page. */
+/**
+ * Dim warnings: route hints that no longer match a page, and the misses of
+ * `severity: warning` questions.
+ */
 export const warningLines = (result: EvalResult, root: string): string[] =>
   result.diagnostics
-    .filter((diagnostic) => diagnostic.code === "BLUME_EVAL_ROUTE_UNKNOWN")
+    .filter((diagnostic) => diagnostic.severity === "warning")
     .map((finding) => {
       const site = finding.file
         ? ` ${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}`
@@ -140,10 +160,11 @@ export const formatEvalReport = (
   }
   lines.push("");
 
-  // The finding tells the author which file fixes which failure.
-  const failures = fixLines(result, root);
-  lines.push(...failures);
-  if (failures.length > 0) {
+  // The finding tells the author which file fixes which failure, warnings
+  // first, as the command prints them.
+  const findings = [...warningLines(result, root), ...fixLines(result, root)];
+  lines.push(...findings);
+  if (findings.length > 0) {
     lines.push("");
   }
 

package/src/eval/run.ts

@@ -40,6 +40,8 @@ export interface QuestionResult {
   question: string;
   routes: string[];
   score?: number;
+  /** The question's severity: a `warning` miss is reported as a warning. */
+  severity?: EvalQuestion["severity"];
   status: QuestionStatus;
 }
 
@@ -99,6 +101,7 @@ const errored = (
   missing: [],
   question: question.question,
   routes: question.routes,
+  severity: question.severity,
   status: "error",
 });
 
@@ -177,6 +180,7 @@ const runQuestion = async (
     question: question.question,
     routes: question.routes,
     score: verdict.score,
+    severity: question.severity,
     status: verdict.pass ? "pass" : "fail",
   };
 };
@@ -250,6 +254,7 @@ export const runEval = async (options: EvalRunOptions): Promise<EvalResult> => {
         missing: [],
         question: question.question,
         routes: question.routes,
+        severity: question.severity,
         status: "skip",
       });
       continue;

package/src/markdown/relative-links.ts

@@ -266,8 +266,9 @@ export const relativeLinksPlugin = (
 
   // A component's string `href` (`<Card href="./install">`) is the same kind
   // of link, and `blume validate` reads it as one. Lowercase elements are
-  // raw HTML in a `.md` page, which neither rewrites nor checks, so only
-  // components are rewritten; an expression-valued `href={…}` is left alone.
+  // raw HTML in a `.md` page, which isn't rewritten, so only components are
+  // rewritten (validate reads a raw `<a href>` as the browser does); an
+  // expression-valued `href={…}` is left alone.
   const rewriteHref = (node: JsxNode, ctx: RelativeLinksContext): void => {
     if (!node.name || !COMPONENT_NAME.test(node.name)) {
       return;

package/src/openapi/checks.ts

@@ -0,0 +1,273 @@
+import type { OperationObject } from "@scalar/openapi-types/3.2";
+
+import type { ApiDocument } from "./model.ts";
+import { HTTP_METHODS } from "./model.ts";
+
+/**
+ * Targeted checks for spec mistakes that would otherwise render silently
+ * wrong. This is not a validator — a linter such as Spectral or oasdiff owns
+ * that — and a spec that fails one of these still renders. Each check names
+ * one mistake whose symptom on the page is quiet: a path placeholder sent
+ * as literal text, a Try it input whose value goes nowhere, an Authorization
+ * section with no scheme to describe. Each reports a warning with its own
+ * code, so a build says what's wrong instead of shipping it.
+ */
+
+/** One mistake a check found, before the source names the spec it's in. */
+export interface SpecIssue {
+  code: string;
+  message: string;
+  suggestion: string;
+}
+
+/** A `{name}` placeholder in a path template. */
+const PATH_TEMPLATE = /\{(?<name>[^{}]+)\}/gu;
+
+/** A reference to a parameter declared under `components.parameters`. */
+const PARAMETER_REF = /^#\/components\/parameters\/(?<name>[^/]+)$/u;
+
+/** How many operations a security-scheme warning names before counting the rest. */
+const NAMED_OPERATIONS = 3;
+
+/** One entry of a document's `paths` or `webhooks` map. */
+type PathItem = NonNullable<ApiDocument["paths"]>[string];
+
+/**
+ * How a warning names an operation: `GET /pets`, or a webhook by its name
+ * (`Webhook "newPet" (POST)`), since a webhook has no path.
+ */
+export const operationLabel = (
+  name: string,
+  method: string,
+  webhook: boolean
+): string =>
+  webhook
+    ? `Webhook "${name}" (${method.toUpperCase()})`
+    : `${method.toUpperCase()} ${name}`;
+
+/** One entry of a `parameters` list: a parameter, or a `$ref` to one. */
+type ParameterEntry = NonNullable<OperationObject["parameters"]>[number];
+
+// The runtime object checks stand guard because the document was parsed from
+// arbitrary YAML/JSON: a spec can put a scalar or null where the type
+// promises an object.
+const isObject = <Value>(value: Value): value is NonNullable<Value> =>
+  typeof value === "object" && value !== null;
+
+/** A parameter's `name`, which a hand-written spec may have left out or mistyped. */
+const isName = <Value>(value: Value): value is Value & string =>
+  typeof value === "string";
+
+/** A declared list, or none when a hand-written spec put something else there. */
+const listOf = <Item>(value: Item[] | undefined): NonNullable<Item>[] =>
+  Array.isArray(value) ? value.filter(isObject) : [];
+
+/** A resolved parameter: one that isn't a `$ref`. */
+type Parameter = Extract<ParameterEntry, { in: string }>;
+
+/**
+ * The parameters an operation declares, its path item's and its own, `$ref`s
+ * resolved through `components.parameters`. One that doesn't resolve is left
+ * out, as the page leaves it out.
+ */
+const parametersOf = (
+  document: ApiDocument,
+  item: PathItem,
+  operation: OperationObject
+): Parameter[] => {
+  const components = document.components?.parameters ?? {};
+  return [
+    ...listOf<ParameterEntry>(item.parameters),
+    ...listOf<ParameterEntry>(operation.parameters),
+  ].flatMap((entry) => {
+    const refName =
+      "$ref" in entry
+        ? PARAMETER_REF.exec(entry.$ref)?.groups?.name
+        : undefined;
+    const parameter = (refName ? components[refName] : undefined) ?? entry;
+    return "in" in parameter ? [parameter] : [];
+  });
+};
+
+/**
+ * A path whose template and declared path parameters disagree. A placeholder
+ * with no parameter gets no input in Try it and no row in the table, so
+ * requests send it as literal text; a parameter with no placeholder gets an
+ * input whose value never reaches the URL.
+ */
+const pathParameterIssues = (
+  signature: string,
+  path: string,
+  declared: Set<string>
+): SpecIssue[] => {
+  const used = new Set(
+    [...path.matchAll(PATH_TEMPLATE)].map((match) => match.groups?.name ?? "")
+  );
+  const issues: SpecIssue[] = [];
+  for (const name of used) {
+    if (!declared.has(name)) {
+      issues.push({
+        code: "BLUME_OPENAPI_PATH_PARAMETER_MISSING",
+        message: `${signature} has {${name}} in its path but declares no "${name}" path parameter, so Try it and the code samples send "{${name}}" as literal text.`,
+        suggestion: `Declare "${name}" in the operation's or the path's \`parameters\`, with \`in: path\` and \`required: true\`.`,
+      });
+    }
+  }
+  for (const name of declared) {
+    if (!used.has(name)) {
+      issues.push({
+        code: "BLUME_OPENAPI_PATH_PARAMETER_UNUSED",
+        message: `${signature} declares a "${name}" path parameter, but its path has no {${name}}, so the value Try it asks for never reaches the URL.`,
+        suggestion: `Add {${name}} to the path, or remove the parameter.`,
+      });
+    }
+  }
+  return issues;
+};
+
+/** The scheme names a `security` list requires. */
+const requiredSchemes = (
+  security: ApiDocument["security"] | OperationObject["security"]
+): string[] =>
+  listOf(security).flatMap((requirement) => Object.keys(requirement));
+
+/**
+ * Security schemes a requirement names that `components.securitySchemes`
+ * doesn't define: the Authorization section has nothing to describe, and
+ * Try it and the code samples send no credential for them. One warning per
+ * scheme, naming where it's required: the root `security`, or the first few
+ * operations that list it.
+ */
+const securitySchemeIssues = (
+  document: ApiDocument,
+  requirers: Map<string, string[]>
+): SpecIssue[] => {
+  const defined = document.components?.securitySchemes ?? {};
+  const root = new Set(requiredSchemes(document.security));
+  const issues: SpecIssue[] = [];
+  for (const scheme of new Set([...root, ...requirers.keys()])) {
+    if (Object.hasOwn(defined, scheme)) {
+      continue;
+    }
+    const operations = requirers.get(scheme) ?? [];
+    const more = operations.length - NAMED_OPERATIONS;
+    const where = root.has(scheme)
+      ? "The spec's root `security` requires"
+      : `${operations.slice(0, NAMED_OPERATIONS).join(", ")}${more > 0 ? ` and ${more} more` : ""} ${operations.length === 1 ? "requires" : "require"}`;
+    issues.push({
+      code: "BLUME_OPENAPI_UNKNOWN_SECURITY_SCHEME",
+      message: `${where} the "${scheme}" security scheme, which \`components.securitySchemes\` doesn't define, so the Authorization section can't describe it and requests send no credential for it.`,
+      suggestion: `Define "${scheme}" in \`components.securitySchemes\`, or fix the name in \`security\`.`,
+    });
+  }
+  return issues;
+};
+
+/**
+ * An operation's `in: querystring` parameter (OpenAPI 3.2: the whole query
+ * string as one value), which the parameter table, Try it, and the code
+ * samples don't handle yet, so it would drop out of all three without a sign.
+ */
+const querystringIssue = (signature: string): SpecIssue => ({
+  code: "BLUME_OPENAPI_UNSUPPORTED",
+  message: `${signature} declares an \`in: querystring\` parameter, which OpenAPI 3.2 added and Blume doesn't render yet, so the page has no row or Try it input for it and the code samples leave the query string out.`,
+  suggestion:
+    "Describe the query string in the operation's `description` until Blume renders `querystring` parameters.",
+});
+
+/**
+ * A path item's `additionalOperations` (OpenAPI 3.2: methods beyond the ones
+ * a path item names directly, like `COPY`), which the extractor doesn't
+ * collect yet, so those operations would be missing without a sign.
+ */
+const additionalOperationsIssue = (
+  label: string,
+  methods: string[]
+): SpecIssue => ({
+  code: "BLUME_OPENAPI_UNSUPPORTED",
+  message: `${label} declares \`additionalOperations\` (${methods.join(", ")}), which OpenAPI 3.2 added and Blume doesn't render yet, so those operations are missing from the reference.`,
+  suggestion:
+    "Document those operations on a page of their own until Blume renders `additionalOperations`.",
+});
+
+/**
+ * A root `x-webhooks` and no `webhooks`. It's the extension OpenAPI 3.0 tools
+ * read for webhooks, and upgrading a 3.0 spec renames it to `webhooks`; a spec
+ * already at 3.1 or later isn't upgraded, so it stays an extension and its
+ * webhooks would be missing without a sign.
+ */
+const xWebhooksIssues = (document: ApiDocument): SpecIssue[] =>
+  "x-webhooks" in document && document.webhooks === undefined
+    ? [
+        {
+          code: "BLUME_OPENAPI_X_WEBHOOKS",
+          message:
+            "The spec declares its webhooks under `x-webhooks`, the extension OpenAPI 3.0 tools read. Blume reads it only from a 3.0 spec, which it upgrades, so these webhooks are missing from the reference.",
+          suggestion:
+            "Rename `x-webhooks` to `webhooks`, the field OpenAPI 3.1 added for them.",
+        },
+      ]
+    : [];
+
+/**
+ * Check a parsed document for the mistakes above, and for the OpenAPI 3.2
+ * features Blume doesn't render yet. `$ref` path items are skipped: the
+ * extractor already reports them as missing from the reference.
+ */
+export const specIssues = (document: ApiDocument): SpecIssue[] => {
+  const issues: SpecIssue[] = [...xWebhooksIssues(document)];
+  // Scheme name -> the operations whose own `security` lists it.
+  const requirers = new Map<string, string[]>();
+  const visit = (name: string, item: PathItem, webhook: boolean): void => {
+    if (!isObject(item) || "$ref" in item) {
+      return;
+    }
+    const additional = isObject(item.additionalOperations)
+      ? Object.keys(item.additionalOperations)
+      : [];
+    if (additional.length > 0) {
+      issues.push(
+        additionalOperationsIssue(
+          webhook ? `Webhook "${name}"` : `Path "${name}"`,
+          additional
+        )
+      );
+    }
+    for (const method of HTTP_METHODS) {
+      const operation = item[method];
+      if (!isObject(operation)) {
+        continue;
+      }
+      const signature = operationLabel(name, method, webhook);
+      for (const scheme of requiredSchemes(operation.security)) {
+        requirers.set(scheme, [...(requirers.get(scheme) ?? []), signature]);
+      }
+      const parameters = parametersOf(document, item, operation);
+      // A webhook is keyed by its name, not a path: it has no template.
+      if (!webhook) {
+        issues.push(
+          ...pathParameterIssues(
+            signature,
+            name,
+            new Set(
+              parameters
+                .filter((parameter) => parameter.in === "path")
+                .map((parameter) => parameter.name)
+                .filter(isName)
+            )
+          )
+        );
+      }
+      if (parameters.some((parameter) => parameter.in === "querystring")) {
+        issues.push(querystringIssue(signature));
+      }
+    }
+  };
+  for (const [path, item] of Object.entries(document.paths ?? {})) {
+    visit(path, item, false);
+  }
+  for (const [name, item] of Object.entries(document.webhooks ?? {})) {
+    visit(name, item, true);
+  }
+  return [...issues, ...securitySchemeIssues(document, requirers)];
+};

package/src/openapi/graphql-build.ts

@@ -26,6 +26,7 @@ import {
   parse,
   print,
   specifiedDirectives,
+  valueFromASTUntyped,
   visit,
 } from "graphql";
 
@@ -34,6 +35,7 @@ import type {
   GraphqlEnumValue,
   GraphqlField as ModelField,
   GraphqlInputValue,
+  GraphqlJsonValue,
   GraphqlTypeDef,
   GraphqlTypeRef,
 } from "./graphql.ts";
@@ -68,6 +70,21 @@ const defaultLiteral = (
   return literal ? print(literal) : undefined;
 };
 
+/**
+ * The same default as the JSON a client sends in `variables`, which the
+ * example variables start from.
+ */
+const defaultJson = (
+  input: GraphQLArgument | GraphQLInputField
+): GraphqlJsonValue | undefined => {
+  const literal = input.default?.literal;
+  // SAFETY: a const literal has no variables, so it lowers to JSON values
+  // only: numbers, strings, booleans, null, enum names, lists and objects.
+  return literal
+    ? (valueFromASTUntyped(literal) as GraphqlJsonValue)
+    : undefined;
+};
+
 const inputValue = (
   input: GraphQLArgument | GraphQLInputField
 ): GraphqlInputValue => {
@@ -80,6 +97,10 @@ const inputValue = (
   if (printed !== undefined) {
     value.default = printed;
   }
+  const json = defaultJson(input);
+  if (json !== undefined) {
+    value.defaultValue = json;
+  }
   // `deprecationReason` is null on non-deprecated members in some builds;
   // absent means "not deprecated" downstream, so both collapse to absent.
   if (

package/src/openapi/graphql.ts

@@ -58,6 +58,15 @@ export interface GraphqlTypeRef {
   name: string;
 }
 
+/** A GraphQL input value as JSON carries it in `variables` (enums by name). */
+export type GraphqlJsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | GraphqlJsonValue[]
+  | { [key: string]: GraphqlJsonValue };
+
 /** An argument or input-object field. */
 export interface GraphqlInputValue {
   name: string;
@@ -65,6 +74,8 @@ export interface GraphqlInputValue {
   type: GraphqlTypeRef;
   /** Default value printed as a GraphQL literal (`10`, `"asc"`, `[1, 2]`). */
   default?: string;
+  /** The same default as JSON, for the example variables. */
+  defaultValue?: GraphqlJsonValue;
   deprecationReason?: string;
 }
 

package/src/openapi/model.ts

@@ -1,5 +1,6 @@
-import type { Document, OperationObject } from "@scalar/openapi-types/3.1";
+import type { Document, OperationObject } from "@scalar/openapi-types/3.2";
 
+import type { GraphqlAuthOptions } from "../reference/options.ts";
 import type {
   AsyncApiAction,
   AsyncApiDocument,
@@ -26,10 +27,18 @@ export { slugify } from "./references.ts";
  * per-operation route so it becomes a first-class Blume page.
  */
 
-/** A normalized OpenAPI 3.1 document, internal `$ref`s intact. */
+/**
+ * A normalized OpenAPI document, internal `$ref`s intact. Typed as 3.2, which
+ * is backward compatible with 3.1, so one type covers both the upgraded 3.1
+ * documents and the 3.2 ones read as written.
+ */
 export type ApiDocument = Document;
 
-/** The HTTP methods an OpenAPI path item may declare, in spec order. */
+/**
+ * The HTTP methods an OpenAPI path item may declare, in spec order. `query`
+ * is OpenAPI 3.2's (the QUERY method: a safe request that carries a body); a
+ * 3.1 path item can't declare it, so listing it changes nothing there.
+ */
 export const HTTP_METHODS = [
   "get",
   "put",
@@ -39,6 +48,7 @@ export const HTTP_METHODS = [
   "head",
   "patch",
   "trace",
+  "query",
 ] as const;
 
 export type HttpMethod = (typeof HTTP_METHODS)[number];
@@ -140,6 +150,8 @@ export interface ApiSpecData {
    * (GraphQL only; OpenAPI documents carry their servers in the document).
    */
   endpoint?: string;
+  /** How the GraphQL endpoint authenticates (GraphQL only). */
+  auth?: GraphqlAuthOptions;
   /** Operations keyed by {@link ApiOperationRef.key}. */
   operations: Record<string, ApiOperationRef>;
   tags: ApiTagRef[];