@mintlify/cli 4.0.1733 → 4.0.1735

package/src/agent-claim.ts

@@ -56,7 +56,6 @@ const signupSchema = z.object({ orgId: z.string().min(1), claimed: z.literal(tru
 
 export async function claimDeployment(
   options: { apiUrl: string; directory: string; code?: string; email?: string },
-  warn: (message: string) => void,
   progress: (message: string) => void = () => {}
 ): Promise<string> {
   const { apiUrl, directory } = options;
@@ -81,15 +80,7 @@ export async function claimDeployment(
   if (new URL(discovery.issuer).origin !== new URL(credentials.issuer).origin) {
     throw new Error('Saved agent credentials belong to another issuer.');
   }
-  const save = async (value: AgentCredentials) => {
-    try {
-      await saveAgentCredentials(apiUrl, directory, value);
-    } catch {
-      warn(
-        'Could not save claimed credentials to the keyring. Continuing for this run; a retry may not recover the completed claim.'
-      );
-    }
-  };
+  const save = (value: AgentCredentials) => saveAgentCredentials(apiUrl, directory, value);
   if (!credentials.claimVerified) {
     progress('Completing the WorkOS claim...');
     const completed = await agentResponse(

package/src/agent-credentials.ts

@@ -1,10 +1,13 @@
 import { createHash } from 'node:crypto';
+import fs from 'node:fs/promises';
 import path from 'node:path';
 import { z } from 'zod';
 
+import { CONFIG_DIR } from './constants.js';
 import { getKeytar } from './keyring.js';
 
 const service = 'mintlify-agent';
+const fallbackFile = () => path.join(CONFIG_DIR, 'agent-credentials.json');
 
 export const agentCredentialsSchema = z.object({
   registrationId: z.string().min(1),
@@ -28,14 +31,59 @@ function registrationAccount(apiUrl: string, registrationId: string): string {
   return `${new URL(apiUrl).origin}:${registrationId}`;
 }
 
+async function readFallback(): Promise<Record<string, string>> {
+  try {
+    return z
+      .record(z.string(), z.string())
+      .parse(JSON.parse(await fs.readFile(fallbackFile(), 'utf-8')));
+  } catch {
+    return {};
+  }
+}
+
+async function writeFallback(entries: Record<string, string>): Promise<void> {
+  await fs.mkdir(CONFIG_DIR, { recursive: true });
+  const temp = `${fallbackFile()}.${process.pid}.tmp`;
+  await fs.writeFile(temp, JSON.stringify(entries, null, 2), { mode: 0o600 });
+  await fs.chmod(temp, 0o600);
+  await fs.rename(temp, fallbackFile());
+}
+
+async function getSecret(account: string): Promise<string | null> {
+  let keyringError: unknown;
+  try {
+    const value = await (await getKeytar()).getPassword(service, account);
+    if (value) return value;
+  } catch (error) {
+    keyringError = error;
+  }
+  const value = (await readFallback())[account];
+  if (value) return value;
+  if (keyringError) throw keyringError;
+  return null;
+}
+
+async function setSecret(account: string, value: string): Promise<void> {
+  try {
+    await (await getKeytar()).setPassword(service, account, value);
+  } catch {
+    await writeFallback({ ...(await readFallback()), [account]: value });
+    await getKeytar()
+      .then((keytar) => keytar.deletePassword(service, account))
+      .catch(() => {});
+    return;
+  }
+  const { [account]: stale, ...rest } = await readFallback();
+  if (stale) await writeFallback(rest).catch(() => {});
+}
+
 export async function loadAgentCredentials(
   apiUrl: string,
   directory: string
 ): Promise<AgentCredentials | null> {
-  const keytar = await getKeytar();
-  const registrationId = await keytar.getPassword(service, projectAccount(apiUrl, directory));
+  const registrationId = await getSecret(projectAccount(apiUrl, directory));
   if (!registrationId) return null;
-  const value = await keytar.getPassword(service, registrationAccount(apiUrl, registrationId));
+  const value = await getSecret(registrationAccount(apiUrl, registrationId));
   if (!value)
     throw new Error('Saved agent credentials are missing. Use --new to create another site.');
   const credentials = agentCredentialsSchema.parse(JSON.parse(value));
@@ -51,11 +99,6 @@ export async function saveAgentCredentials(
   credentials: AgentCredentials
 ): Promise<void> {
   const value = agentCredentialsSchema.parse(credentials);
-  const keytar = await getKeytar();
-  await keytar.setPassword(
-    service,
-    registrationAccount(apiUrl, value.registrationId),
-    JSON.stringify(value)
-  );
-  await keytar.setPassword(service, projectAccount(apiUrl, directory), value.registrationId);
+  await setSecret(registrationAccount(apiUrl, value.registrationId), JSON.stringify(value));
+  await setSecret(projectAccount(apiUrl, directory), value.registrationId);
 }

package/src/cli.tsx

@@ -56,6 +56,7 @@ import { sendPublishPrompt, signup } from './signup.js';
 import { status, getCliSubdomains } from './status.js';
 import { trackTelemetryPreferenceChange } from './telemetry/track.js';
 import { update } from './update.js';
+import { runLintStep, splitList } from './validate/command.js';
 import { renderWelcome } from './welcome.js';
 import { workflowsBuilder } from './workflows/index.js';
 
@@ -142,7 +143,6 @@ export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
           try {
             const message = await claimDeployment(
               { apiUrl: API_URL, directory: argv.directory, code: argv.code, email: argv.email },
-              (message) => process.stderr.write(`Warning: ${message}\n`),
               (message) => process.stderr.write(`${message}\n`)
             );
             process.stdout.write(`${message}\n`);
@@ -379,9 +379,32 @@ export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
               default: false,
               description: 'Disable OpenAPI file generation',
             })
+            .option('files', {
+              type: 'string',
+              array: true,
+              description:
+                'Limit page checks to these files or globs; site-wide checks and the build still cover the whole site',
+            })
+            .option('enable', {
+              type: 'string',
+              array: true,
+              coerce: splitList,
+              description:
+                'Turn on rules or groups (comma-separated), for example after --disable all',
+            })
+            .option('disable', {
+              type: 'string',
+              array: true,
+              coerce: splitList,
+              description:
+                'Turn off rules or groups (comma-separated), or all; parse errors are still reported',
+            })
             .usage('usage: mintlify validate [options]')
             .example('mintlify validate', 'validate the build'),
         async (argv) => {
+          const lintPassed = await runLintStep(argv);
+          if (!lintPassed) return;
+
           if (!argv['disable-openapi']) {
             let openApiFilenames: string[] = [];
 

package/src/deploy.ts

@@ -62,15 +62,8 @@ export async function prepareAgentDeployment(
   if (!subdomain) throw new Error('Choose a site name with --subdomain <name>.');
   const discovery = await discoverAgentAuth(apiUrl);
 
-  const save = async (credentials: AgentCredentials) => {
-    try {
-      await saveAgentCredentials(apiUrl, directory, credentials);
-    } catch {
-      warn(
-        'Could not save agent credentials to the keyring. Continuing for this run; later commands may not recover access.'
-      );
-    }
-  };
+  const save = (credentials: AgentCredentials) =>
+    saveAgentCredentials(apiUrl, directory, credentials);
 
   for (let attempt = 0; attempt < 2; attempt++) {
     const credentials = saved ?? (await registerAgent(discovery));

package/src/resolveFiles.ts

@@ -66,6 +66,8 @@ async function walk(baseDir: string, dir = ''): Promise<string[]> {
   return files;
 }
 
+export class NoMatchingFilesError extends Error {}
+
 export async function resolveExplicitFiles(baseDir: string, args: string[]): Promise<string[]> {
   let allFiles: string[] | null = null;
   const matched: string[] = [];
@@ -90,7 +92,7 @@ export async function resolveExplicitFiles(baseDir: string, args: string[]): Pro
   }
 
   if (unmatched.length > 0) {
-    throw new Error(
+    throw new NoMatchingFilesError(
       `No matching files for: ${unmatched.join(', ')} (globs match .md and .mdx files)`
     );
   }

package/src/validate/command.tsx

@@ -0,0 +1,71 @@
+import { addLog, clearLogs, ErrorLog, SpinnerLog } from '@mintlify/previewing';
+import path from 'node:path';
+
+import { CMD_EXEC_PATH, terminate } from '../helpers.js';
+import { NoMatchingFilesError, resolveExplicitFiles } from '../resolveFiles.js';
+import { checkSite, selectRules, UsageError } from './engine.js';
+import { logDiagnostics } from './output.js';
+
+export type ValidateArgs = {
+  'local-schema': boolean;
+  'client-version'?: string;
+  groups?: (string | number)[];
+  files?: (string | number)[];
+  enable?: string[];
+  disable?: string[];
+};
+
+export const EXIT = { passed: 0, failed: 1, crashed: 2 } as const;
+
+export const splitList = (values: string[]) =>
+  values.flatMap((value) =>
+    value
+      .split(',')
+      .map((item) => item.trim())
+      .filter(Boolean)
+  );
+
+const errorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error));
+
+const resolveSelection = async (baseDir: string, values: (string | number)[] | undefined) => {
+  const args = (values ?? []).map(String).filter(Boolean);
+  if (args.length === 0) return undefined;
+  const files = await resolveExplicitFiles(CMD_EXEC_PATH, args);
+  return new Set(
+    files.map((file) =>
+      path.relative(baseDir, path.resolve(CMD_EXEC_PATH, file)).split(path.sep).join('/')
+    )
+  );
+};
+
+export const runLintStep = async (
+  argv: ValidateArgs,
+  baseDir = CMD_EXEC_PATH
+): Promise<boolean> => {
+  try {
+    const rules = selectRules({
+      enable: argv.enable,
+      disable: argv.disable ?? [],
+    });
+    const selected = await resolveSelection(baseDir, argv.files);
+    addLog(<SpinnerLog message="validating..." />);
+
+    const { diagnostics } = await checkSite(baseDir, rules, {
+      selected,
+      localSchema: argv['local-schema'],
+    });
+    clearLogs();
+    const failed = logDiagnostics(diagnostics);
+    if (failed) {
+      await terminate(EXIT.failed);
+      return false;
+    }
+    return true;
+  } catch (error) {
+    const usage = error instanceof UsageError || error instanceof NoMatchingFilesError;
+    clearLogs();
+    addLog(<ErrorLog message={errorMessage(error)} />);
+    await terminate(usage ? EXIT.failed : EXIT.crashed);
+    return false;
+  }
+};

package/src/validate/engine.ts

@@ -0,0 +1,93 @@
+import { checkFiles } from './files.js';
+import { RULES } from './rules/index.js';
+import { PARSE_ERROR } from './rules/parseError.js';
+import { loadSite } from './site.js';
+import {
+  GROUPS,
+  type Diagnostic,
+  type DiagnosticSource,
+  type FileRule,
+  type Finding,
+  type Rule,
+  type Site,
+} from './types.js';
+
+export class UsageError extends Error {}
+
+const ruleList = () => RULES.map(({ id }) => id).join(', ');
+
+const matchToken = (token: string): Rule[] => {
+  if (token === 'all') return RULES;
+  const rule = RULES.find(({ id }) => id === token);
+  if (rule) return [rule];
+  if ((GROUPS as string[]).includes(token)) {
+    return RULES.filter(({ group }) => group === token);
+  }
+  throw new UsageError(
+    `unknown rule or group "${token}". Groups: all, ${GROUPS.join(', ')}. Rules: ${ruleList()}`
+  );
+};
+
+/** Every rule, minus --disable, plus --enable. parse-error is the engine's own, not selectable. */
+export const selectRules = ({
+  enable = [],
+  disable = [],
+}: {
+  enable?: string[];
+  disable?: string[];
+}): Rule[] => {
+  if ([...disable, ...enable].includes(PARSE_ERROR.id)) {
+    throw new UsageError(`${PARSE_ERROR.id} always runs and can't be disabled`);
+  }
+  const selected = new Set(RULES);
+  for (const token of disable) for (const rule of matchToken(token)) selected.delete(rule);
+  for (const token of enable) for (const rule of matchToken(token)) selected.add(rule);
+  return RULES.filter((rule) => selected.has(rule));
+};
+
+const toDiagnostic = (source: DiagnosticSource, finding: Finding): Diagnostic => ({
+  ...finding,
+  rule: source.id,
+  group: source.group,
+  severity: finding.severity ?? source.severity,
+});
+
+const RULE_ORDER = new Map(RULES.map(({ id }, index) => [id, index]));
+
+const sortDiagnostics = (diagnostics: Diagnostic[]) =>
+  diagnostics.sort(
+    (a, b) =>
+      (a.file < b.file ? -1 : a.file > b.file ? 1 : 0) ||
+      (a.line ?? 0) - (b.line ?? 0) ||
+      (a.column ?? 0) - (b.column ?? 0) ||
+      (RULE_ORDER.get(a.rule) ?? 0) - (RULE_ORDER.get(b.rule) ?? 0)
+  );
+
+/** Runs every selected rule that doesn't need the build. */
+export const runChecks = async (site: Site, rules: Rule[]) => {
+  const fileRules = rules.filter((rule): rule is FileRule => rule.scope === 'file');
+  const diagnostics = await checkFiles(site, fileRules);
+
+  for (const rule of rules) {
+    if (rule.scope !== 'site' || rule.needsBuild) continue;
+    for (const finding of await rule.check(site)) diagnostics.push(toDiagnostic(rule, finding));
+  }
+  return sortDiagnostics(diagnostics);
+};
+
+/** Loads the site and runs the selected rules. */
+export const checkSite = async (
+  baseDir: string,
+  rules: Rule[],
+  options: { selected?: ReadonlySet<string>; localSchema: boolean }
+) => {
+  const site = await loadSite(baseDir, options);
+  const { selected } = options;
+  if (selected && !site.files.some(({ file }) => selected.has(file))) {
+    throw new UsageError(
+      `none of the given files are pages or snippets of this site: ${[...selected].join(', ')}`
+    );
+  }
+  const diagnostics = await runChecks(site, rules);
+  return { site, diagnostics };
+};

package/src/validate/files.ts

@@ -0,0 +1,212 @@
+import { getAST, getFileCategory, parseFrontmatter, replaceVariables } from '@mintlify/common';
+import {
+  categorizeMdxFiles,
+  getErrorPosition,
+  getMintIgnore,
+  listContentFiles,
+} from '@mintlify/prebuild';
+import type { Root } from 'mdast';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { applyEdits, parseContent, relocate, selectEdits, unmapEdit } from './mdx.js';
+import { PARSE_ERROR } from './rules/parseError.js';
+import type {
+  Diagnostic,
+  Edit,
+  FileContext,
+  FileFinding,
+  FileKind,
+  FileRule,
+  Site,
+  SiteFile,
+} from './types.js';
+
+const CONCURRENCY = 16;
+
+const mapLimit = async <T, R>(items: T[], task: (item: T) => Promise<R>): Promise<R[]> => {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(CONCURRENCY, items.length) }, async () => {
+      while (next < items.length) {
+        const index = next++;
+        results[index] = await task(items[index] as T);
+      }
+    })
+  );
+  return results;
+};
+
+const CHECKED_CATEGORIES = new Set<string | null>(['page', 'snippet', 'snippet-v2']);
+
+export const readSiteFiles = async (baseDir: string): Promise<SiteFile[]> => {
+  const { mdxCandidates: candidates } = await listContentFiles(
+    baseDir,
+    await getMintIgnore(baseDir)
+  );
+  const { snippets, snippetsV2 } = await categorizeMdxFiles(baseDir, candidates);
+  const snippetFiles = new Set([...snippets, ...snippetsV2]);
+  return candidates
+    .filter((name) => /\.mdx?$/.test(name) && CHECKED_CATEGORIES.has(getFileCategory(name)))
+    .map((name) => ({
+      file: name.replace(/^\//, ''),
+      kind: (snippetFiles.has(name) ? 'snippet' : 'page') as FileKind,
+    }));
+};
+
+const readFrontmatter = (content: string): Record<string, unknown> => {
+  try {
+    const { attributes } = parseFrontmatter<unknown>(content);
+    return attributes && typeof attributes === 'object' ? { ...attributes } : {};
+  } catch {
+    return {};
+  }
+};
+
+type ParseError = { line?: number; column?: number; message: string };
+
+const REASON_POSITION = /\s*\(\d+:\d+(?:-\d+:\d+)?\)$/;
+
+export const toParseError = (error: unknown): ParseError => {
+  const reason = (error as { reason?: unknown } | null)?.reason;
+  const fallback = error instanceof Error ? error.message : 'could not parse this file';
+  const text = typeof reason === 'string' ? reason : fallback;
+  const position = error && typeof error === 'object' ? getErrorPosition(error) : null;
+  return {
+    ...(position?.line != null && { line: position.line }),
+    ...(position?.column != null && { column: position.column }),
+    message: text.replace(REASON_POSITION, ''),
+  };
+};
+
+const parseErrorDiagnostic = (file: string, error: ParseError): Diagnostic => ({
+  ...error,
+  rule: PARSE_ERROR.id,
+  group: PARSE_ERROR.group,
+  severity: PARSE_ERROR.severity,
+  file,
+  message: `${error.message} (other checks were skipped for this file)`,
+});
+
+type Parsed = { context: FileContext; approximate: boolean } | { error: ParseError };
+
+const tryParseContent = (content: string): Root | undefined => {
+  try {
+    return parseContent(content);
+  } catch {
+    return undefined;
+  }
+};
+
+const parseFile = (file: string, content: string, baseDir: string): Parsed => {
+  const context = { file, content, baseDir, frontmatter: readFrontmatter(content) };
+  const tree = tryParseContent(content);
+  if (tree) return { context: { ...context, tree }, approximate: false };
+  try {
+    return { context: { ...context, tree: getAST(content, file) }, approximate: true };
+  } catch (error) {
+    return { error: toParseError(error) };
+  }
+};
+
+const EMPTY_TREE = { type: 'root' as const, children: [] };
+
+const SOURCE_STAGE = 1;
+const AFTER_RAW_STAGE = 2;
+
+type Stage = { content: string; edits: Edit[] };
+
+const VARIABLE = /\{\{\s*([\w.\-]+)\s*\}\}/g;
+
+const variableEdits = (content: string, variables: Record<string, string> = {}): Edit[] =>
+  [...content.matchAll(VARIABLE)]
+    .filter((match) => match[1] !== undefined && variables[match[1]] !== undefined)
+    .map((match) => ({
+      start: match.index,
+      end: match.index + match[0].length,
+      text: replaceVariables(match[0], variables),
+    }));
+
+const toDiagnostic = (
+  rule: FileRule,
+  file: string,
+  { edit, ...finding }: FileFinding,
+  stages: Stage[],
+  stage: number
+): Diagnostic => {
+  const original = edit && unmapEdit(stages, stage, edit);
+  return {
+    ...finding,
+    ...relocate(stages, stage, finding),
+    rule: rule.id,
+    group: rule.group,
+    severity: rule.severity,
+    file,
+    ...(original && { fix: { file, ...original } }),
+  };
+};
+
+const checkFile = (
+  allRules: FileRule[],
+  { file, kind }: SiteFile,
+  content: string,
+  baseDir: string,
+  variables?: Record<string, string>
+): Diagnostic[] => {
+  const rules = kind === 'snippet' ? allRules.filter((rule) => !rule.pagesOnly) : allRules;
+  const raw = rules.filter((rule) => rule.raw);
+  const treeRules = rules.filter((rule) => !rule.raw);
+  const varEdits = kind === 'snippet' ? variableEdits(content, variables) : [];
+  const source = applyEdits(content, varEdits);
+  const rawContext = { file, content: source, baseDir, frontmatter: {}, tree: EMPTY_TREE };
+
+  const rawFound = raw.flatMap((rule) =>
+    rule.check(rawContext).map((finding) => ({ rule, finding }))
+  );
+  const rawEdits = selectEdits(
+    rawFound.flatMap(({ finding }) => (finding.edit ? [finding.edit] : []))
+  );
+  const afterRaw = applyEdits(source, rawEdits);
+  const stages: Stage[] = [
+    { content, edits: varEdits },
+    { content: source, edits: rawEdits },
+    { content: afterRaw, edits: [] },
+  ];
+  const diagnostics = rawFound.map(({ rule, finding }) =>
+    toDiagnostic(rule, file, finding, stages, SOURCE_STAGE)
+  );
+
+  const parsed = parseFile(file, afterRaw, baseDir);
+  if ('error' in parsed) {
+    diagnostics.unshift(
+      parseErrorDiagnostic(file, {
+        ...parsed.error,
+        ...relocate(stages, AFTER_RAW_STAGE, parsed.error),
+      })
+    );
+    return diagnostics;
+  }
+  for (const rule of treeRules) {
+    for (const finding of rule.check(parsed.context)) {
+      const found = parsed.approximate ? { ...finding, edit: undefined } : finding;
+      diagnostics.push(toDiagnostic(rule, file, found, stages, AFTER_RAW_STAGE));
+    }
+  }
+  return diagnostics;
+};
+
+export const checkFiles = async (site: Site, rules: FileRule[]): Promise<Diagnostic[]> => {
+  const targets = site.files.filter(({ file }) => !site.selected || site.selected.has(file));
+  const results = await mapLimit(targets, async (entry) => {
+    let content: string;
+    try {
+      content = await fs.readFile(path.join(site.baseDir, entry.file), 'utf-8');
+    } catch (error) {
+      const message = error instanceof Error ? error.message : 'could not read this file';
+      return [parseErrorDiagnostic(entry.file, { message })];
+    }
+    return checkFile(rules, entry, content, site.baseDir, site.variables);
+  });
+  return results.flat();
+};