@mintlify/cli 4.0.1733 → 4.0.1735

package/src/validate/mdx.ts

@@ -0,0 +1,252 @@
+import {
+  coreRemark,
+  FRONTMATTER_RE,
+  getCodeRanges,
+  HEADING_ID_RE,
+  JSXParser,
+} from '@mintlify/common';
+import type { Root } from 'mdast';
+
+import type { Edit } from './types.js';
+
+const HEADING_ID = new RegExp(HEADING_ID_RE.source, 'gm');
+const ESM_START = /^(?:import|export)(?=[\s{*])/;
+const JSX_TAG = /<[A-Za-z][\w.:-]*(?:"[^"]*"|'[^']*'|\{[^{}]*\}|[^<>"'{}])*>/g;
+const ATTR_VALUE = /"[^"]*"|'[^']*'|\{[^{}]*\}/g;
+
+export const hasCustomHeadingId = (line: string) => /\{\s*#[^}]+?\s*\}\s*$/.test(line);
+
+export const parseContent = (content: string): Root => {
+  const padded = content.replace(
+    HEADING_ID,
+    (match, indent: string, hashes: string, text: string) => {
+      const end = match.indexOf(text, indent.length + hashes.length) + text.length;
+      return match.slice(0, end).padEnd(match.length, ' ');
+    }
+  );
+  return coreRemark().parse(padded);
+};
+
+type Range = [number, number];
+
+const blankRanges = (content: string, ranges: Range[]) => {
+  let result = '';
+  let cursor = 0;
+  for (const [from, to] of [...ranges].sort((a, b) => a[0] - b[0])) {
+    const start = Math.max(from, cursor);
+    if (to <= start) continue;
+    result += content.slice(cursor, start) + content.slice(start, to).replace(/[^\n]/g, ' ');
+    cursor = to;
+  }
+  return result + content.slice(cursor);
+};
+
+const maskWith = (content: string, maskers: ((content: string) => Range[])[]) =>
+  maskers.reduce((text, ranges) => blankRanges(text, ranges(text)), content);
+
+const isCompleteEsm = (text: string) => {
+  try {
+    JSXParser.parse(text, { ecmaVersion: 'latest', sourceType: 'module' });
+    return true;
+  } catch (error) {
+    const pos = (error as { pos?: unknown }).pos;
+    return typeof pos !== 'number' || pos < text.trimEnd().length;
+  }
+};
+
+const frontmatterRanges = (content: string): Range[] => {
+  const match = FRONTMATTER_RE.exec(content);
+  return match ? [[match.index, match.index + match[0].length]] : [];
+};
+
+const esmRanges = (content: string, codeRanges: Range[]): Range[] => {
+  let esm: string[] | undefined;
+  let previousBlank = true;
+  const ranges: Range[] = [];
+  let offset = 0;
+  for (const line of content.split('\n')) {
+    const isBlank = line.trim() === '';
+    const wasBlank = previousBlank;
+    previousBlank = isBlank;
+    if (esm) {
+      if (isBlank && isCompleteEsm(esm.join('\n'))) esm = undefined;
+      else {
+        esm.push(line);
+        ranges.push([offset, offset + line.length]);
+      }
+    } else if (
+      wasBlank &&
+      ESM_START.test(line) &&
+      !codeRanges.some(([start, end]) => start <= offset && offset < end)
+    ) {
+      esm = [line];
+      ranges.push([offset, offset + line.length]);
+    }
+    offset += line.length + 1;
+  }
+  return ranges;
+};
+
+export const maskFrontmatter = (content: string): string =>
+  blankRanges(content, frontmatterRanges(content));
+
+export const maskCode = (content: string, { keepEsm = false } = {}): string => {
+  const codeRanges = getCodeRanges(content);
+  return blankRanges(content, [
+    ...frontmatterRanges(content),
+    ...codeRanges,
+    ...(keepEsm ? [] : esmRanges(content, codeRanges)),
+  ]);
+};
+
+const OPENS = new Set(['{', '${']);
+
+const maskExpression = (content: string, start: number): { ranges: Range[]; end: number } => {
+  const ranges: Range[] = [];
+  let depth = 0;
+  try {
+    const tokens = JSXParser.tokenizer(content.slice(start), {
+      ecmaVersion: 'latest',
+      onComment: (block, _text, from, to) =>
+        ranges.push([start + from + 2, start + to - (block ? 2 : 0)]),
+    });
+    for (const token of tokens) {
+      const label = token.type.label;
+      if (label === 'template') ranges.push([start + token.start, start + token.end]);
+      else if (label === 'string') ranges.push([start + token.start + 1, start + token.end - 1]);
+      else if (label === 'regexp') {
+        const raw = content.slice(start + token.start, start + token.end);
+        ranges.push([start + token.start + 1, start + token.start + raw.lastIndexOf('/')]);
+      } else if (OPENS.has(label)) depth++;
+      else if (label === '}' && depth-- === 0) return { ranges, end: start + token.end };
+    }
+  } catch (error) {
+    if (!(error instanceof SyntaxError && 'pos' in error && typeof error.pos === 'number')) {
+      return { ranges, end: content.length };
+    }
+    const pos = start + error.pos;
+    if (!error.message.startsWith('Unterminated string')) return { ranges, end: pos };
+    const lineEnd = content.indexOf('\n', pos);
+    ranges.push([pos + 1, lineEnd === -1 ? content.length : lineEnd]);
+  }
+  return { ranges, end: content.length };
+};
+
+const expressionRanges = (content: string): Range[] => {
+  const ranges: Range[] = [];
+  for (let index = 0; index < content.length; index++) {
+    const char = content[index];
+    if (char === '\\') index++;
+    else if (char === '{') {
+      const expression = maskExpression(content, index + 1);
+      ranges.push(...expression.ranges);
+      index = expression.end - 1;
+    }
+  }
+  return ranges;
+};
+
+const attributeRanges = (content: string): Range[] =>
+  [...content.matchAll(JSX_TAG)].flatMap(({ index, 0: tag }) =>
+    [...tag.matchAll(ATTR_VALUE)].map(
+      ({ index: at, 0: value }): Range => [index + at + 1, index + at + value.length - 1]
+    )
+  );
+
+const escapedTagRanges = (content: string): Range[] =>
+  [...content.matchAll(/\\[\s\S]/g)].flatMap(({ index, 0: escape }): Range[] =>
+    escape === '\\<' ? [[index + 1, index + 2]] : []
+  );
+
+export const maskNonMarkup = (content: string): string =>
+  maskWith(maskCode(content), [attributeRanges, escapedTagRanges, expressionRanges]);
+
+export const selectEdits = <T extends Edit>(edits: T[]): T[] => {
+  const ordered = edits
+    .map((edit, index) => ({ edit, index }))
+    .sort((a, b) => b.edit.start - a.edit.start || b.edit.end - a.edit.end || b.index - a.index);
+  const selected: T[] = [];
+  let floor = Infinity;
+  for (const { edit } of ordered) {
+    if (edit.end > floor) continue;
+    selected.push(edit);
+    floor = edit.start;
+  }
+  return selected;
+};
+
+export const applyEdits = (content: string, edits: Edit[]): string =>
+  selectEdits(edits).reduce(
+    (result, edit) => result.slice(0, edit.start) + edit.text + result.slice(edit.end),
+    content
+  );
+
+const toOffset = (content: string, line: number, column = 1) => {
+  let offset = 0;
+  for (let current = 1; current < line; current++) {
+    const next = content.indexOf('\n', offset);
+    if (next === -1) break;
+    offset = next + 1;
+  }
+  return offset + column - 1;
+};
+
+export const toPosition = (content: string, offset: number) => {
+  const before = content.slice(0, offset).split('\n');
+  return { line: before.length, column: (before.pop()?.length ?? 0) + 1 };
+};
+
+const byStart = (a: Edit, b: Edit) => a.start - b.start;
+
+/** Maps an offset in edited content back to the content before the edits, or undefined inside an edit. */
+function toOriginal(edits: Edit[], offset: number, inside: 'snap'): number;
+function toOriginal(edits: Edit[], offset: number, inside: 'reject'): number | undefined;
+function toOriginal(edits: Edit[], offset: number, inside: 'snap' | 'reject') {
+  let delta = 0;
+  for (const edit of edits) {
+    const start = edit.start + delta;
+    if (inside === 'snap' ? offset < start : offset <= start) break;
+    if (offset < start + edit.text.length) return inside === 'snap' ? edit.start : undefined;
+    delta += edit.text.length - (edit.end - edit.start);
+  }
+  return offset - delta;
+}
+
+/** Maps an edit made against a later stage back to the original content, or undefined when it touches an earlier fix. */
+export const unmapEdit = (
+  stages: { content: string; edits: Edit[] }[],
+  index: number,
+  edit: Edit
+): Edit | undefined => {
+  let { start, end } = edit;
+  for (const { edits } of stages.slice(0, index).reverse()) {
+    const sorted = [...edits].sort(byStart);
+    const mappedStart = toOriginal(sorted, start, 'reject');
+    const mappedEnd = toOriginal(sorted, end, 'reject');
+    if (mappedStart === undefined || mappedEnd === undefined) return undefined;
+    if (sorted.some((earlier) => earlier.start < mappedEnd && earlier.end > mappedStart)) {
+      return undefined;
+    }
+    start = mappedStart;
+    end = mappedEnd;
+  }
+  return { start, end, text: edit.text };
+};
+
+export const relocate = (
+  stages: { content: string; edits: Edit[] }[],
+  index: number,
+  position: { line?: number; column?: number }
+) => {
+  const stage = stages[index];
+  const original = stages[0];
+  if (!stage || !original || position.line === undefined) return {};
+  const offset = stages
+    .slice(0, index)
+    .reduceRight(
+      (current, { edits }) => toOriginal([...edits].sort(byStart), current, 'snap'),
+      toOffset(stage.content, position.line, position.column)
+    );
+  const mapped = toPosition(original.content, offset);
+  return position.column === undefined ? { line: mapped.line } : mapped;
+};

package/src/validate/output.tsx

@@ -0,0 +1,56 @@
+import { addLog, ErrorLog } from '@mintlify/previewing';
+import { Box, Text } from 'ink';
+
+import type { Diagnostic } from './types.js';
+
+const location = ({ line, column }: Pick<Diagnostic, 'line' | 'column'>) =>
+  line ? `${line}${column ? `:${column}` : ''}` : '';
+
+const DiagnosticLine = ({ diagnostic }: { diagnostic: Diagnostic }) => {
+  const at = location(diagnostic);
+  return (
+    <Text>
+      {'  '}
+      {diagnostic.severity === 'error' ? (
+        <Text color="red">✗</Text>
+      ) : (
+        <Text color="yellow">!</Text>
+      )}{' '}
+      {at && <Text dimColor>{at} </Text>}
+      <Text dimColor>{diagnostic.rule}</Text> {diagnostic.message}
+      {diagnostic.fix && <Text dimColor> (fixable)</Text>}
+    </Text>
+  );
+};
+
+const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? '' : 's'}`;
+
+export const logDiagnostics = (all: Diagnostic[]) => {
+  const byFile = new Map<string, Diagnostic[]>();
+  for (const diagnostic of all) {
+    byFile.get(diagnostic.file)?.push(diagnostic) ?? byFile.set(diagnostic.file, [diagnostic]);
+  }
+  for (const [file, diagnostics] of byFile) {
+    addLog(
+      <Box flexDirection="column">
+        <Text bold>{file}</Text>
+        {diagnostics.map((diagnostic, index) => (
+          <DiagnosticLine key={index} diagnostic={diagnostic} />
+        ))}
+      </Box>
+    );
+  }
+
+  const errors = all.filter(({ severity }) => severity === 'error').length;
+  const warnings = all.length - errors;
+  if (errors > 0) {
+    addLog(
+      <ErrorLog
+        message={`${plural(errors, 'error')}${warnings > 0 ? `, ${plural(warnings, 'warning')}` : ''}`}
+      />
+    );
+  } else if (warnings > 0) {
+    addLog(<Text dimColor>{plural(warnings, 'warning')}</Text>);
+  }
+  return errors > 0;
+};

package/src/validate/rules/content/voidTags.ts

@@ -0,0 +1,35 @@
+import { maskNonMarkup } from '../../mdx.js';
+import type { FileRule, FileFinding } from '../../types.js';
+
+const VOID_TAG = /<(br|hr|img)(?=[\s/>])((?:"[^"]*"|'[^']*'|\{[^{}]*\}|[^<>"'{}])*?)(\/?)>/g;
+
+export const voidTags: FileRule = {
+  id: 'void-tags',
+  group: 'content',
+  severity: 'error',
+  scope: 'file',
+  description: "self-close void HTML tags (<br>, <hr>, <img>); MDX can't parse them unclosed",
+  raw: true,
+  check: ({ content }) => {
+    const masked = maskNonMarkup(content);
+    const issues: FileFinding[] = [];
+    for (const match of masked.matchAll(VOID_TAG)) {
+      if (match[3]) continue;
+      const start = match.index;
+      const end = start + match[0].length;
+      if (new RegExp(`^\\s*</${match[1]}\\s*>`).test(masked.slice(end))) continue;
+      const before = content.slice(0, start).split('\n');
+      issues.push({
+        line: before.length,
+        column: (before.pop()?.length ?? 0) + 1,
+        message: `<${match[1]}> must be self-closed`,
+        edit: {
+          start,
+          end,
+          text: `${content.slice(start, end - 1).trimEnd()} />`,
+        },
+      });
+    }
+    return issues;
+  },
+};

package/src/validate/rules/index.ts

@@ -0,0 +1,5 @@
+import type { Rule } from '../types.js';
+import { voidTags } from './content/voidTags.js';
+
+/** Every check mint validate runs, in report order. */
+export const RULES: Rule[] = [voidTags];

package/src/validate/rules/parseError.ts

@@ -0,0 +1,9 @@
+import type { DiagnosticSource } from '../types.js';
+
+/** The engine's built-in diagnostic for files neither parser can parse; not a selectable rule. */
+export const PARSE_ERROR: DiagnosticSource = {
+  id: 'parse-error',
+  group: 'content',
+  severity: 'error',
+  description: "files MDX can't parse; other checks skip them",
+};

package/src/validate/site.ts

@@ -0,0 +1,52 @@
+import { existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+
+import { readSiteFiles } from './files.js';
+import { toPosition } from './mdx.js';
+import type { Site } from './types.js';
+
+const readVariables = (config: string): Record<string, string> | undefined => {
+  try {
+    const { variables } = JSON.parse(config) as { variables?: unknown };
+    return variables && typeof variables === 'object'
+      ? (variables as Record<string, string>)
+      : undefined;
+  } catch {
+    return undefined;
+  }
+};
+
+/** Lists the site's pages and snippets; locate reads files from disk on demand. */
+export const loadSite = async (
+  baseDir: string,
+  { selected, localSchema }: { selected?: ReadonlySet<string>; localSchema: boolean }
+): Promise<Site> => {
+  const files = await readSiteFiles(baseDir);
+  const byPath = new Map<string, string>();
+  const configFile = existsSync(path.join(baseDir, 'docs.json')) ? 'docs.json' : 'mint.json';
+
+  const read = (file: string) => {
+    if (!byPath.has(file)) {
+      try {
+        byPath.set(file, readFileSync(path.join(baseDir, file), 'utf-8'));
+      } catch {
+        byPath.set(file, '');
+      }
+    }
+    return byPath.get(file) ?? '';
+  };
+
+  return {
+    baseDir,
+    files,
+    selected,
+    configFile,
+    variables: readVariables(read(configFile)),
+    localSchema,
+    locate: (file, text) => {
+      const content = read(file);
+      const offset = text ? content.indexOf(text) : -1;
+      return offset === -1 ? {} : toPosition(content, offset);
+    },
+  };
+};

package/src/validate/types.ts

@@ -0,0 +1,88 @@
+import type { BuildValidationResult } from '@mintlify/previewing';
+import type { Root } from 'mdast';
+
+export type Group = 'content' | 'nav' | 'config' | 'openapi';
+
+export const GROUPS: Group[] = ['content', 'nav', 'config', 'openapi'];
+
+export type Severity = 'error' | 'warning';
+
+export type Edit = { start: number; end: number; text: string };
+
+export type FileEdit = Edit & { file: string };
+
+export type Diagnostic = {
+  rule: string;
+  group: Group;
+  severity: Severity;
+  file: string;
+  line?: number;
+  column?: number;
+  message: string;
+  fix?: FileEdit;
+};
+
+/** What a rule reports; the engine adds its id and group, and its severity unless overridden. */
+export type Finding = Omit<Diagnostic, 'rule' | 'group' | 'severity'> & { severity?: Severity };
+
+type RuleMeta = {
+  id: string;
+  group: Group;
+  severity: Severity;
+  needsBuild?: boolean;
+  description: string;
+};
+
+export type DiagnosticSource = Pick<RuleMeta, 'id' | 'group' | 'severity' | 'description'>;
+
+export type FileKind = 'page' | 'snippet';
+
+export type FileContext = {
+  file: string;
+  content: string;
+  baseDir: string;
+  frontmatter: Record<string, unknown>;
+  tree: Root;
+};
+
+/** A finding in one file, with an optional edit against the content the rule saw. */
+export type FileFinding = {
+  line?: number;
+  column?: number;
+  message: string;
+  edit?: Edit;
+};
+
+/** Checks one page or snippet at a time. Raw rules read text; the rest also get the MDX tree. */
+export type FileRule = RuleMeta & {
+  scope: 'file';
+  raw?: boolean;
+  pagesOnly?: boolean;
+  needsAnchorLinks?: boolean;
+  check: (context: FileContext) => FileFinding[];
+};
+
+export type SiteFile = { file: string; kind: FileKind };
+
+export type BuildResult = BuildValidationResult;
+
+export type Site = {
+  baseDir: string;
+  files: SiteFile[];
+  /** Set with --files: rules report on these files only. */
+  selected?: ReadonlySet<string>;
+  configFile: string;
+  variables?: Record<string, string>;
+  build?: BuildResult;
+  /** The line and column of the first occurrence of text in a file. */
+  locate: (file: string, text: string) => { line?: number; column?: number };
+  localSchema: boolean;
+};
+
+/** Checks the site as a whole. */
+export type SiteRule = RuleMeta & {
+  scope: 'site';
+  check: (site: Site) => Finding[] | Promise<Finding[]>;
+};
+
+export type Rule = FileRule | SiteRule;