failproofai 0.0.14-beta.0 → 0.0.14-beta.2

package/scripts/translate-docs/types.ts

@@ -43,4 +43,10 @@ export interface TranslationResult {
   inputTokens: number;
   outputTokens: number;
   cached: boolean;
+  /**
+   * Total translation attempts spent (1 = valid on first try). Optional so the
+   * cached / dry-run result literals need no change; absent means "not
+   * translated this run" (cached) or "1".
+   */
+  attempts?: number;
 }

package/scripts/translate-docs/validate-translation.ts

@@ -0,0 +1,132 @@
+/**
+ * Generation-time validation for a freshly translated page.
+ *
+ * `findTranslationError` runs the exact checks the Mintlify deploy performs —
+ * frontmatter YAML then MDX body — plus one the deploy cannot: frontmatter
+ * KEY-PARITY against the English source. It is called on the exact bytes about
+ * to be written to disk (post-sanitize, post-link-rewrite), so a page that
+ * passes here is a page `mintlify validate` and the deploy accept.
+ *
+ * Why a translation-specific wrapper instead of `findPageError` alone:
+ *  - Frontmatter YAML (class A): the model re-emits an inner `"` unescaped into
+ *    a double-quoted `title:`/`description:` value, breaking the YAML. This is
+ *    the class that failed run 29575781632; `findFrontmatterError` catches it.
+ *  - Key parity (class B): the model drops the frontmatter block entirely, or
+ *    renames a key. A dropped block is still *valid YAML* (mintlify tolerates
+ *    it, deriving the title from the slug), so only comparing against the
+ *    source's keys catches it. This is deliberately stricter than mintlify —
+ *    the keys are prompt-forbidden to change, so it fails ~never but converts
+ *    a silent content regression into a retry.
+ *  - MDX body (classes C+): stray `<slug>`, `{#anchor}`, unmatched fence,
+ *    swallowed `</div>` — `findMdxParseError` catches these.
+ *
+ * The message returned is written for a MODEL to act on: it names the failing
+ * construct and, for a body error, quotes the offending line, so the retry has
+ * a concrete defect to fix rather than a bare line number the RTL/README
+ * wrappers would have offset anyway.
+ */
+import { findFrontmatterError, findMdxParseError } from "../validate-mdx";
+import YAML from "yaml";
+
+// Same matcher as validate-mdx.ts; duplicated locally so this module owns its
+// frontmatter extraction and does not depend on an internal export.
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
+
+/**
+ * The sorted top-level keys of a page's frontmatter, or `null` when the page
+ * has no frontmatter block or its block is not a key/value mapping (an
+ * unparseable or scalar block). Callers distinguish "no block" from a YAML
+ * error separately via `findFrontmatterError`.
+ */
+function frontmatterKeys(page: string): string[] | null {
+  const match = FRONTMATTER_RE.exec(page);
+  if (!match) return null;
+  try {
+    const doc = YAML.parse(match[1]);
+    if (doc && typeof doc === "object" && !Array.isArray(doc)) {
+      return Object.keys(doc).sort();
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * A ±2-line window around `line` (1-based), the failing line prefixed `> ` and
+ * its neighbours `  `. Empty string when `line` is undefined.
+ */
+function excerpt(page: string, line?: number): string {
+  if (!line) return "";
+  const lines = page.split("\n");
+  const start = Math.max(1, line - 2);
+  const end = Math.min(lines.length, line + 2);
+  const out: string[] = [];
+  for (let n = start; n <= end; n++) {
+    out.push(`${n === line ? "> " : "  "}${lines[n - 1]}`);
+  }
+  return out.join("\n");
+}
+
+const backticked = (keys: string[]): string =>
+  keys.map((k) => `\`${k}\``).join(", ");
+
+/**
+ * Validate a rendered translation against its English `source`. Returns a
+ * model-actionable error message, or `null` when the page is publishable.
+ */
+export async function findTranslationError(
+  rendered: string,
+  source: string,
+): Promise<string | null> {
+  // Validate the rendered frontmatter block FIRST, for every source shape —
+  // even when the source had none. findMdxParseError (below) blanks a leading
+  // `---` block before compiling, so a malformed block the model *added* to a
+  // frontmatter-less page would otherwise be invisible here yet still break the
+  // Mintlify deploy. Closing exactly that blind spot is this module's job, so
+  // it must hold regardless of whether the source has frontmatter.
+  const fm = findFrontmatterError(rendered);
+  if (fm) {
+    return (
+      "The YAML frontmatter (the `---` block at the top of the file) does " +
+      `not parse:\n\n${fm.message}`
+    );
+  }
+
+  const sourceKeys = frontmatterKeys(source);
+  if (sourceKeys) {
+    // The source has frontmatter, so the translation must carry the same keys.
+    // (Its block, if present, already parsed cleanly above.)
+    const keys = frontmatterKeys(rendered);
+    if (keys === null) {
+      return (
+        "The YAML frontmatter is missing. The English source starts with a " +
+        `\`---\` block containing ${backticked(sourceKeys)}; the translation ` +
+        "must start with the same block — translate the values, keep the keys."
+      );
+    }
+    // Subset check BY DESIGN: a missing or renamed key is a content regression
+    // (the page loses its title/description), so reject it. An *extra* key is
+    // harmless — Mintlify ignores unknown frontmatter keys — so tolerate it
+    // rather than burn a retry (and risk exhausting the whole batch) on a page
+    // that would deploy fine. Reject content loss; allow harmless additions.
+    const missing = sourceKeys.filter((k) => !keys.includes(k));
+    if (missing.length > 0) {
+      return (
+        `The YAML frontmatter keys changed. Expected ${backticked(sourceKeys)} ` +
+        `but got ${backticked(keys)}. Translate the values, never rename or ` +
+        "drop a key."
+      );
+    }
+  }
+
+  const body = await findMdxParseError(rendered);
+  if (body) {
+    const snippet = excerpt(rendered, body.line);
+    return `The MDX body does not parse: ${body.message}${
+      snippet ? `\n\n${snippet}` : ""
+    }`;
+  }
+
+  return null;
+}

package/scripts/validate-mdx.ts

@@ -1,13 +1,13 @@
 /**
- * Validate that every docs MDX page parses with the same MDX engine Mintlify
- * runs at deploy time.
+ * Validate that every docs page (`.mdx` and `.md` — Mintlify parses both as
+ * MDX) parses with the same MDX engine Mintlify runs at deploy time.
  *
- * Why this exists: `mintlify validate` (the existing `docs` CI job) only checks
- * `docs.json` structure and nav-link resolution — it does NOT parse page
- * content. A page that is structurally referenced but contains an MDX syntax
- * error (e.g. a `<slug>` that escaped its surrounding backticks because a
- * translation dropped a closing `` ` ``) passes `mintlify validate` but fails
- * the Mintlify deploy with:
+ * Why this exists: `mintlify validate` (the existing `docs` CI job) checks
+ * `docs.json` structure, nav-link resolution, and frontmatter YAML — but it
+ * does NOT report MDX *body* syntax errors. A page whose frontmatter is valid
+ * but whose body contains an MDX syntax error (e.g. a `<slug>` that escaped its
+ * surrounding backticks because a translation dropped a closing `` ` ``) passes
+ * `mintlify validate` but fails the Mintlify deploy with:
  *
  *   Failed to parse page content at path tr/cli/audit.mdx:
  *   Expected a closing tag for `<slug>` (61:127-61:133) before the end of `paragraph`
@@ -21,12 +21,16 @@
  *
  * The error string above is emitted by `@mdx-js/mdx`'s micromark MDX layer,
  * which is the same engine Mintlify uses, so compiling here reproduces the
- * deploy-time parse faithfully.
+ * deploy-time parse faithfully. `main()` validates the frontmatter too (via
+ * `findPageError`), so this net is a strict superset of `mintlify validate`:
+ * it catches both the frontmatter YAML class that fails `mintlify validate`
+ * and the body-MDX class that `mintlify validate` lets through to deploy.
  */
 import { readdirSync, statSync, readFileSync } from "node:fs";
 import { dirname, join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
 import { compile } from "@mdx-js/mdx";
+import YAML from "yaml";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const DOCS_DIR = join(__dirname, "..", "docs");
@@ -37,21 +41,86 @@ export interface MdxParseError {
   column?: number;
 }
 
+/**
+ * Shared frontmatter matcher. The capture group is the YAML block *body* (the
+ * text between the fences); `match[0]` is still the whole block including the
+ * fences and trailing newline, which is what `stripFrontmatter` blanks.
+ */
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
+
 /**
  * Replace a leading YAML frontmatter block (`--- … ---`) with blank lines.
  *
  * Mintlify parses frontmatter as YAML, not MDX, so it never causes an MDX parse
- * error. We blank it rather than delete it so the remaining content keeps its
- * original line numbers — error positions then match the real file.
+ * error — but it CAN contain a YAML syntax error, which blanking here hides
+ * from `findMdxParseError`. That gap is covered separately by
+ * `findFrontmatterError`; `findPageError` runs both. We blank rather than delete
+ * so the remaining body keeps its original line numbers — body error positions
+ * then match the real file.
  */
 export function stripFrontmatter(source: string): string {
-  const match = /^---\r?\n[\s\S]*?\r?\n---[ \t]*\r?\n?/.exec(source);
+  const match = FRONTMATTER_RE.exec(source);
   if (!match) return source;
   // Keep newlines, drop every other character, so line numbers stay aligned.
   const blanked = match[0].replace(/[^\n]/g, "");
   return blanked + source.slice(match[0].length);
 }
 
+/**
+ * Parse the leading YAML frontmatter block and return its parse error, or
+ * `null` when the block is absent (legal — some pages and every i18n README
+ * have none) or valid.
+ *
+ * This is the half of page validation that `findMdxParseError` structurally
+ * cannot do: it compiles only the *body* (frontmatter blanked), so a YAML
+ * syntax error in `title:`/`description:` — the exact class that failed the
+ * `consolidate` job's `mintlify validate` step, and that this repo's own
+ * `validate:mdx` net could not see — sailed straight through. `mintlify`
+ * parses the frontmatter as YAML, so parsing it here reproduces that check.
+ *
+ * The reported `line` is FILE-relative (the opening `---` is file line 1), so
+ * it matches the convention `findMdxParseError` already uses for body errors.
+ * That is deliberately one greater than the block-relative number `mintlify`
+ * prints — do NOT "fix" it to match mintlify; file-relative is what points a
+ * reader (or a model asked to repair the page) at the right line.
+ */
+export function findFrontmatterError(source: string): MdxParseError | null {
+  const match = FRONTMATTER_RE.exec(source);
+  if (!match) return null;
+  try {
+    YAML.parse(match[1]);
+    return null;
+  } catch (err) {
+    const e = err as {
+      message?: string;
+      linePos?: Array<{ line: number; col: number }>;
+    };
+    const raw = e.message ?? String(err);
+    return {
+      // Drop yaml's block-relative "at line N, column N:" phrase (it would
+      // contradict the file-relative line we report) but keep the caret-
+      // underlined excerpt after it — the single most useful signal for a
+      // model asked to repair the offending line.
+      message: raw.replace(/ at line \d+, column \d+:/, ":"),
+      line: e.linePos?.[0] ? e.linePos[0].line + 1 : undefined,
+      column: e.linePos?.[0]?.col,
+    };
+  }
+}
+
+/**
+ * Validate a full page the way the Mintlify deploy does: frontmatter YAML
+ * first, then the MDX body. Returns the first error found, or `null`. This is
+ * the single entry point shared by `validate:mdx` (`main()` below) and the
+ * translation pipeline's generation-time gate, so the two can never disagree
+ * about what is publishable.
+ */
+export async function findPageError(
+  source: string,
+): Promise<MdxParseError | null> {
+  return findFrontmatterError(source) ?? (await findMdxParseError(source));
+}
+
 /**
  * Compile one MDX source string with the deploy-time parser. Returns `null`
  * when it parses cleanly, or the parse error (with position) otherwise.
@@ -91,12 +160,20 @@ export function encodeAnnotation(value: string): string {
     .replace(/\n/g, "%0A");
 }
 
-function collectMdxFiles(dir: string): string[] {
+/**
+ * Collect every Mintlify content page under `dir`. Mintlify runs BOTH `.mdx`
+ * and `.md` files through the same MDX pipeline at deploy time, so a `.md` page
+ * with an MDX syntax error (e.g. an HTML `<!-- -->` comment, which MDX rejects)
+ * fails the deploy exactly like a broken `.mdx` would. The docs/i18n README
+ * translations are `.md`, so restricting this walk to `.mdx` let their breakage
+ * sail past this safety net and reach the post-merge deploy — collect both.
+ */
+export function collectMdxFiles(dir: string): string[] {
   const out: string[] = [];
   for (const entry of readdirSync(dir)) {
     const full = join(dir, entry);
     if (statSync(full).isDirectory()) out.push(...collectMdxFiles(full));
-    else if (entry.endsWith(".mdx")) out.push(full);
+    else if (entry.endsWith(".mdx") || entry.endsWith(".md")) out.push(full);
   }
   return out;
 }
@@ -106,7 +183,7 @@ async function main(): Promise<void> {
   const failures: Array<{ file: string; error: MdxParseError }> = [];
 
   for (const file of files) {
-    const error = await findMdxParseError(readFileSync(file, "utf-8"));
+    const error = await findPageError(readFileSync(file, "utf-8"));
     if (error) failures.push({ file: relative(process.cwd(), file), error });
   }
 

package/src/audit/cli.ts

@@ -22,6 +22,7 @@ import { trackHookEvent } from "../hooks/hook-telemetry";
 import { getInstanceId } from "../../lib/telemetry-id";
 import { sanitizeErrorMessage } from "../../lib/telemetry-sanitize";
 import { openWhenReady } from "./open-browser";
+import { brandAnsi, ANSI_RESET, ANSI_BOLD, ANSI_DIM } from "../hooks/tui";
 
 /** Port the bundled dashboard binds to. Matches `scripts/launch.ts`'s default
  *  for `start` mode, which `failproofai` (bare) already uses. */
@@ -59,12 +60,17 @@ WHAT IT DOES
 `.trimStart();
 
 // ── ANSI helpers ────────────────────────────────────────────────────────────
-const RESET = "\x1b[0m";
-const DIM = "\x1b[2m";
-const BOLD = "\x1b[1m";
-const PINK = "\x1b[38;5;204m";
-const GREEN = "\x1b[38;5;120m";
-const CYAN = "\x1b[38;5;81m";
+// Colours come from the shared brand palette in hooks/tui.ts, so `audit` reads
+// as the same product as `config`. This file used to define its own 256-colour
+// set (a green and a blue that appear nowhere in the brand). The palette has
+// two accents rather than three, so success and command/URL text share the
+// teal — the glyph (✓) and the surrounding copy already distinguish them.
+const RESET = ANSI_RESET;
+const DIM = ANSI_DIM;
+const BOLD = ANSI_BOLD;
+const PINK = brandAnsi("pink");
+const GREEN = brandAnsi("guide"); // success — brand teal
+const CYAN = brandAnsi("guide"); // commands & URLs — brand teal
 const SPINNER = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
 
 function colorOn(): boolean {
@@ -237,6 +243,26 @@ function printSummary(result: AuditResult): void {
   for (const line of buildSummary(result)) process.stdout.write(`  ${line}\n`);
 }
 
+// ── Audit telemetry ──────────────────────────────────────────────────────────
+
+/**
+ * Which entry point ran the audit. `onboarding` is the automatic post-setup run;
+ * `cli` is an explicit `failproofai audit`. Carried on every cli_audit_* event so
+ * the first audit a user ever runs is distinguishable from later deliberate ones.
+ */
+type AuditSource = "cli" | "onboarding";
+
+/** Shared so both entry points report cli_audit_completed identically. */
+function auditCompletedProps(source: AuditSource, result: AuditResult) {
+  return {
+    source,
+    events_scanned: result.eventsScanned,
+    sessions_scanned: result.transcripts.scanned,
+    total_hits: result.totals.hits,
+    findings: result.results.length,
+  };
+}
+
 // ── Post-setup background audit ────────────────────────────────────────────────
 
 /**
@@ -252,7 +278,13 @@ function printSummary(result: AuditResult): void {
  */
 export async function runPostSetupAudit(): Promise<void> {
   if (process.env.FAILPROOFAI_NO_AUTO_AUDIT === "1") return;
-  
+
+  const instanceId = getInstanceId();
+  // Fire-and-forget, as in runAuditCli: the multi-second scan below keeps the
+  // process alive long enough for this to land, and the completed/failed event
+  // that follows is awaited.
+  void trackHookEvent(instanceId, "cli_audit_started", { source: "onboarding" });
+
   process.stdout.write(
     `\n  ${c(PINK, "✦")} ${c(BOLD, "failproofai audit now running")}  ${c(DIM, "· ctrl+c to stop")}\n\n`,
   );
@@ -260,13 +292,25 @@ export async function runPostSetupAudit(): Promise<void> {
   let result: AuditResult;
   try {
     result = await runWithProgress({});
-  } catch {
+  } catch (err) {
+    // Awaited: this function returns straight into the dashboard boot, and a
+    // fire-and-forget fetch would race it.
+    await trackHookEvent(instanceId, "cli_audit_failed", {
+      source: "onboarding",
+      error_type: err instanceof Error ? err.name : "unknown",
+      error_message: sanitizeErrorMessage(err),
+    });
     process.stdout.write(
       `  ${c(PINK, "!")} ${c(DIM, "audit couldn't finish — run")} ${c(CYAN, "failproofai audit")} ${c(DIM, "later.")}\n\n`,
     );
     return;
   }
 
+  // Reported before the empty-history return below, so an onboarding audit that
+  // finds nothing is still counted — matching runAuditCli, which reports
+  // completed regardless of what the scan turned up.
+  await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("onboarding", result));
+
   if (result.eventsScanned === 0) {
     process.stdout.write(
       `\n  ${c(DIM, "no agent sessions to audit yet — come back after using your agent.")}\n\n`,
@@ -328,13 +372,7 @@ export async function runAuditCli(args: string[]): Promise<void> {
   // would otherwise drop this event. On the dashboard path launch() keeps the
   // process alive, but awaiting makes delivery reliable on every exit path.
   // Bounded (5s) and never throws.
-  await trackHookEvent(instanceId, "cli_audit_completed", {
-    source: "cli",
-    events_scanned: result.eventsScanned,
-    sessions_scanned: result.transcripts.scanned,
-    total_hits: result.totals.hits,
-    findings: result.results.length,
-  });
+  await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("cli", result));
 
   // No sessions on disk — guide the user instead of opening an empty dashboard.
   if (result.eventsScanned === 0) {

package/src/auth/cli.ts

@@ -28,6 +28,7 @@ import {
   writeAuth,
 } from "../../lib/auth/auth-store";
 import { CliError } from "../cli-error";
+import { brandAnsi, colorsEnabled, ANSI_RESET, ANSI_DIM } from "../hooks/tui";
 import { trackHookEvent } from "../hooks/hook-telemetry";
 import { getInstanceId } from "../../lib/telemetry-id";
 
@@ -123,11 +124,19 @@ function prompt(question: string, opts: { hidden?: boolean } = {}): Promise<stri
   });
 }
 
-const DIM = "\x1b[2m";
-const RESET = "\x1b[0m";
-const PINK = "\x1b[38;5;204m";
-const GREEN = "\x1b[38;5;120m";
-const RED = "\x1b[38;5;197m";
+// Brand palette (see hooks/tui.ts) rather than this file's former private
+// 256-colour set, so `auth` matches `config` and `audit`. Resolved once here
+// and blanked when colour is off — these are interpolated directly into
+// template literals at ~15 call sites, which previously meant `auth` emitted
+// escape codes even under NO_COLOR. Blanking the constants fixes every call
+// site at once. The palette has no red, and both uses are recoverable input
+// errors ("try again") rather than failures, so they take the amber warn hue.
+const COLOR = colorsEnabled(process.stdout);
+const DIM = COLOR ? ANSI_DIM : "";
+const RESET = COLOR ? ANSI_RESET : "";
+const PINK = COLOR ? brandAnsi("pink") : "";
+const GREEN = COLOR ? brandAnsi("guide") : "";
+const RED = COLOR ? brandAnsi("warn") : "";
 
 function emailLooksValid(email: string): boolean {
   return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email);