flecto 3.0.2 → 4.0.0

package/src/pr-comment.js

@@ -62,6 +62,35 @@ function truncate(text, max) {
   return text.length > max ? `${text.slice(0, max)}…` : text;
 }
 
+/**
+ * The `flecto explain` narration (#143), rendered so it cannot pass for Flecto's
+ * own output and cannot do anything but be read.
+ *
+ * The model saw pull-request-authored text, so its output is untrusted. It goes
+ * inside a fenced block, fence wider than any backtick run in it, which is the
+ * one markdown construct where links, images, `@`-mentions, and HTML all render
+ * as literal text.
+ * @param {{ text: string, provider: string, model: string, cached?: boolean, truncated?: boolean }} narration
+ * @returns {string[]}
+ */
+function narrationSection(narration) {
+  const text = String(narration.text);
+  const runs = text.match(/`+/g) ?? [];
+  const fence = '`'.repeat(Math.max(3, ...runs.map((run) => run.length + 1)));
+  return [
+    '### Model-generated narration (advisory)',
+    '',
+    `<sub>Written by ${inlineCode(`${narration.provider} ${narration.model}`)}`
+    + `${narration.cached ? ' (cached)' : ''} from the masked semantic diff. Not computed by Flecto,`
+    + ' not a policy finding, and never part of this check\'s result.</sub>',
+    '',
+    `${fence}text`,
+    text,
+    fence,
+    ...(narration.truncated ? ['', '_Cut off at the output token limit._'] : []),
+  ];
+}
+
 /**
  * Format a change value for display, or return null when the side is absent.
  * @param {unknown} value
@@ -119,7 +148,8 @@ function findingsOf(result) {
  * Pure: it reads nothing but its arguments, so the exact body posted to GitHub
  * is the body printed to stdout.
  * @param {CiResult[]} results
- * @param {{ cwd?: string, failed?: boolean, marker?: string, maxInlineChanges?: number, maxBodyChars?: number }} [options]
+ * @param {{ cwd?: string, failed?: boolean, marker?: string, maxInlineChanges?: number, maxBodyChars?: number,
+ *   narration?: { text: string, provider: string, model: string, cached?: boolean, truncated?: boolean } | null }} [options]
  * @returns {string} Markdown, always beginning with the sticky marker
  */
 export function renderPrComment(results, options = {}) {
@@ -195,6 +225,8 @@ export function renderPrComment(results, options = {}) {
     }
   }
 
+  if (options.narration) lines.push('', ...narrationSection(options.narration));
+
   if (totalChanges > 0) {
     const collapse = totalChanges > maxInlineChanges;
     lines.push('', '### Changes');
@@ -296,6 +328,33 @@ async function errorDetail(response, token) {
   return safe ? `: ${safe}` : '';
 }
 
+/**
+ * @param {Response} response
+ * @returns {boolean}
+ */
+function isRedirect(response) {
+  // `redirect: 'manual'` surfaces the 3xx itself in Node; a browser-shaped
+  // opaque redirect reports status 0 with `type: 'opaqueredirect'`.
+  return (response.status >= 300 && response.status < 400) || response.type === 'opaqueredirect';
+}
+
+/**
+ * The origin a redirect points at, for the error message. Only the origin: the
+ * rest of a `Location` is not worth echoing into a log.
+ * @param {Response} response
+ * @param {string} url
+ * @returns {string}
+ */
+function redirectTarget(response, url) {
+  const location = response.headers?.get?.('location');
+  if (!location) return '';
+  try {
+    return ` to ${new URL(location, url).origin}`;
+  } catch {
+    return '';
+  }
+}
+
 /**
  * @param {{ fetchImpl: typeof fetch, provider: object, url: string, method: string, token: string, body?: unknown, timeoutMs: number }} request
  * @returns {Promise<Response>}
@@ -315,6 +374,13 @@ async function apiRequest({ fetchImpl, provider, url, method, token, body, timeo
       },
       body: body === undefined ? undefined : JSON.stringify(body),
       signal: controller.signal,
+      // Never follow a redirect with a credential attached. `fetch` strips
+      // `Authorization` when a redirect crosses origins, but it strips *only*
+      // that header — GitLab authenticates with `PRIVATE-TOKEN`, which is
+      // forwarded to wherever the API host points, verified against a local
+      // server. Refusing the redirect is the whole fix: these endpoints do not
+      // legitimately redirect, and one that does is worth seeing.
+      redirect: 'manual',
     });
   } catch (err) {
     const reason = err?.name === 'AbortError'
@@ -325,6 +391,14 @@ async function apiRequest({ fetchImpl, provider, url, method, token, body, timeo
     clearTimeout(timer);
   }
 
+  if (isRedirect(response)) {
+    throw new Error(
+      `${provider.label} API ${method} was redirected (HTTP ${response.status}`
+      + `${redirectTarget(response, url)}), and Flecto does not follow a redirect with an API`
+      + ' token attached. Point the API URL at the host that answers directly.',
+    );
+  }
+
   if (!response.ok) {
     throw new Error(
       `${provider.label} API ${method} returned HTTP ${response.status}${await errorDetail(response, token)}`,

package/src/pr-providers.js

@@ -229,8 +229,15 @@ const bitbucket = {
   readOne: (payload) => ({ url: payload?.links?.html?.href }),
 };
 
+/**
+ * Comments collection for the pull request. The workspace and slug are encoded
+ * for the same reason GitLab's project id is: they are interpolated into a URL
+ * path, and a value carrying `/`, `?`, or `#` would otherwise restructure the
+ * request rather than name a repository.
+ */
 function bitbucketCommentsBase(c) {
-  return `${c.apiUrl}/repositories/${c.workspace}/${c.repoSlug}/pullrequests/${c.prNumber}/comments`;
+  return `${c.apiUrl}/repositories/${encodeURIComponent(c.workspace)}`
+    + `/${encodeURIComponent(c.repoSlug)}/pullrequests/${c.prNumber}/comments`;
 }
 
 /** Detection order. GitHub stays first so its behavior is unchanged. */

package/src/regex-engine.js

@@ -0,0 +1,138 @@
+import { RE2JS } from 're2js';
+
+/**
+ * Regular-expression compilation, split by who wrote the pattern.
+ *
+ * Policy packs accept user-supplied regexes (`match.path`, `afterMatches`,
+ * `afterAnyMatches`), and on an untrusted pull request the pack file is
+ * attacker-controlled: a committed `policies/evil.json` plus a `.flectorc`
+ * selecting it is all it takes. JavaScript's own engine backtracks, so
+ * `^(a+)+$` against a 45-character string is not slow but effectively
+ * non-terminating -- measured here at **97 seconds** where RE2 answers in 3ms.
+ * A CI job that never finishes is a denial of service against the merge gate
+ * itself, and no timeout inside the process helps, because the backtracking
+ * happens inside a single uninterruptible call into the engine.
+ *
+ * So patterns are compiled by provenance:
+ *
+ * - **Trusted** -- the packs Flecto ships in `src/packs/`. These are reviewed,
+ *   change only in a release, and are not reachable by a pull request. They
+ *   keep the native engine, which costs nothing and leaves their existing
+ *   syntax (including the negative lookahead in `github-actions.json`) working.
+ * - **Untrusted** -- anything loaded from a repository's `policies/` directory
+ *   or added by `flecto policies add`. These compile with RE2, whose matching
+ *   is linear in the length of the input by construction.
+ *
+ * The split is provenance, not content: a local pack that *overrides* a
+ * built-in id is still local, and still untrusted.
+ *
+ * RE2 deliberately omits lookaround and backreferences, because neither is a
+ * regular operation and both are what make backtracking unbounded. A pack
+ * using them now fails to *load*, with a message naming the rule, rather than
+ * hanging at match time. That is the breaking part of this change, and it is
+ * why it ships in a major version.
+ */
+
+/**
+ * A compiled pattern, with the only operation the policy engine performs.
+ * @typedef {{ test: (value: string) => boolean, source: string, engine: 'native' | 're2' }} CompiledPattern
+ */
+
+/**
+ * Translate JavaScript regex flags into RE2 flags.
+ *
+ * `g` and `y` are accepted and dropped: both only mean anything to a stateful
+ * `lastIndex`, which `test()`-style matching does not use, and RE2's matcher
+ * searches the whole input anyway. `u` and `v` are accepted and dropped
+ * because RE2 is Unicode-aware natively. Anything else is refused rather than
+ * silently ignored, so a pack asking for behaviour it will not get finds out.
+ * @param {string} flags
+ * @returns {number}
+ */
+function re2Flags(flags) {
+  let out = 0;
+  for (const flag of flags) {
+    if (flag === 'i') out |= RE2JS.CASE_INSENSITIVE;
+    else if (flag === 'm') out |= RE2JS.MULTILINE;
+    else if (flag === 's') out |= RE2JS.DOTALL;
+    else if (flag === 'g' || flag === 'y' || flag === 'u' || flag === 'v') continue;
+    else throw new Error(`unsupported regular expression flag "${flag}"`);
+  }
+  return out;
+}
+
+/**
+ * Compile a pattern written by whoever controls the pack file.
+ *
+ * @param {string} pattern
+ * @param {string} [flags]
+ * @param {{ trusted?: boolean }} [options] `trusted` only for packs Flecto ships
+ * @returns {CompiledPattern}
+ * @throws {Error} when the pattern does not compile on the chosen engine
+ */
+export function compilePattern(pattern, flags = '', options = {}) {
+  if (options.trusted) {
+    const re = new RegExp(pattern, flags);
+    // A `g`/`y` pattern carries a mutable lastIndex, and .test() advances it,
+    // so the same regex reused across values would skip matches. Reset per
+    // call rather than per pack: packs are cached and shared between files.
+    return {
+      source: pattern,
+      engine: 'native',
+      test: (value) => {
+        re.lastIndex = 0;
+        return re.test(value);
+      },
+    };
+  }
+  const compiled = RE2JS.compile(pattern, re2Flags(flags));
+  return {
+    source: pattern,
+    engine: 're2',
+    test: (value) => compiled.matcher(value).find(),
+  };
+}
+
+/**
+ * Whether a pattern compiles, without throwing.
+ *
+ * Used by pack validation so a bad pattern is reported with its rule location
+ * rather than as a bare engine error.
+ * @param {string} pattern
+ * @param {string} [flags]
+ * @param {{ trusted?: boolean }} [options]
+ * @returns {{ ok: true } | { ok: false, reason: string }}
+ */
+export function checkPattern(pattern, flags = '', options = {}) {
+  try {
+    compilePattern(pattern, flags, options);
+    return { ok: true };
+  } catch (error) {
+    return { ok: false, reason: error?.message ?? String(error) };
+  }
+}
+
+/**
+ * The advice appended when an untrusted pack uses syntax RE2 does not support.
+ *
+ * Worth being specific: "invalid regular expression" sends someone hunting for
+ * a typo in a pattern that is perfectly valid JavaScript.
+ * @param {string} reason
+ * @returns {string}
+ */
+export function explainPatternFailure(reason) {
+  // Order matters: an escape failure also contains "invalid escape sequence",
+  // and telling someone to remove lookahead from a pattern that has none is
+  // worse than saying nothing.
+  if (/invalid escape sequence: `\\[ucC]/.test(reason)) {
+    return `${reason}. RE2 spells a unicode escape \`\\x{41}\`, not \`\\u0041\`, and has no`
+      + ' control-character escape. Policy packs outside src/packs/ are matched with RE2.';
+  }
+  if (/Perl syntax|invalid escape sequence|lookbehind|invalid named capture/i.test(reason)) {
+    return `${reason}. Policy packs outside src/packs/ are matched with RE2, which does not`
+      + ' support lookahead, lookbehind, or backreferences -- they are what make backtracking'
+      + ' unbounded, and a pack is attacker-controlled on an untrusted pull request. Rewrite the'
+      + ' pattern without them.';
+  }
+  return reason;
+}

package/src/renderer.js

@@ -1,20 +1,8 @@
 import chalk from 'chalk';
-import { redactSecretString } from './secrets.js';
+import { looksLikeSecretPath, redactSecretString } from './secrets.js';
 import { ENCRYPTED_DISPLAY, displayEncrypted, isEncryptedSentinel } from './encrypted.js';
 import { secretMatchPath } from './differ.js';
 
-/**
- * Key names that mean "this value is a credential".
- *
- * Matched against the *configuration* path only. A multi-document file prefixes
- * every path with the document's identity — `Deployment/prod/token-service.…` —
- * and that identity is a resource name the user chose, not a key name. Letting
- * it match here masked every value in the document, numbers and booleans
- * included, so the path reaching this regex is always the one
- * {@link secretMatchPath} produced.
- */
-const SECRET_PATH_RE = /(secret|token|password|api[_-]?key|private[_-]?key|credential)/i;
-
 /**
  * Format a scalar value for display. Strings get quoted; others are JSON-stringified.
  * @param {unknown} v
@@ -28,7 +16,7 @@ function fmt(v, opts = {}) {
   if (isEncryptedSentinel(v)) return chalk.dim(ENCRYPTED_DISPLAY);
   let value = displayEncrypted(v);
   if (opts.maskSecrets) {
-    if (opts.path && SECRET_PATH_RE.test(opts.path)) {
+    if (opts.path && looksLikeSecretPath(opts.path)) {
       return chalk.dim('"***"');
     }
     // The changed path itself can look benign while the value carries secrets,
@@ -202,7 +190,7 @@ export function renderPolicyFindings(findings) {
  * @returns {unknown}
  */
 export function maskSensitiveValue(value, path = '') {
-  if (SECRET_PATH_RE.test(path)) return '***';
+  if (looksLikeSecretPath(path)) return '***';
   if (Array.isArray(value)) {
     return value.map((v, i) => maskSensitiveValue(v, `${path}[${i}]`));
   }
@@ -223,6 +211,30 @@ export function maskSensitiveValue(value, path = '') {
   return value;
 }
 
+/**
+ * Redact secret-shaped text from policy messages. A rule using
+ * `messageTemplate` can interpolate `{before}` / `{after}`, so a finding can
+ * carry a credential even when the change events beside it are masked. Replace
+ * exact interpolated values using the same path-aware masking as change events,
+ * then catch any other recognizable secret fragments in free-form messages.
+ * @param {import('./policy.js').PolicyFinding[]} findings
+ * @param {import('./differ.js').ChangeEvent[]} changes the unmasked events
+ * @returns {import('./policy.js').PolicyFinding[]}
+ */
+export function maskFindings(findings, changes) {
+  return findings.map((finding) => {
+    let message = String(finding.message ?? '');
+    for (const change of changes.filter((event) => event.path === finding.path)) {
+      for (const value of [change.before, change.after]) {
+        const original = String(value);
+        const masked = String(maskSensitiveValue(value, secretMatchPath(change)));
+        if (original && original !== masked) message = message.replaceAll(original, masked);
+      }
+    }
+    return { ...finding, message: redactSecretString(message) };
+  });
+}
+
 /**
  * @param {import('./differ.js').ChangeEvent} event
  * @returns {import('./differ.js').ChangeEvent}

package/src/report.js

@@ -31,7 +31,8 @@ import { isAbsolute, relative } from 'path';
  *   cwd?: string,
  *   version?: string,
  *   limit?: number,
- *   maskSecrets?: boolean
+ *   maskSecrets?: boolean,
+ *   store?: string
  * }} ReportData
  */
 
@@ -657,7 +658,8 @@ function footer(data) {
   const limit = Number.isInteger(data.limit)
     ? ` Limited to the ${escapeHtml(plural(data.limit, 'most recent snapshot'))}.`
     : '';
-  return '<footer>Generated by Flecto from <code>.flecto-snapshots/</code>.'
+  const store = escapeHtml(String(data.store ?? '.flecto-snapshots/'));
+  return `<footer>Generated by Flecto from <code>${store}</code>.`
     + ' This file is self-contained: no external scripts, fonts, or images, and nothing'
     + ` is sent anywhere when you open it.${limit}</footer>`;
 }

package/src/secrets.js

@@ -330,6 +330,33 @@ export function redactSecretString(value) {
   return out + value.slice(cursor);
 }
 
+/**
+ * Key names that mean "this value is a credential".
+ *
+ * Matched against the *configuration* path only. A multi-document file prefixes
+ * every path with the document's identity — `Deployment/prod/token-service.…` —
+ * and that identity is a resource name the user chose, not a key name. Letting
+ * it match masks every value in the document, numbers and booleans included, so
+ * callers strip the document prefix before matching: see `secretMatchPath` in
+ * differ.js, and the document handling in `maskState`.
+ */
+export const SECRET_PATH_RE = /(secret|token|password|api[_-]?key|private[_-]?key|credential)/i;
+
+/**
+ * True when a configuration path names a credential.
+ *
+ * Lives here rather than beside either caller because there are two of them —
+ * the renderer masking a diff for display, and the snapshot store masking a
+ * state for a commit — and a store that recognized fewer key names than the
+ * terminal did would write into git history exactly the values the terminal
+ * thought were too sensitive to print.
+ * @param {string} path
+ * @returns {boolean}
+ */
+export function looksLikeSecretPath(path) {
+  return SECRET_PATH_RE.test(path);
+}
+
 /**
  * True when a value — or any string nested inside a plain object or array —
  * looks like a secret.