flecto 3.1.0 → 4.0.0

package/src/pr-comment.js

@@ -62,6 +62,35 @@ function truncate(text, max) {
   return text.length > max ? `${text.slice(0, max)}…` : text;
 }
 
+/**
+ * The `flecto explain` narration (#143), rendered so it cannot pass for Flecto's
+ * own output and cannot do anything but be read.
+ *
+ * The model saw pull-request-authored text, so its output is untrusted. It goes
+ * inside a fenced block, fence wider than any backtick run in it, which is the
+ * one markdown construct where links, images, `@`-mentions, and HTML all render
+ * as literal text.
+ * @param {{ text: string, provider: string, model: string, cached?: boolean, truncated?: boolean }} narration
+ * @returns {string[]}
+ */
+function narrationSection(narration) {
+  const text = String(narration.text);
+  const runs = text.match(/`+/g) ?? [];
+  const fence = '`'.repeat(Math.max(3, ...runs.map((run) => run.length + 1)));
+  return [
+    '### Model-generated narration (advisory)',
+    '',
+    `<sub>Written by ${inlineCode(`${narration.provider} ${narration.model}`)}`
+    + `${narration.cached ? ' (cached)' : ''} from the masked semantic diff. Not computed by Flecto,`
+    + ' not a policy finding, and never part of this check\'s result.</sub>',
+    '',
+    `${fence}text`,
+    text,
+    fence,
+    ...(narration.truncated ? ['', '_Cut off at the output token limit._'] : []),
+  ];
+}
+
 /**
  * Format a change value for display, or return null when the side is absent.
  * @param {unknown} value
@@ -119,7 +148,8 @@ function findingsOf(result) {
  * Pure: it reads nothing but its arguments, so the exact body posted to GitHub
  * is the body printed to stdout.
  * @param {CiResult[]} results
- * @param {{ cwd?: string, failed?: boolean, marker?: string, maxInlineChanges?: number, maxBodyChars?: number }} [options]
+ * @param {{ cwd?: string, failed?: boolean, marker?: string, maxInlineChanges?: number, maxBodyChars?: number,
+ *   narration?: { text: string, provider: string, model: string, cached?: boolean, truncated?: boolean } | null }} [options]
  * @returns {string} Markdown, always beginning with the sticky marker
  */
 export function renderPrComment(results, options = {}) {
@@ -195,6 +225,8 @@ export function renderPrComment(results, options = {}) {
     }
   }
 
+  if (options.narration) lines.push('', ...narrationSection(options.narration));
+
   if (totalChanges > 0) {
     const collapse = totalChanges > maxInlineChanges;
     lines.push('', '### Changes');

package/src/regex-engine.js

@@ -0,0 +1,138 @@
+import { RE2JS } from 're2js';
+
+/**
+ * Regular-expression compilation, split by who wrote the pattern.
+ *
+ * Policy packs accept user-supplied regexes (`match.path`, `afterMatches`,
+ * `afterAnyMatches`), and on an untrusted pull request the pack file is
+ * attacker-controlled: a committed `policies/evil.json` plus a `.flectorc`
+ * selecting it is all it takes. JavaScript's own engine backtracks, so
+ * `^(a+)+$` against a 45-character string is not slow but effectively
+ * non-terminating -- measured here at **97 seconds** where RE2 answers in 3ms.
+ * A CI job that never finishes is a denial of service against the merge gate
+ * itself, and no timeout inside the process helps, because the backtracking
+ * happens inside a single uninterruptible call into the engine.
+ *
+ * So patterns are compiled by provenance:
+ *
+ * - **Trusted** -- the packs Flecto ships in `src/packs/`. These are reviewed,
+ *   change only in a release, and are not reachable by a pull request. They
+ *   keep the native engine, which costs nothing and leaves their existing
+ *   syntax (including the negative lookahead in `github-actions.json`) working.
+ * - **Untrusted** -- anything loaded from a repository's `policies/` directory
+ *   or added by `flecto policies add`. These compile with RE2, whose matching
+ *   is linear in the length of the input by construction.
+ *
+ * The split is provenance, not content: a local pack that *overrides* a
+ * built-in id is still local, and still untrusted.
+ *
+ * RE2 deliberately omits lookaround and backreferences, because neither is a
+ * regular operation and both are what make backtracking unbounded. A pack
+ * using them now fails to *load*, with a message naming the rule, rather than
+ * hanging at match time. That is the breaking part of this change, and it is
+ * why it ships in a major version.
+ */
+
+/**
+ * A compiled pattern, with the only operation the policy engine performs.
+ * @typedef {{ test: (value: string) => boolean, source: string, engine: 'native' | 're2' }} CompiledPattern
+ */
+
+/**
+ * Translate JavaScript regex flags into RE2 flags.
+ *
+ * `g` and `y` are accepted and dropped: both only mean anything to a stateful
+ * `lastIndex`, which `test()`-style matching does not use, and RE2's matcher
+ * searches the whole input anyway. `u` and `v` are accepted and dropped
+ * because RE2 is Unicode-aware natively. Anything else is refused rather than
+ * silently ignored, so a pack asking for behaviour it will not get finds out.
+ * @param {string} flags
+ * @returns {number}
+ */
+function re2Flags(flags) {
+  let out = 0;
+  for (const flag of flags) {
+    if (flag === 'i') out |= RE2JS.CASE_INSENSITIVE;
+    else if (flag === 'm') out |= RE2JS.MULTILINE;
+    else if (flag === 's') out |= RE2JS.DOTALL;
+    else if (flag === 'g' || flag === 'y' || flag === 'u' || flag === 'v') continue;
+    else throw new Error(`unsupported regular expression flag "${flag}"`);
+  }
+  return out;
+}
+
+/**
+ * Compile a pattern written by whoever controls the pack file.
+ *
+ * @param {string} pattern
+ * @param {string} [flags]
+ * @param {{ trusted?: boolean }} [options] `trusted` only for packs Flecto ships
+ * @returns {CompiledPattern}
+ * @throws {Error} when the pattern does not compile on the chosen engine
+ */
+export function compilePattern(pattern, flags = '', options = {}) {
+  if (options.trusted) {
+    const re = new RegExp(pattern, flags);
+    // A `g`/`y` pattern carries a mutable lastIndex, and .test() advances it,
+    // so the same regex reused across values would skip matches. Reset per
+    // call rather than per pack: packs are cached and shared between files.
+    return {
+      source: pattern,
+      engine: 'native',
+      test: (value) => {
+        re.lastIndex = 0;
+        return re.test(value);
+      },
+    };
+  }
+  const compiled = RE2JS.compile(pattern, re2Flags(flags));
+  return {
+    source: pattern,
+    engine: 're2',
+    test: (value) => compiled.matcher(value).find(),
+  };
+}
+
+/**
+ * Whether a pattern compiles, without throwing.
+ *
+ * Used by pack validation so a bad pattern is reported with its rule location
+ * rather than as a bare engine error.
+ * @param {string} pattern
+ * @param {string} [flags]
+ * @param {{ trusted?: boolean }} [options]
+ * @returns {{ ok: true } | { ok: false, reason: string }}
+ */
+export function checkPattern(pattern, flags = '', options = {}) {
+  try {
+    compilePattern(pattern, flags, options);
+    return { ok: true };
+  } catch (error) {
+    return { ok: false, reason: error?.message ?? String(error) };
+  }
+}
+
+/**
+ * The advice appended when an untrusted pack uses syntax RE2 does not support.
+ *
+ * Worth being specific: "invalid regular expression" sends someone hunting for
+ * a typo in a pattern that is perfectly valid JavaScript.
+ * @param {string} reason
+ * @returns {string}
+ */
+export function explainPatternFailure(reason) {
+  // Order matters: an escape failure also contains "invalid escape sequence",
+  // and telling someone to remove lookahead from a pattern that has none is
+  // worse than saying nothing.
+  if (/invalid escape sequence: `\\[ucC]/.test(reason)) {
+    return `${reason}. RE2 spells a unicode escape \`\\x{41}\`, not \`\\u0041\`, and has no`
+      + ' control-character escape. Policy packs outside src/packs/ are matched with RE2.';
+  }
+  if (/Perl syntax|invalid escape sequence|lookbehind|invalid named capture/i.test(reason)) {
+    return `${reason}. Policy packs outside src/packs/ are matched with RE2, which does not`
+      + ' support lookahead, lookbehind, or backreferences -- they are what make backtracking'
+      + ' unbounded, and a pack is attacker-controlled on an untrusted pull request. Rewrite the'
+      + ' pattern without them.';
+  }
+  return reason;
+}

package/src/renderer.js

@@ -211,6 +211,30 @@ export function maskSensitiveValue(value, path = '') {
   return value;
 }
 
+/**
+ * Redact secret-shaped text from policy messages. A rule using
+ * `messageTemplate` can interpolate `{before}` / `{after}`, so a finding can
+ * carry a credential even when the change events beside it are masked. Replace
+ * exact interpolated values using the same path-aware masking as change events,
+ * then catch any other recognizable secret fragments in free-form messages.
+ * @param {import('./policy.js').PolicyFinding[]} findings
+ * @param {import('./differ.js').ChangeEvent[]} changes the unmasked events
+ * @returns {import('./policy.js').PolicyFinding[]}
+ */
+export function maskFindings(findings, changes) {
+  return findings.map((finding) => {
+    let message = String(finding.message ?? '');
+    for (const change of changes.filter((event) => event.path === finding.path)) {
+      for (const value of [change.before, change.after]) {
+        const original = String(value);
+        const masked = String(maskSensitiveValue(value, secretMatchPath(change)));
+        if (original && original !== masked) message = message.replaceAll(original, masked);
+      }
+    }
+    return { ...finding, message: redactSecretString(message) };
+  });
+}
+
 /**
  * @param {import('./differ.js').ChangeEvent} event
  * @returns {import('./differ.js').ChangeEvent}

package/src/snapshot-store.js

@@ -9,7 +9,7 @@ import {
 } from 'fs';
 import { createHash } from 'crypto';
 import { execFileSync } from 'child_process';
-import { dirname, join, relative, resolve, sep } from 'path';
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'path';
 
 import { containsSecret, looksLikeSecretPath } from './secrets.js';
 import { documentKeysOf, withDocumentKeys } from './documents.js';
@@ -374,12 +374,16 @@ function createSharedStore({ root, projectRoot, retention, maskMode }) {
    * slashed. A file outside the project has no such key, and inventing one (a
    * hash, an absolute path) would produce a store entry that is meaningless on
    * any other checkout — so it is refused rather than written somewhere useless.
+   *
+   * "Outside" includes a file `relative` cannot reach at all: on Windows a target
+   * on another drive or a UNC share comes back *absolute* (`D:\etc\hosts`), not
+   * `..`-prefixed, and would otherwise be accepted as a key.
    * @param {string} absFile
    * @returns {string}
    */
   const keyFor = (absFile) => {
     const rel = relative(projectRoot, resolve(absFile));
-    if (!rel || rel.startsWith('..') || rel.startsWith(`..${sep}`)) {
+    if (!rel || rel.startsWith('..') || isAbsolute(rel)) {
       throw new Error(
         `The shared snapshot store keys snapshots by repo-relative path, and "${absFile}" is`
         + ` outside ${projectRoot}. Run Flecto from the repository that holds the file, or use`