@mmerterden/multi-agent-pipeline 20.23.2 → 20.25.0

package/pipeline/scripts/_plan-overlap.mjs

@@ -0,0 +1,231 @@
+/**
+ * @file _plan-overlap.mjs  -  where two plan tasks meet, and whether they agree.
+ *
+ * Phase 1 hands Phase 2 a list of tasks that may run in parallel. Two tasks
+ * that touch the same file, or name the same interface, are where a plan
+ * contradicts itself without any single task being wrong: both create the
+ * same new file, one deletes or renames what the other still uses, or a task
+ * consumes something its producer is not ordered before.
+ *
+ * Every pair that shares a name becomes an anchored row:
+ *
+ *   { a, b, kind: "file"|"symbol", name, aRole, bRole, order, finding }
+ *
+ * `a` precedes `b` in plan order. A role is what the task does to the name:
+ *
+ *   file    creates (in `creates`) | removes (in `removes`) | edits (in `files`)
+ *   symbol  produces (in `produces`) | removes (in `removes`) | consumes (in `consumes`)
+ *
+ * `removes` covers deletion and the old side of a rename, so one field holds
+ * files and symbols alike. A name is a file when it contains a `/`, ends in a
+ * known file extension, is a well-known extensionless file (Makefile,
+ * Dockerfile, Podfile, ...) or is a dotfile; anything else, `UserService.fetch`
+ * included, is a symbol. File names are compared normalised, so `./a` and `a`
+ * are one file. A task that names something in more than one field takes its
+ * strongest role: removes, then creates / produces, then edits / consumes.
+ *
+ * `order` comes from `dependsOn`, transitively: `a-first`, `b-first`, or
+ * `unordered` when neither task waits for the other.
+ *
+ * Findings:
+ *
+ *   double-create                two tasks create the same file. Contradiction:
+ *                                the second one overwrites the first.
+ *   removes-used                 one task removes what the other edits,
+ *                                consumes, creates or produces. Warning.
+ *   consumer-not-after-producer  the user of a created file or produced symbol
+ *                                is not ordered after its producer. Warning.
+ *   unordered-edit               two tasks edit one file with no order between
+ *                                them, so parallel workers collide. Warning.
+ *
+ * Only the roles the plan states are judged. A plan without `creates`,
+ * `removes`, `produces` or `consumes` still gets its shared-file rows, and the
+ * unordered-edit finding, from `files` alone.
+ *
+ * Zero runtime dependencies (ADR-0004).
+ *
+ * @module pipeline/scripts/_plan-overlap
+ */
+
+const RANK = { removes: 3, creates: 2, produces: 2, edits: 1, consumes: 1 };
+
+const list = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === "string" && x) : []);
+
+const FILE_EXTENSIONS = new Set(
+  (
+    "swift m mm h hh hpp c cc cpp cxx kt kts java gradle groovy scala js mjs cjs jsx ts tsx mts cts " +
+    "vue svelte py pyi rb go rs php cs dart lua pl sh bash zsh fish ps1 bat cmd sql graphql gql " +
+    "proto json jsonc json5 yaml yml toml ini cfg conf xml plist html htm css scss sass less md " +
+    "mdx markdown txt rst adoc csv tsv lock resolved xcconfig xcstrings strings stringsdict " +
+    "storyboard xib entitlements pbxproj xcscheme xcworkspacedata xcassets xcodeproj xcworkspace " +
+    "xcframework xctestplan png jpg jpeg gif svg webp heic pdf ico ttf otf woff woff2 mp3 mp4 mov " +
+    "wav zip gz tgz tar jar aar apk aab ipa dylib so tf hcl cmake mk patch diff pem p12 mobileprovision"
+  ).split(" "),
+);
+const FILE_NAMES = new Set([
+  "Makefile",
+  "GNUmakefile",
+  "Dockerfile",
+  "Containerfile",
+  "Podfile",
+  "Gemfile",
+  "Rakefile",
+  "Fastfile",
+  "Appfile",
+  "Matchfile",
+  "Pluginfile",
+  "Brewfile",
+  "Cartfile",
+  "Mintfile",
+  "Procfile",
+  "Jenkinsfile",
+  "Vagrantfile",
+  "Pipfile",
+  "Justfile",
+  "Package.swift",
+  "Package.resolved",
+  "Podfile.lock",
+  "Gemfile.lock",
+  "go.mod",
+  "go.sum",
+  "gradlew",
+  "mvnw",
+  "BUILD",
+  "WORKSPACE",
+  "LICENSE",
+  "README",
+  "CHANGELOG",
+  "CODEOWNERS",
+]);
+
+/** `./a`, `a` and `a//b/.` name the same file. */
+export function normalizeFile(name) {
+  const parts = [];
+  for (const seg of name.split("/")) {
+    if (seg === "" || seg === ".") continue;
+    if (seg === ".." && parts.length && parts.at(-1) !== "..") parts.pop();
+    else parts.push(seg);
+  }
+  const joined = parts.join("/");
+  return name.startsWith("/") ? `/${joined}` : joined || name;
+}
+
+export const isFileName = (name) => {
+  if (name.includes("/")) return true;
+  if (FILE_NAMES.has(name) || name.startsWith(".")) return true;
+  const dot = name.lastIndexOf(".");
+  return dot > 0 && FILE_EXTENSIONS.has(name.slice(dot + 1).toLowerCase());
+};
+
+/** name -> role for one task, per kind, keeping the strongest role. */
+function rolesOf(task) {
+  const files = new Map();
+  const symbols = new Map();
+  const put = (map, name, role) => {
+    const cur = map.get(name);
+    if (!cur || RANK[role] > RANK[cur]) map.set(name, role);
+  };
+  for (const f of list(task.files)) put(files, normalizeFile(f), "edits");
+  for (const f of list(task.creates)) put(files, normalizeFile(f), "creates");
+  for (const s of list(task.consumes)) put(symbols, s, "consumes");
+  for (const s of list(task.produces)) put(symbols, s, "produces");
+  for (const n of list(task.removes)) {
+    if (isFileName(n)) put(files, normalizeFile(n), "removes");
+    else put(symbols, n, "removes");
+  }
+  return { file: files, symbol: symbols };
+}
+
+/** id -> set of every id it waits for, through dependsOn. */
+function ancestors(tasks) {
+  const deps = new Map(tasks.map((t) => [String(t.id), list(t.dependsOn)]));
+  const memo = new Map();
+  const walk = (id, trail = new Set()) => {
+    if (memo.has(id)) return memo.get(id);
+    const out = new Set();
+    if (trail.has(id)) return out;
+    trail.add(id);
+    for (const d of deps.get(id) || []) {
+      out.add(d);
+      for (const x of walk(d, trail)) out.add(x);
+    }
+    trail.delete(id);
+    memo.set(id, out);
+    return out;
+  };
+  return (id) => walk(id);
+}
+
+const PRODUCER = new Set(["creates", "produces"]);
+
+function judge(aRole, bRole, order) {
+  if (aRole === "creates" && bRole === "creates") return "double-create";
+  if ((aRole === "removes") !== (bRole === "removes")) return "removes-used";
+  if (aRole === "removes") return null;
+  const aMakes = PRODUCER.has(aRole);
+  const bMakes = PRODUCER.has(bRole);
+  if (aMakes && !bMakes && order !== "a-first") return "consumer-not-after-producer";
+  if (bMakes && !aMakes && order !== "b-first") return "consumer-not-after-producer";
+  if (aRole === "edits" && bRole === "edits" && order === "unordered") return "unordered-edit";
+  return null;
+}
+
+function sentence(row) {
+  const { a, b, name, aRole, bRole, order, finding } = row;
+  switch (finding) {
+    case "double-create":
+      return order === "unordered"
+        ? `${a} and ${b} both create ${name}; one task must create it and the other depend on it`
+        : `${a} and ${b} both create ${name}; they are already ordered, so drop ${name} from the creates of one (the later task edits it)`;
+    case "removes-used": {
+      const [rm, user, role] = aRole === "removes" ? [a, b, bRole] : [b, a, aRole];
+      return `${a} and ${b}: ${rm} removes ${name}, which ${user} ${role}`;
+    }
+    case "consumer-not-after-producer": {
+      const [maker, user, role] = PRODUCER.has(aRole) ? [a, b, bRole] : [b, a, aRole];
+      return `${a} and ${b}: ${user} ${role} ${name} but is not ordered after ${maker}, which ${PRODUCER.has(aRole) ? aRole : bRole} it`;
+    }
+    case "unordered-edit":
+      return `${a} and ${b} both edit ${name} with no dependsOn between them`;
+    default:
+      return null;
+  }
+}
+
+/**
+ * @param {Array<{id: string, files?: string[], creates?: string[], removes?: string[],
+ *   produces?: string[], consumes?: string[], dependsOn?: string[]}>} tasks
+ * @returns {{
+ *   rows: Array<{a: string, b: string, kind: "file"|"symbol", name: string, aRole: string,
+ *     bRole: string, order: "a-first"|"b-first"|"unordered", finding: string|null}>,
+ *   contradictions: string[],
+ *   warnings: string[],
+ * }}
+ */
+export function planOverlaps(tasks) {
+  const ts = (Array.isArray(tasks) ? tasks : []).filter((t) => t && t.id != null);
+  const roles = ts.map(rolesOf);
+  const before = ancestors(ts);
+  const rows = [];
+  for (const kind of ["file", "symbol"]) {
+    for (let i = 0; i < ts.length; i++) {
+      for (let j = i + 1; j < ts.length; j++) {
+        const a = String(ts[i].id);
+        const b = String(ts[j].id);
+        const order = before(b).has(a) ? "a-first" : before(a).has(b) ? "b-first" : "unordered";
+        for (const [name, aRole] of roles[i][kind]) {
+          const bRole = roles[j][kind].get(name);
+          if (!bRole) continue;
+          rows.push({ a, b, kind, name, aRole, bRole, order, finding: judge(aRole, bRole, order) });
+        }
+      }
+    }
+  }
+  const contradictions = [];
+  const warnings = [];
+  for (const row of rows) {
+    if (!row.finding) continue;
+    (row.finding === "double-create" ? contradictions : warnings).push(sentence(row));
+  }
+  return { rows, contradictions, warnings };
+}

package/pipeline/scripts/_redact.mjs

@@ -51,22 +51,136 @@ const SYSTEM_PATH = /(?<![\w.~\-/])\/(?:Volumes|private|var|opt|tmp)(?![\w.-])\S
 /** Any user's home directory, reduced to ~ so the rest of the path still reads. */
 const HOME_PATH = /(?<![\w.~\-/])\/(?:Users|home)(?:\/[^\s/)"'\]]+)?(?![\w.-])/g;
 
+/** The replacement chain, in order, with the kind each step counts as. */
+const STEPS = [
+  [SCHEME_LINK, "[link]", "link"],
+  [WWW_LINK, "[link]", "link"],
+  [EMAIL, "[email]", "email"],
+  [BARE_LINK, "[link]", "link"],
+  [IPV4, "[host]", "host"],
+  [DOTTED_NAME, hostOrSelf, "host"],
+  [SYSTEM_PATH, "[path]", "path"],
+  [HOME_PATH, "~", "home-path"],
+];
+
+/**
+ * scrub, plus how many replacements of each kind it made. A dotted name that
+ * hostOrSelf leaves alone is not counted.
+ *
+ * @returns {{text: string, counts: Record<string, number>}}
+ */
+export function scrubCounted(text) {
+  const counts = {};
+  let out = String(text ?? "");
+  for (const [re, replacement, kind] of STEPS) {
+    out = out.replace(re, (match, ...groups) => {
+      const next = typeof replacement === "function" ? replacement(match, ...groups) : replacement;
+      if (next !== match) counts[kind] = (counts[kind] || 0) + 1;
+      return next;
+    });
+  }
+  return { text: out.trim(), counts };
+}
+
 /**
  * Free text with URLs (with or without a scheme), e-mail addresses, IPv4
  * addresses, host names, absolute machine paths and home directories replaced
  * by markers.
  */
 export function scrub(text) {
-  return String(text ?? "")
-    .replace(SCHEME_LINK, "[link]")
-    .replace(WWW_LINK, "[link]")
-    .replace(EMAIL, "[email]")
-    .replace(BARE_LINK, "[link]")
-    .replace(IPV4, "[host]")
-    .replace(DOTTED_NAME, hostOrSelf)
-    .replace(SYSTEM_PATH, "[path]")
-    .replace(HOME_PATH, "~")
-    .trim();
+  return scrubCounted(text).text;
+}
+
+// --- credentials the provider rules (lib/redact.mjs) have no prefix for ------
+
+/**
+ * A variable, flag or header name that holds a credential: any word ending in
+ * TOKEN, SECRET, PASSWORD, PASSWD, PWD or API_KEY (PGPASSWORD, MYSQL_PWD,
+ * myAppToken, db-password, x-api-key), or a separated word ending in KEY.
+ */
+const SECRET_NAME = String.raw`[A-Za-z0-9_-]*(?:TOKEN|SECRET|PASSWORD|PASSWD|PWD|API[_-]?KEY)|(?:[A-Za-z0-9]+[_-])+KEY`;
+const NOT_MARKER = String.raw`(?!\[redacted:)`;
+const PEM_END = String.raw`-----END [A-Z ]*PRIVATE KEY-----`;
+const JSON_KEY = String.raw`[A-Za-z0-9_-]*(?:password|passwd|secret|token|api[_-]?key|private[_-]?key|credentials?)`;
+
+/**
+ * Credential shapes the provider rules miss, as [pattern, replacer, kind].
+ * Each keeps the name or header in front of the value so the line still says
+ * what was configured, and drops only the value. They run after
+ * lib/redact.mjs, so they also finish what its markers leave behind: the body
+ * of a private key whose BEGIN line it replaced, and the host and port after
+ * a credential URL.
+ */
+const CREDENTIAL_STEPS = [
+  [
+    new RegExp(
+      String.raw`(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|\[redacted:private-key-block\])[\s\S]*?(?:${PEM_END}|$(?![\s\S]))`,
+      "g",
+    ),
+    () => "[redacted:private-key-block]",
+    "private-key-block",
+  ],
+  [
+    /\b[a-z][a-z0-9+.-]*:\/\/[^\s/:@]+:[^\s/@]+@\S*|\[redacted:url-with-credentials\]@\S*/gi,
+    () => "[redacted:url-with-credentials]",
+    "url-with-credentials",
+  ],
+  [/\bAT(?:ATT|CTT)[A-Za-z0-9_=-]{20,}/g, () => "[redacted:atlassian-token]", "atlassian-token"],
+  [
+    new RegExp(
+      String.raw`\b(Authorization\s*:\s*)${NOT_MARKER}(?:(?:Bearer|Basic|Digest|token)\s+)?${NOT_MARKER}[^\s"',;]+`,
+      "gi",
+    ),
+    (_, head) => `${head}[redacted:authorization-header]`,
+    "authorization-header",
+  ],
+  [
+    /\b(Bearer\s+)(?!\[redacted:)(?=[A-Za-z0-9._~+/=-]*\d)([A-Za-z0-9._~+/=-]{12,})/g,
+    (_, head) => `${head}[redacted:bearer-token]`,
+    "bearer-token",
+  ],
+  [
+    new RegExp(
+      String.raw`(\\?")(${JSON_KEY})\1(\s*:\s*)\1${NOT_MARKER}(?:\\[^"]|[^"\\])+?\1`,
+      "gi",
+    ),
+    (_, q, key, sep) => `${q}${key}${q}${sep}${q}[redacted:json-secret]${q}`,
+    "json-secret",
+  ],
+  [
+    new RegExp(
+      String.raw`(?<![\w-])(${SECRET_NAME})(\s*[=:]\s*)${NOT_MARKER}("[^"\n]+"|'[^'\n]+'|[^\s"',;]+)`,
+      "gi",
+    ),
+    (_, name, sep) => `${name}${sep}[redacted:env-secret]`,
+    "env-secret",
+  ],
+  [
+    /(\bcurl\b[^\n]*?\s(?:-u\s*|--user[=\s]\s*)(["']?)[^\s:"']+:)(?!\[redacted:)[^\s"']+\2/g,
+    (_, head, q) => `${head}[redacted:basic-auth]${q}`,
+    "basic-auth",
+  ],
+  [
+    /\b((?:mysql|mysqldump|mysqladmin|mariadb|mariadb-dump)\b[^\n]*?\s-p)(?!\s)(?!\[redacted:)\S+/g,
+    (_, head) => `${head}[redacted:password-flag]`,
+    "password-flag",
+  ],
+  [
+    /(--password[= ])(?!\[redacted:)\S+/g,
+    (_, head) => `${head}[redacted:password-flag]`,
+    "password-flag",
+  ],
+];
+
+/**
+ * `text` with the credentials CREDENTIAL_STEPS recognise replaced by
+ * `[redacted:<kind>]` markers. Meant to run after lib/redact.mjs; markers it
+ * already wrote are never redacted a second time.
+ */
+export function redactCredentials(text) {
+  let out = String(text ?? "");
+  for (const [re, replacer] of CREDENTIAL_STEPS) out = out.replace(re, replacer);
+  return out;
 }
 
 /** A copy of `obj` without `keys`. */