@mnaimfaizy/agent-kit 0.0.0-stage → 1.0.0-beta.1

package/src/checks/rulesets.mjs

@@ -0,0 +1,346 @@
+// The rulesets docs/repository-hardening.md sets up, which keep the Claude
+// GitHub App token away from the default branch, release branches and
+// release tags. Each gap is a warning: the flows still run without them.
+//
+// A rule only counts when no one the App can act as is on the ruleset's
+// bypass list: rulesets stack, so an actor on one ruleset's bypass list is
+// still held by any other ruleset with the same rule that does not list it.
+
+import { stripComment } from "../yaml-subset.mjs";
+import { result } from "./result.mjs";
+
+const HARDENING = "https://github.com/mnaimfaizy/agent-kit/blob/main/docs/repository-hardening.md";
+
+/**
+ * fnmatch with FNM_PATHNAME, the matching GitHub uses for ruleset ref
+ * patterns: `*` and `?` stay within one path segment, a double star
+ * followed by a slash matches zero or more whole segments, `[...]` is a character class (`[!...]` or `[^...]`
+ * negates it), and `\` escapes the next character.
+ */
+export function fnmatchToRegExp(pattern) {
+  let re = "";
+  let i = 0;
+  const esc = (c) => c.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&");
+  while (i < pattern.length) {
+    const c = pattern[i];
+    if (c === "*" && pattern[i + 1] === "*" && pattern[i + 2] === "/") {
+      re += "(?:[^/]*/)*";
+      i += 3;
+    } else if (c === "*" && pattern[i + 1] === "*") {
+      re += "[^/]*";
+      i += 2;
+    } else if (c === "*") {
+      re += "[^/]*";
+      i += 1;
+    } else if (c === "?") {
+      re += "[^/]";
+      i += 1;
+    } else if (c === "\\" && i + 1 < pattern.length) {
+      re += esc(pattern[i + 1]);
+      i += 2;
+    } else if (c === "[") {
+      const close = pattern.indexOf("]", i + 2);
+      if (close === -1) {
+        re += "\\[";
+        i += 1;
+        continue;
+      }
+      let body = pattern.slice(i + 1, close);
+      let negate = false;
+      if (body.startsWith("!") || body.startsWith("^")) {
+        negate = true;
+        body = body.slice(1);
+      }
+      const cls = body.replace(/[\\\]^]/g, "\\$&");
+      re += negate ? `(?!/)[^${cls}]` : `[${cls}]`;
+      i = close + 1;
+    } else {
+      re += esc(c);
+      i += 1;
+    }
+  }
+  return new RegExp(`^${re}$`);
+}
+
+/** Whether a ruleset's `conditions.ref_name` covers `ref`. */
+export function coversRef(ruleset, ref, { defaultBranch } = {}) {
+  // A ruleset with a pattern doctor cannot interpret is not counted at all.
+  if (unreadablePatterns(ruleset).length) return false;
+  const cond = ruleset.conditions?.ref_name ?? {};
+  const matches = (p) =>
+    p === "~ALL" ||
+    (p === "~DEFAULT_BRANCH" && defaultBranch && ref === `refs/heads/${defaultBranch}`) ||
+    fnmatchToRegExp(p).test(ref);
+  return (cond.include ?? []).some(matches) && !(cond.exclude ?? []).some(matches);
+}
+
+/** The ref patterns of a ruleset that do not compile (an invalid range such as `[z-a]`). */
+export function unreadablePatterns(ruleset) {
+  const cond = ruleset.conditions?.ref_name ?? {};
+  return [...(cond.include ?? []), ...(cond.exclude ?? [])].filter((p) => {
+    if (typeof p !== "string") return true;
+    if (p === "~ALL" || p === "~DEFAULT_BRANCH") return false;
+    try {
+      fnmatchToRegExp(p);
+      return false;
+    } catch {
+      return true;
+    }
+  });
+}
+
+/**
+ * Note the rulesets of `target` that were not counted because a pattern could
+ * not be interpreted. Rulesets only add restrictions, so a check that holds
+ * without them stays a pass; one that does not is already a warning.
+ */
+function withPatternNotes(res, rulesets, target) {
+  const notes = rulesets
+    .filter((r) => r.target === target)
+    .flatMap((r) =>
+      unreadablePatterns(r).map(
+        (p) =>
+          `${label(r)}: pattern ${JSON.stringify(p)} could not be interpreted, so it was not counted.`,
+      ),
+    );
+  if (notes.length === 0) return res;
+  return { ...res, details: [...notes, ...res.details] };
+}
+
+/**
+ * Active rulesets with their full rules and bypass lists. `bypass_actors` is
+ * only returned by the single-ruleset endpoint, and only to an admin, which
+ * the local checks require. Throws GhError on any API failure.
+ */
+export function loadRulesets(github, repo) {
+  const ids = github.getLines(`repos/${repo}/rulesets?includes_parents=true`, ".[].id");
+  return ids
+    .map((id) => github.getJson(`repos/${repo}/rulesets/${Number(id)}`))
+    .filter((r) => r.enforcement === "active");
+}
+
+// Base repository roles as ruleset bypass actors (docs/repository-hardening.md).
+const ADMIN_ROLE = 5;
+const MAINTAIN_ROLE = 2;
+const ROLE_NAMES = { 1: "Read", 2: "Maintain", 3: "Triage", 4: "Write", 5: "Admin" };
+
+/**
+ * Bypass actors that undo the ruleset's protection against the Claude App:
+ * any GitHub App (the Claude App is an `Integration`), any repository role
+ * below Maintain (the Write role, or a custom role), and, on a branch
+ * ruleset, deploy keys. Admin, Maintain, teams and organization admins are
+ * the intended bypass list; a DeployKey bypass on a tag ruleset is the
+ * documented personal-account pattern.
+ */
+export function riskyBypassActors(ruleset) {
+  // Only an admin sees bypass lists, and an organization ruleset's may be
+  // hidden even then: an unseen list is never assumed to be clean.
+  if (!Array.isArray(ruleset.bypass_actors)) return ["a bypass list not visible to this login"];
+  const risky = [];
+  for (const actor of ruleset.bypass_actors) {
+    const type = actor.actor_type;
+    if (type === "Integration") {
+      risky.push(`a GitHub App (Integration ${actor.actor_id})`);
+    } else if (type === "RepositoryRole" && ![ADMIN_ROLE, MAINTAIN_ROLE].includes(actor.actor_id)) {
+      const name = ROLE_NAMES[actor.actor_id];
+      risky.push(name ? `the ${name} role` : `custom repository role ${actor.actor_id}`);
+    } else if (type === "DeployKey" && ruleset.target !== "tag") {
+      risky.push("deploy keys");
+    }
+  }
+  return risky;
+}
+
+const hasRule = (ruleset, type) => (ruleset.rules ?? []).some((x) => x.type === type);
+const label = (r) => `"${r.name ?? `ruleset ${r.id}`}"`;
+
+/**
+ * For one rule type: the covering rulesets that have it, and whether any of
+ * them has a clean bypass list.
+ */
+function ruleStatus(covering, type) {
+  const withRule = covering.filter((r) => hasRule(r, type));
+  const clean = withRule.filter((r) => riskyBypassActors(r).length === 0);
+  const bypassed = withRule
+    .filter((r) => !clean.includes(r))
+    .map((r) => `${label(r)} lets ${riskyBypassActors(r).join(", ")} bypass its ${type} rule.`);
+  return { present: withRule.length > 0, held: clean.length > 0, bypassed };
+}
+
+const BYPASS_ADVICE =
+  "Keep the bypass list to Admin and Maintain: never a GitHub App, the Write role, or a team the App acts through.";
+
+function updateRule(rulesets, { defaultBranch }) {
+  const ID = "ruleset-update";
+  const TITLE = "Default branch has an update rule";
+  const recipe = `Recipe: ${HARDENING}#1-default-branch-an-update-rule`;
+  const ref = `refs/heads/${defaultBranch}`;
+  const covering = rulesets.filter(
+    (r) => r.target === "branch" && coversRef(r, ref, { defaultBranch }),
+  );
+  const s = ruleStatus(covering, "update");
+  if (s.held) return result(ID, TITLE, "pass", `${defaultBranch} has an active update rule.`);
+  if (s.present) {
+    return result(ID, TITLE, "warn", `The update rule on ${defaultBranch} can be bypassed.`, {
+      details: [...s.bypassed, BYPASS_ADVICE, recipe],
+    });
+  }
+  return result(ID, TITLE, "warn", `No active update rule on ${defaultBranch}.`, {
+    details: [
+      "A required pull request with zero approvals does not stop the Claude App merging its own pull request; an update rule does.",
+      recipe,
+    ],
+  });
+}
+
+// Release-branch names a creation rule should cover; any one is enough.
+const RELEASE_BRANCH_REFS = [
+  "refs/heads/release/1.0.0",
+  "refs/heads/releases/1.0.0",
+  "refs/heads/release-1.0.0",
+];
+
+function creationRule(rulesets) {
+  const ID = "ruleset-creation";
+  const TITLE = "Release branches have a creation rule";
+  const recipe = `Recipe: ${HARDENING}#2-release-branches-a-creation-rule`;
+  const branchRulesets = rulesets.filter((r) => r.target === "branch");
+  const covering = branchRulesets.filter((r) =>
+    RELEASE_BRANCH_REFS.some((ref) => coversRef(r, ref)),
+  );
+  const s = ruleStatus(covering, "creation");
+  if (s.held) {
+    return result(ID, TITLE, "pass", "An active creation rule covers release branches.");
+  }
+  if (s.present) {
+    return result(ID, TITLE, "warn", "Every release-branch creation rule can be bypassed.", {
+      details: [...s.bypassed, BYPASS_ADVICE, recipe],
+    });
+  }
+  const elsewhere = ruleStatus(branchRulesets, "creation").present;
+  return result(
+    ID,
+    TITLE,
+    "warn",
+    elsewhere
+      ? "A branch creation rule exists, but none covers a release/* branch."
+      : "No active branch ruleset has a creation rule.",
+    {
+      details: [
+        "If you cut releases from branches, stop anyone outside the bypass list creating one.",
+        `Checked against ${RELEASE_BRANCH_REFS.join(", ")}.`,
+        recipe,
+      ],
+    },
+  );
+}
+
+const TAG_RULES = ["creation", "update", "deletion"];
+
+// A stable and a pre-release tag: the v* ruleset must cover both.
+const TAG_REFS = ["refs/tags/v1.0.0", "refs/tags/v1.0.0-alpha.1"];
+
+function tagRuleset(rulesets) {
+  const ID = "ruleset-tags";
+  const TITLE = "Release tags have a v* tag ruleset";
+  const recipe = `Recipe: ${HARDENING}#3-release-tags-a-v-tag-ruleset`;
+  const problems = [];
+  const details = new Set();
+  for (const ref of TAG_REFS) {
+    const covering = rulesets.filter((r) => r.target === "tag" && coversRef(r, ref));
+    if (covering.length === 0) {
+      problems.push(`no active tag ruleset covers ${ref.slice("refs/tags/".length)}`);
+      continue;
+    }
+    const statuses = TAG_RULES.map((type) => [type, ruleStatus(covering, type)]);
+    const missing = statuses.filter(([, st]) => !st.present).map(([t]) => t);
+    const bypassed = statuses.filter(([, st]) => st.present && !st.held);
+    const tag = ref.slice("refs/tags/".length);
+    if (missing.length) problems.push(`${tag} lacks: ${missing.join(", ")}`);
+    if (bypassed.length) {
+      problems.push(`${tag} can be bypassed: ${bypassed.map(([t]) => t).join(", ")}`);
+      for (const [, st] of bypassed) for (const d of st.bypassed) details.add(d);
+      details.add(BYPASS_ADVICE);
+    }
+  }
+  if (problems.length) {
+    const unique = [...new Set(problems)];
+    return result(ID, TITLE, "warn", `The v* tag ruleset: ${unique.join("; ")}.`, {
+      details: [...details, recipe],
+    });
+  }
+  return result(
+    ID,
+    TITLE,
+    "pass",
+    "An active tag ruleset blocks creating, moving and deleting v* tags, pre-releases included.",
+  );
+}
+
+export function checkUpdateRule(rulesets, opts) {
+  return withPatternNotes(updateRule(rulesets, opts), rulesets, "branch");
+}
+
+export function checkCreationRule(rulesets) {
+  return withPatternNotes(creationRule(rulesets), rulesets, "branch");
+}
+
+export function checkTagRuleset(rulesets) {
+  return withPatternNotes(tagRuleset(rulesets), rulesets, "tag");
+}
+
+/**
+ * Workflows that push tags with GITHUB_TOKEN: a `git push` of tags, with no
+ * SSH deploy key in the same file.
+ */
+export function workflowsPushingTagsWithToken(files) {
+  return files
+    .filter(({ text }) => {
+      const body = String(text)
+        .replace(/\r\n?/g, "\n")
+        .split("\n")
+        .filter((l) => !l.trimStart().startsWith("#"))
+        .map(stripComment)
+        .join("\n");
+      const pushes = body
+        .split("\n")
+        .some(
+          (l) =>
+            /\bgit\b.*\bpush\b/.test(l) &&
+            /--tags|--follow-tags|refs\/tags\/|\bTAG\b|\btag\b/i.test(l),
+        );
+      const tagged = /\bgit\b[^\n]*\btag\b/.test(body) && /\bgit\b[^\n]*\bpush\b/.test(body);
+      const ssh = /GIT_SSH_COMMAND|git@github\.com:|ssh:\/\//.test(body);
+      return (pushes || tagged) && !ssh;
+    })
+    .map((f) => f.path);
+}
+
+export function checkPersonalTagPush({ ownerType, files }) {
+  const ID = "release-tag-push";
+  const TITLE = "Release tags can be pushed under the tag ruleset";
+  if (ownerType !== "User") {
+    return result(
+      ID,
+      TITLE,
+      "pass",
+      "Owned by an organization: the GitHub Actions app can be a ruleset bypass actor.",
+    );
+  }
+  const offenders = workflowsPushingTagsWithToken(files);
+  if (offenders.length === 0)
+    return result(ID, TITLE, "pass", "No workflow pushes tags with GITHUB_TOKEN.");
+  return result(
+    ID,
+    TITLE,
+    "warn",
+    "A personal-account repository whose workflow pushes tags with GITHUB_TOKEN.",
+    {
+      details: [
+        ...offenders,
+        "On a personal account the GitHub Actions app cannot bypass a tag ruleset, so these pushes are refused once v* tags are protected.",
+        `Fix: ${HARDENING}#personal-account-repositories-pushing-release-tags`,
+      ],
+    },
+  );
+}

package/src/checks/secrets-and-variables.mjs

@@ -0,0 +1,252 @@
+// Secrets and repository variables, worked out from the Consumer's Callers.
+// A secret is required when the reusable workflow at the pinned tag declares
+// the secret it is passed to as `required: true`; otherwise it is optional.
+// A variable is optional when every read of it falls back with `||`.
+
+import { secretNames } from "../callers.mjs";
+import { parseWorkflowCallSecrets } from "../workflow-call.mjs";
+import { byHand, command, NAME_PATTERN, result } from "./result.mjs";
+
+/** GitHub provides `secrets.GITHUB_TOKEN`; no `GITHUB_*` secret can be set by hand. */
+const isProvided = (name) => /^github_/i.test(name);
+
+/** Caller parts doctor could not read fully: reported, never assumed fine. */
+function unreadable(callers) {
+  const out = [];
+  for (const caller of callers) {
+    if (caller.parseIncomplete) out.push(`${caller.path}: ${caller.parseIncomplete}`);
+    for (const job of caller.jobs) {
+      if (job.secretsUnsupported) {
+        out.push(
+          `${caller.path}:${job.name}: ${job.secretsUnsupported}; its secrets were not checked.`,
+        );
+      }
+    }
+  }
+  return out;
+}
+
+/** The variables every Caller reads. Needs no tag. */
+export function deriveVariables(callers) {
+  const variables = new Map();
+  for (const caller of callers) {
+    for (const v of caller.variables) {
+      const prev = variables.get(v.name);
+      variables.set(v.name, {
+        name: v.name,
+        optional: prev ? prev.optional && v.optional : v.optional,
+        where: [...(prev?.where ?? []), caller.path],
+      });
+    }
+  }
+  return [...variables.values()];
+}
+
+/**
+ * @param {{ path: string, jobs: object[], variables: { name: string, optional: boolean }[] }[]} callers
+ * @param {(workflowFile: string) => string | null} readReusable reads .github/workflows/<file> at the pinned tag
+ */
+export function deriveRequirements(callers, readReusable) {
+  const secrets = new Map();
+  const problems = [];
+  const specs = new Map();
+  const specFor = (file) => {
+    if (!specs.has(file)) {
+      const text = readReusable(file);
+      specs.set(file, text === null ? null : parseWorkflowCallSecrets(text));
+    }
+    return specs.get(file);
+  };
+  const addSecret = (names, required, where) => {
+    // A GITHUB_* secret is always set, so an expression that falls back to one is too.
+    if (names.some(isProvided)) return;
+    const key = names.join(" || ");
+    const prev = secrets.get(key);
+    secrets.set(key, {
+      names,
+      required: Boolean(prev?.required || required),
+      where: [...(prev?.where ?? []), where],
+    });
+  };
+
+  for (const caller of callers) {
+    for (const job of caller.jobs) {
+      const where = `${caller.path}:${job.name}`;
+      if (job.secretsUnsupported) continue;
+      const spec = specFor(job.uses.file);
+      if (spec === null) {
+        problems.push(`${where}: ${job.uses.file} is not a reusable workflow at the pinned tag`);
+        continue;
+      }
+      if (job.secrets === "inherit") {
+        for (const { name, required } of spec.values()) {
+          addSecret([name.toUpperCase()], required, where);
+        }
+        continue;
+      }
+      const passed = new Set(Object.keys(job.secrets).map((k) => k.toLowerCase()));
+      for (const [input, value] of Object.entries(job.secrets)) {
+        const names = secretNames(value);
+        if (names.length === 0) continue;
+        const declared = spec.get(input.toLowerCase());
+        if (!declared) {
+          problems.push(
+            `${where}: passes secret ${JSON.stringify(input)}, which ${job.uses.file} does not declare`,
+          );
+          continue;
+        }
+        addSecret(names, declared.required, `${where} (${input})`);
+      }
+      for (const { name, required } of spec.values()) {
+        if (required && !passed.has(name.toLowerCase())) {
+          problems.push(
+            `${where}: does not pass required secret ${JSON.stringify(name)} to ${job.uses.file}`,
+          );
+        }
+      }
+    }
+  }
+  return {
+    secrets: [...secrets.values()],
+    variables: deriveVariables(callers),
+    problems,
+    unsupported: unreadable(callers),
+  };
+}
+
+function listNames(github, repo, kind, ownerType) {
+  const key = kind === "secret" ? "secrets" : "variables";
+  const names = new Set(github.getLines(`repos/${repo}/actions/${key}`, `.${key}[].name`));
+  let orgNote = null;
+  if (ownerType === "Organization") {
+    try {
+      for (const n of github.getLines(
+        `repos/${repo}/actions/organization-${key}`,
+        `.${key}[].name`,
+      )) {
+        names.add(n);
+      }
+    } catch {
+      orgNote = `Organization ${key} could not be read; one reported missing may be set at the organization.`;
+    }
+  }
+  return { names: new Set([...names].map((n) => n.toUpperCase())), orgNote };
+}
+
+/** `gh secret set` / `gh variable set` for a valid name, or a by-hand note. */
+function setCommand(kind, name, repo, extra = []) {
+  if (!NAME_PATTERN.test(name)) return { note: byHand(`${kind} named`, name) };
+  return { cmd: command("gh", kind, "set", name, "--repo", repo, ...extra) };
+}
+
+function splitFixes(items) {
+  return {
+    fix: items.filter((i) => i.cmd).map((i) => i.cmd),
+    notes: items.filter((i) => i.note).map((i) => i.note),
+  };
+}
+
+export function checkSecrets({ github, repo, ownerType, requirements }) {
+  const ID = "secrets";
+  const TITLE = "Secrets the Callers pass are set";
+  const { problems, unsupported = [] } = requirements;
+  let listed;
+  try {
+    listed = listNames(github, repo, "secret", ownerType);
+  } catch (err) {
+    // Problems found in the Caller files need no API call: they still fail.
+    if (problems.length) {
+      return result(ID, TITLE, "fail", `${problems.length} Caller problem(s) with secrets.`, {
+        details: [
+          ...problems,
+          ...unsupported,
+          `Could not list repository secrets, so unset secrets were not checked: ${err.message}`,
+        ],
+      });
+    }
+    return result(ID, TITLE, "skip", `Could not list repository secrets: ${err.message}`, {
+      details: unsupported,
+    });
+  }
+  const isSet = (x) => x.names.some((n) => listed.names.has(n.toUpperCase()));
+  const missingRequired = requirements.secrets.filter((x) => x.required && !isSet(x));
+  const missingOptional = requirements.secrets.filter((x) => !x.required && !isSet(x));
+  const describe = (x) => `${x.names.join(" or ")} (${x.where.join(", ")})`;
+  // No --body: gh prompts for the value, so it never lands in shell history.
+  const { fix, notes } = splitFixes(
+    [...missingRequired, ...missingOptional].map((x) => setCommand("secret", x.names[0], repo)),
+  );
+  const details = [
+    ...problems,
+    ...missingRequired.map((x) => `required, not set: ${describe(x)}`),
+    ...missingOptional.map((x) => `optional, not set: ${describe(x)}`),
+    ...unsupported,
+    ...notes,
+  ];
+  if (listed.orgNote && (missingRequired.length || missingOptional.length)) {
+    details.push(listed.orgNote);
+  }
+  if (missingRequired.length || problems.length) {
+    const summary = missingRequired.length
+      ? `${missingRequired.length} required secret(s) missing.`
+      : `${problems.length} Caller problem(s) with secrets.`;
+    return result(ID, TITLE, "fail", summary, { details, fix });
+  }
+  if (missingOptional.length) {
+    return result(
+      ID,
+      TITLE,
+      "warn",
+      `${missingOptional.length} optional secret(s) referenced but not set.`,
+      { details, fix },
+    );
+  }
+  if (unsupported.length) {
+    return result(
+      ID,
+      TITLE,
+      "skip",
+      `${unsupported.length} Caller part(s) doctor could not read; their secrets were not checked.`,
+      { details: unsupported },
+    );
+  }
+  return result(
+    ID,
+    TITLE,
+    "pass",
+    `All ${requirements.secrets.length} secret(s) the Callers pass are set.`,
+  );
+}
+
+export function checkVariables({ github, repo, ownerType, requirements }) {
+  const ID = "variables";
+  const TITLE = "Repository variables the Callers read are set";
+  let listed;
+  try {
+    listed = listNames(github, repo, "variable", ownerType);
+  } catch (err) {
+    return result(ID, TITLE, "skip", `Could not list repository variables: ${err.message}`);
+  }
+  const required = requirements.variables.filter((v) => !v.optional && !isProvided(v.name));
+  const missing = required.filter((v) => !listed.names.has(v.name.toUpperCase()));
+  if (missing.length === 0) {
+    return result(ID, TITLE, "pass", `All ${required.length} required variable(s) are set.`);
+  }
+  const killSwitch = (name) => /_ENABLED$/.test(name);
+  const details = missing.map(
+    (v) =>
+      `not set: ${v.name} (${v.where.join(", ")})` +
+      (killSwitch(v.name) ? " — the Kill switch: the flow never runs until it is set to true" : ""),
+  );
+  if (listed.orgNote) details.push(listed.orgNote);
+  // The Kill switch takes true; for anything else, gh prompts for the value.
+  const { fix, notes } = splitFixes(
+    missing.map((v) =>
+      setCommand("variable", v.name, repo, killSwitch(v.name) ? ["--body", "true"] : []),
+    ),
+  );
+  return result(ID, TITLE, "fail", `${missing.length} required variable(s) missing.`, {
+    details: [...details, ...notes],
+    fix,
+  });
+}