@humwick/server 0.0.0-stage → 0.130.1

package/src/ci/monitor.ts

@@ -0,0 +1,528 @@
+import { selectedJobsVerdict } from "./job-verdict";
+import type { CiMonitorConfig, CiMonitorWatch } from "@humwick/sdk/ci-monitor";
+import { errMessage, stringValue, type Project, type ProjectRoot } from "@humwick/sdk";
+import { CI_MONITOR_FILE, cleanCiMonitorConfig } from "../ci-monitor.ts";
+import { projectFileRoots, readProjectRootFile } from "../project-file-source";
+import { listProjects, listWorkspaces, rootsForProject } from "../db";
+import type { CiShaObservation } from "../db/ci-incidents.ts";
+import { CI_GATE_RUN_RETENTION_MS, type CiGateBackfill, type CiGateRunRecord } from "../db/ci-gate-runs.ts";
+import { attemptOf, completedWatchRuns, gateRunRecord, runIdOf } from "./gate-history.ts";
+import { getGateway } from "../db/gateway.ts";
+import { isPathWithinBase } from "../utils";
+import { computeCiWatchKey, isFailingJob } from "./incidents.ts";
+import { ghApi, type GitHubApi } from "../issue-tracker/github.ts";
+import { resolveCoordinatorAgentId } from "../steward-identity.ts";
+import { notifyCiRootMissing, notifyCiRootMissingAsync, type CiRootMissing } from "./root-missing.ts";
+import { notifyCiTransition, notifyCiTransitionAsync, type CiTransitionNotifyResult } from "./transition-notify.ts";
+import { workflowNotTriggered } from "./not-triggered.ts";
+import type { CiJobRef, CiObservation, CiObservationStatus, CiRunRef } from "./provider.ts";
+
+export const GITHUB_API_TIMEOUT_MS = 15_000;
+export const CI_HISTORY_API_CALLS_PER_TICK = 8;
+export const CI_HISTORY_BUDGET_MS = 30_000;
+export const CI_HISTORY_API_TIMEOUT_MS = 5_000;
+
+class HistoryBudgetExhausted extends Error {}
+
+// The cursor rotates only admission order; persisted attempts/backfill pages own progress across restarts.
+let historyWatchOffset = 0;
+
+function historyBudget(deps: CiMonitorDeps) {
+  const now = deps.now ?? Date.now;
+  const deadline = now() + CI_HISTORY_BUDGET_MS;
+  let calls = 0;
+  const available = (count = 1) => !deps.signal?.aborted && calls + count <= CI_HISTORY_API_CALLS_PER_TICK && now() < deadline;
+  const api: GitHubApi = async (request) => {
+    const remainingMs = deadline - now();
+    if (!available() || remainingMs <= 0) throw new HistoryBudgetExhausted();
+    calls += 1;
+    return deps.api({ ...request, timeoutMs: Math.min(CI_HISTORY_API_TIMEOUT_MS, remainingMs) });
+  };
+  return { available, api, calls: () => calls };
+}
+
+export type GitHubWorkflowRun = {
+  id?: unknown;
+  name?: unknown;
+  html_url?: unknown;
+  head_sha?: unknown;
+  head_branch?: unknown;
+  event?: unknown;
+  status?: unknown;
+  conclusion?: unknown;
+  created_at?: unknown;
+  updated_at?: unknown;
+  run_attempt?: unknown;
+};
+
+type GitHubJob = {
+  id?: unknown;
+  name?: unknown;
+  html_url?: unknown;
+  status?: unknown;
+  conclusion?: unknown;
+  started_at?: unknown;
+};
+
+export interface GitHubRepository {
+  owner: string;
+  repo: string;
+}
+
+/**
+ * Parse only canonical github.com clone URLs. This is intentionally stricter
+ * than the generic git-remote parser: gitlab/foo or github.com.evil must never
+ * lead the monitor to issue a GitHub API request (and therefore never emit a
+ * misleading "repo not found" failure).
+ */
+export function parseGitHubRepositoryRemote(remoteUrl: string | undefined): GitHubRepository | null {
+  const raw = remoteUrl?.trim();
+  if (!raw) return null;
+  let path: string | undefined;
+  const scp = raw.match(/^(?:[^@/\s]+@)?github\.com:([^\s]+)$/i);
+  if (scp) {
+    path = scp[1] ?? "";
+  } else {
+    try {
+      const url = new URL(raw);
+      if (url.hostname.toLowerCase() !== "github.com") return null;
+      path = url.pathname;
+    } catch {
+      return null;
+    }
+  }
+  const parts = path.replace(/^\/+|\/+$/g, "").split("/");
+  if (parts.length !== 2) return null;
+  const [owner, repoWithSuffix] = parts;
+  if (!owner || !repoWithSuffix) return null;
+  const repo = repoWithSuffix.replace(/\.git$/i, "");
+  const component = /^[A-Za-z0-9_.-]+$/;
+  return component.test(owner) && component.test(repo) ? { owner, repo } : null;
+}
+
+function runId(run: GitHubWorkflowRun): string {
+  return typeof run.id === "number" || typeof run.id === "string" ? String(run.id) : "unknown";
+}
+
+function jobRef(job: GitHubJob): CiJobRef {
+  const startedAt = Date.parse(stringValue(job.started_at) ?? "");
+  return {
+    id: typeof job.id === "number" || typeof job.id === "string" ? String(job.id) : stringValue(job.html_url) ?? stringValue(job.name) ?? "unknown",
+    name: stringValue(job.name) ?? "unknown",
+    ...(stringValue(job.html_url) ? { url: stringValue(job.html_url) } : {}),
+    ...(stringValue(job.status) ? { status: stringValue(job.status) } : {}),
+    ...(stringValue(job.conclusion) ? { conclusion: stringValue(job.conclusion) } : {}),
+    ...(Number.isFinite(startedAt) ? { startedAt } : {}),
+  };
+}
+
+function runRef(run: GitHubWorkflowRun, jobs: CiJobRef[]): CiRunRef {
+  return {
+    id: runId(run),
+    ...(stringValue(run.html_url) ? { url: stringValue(run.html_url) } : {}),
+    status: stringValue(run.status) ?? "unknown",
+    ...(stringValue(run.conclusion) ? { conclusion: stringValue(run.conclusion) } : {}),
+    jobs,
+  };
+}
+
+function completionStatus(run: GitHubWorkflowRun): CiObservationStatus {
+  if (stringValue(run.status) !== "completed") return "pending";
+  const conclusion = stringValue(run.conclusion);
+  if (["success", "skipped", "neutral"].includes(conclusion ?? "")) return "green";
+  if (conclusion === "cancelled") return "cancelled";
+  return "failing";
+}
+
+function statusForSelectedJobs(jobs: CiJobRef[], selectedNames: string[], run: GitHubWorkflowRun): CiObservationStatus {
+  if (!jobs.some(job => selectedNames.includes(job.name))) {
+    const runStatus = completionStatus(run);
+    return jobs.length === 0 && (runStatus === "failing" || runStatus === "cancelled") ? runStatus : "pending";
+  }
+  const verdict = selectedJobsVerdict(jobs.map(({ name, status, conclusion }) => ({ name, status, conclusion })), selectedNames, stringValue(run.conclusion));
+  if (!jobs.length && stringValue(run.status) !== "completed") return "pending";
+  return verdict.status === "success" ? "green" : verdict.status === "cancelled" ? "cancelled"
+    : verdict.status === "pending" ? "pending" : "failing";
+}
+
+/** Every existing config omits `events`, so this is the exact legacy push-only behavior. */
+const DEFAULT_CI_MONITOR_EVENTS: readonly string[] = ["push"];
+
+export function matchingLatestRun(value: unknown, watch: CiMonitorWatch, branch: string): GitHubWorkflowRun | undefined {
+  const runs = workflowRuns(value);
+  const events = watch.events?.length ? watch.events : DEFAULT_CI_MONITOR_EVENTS;
+  // Creation order is authoritative: rerunning an old commit can update an old run today.
+  return runs.filter((run) => stringValue(run.name) === watch.workflow && stringValue(run.head_branch) === branch && events.includes(stringValue(run.event) ?? ""))
+    .sort((a, b) => {
+      const createdA = Date.parse(stringValue(a.created_at) ?? "");
+      const createdB = Date.parse(stringValue(b.created_at) ?? "");
+      return (Number.isFinite(createdA) && Number.isFinite(createdB) ? createdB - createdA : 0)
+        || Number(runId(b)) - Number(runId(a));
+    })[0];
+}
+
+export async function jobsForRun(repository: GitHubRepository, id: string, api: GitHubApi, attempt?: number): Promise<CiJobRef[]> {
+  const payload = await api({
+    method: "GET",
+    path: `/repos/${repository.owner}/${repository.repo}/actions/runs/${encodeURIComponent(id)}${attempt ? `/attempts/${attempt}` : ""}/jobs?per_page=100`,
+    timeoutMs: GITHUB_API_TIMEOUT_MS,
+  });
+  const jobs = payload && typeof payload === "object" && Array.isArray((payload as { jobs?: unknown }).jobs)
+    ? (payload as { jobs: GitHubJob[] }).jobs
+    : [];
+  return jobs.map(jobRef);
+}
+
+const RUNS_PER_PAGE = 100;
+/** The one-time backfill covers the whole 30d Gates window. */
+const BACKFILL_WINDOW_MS = 30 * 86_400_000;
+
+/** The branch's newest 100 runs; readiness and gate history both read this one page. */
+function fetchWatchRuns(repository: GitHubRepository, watch: CiMonitorWatch, api: GitHubApi, backfill?: { since: number; page: number }): Promise<unknown> {
+  // Events are matched client-side; a crowded 100-run page can yield missing (#1616).
+  const query = new URLSearchParams({ branch: watch.branch || "main", per_page: String(RUNS_PER_PAGE) });
+  if (backfill) {
+    query.set("status", "completed");
+    query.set("created", `>=${new Date(backfill.since).toISOString().replace(/\.\d+Z$/, "Z")}`);
+    query.set("page", String(backfill.page));
+  }
+  return api({ method: "GET", path: `/repos/${repository.owner}/${repository.repo}/actions/runs?${query.toString()}`, timeoutMs: GITHUB_API_TIMEOUT_MS });
+}
+
+function workflowRuns(value: unknown): GitHubWorkflowRun[] {
+  return value && typeof value === "object" && Array.isArray((value as { workflow_runs?: unknown }).workflow_runs)
+    ? (value as { workflow_runs: GitHubWorkflowRun[] }).workflow_runs
+    : [];
+}
+
+/**
+ * Record every new completed attempt on the page (#3370). Each attempt costs one jobs lookup, once: it is the only source
+ * of the first job start. A lookup failure skips that run until the next poll and returns false.
+ */
+async function recordGateHistory(repository: GitHubRepository, watch: CiMonitorWatch, watchKey: string, payload: unknown, deps: CiMonitorDeps, budget: ReturnType<typeof historyBudget>): Promise<boolean> {
+  const runs = completedWatchRuns(workflowRuns(payload), watch, watch.branch || "main", watch.events?.length ? watch.events : DEFAULT_CI_MONITOR_EVENTS,
+    (deps.now?.() ?? Date.now()) - CI_GATE_RUN_RETENTION_MS);
+  if (runs.length === 0) return true;
+  let complete = true;
+  const stored = new Set(await deps.storedRuns({ watchKey, runIds: runs.map((run) => runIdOf(run)!) }));
+  const records: CiGateRunRecord[] = [];
+  for (const latest of runs) {
+    if (deps.signal?.aborted) { complete = false; break; }
+    const id = runIdOf(latest)!;
+    try {
+      for (let attempt = 1; attempt <= attemptOf(latest); attempt += 1) {
+        if (stored.has(`${id}:${attempt}`)) continue;
+        if (!budget.available(attempt === attemptOf(latest) ? 1 : 2)) { complete = false; break; }
+        const run = attempt === attemptOf(latest) ? latest : await deps.api({ method: "GET", timeoutMs: GITHUB_API_TIMEOUT_MS,
+          path: `/repos/${repository.owner}/${repository.repo}/actions/runs/${encodeURIComponent(id)}/attempts/${attempt}` }) as GitHubWorkflowRun;
+        const jobs = await jobsForRun(repository, id, deps.api, attempt);
+        const record = gateRunRecord(watchKey, run, jobs, watch.jobs?.length ? statusForSelectedJobs(jobs, watch.jobs, run) : completionStatus(run), watch.jobs);
+        if (record) records.push(record);
+        else complete = false;
+      }
+    } catch (error) {
+      complete = false;
+      if (!(error instanceof HistoryBudgetExhausted)) console.warn(`ci monitor gate history ${repository.owner}/${repository.repo} run ${id} skipped until next poll: ${errMessage(error)}`);
+    }
+  }
+  if (records.length > 0) await deps.recordRuns({ runs: records });
+  return complete;
+}
+
+/** Begin the watch's backfill on first sight, which also drops pre-fix rows (#3384); returns where it stands. */
+async function startedBackfill(watchKey: string, deps: CiMonitorDeps): Promise<CiGateBackfill> {
+  const state = await deps.backfillState({ watchKey });
+  if (state) return state;
+  const since = (deps.now?.() ?? Date.now()) - BACKFILL_WINDOW_MS;
+  await deps.startBackfill({ watchKey, since });
+  return { since, nextPage: 1, doneAt: null };
+}
+
+/** Partial pages persist their completed attempts; only a fully recorded page advances the cursor. */
+async function backfillGateHistory(repository: GitHubRepository, watch: CiMonitorWatch, watchKey: string, state: CiGateBackfill, deps: CiMonitorDeps, budget: ReturnType<typeof historyBudget>): Promise<void> {
+  const payload = await fetchWatchRuns(repository, watch, deps.api, { since: state.since, page: state.nextPage });
+  if (!await recordGateHistory(repository, watch, watchKey, payload, deps, budget) || deps.signal?.aborted) return;
+  const last = workflowRuns(payload).length < RUNS_PER_PAGE;
+  await deps.advanceBackfill({ watchKey, nextPage: last ? state.nextPage : state.nextPage + 1, ...(last ? { doneAt: deps.now?.() ?? Date.now() } : {}) });
+}
+
+/** Query the newest push run for one configured GitHub Actions workflow. */
+export async function observeGitHubCiMonitorWatch(
+  repository: GitHubRepository,
+  watch: CiMonitorWatch,
+  api: GitHubApi = ghApi,
+  runsPayload?: unknown,
+): Promise<CiObservation> {
+  const branch = watch.branch || "main";
+  const payload = runsPayload ?? await fetchWatchRuns(repository, watch, api);
+  const run = matchingLatestRun(payload, watch, branch);
+  if (!run) {
+    return {
+      provider: "github-actions", repository: `${repository.owner}/${repository.repo}`, branch, workflow: watch.workflow,
+      ...(watch.jobs ? { jobs: watch.jobs } : {}), headSha: "unknown", status: "missing", runs: [], failingRuns: [], observedAt: Date.now(),
+    };
+  }
+  // Validate both red and green against the live branch tip, including the first poll
+  // after restart. A stale run is pending evidence, never an incident transition (#2735).
+  const branchPayload = await api({
+    method: "GET",
+    path: `/repos/${repository.owner}/${repository.repo}/branches/${encodeURIComponent(branch)}`,
+    timeoutMs: GITHUB_API_TIMEOUT_MS,
+  });
+  const tipSha = branchPayload && typeof branchPayload === "object"
+    ? stringValue((branchPayload as { commit?: { sha?: unknown } }).commit?.sha)
+    : undefined;
+  const isCurrentHead = Boolean(tipSha && tipSha === stringValue(run.head_sha));
+  const initialStatus = isCurrentHead ? completionStatus(run) : "pending";
+  const needsJobs = isCurrentHead && (Boolean(watch.jobs?.length) || initialStatus === "failing");
+  const jobs = needsJobs ? await jobsForRun(repository, runId(run), api) : [];
+  let status = isCurrentHead && watch.jobs?.length ? statusForSelectedJobs(jobs, watch.jobs, run) : initialStatus;
+  const updatedAt = Date.parse(stringValue(run.updated_at) ?? "");
+  if (status === "green" && !Number.isFinite(updatedAt)) status = "pending";
+  const ref = runRef(run, jobs);
+  const selectionAbsent = isCurrentHead && Boolean(watch.jobs?.length) && jobs.length > 0 && completionStatus(run) !== "pending"
+    && !jobs.some((job) => watch.jobs?.includes(job.name));
+  return {
+    provider: "github-actions",
+    repository: `${repository.owner}/${repository.repo}`,
+    branch,
+    workflow: watch.workflow,
+    ...(watch.jobs ? { jobs: watch.jobs } : {}),
+    headSha: stringValue(run.head_sha) ?? "unknown",
+    status,
+    ...(stringValue(run.conclusion) ? { conclusion: stringValue(run.conclusion) } : {}),
+    runs: [ref],
+    failingRuns: status === "failing" ? [ref] : [],
+    observedAt: Date.now(),
+    ...(status === "green" ? { runUpdatedAt: updatedAt } : {}),
+    ...(selectionAbsent && watch.jobs ? { unmatchedJobs: watch.jobs } : {}),
+  };
+}
+
+/**
+ * One watch's verdict on one exact commit (#3045 H5): its newest matching run for that head_sha, never the branch tip.
+ * No run: `wait` (when the wait began, and the branch's runs) decides whether one will never come (#3622).
+ */
+export async function observeGitHubCiSha(
+  repository: GitHubRepository,
+  watch: CiMonitorWatch,
+  headSha: string,
+  api: GitHubApi = ghApi,
+  wait?: { since: number; branchRuns: unknown; now: number },
+): Promise<Omit<CiShaObservation, "watchKey">> {
+  const branch = watch.branch || "main";
+  const payload = await api({
+    method: "GET",
+    path: `/repos/${repository.owner}/${repository.repo}/actions/runs?${new URLSearchParams({ head_sha: headSha, per_page: "100" }).toString()}`,
+    timeoutMs: GITHUB_API_TIMEOUT_MS,
+  });
+  const run = matchingLatestRun(payload, watch, branch);
+  if (!run || stringValue(run.head_sha) !== headSha) {
+    const notTriggered = wait && await workflowNotTriggered({ ...repository, watch, sha: headSha, api, ...wait });
+    return { headSha, status: notTriggered ? "not-triggered" : "pending", observedAt: Date.now() };
+  }
+  const runStatus = completionStatus(run);
+  const jobs = watch.jobs?.length || runStatus === "failing" ? await jobsForRun(repository, runId(run), api) : [];
+  const status = watch.jobs?.length ? statusForSelectedJobs(jobs, watch.jobs, run) : runStatus;
+  const failingJobs = status === "failing"
+    ? jobs.filter((job) => isFailingJob(job) && (!watch.jobs?.length || watch.jobs.includes(job.name))).map((job) => ({ name: job.name, ...(job.url ? { url: job.url } : {}) }))
+    : [];
+  return { headSha, status: status === "missing" ? "pending" : status, ...(stringValue(run.html_url) ? { runUrl: stringValue(run.html_url) } : {}), ...(failingJobs.length ? { failingJobs } : {}), observedAt: Date.now() };
+}
+
+/** A commit a workflow waits on for CI (#3045 H5), and the project root it runs in. */
+export interface CiShaWait {
+  sha: string;
+  cwd: string;
+  /** When the wait began. */
+  since: number;
+}
+
+export interface CiMonitorDeps {
+  listProjects: () => Project[];
+  rootsForProject: (projectId: string) => ProjectRoot[];
+  loadConfig: (root: ProjectRoot) => CiMonitorConfig | null | Promise<CiMonitorConfig | null>;
+  api: GitHubApi;
+  notify: typeof notifyCiTransition;
+  resolveCoordinator: (rootPath: string) => string | undefined;
+  /** Commits workflows wait on: each watch of their root reports its verdict on them too. */
+  shaWaits: () => Promise<CiShaWait[]>;
+  recordSha: (observation: CiShaObservation) => Promise<unknown>;
+  /** Gate history (#3370): each finished branch-tip run attempt, waited on or not. */
+  recordRuns: (args: { runs: CiGateRunRecord[] }) => Promise<unknown>;
+  /** The watch's one-time history backfill (#3384): its state, its start (which clears pre-fix rows), and its page cursor. */
+  backfillState: (args: { watchKey: string }) => Promise<CiGateBackfill | null>;
+  startBackfill: (args: { watchKey: string; since: number }) => Promise<unknown>;
+  advanceBackfill: (args: { watchKey: string; nextPage: number; doneAt?: number }) => Promise<unknown>;
+  /** "runId:attempt" keys already in gate history, read once per watch before any job lookup. */
+  storedRuns: (args: { watchKey: string; runIds: string[] }) => Promise<string[]>;
+  /** Drops per-commit observations past the longest wait; each tick runs it, so idle periods prune too. */
+  pruneShas: () => Promise<unknown>;
+  signal?: AbortSignal;
+  /** Overridable clock, so tests can advance past the never-matched grace period deterministically. */
+  now?: () => number;
+}
+
+function defaultCoordinator(rootPath: string): string | undefined {
+  const owner = listWorkspaces({ repoRoot: rootPath })[0]?.ownerAgentId;
+  return resolveCoordinatorAgentId(owner) ?? undefined;
+}
+
+const defaultDeps: CiMonitorDeps = {
+  listProjects,
+  rootsForProject,
+  loadConfig: async (root) => {
+    const content = await readProjectRootFile(root, CI_MONITOR_FILE);
+    return content === null ? null : cleanCiMonitorConfig(JSON.parse(content));
+  },
+  api: ghApi,
+  notify: notifyCiTransition,
+  resolveCoordinator: defaultCoordinator,
+  shaWaits: () => getGateway().read("workflowTask.ciWaits", undefined),
+  recordSha: (observation) => getGateway().write("ciIncidents.sha.record", observation),
+  recordRuns: (args) => getGateway().write("ciIncidents.gateRun.record", args),
+  storedRuns: (args) => getGateway().read("ciIncidents.gateRun.stored", args),
+  backfillState: (args) => getGateway().read("ciIncidents.gateBackfill.get", args),
+  startBackfill: (args) => getGateway().write("ciIncidents.gateBackfill.start", args),
+  advanceBackfill: (args) => getGateway().write("ciIncidents.gateBackfill.advance", args),
+  // The reader precheck keeps an idle tick off the writer lock.
+  pruneShas: async () => {
+    const now = Date.now();
+    if (await getGateway().read("ciIncidents.sha.hasExpired", { now })) await getGateway().write("ciIncidents.sha.prune", { now });
+  },
+};
+
+export interface CiMonitorRunResult {
+  projects: number;
+  roots: number;
+  watches: number;
+  skippedRoots: number;
+  errors: number;
+  /** Per-commit verdicts recorded for waiting workflows (#3045 H5). */
+  shaObservations: number;
+  /** Repositories with CI watch state whose checkout/config is gone (#3004). */
+  missingRoots: number;
+  readinessMs: number;
+  historyMs: number;
+  historyApiCalls: number;
+  transitions: Record<"went_red" | "back_green" | "never_matched" | "config_mismatch" | "none", number>;
+}
+
+/** Poll every opt-in project root. Individual roots/watches degrade independently. */
+export async function runCiMonitor(input: Partial<CiMonitorDeps> = {}): Promise<CiMonitorRunResult> {
+  const deps = { ...defaultDeps, ...input };
+  const now = deps.now ?? Date.now;
+  const startedAt = now();
+  const history = new Map<string, { repository: GitHubRepository; watch: CiMonitorWatch; runsPayload: unknown }>();
+  const result: CiMonitorRunResult = {
+    projects: 0, roots: 0, watches: 0, skippedRoots: 0, errors: 0, shaObservations: 0, missingRoots: 0,
+    readinessMs: 0, historyMs: 0, historyApiCalls: 0,
+    transitions: { went_red: 0, back_green: 0, never_matched: 0, config_mismatch: 0, none: 0 },
+  };
+  const useAsync = deps.notify === notifyCiTransition && !deps.now;
+  // A repository counts as missing only if no root of it has the config, so a duplicate root cannot flap the notice.
+  const missingRoots = new Map<string, CiRootMissing>();
+  const presentRepositories = new Set<string>();
+  const shaWaits = await deps.shaWaits().catch((error: unknown) => {
+    result.errors += 1;
+    console.warn(`ci monitor workflow waits skipped: ${errMessage(error)}`);
+    return [] as CiShaWait[];
+  });
+  const projects = deps.listProjects();
+  result.projects = projects.length;
+  for (const project of projects) {
+    if (deps.signal?.aborted) break;
+    for (const root of projectFileRoots(deps.rootsForProject(project.id))) {
+      if (deps.signal?.aborted) break;
+      result.roots += 1;
+      // This check comes before config parsing/API work so non-GitHub roots are
+      // a truly silent no-op, including roots whose files are inaccessible.
+      const repository = parseGitHubRepositoryRemote(root.remoteUrl);
+      if (!repository) {
+        result.skippedRoots += 1;
+        continue;
+      }
+      let config;
+      try {
+        config = await deps.loadConfig(root);
+      } catch (error) {
+        result.errors += 1;
+        console.warn(`ci monitor config ${root.rootPath} skipped: ${errMessage(error)}`);
+        continue;
+      }
+      const repositoryKey = `${repository.owner}/${repository.repo}`.toLowerCase();
+      if (config !== null) presentRepositories.add(repositoryKey);
+      else missingRoots.set(repositoryKey, { repository: `${repository.owner}/${repository.repo}`, rootPath: root.rootPath });
+      // Each waited commit once, from its earliest wait.
+      const shas = new Map<string, number>();
+      for (const wait of shaWaits) if (isPathWithinBase(wait.cwd, root.rootPath)) shas.set(wait.sha, Math.min(wait.since, shas.get(wait.sha) ?? wait.since));
+      for (const watch of config?.watches ?? []) {
+        if (deps.signal?.aborted) break;
+        result.watches += 1;
+        try {
+          const runsPayload = await fetchWatchRuns(repository, watch, deps.api);
+          const observation = await observeGitHubCiMonitorWatch(repository, watch, deps.api, runsPayload);
+          if (deps.signal?.aborted) break;
+          const notificationDeps = { resolveCoordinator: () => deps.resolveCoordinator(root.rootPath), ...(deps.now ? { now: deps.now() } : {}) };
+          const transition = useAsync
+            ? await notifyCiTransitionAsync(observation, notificationDeps)
+            : deps.notify(observation, notificationDeps);
+          result.transitions[transition.transition] += 1;
+          // The branch-tip row keys the watch; the same key carries its verdict on each waited commit.
+          const watchKey = computeCiWatchKey(observation);
+          history.set(watchKey, { repository, watch, runsPayload });
+          for (const [sha, since] of shas) {
+            if (deps.signal?.aborted) break;
+            const shaObservation = await observeGitHubCiSha(repository, watch, sha, deps.api, { since, branchRuns: runsPayload, now: now() });
+            if (deps.signal?.aborted) break;
+            await deps.recordSha({ watchKey, ...shaObservation });
+            result.shaObservations += 1;
+          }
+        } catch (error) {
+          result.errors += 1;
+          console.warn(`ci monitor ${repository.owner}/${repository.repo} ${watch.workflow}@${watch.branch} skipped: ${errMessage(error)}`);
+        }
+      }
+    }
+  }
+  try {
+    await deps.pruneShas();
+  } catch (error) {
+    result.errors += 1;
+    console.warn(`ci monitor per-commit observation prune skipped: ${errMessage(error)}`);
+  }
+  for (const [key, missing] of missingRoots) {
+    if (presentRepositories.has(key) || deps.signal?.aborted) continue;
+    const notifyDeps = { resolveCoordinator: () => deps.resolveCoordinator(missing.rootPath), ...(deps.now ? { now: deps.now() } : {}) };
+    try {
+      if (useAsync ? await notifyCiRootMissingAsync(missing, notifyDeps) : notifyCiRootMissing(missing, notifyDeps)) result.missingRoots += 1;
+    } catch (error) {
+      result.errors += 1;
+      console.warn(`ci monitor ${missing.repository} root-missing notice skipped: ${errMessage(error)}`);
+    }
+  }
+  // Readiness for ALL watches/commits precedes history, even if a history API or writer fails (#3396).
+  result.readinessMs = now() - startedAt;
+  const historyStartedAt = now();
+  const budget = historyBudget(deps);
+  const historyDeps = { ...deps, api: budget.api };
+  const entries = [...history];
+  const offset = historyWatchOffset % (entries.length || 1);
+  for (let i = 0; i < entries.length && budget.available(); i += 1) {
+    const index = (offset + i) % entries.length;
+    historyWatchOffset = index + 1;
+    const [watchKey, { repository, watch, runsPayload }] = entries[index]!;
+    try {
+      const backfill = await startedBackfill(watchKey, historyDeps);
+      await recordGateHistory(repository, watch, watchKey, runsPayload, historyDeps, budget);
+      if (backfill.doneAt === null && budget.available()) await backfillGateHistory(repository, watch, watchKey, backfill, historyDeps, budget);
+    } catch (error) {
+      if (error instanceof HistoryBudgetExhausted) break;
+      result.errors += 1;
+      console.warn(`ci monitor gate history ${repository.owner}/${repository.repo} retried next poll: ${errMessage(error)}`);
+    }
+  }
+  result.historyMs = now() - historyStartedAt;
+  result.historyApiCalls = budget.calls();
+  return result;
+}

package/src/ci/not-triggered.ts

@@ -0,0 +1,116 @@
+// #3622 — a main CI workflow filtered by `on.push.paths` (or skipped) never gets a run for a commit it does
+// not match, so a ci-main wait on that commit would sit until its deadline as a false "no verdict". With no
+// run past a short grace, either signal below says none will come. Any doubt answers "may still run": the
+// wait then keeps the old path to the real verdict or the deadline.
+import { errMessage, isRecord, stringValue } from "@humwick/sdk";
+import type { CiMonitorWatch } from "@humwick/sdk/ci-monitor";
+import type { GitHubApi } from "../issue-tracker/github.ts";
+
+/** GitHub creates a push run within seconds; this covers a slow one before no run means none will come. */
+export const NOT_TRIGGERED_GRACE_MS = 3 * 60_000;
+const TIMEOUT_MS = 15_000;
+/** The single-commit API lists at most this many files per page; a fuller list may be cut short. */
+const COMMIT_FILES_PAGE = 300;
+
+export interface NotTriggeredInput {
+  owner: string;
+  repo: string;
+  watch: CiMonitorWatch;
+  sha: string;
+  /** When the wait on `sha` began: the commit was on main by then. */
+  since: number;
+  /** The branch's newest workflow runs, as the monitor's tick already fetched them. */
+  branchRuns: unknown;
+  now: number;
+  api: GitHubApi;
+}
+
+/** Whether the watched workflow will not run on `sha`; called only when no run of it exists for `sha`. */
+export async function workflowNotTriggered(input: NotTriggeredInput): Promise<boolean> {
+  if (input.now - input.since < NOT_TRIGGERED_GRACE_MS) return false;
+  if (laterPushRun(input)) return true;
+  try {
+    return await pathFiltersSkip(input);
+  } catch (error) {
+    console.warn(`ci monitor ${input.owner}/${input.repo} ${input.watch.workflow}: path filter check on ${input.sha} skipped: ${errMessage(error)}`);
+    return false;
+  }
+}
+
+/** A later push to the branch already has a run of this workflow, so this commit's push was passed over. */
+function laterPushRun({ watch, sha, since, branchRuns }: NotTriggeredInput): boolean {
+  const runs = isRecord(branchRuns) && Array.isArray(branchRuns.workflow_runs) ? branchRuns.workflow_runs as unknown[] : [];
+  return runs.some((run) => isRecord(run) && stringValue(run.name) === watch.workflow && stringValue(run.head_branch) === (watch.branch || "main")
+    && stringValue(run.event) === "push" && stringValue(run.head_sha) !== sha && Date.parse(stringValue(run.created_at) ?? "") > since);
+}
+
+async function pathFiltersSkip({ owner, repo, watch, sha, api }: NotTriggeredInput): Promise<boolean> {
+  const base = `/repos/${owner}/${repo}`;
+  const listed = await api({ method: "GET", path: `${base}/actions/workflows?per_page=100`, timeoutMs: TIMEOUT_MS });
+  const workflows = isRecord(listed) && Array.isArray(listed.workflows) ? listed.workflows.filter(isRecord) : [];
+  const named = workflows.filter((workflow) => stringValue(workflow.name) === watch.workflow);
+  const file = named.length === 1 ? stringValue(named[0]!.path) : undefined;
+  if (!file) return false;
+  const contents = await api({ method: "GET", path: `${base}/contents/${file.split("/").map(encodeURIComponent).join("/")}?ref=${sha}`, timeoutMs: TIMEOUT_MS });
+  const encoded = isRecord(contents) ? stringValue(contents.content) : undefined;
+  if (!encoded) return false;
+  const filter = pushPathFilter(Bun.YAML.parse(Buffer.from(encoded, "base64").toString("utf8")));
+  if (!filter) return false;
+  const commit = await api({ method: "GET", path: `${base}/commits/${sha}`, timeoutMs: TIMEOUT_MS });
+  const files = isRecord(commit) && Array.isArray(commit.files) ? commit.files.filter(isRecord) : [];
+  if (files.length === 0 || files.length >= COMMIT_FILES_PAGE) return false;
+  // A rename is checked under both names, which can only find a match.
+  const paths = files.flatMap((entry) => [stringValue(entry.filename), stringValue(entry.previous_filename)].filter((path): path is string => !!path));
+  return !pushPathsMatch(filter, paths);
+}
+
+export type PushPathFilter = { paths: string[] } | { ignore: string[] };
+
+/** The workflow's `on.push` path filter, or undefined when push has none or the filter is outside what this models. */
+export function pushPathFilter(workflow: unknown): PushPathFilter | undefined {
+  const on = isRecord(workflow) ? workflow.on : undefined;
+  const push = isRecord(on) ? on.push : undefined;
+  if (!isRecord(push)) return undefined;
+  const paths = patterns(push.paths);
+  const ignore = patterns(push["paths-ignore"]);
+  if (paths && !ignore) return { paths };
+  // A `!` inside paths-ignore is undocumented; leave it to the real run.
+  if (ignore && !paths && !ignore.some((pattern) => pattern.startsWith("!"))) return { ignore };
+  return undefined;
+}
+
+// GitHub's `?` and `+` quantify the preceding character and `[...]` is a class: not modeled, so such a filter is unknown.
+const UNMODELED = /[?+[\]{}]/;
+
+function patterns(value: unknown): string[] | undefined {
+  if (!Array.isArray(value) || value.length === 0) return undefined;
+  const list = value.map((entry) => (typeof entry === "string" ? entry.trim() : ""));
+  return list.every((pattern) => pattern && !UNMODELED.test(pattern)) ? list : undefined;
+}
+
+/**
+ * GitHub filter globs: `*` stays within a path segment, `**` crosses them, `**\/` may match no directory.
+ * `loose` lets a literal path also cover what is under it: used only where over-matching keeps the wait.
+ */
+function globRegex(pattern: string, loose: boolean): RegExp {
+  let source = "";
+  for (let i = 0; i < pattern.length; i += 1) {
+    if (pattern.startsWith("**/", i)) { source += "(?:.*/)?"; i += 2; }
+    else if (pattern.startsWith("**", i)) { source += ".*"; i += 1; }
+    else if (pattern[i] === "*") source += "[^/]*";
+    else source += pattern[i]!.replace(/[.^$()|\\/]/g, "\\$&");
+  }
+  return new RegExp(`^${source}${loose && !pattern.includes("*") ? "(?:/.*)?" : ""}$`);
+}
+
+/** Whether a push changing `files` triggers the workflow under `filter` (patterns in order; a later `!` excludes). */
+export function pushPathsMatch(filter: PushPathFilter, files: string[]): boolean {
+  if ("ignore" in filter) {
+    const ignored = filter.ignore.map((pattern) => globRegex(pattern, false));
+    return files.some((file) => !ignored.some((regex) => regex.test(file)));
+  }
+  const rules = filter.paths.map((pattern) => pattern.startsWith("!")
+    ? { include: false, regex: globRegex(pattern.slice(1), false) }
+    : { include: true, regex: globRegex(pattern, true) });
+  return files.some((file) => rules.reduce((included, rule) => (rule.regex.test(file) ? rule.include : included), false));
+}