@opsee/cli 0.11.13 → 0.11.18

package/src/commands/foreman-up.ts

@@ -19,18 +19,22 @@
  */
 import { mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
+import type { Account } from "../foreman/account.js";
 import type { AccountStore } from "../foreman/account-store.js";
 import { OutboxTracker } from "../foreman/core/outbox-tracker.js";
 import type { RunRequestRow } from "../foreman/core/process-table.js";
 import { ForemanError, PAUSE_POLL_MS, reconcileTick, runForeman } from "../foreman/core/run.js";
-import { count } from "../foreman/core/text.js";
+import { count, printableOneLine } from "../foreman/core/text.js";
 import { sleepWarning } from "../foreman/host.js";
 import { EXIT_ANOTHER_FOREMAN } from "../foreman/service-unit.js";
 import { errorMessage } from "../foreman/worker-process.js";
 
 /** The daemon's exit when another Foreman holds the pid file (defined beside the units that read it). */
 export { EXIT_ANOTHER_FOREMAN };
-import { checkLearningsFile, chooseAccount, liveDaemonPid, reconciled, runDepsFor, summarize, type ForemanLocal, type SharedDeps } from "./foreman.js";
+import { checkLearningsFile, chooseAccount, liveDaemonPid, reconciled, runDepsFor, summarize, type ForemanLocal, type RankingArgs, type SharedDeps } from "./foreman.js";
+import { allBackedOffUntil, AnthropicUsageSource, chooseToPoll, pollable, pollUsage, type UsageSource } from "../foreman/usage-poller.js";
+import { describeUntil } from "../foreman/usage.js";
+import { usageByAccount, type UsageStore } from "../foreman/usage-store.js";
 
 /** How long the daemon idles between ticks when no Run is requested; the same interval a
  * foreground Run waits through a pause at. */
@@ -48,6 +52,15 @@ export interface ForemanUpDeps extends SharedDeps {
   tickMs?: number;
   sleep?: (ms: number) => Promise<void>;
   pid?: number;
+  /** The usage source the tick polls through; the real Anthropic one by default, a fake in tests.
+   * Nothing but this reads a credential (ADR-0013 as amended, and usage-poller.ts). */
+  usageSource?: UsageSource;
+  /** How often usage is refreshed while a Run is in flight; `USAGE_POLL_INTERVAL_MS` when unset. */
+  usagePollMs?: number;
+  /** How the daemon's Runs order the Lanes a Ready Task may use (`RankingArgs`). The daemon is
+   * where this matters most: its Runs are the ones going at 3am with nobody to notice an Account
+   * being drained into a refusal. */
+  ranking?: RankingArgs;
 }
 
 export async function runForemanUp(deps: ForemanUpDeps): Promise<number> {
@@ -77,6 +90,27 @@ export async function runForemanUp(deps: ForemanUpDeps): Promise<number> {
   out("Waiting for Runs: opsee foreman run <initiativeId> [--account <name>] [--once] [--task <id>] from another terminal");
   if (await deps.host.isLaptop()) for (const line of sleepWarning(deps.host.platform)) out(line);
 
+  // Across a Run, not only between them. `serveRun` below blocks for as long as the Run lasts, and
+  // a Run is exactly when knowing an idle Account's Headroom matters: those are the Accounts that
+  // dispatch is choosing between. Unref'd so it can never hold the daemon open, and guarded so a
+  // slow poll cannot overlap itself.
+  let polling = false;
+  /** One at a time, whoever asked. The guard used to defend the timer from itself while the loop
+   * below called the same function on its own tick: both read the store before either wrote, and
+   * `chooseToPoll` is deterministic, so they picked the same Accounts and asked about them twice. */
+  const pollOnce = () => {
+    if (polling) return;
+    polling = true;
+    void pollUsageTick(deps).finally(() => {
+      polling = false;
+    });
+  };
+  // Once at start, so a daemon that has just come up does not wait a whole interval to learn
+  // anything, and then on the timer alone.
+  pollOnce();
+  const usageTimer = setInterval(pollOnce, deps.usagePollMs ?? USAGE_POLL_INTERVAL_MS);
+  usageTimer.unref?.();
+
   try {
     while (!stopping) {
       await idleReconcile(deps, repoRoot);
@@ -89,12 +123,152 @@ export async function runForemanUp(deps: ForemanUpDeps): Promise<number> {
       await Promise.race([sleep(tickMs), deps.untilStop]);
     }
   } finally {
+    clearInterval(usageTimer);
     rmSync(local.pidFile, { force: true });
   }
   out("Foreman down");
   return 0;
 }
 
+/** How often the daemon refreshes usage while it is otherwise busy. `chooseToPoll` decides whether
+ * there is anything worth asking, so a tick that finds nothing due costs one store read. */
+export const USAGE_POLL_INTERVAL_MS = 60_000;
+
+/**
+ * What the last pass said, per daemon and per kind of thing there is to say.
+ *
+ * Keyed by the deps object, which is one per daemon, the same way `core/run.ts` remembers the
+ * threshold holds it has already reported. Without it a poller that cannot reach the endpoint says
+ * so on every pass for as long as that lasts: one real overnight Run produced a hundred and forty
+ * identical lines, which is not a log, it is a wall.
+ *
+ * Per kind, not one slot for the whole daemon, because the kinds interleave: a pass that polls and
+ * fails and a pass that finds nothing due alternate all night, and a single slot would find each
+ * one different from the last and say both, every cycle.
+ */
+const lastSaid = new WeakMap<object, Map<SaidKind, string>>();
+
+type SaidKind = "failure" | "idle" | "threw";
+
+/**
+ * Says `line` when `state` differs from what this kind last reported, and remembers `state` either
+ * way.
+ *
+ * The separation of the two arguments is the whole point, and it was learned the hard way: the
+ * first version of this compared the rendered line, which carries a relative time. At the daemon's
+ * one-minute tick "the first lifts in 58m" becomes "in 57m" and never matches itself, so the fix
+ * for a hundred and forty lines produced four hundred and eighty. **`state` must describe which
+ * Accounts and why, and never when.** A time that is only rendered is a time that cannot flood.
+ *
+ * `line` may be undefined with a `state` that is defined: that is how a kind records that its
+ * trouble is over without announcing it, so the next occurrence is news again rather than a repeat.
+ */
+function say(deps: ForemanUpDeps, kind: SaidKind, state: string, line: () => string | undefined): void {
+  let said = lastSaid.get(deps);
+  if (!said) lastSaid.set(deps, (said = new Map()));
+  if (said.get(kind) === state) return;
+  said.set(kind, state);
+  const text = line();
+  if (text !== undefined) deps.out(text);
+}
+
+/**
+ * Refreshes what is known about a couple of Accounts' rate-limit windows (usage-poller.ts).
+ *
+ * In the daemon because this is the process that is up when nothing is running: in-band usage only
+ * arrives for an Account that has a Worker going, so without this an idle Account is never measured
+ * and `best` has nothing to rank it by. `chooseToPoll` decides which are worth a request — never
+ * more than `POLL_BATCH` at a time, and never an Account that has nothing to report.
+ *
+ * Every failure here is swallowed on purpose. Usage is an optimisation: a daemon that stopped
+ * Reconciling because a usage request timed out would have traded the thing that matters for the
+ * thing that helps.
+ *
+ * Exported for the test only: what this says, and how often, is the whole of what needed fixing.
+ */
+export async function pollUsageTick(deps: ForemanUpDeps): Promise<void> {
+  const store = deps.usage;
+  if (!store) return;
+  try {
+    const source = deps.usageSource ?? new AnthropicUsageSource();
+    const registered = deps.store.load();
+    const now = Date.now();
+    const accounts = chooseToPoll(registered, usageByAccount(store), now);
+    if (accounts.length === 0) {
+      // Not "nothing happened": the one empty pass worth a word is the one the poller cannot leave
+      // by itself, which from outside is indistinguishable from working normally.
+      const lifts = allBackedOffUntil(registered, usageByAccount(store), now);
+      const stuck = lifts === undefined ? [] : registered.filter(pollable).map((a) => a.name);
+      say(deps, "idle", stuck.join(","), () => (lifts === undefined ? undefined : idleLine(lifts, now)));
+      return;
+    }
+    await pollUsage({ store, source, accounts });
+    // Read back after the requests rather than before: the failures being described were written
+    // during them, and a `now` from before would date them by however long the endpoint took.
+    const trouble = troubled(store, registered, Date.now());
+    say(
+      deps,
+      "failure",
+      trouble.map((t) => `${t.name}=${t.reason}`).join(";"),
+      () => (trouble.length === 0 ? undefined : failureLine(trouble)),
+    );
+  } catch (error) {
+    say(deps, "threw", errorMessage(error), () => `usage: this pass failed, retrying next tick: ${errorMessage(error)}`);
+  }
+}
+
+/** An Account the poller could not measure: what refused it, and when it will be asked again. */
+interface Troubled {
+  name: string;
+  reason: string;
+  again: string | undefined;
+}
+
+/**
+ * Every pollable Account whose last attempt failed, by name.
+ *
+ * The whole fleet rather than the batch just asked, so what is said describes a state rather than
+ * an event: an Account that failed two passes ago and has not been retried since is still failing,
+ * and dropping it would make the line change — and so be said again — every time the batch rotates.
+ *
+ * A failure with no reason recorded still counts. The reason is redacted before it is stored and
+ * redaction can empty it, and an Account silently missing from the line is worse than one whose
+ * cause is unknown.
+ */
+function troubled(store: UsageStore, registered: readonly Account[], now: number): Troubled[] {
+  const trouble: Troubled[] = [];
+  for (const account of registered) {
+    if (!pollable(account)) continue;
+    const record = store.loadOne(account.name);
+    if (!record?.pollFailedAt) continue;
+    const until = record.pollBackoffUntil ? Date.parse(record.pollBackoffUntil) : Number.NaN;
+    trouble.push({
+      name: account.name,
+      reason: printableOneLine(record.pollFailure ?? "") || "no reason recorded",
+      again: Number.isFinite(until) && until > now ? describeUntil(record.pollBackoffUntil!, now) : undefined,
+    });
+  }
+  return trouble;
+}
+
+/**
+ * What went wrong, per Account, and when each will be asked again.
+ *
+ * The line it replaces was `measured 0 of 1 Accounts`, which answers neither question a reader has.
+ * A count is only useful when the interesting part is how many; here the interesting part is which,
+ * why, and whether it is going to sort itself out.
+ */
+function failureLine(trouble: readonly Troubled[]): string {
+  const said = trouble.map((t) => `${t.name} (${t.reason}${t.again ? `, asking again ${t.again}` : ""})`);
+  return `usage: could not measure ${said.join("; ")}. What is already known is shown by "opsee foreman account usage".`;
+}
+
+/** Said once when every Account the poller speaks to is inside a rate-limit window the endpoint
+ * imposed, so that a reader knows the silence that follows is a wall and not a working daemon. */
+function idleLine(lifts: number, now: number): string {
+  return `usage: every Account is inside a rate-limit window the endpoint imposed, so nothing more can be measured; the first lifts ${describeUntil(new Date(lifts).toISOString(), now)}. Runs are unaffected — dispatch falls back to reacting to limits as they arrive.`;
+}
+
 /** The idle tick: every (Initiative, Account) the Process Table has rows for gets a Reconcile, and
  * whatever the outbox holds is offered to the backend. One group failing never stops the others. */
 async function idleReconcile(deps: ForemanUpDeps, repoRoot: string): Promise<void> {
@@ -135,8 +309,14 @@ async function serveRun(deps: ForemanUpDeps, repoRoot: string, request: RunReque
   }
   out(`${label}: starting`);
   try {
-    const account = chooseAccount(deps.store.load(), request.account);
-    const outcome = await (deps.run ?? runForeman)(await runDepsFor(deps, account, repoRoot, local, { maxTurns: request.maxTurns, stallTimeoutMs: request.stallTimeoutMs }), {
+    // `mustChoose`: there is no operator here to answer "which Account". A queued Run was accepted
+    // with exit 0 in somebody's terminal, and refusing it now would mark it failed on this one.
+    const account = chooseAccount(deps.store.load(), request.account, {
+      strategy: deps.ranking?.strategy,
+      usage: usageByAccount(deps.usage),
+      mustChoose: true,
+    });
+    const outcome = await (deps.run ?? runForeman)(await runDepsFor(deps, account, repoRoot, local, { maxTurns: request.maxTurns, stallTimeoutMs: request.stallTimeoutMs, ranking: deps.ranking }), {
       initiativeId: request.initiativeId,
       once: request.once,
       taskId: request.taskId,

package/src/commands/foreman.ts

@@ -1,8 +1,11 @@
 import { execFileSync } from "node:child_process";
 import { existsSync, readFileSync, statSync } from "node:fs";
 import { uptime } from "node:os";
-import { AccountError, type Account } from "../foreman/account.js";
+import { AccountError, isUsableNow, type Account } from "../foreman/account.js";
 import type { AccountStore } from "../foreman/account-store.js";
+import { usageByAccount, type UsageStore } from "../foreman/usage-store.js";
+import type { AccountUsage } from "../foreman/usage.js";
+import { DEFAULT_RANKING_POLICY, rankLanes, type DispatchStrategy, type RankingPolicy } from "../foreman/core/scheduler.js";
 import { defectFilerWith } from "../foreman/core/defects.js";
 import { baseGateCommands, gatesWith, noGatesWarning } from "../foreman/core/gates.js";
 import { handOffWith } from "../foreman/core/handoff.js";
@@ -72,6 +75,10 @@ export interface SharedDeps {
    * commands have one; it is optional here so a test can build a Run's deps without a store, which
    * then Pauses nothing and Fails over nowhere. */
   store?: AccountStore;
+  /** Where each Account's rate-limit windows are kept (usage-store.ts). Both commands have one;
+   * optional here so a test can build a Run's deps without it, which then ranks Lanes in
+   * registration order and holds nothing back — exactly the behaviour before usage existed. */
+  usage?: UsageStore;
   /** Test seams; the real Workspace manager, code host, Hand-off and loop by default. Every git
    * call the commands make goes through `git`, the guarded runner (ADR-0003). */
   workspacesFor?: (repoRoot: string, log: (line: string) => void, git: GitRunner) => RunDeps["workspaces"];
@@ -92,7 +99,19 @@ export interface SharedDeps {
 }
 
 /** Options of `opsee foreman run`. */
-export interface ForemanRunArgs {
+/**
+ * How a Run orders the Lanes a Ready Task may use (core/scheduler.ts `rankLanes`), as the command
+ * line states it. Shared by `foreman run` and `foreman up` because the daemon's Runs want it just
+ * as much as a foreground one — more, since they are the ones running at 3am.
+ */
+export interface RankingArgs {
+  /** "order" (the default), "best", or "consume-first"; refused by the parser otherwise. */
+  strategy?: DispatchStrategy;
+  hysteresisPct?: number;
+  cooldownMs?: number;
+}
+
+export interface ForemanRunArgs extends RankingArgs {
   initiativeId: number;
   account?: string;
   once: boolean;
@@ -101,6 +120,18 @@ export interface ForemanRunArgs {
   stallTimeoutMs?: number;
 }
 
+/** Options of `opsee foreman up`: the daemon takes the dispatch policy and nothing else. */
+export type ForemanUpArgs = RankingArgs;
+
+/** The policy a Run uses, from what the command line said and the defaults for the rest. */
+export function rankingPolicyFrom(args: RankingArgs): RankingPolicy {
+  return {
+    strategy: args.strategy ?? DEFAULT_RANKING_POLICY.strategy,
+    hysteresisPct: args.hysteresisPct ?? DEFAULT_RANKING_POLICY.hysteresisPct,
+    cooldownMs: args.cooldownMs ?? DEFAULT_RANKING_POLICY.cooldownMs,
+  };
+}
+
 export interface ForemanRunDeps extends SharedDeps {
   store: AccountStore;
   /** The repository the Run works on: the checkout the command runs in. */
@@ -175,7 +206,43 @@ export function refuseQuarantinedRun(account: Account): void {
 }
 
 /** Picks the Account a Run uses: the named one, or the only one registered. */
-export function chooseAccount(accounts: Account[], name: string | undefined): Account {
+/** What `chooseAccount` may use to make the choice itself, when nobody named an Account. */
+export interface AccountChoice {
+  /** The Run's dispatch strategy. `best` and `consume-first` are answers to "which Account"; under
+   * `order` nothing has said, so an ambiguous fleet is refused unless nobody can be asked. */
+  strategy?: DispatchStrategy;
+  usage?: ReadonlyMap<string, AccountUsage>;
+  /**
+   * There is no human to ask, so an Account must come back whatever the strategy.
+   *
+   * What a daemon serving a queued Run passes. The refusal exists so an operator standing at a
+   * terminal can say which Account they meant; throwing where nobody is standing turns a Run that
+   * was accepted with exit 0 into one marked failed minutes later on somebody else's terminal.
+   */
+  mustChoose?: boolean;
+  /** Which Accounts another Foreman already has Workers on (the Process Table), where the caller
+   * can say. Choosing one of those is choosing a refusal a moment later. */
+  busy?: (name: string) => boolean;
+  /** The rest of the refusal's sentence, for callers that have another way out to offer. Passed
+   * rather than assumed so a command cannot advertise a flag its own parser rejects. */
+  hint?: string;
+  /** Test seam; `Date.now` otherwise, as everywhere else in this package. */
+  now?: number;
+}
+
+/**
+ * The Account a Run starts on: Lane 0.
+ *
+ * A name always wins — a human who said which Account they meant is not overruled. Otherwise, with
+ * several registered, the strategy decides where it can: `best` and `consume-first` are already
+ * statements about which Account to prefer, so making the operator repeat that by hand was ceremony
+ * over a choice the policy had made. Under `order` nothing has said, and the refusal stands.
+ *
+ * What this picks matters less than it looks: every sibling of the same vendor becomes a Failover
+ * Lane regardless (`failoverAccountsFor`), and `rankLanes` reorders them before any Task is placed.
+ * Lane 0 is where the list starts, not where the work goes.
+ */
+export function chooseAccount(accounts: Account[], name: string | undefined, choice: AccountChoice = {}): Account {
   if (accounts.length === 0) {
     throw new AccountError("No Account registered; register one with: opsee foreman account add --vendor claude --config-dir <dir>");
   }
@@ -184,10 +251,34 @@ export function chooseAccount(accounts: Account[], name: string | undefined): Ac
     if (!found) throw new AccountError(`No Account named "${name}" (have: ${accounts.map((a) => a.name).join(", ")})`);
     return found;
   }
-  if (accounts.length > 1) {
-    throw new AccountError(`Several Accounts are registered; pass --account <name> (have: ${accounts.map((a) => a.name).join(", ")})`);
+  if (accounts.length === 1) return accounts[0];
+
+  const ranked = choice.strategy === "best" || choice.strategy === "consume-first";
+  if (!ranked && !choice.mustChoose) {
+    throw new AccountError(
+      `Several Accounts are registered, and nothing has said which to start on (have: ${accounts.map((a) => a.name).join(", ")}). ` +
+        `Name one with --account <name>${choice.hint ? `, ${choice.hint}` : ""}.`,
+    );
   }
-  return accounts[0];
+
+  const now = choice.now ?? Date.now();
+  // An Account that cannot take work would start the Run on a Lane with no Slots, or on one another
+  // Foreman is already running — which `refuseSecondRunOnAccount` then hard-fails a moment later,
+  // where an idle sibling would have worked. Passed over unless they all are, in which case the Run
+  // still starts and reports the state properly (`refuseQuarantinedRun`, core/run.ts).
+  const usable = accounts.filter((a) => isUsableNow(a, now, choice.busy));
+  const among = usable.length > 0 ? usable : accounts;
+  // `order` is registration order, so under it — or under no strategy at all, which is what a
+  // daemon serving a queued Run has — the first registered Account is the answer, not a refusal.
+  const strategy: DispatchStrategy = ranked ? choice.strategy! : "order";
+  const { lanes } = rankLanes(
+    among.map((account) => ({ account })),
+    choice.usage ?? new Map(),
+    { ...DEFAULT_RANKING_POLICY, strategy },
+    {},
+    now,
+  );
+  return lanes[0].account;
 }
 
 /** The repository's learnings file, or the default with the reason the configured one is refused
@@ -210,7 +301,7 @@ export async function runDepsFor(
   account: Account,
   repoRoot: string,
   local: ForemanLocal | undefined,
-  options: { maxTurns?: number; stallTimeoutMs?: number; failover?: boolean },
+  options: { maxTurns?: number; stallTimeoutMs?: number; failover?: boolean; ranking?: RankingArgs },
 ): Promise<RunDeps> {
   const outbox = local ? new OutboxTracker(shared.tracker, local.table, shared.out) : undefined;
   const git = guardedGit(runGit, repoRoot);
@@ -279,6 +370,10 @@ export async function runDepsFor(
     failover,
     pinnable,
     accounts: shared.store,
+    usage: shared.usage,
+    // The dispatch policy the command line asked for, defaults filled in. The daemon's idle
+    // Reconcile passes none and gets the defaults, which it never uses: it dispatches nothing.
+    ranking: rankingPolicyFrom(options.ranking ?? {}),
     workspaces,
     install: shared.install,
     handOff,
@@ -422,25 +517,43 @@ export function reconciled(results: Array<Dispatch | BlockedTask>): RunOutcome {
  * push...), so a script can tell the two apart. The code host is chosen from origin before the
  * first dispatch: a repository the Foreman could never hand off from is refused up front. */
 export async function runForemanRun(deps: ForemanRunDeps, args: ForemanRunArgs): Promise<number> {
-  const account = chooseAccount(deps.store.load(), args.account);
   const repoRoot = await deps.repoRoot();
   if (!repoRoot) throw new ForemanError("Not inside a git repository; run foreman run from the checkout the Initiative's Tasks are about.");
 
   const daemon = deps.local ? liveDaemonPid(deps.local.pidFile, deps.local.probe) : undefined;
   if (daemon !== undefined && deps.local) {
+    // Resolved here when it is unambiguous — a name was given, or there is only one Account — and
+    // left to the daemon when it is not. A `RunRequestRow` carries no strategy, so the daemon's
+    // policy is what will apply; deciding here would pick under this terminal's flags and run under
+    // the daemon's, which is the mismatch that made `foreman run` demand `--account` from a fleet
+    // the daemon was already ranking. A name is still validated now, so a typo is refused by the
+    // terminal that made it rather than by a daemon somewhere else an hour later.
+    const registered = deps.store.load();
+    const queued = args.account !== undefined ? chooseAccount(registered, args.account).name : registered.length === 1 ? registered[0].name : undefined;
     const request = deps.local.table.requestRun({
       initiativeId: args.initiativeId,
-      account: account.name,
+      account: queued,
       once: args.once,
       taskId: args.taskId,
       repoRoot,
       maxTurns: args.maxTurns,
       stallTimeoutMs: args.stallTimeoutMs,
     });
-    deps.out(`Run #${request.id} queued for the Foreman daemon (pid ${daemon}) on Account "${account.name}"; it starts on the daemon's next tick and its output goes to the daemon's terminal.`);
+    deps.out(
+      `Run #${request.id} queued for the Foreman daemon (pid ${daemon}) ${queued === undefined ? "on the Account its own strategy picks" : `on Account "${queued}"`}; ` +
+        `it starts on the daemon's next tick and its output goes to the daemon's terminal.`,
+    );
     return 0;
   }
 
+  // In the foreground this terminal's own flags are the policy, so they decide Lane 0 too.
+  const account = chooseAccount(deps.store.load(), args.account, {
+    strategy: args.strategy,
+    usage: usageByAccount(deps.usage),
+    busy: busyAccounts(deps),
+    hint: "or let the policy choose with --strategy best, which starts on whichever Account has the most Headroom",
+  });
+
   // In the foreground this process is the one running the Workers, so it is the one that must not
   // double the Account's cap on top of another Foreman's.
   if (deps.local) refuseSecondRunOnAccount(deps.local.table, account, deps.isAlive ?? pidIsAlive);
@@ -450,7 +563,7 @@ export async function runForemanRun(deps: ForemanRunDeps, args: ForemanRunArgs):
   const learningsConfig = checkLearningsFile(repoRoot);
   if (learningsConfig.refused) deps.out(`learnings: ${learningsConfig.refused}`);
   // In the foreground there is no daemon to hand a paused Run back to, so it waits for the resume.
-  const outcome = await (deps.run ?? runForeman)(await runDepsFor(deps, account, repoRoot, deps.local, { maxTurns: args.maxTurns, stallTimeoutMs: args.stallTimeoutMs }), {
+  const outcome = await (deps.run ?? runForeman)(await runDepsFor(deps, account, repoRoot, deps.local, { maxTurns: args.maxTurns, stallTimeoutMs: args.stallTimeoutMs, ranking: args }), {
     initiativeId: args.initiativeId,
     once: args.once,
     taskId: args.taskId,
@@ -461,5 +574,14 @@ export async function runForemanRun(deps: ForemanRunDeps, args: ForemanRunArgs):
   return needingAttention === 0 && blocked === 0 ? 0 : 1;
 }
 
+/** Which Accounts another Foreman has live Workers on, for the auto-pick to pass over. Empty when
+ * this process keeps no machine-local record, which is every test that hands in no `local`. */
+function busyAccounts(deps: Pick<ForemanRunDeps, "local" | "isAlive">): ((name: string) => boolean) | undefined {
+  const table = deps.local?.table;
+  if (!table) return undefined;
+  const isAlive = deps.isAlive ?? pidIsAlive;
+  return (name) => otherForemanWorkersOn(table, name, Date.now(), { isAlive }).length > 0;
+}
+
 /** The repository the current directory is in, for the real command. */
 export const currentRepoRoot = () => repoRootOf(process.cwd());