@opsee/cli 0.11.9

package/src/foreman/account.ts

@@ -0,0 +1,474 @@
+import { constants } from "node:fs";
+import { homedir } from "node:os";
+import { resolve } from "node:path";
+import { isVendor, VENDORS, type Vendor } from "./vendor.js";
+import type { AccountStore } from "./account-store.js";
+import { oneLine, printableOneLine } from "./core/text.js";
+
+/**
+ * An Account (see ../../CONTEXT.md): one identity for one vendor, registered by path or key
+ * reference only. What is stored here is the whole of what the Foreman knows about a credential;
+ * the credential itself is never read (ADR-0013), and account-boundary.test.ts holds it to that.
+ */
+export type AccountType = "subscription" | "api_key";
+
+/**
+ * Concurrency cap a new Account gets (story 27). Deliberately low: two Slots is one implementer and
+ * the reserved Verifier Slot beside it (core/scheduler.ts), which is what one person at one keyboard
+ * looks like to the vendor. Raising it is always an explicit act — `--cap` at `foreman account add`,
+ * or `foreman account set <name> --cap <n>` afterwards — never something the Foreman does itself.
+ */
+export const DEFAULT_CAP = 2;
+
+/** The most Slots one Account may be given. An Account is one vendor identity (ADR-0013) and the
+ * cap is what that identity may run at once, so a number far past this is a typo — a `--cap 100`
+ * meant as `--cap 10`, a millisecond value in the wrong flag — much more often than an intent, and
+ * nothing below would refuse it: the Run would obligingly put a hundred Workers on one login. */
+export const MAX_CAP = 32;
+
+/** The least a stall timeout may be set to. Below this a Worker that is only thinking, or running
+ * the project's test suite, is stopped as `stalled` every time, and the Task spends its attempts
+ * (`DEFAULT_ATTEMPT_RETRIES`) in seconds without a single turn having had a chance. */
+export const MIN_STALL_TIMEOUT_MS = 10_000;
+
+/** Set while the vendor has reported a rate limit (story 30): when the pause lifts, and what the
+ * vendor said. Written by the Run that saw the limit (core/run.ts) through the AccountStore, read
+ * back by every Run that fills Slots, so a pause one Foreman hit is a pause the next one honours. */
+export interface PausedState {
+  until: string;
+  reason?: string;
+}
+
+/**
+ * Set once the Account's credential is judged dead (OPS-288): when it happened, and the vendor's
+ * own words or the Foreman's for why. Unlike `paused` it carries no reset, because there is nothing
+ * to wait for — a quarantine is left only when a human re-adds the Account or clears it by hand
+ * (`clearQuarantine`, `opsee foreman account resume`).
+ */
+export interface QuarantinedState {
+  at: string;
+  reason: string;
+}
+
+/**
+ * Credential failures in a row on this Account, and what the last one said (OPS-288). Reset by any
+ * turn that completes on it, so this counts a streak and never a total.
+ *
+ * In the store rather than in a Run because the fact outlives the Run, for the same reason `paused`
+ * is: a dead login is still dead after a restart, and the second failure that proves it may well be
+ * the next Foreman's. Written only while a streak is open, so an Account that is working pays no
+ * store write per turn.
+ */
+export interface CredentialFailures {
+  count: number;
+  at: string;
+  reason: string;
+}
+
+/**
+ * How many credential failures in a row quarantine an Account.
+ *
+ * Two, not one. Every pattern the adapters match is a guess about a vendor's prose, and a false
+ * positive costs more than a false negative here: it pulls a working Account out of rotation until
+ * a human re-adds it, which overnight is the whole Lane, where a miss only spends the Task's retry
+ * cap. Two consecutive failures on one Account is a shape a transient 401 does not have; a genuinely
+ * dead login has it within seconds, because the second Task is dispatched as soon as a Slot frees.
+ *
+ * A `CredentialError` does not count against this and quarantines at once: it is not a guess about
+ * prose but a fact the Foreman established about its own Account record before spawning anything.
+ */
+export const CREDENTIAL_FAILURES_BEFORE_QUARANTINE = 2;
+
+/** A lift a human made that no Run has reported yet: when, and which state it ended. */
+export interface ResumedState {
+  at: string;
+  /** What the Account was in when the human lifted it, for the event's reason line. */
+  from: "paused" | "quarantined" | "both";
+}
+
+/** How long an Account is Paused when the vendor refused the turn without saying when the limit
+ * lifts. Long enough that the Foreman is not straight back at a limit it cannot see the shape of,
+ * short enough that a night is not lost to one refusal whose window was five minutes. */
+export const DEFAULT_PAUSE_MS = 60 * 60_000;
+
+/** The shortest a pause may be. A window of zero is not a short pause, it is no pause: `until`
+ * would equal the moment the limit was seen, which `pausedUntil` reads as an Account that was never
+ * Paused, and the Run would go straight back at the limit that had just refused it. */
+export const MIN_PAUSE_MS = 1_000;
+
+/** The furthest ahead a vendor's reported reset is taken at face value. `resetAt` is parsed out of
+ * the vendor's own text in places (claude-worker-adapter.ts `resetAtFromLimitText`), so a stray
+ * year in a message would otherwise Pause an Account past any night the Foreman is running. */
+export const MAX_PAUSE_MS = 24 * 60 * 60_000;
+
+/**
+ * When the Account's pause lifts, or undefined when it is not Paused at `now` (Paused, CONTEXT.md).
+ *
+ * Resume is time-based and takes no operator action: a stored pause whose reset has passed is not
+ * a pause, so the tick after it lapses fills the Account's Slots again by itself (story 30).
+ */
+export function pausedUntil(account: Pick<Account, "paused">, now: number): Date | undefined {
+  if (!account.paused) return undefined;
+  const until = new Date(account.paused.until);
+  // An `until` that is not a date at all — a hand-edited accounts file — would compare false
+  // against every clock and Pause the Account for ever; it is read as no pause instead.
+  if (!Number.isFinite(until.getTime())) return undefined;
+  return until.getTime() > now ? until : undefined;
+}
+
+export function isPaused(account: Pick<Account, "paused">, now: number): boolean {
+  return pausedUntil(account, now) !== undefined;
+}
+
+/** When a rate limit seen at `now` lifts: the vendor's own reset when it gave one that is ahead of
+ * now and inside `MAX_PAUSE_MS`, else `defaultMs` from now. A reset the vendor reported as already
+ * past is no reset: the turn was refused all the same, so the default window is used rather than
+ * going straight back at the limit. */
+export function pauseUntilFrom(now: number, resetAt: string | undefined, defaultMs: number = DEFAULT_PAUSE_MS): Date {
+  const reported = resetAt === undefined ? Number.NaN : new Date(resetAt).getTime();
+  if (!Number.isFinite(reported) || reported <= now) return new Date(now + defaultMs);
+  return new Date(Math.min(reported, now + MAX_PAUSE_MS));
+}
+
+/** Marks the Account Paused until `until` (story 30). The store is re-read rather than written from
+ * a snapshot, so a cap this process set in between survives; an Account that has since been removed
+ * is a no-op and says so by returning undefined.
+ *
+ * Re-reading is not a lock: load and save are two calls, so a `foreman account set --cap` in
+ * another terminal between them is overwritten whole (FileAccountStore.save says what its atomic
+ * rename does and does not buy). Within one process — the daemon, where pauses are written — the
+ * two calls are synchronous with nothing between them, and the guarantee holds. */
+export function pauseAccount(store: AccountStore, name: string, until: Date, reason: string): Account | undefined {
+  return updateAccount(store, name, (account) => ({ ...account, paused: { until: until.toISOString(), reason } }));
+}
+
+/** Clears the Paused state. Resume is time-based, so this is for the operator's own hand and for a
+ * Run tidying a pause it has already outlived; nothing waits on it. */
+export function resumeAccount(store: AccountStore, name: string): Account | undefined {
+  return updateAccount(store, name, (account) => ({ ...account, paused: null }));
+}
+
+/** True while the Account is quarantined (OPS-288). No `now`, unlike `isPaused`: a quarantine has no
+ * reset, so nothing about it changes with the clock. */
+export function isQuarantined(account: Pick<Account, "quarantined">): boolean {
+  return account.quarantined != null;
+}
+
+/** Quarantines the Account and closes its credential-failure streak, since the streak has done its
+ * job. Re-reads the store for the reason `pauseAccount` gives; an Account that has since been
+ * removed is a no-op and says so by returning undefined. */
+export function quarantineAccount(store: AccountStore, name: string, at: Date, reason: string): Account | undefined {
+  return updateAccount(store, name, (account) => ({ ...account, quarantined: { at: at.toISOString(), reason }, credentialFailures: undefined }));
+}
+
+/** Lifts a quarantine and forgets the streak behind it: what `opsee foreman account resume` does,
+ * and what re-registering the Account achieves by starting a fresh record. */
+export function clearQuarantine(store: AccountStore, name: string): Account | undefined {
+  return updateAccount(store, name, (account) => ({ ...account, quarantined: null, credentialFailures: undefined }));
+}
+
+/** Stamps the moment a human brought the Account back, for the next Run to put on the Run Record.
+ * Only the newest lift is kept: a second `account resume` before any Run has reported the first
+ * describes the same Account being available, and one event says that as well as two. */
+export function noteResumed(store: AccountStore, name: string, at: Date, from: ResumedState["from"]): Account | undefined {
+  return updateAccount(store, name, (account) => ({ ...account, resumedAt: { at: at.toISOString(), from } }));
+}
+
+/** Forgets a stamped lift once a Run has reported it. */
+export function clearResumed(store: AccountStore, name: string): Account | undefined {
+  return updateAccount(store, name, (account) => ({ ...account, resumedAt: undefined }));
+}
+
+/**
+ * Records one more credential failure in a row and returns the Account with the new count, so the
+ * caller can see whether `CREDENTIAL_FAILURES_BEFORE_QUARANTINE` is reached. Does not quarantine:
+ * deciding that is the Run's, which has the log and the Run Record to write alongside it.
+ */
+export function noteCredentialFailure(store: AccountStore, name: string, at: Date, reason: string): Account | undefined {
+  return updateAccount(store, name, (account) => ({
+    ...account,
+    credentialFailures: { count: (account.credentialFailures?.count ?? 0) + 1, at: at.toISOString(), reason },
+  }));
+}
+
+/** Forgets an open credential-failure streak after a turn completed on the Account: the failures
+ * this counts are consecutive ones, and a turn the vendor accepted proves the credential is live.
+ * A no-op — and, deliberately, no store write at all — when there is no streak to clear, which is
+ * every turn of a healthy Account. */
+export function clearCredentialFailures(store: AccountStore, name: string): Account | undefined {
+  const account = store.load().find((a) => a.name === name);
+  if (!account?.credentialFailures) return account;
+  return updateAccount(store, name, (a) => ({ ...a, credentialFailures: undefined }));
+}
+
+function updateAccount(store: AccountStore, name: string, change: (account: Account) => Account): Account | undefined {
+  const existing = store.load();
+  const account = existing.find((a) => a.name === name);
+  if (!account) return undefined;
+  const updated = change(account);
+  store.save(existing.map((a) => (a.name === name ? updated : a)));
+  return updated;
+}
+
+interface AccountBase {
+  name: string;
+  vendor: Vendor;
+  cap: number;
+  paused: PausedState | null;
+  /** Set while the Account's credential is dead (OPS-288). Optional rather than nullable, unlike
+   * `paused`, so that every accounts file written before this existed reads as a healthy Account
+   * instead of being refused for a missing field. */
+  quarantined?: QuarantinedState | null;
+  /** The open credential-failure streak, absent when there is none (`noteCredentialFailure`). */
+  credentialFailures?: CredentialFailures;
+  /**
+   * When a human last brought this Account back by hand, until a Run has put that on a Run Record
+   * (`RunAccountEvent`, state `active`).
+   *
+   * Stamped here rather than appended there because `foreman account resume` has no Initiative: it
+   * is a local command over the accounts file, and a Run Record belongs to an Initiative. So the
+   * moment is recorded where it happens and reported by the next Run that builds a Lane on this
+   * Account, carrying this timestamp as the event's `occurredAt` — which is what that field is for,
+   * the Run Record already expecting facts that reach it late.
+   *
+   * Cleared once reported. Until then the lift is simply missing from the Record, which is honest:
+   * nothing has read it yet.
+   */
+  resumedAt?: ResumedState;
+  /** Cap on agentic turns within one Worker turn on this Account, where the vendor supports one
+   * (story 33). Unset leaves it to the Run's own `--max-turns`, and to the vendor's default past
+   * that. A subscription that bills by turn is what this is for. */
+  maxTurns?: number;
+  /** How long a Worker on this Account may go silent before its turn is stopped as `stalled`
+   * (story 33). Unset falls back to the Run's `--stall-timeout` and then to the CLI's default. */
+  stallTimeoutMs?: number;
+}
+
+/** A subscription the user has already signed into, isolated in its own config directory; the
+ * Worker Adapter points the vendor's config-directory variable at it. */
+export interface SubscriptionAccount extends AccountBase {
+  type: "subscription";
+  configDir: string;
+}
+
+/** An API key named by reference. In v1 `keyRef` is the name of an environment variable the
+ * Worker is launched with; its value never passes through the Foreman. */
+export interface ApiKeyAccount extends AccountBase {
+  type: "api_key";
+  keyRef: string;
+}
+
+export type Account = SubscriptionAccount | ApiKeyAccount;
+
+/**
+ * The only file-system access registration is allowed: metadata of the directory itself. There is
+ * deliberately no readdir, readFile or open on this seam, so the type system already rules out
+ * looking inside; the boundary test rules it out for the real fs behind it too.
+ */
+export interface ConfigDirFs {
+  statSync(path: string): { isDirectory(): boolean };
+  accessSync(path: string, mode: number): void;
+}
+
+/** A registration the user got wrong; reported as a plain message, never a stack. */
+export class AccountError extends Error {}
+
+/** What `account add` collects from the command line, before validation. */
+export interface AccountSpec {
+  vendor: string;
+  name?: string;
+  configDir?: string;
+  keyRef?: string;
+  cap?: number;
+}
+
+const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+function expandHome(path: string): string {
+  return path === "~" || path.startsWith("~/") ? homedir() + path.slice(1) : path;
+}
+
+function describeSource(account: Account): string {
+  return account.type === "subscription" ? account.configDir : `$${account.keyRef}`;
+}
+
+/** Checks that `dir` is a directory the user can enter and read, touching nothing but the
+ * directory entry itself. Returns the absolute path that will be stored. */
+export function validateConfigDir(fs: ConfigDirFs, dir: string): string {
+  const path = resolve(expandHome(dir));
+  let stat: { isDirectory(): boolean };
+  try {
+    stat = fs.statSync(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      throw new AccountError(`Config directory does not exist: ${path}`);
+    }
+    throw error;
+  }
+  if (!stat.isDirectory()) {
+    throw new AccountError(`Config directory is not a directory: ${path}`);
+  }
+  try {
+    fs.accessSync(path, constants.R_OK | constants.X_OK);
+  } catch {
+    throw new AccountError(`Config directory is not readable by you: ${path}`);
+  }
+  return path;
+}
+
+/** Validates the spec against the file system and the existing Accounts, then stores the new one.
+ * Names are unique across vendors; a directory or key may back only one Account per vendor. */
+export function registerAccount(store: AccountStore, fs: ConfigDirFs, spec: AccountSpec): Account {
+  if (!isVendor(spec.vendor)) {
+    throw new AccountError(`Unknown vendor "${spec.vendor}". Vendors: ${VENDORS.join(", ")}`);
+  }
+  const vendor: Vendor = spec.vendor;
+  if ((spec.configDir === undefined) === (spec.keyRef === undefined)) {
+    throw new AccountError("An Account needs either --config-dir (subscription) or --key-env (API key), not both");
+  }
+  const cap = spec.cap ?? DEFAULT_CAP;
+  if (!Number.isInteger(cap) || cap < 1) {
+    throw new AccountError("--cap must be a positive integer");
+  }
+  if (cap > MAX_CAP) {
+    throw new AccountError(`--cap is at most ${MAX_CAP}; a cap is how many Workers one vendor identity runs at once`);
+  }
+  const name = spec.name ?? vendor;
+  if (!NAME_PATTERN.test(name)) {
+    throw new AccountError("Account names are letters, digits, '.', '_' and '-' and start with a letter or digit");
+  }
+
+  const existing = store.load();
+  if (existing.some((a) => a.name === name)) {
+    throw new AccountError(
+      spec.name === undefined
+        ? `An Account named "${name}" already exists; pass --name to register a second ${vendor} Account`
+        : `An Account named "${name}" already exists`,
+    );
+  }
+
+  // A registration is a fresh record, so re-adding an Account is how a human lifts a quarantine
+  // (OPS-288): the state and its streak are not carried over from anything.
+  const base = { name, vendor, cap, paused: null, quarantined: null };
+  let account: Account;
+  if (spec.configDir !== undefined) {
+    const configDir = validateConfigDir(fs, spec.configDir);
+    account = { ...base, type: "subscription", configDir };
+  } else {
+    const keyRef = spec.keyRef!;
+    if (!ENV_NAME_PATTERN.test(keyRef)) {
+      // Deliberately not echoed: the likeliest mistake is pasting the key itself here.
+      throw new AccountError("--key-env must be an environment variable name (letters, digits and '_'), not the key");
+    }
+    account = { ...base, type: "api_key", keyRef };
+  }
+
+  // One login or key is one identity. Two Accounts on it would each carry their own cap and so
+  // double the concurrency the vendor sees on that identity, which is what per-Account caps exist
+  // to keep person-shaped (ADR-0013).
+  const twin = existing.find((a) => a.vendor === vendor && describeSource(a) === describeSource(account));
+  if (twin) {
+    throw new AccountError(`${describeSource(account)} is already registered as Account "${twin.name}" for ${vendor}`);
+  }
+
+  store.save([...existing, account]);
+  return account;
+}
+
+/** What `account set` may change: the scheduling limits, never the identity behind the Account.
+ * A field left out is left alone; there is deliberately no way to clear one back to unset here,
+ * since every value has a meaning and "unset" is only ever the shape a fresh Account starts in. */
+export interface AccountSettings {
+  cap?: number;
+  maxTurns?: number;
+  stallTimeoutMs?: number;
+}
+
+/** Changes an existing Account's scheduling limits (story 27): the cap that decides how many Slots
+ * it has, and the per-turn limits its Workers run under. Raising a cap is this call and nothing
+ * else — the Foreman never widens one on its own. */
+export function setAccountLimits(store: AccountStore, name: string, settings: AccountSettings): Account {
+  const existing = store.load();
+  const account = existing.find((a) => a.name === name);
+  if (!account) {
+    throw new AccountError(`No Account named "${name}"`);
+  }
+  if (settings.cap === undefined && settings.maxTurns === undefined && settings.stallTimeoutMs === undefined) {
+    throw new AccountError("foreman account set needs at least one of --cap, --max-turns, --stall-timeout");
+  }
+  for (const [flag, value] of [
+    ["--cap", settings.cap],
+    ["--max-turns", settings.maxTurns],
+    ["--stall-timeout", settings.stallTimeoutMs],
+  ] as const) {
+    if (value !== undefined && (!Number.isInteger(value) || value < 1)) {
+      throw new AccountError(`${flag} must be a positive integer`);
+    }
+  }
+  // Both ends of the range a number can be wrong at: a cap past what one identity should ever run,
+  // and a stall timeout so short that every turn is stopped as stalled.
+  if (settings.cap !== undefined && settings.cap > MAX_CAP) {
+    throw new AccountError(`--cap is at most ${MAX_CAP}; a cap is how many Workers one vendor identity runs at once`);
+  }
+  if (settings.stallTimeoutMs !== undefined && settings.stallTimeoutMs < MIN_STALL_TIMEOUT_MS) {
+    throw new AccountError(`--stall-timeout is at least ${MIN_STALL_TIMEOUT_MS}ms; below that a Worker that is only thinking is stopped as stalled`);
+  }
+  const updated: Account = {
+    ...account,
+    cap: settings.cap ?? account.cap,
+    maxTurns: settings.maxTurns ?? account.maxTurns,
+    stallTimeoutMs: settings.stallTimeoutMs ?? account.stallTimeoutMs,
+  };
+  store.save(existing.map((a) => (a === account ? updated : a)));
+  return updated;
+}
+
+export function removeAccount(store: AccountStore, name: string): Account {
+  const existing = store.load();
+  const account = existing.find((a) => a.name === name);
+  if (!account) {
+    throw new AccountError(`No Account named "${name}"`);
+  }
+  store.save(existing.filter((a) => a !== account));
+  return account;
+}
+
+/** A fixed-width table, one line per Account. Keys appear by variable name only. The STATE column
+ * is the Paused state with its reset (story 30); a pause whose reset has passed reads as active,
+ * because it is — resume is time-based — with the lapsed reset kept in view so a reader can tell a
+ * rate-limited Account that has just come back from one that was never Paused.
+ *
+ * Quarantine (OPS-288) is reported ahead of a pause, and swallows it: an Account may well have been
+ * Paused before its credential died, and of the two facts only one still needs the reader to do
+ * something. It names the remedy for the same reason, since nothing lifts it by itself. */
+export function formatAccountTable(accounts: Account[], now: number = Date.now()): string[] {
+  const state = (a: Account): string => {
+    if (a.quarantined) return `quarantined since ${a.quarantined.at} (${printableOneLine(a.quarantined.reason)}); re-add it or: opsee foreman account resume ${a.name}`;
+    const until = pausedUntil(a, now);
+    if (until) return `paused until ${a.paused!.until}${a.paused!.reason ? ` (${oneLine(a.paused!.reason)})` : ""}`;
+    return a.paused ? `active (pause lapsed ${a.paused.until})` : "active";
+  };
+  const rows = [
+    ["NAME", "VENDOR", "TYPE", "SOURCE", "CAP", "STATE"],
+    ...accounts.map((a) => [a.name, a.vendor, a.type, describeSource(a), String(a.cap), state(a)]),
+  ];
+  const widths = rows[0].map((_, i) => Math.max(...rows.map((r) => r[i].length)));
+  return rows.map((r) => r.map((cell, i) => (i === r.length - 1 ? cell : cell.padEnd(widths[i]))).join("  "));
+}
+
+export function describeAccount(account: Account): string {
+  const source = account.type === "subscription" ? `config dir ${account.configDir}` : `key from $${account.keyRef}`;
+  return `${account.type} Account "${account.name}" for ${account.vendor} (${source}, ${describeLimits(account)})`;
+}
+
+/** The Account's scheduling limits in words: the cap and how its Slots divide, plus the per-turn
+ * limits when it sets its own. */
+export function describeLimits(account: Account): string {
+  const slots = account.cap > 1 ? `${account.cap - 1} implementer${account.cap - 1 === 1 ? "" : "s"} + 1 reserved for Verifiers` : "1 implementer, no Slot to reserve for Verifiers";
+  const perTurn = [
+    account.maxTurns === undefined ? "" : `max turns ${account.maxTurns}`,
+    account.stallTimeoutMs === undefined ? "" : `stall timeout ${account.stallTimeoutMs}ms`,
+  ].filter(Boolean);
+  return `cap ${account.cap} (${slots})${perTurn.length ? `, ${perTurn.join(", ")}` : ""}`;
+}