@mmerterden/multi-agent-pipeline 20.2.1 → 20.3.1

package/pipeline/scripts/autopilot-arming.mjs

@@ -10,7 +10,9 @@
  *   Spend. A rolling 24-hour total from the attempt log against
  *   `costCeilingUsd`. Past the ceiling, dispatch refuses and says so. Work
  *   already running is NEVER killed mid-item: stopping between a commit and a
- *   PR leaves a branch nobody asked for, and the money is already spent.
+ *   PR leaves a branch nobody asked for, and the money is already spent. A run
+ *   in the window whose cost was not measured (`usd: null`) blocks too, as
+ *   `cost-unknown`: the ceiling cannot be checked against a total with a hole.
  *
  *   Credentials. `prefs.global.serviceStatus.<service>` must not be a recorded
  *   failure for the service an item needs. Without this, an expired Jira token
@@ -22,35 +24,157 @@
  * run yet on this machine, and refusing on absence would mean the queue never
  * starts. Only a recorded `ok: false` blocks.
  *
+ * Readable, not only recorded. `probe` asks every credential the configured
+ * repos need whether it can be read right now without a prompt: `gh auth
+ * status` for GitHub, and `credential-store.sh probe` for the logical keys a
+ * source reads from `prefs.global.keychainMapping` (Jira today). The value is
+ * never read into this process - the store answers one word. autopilot-on runs
+ * it before the schedule is offered, and every tick runs the item's own half
+ * through `check --probe`, because a tick runs in the background session, where
+ * a keychain that locked on sleep answers differently from the terminal that
+ * armed it. A missing, unmapped or locked credential refuses; a keychain set to
+ * lock on sleep or after a timeout is a warning, since it is readable now.
+ *
+ * A check that cannot finish answers `armed: false` with `blockedBy:
+ * "internal"`. This verdict stands between an unattended queue and money, so
+ * the absence of an answer has to cost nothing rather than read as yes.
+ *
  * Usage:
  *   autopilot-arming.mjs check [--source github|jira]   KEY=VALUE lines, exit 0
  *   autopilot-arming.mjs check --json
+ *   autopilot-arming.mjs check --source jira --probe --json   also probe the item's credential
+ *   autopilot-arming.mjs check --source jira --repo <owner/name> --probe --json
+ *                                                         and the repo's publish credential
+ *   autopilot-arming.mjs probe [--json]                        every credential the config needs
  *
  * Exit: 0 whether or not it is armed - "not armed" is an answer. 2 on usage.
  */
 
+import { spawnSync } from "node:child_process";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { attemptSpend } from "./_cost.mjs";
+import { configRefusal } from "./_autopilot-config.mjs";
+import { hostCommand, messages as languageHalf } from "../lib/user-facing.mjs";
 
 const ROOT = process.env.MA_AUTOPILOT_ROOT || join(homedir(), ".claude", "autopilot");
+// `$MULTI_AGENT_PREFS` is the name every other shipped script honours, so a test
+// that points one half of a status line at a fixture points the other half at
+// the same one. `$MA_PREFS_FILE` stays readable for callers that already set it.
 const PREFS =
-  process.env.MA_PREFS_FILE || join(homedir(), ".claude", "multi-agent-preferences.json");
+  process.env.MULTI_AGENT_PREFS ||
+  process.env.MA_PREFS_FILE ||
+  join(homedir(), ".claude", "multi-agent-preferences.json");
+
+// Beside this script in every install, as lib/ sits beside scripts/.
+const STORE =
+  process.env.MA_AP_CREDENTIAL_STORE ||
+  join(import.meta.dirname, "..", "lib", "credential-store.sh");
+// A store that has not answered in this long is waiting on a prompt nobody can
+// answer from a background session.
+const PROBE_TIMEOUT_MS = Number(process.env.MA_AP_PROBE_TIMEOUT_MS || 20000);
 
 /** Which service an item from this source cannot proceed without. */
 export const SOURCE_SERVICE = { github: "github", jira: "jira" };
 
+/**
+ * A blocked queue explains itself in `prefs.global.outputLanguage`, the same
+ * preference every other human-facing surface reads; English is the default and
+ * the fallback for a language this table does not carry.
+ */
+const MESSAGES = {
+  en: {
+    credential: (svc, at, cmd) =>
+      `the ${svc} credential is not valid (${at}) - refresh it with ${cmd}`,
+    cost: (spent, ceiling) =>
+      `$${spent} spent in the last 24 hours, ceiling $${ceiling} - work in flight is finished, no new item is taken`,
+    costUnknown: (runs) =>
+      `${runs} run(s) in the last 24 hours have a cost that was not measured, so the spend ceiling cannot be checked - no new item is taken until they leave the window`,
+    internal: (detail) =>
+      `the arming check could not finish (${detail}) - no new item is taken until it can`,
+    config: (detail) => `${detail} - no new item is taken until config.json is corrected`,
+    ghMissing: () => "GitHub: gh is not installed - install it, then run gh auth login",
+    ghUnauth: () =>
+      "GitHub: gh is not authenticated for a non-interactive session - run gh auth login",
+    unmapped: (key, cmd) =>
+      `credential "${key}": prefs.global.keychainMapping.${key} is not set - save the token with ${cmd}`,
+    noHost: (key, cmd) =>
+      `credential "${key}": prefs.global.hosts.${key} is not set - record the host with ${cmd}`,
+    missing: (key, item, cmd) =>
+      `credential "${key}" (stored as "${item}") is not in the credential store - save it with ${cmd}`,
+    locked: (key) =>
+      `credential "${key}" cannot be read without a prompt: the keychain is locked for this session - unlock it (security unlock-keychain), or stop it locking on sleep and after a timeout`,
+    noAnswer: (key, sec) =>
+      `credential "${key}": the credential store did not answer in ${sec}s - it is most likely waiting on a keychain prompt nobody can answer in the background; unlock the keychain`,
+    storeError: (key) =>
+      `credential "${key}": the credential store could not be read - run credential-store.sh doctor`,
+    lockWarning: (when) =>
+      `the login keychain locks ${when}; a background tick after that finds it locked and refuses items until it is unlocked - Keychain Access > login > Change Settings, or security set-keychain-settings with no options`,
+    afterSleep: "after sleep",
+    afterIdle: (min) => `after ${min} min idle`,
+    and: " and ",
+  },
+  tr: {
+    credential: (svc, at, cmd) => `${svc} kimlik bilgisi geçersiz (${at}) - ${cmd} ile yenile`,
+    cost: (spent, ceiling) =>
+      `son 24 saatte $${spent} harcandı, tavan $${ceiling} - koşan iş bitirilir, yeni madde alınmaz`,
+    costUnknown: (runs) =>
+      `son 24 saatte ${runs} koşunun maliyeti ölçülmedi, harcama tavanı denetlenemiyor - bu koşular pencereden çıkana kadar yeni madde alınmaz`,
+    internal: (detail) =>
+      `hazırlık denetimi tamamlanamadı (${detail}) - tamamlanana kadar yeni madde alınmaz`,
+    config: (detail) => `${detail} - config.json düzeltilene kadar yeni madde alınmaz`,
+    ghMissing: () => "GitHub: gh kurulu değil - kur, sonra gh auth login çalıştır",
+    ghUnauth: () => "GitHub: gh etkileşimsiz oturum için giriş yapmamış - gh auth login çalıştır",
+    unmapped: (key, cmd) =>
+      `"${key}" kimlik bilgisi: prefs.global.keychainMapping.${key} tanımlı değil - token'ı ${cmd} ile kaydet`,
+    noHost: (key, cmd) =>
+      `"${key}" kimlik bilgisi: prefs.global.hosts.${key} tanımlı değil - host'u ${cmd} ile kaydet`,
+    missing: (key, item, cmd) =>
+      `"${key}" kimlik bilgisi ("${item}" adıyla) kimlik deposunda yok - ${cmd} ile kaydet`,
+    locked: (key) =>
+      `"${key}" kimlik bilgisi sorusuz okunamıyor: anahtar zinciri bu oturum için kilitli - kilidi aç (security unlock-keychain) ya da uykuda ve zaman aşımında kilitlenmesini kapat`,
+    noAnswer: (key, sec) =>
+      `"${key}" kimlik bilgisi: kimlik deposu ${sec}sn içinde cevap vermedi - büyük olasılıkla arka planda kimsenin cevaplayamayacağı bir anahtar zinciri sorusunu bekliyor; kilidi aç`,
+    storeError: (key) =>
+      `"${key}" kimlik bilgisi: kimlik deposu okunamadı - credential-store.sh doctor çalıştır`,
+    lockWarning: (when) =>
+      `giriş anahtar zinciri ${when} kilitleniyor; sonrasındaki arka plan turu onu kilitli bulur ve kilit açılana kadar madde almaz - Anahtar Zinciri Erişimi > login > Ayarları Değiştir, ya da seçeneksiz security set-keychain-settings`,
+    afterSleep: "uykudan sonra",
+    afterIdle: (min) => `${min} dk boşta kaldıktan sonra`,
+    and: " ve ",
+  },
+};
+
+/**
+ * Every key in the table resolves, whatever the preference says.
+ *
+ * The half comes from an own-property lookup, so a code that spells an
+ * inherited name is a language this table does not carry rather than a
+ * function. English is then laid underneath it, so a key carried by one half
+ * and not the other renders in English instead of arriving as `undefined`
+ * where the caller expects a sentence.
+ */
+export function messages(language) {
+  return { ...MESSAGES.en, ...languageHalf(MESSAGES, language) };
+}
+
 /**
  * Spend inside the trailing window. Reads the append-only attempt log rather
  * than a running total in a file: a counter can be lost, double-counted, or
  * left behind by a crash, and the log is the thing that already has to be
  * written for retry to work at all.
+ *
+ * `usd` is cumulative per run, so a run recorded twice counts once, at its
+ * largest figure (_cost.mjs attemptSpend). A run whose cost was never measured
+ * (`usd: null`) is counted in `unknown`, never as 0. A row with no `usd` at all
+ * is one where nothing ran - an arming refusal, a checkout that is not there -
+ * and costs nothing.
  */
 export function spendWindow(lines, { now = Date.now() / 1000, windowSeconds = 86400 } = {}) {
   const cutoff = now - windowSeconds;
-  let usd = 0;
-  let items = 0;
+  const rows = [];
   for (const raw of lines) {
     const s = raw.trim();
     if (!s) continue;
@@ -60,18 +184,33 @@ export function spendWindow(lines, { now = Date.now() / 1000, windowSeconds = 86
     } catch {
       continue;
     }
-    if ((rec.at || 0) < cutoff) continue;
-    usd += Number(rec.usd || 0);
-    items += 1;
+    if (!rec || typeof rec !== "object" || (rec.at || 0) < cutoff) continue;
+    rows.push(rec);
   }
-  return { usd: Math.round(usd * 100) / 100, items };
+  const { usd, unknown } = attemptSpend(rows);
+  return { usd: Math.round(usd * 100) / 100, items: rows.length, unknown };
+}
+
+/**
+ * The service the runner publishes a repo's verified branch with, when it is a
+ * credential this script can probe: `gh` for a GitHub-hosted repo (the default
+ * host). A Bitbucket push goes through the operator's git credential helper,
+ * which has no probe that answers without a network round trip.
+ *
+ * @returns {string|null}
+ */
+export function publishService(config, repo) {
+  if (!repo) return null;
+  const cfg = (config?.repos || []).find((r) => r && r.nameWithOwner === repo);
+  if (!cfg) return null;
+  return (cfg.host ?? "github") === "github" ? "github" : null;
 }
 
 /**
  * @returns {{ok: boolean, reason: string|null}} - `reason` is written for a
  * human to read in the menu bar, so it says what to do, not what failed.
  */
-export function credentialGate(serviceStatus, source) {
+export function credentialGate(serviceStatus, source, language = "en") {
   const svc = SOURCE_SERVICE[source];
   if (!svc) return { ok: true, reason: null };
   const rec = serviceStatus?.[svc];
@@ -79,22 +218,187 @@ export function credentialGate(serviceStatus, source) {
   if (rec.ok) return { ok: true, reason: null };
   return {
     ok: false,
-    reason: `${svc} kimlik bilgisi geçersiz (${rec.checkedAt || "?"}) - /multi-agent:setup ile yenile`,
+    reason: messages(language).credential(svc, rec.checkedAt || "?", hostCommand("setup")),
   };
 }
 
-export function arming({ lines, prefs, config, source, now }) {
-  const ceiling = Number(config?.costCeilingUsd ?? 25);
+/**
+ * The services the configured repos cannot run without. GitHub for a repo that
+ * takes GitHub issues or publishes to GitHub (the default host); Jira for a repo
+ * that takes Jira items. A repo with no `sources` takes GitHub issues.
+ */
+export function requiredServices(config) {
+  const need = new Set();
+  for (const r of config?.repos || []) {
+    const sources = Array.isArray(r?.sources) && r.sources.length ? r.sources : ["github"];
+    for (const s of sources) if (SOURCE_SERVICE[s]) need.add(SOURCE_SERVICE[s]);
+    if ((r?.host ?? "github") === "github") need.add("github");
+  }
+  return [...need].sort();
+}
+
+function timedOut(r) {
+  return r.error?.code === "ETIMEDOUT" || (r.status === null && Boolean(r.signal));
+}
+
+/**
+ * One service, answered without its value: `gh auth status` for GitHub, the
+ * store's `probe` for a mapped key. Nothing either prints is kept.
+ *
+ * @returns {{service:string, key?:string, status:string, message?:string}}
+ */
+export function probeService(service, { prefs, env = process.env, language = "en" } = {}) {
+  const m = messages(language);
+  if (service === "github") {
+    const gh = env.MA_AP_GH_BIN || "gh";
+    const r = spawnSync(gh, ["auth", "status"], {
+      env,
+      stdio: "ignore",
+      timeout: PROBE_TIMEOUT_MS,
+    });
+    if (r.error?.code === "ENOENT")
+      return { service, status: "missing-tool", message: m.ghMissing() };
+    if (timedOut(r)) {
+      return {
+        service,
+        status: "no-answer",
+        message: m.noAnswer("github", PROBE_TIMEOUT_MS / 1000),
+      };
+    }
+    return r.status === 0
+      ? { service, status: "readable" }
+      : { service, status: "unauthenticated", message: m.ghUnauth() };
+  }
+  const key = service;
+  const setup = hostCommand("setup");
+  const item = prefs?.global?.keychainMapping?.[key];
+  if (typeof item !== "string" || !item.trim()) {
+    return { service, key, status: "unmapped", message: m.unmapped(key, setup) };
+  }
+  const host = prefs?.global?.hosts?.[key];
+  if (typeof host !== "string" || !host.trim()) {
+    return { service, key, status: "no-host", message: m.noHost(key, setup) };
+  }
+  const r = spawnSync("bash", [STORE, "probe", key], {
+    env: { ...env, MULTI_AGENT_PREFS: PREFS },
+    encoding: "utf-8",
+    stdio: ["ignore", "pipe", "ignore"],
+    timeout: PROBE_TIMEOUT_MS,
+  });
+  if (timedOut(r)) {
+    return { service, key, status: "no-answer", message: m.noAnswer(key, PROBE_TIMEOUT_MS / 1000) };
+  }
+  const word = String(r.stdout || "").trim();
+  if (r.status === 0 && word === "readable") return { service, key, status: "readable" };
+  if (r.status === 5 || word === "locked") {
+    return { service, key, status: "locked", message: m.locked(key) };
+  }
+  if (r.status === 1 || word === "missing") {
+    return { service, key, status: "missing", message: m.missing(key, item.trim(), setup) };
+  }
+  return { service, key, status: "error", message: m.storeError(key) };
+}
+
+/**
+ * The login keychain's lock settings, read with `security show-keychain-info`.
+ * Only macOS has them. A keychain that locks on sleep or after a timeout is
+ * readable from the terminal that armed the mode and locked for the tick that
+ * fires after the machine slept.
+ *
+ * @returns {{locked:boolean, lockOnSleep:boolean, timeoutSec:number|null}|null}
+ */
+export function keychainLock({ env = process.env, platform } = {}) {
+  if ((platform || env.MA_AP_PLATFORM || process.platform) !== "darwin") return null;
+  const r = spawnSync(env.MA_AP_SECURITY_BIN || "/usr/bin/security", ["show-keychain-info"], {
+    env,
+    encoding: "utf-8",
+    stdio: ["ignore", "pipe", "pipe"],
+    timeout: PROBE_TIMEOUT_MS,
+  });
+  if (r.error) return null;
+  const text = `${r.stdout || ""}${r.stderr || ""}`;
+  if (r.status === 36 || /interaction is not allowed/i.test(text)) {
+    return { locked: true, lockOnSleep: false, timeoutSec: null };
+  }
+  if (r.status !== 0) return null;
+  const t = /timeout=(\d+)s/.exec(text);
+  return {
+    locked: false,
+    lockOnSleep: /lock-on-sleep/.test(text),
+    timeoutSec: t ? Number(t[1]) : null,
+  };
+}
+
+/**
+ * Every credential the config needs, probed. `ready` is false when any one is
+ * missing, unmapped, locked or unanswered; lock settings are warnings.
+ */
+export function probeCredentials({ config, prefs, env = process.env, platform } = {}) {
+  const language = prefs?.global?.outputLanguage ?? "en";
+  const m = messages(language);
+  const checked = requiredServices(config).map((svc) =>
+    probeService(svc, { prefs, env, language }),
+  );
+  const problems = checked.filter((c) => c.status !== "readable");
+  const keychain = keychainLock({ env, platform });
+  const warnings = [];
+  if (keychain && (keychain.lockOnSleep || keychain.timeoutSec)) {
+    const when = [
+      keychain.lockOnSleep ? m.afterSleep : null,
+      keychain.timeoutSec ? m.afterIdle(Math.round(keychain.timeoutSec / 60)) : null,
+    ]
+      .filter(Boolean)
+      .join(m.and);
+    warnings.push(m.lockWarning(when));
+  }
+  return { ready: problems.length === 0, checked, problems, warnings, keychain };
+}
+
+export function arming({ lines, prefs, config, source, now, probe = null }) {
+  const raw = config?.costCeilingUsd === undefined ? 25 : config.costCeilingUsd;
   const spend = spendWindow(lines, { now });
-  const cred = credentialGate(prefs?.global?.serviceStatus, source);
+  const language = prefs?.global?.outputLanguage ?? "en";
+  // Every comparison with NaN is false, so a ceiling that is not a number would
+  // read as no ceiling at all. It is refused instead, and named.
+  if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) {
+    return {
+      armed: false,
+      blockedBy: "config",
+      reason: messages(language).config(
+        `costCeilingUsd must be a number >= 0, got ${JSON.stringify(config?.costCeilingUsd)}`,
+      ),
+      spend,
+      ceiling: 0,
+    };
+  }
+  const ceiling = raw;
+  const cred = credentialGate(prefs?.global?.serviceStatus, source, language);
 
   if (!cred.ok)
     return { armed: false, blockedBy: "credential", reason: cred.reason, spend, ceiling };
+  // A credential recorded as fine can still be unreadable here: the tick runs in
+  // the background session, and a keychain that locked on sleep is locked there.
+  const unreadable = [probe].flat().find((p) => p && p.status !== "readable");
+  if (unreadable) {
+    return { armed: false, blockedBy: "credential", reason: unreadable.message, spend, ceiling };
+  }
   if (spend.usd >= ceiling) {
     return {
       armed: false,
       blockedBy: "cost",
-      reason: `son 24 saatte $${spend.usd.toFixed(2)} harcandı, tavan $${ceiling.toFixed(2)} - koşan iş bitirilir, yeni madde alınmaz`,
+      reason: messages(language).cost(spend.usd.toFixed(2), ceiling.toFixed(2)),
+      spend,
+      ceiling,
+    };
+  }
+  // A ceiling checked against a total with holes in it reads as protection and
+  // is not: an unmeasured run may have cost more than the whole ceiling. Refuse
+  // until the unmeasured runs leave the window, rather than counting them as 0.
+  if (spend.unknown > 0) {
+    return {
+      armed: false,
+      blockedBy: "cost-unknown",
+      reason: messages(language).costUnknown(spend.unknown),
       spend,
       ceiling,
     };
@@ -102,6 +406,8 @@ export function arming({ lines, prefs, config, source, now }) {
   return { armed: true, blockedBy: null, reason: null, spend, ceiling };
 }
 
+class ConfigRefused extends Error {}
+
 function readJson(path, fallback) {
   if (!existsSync(path)) return fallback;
   try {
@@ -111,20 +417,81 @@ function readJson(path, fallback) {
   }
 }
 
+function probeMain(args) {
+  const prefs = readJson(PREFS, {});
+  const language = prefs?.global?.outputLanguage ?? "en";
+  let result;
+  try {
+    result = probeCredentials({ config: readJson(join(ROOT, "config.json"), {}), prefs });
+  } catch (e) {
+    const reason = messages(language).internal(e?.message || String(e));
+    result = {
+      ready: false,
+      checked: [],
+      problems: [{ status: "error", message: reason }],
+      warnings: [],
+      keychain: null,
+    };
+  }
+  if (args.includes("--json")) {
+    process.stdout.write(JSON.stringify(result) + "\n");
+    return 0;
+  }
+  const out = [`PROBE_READY=${result.ready}`];
+  for (const p of result.problems) out.push(`PROBE_PROBLEM=${p.message}`);
+  for (const w of result.warnings) out.push(`PROBE_WARNING=${w}`);
+  process.stdout.write(out.join("\n") + "\n");
+  return 0;
+}
+
 function main(argv) {
   const args = argv.slice(2);
+  if (args[0] === "probe") return probeMain(args);
   if (args[0] !== "check") {
-    process.stderr.write("usage: autopilot-arming.mjs check [--source github|jira] [--json]\n");
+    process.stderr.write(
+      "usage: autopilot-arming.mjs check [--source github|jira] [--repo <owner/name>] [--probe] [--json] | probe [--json]\n",
+    );
     return 2;
   }
   const source = args.includes("--source") ? args[args.indexOf("--source") + 1] : null;
+  const repo = args.includes("--repo") ? args[args.indexOf("--repo") + 1] : null;
   const attempts = join(ROOT, "attempted.jsonl");
-  const result = arming({
-    lines: existsSync(attempts) ? readFileSync(attempts, "utf-8").split("\n") : [],
-    prefs: readJson(PREFS, {}),
-    config: readJson(join(ROOT, "config.json"), {}),
-    source,
-  });
+  const prefs = readJson(PREFS, {});
+  const language = prefs?.global?.outputLanguage ?? "en";
+
+  // An arming check that cannot finish is not an armed queue. This verdict is
+  // the last thing between the queue and money, so the answer to "I could not
+  // work it out" is the one that spends nothing and says why.
+  let result;
+  try {
+    const config = readJson(join(ROOT, "config.json"), {});
+    const refusal = existsSync(join(ROOT, "config.json")) ? configRefusal(config) : null;
+    if (refusal) throw new ConfigRefused(refusal);
+    // The item's source credential, and the one its repo publishes with: an
+    // expired gh stops a Jira item in a GitHub repo at the push, after the run
+    // has been paid for, unless it is asked here first.
+    const services = args.includes("--probe")
+      ? [...new Set([SOURCE_SERVICE[source], publishService(config, repo)].filter(Boolean))]
+      : [];
+    result = arming({
+      lines: existsSync(attempts) ? readFileSync(attempts, "utf-8").split("\n") : [],
+      prefs,
+      config,
+      source,
+      probe: services.map((svc) => probeService(svc, { prefs, language })),
+    });
+  } catch (e) {
+    const refused = e instanceof ConfigRefused;
+    result = {
+      armed: false,
+      blockedBy: refused ? "config" : "internal",
+      reason: refused
+        ? messages(language).config(e.message)
+        : messages(language).internal(e?.message || String(e)),
+      spend: { usd: 0, items: 0, unknown: 0 },
+      ceiling: 0,
+    };
+  }
 
   if (args.includes("--json")) {
     process.stdout.write(JSON.stringify(result) + "\n");
@@ -137,6 +504,7 @@ function main(argv) {
       `ARM_REASON=${result.reason ?? ""}`,
       `ARM_SPEND_USD=${result.spend.usd}`,
       `ARM_SPEND_ITEMS=${result.spend.items}`,
+      `ARM_SPEND_UNKNOWN=${result.spend.unknown ?? 0}`,
       `ARM_CEILING_USD=${result.ceiling}`,
     ].join("\n") + "\n",
   );