@edgehero/pi-dispatch 4.0.1 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/doctor.mjs CHANGED
@@ -66,12 +66,14 @@ import { isTransientOverlayRead, overlayProviderProblem } from "./model-catalog.
66
66
  import { EMPTY_USD_FINGERPRINT, usdFingerprint } from "./dollar-fingerprint.mjs";
67
67
  import { splitModelEntry } from "./model-ref.mjs";
68
68
  import { ignoredOutputCapModels, outputCapView, outputUnboundable } from "./output-cap.mjs";
69
- import { KEYLESS_API_KEY, KEYLESS_HOW, MODEL_ENDPOINTS_FILE_NAME, MODEL_ENDPOINTS_INCLUDE_NAME, MODEL_ENDPOINT_ID_RE, OVERLAY_LINK_FIX, OVERLAY_NOT_A_FILE_FIX, RENAMED_PROVIDERS, baseUrlTarget, keylessVerdict, loadModelEndpoints, providerRenameHint, readOverlayModels, renderEndpointsInclude, unreportedUsageModels } from "./model-endpoints.mjs";
69
+ import { KEYLESS_API_KEY, KEYLESS_HOW, MODEL_ENDPOINTS_FILE_NAME, MODEL_ENDPOINTS_INCLUDE_NAME, MODEL_ENDPOINT_ID_RE, OVERLAY_LINK_FIX, OVERLAY_NOT_A_FILE_FIX, RENAMED_PROVIDERS, baseUrlTarget, endpointsAt, keylessVerdict, loadModelEndpoints, providerRenameHint, readOverlayModels, renderEndpointsInclude, unreportedUsageModels } from "./model-endpoints.mjs";
70
70
  import { declaredEndpointsIn, endpointsDeclaredIn, reloadCommand, rulesFileIncludes, rulesPredateEndpointsLine } from "./egress-cli.mjs";
71
71
  import { loadPauseWindows, parseScopeString } from "./pause-windows.mjs";
72
72
  import { WAIT_AFTER_MAX_DEFAULT_MS, afterInstantMs, parseWaitProfiles } from "./wait-for.mjs";
73
73
  import { isForgeKind } from "./forges.mjs";
74
74
  import { findLiteralSecret, ADMIN_RE } from "./import-pi.mjs";
75
+ import { mcpServerWhere, readOverlayMcp } from "./mcp-config.mjs";
76
+ import { allowlistEntryFor, egressRefusalFor, packageRunnerHosts, parseAllowlist } from "./egress-allowlist.mjs";
75
77
  import { agentDirFrom, readHostPi } from "./host-pi.mjs";
76
78
  import { PACKAGES_SUBDIR, readStagedSkills, readStageManifest } from "./packages.mjs";
77
79
  import { copySkillTree } from "./copy-tree.mjs";
@@ -99,6 +101,8 @@ import { CONTAINER_HOME, SHIPPED_IMAGE_UID, SIZE_LABEL_CPU, SIZE_LABEL_MEM } fro
99
101
  import { DEFAULT_JOB_SIZE, cpuCeilingCenti, formatCpus, formatMemory, jobSizeDefaults, resolveJobSize } from "./job-size.mjs";
100
102
  import { SUGGEST_WINDOW_DAYS, cpusText, hostCap, refusalWords, sizeRefusal, suggestSize, suggestionCall, suggestionEvidence } from "./size-suggest.mjs";
101
103
  import { SIZING_RECORD_MAX_BYTES, readSizingRecords } from "./size-records.mjs";
104
+ import { CAPACITY_WINDOWS, computeCapacity } from "./capacity.mjs";
105
+ import { JOBS_ONLY, durationText, milliText, notSharedWhy, percentText, shareText } from "./capacity-cli.mjs";
102
106
  import { CGROUP_PARENT, cgroupParentFor, operatorQuotaCommand, readQuota, reservePlan, userQuotaCommand } from "./cpu-reserve.mjs";
103
107
  import { HOST_BUDGET_KEYS, computeHostBudget, hostBudgetSettings, largestFit, neverFits, projectBudgetRow, publishedBudget, readUserServiceLimits } from "./host-budget.mjs";
104
108
  import { makeImagePreflight, normalizeImageId } from "./image-preflight.mjs";
@@ -220,6 +224,9 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
220
224
  dollarKeysExist,
221
225
  // Issue #504 part B: the applied split's envelope digest (`alloc:plan`), read once. Undefined means the default.
222
226
  readAppliedSplit,
227
+ // Issue #599, phase 2: the capacity line's read of the run history (the run mirror and the logs directory).
228
+ // Undefined means the default, which reads the deployment's real Valkey and files.
229
+ readCapacity,
223
230
  // --live (issue #278, INT-LIVE-PROBE-CONTRACT): read the backend declarations back off short-lived real containers.
224
231
  // STRICTLY `=== true`, so only the CLI's own flag arms it: a truthy string from a caller that forwarded an
225
232
  // option bag runs nothing. The fs, PID-liveness and nonce are seams so the sequence is driven without Docker.
@@ -383,7 +390,7 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
383
390
  return { ...(await valkeyAuthState(url, { context, withoutPassword })), passwordSet: Boolean(sent.password), from: sent.from };
384
391
  }
385
392
  : null;
386
- const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, ...(readRunRecords ? { readRunRecords } : {}), venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
393
+ const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), ...(readCapacity ? { readCapacity } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, ...(readRunRecords ? { readRunRecords } : {}), venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
387
394
  // Issue #471: every other service key, resolved ONCE for the whole run (the fix pass's re-collect and `--live` judge the
388
395
  // same resolution). THE RULE (PR #474's round cap, after three rounds of trust patches): no program doctor starts is
389
396
  // handed anything from `.env`. Every child gets this shell's own environment, the one it had before #471; a `.env`
@@ -595,7 +602,7 @@ export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "P
595
602
  export const GITHUB_SERVICE_KEYS = Object.freeze(["GITHUB_AUTH_SOURCE", "GITHUB_APP_ID", "GITHUB_APP_INSTALLATION_ID", "GITHUB_APP_PRIVATE_KEY_PATH", "GITHUB_APP_PRIVATE_KEY"]);
596
603
  /** Issue #471: the worker's settings doctor judges, which it read from this shell alone while the service read them from
597
604
  * `.env`. TEMP is TMPDIR's twin in the worker's temp root; PI_CODING_AGENT_DIR is where the worker reads auth.json. */
598
- export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_JOB_MEMORY", "PI_JOB_CPUS", "PI_CONCURRENCY", ...Object.values(HOST_BUDGET_KEYS), "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_SECRET_PROFILES", "PI_SECRET_RESOLVER_ROOTS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "PI_ALLOWED_MODELS", "PI_DISPATCH_RUN_ROOTS", "GITHUB_PAT_VAR", "TEMP", ...Object.values(DOLLAR_ENV_NAMES)]);
605
+ export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_JOB_MEMORY", "PI_JOB_CPUS", "PI_CONCURRENCY", ...Object.values(HOST_BUDGET_KEYS), "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_LOG_RETENTION_DAYS", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_SECRET_PROFILES", "PI_SECRET_RESOLVER_ROOTS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "PI_ALLOWED_MODELS", "PI_DISPATCH_RUN_ROOTS", "GITHUB_PAT_VAR", "TEMP", ...Object.values(DOLLAR_ENV_NAMES)]);
599
606
  /** Issue #471: the receiver's keys doctor judges its boot by (the receiver's unit reads the same `.env`). */
600
607
  export const RECEIVER_SERVICE_KEYS = Object.freeze(["WEBHOOK_SECRET", "RECEIVER_PORT", "GITLAB_TOKEN", "GITLAB_URL", "GITLAB_WEBHOOK_MODE", "GITLAB_WEBHOOK_SECRET", "FORGEJO_URL", "FORGEJO_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_ORG_URL", "AZURE_TOKEN", "AZURE_WEBHOOK_MODE", "AZURE_WEBHOOK_SECRET", "AZURE_WEBHOOK_HEADER"]);
601
608
  /**
@@ -2594,6 +2601,11 @@ export async function collectChecks(shellVars, seams) {
2594
2601
  // Said, rather than silently absent: "no peers" and "could not ask" are different facts.
2595
2602
  checks.push({ ok: true, label: `Fleet: could not read the host registry (${printable(fleet.unreachable)})` });
2596
2603
  }
2604
+ // Issue #599, phase 2: how busy each host was over the last 7 days, one fact line per host, never a warning (nothing
2605
+ // decides on it). On every deployment, a single host too: its own files are read when the run mirror is not. Read from
2606
+ // the same Valkey the fleet was, with the registry rows just read (their running jobs count up to now), every Valkey
2607
+ // call bounded and the whole read under `CAPACITY_DOCTOR_TIMEOUT_MS`, so a slow mirror costs this line, not the run.
2608
+ checks.push(...(await doctorCapacity({ seams, env, home, url: valkeyUsable ? valkeyTalkUrl : null, hosts: fleet.hosts ?? [], localHost: workerNameOf(declaredWorkerName) })));
2597
2609
  // Issue #504 part B: the APPLIED split names the envelope it was made for, and every host
2598
2610
  // whose envelope differs refuses its governed jobs. Read whenever this command may talk to the Valkey (above), from
2599
2611
  // the same Valkey the fleet was read from; with no split there, or no answer, nothing is said.
@@ -2718,6 +2730,23 @@ export async function collectChecks(shellVars, seams) {
2718
2730
  },
2719
2731
  },
2720
2732
  });
2733
+ // The overlay's MCP servers (REQ-MCP-OPERATOR-SERVERS), through the one parser (mcp-config.mjs).
2734
+ let mcpAllowlist = [];
2735
+ try {
2736
+ mcpAllowlist = parseAllowlist(keylessIo.readFileSync(join(seams.cwd, "egress-allowlist.conf"), "utf8"));
2737
+ } catch {
2738
+ // No allowlist in this folder: every server outside a declared endpoint reads as unreachable, which it is.
2739
+ }
2740
+ checks.push(
2741
+ ...overlayMcpChecks({
2742
+ overlay,
2743
+ env,
2744
+ fileExists,
2745
+ read: () => readOverlayMcp(overlay, { readFileSync: seams.readOverlayFile ?? ((p, enc) => readFileSync(p, enc)), ...(seams.lstatOverlayFile ? { lstatSync: seams.lstatOverlayFile } : {}) }),
2746
+ allowlist: mcpAllowlist,
2747
+ endpoints: () => (seams.declaredEndpoints ?? ((a) => declaredEndpointsIn({ ...a, fs: keylessIo })))({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform, valkeyUrl: env.VALKEY_URL ?? DEFAULT_VALKEY_URL }),
2748
+ }),
2749
+ );
2721
2750
  const modelsPath = join(overlay, "models.json");
2722
2751
  // Through the one reader (`readOverlayModels`, PR #520): an exists test answered false for a file under an
2723
2752
  // unreadable directory, so this line passed in silence on a file nobody had read. Absent is a pass (nothing to
@@ -4722,21 +4751,15 @@ export function appliedSplitChecks(applied, mine, myName, peers) {
4722
4751
  */
4723
4752
  export async function defaultReadAppliedSplit(url) {
4724
4753
  try {
4725
- const { makeRedisClient } = await import("./connection.mjs");
4726
- const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
4727
- client.on("error", () => {});
4728
- try {
4729
- await client.connect();
4730
- const text = await client.get(ALLOC_PLAN_KEY);
4754
+ return await withDoctorClient(url, async (client) => {
4755
+ const text = await boundedOp(client.get(ALLOC_PLAN_KEY), DOCTOR_VALKEY_OP_TIMEOUT_MS);
4731
4756
  if (text === null || text === undefined) return null;
4732
4757
  let digest = null;
4733
4758
  try {
4734
4759
  digest = JSON.parse(text)?.envelopeDigest;
4735
4760
  } catch {}
4736
4761
  return typeof digest === "string" && /^[0-9a-f]{16}$/.test(digest) ? { digest } : { undecodable: true };
4737
- } finally {
4738
- client.disconnect();
4739
- }
4762
+ });
4740
4763
  } catch {
4741
4764
  return null;
4742
4765
  }
@@ -5900,6 +5923,79 @@ export function forgeUrlEgressChecks(env, forges = []) {
5900
5923
  return checks;
5901
5924
  }
5902
5925
 
5926
+ /**
5927
+ * The overlay's MCP servers (REQ-MCP-OPERATOR-SERVERS): whether `mcp.json` reads, each server a job may be offered,
5928
+ * each one refused and why, no `mcp-auth.json` beside it, and, with the egress policy on, whether each streamable HTTP
5929
+ * server is one the proxy lets a job reach. A refused server is one no job can use, so each is ✗; an unreachable one
5930
+ * is ⚠, since the policy may be about to change.
5931
+ * `read` throws the reader's errors; `endpoints` is called only when a server needs it.
5932
+ */
5933
+ export function overlayMcpChecks({ overlay, env, fileExists, read, allowlist = [], endpoints = () => [] }) {
5934
+ const checks = [];
5935
+ const authPath = join(overlay, "mcp-auth.json");
5936
+ if (fileExists(authPath)) {
5937
+ checks.push({
5938
+ ok: false,
5939
+ label: "Overlay holds mcp-auth.json, pi's MCP sign-ins, which no job may hold",
5940
+ fix: "delete mcp-auth.json from the overlay; a job's MCP server takes its credential as a ${NAME} reference from run.secrets: docs/mcp.md",
5941
+ fixAction: {
5942
+ tier: "prompt",
5943
+ describe: `rm ${authPath}`,
5944
+ run: async ({ rm }) => {
5945
+ rm(authPath);
5946
+ return { ok: true };
5947
+ },
5948
+ },
5949
+ });
5950
+ }
5951
+ let parsed;
5952
+ try {
5953
+ parsed = read();
5954
+ } catch (error) {
5955
+ const why = error?.mcpConfig ? error.message : `cannot be read (${error?.code ?? "unreadable"})`;
5956
+ checks.push({ ok: false, label: `Overlay mcp.json: ${why}`, fix: "re-run `pi-dispatch import-pi`, or fix the file: docs/mcp.md. Until it reads, no job can use an MCP server" });
5957
+ return checks;
5958
+ }
5959
+ if (parsed === null) return checks;
5960
+ const names = Object.keys(parsed.servers);
5961
+ checks.push({ ok: true, label: names.length === 0 ? "Overlay mcp.json offers no MCP server" : `Overlay mcp.json offers ${names.length} MCP server${names.length === 1 ? "" : "s"} to triggers that name them (${names.join(", ")})` });
5962
+ for (const { name, problem } of parsed.refused) {
5963
+ checks.push({ ok: false, label: `Overlay mcp.json: server ${quotedShown(name)} is refused: ${problem}`, fix: "fix the entry and re-run `pi-dispatch import-pi`: docs/mcp.md. Until then no job can use it" });
5964
+ }
5965
+ let armed = false;
5966
+ try {
5967
+ armed = egressArmed(env) === true;
5968
+ } catch {
5969
+ // A malformed PI_EGRESS: the .env check reports it.
5970
+ }
5971
+ let declared = null;
5972
+ for (const name of names) {
5973
+ const server = parsed.servers[name];
5974
+ if (server.packageRunner) {
5975
+ const missing = packageRunnerHosts(server.command).filter((host) => allowlistEntryFor(allowlist, host) === null);
5976
+ const listed = missing.length === 0;
5977
+ const registry = missing.join(" and ");
5978
+ checks.push({
5979
+ ok: false,
5980
+ warn: true,
5981
+ label: `MCP server ${name} runs ${mcpServerWhere(server)}, which fetches the server from a package registry at the start of every job that uses it`,
5982
+ fix: armed && !listed ? `install the server in the job image instead, or allowlist ${registry} in egress-allowlist.conf: docs/mcp.md` : "install the server in the job image instead, so every job runs the version you chose: docs/mcp.md",
5983
+ });
5984
+ }
5985
+ if (!armed || server.transport !== "http" || server.enabled === false) continue;
5986
+ declared ??= endpoints();
5987
+ const refusal = egressRefusalFor(server.url, { allowlist, endpoints: declared });
5988
+ if (refusal === null) continue;
5989
+ checks.push({
5990
+ ok: false,
5991
+ warn: true,
5992
+ label: `MCP server ${name} (${mcpServerWhere(server)}) is not reachable with the egress policy on: ${refusal}`,
5993
+ fix: "use an https:// url and allowlist its host for 443, or declare its host and port in model-endpoints.json: docs/egress.md. Until then no job can reach it",
5994
+ });
5995
+ }
5996
+ return checks;
5997
+ }
5998
+
5903
5999
  /** The canary's three probes. ONE list: the loop names its containers from it and the sweep matches on it. */
5904
6000
  export const CANARY_PROBE_SLUGS = Object.freeze(["provider", "unlisted", "plainhttp"]);
5905
6001
 
@@ -6149,6 +6245,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
6149
6245
  const { declaredEndpoints = ({ env: e, cwd, platform }) => declaredEndpointsIn({ env: e, cwd, fs: { readFileSync, existsSync }, platform }), hostAddresses = () => lanIPv4Addresses(networkInterfaces()) } = seams;
6150
6246
  const declared = declaredEndpoints({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform });
6151
6247
  const endpoints = declared.length > 0 && !(!custom && inFolder && !needs.rulesInclude) ? declared : [];
6248
+ const mcpOnly = endpoints.length > 0 ? mcpOnlyEndpoints({ endpoints, ...overlayEndpointUsers(env, seams) }) : new Map();
6152
6249
  if (endpoints.length > 0) {
6153
6250
  const runtime = routeRuntimeFromFacts(seams.readFactsOnce ? await seams.readFactsOnce() : null, seams.platform ?? process.platform);
6154
6251
  const addresses = hostAddresses();
@@ -6255,7 +6352,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
6255
6352
  // script. docker's probes keep `runCmdCapture` (its 30 s bound, stderr merged, SIGTERM at the bound) rather than the
6256
6353
  // bounded step runner, so docker's spawns, and what a probe that never launched leaves for the teardown, are exactly
6257
6354
  // what they were: pinned in doctor.test.mjs against the output and argv captured before the move.
6258
- const canary = await runEgressCanary({ run: liveRunVia(spawn), probeRun: (args) => runCmdCapture(spawn, "docker", args), proxy, image: jobImage, pid, gate, endpoints });
6355
+ const canary = await runEgressCanary({ run: liveRunVia(spawn), probeRun: (args) => runCmdCapture(spawn, "docker", args), proxy, image: jobImage, pid, gate, endpoints, mcpOnly });
6259
6356
  // Through a stale proxy the canary reads THAT proxy's policy (exec round 3): a failure is the proxy's to fix, never
6260
6357
  // this deployment's allowlist, which it may not even mount.
6261
6358
  checks.push(...(staleRunning ? canary.checks.map((c) => (c.ok ? c : { ...c, fix: `the canary ran through ${proxy}, which is not this deployment's proxy (above), so this says nothing about egress-allowlist.conf: ${replaceFix}, then re-run doctor` })) : canary.checks));
@@ -6349,8 +6446,10 @@ const CANARY_NO_WORKSPACE = "/nonexistent/pi-dispatch-egress-canary-mounts-nothi
6349
6446
  *
6350
6447
  * `endpoints` (issue #503) are the declared model endpoints to prove on the same network after the three
6351
6448
  * (`runEndpointProbes`), `[]` by default, so the conformance script and a deployment with none run exactly the three.
6449
+ * `mcpOnly` maps the ids of those only an overlay MCP server uses (`mcpOnlyEndpoints`), whose `models` probe proves
6450
+ * the route and is not judged as a model server's answer.
6352
6451
  */
6353
- export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = process.pid, user = null, probeRun = null, endpoints = [], size = DEFAULT_JOB_SIZE, hostCpus = null, gate = makeDetachGate((args, opts) => run(args, { timeoutMs: opts?.timeoutMs ?? CANARY_STEP_TIMEOUT_MS }), { bin }) }) {
6452
+ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = process.pid, user = null, probeRun = null, endpoints = [], mcpOnly = new Map(), size = DEFAULT_JOB_SIZE, hostCpus = null, gate = makeDetachGate((args, opts) => run(args, { timeoutMs: opts?.timeoutMs ?? CANARY_STEP_TIMEOUT_MS }), { bin }) }) {
6354
6453
  const venue = canaryVenueFor(bin);
6355
6454
  const probe = probeRun ?? ((args) => run(args, { timeoutMs: bin === "podman" ? PODMAN_FIRST_START_TIMEOUT_MS : RUN_TIMEOUTS.cmd }));
6356
6455
  const checks = [];
@@ -6505,7 +6604,7 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
6505
6604
  if (staleRunner) {
6506
6605
  checks.push({ ok: false, warn: true, label: `${venue.prefix}Model endpoints: not probed, because the job image has no runner module (above), and their probes take the runner's route`, fix: "use a job image built after issue #427, then re-run doctor" });
6507
6606
  } else {
6508
- checks.push(...(await runEndpointProbes({ probe, bin, pid, network: net, proxy, image, user, endpoints, venue, unfinished, size, hostCpus })));
6607
+ checks.push(...(await runEndpointProbes({ probe, bin, pid, network: net, proxy, image, user, endpoints, mcpOnly, venue, unfinished, size, hostCpus })));
6509
6608
  }
6510
6609
  }
6511
6610
  } finally {
@@ -6609,6 +6708,66 @@ function endpointsById(endpoints) {
6609
6708
  return [...endpoints].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
6610
6709
  }
6611
6710
 
6711
+ /**
6712
+ * The declared endpoints that an overlay MCP server dials (an enabled streamable HTTP server's `url` in `mcp.json`, the
6713
+ * servers doctor's own MCP check and the pickup judge) and no overlay model does (a provider's or a model's `baseUrl` in `models.json`), each matched by `endpointsAt`, as the
6714
+ * pickup's egress check matches an MCP server to an endpoint. docs/egress.md has an operator declare an MCP server off
6715
+ * port 443 there, and such a server does not serve `/v1/models`: it answers 404 or 405, and the `models` probe used to
6716
+ * warn on every run that the port was not the model server's. Its tunnel is still opened and its route proved; only its
6717
+ * answer is not judged as a model server's. An endpoint both use is probed as a model server, since a model needs that.
6718
+ * A models.json that could not be read is not evidence that no model uses an endpoint, so nothing is MCP-only then.
6719
+ * Returns a Map of endpoint id to `{ tls }`, `tls` when a server's url on it is `https:`: the probe's plain request then
6720
+ * meets a TLS server, which closes the socket rather than answer (`endpointProbeCheck`).
6721
+ */
6722
+ export function mcpOnlyEndpoints({ endpoints, mcp = null, models = null, modelsUnread = false }) {
6723
+ if (modelsUnread || !Array.isArray(endpoints) || endpoints.length === 0) return new Map();
6724
+ const idsAt = (urls) => new Set(urls.flatMap((url) => endpointsAt(url, endpoints).map((e) => e.id)));
6725
+ const mcpUrls = Object.values(mcp?.servers ?? {})
6726
+ .filter((server) => server.transport === "http" && server.enabled !== false)
6727
+ .map((server) => server.url);
6728
+ const modelUrls = [];
6729
+ const providers = models?.providers;
6730
+ for (const entry of providers !== null && typeof providers === "object" ? Object.values(providers) : []) {
6731
+ if (entry === null || typeof entry !== "object") continue;
6732
+ modelUrls.push(entry.baseUrl);
6733
+ for (const model of Array.isArray(entry.models) ? entry.models : []) modelUrls.push(model?.baseUrl);
6734
+ }
6735
+ const byModels = idsAt(modelUrls);
6736
+ const only = new Map();
6737
+ for (const url of mcpUrls) {
6738
+ for (const { id } of endpointsAt(url, endpoints)) {
6739
+ if (byModels.has(id)) continue;
6740
+ only.set(id, { tls: only.get(id)?.tls === true || new URL(url).protocol === "https:" });
6741
+ }
6742
+ }
6743
+ return only;
6744
+ }
6745
+
6746
+ /**
6747
+ * The overlay's mcp.json and models.json for `mcpOnlyEndpoints`, through the readers the overlay section uses and its
6748
+ * seams. An mcp.json that cannot be read offers no server here (its own line says why); a models.json that cannot be read
6749
+ * is `modelsUnread`, which keeps every endpoint probed as a model server.
6750
+ */
6751
+ function overlayEndpointUsers(env, seams) {
6752
+ const overlay = env.PI_GLOBAL_PI_DIR;
6753
+ if (typeof overlay !== "string" || !isAbsolute(overlay)) return {};
6754
+ const io = { readFileSync: seams.readOverlayFile ?? ((p, enc) => readFileSync(p, enc)), ...(seams.lstatOverlayFile ? { lstatSync: seams.lstatOverlayFile } : {}) };
6755
+ let mcp = null;
6756
+ try {
6757
+ mcp = readOverlayMcp(overlay, io);
6758
+ } catch {
6759
+ // Named by the overlay section's own line.
6760
+ }
6761
+ try {
6762
+ return { mcp, models: readOverlayModels(overlay, io) };
6763
+ } catch {
6764
+ return { mcp, modelsUnread: true };
6765
+ }
6766
+ }
6767
+
6768
+ /** The `blocked` codes of a socket the far end closed: what an https server does with the probe's plain request. */
6769
+ const SOCKET_CLOSED_CODES = Object.freeze(["UND_ERR_SOCKET", "ECONNRESET"]);
6770
+
6612
6771
  /** The exits each probe's script ends a READING with; any other exit is a container that did not run it. */
6613
6772
  const ENDPOINT_PROBE_READ_EXITS = Object.freeze({ models: [0, 3, 6], nextport: [0, 3, 6], plain: [0, 3] });
6614
6773
 
@@ -6621,8 +6780,8 @@ const ENDPOINT_PROBE_READ_EXITS = Object.freeze({ models: [0, 3, 6], nextport: [
6621
6780
  * - `plain`: a 403 carrying squid's `X-Squid-Error: ERR_ACCESS_DENIED`. A bare 403 may be the server's own answer to a
6622
6781
  * request the proxy forwarded.
6623
6782
  */
6624
- function endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next }) {
6625
- const head = `${venue.prefix}Model endpoint ${endpoint.id}`;
6783
+ function endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next, mcpOnly = null }) {
6784
+ const head = mcpOnly ? `${venue.prefix}MCP server endpoint ${endpoint.id}` : `${venue.prefix}Model endpoint ${endpoint.id}`;
6626
6785
  const at = endpointAddress(endpoint.host, endpoint.port);
6627
6786
  const reading = endpointProbeReading(answer);
6628
6787
  const tag = { id: endpoint.id, probe: slug, kind: reading?.kind ?? null, status: reading?.status ?? null };
@@ -6637,6 +6796,19 @@ function endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next })
6637
6796
  }
6638
6797
  const reload = reloadCommand(bin, proxy);
6639
6798
  if (slug === "models") {
6799
+ // An endpoint only an overlay MCP server uses: any answer through the tunnel proves the route and the rule, and
6800
+ // what it says to /v1/models is not a model server's answer (`mcpOnlyEndpoints`). Not reaching it still fails below.
6801
+ if (mcpOnly && reading.kind === "reached") {
6802
+ return { ok: true, warn: true, label: `${head}: the route through the proxy works (the tunnel to ${at} opened and the server answered ${reading.status}); no overlay model uses it, so it is not probed as a model server`, endpointProbe: tag };
6803
+ }
6804
+ // An https MCP server meets the probe's plain request as TLS servers do: it closes the socket (measured: undici reads
6805
+ // UND_ERR_SOCKET, "other side closed"; a reset is the same fact). Only a CONNECT the proxy let through reaches the
6806
+ // server to be closed by it, since a refused tunnel reads as `tunnel`, so the route is proved. Not a timeout, and
6807
+ // not a code that says the proxy itself was not reached (ECONNREFUSED to it, a name it could not resolve).
6808
+ if (mcpOnly?.tls && reading.kind === "blocked" && SOCKET_CLOSED_CODES.includes(reading.detail)) {
6809
+ return { ok: true, warn: true, label: `${head}: the route through the proxy works (the tunnel to ${at} opened and the server closed a plain request, as a TLS server does: ${reading.detail}); no overlay model uses it, so it is not probed as a model server`, endpointProbe: tag };
6810
+ }
6811
+ const server = mcpOnly ? "MCP server" : "model server";
6640
6812
  if (reading.kind === "reached" && reading.status === 200) return { ok: true, warn: true, label: `${head} answers through the proxy (GET http://${at}/v1/models through a CONNECT tunnel: 200)`, endpointProbe: tag };
6641
6813
  // The SERVER refused the request for want of a key (vLLM --api-key, a gateway in front): the route and the rule are
6642
6814
  // proved, since only an open tunnel carries the server's own answer. Said apart from a fault, still warn-tier.
@@ -6669,10 +6841,12 @@ function endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next })
6669
6841
  : reading.kind === "reached"
6670
6842
  ? "check that this port is the model server's and that it serves the OpenAI-compatible API under /v1 (Ollama, llama-server, vLLM and LM Studio do)"
6671
6843
  : timedOut
6672
- ? "the server is there but stuck or busy (a model still loading, every slot taken): check its own log, then re-run doctor"
6844
+ ? mcpOnly
6845
+ ? "the server is there but stuck or busy: check its own log, then re-run doctor"
6846
+ : "the server is there but stuck or busy (a model still loading, every slot taken): check its own log, then re-run doctor"
6673
6847
  : bin === "podman"
6674
- ? `start the model server, bound to this host's LAN address or 0.0.0.0 (never 127.0.0.1 alone), and declare a server on this host as host.containers.internal (docs/egress.md, "Local model servers"). A job meets this as "Connection error"`
6675
- : `start the model server, and check it listens where the proxy reaches it (docs/egress.md, "Local model servers"; on Docker Engine that is 172.17.0.1 or 0.0.0.0, never 127.0.0.1 alone). A job meets this as "Connection error"`;
6848
+ ? `start the ${server}, bound to this host's LAN address or 0.0.0.0 (never 127.0.0.1 alone), and declare a server on this host as host.containers.internal (docs/egress.md, "Local model servers"). A job meets this as "Connection error"`
6849
+ : `start the ${server}, and check it listens where the proxy reaches it (docs/egress.md, "Local model servers"; on Docker Engine that is 172.17.0.1 or 0.0.0.0, never 127.0.0.1 alone). A job meets this as "Connection error"`;
6676
6850
  return { ok: false, warn: true, label: `${head} does NOT answer: ${why}`, fix, endpointProbe: tag };
6677
6851
  }
6678
6852
  const wider = `the proxy's rules are wider than model-endpoints.conf renders: compare the include inside it (\`${bin} exec ${proxy} cat ${ENDPOINTS_INCLUDE_IN_PROXY}\`) with \`pi-dispatch egress render\`'s, check egress-allowlist.conf does not list ${endpoint.host}, then \`${reload}\``;
@@ -6721,7 +6895,7 @@ export function undeclaredPortNear(endpoint, endpoints) {
6721
6895
  * variable as well on docker: the runner's dispatcher sends an `http://` origin to HTTP_PROXY, which docker's canary
6722
6896
  * argv does not otherwise carry (podman's is a job's environment and has it).
6723
6897
  */
6724
- async function runEndpointProbes({ probe, bin, pid, network, proxy, image, user, endpoints, venue, unfinished, size = DEFAULT_JOB_SIZE, hostCpus = null }) {
6898
+ async function runEndpointProbes({ probe, bin, pid, network, proxy, image, user, endpoints, mcpOnly = new Map(), venue, unfinished, size = DEFAULT_JOB_SIZE, hostCpus = null }) {
6725
6899
  const checks = [];
6726
6900
  for (const endpoint of endpointsById(endpoints)) {
6727
6901
  const next = undeclaredPortNear(endpoint, endpoints);
@@ -6736,7 +6910,7 @@ async function runEndpointProbes({ probe, bin, pid, network, proxy, image, user,
6736
6910
  const name = egressEndpointProbe(slug, endpoint.id, pid);
6737
6911
  const answer = await probe(egressCanaryProbeArgs({ bin, slug, name, pid, network, proxy, image, url, user, script, httpProxy: true, size, hostCpus }));
6738
6912
  if (answer?.code === null && answer.ended !== "error") unfinished.push(name);
6739
- checks.push(endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next }));
6913
+ checks.push(endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next, mcpOnly: mcpOnly.get(endpoint.id) ?? null }));
6740
6914
  }
6741
6915
  }
6742
6916
  return checks;
@@ -7389,16 +7563,9 @@ function workerNameOf(declared) {
7389
7563
 
7390
7564
  async function defaultReadHosts(url) {
7391
7565
  try {
7392
- const { makeRedisClient } = await import("./connection.mjs");
7393
7566
  const { readLiveHosts } = await import("./host-registry.mjs");
7394
- const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
7395
- client.on("error", () => {});
7396
- try {
7397
- await client.connect();
7398
- return await readLiveHosts(client);
7399
- } finally {
7400
- client.disconnect();
7401
- }
7567
+ // readLiveHosts bounds each of its own commands.
7568
+ return await withDoctorClient(url, (client) => readLiveHosts(client));
7402
7569
  } catch (err) {
7403
7570
  return { unreachable: err?.message ?? "registry unreadable" };
7404
7571
  }
@@ -7447,35 +7614,60 @@ export async function dollarKeysExistWith(client, { now = () => new Date(), dead
7447
7614
  /** `dollarKeysExistWith` over a fail-fast client on `url`, always disconnected. */
7448
7615
  export async function defaultDollarKeysExist(url) {
7449
7616
  try {
7450
- const { makeRedisClient } = await import("./connection.mjs");
7451
- const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
7452
- client.on("error", () => {});
7453
- try {
7454
- await client.connect();
7455
- return await dollarKeysExistWith(client);
7456
- } finally {
7457
- client.disconnect();
7458
- }
7617
+ // dollarKeysExistWith bounds its whole read.
7618
+ return await withDoctorClient(url, (client) => dollarKeysExistWith(client));
7459
7619
  } catch {
7460
7620
  return false;
7461
7621
  }
7462
7622
  }
7463
7623
 
7464
7624
  async function defaultProbeValkey(url) {
7465
- const { makeRedisClient } = await import("./connection.mjs");
7466
- const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
7467
- client.on("error", () => {}); // swallow connect errors + retries; reachability is the ✓/✗, not a trace
7468
7625
  try {
7469
- await client.connect();
7470
- await client.ping();
7471
- return true;
7626
+ return await withDoctorClient(url, async (client) => (await boundedOp(client.ping(), DOCTOR_VALKEY_OP_TIMEOUT_MS), true));
7472
7627
  } catch {
7473
7628
  return false;
7629
+ }
7630
+ }
7631
+
7632
+ /** How long doctor waits for a Valkey connection to be ready (the TCP connect AND the ready check), and for one command. */
7633
+ export const DOCTOR_VALKEY_CONNECT_TIMEOUT_MS = 5_000;
7634
+ export const DOCTOR_VALKEY_OP_TIMEOUT_MS = 5_000;
7635
+
7636
+ /**
7637
+ * One fail-fast client for one of doctor's own Valkey reads, its connection bounded, always dropped. Measured: a
7638
+ * server that accepts the connection and never answers holds ioredis in its ready check forever (`failFast` bounds
7639
+ * only the TCP connect), so doctor never exited; and `disconnect()` arms a 2 s timer a closed socket never clears, so
7640
+ * the client is built with a disconnect timeout of 0 and its stream destroyed. Throws when the connection is not ready
7641
+ * in time; `fn`'s own commands are the caller's to bound (`boundedOp`).
7642
+ */
7643
+ export async function withDoctorClient(url, fn, { connectTimeoutMs = DOCTOR_VALKEY_CONNECT_TIMEOUT_MS, makeClient } = {}) {
7644
+ const make = makeClient ?? (await import("./connection.mjs")).makeRedisClient;
7645
+ const client = make(url, { failFast: true, lazyConnect: true, disconnectTimeoutMs: 0 });
7646
+ client.on?.("error", () => {}); // swallow connect errors + retries; reachability is the ✓/✗, not a trace
7647
+ try {
7648
+ await connectWithin(client, connectTimeoutMs);
7649
+ return await fn(client);
7474
7650
  } finally {
7475
- client.disconnect();
7651
+ try {
7652
+ client.disconnect?.();
7653
+ client.stream?.destroy?.();
7654
+ } catch {
7655
+ // a release that failed has stopped mattering
7656
+ }
7476
7657
  }
7477
7658
  }
7478
7659
 
7660
+ /** A command's reply, or a rejection after `ms` (the client is dropped by `withDoctorClient` either way). */
7661
+ export function boundedOp(promise, ms) {
7662
+ return new Promise((resolve, reject) => {
7663
+ const t = setTimeout(() => reject(new Error("timeout")), ms);
7664
+ Promise.resolve(promise).then(
7665
+ (v) => (clearTimeout(t), resolve(v)),
7666
+ (e) => (clearTimeout(t), reject(e)),
7667
+ );
7668
+ });
7669
+ }
7670
+
7479
7671
  /**
7480
7672
  * The deployment's default job size as doctor reads it (issue #596): `PI_JOB_MEMORY` and `PI_JOB_CPUS` through the
7481
7673
  * worker's own rule (`jobSizeDefaults`), or the built-in 4g and 2 when they do not parse (`jobSizeChecks` reports that as
@@ -7756,6 +7948,162 @@ export function hostBudgetChecks(view, { concurrency = 3, limits = [], env = {},
7756
7948
  return checks;
7757
7949
  }
7758
7950
 
7951
+ /** The most the capacity read may take in doctor, all of it: a full mirror reads in well under a second. */
7952
+ export const CAPACITY_DOCTOR_TIMEOUT_MS = 5_000;
7953
+
7954
+ /**
7955
+ * The capacity lines (issue #599, phase 2): the report over the last 7 days (`capacity.mjs`, the CLI's function), read
7956
+ * by `seams.readCapacity` (default: `readCapacityRecords` through a fail-fast client, `prune`-free and writing nothing).
7957
+ * Never throws and never warns: an unreadable history is one line saying so.
7958
+ */
7959
+ export async function doctorCapacity({ seams, env, home, url, hosts, localHost }) {
7960
+ try {
7961
+ const window = CAPACITY_WINDOWS["7d"];
7962
+ const nowMs = (typeof seams.wallClock === "function" ? seams.wallClock : Date.now)();
7963
+ const retention = typeof env.PI_LOG_RETENTION_DAYS === "string" && /^\d{1,6}$/.test(env.PI_LOG_RETENTION_DAYS) ? Number(env.PI_LOG_RETENTION_DAYS) : 30;
7964
+ // The read is told to stop when the bound passes (`signal`): giving up on it is not enough, since a client still
7965
+ // waiting on its connection would keep this process alive after doctor has printed everything.
7966
+ const stop = new AbortController();
7967
+ const args = { url, logsDir: logsDirPath(env, home), sinceMs: nowMs - window.ms, nowMs, retentionDays: retention, localHost, signal: stop.signal };
7968
+ let timer;
7969
+ const read = await Promise.race([
7970
+ Promise.resolve()
7971
+ .then(() => (seams.readCapacity ?? defaultReadCapacity)(args))
7972
+ .finally(() => clearTimeout(timer)),
7973
+ new Promise((resolve) => {
7974
+ timer = setTimeout(() => {
7975
+ stop.abort();
7976
+ resolve(null);
7977
+ }, seams.capacityTimeoutMs ?? CAPACITY_DOCTOR_TIMEOUT_MS);
7978
+ }),
7979
+ ]);
7980
+ if (read === null) return [{ ok: true, label: "Capacity: the run history did not answer in time, so nothing is shown (pi-dispatch capacity waits up to 2 s per Valkey call)" }];
7981
+ const report = computeCapacity({ records: read.records, live: hosts, windowStartMs: nowMs - window.ms, nowMs, bucketMs: window.bucketMs, coverage: read.coverage });
7982
+ return capacityChecks(report, { since: "7d" });
7983
+ } catch (err) {
7984
+ if (err?.[FORBIDDEN_READ]) throw err;
7985
+ return [{ ok: true, label: `Capacity: not read (${printable(err?.message ?? "error")})` }];
7986
+ }
7987
+ }
7988
+
7989
+ /**
7990
+ * Set by the test helper (test/helpers/doctor.mjs): from then on the DEFAULT capacity read throws an error doctor does
7991
+ * not swallow, so a test that reaches the developer's real Valkey and logs directory fails loudly instead of printing
7992
+ * whatever that machine holds.
7993
+ */
7994
+ let defaultCapacityReadForbidden = false;
7995
+ export function forbidDefaultCapacityRead() {
7996
+ defaultCapacityReadForbidden = true;
7997
+ }
7998
+ const FORBIDDEN_READ = Symbol("forbidden-read");
7999
+
8000
+ /** How long the default read waits for its Valkey connection to be ready (the TCP connect and the ready check). */
8001
+ export const CAPACITY_CONNECT_TIMEOUT_MS = 2_000;
8002
+
8003
+ /**
8004
+ * The default capacity read: a fail-fast client, its connection bounded (`connectWithin`: a server that accepts TCP and
8005
+ * never answers holds ioredis in its ready check forever) and DISCONNECTED when `signal` aborts, so a read doctor gave
8006
+ * up on cannot keep the process alive.
8007
+ */
8008
+ async function defaultReadCapacity({ url, logsDir, sinceMs, nowMs, retentionDays, localHost, signal }) {
8009
+ if (defaultCapacityReadForbidden) throw Object.assign(new Error("doctor reached its real capacity read in a test: pass readCapacity"), { [FORBIDDEN_READ]: true });
8010
+ const { readCapacityRecords } = await import("./capacity-records.mjs");
8011
+ let client = null;
8012
+ // Measured on ioredis 5.11: `disconnect()` ends the socket and arms a timer (`disconnectTimeout`, 2 s by default)
8013
+ // that destroys it unless the socket's `close` clears it first, and the client's own failed ready check disconnects
8014
+ // again after the close, arming one that nothing clears: a server that never answered held the process 2 s past
8015
+ // doctor's bound. So the client is built with a disconnect timeout of 0 (no timer outlives the socket), its stream
8016
+ // is destroyed in the same tick (the socket is closed by the time doctor prints), and it is dropped once.
8017
+ const drop = () => {
8018
+ const c = client;
8019
+ client = null;
8020
+ try {
8021
+ c?.disconnect?.();
8022
+ c?.stream?.destroy?.();
8023
+ } catch {
8024
+ // a release that failed has stopped mattering
8025
+ }
8026
+ };
8027
+ signal?.addEventListener?.("abort", drop, { once: true });
8028
+ if (url && !signal?.aborted) {
8029
+ try {
8030
+ const { makeRedisClient } = await import("./connection.mjs");
8031
+ client = makeRedisClient(url, { failFast: true, lazyConnect: true, disconnectTimeoutMs: 0 });
8032
+ client.on("error", () => {});
8033
+ await connectWithin(client, CAPACITY_CONNECT_TIMEOUT_MS, signal);
8034
+ } catch {
8035
+ drop();
8036
+ }
8037
+ }
8038
+ try {
8039
+ return await readCapacityRecords({ redis: client, logsDir, sinceMs, nowMs, retentionDays, localHost, noMirrorReason: url ? "the Valkey did not answer: only this host's files were read" : "no Valkey to read: only this host's files were read" });
8040
+ } finally {
8041
+ signal?.removeEventListener?.("abort", drop);
8042
+ drop();
8043
+ }
8044
+ }
8045
+
8046
+ /**
8047
+ * `client.connect()`, or a rejection after `ms` or as soon as `signal` aborts (the client is then the caller's to
8048
+ * disconnect). The timer goes with an abort, so nothing of a read doctor gave up on holds the process.
8049
+ */
8050
+ export function connectWithin(client, ms, signal) {
8051
+ return new Promise((resolve, reject) => {
8052
+ const stop = () => (clearTimeout(t), reject(new Error("connect stopped")));
8053
+ const t = setTimeout(() => (signal?.removeEventListener?.("abort", stop), reject(new Error("connect timeout"))), ms);
8054
+ signal?.addEventListener?.("abort", stop, { once: true });
8055
+ Promise.resolve(client.connect()).then(
8056
+ (v) => (clearTimeout(t), signal?.removeEventListener?.("abort", stop), resolve(v)),
8057
+ (e) => (clearTimeout(t), signal?.removeEventListener?.("abort", stop), reject(e)),
8058
+ );
8059
+ });
8060
+ }
8061
+
8062
+ /**
8063
+ * One fact line per host of a capacity report (`INT-CAPACITY-REPORT`), `{ ok: true, label }` each, never a warning:
8064
+ * `Host a: last 7d busy 63% (avg 2.1 of 4 slots, full 12%), promised 48% memory / 40% CPU, used 18% CPU of 8, wait p50
8065
+ * 40s p95 6m, most busy: web`, each part only when it is known, then what the line cannot see: the running jobs not
8066
+ * counted, and the coverage when the history is cut, not shared, or this host's files only. Every control character is
8067
+ * removed (the report admits none; this is the printer not relying on it).
8068
+ */
8069
+ export function capacityChecks(report, { since = "7d" } = {}) {
8070
+ const out = [];
8071
+ const localOnly = report?.coverage?.source === "local";
8072
+ for (const h of Array.isArray(report?.hosts) ? report.hosts : []) {
8073
+ const notes = [];
8074
+ let line;
8075
+ if (h.coveredMs === 0) {
8076
+ line = `Host ${h.name}: last ${since} no history here`;
8077
+ if (!h.shared) notes.push(notSharedWhy(h, report.coverage));
8078
+ } else {
8079
+ const c = h.capacity ?? {};
8080
+ const slots = Number.isSafeInteger(c.slots) ? ` of ${c.slots} slots` : " at once";
8081
+ const full = h.fullMs !== null ? `, full ${shareText(h.fullMs, h.coveredMs)}` : "";
8082
+ const parts = [`busy ${shareText(h.busyMs, h.coveredMs)} (avg ${milliText(h.avgMilli ?? 0)}${slots}${full})`];
8083
+ const promised = [h.promisedMemPerMille !== null ? `${percentText(h.promisedMemPerMille)} memory` : null, h.promisedCpuPerMille !== null ? `${percentText(h.promisedCpuPerMille)} CPU` : null].filter(Boolean);
8084
+ if (promised.length > 0) parts.push(`promised ${promised.join(" / ")}`);
8085
+ if (h.usedCpuPerMille !== null) parts.push(`used ${percentText(h.usedCpuPerMille)} CPU${Number.isSafeInteger(c.cpus) ? ` of ${c.cpus}` : ""}`);
8086
+ if (h.waits?.n > 0) parts.push(`wait p50 ${durationText(h.waits.p50Ms)} p95 ${durationText(h.waits.p95Ms)}`);
8087
+ if (h.projects?.length > 0) parts.push(`most busy: ${h.projects[0].project ?? "(no project)"}`);
8088
+ line = `Host ${h.name}: last ${since} ${parts.join(", ")}`;
8089
+ if (h.coverage.live > 0) notes.push(`${h.coverage.live} running now, counted to now`);
8090
+ if (h.missingMs > 0) notes.push(`history from ${new Date(h.coverage.fromMs).toISOString().slice(0, 16).replace("T", " ")} UTC only${h.coverage.truncated ? " (the run mirror holds nothing older)" : ""}, earlier time counted as neither busy nor idle`);
8091
+ // The reason with it: a run mirror that did not answer leaves a busy fleet reading nearly idle here.
8092
+ // The reasons end in "only this host's files were read", which this clause already says.
8093
+ const why = typeof report.coverage.reason === "string" ? report.coverage.reason.replace(/: only this host's files were read$/, "") : "";
8094
+ if (localOnly) notes.push(`this host's files only${why ? ` (${why})` : ""}`);
8095
+ }
8096
+ if (h.coverage?.liveNotCounted > 0) notes.push(`${h.coverage.liveNotCounted} running now not counted`);
8097
+ if (h.coverage?.liveUnreadable > 0) notes.push("its running jobs could not be read, not counted");
8098
+ if (report.coverage?.liveRowMissing === h.name) notes.push("no live row read for it, so its running jobs are not known");
8099
+ out.push({ ok: true, label: `${line}${notes.length > 0 ? `; ${notes.join("; ")}` : ""}`.replace(/[\u0000-\u001f\u007f-\u009f]/g, "") });
8100
+ }
8101
+ // What none of these lines can see, in the words every surface of the report uses (REQ-CAPACITY-INSIGHTS): a fact,
8102
+ // never a warning, once after the hosts.
8103
+ if (out.length > 0) out.push({ ok: true, label: JOBS_ONLY.replace(/\.$/, "") });
8104
+ return out;
8105
+ }
8106
+
7759
8107
  /**
7760
8108
  * The fleet's budgets (issue #596, phase 2), from the registry rows (this host's own included): one line per host that
7761
8109
  * publishes a budget (its budget, what its jobs hold, what its holds keep, and the largest project size that fits it),
@@ -8528,6 +8876,7 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
8528
8876
  // Issue #503: the declared model endpoints `--live` probes on this venue, `[]` for none; and whether the rules the
8529
8877
  // proxy runs include their file (only the shipped proxy's account copy is read; another proxy must include it).
8530
8878
  let endpoints = [];
8879
+ let mcpOnly = new Map();
8531
8880
  let endpointRulesInclude = true;
8532
8881
  if (armed !== false) {
8533
8882
  // `.State.Status` (issue #453): only `running` carries traffic; Podman reports paused and between-restarts states
@@ -8594,6 +8943,7 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
8594
8943
  const declaredList = declaredEndpoints({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform });
8595
8944
  if (declaredList.length > 0 && endpointRulesInclude) {
8596
8945
  endpoints = declaredList;
8946
+ mcpOnly = mcpOnlyEndpoints({ endpoints, ...overlayEndpointUsers(env, seams) });
8597
8947
  // The helper as Podman 5 names it in `podman info` (`rootlessNetworkCmd`), else as this account's running rootless
8598
8948
  // network shows it (`observeRootlessNetns`, the record or the 4.x argv the worker's widening check reads), else
8599
8949
  // not known, which the route row says as such.
@@ -8617,7 +8967,7 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
8617
8967
  if (keeperNetwork) checks.push(keeperNetwork);
8618
8968
  }
8619
8969
 
8620
- forLive = { run: true, user, relabel, info, imagePresent, egress: { armed, proxy, proxyRunning, keeperBlocked, endpoints } };
8970
+ forLive = { run: true, user, relabel, info, imagePresent, egress: { armed, proxy, proxyRunning, keeperBlocked, endpoints, mcpOnly } };
8621
8971
  return { checks, observed, relabel, imagesReadable, imageDigest, forLive };
8622
8972
  }
8623
8973
 
@@ -9142,7 +9492,7 @@ async function podmanEgressCanary({ podman, readInfo, run, image, pid, isAlive,
9142
9492
  const endpoints = podman.egress.endpoints ?? [];
9143
9493
  const more = endpoints.length > 0 ? `; then three per declared model endpoint (${endpointsById(endpoints).map((e) => e.id).join(", ")}), named ${EGRESS_ENDPOINT_PROBE_PREFIX}<probe>-<id>-${pid}, removed the same way` : "";
9144
9494
  announce(`starting ${probes.slice(0, -1).join(", ")} and ${probes.at(-1)} from ${image} (as the job user ${podman.user}) on the --internal network ${egressCanaryNetwork(pid)}, with ${podman.egress.proxy} attached, to read the egress allowlist back; all three are removed when the canary ends${more}`);
9145
- const canary = await runEgressCanary({ run, bin: "podman", proxy: podman.egress.proxy, image, pid, user: podman.user, gate, endpoints, size, hostCpus: again.info?.hostCpus ?? null });
9495
+ const canary = await runEgressCanary({ run, bin: "podman", proxy: podman.egress.proxy, image, pid, user: podman.user, gate, endpoints, mcpOnly: podman.egress.mcpOnly ?? new Map(), size, hostCpus: again.info?.hostCpus ?? null });
9146
9496
  return { checks: [...checks, ...canary.checks], results: canary.results };
9147
9497
  }
9148
9498