@edgehero/pi-dispatch 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +94 -4
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +199 -30
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1663 -198
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +103 -9
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-auth.mjs +4 -0
- package/src/forgejo-host.mjs +127 -31
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +73 -15
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +740 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1554 -167
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +296 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +782 -40
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/doctor.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
/**
|
|
2
3
|
* `pi-dispatch doctor` — preflight the host before the first job. Prints a ✓/⚠/✗ line per prerequisite
|
|
3
4
|
* with a one-line fix, and exits non-zero if any hard check fails, so it is usable in a setup script.
|
|
@@ -52,30 +53,33 @@
|
|
|
52
53
|
import { accessSync, chmodSync, closeSync, constants as fsConstants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
53
54
|
import { lookup as dnsLookup } from "node:dns/promises";
|
|
54
55
|
import { homedir, networkInterfaces, release as osRelease, tmpdir, userInfo } from "node:os";
|
|
56
|
+
import { BlockList, isIPv4, isIPv6 } from "node:net";
|
|
57
|
+
import * as nodeFs from "node:fs";
|
|
55
58
|
import { NETNS_KEEPER, NETNS_KEEPER_FORMAT, NETNS_KEEPER_START, STACK_KEYS, STARTED_AT_FORMAT, VALKEY_SHARED_KEY, judgeNetnsKeeper, judgeValkeyListeners, netnsKeeperRemedy, pinnedValkeyUrl, podmanNeedsNetnsKeeper, probeTcpAddress, proxyConfCopyPath, readSubuidRanges, proxyRestartAdvice, readLinger, readValkeyKeys, valkeySharedOn, valkeyTarget } from "./podman-stack.mjs";
|
|
56
|
-
import { basename, dirname, isAbsolute, join,
|
|
59
|
+
import { basename, dirname, isAbsolute, join, posix, resolve, win32 } from "node:path";
|
|
57
60
|
import { fileURLToPath } from "node:url";
|
|
58
61
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
59
62
|
import { randomBytes } from "node:crypto";
|
|
60
|
-
import { DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_VALKEY_URL, accountTempRoot, allowedModelsFrom, defaultLogsDir, defaultSandboxDir, defaultSettingsFile, defaultWorkerName, globalExtensionsEnabled, jobsDirOwnerFix, jobsDirPath, sandboxDirOwnerFix, legacyTempStateDir, logsDirPath, modelEndpointsFilePath, delimitedList, envelopeFilePath, pauseWindowsFilePath, projectsFilePath, safeHomeDir, scopedLimitsFilePath, settingsFilePath, underOsTempDir } from "./config.mjs";
|
|
63
|
+
import { DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_VALKEY_URL, accountTempRoot, checkpointStoreCollision, checkpointsDirPath, allowedModelsFrom, modelFrom, providerFrom, defaultLogsDir, defaultSandboxDir, defaultSettingsFile, defaultWorkerName, globalExtensionsEnabled, jobsDirOwnerFix, jobsDirPath, sandboxDirOwnerFix, legacyTempStateDir, logsDirPath, modelEndpointsFilePath, delimitedList, envelopeFilePath, pauseWindowsFilePath, projectsFilePath, safeHomeDir, scopedLimitsFilePath, settingsFilePath, underOsTempDir, workerName as workerNameFrom } from "./config.mjs";
|
|
61
64
|
import { SYSTEMD_HAZARD_SHAPES, decodeEnvFile, envFileHazard, envValueShown, quotedRegions, readEnvAssignments, renderEnvValue, envFileWrapperInternal, wrapperInternalSentence } from "./env-file.mjs";
|
|
62
65
|
import { canonicalScope, danglingProjectRows, dollarRowsBelowJobCap, dollarRowsWithoutCap, isModelScope, isProjectScope, loadScopedLimits, parseScopedLimits, scopedLimitsVersionFor } from "./scoped-limits.mjs";
|
|
63
|
-
import { EMPTY_PROJECTS_FINGERPRINT, loadProjects, projectsFingerprint } from "./projects.mjs";
|
|
66
|
+
import { EMPTY_PROJECTS_FINGERPRINT, escapeControls, loadProjects, projectOf, projectsFingerprint } from "./projects.mjs";
|
|
64
67
|
import { parseModelsJson, stripBom, stripJsonComments } from "./models-json.mjs";
|
|
65
|
-
import { isTransientOverlayRead, overlayProviderProblem } from "./model-catalog.mjs";
|
|
68
|
+
import { FIRST_CALL_MIN_CONTEXT_BYTES, FIRST_CALL_OVERHEAD_TOKENS, boundModelOf, firstCallFloorMicros, isTransientOverlayRead, overlayProviderProblem, serviceTierMultiplier } from "./model-catalog.mjs";
|
|
66
69
|
import { EMPTY_USD_FINGERPRINT, usdFingerprint } from "./dollar-fingerprint.mjs";
|
|
67
70
|
import { splitModelEntry } from "./model-ref.mjs";
|
|
71
|
+
import { resolveConfiguredModelRuns as resolveDoctorModelRuns, cooldownModelRefs, RADIUS_COST_CAP_LABEL, RADIUS_COST_CAP_FIX } from "./model-observation.mjs";
|
|
68
72
|
import { ignoredOutputCapModels, outputCapView, outputUnboundable } from "./output-cap.mjs";
|
|
69
73
|
import { KEYLESS_API_KEY, KEYLESS_HOW, MODEL_ENDPOINTS_FILE_NAME, MODEL_ENDPOINTS_INCLUDE_NAME, MODEL_ENDPOINT_ID_RE, OVERLAY_LINK_FIX, OVERLAY_NOT_A_FILE_FIX, RENAMED_PROVIDERS, baseUrlTarget, endpointsAt, keylessVerdict, loadModelEndpoints, providerRenameHint, readOverlayModels, renderEndpointsInclude, unreportedUsageModels } from "./model-endpoints.mjs";
|
|
70
74
|
import { declaredEndpointsIn, endpointsDeclaredIn, reloadCommand, rulesFileIncludes, rulesPredateEndpointsLine } from "./egress-cli.mjs";
|
|
71
|
-
import { loadPauseWindows, parseScopeString } from "./pause-windows.mjs";
|
|
75
|
+
import { loadPauseWindows, parsePauseWindows, parseScopeString, pauseWindowsVersionFor } from "./pause-windows.mjs";
|
|
72
76
|
import { WAIT_AFTER_MAX_DEFAULT_MS, afterInstantMs, parseWaitProfiles } from "./wait-for.mjs";
|
|
73
77
|
import { isForgeKind } from "./forges.mjs";
|
|
74
78
|
import { findLiteralSecret, ADMIN_RE } from "./import-pi.mjs";
|
|
75
79
|
import { mcpServerWhere, readOverlayMcp } from "./mcp-config.mjs";
|
|
76
80
|
import { allowlistEntryFor, egressRefusalFor, packageRunnerHosts, parseAllowlist } from "./egress-allowlist.mjs";
|
|
77
81
|
import { agentDirFrom, readHostPi } from "./host-pi.mjs";
|
|
78
|
-
import { PACKAGES_SUBDIR, readStagedSkills, readStageManifest } from "./packages.mjs";
|
|
82
|
+
import { PACKAGES_SUBDIR, readStagedSkills, readStageManifest, STAGE_MANIFEST } from "./packages.mjs";
|
|
79
83
|
import { copySkillTree } from "./copy-tree.mjs";
|
|
80
84
|
import { SKILL_NAME_RE } from "./flow-gate.mjs";
|
|
81
85
|
import { GIT_READ_FLAGS } from "./git-hardening.mjs";
|
|
@@ -84,21 +88,21 @@ import { deploymentVenueEnv, sharedShellIgnored } from "./deployment-venue.mjs";
|
|
|
84
88
|
import { readDeploymentEnv, resolveServiceEnv, serviceEnvFileOf, serviceEnvLoader } from "./service-env.mjs";
|
|
85
89
|
import { imageRefProblem, jobImageFix, pullOffered } from "./image-ref.mjs";
|
|
86
90
|
import { PROXY_STATE_FORMAT, parseProxyState, rulesIncludeEndpoints, shippedProxyDrift } from "./egress-proxy-state.mjs";
|
|
87
|
-
import { PACKAGED_EGRESS_PROXY_CONF, judgeProxyConfCopy, packageCopyName, readPackagedProxyConf } from "./egress-conf-copy.mjs";
|
|
88
|
-
import { ABSENT, ASSERTED, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OBSERVATION_FIX, OBSERVATIONS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_ROOTFUL_WIDENING_KEYS, PODMAN_SERVICE_LOCAL, PROPERTY_NAMES, RUNTIME_ADDS_NO_MOUNTS, HOST_ROUTE_LAN, HOST_ROUTE_REFUTED, HOST_ROUTE_WORKS, declarationOf, floorShortfall, hostRouteFor, isProxyLocalHost, parseBackendFloor, parseBackendList, unarmedFloor, unobservedFloor, venuesOf } from "./backends.mjs";
|
|
89
|
-
import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_FIRST_START_TIMEOUT_MS, PODMAN_INFO_TIMEOUT_MS, PODMAN_JOB_USER_FIX, decidePodmanJobUser, makePodmanInfoReader, observePodman, observeRootlessNetns, podmanConfFix, podmanConfWidening, resolvePodmanImageUser } from "./backend-podman.mjs";
|
|
90
|
-
import { PODMAN_PINNED_FLAGS, buildPodmanRunArgs, containerSpec, podmanArgsFromSpec } from "./docker-run.mjs";
|
|
91
|
+
import { PACKAGED_EGRESS_PROXY_CONF, PACKAGE_VERSION, judgeProxyConfCopy, packageCopyName, readPackagedProxyConf } from "./egress-conf-copy.mjs";
|
|
92
|
+
import { ABSENT, ASSERTED, BACKENDS, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OBSERVATION_FIX, OBSERVATIONS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_ROOTFUL_WIDENING_KEYS, PODMAN_SERVICE_LOCAL, PROPERTY_NAMES, RUNTIME_ADDS_NO_MOUNTS, HOST_ROUTE_LAN, HOST_ROUTE_REFUTED, HOST_ROUTE_WORKS, backendRefusals, declarationOf, floorShortfall, hostRouteFor, isProxyLocalHost, openshellSetupOf, parseBackendFloor, parseBackendList, unarmedFloor, unobservedFloor, venuesOf } from "./backends.mjs";
|
|
93
|
+
import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_FIRST_START_TIMEOUT_MS, PODMAN_INFO_TIMEOUT_MS, PODMAN_JOB_USER_FIX, decidePodmanJobUser, makePodmanInfoReader, podmanPausable, observePodman, observeRootlessNetns, podmanConfFix, podmanConfWidening, resolvePodmanImageUser } from "./backend-podman.mjs";
|
|
94
|
+
import { PODMAN_PINNED_FLAGS, buildDockerRunArgs, buildPodmanRunArgs, containerSpec, podmanArgsFromSpec } from "./docker-run.mjs";
|
|
91
95
|
import { PODMAN_SERVICE_TIMEOUT_MS, PODMAN_SERVICE_UNIT, makePodmanServiceReader, observeHost, observeRootfulConf, readRootfulService, rootfulConfFix, rootfulConfRetries, rootfulConfResidual, rootfulUnreadList } from "./runtime-observations.mjs";
|
|
92
96
|
import { JOB_NAME_PREFIX, endpointShown, makeDockerEndpointResolver, quotedShown } from "./backend-local.mjs";
|
|
93
97
|
import { DEFAULT_EGRESS_PROXY, STOPPED_PROXY_STATES, EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, EGRESS_ENDPOINT_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEndpointProbe, egressEnv, egressProxyName, egressProxyUrl, networkEndpoints, removeNetworkOrSay } from "./egress.mjs";
|
|
94
98
|
import { detachBlockedSentence, makeDetachGate, runtimeFromFacts } from "./netns-keeper.mjs";
|
|
95
99
|
import { runLiveProbes } from "./live-probes.mjs";
|
|
96
|
-
import { VALKEY_PASSWORD_KEY, VALKEY_PASSWORD_HOWTO, VALKEY_PORT_KEY, isLoopbackHost, valkeyPasswordProblem, valkeyPortConflict } from "./valkey-auth.mjs";
|
|
100
|
+
import { COMPOSE_VALKEY_OVERRIDE, VALKEY_PASSWORD_KEY, VALKEY_PASSWORD_HOWTO, VALKEY_PORT_KEY, isLoopbackHost, valkeyContainerOwner, valkeyPasswordProblem, valkeyPortConflict } from "./valkey-auth.mjs";
|
|
97
101
|
import { urlShown, valkeyContextFromResolution, valkeyPasswordFor, valkeyUrlProblem } from "./valkey-endpoint.mjs";
|
|
98
102
|
import { SANDBOX_TOMBSTONE_STUCK_MS, isSandboxTombstone, sandboxTombstoneAge } from "./sandbox-store.mjs";
|
|
99
|
-
import { installedUnitPaths, readUnitSeam, readUnitUser } from "./service.mjs";
|
|
103
|
+
import { installedConsoleUnitPath, installedUnitPaths, readUnitSeam, readUnitUser } from "./service.mjs";
|
|
100
104
|
import { CONTAINER_HOME, SHIPPED_IMAGE_UID, SIZE_LABEL_CPU, SIZE_LABEL_MEM } from "./container-spec.mjs";
|
|
101
|
-
import { DEFAULT_JOB_SIZE, cpuCeilingCenti, formatCpus, formatMemory, jobSizeDefaults, resolveJobSize } from "./job-size.mjs";
|
|
105
|
+
import { DEFAULT_JOB_SIZE, codemodeWorkersOf, cpuCeilingCenti, formatCpus, formatMemory, jobSizeDefaults, resolveJobSize } from "./job-size.mjs";
|
|
102
106
|
import { SUGGEST_WINDOW_DAYS, cpusText, hostCap, refusalWords, sizeRefusal, suggestSize, suggestionCall, suggestionEvidence } from "./size-suggest.mjs";
|
|
103
107
|
import { SIZING_RECORD_MAX_BYTES, readSizingRecords } from "./size-records.mjs";
|
|
104
108
|
import { CAPACITY_WINDOWS, computeCapacity } from "./capacity.mjs";
|
|
@@ -107,19 +111,38 @@ import { CGROUP_PARENT, cgroupParentFor, operatorQuotaCommand, readQuota, reserv
|
|
|
107
111
|
import { HOST_BUDGET_KEYS, computeHostBudget, hostBudgetSettings, largestFit, neverFits, projectBudgetRow, publishedBudget, readUserServiceLimits } from "./host-budget.mjs";
|
|
108
112
|
import { makeImagePreflight, normalizeImageId } from "./image-preflight.mjs";
|
|
109
113
|
import { BOOT_REFUSING_JOB_USER_CAUSES, DAEMON_FACTS_TIMEOUT_MS, JOB_USER_FIX, makeDaemonFactsReader, makeJobUserResolver, relabelsPrivateMounts, resolveImageUser } from "./job-user.mjs";
|
|
110
|
-
import { parseSecretProfiles } from "./secret-profiles.mjs";
|
|
114
|
+
import { mergeSecretProfiles, parseSecretProfiles, servableSecretProfiles } from "./secret-profiles.mjs";
|
|
111
115
|
// The OAuth-suffix rule and the variable it selects live in their own import-free module so the worker
|
|
112
116
|
// can share them: doctor NAMES a variable and env-allowlist WRITES one, and they must never differ.
|
|
113
117
|
import { apiKeyVariable, nonApiKeyKind } from "./provider-key.mjs";
|
|
114
|
-
import {
|
|
115
|
-
import {
|
|
116
|
-
import {
|
|
118
|
+
import { WORKER_NAME_RE } from "./worker-name.mjs";
|
|
119
|
+
import { parseTriggersDocument, triggersFileVersionFor } from "./triggers.mjs";
|
|
120
|
+
import { jobDataVersionFor } from "./job-data-version.mjs";
|
|
121
|
+
import { secretBundleChecks } from "./doctor-bundles.mjs";
|
|
122
|
+
import { doctorModelProposalChecks } from "./doctor-model-proposals.mjs";
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* How long `doctor --secrets` waits for one bundle resolver (issue #704): doctor's own 30 s capture bound, not the
|
|
126
|
+
* job path's tighter PI_SECRET_RESOLVE_TIMEOUT_MS. A job's read sits in front of a paid container and holds a slot;
|
|
127
|
+
* doctor's is an operator at a prompt asking whether the wiring works, and a slow manager is an answer, not a fault.
|
|
128
|
+
*/
|
|
129
|
+
const BUNDLE_DOCTOR_TIMEOUT_MS = 30_000;
|
|
130
|
+
import { readOverlay, readOverlayDocument, resolveSettings, settingsVersionFor } from "./runtime-settings.mjs";
|
|
131
|
+
import { DEPLOYMENT_WINDOWS, DOLLAR_KEY_PREFIX, deploymentWindowsAt, dollarWindowCaps, dollarWindowFloors, valkeyEndpointTag } from "./dollar-budget.mjs";
|
|
117
132
|
import { dayKey, monthKey, weekKey } from "./budget.mjs";
|
|
118
133
|
import { DOLLAR_ENV_NAMES, DOLLAR_SETTING_KEYS, checkDollarInvariant, effectiveCostCapMicros, formatMicros, optionalUsdMicros, parseUsdMicros } from "./money.mjs";
|
|
119
134
|
import { cronPlacement, envelopeJobPaths } from "./schedules.mjs";
|
|
120
|
-
import { envelopeDigest, loadEnvelopeChecked } from "./envelope.mjs";
|
|
135
|
+
import { envelopeDigest, envelopeInsideJobPaths, loadEnvelopeChecked } from "./envelope.mjs";
|
|
136
|
+
import { keptTokens, operatorCount, readTokensFile, tokensFilePath } from "./console-tokens.mjs";
|
|
137
|
+
import { RUN_MIRROR_MAX_RUNS_DEFAULT, RUN_MIRROR_MAX_RUNS_MAX, RUN_MIRROR_MAX_RUNS_MIN, runMirrorMaxRunsOf } from "./retention.mjs";
|
|
138
|
+
import { verifyConsoleAudit } from "./console-audit.mjs";
|
|
121
139
|
import { OTHER } from "./priorities.mjs";
|
|
122
140
|
import { ALLOC_PLAN_KEY, NO_ENVELOPE_FINGERPRINT } from "./allocation.mjs";
|
|
141
|
+
import { assignCheckIds, buildDoctorRecord, doctorIntervalHours, publishRedactor, secretValuesIn } from "./doctor-results.mjs";
|
|
142
|
+
import { olderStateFindings as olderStateFindingsReal, VALKEY_CONTAINER } from "./older-state.mjs";
|
|
143
|
+
import { DOCTOR_PUBLISH_SKIPPED_PREFIX } from "./doctor-spawner.mjs";
|
|
144
|
+
import { extensionsNotHere } from "./loaded-record.mjs";
|
|
145
|
+
import { buildHostLoads, flowProbes, flowTierOf, mcpRowsOf, pathIsThere, readOverlayExtensionRows, readOverlaySkillNames } from "./host-loads.mjs";
|
|
123
146
|
|
|
124
147
|
const NODE_FLOOR = [22, 19]; // pi's engine floor (22.19.0)
|
|
125
148
|
|
|
@@ -181,7 +204,8 @@ const ghProbeArgs = (image, bin) => ["run", "--rm", "-i", "--pull=never", ...(bi
|
|
|
181
204
|
export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
182
205
|
const {
|
|
183
206
|
cwd = process.cwd(),
|
|
184
|
-
out = (s) => process.stdout.write(s),
|
|
207
|
+
out: output = (s) => process.stdout.write(s),
|
|
208
|
+
json = false,
|
|
185
209
|
spawn = nodeSpawn,
|
|
186
210
|
// The bounds every `runCmd` goes through (issue #397). A seam so a test can drive the timeout path
|
|
187
211
|
// in milliseconds; nothing else passes it.
|
|
@@ -217,20 +241,55 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
217
241
|
// without uninstalling a dependency. Threaded like every other seam: a seam collectChecks honours
|
|
218
242
|
// and runDoctor silently drops is a seam that cannot pin an EXIT CODE, only a check object.
|
|
219
243
|
providerOracle = defaultProviderOracle,
|
|
244
|
+
subscriptionFactory,
|
|
220
245
|
// Issues #501 and #502: the worker's model catalog and pi's own model loader, both imported lazily. Seams, so a test
|
|
221
246
|
// can drive the not-installed arm and a disagreement without editing pi. Undefined means the defaults.
|
|
222
247
|
modelCatalog,
|
|
223
248
|
piModelLoader,
|
|
249
|
+
modelProposals,
|
|
224
250
|
dollarKeysExist,
|
|
225
251
|
// Issue #504 part B: the applied split's envelope digest (`alloc:plan`), read once. Undefined means the default.
|
|
226
252
|
readAppliedSplit,
|
|
227
253
|
// Issue #599, phase 2: the capacity line's read of the run history (the run mirror and the logs directory).
|
|
228
254
|
// Undefined means the default, which reads the deployment's real Valkey and files.
|
|
229
255
|
readCapacity,
|
|
256
|
+
// Issue #761: the dollar window check's two reads, this host's records and then the counters (`doctorDollarWindows`),
|
|
257
|
+
// and the stores an older release left behind (`olderStateFindings`, older-state.mjs). Undefined means the defaults,
|
|
258
|
+
// which read the deployment's real files, Valkey and Docker.
|
|
259
|
+
readDollarRecords,
|
|
260
|
+
readDollarCounters,
|
|
261
|
+
olderStateFindings,
|
|
262
|
+
// Issue #708: whether the operator console answers (`probeConsole`, console-probe.mjs). Undefined means the default,
|
|
263
|
+
// which sends one GET /healthz to a loopback address or the socket, so a test passes its own.
|
|
264
|
+
consoleProbe,
|
|
230
265
|
// --live (issue #278, INT-LIVE-PROBE-CONTRACT): read the backend declarations back off short-lived real containers.
|
|
231
266
|
// STRICTLY `=== true`, so only the CLI's own flag arms it: a truthy string from a caller that forwarded an
|
|
232
267
|
// option bag runs nothing. The fs, PID-liveness and nonce are seams so the sequence is driven without Docker.
|
|
233
268
|
live = false,
|
|
269
|
+
// The granted browser action publishes observations without starting/sweeping probe containers (#774).
|
|
270
|
+
setupReadOnly = false,
|
|
271
|
+
// Issue #683: `--openshell`, the openshell venue's lines (doctor-openshell.mjs), strictly `=== true` as `live`.
|
|
272
|
+
// `openshellRun` is their spawn seam, for the tests.
|
|
273
|
+
openshell = false,
|
|
274
|
+
// Issue #704: `--secrets`, resolve each project secret bundle on this host and print its variable names and count,
|
|
275
|
+
// never a value (doctor-bundles.mjs). Strictly `=== true`, as `live`: it spawns every bundle resolver, which reads
|
|
276
|
+
// the operator's manager. `bundleSpawn` is its spawn seam, for the tests.
|
|
277
|
+
secrets = false,
|
|
278
|
+
bundleSpawn,
|
|
279
|
+
// Issue #712: publish this run's results to Valkey for the console and the panel (`doctor-results.mjs`). Strictly
|
|
280
|
+
// `=== true`, as `live`: the CLI passes it unless `--no-publish`, and a caller that hands runDoctor an option bag
|
|
281
|
+
// (every test) publishes nothing unless it says so. `publishDoctor(url, name, record, loads)` is the write, a seam so a
|
|
282
|
+
// test drives it against a fake Valkey; `writeErr` takes the one line said when it could not publish; `publishNow`
|
|
283
|
+
// stamps the record.
|
|
284
|
+
publish = false,
|
|
285
|
+
// `{ worker: true, every }` when the worker's own spawn ran this (`--by-worker=<hours>`): the record then carries the
|
|
286
|
+
// interval the worker actually runs at, not this resolution's reading of it.
|
|
287
|
+
publishBy = null,
|
|
288
|
+
publishDoctor = defaultPublishDoctor,
|
|
289
|
+
hostLoadsOf = doctorHostLoads,
|
|
290
|
+
writeErr = (s) => process.stderr.write(s),
|
|
291
|
+
publishNow = Date.now,
|
|
292
|
+
openshellRun,
|
|
234
293
|
liveFs = { chmodSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, statSync },
|
|
235
294
|
// Issue #341: who a job on this host would run as, decided from the same facts the worker reads. Seams because
|
|
236
295
|
// the answer is this process's own ids and a daemon, neither of which belongs in a unit test. `stat` reads the
|
|
@@ -247,6 +306,8 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
247
306
|
readEnvFile = (path) => readFileSync(path),
|
|
248
307
|
// Issue #345: the host files the runtime-mounts observation reads (Podman's mounts.conf and containers.conf).
|
|
249
308
|
observationFs = { statSync, readFileSync, readdirSync },
|
|
309
|
+
// Issue #800: what the flow tiers read, so a test can stand in for a folder at mode 000, which root reads straight through.
|
|
310
|
+
flowFs = { statSync, readFileSync, readdirSync },
|
|
250
311
|
// Issue #464: the jobs dir check's reads (its owner, whether this account may write it, the old shared default's
|
|
251
312
|
// retained workspaces). Never a write: doctor says what the worker will meet, and the worker creates the dir.
|
|
252
313
|
jobsDirFs = { accessSync, lstatSync, readdirSync, statSync },
|
|
@@ -315,6 +376,8 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
315
376
|
// against), and the text handed to the first `.env` read the checks below make, so a run still reads the file once.
|
|
316
377
|
// A file that is there and cannot be read is said, as `up` says it (gate round 2): the venue then comes from this
|
|
317
378
|
// shell alone. An absent one (ENOENT) is simply absent.
|
|
379
|
+
// JSON owns stdout; the same diagnostics and fixes remain visible on stderr.
|
|
380
|
+
const out = json === true ? writeErr : output;
|
|
318
381
|
const envPath = join(cwd, ".env");
|
|
319
382
|
// BYTES (issue #447): what systemd refuses to load is judged before decoding. Through ONE descriptor since gate round
|
|
320
383
|
// 2 (`readDeploymentEnv`, the panel's reader too): judged by `stat` and read by path, a folder another account could
|
|
@@ -390,7 +453,7 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
390
453
|
return { ...(await valkeyAuthState(url, { context, withoutPassword })), passwordSet: Boolean(sent.password), from: sent.from };
|
|
391
454
|
}
|
|
392
455
|
: null;
|
|
393
|
-
const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), ...(readCapacity ? { readCapacity } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, ...(readRunRecords ? { readRunRecords } : {}), venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
|
|
456
|
+
const seams = { setupReadOnly: setupReadOnly === true, ...(subscriptionFactory ? { subscriptionFactory } : {}), ...(modelProposals ? { modelProposals } : {}), secrets: secrets === true, ...(bundleSpawn ? { bundleSpawn } : {}), openshell: openshell === true, ...(openshellRun ? { openshellRun } : {}), cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), ...(readCapacity ? { readCapacity } : {}), ...(readDollarRecords ? { readDollarRecords } : {}), ...(readDollarCounters ? { readDollarCounters } : {}), ...(olderStateFindings ? { olderStateFindings } : {}), ...(consoleProbe ? { consoleProbe } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, flowFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, ...(readRunRecords ? { readRunRecords } : {}), venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
|
|
394
457
|
// Issue #471: every other service key, resolved ONCE for the whole run (the fix pass's re-collect and `--live` judge the
|
|
395
458
|
// same resolution). THE RULE (PR #474's round cap, after three rounds of trust patches): no program doctor starts is
|
|
396
459
|
// handed anything from `.env`. Every child gets this shell's own environment, the one it had before #471; a `.env`
|
|
@@ -425,6 +488,8 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
425
488
|
}
|
|
426
489
|
}
|
|
427
490
|
|
|
491
|
+
// The --live read-backs, kept for the published record (issue #712).
|
|
492
|
+
const liveRendered = [];
|
|
428
493
|
// ONCE, and after --fix: the probes read the host as the fix pass left it, so a fix that pulled the job image is
|
|
429
494
|
// read back rather than reported absent. Rendered like every other check and judged by the same rule.
|
|
430
495
|
if (live === true) {
|
|
@@ -433,8 +498,26 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
433
498
|
// Each venue's results are rendered BEFORE the next venue announces its containers, so what podman is about to
|
|
434
499
|
// start never lands between local's announcement and local's verdicts.
|
|
435
500
|
const liveSeams = { ...seams, liveFs, isAlive, pid, nonce, ...(now ? { now } : {}), ...(delay ? { delay } : {}) };
|
|
436
|
-
if (facts.localUsed !== false
|
|
437
|
-
|
|
501
|
+
if (facts.localUsed !== false) {
|
|
502
|
+
const read = await liveChecks(env, liveSeams, facts);
|
|
503
|
+
liveRendered.push(...read);
|
|
504
|
+
if (render(read, out)) failed = true;
|
|
505
|
+
}
|
|
506
|
+
if (facts.podman) {
|
|
507
|
+
const read = await podmanLiveChecks(env, liveSeams, facts);
|
|
508
|
+
liveRendered.push(...read);
|
|
509
|
+
if (render(read, out)) failed = true;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Issue #712: what was judged, published for the console and the panel, after every check is in (the `--fix`
|
|
514
|
+
// re-collect and the `--live` read-backs included) and before the closing line. Best effort BY CONTRACT: whatever
|
|
515
|
+
// happens here leaves `failed`, and so the exit code, as it was, and a fault is one line on stderr, never on stdout,
|
|
516
|
+
// where a script reads doctor's verdict.
|
|
517
|
+
const record = doctorResultsRecord({ checks: [...checks, ...liveRendered], liveRendered, seams, env, at: publishNow(), by: publishBy });
|
|
518
|
+
if (json === true) output(`${JSON.stringify(record)}\n`);
|
|
519
|
+
if (publish === true) {
|
|
520
|
+
await publishResults({ seams, publishDoctor, hostLoadsOf, writeErr, record, env });
|
|
438
521
|
}
|
|
439
522
|
|
|
440
523
|
// Issue #477: whether anything doctor judged came from the deployment's `.env`, which a hand-started worker never reads.
|
|
@@ -449,6 +532,117 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
|
|
|
449
532
|
return failed ? 1 : 0;
|
|
450
533
|
}
|
|
451
534
|
|
|
535
|
+
/**
|
|
536
|
+
* Issue #712: publish one run's results (`INT-DOCTOR-RESULTS-KEY`). Never throws and never decides anything: a Valkey
|
|
537
|
+
* this run may not talk to, or one that does not answer, is ONE line on stderr whose wording starts with
|
|
538
|
+
* `DOCTOR_PUBLISH_SKIPPED_PREFIX` (the worker's spawner reads that prefix and nothing else), and no URL, path or value
|
|
539
|
+
* is in it. A record that landed without its `doctor:loads` beside it (issue #800) is one line in other words, for the same
|
|
540
|
+
* reader: it was published, and saying it was not would be false.
|
|
541
|
+
* The secrets the record must not carry are every secret-named value of the resolution (`env`, which holds
|
|
542
|
+
* this shell's whole environment with the service's `.env` values over it), plus the `.env` values the resolution did
|
|
543
|
+
* not take (a provider key resolved later, a value this shell disagrees with), so a value the file holds is as absent
|
|
544
|
+
* as one this shell exports.
|
|
545
|
+
*/
|
|
546
|
+
export function doctorResultsRecord({ checks, liveRendered = [], seams, env, at, by = null }) {
|
|
547
|
+
const live = new Set(liveRendered);
|
|
548
|
+
const ids = assignCheckIds(checks, (c) => (live.has(c) ? "live" : (seams.checkSection?.(c) ?? "env")));
|
|
549
|
+
const service = seams.serviceEnv ?? {};
|
|
550
|
+
const fileSide = Object.fromEntries((service.disagreements ?? []).map((d) => [d.key, d.file]));
|
|
551
|
+
const secrets = secretValuesIn(env, service.fromFile, service.extraFromFile, fileSide);
|
|
552
|
+
const worker = by?.worker === true;
|
|
553
|
+
const every = worker && Number.isSafeInteger(by.every) ? by.every : seams.publishTo?.every;
|
|
554
|
+
return buildDoctorRecord({ checks, ids, at, version: PACKAGE_VERSION, openshell: seams.openshellJudged === true, every, secrets, by: { uid: process.geteuid?.() ?? null, worker } });
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Issue #800: what this host's jobs can load, for `doctor:loads:<name>` (`host-loads.mjs`), read when it is published and
|
|
559
|
+
* through the readers a job's worker uses, so a run by `--no-publish` reads nothing more. A part that cannot be read is
|
|
560
|
+
* named in `unread`, which a reader takes as unknown, and an empty list would read as none. Only ABSENT_CODES are absence
|
|
561
|
+
* (`pathIsThere`): an overlay this uid cannot enter is not an overlay with nothing in it. A throw that gets past that is
|
|
562
|
+
* the caller's to say: the doctor record goes out either way.
|
|
563
|
+
*/
|
|
564
|
+
export function doctorHostLoads({ seams, env, record, fs = { readFileSync, readdirSync, statSync } }) {
|
|
565
|
+
const overlay = typeof env.PI_GLOBAL_PI_DIR === "string" ? env.PI_GLOBAL_PI_DIR : "";
|
|
566
|
+
const unread = [];
|
|
567
|
+
const read = (part, fn, fallback) => {
|
|
568
|
+
try {
|
|
569
|
+
const value = fn();
|
|
570
|
+
if (value !== null) return value;
|
|
571
|
+
} catch {
|
|
572
|
+
// named below, by part alone: what failed is the doctor line's to say
|
|
573
|
+
}
|
|
574
|
+
unread.push(part);
|
|
575
|
+
return fallback;
|
|
576
|
+
};
|
|
577
|
+
const service = seams.serviceEnv ?? {};
|
|
578
|
+
const fileSide = Object.fromEntries((service.disagreements ?? []).map((d) => [d.key, d.file]));
|
|
579
|
+
const redact = publishRedactor(secretValuesIn(env, service.fromFile, service.extraFromFile, fileSide));
|
|
580
|
+
const extensions = read("extensions", () => readOverlayExtensionRows(overlay, { fs }), []);
|
|
581
|
+
const skills = read("skills", () => readOverlaySkillNames(overlay, { fs }), []);
|
|
582
|
+
const mcp = read("mcp", () => mcpRowsOf(readOverlayMcp(overlay, { readFileSync: seams.readOverlayFile ?? ((p, enc) => readFileSync(p, enc)), ...(seams.lstatOverlayFile ? { lstatSync: seams.lstatOverlayFile } : {}) })?.servers, { redact }), []);
|
|
583
|
+
// A manifest that is there and does not read is not "nothing staged": the job's own reader degrades, this one says so.
|
|
584
|
+
const packages = read("packages", () => {
|
|
585
|
+
if (overlay === "" || !pathIsThere(join(overlay, PACKAGES_SUBDIR, STAGE_MANIFEST), fs)) return [];
|
|
586
|
+
return readStageManifest({ globalPiDir: overlay, readFile: (p) => fs.readFileSync(p, "utf8"), fileExists: () => true })?.packages ?? null;
|
|
587
|
+
}, []);
|
|
588
|
+
// `none` says every tier was checked, and a tier in `unread` was not. Doctor's line stays as it is, since "visible here" is
|
|
589
|
+
// still true of it. What is published is what can be said: not that the flow is nowhere, only that it was not found.
|
|
590
|
+
const blind = unread.includes("skills") || unread.includes("packages");
|
|
591
|
+
const flows = (seams.flowRows ?? []).map((row) => (blind && row.tier === "none" && row.flow !== null ? { ...row, tier: "unknown" } : row));
|
|
592
|
+
return buildHostLoads({ at: record.at, every: record.every, by: record.by, overlayExtensionsAllowed: env.PI_GLOBAL_ALLOW_EXTENSIONS !== "0", extensions, skills, mcp, packages, flows, unread });
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
async function publishResults({ seams, publishDoctor, hostLoadsOf, writeErr, record, env }) {
|
|
596
|
+
const line = (text) => {
|
|
597
|
+
try {
|
|
598
|
+
writeErr(`${text}\n`);
|
|
599
|
+
} catch {
|
|
600
|
+
// A closed stderr cannot change the verdict either.
|
|
601
|
+
}
|
|
602
|
+
};
|
|
603
|
+
const say = (why) => line(`${DOCTOR_PUBLISH_SKIPPED_PREFIX} (${why}); the console and the panel keep the last published results`);
|
|
604
|
+
// Not that prefix, on purpose: the doctor record DID land, and the worker's spawner reads the prefix as "it did not". And not
|
|
605
|
+
// "readers say unknown", which is false: the last list stays for its seven days and reads as current until it is old.
|
|
606
|
+
const sayLoads = (why) => line(`doctor: results published, but not what this host's jobs can load (${why}); the console keeps the last list it had`);
|
|
607
|
+
const target = seams.publishTo;
|
|
608
|
+
if (!target) return say("doctor talked to no Valkey on this run");
|
|
609
|
+
if (target.refused) return say(target.refused);
|
|
610
|
+
let loads = null;
|
|
611
|
+
let loadsThrew = false;
|
|
612
|
+
try {
|
|
613
|
+
loads = hostLoadsOf({ seams, env, record });
|
|
614
|
+
} catch {
|
|
615
|
+
// Each part's own read fails into `unread`, so this is the build of it failing. The doctor record is the older promise
|
|
616
|
+
// and goes out on its own.
|
|
617
|
+
loadsThrew = true;
|
|
618
|
+
}
|
|
619
|
+
try {
|
|
620
|
+
await publishDoctor(target.url, target.name, record, loads);
|
|
621
|
+
} catch (error) {
|
|
622
|
+
return error?.doctorRecordLanded === true ? sayLoads("Valkey did not take them") : say("Valkey did not take them");
|
|
623
|
+
}
|
|
624
|
+
if (loadsThrew) sayLoads("this run could not read them");
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* The real write: doctor's own fail-fast client, bounded, always dropped (`withDoctorClient`). A fault on the second write says
|
|
629
|
+
* the first landed (`doctorRecordLanded`), since "nothing was published" would then be false of a record readers can see.
|
|
630
|
+
*/
|
|
631
|
+
export async function defaultPublishDoctor(url, name, record, loads = null, { makeClient } = {}) {
|
|
632
|
+
const { publishDoctorRecord } = await import("./doctor-results.mjs");
|
|
633
|
+
const { publishHostLoads } = await import("./host-loads.mjs");
|
|
634
|
+
await withDoctorClient(url, async (client) => {
|
|
635
|
+
await publishDoctorRecord(client, name, record);
|
|
636
|
+
if (loads === null) return;
|
|
637
|
+
// After the doctor record, whose write puts this host under the index both share.
|
|
638
|
+
try {
|
|
639
|
+
await publishHostLoads(client, name, loads);
|
|
640
|
+
} catch (cause) {
|
|
641
|
+
throw Object.assign(new Error("the doctor record landed and the loads did not", { cause }), { doctorRecordLanded: true });
|
|
642
|
+
}
|
|
643
|
+
}, { makeClient });
|
|
644
|
+
}
|
|
645
|
+
|
|
452
646
|
/**
|
|
453
647
|
* Issue #477: how to start a worker that runs with what doctor judged. `pi-dispatch worker` started by hand reads this
|
|
454
648
|
* shell's environment and no `.env` (docs/secrets.md: the worker parses no `.env` file), so where doctor took any value
|
|
@@ -601,10 +795,12 @@ export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "P
|
|
|
601
795
|
* the key's path, neither a secret); the inline key is only ever sniffed for its first bytes. */
|
|
602
796
|
export const GITHUB_SERVICE_KEYS = Object.freeze(["GITHUB_AUTH_SOURCE", "GITHUB_APP_ID", "GITHUB_APP_INSTALLATION_ID", "GITHUB_APP_PRIVATE_KEY_PATH", "GITHUB_APP_PRIVATE_KEY"]);
|
|
603
797
|
/** Issue #471: the worker's settings doctor judges, which it read from this shell alone while the service read them from
|
|
604
|
-
* `.env`. TEMP is TMPDIR's twin in the worker's temp root; PI_CODING_AGENT_DIR is where the worker reads auth.json.
|
|
605
|
-
|
|
798
|
+
* `.env`. TEMP is TMPDIR's twin in the worker's temp root; PI_CODING_AGENT_DIR is where the worker reads auth.json.
|
|
799
|
+
* Issue #761: PI_CONSOLE_AUDIT_RETENTION_DAYS is here and not in `CONSOLE_ENV_KEYS` although it is named for the
|
|
800
|
+
* console: the worker reads it (it runs the audit reaper), and a console key set alone makes doctor judge a console. */
|
|
801
|
+
export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_JOB_MEMORY", "PI_JOB_CPUS", "PI_CONCURRENCY", ...Object.values(HOST_BUDGET_KEYS), "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_LOG_RETENTION_DAYS", "PI_CONSOLE_AUDIT_RETENTION_DAYS", "PI_RUN_MIRROR_MAX_RUNS", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_CHECKPOINTS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_AUTH_CONNECTIONS_FILE", "PI_AUTH_STORE_DIR", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_OPENSHELL_ENGINE", "PI_OPENSHELL_GATEWAY", "PI_SECRET_PROFILES", "PI_SECRET_BUNDLES", "PI_SECRET_RESOLVER_ROOTS", "PI_SECRET_RESOLVE_TIMEOUT_MS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "PI_DOCTOR_INTERVAL_HOURS", "PI_ALLOWED_MODELS", "PI_DISPATCH_RUN_ROOTS", "GITHUB_PAT_VAR", "TEMP", ...Object.values(DOLLAR_ENV_NAMES)]);
|
|
606
802
|
/** Issue #471: the receiver's keys doctor judges its boot by (the receiver's unit reads the same `.env`). */
|
|
607
|
-
export const RECEIVER_SERVICE_KEYS = Object.freeze(["WEBHOOK_SECRET", "RECEIVER_PORT", "GITLAB_TOKEN", "GITLAB_URL", "GITLAB_WEBHOOK_MODE", "GITLAB_WEBHOOK_SECRET", "FORGEJO_URL", "FORGEJO_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_ORG_URL", "AZURE_TOKEN", "AZURE_WEBHOOK_MODE", "AZURE_WEBHOOK_SECRET", "AZURE_WEBHOOK_HEADER"]);
|
|
803
|
+
export const RECEIVER_SERVICE_KEYS = Object.freeze(["WEBHOOK_SECRET", "RECEIVER_PORT", "GITLAB_TOKEN", "GITLAB_URL", "GITLAB_WEBHOOK_MODE", "GITLAB_WEBHOOK_SECRET", "FORGEJO_URL", "FORGEJO_TOKEN", "FORGEJO_PERMISSION_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_ORG_URL", "AZURE_TOKEN", "AZURE_WEBHOOK_MODE", "AZURE_WEBHOOK_SECRET", "AZURE_WEBHOOK_HEADER"]);
|
|
608
804
|
/**
|
|
609
805
|
* Issue #471 (follow-up): the variables the CLIs doctor spawns read from their environment, by the CLI that reads them.
|
|
610
806
|
* The worker starts podman, docker and gh with its OWN environment (no `env` option: `image-preflight.mjs`,
|
|
@@ -624,9 +820,17 @@ export const CLI_SERVICE_KEYS = Object.freeze({
|
|
|
624
820
|
podman: Object.freeze(["CONTAINERS_CONF", "CONTAINERS_CONF_OVERRIDE", "CONTAINERS_STORAGE_CONF", "CONTAINERS_REGISTRIES_CONF", "CONTAINER_HOST", "CONTAINER_CONNECTION", "XDG_CONFIG_HOME", "XDG_RUNTIME_DIR", "HOME"]),
|
|
625
821
|
docker: Object.freeze(["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "HOME"]),
|
|
626
822
|
gh: Object.freeze(["GH_TOKEN", "GITHUB_TOKEN", "GH_CONFIG_DIR", "GH_HOST", "XDG_CONFIG_HOME", "HOME"]),
|
|
823
|
+
// Issue #707: the openshell CLI keeps its gateway registrations (and so which gateway is the active one) under this
|
|
824
|
+
// account's config dir, XDG_CONFIG_HOME or else HOME/.config (measured on 0.1.4: `gateway add` writes there), so a
|
|
825
|
+
// `.env` that moves either points the SERVICE's CLI at another registration than doctor's. PATH is not listed: every
|
|
826
|
+
// shell sets one, which would make the line fire on every run, and where the CLI is found is `service install`'s to
|
|
827
|
+
// put in the unit (the plist's PATH carries its directory on macOS).
|
|
828
|
+
openshell: Object.freeze(["XDG_CONFIG_HOME", "HOME"]),
|
|
627
829
|
});
|
|
628
830
|
const CLI_KEY_NAMES = [...new Set(Object.values(CLI_SERVICE_KEYS).flat())];
|
|
629
|
-
|
|
831
|
+
// A Set, since issue #707 made two of the worker's keys (PI_OPENSHELL_ENGINE, PI_OPENSHELL_GATEWAY) venue keys too: each
|
|
832
|
+
// key once, in the order it is first named.
|
|
833
|
+
export const SERVICE_ENV_KEYS = Object.freeze([...new Set([...STACK_KEYS, "VALKEY_URL", VALKEY_SHARED_KEY, VALKEY_PASSWORD_KEY, VALKEY_PORT_KEY, "PI_PROVIDER", "PI_MODEL", ...GITHUB_SERVICE_KEYS, "PI_JOBS_DIR", "PI_SANDBOX_DIR", "TMPDIR", "PI_WORKER_NAME", ...WORKER_SERVICE_KEYS, ...RECEIVER_SERVICE_KEYS, ...CLI_KEY_NAMES])]);
|
|
630
834
|
|
|
631
835
|
/** Issue #471 (gate round 1): what a service manager gives every service itself, so a `.env` need not carry it. */
|
|
632
836
|
const AMBIENT_SERVICE_KEYS = Object.freeze(["TMPDIR", "TEMP", "XDG_RUNTIME_DIR", "HOME"]);
|
|
@@ -651,6 +855,33 @@ function installedWorkerUnit(seams, cwd) {
|
|
|
651
855
|
return null;
|
|
652
856
|
}
|
|
653
857
|
|
|
858
|
+
/**
|
|
859
|
+
* Issue #707: whether the worker unit installed for this folder pulls in the OpenShell gateway's own user unit
|
|
860
|
+
* (`Wants=openshell-gateway.service`, which the 0.1.4 deb ships), as `service install` writes it for an openshell venue,
|
|
861
|
+
* and is NOT ordered after it: that unit is `After=default.target`, so an `After=` from a worker that is WantedBy
|
|
862
|
+
* default.target is an ordering cycle, and systemd deletes the worker's start job to break it (review round 1 of #715,
|
|
863
|
+
* measured). Read only on Linux, from the unit file itself (`installedWorkerUnit`); no unit for this folder says nothing,
|
|
864
|
+
* since the worker may run by hand. A ⚠, not a ✗: a missing `Wants=` costs a gateway the worker's boot only logs as
|
|
865
|
+
* down, and `service install --force` rewrites the unit.
|
|
866
|
+
*/
|
|
867
|
+
export function openshellUnitOrderChecks(seams, cwd) {
|
|
868
|
+
if (seams.platform !== "linux" || typeof seams.fileExists !== "function") return [];
|
|
869
|
+
const unit = installedWorkerUnit(seams, cwd);
|
|
870
|
+
if (!unit) return [];
|
|
871
|
+
if (unit.scope === "system") {
|
|
872
|
+
return [{ ok: true, warn: true, label: `openshell: the worker's unit (${unit.path}) is a system unit, which cannot pull in the gateway's user unit (openshell-gateway.service)`, fix: "install the worker in user scope (`pi-dispatch service install`, without --system), in the account whose user manager runs the gateway and whose home holds the CLI's gateway registration" }];
|
|
873
|
+
}
|
|
874
|
+
let text;
|
|
875
|
+
try {
|
|
876
|
+
text = String((seams.readUnit ?? ((path) => readFileSync(path, "utf8")))(unit.path));
|
|
877
|
+
} catch {
|
|
878
|
+
return [{ ok: true, warn: true, label: `openshell: whether the worker's unit (${unit.path}) pulls in the gateway could not be read`, fix: "make the unit file readable by this account, then re-run doctor" }];
|
|
879
|
+
}
|
|
880
|
+
const names = (key) => new RegExp(`^${key}=(?:.*\\s)?openshell-gateway\\.service(?:\\s|$)`, "m").test(text);
|
|
881
|
+
if (names("After")) return [{ ok: true, warn: true, label: `openshell: the worker's unit (${unit.path}) is After=openshell-gateway.service, an ordering cycle (the gateway's unit is After=default.target), which systemd breaks by deleting the worker's start job at boot`, fix: "`pi-dispatch service install --force` in the deployment folder rewrites the unit with Wants=openshell-gateway.service and no After= on it" }];
|
|
882
|
+
return [names("Wants") ? { ok: true, label: "openshell: the worker's unit pulls in the gateway's (Wants=openshell-gateway.service)" } : { ok: true, warn: true, label: `openshell: the worker's unit (${unit.path}) does not pull in the gateway's (openshell-gateway.service), so at boot the gateway starts only if something else starts it`, fix: "`pi-dispatch service install --force` in the deployment folder rewrites the unit with Wants=openshell-gateway.service" }];
|
|
883
|
+
}
|
|
884
|
+
|
|
654
885
|
/** The CLI variables that name a path, whose value a ⚠ may show; the rest (endpoints, names, tokens) it never shows. */
|
|
655
886
|
const CLI_PATH_KEYS = Object.freeze(["CONTAINERS_CONF", "CONTAINERS_CONF_OVERRIDE", "CONTAINERS_STORAGE_CONF", "CONTAINERS_REGISTRIES_CONF", "XDG_CONFIG_HOME", "XDG_RUNTIME_DIR", "DOCKER_CONFIG", "GH_CONFIG_DIR", "HOME"]);
|
|
656
887
|
/** The ones the worker's containers.conf chain check reads, which doctor runs in-process on the resolved values. */
|
|
@@ -662,11 +893,13 @@ const PODMAN_CHAIN_KEYS = Object.freeze(["CONTAINERS_CONF", "CONTAINERS_CONF_OVE
|
|
|
662
893
|
* context name, or a token. Says that doctor's probes through that CLI describe this shell's view, and, for the keys
|
|
663
894
|
* the containers.conf chain check reads, that the check itself judged the file's value, in-process, as the worker does.
|
|
664
895
|
*/
|
|
665
|
-
export function cliNotHandedLines(service, envPath) {
|
|
896
|
+
export function cliNotHandedLines(service, envPath, { openshellUsed = false } = {}) {
|
|
666
897
|
const listed = (items) => (items.length < 2 ? items.join("") : `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`);
|
|
667
898
|
// Issue #481: an empty value is left out only where the tool's own source reads it as unset (`fileConfigures`).
|
|
668
899
|
return CLI_KEY_NAMES.filter((k) => Object.hasOwn(service.fromFile, k) && fileConfigures(k, service.fromFile[k])).map((k) => {
|
|
669
|
-
|
|
900
|
+
// The openshell CLI is named only where an openshell venue is blessed (review round 1 of #715): elsewhere no probe
|
|
901
|
+
// of it runs, and the service never starts it.
|
|
902
|
+
const names = Object.entries(CLI_SERVICE_KEYS).filter(([cli, ks]) => ks.includes(k) && (cli !== "openshell" || openshellUsed)).map(([cli]) => cli);
|
|
670
903
|
const clis = listed(names);
|
|
671
904
|
// Grammar by count (round-cap re-review, D2): "podman, docker and gh use it", "podman uses it".
|
|
672
905
|
const verb = names.length > 1 ? "use" : "uses";
|
|
@@ -691,6 +924,7 @@ export const STEERING_SERVICE_KEYS = Object.freeze({
|
|
|
691
924
|
PI_BACKENDS: "which runtime doctor spawns: judged by the worker's own list parser, a shell/.env disagreement a ✗ (deploymentVenueEnv)",
|
|
692
925
|
PI_EGRESS: "whether the egress canary runs: the worker's own reading, a disagreement a ✗ (deploymentVenueEnv)",
|
|
693
926
|
PI_EGRESS_PROXY: "which proxy container doctor inspects: the worker's own name resolution, a disagreement a ✗ (deploymentVenueEnv)",
|
|
927
|
+
PI_OPENSHELL_ENGINE: "which engine the openshell lines spawn (its version, the job image in its store): docker or podman only, anything else named and nothing spawned (doctor-openshell.mjs), a disagreement a ✗ (deploymentVenueEnv, issue #707)",
|
|
694
928
|
VALKEY_URL: "which Valkey doctor connects to: the owner rule the worker applies at boot, and a line doctor cannot read makes it contact none",
|
|
695
929
|
PI_JOB_IMAGE: "the image doctor inspects and runs (the canary, the gh probe, --live): the worker's `||` default and the one image rule its boot applies (image-ref.mjs: blank, padded, a leading dash, a control character), a refused value named and not used",
|
|
696
930
|
GITHUB_AUTH_SOURCE: "whether doctor runs gh on this host: the worker's own three values",
|
|
@@ -708,6 +942,13 @@ export const STEERING_SERVICE_KEYS = Object.freeze({
|
|
|
708
942
|
GH_CONFIG_DIR: "which gh login the SERVICE's gh reads: never handed to doctor's gh, named in a ⚠",
|
|
709
943
|
});
|
|
710
944
|
|
|
945
|
+
/**
|
|
946
|
+
* The operator console's environment (issue #640): the keys `pi-dispatch-console` reads, resolved by the console's own
|
|
947
|
+
* rule (this shell's value where it sets one, else the deployment `.env`'s, as `panelEnv` does for the panel). Not
|
|
948
|
+
* service keys: no service reads them, so they are kept out of SERVICE_ENV_KEYS and its "set in this shell only" line.
|
|
949
|
+
*/
|
|
950
|
+
export const CONSOLE_ENV_KEYS = Object.freeze(["PI_CONSOLE_LISTEN", "PI_CONSOLE_TOKENS_FILE", "PI_CONSOLE_ALLOWED_HOSTS", "PI_CONSOLE_READ_ONLY", "PI_CONSOLE_BIND_PUBLIC", "PI_CONSOLE_IDLE_MINUTES", "PI_CONSOLE_MAX_HOURS", "PI_CONSOLE_MAX_STREAMS"]);
|
|
951
|
+
|
|
711
952
|
/**
|
|
712
953
|
* Issue #471: the variables doctor reads from THIS shell on purpose, each with why, so the bolt test can tell a read
|
|
713
954
|
* that went back to the shell from one that was always meant to be there. A name read as `env.NAME` in doctor.mjs is in
|
|
@@ -723,11 +964,12 @@ export const DOCTOR_SHELL_KEYS = Object.freeze({
|
|
|
723
964
|
PI_MODEL_ENDPOINTS_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS), #503",
|
|
724
965
|
PI_ENVELOPE_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS), #504",
|
|
725
966
|
[VALKEY_SHARED_KEY]: "read from .env ONLY (#464); a value in this shell is named as ignored",
|
|
967
|
+
...Object.fromEntries(CONSOLE_ENV_KEYS.map((k) => [k, "the operator console's, not a service's: resolved by the console's own rule (this shell, else the deployment .env) apart from the service record, #640"])),
|
|
726
968
|
});
|
|
727
969
|
|
|
728
970
|
/** Issue #471: the service keys whose values a disagreement line never prints (set or unset only); URLs print as
|
|
729
971
|
* `urlShown` does. Every other key's two values are shown, quoted. */
|
|
730
|
-
const SECRET_SERVICE_KEYS = new Set([VALKEY_PASSWORD_KEY, "GH_TOKEN", "GITHUB_TOKEN", "GITHUB_APP_PRIVATE_KEY", "WEBHOOK_SECRET", "GITLAB_TOKEN", "GITLAB_WEBHOOK_SECRET", "FORGEJO_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_TOKEN", "AZURE_WEBHOOK_SECRET"]);
|
|
972
|
+
const SECRET_SERVICE_KEYS = new Set([VALKEY_PASSWORD_KEY, "GH_TOKEN", "GITHUB_TOKEN", "GITHUB_APP_PRIVATE_KEY", "WEBHOOK_SECRET", "GITLAB_TOKEN", "GITLAB_WEBHOOK_SECRET", "FORGEJO_TOKEN", "FORGEJO_PERMISSION_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_TOKEN", "AZURE_WEBHOOK_SECRET"]);
|
|
731
973
|
const URL_SERVICE_KEYS = new Set(["VALKEY_URL", "GITLAB_URL", "FORGEJO_URL", "AZURE_ORG_URL"]);
|
|
732
974
|
|
|
733
975
|
/** `keys` narrowed to `SERVICE_ENV_KEYS` plus the provider's own key variables. */
|
|
@@ -830,6 +1072,7 @@ export const EMPTY_READ_AS_UNSET = Object.freeze({
|
|
|
830
1072
|
GITLAB_WEBHOOK_SECRET: "empty",
|
|
831
1073
|
FORGEJO_URL: "empty",
|
|
832
1074
|
FORGEJO_TOKEN: "empty",
|
|
1075
|
+
FORGEJO_PERMISSION_TOKEN: "blank",
|
|
833
1076
|
FORGEJO_WEBHOOK_SECRET: "empty",
|
|
834
1077
|
AZURE_ORG_URL: "empty",
|
|
835
1078
|
AZURE_TOKEN: "empty",
|
|
@@ -1020,7 +1263,7 @@ export function envFileKeys(path, keys, { fileExists, readEnvFile, statFile = st
|
|
|
1020
1263
|
* loaders throw a `configError` on a path that does not exist and on content that does not parse, which is
|
|
1021
1264
|
* exactly the boot this check is predicting.
|
|
1022
1265
|
*/
|
|
1023
|
-
const BOOT_FILES = Object.freeze([
|
|
1266
|
+
export const BOOT_FILES = Object.freeze([
|
|
1024
1267
|
Object.freeze({
|
|
1025
1268
|
key: "PI_PAUSE_WINDOWS_FILE",
|
|
1026
1269
|
noun: "scoped pauses",
|
|
@@ -1119,6 +1362,15 @@ const BOOT_FILES = Object.freeze([
|
|
|
1119
1362
|
}),
|
|
1120
1363
|
]);
|
|
1121
1364
|
|
|
1365
|
+
/**
|
|
1366
|
+
* A boot file's published check id (issue #712, `NAMED_CHECK_IDS`): `<scaffold name>.file` for the SERVICE's verdict on
|
|
1367
|
+
* the file it reads, `<scaffold name>.file-shell` for this shell's. Derived from the scaffold name so a sixth boot file
|
|
1368
|
+
* names itself, and the test holding `NAMED_CHECK_IDS` to the source fails until it is listed.
|
|
1369
|
+
*/
|
|
1370
|
+
export function bootFileId(spec, subject = "service") {
|
|
1371
|
+
return `${spec.scaffold.replace(/\.json$/, "")}.file${subject === "shell" ? "-shell" : ""}`;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1122
1374
|
/** A key's value when it is a non-empty string, else null: the boot reads an unset key as off. */
|
|
1123
1375
|
function nonEmpty(value) {
|
|
1124
1376
|
return typeof value === "string" && value.trim() !== "" ? value : null;
|
|
@@ -1225,8 +1477,159 @@ function loadVerdict(spec, rawPath, cwd, io, platform = process.platform) {
|
|
|
1225
1477
|
spec.load(path, io.loaderIo, io.env);
|
|
1226
1478
|
return { ok: true };
|
|
1227
1479
|
} catch (err) {
|
|
1228
|
-
|
|
1480
|
+
// Issue #761: the loader's own flag says a newer pi-dispatch wrote the file (never the message's wording), so the
|
|
1481
|
+
// fix can say to upgrade rather than to edit what that release wrote.
|
|
1482
|
+
return { ok: false, reason: envValueShown(err?.message ?? String(err)), ...(err?.newer === true ? { newer: true } : {}) };
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
/**
|
|
1487
|
+
* The fix for a file a newer pi-dispatch wrote (issue #761), whichever store: upgrade `who` and every writer of the file,
|
|
1488
|
+
* never edit or delete it, which would lose what that release wrote. The one exception is a `version` an operator
|
|
1489
|
+
* wrote by hand (earlier releases ignored the key): no release wrote that file, so removing the key is the fix. One
|
|
1490
|
+
* sentence for the settings overlay, the pause windows file and the triggers file, so the three cannot drift apart.
|
|
1491
|
+
*/
|
|
1492
|
+
export function newerFileFix(who) {
|
|
1493
|
+
return `upgrade ${who}, and every panel and console that writes this file, to the release that wrote it; do not edit or delete the file, which would lose what that release wrote, unless you added \`version\` to it yourself: then remove that key`;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
/** The oldest git the checkpoint store accepts (issue #645), as `[major, minor]`. */
|
|
1497
|
+
export const CHECKPOINT_GIT_MIN = Object.freeze([2, 32]);
|
|
1498
|
+
|
|
1499
|
+
/** `[major, minor]` out of `git --version`'s answer (`git version 2.43.0`, `git version 2.39.3 (Apple Git-146)`), or null. */
|
|
1500
|
+
export function parseGitVersion(text) {
|
|
1501
|
+
const m = /\bgit version (\d{1,4})\.(\d{1,4})/.exec(String(text ?? ""));
|
|
1502
|
+
return m ? [Number(m[1]), Number(m[2])] : null;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
/**
|
|
1506
|
+
* The checkpoint store's group (issue #645): where it is, whether it is a private directory of this account's, whether it
|
|
1507
|
+
* sits under the OS temp dir, whether git is new enough, and whether each venue that runs jobs here can freeze one while
|
|
1508
|
+
* its workspace is read. Pure over its reads, so every arm is driven without a host.
|
|
1509
|
+
*
|
|
1510
|
+
* WARN AT MOST, with ONE exception, and that is deliberate rather than lenient: the store is written only by jobs whose
|
|
1511
|
+
* trigger arms `run.recover` (the worker's run checkpointer, issue #645), so a red doctor here would fail every deployment
|
|
1512
|
+
* that configured the store and armed nothing, and a problem with it costs a durable run its recovery (the run falls back
|
|
1513
|
+
* to today's retry), never a job its run. The exception is a store that
|
|
1514
|
+
* COLLIDES with the session store (`checkpointStoreCollision`), which is ✗ because the worker itself refuses to load
|
|
1515
|
+
* that configuration: doctor's verdict follows the worker's.
|
|
1516
|
+
*
|
|
1517
|
+
* 0700 AND THIS ACCOUNT'S, for the transcript store's reason: a checkpoint is a job's whole workspace, so as PII-bearing
|
|
1518
|
+
* as a transcript, and an owner other than the worker could swap what a restore reads. A symbolic link is refused for the
|
|
1519
|
+
* same reason (the store's contents would live wherever it points). Under the OS temp dir it is swept by the OS, which
|
|
1520
|
+
* turns a durable run's resume into a silent cold start.
|
|
1521
|
+
*
|
|
1522
|
+
* GIT 2.32 because that is the first release that reads `GIT_CONFIG_GLOBAL`, which the store's git environment
|
|
1523
|
+
* (`git-hardening.mjs`, `storeGitEnv`) points at /dev/null so no global config (a hook path, a filter, an fsmonitor)
|
|
1524
|
+
* runs inside the store. An older git ignores the variable; the empty HOME beside it still hides `~/.gitconfig`, so the
|
|
1525
|
+
* line warns rather than refuses, but the store then rests on one guard where it was built with two.
|
|
1526
|
+
*
|
|
1527
|
+
* PAUSABLE IS A FACT LINE (`ok: true`) either way, because nothing the operator does short of changing hosts clears it.
|
|
1528
|
+
* What it costs is said in the line: a venue that cannot freeze a running job gets EXIT checkpoints only, taken once the
|
|
1529
|
+
* container is gone, so a job there is not recovered from a worker that crashed mid-run. A checkpoint is never read
|
|
1530
|
+
* from a workspace the job is still writing.
|
|
1531
|
+
*/
|
|
1532
|
+
export function checkpointStoreChecks({ dir, explicit = false, sessionsDir = null, fs = { lstatSync, statSync, accessSync }, uid, underTemp = () => false, gitVersion = null, venues = [DEFAULT_BACKEND], podmanRead = null }) {
|
|
1533
|
+
const checks = [];
|
|
1534
|
+
const shown = envValueShown(dir);
|
|
1535
|
+
const where = explicit ? "PI_CHECKPOINTS_DIR" : "the default inside PI_SESSIONS_DIR";
|
|
1536
|
+
const collision = checkpointStoreCollision(sessionsDir, dir);
|
|
1537
|
+
if (collision) {
|
|
1538
|
+
checks.push({
|
|
1539
|
+
ok: false,
|
|
1540
|
+
label: `checkpoint store ${shown} and the session store ${envValueShown(resolve(sessionsDir))} collide: ${collision}, so the session reaper and the checkpoint sweep would delete each other's folders, and the worker refuses to start with this configuration`,
|
|
1541
|
+
fix: "unset PI_CHECKPOINTS_DIR (the default is <PI_SESSIONS_DIR>/.checkpoints) or point it at a directory outside PI_SESSIONS_DIR",
|
|
1542
|
+
});
|
|
1543
|
+
}
|
|
1544
|
+
let st = null;
|
|
1545
|
+
let missing = false;
|
|
1546
|
+
try {
|
|
1547
|
+
st = fs.lstatSync(dir);
|
|
1548
|
+
} catch (err) {
|
|
1549
|
+
if (err?.code === "ENOENT") missing = true;
|
|
1550
|
+
else checks.push({ ok: false, warn: true, label: `checkpoint store ${shown} (${where}) could not be read (${err?.code ?? "error"})`, fix: "make it a directory this account can read, or point PI_CHECKPOINTS_DIR at one" });
|
|
1551
|
+
}
|
|
1552
|
+
if (missing) {
|
|
1553
|
+
// The nearest ancestor that exists decides whether the checkpointer can create it, as `mkdir -p` would. One that
|
|
1554
|
+
// exists and is NOT a directory (a file where a parent should be) is its own answer: `mkdir -p` fails on it.
|
|
1555
|
+
let parent = dirname(dir);
|
|
1556
|
+
let ancestor = null;
|
|
1557
|
+
let blockedBy = null;
|
|
1558
|
+
for (let i = 0; i < 64 && ancestor === null && blockedBy === null; i++) {
|
|
1559
|
+
let pst = null;
|
|
1560
|
+
try {
|
|
1561
|
+
pst = fs.statSync(parent);
|
|
1562
|
+
} catch {
|
|
1563
|
+
pst = null;
|
|
1564
|
+
}
|
|
1565
|
+
if (pst?.isDirectory()) ancestor = parent;
|
|
1566
|
+
else if (pst) blockedBy = parent;
|
|
1567
|
+
else {
|
|
1568
|
+
const up = dirname(parent);
|
|
1569
|
+
if (up === parent) break;
|
|
1570
|
+
parent = up;
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
let writable = false;
|
|
1574
|
+
if (ancestor) {
|
|
1575
|
+
try {
|
|
1576
|
+
fs.accessSync(ancestor, fsConstants.W_OK | fsConstants.X_OK);
|
|
1577
|
+
writable = true;
|
|
1578
|
+
} catch {
|
|
1579
|
+
writable = false;
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
checks.push(
|
|
1583
|
+
writable
|
|
1584
|
+
? { ok: true, label: `checkpoint store ${shown} (${where}) does not exist yet: it is created with mode 0700 inside ${envValueShown(ancestor)} when a durable run first writes one` }
|
|
1585
|
+
: { ok: false, warn: true, label: `checkpoint store ${shown} (${where}) does not exist and cannot be created: ${ancestor ? `${envValueShown(ancestor)} is not writable by this account` : blockedBy ? `${envValueShown(blockedBy)} above it is not a directory` : "no directory above it exists"}`, fix: `create it as the worker's account: mkdir -p ${shown} && chmod 700 ${shown}` },
|
|
1586
|
+
);
|
|
1587
|
+
} else if (st) {
|
|
1588
|
+
if (st.isSymbolicLink()) {
|
|
1589
|
+
checks.push({ ok: false, warn: true, label: `checkpoint store ${shown} (${where}) is a symbolic link, so what a restore reads lives wherever it points`, fix: `replace it with a real directory: rm ${shown} && mkdir -p ${shown} && chmod 700 ${shown}` });
|
|
1590
|
+
} else if (!st.isDirectory()) {
|
|
1591
|
+
checks.push({ ok: false, warn: true, label: `checkpoint store ${shown} (${where}) is not a directory`, fix: `move it aside and create a directory there: mkdir -p ${shown} && chmod 700 ${shown}` });
|
|
1592
|
+
} else {
|
|
1593
|
+
// No uid (Windows): ownership and the mode bits mean nothing there, so neither is judged.
|
|
1594
|
+
const problems = [];
|
|
1595
|
+
if (Number.isInteger(uid) && st.uid !== uid) problems.push(`owned by uid ${st.uid}, not this account (uid ${uid})`);
|
|
1596
|
+
if (Number.isInteger(uid) && (st.mode & 0o777) !== 0o700) problems.push(`mode ${(st.mode & 0o777).toString(8).padStart(4, "0")}, not 0700`);
|
|
1597
|
+
checks.push(
|
|
1598
|
+
problems.length === 0
|
|
1599
|
+
? { ok: true, label: `checkpoint store ${shown} (${where}) is a private directory of this account's` }
|
|
1600
|
+
: { ok: false, warn: true, label: `checkpoint store ${shown} (${where}) is ${problems.join(" and ")}: a checkpoint holds a job's whole workspace, as private as a transcript`, fix: `as the worker's account: chown "$(id -u)" ${shown} && chmod 700 ${shown}` },
|
|
1601
|
+
);
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
if (underTemp(dir)) {
|
|
1605
|
+
checks.push({ ok: false, warn: true, label: `checkpoint store ${shown} is under the OS temp dir, which the OS sweeps: a durable run would resume from nothing after a reboot or a cleaner pass`, fix: "point PI_CHECKPOINTS_DIR (or PI_SESSIONS_DIR) at a durable private directory" });
|
|
1229
1606
|
}
|
|
1607
|
+
const version = parseGitVersion(gitVersion);
|
|
1608
|
+
const [minMajor, minMinor] = CHECKPOINT_GIT_MIN;
|
|
1609
|
+
if (version === null) {
|
|
1610
|
+
checks.push({ ok: false, warn: true, label: "checkpoint store: `git --version` did not answer, so whether git is 2.32 or newer is not known", fix: "install git 2.32 or newer on the worker's PATH" });
|
|
1611
|
+
} else if (version[0] < minMajor || (version[0] === minMajor && version[1] < minMinor)) {
|
|
1612
|
+
checks.push({ ok: false, warn: true, label: `checkpoint store: git ${version.join(".")} is older than 2.32, which ignores GIT_CONFIG_GLOBAL, so the store's git keeps one of its two guards against this host's own config`, fix: "install git 2.32 or newer on the worker's PATH" });
|
|
1613
|
+
} else {
|
|
1614
|
+
checks.push({ ok: true, label: `checkpoint store: git ${version.join(".")} (2.32 or newer)` });
|
|
1615
|
+
}
|
|
1616
|
+
for (const venue of venues) {
|
|
1617
|
+
const word = BACKENDS[venue]?.declares?.pausable ?? ABSENT;
|
|
1618
|
+
let label;
|
|
1619
|
+
const exitOnly = "so its jobs get exit checkpoints only, taken once the container is gone, and are not recovered from a worker crash mid-run";
|
|
1620
|
+
if (word === ABSENT) label = `checkpoint store: ${venue} declares no freeze, ${exitOnly}`;
|
|
1621
|
+
else if (venue === PODMAN_BACKEND) {
|
|
1622
|
+
const can = podmanPausable(podmanRead);
|
|
1623
|
+
label =
|
|
1624
|
+
can.pausable === true
|
|
1625
|
+
? `checkpoint store: ${venue} can freeze a job while its workspace is read (podman pause, cgroup v2)`
|
|
1626
|
+
: can.pausable === false
|
|
1627
|
+
? `checkpoint store: ${venue} cannot freeze a job here (rootless Podman on cgroup v1), ${exitOnly}`
|
|
1628
|
+
: `checkpoint store: whether ${venue} can freeze a job is not known (podman info did not answer: ${can.reason}); on cgroup v1 it cannot, ${exitOnly}`;
|
|
1629
|
+
} else label = `checkpoint store: ${venue} can freeze a job while its workspace is read (docker pause, the cgroup freezer)`;
|
|
1630
|
+
checks.push({ ok: true, label });
|
|
1631
|
+
}
|
|
1632
|
+
return checks;
|
|
1230
1633
|
}
|
|
1231
1634
|
|
|
1232
1635
|
/**
|
|
@@ -1276,6 +1679,22 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1276
1679
|
// ioredis's SELECT. It is a ✗ of its own, the worker's refusal (exit 2), and nothing below contacts a Valkey.
|
|
1277
1680
|
const valkeyDbProblem = unreadValkey ? null : valkeyUrlProblem(valkeyUrl);
|
|
1278
1681
|
const provider = env.PI_PROVIDER ?? "anthropic";
|
|
1682
|
+
// Issue #673: is the service's PI_MODEL or PI_PROVIDER outside the one id shape (`modelFrom`, `providerFrom`)? Judged
|
|
1683
|
+
// once, here, because the answer decides more than its own row (review round 1): the older rows that judge the same
|
|
1684
|
+
// value (the provider-key row, the trigger-secret clash row and the deployment default's catalog row) are skipped for
|
|
1685
|
+
// a malformed one. They print the value, so a control byte would reach the terminal, and their fix ("set PI_PROVIDER to
|
|
1686
|
+
// a provider pi has") is not the one the boot refusal asks for; one root cause prints one row. A well-shaped value is
|
|
1687
|
+
// judged by them unchanged. Null when the setting is unset or well-shaped, else the worker's own refusal message.
|
|
1688
|
+
const envShapeProblem = {};
|
|
1689
|
+
for (const [name, judge] of [["PI_MODEL", modelFrom], ["PI_PROVIDER", providerFrom]]) {
|
|
1690
|
+
envShapeProblem[name] = null;
|
|
1691
|
+
if (typeof env[name] !== "string") continue;
|
|
1692
|
+
try {
|
|
1693
|
+
judge(env);
|
|
1694
|
+
} catch (err) {
|
|
1695
|
+
envShapeProblem[name] = err?.message ?? "invalid";
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1279
1698
|
// Issue #464 (gate round 1): this host's declared worker name, resolved with every other service key. Every reader
|
|
1280
1699
|
// below (the fleet line and its routing warning, the per-host backend wording, which cron triggers are scheduled
|
|
1281
1700
|
// here) uses this value: read from the shell alone, a deployment whose `.env` names its worker was told host routing
|
|
@@ -1290,8 +1709,25 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1290
1709
|
let patFromFile = null;
|
|
1291
1710
|
|
|
1292
1711
|
const checks = [];
|
|
1712
|
+
// Issue #712: the section each check was added in, for its published id (`assignCheckIds`, doctor-results.mjs). Each
|
|
1713
|
+
// `section(name)` call hands every check added since the last one to the section that was current, then opens `name`.
|
|
1714
|
+
// A WeakMap rather than a field on the check: a check may be a shared or frozen object, and a write to a frozen one
|
|
1715
|
+
// throws in this strict module. Read back through `seams.checkSection`, set again on every collection (`--fix`
|
|
1716
|
+
// re-collects).
|
|
1717
|
+
const sectionOf = new WeakMap();
|
|
1718
|
+
let currentSection = "node";
|
|
1719
|
+
const tagSection = (list, name) => {
|
|
1720
|
+
for (const c of list) if (c !== null && typeof c === "object" && !sectionOf.has(c)) sectionOf.set(c, name);
|
|
1721
|
+
};
|
|
1722
|
+
const section = (name) => {
|
|
1723
|
+
tagSection(checks, currentSection);
|
|
1724
|
+
currentSection = name;
|
|
1725
|
+
};
|
|
1726
|
+
seams.checkSection = (c) => sectionOf.get(c) ?? null;
|
|
1727
|
+
section("node");
|
|
1293
1728
|
checks.push(nodeCheck(nodeVersion));
|
|
1294
1729
|
|
|
1730
|
+
section("env");
|
|
1295
1731
|
checks.push({
|
|
1296
1732
|
ok: fileExists(join(cwd, ".env")),
|
|
1297
1733
|
warn: true, // advisory: env may be supplied by a service manager instead of a file
|
|
@@ -1390,10 +1826,11 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1390
1826
|
// disagreement is said once at the end rather than lost.
|
|
1391
1827
|
const envPathShown = serviceEnvFile?.path ?? join(cwd, ".env");
|
|
1392
1828
|
// PR #474's round cap: each CLI variable the file sets, which no program doctor starts is handed.
|
|
1393
|
-
checks.push(...cliNotHandedLines(service, envPathShown));
|
|
1829
|
+
checks.push(...cliNotHandedLines(service, envPathShown, { openshellUsed: venuesOf(env).openshellUsed }));
|
|
1394
1830
|
const saidAtTop = { disagreements: service.disagreements.length, unread: service.unread.length };
|
|
1395
1831
|
// A CLI variable is not listed as read: no program doctor starts used it, and its own ⚠ says so.
|
|
1396
1832
|
checks.push(...serviceEnvLines(service, envPathShown, { said: ["VALKEY_URL", "PI_PROVIDER", ...GITHUB_SERVICE_KEYS, "PI_JOBS_DIR", "PI_SANDBOX_DIR", "TMPDIR", ...CLI_KEY_NAMES] }));
|
|
1833
|
+
section("image");
|
|
1397
1834
|
if (jobImageRefused) {
|
|
1398
1835
|
checks.push({
|
|
1399
1836
|
ok: false,
|
|
@@ -1405,6 +1842,12 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1405
1842
|
// Issue #354: which venues run jobs here. An unparseable PI_BACKENDS reads as the unset default, so every docker line
|
|
1406
1843
|
// below is exactly what it always was, and the backend section is what fails on it (the worker refuses to boot).
|
|
1407
1844
|
const { localUsed, podmanUsed, podmanDefault } = venuesOf(env);
|
|
1845
|
+
// Issue #707: where an openshell-only deployment's Valkey runs (`openshellSetupOf`, the rule `up` and `service install`
|
|
1846
|
+
// follow): on the gateway's engine, docker by default. The Valkey lines' fixes name THAT engine's way to start it, as
|
|
1847
|
+
// they name docker's where `local` is blessed and the Quadlet unit's where `podman` is.
|
|
1848
|
+
const openshellSetup = openshellSetupOf(venuesOf(env), env);
|
|
1849
|
+
const valkeyVenues = { localUsed: localUsed || openshellSetup.valkeyEngine === "docker", podmanUsed: podmanUsed || openshellSetup.valkeyEngine === "podman" };
|
|
1850
|
+
section("docker");
|
|
1408
1851
|
// THE FIRST DOCKER CALL OF THE RUN, which is why it was the one that hung (issue #397): a daemon that
|
|
1409
1852
|
// accepts the connection and never answers held doctor here, before it had printed anything. Bounded
|
|
1410
1853
|
// now, and the three outcomes are three different sentences, because telling someone whose daemon is
|
|
@@ -1446,9 +1889,17 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1446
1889
|
// image checks just below, and `optingOut`/`requiring` colour the staged-packages lines further down.
|
|
1447
1890
|
// `optingOut` counts the only value that withholds the staged set; `requiring` counts an explicit
|
|
1448
1891
|
// run.packages: true, which arms nothing any more but is still an operator statement of intent.
|
|
1449
|
-
|
|
1892
|
+
// Issue #761: the version tables, and the versions this build reads, that the store-version needs are judged by,
|
|
1893
|
+
// per store: `{ settings: { supported, table, valueTable }, pauseWindows: { supported, table }, triggers: { supported,
|
|
1894
|
+
// runTable, blockTable }, jobData: { table } }`, each part the store module's own when absent. This seam is how a test
|
|
1895
|
+
// drives a need above 1 through the real collection, for a store whose table has no row of its own and at a version no row holds yet.
|
|
1896
|
+
const versionTables = seams.versionTables ?? {};
|
|
1897
|
+
const { watchSources: triggerWatchSources, watchRules: triggerWatchRules, modelRoles: triggerModelRoles, secretBundles: triggerBundles, requiring, waiting, listing, waitProfiles, waitAfters, optingOut, resuming, recovering, replicating, instructing, commands, secreting, onceArmed, onceSpent, secretProfiles, localSecretFolders, secretNames, folders, images, imageRoutes, namedBackends, skillsDirs, forges, repositories, flows, excludedNames, costCaps, modelRuns, triggersVersionNeed, jobDataVersionNeed, parseError, parseNewer: triggersFileIsNewer, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd, declaredWorkerName, versionTables);
|
|
1898
|
+
const modelRoleProjects = readProjectFacts(env, fileExists).projects;
|
|
1450
1899
|
const scopedLimitFacts = readScopedLimitFacts(env, fileExists);
|
|
1451
|
-
const pauseWindowFacts = readPauseWindowFacts(env, fileExists);
|
|
1900
|
+
const pauseWindowFacts = readPauseWindowFacts(env, fileExists, versionTables.pauseWindows);
|
|
1901
|
+
section("triggers");
|
|
1902
|
+
checks.push(...watchDoctorChecks({ sources: triggerWatchSources, rules: triggerWatchRules, env }));
|
|
1452
1903
|
// FIRST, and fail rather than warn: every check below this line reads counts that a parse failure
|
|
1453
1904
|
// zeroed, so a green run here would be reporting on a file nobody could read. The receiver loads this
|
|
1454
1905
|
// file unconditionally and refuses to start without it, which is the consequence worth naming.
|
|
@@ -1461,6 +1912,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1461
1912
|
if (triggersSet !== undefined && !fileExists(triggersPath(env, cwd))) {
|
|
1462
1913
|
const note = fromFileNote(fileSays("PI_TRIGGERS_FILE"));
|
|
1463
1914
|
checks.push({
|
|
1915
|
+
id: "triggers.file-missing",
|
|
1464
1916
|
ok: false,
|
|
1465
1917
|
label: triggersSet === ""
|
|
1466
1918
|
? `PI_TRIGGERS_FILE is set to an empty value, which neither process reads as unset: the worker and the receiver refuse to start on it (exit 2)${note}`
|
|
@@ -1470,17 +1922,24 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1470
1922
|
}
|
|
1471
1923
|
if (parseError) {
|
|
1472
1924
|
checks.push({
|
|
1925
|
+
// Named (issue #712): the console quotes it beside "the triggers file does not load".
|
|
1926
|
+
id: "triggers.file",
|
|
1473
1927
|
ok: false,
|
|
1474
1928
|
// "is refused at load" rather than "does not parse", because since issue #313 it is no longer
|
|
1475
1929
|
// only a syntax error: a duplicate key parses perfectly and is refused anyway. The loader's own
|
|
1476
1930
|
// message says which, and the old wording sent an operator hunting for a syntax error that was
|
|
1477
1931
|
// not there. It still covers the JSON case, whose message begins "is not valid JSON".
|
|
1478
1932
|
label: `triggers file is refused at load -- the receiver will refuse to start: ${parseError}`,
|
|
1479
|
-
|
|
1933
|
+
// Issue #761: a file a newer pi-dispatch wrote is no broken file, and editing it would lose what that release wrote.
|
|
1934
|
+
fix: triggersFileIsNewer
|
|
1935
|
+
? `${newerFileFix("this host's worker and receiver")} -- every trigger-derived check below is skipped until this host reads it`
|
|
1936
|
+
: `fix ${triggersFilePath} so it loads (the message above names the entry and the reason), then re-run doctor -- every trigger-derived check below is skipped until it loads`,
|
|
1480
1937
|
});
|
|
1481
1938
|
}
|
|
1939
|
+
section("dollars");
|
|
1482
1940
|
checks.push(...dollarChecks(env, costCaps, triggersSet !== undefined));
|
|
1483
1941
|
|
|
1942
|
+
section("image");
|
|
1484
1943
|
// Only meaningful if docker itself responds; otherwise the image check is noise on top of a down daemon. Issue #433:
|
|
1485
1944
|
// and never without `local`, where the podman section's line (the image in THIS ACCOUNT'S store) is the image check.
|
|
1486
1945
|
const imageRun = dockerRun === null ? null : dockerCode === 0 ? await runCmd(spawn, "docker", ["image", "inspect", jobImage], runTimeouts.cmd) : { code: null, ended: "error" };
|
|
@@ -1526,6 +1985,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1526
1985
|
: {}),
|
|
1527
1986
|
});
|
|
1528
1987
|
|
|
1988
|
+
section("triggers");
|
|
1529
1989
|
// REQ-PER-TRIGGER-SKILLS (issue #60). Every distinct `run.skillsDir`, checked BEFORE anything fires,
|
|
1530
1990
|
// because the worker's own gate for these refuses at job time -- correct, but at 03:00 in a log nobody
|
|
1531
1991
|
// is reading. A deployment naming none adds no lines at all, so its output is byte-identical.
|
|
@@ -1584,14 +2044,20 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1584
2044
|
// (DES-FLOW-RESOLUTION-TWO-ADVISORY-LAYERS): these probes read dir names, and a frontmatter
|
|
1585
2045
|
// `name:` rename is invisible to them, so a ⚠ here can be wrong in only the loud direction.
|
|
1586
2046
|
// A deployment with no triggers adds no lines at all, so its output is byte-identical.
|
|
2047
|
+
// Issue #800: the same answer per trigger, for the record of what this host's jobs can load (`publishResults`). Set afresh
|
|
2048
|
+
// on every collection, so a `--fix` re-collect replaces it, as `seams.checkSection` is.
|
|
2049
|
+
seams.flowRows = [];
|
|
1587
2050
|
if (flows.length > 0) {
|
|
1588
2051
|
// One stage read for the whole run; each tuple's staged probe is a lookup on its result.
|
|
1589
|
-
const
|
|
2052
|
+
const flowFs = seams.flowFs ?? { statSync, readFileSync, readdirSync };
|
|
2053
|
+
const stagedProbes = flowProbes({ fileExists, fs: flowFs });
|
|
2054
|
+
const staged = readStagedSkills({ globalPiDir: env.PI_GLOBAL_PI_DIR, readFile: stagedProbes.readFile, fileExists: stagedProbes.fileExists, readDir: stagedProbes.readDir });
|
|
1590
2055
|
const groups = new Map();
|
|
1591
2056
|
for (const f of flows) {
|
|
1592
|
-
const key = JSON.stringify([f.flow, f.folder, f.skillsDir, f.packages]);
|
|
1593
|
-
if (!groups.has(key)) groups.set(key, { ...f, labels: [] });
|
|
2057
|
+
const key = JSON.stringify([f.flow, f.folder, f.skillsDir, f.packages, f.excludeExtensions]);
|
|
2058
|
+
if (!groups.has(key)) groups.set(key, { ...f, labels: [], triggers: [] });
|
|
1594
2059
|
groups.get(key).labels.push(f.label);
|
|
2060
|
+
groups.get(key).triggers.push(f.index);
|
|
1595
2061
|
}
|
|
1596
2062
|
for (const g of groups.values()) {
|
|
1597
2063
|
const at = g.labels.join(", ");
|
|
@@ -1605,37 +2071,18 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1605
2071
|
label: `Trigger flow ${JSON.stringify(g.flow)} fails the skill name charset (${at})`,
|
|
1606
2072
|
fix: "a flow name must match the skill charset (lowercase alphanumerics, - and _, 64 max), or no tier can ever hold it -- fix run.flow",
|
|
1607
2073
|
});
|
|
2074
|
+
// No tier can hold it, and a name that fails the charset is not published.
|
|
2075
|
+
for (const trigger of g.triggers) seams.flowRows.push({ trigger, flow: null, tier: "none", package: null, repo: "unknown" });
|
|
1608
2076
|
continue;
|
|
1609
2077
|
}
|
|
1610
|
-
|
|
1611
|
-
const
|
|
1612
|
-
const unknown =
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
} else {
|
|
1619
|
-
unknown.push("repo (a forge clone, not on this host)");
|
|
1620
|
-
}
|
|
1621
|
-
if (!resolved && g.skillsDir) {
|
|
1622
|
-
if (fileExists(join(g.skillsDir, g.flow, "SKILL.md"))) resolved = `injected run.skillsDir ${g.skillsDir}`;
|
|
1623
|
-
else checked.push("injected run.skillsDir");
|
|
1624
|
-
}
|
|
1625
|
-
if (!resolved && env.PI_GLOBAL_PI_DIR) {
|
|
1626
|
-
if (fileExists(join(env.PI_GLOBAL_PI_DIR, "skills", g.flow, "SKILL.md"))) resolved = "the overlay skills/";
|
|
1627
|
-
else checked.push("overlay skills/");
|
|
1628
|
-
}
|
|
1629
|
-
if (!resolved) {
|
|
1630
|
-
if (!g.packages) {
|
|
1631
|
-
checked.push("staged packages (withheld: run.packages false)");
|
|
1632
|
-
} else {
|
|
1633
|
-
const hit = staged.skills.find((s) => s.name === g.flow);
|
|
1634
|
-
if (hit) resolved = `staged package ${hit.package}`;
|
|
1635
|
-
else if (staged.unenumerable.length > 0) unknown.push(`staged package(s) ${staged.unenumerable.join(", ")} (manifest patterns, not enumerable here)`);
|
|
1636
|
-
else checked.push("staged packages");
|
|
1637
|
-
}
|
|
1638
|
-
}
|
|
2078
|
+
const probes = flowProbes({ fileExists, fs: flowFs });
|
|
2079
|
+
const found = await flowTierOf(g, { repoFlowAtHead: (folder, flow) => repoFlowAtHead(spawn, folder, flow), fileExists: probes.fileExists, overlay: env.PI_GLOBAL_PI_DIR, staged });
|
|
2080
|
+
const { resolved, checked, unknown } = found;
|
|
2081
|
+
// `none` says every tier was checked. Per question, and the stage's read only where this one looks in the stage: a
|
|
2082
|
+
// trigger that withholds the packages did not, and another trigger's unreadable folder says nothing of this one's.
|
|
2083
|
+
const blind = probes.blind() || (g.packages !== false && stagedProbes.blind());
|
|
2084
|
+
const tier = found.tier === "none" && blind ? "unknown" : found.tier;
|
|
2085
|
+
for (const trigger of g.triggers) seams.flowRows.push({ trigger, flow: g.flow, tier, package: found.package, repo: found.repo });
|
|
1639
2086
|
if (resolved) {
|
|
1640
2087
|
checks.push({ ok: true, label: `Trigger flow "${g.flow}" resolves (${at}: ${resolved})` });
|
|
1641
2088
|
} else {
|
|
@@ -1643,7 +2090,38 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1643
2090
|
ok: false,
|
|
1644
2091
|
warn: true,
|
|
1645
2092
|
label: `Trigger flow "${g.flow}" resolves in NO tier visible here (${at})`,
|
|
1646
|
-
fix: `checked: ${checked.join(", ") || "nothing checkable"}${unknown.length > 0 ? `; not checkable here: ${unknown.join(", ")}` : ""} -- commit .pi/skills/${g.flow}/SKILL.md, add the skill to run.skillsDir or the overlay skills/, or stage a package shipping it; a job of this trigger runs without the flow it names (the runner logs flow_not_loaded) and still exits 0`,
|
|
2093
|
+
fix: `checked: ${checked.join(", ") || "nothing checkable"}${unknown.length > 0 ? `; not checkable here: ${unknown.join(", ")}` : ""} -- commit .pi/skills/${g.flow}/SKILL.md, add the skill to run.skillsDir or the overlay skills/, or stage a package shipping it (one already staged must be off this trigger's run.excludeExtensions, and run.packages must not be false); a job of this trigger runs without the flow it names (the runner logs flow_not_loaded) and still exits 0`,
|
|
2094
|
+
});
|
|
2095
|
+
}
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
// Issue #800: a name in run.excludeExtensions that this host holds neither way. No job refuses over it and nothing loads for it, since a host's overlay
|
|
2100
|
+
// and stage are its own and the file is shared, but a typo in a list that keeps code out of a job reads as kept out. ⚠ and never ✗. Only against what
|
|
2101
|
+
// was read: an overlay or manifest that cannot be read says nothing of a name, and has its own line. A trigger that withholds the packages is passed
|
|
2102
|
+
// over: its name may be a package's, which that trigger never looks for.
|
|
2103
|
+
{
|
|
2104
|
+
const overlayDir = env.PI_GLOBAL_PI_DIR;
|
|
2105
|
+
let held = null;
|
|
2106
|
+
try {
|
|
2107
|
+
const entries = readOverlayExtensionRows(overlayDir, { fs: seams.flowFs ?? { readFileSync, readdirSync, statSync } });
|
|
2108
|
+
const manifest = overlayDir && fileExists(join(overlayDir, PACKAGES_SUBDIR, STAGE_MANIFEST)) ? readStageManifest({ globalPiDir: overlayDir, readFile: (p) => readFileSync(p, "utf8"), fileExists }) : { packages: [] };
|
|
2109
|
+
if (entries !== null && manifest !== null) held = { overlay: entries.map((row) => row.name), packages: manifest.packages };
|
|
2110
|
+
} catch {
|
|
2111
|
+
held = null;
|
|
2112
|
+
}
|
|
2113
|
+
if (held !== null) {
|
|
2114
|
+
const lines = new Map();
|
|
2115
|
+
for (const { label, names, packages } of excludedNames) {
|
|
2116
|
+
if (!packages) continue;
|
|
2117
|
+
for (const name of extensionsNotHere(names, held)) lines.set(name, [...(lines.get(name) ?? []), label]);
|
|
2118
|
+
}
|
|
2119
|
+
for (const [name, labels] of lines) {
|
|
2120
|
+
checks.push({
|
|
2121
|
+
ok: false,
|
|
2122
|
+
warn: true,
|
|
2123
|
+
label: `run.excludeExtensions names ${JSON.stringify(name)}, and this host has no staged package or overlay extension of that name (${labels.join(", ")})`,
|
|
2124
|
+
fix: "nothing loads for it here, so a job is not refused: fix the name if it is a typo, since a misspelled name keeps nothing out on a host that has the real one",
|
|
1647
2125
|
});
|
|
1648
2126
|
}
|
|
1649
2127
|
}
|
|
@@ -1660,6 +2138,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1660
2138
|
checks.push({ ok: true, label: `${commands} command trigger(s): a command is only verifiable in-container -- the runner refuses an unregistered one before the prompt is sent (command-unregistered)` });
|
|
1661
2139
|
}
|
|
1662
2140
|
|
|
2141
|
+
section("venues");
|
|
1663
2142
|
// Issue #41: every DISTINCT image a trigger names in run.image, minus the deployment default already
|
|
1664
2143
|
// checked above. Two silent-failure modes, and both used to be impossible because there was one image.
|
|
1665
2144
|
// 1. the image was never built -- a job that refuses pre-spend at 03:00 in a log nobody is reading, and
|
|
@@ -1722,6 +2201,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1722
2201
|
// of the default image is what gates their podman half; its lines are still printed after the backend section, where
|
|
1723
2202
|
// they always were, so moving the read changed the order of spawns and not a byte of output.
|
|
1724
2203
|
const podman = podmanUsed ? await podmanChecks(env, seams, { jobImage, jobImageNote }) : null;
|
|
2204
|
+
section("images");
|
|
1725
2205
|
for (const img of images.filter((i) => i !== jobImage)) {
|
|
1726
2206
|
if (dockerRun !== null && dockerCode === 0 && dockerRouted.has(img)) {
|
|
1727
2207
|
const run = await runCmd(spawn, "docker", ["image", "inspect", img], runTimeouts.cmd);
|
|
@@ -1759,6 +2239,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1759
2239
|
}
|
|
1760
2240
|
}
|
|
1761
2241
|
|
|
2242
|
+
section("egress");
|
|
1762
2243
|
// REQ-EGRESS-ALLOWLIST (issue #202). [] when PI_EGRESS=0, so a deployment that declined it gets
|
|
1763
2244
|
// byte-identical output. Gated on docker and the image, because two of these checks run a container and
|
|
1764
2245
|
// the rest are noise on top of a down daemon.
|
|
@@ -1841,7 +2322,9 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1841
2322
|
: dockerCode === null
|
|
1842
2323
|
? { answered: false, reason: "docker-not-found", transient: true }
|
|
1843
2324
|
: null);
|
|
2325
|
+
section("backends");
|
|
1844
2326
|
checks.push(...backendChecks(env, { endpoint, daemon, fs: seams.observationFs, unit: jobUser.unit, ...(podman ? { podman: podman.observed } : {}) }));
|
|
2327
|
+
section("sizing");
|
|
1845
2328
|
// Issue #596: the default job size, and what this host's runtime says about the bounds a size becomes.
|
|
1846
2329
|
// Each venue this deployment runs, from that venue's own read: `undefined` leaves a venue out, null is "not read".
|
|
1847
2330
|
// Issue #596, phase 2: the host budget a worker started now would compute, from the same two reads, and what it means
|
|
@@ -1874,6 +2357,8 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1874
2357
|
const total = { memMiB: least(f.memTotalMiB, f.userMemMiB), cpuCenti: least(Number.isSafeInteger(f.hostCpus) ? f.hostCpus * 100 : null, f.userCpuCenti) };
|
|
1875
2358
|
checks.push(...sizeSuggestionChecks({ projects: sizingProjects, limits: budgetChecksArgs.limits, env, records: read.records, budget: { memMiB: budgetView.memMiB, cpuCenti: budgetView.cpuCenti }, total, nowMs }));
|
|
1876
2359
|
if (read.skipped > 0) checks.push({ ok: true, label: `size suggestions: ${read.skipped} run record${read.skipped === 1 ? "" : "s"} over ${SIZING_RECORD_MAX_BYTES / 1024} KiB skipped, not read` });
|
|
2360
|
+
// Issue #761: a record a newer pi-dispatch wrote is not folded into a suggestion (`readSizingRecords`), said so.
|
|
2361
|
+
if (read.newerSkipped > 0) checks.push({ ok: true, label: `size suggestions: ${read.newerSkipped} run record${read.newerSkipped === 1 ? "" : "s"} from a newer pi-dispatch not read` });
|
|
1877
2362
|
}
|
|
1878
2363
|
}
|
|
1879
2364
|
// Issue #596, phase 2: the aggregate CPU reserve per venue, read (never written) the way the worker reads it.
|
|
@@ -1889,8 +2374,29 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1889
2374
|
});
|
|
1890
2375
|
checks.push(...cpuReserveChecks(reserveReads, budgetView.cpuCenti));
|
|
1891
2376
|
}
|
|
2377
|
+
section("job-user");
|
|
1892
2378
|
checks.push(...jobUser.checks);
|
|
2379
|
+
section("podman");
|
|
1893
2380
|
if (podman) checks.push(...podman.checks);
|
|
2381
|
+
section("openshell");
|
|
2382
|
+
// Issue #683: the openshell venue's lines, asked by `--openshell` (before a deployment blesses it), or on every run
|
|
2383
|
+
// once PI_BACKENDS does (blessable since #684).
|
|
2384
|
+
// Issue #686 part 3: either OpenShell venue runs the section; copy mode's image line where `openshell` is blessed (or
|
|
2385
|
+
// asked for with neither blessed), bind mode's lines where `openshell-bind` is.
|
|
2386
|
+
const openshellVenues = venuesOf(env);
|
|
2387
|
+
// Issue #712: whether this run judged the openshell venue, for the published record.
|
|
2388
|
+
seams.openshellJudged = seams.openshell === true || Boolean(openshellVenues.openshellUsed);
|
|
2389
|
+
if (seams.openshell === true || openshellVenues.openshellUsed) {
|
|
2390
|
+
const { openshellChecks } = await import("./doctor-openshell.mjs");
|
|
2391
|
+
const modes = { copy: openshellVenues.openshellCopyUsed || !openshellVenues.openshellUsed, bind: openshellVenues.openshellBindUsed };
|
|
2392
|
+
// Bind mode's job-user line reads the same identity doctor's local and podman lines read.
|
|
2393
|
+
const ids = seams.jobUserIdentity ? { ids: { euid: seams.jobUserIdentity.euid ?? null, egid: seams.jobUserIdentity.egid ?? null } } : {};
|
|
2394
|
+
const osPlatform = seams.jobUserIdentity?.platform ?? seams.platform;
|
|
2395
|
+
// The probe's source under the jobs dir the service will use (review round 2 of #714).
|
|
2396
|
+
const probeJobsDir = Number.isInteger(seams.jobsDirUid) ? jobsDirPath(env, seams.jobsDirUid) : null;
|
|
2397
|
+
checks.push(...(await openshellChecks(env, { image: jobImage, modes, ...ids, jobsDir: probeJobsDir || null, ...(osPlatform ? { platform: osPlatform } : {}), ...(seams.openshellRun ? { run: seams.openshellRun } : {}), ...(seams.openshellMkdtemp ? { mkdtemp: seams.openshellMkdtemp, rmdir: () => {} } : {}) })));
|
|
2398
|
+
if (openshellVenues.openshellUsed) checks.push(...openshellUnitOrderChecks(seams, cwd));
|
|
2399
|
+
}
|
|
1894
2400
|
if (facts) facts.jobUser = jobUser.forLive;
|
|
1895
2401
|
// Issue #596, phase 2: the CPU budget the parent's quota is read back against by `--live`.
|
|
1896
2402
|
if (facts) facts.cpuBudgetCenti = budgetView.cpuCenti ?? null;
|
|
@@ -1898,6 +2404,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1898
2404
|
if (facts) facts.daemon = jobUser.daemon;
|
|
1899
2405
|
// Issue #354: which read-backs `--live` runs, and the podman venue's facts for its own.
|
|
1900
2406
|
if (facts) Object.assign(facts, { localUsed, podman: podman?.forLive ?? null });
|
|
2407
|
+
section("selinux");
|
|
1901
2408
|
// Issue #355: on a Podman host that confines containers with SELinux, the directories a job mounts that the worker
|
|
1902
2409
|
// did NOT make, which it therefore never relabels. Read from the same facts and endpoint as the line above. Issue #354:
|
|
1903
2410
|
// the podman venue relabels on the same rule (its own directories only), so the same lines, ONCE when both venues do.
|
|
@@ -1905,6 +2412,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
1905
2412
|
checks.push(...(await selinuxLabelChecks({ folders, overlay: env.PI_GLOBAL_PI_DIR || null, spawn: seams.spawn })));
|
|
1906
2413
|
}
|
|
1907
2414
|
|
|
2415
|
+
section("forges");
|
|
1908
2416
|
// The receiver itself, when the triggers file names ANY forge (issue #80). Only forge deliveries need
|
|
1909
2417
|
// the receiver at all, so a cron/local-only deployment gets no receiver noise here. WARNS rather than
|
|
1910
2418
|
// fails, same doctrine as the gitlab block below: a deployment can legitimately be mid-setup (or run
|
|
@@ -2005,6 +2513,27 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2005
2513
|
} else {
|
|
2006
2514
|
checks.push({ ok: true, label: `forgejo triggers configured (${urlShown(env.FORGEJO_URL)})` });
|
|
2007
2515
|
}
|
|
2516
|
+
// Measured on Forgejo 14.0.5: the author gate's permission lookup answers 403 unless the token's account is a site
|
|
2517
|
+
// admin, a repository admin, or the actor themselves. Which token asks is the receiver's choice (OQ-043):
|
|
2518
|
+
// FORGEJO_PERMISSION_TOKEN when set, a read:repository token of an admin account that no job ever holds; else
|
|
2519
|
+
// FORGEJO_TOKEN, every job's token, whose account must then be an admin. Doctor cannot ask Forgejo either way
|
|
2520
|
+
// (forgejo triggers name no repository, each delivery does), so it says which token is in use: a FACT LINE when the
|
|
2521
|
+
// dedicated one is set, with the requirement in the label (issue #462's form), and a WARNING on the fallback,
|
|
2522
|
+
// because the fallback is the configuration that puts an admin's token into every job.
|
|
2523
|
+
const permissionToken = typeof env.FORGEJO_PERMISSION_TOKEN === "string" && env.FORGEJO_PERMISSION_TOKEN.trim() !== "";
|
|
2524
|
+
if (permissionToken) {
|
|
2525
|
+
checks.push({
|
|
2526
|
+
ok: true,
|
|
2527
|
+
label: "forgejo author-gate lookups use FORGEJO_PERMISSION_TOKEN: its account must be an admin (or the owner) of every repository the triggers serve and the token needs read:repository only, while FORGEJO_TOKEN, which every job carries, stays a write collaborator's (docs/forgejo.md)",
|
|
2528
|
+
});
|
|
2529
|
+
} else {
|
|
2530
|
+
checks.push({
|
|
2531
|
+
ok: false,
|
|
2532
|
+
warn: true,
|
|
2533
|
+
label: "FORGEJO_PERMISSION_TOKEN is unset, so forgejo author-gate lookups use FORGEJO_TOKEN: Forgejo answers them only to a repository admin, so that account must be one, and then every forgejo job carries an admin's token that can change repository settings and branch protection (OQ-043)",
|
|
2534
|
+
fix: "keep FORGEJO_TOKEN's account a write collaborator, and set FORGEJO_PERMISSION_TOKEN to a read:repository token of an account that is an admin (or the owner) of the repositories; only the receiver reads it (docs/forgejo.md)",
|
|
2535
|
+
});
|
|
2536
|
+
}
|
|
2008
2537
|
}
|
|
2009
2538
|
|
|
2010
2539
|
// Issue #508: with the egress policy on, a job reaches a forge only over https on port 443.
|
|
@@ -2061,6 +2590,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2061
2590
|
}
|
|
2062
2591
|
}
|
|
2063
2592
|
|
|
2593
|
+
section("github");
|
|
2064
2594
|
// The default source `gh` mints job tokens from the operator's gh login, so the FULL-scope login token
|
|
2065
2595
|
// reaches every token-carrying job container — the opposite of the App path's per-repo short-lived
|
|
2066
2596
|
// tokens (CONST-TOKEN-SCOPED-PER-JOB). Both checks below warn, never fail: a local-only deployment with
|
|
@@ -2269,7 +2799,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2269
2799
|
// is podman's answer to the question docker's endpoint read answers below: the token must not ride to another
|
|
2270
2800
|
// machine. A deployment with `local` probes on docker exactly as before, and only there: one hand-off of the token per
|
|
2271
2801
|
// doctor run, not one per venue, and that deployment's output unchanged.
|
|
2272
|
-
const ghProbeBin = localUsed
|
|
2802
|
+
const ghProbeBin = seams.setupReadOnly ? null : localUsed
|
|
2273
2803
|
? dockerCode === 0 && imageCode === 0
|
|
2274
2804
|
? "docker"
|
|
2275
2805
|
: null
|
|
@@ -2358,6 +2888,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2358
2888
|
}
|
|
2359
2889
|
}
|
|
2360
2890
|
|
|
2891
|
+
section("valkey");
|
|
2361
2892
|
// Issue #464 (gate rounds 1 and 2): WHOSE Valkey that is, on the podman venue, by the one rule the worker applies at
|
|
2362
2893
|
// boot and `service install` and `up` apply (`judgeValkeyListeners`). Reachable is not enough: on a shared host a
|
|
2363
2894
|
// VALKEY_URL that reaches another account's Valkey (its port, its ::1 for a `localhost` URL, or 0.0.0.0) passed here,
|
|
@@ -2375,7 +2906,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2375
2906
|
})();
|
|
2376
2907
|
if (!unreadValkey && !valkeyDbProblem && seams.valkeyOwner) {
|
|
2377
2908
|
if (typeof env[VALKEY_SHARED_KEY] === "string") checks.push({ ok: false, warn: true, label: sharedShellIgnored(env[VALKEY_SHARED_KEY], join(cwd, ".env")), fix: "put PI_VALKEY_SHARED=1 in the deployment's .env if that Valkey is shared on purpose, and unset it in this shell" });
|
|
2378
|
-
ownerVerdict = await seams.valkeyOwner(valkeyUrl, { shared: valkeySharedOn(sharedInFile), user: userNameOf(seams), envPath: join(cwd, ".env"), rootOk: !(podmanUsed && !localUsed) });
|
|
2909
|
+
ownerVerdict = await seams.valkeyOwner(valkeyUrl, { shared: valkeySharedOn(sharedInFile), user: userNameOf(seams), envPath: join(cwd, ".env"), rootOk: !((podmanUsed && !localUsed) || openshellSetup.valkeyEngine === "podman") });
|
|
2379
2910
|
}
|
|
2380
2911
|
// A name that does not resolve here cannot be judged (gate round 3): said, and talked to no more than a refused one.
|
|
2381
2912
|
const valkeyUnresolved = ownerVerdict?.unresolved ? `VALKEY_URL's host ${ownerVerdict.unresolved} did not resolve here (${ownerVerdict.why}), so whose Valkey it reaches cannot be judged; the worker waits for it and retries` : null;
|
|
@@ -2409,7 +2940,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2409
2940
|
} else if (valkeyAuthVerdict && (valkeyAuthVerdict.state === "noauth" || valkeyAuthVerdict.state === "wrongpass")) {
|
|
2410
2941
|
// Issue #468: a Valkey that refuses this deployment's credential answers, so "not reachable" would be the wrong
|
|
2411
2942
|
// sentence: it is the password, named by its key and where it came from, never its value.
|
|
2412
|
-
checks.push({ ok: false, label: `Valkey (${urlShown(valkeyUrl)}) answers, and ${valkeyAuthVerdict.state === "noauth" ? "requires a password this deployment does not send" : `refuses the ${VALKEY_PASSWORD_KEY} this deployment sends`}: the worker refuses to start on it (exit 2)`, fix: valkeyAuthFix(valkeyAuthVerdict.state, {
|
|
2943
|
+
checks.push({ ok: false, label: `Valkey (${urlShown(valkeyUrl)}) answers, and ${valkeyAuthVerdict.state === "noauth" ? "requires a password this deployment does not send" : `refuses the ${VALKEY_PASSWORD_KEY} this deployment sends`}: the worker refuses to start on it (exit 2)`, fix: valkeyAuthFix(valkeyAuthVerdict.state, { ...valkeyVenues, envPath: join(cwd, ".env") }) });
|
|
2413
2944
|
} else checks.push({
|
|
2414
2945
|
ok: await probeValkey(valkeyTalkUrl),
|
|
2415
2946
|
label: `Valkey reachable (${urlShown(valkeyUrl)})${fromFileNote(fileSays("VALKEY_URL"))}`,
|
|
@@ -2418,9 +2949,11 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2418
2949
|
// the deployment's VALKEY_PASSWORD in the docker CLI's environment, and doctor hands no program anything from
|
|
2419
2950
|
// `.env`). It names the step: `pi-dispatch up`, which starts it with the password, labels its container and volume
|
|
2420
2951
|
// with this folder, and asks before it uses a volume it cannot attribute. The words hold on both venues.
|
|
2421
|
-
fix: localUsed
|
|
2952
|
+
fix: valkeyVenues.localUsed
|
|
2422
2953
|
? "run `pi-dispatch up` in the deployment folder: it starts Valkey with the deployment's VALKEY_PASSWORD (or, in a folder /dispatch setup handed to compose, compose's own: `docker compose -p <folder name> --env-file .env -f deploy/docker-compose.yml -f deploy/docker-compose.valkey.yml up -d valkey`)"
|
|
2423
|
-
:
|
|
2954
|
+
: openshellSetup.valkeyEngine !== "podman"
|
|
2955
|
+
? "run `pi-dispatch up` (or `pi-dispatch service install`) as this account: on the podman venue both start Valkey as a Quadlet unit under its Podman (docs/podman.md, setup step 6); a Valkey you run yourself, a distribution package say, works too"
|
|
2956
|
+
: "run `pi-dispatch up` (or `pi-dispatch service install`) as this account: with PI_OPENSHELL_ENGINE=podman both start Valkey as a Quadlet unit under its Podman (the openshell venue's engine; no egress proxy is started for it); a Valkey you run yourself, a distribution package say, works too",
|
|
2424
2957
|
});
|
|
2425
2958
|
|
|
2426
2959
|
// Round 3 of PR #475's review: PI_VALKEY_PORT is the port compose publishes its Valkey on, and VALKEY_URL's is the one
|
|
@@ -2443,11 +2976,12 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2443
2976
|
if (ownerVerdict?.heldBy) {
|
|
2444
2977
|
checks.push({ ok: true, label: `Valkey (${urlShown(valkeyUrl)}) answers from a listener of ${ownerVerdict.heldBy}${ownerVerdict.chosen ? `; the worker connects to ${ownerVerdict.chosen} only` : ""}` });
|
|
2445
2978
|
}
|
|
2446
|
-
checks.push(...(await valkeyPasswordChecks({ verdict: valkeyAuthVerdict, context: valkeyContext, seams, valkeyUrl, valkeyTalkUrl,
|
|
2979
|
+
checks.push(...(await valkeyPasswordChecks({ verdict: valkeyAuthVerdict, context: valkeyContext, seams, valkeyUrl, valkeyTalkUrl, ...valkeyVenues, platform, statSeam, fileValue })));
|
|
2447
2980
|
for (const other of ownerVerdict?.elsewhere ?? []) {
|
|
2448
2981
|
checks.push({ ok: false, warn: true, label: `another account also listens on an address VALKEY_URL's host resolves to: ${other}. The worker connects only to the address judged this account's, so it never reaches that one`, fix: "nothing to change for this deployment; a client that resolves the name itself (`pi-dispatch run` from a shell) may still reach it, so prefer VALKEY_URL=redis://127.0.0.1:<port>" });
|
|
2449
2982
|
}
|
|
2450
2983
|
|
|
2984
|
+
section("fleet");
|
|
2451
2985
|
// --- the fleet (issue #57) -------------------------------------------------------------------------
|
|
2452
2986
|
//
|
|
2453
2987
|
// Every line here is gated on a peer actually existing, so a single-host deployment's output is
|
|
@@ -2463,7 +2997,12 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2463
2997
|
// Valkey doctor reads, a host row is another party's text, and a control byte in a name or zone must not reach the
|
|
2464
2998
|
// terminal. The registry's own charset already refuses them at the source; this is the reader not relying on it.
|
|
2465
2999
|
const peers = (fleet.hosts ?? []).map((h) => ({ ...h, name: printable(h.name), tz: h.tz ? printable(h.tz) : h.tz })).filter((h) => h.name !== workerNameOf(declaredWorkerName));
|
|
2466
|
-
if (!env.PI_WORKER_NAME && peers.length > 0)
|
|
3000
|
+
if (!env.PI_WORKER_NAME && peers.length > 0) {
|
|
3001
|
+
// The budget lines are re-said for a fleet IN PLACE, so they keep the section they were first said in (issue #712).
|
|
3002
|
+
const again = hostBudgetChecks(budgetView, { ...budgetChecksArgs, peers: true });
|
|
3003
|
+
tagSection(again, "sizing");
|
|
3004
|
+
checks.splice(budgetChecksAt, budgetChecksHere.length, ...again);
|
|
3005
|
+
}
|
|
2467
3006
|
// Issue #596, phase 2: this host's own row, when its worker runs and publishes one. Its host budget ledger is held
|
|
2468
3007
|
// against the size labels of the job containers each venue's runtime lists, read only when the row carries the ledger.
|
|
2469
3008
|
const selfRow = (fleet.hosts ?? []).find((h) => printable(h.name) === workerNameOf(declaredWorkerName)) ?? null;
|
|
@@ -2494,6 +3033,17 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2494
3033
|
// no envelope that refuses every job is told why. `{ digest }`, `{ undecodable: true }` for a key that exists and does
|
|
2495
3034
|
// not decode (the worker's EXISTS still counts it as governed), or null (no split, or no answer: nothing is said).
|
|
2496
3035
|
const valkeyUsable = !(unreadValkey || valkeyRefused || valkeyDbProblem || valkeyAuthVerdict?.state === "dbrange");
|
|
3036
|
+
// Issue #712: where this run's results are published, by the rule every read above obeys: the Valkey doctor may talk
|
|
3037
|
+
// to, at the address the worker pins, or none. Named for this host as the worker names it, so a record and its
|
|
3038
|
+
// registry row share a name.
|
|
3039
|
+
// A declared name the worker refuses at boot (review round 1) is never a key: the worker's own rule, not a copy.
|
|
3040
|
+
let publishName = null;
|
|
3041
|
+
try {
|
|
3042
|
+
publishName = workerNameFrom({ PI_WORKER_NAME: declaredWorkerName });
|
|
3043
|
+
} catch {
|
|
3044
|
+
publishName = null;
|
|
3045
|
+
}
|
|
3046
|
+
seams.publishTo = !valkeyUsable ? null : publishName === null ? { refused: "PI_WORKER_NAME is a name the worker refuses at boot" } : { url: valkeyTalkUrl, name: publishName, every: doctorIntervalHours(env) };
|
|
2497
3047
|
const appliedSplit = valkeyUsable ? await (seams.readAppliedSplit ?? defaultReadAppliedSplit)(valkeyTalkUrl) : null;
|
|
2498
3048
|
// Read only when there is a peer to compare against. Every line below is gated on a peer existing, and
|
|
2499
3049
|
// the SUBPROCESS has to be too: otherwise every `doctor` run on every single-host deployment spawns an
|
|
@@ -2515,6 +3065,17 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2515
3065
|
// The one thing that is silently WRONG rather than merely undeclared. Without a declared name this
|
|
2516
3066
|
// host enqueues its own folder work to the SHARED queue, where a peer that has no such folder can
|
|
2517
3067
|
// pop it -- so the routing that makes a fleet safe is simply off, and nothing else says so.
|
|
3068
|
+
// Issue #645: a durable run's checkpoint lives on ONE host, and its recovery is routed there through that host's own
|
|
3069
|
+
// queue. Without a declared name the recovery goes to the SHARED queue, where a peer that holds no checkpoint can pop
|
|
3070
|
+
// it and refuse it (`recovery-checkpoint-absent`), so every crash on this host ends the run instead of continuing it.
|
|
3071
|
+
if (!declaredWorkerName && recovering > 0) {
|
|
3072
|
+
checks.push({
|
|
3073
|
+
ok: false,
|
|
3074
|
+
warn: true,
|
|
3075
|
+
label: `${recovering} trigger(s) arm run.recover, but this worker has peers and no PI_WORKER_NAME, so a crashed run's recovery is queued where a host without its checkpoint can take it`,
|
|
3076
|
+
fix: "set PI_WORKER_NAME in this host's .env and restart: a recovery is routed to the host holding the run's checkpoint through that host's own queue, which only a named worker drains",
|
|
3077
|
+
});
|
|
3078
|
+
}
|
|
2518
3079
|
if (!declaredWorkerName) {
|
|
2519
3080
|
checks.push({
|
|
2520
3081
|
ok: false,
|
|
@@ -2589,6 +3150,10 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2589
3150
|
// no edit to the file applies on it. Judged on the DECLARED version, or the derived one if that is higher (gate
|
|
2590
3151
|
// round 2). SKIPPED when this host's file does not load.
|
|
2591
3152
|
if (scopedLimitFacts.parseError === null) checks.push(...fleetSizeChecks(Math.max(Number(scopedLimitFacts.declaredVersion) || 0, scopedLimitsVersionFor(scopedLimitFacts.limits)), peers));
|
|
3153
|
+
// Issue #761: the version each of this host's own stores needs (the higher of what the file declares and what its
|
|
3154
|
+
// content needs, the size check's rule), by registry field, against what every peer publishes. A store whose file
|
|
3155
|
+
// does not load here is left out: its own line already fails, and comparing would name the wrong hosts.
|
|
3156
|
+
checks.push(...fleetStoreVersionChecks(storeVersionNeeds({ settingsFile: settingsFilePath(env, home), fileExists, pauseWindowFacts, triggersVersionNeed, jobDataVersionNeed, triggersParsed: parseError === null, settingsVersions: versionTables.settings }), peers));
|
|
2592
3157
|
// Issue #596, phase 2: each host's budget and use, and which hosts each project's size fits on, from every row that
|
|
2593
3158
|
// publishes a budget (this host's own included). SKIPPED when this host's scoped-limits file does not load.
|
|
2594
3159
|
if (scopedLimitFacts.parseError === null) checks.push(...fleetBudgetChecks((fleet.hosts ?? []).map((h) => ({ ...h, name: printable(h.name) })), { limits: scopedLimitFacts.limits, env }));
|
|
@@ -2601,6 +3166,16 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2601
3166
|
// Said, rather than silently absent: "no peers" and "could not ask" are different facts.
|
|
2602
3167
|
checks.push({ ok: true, label: `Fleet: could not read the host registry (${printable(fleet.unreachable)})` });
|
|
2603
3168
|
}
|
|
3169
|
+
// Issue #761: the pi-dispatch versions the live workers report, and this doctor's own against this host's running
|
|
3170
|
+
// worker, after the peers block so the second is said on a single host too. Workers only: a receiver's status row is
|
|
3171
|
+
// never a doctor input (INT-RECEIVER-STATUS-KEY).
|
|
3172
|
+
checks.push(...fleetVersionChecks({ installed: PACKAGE_VERSION, self: selfRow, selfName: workerNameOf(declaredWorkerName), peers }));
|
|
3173
|
+
// Issue #761: a dollar window counter below what this host's own records settled into it (`doctorDollarWindows`).
|
|
3174
|
+
// Re-opened here, where whether this command may talk to the Valkey is known, from the same Valkey at the address the
|
|
3175
|
+
// worker pins.
|
|
3176
|
+
section("dollars");
|
|
3177
|
+
checks.push(...(await doctorDollarWindows({ seams, env, home, url: valkeyUsable ? valkeyTalkUrl : null, localHost: workerNameOf(declaredWorkerName), fileExists })));
|
|
3178
|
+
section("capacity");
|
|
2604
3179
|
// Issue #599, phase 2: how busy each host was over the last 7 days, one fact line per host, never a warning (nothing
|
|
2605
3180
|
// decides on it). On every deployment, a single host too: its own files are read when the run mirror is not. Read from
|
|
2606
3181
|
// the same Valkey the fleet was, with the registry rows just read (their running jobs count up to now), every Valkey
|
|
@@ -2614,6 +3189,8 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2614
3189
|
if (envelopeHere.parseError === null) checks.push(...appliedSplitChecks(appliedSplit, envelopeHere.envelope ? envelopeDigest(envelopeHere.envelope) : NO_ENVELOPE_FINGERPRINT, workerNameOf(declaredWorkerName), peers));
|
|
2615
3190
|
}
|
|
2616
3191
|
|
|
3192
|
+
section("provider");
|
|
3193
|
+
const subscriptionCredentials = await doctorSubscriptionCredentials(env, { cwd, home, now: seams.wallClock ?? Date.now, configure: seams.subscriptionFactory });
|
|
2617
3194
|
// Which variable holds a provider's key is PI'S fact, asked of pi rather than copied (issue #286).
|
|
2618
3195
|
// The copy this replaced had anthropic's two variables in the WRONG precedence order, invented
|
|
2619
3196
|
// GOOGLE_API_KEY, and invented a `gemini` provider pi has never had -- three ways for doctor to bless
|
|
@@ -2667,7 +3244,8 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2667
3244
|
hintUnread = true;
|
|
2668
3245
|
}
|
|
2669
3246
|
}
|
|
2670
|
-
const
|
|
3247
|
+
const subscriptionDefault = await subscriptionCredentials.check(provider);
|
|
3248
|
+
const keyCheck = subscriptionDefault.managed ? subscriptionCredentialCheck(provider, subscriptionDefault) : providerKeyCheck({ provider, env: { ...env, ...keyFromFile }, agentDir: keyAgentDir, oracle, nodeOk: checks[0]?.ok, keyless: { endpoints: keylessEndpoints, models: keylessModels, unreadable: keylessUnreadable, hintModels, hintUnread } });
|
|
2671
3249
|
const keyNamed = Object.keys(keyFromFile).filter((name) => keyCheck.label?.includes(`: ${name})`));
|
|
2672
3250
|
// Issue #481 (PR #485 review round 2): a key found nowhere doctor can look is one an env-setup script may export, which
|
|
2673
3251
|
// is the documented home for a provider key fetched from a secrets manager (docs/secrets.md).
|
|
@@ -2678,9 +3256,12 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2678
3256
|
: setupSupplies(["worker"])
|
|
2679
3257
|
? credentialAbsent({ subject: `Provider key (${provider}: ${keyAbsent.join(" or ")}) not set`, names: keyNames, readers: ["worker"], refusal: "jobs cannot authenticate to the provider", fix: keyShown.fix })
|
|
2680
3258
|
: { ...keyShown, fix: `${keyShown.fix}${notSupplied(keyNames, ["worker"])}` };
|
|
2681
|
-
|
|
3259
|
+
// Not for a malformed PI_PROVIDER: its shape row below is the one line (issue #673, review round 1).
|
|
3260
|
+
// Issue #737: the console reads whether the key is found by this id (`NAMED_CHECK_IDS`).
|
|
3261
|
+
if (envShapeProblem.PI_PROVIDER === null) checks.push({ ...keyLine, id: "provider.key", label: `${keyLine.label}${fromFileNote([...fileSays("PI_PROVIDER"), ...keyNamed, ...(keyAgentDir !== agentDir ? ["PI_CODING_AGENT_DIR"] : [])])}` });
|
|
2682
3262
|
|
|
2683
3263
|
|
|
3264
|
+
section("overlay");
|
|
2684
3265
|
// REQ-GLOBAL-PI-OVERLAY: read the extensions opt-out through the WORKER's own parser, so doctor reports
|
|
2685
3266
|
// the exact posture the worker will boot with and refuses the exact values it refuses. Checked with or
|
|
2686
3267
|
// without an overlay configured, because a malformed knob stops boot either way -- and a `false` an
|
|
@@ -2893,7 +3474,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2893
3474
|
// A model a capped job may use is named by the cost-cap line below (`costCapFitChecks`), which runs on the
|
|
2894
3475
|
// same catalog seam, so this line names only the rest: one line per model.
|
|
2895
3476
|
const capEndpoints = (seams.declaredEndpoints ?? ((a) => declaredEndpointsIn({ ...a, fs: keylessIo })))({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform, valkeyUrl: keylessValkey });
|
|
2896
|
-
const capped = usageCatalog ? cappedModelNames(env, { runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists) }) : new Set();
|
|
3477
|
+
const capped = usageCatalog ? cappedModelNames(env, { runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists), modelRoles: triggerModelRoles, projects: modelRoleProjects }) : new Set();
|
|
2897
3478
|
const uncapped = ignoredOutputCapModels({ models: overlayModels, endpoints: capEndpoints, builtinModel: usageCatalog?.builtinModel, builtinChatModels: usageCatalog?.builtinChatModels }).filter((m) => !capped.has(`${m.provider}/${m.modelId}`));
|
|
2898
3479
|
if (uncapped.length > 0) {
|
|
2899
3480
|
const SHOWN = 5;
|
|
@@ -2927,7 +3508,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2927
3508
|
}
|
|
2928
3509
|
|
|
2929
3510
|
// Staged pi packages (REQ-GLOBAL-PI-OVERLAY): pinned third-party code the operator staged with
|
|
2930
|
-
// `import-pi --with-packages`, loaded by every job whose trigger did not set `run.packages: false`.
|
|
3511
|
+
// `import-pi --with-packages`, loaded by every job whose trigger did not set `run.packages: false` or name it in `run.excludeExtensions`.
|
|
2931
3512
|
// Keyed on the dir the same way the extensions pair above is, so a deployment that stages none
|
|
2932
3513
|
// prints nothing here.
|
|
2933
3514
|
const packagesDir = join(overlay, PACKAGES_SUBDIR);
|
|
@@ -2983,7 +3564,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2983
3564
|
});
|
|
2984
3565
|
}
|
|
2985
3566
|
// There is no dormant state left to report: a staged, manifested package loads into every
|
|
2986
|
-
// job whose trigger did not opt out -- INCLUDING jobs no trigger file describes at all
|
|
3567
|
+
// job whose trigger did not opt out or leave it out by name -- INCLUDING jobs no trigger file describes at all
|
|
2987
3568
|
// (a matched webhook, `dispatch_run`, the CLI). So "staged" IS "loading", and the honest
|
|
2988
3569
|
// line says so and names how much of the trigger file withholds it. Warn, never fail: this
|
|
2989
3570
|
// is the intended posture, and it is stated so a forgotten stage cannot read as inert.
|
|
@@ -2992,8 +3573,9 @@ export async function collectChecks(shellVars, seams) {
|
|
|
2992
3573
|
checks.push({
|
|
2993
3574
|
ok: false,
|
|
2994
3575
|
warn: true,
|
|
2995
|
-
|
|
2996
|
-
|
|
3576
|
+
// Counted against this host's manifest: a name in the list may be an overlay extension, which withholds no package, and a trigger with packages off is counted above.
|
|
3577
|
+
label: `Staged packages LOAD in every job (${optingOut} trigger(s) opt out with run.packages: false, ${(excludedNames ?? []).filter((listed) => listed.packages && listed.names.some((name) => manifest.packages.some((staged) => staged.name === name))).length} leave one or more out with run.excludeExtensions)`,
|
|
3578
|
+
fix: "they run third-party code against adversarial input with open egress -- vet each, keep every version exactly pinned, and on a trigger that must not load them set run.packages: false, or name one in run.excludeExtensions",
|
|
2997
3579
|
});
|
|
2998
3580
|
|
|
2999
3581
|
// A staged package whose --ignore-scripts build never ran (issue #102, comment 1). The
|
|
@@ -3096,6 +3678,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3096
3678
|
});
|
|
3097
3679
|
}
|
|
3098
3680
|
|
|
3681
|
+
section("waits");
|
|
3099
3682
|
// Issue #230, `run.waitFor`. The PARSE is asked unconditionally, and the rest only when something waits.
|
|
3100
3683
|
// That split is not the usual "only report what this deployment uses": `loadConfig` calls
|
|
3101
3684
|
// `parseWaitProfiles` on every boot whether or not a trigger holds anything, so a garbled variable is a
|
|
@@ -3167,6 +3750,18 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3167
3750
|
});
|
|
3168
3751
|
}
|
|
3169
3752
|
|
|
3753
|
+
section("models");
|
|
3754
|
+
// Issue #673: PI_MODEL and PI_PROVIDER are held to the one model-id and provider-id shape a trigger's run.model and
|
|
3755
|
+
// run.provider are (`modelFrom`, `providerFrom`), and a value outside it is a worker that will not boot. Judged by the
|
|
3756
|
+
// worker's own functions (`envShapeProblem`, above), the PI_ALLOWED_MODELS row's rule below, on the SERVICE's value.
|
|
3757
|
+
// Only a failing row: a good value is judged further by the provider-key and model-catalog lines, which a malformed one
|
|
3758
|
+
// skips. The value is never echoed, as the worker's own refusal never echoes it; the setting's name is all an
|
|
3759
|
+
// operator needs to find the line.
|
|
3760
|
+
for (const name of ["PI_MODEL", "PI_PROVIDER"]) {
|
|
3761
|
+
if (envShapeProblem[name] === null) continue;
|
|
3762
|
+
checks.push({ ok: false, label: `${name} is not a valid ${name === "PI_MODEL" ? "model" : "provider"} id`, fix: `${envShapeProblem[name]} -- the worker refuses to boot until this is fixed` });
|
|
3763
|
+
}
|
|
3764
|
+
|
|
3170
3765
|
// Issue #502, allowed-model lists. The deployment list is judged unconditionally, `PI_WAIT_PROFILES`' rule: a
|
|
3171
3766
|
// malformed or spaced value is a worker that will not boot. Its grammar is the worker's own (`allowedModelsFrom`).
|
|
3172
3767
|
// The value doctor reads is the SERVICE's (`WORKER_SERVICE_KEYS`), so a spaced value in a `.env` a shell sources is
|
|
@@ -3208,6 +3803,10 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3208
3803
|
// pi-ai does not load: the provider key line below already fails on that. A triggers file that did not load
|
|
3209
3804
|
// contributes no trigger (its own line says so), and the deployment's default is still judged for its cap.
|
|
3210
3805
|
{
|
|
3806
|
+
let roleEnvList = null;
|
|
3807
|
+
try { roleEnvList = allowedModelsFrom(env); } catch {}
|
|
3808
|
+
// Role refusals are determinate without pi; a missing catalog must not hide invalid reviewed policy.
|
|
3809
|
+
checks.push(...resolveDoctorModelRuns({ runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists), envList: roleEnvList, modelRoles: triggerModelRoles, projects: modelRoleProjects }).checks);
|
|
3211
3810
|
const catalog = await (seams.modelCatalog ?? defaultModelCatalog)();
|
|
3212
3811
|
if (catalog) {
|
|
3213
3812
|
const overlayDir = typeof env.PI_GLOBAL_PI_DIR === "string" && env.PI_GLOBAL_PI_DIR !== "" ? resolve(cwd, env.PI_GLOBAL_PI_DIR) : null;
|
|
@@ -3224,37 +3823,110 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3224
3823
|
} catch {
|
|
3225
3824
|
// A malformed PI_ALLOWED_MODELS is its own line above (the worker refuses to boot on it).
|
|
3226
3825
|
}
|
|
3227
|
-
const subjects = modelSubjects({ runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists), envList });
|
|
3228
|
-
|
|
3229
|
-
|
|
3826
|
+
const subjects = modelSubjects({ runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists), envList, modelRoles: triggerModelRoles, projects: modelRoleProjects });
|
|
3827
|
+
const subscriptionChecks = new Map();
|
|
3828
|
+
for (const subjectProvider of new Set(subjects.flatMap((subject) => [subject.main.provider, ...subject.list.map((ref) => ref.provider)]))) {
|
|
3829
|
+
const result = await subscriptionCredentials.check(subjectProvider);
|
|
3830
|
+
subscriptionChecks.set(subjectProvider, result);
|
|
3831
|
+
if (subjectProvider !== provider && result.managed) checks.push(subscriptionCredentialCheck(subjectProvider, result));
|
|
3832
|
+
}
|
|
3833
|
+
|
|
3834
|
+
// The deployment default's catalog row prints PI_PROVIDER/PI_MODEL: not for a malformed one, whose shape row is
|
|
3835
|
+
// the one line (issue #673, review round 1).
|
|
3836
|
+
const envShapeBad = envShapeProblem.PI_MODEL !== null || envShapeProblem.PI_PROVIDER !== null;
|
|
3837
|
+
checks.push(...unknownModelChecks(envShapeBad ? subjects.filter((s) => s.deployment !== true) : subjects, catalog.checkModelsKnown, readOverlayDoc));
|
|
3838
|
+
checks.push(...costCapFitChecks(subjects, (ref) => boundModelOf(ref, { builtinModel: catalog.builtinModel, overlay: overlayDoc }), { unboundable: (ref) => outputUnboundable(outputCapView({ models: overlayDoc, provider: ref.provider, modelId: ref.id, builtinModel: catalog.builtinModel, builtinChatModels: catalog.builtinChatModels })), apiOf: (ref) => outputCapView({ models: overlayDoc, provider: ref.provider, modelId: ref.id, builtinModel: catalog.builtinModel, builtinChatModels: catalog.builtinChatModels })?.api }));
|
|
3230
3839
|
checks.push(
|
|
3231
3840
|
...listedProviderCredentialChecks(subjects, {
|
|
3232
3841
|
candidatesOf: (name) => (typeof oracle?.providerKeyCandidates === "function" ? oracle.providerKeyCandidates(name) : []),
|
|
3233
3842
|
forwarded: (env.PI_FORWARD_ENV ?? "").split(",").map((name) => name.trim()).filter((name) => name !== ""),
|
|
3234
3843
|
env,
|
|
3235
3844
|
overlay: overlayDoc,
|
|
3845
|
+
subscriptionChecks,
|
|
3236
3846
|
}),
|
|
3237
3847
|
);
|
|
3238
3848
|
}
|
|
3239
3849
|
}
|
|
3240
3850
|
|
|
3851
|
+
// Read only configured model identities. A keyspace scan would expose retired configuration and make a
|
|
3852
|
+
// doctor run's cost depend on everything else sharing this Valkey.
|
|
3853
|
+
{
|
|
3854
|
+
let envList = null;
|
|
3855
|
+
try { envList = allowedModelsFrom(env); } catch {}
|
|
3856
|
+
const refs = cooldownModelRefs(parseError ? [] : modelRuns, deploymentSettingsOf(env, settingsFilePath(env, home), fileExists), { modelRoles: triggerModelRoles, projects: modelRoleProjects, envList });
|
|
3857
|
+
if (refs.length > 0) {
|
|
3858
|
+
const state = valkeyUsable ? await (seams.readModelCooldowns ?? defaultReadModelCooldowns)(valkeyTalkUrl, refs, { now: seams.wallClock ?? Date.now }) : null;
|
|
3859
|
+
checks.push(...modelCooldownChecks(state));
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
|
|
3863
|
+
checks.push(...await doctorModelProposalChecks(env, { cwd, credentials: service.extra(["ANTHROPIC_API_KEY", "OPENAI_API_KEY"]).env, now: seams.wallClock ?? Date.now, ...(seams.modelProposals ?? {}) }));
|
|
3864
|
+
section("secrets");
|
|
3241
3865
|
// REQ-TRIGGER-SECRETS. Only reported when a trigger actually binds one, on the run.resume block's
|
|
3242
3866
|
// reasoning below: a deployment that uses no secrets should not be told about a variable it has no
|
|
3243
3867
|
// reason to set.
|
|
3244
3868
|
if (secreting > 0) {
|
|
3245
|
-
const { profiles:
|
|
3246
|
-
const
|
|
3869
|
+
const { profiles: envDeclared, error: parseFailure } = parseSecretProfilesSafe(env.PI_SECRET_PROFILES);
|
|
3870
|
+
const roots = delimitedList(env.PI_SECRET_RESOLVER_ROOTS);
|
|
3247
3871
|
if (parseFailure) {
|
|
3248
3872
|
checks.push({ ok: false, label: "PI_SECRET_PROFILES does not parse", fix: `${parseFailure} -- the worker refuses to boot until this is fixed, rather than dropping the entry and leaving you a profile you believe is wired` });
|
|
3249
3873
|
} else {
|
|
3874
|
+
// Issue #704: the worker resolves a profile from PI_SECRET_PROFILES AND the settings overlay's
|
|
3875
|
+
// `secretProfiles`, so a profile declared in the panel is declared, and reading env alone told an
|
|
3876
|
+
// operator it was missing. The worker's rules, reused rather than restated: a name in both sources is
|
|
3877
|
+
// ambiguous (`mergeSecretProfiles`, and the worker then refuses every secrets job), and an overlay
|
|
3878
|
+
// profile counts only when its resolver is an executable file whose realpath sits inside
|
|
3879
|
+
// PI_SECRET_RESOLVER_ROOTS (`servableSecretProfiles`, the lookup the resolver makes). An invalid
|
|
3880
|
+
// overlay contributes nothing here; its own check below already fails doctor.
|
|
3881
|
+
//
|
|
3882
|
+
// The probe is the resolver's own (`secrets.mjs -> realExecutable`), through a dynamic import for the
|
|
3883
|
+
// reason `providerOracle` gives: that module loads pi through `env-allowlist.mjs`, and a static import
|
|
3884
|
+
// would make doctor itself fail to start when pi does not load. Without it only the usability probe is
|
|
3885
|
+
// skipped: the ambiguity check needs no pi and still runs on the raw overlay, and an overlay profile then
|
|
3886
|
+
// counts as declared as written (the pi failure is already its own failing line).
|
|
3887
|
+
//
|
|
3888
|
+
// OWN properties only, in every lookup below: the profile charset admits `toString` and `constructor`.
|
|
3889
|
+
const settingsRead = fileExists(settingsFilePath(env, home)) ? readOverlay(settingsFilePath(env, home)) : { overlay: {} };
|
|
3890
|
+
let realExecutablePath = null;
|
|
3891
|
+
try {
|
|
3892
|
+
({ realExecutable: realExecutablePath } = await import("./secrets.mjs"));
|
|
3893
|
+
} catch {
|
|
3894
|
+
realExecutablePath = null;
|
|
3895
|
+
}
|
|
3896
|
+
const overlayDeclared = settingsRead.invalid ? {} : (settingsRead.overlay?.secretProfiles ?? {});
|
|
3897
|
+
const ambiguousNames = Object.keys(overlayDeclared).filter((name) => Object.hasOwn(envDeclared, name)).sort();
|
|
3898
|
+
const unambiguous = Object.fromEntries(Object.entries(overlayDeclared).filter(([name]) => !Object.hasOwn(envDeclared, name)));
|
|
3899
|
+
const overlayUsable = realExecutablePath === null ? unambiguous : servableSecretProfiles({ envProfiles: {}, overlayProfiles: unambiguous, roots, realExecutablePath }).profiles;
|
|
3900
|
+
const declared = { ...envDeclared, ...overlayUsable };
|
|
3901
|
+
const names = Object.keys(declared).sort();
|
|
3902
|
+
if (mergeSecretProfiles(envDeclared, overlayDeclared).ambiguous) {
|
|
3903
|
+
checks.push({
|
|
3904
|
+
ok: false,
|
|
3905
|
+
label: `${ambiguousNames.length} secret profile name(s) are declared in both PI_SECRET_PROFILES and the settings overlay: ${ambiguousNames.join(", ")}`,
|
|
3906
|
+
fix: "remove each name from one of the two (neither source wins a collision) -- until then the worker refuses every job that binds secrets as secret-profile-ambiguous, whatever profile it names",
|
|
3907
|
+
});
|
|
3908
|
+
}
|
|
3909
|
+
// Named by a trigger, declared in the overlay, and not usable: said apart from "not declared",
|
|
3910
|
+
// because the fix is the resolver or the roots, not a new declaration.
|
|
3911
|
+
const unusable = secretProfiles.filter((name) => Object.hasOwn(unambiguous, name) && !Object.hasOwn(overlayUsable, name));
|
|
3912
|
+
if (unusable.length > 0) {
|
|
3913
|
+
checks.push({
|
|
3914
|
+
ok: false,
|
|
3915
|
+
label: `${unusable.length} named secret profile(s) are declared in the settings overlay but not usable: ${unusable.join(", ")}`,
|
|
3916
|
+
fix:
|
|
3917
|
+
roots.length === 0
|
|
3918
|
+
? "PI_SECRET_RESOLVER_ROOTS is unset, so the worker uses no overlay-declared resolver: set it to the directory the resolver lives in, or declare the profile in PI_SECRET_PROFILES -- these jobs refuse pre-spend as secret-profile-unknown until you do"
|
|
3919
|
+
: "the resolver must be an executable file whose real path sits inside PI_SECRET_RESOLVER_ROOTS -- these jobs refuse pre-spend as secret-profile-unknown until it is",
|
|
3920
|
+
});
|
|
3921
|
+
}
|
|
3250
3922
|
// A HARD FAIL, not a warning, and worded like the run.resume/PI_SESSIONS_DIR check below for the
|
|
3251
3923
|
// same reason: these jobs refuse pre-spend until it is set, deliberately, rather than running
|
|
3252
3924
|
// without their secrets and looking like they worked.
|
|
3253
|
-
const missing = secretProfiles.filter((name) => !(name
|
|
3925
|
+
const missing = secretProfiles.filter((name) => !Object.hasOwn(declared, name) && !unusable.includes(name) && !ambiguousNames.includes(name));
|
|
3254
3926
|
checks.push({
|
|
3255
3927
|
ok: missing.length === 0,
|
|
3256
3928
|
label: missing.length === 0 ? `${secreting} trigger(s) bind secrets, and every profile they name is declared` : `${secreting} trigger(s) bind secrets, but ${missing.length} named profile(s) are not declared: ${missing.join(", ")}`,
|
|
3257
|
-
fix: `declare them in PI_SECRET_PROFILES as name:/absolute/path pairs (a resolver is one line, e.g. \`exec op read --no-newline "$1"\`) -- these jobs refuse pre-spend until you do`,
|
|
3929
|
+
fix: `declare them in PI_SECRET_PROFILES as name:/absolute/path pairs (a resolver is one line, e.g. \`exec op read --no-newline "$1"\`), or through the panel with PI_SECRET_RESOLVER_ROOTS set -- these jobs refuse pre-spend until you do`,
|
|
3258
3930
|
});
|
|
3259
3931
|
// The declared table, so an operator sees what is wired without reading .env. NAMES and paths only:
|
|
3260
3932
|
// this is doctor's own stdout on the operator's host, not a public issue comment.
|
|
@@ -3265,7 +3937,6 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3265
3937
|
// when closed and a disclosure when open. Issue #462: the disclosure is a FACT LINE too, since the operator opened
|
|
3266
3938
|
// it on purpose and nothing but closing it again clears it, so its advice lives in the label (an `ok: true` check
|
|
3267
3939
|
// never prints a fix line).
|
|
3268
|
-
const roots = (env.PI_SECRET_RESOLVER_ROOTS ?? "").split(delimiter).map((r) => r.trim()).filter(Boolean);
|
|
3269
3940
|
checks.push({
|
|
3270
3941
|
ok: true,
|
|
3271
3942
|
...(roots.length === 0
|
|
@@ -3290,7 +3961,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3290
3961
|
// one would be permanently false and this check would silently never run. It is skipped rather than
|
|
3291
3962
|
// failed when pi did not load, because providerKeyCheck above has already reported that as its own ✗
|
|
3292
3963
|
// and one root cause should print one line.
|
|
3293
|
-
if (secretNames.length > 0 && oracle?.providerKeyCandidates) {
|
|
3964
|
+
if (secretNames.length > 0 && oracle?.providerKeyCandidates && envShapeProblem.PI_PROVIDER === null) {
|
|
3294
3965
|
const candidates = oracle.providerKeyCandidates(provider);
|
|
3295
3966
|
const clashing = secretNames.filter((n) => candidates.includes(n));
|
|
3296
3967
|
checks.push({
|
|
@@ -3312,6 +3983,60 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3312
3983
|
}
|
|
3313
3984
|
}
|
|
3314
3985
|
|
|
3986
|
+
// Issue #704: the project secret bundles (doctor-bundles.mjs says what each line is for). Silent on a deployment that
|
|
3987
|
+
// binds no bundle and declares no bundle profile. The overlay and the probe are read the way the run.secrets block
|
|
3988
|
+
// above reads them, and `--secrets` (seams.secrets) resolves each bundle here, names only.
|
|
3989
|
+
{
|
|
3990
|
+
const bundleSettingsPath = settingsFilePath(env, home);
|
|
3991
|
+
const bundleSettings = fileExists(bundleSettingsPath) ? readOverlay(bundleSettingsPath) : { overlay: {} };
|
|
3992
|
+
const overlayBundles = bundleSettings.invalid ? {} : (bundleSettings.overlay?.secretBundles ?? {});
|
|
3993
|
+
const anyBundle = Object.keys(triggerBundles ?? {}).length > 0 || (env.PI_SECRET_BUNDLES ?? "").trim() !== "" || Object.keys(overlayBundles).length > 0;
|
|
3994
|
+
// Review round 2: the named residual. A worker that declares no bundle profile and has no triggers file reads "no
|
|
3995
|
+
// bundles", so on a fleet a project another host's file binds would run here without its environment, with no
|
|
3996
|
+
// skip reason. Doctor cannot see another host's file, so it warns where the shape allows it: a multi-host worker
|
|
3997
|
+
// (PI_WORKER_NAME) with projects and no triggers file of its own.
|
|
3998
|
+
if (!anyBundle && typeof env.PI_WORKER_NAME === "string" && env.PI_WORKER_NAME.trim() !== "" && !fileExists(triggersPath(env, cwd))) {
|
|
3999
|
+
const fleetProjects = readProjectFacts(env, fileExists);
|
|
4000
|
+
if (fleetProjects.parseError === null && fleetProjects.projects.length > 0) {
|
|
4001
|
+
checks.push({ ok: true, warn: true, label: "this fleet worker has projects but no triggers file, so it applies no project secret bundle: a project another host's triggers.json binds would run here without its environment", fix: "keep the same triggers.json on every host (PI_TRIGGERS_FILE), or declare the bundle profiles here, which makes a missing file a retry instead" });
|
|
4002
|
+
}
|
|
4003
|
+
}
|
|
4004
|
+
if (anyBundle) {
|
|
4005
|
+
let realExecutablePath = null;
|
|
4006
|
+
try {
|
|
4007
|
+
({ realExecutable: realExecutablePath } = await import("./secrets.mjs"));
|
|
4008
|
+
} catch {
|
|
4009
|
+
realExecutablePath = null;
|
|
4010
|
+
}
|
|
4011
|
+
const projectFacts = readProjectFacts(env, fileExists);
|
|
4012
|
+
let resolveBundle = null;
|
|
4013
|
+
if (seams.secrets === true) {
|
|
4014
|
+
try {
|
|
4015
|
+
const { makeBundleResolver } = await import("./secret-bundles.mjs");
|
|
4016
|
+
const { parseSecretBundleProfilesSafe } = await import("./doctor-bundles.mjs");
|
|
4017
|
+
const resolver = makeBundleResolver({ envBundles: parseSecretBundleProfilesSafe(env.PI_SECRET_BUNDLES).bundles, roots: delimitedList(env.PI_SECRET_RESOLVER_ROOTS), forwardEnv: (env.PI_FORWARD_ENV ?? "").split(",").map((n) => n.trim()).filter((n) => n !== ""), hostEnv: spawnEnv, timeoutMs: Math.max(BUNDLE_DOCTOR_TIMEOUT_MS, Number(env.PI_SECRET_RESOLVE_TIMEOUT_MS) || 0), ...(seams.bundleSpawn ? { spawnFn: seams.bundleSpawn } : {}) });
|
|
4018
|
+
resolveBundle = (binding) => resolver.resolve({ provider }, binding, { overlayBundles });
|
|
4019
|
+
} catch {
|
|
4020
|
+
checks.push({ ok: false, label: "--secrets could not load the bundle resolver (pi did not load; its own line above says why)" });
|
|
4021
|
+
}
|
|
4022
|
+
}
|
|
4023
|
+
checks.push(
|
|
4024
|
+
...(await secretBundleChecks({
|
|
4025
|
+
bundles: triggerBundles ?? {},
|
|
4026
|
+
triggersPresent: fileExists(triggersPath(env, cwd)),
|
|
4027
|
+
projects: projectFacts.projects,
|
|
4028
|
+
projectsError: projectFacts.parseError,
|
|
4029
|
+
envRaw: env.PI_SECRET_BUNDLES,
|
|
4030
|
+
overlayBundles,
|
|
4031
|
+
roots: delimitedList(env.PI_SECRET_RESOLVER_ROOTS),
|
|
4032
|
+
realExecutablePath,
|
|
4033
|
+
resolveBundle,
|
|
4034
|
+
})),
|
|
4035
|
+
);
|
|
4036
|
+
}
|
|
4037
|
+
}
|
|
4038
|
+
|
|
4039
|
+
section("sessions");
|
|
3315
4040
|
// REQ-RESUMABLE-SESSION. Only reported when a trigger actually asked for it: a deployment that does
|
|
3316
4041
|
// not use resume should not be told about a directory it has no reason to create.
|
|
3317
4042
|
if (resuming > 0) {
|
|
@@ -3389,6 +4114,30 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3389
4114
|
}
|
|
3390
4115
|
}
|
|
3391
4116
|
|
|
4117
|
+
// Issue #645: the checkpoint store, only where one is configured (PI_CHECKPOINTS_DIR, or the default inside
|
|
4118
|
+
// PI_SESSIONS_DIR). Never a ✗: only a job that arms run.recover writes it, a fault costs that run its recovery and never
|
|
4119
|
+
// the job, and a deployment without durable runs must not fail doctor over a store it does not use. One `git --version`, bounded like every other spawn here.
|
|
4120
|
+
const checkpointsDir = checkpointsDirPath(env);
|
|
4121
|
+
if (checkpointsDir) {
|
|
4122
|
+
// The hardening flags lead even here, where no repository is read, because the bolt that pins every host-side git
|
|
4123
|
+
// argv to them (`prepare-local.test.mjs`) is worth more unbroken than one `--version` is without them.
|
|
4124
|
+
const gitRead = await runCmdCapture(spawn, "git", [...GIT_READ_FLAGS, "--version"], { timeoutMs: runTimeouts.cmd, stdoutOnly: true });
|
|
4125
|
+
checks.push(
|
|
4126
|
+
...checkpointStoreChecks({
|
|
4127
|
+
dir: checkpointsDir,
|
|
4128
|
+
explicit: Boolean(env.PI_CHECKPOINTS_DIR),
|
|
4129
|
+
sessionsDir: env.PI_SESSIONS_DIR || null,
|
|
4130
|
+
fs: seams.checkpointFs,
|
|
4131
|
+
uid: Object.hasOwn(seams, "checkpointUid") ? seams.checkpointUid : process.geteuid?.(),
|
|
4132
|
+
underTemp: (path) => underTemp(path, env),
|
|
4133
|
+
gitVersion: gitRead.code === 0 ? gitRead.output : null,
|
|
4134
|
+
venues: blessedList ?? [DEFAULT_BACKEND],
|
|
4135
|
+
podmanRead: podman?.observed?.read ?? null,
|
|
4136
|
+
}),
|
|
4137
|
+
);
|
|
4138
|
+
}
|
|
4139
|
+
|
|
4140
|
+
section("triggers");
|
|
3392
4141
|
// REQ-PER-TRIGGER-INSTRUCTION. A plain fact line, not a warning: standing text is an ordinary operator
|
|
3393
4142
|
// choice. It is reported at all because it changes what EVERY job of that trigger is told, and unlike a
|
|
3394
4143
|
// flow (which lives in the repo, reviewed by a merge) it lives only in triggers.json, so nothing else
|
|
@@ -3443,6 +4192,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3443
4192
|
});
|
|
3444
4193
|
}
|
|
3445
4194
|
|
|
4195
|
+
section("files");
|
|
3446
4196
|
// REQ-SCOPED-PAUSE-WINDOWS, the panel-writes-what-the-worker-ignores trap (issue #99). Three defaults
|
|
3447
4197
|
// that are individually defensible and together silent:
|
|
3448
4198
|
//
|
|
@@ -3639,12 +4389,19 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3639
4389
|
const verdict = loadVerdict(spec, fileRaw, cwd, seamsForLoad, platform);
|
|
3640
4390
|
checks.push(
|
|
3641
4391
|
verdict.ok
|
|
3642
|
-
? { ok: true, label: `${spec.key} is set in ${join(cwd, ".env")} (${envValueShown(fileRaw)})${alsoExported === undefined ? "" : ", for this platform's loader,"} and loads: the service reads that file, this shell does not` }
|
|
4392
|
+
? { id: bootFileId(spec), ok: true, label: `${spec.key} is set in ${join(cwd, ".env")} (${envValueShown(fileRaw)})${alsoExported === undefined ? "" : ", for this platform's loader,"} and loads: the service reads that file, this shell does not` }
|
|
3643
4393
|
: {
|
|
4394
|
+
id: bootFileId(spec),
|
|
3644
4395
|
ok: false,
|
|
3645
4396
|
warn: envSetup !== null,
|
|
3646
4397
|
label: `${spec.key} is set in ${join(cwd, ".env")} (${envValueShown(fileRaw)}) to a file the worker cannot load, so a service started from it ${spec.fails}: ${verdict.reason}`,
|
|
3647
|
-
fix:
|
|
4398
|
+
fix: verdict.newer
|
|
4399
|
+
? envSetup !== null
|
|
4400
|
+
? `${envSetup} runs after that file and may replace it; if it does not, ${newerFileFix("this host's worker")}`
|
|
4401
|
+
: newerFileFix("this host's worker")
|
|
4402
|
+
: envSetup !== null
|
|
4403
|
+
? `${envSetup} runs after that file and may replace it; if it does not, fix ${envValueShown(fileRaw)} or point the key at a file that loads`
|
|
4404
|
+
: `fix ${envValueShown(fileRaw)}, or point ${spec.key} at a file that loads`,
|
|
3648
4405
|
},
|
|
3649
4406
|
);
|
|
3650
4407
|
}
|
|
@@ -3660,9 +4417,10 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3660
4417
|
const verdict = loadVerdict(spec, shellRaw, cwd, seamsForLoad, platform);
|
|
3661
4418
|
if (!verdict.ok) {
|
|
3662
4419
|
checks.push({
|
|
4420
|
+
id: bootFileId(spec, "shell"),
|
|
3663
4421
|
ok: false,
|
|
3664
4422
|
label: `${spec.key} is set in this shell to a file the worker cannot load, so it ${spec.fails}: ${verdict.reason}`,
|
|
3665
|
-
fix: `fix ${envValueShown(shellRaw)}, or point ${spec.key} at a file that loads`,
|
|
4423
|
+
fix: verdict.newer ? newerFileFix("this host's worker") : `fix ${envValueShown(shellRaw)}, or point ${spec.key} at a file that loads`,
|
|
3666
4424
|
});
|
|
3667
4425
|
}
|
|
3668
4426
|
} else if (spec.defaultsToScaffold) {
|
|
@@ -3676,6 +4434,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3676
4434
|
const verdict = loadVerdict(spec, scaffolded, cwd, seamsForLoad, platform);
|
|
3677
4435
|
if (!verdict.ok) {
|
|
3678
4436
|
checks.push({
|
|
4437
|
+
id: bootFileId(spec),
|
|
3679
4438
|
ok: false,
|
|
3680
4439
|
label: `${spec.key} is unset, so ${spec.unsetMeans}, and it does not load: the worker ${spec.fails}: ${verdict.reason}`,
|
|
3681
4440
|
fix: `fix ${envValueShown(scaffolded)}, or empty its "endpoints" list`,
|
|
@@ -3711,6 +4470,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3711
4470
|
// dead folder by one line and a bare repo by the other.
|
|
3712
4471
|
const folderOnly = (s) => scopeFormOf(s) !== "qualified" && (s.startsWith("/") || s.startsWith("./") || s.startsWith("../") || s.includes("\\") || !s.includes("/") || /^[A-Za-z]:/.test(s));
|
|
3713
4472
|
|
|
4473
|
+
section("limits");
|
|
3714
4474
|
// The dead-scope advisory (issue #242), honest about what doctor can actually judge. A forge repo
|
|
3715
4475
|
// always contains "/" and never begins "/", "./" or "../" or carries a backslash, so a scope in any
|
|
3716
4476
|
// of THOSE shapes can only ever be a folder -- and a folder row that matches no trigger's canonical
|
|
@@ -3745,11 +4505,26 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3745
4505
|
if (projectFacts.parseError === null) checks.push(...projectRowChecks(scopedLimitFacts.limits, projectFacts.projects, scopedLimitFacts.path));
|
|
3746
4506
|
}
|
|
3747
4507
|
|
|
4508
|
+
section("envelope");
|
|
3748
4509
|
// Issue #504 part B: the envelope's advisories, on the file the service names, when it loads (a file that does not is
|
|
3749
4510
|
// the BOOT_FILES line's). Warnings only: the worker boots on both.
|
|
3750
4511
|
const envelopeFactsHere = readEnvelopeFacts(env);
|
|
3751
4512
|
if (envelopeFactsHere.envelope) checks.push(...envelopeChecks(envelopeFactsHere.envelope, envelopeFactsHere.projects, envelopeFactsHere.maxCostMicros));
|
|
3752
4513
|
|
|
4514
|
+
section("console");
|
|
4515
|
+
// Issue #640: the operator console, judged with its keys resolved by the console's own rule (this shell's value, else
|
|
4516
|
+
// the deployment `.env`'s; DOCTOR_SHELL_KEYS). This shell's values are the ones `env` already carries (the resolution
|
|
4517
|
+
// starts from a copy of this shell), so nothing here reads the shell a second way; and the resolution is apart from the
|
|
4518
|
+
// service record, so a console key is never reported as a service's.
|
|
4519
|
+
const consoleShell = Object.fromEntries(CONSOLE_ENV_KEYS.filter((k) => Object.hasOwn(env, k)).map((k) => [k, env[k]]));
|
|
4520
|
+
const consoleEnv = { ...env, ...resolveServiceEnv({ env: consoleShell, file: serviceEnvFile, keys: CONSOLE_ENV_KEYS }).env };
|
|
4521
|
+
const consoleLines = consoleChecks(consoleEnv, { cwd, home, platform, folders, skillsDirs });
|
|
4522
|
+
checks.push(...consoleLines);
|
|
4523
|
+
// Issue #708: whether a service keeps the console running, and whether it answers. Only for a deployment that has a
|
|
4524
|
+
// console at all (the lines above), so a deployment that never set one up meets no probe.
|
|
4525
|
+
if (consoleLines.length > 0) checks.push(...(await consoleServiceChecks(consoleEnv, { ...seams, cwd, home, platform })));
|
|
4526
|
+
|
|
4527
|
+
section("limits");
|
|
3753
4528
|
// Issue #498: a BARE repo scope (`acme/web`) matches that repo on every forge, so with triggers on more than one forge
|
|
3754
4529
|
// kind a bare row is one cap (one lease, one count) shared by GitHub's acme/web and Forgejo's, and a bare pause window
|
|
3755
4530
|
// pauses both. That may be meant, so this is a WARNING naming the qualified spellings, never a failure: refusing bare
|
|
@@ -3769,6 +4544,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3769
4544
|
}
|
|
3770
4545
|
}
|
|
3771
4546
|
|
|
4547
|
+
section("dirs");
|
|
3772
4548
|
// Issue #464: the jobs dir is this account's to write. PI_JOBS_DIR, PI_SANDBOX_DIR and TMPDIR as the service reads them
|
|
3773
4549
|
// (TMPDIR in .env is a documented way out of a squatted default root, so doctor judges the root the service will use).
|
|
3774
4550
|
let dirRefused = false;
|
|
@@ -3805,14 +4581,18 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3805
4581
|
checks.push(...sandboxTombstoneChecks(sandboxDir, kept));
|
|
3806
4582
|
}
|
|
3807
4583
|
|
|
4584
|
+
section("state");
|
|
4585
|
+
// Issue #761: how long the console audit trail is kept and how many runs the shared run index holds, each a ✗ where
|
|
4586
|
+
// the worker refuses to boot on its value (`WORKER_INT_SETTINGS`).
|
|
4587
|
+
checks.push(...workerIntSettingChecks(env, (key) => fromFileNote(fileSays(key))));
|
|
3808
4588
|
// issue #290. The two DURABLE stores -- the run history everything folds over, and the overlay holding
|
|
3809
4589
|
// every cap the operator tuned from the panel -- used to default under the OS temp dir, which macOS
|
|
3810
4590
|
// sweeps on its own schedule and which is tmpfs (RAM) on several Linux distros. They now default under
|
|
3811
4591
|
// ~/.pi-dispatch, so this block is normally one green line; it warns only when a path RESOLVES under a
|
|
3812
4592
|
// temp dir, which after the move means an operator put it there.
|
|
3813
4593
|
//
|
|
3814
|
-
// Deliberately NOT checked HERE: PI_JOBS_DIR
|
|
3815
|
-
// retention-bounded
|
|
4594
|
+
// Deliberately NOT checked HERE: PI_JOBS_DIR and PI_SANDBOX_DIR. Those are per-run and
|
|
4595
|
+
// retention-bounded respectively, and they stay under temp on purpose. Warning about a
|
|
3816
4596
|
// directory that is SUPPOSED to be swept is how an operator learns to skim this section, which costs
|
|
3817
4597
|
// more than it buys. Whose the jobs dir is, is checked above (`jobsDirChecks`, issue #464): that is a
|
|
3818
4598
|
// different question, and the one that failed every job of a second account on a shared host.
|
|
@@ -3892,22 +4672,51 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3892
4672
|
// become invalid on upgrade: a duplicate key, which used to take the last value, is now refused. The reason names
|
|
3893
4673
|
// keys only, never values. Absent is the normal empty overlay and says nothing.
|
|
3894
4674
|
if (fileExists(settingsFile)) {
|
|
3895
|
-
|
|
4675
|
+
// The writer's read (issue #761), so the keys this build does not know are named below; its verdict is the
|
|
4676
|
+
// worker's own (`readOverlay` is this read without the carry).
|
|
4677
|
+
const overlay = readOverlayDocument(settingsFile);
|
|
3896
4678
|
if (overlay.invalid) {
|
|
4679
|
+
// Issue #761: a file written by a newer pi-dispatch is no broken file. The worker DEFERS every job
|
|
4680
|
+
// (newer-release) rather than refusing it, and the fix is an upgrade, never an edit or a delete, which would
|
|
4681
|
+
// lose what the newer release wrote. The one exception is a `version` an operator wrote by hand (earlier
|
|
4682
|
+
// releases ignored the key): no release wrote that file, so removing the key is the fix.
|
|
4683
|
+
const newer = overlay.newer === true;
|
|
3897
4684
|
checks.push({
|
|
4685
|
+
// Named (issue #712): the console quotes it beside "the settings overlay does not load".
|
|
4686
|
+
id: "settings.overlay",
|
|
3898
4687
|
ok: false,
|
|
3899
|
-
label:
|
|
4688
|
+
label: newer
|
|
4689
|
+
? `settings overlay ${settingsFile} cannot be read here (${overlay.invalid}) -- the worker defers every job (newer-release) until it is upgraded, and the panel refuses to write over it`
|
|
4690
|
+
: `settings overlay ${settingsFile} is invalid (${overlay.invalid}) -- the worker refuses every job as settings-overlay-invalid until it is fixed, and the panel refuses to write over it`,
|
|
3900
4691
|
// The fix follows the reason: only a duplicate key gets the upgrade note, since only it loaded before #501.
|
|
3901
|
-
fix:
|
|
3902
|
-
? "
|
|
3903
|
-
:
|
|
4692
|
+
fix: newer
|
|
4693
|
+
? newerFileFix("this worker")
|
|
4694
|
+
: /duplicate key/.test(overlay.invalid)
|
|
4695
|
+
? "remove the repeated key from that file, keeping the value you mean (a duplicate key is refused since issue #501; it used to take the last value), or delete the file to start from an empty overlay"
|
|
4696
|
+
: "fix what the reason names in that file, or delete the file to start from an empty overlay",
|
|
3904
4697
|
});
|
|
3905
4698
|
} else {
|
|
4699
|
+
// Issue #761: the keys this build does not know, which the worker ignores and every writer from issue #761's
|
|
4700
|
+
// release on keeps. An earlier panel or console (4.3.1 included) drops every key it does not know when it writes
|
|
4701
|
+
// the file (`secretBundles` included, which it predates), so the fix says to upgrade those first. NAMES only,
|
|
4702
|
+
// sorted and capped, never a value.
|
|
4703
|
+
const unknown = [...(overlay.carry?.unknownKeys ?? [])].map((key) => printable(key).slice(0, UNKNOWN_KEY_NAME_MAX)).sort();
|
|
4704
|
+
if (unknown.length > 0) {
|
|
4705
|
+
const shown = unknown.slice(0, UNKNOWN_KEYS_SHOWN);
|
|
4706
|
+
const more = unknown.length - shown.length;
|
|
4707
|
+
checks.push({
|
|
4708
|
+
id: "settings.unknown-keys",
|
|
4709
|
+
ok: false,
|
|
4710
|
+
warn: true,
|
|
4711
|
+
label: `settings overlay ${settingsFile} holds ${unknown.length === 1 ? "a key" : `${unknown.length} keys`} this build does not know (${shown.join(", ")}${more > 0 ? `, and ${more} more` : ""}): the worker ignores ${unknown.length === 1 ? "it" : "them"}`,
|
|
4712
|
+
fix: "a key from a newer release is ignored here and kept by every panel and console from 5.0.0 on, but an earlier one (4.3.1 included) drops every key it does not know when it writes this file, secretBundles included, so upgrade every panel and console first; a misspelt key is kept too, so remove it if it is a typo",
|
|
4713
|
+
});
|
|
4714
|
+
}
|
|
3906
4715
|
// Issue #501 (PR #542's review): a VALID overlay can still break the dollar invariant once merged over env
|
|
3907
4716
|
// (a window in one, no maxCostUsd in either), which the worker checks per job and answers by refusing every
|
|
3908
4717
|
// job. The env-only half is `dollarChecks`; this is the merged half, said only when env alone is fine.
|
|
3909
4718
|
const merged = overlayDollarProblem(overlay.overlay, env);
|
|
3910
|
-
if (merged) checks.push({ ok: false, label: `settings overlay ${settingsFile}: ${merged} -- the worker refuses every job as settings-overlay-invalid`, fix: "set maxCostUsd in the overlay (or PI_MAX_COST_USD in .env), or remove the dollar window" });
|
|
4719
|
+
if (merged) checks.push({ id: "settings.overlay-dollars", ok: false, label: `settings overlay ${settingsFile}: ${merged} -- the worker refuses every job as settings-overlay-invalid`, fix: "set maxCostUsd in the overlay (or PI_MAX_COST_USD in .env), or remove the dollar window" });
|
|
3911
4720
|
}
|
|
3912
4721
|
}
|
|
3913
4722
|
// PR #549's review: the scoped-limits dollar rows against the deployment's per-job cap, env and overlay merged
|
|
@@ -3970,7 +4779,29 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3970
4779
|
});
|
|
3971
4780
|
}
|
|
3972
4781
|
}
|
|
3973
|
-
|
|
4782
|
+
// Issue #761: a store an older release left in a shape this build no longer writes (older-state.mjs), one warning per
|
|
4783
|
+
// finding, each published under this one id (a second is `state.older-2`, `assignCheckIds`). The id is set here, never
|
|
4784
|
+
// in that module, so it is written at one site. The three older-location hints above keep their own words and places.
|
|
4785
|
+
// The container is judged as `up` judges it (`upValkeyContainerImage`): this deployment's, in a folder not handed to
|
|
4786
|
+
// compose, and only where VALKEY_URL names this host, the one place `up` runs a Valkey. The hand-over is read as `up`
|
|
4787
|
+
// reads it, by the override's presence; null where it cannot be, which names no container.
|
|
4788
|
+
// Only where the `.env` reads cleanly through the reader `up` judges its Valkey by (`readValkeyKeys`, under
|
|
4789
|
+
// `deploymentValkeyEnv`), from the file's bytes, and its VALKEY_URL, where it sets one, is the URL judged here. Not
|
|
4790
|
+
// when that reader errors, this shell's VALKEY_URL set or not: under systemd, a NUL or invalid UTF-8 anywhere in the
|
|
4791
|
+
// file (it loads none of it), or, in a file that spells one of its three Valkey keys, a line the service's loader reads
|
|
4792
|
+
// differently or one of those keys on a line it cannot read. `up` refuses each of those on Linux, and elsewhere judges
|
|
4793
|
+
// this shell's value or the default in the file's place. Nor when the file's VALKEY_URL is not `valkeyUrl` (this
|
|
4794
|
+
// shell sets another, which `up` refuses everywhere). Either way the port judged may not be the one the service
|
|
4795
|
+
// dials, and the fix's `pi-dispatch up` would start no Valkey, so the container is not asked about at all.
|
|
4796
|
+
const upValkeyRead = serviceEnvFile ? readValkeyKeys(serviceEnvFile.raw ?? serviceEnvFile.text, { loader: serviceEnvFile.loader, path: serviceEnvFile.path }) : { keys: {} };
|
|
4797
|
+
const valkeyUrlRead = !upValkeyRead.error && (!Object.hasOwn(upValkeyRead.keys, "VALKEY_URL") || upValkeyRead.keys.VALKEY_URL === valkeyUrl);
|
|
4798
|
+
const valkeyContainer = valkeyContainerReader({ spawn, runTimeouts, asked: localUsed && dockerCode === 0 && valkeyUrlRead, cwd, valkeyUrl });
|
|
4799
|
+
const handedOver = typeof fileExists === "function" && typeof cwd === "string" && cwd !== "" ? Boolean(fileExists(join(cwd, COMPOSE_VALKEY_OVERRIDE))) : null;
|
|
4800
|
+
for (const f of await olderStateOf(seams, { cwd, home, valkeyContainer, handedOver })) {
|
|
4801
|
+
checks.push({ id: "state.older", ok: false, warn: true, label: f.label, fix: f.fix });
|
|
4802
|
+
}
|
|
4803
|
+
|
|
4804
|
+
section("env");
|
|
3974
4805
|
// Issue #471: what a key resolved above this point (the provider's variables, the PAT) added to the record.
|
|
3975
4806
|
checks.push(...serviceEnvLines({ fromFile: {}, disagreements: service.disagreements.slice(saidAtTop.disagreements), unread: service.unread.slice(saidAtTop.unread) }, envPathShown));
|
|
3976
4807
|
// Issue #471 (gate round 1): a key only this shell sets was taken in silence. Where a service for this folder is
|
|
@@ -3995,6 +4826,7 @@ export async function collectChecks(shellVars, seams) {
|
|
|
3995
4826
|
const { hazard } = serviceEnvFile;
|
|
3996
4827
|
checks.push({ ok: false, label: `${serviceEnvFile.path} line ${hazard.line} ${hazard.what}, so doctor read none of ${service.hazardSkipped.join(", ")} from it and judged this shell's values (or the defaults) instead: the service's own are unknown`, fix: hazard.fix });
|
|
3997
4828
|
}
|
|
4829
|
+
tagSection(checks, currentSection);
|
|
3998
4830
|
return checks;
|
|
3999
4831
|
}
|
|
4000
4832
|
|
|
@@ -4414,7 +5246,8 @@ function nodeCheck(version) {
|
|
|
4414
5246
|
* that every image a trigger names is actually on this host -- with `--pull=never` nothing will fetch one at
|
|
4415
5247
|
* job time, so this is the only warning that arrives BEFORE the trigger fires at 03:00.
|
|
4416
5248
|
*
|
|
4417
|
-
* `optingOut` counts `run.packages: false` -- the only thing that
|
|
5249
|
+
* `optingOut` counts `run.packages: false` -- the only thing that withholds the whole staged set from a job. `run.excludeExtensions`
|
|
5250
|
+
* withholds one package at a time, and `excludedNames` holds those names.
|
|
4418
5251
|
* `requiring` counts explicit `run.packages: true`, which arms nothing any more but is still an operator
|
|
4419
5252
|
* asserting "this flow needs those packages"; that assertion is what makes an empty stage a hard failure.
|
|
4420
5253
|
*
|
|
@@ -4616,6 +5449,228 @@ function readProjectFacts(env, fileExists) {
|
|
|
4616
5449
|
}
|
|
4617
5450
|
}
|
|
4618
5451
|
|
|
5452
|
+
// `sun_path` sizes, NUL terminator included, as the console's own `parseListen` (admin/src/console/listen.mjs) holds
|
|
5453
|
+
// them; doctor-console.test.mjs compares the two parsers' verdicts, since the worker cannot import the admin.
|
|
5454
|
+
const CONSOLE_SUN_PATH = { darwin: 104, freebsd: 104, openbsd: 104, netbsd: 104, linux: 108 };
|
|
5455
|
+
const CONSOLE_LOOPBACK = new BlockList();
|
|
5456
|
+
CONSOLE_LOOPBACK.addSubnet("127.0.0.0", 8, "ipv4");
|
|
5457
|
+
CONSOLE_LOOPBACK.addAddress("::1", "ipv6");
|
|
5458
|
+
|
|
5459
|
+
/**
|
|
5460
|
+
* PI_CONSOLE_LISTEN as the console parses it: `{ shown, loopback }` or `{ refusal }`. IP literals only (`ip:port`,
|
|
5461
|
+
* `[ipv6]:port`) or `unix:/abs/path` within the platform's socket path size; loopback is 127.0.0.0/8 and ::1. Unset or
|
|
5462
|
+
* empty is the default, 127.0.0.1:4790. The non-loopback refusal is the caller's (it depends on PI_CONSOLE_BIND_PUBLIC).
|
|
5463
|
+
*/
|
|
5464
|
+
export function consoleListenOf(value, platform = process.platform) {
|
|
5465
|
+
const text = typeof value === "string" ? value.trim() : "";
|
|
5466
|
+
if (text === "") return { shown: "127.0.0.1:4790 (the default)", loopback: true };
|
|
5467
|
+
if (text.startsWith("unix:")) {
|
|
5468
|
+
const path = text.slice("unix:".length);
|
|
5469
|
+
if (!isAbsolute(path)) return { refusal: "a unix socket path must be absolute: write unix:/abs/path" };
|
|
5470
|
+
const limit = CONSOLE_SUN_PATH[platform] ?? 108;
|
|
5471
|
+
if (Buffer.byteLength(path) >= limit) return { refusal: `the unix socket path is ${Buffer.byteLength(path)} bytes, and this platform allows at most ${limit - 1}: choose a shorter path` };
|
|
5472
|
+
return { shown: text, loopback: true };
|
|
5473
|
+
}
|
|
5474
|
+
const m = /^(?:\[([^\]]*)\]|([^:[\]]*)):([0-9]{1,5})$/.exec(text);
|
|
5475
|
+
if (!m || Number(m[3]) > 65535) return { refusal: "it is not ip:port, [ipv6]:port or unix:/abs/path (the port a number from 0 to 65535)" };
|
|
5476
|
+
const host = m[1] ?? m[2];
|
|
5477
|
+
if (m[1] !== undefined ? !isIPv6(host) : !isIPv4(host)) return { refusal: `${host} is not an IP literal; a hostname can resolve anywhere, so write 127.0.0.1 or [::1]` };
|
|
5478
|
+
const loopback = CONSOLE_LOOPBACK.check(host, m[1] !== undefined ? "ipv6" : "ipv4");
|
|
5479
|
+
return { shown: text, loopback };
|
|
5480
|
+
}
|
|
5481
|
+
|
|
5482
|
+
/**
|
|
5483
|
+
* Issue #761: the worker settings read with `config.mjs`'s `nonNegativeInt` or `boundedInt` whose bad value doctor
|
|
5484
|
+
* names, by that same rule (unset or empty is the default; any other value must be exactly the integer, spaces around
|
|
5485
|
+
* it allowed, inside the bounds): a value the worker refuses to boot on (exit 2, a configuration error naming the key)
|
|
5486
|
+
* fails here too, and nothing else does. `want` is the loader's own phrase for the rule.
|
|
5487
|
+
*/
|
|
5488
|
+
export const WORKER_INT_SETTINGS = Object.freeze({
|
|
5489
|
+
PI_CONSOLE_AUDIT_RETENTION_DAYS: Object.freeze({ min: 0, max: undefined, want: "a non-negative integer", fix: "set it to a whole number of days (0 keeps the console audit trail forever), or delete the line so the trail is kept as long as PI_LOG_RETENTION_DAYS keeps run history" }),
|
|
5490
|
+
PI_RUN_MIRROR_MAX_RUNS: Object.freeze({ min: RUN_MIRROR_MAX_RUNS_MIN, max: RUN_MIRROR_MAX_RUNS_MAX, want: `an integer ${RUN_MIRROR_MAX_RUNS_MIN}-${RUN_MIRROR_MAX_RUNS_MAX}`, fix: `set it to a whole number from ${RUN_MIRROR_MAX_RUNS_MIN} to ${RUN_MIRROR_MAX_RUNS_MAX}, or delete the line for the default (${RUN_MIRROR_MAX_RUNS_DEFAULT})` }),
|
|
5491
|
+
});
|
|
5492
|
+
|
|
5493
|
+
/** Whether `raw` is a value `config.mjs`'s `boundedInt` takes for `rule` (see `WORKER_INT_SETTINGS`). */
|
|
5494
|
+
function workerIntAccepted(raw, { min, max }) {
|
|
5495
|
+
if (raw === undefined || raw === "") return true;
|
|
5496
|
+
const n = Number.parseInt(raw, 10);
|
|
5497
|
+
return Number.isInteger(n) && n >= min && (max === undefined || n <= max) && String(n) === String(raw).trim();
|
|
5498
|
+
}
|
|
5499
|
+
|
|
5500
|
+
/**
|
|
5501
|
+
* A ✗ for each `WORKER_INT_SETTINGS` key `env` sets to a value the worker refuses to boot on; `note(key)` says where it
|
|
5502
|
+
* came from. The keys are the table's own, so a setting added there is judged here with nothing else to change.
|
|
5503
|
+
*/
|
|
5504
|
+
export function workerIntSettingChecks(env, note = () => "") {
|
|
5505
|
+
return Object.entries(WORKER_INT_SETTINGS)
|
|
5506
|
+
.filter(([key, rule]) => !workerIntAccepted(env[key], rule))
|
|
5507
|
+
.map(([key, rule]) => ({ ok: false, label: `${key} is ${quotedShown(env[key])}, which is not ${rule.want}: the worker refuses to boot on it (exit 2)${note(key)}`, fix: rule.fix }));
|
|
5508
|
+
}
|
|
5509
|
+
|
|
5510
|
+
/**
|
|
5511
|
+
* The console's lines (issue #640, REQ-CONSOLE-SECURITY). NONE for a deployment that never set the console up: no
|
|
5512
|
+
* tokens file at the resolved path (nothing at all there, by lstat) and no PI_CONSOLE_* key set. Otherwise:
|
|
5513
|
+
* - the tokens file as the console reads it (`readTokensFile`: owner-only regular file, valid), ✗ with its own fix;
|
|
5514
|
+
* - at least one operator token, ✗ (with none, nobody can change anything from the console);
|
|
5515
|
+
* - the tokens file outside every path a job container can see, ✗: a local trigger's run.folder is bind-mounted
|
|
5516
|
+
* read-write, so a job there could read the file or add a token of its own. Judged by `envelopeInsideJobPaths`, the
|
|
5517
|
+
* envelope's identity-based containment check (a firmlink or bind alias of a folder is caught), on the file's real
|
|
5518
|
+
* path, with the local folders, skills dirs, absolute run roots and the overlay;
|
|
5519
|
+
* - entries kept for a role this release does not know (issue #761, `keptTokens`), ⚠: a newer release writes such
|
|
5520
|
+
* entries, and a console of this release signs them in nowhere;
|
|
5521
|
+
* - PI_CONSOLE_LISTEN parseable (✗, the console refuses to start) and PI_CONSOLE_BIND_PUBLIC=1 (⚠, plain HTTP);
|
|
5522
|
+
* - the audit chain (`verifyConsoleAudit`), ⚠ when broken, nothing while the log holds no row; an intact chain with
|
|
5523
|
+
* rows whose shaped action or outcome this release does not know (issue #761, `unknown`; a newer console writes
|
|
5524
|
+
* such rows) is still ok, and its line counts them.
|
|
5525
|
+
* `env` must already carry the console keys resolved (`collectChecks` does it). Seams: `cwd` (the deployment folder),
|
|
5526
|
+
* `fs`, `uid`, `home`, `platform`, `folders` (canonical local run.folders), `skillsDirs`, `realpath`.
|
|
5527
|
+
*/
|
|
5528
|
+
export function consoleChecks(env, { cwd = process.cwd(), fs = nodeFs, uid = process.geteuid?.(), home, platform = process.platform, folders = [], skillsDirs = [], realpath = realpathSync.native } = {}) {
|
|
5529
|
+
const path = tokensFilePath({ PI_CONSOLE_TOKENS_FILE: env.PI_CONSOLE_TOKENS_FILE }, cwd);
|
|
5530
|
+
let present = true;
|
|
5531
|
+
try {
|
|
5532
|
+
fs.lstatSync(path);
|
|
5533
|
+
} catch {
|
|
5534
|
+
present = false;
|
|
5535
|
+
}
|
|
5536
|
+
const keysSet = CONSOLE_ENV_KEYS.filter((k) => typeof env[k] === "string" && env[k] !== "");
|
|
5537
|
+
if (!present && keysSet.length === 0) return [];
|
|
5538
|
+
const checks = [];
|
|
5539
|
+
const shownPath = quotedShown(path);
|
|
5540
|
+
const read = readTokensFile({ path, fs, uid });
|
|
5541
|
+
if (read.refusal) {
|
|
5542
|
+
checks.push({ ok: false, label: `console: the tokens file ${shownPath} cannot be used, so \`pi-dispatch-console serve\` refuses to start`, fix: escapeControls(read.refusal) });
|
|
5543
|
+
} else {
|
|
5544
|
+
const operators = operatorCount(read.file);
|
|
5545
|
+
const viewers = read.file.tokens.length - operators;
|
|
5546
|
+
if (operators === 0) {
|
|
5547
|
+
checks.push({ ok: false, label: `console: the tokens file ${shownPath} holds no operator token, so nobody can make a change from the console`, fix: "run `pi-dispatch-console token add --name <you> --role operator`" });
|
|
5548
|
+
}
|
|
5549
|
+
const inside = consoleTokensInsideJobPaths(path, env, { cwd, fs, folders, skillsDirs, realpath });
|
|
5550
|
+
if (inside.problem) {
|
|
5551
|
+
checks.push({ ok: false, label: `console: the tokens file ${shownPath} ${inside.problem}`, fix: inside.fix });
|
|
5552
|
+
} else if (operators > 0) {
|
|
5553
|
+
checks.push({ ok: true, label: `console: tokens file ${shownPath} is owner-only and outside every job folder (${operators} operator${operators === 1 ? "" : "s"}, ${viewers} viewer${viewers === 1 ? "" : "s"})` });
|
|
5554
|
+
}
|
|
5555
|
+
const kept = keptTokens(read.file).length;
|
|
5556
|
+
if (kept > 0) {
|
|
5557
|
+
const said = kept === 1 ? "1 token carries a role this pi-dispatch does not know, so a console of this release signs it in nowhere" : `${kept} tokens carry a role this pi-dispatch does not know, so a console of this release signs them in nowhere`;
|
|
5558
|
+
checks.push({ ok: false, warn: true, label: `console: ${said}`, fix: "upgrade pi-dispatch and the console" });
|
|
5559
|
+
}
|
|
5560
|
+
}
|
|
5561
|
+
const listen = consoleListenOf(env.PI_CONSOLE_LISTEN, platform);
|
|
5562
|
+
const bindPublic = env.PI_CONSOLE_BIND_PUBLIC === "1";
|
|
5563
|
+
if (listen.refusal) {
|
|
5564
|
+
checks.push({ ok: false, label: `console: PI_CONSOLE_LISTEN ${quotedShown(env.PI_CONSOLE_LISTEN)} cannot be used, so \`pi-dispatch-console serve\` refuses to start: ${listen.refusal}`, fix: "write 127.0.0.1:4790, [::1]:4790 or unix:/abs/path, or unset PI_CONSOLE_LISTEN for the default" });
|
|
5565
|
+
} else if (!listen.loopback && !bindPublic) {
|
|
5566
|
+
checks.push({ ok: false, label: `console: PI_CONSOLE_LISTEN ${quotedShown(env.PI_CONSOLE_LISTEN)} is not a loopback address, so \`pi-dispatch-console serve\` refuses to start`, fix: "listen on 127.0.0.1, [::1] or a unix: socket and reach it with ssh -L; or, to accept plain HTTP on a network, set PI_CONSOLE_BIND_PUBLIC=1" });
|
|
5567
|
+
} else {
|
|
5568
|
+
checks.push({ ok: true, label: `console: listens on ${escapeControls(listen.shown)}` });
|
|
5569
|
+
}
|
|
5570
|
+
if (bindPublic) {
|
|
5571
|
+
checks.push({ ok: false, warn: true, label: "console: PI_CONSOLE_BIND_PUBLIC=1 lets the console listen on a non-loopback address, where tokens and session cookies cross the network in plain HTTP", fix: "keep it only behind a TLS proxy on a network you trust; otherwise unset PI_CONSOLE_BIND_PUBLIC, listen on 127.0.0.1, [::1] or a unix: socket, and reach it with ssh -L" });
|
|
5572
|
+
}
|
|
5573
|
+
const logsDir = logsDirPath(env, home);
|
|
5574
|
+
const audit = verifyConsoleAudit({ logsDir, fs });
|
|
5575
|
+
if (!audit.ok) {
|
|
5576
|
+
const where = audit.broken.file === null ? "" : ` at ${audit.broken.file} line ${audit.broken.line}`;
|
|
5577
|
+
checks.push({ ok: false, warn: true, label: `console: the audit log chain is broken${where}: ${escapeControls(audit.broken.reason)}`, fix: `a row was changed, removed or moved after it was written, or the file was damaged. Keep a copy of ${join(logsDir, "console-audit")} for review; \`pi-dispatch-console audit verify\` names the same link, and a damaged LAST line makes the console refuse every write until that file is moved aside` });
|
|
5578
|
+
} else if (audit.rows > 0) {
|
|
5579
|
+
const start = audit.startsAt ? `, starting at seq ${audit.startsAt.seq} in ${audit.startsAt.file} (older rows reaped)` : "";
|
|
5580
|
+
// Issue #761: what the check knows of such rows, that their words are unknown here and shaped, never who wrote them.
|
|
5581
|
+
// Last, since its colon would otherwise run into the start's clause.
|
|
5582
|
+
const unknown = audit.unknown > 0 ? `, ${audit.unknown} not known to this pi-dispatch: a newer console writes such rows` : "";
|
|
5583
|
+
checks.push({ ok: true, label: `console: audit log chain intact (${audit.rows} row${audit.rows === 1 ? "" : "s"}${start}${unknown})` });
|
|
5584
|
+
}
|
|
5585
|
+
return checks;
|
|
5586
|
+
}
|
|
5587
|
+
|
|
5588
|
+
/** The console's service name on Windows, where nssm keeps one machine-scoped service per name (service.mjs). */
|
|
5589
|
+
const CONSOLE_NSSM_SERVICE = "pi-dispatch-console";
|
|
5590
|
+
|
|
5591
|
+
/**
|
|
5592
|
+
* The console service unit installed for the deployment in `cwd` (issue #708): `{ path }` (the unit file, or the nssm
|
|
5593
|
+
* service's name on win32), else null. Read as the worker's unit is (`installedWorkerUnit`): a unit FILE whose
|
|
5594
|
+
* WorkingDirectory is this folder (`readUnitSeam`), in either scope; on win32 the nssm service by name, since nssm
|
|
5595
|
+
* keeps the folder apart (AppDirectory) and there is one such service per machine.
|
|
5596
|
+
*/
|
|
5597
|
+
async function installedConsoleUnit({ platform, home, cwd, fileExists, readUnit = (path) => readFileSync(path, "utf8"), spawn }) {
|
|
5598
|
+
if (platform === "win32") {
|
|
5599
|
+
if (typeof spawn !== "function") return null;
|
|
5600
|
+
const got = await runCmdCapture(spawn, "nssm", ["status", CONSOLE_NSSM_SERVICE]);
|
|
5601
|
+
return got.code === 0 ? { path: CONSOLE_NSSM_SERVICE } : null;
|
|
5602
|
+
}
|
|
5603
|
+
const path = installedConsoleUnitPath({ platform, home, cwd, fileExists, readUnit });
|
|
5604
|
+
return path === null ? null : { path };
|
|
5605
|
+
}
|
|
5606
|
+
|
|
5607
|
+
/**
|
|
5608
|
+
* The console's service lines (issue #708): whether a service unit keeps the console of this deployment running, and
|
|
5609
|
+
* whether the console answers (`probeConsole`: one GET /healthz to the loopback address or the socket it binds, 1 s).
|
|
5610
|
+
* - a unit installed and the console answering: ✓;
|
|
5611
|
+
* - a unit installed and nothing answering: ✗, since the service exists to keep it answering (`serve` exits 2 on a
|
|
5612
|
+
* refusal and the unit then stays stopped, so the unit's log says which);
|
|
5613
|
+
* - no unit and nothing answering: ⚠, with the install command;
|
|
5614
|
+
* - no unit and the console answering: ✓, saying it was started by hand and stops with its terminal;
|
|
5615
|
+
* - an address the probe may not or cannot ask (port 0, a non-loopback bind): the unit line alone, and why the
|
|
5616
|
+
* answer is not known.
|
|
5617
|
+
* NOTHING when the console could not start on its listen address at all: `consoleChecks` already fails that line, and a
|
|
5618
|
+
* second line saying it does not answer would be the same fact twice. `env` carries the console keys resolved as
|
|
5619
|
+
* `collectChecks` resolves them; no other key is read. Seams: `consoleProbe(env)`, `fileExists`, `readUnit`, `spawn`.
|
|
5620
|
+
*/
|
|
5621
|
+
export async function consoleServiceChecks(env, seams = {}) {
|
|
5622
|
+
const { platform = process.platform, cwd = process.cwd() } = seams;
|
|
5623
|
+
const listen = consoleListenOf(env.PI_CONSOLE_LISTEN, platform);
|
|
5624
|
+
if (listen.refusal || (!listen.loopback && env.PI_CONSOLE_BIND_PUBLIC !== "1")) return [];
|
|
5625
|
+
const unit = await installedConsoleUnit({ ...seams, platform, cwd, fileExists: seams.fileExists ?? existsSync });
|
|
5626
|
+
const probe = seams.consoleProbe ?? (async (e) => (await import("./console-probe.mjs")).probeConsole(e));
|
|
5627
|
+
let answer;
|
|
5628
|
+
try {
|
|
5629
|
+
answer = await probe(env);
|
|
5630
|
+
} catch (err) {
|
|
5631
|
+
answer = { state: "unknown", reason: `the probe failed (${escapeControls(err?.message ?? String(err))})` };
|
|
5632
|
+
}
|
|
5633
|
+
const at = answer.target ? ` on ${escapeControls(answer.target)}` : "";
|
|
5634
|
+
const why = answer.reason ? ` (${escapeControls(answer.reason)})` : "";
|
|
5635
|
+
const unitShown = unit ? quotedShown(unit.path) : null;
|
|
5636
|
+
const logs = platform === "linux" ? "`journalctl --user -u pi-dispatch-console` (or without --user for a system unit)" : platform === "darwin" ? `${join(cwd, "logs", "console.err.log")}` : `${join(cwd, "logs", "console.err.log")} and \`nssm status ${CONSOLE_NSSM_SERVICE}\``;
|
|
5637
|
+
if (answer.state === "unknown") {
|
|
5638
|
+
return [{ ok: true, label: `console: ${unit ? `the service ${unitShown} serves this deployment; ` : "no service unit keeps it running; "}whether it answers is not checked${why}` }];
|
|
5639
|
+
}
|
|
5640
|
+
if (unit && answer.state === "running") return [{ ok: true, label: `console: the service ${unitShown} keeps it running, and it answers${at}` }];
|
|
5641
|
+
if (unit) {
|
|
5642
|
+
const lock = join(logsDirPath(env, seams.home), "console-audit", ".console.lock");
|
|
5643
|
+
return [{ ok: false, label: `console: the service ${unitShown} is installed for this deployment, but nothing answers${at}${why}`, fix: `\`pi-dispatch service status --console\` says whether it runs, and ${logs} says why it stopped: \`pi-dispatch-console serve\` exits 2 on a refusal (no operator token, the lock held by another console, an address in use), and the unit then stays stopped until it is started again (\`pi-dispatch service start --console\`). After a crash the lock ${escapeControls(lock)} can name a pid another process has since taken, which keeps every start refusing: when that pid is not a console, remove the lock, then start the unit` }];
|
|
5644
|
+
}
|
|
5645
|
+
if (answer.state === "running") return [{ ok: true, label: `console: answers${at}, started by hand (no service unit for this deployment, so it stops with the terminal that runs it)` }];
|
|
5646
|
+
return [{ ok: false, warn: true, label: `console: not running${at}${why}, and no service unit keeps it running`, fix: "`pi-dispatch service install --console` keeps it running (it needs an operator token); or start it in a terminal with `pi-dispatch-console serve`" }];
|
|
5647
|
+
}
|
|
5648
|
+
|
|
5649
|
+
/**
|
|
5650
|
+
* Whether a job container can see the console tokens file: `{}` or `{ problem, fix }`. The file's REAL path is judged
|
|
5651
|
+
* (the containment check requires a canonical path, and the console reads whatever the configured one resolves to). A
|
|
5652
|
+
* second hard link is refused as the envelope's is: another name of the file could sit in a job folder unseen. A
|
|
5653
|
+
* relative run root is skipped here (the envelope refuses one; the console has no such rule to borrow), and a path
|
|
5654
|
+
* that cannot be judged is said, not passed.
|
|
5655
|
+
*/
|
|
5656
|
+
function consoleTokensInsideJobPaths(path, env, { cwd, fs, folders, skillsDirs, realpath }) {
|
|
5657
|
+
const fix = "move the tokens file out of every local trigger's run.folder and run.skillsDir, every PI_DISPATCH_RUN_ROOTS root and PI_GLOBAL_PI_DIR, and point PI_CONSOLE_TOKENS_FILE at its new path";
|
|
5658
|
+
try {
|
|
5659
|
+
const st = fs.statSync(path);
|
|
5660
|
+
if (st.nlink > 1) return { problem: `has ${st.nlink} hard links: another name of the file could sit inside a folder a job can see, so keep exactly one`, fix: `remove the other names (\`find / -samefile ${path}\` lists them)` };
|
|
5661
|
+
const real = realpath(path);
|
|
5662
|
+
const runRoots = delimitedList(env.PI_DISPATCH_RUN_ROOTS).filter((r) => isAbsolute(r));
|
|
5663
|
+
const globalPiDir = typeof env.PI_GLOBAL_PI_DIR === "string" && env.PI_GLOBAL_PI_DIR !== "" ? env.PI_GLOBAL_PI_DIR : null;
|
|
5664
|
+
// PWD comes from the process environment inside the check, as on doctor's envelope load (`loadEnvelopeChecked`).
|
|
5665
|
+
const inside = envelopeInsideJobPaths(real, { cronFolders: folders, runRoots, skillsDirs, globalPiDir }, { cwd });
|
|
5666
|
+
if (inside === null) return {};
|
|
5667
|
+
const what = { "cron-folder": "a local trigger's run.folder", "run-root": "a PI_DISPATCH_RUN_ROOTS root", "skills-dir": "a trigger's run.skillsDir", "global-pi-dir": "PI_GLOBAL_PI_DIR" }[inside.kind] ?? inside.kind;
|
|
5668
|
+
return { problem: `lies inside ${what} (${quotedShown(inside.path)}), which a job container can see: a job there could read the token hashes or add a token of its own`, fix };
|
|
5669
|
+
} catch (err) {
|
|
5670
|
+
return { problem: `could not be judged against the folders a job can see (${escapeControls(err?.code ?? "the path could not be resolved")})`, fix: `make ${path} and the trigger folders readable here, then run doctor again` };
|
|
5671
|
+
}
|
|
5672
|
+
}
|
|
5673
|
+
|
|
4619
5674
|
/**
|
|
4620
5675
|
* The envelope facts (issue #504 part B): the envelope as the worker would load it, the projects and the per-job cap it
|
|
4621
5676
|
* was judged against, or a parse error when `PI_ENVELOPE_FILE` is set and does not load (the BOOT_FILES line's).
|
|
@@ -4765,6 +5820,30 @@ export async function defaultReadAppliedSplit(url) {
|
|
|
4765
5820
|
}
|
|
4766
5821
|
}
|
|
4767
5822
|
|
|
5823
|
+
export { cooldownModelRefs } from "./model-observation.mjs";
|
|
5824
|
+
|
|
5825
|
+
/** A failed bounded read is visible, never reported as "no cooldown". */
|
|
5826
|
+
export async function defaultReadModelCooldowns(url, refs, options) {
|
|
5827
|
+
try {
|
|
5828
|
+
// Kept lazy like the other model checks: doctor must start even when pi cannot be imported.
|
|
5829
|
+
const { readModelCooldowns } = await import("./model-failover.mjs");
|
|
5830
|
+
return await withDoctorClient(url, (client) => boundedOp(readModelCooldowns(client, refs, options), DOCTOR_VALKEY_OP_TIMEOUT_MS));
|
|
5831
|
+
} catch {
|
|
5832
|
+
return null;
|
|
5833
|
+
}
|
|
5834
|
+
}
|
|
5835
|
+
|
|
5836
|
+
/** Warn lines are also published to the console's existing Doctor panel, with the observation's age. */
|
|
5837
|
+
export function modelCooldownChecks(rows) {
|
|
5838
|
+
if (!Array.isArray(rows)) return [{ ok: false, warn: true, label: "Model cooldowns could not be read", fix: "check Valkey and re-run doctor; failover jobs retry infrastructure failures before they spend" }];
|
|
5839
|
+
return rows.filter((row) => splitModelEntry(row?.ref) && ["rate-limited", "quota-exhausted", "overloaded"].includes(row.kind) && Number.isSafeInteger(row.until) && row.until >= 0 && row.until <= 253402300799999).map((row) => ({
|
|
5840
|
+
ok: false,
|
|
5841
|
+
warn: true,
|
|
5842
|
+
label: `Model ${row.ref} is cooling down until ${new Date(row.until).toISOString()} (${row.kind})`,
|
|
5843
|
+
fix: "new failover jobs skip this model until then and use only the reviewed fallback list; recovery keeps its checkpoint model",
|
|
5844
|
+
}));
|
|
5845
|
+
}
|
|
5846
|
+
|
|
4768
5847
|
/**
|
|
4769
5848
|
* A FAILURE per scoped-limits file that holds a `project:<id>` row whose id is not a project in `projects` (issue #499
|
|
4770
5849
|
* part B): the worker refuses to start on it, and a live reload that would create one is kept out. Rows are named by
|
|
@@ -4785,15 +5864,24 @@ export function projectRowChecks(limits, projects, path) {
|
|
|
4785
5864
|
/**
|
|
4786
5865
|
* The pause-windows facts (issue #498): the parsed windows when PI_PAUSE_WINDOWS_FILE is set, else none. A file that
|
|
4787
5866
|
* does not load is the BOOT_FILES check's line, so here it only silences the advisories that read the windows.
|
|
5867
|
+
* `versionNeed` (issue #761) is the version the file needs, the higher of what it declares and what its RAW windows'
|
|
5868
|
+
* fields need, for the fleet line that names a peer that cannot read it; 1 without a file. `versions` is the
|
|
5869
|
+
* `{ supported, table, valueTable }` the file is read and its need judged by (collectChecks' `versionTables`), the
|
|
5870
|
+
* module's own by default.
|
|
4788
5871
|
*/
|
|
4789
|
-
function readPauseWindowFacts(env, fileExists) {
|
|
5872
|
+
function readPauseWindowFacts(env, fileExists, versions = {}) {
|
|
4790
5873
|
const path = env.PI_PAUSE_WINDOWS_FILE;
|
|
4791
|
-
if (typeof path !== "string" || path.trim() === "") return { windows: [], parseError: null };
|
|
5874
|
+
if (typeof path !== "string" || path.trim() === "") return { windows: [], versionNeed: 1, parseError: null };
|
|
4792
5875
|
try {
|
|
4793
|
-
if (!fileExists(path) || !statSync(path).isFile()) return { windows: [], parseError: "unreadable" };
|
|
4794
|
-
|
|
5876
|
+
if (!fileExists(path) || !statSync(path).isFile()) return { windows: [], versionNeed: 1, parseError: "unreadable" };
|
|
5877
|
+
// The loader's own parse (`loadPauseWindows` is this read after the existence check above), handed the versions.
|
|
5878
|
+
const text = readFileSync(path, "utf8");
|
|
5879
|
+
const windows = parsePauseWindows(text, path, versions);
|
|
5880
|
+
// The parse just accepted this text, so it parses, and a declared version is an integer.
|
|
5881
|
+
const doc = JSON.parse(text);
|
|
5882
|
+
return { windows, versionNeed: Math.max(Number.isSafeInteger(doc?.version) ? doc.version : 1, pauseWindowsVersionFor(doc?.windows, versions)), parseError: null };
|
|
4795
5883
|
} catch (e) {
|
|
4796
|
-
return { windows: [], parseError: e?.message ?? String(e) };
|
|
5884
|
+
return { windows: [], versionNeed: 1, parseError: e?.message ?? String(e) };
|
|
4797
5885
|
}
|
|
4798
5886
|
}
|
|
4799
5887
|
|
|
@@ -4985,6 +6073,76 @@ export function fleetSizeChecks(fileVersion, peers) {
|
|
|
4985
6073
|
}];
|
|
4986
6074
|
}
|
|
4987
6075
|
|
|
6076
|
+
/** The overlay keys `settings.unknown-keys` names at most (issue #761), and how much of each name it shows. */
|
|
6077
|
+
export const UNKNOWN_KEYS_SHOWN = 8;
|
|
6078
|
+
const UNKNOWN_KEY_NAME_MAX = 64;
|
|
6079
|
+
|
|
6080
|
+
/**
|
|
6081
|
+
* The stores a host publishes a version for (issue #761, `DES-STORE-VERSION-WHEN-NEEDED`), by registry field: what the
|
|
6082
|
+
* store is, what a peer does with this host's file when it publishes a LOWER version (this release's rule: it defers
|
|
6083
|
+
* the job, `newer-release`) and when it publishes NONE (a worker from before issue #761: it reads the file without its
|
|
6084
|
+
* version and drops what it does not know), and who else writes it, for the fix. A test holds the fields to the ones
|
|
6085
|
+
* start.mjs publishes.
|
|
6086
|
+
*
|
|
6087
|
+
* A lower peer that watches the pause windows or the triggers file defers every job from its next reload of it, not
|
|
6088
|
+
* only the jobs that read the file: its worker-wide newer state is read by the triggers gate at the processor's head,
|
|
6089
|
+
* and by the pickup's pause gate and the watch branch's (a job ended before those, by the draft-source or sandbox
|
|
6090
|
+
* check, a runtime's own result, a lost lock's replay or a grouping job's deleted project, spends nothing). Job data
|
|
6091
|
+
* is written by the receivers, the console and the workers' own builders, and a receiver publishes no registry row,
|
|
6092
|
+
* so its fix names the receivers as a set doctor cannot see.
|
|
6093
|
+
*/
|
|
6094
|
+
export const STORE_FLEET_LINES = Object.freeze({
|
|
6095
|
+
settingsVersion: Object.freeze({ store: "the settings overlay", older: "defers every job (newer-release) while it reads this file", silent: "drops the overlay keys it does not know, so a cap they set does not apply on it", writers: "every panel and console that writes the settings overlay (one from before issue #761 drops a key it does not know when it writes)" }),
|
|
6096
|
+
pauseWindowsVersion: Object.freeze({ store: "the pause windows file", older: "defers every job (newer-release) from its next reload of this file, and refuses the file at its next start", silent: "drops the window fields it does not know, so a window pauses at the wrong time on it", writers: "every panel and console that writes the pause windows file (one from before issue #761 drops a key it does not know when it writes)" }),
|
|
6097
|
+
triggersVersion: Object.freeze({ store: "the triggers file", older: "defers every job (newer-release) from its next reload of this file, keeps its last good schedules, and refuses the file at its next start", silent: "drops the run keys it does not know, so a trigger's jobs run wider on it than the file says", writers: "every panel and console that writes the triggers file (one from before issue #761 drops a key it does not know when it writes)" }),
|
|
6098
|
+
jobDataVersion: Object.freeze({ store: "the jobs this host's triggers queue", older: "defers each such job (newer-release) until it is upgraded", silent: "drops the job fields it does not know, so such a job runs wider on it than its trigger says", writers: "every receiver and every console that queues these jobs (a receiver publishes no registry row, so doctor cannot see which receivers are older)" }),
|
|
6099
|
+
});
|
|
6100
|
+
|
|
6101
|
+
/**
|
|
6102
|
+
* The version each of this host's stores needs, by registry field (issue #761): the settings overlay (read by the
|
|
6103
|
+
* worker's own reader), the pause windows file and the triggers file (from the facts doctor already read), and the job
|
|
6104
|
+
* data their triggers queue. A store whose file does not load is left out. `settingsVersions` is the overlay's
|
|
6105
|
+
* `{ supported, table, valueTable }` (collectChecks' `versionTables`), the module's own by default.
|
|
6106
|
+
*/
|
|
6107
|
+
export function storeVersionNeeds({ settingsFile, fileExists, pauseWindowFacts, triggersVersionNeed, jobDataVersionNeed, triggersParsed, settingsVersions = {} }) {
|
|
6108
|
+
const needs = {};
|
|
6109
|
+
const settings = fileExists(settingsFile) ? readOverlayDocument(settingsFile, settingsVersions) : { overlay: {}, carry: null };
|
|
6110
|
+
if (!settings.invalid) {
|
|
6111
|
+
const declared = settings.carry?.doc?.version;
|
|
6112
|
+
needs.settingsVersion = Math.max(Number.isSafeInteger(declared) ? declared : 1, settingsVersionFor(settings.overlay, settingsVersions));
|
|
6113
|
+
}
|
|
6114
|
+
if (pauseWindowFacts.parseError === null) needs.pauseWindowsVersion = pauseWindowFacts.versionNeed;
|
|
6115
|
+
if (triggersParsed) {
|
|
6116
|
+
needs.triggersVersion = triggersVersionNeed;
|
|
6117
|
+
needs.jobDataVersion = jobDataVersionNeed;
|
|
6118
|
+
}
|
|
6119
|
+
return needs;
|
|
6120
|
+
}
|
|
6121
|
+
|
|
6122
|
+
/**
|
|
6123
|
+
* The fleet's store versions (issue #761): for each store whose need (`needs`, registry field -> the version this host's
|
|
6124
|
+
* own file needs) is above 1, a WARNING naming the peers that publish that field below the need (they defer what the
|
|
6125
|
+
* file touches until they are upgraded) and one naming the peers that publish no such field at all (workers from before
|
|
6126
|
+
* the field: they read the file without its version and drop what they do not know, which can widen a job). Nothing
|
|
6127
|
+
* at all while every need is 1, which is a deployment whose files use nothing a table has a row for. Hosts are named,
|
|
6128
|
+
* never a file's content. `fleetSizeChecks` keeps its own line: a peer without `limitsVersion` still reads version 2.
|
|
6129
|
+
*/
|
|
6130
|
+
export function fleetStoreVersionChecks(needs, peers) {
|
|
6131
|
+
const checks = [];
|
|
6132
|
+
const warn = (label, fix) => checks.push({ id: "fleet.store-versions", ok: false, warn: true, label, fix });
|
|
6133
|
+
const fix = (line, need) => `upgrade and restart the workers named, and ${line.writers}, to a release that reads version ${need}`;
|
|
6134
|
+
for (const [field, line] of Object.entries(STORE_FLEET_LINES)) {
|
|
6135
|
+
const need = needs?.[field];
|
|
6136
|
+
if (!Number.isSafeInteger(need) || need <= 1) continue;
|
|
6137
|
+
const silent = peers.filter((h) => h[field] === undefined || h[field] === null || h[field] === "");
|
|
6138
|
+
const older = peers.filter((h) => !silent.includes(h) && !(Number.isSafeInteger(Number(h[field])) && Number(h[field]) >= need));
|
|
6139
|
+
const names = (hosts) => hosts.map((h) => h.name).join(", ");
|
|
6140
|
+
if (older.length > 0) warn(`${names(older)} ${older.length === 1 ? "reads" : "read"} ${line.store} only below version ${need}, the version this host's file needs: ${older.length === 1 ? "it" : "each"} ${line.older}`, fix(line, need));
|
|
6141
|
+
if (silent.length > 0) warn(`${names(silent)} ${silent.length === 1 ? "publishes" : "publish"} no version for ${line.store} (a worker from before issue #761), while this host's file needs version ${need}: ${silent.length === 1 ? "it" : "each"} ${line.silent}`, fix(line, need));
|
|
6142
|
+
}
|
|
6143
|
+
return checks;
|
|
6144
|
+
}
|
|
6145
|
+
|
|
4988
6146
|
export function fleetProjectsChecks(mine, peers) {
|
|
4989
6147
|
const checks = [];
|
|
4990
6148
|
const opinions = peers.filter((h) => typeof h.fpProjects === "string" && h.fpProjects !== "");
|
|
@@ -5010,72 +6168,11 @@ export function fleetProjectsChecks(mine, peers) {
|
|
|
5010
6168
|
return checks;
|
|
5011
6169
|
}
|
|
5012
6170
|
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
|
|
5016
|
-
*/
|
|
5017
|
-
export const FIRST_CALL_OVERHEAD_TOKENS = 8192;
|
|
5018
|
-
/**
|
|
5019
|
-
* The least a job's FIRST call carries before its task, in bytes: pi 0.99.1's default system prompt (1,753 bytes) and
|
|
5020
|
-
* the schemas of its four default tools (2,712 bytes) serialise to 4,465 bytes, and PR #542's lab measured 10,667 for
|
|
5021
|
-
* a short task. 4,096 is below both, so the bound below stays a lower bound for a job that adds nothing.
|
|
5022
|
-
*/
|
|
5023
|
-
export const FIRST_CALL_MIN_CONTEXT_BYTES = 4096;
|
|
6171
|
+
// The first-call bound moved to model-catalog.mjs (issue #800), where the console's preview can read it without loading this
|
|
6172
|
+
// module. Re-exported so every caller and test of doctor's own lines reads the one rule as before.
|
|
6173
|
+
export { FIRST_CALL_MIN_CONTEXT_BYTES, FIRST_CALL_OVERHEAD_TOKENS, boundModelOf, firstCallFloorMicros, serviceTierMultiplier };
|
|
5024
6174
|
|
|
5025
|
-
|
|
5026
|
-
* A LOWER BOUND, in integer micro-dollars, on what the runner's cost guard (`callCostBound`) reserves for a job's
|
|
5027
|
-
* first call on `model`, or null when this cannot say. The guard refuses a call when its bound would pass the cap, so
|
|
5028
|
-
* a per-job cap below this number can never admit a call on the model: with the main model, the job makes no call at
|
|
5029
|
-
* all (PR #542's lab: the default model's first call was bounded at $1.03 under a $1 cap).
|
|
5030
|
-
*
|
|
5031
|
-
* The guard's bound is (request bytes + 8,192) x the dearest input rate + the output limit x the dearest output rate,
|
|
5032
|
-
* times a service-tier multiplier, over the model's table, its tiers and its fallbacks. This takes the parts of that
|
|
5033
|
-
* which cannot be smaller: the model's own table (not its tiers or fallbacks, which only raise the bound), the output
|
|
5034
|
-
* limit `model.maxTokens` (pi sends none of its own, so the guard uses the model's), an input of the overhead plus
|
|
5035
|
-
* `FIRST_CALL_MIN_CONTEXT_BYTES`, and the runner's own service-tier multiplier (`serviceTierMultiplier`). The input
|
|
5036
|
-
* rate is the guard's: the highest of input, cache read and cache write (not the 1h write, which needs long
|
|
5037
|
-
* retention). Null for a table or limit it cannot read.
|
|
5038
|
-
*/
|
|
5039
|
-
export function firstCallFloorMicros(model) {
|
|
5040
|
-
const cost = model?.cost;
|
|
5041
|
-
const usable = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0;
|
|
5042
|
-
if (!cost || !["input", "output", "cacheRead", "cacheWrite"].every((field) => usable(cost[field]))) return null;
|
|
5043
|
-
const maxTokens = model.maxTokens;
|
|
5044
|
-
if (!(typeof maxTokens === "number" && Number.isFinite(maxTokens) && maxTokens > 0)) return null;
|
|
5045
|
-
const inputRate = Math.max(cost.input, cost.cacheRead, cost.cacheWrite);
|
|
5046
|
-
return Math.ceil(((FIRST_CALL_OVERHEAD_TOKENS + FIRST_CALL_MIN_CONTEXT_BYTES) * inputRate + maxTokens * cost.output) * serviceTierMultiplier(model));
|
|
5047
|
-
}
|
|
5048
|
-
|
|
5049
|
-
/**
|
|
5050
|
-
* The runner's service-tier multiplier (`callCostBound` step 7, image/runner/src/usage-meter.mjs), copied like the
|
|
5051
|
-
* overhead above: pi multiplies a settled cost by the tier the RESPONSE reports on these two apis, which can be the
|
|
5052
|
-
* account's default rather than the request's, so the runner always bounds at the dearest tier. Leaving it out
|
|
5053
|
-
* (PR #551's review) let doctor stay silent on a cap the runner refuses every call under: openai/gpt-5.5 under $5.
|
|
5054
|
-
* `fleet-dollars.test.mjs` derives the runner's multiplier from its own bound for every priced api and holds this
|
|
5055
|
-
* equal to it.
|
|
5056
|
-
*/
|
|
5057
|
-
export function serviceTierMultiplier(model) {
|
|
5058
|
-
if (model?.api !== "openai-responses" && model?.api !== "openai-codex-responses") return 1;
|
|
5059
|
-
return model.id === "gpt-5.5" ? 2.5 : 2;
|
|
5060
|
-
}
|
|
5061
|
-
|
|
5062
|
-
/**
|
|
5063
|
-
* The model object the floor above is computed from: the builtin catalog's (`builtinModel`, model-catalog.mjs), or a
|
|
5064
|
-
* CUSTOM model the overlay `models.json` defines (its own `cost` and `maxTokens`). Null, so the model is not judged,
|
|
5065
|
-
* when the overlay redefines or overrides a builtin model: pi composes the two, and this does not copy that rule.
|
|
5066
|
-
*/
|
|
5067
|
-
export function boundModelOf({ provider, id }, { builtinModel, overlay = null }) {
|
|
5068
|
-
const providers = overlay?.providers;
|
|
5069
|
-
const entry = providers !== null && typeof providers === "object" && !Array.isArray(providers) && Object.hasOwn(providers, provider) ? providers[provider] : null;
|
|
5070
|
-
const builtin = builtinModel(provider, id);
|
|
5071
|
-
if (entry !== null && typeof entry === "object") {
|
|
5072
|
-
const overrides = entry.modelOverrides;
|
|
5073
|
-
if (overrides !== null && typeof overrides === "object" && Object.hasOwn(overrides, id)) return null;
|
|
5074
|
-
const definition = Array.isArray(entry.models) ? entry.models.find((m) => m !== null && typeof m === "object" && m.id === id) : undefined;
|
|
5075
|
-
if (definition !== undefined) return builtin ? null : definition;
|
|
5076
|
-
}
|
|
5077
|
-
return builtin;
|
|
5078
|
-
}
|
|
6175
|
+
export { resolveConfiguredModelRuns as resolveDoctorModelRuns } from "./model-observation.mjs";
|
|
5079
6176
|
|
|
5080
6177
|
/**
|
|
5081
6178
|
* Who runs on which models under which cap (issues #501 and #502), as the worker resolves a job: the deployment's own
|
|
@@ -5086,7 +6183,7 @@ export function boundModelOf({ provider, id }, { builtinModel, overlay = null })
|
|
|
5086
6183
|
* trigger's and the deployment's, `effectiveCostCapMicros`) or null, and `named` says the trigger names a provider, a
|
|
5087
6184
|
* model or a list of its own.
|
|
5088
6185
|
*/
|
|
5089
|
-
export function modelSubjects({ runs = [], deployment, envList = null }) {
|
|
6186
|
+
export function modelSubjects({ runs = [], deployment, envList = null, modelRoles = {}, projects = [] }) {
|
|
5090
6187
|
const listOf = (list) => (Array.isArray(list) ? list.map(splitModelEntry).filter((ref) => ref !== null).map((ref) => ({ provider: ref.provider, id: ref.model })) : []);
|
|
5091
6188
|
const capOf = (triggerValue) => {
|
|
5092
6189
|
try {
|
|
@@ -5096,7 +6193,7 @@ export function modelSubjects({ runs = [], deployment, envList = null }) {
|
|
|
5096
6193
|
}
|
|
5097
6194
|
};
|
|
5098
6195
|
const subjects = [{ label: "the deployment default", main: { provider: deployment.provider, id: deployment.model }, list: listOf(envList), cap: capOf(undefined), secretNames: [], named: false, deployment: true }];
|
|
5099
|
-
for (const run of runs) {
|
|
6196
|
+
for (const run of resolveDoctorModelRuns({ runs, deployment, envList, modelRoles, projects }).runs) {
|
|
5100
6197
|
subjects.push({
|
|
5101
6198
|
label: run.label,
|
|
5102
6199
|
main: { provider: run.provider ?? deployment.provider, id: run.model ?? deployment.model },
|
|
@@ -5114,7 +6211,7 @@ export function modelSubjects({ runs = [], deployment, envList = null }) {
|
|
|
5114
6211
|
* `modelSubjects` resolves them), for the overlay's output-cap line to leave to `costCapFitChecks` (issue #507). Empty
|
|
5115
6212
|
* when the settings cannot be read.
|
|
5116
6213
|
*/
|
|
5117
|
-
export function cappedModelNames(env, { runs, deployment }) {
|
|
6214
|
+
export function cappedModelNames(env, { runs, deployment, modelRoles = {}, projects = [] }) {
|
|
5118
6215
|
let envList = null;
|
|
5119
6216
|
try {
|
|
5120
6217
|
envList = allowedModelsFrom(env);
|
|
@@ -5122,7 +6219,7 @@ export function cappedModelNames(env, { runs, deployment }) {
|
|
|
5122
6219
|
// A malformed PI_ALLOWED_MODELS is its own line.
|
|
5123
6220
|
}
|
|
5124
6221
|
const names = new Set();
|
|
5125
|
-
for (const s of modelSubjects({ runs, deployment, envList })) {
|
|
6222
|
+
for (const s of modelSubjects({ runs, deployment, envList, modelRoles, projects })) {
|
|
5126
6223
|
if (s.cap === null || s.cap === undefined) continue;
|
|
5127
6224
|
for (const ref of [s.main, ...s.list]) names.add(`${ref.provider}/${ref.id}`);
|
|
5128
6225
|
}
|
|
@@ -5137,10 +6234,13 @@ export function cappedModelNames(env, { runs, deployment }) {
|
|
|
5137
6234
|
* Issue #507: `unboundable(ref)` says the runner cannot bound the model's output at all (`outputUnboundable`,
|
|
5138
6235
|
* output-cap.mjs: openai-completions to a server off the trusted hosts without `compat.maxTokensField: "max_tokens"`),
|
|
5139
6236
|
* so every call to it is refused under any cap. Such a model gets a line of its own, grouped by model, and no floor.
|
|
6237
|
+
* Radius and every composed pi-messages model have a separate diagnosis: their server-reported billing and input
|
|
6238
|
+
* rewrites are not bounded by the catalog, including rows whose listed rates are zero (issue #693).
|
|
5140
6239
|
*/
|
|
5141
|
-
export function costCapFitChecks(subjects, modelOf, { unboundable = () => false } = {}) {
|
|
6240
|
+
export function costCapFitChecks(subjects, modelOf, { unboundable = () => false, apiOf = (ref) => modelOf(ref)?.api } = {}) {
|
|
5142
6241
|
const groups = new Map();
|
|
5143
6242
|
const unbounded = new Map();
|
|
6243
|
+
const radius = new Map();
|
|
5144
6244
|
for (const s of subjects) {
|
|
5145
6245
|
if (s.cap === null || s.cap === undefined) continue;
|
|
5146
6246
|
const seen = new Set();
|
|
@@ -5148,6 +6248,15 @@ export function costCapFitChecks(subjects, modelOf, { unboundable = () => false
|
|
|
5148
6248
|
const name = `${ref.provider}/${ref.id}`;
|
|
5149
6249
|
if (seen.has(name)) continue;
|
|
5150
6250
|
seen.add(name);
|
|
6251
|
+
// The gateway reports its own prices and may rewrite input, so even a zero-rate catalog row is no bound.
|
|
6252
|
+
const api = apiOf(ref);
|
|
6253
|
+
if (api === "pi-messages" || (api === undefined && ref.provider === "radius")) {
|
|
6254
|
+
if (!radius.has(name)) radius.set(name, { main: false, labels: [] });
|
|
6255
|
+
const r = radius.get(name);
|
|
6256
|
+
r.main ||= ref === s.main;
|
|
6257
|
+
r.labels.push(s.label);
|
|
6258
|
+
continue;
|
|
6259
|
+
}
|
|
5151
6260
|
if (unboundable(ref)) {
|
|
5152
6261
|
if (!unbounded.has(name)) unbounded.set(name, { main: false, labels: [] });
|
|
5153
6262
|
const u = unbounded.get(name);
|
|
@@ -5164,6 +6273,15 @@ export function costCapFitChecks(subjects, modelOf, { unboundable = () => false
|
|
|
5164
6273
|
}
|
|
5165
6274
|
}
|
|
5166
6275
|
const out = [];
|
|
6276
|
+
if (radius.size > 0) {
|
|
6277
|
+
const items = [...radius].map(([name, r]) => `${printable(name)}${r.main ? " (the main model, so such a job makes no call at all)" : ""} (${r.labels.join(", ")})`);
|
|
6278
|
+
out.push({
|
|
6279
|
+
ok: false,
|
|
6280
|
+
warn: true,
|
|
6281
|
+
label: `${RADIUS_COST_CAP_LABEL}: ${items.join("; ")}`,
|
|
6282
|
+
fix: RADIUS_COST_CAP_FIX,
|
|
6283
|
+
});
|
|
6284
|
+
}
|
|
5167
6285
|
if (unbounded.size > 0) {
|
|
5168
6286
|
const items = [...unbounded].map(([name, u]) => `${printable(name)}${u.main ? " (the main model, so such a job makes no call at all)" : ""} (${u.labels.join(", ")})`);
|
|
5169
6287
|
out.push({
|
|
@@ -5248,7 +6366,7 @@ export function unknownModelChecks(subjects, checkModelsKnown, readOverlay) {
|
|
|
5248
6366
|
* Said nothing about a provider it cannot judge: no names at all (Bedrock reads the AWS chain), the keyless marker
|
|
5249
6367
|
* (#503), or an `apiKey` that is not a variable reference.
|
|
5250
6368
|
*/
|
|
5251
|
-
export function listedProviderCredentialChecks(subjects, { candidatesOf, forwarded = [], env = {}, overlay = null }) {
|
|
6369
|
+
export function listedProviderCredentialChecks(subjects, { candidatesOf, forwarded = [], env = {}, overlay = null, subscriptionChecks = new Map() }) {
|
|
5252
6370
|
const providers = overlay?.providers;
|
|
5253
6371
|
const namesFor = (provider) => {
|
|
5254
6372
|
const entry = providers !== null && typeof providers === "object" && !Array.isArray(providers) && Object.hasOwn(providers, provider) ? providers[provider] : null;
|
|
@@ -5264,6 +6382,7 @@ export function listedProviderCredentialChecks(subjects, { candidatesOf, forward
|
|
|
5264
6382
|
for (const s of subjects) {
|
|
5265
6383
|
for (const provider of new Set(s.list.map((ref) => ref.provider))) {
|
|
5266
6384
|
if (provider === s.main.provider) continue;
|
|
6385
|
+
if (subscriptionChecks.get(provider)?.managed) continue;
|
|
5267
6386
|
const names = namesFor(provider);
|
|
5268
6387
|
if (names.length === 0) continue;
|
|
5269
6388
|
if (names.some((n) => s.secretNames.includes(n) || (forwarded.includes(n) && typeof env[n] === "string" && env[n] !== ""))) continue;
|
|
@@ -5306,6 +6425,43 @@ export function deploymentSettingsOf(env, settingsFile, fileExists) {
|
|
|
5306
6425
|
return Object.fromEntries(Object.keys(fromEnv).map((key) => [key, resolved[key] ?? null]));
|
|
5307
6426
|
}
|
|
5308
6427
|
|
|
6428
|
+
/** Subscription diagnostics use only local credential metadata, independently of forge validity or Valkey. */
|
|
6429
|
+
export async function doctorSubscriptionCredentials(env, { cwd = process.cwd(), home = safeHomeDir(), now = Date.now, configure } = {}) {
|
|
6430
|
+
if (!env.PI_AUTH_CONNECTIONS_FILE) return { check: async () => ({ ok: true, managed: false }) };
|
|
6431
|
+
let auth;
|
|
6432
|
+
try {
|
|
6433
|
+
configure ??= (await import("./subscription-config.mjs")).configuredSubscriptions;
|
|
6434
|
+
const config = { authConnectionsFile: resolve(cwd, env.PI_AUTH_CONNECTIONS_FILE), authStoreDir: env.PI_AUTH_STORE_DIR ? resolve(cwd, env.PI_AUTH_STORE_DIR) : null, settingsFile: settingsFilePath(env, home), workerName: workerNameFrom(env), globalPiDir: env.PI_GLOBAL_PI_DIR || null, jobsDir: jobsDirPath(env), sandboxDir: env.PI_SANDBOX_DIR || defaultSandboxDir(env) };
|
|
6435
|
+
auth = configure({ config, redis: null, now })?.auth;
|
|
6436
|
+
if (typeof auth?.check !== "function") throw new Error();
|
|
6437
|
+
} catch { return { check: async () => ({ ok: false, managed: true, reason: "auth-policy-invalid" }) }; }
|
|
6438
|
+
const cache = new Map();
|
|
6439
|
+
return { check(provider) {
|
|
6440
|
+
if (!cache.has(provider)) cache.set(provider, Promise.resolve().then(() => auth.check(provider, { readOnly: true })).catch(() => ({ ok: false, managed: true, reason: "auth-state-unavailable" })));
|
|
6441
|
+
return cache.get(provider);
|
|
6442
|
+
} };
|
|
6443
|
+
}
|
|
6444
|
+
|
|
6445
|
+
export function subscriptionCredentialCheck(provider, row) {
|
|
6446
|
+
const name = printable(provider);
|
|
6447
|
+
const owner = typeof row.ownerHost === "string" && WORKER_NAME_RE.test(row.ownerHost) ? ` on owner ${row.ownerHost}` : "";
|
|
6448
|
+
const id = typeof (row.connectionId ?? row.id) === "string" && WORKER_NAME_RE.test(row.connectionId ?? row.id) ? ` ${row.connectionId ?? row.id}` : " CONNECTION";
|
|
6449
|
+
if (row.ok === true) return { ok: true, label: `Subscription credential selected (${name})${owner}; the host renews access before admitted jobs` };
|
|
6450
|
+
const words = {
|
|
6451
|
+
"auth-owner-host": "belongs to another owner host; this worker defers its subscription jobs",
|
|
6452
|
+
"auth-reauth-required": "requires sign-in on its owner host",
|
|
6453
|
+
"auth-refresh-uncertain": "requires reconnect after an uncertain token refresh",
|
|
6454
|
+
"auth-adapter-unpinned": "requires reviewed adapter staging",
|
|
6455
|
+
"auth-adapter-source-mismatch": "requires reviewed adapter staging",
|
|
6456
|
+
"auth-adapter-version-mismatch": "has an incompatible adapter version",
|
|
6457
|
+
"auth-adapter-unavailable": "has no usable staged adapter",
|
|
6458
|
+
"auth-store-permissions": "has unsafe private credential-store permissions",
|
|
6459
|
+
"auth-credential-invalid": "has an invalid stored subscription credential",
|
|
6460
|
+
"auth-policy-invalid": "has an unavailable or invalid connection policy",
|
|
6461
|
+
};
|
|
6462
|
+
return { ok: false, ...(row.reason === "auth-owner-host" ? { warn: true } : {}), label: `Subscription credential selected (${name}) ${words[row.reason] ?? "has unavailable credential metadata"}`, fix: row.reason === "auth-reauth-required" || row.reason === "auth-refresh-uncertain" ? `run \`pi-dispatch auth login${id}\` on the connection owner host` : "run `pi-dispatch auth status` on the connection owner host and correct its reviewed policy, adapter or private store" };
|
|
6463
|
+
}
|
|
6464
|
+
|
|
5309
6465
|
/** The worker's model catalog (model-catalog.mjs), imported lazily for `defaultProviderOracle`'s reason, or null. */
|
|
5310
6466
|
async function defaultModelCatalog() {
|
|
5311
6467
|
try {
|
|
@@ -5405,8 +6561,8 @@ function triggerLabel(t, index) {
|
|
|
5405
6561
|
return t.on.type === "cron" ? `cron "${t.on.id}"` : `${t.on.type} trigger #${index}`;
|
|
5406
6562
|
}
|
|
5407
6563
|
|
|
5408
|
-
function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
|
|
5409
|
-
const none = { requiring: 0, waiting: 0, listing: 0, waitProfiles: [], waitAfters: [], optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, secreting: 0, onceArmed: 0, onceSpent: 0, secretProfiles: [], localSecretFolders: [], secretNames: [], folders: [], images: [], imageRoutes: [], namedBackends: [], skillsDirs: [], forges: [], repositories: [], flows: [], costCaps: [], modelRuns: [], parseError: null, path: null };
|
|
6564
|
+
function readTriggerFacts(env, fileExists, cwd, declaredWorkerName, versionTables = {}) {
|
|
6565
|
+
const none = { watchSources: {}, watchRules: 0, modelRoles: {}, secretBundles: {}, requiring: 0, waiting: 0, listing: 0, waitProfiles: [], waitAfters: [], optingOut: 0, resuming: 0, recovering: 0, replicating: 0, instructing: 0, commands: 0, secreting: 0, onceArmed: 0, onceSpent: 0, secretProfiles: [], localSecretFolders: [], secretNames: [], folders: [], images: [], imageRoutes: [], namedBackends: [], skillsDirs: [], forges: [], repositories: [], flows: [], excludedNames: [], costCaps: [], modelRuns: [], triggersVersionNeed: 1, jobDataVersionNeed: 1, parseError: null, parseNewer: false, path: null };
|
|
5410
6566
|
try {
|
|
5411
6567
|
// Unset falls back to ./triggers.json in cwd, MIRRORING the receiver's own default
|
|
5412
6568
|
// (receiver/src/config.mjs) -- the two must read the same file, or doctor preflights a deployment
|
|
@@ -5417,14 +6573,16 @@ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
|
|
|
5417
6573
|
// naming a FIFO hangs the command with no output and no timeout that can reach it.
|
|
5418
6574
|
if (!statSync(path).isFile()) return { ...none, parseError: `triggers file is not a regular file: ${path}`, path };
|
|
5419
6575
|
const text = readFileSync(path, "utf8");
|
|
5420
|
-
|
|
6576
|
+
// The document parser, so the file's `secretBundles` (issue #704) is judged by the loaders' one rule too.
|
|
6577
|
+
const { triggers, secretBundles, modelRoles, watchSources } = parseTriggersDocument(text, path, versionTables.triggers);
|
|
5421
6578
|
// The one-shot facts are counted from the RAW entries, not the parsed records, because the
|
|
5422
6579
|
// validator collapses a disarmed entry to a sentinel that carries neither `once` nor
|
|
5423
6580
|
// `disarmed` -- exactly so nothing can match it -- which also erases it from every parsed
|
|
5424
6581
|
// count above. Doctor is the surface that must still SEE the spent entry: "why did nothing
|
|
5425
6582
|
// fire" is answered by a spent row, and only the raw file still holds it. Safe unguarded:
|
|
5426
6583
|
// parseTriggers just accepted this same text, so JSON.parse cannot throw here.
|
|
5427
|
-
const
|
|
6584
|
+
const rawDocument = JSON.parse(text);
|
|
6585
|
+
const rawEntries = rawDocument?.triggers ?? [];
|
|
5428
6586
|
// Issue #433 review rounds 2 and 3: whether THIS HOST'S WORKER schedules a cron trigger, for the unblessed-venue
|
|
5429
6587
|
// line, by the worker's two conditions and nothing else. (a) The worker schedules cron only from a PI_TRIGGERS_FILE
|
|
5430
6588
|
// it was given (`config.triggersFile` is null without one, and `loadSchedules` then returns []), while doctor
|
|
@@ -5438,10 +6596,25 @@ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
|
|
|
5438
6596
|
const fleet = Boolean(declaredWorkerName);
|
|
5439
6597
|
const cronScheduledHere = (t) => env.PI_TRIGGERS_FILE !== undefined && cronPlacement(t.run, { existsSync: fileExists, fleet }) !== "elsewhere";
|
|
5440
6598
|
return {
|
|
6599
|
+
// Issue #761: the version this file needs (the higher of what it declares and what its keys need), and the version
|
|
6600
|
+
// the jobs its triggers queue need (`jobDataVersionFor`): for the fleet line that names a peer that cannot read
|
|
6601
|
+
// either. The loader just accepted this text, so a declared version is an integer. The FILE's need is the raw
|
|
6602
|
+
// keys', since an older loader reads the raw file; the JOBS' need is what the loader KEPT, each parsed entry's
|
|
6603
|
+
// `run`, which is what the builders copy into job data (schedules.mjs, the receiver). A raw run key the loader
|
|
6604
|
+
// drops (a hand-written `run.packageCanary`, which only the Updates page's canary job carries) reaches no job,
|
|
6605
|
+
// so it needs no version, and a line claiming one would send an operator to upgrade for nothing.
|
|
6606
|
+
triggersVersionNeed: Math.max(Number.isSafeInteger(rawDocument?.version) ? rawDocument.version : 1, triggersFileVersionFor(rawDocument, versionTables.triggers)),
|
|
6607
|
+
jobDataVersionNeed: triggers.reduce((need, t) => Math.max(need, jobDataVersionFor(t?.run, versionTables.jobData)), 1),
|
|
6608
|
+
watchSources,
|
|
6609
|
+
watchRules: triggers.filter(t => t.on.type === "watch").length,
|
|
6610
|
+
secretBundles,
|
|
6611
|
+
modelRoles,
|
|
5441
6612
|
onceArmed: rawEntries.filter((t) => t?.on?.once === true && t.on.disarmed === undefined).length,
|
|
5442
6613
|
onceSpent: rawEntries.filter((t) => t?.on?.disarmed !== undefined).length,
|
|
5443
6614
|
requiring: triggers.filter((t) => t.run.packages === true).length,
|
|
5444
6615
|
resuming: triggers.filter((t) => t.run.resume === true).length,
|
|
6616
|
+
// Issue #645: durable runs. Counted for the fleet line: a recovery is routed to the host holding its checkpoint.
|
|
6617
|
+
recovering: triggers.filter((t) => t.run.recover === true).length,
|
|
5445
6618
|
// REQ-PER-TRIGGER-INSTRUCTION. Counted beside `resuming` for the same reason: it is a per-trigger
|
|
5446
6619
|
// choice that changes what every job of it is told, and an operator should see it before it fires.
|
|
5447
6620
|
instructing: triggers.filter((t) => typeof t.run.instructions === "string").length,
|
|
@@ -5519,14 +6692,18 @@ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
|
|
|
5519
6692
|
// entries by raw file position (the admin's trigger:<index> identity).
|
|
5520
6693
|
flows: triggers
|
|
5521
6694
|
.map((t, index) => ({
|
|
6695
|
+
index,
|
|
5522
6696
|
label: triggerLabel(t, index),
|
|
5523
6697
|
flow: t.run.flow,
|
|
5524
6698
|
kind: t.run.kind,
|
|
5525
6699
|
folder: typeof t.run.folder === "string" ? t.run.folder : null,
|
|
5526
6700
|
skillsDir: typeof t.run.skillsDir === "string" ? t.run.skillsDir : null,
|
|
5527
6701
|
packages: t.run.packages !== false,
|
|
6702
|
+
excludeExtensions: Array.isArray(t.run.excludeExtensions) ? t.run.excludeExtensions : [],
|
|
5528
6703
|
}))
|
|
5529
6704
|
.filter((f) => typeof f.flow === "string"),
|
|
6705
|
+
// Issue #800: each trigger's run.excludeExtensions, for the line that says a name is on neither side of this host.
|
|
6706
|
+
excludedNames: triggers.flatMap((t, index) => (Array.isArray(t.run.excludeExtensions) ? [{ label: triggerLabel(t, index), names: t.run.excludeExtensions, packages: t.run.packages !== false }] : [])),
|
|
5530
6707
|
// Issue #501: every trigger that sets run.maxCostUsd, with the label the cap check names it by. The value
|
|
5531
6708
|
// already parsed (the loader validated it), so the check below only compares.
|
|
5532
6709
|
costCaps: triggers.map((t, index) => ({ label: triggerLabel(t, index), maxCostUsd: t.run.maxCostUsd })).filter((r) => r.maxCostUsd !== undefined && r.maxCostUsd !== null),
|
|
@@ -5535,9 +6712,13 @@ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
|
|
|
5535
6712
|
// a model the worker does not know is refused wherever the job runs.
|
|
5536
6713
|
modelRuns: triggers.map((t, index) => ({
|
|
5537
6714
|
label: triggerLabel(t, index),
|
|
6715
|
+
kind: t.run.kind,
|
|
6716
|
+
folder: t.run.folder,
|
|
6717
|
+
repository: t.run.repository,
|
|
5538
6718
|
provider: typeof t.run.provider === "string" ? t.run.provider : undefined,
|
|
5539
6719
|
model: typeof t.run.model === "string" ? t.run.model : undefined,
|
|
5540
6720
|
models: Array.isArray(t.run.models) ? t.run.models : undefined,
|
|
6721
|
+
fallbackModels: Array.isArray(t.run.fallbackModels) ? t.run.fallbackModels : undefined,
|
|
5541
6722
|
maxCostUsd: t.run.maxCostUsd ?? undefined,
|
|
5542
6723
|
secretNames: t.run.secrets !== undefined && t.run.secrets !== null ? Object.keys(t.run.secrets) : [],
|
|
5543
6724
|
})),
|
|
@@ -5562,7 +6743,9 @@ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
|
|
|
5562
6743
|
// a directory), which is not a statement about the file's CONTENT and has no fix an operator can act
|
|
5563
6744
|
// on from this line. Those keep the old silent zeroes.
|
|
5564
6745
|
if (e?.piDispatchConfig !== true) return none;
|
|
5565
|
-
|
|
6746
|
+
// Issue #761: `parseNewer` is the loader's own flag for a file a newer pi-dispatch wrote, so the fix below says to
|
|
6747
|
+
// upgrade rather than to edit it.
|
|
6748
|
+
return { ...none, parseError: e.message, parseNewer: e.newer === true, path: triggersPath(env, cwd) };
|
|
5566
6749
|
}
|
|
5567
6750
|
}
|
|
5568
6751
|
|
|
@@ -6189,7 +7372,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
|
|
|
6189
7372
|
// ONE detach gate for this doctor run (issue #452, gate round 3), shared by the sweep and the canary: `local` can be a
|
|
6190
7373
|
// rootless Podman 4.x reached as `docker`, where both of them detach the running proxy.
|
|
6191
7374
|
const gate = makeDetachGate(canaryDocker, { bin: "docker", ...(seams.readFactsOnce ? { readRuntime: async () => runtimeFromFacts(await seams.readFactsOnce()) } : {}) });
|
|
6192
|
-
checks.push(...(await sweepStaleCanaryNetworks({ run: canaryDocker, pid, isAlive, endpoint, gate })));
|
|
7375
|
+
if (!seams.setupReadOnly) checks.push(...(await sweepStaleCanaryNetworks({ run: canaryDocker, pid, isAlive, endpoint, gate })));
|
|
6193
7376
|
|
|
6194
7377
|
// `docker inspect` on the container, not `ps`: it answers present-vs-absent and running-vs-stopped in
|
|
6195
7378
|
// one call, and those are two different fixes. The FIELD_SEP habit is image-preflight.mjs's -- neither
|
|
@@ -6268,6 +7451,8 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
|
|
|
6268
7451
|
// `--live` reads, so the output never reads healthy right before saying the proxy is not this deployment's.
|
|
6269
7452
|
const staleRunning = up && !custom && judged.drift.length > 0;
|
|
6270
7453
|
const proxyCheck = {
|
|
7454
|
+
// Issue #737: the console reads whether the proxy runs by this id (`NAMED_CHECK_IDS`).
|
|
7455
|
+
id: "egress.proxy",
|
|
6271
7456
|
ok: up,
|
|
6272
7457
|
...(retried ? { warn: true } : {}),
|
|
6273
7458
|
label: up
|
|
@@ -6299,6 +7484,8 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
|
|
|
6299
7484
|
if (!custom) {
|
|
6300
7485
|
if (judged.drift.length > 0) {
|
|
6301
7486
|
checks.push({
|
|
7487
|
+
// This line stands in for the proxy's own (not pushed while `staleRunning`), so it carries that line's id.
|
|
7488
|
+
id: proxyCheck.id,
|
|
6302
7489
|
ok: false,
|
|
6303
7490
|
label: judged.outOfDate ? `Egress proxy is this deployment's proxy but out of date (${proxy}): ${judged.drift.join("; ")}` : `Egress proxy is running but is not this deployment's (${proxy}): ${judged.drift.join("; ")}`,
|
|
6304
7491
|
fix: `${replaceFix}; until then every job's egress runs through a policy this deployment did not ship`,
|
|
@@ -6347,6 +7534,10 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
|
|
|
6347
7534
|
const mounted = readIncludeBindSource(parsed?.mounts);
|
|
6348
7535
|
checks.push(endpointsIncludeCheck({ answer: inside, endpoints, bin: "docker", proxy, prefix: "", mounted, recreate: custom ? `recreate ${proxy} so it mounts the file anew` : `\`docker rm -f -v ${proxy}\`, then \`pi-dispatch up\` starts it on the file anew (a restart cuts running jobs' tunnels)` }));
|
|
6349
7536
|
}
|
|
7537
|
+
if (seams.setupReadOnly) {
|
|
7538
|
+
checks.push({ ok: false, warn: true, label: "Egress canary: not run by browser setup", fix: "run pi-dispatch doctor in the deployment terminal for the container probe" });
|
|
7539
|
+
return checks;
|
|
7540
|
+
}
|
|
6350
7541
|
if (imageCode !== 0) return checks;
|
|
6351
7542
|
// Issue #431: the canary itself is `runEgressCanary`, shared with the podman venue's `--live` and the conformance
|
|
6352
7543
|
// script. docker's probes keep `runCmdCapture` (its 30 s bound, stderr merged, SIGTERM at the bound) rather than the
|
|
@@ -7121,7 +8312,8 @@ export function allowlistHostAliases(text) {
|
|
|
7121
8312
|
}
|
|
7122
8313
|
|
|
7123
8314
|
/**
|
|
7124
|
-
* The
|
|
8315
|
+
* The services installed for THIS folder (the worker, the receiver and, since issue #708, the console), as
|
|
8316
|
+
* `[{ which, source, setup }]` (`setup` null for a unit
|
|
7125
8317
|
* rendered without `--env-setup`): an installed unit whose WorkingDirectory is this folder (systemd, launchd), or the
|
|
7126
8318
|
* nssm service by name on win32, where nssm keeps the folder in a separate AppDirectory property and there is exactly
|
|
7127
8319
|
* one machine-scoped service per name. Read ONCE (issue #481, PR #485's final review) and shared by the env-setup lines
|
|
@@ -7131,7 +8323,7 @@ async function installedServices(seams) {
|
|
|
7131
8323
|
const { cwd, spawn, fileExists, platform, home } = seams;
|
|
7132
8324
|
const found = [];
|
|
7133
8325
|
if (platform === "win32") {
|
|
7134
|
-
for (const which of ["worker", "receiver"]) {
|
|
8326
|
+
for (const which of ["worker", "receiver", "console"]) {
|
|
7135
8327
|
const service = `pi-dispatch-${which}`;
|
|
7136
8328
|
const got = await runCmdCapture(spawn, "nssm", ["get", service, "AppEnvironmentExtra"]);
|
|
7137
8329
|
// Not installed, or nssm not on PATH: silence. Same doctrine as check-ignore below -- a check
|
|
@@ -7700,7 +8892,7 @@ export function jobSizeChecks(env, { daemon = undefined, podman = undefined, cpu
|
|
|
7700
8892
|
return [{ ok: false, label: `job size does not parse: ${error.message}, so the worker REFUSES TO START`, fix: "set PI_JOB_MEMORY like 512m, 1536m or 4g and PI_JOB_CPUS like 0.5 or 2 (or unset them for 4g and 2), then re-run doctor" }];
|
|
7701
8893
|
}
|
|
7702
8894
|
const where = d.memSet || d.cpuSet ? "PI_JOB_MEMORY and PI_JOB_CPUS" : "the built-in default";
|
|
7703
|
-
const checks = [{ ok: true, label: `Job size: ${formatMemory(d.memMiB)} of memory with no swap beyond it, and the CPU weight of ${formatCpus(d.cpuCenti)} CPUs, per job (${where}; a project row's memory and cpus override it, docs/scoped-limits.md)` }];
|
|
8895
|
+
const checks = [{ ok: true, label: `Job size: ${formatMemory(d.memMiB)} of memory with no swap beyond it, and the CPU weight of ${formatCpus(d.cpuCenti)} CPUs, per job (${where}; a project row's memory and cpus override it, docs/scoped-limits.md); codemode workers: ${codemodeWorkersOf(d.memMiB)} (PI_CODEMODE_MAX_WORKERS, half the memory reserved for pi, tools and worker overhead; a sizing heuristic, not an RSS guarantee)` }];
|
|
7704
8896
|
const venues = [];
|
|
7705
8897
|
// `reason` is the reader's own token (`timeout`, `unparseable`, ...), never the runtime's text.
|
|
7706
8898
|
const reasonOf = (read) => (typeof read?.reason === "string" && /^[a-z0-9-]{1,40}$/.test(read.reason) ? read.reason : "not read");
|
|
@@ -7953,22 +9145,26 @@ export const CAPACITY_DOCTOR_TIMEOUT_MS = 5_000;
|
|
|
7953
9145
|
|
|
7954
9146
|
/**
|
|
7955
9147
|
* The capacity lines (issue #599, phase 2): the report over the last 7 days (`capacity.mjs`, the CLI's function), read
|
|
7956
|
-
* by `seams.readCapacity` (default: `readCapacityRecords` through a fail-fast client, `prune`-free and writing nothing
|
|
7957
|
-
*
|
|
9148
|
+
* by `seams.readCapacity` (default: `readCapacityRecords` through a fail-fast client, `prune`-free and writing nothing;
|
|
9149
|
+
* `seams.readCapacityRecords` stands in for `readCapacityRecords` alone, see `defaultReadCapacity`). Never throws and
|
|
9150
|
+
* never warns: an unreadable history is one line saying so.
|
|
7958
9151
|
*/
|
|
7959
9152
|
export async function doctorCapacity({ seams, env, home, url, hosts, localHost }) {
|
|
7960
9153
|
try {
|
|
7961
9154
|
const window = CAPACITY_WINDOWS["7d"];
|
|
7962
9155
|
const nowMs = (typeof seams.wallClock === "function" ? seams.wallClock : Date.now)();
|
|
7963
9156
|
const retention = typeof env.PI_LOG_RETENTION_DAYS === "string" && /^\d{1,6}$/.test(env.PI_LOG_RETENTION_DAYS) ? Number(env.PI_LOG_RETENTION_DAYS) : 30;
|
|
9157
|
+
// The mirror's cap (PI_RUN_MIRROR_MAX_RUNS, issue #761), so an index the workers may grow past the built-in cap is not
|
|
9158
|
+
// read as one a trim cut short. Lenient: a bad value is the worker's refusal, which `workerIntSettingChecks` names.
|
|
9159
|
+
const indexMax = runMirrorMaxRunsOf(env.PI_RUN_MIRROR_MAX_RUNS).value;
|
|
7964
9160
|
// The read is told to stop when the bound passes (`signal`): giving up on it is not enough, since a client still
|
|
7965
9161
|
// waiting on its connection would keep this process alive after doctor has printed everything.
|
|
7966
9162
|
const stop = new AbortController();
|
|
7967
|
-
const args = { url, logsDir: logsDirPath(env, home), sinceMs: nowMs - window.ms, nowMs, retentionDays: retention, localHost, signal: stop.signal };
|
|
9163
|
+
const args = { url, logsDir: logsDirPath(env, home), sinceMs: nowMs - window.ms, nowMs, retentionDays: retention, localHost, indexMax, signal: stop.signal };
|
|
7968
9164
|
let timer;
|
|
7969
9165
|
const read = await Promise.race([
|
|
7970
9166
|
Promise.resolve()
|
|
7971
|
-
.then(() => (seams.readCapacity ?? defaultReadCapacity)(args))
|
|
9167
|
+
.then(() => (seams.readCapacity ?? ((a) => defaultReadCapacity(a, seams.readCapacityRecords)))(args))
|
|
7972
9168
|
.finally(() => clearTimeout(timer)),
|
|
7973
9169
|
new Promise((resolve) => {
|
|
7974
9170
|
timer = setTimeout(() => {
|
|
@@ -8003,11 +9199,14 @@ export const CAPACITY_CONNECT_TIMEOUT_MS = 2_000;
|
|
|
8003
9199
|
/**
|
|
8004
9200
|
* The default capacity read: a fail-fast client, its connection bounded (`connectWithin`: a server that accepts TCP and
|
|
8005
9201
|
* never answers holds ioredis in its ready check forever) and DISCONNECTED when `signal` aborts, so a read doctor gave
|
|
8006
|
-
* up on cannot keep the process alive.
|
|
9202
|
+
* up on cannot keep the process alive. `readRecords` (doctor's `seams.readCapacityRecords`, for a test) stands in for
|
|
9203
|
+
* `readCapacityRecords` and receives exactly what it would, the mirror's cap included. The forbidden-read guard still
|
|
9204
|
+
* runs first, whether or not it is given: it is a seam for a test process that never forbade the real read, and it
|
|
9205
|
+
* lifts the guard for no other test.
|
|
8007
9206
|
*/
|
|
8008
|
-
async function defaultReadCapacity({ url, logsDir, sinceMs, nowMs, retentionDays, localHost, signal }) {
|
|
9207
|
+
async function defaultReadCapacity({ url, logsDir, sinceMs, nowMs, retentionDays, localHost, indexMax, signal }, readRecords = undefined) {
|
|
8009
9208
|
if (defaultCapacityReadForbidden) throw Object.assign(new Error("doctor reached its real capacity read in a test: pass readCapacity"), { [FORBIDDEN_READ]: true });
|
|
8010
|
-
const
|
|
9209
|
+
const readCapacityRecords = readRecords ?? (await import("./capacity-records.mjs")).readCapacityRecords;
|
|
8011
9210
|
let client = null;
|
|
8012
9211
|
// Measured on ioredis 5.11: `disconnect()` ends the socket and arms a timer (`disconnectTimeout`, 2 s by default)
|
|
8013
9212
|
// that destroys it unless the socket's `close` clears it first, and the client's own failed ready check disconnects
|
|
@@ -8036,7 +9235,7 @@ async function defaultReadCapacity({ url, logsDir, sinceMs, nowMs, retentionDays
|
|
|
8036
9235
|
}
|
|
8037
9236
|
}
|
|
8038
9237
|
try {
|
|
8039
|
-
return await readCapacityRecords({ redis: client, logsDir, sinceMs, nowMs, retentionDays, localHost, noMirrorReason: url ? "the Valkey did not answer: only this host's files were read" : "no Valkey to read: only this host's files were read" });
|
|
9238
|
+
return await readCapacityRecords({ redis: client, logsDir, sinceMs, nowMs, retentionDays, localHost, indexMax, noMirrorReason: url ? "the Valkey did not answer: only this host's files were read" : "no Valkey to read: only this host's files were read" });
|
|
8040
9239
|
} finally {
|
|
8041
9240
|
signal?.removeEventListener?.("abort", drop);
|
|
8042
9241
|
drop();
|
|
@@ -8059,6 +9258,239 @@ export function connectWithin(client, ms, signal) {
|
|
|
8059
9258
|
});
|
|
8060
9259
|
}
|
|
8061
9260
|
|
|
9261
|
+
// ---- issue #761: what an upgrade leaves behind ----------------------------------------------------------------------
|
|
9262
|
+
|
|
9263
|
+
/** A version a registry row may carry: npm's characters, at most 32 of them. Anything else, `""` included, is unknown. */
|
|
9264
|
+
const ROW_VERSION_RE = /^[0-9A-Za-z.+-]{1,32}$/;
|
|
9265
|
+
const rowVersion = (v) => (typeof v === "string" && ROW_VERSION_RE.test(v) ? v : null);
|
|
9266
|
+
|
|
9267
|
+
/**
|
|
9268
|
+
* The fleet's versions (issue #761): one `fleet.versions` check, or none. PURE.
|
|
9269
|
+
* - `installed`: this doctor's own package version (`PACKAGE_VERSION`), null when it could not be read;
|
|
9270
|
+
* - `self`: this host's own registry row (its worker's), null when it publishes none; `selfName` its name;
|
|
9271
|
+
* - `peers`: every other live worker's row.
|
|
9272
|
+
* A WARNING when the live workers do not all report one version (an unreadable one, which a worker publishes as `""`
|
|
9273
|
+
* when it cannot read its own manifest, or a row's value outside the charset, is its own group, "unknown"), or when
|
|
9274
|
+
* this doctor's version is not the one this host's worker row reports. The second is worded as exactly that, two facts,
|
|
9275
|
+
* never "not restarted" and never "the running worker": doctor may run from another install (npx, another checkout),
|
|
9276
|
+
* and then it is that, not the service, which differs; and a row outlives a worker that died without deleting it for up
|
|
9277
|
+
* to its TTL (`HOST_TTL_MS`, 90 s), so what reports the version is the row, not a worker known to run. The headline says
|
|
9278
|
+
* "differ" only of versions it read: two readable ones among the workers, or this doctor's against this host's row.
|
|
9279
|
+
* Where the only other group is "unknown" it says the versions "cannot all be read", since an unread version may be the
|
|
9280
|
+
* same one. The fix is the order of docs/upgrading.md for each host: pause, wait for no active job, restart, resume.
|
|
9281
|
+
* Not `service restart --drain`, which covers this host's queues alone (the shared one and its own) and resumes them
|
|
9282
|
+
* after its own restart, so it is not the fleet's pause: a peer's own queue keeps running under it. An ok line when
|
|
9283
|
+
* peers exist and every one, this host's included, reports one known version, naming them as the workers with a live
|
|
9284
|
+
* row (this host's worker may publish none); nothing on a single host whose worker row reports this doctor's version,
|
|
9285
|
+
* or publishes none, so its output is unchanged. Workers only: the receiver's status row is never a doctor input
|
|
9286
|
+
* (INT-RECEIVER-STATUS-KEY).
|
|
9287
|
+
*/
|
|
9288
|
+
export function fleetVersionChecks({ installed, self = null, selfName = null, peers = [] }) {
|
|
9289
|
+
const others = Array.isArray(peers) ? peers : [];
|
|
9290
|
+
const rows = [...(self ? [{ name: printable(String(selfName ?? self.name ?? "")), version: rowVersion(self.version) }] : []), ...others.map((p) => ({ name: printable(String(p?.name ?? "")), version: rowVersion(p?.version) }))];
|
|
9291
|
+
const groups = new Map();
|
|
9292
|
+
for (const r of rows) groups.set(r.version ?? "unknown", [...(groups.get(r.version ?? "unknown") ?? []), r.name]);
|
|
9293
|
+
const parts = [];
|
|
9294
|
+
if (groups.size > 1) parts.push(`the live workers report ${groups.size} versions: ${[...groups.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([v, names]) => `${v} (${names.sort().join(", ")})`).join(", ")}`);
|
|
9295
|
+
const mine = rowVersion(installed);
|
|
9296
|
+
const running = self ? rowVersion(self.version) : null;
|
|
9297
|
+
const apart = mine !== null && running !== null && mine !== running;
|
|
9298
|
+
if (apart) parts.push(`this doctor ${mine}, this host's worker row reports ${running}`);
|
|
9299
|
+
const warn = parts.length > 0;
|
|
9300
|
+
const differ = [...groups.keys()].filter((v) => v !== "unknown").length > 1 || apart;
|
|
9301
|
+
const agreed = !warn && others.length > 0 && groups.size === 1 && !groups.has("unknown");
|
|
9302
|
+
if (!warn && !agreed) return [];
|
|
9303
|
+
const fix = `finish the upgrade in one window (docs/upgrading.md): the same version on every host; on each host \`pi-dispatch pause\`, wait until \`pi-dispatch status\` shows no active job, restart the worker, then \`pi-dispatch resume\`${apart ? ". If this doctor is not the install the service runs (npx, another checkout), run that install's doctor instead" : ""}`;
|
|
9304
|
+
const one = rows.length === 1;
|
|
9305
|
+
const ok = `Fleet versions: the ${rows.length} ${one ? "worker with a live row reports" : "workers with a live row report"} ${[...groups.keys()][0]}`;
|
|
9306
|
+
return [{ id: "fleet.versions", ok: !warn, ...(warn ? { warn: true } : {}), label: warn ? `pi-dispatch versions ${differ ? "differ" : "cannot all be read"}: ${parts.join("; ")}` : ok, ...(warn ? { fix } : {}) }];
|
|
9307
|
+
}
|
|
9308
|
+
|
|
9309
|
+
/** The most the dollar window check may take, its records read and its counter read together. */
|
|
9310
|
+
export const DOLLAR_WINDOWS_DOCTOR_TIMEOUT_MS = 5_000;
|
|
9311
|
+
|
|
9312
|
+
const WINDOW_WORDS = Object.freeze({ day: { its: "today's", cap: "daily" }, week: { its: "this week's", cap: "weekly" }, month: { its: "this month's", cap: "monthly" } });
|
|
9313
|
+
/** A counter value as micro-dollars, sign kept: `-$2.00` for a counter a give-back took below 0. */
|
|
9314
|
+
const usdSigned = (micros) => (micros < 0 ? `-$${formatMicros(-micros)}` : `$${formatMicros(micros)}`);
|
|
9315
|
+
const utcMinute = (ms) => `${new Date(ms).toISOString().slice(0, 16).replace("T", " ")} UTC`;
|
|
9316
|
+
|
|
9317
|
+
/**
|
|
9318
|
+
* The dollar window reset check (issue #761, DES-DOLLAR-RESERVE-AND-SETTLE): whether a deployment dollar window's
|
|
9319
|
+
* counter holds less than this host's own run records settled into it, which only lost history explains (a flushed,
|
|
9320
|
+
* replaced or restored Valkey, an evicted key, a hand edit): the cap then counts from less than was spent and can let
|
|
9321
|
+
* at least the difference more be spent. One warning for every such window, or nothing.
|
|
9322
|
+
*
|
|
9323
|
+
* 1. The deployment's day, week and month caps, overlay over env (`deploymentSettingsOf`, `dollarWindowCaps`, which
|
|
9324
|
+
* throws on a value the worker refuses: that is its own line, and this check says nothing). None set, or no Valkey
|
|
9325
|
+
* this command may talk to (`url` null): nothing is read.
|
|
9326
|
+
* 2. This host's records FIRST (`seams.readDollarRecords`; default: `readLocalWindow` from the start of the earliest
|
|
9327
|
+
* active window, a file at a time until the bound passes), summed per window by `dollarWindowFloors` over this
|
|
9328
|
+
* host's name and the endpoint tag of `url` (`valkeyEndpointTag`, the worker's).
|
|
9329
|
+
* 3. THEN the counters of the windows whose sum is above 0 (`seams.readDollarCounters`; default: one MGET). In this
|
|
9330
|
+
* order every record read was settled before its counter was read, so a counter read after it cannot be
|
|
9331
|
+
* legitimately below it.
|
|
9332
|
+
* 4. A counter that is absent, or an integer below the sum (a negative one included), is a finding; one that is not
|
|
9333
|
+
* an integer is skipped.
|
|
9334
|
+
* Both default reads share one bound (`DOLLAR_WINDOWS_DOCTOR_TIMEOUT_MS`) and are refused in tests
|
|
9335
|
+
* (`forbidDefaultDollarWindowRead`). A read that fails or runs out of time says nothing: this check claims a reset only
|
|
9336
|
+
* where it sees one, and its silence is never a claim that nothing was lost. It sees this host's share only (another
|
|
9337
|
+
* host's records are not here) and only the deployment windows (a scoped, project or model key is a hash per ledger),
|
|
9338
|
+
* from the records written since issue #761 (`dollars.reservedIn`).
|
|
9339
|
+
*/
|
|
9340
|
+
export async function doctorDollarWindows({ seams = {}, env = {}, home, url, localHost, fileExists = existsSync }) {
|
|
9341
|
+
// Resolved first, then judged: `deploymentSettingsOf` never throws and hands back values as written, which
|
|
9342
|
+
// `dollarWindowCaps` may refuse.
|
|
9343
|
+
const settings = deploymentSettingsOf(env, settingsFilePath(env, home), fileExists);
|
|
9344
|
+
let caps = null;
|
|
9345
|
+
try {
|
|
9346
|
+
caps = dollarWindowCaps(settings);
|
|
9347
|
+
} catch {
|
|
9348
|
+
return [];
|
|
9349
|
+
}
|
|
9350
|
+
const valkey = url ? valkeyEndpointTag(url) : null;
|
|
9351
|
+
if (caps === null || valkey === null) return [];
|
|
9352
|
+
try {
|
|
9353
|
+
const nowMs = (typeof seams.wallClock === "function" ? seams.wallClock : Date.now)();
|
|
9354
|
+
const at = deploymentWindowsAt(nowMs);
|
|
9355
|
+
const active = DEPLOYMENT_WINDOWS.filter((w) => caps[w] !== null && caps[w] !== undefined && at[w].bucket !== null).map((w) => ({ window: w, cap: caps[w], ...at[w] }));
|
|
9356
|
+
if (active.length === 0) return [];
|
|
9357
|
+
// The bound runs on the real clock, never the injected one a test fixes: it measures how long this read takes.
|
|
9358
|
+
const clock = typeof seams.dollarWindowsClock === "function" ? seams.dollarWindowsClock : Date.now;
|
|
9359
|
+
const deadlineAt = clock() + (seams.dollarWindowsTimeoutMs ?? DOLLAR_WINDOWS_DOCTOR_TIMEOUT_MS);
|
|
9360
|
+
const read = await (seams.readDollarRecords ?? defaultReadDollarRecords)({ logsDir: logsDirPath(env, home), sinceMs: Math.min(...active.map((w) => w.startMs)), deadlineAt, clock });
|
|
9361
|
+
const floors = dollarWindowFloors(read?.records, { host: localHost, valkey, buckets: Object.fromEntries(active.map((w) => [w.window, w.bucket])) });
|
|
9362
|
+
const due = active.filter((w) => floors[w.window] > 0);
|
|
9363
|
+
if (due.length === 0) return [];
|
|
9364
|
+
const counters = await (seams.readDollarCounters ?? defaultReadDollarCounters)({ url, keys: due.map((w) => w.key), deadlineAt, clock });
|
|
9365
|
+
if (!Array.isArray(counters) || counters.length !== due.length) return [];
|
|
9366
|
+
const found = [];
|
|
9367
|
+
due.forEach((w, i) => {
|
|
9368
|
+
const raw = counters[i];
|
|
9369
|
+
if (raw === null || raw === undefined) return void found.push({ ...w, floor: floors[w.window], counter: null });
|
|
9370
|
+
const text = typeof raw === "number" ? String(raw) : raw;
|
|
9371
|
+
if (typeof text !== "string" || !/^-?\d{1,16}$/.test(text) || !Number.isSafeInteger(Number(text))) return;
|
|
9372
|
+
if (Number(text) < floors[w.window]) found.push({ ...w, floor: floors[w.window], counter: Number(text) });
|
|
9373
|
+
});
|
|
9374
|
+
return dollarWindowResetChecks(found);
|
|
9375
|
+
} catch (err) {
|
|
9376
|
+
if (err?.[FORBIDDEN_READ]) throw err;
|
|
9377
|
+
return [];
|
|
9378
|
+
}
|
|
9379
|
+
}
|
|
9380
|
+
|
|
9381
|
+
/**
|
|
9382
|
+
* The one `dollars.window-reset` warning for `found` (`doctorDollarWindows`), each `{ window, key, cap, floor, counter,
|
|
9383
|
+
* ttlSeconds, endMs }` (`counter` null for an absent key), or none for an empty list. PURE. It says "at least" and
|
|
9384
|
+
* "this host's" wherever it gives an amount: the other hosts' records are not here, so the real shortfall can only be
|
|
9385
|
+
* larger. The fix puts the shortfall back (`INCRBY`, then `EXPIRE` with the window's own TTL, printed, and without
|
|
9386
|
+
* `NX`, which an operator's older Valkey may not take), lowers the cap, or pauses until the window ends. On a fleet
|
|
9387
|
+
* the amount is the sum of every host's settled share less what the counter holds, never the sum of each host's
|
|
9388
|
+
* shortfall: each shortfall subtracts the counter, so adding them subtracts it once per host and puts back too little.
|
|
9389
|
+
* The sum is in micro-dollars, as `INCRBY` takes it, while the label names dollars.
|
|
9390
|
+
*/
|
|
9391
|
+
export function dollarWindowResetChecks(found) {
|
|
9392
|
+
if (!Array.isArray(found) || found.length === 0) return [];
|
|
9393
|
+
const parts = found.map((f) => {
|
|
9394
|
+
const short = f.floor - (f.counter ?? 0);
|
|
9395
|
+
const holds = f.counter === null ? "is missing" : `holds ${usdSigned(f.counter)}`;
|
|
9396
|
+
return `${WINDOW_WORDS[f.window].its} (${f.key}) ${holds} while this host's own run records settled $${formatMicros(f.floor)} into it, so the $${formatMicros(f.cap)} ${WINDOW_WORDS[f.window].cap} cap counts at least $${formatMicros(short)} too little`;
|
|
9397
|
+
});
|
|
9398
|
+
const put = found.map((f) => `\`valkey-cli INCRBY ${f.key} ${f.floor - (f.counter ?? 0)}\` then \`valkey-cli EXPIRE ${f.key} ${f.ttlSeconds}\``);
|
|
9399
|
+
const ends = found.map((f) => `${WINDOW_WORDS[f.window].its} at ${utcMinute(f.endMs)}`);
|
|
9400
|
+
const one = found.length === 1;
|
|
9401
|
+
return [
|
|
9402
|
+
{
|
|
9403
|
+
id: "dollars.window-reset",
|
|
9404
|
+
ok: false,
|
|
9405
|
+
warn: true,
|
|
9406
|
+
label: `${one ? "A dollar window counter is" : "Dollar window counters are"} below what this host's own runs settled into ${one ? "it" : "them"}, so ${one ? "its" : "their"} history was lost (a flushed, replaced or restored Valkey, an evicted key, or a hand edit) and the cap can let at least that much more be spent: ${parts.join("; ")}. Other hosts' runs are not counted here`,
|
|
9407
|
+
fix: `put back at least this host's share for the rest of the window: ${put.join("; ")} (on a fleet, run doctor on every host first, then \`INCRBY\` the sum, in micro-dollars, of what each says its records settled, less what the counter holds now), or lower the cap by what was already spent, or run \`pi-dispatch pause\` and resume once the window ends (${ends.join("; ")}). A counter lowered on purpose is named here until its window ends (docs/costs.md, "When a counter resets")`,
|
|
9408
|
+
},
|
|
9409
|
+
];
|
|
9410
|
+
}
|
|
9411
|
+
|
|
9412
|
+
/**
|
|
9413
|
+
* Set by the test helper (test/helpers/doctor.mjs), as `forbidDefaultCapacityRead` is: from then on the dollar window
|
|
9414
|
+
* check's DEFAULT reads throw an error doctor does not swallow, so a test that reaches the developer's real logs
|
|
9415
|
+
* directory and Valkey fails loudly instead of judging whatever that machine holds.
|
|
9416
|
+
*/
|
|
9417
|
+
let defaultDollarWindowReadForbidden = false;
|
|
9418
|
+
export function forbidDefaultDollarWindowRead() {
|
|
9419
|
+
defaultDollarWindowReadForbidden = true;
|
|
9420
|
+
}
|
|
9421
|
+
const forbiddenDollarRead = () => Object.assign(new Error("doctor reached its real dollar window read in a test: pass readDollarRecords and readDollarCounters"), { [FORBIDDEN_READ]: true });
|
|
9422
|
+
|
|
9423
|
+
/** The default records read: this host's files, a file at a time until `deadlineAt` (`readLocalWindow`). */
|
|
9424
|
+
async function defaultReadDollarRecords({ logsDir, sinceMs, deadlineAt, clock }) {
|
|
9425
|
+
if (defaultDollarWindowReadForbidden) throw forbiddenDollarRead();
|
|
9426
|
+
const { readLocalWindow } = await import("./capacity-records.mjs");
|
|
9427
|
+
return readLocalWindow(logsDir, { sinceMs, deadlineAt, clock });
|
|
9428
|
+
}
|
|
9429
|
+
|
|
9430
|
+
/**
|
|
9431
|
+
* The default counters read: one MGET through a fail-fast client, its connection and its command each bounded by what
|
|
9432
|
+
* is left before `deadlineAt`, so the whole check stays inside its bound. Null when it could not be read.
|
|
9433
|
+
*/
|
|
9434
|
+
async function defaultReadDollarCounters({ url, keys, deadlineAt, clock }) {
|
|
9435
|
+
if (defaultDollarWindowReadForbidden) throw forbiddenDollarRead();
|
|
9436
|
+
const left = () => Math.max(0, deadlineAt - clock());
|
|
9437
|
+
if (left() === 0) return null;
|
|
9438
|
+
try {
|
|
9439
|
+
return await withDoctorClient(url, (client) => boundedOp(client.mget(...keys), left()), { connectTimeoutMs: left() });
|
|
9440
|
+
} catch {
|
|
9441
|
+
return null;
|
|
9442
|
+
}
|
|
9443
|
+
}
|
|
9444
|
+
|
|
9445
|
+
/**
|
|
9446
|
+
* The findings of older-state.mjs (`olderStateFindings({ cwd, home, readFile, valkeyContainer, handedOver })`, a promise
|
|
9447
|
+
* of `{ store, label, fix }` each), through `seams.olderStateFindings` when given. Never throws: an answer of another
|
|
9448
|
+
* shape is no finding, and every label and fix is made printable here, the printer not relying on it.
|
|
9449
|
+
*/
|
|
9450
|
+
async function olderStateOf(seams, { cwd, home, valkeyContainer, handedOver }) {
|
|
9451
|
+
try {
|
|
9452
|
+
const find = typeof seams.olderStateFindings === "function" ? seams.olderStateFindings : olderStateFindingsReal;
|
|
9453
|
+
const found = await find({ cwd, home, readFile: (path) => readFileSync(path, "utf8"), valkeyContainer, handedOver });
|
|
9454
|
+
return (Array.isArray(found) ? found : []).filter((f) => typeof f?.label === "string" && typeof f?.fix === "string").map((f) => ({ label: printable(f.label), fix: printable(f.fix) }));
|
|
9455
|
+
} catch {
|
|
9456
|
+
return [];
|
|
9457
|
+
}
|
|
9458
|
+
}
|
|
9459
|
+
|
|
9460
|
+
/**
|
|
9461
|
+
* older-state.mjs's `valkeyContainer`: `valkeyContainerOwner`'s verdict on the container `up` starts the deployment's
|
|
9462
|
+
* Valkey as (`docker container inspect pi-dispatch-valkey`, judged by up's own rule with this folder, as given and as its
|
|
9463
|
+
* real path, and VALKEY_URL's port), or null; it never throws. A container docker cannot describe, or none, is a verdict
|
|
9464
|
+
* that is not `ours`, which names nothing. Null itself, so the container is not asked about at all, unless:
|
|
9465
|
+
* - `asked`: this deployment runs jobs on Docker and the daemon answered `docker info`, since doctor spawns no docker
|
|
9466
|
+
* anywhere else (issue #433), and VALKEY_URL was read as the service reads it (the caller's gate);
|
|
9467
|
+
* - and VALKEY_URL names this host by an IPv4 loopback address or `localhost`, where `up` runs a Valkey (it adds none
|
|
9468
|
+
* for another host's, and refuses an IPv6 literal). Elsewhere a `pi-dispatch-valkey` here is not the queue the
|
|
9469
|
+
* worker dials, whatever port it publishes, and the fix would remove it.
|
|
9470
|
+
*/
|
|
9471
|
+
function valkeyContainerReader({ spawn, runTimeouts, asked, cwd, valkeyUrl }) {
|
|
9472
|
+
if (!asked || typeof cwd !== "string" || cwd === "") return null;
|
|
9473
|
+
const target = valkeyTarget(valkeyUrl);
|
|
9474
|
+
if (target.error || !isLoopbackHost(target.host) || target.host.includes(":")) return null;
|
|
9475
|
+
let deployment = cwd;
|
|
9476
|
+
try {
|
|
9477
|
+
deployment = realpathSync(cwd);
|
|
9478
|
+
} catch {
|
|
9479
|
+
// Unresolvable: the folder as given, as `up` does.
|
|
9480
|
+
}
|
|
9481
|
+
const query = async (cmd, args) => {
|
|
9482
|
+
const r = await runCmdCapture(spawn, cmd, args, { stdoutOnly: true, timeoutMs: runTimeouts?.cmd ?? RUN_TIMEOUTS.cmd });
|
|
9483
|
+
return { code: r.code, stdout: r.output, stderr: "" };
|
|
9484
|
+
};
|
|
9485
|
+
return async () => {
|
|
9486
|
+
try {
|
|
9487
|
+
return await valkeyContainerOwner(VALKEY_CONTAINER, { dirs: [deployment, cwd], port: target.port, query });
|
|
9488
|
+
} catch {
|
|
9489
|
+
return null;
|
|
9490
|
+
}
|
|
9491
|
+
};
|
|
9492
|
+
}
|
|
9493
|
+
|
|
8062
9494
|
/**
|
|
8063
9495
|
* One fact line per host of a capacity report (`INT-CAPACITY-REPORT`), `{ ok: true, label }` each, never a warning:
|
|
8064
9496
|
* `Host a: last 7d busy 63% (avg 2.1 of 4 slots, full 12%), promised 48% memory / 40% CPU, used 18% CPU of 8, wait p50
|
|
@@ -8329,6 +9761,13 @@ export function backendChecks(env, { endpoint = null, daemon = null, fs = { stat
|
|
|
8329
9761
|
// that could not start.
|
|
8330
9762
|
checks.push({ ok: false, label: `PI_EGRESS does not parse, so what this deployment actually gets cannot be determined: ${error.message}`, fix: 'set PI_EGRESS to exactly "0" (off) or "1"/unset (on)' });
|
|
8331
9763
|
}
|
|
9764
|
+
// Issue #684 (review): the worker's own boot rule for a layer venue with egress off, replayed from `backendRefusals`
|
|
9765
|
+
// so doctor and the worker say one sentence. Only that rule: the floor rules are this section's own lines below.
|
|
9766
|
+
if (switches.PI_EGRESS === false) {
|
|
9767
|
+
for (const refusal of backendRefusals({ backends, backendFloor: {}, egress: false }).filter((r) => r.startsWith("PI_EGRESS=0 cannot run"))) {
|
|
9768
|
+
checks.push({ ok: false, label: refusal, fix: "unset PI_EGRESS (armed is the default), or remove that backend from PI_BACKENDS: the worker refuses to boot otherwise" });
|
|
9769
|
+
}
|
|
9770
|
+
}
|
|
8332
9771
|
|
|
8333
9772
|
// The dispatch landed in slice 4, so the "nothing selects yet" qualifier came off -- and it came off HERE
|
|
8334
9773
|
// as well as in the code, because a stale caveat on the one surface that makes the table admissible is
|
|
@@ -8340,8 +9779,12 @@ export function backendChecks(env, { endpoint = null, daemon = null, fs = { stat
|
|
|
8340
9779
|
const d = declarationOf(name, property);
|
|
8341
9780
|
if (!d) continue;
|
|
8342
9781
|
// FIRST, ahead of the gate: a control that does not exist is not a control that is unarmed.
|
|
9782
|
+
// A WARNING since issue #707, no longer a ✗: since the openshell venues, a blessed venue legitimately declares
|
|
9783
|
+
// properties absent (`localFolders`, `pausable`), and a ✗ here failed doctor, and so `up`, on every such
|
|
9784
|
+
// deployment. Whether the deployment NEEDS the property is the floor's to say, and that refusal is its own ✗
|
|
9785
|
+
// below (`PI_BACKEND_FLOOR is not met`), as the worker's boot refusal is unchanged.
|
|
8343
9786
|
if (d.word === ABSENT) {
|
|
8344
|
-
checks.push({ ok: false, label: `${name}: ${property} is ABSENT -- ${d.question}`, fix: `this backend does not provide ${property}
|
|
9787
|
+
checks.push({ ok: false, warn: true, label: `${name}: ${property} is ABSENT -- ${d.question}`, fix: `this backend does not provide ${property}: a PI_BACKEND_FLOOR naming ${property} refuses the boot while ${name} is blessed, so a deployment that needs it must not run jobs on ${name}` });
|
|
8345
9788
|
continue;
|
|
8346
9789
|
}
|
|
8347
9790
|
if (d.armedBy && switches[d.armedBy] === undefined) {
|
|
@@ -8894,6 +10337,8 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
|
|
|
8894
10337
|
const crashLoop = status === "stopped" || status === "restarting";
|
|
8895
10338
|
const retried = state.code === 0 && !proxyRunning && /^[a-z]{1,20}$/.test(status ?? "") && !STOPPED_PROXY_STATES.has(status) && !crashLoop;
|
|
8896
10339
|
checks.push({
|
|
10340
|
+
// Issue #737: docker's proxy line's id, for this venue's proxy (`REPEATED_CHECK_IDS`).
|
|
10341
|
+
id: "egress.proxy",
|
|
8897
10342
|
ok: proxyRunning,
|
|
8898
10343
|
...(retried ? { warn: true } : {}),
|
|
8899
10344
|
label: proxyRunning ? `podman: egress proxy running under this account's Podman (${proxy})` : state.code === 0 ? `podman: egress proxy is ${/^[a-z]{1,20}$/.test(status ?? "") ? status : "in a state it did not report"} under this account's Podman (${proxy})` : `podman: egress proxy is not under this account's Podman (${proxy})`,
|
|
@@ -9244,6 +10689,9 @@ export async function liveChecks(env, seams, facts) {
|
|
|
9244
10689
|
// endpoint the job-user line was decided from. The fixture's global directory is never relabelled, like a job's.
|
|
9245
10690
|
const relabel = relabelsPrivateMounts(facts.daemon?.answered ? facts.daemon.facts : null, facts.endpoint, ids.platform ?? seams.platform);
|
|
9246
10691
|
const result = await runLiveProbes({
|
|
10692
|
+
// Issue #681: the local venue's builder and CLI, named rather than defaulted.
|
|
10693
|
+
buildArgs: buildDockerRunArgs,
|
|
10694
|
+
bin: "docker",
|
|
9247
10695
|
image: facts.jobImage ?? jobImageOf(env).image,
|
|
9248
10696
|
// Issue #596: the deployment's default size, and the `--cpus` ceiling from the same `docker info` answer, so the
|
|
9249
10697
|
// probe is built at the size a job gets and reads back memory, swap, weight and ceiling against it.
|
|
@@ -9838,3 +11286,20 @@ export function liveRunVia(spawn, { bin = "docker" } = {}) {
|
|
|
9838
11286
|
child.on("close", (code) => finish(code, "close"));
|
|
9839
11287
|
});
|
|
9840
11288
|
}
|
|
11289
|
+
|
|
11290
|
+
/** Diagnose reviewed source bindings without resolving or printing credential values. */
|
|
11291
|
+
export function watchDoctorChecks({ sources = {}, rules = 0, env = {} } = {}) {
|
|
11292
|
+
const entries = Object.entries(sources);
|
|
11293
|
+
if (!entries.length) return [];
|
|
11294
|
+
const checks = [{ id: "watch.sources", ok: true, label: `${entries.length} watch source(s), ${rules} watch rule(s); daily caps are required at load` }];
|
|
11295
|
+
for (const [source, declaration] of entries) {
|
|
11296
|
+
const identity = createHash("sha256").update(source).digest("hex").slice(0, 16);
|
|
11297
|
+
const present = typeof env[declaration.secretEnv] === "string" && env[declaration.secretEnv].length > 0;
|
|
11298
|
+
checks.push({ id: `watch.secret-${identity}`, ok: present, ...(!present ? { warn: true } : {}), label: present ? `watch source ${source}: signing secret is available` : `watch source ${source}: signing secret ${declaration.secretEnv} is missing; push requests return 503`, ...(!present ? { fix: `Set ${declaration.secretEnv} on the receiver host, or use the configured pull source without serving push requests.` } : {}) });
|
|
11299
|
+
if (declaration.pull?.secretEnv) {
|
|
11300
|
+
const name = declaration.pull.secretEnv; const available = typeof env[name] === "string" && env[name].length > 0;
|
|
11301
|
+
checks.push({ id: `watch.pull-${identity}`, ok: available, ...(!available ? { warn: true } : {}), label: available ? `watch source ${source}: pull credential is available` : `watch source ${source}: pull credential ${name} is missing`, ...(!available ? { fix: `Set ${name} on the watch collector host.` } : {}) });
|
|
11302
|
+
}
|
|
11303
|
+
}
|
|
11304
|
+
return checks;
|
|
11305
|
+
}
|