@edgehero/pi-dispatch 4.3.1 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +84 -3
- package/README.md +21 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +118 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +136 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +190 -29
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +325 -0
- package/src/console-client.mjs +848 -0
- package/src/console-devices.mjs +372 -0
- package/src/console-install.mjs +239 -0
- package/src/console-open.mjs +99 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1657 -196
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +96 -6
- package/src/env-file.mjs +6 -4
- package/src/envelope.mjs +5 -3
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-host.mjs +42 -5
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +33 -3
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +734 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +4 -2
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +12 -2
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1550 -173
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +264 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +36 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +780 -38
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +402 -53
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/start.mjs
CHANGED
|
@@ -1,3 +1,32 @@
|
|
|
1
|
+
import { subscriptionRoutes } from "./subscription-runtime.mjs";
|
|
2
|
+
import { configuredSubscriptions } from "./subscription-config.mjs";
|
|
3
|
+
import { publishSubscriptionStatus } from "./subscription-status.mjs";
|
|
4
|
+
import { readBusy as canaryBusy } from "./canary-lock.mjs";
|
|
5
|
+
import { makeReadModelRoles } from "./model-roles-file.mjs";
|
|
6
|
+
import { makeWatchStore } from "./watch-store.mjs";
|
|
7
|
+
import { makeTouchStore } from "./touch-state.mjs";
|
|
8
|
+
import { makeTouchOutbox } from "./touch-outbox.mjs";
|
|
9
|
+
import { makeQuestionGroupStore } from "./question-groups.mjs";
|
|
10
|
+
import { makeQuestionGroupRuntime } from "./question-group-runtime.mjs";
|
|
11
|
+
import { makeAnswerDraftStore } from "./answerer.mjs";
|
|
12
|
+
import { makeAnswerDraftRuntime, readAnswerPending } from "./answerer-runtime.mjs";
|
|
13
|
+
import { forgeApiRoot } from "./forges.mjs";
|
|
14
|
+
import { enqueueAnswerDraftJob } from "./queue.mjs";
|
|
15
|
+
import { makeWatchRuntime } from "./watch-runtime.mjs";
|
|
16
|
+
import { makeWatchDrainer } from "./watch-drainer.mjs";
|
|
17
|
+
import { makeWatchIssuePublisher } from "./watch-issue.mjs";
|
|
18
|
+
import { makeWatchIssueAdapter } from "./watch-issue-adapter.mjs";
|
|
19
|
+
import { checkFailoverCandidate, failoverEgressRequired } from "./model-failover.mjs";
|
|
20
|
+
import { parseTriggers, parseTriggersDocument, TRIGGERS_FILE_VERSION } from "./triggers.mjs";
|
|
21
|
+
import { watchDurationMs } from "./watch-contract.mjs";
|
|
22
|
+
import { makeApprovalSessionCheck } from "./approval-session.mjs";
|
|
23
|
+
import { APPROVAL_FINISHED_TTL_MS, makeApprovalStore } from "./approval-state.mjs";
|
|
24
|
+
import { makeApprovalRuntime, approvalExpiryJobId } from "./approval-runtime.mjs";
|
|
25
|
+
import { makeCheckApprovalPolicy } from "./approval-policy.mjs";
|
|
26
|
+
import { APPROVAL_LABEL } from "./parked.mjs";
|
|
27
|
+
import { createForkStore } from "./fork-store.mjs";
|
|
28
|
+
import { makeCheckForkPolicy } from "./fork-policy.mjs";
|
|
29
|
+
import { thinkingLevelAllowed } from "./thinking-level.mjs";
|
|
1
30
|
import { readdirSync, readFileSync, statSync, watch } from "node:fs";
|
|
2
31
|
import { lookup as dnsLookup } from "node:dns/promises";
|
|
3
32
|
import { homedir, networkInterfaces, release as osRelease, userInfo } from "node:os";
|
|
@@ -8,7 +37,9 @@ import { valkeyClientContext } from "./valkey-endpoint.mjs";
|
|
|
8
37
|
import { authRefusalFor, makeRedisClient, onValkeyError, parseConnection, valkeyAuthState, valkeyPasswordFor } from "./connection.mjs";
|
|
9
38
|
import { reconcileGated, reloadSchedules } from "./cron.mjs";
|
|
10
39
|
import { makeGitHubAuth } from "./get-token.mjs";
|
|
11
|
-
import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING } from "./processor.mjs";
|
|
40
|
+
import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, checkDraftAnswerSource } from "./processor.mjs";
|
|
41
|
+
import { consoleDraftOf } from "./console-answer.mjs";
|
|
42
|
+
import { FAILED_COMMENT, FAILED_COMMENT_BY_REASON, RECOVERY_HOST_LOST_COMMENT } from "./comment-tables.mjs";
|
|
12
43
|
import { transientError } from "./transient.mjs";
|
|
13
44
|
import { makeGitHubHost } from "./github-host.mjs";
|
|
14
45
|
import { githubFailureFields } from "./octokit-log.mjs";
|
|
@@ -24,8 +55,10 @@ import { MAX_SLOTS, loadModelEndpoints, modelEndpointsPath, readOverlayModels }
|
|
|
24
55
|
import { builtinModel, checkModelsKnown } from "./model-catalog.mjs";
|
|
25
56
|
import { capabilityTokens, serializeCaps } from "./capabilities.mjs";
|
|
26
57
|
import { cronFingerprint } from "./fingerprint.mjs";
|
|
27
|
-
import { makeHostRegistry } from "./host-registry.mjs";
|
|
58
|
+
import { makeHostRegistry, readLiveHosts } from "./host-registry.mjs";
|
|
59
|
+
import { endpointStatusFields, ENDPOINT_STATUS_VERSION } from "./endpoint-status.mjs";
|
|
28
60
|
import { liveJobsFields, liveJobsOf } from "./live-jobs.mjs";
|
|
61
|
+
import { makeSandboxTelemetry, parseBudgetFrom, parseCpuReserve } from "./host-telemetry.mjs";
|
|
29
62
|
import { budgetField, readUserServiceLimits } from "./host-budget.mjs";
|
|
30
63
|
import { makeCpuReserve, reservePlan } from "./cpu-reserve.mjs";
|
|
31
64
|
import { makeImagePreflight } from "./image-preflight.mjs";
|
|
@@ -35,44 +68,73 @@ import { unenforcedSizeFlags } from "./job-size.mjs";
|
|
|
35
68
|
import { makeCollectChain } from "./outbox.mjs";
|
|
36
69
|
import { makeCollectPlan } from "./outbox-plan.mjs";
|
|
37
70
|
import { makePortfolioSnapshot } from "./portfolio-snapshot.mjs";
|
|
38
|
-
import { containerPackagePaths, readStageManifest } from "./packages.mjs";
|
|
71
|
+
import { containerPackagePaths, PACKAGES_SUBDIR, readStageManifest, STAGE_MANIFEST } from "./packages.mjs";
|
|
39
72
|
import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare.mjs";
|
|
40
|
-
import { listRunningSandboxes, makeSandboxNetworkSweeper, makeSandboxRuntimeWatch } from "./sandbox.mjs";
|
|
73
|
+
import { listRunningSandboxes, makeSandboxNetworkSweeper, makeSandboxRuntimeWatch, sandboxSyncRefusal } from "./sandbox.mjs";
|
|
74
|
+
import { createSandboxPreparer } from "./sandbox-prepare.mjs";
|
|
75
|
+
import { checkSandboxWorkspace } from "./sandbox-workspace-guard.mjs";
|
|
76
|
+
import { startSandboxPrepareWorker } from "./sandbox-prepare-requests.mjs";
|
|
77
|
+
import { makeSandboxInventory } from "./sandbox-inventory.mjs";
|
|
41
78
|
import { makeRetentionSweep } from "./retention-sweep.mjs";
|
|
42
|
-
import {
|
|
79
|
+
import { FORGE_JOB_RETENTION_S } from "./retention.mjs";
|
|
80
|
+
import { makeDoctorSpawner } from "./doctor-spawner.mjs";
|
|
81
|
+
import { hostHasPending, publishCanaryFacts, UPDATES_LOOK_EVERY_HOURS } from "./updates.mjs";
|
|
82
|
+
import { makePackageCanaryRuntime } from "./package-canary.mjs";
|
|
83
|
+
|
|
84
|
+
/** How often a host asks whether a person wants something of it, and how long applying it may take: several npm installs on a slow link. */
|
|
85
|
+
const UPDATES_RECONCILE_EVERY_MS = 2 * 60 * 1000;
|
|
86
|
+
const UPDATES_RECONCILE_TIMEOUT_MS = 15 * 60 * 1000;
|
|
87
|
+
import { fileURLToPath } from "node:url";
|
|
88
|
+
import { listSandboxes, sandboxDeadline, makeSandboxReaper } from "./sandbox-store.mjs";
|
|
43
89
|
import { makeSessionStore } from "./session-store.mjs";
|
|
44
90
|
import { scrubCredentials } from "./redact.mjs";
|
|
45
|
-
import { makeCheckOnceSpent, makeCheckPortfolioFlag, makeCheckWaitSkew, makeDisarmOnce } from "./triggers-file.mjs";
|
|
91
|
+
import { makeCheckOnceSpent, makeCheckPortfolioFlag, makeCheckWaitSkew, makeDisarmOnce, makeFindSecretBundle } from "./triggers-file.mjs";
|
|
92
|
+
import { makeBundleResolver } from "./secret-bundles.mjs";
|
|
46
93
|
import { WATCH_DEBOUNCE_MS, changedWhileArming, makeWatchCloser, readBeforeArming } from "./watch-closer.mjs";
|
|
47
|
-
import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
|
|
48
|
-
import { checkProjectRows, danglingProjectRows, dollarRowsWithoutCap, loadScopedLimits, SCOPED_LIMITS_VERSION, scopeClaimRows } from "./scoped-limits.mjs";
|
|
94
|
+
import { loadPauseWindows, PAUSE_WINDOWS_VERSION, pauseUntilMs } from "./pause-windows.mjs";
|
|
95
|
+
import { checkProjectRows, danglingProjectRows, dollarRowsWithoutCap, makeInFlight, loadScopedLimits, SCOPED_LIMITS_VERSION, scopeClaimRows } from "./scoped-limits.mjs";
|
|
49
96
|
import { escapeControls, loadProjects, projectOf, projectsFingerprint } from "./projects.mjs";
|
|
50
97
|
import { envelopeDigest, envelopeInsideJobPaths, loadEnvelopeChecked } from "./envelope.mjs";
|
|
51
98
|
import { NO_ENVELOPE_FINGERPRINT, makeAllocationAudit, makeAllocationLogReaper, makeAllocationState } from "./allocation.mjs";
|
|
99
|
+
import { makeConsoleAuditReaper } from "./console-audit.mjs";
|
|
52
100
|
import { optionalUsdMicros } from "./money.mjs";
|
|
53
101
|
import { makeOnFailure } from "./on-failure.mjs";
|
|
102
|
+
import { makeHookRunsPublisher } from "./hook-runs.mjs";
|
|
54
103
|
import { makeWaitChecker } from "./wait-check.mjs";
|
|
55
104
|
import { makeWaitState } from "./wait-state.mjs";
|
|
56
|
-
import { hostQueueName, makeQueue } from "./queue.mjs";
|
|
57
|
-
import { endpointShown, execDockerBounded,
|
|
105
|
+
import { hostQueueName, makeQueue, enqueueWatchJob, discoverHostQueuesReporting, QUEUE } from "./queue.mjs";
|
|
106
|
+
import { endpointShown, execDockerBounded, makeDockerEndpointResolver, makeJobContainerLister, makeLocalBackend, makeReaper, makeStopContainer, quotedShown } from "./backend-local.mjs";
|
|
58
107
|
import { NETNS_KEEPER_MIN_AGE_MS, NETNS_KEEPER_YOUNG_MARGIN_MS, runtimeFromFacts } from "./netns-keeper.mjs";
|
|
59
108
|
import { makeBackendRegistry, reapAll, resolveBackendName } from "./backend-registry.mjs";
|
|
60
|
-
import { DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, backendFor, isPerMachineHost, observationRefusalIsTransient, observationRefusals, unobservedFloor } from "./backends.mjs";
|
|
109
|
+
import { DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OPENSHELL_BACKEND, OPENSHELL_BIND_BACKEND, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, backendFor, hostBackendsField, isPerMachineHost, observationRefusalIsTransient, observationRefusals, unobservedFloor } from "./backends.mjs";
|
|
110
|
+
import { makeOpenshellBackend, makeOpenshellReaper, openshellBootVersions } from "./backend-openshell.mjs";
|
|
61
111
|
import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_INFO_TIMEOUT_MS, cachedPodmanInfo, decidePodmanJobUser, makePodmanBackend, makePodmanInfoReader, makePodmanReaper, observePodman, podmanConfRefusal, unavailableFor, podmanConfWidening, podmanJobUserRefusal, resolvePodmanImageUser } from "./backend-podman.mjs";
|
|
62
|
-
import {
|
|
112
|
+
import { makePodmanServiceReader, onceFs, makeRootfulMemory, observeHost, observeRootfulConf, readRootfulService, rootfulConfRefusal, rootfulConfRetries, rootfulUnreadList, runtimeObservationKey } from "./runtime-observations.mjs";
|
|
63
113
|
|
|
64
114
|
import { makeRunContainer } from "./run-container.mjs";
|
|
65
115
|
import { piProviders, providerKeyCandidates, resolveProviderCredential } from "./env-allowlist.mjs";
|
|
66
116
|
import { parseAllowlist } from "./egress-allowlist.mjs";
|
|
67
|
-
import {
|
|
68
|
-
import {
|
|
69
|
-
import {
|
|
70
|
-
import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, newerRecord, RUNNER_POLICY_REASONS, sanitizeJobId, UNREADABLE_RECORD } from "./run-history.mjs";
|
|
117
|
+
import { makeMcpCheck, reservedJobNames } from "./mcp-preflight.mjs";
|
|
118
|
+
import { makeSecretsResolver, realExecutable } from "./secrets.mjs";
|
|
119
|
+
import { servableSecretProfiles } from "./secret-profiles.mjs";
|
|
120
|
+
import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, newerRecord, recordOfThisJob, recordReadable, RUNNER_POLICY_REASONS, sanitizeJobId, UNREADABLE_RECORD } from "./run-history.mjs";
|
|
71
121
|
import { makeRunMirror, readMirroredRecord } from "./run-mirror.mjs";
|
|
72
|
-
import {
|
|
122
|
+
import { makeLiveTranscriptOpener } from "./live-transcript.mjs";
|
|
123
|
+
import { makeInboxRelay } from "./inbox-relay.mjs";
|
|
124
|
+
import { makeLateForge, settleInbox } from "./inbox-settle.mjs";
|
|
125
|
+
import { makeHeldSettle } from "./held-settle.mjs";
|
|
126
|
+
import { readProgressKey } from "./run-progress.mjs";
|
|
127
|
+
import { makeStatusCommenter, makeStatusCommentRegistry, renderStatusText, STATUS_OUTCOMES, STATUS_RETRYING } from "./status-comment.mjs";
|
|
128
|
+
import { readOverlay, resolveSettings, SETTINGS_VERSION } from "./runtime-settings.mjs";
|
|
129
|
+
import { JOB_DATA_VERSION } from "./job-data-version.mjs";
|
|
73
130
|
import { usdFingerprint } from "./dollar-fingerprint.mjs";
|
|
74
131
|
import { authoredCron, envelopeJobPaths, loadSchedules, servedSchedules } from "./schedules.mjs";
|
|
75
132
|
import { makeStallGuard } from "./scheduler-stall-guard.mjs";
|
|
133
|
+
import { makeWorkspaceCheckpointer } from "./workspace-checkpoint.mjs";
|
|
134
|
+
import { makeRunBudgetLedger } from "./run-budget-ledger.mjs";
|
|
135
|
+
import { normalizeHost } from "./run-checkpoint.mjs";
|
|
136
|
+
import { isRecoveryArmed, makeHostLostSweep, makeRecoveryEnqueuer, makeSalvager, pointedJobDirs, runIdOf, sweepOrphanJobDirs } from "./recovery-queue.mjs";
|
|
137
|
+
import { finishModelAttempt, modelAttemptIdOf } from "./model-attempts.mjs";
|
|
76
138
|
|
|
77
139
|
|
|
78
140
|
/** How long boot will wait for `docker image inspect` before shipping without a digest. */
|
|
@@ -142,6 +204,18 @@ const WORKER_VERSION = (() => {
|
|
|
142
204
|
// the same handle); re-exported here so every existing importer keeps its address.
|
|
143
205
|
export { makeWatchCloser } from "./watch-closer.mjs";
|
|
144
206
|
|
|
207
|
+
/** The bound on `recordRun`'s final status edit for an attempt that registered no commenter, as the commenter's own. */
|
|
208
|
+
const STATUS_UNREGISTERED_EDIT_MS = 4_000;
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* The approval expiry job's options (issue #649): its id is the request's own (`approvalExpiryJobId`), and it is a
|
|
212
|
+
* forge-kind job built from the request's `resumeJob`, so it is kept as long as a forge job (`FORGE_JOB_RETENTION_S`,
|
|
213
|
+
* in SECONDS: BullMQ's `removeOn*.age`), the window in which its id still answers a second enqueue as the same job.
|
|
214
|
+
*/
|
|
215
|
+
export function approvalExpiryJobOpts(jobId) {
|
|
216
|
+
return { jobId, attempts: 2, backoff: { type: "exponential", delay: 5000 }, removeOnComplete: { age: FORGE_JOB_RETENTION_S }, removeOnFail: { age: FORGE_JOB_RETENTION_S } };
|
|
217
|
+
}
|
|
218
|
+
|
|
145
219
|
/**
|
|
146
220
|
* Race a read against a fuse, and CLEAN THE FUSE UP whichever side wins (issue #300). The fuse is
|
|
147
221
|
* unref'd, deliberately: it exists so a wedged docker daemon cannot hold boot, and it must never itself
|
|
@@ -174,14 +248,175 @@ export async function settleWithin(promise, ms, fallback) {
|
|
|
174
248
|
}
|
|
175
249
|
}
|
|
176
250
|
|
|
251
|
+
/**
|
|
252
|
+
* How long a queue's close waits for each of its two steps before it stops waiting (issue #813): first for the
|
|
253
|
+
* connection's own start, then for the close itself, after which the queue is disconnected. Five seconds, the bound the
|
|
254
|
+
* backend registry refusal already gave the same handles, which reads it from here. Against a live server the start
|
|
255
|
+
* settles in milliseconds (it is the INFO below) and a queue's close in about as long, so neither bound is reached by a
|
|
256
|
+
* queue's close that would have ended. A Worker's close also waits for the job it is running, so the one Worker closed
|
|
257
|
+
* this way gets a longer close step (`SANDBOX_CONTROL_CLOSE_BOUND_MS`).
|
|
258
|
+
*/
|
|
259
|
+
export const QUEUE_CLOSE_BOUND_MS = 5_000;
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* The sandbox control Worker's bound for its CLOSE step (issue #813); its start keeps `QUEUE_CLOSE_BOUND_MS`. A
|
|
263
|
+
* Worker's close waits for the job it is running, and nothing aborts that job first: the control Worker is not among
|
|
264
|
+
* the workers `createWorker`'s stop cancels, and a human's preparation (a clone, a checkout) is not a paid run to cut
|
|
265
|
+
* short. Under the queues' five seconds a preparation still running was cut off with Valkey up (measured: an 8 s job
|
|
266
|
+
* left `active` at 5001 ms, where the unbounded close waited 8006 ms and the job completed). Twenty seconds, so the two
|
|
267
|
+
* waits together (25 s) stay inside the 30 s that systemd (`TimeoutStopSec`) and launchd (`ExitTimeOut`) give a stop
|
|
268
|
+
* (`service.mjs`). nssm waits 15 s after its Ctrl-C (`AppStopMethodConsole 15000`), so on Windows a preparation past
|
|
269
|
+
* about 15 s is still ended by nssm's later stop methods, as it was under the unbounded wait. One that outlasts this
|
|
270
|
+
* bound is disconnected like a queue, its job left `active` for BullMQ's stalled check.
|
|
271
|
+
*/
|
|
272
|
+
export const SANDBOX_CONTROL_CLOSE_BOUND_MS = 20_000;
|
|
273
|
+
|
|
274
|
+
const TIMED_OUT = Symbol("timed out");
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* `promise`'s own outcome, `{ value }` or `{ error }`, or `TIMED_OUT` once `ms` has passed; never rejects. The timer is
|
|
278
|
+
* REF'D, unlike `settleWithin`'s fuse, and cleared whichever side settles: a queue is closed where nothing else may hold
|
|
279
|
+
* the event loop (a refused boot draining to its exit code, a test's teardown), and an unref'd timer awaited there never
|
|
280
|
+
* fires, so node would abandon the await with the queue still open. Cleared, it holds the loop no longer than the wait.
|
|
281
|
+
*/
|
|
282
|
+
async function outcomeWithin(promise, ms) {
|
|
283
|
+
let timer = null;
|
|
284
|
+
try {
|
|
285
|
+
return await Promise.race([
|
|
286
|
+
Promise.resolve(promise).then(
|
|
287
|
+
(value) => ({ value }),
|
|
288
|
+
(error) => ({ error }),
|
|
289
|
+
),
|
|
290
|
+
new Promise((resolve) => {
|
|
291
|
+
timer = setTimeout(() => resolve(TIMED_OUT), ms);
|
|
292
|
+
}),
|
|
293
|
+
]);
|
|
294
|
+
} finally {
|
|
295
|
+
clearTimeout(timer);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* The close `closeQueueSettled` and `settleCloseOf` share. It waits for the queue's start (`waitUntilReady`), at most
|
|
301
|
+
* `readyMs`, its failure ignored; then runs `close`, at most `closeMs`. A close that settled in time answers with its
|
|
302
|
+
* own value or rejection. One that did not is left pending and the queue is disconnected, as the backend registry
|
|
303
|
+
* refusal does: a queue whose connection came up closes by sending QUIT and awaiting the reply, which a server that
|
|
304
|
+
* stopped answering never sends. The disconnect is not awaited (BullMQ's awaits the start, which may never come), and
|
|
305
|
+
* its rejection is observed so it cannot surface as an unhandled one.
|
|
306
|
+
*
|
|
307
|
+
* After the close, whatever its outcome, each of the queue's connections gets a no-op `error` listener. BullMQ's
|
|
308
|
+
* `RedisConnection` constructor re-emits its start's rejection as an `error` on itself (`initializing.catch`), and its
|
|
309
|
+
* close strips every listener from it first (`removeAllListeners`). When the close lands during a connect attempt
|
|
310
|
+
* after a failed one, on a connection that never became ready, the start then rejects with the failed attempt's error
|
|
311
|
+
* and the emit finds no listener: an unhandled rejection (measured: now and then against a refused port, and every
|
|
312
|
+
* close after about 10 s against a host that drops packets). Attached before the close, the listener would be stripped
|
|
313
|
+
* with the rest. Each in a try: the connections are BullMQ's own objects, and nothing here may replace the close's
|
|
314
|
+
* answer.
|
|
315
|
+
*/
|
|
316
|
+
async function closeWithin(queue, close, { readyMs, closeMs }) {
|
|
317
|
+
await outcomeWithin(Promise.resolve().then(() => queue.waitUntilReady?.()), readyMs);
|
|
318
|
+
const closed = await outcomeWithin(Promise.resolve().then(close), closeMs);
|
|
319
|
+
for (const connection of [queue.connection, queue.blockingConnection]) {
|
|
320
|
+
try {
|
|
321
|
+
connection?.on?.("error", () => {});
|
|
322
|
+
} catch {
|
|
323
|
+
// a connection that takes no listener has none to re-emit to
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
if (closed !== TIMED_OUT) {
|
|
327
|
+
if ("error" in closed) throw closed.error;
|
|
328
|
+
return closed.value;
|
|
329
|
+
}
|
|
330
|
+
try {
|
|
331
|
+
void Promise.resolve(queue.disconnect?.()).catch(() => {});
|
|
332
|
+
} catch {
|
|
333
|
+
// a queue with nothing left to drop
|
|
334
|
+
}
|
|
335
|
+
return undefined;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* Closes a BullMQ Queue only after its connection's own start has settled, and never throws. BullMQ's
|
|
340
|
+
* `RedisConnection.init` sends an INFO (the server's version and type) that nothing in BullMQ awaits; a queue closed
|
|
341
|
+
* while that INFO is still pending on a fail-fast client (`enableOfflineQueue: false`) rejects it as an UNHANDLED
|
|
342
|
+
* rejection, inside BullMQ ("Stream isn't writeable and enableOfflineQueue options is false", measured in CI on the
|
|
343
|
+
* boot that refuses right after making the reconcile queue, issue #645's PR). Waiting for `waitUntilReady` (its failure
|
|
344
|
+
* swallowed here) lets that INFO land or fail inside the init's own handling first.
|
|
345
|
+
*
|
|
346
|
+
* BOUNDED, by `QUEUE_CLOSE_BOUND_MS` (issue #813). A long-lived queue's start never ends when nothing answers
|
|
347
|
+
* VALKEY_URL: BullMQ's `waitUntilReady` resolves on `ready` and rejects only on `end`, and the client retries forever,
|
|
348
|
+
* so the unbounded wait held a stopping worker until its supervisor killed it, and the release workflow's suite for six
|
|
349
|
+
* hours. A fail-fast client's start does end, but not always inside the bound: against a host that drops packets the
|
|
350
|
+
* reconcile queue's client gives up after about 6.4 s (three 2 s connect timeouts and two 200 ms retry delays,
|
|
351
|
+
* `parseConnection`). Once the bound has passed the queue is closed anyway: BullMQ's close of a connection still
|
|
352
|
+
* starting disconnects it and swallows the start's own rejection, and the start's re-emitted `error` meets the
|
|
353
|
+
* listener `closeWithin` leaves behind. It is the hazard `receiver/src/start.mjs` avoids by releasing the raw client
|
|
354
|
+
* instead. Exported for the test, with the bound as an option for the same reason.
|
|
355
|
+
*/
|
|
356
|
+
export async function closeQueueSettled(queue, { boundMs = QUEUE_CLOSE_BOUND_MS } = {}) {
|
|
357
|
+
if (!queue) return;
|
|
358
|
+
try {
|
|
359
|
+
await closeWithin(queue, () => queue.close?.(), { readyMs: boundMs, closeMs: boundMs });
|
|
360
|
+
} catch {}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Routes a queue's own `close` through `closeQueueSettled`'s bounded wait, in place, so every closer that holds the
|
|
365
|
+
* queue (the closer list, the worker's shutdown) gets the settled close without a wrapper object. Unlike
|
|
366
|
+
* `closeQueueSettled` it keeps the close's own answer, a rejection included, when the close settles in time. Idempotent,
|
|
367
|
+
* and a queue with no `close` is left as it is. `closeBoundMs` bounds the close step alone, `boundMs` by default: a
|
|
368
|
+
* Worker whose close waits for a job it must let finish passes a longer one (`SANDBOX_CONTROL_CLOSE_BOUND_MS`).
|
|
369
|
+
*/
|
|
370
|
+
export function settleCloseOf(queue, { boundMs = QUEUE_CLOSE_BOUND_MS, closeBoundMs = boundMs } = {}) {
|
|
371
|
+
if (!queue || typeof queue.close !== "function" || queue.close.settled === true) return queue;
|
|
372
|
+
const close = queue.close.bind(queue);
|
|
373
|
+
const settled = (...args) => closeWithin(queue, () => close(...args), { readyMs: boundMs, closeMs: closeBoundMs });
|
|
374
|
+
settled.settled = true;
|
|
375
|
+
queue.close = settled;
|
|
376
|
+
return queue;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* The triggers file's live reload (issue #761), EXPORTED apart from its watcher like `reloadPauseWindows`. The cron
|
|
381
|
+
* reload is `reloadSchedules`'s, unchanged: a file that does not load keeps the running schedules
|
|
382
|
+
* (`schedules_reload_invalid`). One state rides beside it, `state.newer`: SET when the loader refused the file as one
|
|
383
|
+
* a newer pi-dispatch wrote (its configError's `.newer`), logged `triggers_reload_newer`, and CLEARED the next time the
|
|
384
|
+
* file loads. An edit that breaks the file leaves it as it was: that says nothing about what the newer file held. The
|
|
385
|
+
* processor defers every job while it is set (`triggersNewer`), since the last good copy every reader would fall back
|
|
386
|
+
* to is no longer what the operator wrote. `reload` and `load` are injectable for tests; the rest is
|
|
387
|
+
* `reloadSchedules`' own options.
|
|
388
|
+
*/
|
|
389
|
+
export async function reloadTriggersFile(config, queue, state, { reload = reloadSchedules, load = loadSchedules, ...options } = {}) {
|
|
390
|
+
let loaded = false;
|
|
391
|
+
let thrown = null;
|
|
392
|
+
const loadFn = (cfg, opts) => {
|
|
393
|
+
try {
|
|
394
|
+
const schedules = load(cfg, opts);
|
|
395
|
+
loaded = true;
|
|
396
|
+
return schedules;
|
|
397
|
+
} catch (error) {
|
|
398
|
+
thrown = error;
|
|
399
|
+
throw error;
|
|
400
|
+
}
|
|
401
|
+
};
|
|
402
|
+
const result = await reload(config, queue, { ...options, loadFn });
|
|
403
|
+
if (loaded) {
|
|
404
|
+
state.newer = false;
|
|
405
|
+
} else if (thrown?.newer === true) {
|
|
406
|
+
state.newer = true;
|
|
407
|
+
options.log?.("triggers_reload_newer", { reason: thrown?.message ?? String(thrown) });
|
|
408
|
+
}
|
|
409
|
+
return result;
|
|
410
|
+
}
|
|
411
|
+
|
|
177
412
|
/**
|
|
178
413
|
* Watch the DIRECTORY holding the triggers file (robust to the admin's atomic tmp+rename, which swaps the
|
|
179
414
|
* inode a file-watch would lose), debounce, and re-reconcile the cron schedulers on change via
|
|
180
|
-
* `
|
|
181
|
-
* schedulers. The FSWatcher is unref'd (the debounce it arms is NOT), and the returned
|
|
182
|
-
* `startWorker` registers so the watch dies with the worker that armed it (issue #295).
|
|
415
|
+
* `reloadTriggersFile` (which keeps `state.newer`, issue #761). Best-effort: a platform without `fs.watch` logs and
|
|
416
|
+
* the worker keeps its boot-time schedulers. The FSWatcher is unref'd (the debounce it arms is NOT), and the returned
|
|
417
|
+
* closer is what `startWorker` registers so the watch dies with the worker that armed it (issue #295).
|
|
183
418
|
*/
|
|
184
|
-
function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot, afterReload = () => {}) {
|
|
419
|
+
function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot, afterReload = () => {}, state = { newer: false }, watchFn = watch) {
|
|
185
420
|
const path = config.triggersFile;
|
|
186
421
|
const dir = dirname(path) || ".";
|
|
187
422
|
const file = basename(path);
|
|
@@ -193,13 +428,13 @@ function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot,
|
|
|
193
428
|
// (issue #386). A baseline taken here would measure the arming instead.
|
|
194
429
|
readBeforeArming(handles, readFile, atBoot);
|
|
195
430
|
try {
|
|
196
|
-
handles.watcher =
|
|
431
|
+
handles.watcher = watchFn(dir, (_event, changed) => {
|
|
197
432
|
if (handles.closed) return; // see makeWatchCloser: by construction, not by a delivery rule
|
|
198
433
|
if (changed && changed !== file) return; // only our file (a null name -> reload to be safe)
|
|
199
434
|
clearTimeout(handles.timer);
|
|
200
435
|
// Issue #504 part B: a cron folder or a skills dir an edit adds is a new job path, so the envelope's place is
|
|
201
436
|
// judged again after every triggers reload.
|
|
202
|
-
handles.timer = setTimeout(() => void
|
|
437
|
+
handles.timer = setTimeout(() => void reloadTriggersFile(config, queue, state, { log: closer.reloadLog, ref, registry, tz, fleet }).then(() => afterReload(closer.reloadLog)), WATCH_DEBOUNCE_MS);
|
|
203
438
|
});
|
|
204
439
|
handles.watcher.unref?.();
|
|
205
440
|
log("triggers_watching", { path });
|
|
@@ -211,35 +446,54 @@ function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot,
|
|
|
211
446
|
// changed.
|
|
212
447
|
if (changedWhileArming(handles, readFile)) {
|
|
213
448
|
log("triggers_reread_after_arming", { path });
|
|
214
|
-
void
|
|
449
|
+
void reloadTriggersFile(config, queue, state, { log: closer.reloadLog, ref, registry, tz, fleet }).then(() => afterReload(closer.reloadLog));
|
|
215
450
|
}
|
|
216
451
|
return closer;
|
|
217
452
|
}
|
|
218
453
|
|
|
454
|
+
/**
|
|
455
|
+
* The pause-windows reload, EXPORTED apart from its watcher so its two states are testable without fs.watch (the
|
|
456
|
+
* scoped-limits reload's precedent). A good file replaces `ref.current` and clears `ref.newer`. A bad edit keeps the
|
|
457
|
+
* last-good windows in effect (OQ-008 live-edit safety), logged `pause_windows_reload_invalid`: the pause gate never
|
|
458
|
+
* loses its config to a typo.
|
|
459
|
+
*
|
|
460
|
+
* A file written by a newer pi-dispatch (issue #761, the loader's configError with `.newer`) keeps the last-good
|
|
461
|
+
* windows too, and ALSO sets `ref.newer`, logged `pause_windows_reload_newer`: it is no typo, and those windows are not
|
|
462
|
+
* what the operator wrote any more, since the newer file may pause a job they would run. The processor defers every job
|
|
463
|
+
* (`newer-release`) while it is set, and only the next good reload clears it: an edit that then breaks the file says
|
|
464
|
+
* nothing about what the newer one paused.
|
|
465
|
+
*/
|
|
466
|
+
export function reloadPauseWindows(config, ref, log, { load = loadPauseWindows } = {}) {
|
|
467
|
+
try {
|
|
468
|
+
ref.current = load(config);
|
|
469
|
+
ref.newer = false;
|
|
470
|
+
log("pause_windows_reloaded", { count: ref.current.length });
|
|
471
|
+
} catch (err) {
|
|
472
|
+
if (err?.newer === true) {
|
|
473
|
+
ref.newer = true;
|
|
474
|
+
log("pause_windows_reload_newer", { reason: err?.message });
|
|
475
|
+
} else {
|
|
476
|
+
log("pause_windows_reload_invalid", { reason: err?.message });
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
219
481
|
/**
|
|
220
482
|
* Watch the DIRECTORY holding the pause-windows file (same atomic-rename robustness as the triggers watch)
|
|
221
|
-
* and hot-swap the in-memory windows in `ref.current` on change.
|
|
222
|
-
* effect (OQ-008 live-edit safety) — the pause gate never loses its config to a typo. Best-effort; the
|
|
483
|
+
* and hot-swap the in-memory windows in `ref.current` on change (`reloadPauseWindows`). Best-effort; the
|
|
223
484
|
* FSWatcher is unref'd and the returned closer stops the watch with the worker (issue #295).
|
|
224
485
|
*/
|
|
225
|
-
function watchPauseWindowsFile(config, ref, log, atBoot) {
|
|
486
|
+
function watchPauseWindowsFile(config, ref, log, atBoot, watchFn = watch) {
|
|
226
487
|
const path = config.pauseWindowsFile;
|
|
227
488
|
const dir = dirname(path) || ".";
|
|
228
489
|
const file = basename(path);
|
|
229
490
|
const handles = { watcher: null, timer: null, closed: false };
|
|
230
491
|
const closer = makeWatchCloser(handles, log);
|
|
231
|
-
const reload = () =>
|
|
232
|
-
try {
|
|
233
|
-
ref.current = loadPauseWindows(config);
|
|
234
|
-
closer.reloadLog("pause_windows_reloaded", { count: ref.current.length });
|
|
235
|
-
} catch (err) {
|
|
236
|
-
closer.reloadLog("pause_windows_reload_invalid", { reason: err?.message });
|
|
237
|
-
}
|
|
238
|
-
};
|
|
492
|
+
const reload = () => reloadPauseWindows(config, ref, closer.reloadLog);
|
|
239
493
|
const readFile = () => readFileSync(path, "utf8");
|
|
240
494
|
readBeforeArming(handles, readFile, atBoot); // the BOOT LOAD's own bytes; see watch-closer (issue #386)
|
|
241
495
|
try {
|
|
242
|
-
handles.watcher =
|
|
496
|
+
handles.watcher = watchFn(dir, (_event, changed) => {
|
|
243
497
|
if (handles.closed) return; // see makeWatchCloser: by construction, not by a delivery rule
|
|
244
498
|
if (changed && changed !== file) return;
|
|
245
499
|
clearTimeout(handles.timer);
|
|
@@ -563,6 +817,36 @@ function watchModelEndpointsFile(config, ref, log, atBoot) {
|
|
|
563
817
|
* per job (mintToken throws configError) rather than blocking startup. Collaborators are injectable
|
|
564
818
|
* (defaulting to the real ones) so the wiring is testable offline with no Redis and no forge.
|
|
565
819
|
*/
|
|
820
|
+
/**
|
|
821
|
+
* Whether a status comment for `jobId` could contain `phrase` (issue #644). The receiver starts a job on a comment that
|
|
822
|
+
* merely CONTAINS the phrase (receiver filter.mjs `body.includes`), so a bot comment matching its own deployment's phrase
|
|
823
|
+
* (a phrase like "pi-dispatch" or "Turn") would be read as a new request on every edit. The text is fixed words, the job
|
|
824
|
+
* id and integers, so this renders it for every word form it can take (singular and plural counts, each failure part,
|
|
825
|
+
* each phase form, running, every outcome and the retry) and compares with every digit run collapsed to one `0` on both sides: the
|
|
826
|
+
* integers are the only part that varies, and a phrase holding a number matches any number there. Conservative on
|
|
827
|
+
* purpose: a false yes costs one comment, a false no a comment loop.
|
|
828
|
+
*/
|
|
829
|
+
export function statusTextMayMatch(phrase, jobId, render = renderStatusText) {
|
|
830
|
+
if (typeof phrase !== "string" || phrase === "") return false;
|
|
831
|
+
const norm = (t) => String(t).replace(/[0-9]+/g, "0");
|
|
832
|
+
const needle = norm(phrase);
|
|
833
|
+
for (const c of [1, 2]) {
|
|
834
|
+
for (const failed of [0, c]) {
|
|
835
|
+
for (const interrupted of [0, c]) {
|
|
836
|
+
for (const phase of [null, { index: c - 1, done: null, total: null }, { index: c - 1, done: c, total: null }, { index: c - 1, done: c, total: c }]) {
|
|
837
|
+
for (const phases of [[], Array.from({ length: c + 1 }, () => "")]) {
|
|
838
|
+
for (const outcome of [undefined, ...STATUS_OUTCOMES]) {
|
|
839
|
+
const snapshot = { turn: c, steps: c, failed, interrupted, phase, phases };
|
|
840
|
+
if (norm(render({ jobId, snapshot, outcome })).includes(needle)) return true;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return false;
|
|
848
|
+
}
|
|
849
|
+
|
|
566
850
|
export async function startWorker(
|
|
567
851
|
env = process.env,
|
|
568
852
|
{
|
|
@@ -582,6 +866,7 @@ export async function startWorker(
|
|
|
582
866
|
makeAuth = makeGitHubAuth,
|
|
583
867
|
makeHost = makeGitHubHost,
|
|
584
868
|
createWorkerFn = createWorker,
|
|
869
|
+
makeRuntimeQueue: makeRuntimeQueueFn = makeQueue,
|
|
585
870
|
makeCpuReserve: makeCpuReserveFn = makeCpuReserve,
|
|
586
871
|
makeReaper: makeReaperFn = makeReaper,
|
|
587
872
|
makeBackendRegistry: makeBackendRegistryFn = makeBackendRegistry,
|
|
@@ -592,16 +877,40 @@ export async function startWorker(
|
|
|
592
877
|
makeLogSink: makeLogSinkFn = makeLogSink,
|
|
593
878
|
makeRecordWriter: makeRecordWriterFn = makeRecordWriter,
|
|
594
879
|
makeRunMirror: makeRunMirrorFn = makeRunMirror,
|
|
880
|
+
// Issue #761: the portfolio snapshot's factory, a seam so a wiring test sees the run index cap it is handed.
|
|
881
|
+
makePortfolioSnapshot: makePortfolioSnapshotFn = makePortfolioSnapshot,
|
|
595
882
|
// The fleet copy's by-id read, seamed so a wiring test can answer it without a live mirror.
|
|
596
883
|
readMirroredRecord: readMirroredRecordFn = readMirroredRecord,
|
|
597
884
|
makeLogReaper: makeLogReaperFn = makeLogReaper,
|
|
598
885
|
makeSandboxReaper: makeSandboxReaperFn = makeSandboxReaper,
|
|
886
|
+
createSandboxPreparer: createSandboxPreparerFn = createSandboxPreparer,
|
|
887
|
+
startSandboxPrepareWorker: startSandboxPrepareWorkerFn = startSandboxPrepareWorker,
|
|
888
|
+
makeSandboxInventory: makeSandboxInventoryFn = makeSandboxInventory,
|
|
889
|
+
listSandboxes: listSandboxesFn = listSandboxes,
|
|
599
890
|
makeSandboxNetworkSweeper: makeSandboxNetworkSweeperFn = makeSandboxNetworkSweeper,
|
|
600
891
|
// Issue #429: the one call that asks a runtime which sandboxes are open, seamed so a wiring test can drive the real
|
|
601
892
|
// sandbox reaper against a real retention root without spawning docker or podman.
|
|
602
893
|
listRunningSandboxes: listRunningSandboxesFn = listRunningSandboxes,
|
|
603
894
|
makeRetentionSweep: makeRetentionSweepFn = makeRetentionSweep,
|
|
895
|
+
makeTouchStore: makeTouchStoreFn = makeTouchStore,
|
|
896
|
+
makeTouchOutbox: makeTouchOutboxFn = makeTouchOutbox,
|
|
897
|
+
makeQuestionGroupStore: makeQuestionGroupStoreFn = makeQuestionGroupStore,
|
|
898
|
+
makeQuestionGroupRuntime: makeQuestionGroupRuntimeFn = makeQuestionGroupRuntime,
|
|
899
|
+
// Issue #712: the worker's own doctor runs, a seam so a wiring test sees the interval and never spawns a doctor.
|
|
900
|
+
makeDoctorSpawner: makeDoctorSpawnerFn = makeDoctorSpawner,
|
|
901
|
+
makeUpdatesRunner: makeUpdatesRunnerFn = makeDoctorSpawner,
|
|
604
902
|
makeRunContainer: makeRunContainerFn = makeRunContainer,
|
|
903
|
+
// Issue #643: the preparer factory, a seam so a test can see what the wiring hands it (`stageLive`).
|
|
904
|
+
makePrepareWorkspace: makePrepareWorkspaceFn = makePrepareWorkspace,
|
|
905
|
+
// Issue #644: the opener and the status commenter factories, seams so a wiring test sees the switches and the
|
|
906
|
+
// forge calls the wiring hands them.
|
|
907
|
+
makeLiveTranscriptOpener: makeLiveTranscriptOpenerFn = makeLiveTranscriptOpener,
|
|
908
|
+
// Issue #646: the steer inbox relay's factory, a seam so a wiring test sees the client and host it is bound to.
|
|
909
|
+
makeInboxRelay: makeInboxRelayFn = makeInboxRelay,
|
|
910
|
+
makeStatusCommenter: makeStatusCommenterFn = makeStatusCommenter,
|
|
911
|
+
// Issue #644: the progress key's bounded read, a seam so a wiring test can hand `recordRun` an earlier attempt's
|
|
912
|
+
// comment id without a Valkey.
|
|
913
|
+
readProgressKey: readProgressKeyFn = readProgressKey,
|
|
605
914
|
makeSecretsResolver: makeSecretsResolverFn = makeSecretsResolver,
|
|
606
915
|
makeImagePreflight: makeImagePreflightFn = makeImagePreflight,
|
|
607
916
|
makeScopeClaimSweeper: makeScopeClaimSweeperFn = makeScopeClaimSweeper,
|
|
@@ -636,6 +945,11 @@ export async function startWorker(
|
|
|
636
945
|
readPodmanInfo: readPodmanInfoFn = makePodmanInfoReader(),
|
|
637
946
|
makePodmanReaper: makePodmanReaperFn = makePodmanReaper,
|
|
638
947
|
makePodmanBackend: makePodmanBackendFn = makePodmanBackend,
|
|
948
|
+
// Issue #684: the openshell venue's three, podman's pattern: the boot's version read, its reaper and its bundle,
|
|
949
|
+
// each a seam because the real ones spawn `openshell`, and only while the venue is blessed.
|
|
950
|
+
readOpenshellVersions: readOpenshellVersionsFn = openshellBootVersions,
|
|
951
|
+
makeOpenshellReaper: makeOpenshellReaperFn = makeOpenshellReaper,
|
|
952
|
+
makeOpenshellBackend: makeOpenshellBackendFn = makeOpenshellBackend,
|
|
639
953
|
makeGitLabAuth: makeGitLabAuthFn = makeGitLabAuth,
|
|
640
954
|
makeGitLabHost: makeGitLabHostFn = makeGitLabHost,
|
|
641
955
|
makeForgejoAuth: makeForgejoAuthFn = makeForgejoAuth,
|
|
@@ -663,6 +977,9 @@ export async function startWorker(
|
|
|
663
977
|
// literal address every Valkey client of this worker then connects to. A seam: the real one reads /proc, probes
|
|
664
978
|
// this host's addresses and resolves the name, none of which belongs in a wiring test.
|
|
665
979
|
judgeValkey = defaultJudgeValkey,
|
|
980
|
+
// `fs.watch` for the triggers and pause-windows watchers, so a test can deliver a change itself: macOS can lose a
|
|
981
|
+
// real event outright (the rebuilt FSEvents stream, watch-closer.mjs), which is not what those tests are about.
|
|
982
|
+
watchFs = watch,
|
|
666
983
|
// The scoped-limits watcher, injectable so a test can see what it is armed with (PR #549's review: the cap
|
|
667
984
|
// thunk the reload warning reads). Production passes nothing and gets the real watcher.
|
|
668
985
|
watchScopedLimits: watchScopedLimitsFn = watchScopedLimitsFile,
|
|
@@ -670,6 +987,21 @@ export async function startWorker(
|
|
|
670
987
|
watchProjects: watchProjectsFn = watchProjectsFile,
|
|
671
988
|
// The envelope watcher (issue #504 part B), injectable for the same reason.
|
|
672
989
|
watchEnvelope: watchEnvelopeFn = watchEnvelopeFile,
|
|
990
|
+
// Issue #645 (durable runs): the checkpoint collaborators, seams so a wiring test sees what the boot hands each and
|
|
991
|
+
// what the processor and the opener are given. `loadRunCheckpoint` is the module holding the run checkpointer and
|
|
992
|
+
// the pointer reads (run-checkpoint.mjs), imported at boot rather than statically, so a test can hand in a double.
|
|
993
|
+
makeWorkspaceCheckpointer: makeWorkspaceCheckpointerFn = makeWorkspaceCheckpointer,
|
|
994
|
+
loadRunCheckpoint = () => import("./run-checkpoint.mjs"),
|
|
995
|
+
makeRunBudgetLedger: makeRunBudgetLedgerFn = makeRunBudgetLedger,
|
|
996
|
+
makeRecoveryEnqueuer: makeRecoveryEnqueuerFn = makeRecoveryEnqueuer,
|
|
997
|
+
// The host-lost sweep's factory: a wiring test reaches its `onLost` (the comment, the page, the record) without a
|
|
998
|
+
// live queue, since the harness's clients are closed by the time a test could drive a sweep.
|
|
999
|
+
makeHostLostSweep: makeHostLostSweepFn = makeHostLostSweep,
|
|
1000
|
+
// The boot sweep of orphaned job directories, a seam so a wiring test drives it against a directory of its own.
|
|
1001
|
+
sweepOrphanJobDirs: sweepOrphanJobDirsFn = sweepOrphanJobDirs,
|
|
1002
|
+
// Issue #812: the expiry a finished job's model attempt key is given, a seam so a wiring test sees which id the
|
|
1003
|
+
// listeners below hand it, since the harness's client is closed by the time a test drives one.
|
|
1004
|
+
finishModelAttempt: finishModelAttemptFn = finishModelAttempt,
|
|
673
1005
|
} = {},
|
|
674
1006
|
) {
|
|
675
1007
|
const config = loadConfigFn(env);
|
|
@@ -684,6 +1016,12 @@ export async function startWorker(
|
|
|
684
1016
|
// read, its reaper and its bundle exist only while it is blessed, so a deployment that never chose it spawns no
|
|
685
1017
|
// `podman` and pays nothing for it being in the table.
|
|
686
1018
|
const podmanBlessed = config.backends.includes(PODMAN_BACKEND);
|
|
1019
|
+
// Issue #684: and for the openshell venue, blessable since this issue: its CLI is spawned only while it is blessed.
|
|
1020
|
+
// Issue #686 part 3: either of its two modes, which share the gateway, the CLI, the version read and the boot reaper;
|
|
1021
|
+
// each mode's own bundle is built only while that mode is blessed.
|
|
1022
|
+
const openshellCopyBlessed = config.backends.includes(OPENSHELL_BACKEND);
|
|
1023
|
+
const openshellBindBlessed = config.backends.includes(OPENSHELL_BIND_BACKEND);
|
|
1024
|
+
const openshellBlessed = openshellCopyBlessed || openshellBindBlessed;
|
|
687
1025
|
// Issue #354: an ABSENT `observationPreflight` admits every job, which is the right answer only for a venue whose
|
|
688
1026
|
// words hold without observing anything. A venue whose table entry names an `observedBy` and carries no preflight
|
|
689
1027
|
// would have those words hold with nothing looking, and a floor naming one would pass on capability alone: the
|
|
@@ -719,7 +1057,10 @@ export async function startWorker(
|
|
|
719
1057
|
// worker judges it itself, before it contacts Valkey at all, with the same function (`resolveWorkerValkey`), and
|
|
720
1058
|
// every client below connects to the judged literal address, never to a name resolved again later. A refusal is a
|
|
721
1059
|
// configError (exit 2, not restarted into the same answer); nothing answering is a plain error (exit 1, restarted).
|
|
722
|
-
|
|
1060
|
+
// `openshellUsed` (issue #707, review round 2 of #715): an openshell venue on Podman is the queue's engine where no
|
|
1061
|
+
// `local` or `podman` venue is (`openshellSetupOf`), and the judgement refuses root's listener there as `up` does. Left
|
|
1062
|
+
// out, as it was, the rule saw no openshell venue and the boot took a root-held Valkey `up` had refused.
|
|
1063
|
+
const valkey = await judgeValkey({ url: config.valkeyUrl, venues: { localUsed: localBlessed, podmanUsed: podmanBlessed, openshellUsed: openshellBlessed }, env });
|
|
723
1064
|
for (const note of valkey.notes ?? []) log("valkey_note", { note });
|
|
724
1065
|
if (valkey.pinned) log("valkey_pinned", { address: valkey.pinned.address, port: valkey.pinned.port, heldBy: valkey.pinned.heldBy });
|
|
725
1066
|
// Every client below connects through connection.mjs' JudgedConnector, which judges the (pinned) address again on
|
|
@@ -751,11 +1092,16 @@ export async function startWorker(
|
|
|
751
1092
|
},
|
|
752
1093
|
});
|
|
753
1094
|
const schedules = { current: loadSchedules(config, { fleet: config.workerNameDeclared, ...recording("triggers", config.triggersFile) }) };
|
|
1095
|
+
// Issue #761: whether the triggers file on disk was written by a newer pi-dispatch, kept by the triggers watcher's
|
|
1096
|
+
// reload (`reloadTriggersFile`). At boot such a file is the loader's configError, so it starts false.
|
|
1097
|
+
const triggersState = { newer: false };
|
|
754
1098
|
|
|
755
1099
|
// REQ-SCOPED-PAUSE-WINDOWS: load + validate the pause-windows file with the operator present and before any
|
|
756
1100
|
// Valkey contact, so a malformed file refuses startup (configError) rather than silently disabling scoped
|
|
757
1101
|
// pauses. Held in a mutable ref so the live-reload watcher can hot-swap it. [] means no scoped pauses.
|
|
758
|
-
|
|
1102
|
+
// `newer` (issue #761) is set by a live reload that met a file written by a newer pi-dispatch, which keeps these
|
|
1103
|
+
// windows (`reloadPauseWindows`); at boot such a file is the loader's configError, so it starts false.
|
|
1104
|
+
const pauseWindows = { current: loadPauseWindows(config, recording("pauseWindows", config.pauseWindowsFile)), newer: false };
|
|
759
1105
|
|
|
760
1106
|
// Issue #242: same posture for the scoped-limits file -- fail-loud with the operator present, mutable
|
|
761
1107
|
// ref for the live-reload watcher, [] when unset (the folder mutex is code and needs no file).
|
|
@@ -889,6 +1235,18 @@ export async function startWorker(
|
|
|
889
1235
|
|
|
890
1236
|
const bootRefusal = jobUserBootRefusal(bootDecision, config.defaultBackend);
|
|
891
1237
|
if (bootRefusal) throw configError(bootRefusal);
|
|
1238
|
+
// Issue #684: the openshell venue's releases, read once at boot (the half #683 left to the blessing). A CLI or a gateway
|
|
1239
|
+
// at another release than the one this venue was measured on refuses the boot (exit 2); a CLI that does not answer is
|
|
1240
|
+
// restarted into (exit 1); a gateway that does not answer refuses nothing, since every job's preflight reads it again.
|
|
1241
|
+
// Issue #712: whether the gateway is on this host, as the boot read it, for the registry row's `backends` word.
|
|
1242
|
+
// `unknown` while not read (a gateway down at boot): a word published per beat must not spawn the CLI per beat.
|
|
1243
|
+
let openshellGatewayWord = "unknown";
|
|
1244
|
+
if (openshellBlessed) {
|
|
1245
|
+
const versions = await readOpenshellVersionsFn({ gateway: config.openshell.gateway });
|
|
1246
|
+
if (versions?.refusal) throw versions.transient ? new Error(versions.refusal) : configError(versions.refusal);
|
|
1247
|
+
if (versions?.gatewayDown) log("openshell_gateway_down_at_boot", { gateway: config.openshell.gateway });
|
|
1248
|
+
if (typeof versions?.gatewayLocal === "boolean") openshellGatewayWord = versions.gatewayLocal ? "local" : "remote";
|
|
1249
|
+
}
|
|
892
1250
|
// Issue #448: where a local job runs on rootful Podman's Docker API service on this host, the containers.conf that
|
|
893
1251
|
// service reads (the system chain, root's own, the unit's CONTAINERS_CONF) and whether the running service started
|
|
894
1252
|
// before it last changed. A VENUE refusal, as the podman venue's #428 one is and for its reason: with egress off no
|
|
@@ -1176,12 +1534,18 @@ export async function startWorker(
|
|
|
1176
1534
|
// boot reaper for a venue it does not hold.
|
|
1177
1535
|
// Issue #354: podman's the same way and for the same reason, `makeReaper` with `bin: "podman"`, so the sweep lists
|
|
1178
1536
|
// the store this account's jobs actually ran in. Docker's listing says nothing about it, and the reverse.
|
|
1537
|
+
// Issue #684: the openshell venue's, by its owner label: crashed jobs' sandboxes, then their providers. Issue #686 part
|
|
1538
|
+
// 3: ONE reaper for both modes, registered under each blessed name and run once: the owner label is shared, a crash's
|
|
1539
|
+
// leftover of either mode is an orphan alike, and at boot no job of this process is live in either.
|
|
1540
|
+
const openshellReap = openshellBlessed ? makeOpenshellReaperFn({ gateway: config.openshell.gateway, log }) : null;
|
|
1179
1541
|
backendReaps = {
|
|
1180
1542
|
...(localBlessed ? { [DEFAULT_BACKEND]: makeReaperFn({ log }) } : {}),
|
|
1181
1543
|
...(podmanBlessed ? { [PODMAN_BACKEND]: makePodmanReaperFn({ log }) } : {}),
|
|
1544
|
+
...(openshellCopyBlessed ? { [OPENSHELL_BACKEND]: openshellReap } : {}),
|
|
1545
|
+
...(openshellBindBlessed ? { [OPENSHELL_BIND_BACKEND]: openshellReap } : {}),
|
|
1182
1546
|
...Object.fromEntries(extraBackends.map((b) => [b?.name, b?.reap])),
|
|
1183
1547
|
};
|
|
1184
|
-
const swept = (await reapAll(Object.values(backendReaps), { log }))?.reaped === true;
|
|
1548
|
+
const swept = (await reapAll([...new Set(Object.values(backendReaps))], { log }))?.reaped === true;
|
|
1185
1549
|
// PROVEN FOR THE WHOLE HOST, or not at all (issue #354). `reapAll` is conservative over the reapers it is
|
|
1186
1550
|
// handed, and two gaps sit outside that. A BLESSED venue with no reaper here is refused by the registry, but
|
|
1187
1551
|
// only further down, after the scope sweep has already acted on this answer. And a host without `local` can
|
|
@@ -1216,6 +1580,13 @@ export async function startWorker(
|
|
|
1216
1580
|
// reaper above never reaches (it reaps the top-level `.log` and `.json` only). Never throws, so no double wrap.
|
|
1217
1581
|
const reapAllocationLogs = makeAllocationLogReaper({ logsDir: config.logsDir, retentionDays: config.logRetentionDays, log });
|
|
1218
1582
|
reapAllocationLogs();
|
|
1583
|
+
// Issue #640: the operator console's audit files (`console-audit/YYYY-MM.jsonl`). The worker runs it, not the console,
|
|
1584
|
+
// because the worker always runs and the console may never; it never reaps the newest file (the chain's head). Never
|
|
1585
|
+
// throws. Issue #761: on its OWN retention (`PI_CONSOLE_AUDIT_RETENTION_DAYS`, which inherits the run history's when
|
|
1586
|
+
// unset), so shortening the run history for privacy does not shorten the security trail. The periodic sweep below
|
|
1587
|
+
// re-runs this closure, so boot and every tick read the one value.
|
|
1588
|
+
const reapConsoleAudit = makeConsoleAuditReaper({ logsDir: config.logsDir, retentionDays: config.consoleAuditRetentionDays, log });
|
|
1589
|
+
reapConsoleAudit();
|
|
1219
1590
|
|
|
1220
1591
|
// REQ-RESURRECTABLE-SANDBOX: sweep retained per-job directories past their window, so what `cleanup`
|
|
1221
1592
|
// kept for re-opening stays bounded. Third in the row and deliberately its own sweep -- a different
|
|
@@ -1264,9 +1635,17 @@ export async function startWorker(
|
|
|
1264
1635
|
// One raw Redis client, shared by the budget (via the worker) and the scheduler stall guard, so it is
|
|
1265
1636
|
// hoisted out of the createWorkerFn arg object.
|
|
1266
1637
|
const redis = makeRedisClient(valkey.url, { servername: valkey.servername, context: valkeyContext });
|
|
1638
|
+
const subscriptions = configuredSubscriptions({ config, redis, now: () => Number(now()) });
|
|
1267
1639
|
// Its errors as one message-only line (PR #475's review, round 2), like every Queue and Worker's: without a listener
|
|
1268
1640
|
// ioredis printed a stack per reconnect attempt.
|
|
1269
1641
|
onValkeyError(redis, "shared client");
|
|
1642
|
+
// Accounting is repaired independently of work: publication failure must never replay a human action.
|
|
1643
|
+
const touchStore = makeTouchStoreFn({ redis, now, retentionDays: config.logRetentionDays });
|
|
1644
|
+
const touchOutbox = makeTouchOutboxFn({ dir: join(config.logsDir, "touch-outbox"), store: touchStore, log });
|
|
1645
|
+
const touchInitialized = await touchStore.initialize();
|
|
1646
|
+
if (!touchInitialized.ok) log("touch_tracking_unavailable", { reason: touchInitialized.error });
|
|
1647
|
+
await touchOutbox.repair();
|
|
1648
|
+
const questionGroupStore = makeQuestionGroupStoreFn({ redis, now, retentionDays: config.logRetentionDays });
|
|
1270
1649
|
|
|
1271
1650
|
// Issue #504 part B: the applied split lives in Valkey (`alloc:plan`), shared by every host. Only with an envelope.
|
|
1272
1651
|
// The boot reconcile seeds the neutral split when there is none, expires a plan past its life and re-bases on an
|
|
@@ -1311,7 +1690,9 @@ export async function startWorker(
|
|
|
1311
1690
|
// collector enqueues chained children onto it -- the same pi-jobs queue, so one handle serves both.
|
|
1312
1691
|
// Non-failFast: a long-lived handle rides out a Valkey blip. Registered as an extraCloser so shutdown
|
|
1313
1692
|
// drains it after the worker.
|
|
1314
|
-
|
|
1693
|
+
// Install the settled close before ANY later boot refusal can reach these handles. Waiting until
|
|
1694
|
+
// the worker handoff left the backend registry's refusal racing BullMQ's initial INFO (#664).
|
|
1695
|
+
const runtimeQueue = settleCloseOf(makeRuntimeQueueFn(valkeyConn()));
|
|
1315
1696
|
|
|
1316
1697
|
// THE HOST QUEUE (issue #57): work only this machine can do, because the folder lives here.
|
|
1317
1698
|
//
|
|
@@ -1327,7 +1708,80 @@ export async function startWorker(
|
|
|
1327
1708
|
// live triggers-file edit lands on the same queue the boot reconcile used; otherwise the shared
|
|
1328
1709
|
// runtime queue, exactly as before. Registered as an extraCloser only when it is a NEW handle --
|
|
1329
1710
|
// closing `runtimeQueue` twice would be closing another owner's connection.
|
|
1330
|
-
const cronQueue = hostQueue ?
|
|
1711
|
+
const cronQueue = hostQueue ? settleCloseOf(makeRuntimeQueueFn(valkeyConn(), { name: hostQueue })) : runtimeQueue;
|
|
1712
|
+
|
|
1713
|
+
// Issue #645 (durable runs): the checkpoint collaborators, built once here and handed to the processor and the live
|
|
1714
|
+
// transcript opener. Only with a store (`config.checkpointsDir`, PI_CHECKPOINTS_DIR else <PI_SESSIONS_DIR>/.checkpoints):
|
|
1715
|
+
// without one an armed job is refused by the processor before it spends (`recover-store-unset`), so nothing below may
|
|
1716
|
+
// pretend to checkpoint into nowhere. The run checkpointer's module is loaded unconditionally, because its pointer read
|
|
1717
|
+
// is what the STALL listener needs on every host of a fleet, store or not: the host that notices a stall is any host.
|
|
1718
|
+
let runCheckpointModule = null;
|
|
1719
|
+
try {
|
|
1720
|
+
runCheckpointModule = await loadRunCheckpoint();
|
|
1721
|
+
} catch (err) {
|
|
1722
|
+
// Fail loudly: every armed job on this host then falls back to today's path, and the operator is told why at boot.
|
|
1723
|
+
log("run_checkpoint_unavailable", { reason: scrubCredentials(err?.message) });
|
|
1724
|
+
}
|
|
1725
|
+
const durableHost = config.workerName;
|
|
1726
|
+
// `snapshotTimeoutMs` 20 s, not the module's 60 s default: a LIVE checkpoint snapshots a PAUSED container, so this bound
|
|
1727
|
+
// is how long a paid job can sit frozen (its provider stream stalled, its tool timers running on the clock). A tree too
|
|
1728
|
+
// big to walk in 20 s is skipped (`timeout`) and the run goes on, with its exit checkpoint as the fallback; 60 s of
|
|
1729
|
+
// freeze per checkpoint would be long enough for a provider to drop the stream the agent is waiting on.
|
|
1730
|
+
const forkStore = config.sessionsDir ? createForkStore({ dir: join(config.sessionsDir, ".fork-sources"), now, hostName: durableHost, hostQueue }) : null;
|
|
1731
|
+
const workspaceCheckpointer = config.checkpointsDir ? makeWorkspaceCheckpointerFn({ storeDir: config.checkpointsDir, now, log, hostName: durableHost, limits: { snapshotTimeoutMs: 20_000 } }) : null;
|
|
1732
|
+
const runCheckpointer =
|
|
1733
|
+
workspaceCheckpointer && typeof runCheckpointModule?.makeRunCheckpointer === "function"
|
|
1734
|
+
? runCheckpointModule.makeRunCheckpointer({ forkStore, checkpointer: workspaceCheckpointer, redis, hostName: durableHost, hostQueue, now, log })
|
|
1735
|
+
: null;
|
|
1736
|
+
// The per-run dollar ledger (run-budget-ledger.mjs): on every host, since a recovery settles the crashed attempt's hold
|
|
1737
|
+
// wherever it runs, and an unarmed job never calls it.
|
|
1738
|
+
const runBudgetLedger = makeRunBudgetLedgerFn({ redis, now, log });
|
|
1739
|
+
const readCheckpointPointer = async ({ runId }) => {
|
|
1740
|
+
if (typeof runCheckpointModule?.readCheckpointPointer !== "function") return null;
|
|
1741
|
+
try {
|
|
1742
|
+
return await runCheckpointModule.readCheckpointPointer({ redis, runId, timeoutMs: 2_000 });
|
|
1743
|
+
} catch (err) {
|
|
1744
|
+
log("checkpoint_pointer_read_failed", { code: typeof err?.code === "string" ? err.code : "error" });
|
|
1745
|
+
return null;
|
|
1746
|
+
}
|
|
1747
|
+
};
|
|
1748
|
+
// The queues a recovery can be routed to: the shared one, this host's own, or another host's (the crashed host's,
|
|
1749
|
+
// when THIS host noticed the stall). Another host's handle is opened on first use and joins the closers then; the
|
|
1750
|
+
// array is read at shutdown, not at handoff, which is what makes a late push sound (see `extraClosers` below).
|
|
1751
|
+
const otherHostQueues = new Map();
|
|
1752
|
+
const recoveryQueues = {
|
|
1753
|
+
shared: () => runtimeQueue,
|
|
1754
|
+
host: (name) => {
|
|
1755
|
+
if (name === hostQueue) return cronQueue;
|
|
1756
|
+
let q = otherHostQueues.get(name);
|
|
1757
|
+
if (!q) {
|
|
1758
|
+
q = makeQueue(valkeyConn(), { name });
|
|
1759
|
+
otherHostQueues.set(name, q);
|
|
1760
|
+
extraClosers.push({ close: () => closeQueueSettled(q) });
|
|
1761
|
+
}
|
|
1762
|
+
return q;
|
|
1763
|
+
},
|
|
1764
|
+
};
|
|
1765
|
+
const enqueueRecovery = makeRecoveryEnqueuerFn({ redis, queues: recoveryQueues, hostQueueName, now, log });
|
|
1766
|
+
// Issue #686 part 3, review round 1 (#714): on the openshell-bind venue a lost attempt's job dir is a sandbox's bind,
|
|
1767
|
+
// and that sandbox may still be writing it, so the salvage first has the venue end it and confirm it gone. Resolved at
|
|
1768
|
+
// call time: the bundle is built further down, and a salvage runs only once jobs are consumed.
|
|
1769
|
+
const salvageCheckpoint = makeSalvager({ jobsDir: config.jobsDir, hostName: durableHost, workspaceCheckpointer, runCheckpointer, log, settleJobDir: (name) => (openshellBindBackend ? openshellBindBackend.settleJobDir(name) : true) });
|
|
1770
|
+
// The boot sweep of orphaned `jobs/job-*` directories: a crashed worker's processor never cleaned up after its jobs.
|
|
1771
|
+
// Here, before `createWorker` starts consuming, so no job of THIS process owns any of them; the age bound (the job
|
|
1772
|
+
// timeout plus the abort grace's margin) covers another process on this host. A directory a checkpoint pointer of this
|
|
1773
|
+
// host still names is kept for its recovery's salvage, and when those pointers cannot be read the sweep is skipped
|
|
1774
|
+
// rather than run blind.
|
|
1775
|
+
// The pointer read is BOUNDED: a Valkey that does not answer must not hold the boot (every command of this client waits
|
|
1776
|
+
// for a connection rather than failing), and an unanswered read is the same skip as a failed one.
|
|
1777
|
+
try {
|
|
1778
|
+
const keep = workspaceCheckpointer ? await settleWithin(pointedJobDirs({ redis, hostName: durableHost }), 5_000, null) : new Set();
|
|
1779
|
+
if (keep === null) throw new Error("the checkpoint pointers were not read within 5 s");
|
|
1780
|
+
await sweepOrphanJobDirsFn({ jobsDir: config.jobsDir, olderThanMs: JOB_TIMEOUT_MS + 5 * 60 * 1000, now, keep, log });
|
|
1781
|
+
} catch (err) {
|
|
1782
|
+
log("job_dir_sweep_skipped", { reason: scrubCredentials(err?.message) });
|
|
1783
|
+
}
|
|
1784
|
+
log("durable_runs", { checkpointsDir: config.checkpointsDir ?? null, checkpointer: Boolean(runCheckpointer), pointerReads: typeof runCheckpointModule?.readCheckpointPointer === "function" });
|
|
1331
1785
|
|
|
1332
1786
|
// REQ-LOCAL-JOB-VISIBILITY durable run history, all host-side. The raw `.log` sink is gated on
|
|
1333
1787
|
// captureJobLogs (raw container output is user-authored data, opt-in per no-pii-in-logs); the id-only
|
|
@@ -1371,8 +1825,138 @@ export async function startWorker(
|
|
|
1371
1825
|
// a worker name: an unnamed one is a single host, its own files ARE the whole history, and a mirror
|
|
1372
1826
|
// would be bytes nothing reads. That is also what keeps a single-host deployment byte-identical, since
|
|
1373
1827
|
// no job then issues a single extra Valkey command.
|
|
1374
|
-
|
|
1375
|
-
|
|
1828
|
+
// Issue #761: trimmed to this host's `PI_RUN_MIRROR_MAX_RUNS`; every writer trims the shared index to its own cap, so a
|
|
1829
|
+
// fleet's lowest wins, and each trim raises the horizon a reader sees the cut by.
|
|
1830
|
+
const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, indexMax: config.runMirrorMaxRuns, log }) : null;
|
|
1831
|
+
// Issue #644: the forge status comment's two calls, the `comment` adapter's shape below (forgeFor, the auth re-resolve,
|
|
1832
|
+
// a token minted per call) and its rule: they NEVER throw, since the comment decides nothing. `statusPost` answers
|
|
1833
|
+
// `{ id }` or null; `statusEdit` answers `{ ok: true }`, `{ stop }` for a 403, 404 or 429 (the comment is gone, the
|
|
1834
|
+
// token may not edit it, or the forge is rate limiting: editing on would only repeat the refusal) and `{ ok: false }`
|
|
1835
|
+
// for any other failure, which the commenter tries again on its next edit. Logs carry the job id and the status.
|
|
1836
|
+
const draftReplyMayPost = async job => {
|
|
1837
|
+
if (!consoleDraftOf(job?.console)) return true;
|
|
1838
|
+
try { return await checkDraftAnswerSource(job, { answerDraftStore, answerDraftRuntime }) === null; }
|
|
1839
|
+
catch { return false; }
|
|
1840
|
+
};
|
|
1841
|
+
const statusAuth = async (job) => {
|
|
1842
|
+
const forge = forgeFor(job);
|
|
1843
|
+
if (!forge?.host) return null;
|
|
1844
|
+
let auth = forge.auth ?? null;
|
|
1845
|
+
if (!auth) {
|
|
1846
|
+
try {
|
|
1847
|
+
auth = await ensureAuth(job.kind);
|
|
1848
|
+
} catch {
|
|
1849
|
+
auth = null;
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
return auth ? { forge, auth } : null;
|
|
1853
|
+
};
|
|
1854
|
+
const statusPost = async (job, text) => {
|
|
1855
|
+
if (job?.answerDraft || job?.purpose === "answer-draft") return null;
|
|
1856
|
+
try {
|
|
1857
|
+
if (!await draftReplyMayPost(job)) return null;
|
|
1858
|
+
const ready = await statusAuth(job);
|
|
1859
|
+
if (!ready || typeof ready.forge.host.postStatusComment !== "function") return null;
|
|
1860
|
+
const token = await ready.auth.mintToken(job);
|
|
1861
|
+
const posted = await ready.forge.host.postStatusComment(job, job.target, text, token);
|
|
1862
|
+
return typeof posted?.id === "string" && posted.id !== "" ? { id: posted.id } : null;
|
|
1863
|
+
} catch (err) {
|
|
1864
|
+
log("status_comment_failed", { jobId: job?.id, op: "post", status: Number.isSafeInteger(err?.status) ? err.status : null });
|
|
1865
|
+
return null;
|
|
1866
|
+
}
|
|
1867
|
+
};
|
|
1868
|
+
const statusEdit = async (job, id, text) => {
|
|
1869
|
+
if (job?.answerDraft || job?.purpose === "answer-draft") return { stop: "answer-draft" };
|
|
1870
|
+
try {
|
|
1871
|
+
if (!await draftReplyMayPost(job)) return { stop: "answer-source-changed" };
|
|
1872
|
+
const ready = await statusAuth(job);
|
|
1873
|
+
if (!ready) return { ok: false }; // auth unreachable just now: the next edit asks again
|
|
1874
|
+
if (typeof ready.forge.host.editStatusComment !== "function") return { stop: "no-edit" };
|
|
1875
|
+
const token = await ready.auth.mintToken(job);
|
|
1876
|
+
await ready.forge.host.editStatusComment(job, job.target, id, text, token);
|
|
1877
|
+
return { ok: true };
|
|
1878
|
+
} catch (err) {
|
|
1879
|
+
if (err?.status === 403 || err?.status === 404 || err?.status === 429) return { stop: `http-${err.status}` };
|
|
1880
|
+
log("status_comment_failed", { jobId: job?.id, op: "edit", status: Number.isSafeInteger(err?.status) ? err.status : null });
|
|
1881
|
+
return { ok: false };
|
|
1882
|
+
}
|
|
1883
|
+
};
|
|
1884
|
+
// The commenter for one job, asked by the opener only when its trigger set `run.statusComment` and progress is on;
|
|
1885
|
+
// null for a job with no forge to post to (a local or cron job without a target) or a host that cannot edit, so such
|
|
1886
|
+
// a job runs with no comment rather than with a comment it could never update.
|
|
1887
|
+
// The commenters of the running jobs, by job id: the sink hands each its final snapshot when the container ends, and
|
|
1888
|
+
// `recordRun` below makes the final edit with the record's outcome (status-comment.mjs says why not the exit's).
|
|
1889
|
+
const statusComments = makeStatusCommentRegistry();
|
|
1890
|
+
const makeProgressCommenter = ({ job, jobId, commentId, onId, finalTimeoutMs }) => {
|
|
1891
|
+
const host = forgeFor(job)?.host;
|
|
1892
|
+
if (!host || typeof host.editStatusComment !== "function" || typeof host.postStatusComment !== "function" || !job?.target) {
|
|
1893
|
+
log("status_comment_skipped", { jobId: job?.id ?? null, reason: "no-forge-target" });
|
|
1894
|
+
return null;
|
|
1895
|
+
}
|
|
1896
|
+
// The comment must never trigger a job of its own: skipped, and said once for this job, when its text could hold
|
|
1897
|
+
// the phrase the job was started with. A job started otherwise carries no phrase (`matched.phrase` is the comment
|
|
1898
|
+
// route's), so a deployment whose comment trigger has a colliding phrase is caught only on its comment jobs.
|
|
1899
|
+
const phrase = job?.trigger?.matched?.phrase;
|
|
1900
|
+
if (statusTextMayMatch(phrase, jobId)) {
|
|
1901
|
+
log("status_comment_skipped", { jobId: job?.id ?? null, reason: "phrase-collision" });
|
|
1902
|
+
return null;
|
|
1903
|
+
}
|
|
1904
|
+
const commenter = makeStatusCommenterFn({ jobId, post: (text) => statusPost(job, text), edit: (id, text) => statusEdit(job, id, text), commentId, onId, log, ...(finalTimeoutMs ? { finalTimeoutMs } : {}) });
|
|
1905
|
+
return commenter ? statusComments.register(jobId, commenter) : null;
|
|
1906
|
+
};
|
|
1907
|
+
// Issue #643 (DES-LIVE-TRANSCRIPT-STREAM): every running job's transcript, redacted, to its own capped Valkey stream,
|
|
1908
|
+
// on every deployment unless PI_LIVE_TRANSCRIPTS=0. Not gated on a worker name, unlike the mirror above: the panel on
|
|
1909
|
+
// a single host reads the same stream. Handed to every venue's runContainer, which publish only beside a staged file,
|
|
1910
|
+
// or, on openshell (issue #686), from the exec's stderr records.
|
|
1911
|
+
//
|
|
1912
|
+
// Issue #644 (DES-RUN-PROGRESS): the same opener reads each job's progress off the same file unless PI_RUN_PROGRESS=0,
|
|
1913
|
+
// so it is built when either switch is on; with transcripts off it tails the file and writes no stream.
|
|
1914
|
+
//
|
|
1915
|
+
// Issue #645: the run checkpointer is the file's third reader (an armed job's quiet points schedule its checkpoints), so
|
|
1916
|
+
// with a checkpointer the opener is built even with both switches off; it then tails the file for the checkpoint alone
|
|
1917
|
+
// (no stream, no progress key). `checkpoint` is called per attempt with the open() arguments the attempt supplies.
|
|
1918
|
+
//
|
|
1919
|
+
// Issue #646: a steer-armed job's delivery acks are read off the same file (steer-ack.mjs, the fourth reader), so with a
|
|
1920
|
+
// Valkey client the opener is built even with every other reader off (`steer: true`); a job that is not armed hands no
|
|
1921
|
+
// ack consumer and is tailed exactly as before (or not at all).
|
|
1922
|
+
const steerWired = Boolean(redis);
|
|
1923
|
+
const openLiveTranscript =
|
|
1924
|
+
config.liveTranscripts || config.runProgress || runCheckpointer || steerWired
|
|
1925
|
+
? makeLiveTranscriptOpenerFn({ redis, log, transcripts: config.liveTranscripts, progress: config.runProgress, makeCommenter: makeProgressCommenter, ...(runCheckpointer ? { checkpoint: (args) => runCheckpointer.open(args) } : {}), ...(steerWired ? { steer: true } : {}) })
|
|
1926
|
+
: null;
|
|
1927
|
+
// Issue #646 (DES-STEER-VIA-READONLY-INBOX-MOUNT): the inbox relay's opener, handed to every venue's runContainer, which
|
|
1928
|
+
// opens one per steer-ARMED attempt (on openshell, issue #686, one writing frames on the exec's stdin). Bound here to the worker's client and to the host name the run checkpointer
|
|
1929
|
+
// writes (`durableHost`), the one this host is known by in the fleet: the open key names it.
|
|
1930
|
+
const openInboxRelay = steerWired ? (args) => makeInboxRelayFn({ redis, host: durableHost, now, log, ...args,
|
|
1931
|
+
onDelivered: async ({ runId, id, at }) => {
|
|
1932
|
+
const intent = await touchOutbox.begin({ runId, cause: "steer", at, nonce: id });
|
|
1933
|
+
const result = await touchOutbox.applied(intent);
|
|
1934
|
+
if (!result.ok) log("touch_record_pending", { reason: result.error });
|
|
1935
|
+
},
|
|
1936
|
+
}) : null;
|
|
1937
|
+
// Issue #644: the final edit for an attempt that registered no commenter, so nothing else would edit the comment again.
|
|
1938
|
+
// That is an attempt refused before its container (a cost cap, a policy gate) after an earlier attempt posted the
|
|
1939
|
+
// comment and left it saying "attempt ended, retrying": with no edit here it said so for good. The comment's id is
|
|
1940
|
+
// the one the earlier attempt's final write put in the progress key; an attempt 1 refused before its container
|
|
1941
|
+
// finds none (nothing was posted) and edits nothing. One edit, from the record's outcome and the key's last snapshot
|
|
1942
|
+
// (the earlier attempt's counters, the last the run reached), bounded and never throwing; skipped, as the commenter
|
|
1943
|
+
// is, when the text could hold the job's own phrase.
|
|
1944
|
+
const settleUnregisteredStatus = async (job, outcome) => {
|
|
1945
|
+
try {
|
|
1946
|
+
const data = job?.data ?? {};
|
|
1947
|
+
// Issue #645: the progress key and the comment are the RUN's, so a recovery (`.rN`) finds the comment its run posted.
|
|
1948
|
+
const runId = runIdOf(job);
|
|
1949
|
+
if (statusTextMayMatch(data.trigger?.matched?.phrase, runId)) return;
|
|
1950
|
+
const key = await readProgressKeyFn({ redis, jobId: runId });
|
|
1951
|
+
if (!key?.commentId) return;
|
|
1952
|
+
const text = renderStatusText({ jobId: runId, snapshot: key.snapshot, outcome });
|
|
1953
|
+
const done = await settleWithin(statusEdit(data, key.commentId, text), STATUS_UNREGISTERED_EDIT_MS, null);
|
|
1954
|
+
if (done === null) log("status_comment_final_timeout", { jobId: job.id });
|
|
1955
|
+
} catch {
|
|
1956
|
+
// The comment decides nothing: a failure here costs a stale comment, never the record path.
|
|
1957
|
+
}
|
|
1958
|
+
};
|
|
1959
|
+
const recordRun = ({ job, result, error, startedAt, endedAt, project, size = null, capacity = null, earlier = null, maxTurns = null }) => {
|
|
1376
1960
|
// The project (issue #499) was resolved at the pickup gate and rides here as `project` (an id or null), so a live
|
|
1377
1961
|
// edit of projects.json mid-run cannot make the record disagree with what the job was counted against. A record
|
|
1378
1962
|
// path that ends BEFORE the pickup gate (the wait gate's refusals) passes none, and resolves from the live ref
|
|
@@ -1382,8 +1966,24 @@ export async function startWorker(
|
|
|
1382
1966
|
// four `recordRun` call sites byte-unchanged and `buildRecord` a pure function of its arguments.
|
|
1383
1967
|
// The default venue rides the same way and for the same reason (#277): it is the value the registry
|
|
1384
1968
|
// below is built with, so the record resolves a job's venue exactly as dispatch does.
|
|
1385
|
-
const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size, capacity, earlier });
|
|
1969
|
+
const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size, capacity, earlier, maxTurns, touchTracking: true });
|
|
1386
1970
|
writeRecord(record);
|
|
1971
|
+
// Issue #644: the status comment's last edit, from the record's classification. An infrastructure failure the queue
|
|
1972
|
+
// will retry says the attempt ended and is being retried, not "finished": BullMQ retries an error that is not
|
|
1973
|
+
// unrecoverable while attempts are left, and this record is written while the attempt still counts as running
|
|
1974
|
+
// (`attemptsMade` is the attempts finished before it). Never throws, never awaited; a job with no commenter is a
|
|
1975
|
+
// no-op.
|
|
1976
|
+
try {
|
|
1977
|
+
const attempts = Number.isSafeInteger(job?.opts?.attempts) ? job.opts.attempts : 1;
|
|
1978
|
+
// Issue #645: an attempt whose run continues as a recovery (`recovery-enqueued`) is retrying too, not failed: the
|
|
1979
|
+
// recovery picks the same comment up (it is the run's) and ends it with its own outcome.
|
|
1980
|
+
const retrying = (!result && error instanceof InfraRetry && record.attempt < attempts) || record.reason === "recovery-enqueued";
|
|
1981
|
+
const outcome = retrying ? STATUS_RETRYING : record.outcome;
|
|
1982
|
+
// The commenters are registered by the RUN id (the opener is handed it), so a `.rN` record settles its run's.
|
|
1983
|
+
const runId = runIdOf(job);
|
|
1984
|
+
if (statusComments.has(runId)) void statusComments.settle(runId, outcome);
|
|
1985
|
+
else if (job?.data?.statusComment === true) void settleUnregisteredStatus(job, outcome);
|
|
1986
|
+
} catch {}
|
|
1387
1987
|
// STRICTLY AFTER the file, and deliberately not awaited. After, because a crash between the two must
|
|
1388
1988
|
// leave a record with no fleet row rather than a fleet row with no record -- the mirror is a VIEW,
|
|
1389
1989
|
// and a view that can outlive its source is a second source of truth. Not awaited, because this is
|
|
@@ -1408,17 +2008,39 @@ export async function startWorker(
|
|
|
1408
2008
|
readRecord,
|
|
1409
2009
|
readMirrored: runMirror ? (jobId) => readMirroredRecordFn(redis, sanitizeJobId(jobId)) : null,
|
|
1410
2010
|
});
|
|
2011
|
+
// Issue #645: a run's record by job id, this host's file then the fleet's copy where a mirror is armed, for the
|
|
2012
|
+
// processor's recovery gate (a recovery whose base run already settled is superseded, not run). A recovery may land on
|
|
2013
|
+
// a host that never ran its base attempt, so a local-only read would miss exactly the fleet case. Never rejects;
|
|
2014
|
+
// unreadable or unreachable is null.
|
|
2015
|
+
// `keep`, when given, is the rule the caller's record must pass (the processor's: this run's settled record, issue
|
|
2016
|
+
// #811), applied to EACH side: this host's file answers only when it passes, and the fleet's copy is asked otherwise.
|
|
2017
|
+
// Without it a stale record an older job left in this host's file under a reused id would be the answer, and the
|
|
2018
|
+
// settled record in the mirror would never be looked at. Without `keep`, any record this host holds answers, as before.
|
|
2019
|
+
const readRunRecord = async (jobId, { keep = null } = {}) => {
|
|
2020
|
+
const kept = (record) => (record && record !== UNREADABLE_RECORD && (typeof keep !== "function" || keep(record)) ? record : null);
|
|
2021
|
+
const local = kept(readRecord(jobId));
|
|
2022
|
+
if (local) return local;
|
|
2023
|
+
if (!runMirror) return null;
|
|
2024
|
+
const mirrored = await readMirroredRecordFn(redis, sanitizeJobId(jobId)).catch(() => null);
|
|
2025
|
+
return kept(mirrored);
|
|
2026
|
+
};
|
|
1411
2027
|
// The record a retry (or a pickup after a stall) is about to replace (issue #599), read BOTH ways: this host's file and
|
|
1412
2028
|
// the fleet's copy where a mirror is armed. Both, because each host keeps its own file: a job that ran on A, then B,
|
|
1413
2029
|
// then A again finds A's stale first attempt locally while B's second is in the mirror. `newerRecord` keeps the higher
|
|
1414
|
-
// attempt (a tie: the later end). Its slot interval rides the new record as `earlier`.
|
|
1415
|
-
//
|
|
1416
|
-
|
|
2030
|
+
// attempt (a tie: the later end). Its slot interval rides the new record as `earlier`. The failed listener's lost-lock
|
|
2031
|
+
// check reads a job's own failed record through it too, for the same reason (issue #761). Never rejects: unreadable
|
|
2032
|
+
// or unreachable is no record from that side.
|
|
2033
|
+
// `keep`, when given, is the rule a caller's record must pass (`recordOfThisJob` for one attempt, pure and never
|
|
2034
|
+
// throwing), applied to EACH side before the higher attempt is chosen. Applied after, it would judge only the side
|
|
2035
|
+
// chosen: ids are reused, so a stale record an older job left at a higher attempt would win on attempt and then fail
|
|
2036
|
+
// the rule, and this attempt's own record on the other side would never be looked at.
|
|
2037
|
+
const previousRecord = async (jobId, { keep = null } = {}) => {
|
|
2038
|
+
const kept = (record) => (record === null || typeof keep !== "function" || keep(record) ? record : null);
|
|
1417
2039
|
const read = readRecord(jobId);
|
|
1418
|
-
const local = read === UNREADABLE_RECORD ? null : read;
|
|
2040
|
+
const local = kept(read === UNREADABLE_RECORD ? null : read);
|
|
1419
2041
|
if (!runMirror) return local;
|
|
1420
2042
|
const mirrored = await readMirroredRecordFn(redis, sanitizeJobId(jobId)).catch(() => null);
|
|
1421
|
-
return newerRecord(local, mirrored === UNREADABLE_RECORD ? null : mirrored);
|
|
2043
|
+
return newerRecord(local, kept(mirrored === UNREADABLE_RECORD ? null : mirrored));
|
|
1422
2044
|
};
|
|
1423
2045
|
|
|
1424
2046
|
// INT-CONFIG-OVERLAY-CONTRACT: the worker reads the runtime-settings overlay at EACH job start, so this
|
|
@@ -1432,6 +2054,12 @@ export async function startWorker(
|
|
|
1432
2054
|
// (issue #501), so an overlay window with an env per-job cap is valid. `secretProfiles` rides alongside the
|
|
1433
2055
|
// effective keys; that function says why.
|
|
1434
2056
|
const res = resolveSettings(config, readOverlay(settingsFile, { log }));
|
|
2057
|
+
// Issue #761: a file written by a newer pi-dispatch keeps `newer: true`, so the processor DEFERS the job
|
|
2058
|
+
// (`newer-release`) instead of refusing it as settings-overlay-invalid. Its own line: it is no broken file.
|
|
2059
|
+
if (res.invalid && res.newer === true) {
|
|
2060
|
+
log("settings_overlay_newer", { reason: res.invalid, settingsFile });
|
|
2061
|
+
return { invalid: res.invalid, newer: true };
|
|
2062
|
+
}
|
|
1435
2063
|
if (res.invalid) {
|
|
1436
2064
|
log("settings_overlay_invalid", { reason: res.invalid, settingsFile });
|
|
1437
2065
|
return { invalid: res.invalid };
|
|
@@ -1460,6 +2088,52 @@ export async function startWorker(
|
|
|
1460
2088
|
// (`OQ-009`), so the working tree it needs is the one this machine just used: routing it anywhere
|
|
1461
2089
|
// else would enqueue a job only this host can run onto a queue every host drains.
|
|
1462
2090
|
const collectChain = makeCollectChain({ queue: cronQueue, config, log });
|
|
2091
|
+
const watchStore = makeWatchStore(redis, { now });
|
|
2092
|
+
const watchIssueAdapter = makeWatchIssueAdapter({ getContext: async target => {
|
|
2093
|
+
const auth = forges[target.kind]?.auth ?? await ensureAuth(target.kind);
|
|
2094
|
+
if (!auth) throw configError("watch: configure the target forge credentials");
|
|
2095
|
+
const token = await auth.mintToken({ kind: target.kind, repo: target.repo, permissions: { issues: "write", metadata: "read" } });
|
|
2096
|
+
return { apiUrl: target.kind === "github" ? "https://api.github.com" : config[target.kind]?.apiUrl, token };
|
|
2097
|
+
} });
|
|
2098
|
+
const watchPublisher = makeWatchIssuePublisher({ redis, adapter: watchIssueAdapter, now,
|
|
2099
|
+
canClose: async mapping => {
|
|
2100
|
+
const doc = parseTriggersDocument(readFileSync(onceTriggersFile, "utf8"), onceTriggersFile);
|
|
2101
|
+
const rule = doc.triggers.find(t => t.on.type === "watch" && t.on.id === mapping.ruleId && t.on.source === mapping.source);
|
|
2102
|
+
const target = rule?.run.issue;
|
|
2103
|
+
return Boolean(doc.watchSources?.[mapping.source] && target?.closeAfter && target.kind === mapping.target.kind && target.repo === mapping.target.repo && watchDurationMs(target.closeAfter) === mapping.closeAfterMs);
|
|
2104
|
+
},
|
|
2105
|
+
reportClose: async (outcome, mapping) => { if (mapping?.ruleId) await watchStore.recordIssueOutcome(mapping.ruleId, outcome); },
|
|
2106
|
+
});
|
|
2107
|
+
const watchRuntime = makeWatchRuntime({ store: watchStore, triggersPath: onceTriggersFile, now, log, getSettings, pauseUntil: (job, at) => pauseUntilMs(pauseWindows.current, job, at), enqueue: (fields, opts) => enqueueWatchJob(cronQueue, fields, { now, ...opts }), publishIssue: watchPublisher });
|
|
2108
|
+
const questionGroupRuntime = makeQuestionGroupRuntimeFn({ store: questionGroupStore, triggersPath: onceTriggersFile, getSettings,
|
|
2109
|
+
resolveClassifier: ({ provider, id }) => {
|
|
2110
|
+
const models = readOverlayModels(config.globalPiDir);
|
|
2111
|
+
const entry = models?.providers?.[provider];
|
|
2112
|
+
const custom = entry?.models?.find((model) => model.id === id);
|
|
2113
|
+
if (custom) return custom;
|
|
2114
|
+
const base = builtinModel(provider, id);
|
|
2115
|
+
const override = entry?.modelOverrides?.[id];
|
|
2116
|
+
return base ? { ...base, ...override, cost: { ...base.cost, ...override?.cost } } : null;
|
|
2117
|
+
},
|
|
2118
|
+
});
|
|
2119
|
+
const answerDraftStore = makeAnswerDraftStore({ redis, now, retentionDays: config.logRetentionDays });
|
|
2120
|
+
const answerDraftRuntime = makeAnswerDraftRuntime({ store: answerDraftStore, triggersPath: onceTriggersFile, now, log,
|
|
2121
|
+
enqueue: fields => enqueueAnswerDraftJob(runtimeQueue, fields),
|
|
2122
|
+
readParkedInfo: sessionStore.readParkedInfo,
|
|
2123
|
+
readApproval: source => approvalStore.find(source),
|
|
2124
|
+
readPendingAnswer: async source => {
|
|
2125
|
+
const found = await discoverHostQueuesReporting(redis);
|
|
2126
|
+
if (found.incomplete || found.names.length > 64) return { unknown: true };
|
|
2127
|
+
const queues = [runtimeQueue, ...found.names.map(name => recoveryQueues.host(name))];
|
|
2128
|
+
return readAnswerPending({ queues, source, runId: source.runId });
|
|
2129
|
+
},
|
|
2130
|
+
// The API root is the forge table's (`forgeApiRoot`), the one the console binds a completed draft to. Raw, as
|
|
2131
|
+
// configured: the runner fetches under it and checks Forgejo's issue_url against it.
|
|
2132
|
+
identity: async job => {
|
|
2133
|
+
const auth = forgeFor(job)?.auth ?? await ensureAuth(job.kind);
|
|
2134
|
+
return auth ? { selfId: job.kind === "azure" ? auth.selfId?.id : auth.selfId, apiUrl: forgeApiRoot(job.kind, config) } : null;
|
|
2135
|
+
},
|
|
2136
|
+
});
|
|
1463
2137
|
|
|
1464
2138
|
// Issue #505. One live-file check, shared by the pickup gate, the snapshot and the plan collector, so the three ask one
|
|
1465
2139
|
// file by one rule: the same file and rule as the one-shot checks below (`onceTriggersFile`). `pi-dispatch run
|
|
@@ -1471,7 +2145,8 @@ export async function startWorker(
|
|
|
1471
2145
|
const collectPlan = makeCollectPlan({ allocation: allocationState, governing: governingNow, projects: () => projects.current, checkPortfolioFlag, log });
|
|
1472
2146
|
// The snapshot a confirmed portfolio job reads as /job/portfolio.json, built at prepare. The run counts are complete
|
|
1473
2147
|
// only with the run mirror, which a declared worker name arms (the `runMirror` rule below).
|
|
1474
|
-
|
|
2148
|
+
// `indexMax` (issue #761): the snapshot reads the whole index, which this host's own trim keeps at or under its cap.
|
|
2149
|
+
const portfolioSnapshot = makePortfolioSnapshotFn({ checkPortfolioFlag, governing: governingNow, projects: () => projects.current, limits: () => scopedLimits.current, allocation: allocationState, redis, logsDir: config.logsDir, mirror: config.workerNameDeclared === true, indexMax: config.runMirrorMaxRuns, log });
|
|
1475
2150
|
|
|
1476
2151
|
// REQ-GLOBAL-PI-OVERLAY staged packages: read the operator's stage manifest at EACH job start, like
|
|
1477
2152
|
// getSettings above and the pause-window ref below.
|
|
@@ -1499,6 +2174,21 @@ export async function startWorker(
|
|
|
1499
2174
|
if (lastPackageKey !== null && key !== lastPackageKey) log("packages_stage_changed", { count: key === "" ? 0 : key.split(":").length });
|
|
1500
2175
|
lastPackageKey = key;
|
|
1501
2176
|
};
|
|
2177
|
+
// Issue #800: the name and version each path of the last good read stood for, so the rows a job's record names are the
|
|
2178
|
+
// read that chose its paths. Beside last-known-good, and replaced with it.
|
|
2179
|
+
let stagedRowByPath = new Map();
|
|
2180
|
+
// Issue #800: whether the set this returned says what is staged here, which the record needs before it calls a name absent. `readStageManifest` answers null
|
|
2181
|
+
// for a manifest that is not there and for one that does not read, and only the first is a stage with nothing in it: with no last-good set the second returns
|
|
2182
|
+
// [] too, and a package's real name would be recorded as one this host does not hold.
|
|
2183
|
+
let stageRead = true;
|
|
2184
|
+
const manifestIsAbsent = () => {
|
|
2185
|
+
try {
|
|
2186
|
+
statSync(join(config.globalPiDir, PACKAGES_SUBDIR, STAGE_MANIFEST));
|
|
2187
|
+
return false;
|
|
2188
|
+
} catch (error) {
|
|
2189
|
+
return error?.code === "ENOENT";
|
|
2190
|
+
}
|
|
2191
|
+
};
|
|
1502
2192
|
const getPackagePaths = () => {
|
|
1503
2193
|
if (!config.globalPiDir) return [];
|
|
1504
2194
|
const staged = readStageManifest({ globalPiDir: config.globalPiDir });
|
|
@@ -1508,13 +2198,18 @@ export async function startWorker(
|
|
|
1508
2198
|
return lastGoodPackagePaths;
|
|
1509
2199
|
}
|
|
1510
2200
|
notePackageKey("");
|
|
2201
|
+
stageRead = manifestIsAbsent();
|
|
1511
2202
|
return [];
|
|
1512
2203
|
}
|
|
2204
|
+
stageRead = true;
|
|
1513
2205
|
const paths = containerPackagePaths(staged);
|
|
1514
2206
|
notePackageKey(paths.join(":"));
|
|
1515
2207
|
lastGoodPackagePaths = paths;
|
|
2208
|
+
stagedRowByPath = new Map(staged.packages.map((p) => [containerPackagePaths({ packages: [p] })[0], { name: p.name, version: p.version }]));
|
|
1516
2209
|
return paths;
|
|
1517
2210
|
};
|
|
2211
|
+
getPackagePaths.rowOf = (path) => stagedRowByPath.get(path) ?? null;
|
|
2212
|
+
getPackagePaths.stageRead = () => stageRead;
|
|
1518
2213
|
// One read at boot, for the same log line the boot read always emitted, and to seed last-known-good.
|
|
1519
2214
|
if (config.globalPiDir && !readStageManifest({ globalPiDir: config.globalPiDir })) log("packages_manifest_absent", { overlay: config.globalPiDir });
|
|
1520
2215
|
getPackagePaths();
|
|
@@ -1547,6 +2242,8 @@ export async function startWorker(
|
|
|
1547
2242
|
egress: config.egress,
|
|
1548
2243
|
egressProxy: config.egressProxy,
|
|
1549
2244
|
openJobLog,
|
|
2245
|
+
...(openLiveTranscript ? { openLiveTranscript } : {}),
|
|
2246
|
+
...(openInboxRelay ? { openInboxRelay } : {}),
|
|
1550
2247
|
globalPiDir: config.globalPiDir,
|
|
1551
2248
|
allowGlobalExtensions: config.allowGlobalExtensions,
|
|
1552
2249
|
packagePaths: getPackagePaths,
|
|
@@ -1555,6 +2252,8 @@ export async function startWorker(
|
|
|
1555
2252
|
forgeHosts: { gitlab: config.gitlab?.apiUrl ?? null, forgejo: config.forgejo?.apiUrl ?? null, azure: config.azure?.orgUrl ?? null },
|
|
1556
2253
|
backendFloor: config.backendFloor,
|
|
1557
2254
|
reap: backendReaps[PODMAN_BACKEND],
|
|
2255
|
+
// Issue #681: the budget's leftover listing, through the same seam local's takes.
|
|
2256
|
+
listJobContainers: listJobContainersFn("podman"),
|
|
1558
2257
|
readInfo: podmanInfo,
|
|
1559
2258
|
platform: jobUserIdentity.platform,
|
|
1560
2259
|
euid: jobUserIdentity.euid,
|
|
@@ -1593,8 +2292,52 @@ export async function startWorker(
|
|
|
1593
2292
|
// Its own sentence (PR #463 round 3): the job-shaped one says "this job is retried", and at boot there is no job.
|
|
1594
2293
|
if (typeof bootKeeper?.keeper === "string") log("netns_keeper_not_holding_at_boot", { reason: bootKeeper.keeperAtBoot ?? bootKeeper.keeper });
|
|
1595
2294
|
}
|
|
1596
|
-
//
|
|
1597
|
-
|
|
2295
|
+
// Issue #684: the openshell bundle, while blessed. Its policy reads the same two sources the squid venues' proxy reads
|
|
2296
|
+
// and the MCP check reads (this folder's egress-allowlist.conf, a missing one admitting nothing, and the pickup's
|
|
2297
|
+
// endpoint snapshot, or the boot-loaded one where a job carries none); its reaper is the boot sweep's own.
|
|
2298
|
+
// ONE reader of this folder's egress-allowlist.conf for the MCP check and the openshell policy, so the two cannot
|
|
2299
|
+
// disagree about what the proxy admits. No allowlist here: every host outside a declared endpoint is unreachable,
|
|
2300
|
+
// which it is. Any other failure is the caller's to classify (a transient errno is retried).
|
|
2301
|
+
const readEgressAllowlist = () => {
|
|
2302
|
+
try {
|
|
2303
|
+
return parseAllowlist(readFileSync(join(process.cwd(), "egress-allowlist.conf"), "utf8"));
|
|
2304
|
+
} catch (error) {
|
|
2305
|
+
if (error?.code === "ENOENT") return [];
|
|
2306
|
+
throw error;
|
|
2307
|
+
}
|
|
2308
|
+
};
|
|
2309
|
+
// Issue #686 part 3: one option set, so the two modes cannot drift apart in anything but their mode (and their key in
|
|
2310
|
+
// the boot sweep's map, which holds one reaper for both).
|
|
2311
|
+
const readOpenshellEndpoints = () => modelEndpoints.current ?? [];
|
|
2312
|
+
const openshellOptions = (mode) => ({
|
|
2313
|
+
mode,
|
|
2314
|
+
image: config.jobImage,
|
|
2315
|
+
// Review round 1 (#714): the bind probe's empty source directory is made under the jobs dir.
|
|
2316
|
+
jobsDir: config.jobsDir,
|
|
2317
|
+
engine: config.openshell.engine,
|
|
2318
|
+
gateway: config.openshell.gateway,
|
|
2319
|
+
readAllowlist: readEgressAllowlist,
|
|
2320
|
+
readEndpoints: readOpenshellEndpoints,
|
|
2321
|
+
hostEnv: env,
|
|
2322
|
+
openJobLog,
|
|
2323
|
+
// Issue #686: the same opener the binding venues get, fed from the exec's stderr records, and the same inbox relay
|
|
2324
|
+
// opener, whose relay writes a steered job's messages as frames on the exec's stdin there (part 2).
|
|
2325
|
+
...(openLiveTranscript ? { openLiveTranscript } : {}),
|
|
2326
|
+
...(openInboxRelay ? { openInboxRelay } : {}),
|
|
2327
|
+
globalPiDir: config.globalPiDir,
|
|
2328
|
+
allowGlobalExtensions: config.allowGlobalExtensions,
|
|
2329
|
+
packagePaths: getPackagePaths,
|
|
2330
|
+
forwardEnv: config.forwardEnv,
|
|
2331
|
+
authFromPi: config.authFromPi,
|
|
2332
|
+
forgeHosts: { gitlab: config.gitlab?.apiUrl ?? null, forgejo: config.forgejo?.apiUrl ?? null, azure: config.azure?.orgUrl ?? null },
|
|
2333
|
+
reap: backendReaps[mode === "bind" ? OPENSHELL_BIND_BACKEND : OPENSHELL_BACKEND],
|
|
2334
|
+
log,
|
|
2335
|
+
});
|
|
2336
|
+
const openshellBackend = openshellCopyBlessed ? makeOpenshellBackendFn(openshellOptions("copy")) : null;
|
|
2337
|
+
const openshellBindBackend = openshellBindBlessed ? makeOpenshellBackendFn(openshellOptions("bind")) : null;
|
|
2338
|
+
// Every bundle this boot built beside local's, in registration order: podman, openshell, openshell-bind, then the
|
|
2339
|
+
// injected ones.
|
|
2340
|
+
const builtBackends = [...(podmanBackend ? [podmanBackend] : []), ...(openshellBackend ? [openshellBackend] : []), ...(openshellBindBackend ? [openshellBindBackend] : []), ...extraBackends];
|
|
1598
2341
|
// BOUNDED, because `.catch()` cannot rescue a promise that never settles: `runDocker` resolves only on
|
|
1599
2342
|
// the child's `close` or `error` and has no timeout of its own, so a wedged daemon would hang boot
|
|
1600
2343
|
// here. This read is a nicety -- a digest for the boot line and the registry -- and a nicety may
|
|
@@ -1629,7 +2372,10 @@ export async function startWorker(
|
|
|
1629
2372
|
}
|
|
1630
2373
|
// Resolved once: `Intl` is not free, and this value cannot change without a restart.
|
|
1631
2374
|
const hostTz = Intl.DateTimeFormat().resolvedOptions().timeZone ?? "";
|
|
2375
|
+
const endpointSlots = makeInFlight();
|
|
2376
|
+
const endpointStatusNow = () => endpointStatusFields(modelEndpoints.current, endpointSlots, config.workerNameDeclared);
|
|
1632
2377
|
const registry = makeHostRegistryFn({ redis, name: config.workerName, log });
|
|
2378
|
+
const retainedNow = makeSandboxTelemetry({ list: () => listSandboxesFn({ sandboxDir: config.sandboxDir, throwOnReadFailure: true }), deadline: sandboxDeadline, retentionHours: config.sandboxRetentionHours, canOpen: (manifest) => !sandboxSyncRefusal({ jobId: manifest.jobId, manifest, blessed: config.backends }), now, log });
|
|
1633
2379
|
// Issue #596, phase 2: one integer of the host budget's snapshot for the beat, or "" before the worker exists.
|
|
1634
2380
|
const snapshotField = (key) => {
|
|
1635
2381
|
const snap = worker?.hostBudget?.snapshot?.();
|
|
@@ -1645,7 +2391,7 @@ export async function startWorker(
|
|
|
1645
2391
|
return [];
|
|
1646
2392
|
}
|
|
1647
2393
|
};
|
|
1648
|
-
return liveJobsFields(liveJobsOf({ running: read(() => worker?.runningJobs?.list?.()), budgetEntries: read(() => worker?.hostBudget?.entries?.()) }));
|
|
2394
|
+
return liveJobsFields(liveJobsOf({ defaultVenue: config.defaultBackend, running: read(() => worker?.runningJobs?.list?.()), budgetEntries: read(() => worker?.hostBudget?.entries?.()) }));
|
|
1649
2395
|
};
|
|
1650
2396
|
// NOT awaited, and that is load-bearing rather than an optimisation. `makeRedisClient` sets
|
|
1651
2397
|
// `maxRetriesPerRequest: null` -- required for BullMQ's blocking connections -- which means a command
|
|
@@ -1671,7 +2417,20 @@ export async function startWorker(
|
|
|
1671
2417
|
//
|
|
1672
2418
|
// Recomputed per beat rather than frozen at boot, for the reason the digest is: a host that gains a
|
|
1673
2419
|
// profile on restart must start attracting that work within one beat, and one that loses it must stop.
|
|
1674
|
-
|
|
2420
|
+
//
|
|
2421
|
+
// Issue #704: the secret half is the profiles this host would RESOLVE, not only the env-declared ones. The
|
|
2422
|
+
// overlay is read here each beat (without a log, `fpUsd`'s reason), so a profile declared in the panel is
|
|
2423
|
+
// advertised within one beat of the write when its resolver sits inside PI_SECRET_RESOLVER_ROOTS, and
|
|
2424
|
+
// never when it does not. A name declared in both sources makes this host refuse every secrets job, so it
|
|
2425
|
+
// then advertises no secret profile at all. An overlay the job path refuses advertises the env profiles,
|
|
2426
|
+
// the row it always published; every job on such a host refuses as settings-overlay-invalid anyway.
|
|
2427
|
+
caps: () => {
|
|
2428
|
+
// `resolveSettings`, the job path's own rule (getSettings): an overlay that refuses every job as
|
|
2429
|
+
// settings-overlay-invalid, by its own validation or by the merged dollar rule, contributes no profile.
|
|
2430
|
+
const settings = resolveSettings(config, readOverlay(settingsFile));
|
|
2431
|
+
const served = servableSecretProfiles({ envProfiles: config.secretProfiles, overlayProfiles: settings.invalid ? {} : (settings.secretProfiles ?? {}), roots: config.secretResolverRoots, realExecutablePath: realExecutable });
|
|
2432
|
+
return serializeCaps(capabilityTokens({ secretProfiles: served.ambiguous ? {} : served.profiles, waitProfiles: config.waitProfiles }));
|
|
2433
|
+
},
|
|
1675
2434
|
// The host's IANA zone, because a cron PATTERN carries none: `triggers.json` has no `tz` field and
|
|
1676
2435
|
// BullMQ hands the pattern to cron-parser with no zone, so it resolves in each worker's LOCAL time.
|
|
1677
2436
|
// On one host that is exactly what an operator means; on two in different zones the same pattern is
|
|
@@ -1703,9 +2462,21 @@ export async function startWorker(
|
|
|
1703
2462
|
// Issue #596: the highest scoped-limits version this build reads, so doctor can name a worker that would keep its
|
|
1704
2463
|
// last good file (so no size and no later edit to the file applies on it) once the file is version 3. An integer.
|
|
1705
2464
|
limitsVersion: SCOPED_LIMITS_VERSION,
|
|
2465
|
+
// Issue #761 (DES-STORE-VERSION-WHEN-NEEDED): the highest version this build reads of the settings overlay, the pause
|
|
2466
|
+
// windows file, the triggers file and a queued job's data, integers, on `limitsVersion`'s model: only doctor reads
|
|
2467
|
+
// them, to name a peer that would defer (or, from before these fields, misread) what this host's files now need.
|
|
2468
|
+
settingsVersion: SETTINGS_VERSION,
|
|
2469
|
+
pauseWindowsVersion: PAUSE_WINDOWS_VERSION,
|
|
2470
|
+
triggersVersion: TRIGGERS_FILE_VERSION,
|
|
2471
|
+
jobDataVersion: JOB_DATA_VERSION,
|
|
1706
2472
|
// Issue #596, phase 2: this host's budget and what its jobs hold against it, integers (MiB and hundredths of a CPU),
|
|
1707
2473
|
// `off` for a budget switched off and "" while unknown, so doctor and a forge job's never-fits check can read which
|
|
1708
2474
|
// host a size fits on. Thunks over the worker's one budget, so every beat says what is held now.
|
|
2475
|
+
sandboxes: () => retainedNow().sandboxes,
|
|
2476
|
+
sandboxesMore: () => retainedNow().sandboxesMore,
|
|
2477
|
+
sandboxRetentionHours: () => retainedNow().sandboxRetentionHours,
|
|
2478
|
+
budgetFrom: () => { const value = parseBudgetFrom(worker?.hostBudget?.detail?.()); return value ? JSON.stringify(value) : ""; },
|
|
2479
|
+
cpuReserve: () => JSON.stringify(parseCpuReserve(cpuReserve.states?.()) ?? []),
|
|
1709
2480
|
budgetMemMiB: () => budgetField(worker?.hostBudget?.current().memMiB ?? null),
|
|
1710
2481
|
budgetCpuCenti: () => budgetField(worker?.hostBudget?.current().cpuCenti ?? null),
|
|
1711
2482
|
usedMemMiB: () => snapshotField("usedMemMiB"),
|
|
@@ -1721,6 +2492,10 @@ export async function startWorker(
|
|
|
1721
2492
|
waiters: () => snapshotField("waiters"),
|
|
1722
2493
|
jobs: () => (worker ? liveJobsNow().jobs : ""),
|
|
1723
2494
|
jobsMore: () => (worker ? liveJobsNow().jobsMore : ""),
|
|
2495
|
+
endpointsVersion: ENDPOINT_STATUS_VERSION,
|
|
2496
|
+
endpointsStartedAt: endpointStatusNow().endpointsStartedAt,
|
|
2497
|
+
endpoints: () => endpointStatusNow().endpoints,
|
|
2498
|
+
endpointsMore: () => endpointStatusNow().endpointsMore,
|
|
1724
2499
|
budgetHolds: () => snapshotField("holds"),
|
|
1725
2500
|
budgetOrphans: () => snapshotField("orphans"),
|
|
1726
2501
|
// whether the boot listing of the job containers left from before this worker started has been read
|
|
@@ -1731,6 +2506,10 @@ export async function startWorker(
|
|
|
1731
2506
|
if (typeof snap?.seeded !== "boolean") return "";
|
|
1732
2507
|
return snap.seeded ? "listed" : `unlisted:${(snap.unseeded ?? []).join(",")}`;
|
|
1733
2508
|
},
|
|
2509
|
+
// Issue #712: the venues this host blesses (the first is its default), its floor and its OpenShell engine and
|
|
2510
|
+
// gateway word, for the console's and the panel's Venues view (`hostBackendsField`: names and fixed words only).
|
|
2511
|
+
// A value, not a thunk: every part is boot configuration or the boot's gateway read, none changes without a restart.
|
|
2512
|
+
backends: hostBackendsField({ backends: config.backends, backendFloor: config.backendFloor, openshellEngine: config.openshell?.engine ?? null, gatewayWord: openshellGatewayWord }),
|
|
1734
2513
|
});
|
|
1735
2514
|
|
|
1736
2515
|
|
|
@@ -1864,6 +2643,8 @@ export async function startWorker(
|
|
|
1864
2643
|
// job containers, and an unproven answer must never free a slot.
|
|
1865
2644
|
stopContainer: makeStopContainer(),
|
|
1866
2645
|
reap: backendReaps[DEFAULT_BACKEND],
|
|
2646
|
+
// Issue #681: the host budget's leftover listing rides on the bundle, so the boot seed asks each venue its own.
|
|
2647
|
+
listJobContainers: listJobContainersFn("docker"),
|
|
1867
2648
|
// One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
|
|
1868
2649
|
// image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
|
|
1869
2650
|
// `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
|
|
@@ -1884,6 +2665,8 @@ export async function startWorker(
|
|
|
1884
2665
|
egress: config.egress, // REQ-EGRESS-ALLOWLIST: the per-job network and the proxy variables
|
|
1885
2666
|
egressProxy: config.egressProxy,
|
|
1886
2667
|
openJobLog,
|
|
2668
|
+
openLiveTranscript, // issues #643 and #644: null when PI_LIVE_TRANSCRIPTS=0 and PI_RUN_PROGRESS=0 (and no Valkey client, #646)
|
|
2669
|
+
openInboxRelay, // issue #646: the steer inbox relay, null without a Valkey client
|
|
1887
2670
|
globalPiDir: config.globalPiDir, // REQ-GLOBAL-PI-OVERLAY: :ro overlay mount when configured
|
|
1888
2671
|
allowGlobalExtensions: config.allowGlobalExtensions,
|
|
1889
2672
|
// REQ-GLOBAL-PI-OVERLAY: staged package paths; every job receives them unless its trigger set
|
|
@@ -1941,11 +2724,14 @@ export async function startWorker(
|
|
|
1941
2724
|
const opened = [registry, runtimeQueue, ...(cronQueue !== runtimeQueue ? [cronQueue] : [])];
|
|
1942
2725
|
// Bounded, then forced: a queue whose connection came up closes by sending QUIT and awaiting the reply, and against a
|
|
1943
2726
|
// server that stopped answering that wait never ends (measured through a stalling proxy), so the refusal itself would
|
|
1944
|
-
// never travel.
|
|
1945
|
-
|
|
2727
|
+
// never travel. `QUEUE_CLOSE_BOUND_MS` (five seconds) covers the host registry's own bounded close, and is the bound
|
|
2728
|
+
// each queue's own close waits for each of its steps; whatever is still open is disconnected.
|
|
2729
|
+
await settleWithin(Promise.allSettled(opened.map((handle) => Promise.resolve().then(() => handle?.close?.()))), QUEUE_CLOSE_BOUND_MS);
|
|
1946
2730
|
for (const handle of opened) {
|
|
1947
2731
|
try {
|
|
1948
|
-
|
|
2732
|
+
// BullMQ's disconnect awaits initialization and can reject asynchronously when that start failed.
|
|
2733
|
+
// Observe that cleanup rejection without extending the bound or replacing the boot's refusal.
|
|
2734
|
+
void Promise.resolve(handle?.disconnect?.()).catch(() => {});
|
|
1949
2735
|
} catch {
|
|
1950
2736
|
// a handle with nothing left to drop
|
|
1951
2737
|
}
|
|
@@ -1965,10 +2751,23 @@ export async function startWorker(
|
|
|
1965
2751
|
// a spread, a freeze, a snapshot inside `createWorker` -- un-registers the watchers in SILENCE and puts
|
|
1966
2752
|
// issue #295 back. Append only: two tests pin `[0]` as the runtime queue and `[1]` as the registry.
|
|
1967
2753
|
const extraClosers = [runtimeQueue, registry, ...(cronQueue === runtimeQueue ? [] : [cronQueue])];
|
|
2754
|
+
if (subscriptions) {
|
|
2755
|
+
const publish = () => publishSubscriptionStatus(redis, config.workerName, subscriptions.runtime).catch(() => log("subscription_status_unavailable", {}));
|
|
2756
|
+
await publish();
|
|
2757
|
+
const tick = setInterval(publish, 15_000);
|
|
2758
|
+
tick.unref?.();
|
|
2759
|
+
extraClosers.push({ close: () => clearInterval(tick) });
|
|
2760
|
+
}
|
|
2761
|
+
// A first source can arrive through live configuration after boot. Recovery and quiet-close
|
|
2762
|
+
// draining must already exist then, even when the initial source registry was empty.
|
|
2763
|
+
extraClosers.push(makeWatchDrainer({ runtime: watchRuntime, log }));
|
|
2764
|
+
extraClosers.push(makeWatchDrainer({ runtime: answerDraftRuntime, log: () => log("answer_handoff_unavailable", {}) }));
|
|
1968
2765
|
|
|
1969
2766
|
// HOISTED out of the deps literal (issue #288): the terminal failed listener below needs the same
|
|
1970
2767
|
// adapter the processor gets, and two bodies would drift exactly where drift costs a public comment.
|
|
1971
2768
|
const comment = async (job, text) => {
|
|
2769
|
+
if (job?.answerDraft || job?.purpose === "answer-draft") return;
|
|
2770
|
+
if (!await draftReplyMayPost(job)) return;
|
|
1972
2771
|
// Best-effort: the processor awaits comment() inside its try, so a rejection here would
|
|
1973
2772
|
// corrupt the job outcome and could drive a wrong retry / second PR (CONST-RETRY-INFRA-ONLY).
|
|
1974
2773
|
// This adapter NEVER throws.
|
|
@@ -2002,11 +2801,27 @@ export async function startWorker(
|
|
|
2002
2801
|
log("comment", { jobId: job?.id, text });
|
|
2003
2802
|
};
|
|
2004
2803
|
|
|
2804
|
+
// Issue #646 (DES-STEER-LATE-MESSAGE-RESUME-OR-REPLY): what a late forge message becomes at a steer-armed run's final
|
|
2805
|
+
// outcome, for all three settle sites (the processor, the host-lost sweep, the failed listener): the run's semantic
|
|
2806
|
+
// window released, then a resumed job on the queue that ran the run or one fixed reply through `comment` above. The
|
|
2807
|
+
// recovery queues are the same handles a #645 recovery is routed through, so a host queue is reached by its name.
|
|
2808
|
+
const lateForge = makeLateForge({ redis, queues: recoveryQueues, comment, log, now, canPublish: draftReplyMayPost });
|
|
2809
|
+
// The host-lost sweep's run ended on a host that is gone, so its resumed job goes to the shared queue, never to that
|
|
2810
|
+
// host's queue, where nothing would ever take it.
|
|
2811
|
+
const lateForgeShared = (entries, context) => lateForge(entries, { ...context, queueName: QUEUE });
|
|
2812
|
+
// Issue #638 (DES-AWAIT-INPUT-ENDS-THE-JOB): what an awaitInput run's held replies become. `take` runs inside the
|
|
2813
|
+
// processor's park sequence and starts the first recent one as the resumed job, with the window; the settle at the
|
|
2814
|
+
// same three final-outcome sites is the backstop that drops them for any run that did not park (held-settle.mjs).
|
|
2815
|
+
// The host-lost sweep's run can never have parked, so there it only ever drops.
|
|
2816
|
+
const heldSettle = makeHeldSettle({ redis, queues: recoveryQueues, comment, log, now });
|
|
2817
|
+
|
|
2005
2818
|
// Issue #625: the forge's parked marker, set on a park and cleared when it no longer applies. comment()'s shape and its
|
|
2006
2819
|
// rule: it NEVER throws, because the marker decides nothing (the session store's sidecar does) and a forge that refuses
|
|
2007
2820
|
// the label must not flip an outcome. A local job has no forge and no marker. Issues only: a pull request's reply is
|
|
2008
2821
|
// routed on the phrase (parked.mjs).
|
|
2009
|
-
const parkedLabel = async (job, on) => {
|
|
2822
|
+
const parkedLabel = async (job, on, approval = false) => {
|
|
2823
|
+
if (job?.answerDraft || job?.purpose === "answer-draft") return;
|
|
2824
|
+
if (!await draftReplyMayPost(job)) return;
|
|
2010
2825
|
const forge = forgeFor(job);
|
|
2011
2826
|
if (!forge?.host?.setParkedLabel || job?.target?.type !== "issue") return;
|
|
2012
2827
|
try {
|
|
@@ -2014,11 +2829,58 @@ export async function startWorker(
|
|
|
2014
2829
|
if (!auth) return;
|
|
2015
2830
|
const token = await auth.mintToken(job);
|
|
2016
2831
|
await forge.host.setParkedLabel(job, job.target, on, token);
|
|
2832
|
+
if (approval || !on) await forge.host.setParkedLabel(job, job.target, on && approval, token, APPROVAL_LABEL);
|
|
2017
2833
|
} catch (err) {
|
|
2018
2834
|
log("parked_label_failed", { jobId: job?.id, on, reason: err?.message });
|
|
2019
2835
|
}
|
|
2020
2836
|
};
|
|
2021
2837
|
|
|
2838
|
+
const approvalSessionCheck = makeApprovalSessionCheck({ sessionsDir: config.sessionsDir, maxBytes: config.sessionMaxBytes, ttlDays: config.sessionsTtlDays, maxAgeDays: config.sessionMaxAgeDays, maxResumeChain: config.sessionMaxResumeChain, maxContextPct: config.sessionMaxContextPct, now });
|
|
2839
|
+
const approvalStore = makeApprovalStore({ redis, now });
|
|
2840
|
+
const approvalRuntime = makeApprovalRuntime({
|
|
2841
|
+
store: approvalStore,
|
|
2842
|
+
touchStore,
|
|
2843
|
+
touchOutbox,
|
|
2844
|
+
checkPolicy: makeCheckApprovalPolicy({ triggersPath: onceTriggersFile, parseTriggers }),
|
|
2845
|
+
now,
|
|
2846
|
+
host: durableHost,
|
|
2847
|
+
hostQueue,
|
|
2848
|
+
requireSessionDigest: true,
|
|
2849
|
+
enqueueExpiry: async (request) => {
|
|
2850
|
+
const queue = hostQueue ? cronQueue : runtimeQueue;
|
|
2851
|
+
const jobId = approvalExpiryJobId(request.requestId);
|
|
2852
|
+
await queue.add(request.kind, {
|
|
2853
|
+
...request.resumeJob,
|
|
2854
|
+
approvalRequestId: request.requestId,
|
|
2855
|
+
trigger: { ...request.resumeJob.trigger, deliveryId: `expiry-${request.requestId}`, comment: { body: "/decline" } },
|
|
2856
|
+
}, approvalExpiryJobOpts(jobId));
|
|
2857
|
+
return { ok: true };
|
|
2858
|
+
},
|
|
2859
|
+
comment: async (request, text) => {
|
|
2860
|
+
// One expiry comment per request: the mark lives as long as the request's finished state (MILLISECONDS, PX).
|
|
2861
|
+
if (await redis.set(`run:approval:expiry-comment:${request.requestId}`, "1", "PX", APPROVAL_FINISHED_TTL_MS, "NX") === "OK") await comment(request.resumeJob, text);
|
|
2862
|
+
},
|
|
2863
|
+
});
|
|
2864
|
+
|
|
2865
|
+
let approvalSweep = null;
|
|
2866
|
+
const startApprovalSweep = () => {
|
|
2867
|
+
if (approvalSweep) return;
|
|
2868
|
+
approvalSweep = makeRetentionSweepFn({
|
|
2869
|
+
reapers: [{ name: "approval", reap: async () => {
|
|
2870
|
+
const result = await approvalRuntime.sweep();
|
|
2871
|
+
if (!result.ok) log("approval_expiry_unavailable", { failures: result.failures?.length ?? 1 });
|
|
2872
|
+
} }], intervalMs: 60_000, log,
|
|
2873
|
+
});
|
|
2874
|
+
approvalSweep.start();
|
|
2875
|
+
extraClosers.push(approvalSweep);
|
|
2876
|
+
};
|
|
2877
|
+
const publishApproval = approvalRuntime.afterPark;
|
|
2878
|
+
approvalRuntime.afterPark = async (options) => {
|
|
2879
|
+
const result = await publishApproval(options);
|
|
2880
|
+
if (result.ok) startApprovalSweep();
|
|
2881
|
+
return result;
|
|
2882
|
+
};
|
|
2883
|
+
|
|
2022
2884
|
// The operator's failure hook (issue #288, INT-ON-FAILURE-HOOK-CONTRACT). Constructed ONLY when the
|
|
2023
2885
|
// knob is set: an unset PI_ON_FAILURE builds nothing, spawns nothing, and logs nothing -- the
|
|
2024
2886
|
// byte-identical guarantee. Fired from the two terminal listeners below, never from the processor:
|
|
@@ -2027,8 +2889,10 @@ export async function startWorker(
|
|
|
2027
2889
|
// `hostEnv: env`, not the default process.env -- the #309 rule every spawner in this file follows
|
|
2028
2890
|
// (runContainer, resolveSecrets): a subprocess runs with THIS worker's env, identical on the real
|
|
2029
2891
|
// path and divergent only under an injected one, which is exactly where the difference would hide.
|
|
2892
|
+
const hookRuns = config.onFailure ? makeHookRunsPublisher({ redis, name: config.workerName, now, log }) : null;
|
|
2893
|
+
if (hookRuns) extraClosers.push(hookRuns);
|
|
2030
2894
|
const onFailure = config.onFailure
|
|
2031
|
-
? makeOnFailure({ command: config.onFailure, timeoutMs: config.onFailureTimeoutMs, host: config.workerName ?? "", hostEnv: env, log })
|
|
2895
|
+
? makeOnFailure({ command: config.onFailure, timeoutMs: config.onFailureTimeoutMs, host: config.workerName ?? "", hostEnv: env, log, publish: hookRuns.publish })
|
|
2032
2896
|
: null;
|
|
2033
2897
|
// Which POLICY reasons page the operator. Paid terminals only: worker-abort, runner-policy and every
|
|
2034
2898
|
// named runner reason cost a container and ended wrong. The runner's named reasons are SPREAD from
|
|
@@ -2043,24 +2907,45 @@ export async function startWorker(
|
|
|
2043
2907
|
// One predicate for the completed listener and the lost-lock path below, so a record replays exactly the page its
|
|
2044
2908
|
// result would have sent.
|
|
2045
2909
|
const pagesAsPolicy = (result) => Boolean(onFailure) && result?.outcome === "policy" && HOOK_POLICY_REASONS.has(result.reason) && result.budgetReserved !== false;
|
|
2046
|
-
//
|
|
2047
|
-
//
|
|
2048
|
-
//
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
//
|
|
2052
|
-
//
|
|
2053
|
-
//
|
|
2054
|
-
const
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
//
|
|
2060
|
-
|
|
2910
|
+
// Issue #645: the one RETURNED failure that pages. A recovery whose container ran, ended infra and could not queue
|
|
2911
|
+
// another recovery returns `failed`/`recovery-failed` (the processor posts its own sentence), so it reaches the
|
|
2912
|
+
// completed listener and never the failed one: without this it would be the only paid terminal that pages nobody.
|
|
2913
|
+
const pagesAsReturnedFailure = (result) => Boolean(onFailure) && result?.outcome === "failed" && result.reason === "recovery-failed";
|
|
2914
|
+
// The infra-terminal sentences (FAILED_COMMENT, FAILED_COMMENT_BY_REASON, issue #288) and the host-lost one
|
|
2915
|
+
// (RECOVERY_HOST_LOST_COMMENT, issue #645) are module constants in comment-tables.mjs since issue #710, with why each is fixed.
|
|
2916
|
+
// Fleet-wide, on every host: whichever host's sweep claims a due recovery first tells its run. The record says it too,
|
|
2917
|
+
// so the panel's history shows the run ended rather than a recovery that vanished.
|
|
2918
|
+
const sweepHostLost = makeHostLostSweepFn({
|
|
2919
|
+
redis,
|
|
2920
|
+
queues: recoveryQueues,
|
|
2921
|
+
now,
|
|
2922
|
+
log,
|
|
2923
|
+
// The registry's live rows (host-registry.mjs heartbeats, HOST_TTL_MS): a recovery whose host is still beating is
|
|
2924
|
+
// delayed (a pause window, a full host), not lost. Names in the run checkpointer's spelling, as the pointer has them;
|
|
2925
|
+
// null when the registry could not be read, which extends rather than fails.
|
|
2926
|
+
liveHosts: async () => {
|
|
2927
|
+
const read = await readLiveHosts(redis, { now, prune: false });
|
|
2928
|
+
if (read?.unreachable || !Array.isArray(read?.hosts)) return null;
|
|
2929
|
+
return new Set(read.hosts.map((h) => normalizeHost(h.name)));
|
|
2930
|
+
},
|
|
2931
|
+
onLost: async (job) => {
|
|
2932
|
+
const at = new Date(now()).toISOString();
|
|
2933
|
+
// Issue #646: the run's final outcome, so its inbox is closed and counted here too (index.mjs's rule for every
|
|
2934
|
+
// record it writes); a steer job's waiting messages are then `late`, never left `queued` until the key expires.
|
|
2935
|
+
const lostError = { reason: "recovery-host-lost" };
|
|
2936
|
+
if (job?.data?.steer === true) {
|
|
2937
|
+
const steered = await settleInbox({ redis, job, error: lostError, log, onLateForge: lateForgeShared });
|
|
2938
|
+
if (steered) lostError.steered = steered;
|
|
2939
|
+
}
|
|
2940
|
+
if (job?.data?.awaitInput === true) await heldSettle({ job, error: lostError, queueName: QUEUE });
|
|
2941
|
+
recordRun({ job: { id: job.id, name: job.name, data: job.data, opts: job.opts, attemptsMade: 0, timestamp: job.timestamp }, error: lostError, startedAt: null, endedAt: at });
|
|
2942
|
+
await comment({ ...job.data, id: job.id }, RECOVERY_HOST_LOST_COMMENT);
|
|
2943
|
+
onFailure?.({ jobId: job.id, outcome: "failed", reason: "recovery-host-lost" });
|
|
2944
|
+
},
|
|
2061
2945
|
});
|
|
2062
2946
|
|
|
2063
2947
|
const worker = createWorkerFn({
|
|
2948
|
+
endpointSlots,
|
|
2064
2949
|
connection: valkeyConn(),
|
|
2065
2950
|
// Issue #596, phase 2 (DES-HOST-BUDGET): the host budget's inputs. `createWorker` builds the ONE budget from them and
|
|
2066
2951
|
// shares it between both queues' processors. The settings and the default size were refused at boot if bad.
|
|
@@ -2069,7 +2954,10 @@ export async function startWorker(
|
|
|
2069
2954
|
jobDefault: { memMiB: config.jobSize.memMiB, cpuCenti: config.jobSize.cpuCenti },
|
|
2070
2955
|
readFacts: readHostFacts,
|
|
2071
2956
|
onRefresh: syncCpuReserve,
|
|
2072
|
-
|
|
2957
|
+
// Issue #681: asked of the venue that holds the container, through the registry. This used to pick the CLI from the
|
|
2958
|
+
// venue's name and send every venue but podman to docker, so a third venue's orphan was either freed while it still
|
|
2959
|
+
// ran (docker lists no such name) or never freed (no docker on the host).
|
|
2960
|
+
containerGone: containerGoneFn ?? backends.containerGone,
|
|
2073
2961
|
// AFTER the boot reaper (above), every blessed venue's remaining job containers, seeded into the ledger
|
|
2074
2962
|
// as orphans from their size labels. PER VENUE (gate round 2 of phase 2): a venue that cannot be listed
|
|
2075
2963
|
// stops only its own jobs until a tick reads it, because a container nobody counted is an overcommit, while the
|
|
@@ -2078,18 +2966,26 @@ export async function startWorker(
|
|
|
2078
2966
|
// cause that refuses a boot (the venue's boot-refusing set: every job on it is refused before a container, so this
|
|
2079
2967
|
// worker starts none there). A cause decided from one unreadable answer is re-decided per job and can clear, so
|
|
2080
2968
|
// it leaves the venue unread.
|
|
2969
|
+
// Issue #681: every BLESSED registered venue, each listed by its own bundle; it used to be local and podman by name,
|
|
2970
|
+
// so a third venue was never seeded and its leftover containers were never counted. Only local and podman have a
|
|
2971
|
+
// boot job-user decision; another venue's holds-none rule is ENOENT alone.
|
|
2081
2972
|
survivors: Object.fromEntries(
|
|
2082
|
-
|
|
2083
|
-
backend
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2973
|
+
Object.entries(backends.listers())
|
|
2974
|
+
.filter(([backend]) => config.backends.includes(backend))
|
|
2975
|
+
.map(([backend, list]) => {
|
|
2976
|
+
const [decision, stable] = backend === DEFAULT_BACKEND ? [bootDecision, BOOT_REFUSING_JOB_USER_CAUSES] : backend === PODMAN_BACKEND ? [bootPodmanDecision, PODMAN_BOOT_REFUSING_CAUSES] : [null, new Set()];
|
|
2977
|
+
return [
|
|
2978
|
+
backend,
|
|
2979
|
+
async () => {
|
|
2980
|
+
try {
|
|
2981
|
+
return (await list()).map((c) => ({ ...c, venue: { backend } }));
|
|
2982
|
+
} catch (error) {
|
|
2983
|
+
if (error?.code === "ENOENT" || (decision?.mode === "unmappable" && stable.has(decision.cause))) return [];
|
|
2984
|
+
throw error;
|
|
2985
|
+
}
|
|
2986
|
+
},
|
|
2987
|
+
];
|
|
2988
|
+
}),
|
|
2093
2989
|
),
|
|
2094
2990
|
defaultVenue: config.defaultBackend,
|
|
2095
2991
|
now,
|
|
@@ -2110,6 +3006,7 @@ export async function startWorker(
|
|
|
2110
3006
|
name: config.workerName,
|
|
2111
3007
|
concurrency: bootConcurrency,
|
|
2112
3008
|
getSettings,
|
|
3009
|
+
readModelRoles: makeReadModelRoles({ triggersPath: onceTriggersFile, log }),
|
|
2113
3010
|
redis,
|
|
2114
3011
|
recordRun,
|
|
2115
3012
|
settledRecord,
|
|
@@ -2118,6 +3015,12 @@ export async function startWorker(
|
|
|
2118
3015
|
// REQ-SCOPED-PAUSE-WINDOWS: the processor defers a job whose folder/repo is inside an active window.
|
|
2119
3016
|
// Reads the live-reloaded ref, so an operator edit takes effect on the next job without a restart.
|
|
2120
3017
|
pauseUntil: (job, now) => pauseUntilMs(pauseWindows.current, job, now),
|
|
3018
|
+
// Issue #761: while the file on disk was written by a newer pi-dispatch, `pauseUntil` above reads the last good
|
|
3019
|
+
// windows, which may not pause what the file pauses, so every job waits for this host's upgrade instead.
|
|
3020
|
+
pauseWindowsNewer: () => pauseWindows.newer === true,
|
|
3021
|
+
// Issue #761: while the triggers file on disk was written by a newer pi-dispatch, every reader of it holds a last
|
|
3022
|
+
// good copy or refuses, so every job waits for this host's upgrade instead (the processor's first gates).
|
|
3023
|
+
triggersNewer: () => triggersState.newer === true,
|
|
2121
3024
|
// Issue #242: the scoped-limits snapshot the pickup gate and the scoped budget read, once per
|
|
2122
3025
|
// pickup, from the live-reloaded ref -- same next-job grain as pauseUntil above.
|
|
2123
3026
|
scopedLimits: () => scopedLimits.current,
|
|
@@ -2174,6 +3077,26 @@ export async function startWorker(
|
|
|
2174
3077
|
deps: {
|
|
2175
3078
|
collectChain,
|
|
2176
3079
|
collectPlan,
|
|
3080
|
+
// Issue #646: the late-message branches at the processor's settle (index.mjs `settleSteer`).
|
|
3081
|
+
onLateForge: lateForge,
|
|
3082
|
+
watchRuntime,
|
|
3083
|
+
questionGroupRuntime,
|
|
3084
|
+
questionGroupStore,
|
|
3085
|
+
questionCaptureEnabled: () => {
|
|
3086
|
+
try { return parseTriggersDocument(readFileSync(onceTriggersFile, "utf8"), onceTriggersFile).zeroTouch?.capture === true; }
|
|
3087
|
+
catch { log("question_capture_unavailable", { reason: "configuration-unreadable" }); return false; }
|
|
3088
|
+
},
|
|
3089
|
+
touchStore,
|
|
3090
|
+
touchOutbox,
|
|
3091
|
+
touchNow: now,
|
|
3092
|
+
collectWatch: watchRuntime.collect,
|
|
3093
|
+
// Issue #638: held replies at the processor's settle (index.mjs `settleSteer` calls it first). Wired, it also arms
|
|
3094
|
+
// the keeper that makes an awaitInput run reachable while it runs.
|
|
3095
|
+
settleHeld: heldSettle,
|
|
3096
|
+
// The take inside the park sequence (processor.mjs), which starts the held answer before the parked comment.
|
|
3097
|
+
takeHeld: heldSettle.take,
|
|
3098
|
+
answerDraftRuntime,
|
|
3099
|
+
answerDraftStore,
|
|
2177
3100
|
// The one-shot pre-spend check (issue #231): reads the same file the disarm writes, refuses
|
|
2178
3101
|
// only on a FOREIGN positive mark (index.mjs binds the real queue jobId so a retry of the
|
|
2179
3102
|
// spending delivery is excused). In the compose topology this check is the once-enforcement
|
|
@@ -2191,24 +3114,53 @@ export async function startWorker(
|
|
|
2191
3114
|
// A PROBE: whatever it resolves is dropped on the floor. The credential itself is read where it always
|
|
2192
3115
|
// was, inside buildContainerEnv, so no live key is ever in scope in the processor.
|
|
2193
3116
|
// Issue #503: `modelEndpoints` is the pickup's snapshot, the same one runContainer hands buildContainerEnv.
|
|
3117
|
+
subscriptionRuntime: subscriptions?.runtime ?? null,
|
|
3118
|
+
subscriptionAuthRequired: async (job) => {
|
|
3119
|
+
if (!subscriptions) return false;
|
|
3120
|
+
for (const ref of subscriptionRoutes(job)) if ((await subscriptions.auth.check(ref.provider)).managed) return true;
|
|
3121
|
+
return false;
|
|
3122
|
+
},
|
|
2194
3123
|
checkProviderCredential: (job, { modelEndpoints = null } = {}) => {
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
3124
|
+
const legacy = () => {
|
|
3125
|
+
try {
|
|
3126
|
+
resolveProviderCredential({ provider: job.provider, hostEnv: env, authFromPi: config.authFromPi, forwardEnv: job.answerDraft || job.watch || job.questionGrouping ? [] : config.forwardEnv, modelEndpoints });
|
|
3127
|
+
return { ok: true };
|
|
3128
|
+
} catch (error) {
|
|
3129
|
+
// Only OUR determinate refusal. Anything else (a bug here, an fs fault the module does not model)
|
|
3130
|
+
// must not become a policy refusal on the operator's issue: it rethrows into runJob's catch, which
|
|
3131
|
+
// classifies it the way it always did.
|
|
3132
|
+
// Issue #503: a transient overlay read at this pickup is no verdict; the processor retries it as infra.
|
|
3133
|
+
if (error?.piDispatchTransient === true) return { ok: false, unavailable: error.code ?? "unreadable" };
|
|
3134
|
+
if (error?.piDispatchConfig !== true) throw error;
|
|
3135
|
+
return { ok: false, message: error.message };
|
|
3136
|
+
}
|
|
3137
|
+
};
|
|
3138
|
+
return subscriptions ? subscriptions.auth.check(job.provider).then((checked) => checked.managed ? checked : legacy()) : legacy();
|
|
2207
3139
|
},
|
|
2208
3140
|
// Issue #502. The model-exists gate: pi's builtin catalog, then the overlay models.json, read at most once per
|
|
2209
3141
|
// job and only when a model is not builtin, so the operator's edits apply without a restart. The SAME file
|
|
2210
3142
|
// the endpoint gate reads, through the same reader, so absent, unreadable and unparseable mean one thing.
|
|
2211
3143
|
checkModelsKnown: (refs) => checkModelsKnown(refs, { readOverlay: () => readOverlayModels(config.globalPiDir) }),
|
|
3144
|
+
checkFailoverCandidate: (job, { models, endpoints }) => {
|
|
3145
|
+
const egress = failoverEgressRequired(job, { egress: config.egress, defaultBackend: config.backends[0] });
|
|
3146
|
+
return checkFailoverCandidate(job, {
|
|
3147
|
+
models, endpoints, egress, allowlist: egress ? readEgressAllowlist() : [],
|
|
3148
|
+
credential: async (candidate, { modelEndpoints }) => {
|
|
3149
|
+
if (subscriptions) {
|
|
3150
|
+
const checked = await subscriptions.auth.check(candidate.provider);
|
|
3151
|
+
if (checked.managed) return checked;
|
|
3152
|
+
}
|
|
3153
|
+
try {
|
|
3154
|
+
resolveProviderCredential({ provider: candidate.provider, hostEnv: env, authFromPi: config.authFromPi, forwardEnv: config.forwardEnv, modelEndpoints });
|
|
3155
|
+
return { ok: true };
|
|
3156
|
+
} catch (error) {
|
|
3157
|
+
if (error?.piDispatchTransient) return { unavailable: true };
|
|
3158
|
+
if (error?.piDispatchConfig) return { ok: false };
|
|
3159
|
+
throw error;
|
|
3160
|
+
}
|
|
3161
|
+
},
|
|
3162
|
+
});
|
|
3163
|
+
},
|
|
2212
3164
|
// Issue #503 part 7: the builtin catalog's model object, for the zero-rated check that lets a job on local
|
|
2213
3165
|
// zero-rated models reserve nothing in the dollar windows (processor.mjs, `zeroRatedVerdict`).
|
|
2214
3166
|
builtinModel,
|
|
@@ -2242,9 +3194,9 @@ export async function startWorker(
|
|
|
2242
3194
|
const held = await runtimeQueue.getJob(id);
|
|
2243
3195
|
if (!held) return false;
|
|
2244
3196
|
// EXISTENCE IS NOT LIVENESS, and the difference decides whether a target stays deafened:
|
|
2245
|
-
// `removeOnComplete`/`removeOnFail` keep a finished job's hash for
|
|
2246
|
-
// can never wake again would answer "still waiting" for a month. Only a state it can
|
|
2247
|
-
// picked up from counts.
|
|
3197
|
+
// `removeOnComplete`/`removeOnFail` keep a finished job's hash for `FORGE_JOB_RETENTION_S`, so a
|
|
3198
|
+
// holder that can never wake again would answer "still waiting" for a month. Only a state it can
|
|
3199
|
+
// still be picked up from counts.
|
|
2248
3200
|
const state = await held.getState();
|
|
2249
3201
|
return state === "delayed" || state === "waiting" || state === "active" || state === "prioritized" || state === "waiting-children";
|
|
2250
3202
|
},
|
|
@@ -2255,10 +3207,17 @@ export async function startWorker(
|
|
|
2255
3207
|
// `local`'s are the closures built beside its bundle below, unchanged.
|
|
2256
3208
|
observationPreflight: backends.observationPreflight,
|
|
2257
3209
|
jobUserPreflight: backends.jobUserPreflight,
|
|
3210
|
+
// Issue #704 part 3: the venue's judgement of a project secret bundle, dispatched like the two above.
|
|
3211
|
+
bundleGate: backends.bundleGate,
|
|
2258
3212
|
// Completed-only, so a policy or infra exit leaves the canonical transcript byte-identical and a
|
|
2259
3213
|
// retry starts from what the first attempt did (CONST-RETRY-INFRA-ONLY).
|
|
2260
3214
|
promoteSession: sessionStore.promoteSession,
|
|
3215
|
+
// Issue #645: a recovery stages its checkpoint's transcript as the job's session (`reason: "recovered"`).
|
|
3216
|
+
...(typeof sessionStore.stageRecovered === "function" ? { stageRecovered: sessionStore.stageRecovered } : {}),
|
|
2261
3217
|
isParked: sessionStore.isParked,
|
|
3218
|
+
readParkedInfo: sessionStore.readParkedInfo,
|
|
3219
|
+
approvalRuntime,
|
|
3220
|
+
approvalSessionCheck,
|
|
2262
3221
|
parkedLabel,
|
|
2263
3222
|
// The same value the store above was built from, passed explicitly so the processor's fail-closed
|
|
2264
3223
|
// `run.resume` gate answers from THIS config rather than from its own env default. Identical on the
|
|
@@ -2278,6 +3237,30 @@ export async function startWorker(
|
|
|
2278
3237
|
// injected env they were not, which is the divergence this file already calls out by name for
|
|
2279
3238
|
// sessionsDir. One deployment value, one place, same rule as the profiles below it.
|
|
2280
3239
|
resolveSecrets: makeSecretsResolverFn({ envProfiles: config.secretProfiles, roots: config.secretResolverRoots, timeoutMs: config.secretResolveTimeoutMs, forwardEnv: config.forwardEnv, hostEnv: env, log }),
|
|
3240
|
+
// Issue #704, project secret bundles. The binding is read from the SAME triggers file the one-shot, skew and
|
|
3241
|
+
// portfolio checks read (`onceTriggersFile`), by the worker, never carried by the receiver, and answered from
|
|
3242
|
+
// the last good read while the file is broken (triggers-file.mjs says why). The resolver is the run.secrets
|
|
3243
|
+
// seam's sibling, with the same roots, timeout, forwarded names and host env.
|
|
3244
|
+
// `bundlesDeclared`: whether this host declares any bundle profile, env or panel. On such a host a MISSING
|
|
3245
|
+
// triggers file is a failure (last good read, else retry), never "no bundles" (triggers-file.mjs says why).
|
|
3246
|
+
canaryBusy: () => canaryBusy(redis),
|
|
3247
|
+
findSecretBundle: makeFindSecretBundle({
|
|
3248
|
+
triggersPath: onceTriggersFile,
|
|
3249
|
+
log,
|
|
3250
|
+
bundlesDeclared: () => {
|
|
3251
|
+
if (Object.keys(config.secretBundleProfiles ?? {}).length > 0) return true;
|
|
3252
|
+
const settings = getSettings();
|
|
3253
|
+
return !settings.invalid && Object.keys(settings.secretBundles ?? {}).length > 0;
|
|
3254
|
+
},
|
|
3255
|
+
}),
|
|
3256
|
+
secretBundles: makeBundleResolver({ envBundles: config.secretBundleProfiles ?? {}, roots: config.secretResolverRoots, timeoutMs: config.secretResolveTimeoutMs, forwardEnv: config.forwardEnv, hostEnv: env, log }),
|
|
3257
|
+
// The fork check's seam: the job's own forge host, the one prepare hands run.resume's fork gate. A forge with
|
|
3258
|
+
// no host (none configured for that kind) answers nothing, which the processor reads as a fork: no bundle.
|
|
3259
|
+
resolvePullRequestHead: async (job, token) => {
|
|
3260
|
+
const host = forgeFor(job)?.host;
|
|
3261
|
+
if (typeof host?.resolvePullRequestHead !== "function") return null;
|
|
3262
|
+
return await host.resolvePullRequestHead(job, token);
|
|
3263
|
+
},
|
|
2281
3264
|
// REQ-MCP-OPERATOR-SERVERS: the overlay's mcp.json read once per job that names a server, never at boot (an
|
|
2282
3265
|
// operator who re-runs import-pi must not have to restart the worker). The allowlist is the one the proxy
|
|
2283
3266
|
// mounts, this folder's egress-allowlist.conf on both venues (up.mjs, podman-stack.mjs), and the endpoints
|
|
@@ -2289,29 +3272,9 @@ export async function startWorker(
|
|
|
2289
3272
|
egress: config.egress,
|
|
2290
3273
|
// DES-FORGE-MCP: the GitLab a gitlab job's forge server calls, the same URL its clone and its glab use.
|
|
2291
3274
|
gitlabUrl: config.gitlab?.apiUrl ?? null,
|
|
2292
|
-
// The credentials the job is set up to spend with (
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
reserved: (job) => {
|
|
2296
|
-
let overlayModels = null;
|
|
2297
|
-
try {
|
|
2298
|
-
overlayModels = readOverlayModels(config.globalPiDir);
|
|
2299
|
-
} catch (error) {
|
|
2300
|
-
if (MCP_TRANSIENT_READ_CODES.has(error?.code)) throw error;
|
|
2301
|
-
}
|
|
2302
|
-
const providers = jobProviders(job, { defaultProvider: config.provider, allowedModels: config.allowedModels, everyProvider: piProviders() });
|
|
2303
|
-
return new Set([...RESERVED_ENV_NAMES, ...providerKeyNames({ providers, candidatesOf: providerKeyCandidates, overlayModels })]);
|
|
2304
|
-
},
|
|
2305
|
-
allowlist: () => {
|
|
2306
|
-
try {
|
|
2307
|
-
return parseAllowlist(readFileSync(join(process.cwd(), "egress-allowlist.conf"), "utf8"));
|
|
2308
|
-
} catch (error) {
|
|
2309
|
-
// No allowlist here: every server outside a declared endpoint is unreachable, which it is. Any other
|
|
2310
|
-
// failure is the check's to classify (a transient errno is retried).
|
|
2311
|
-
if (error?.code === "ENOENT") return [];
|
|
2312
|
-
throw error;
|
|
2313
|
-
}
|
|
2314
|
-
},
|
|
3275
|
+
// The credentials the job is set up to spend with (`reservedJobNames`, which the console's preview asks too).
|
|
3276
|
+
reserved: (job) => reservedJobNames(job, { readModels: () => readOverlayModels(config.globalPiDir), defaultProvider: config.provider, allowedModels: config.allowedModels, everyProvider: piProviders(), candidatesOf: providerKeyCandidates }),
|
|
3277
|
+
allowlist: readEgressAllowlist,
|
|
2315
3278
|
endpoints: () => modelEndpoints.current,
|
|
2316
3279
|
}),
|
|
2317
3280
|
// #227. What PI_BACKENDS blessed, so a trigger naming an unblessed venue refuses pre-spend. The
|
|
@@ -2329,8 +3292,14 @@ export async function startWorker(
|
|
|
2329
3292
|
// Off the REGISTRY's surface, like every other per-job backend fact. Rebuilding it at the call site
|
|
2330
3293
|
// is how a fact ends up dispatched on one path and hardcoded on another.
|
|
2331
3294
|
neverStartedExits: backends.neverStartedExits,
|
|
2332
|
-
|
|
3295
|
+
checkSandboxWorkspace: (job) => checkSandboxWorkspace(job, { sandboxDir: config.sandboxDir }),
|
|
3296
|
+
packageCanary: makePackageCanaryRuntime({ globalPiDir: config.globalPiDir, env, host: config.workerName, log, publishFacts: (facts) => publishCanaryFacts(redis, facts) }),
|
|
3297
|
+
prepareWorkspace: makePrepareWorkspaceFn({
|
|
2333
3298
|
jobsDir: config.jobsDir,
|
|
3299
|
+
// Issue #643: with PI_LIVE_TRANSCRIPTS=0 no live file is staged either, so a job without a resume transcript
|
|
3300
|
+
// runs exactly as it did before the feature (its session in memory, no /session mount). Since issue #644 the
|
|
3301
|
+
// progress reader needs the same file, so it is left out only when both switches are off.
|
|
3302
|
+
...(config.liveTranscripts || config.runProgress ? {} : { stageLive: null }),
|
|
2334
3303
|
// Issue #464: the boot's own check, asked again before every job against the same env.
|
|
2335
3304
|
ensureDir: (dir) => ensureJobsDirFn(dir, { env }),
|
|
2336
3305
|
forgeFor,
|
|
@@ -2341,7 +3310,7 @@ export async function startWorker(
|
|
|
2341
3310
|
defaultBackend: config.defaultBackend,
|
|
2342
3311
|
// Issue #504 part B: a local job's folder is resolved at prepare and mounted resolved; one named inside a run
|
|
2343
3312
|
// root or a cron folder must still resolve inside it, and none may be the envelope's folder or above it.
|
|
2344
|
-
localPlacement: { jobPaths: envelopeJobPathsNow, envelopeFile: config.envelopeFile ?? null },
|
|
3313
|
+
localPlacement: { jobPaths: envelopeJobPathsNow, envelopeFile: config.envelopeFile ?? null, sandboxDir: config.sandboxDir, authStoreDir: subscriptions?.storeDir ?? null },
|
|
2345
3314
|
// Issue #505: /job/portfolio.json for a confirmed portfolio job.
|
|
2346
3315
|
portfolioSnapshot,
|
|
2347
3316
|
preparers: makeForgePreparers({ gitlabApiUrl: config.gitlab?.apiUrl ?? null, forgejoApiUrl: config.forgejo?.apiUrl ?? null, azureOrgUrl: config.azure?.orgUrl ?? null }),
|
|
@@ -2395,6 +3364,21 @@ export async function startWorker(
|
|
|
2395
3364
|
if (!host) throw configError(`no forge host is configured for job kind ${JSON.stringify(job?.kind)}`);
|
|
2396
3365
|
return await host.isDefaultBranchProtected(job, token);
|
|
2397
3366
|
},
|
|
3367
|
+
// Issue #645 (durable runs), built above. Null where this host has no store (`recover-store-unset` refuses an
|
|
3368
|
+
// armed job before it spends); the ledger, the pointer read and the enqueue exist on every host.
|
|
3369
|
+
workspaceCheckpointer,
|
|
3370
|
+
forkStore,
|
|
3371
|
+
checkForkPolicy: makeCheckForkPolicy({ triggersPath: onceTriggersFile }),
|
|
3372
|
+
checkThinking: (job) => thinkingLevelAllowed(job, { models: readOverlayModels(config.globalPiDir), builtinModel }),
|
|
3373
|
+
runCheckpointer,
|
|
3374
|
+
runBudgetLedger,
|
|
3375
|
+
readCheckpointPointer,
|
|
3376
|
+
enqueueRecovery,
|
|
3377
|
+
// A recovery landing on the host that crashed takes one more checkpoint from the crashed attempt's job directory
|
|
3378
|
+
// before it restores (recovery-queue.mjs `makeSalvager`): `salvageCheckpoint({ runId, pointer, attempt })`.
|
|
3379
|
+
salvageCheckpoint,
|
|
3380
|
+
readRecord: readRunRecord,
|
|
3381
|
+
hostName: durableHost,
|
|
2398
3382
|
},
|
|
2399
3383
|
});
|
|
2400
3384
|
|
|
@@ -2406,6 +3390,15 @@ export async function startWorker(
|
|
|
2406
3390
|
// the live Worker held the loop forever. The catch below covers the REGION, not a list of calls,
|
|
2407
3391
|
// so a step added tomorrow is covered the day it is added.
|
|
2408
3392
|
try {
|
|
3393
|
+
// Human preparation uses its own bounded control worker, outside the paid processor and its slot ledgers.
|
|
3394
|
+
const prepareSandbox = createSandboxPreparerFn({ config, env, forgeFor, readTriggers: () => readFileSync(onceTriggersFile, "utf8"), projects: () => projects.current, limits: () => scopedLimits.current, now });
|
|
3395
|
+
const sandboxControl = startSandboxPrepareWorkerFn({ connection: valkeyConn(), host: config.workerName, config, prepare: prepareSandbox, now, log });
|
|
3396
|
+
// Its close waits for the preparation it is running, which nothing aborts first, so that step gets the longer bound.
|
|
3397
|
+
settleCloseOf(sandboxControl, { closeBoundMs: SANDBOX_CONTROL_CLOSE_BOUND_MS });
|
|
3398
|
+
extraClosers.push(sandboxControl);
|
|
3399
|
+
const sandboxInventory = makeSandboxInventoryFn({ redis, host: config.workerName, read: retainedNow, settings: { image: config.jobImage, defaultBackend: config.defaultBackend, backends: config.backends, egress: config.egress, egressProxy: config.egressProxy, sandboxRetentionHours: config.sandboxRetentionHours, sandboxPinDays: config.sandboxPinDays, jobMemory: env.PI_JOB_MEMORY, jobCpus: env.PI_JOB_CPUS }, now, log });
|
|
3400
|
+
sandboxInventory.start();
|
|
3401
|
+
extraClosers.push(sandboxInventory);
|
|
2409
3402
|
|
|
2410
3403
|
// REQ-LOCAL-JOB-VISIBILITY: exactly one terminal line per job, carrying the job id and outcome,
|
|
2411
3404
|
// where the operator is already looking. This is the local counterpart of the GitHub issue
|
|
@@ -2419,7 +3412,19 @@ export async function startWorker(
|
|
|
2419
3412
|
// BOTH workers, or a cron job on the host queue produces no `job_completed` line at all -- and
|
|
2420
3413
|
// REQ-LOCAL-JOB-VISIBILITY's whole point is that a missing line is what tells a human a run did nothing.
|
|
2421
3414
|
const allWorkers = [worker, ...(worker.hostWorker ? [worker.hostWorker] : [])];
|
|
3415
|
+
// Issue #812: a finished job's model attempt key gets its expiry (model-attempts.mjs), on the queue's own word that
|
|
3416
|
+
// the job finished: this event, and `failed` with `finishedOn` below. Fire and forget, inside the existing listener
|
|
3417
|
+
// bodies: an expiry that fails leaves the key as it was, without one, and the fixed line says so. The id is the
|
|
3418
|
+
// processor's (index.mjs `modelAttemptId`); a job with no queue name to read names no key, so nothing is sent.
|
|
3419
|
+
const finishAttempt = (w, job) => {
|
|
3420
|
+
const queueName = job?.queueName ?? w.name;
|
|
3421
|
+
if (typeof queueName !== "string" || queueName === "" || job?.id === undefined || job?.id === null) return;
|
|
3422
|
+
void Promise.resolve()
|
|
3423
|
+
.then(() => finishModelAttemptFn(redis, modelAttemptIdOf(queueName, job.id)))
|
|
3424
|
+
.catch(() => log("model_attempt_finish_failed", { jobId: String(job.id) }));
|
|
3425
|
+
};
|
|
2422
3426
|
for (const w of allWorkers) w.on("completed", (job, result) => {
|
|
3427
|
+
finishAttempt(w, job);
|
|
2423
3428
|
log("job_completed", { jobId: job?.id, outcome: result?.outcome, ...(result?.reason ? { reason: result.reason } : {}) });
|
|
2424
3429
|
// The hook's POLICY half (issue #288): worker-abort, runner-policy and provider-auth-refused RETURN, so they land here
|
|
2425
3430
|
// and never in the failed listener -- a failed-only mount would miss exactly the paid terminals
|
|
@@ -2431,11 +3436,26 @@ export async function startWorker(
|
|
|
2431
3436
|
if (pagesAsPolicy(result)) {
|
|
2432
3437
|
onFailure({ jobId: job?.id, outcome: "policy", reason: result.reason });
|
|
2433
3438
|
}
|
|
3439
|
+
if (pagesAsReturnedFailure(result)) onFailure({ jobId: job?.id, outcome: "failed", reason: result.reason });
|
|
2434
3440
|
});
|
|
2435
3441
|
// The failed listener's body, for a failure the queue decided. Split out only so the lost-lock check below can
|
|
2436
3442
|
// run it after an await; a failure of any other reason runs it synchronously, exactly as before.
|
|
2437
|
-
|
|
2438
|
-
|
|
3443
|
+
//
|
|
3444
|
+
// `settledElsewhere` (issue #761) is a terminal failure whose attempt another host already ended and recorded,
|
|
3445
|
+
// in a record this build can read only by its core (the lost-lock check below says which). It runs the terminal
|
|
3446
|
+
// duties below, so nothing the job held is left open: the processor never closes the approval claim of a failed
|
|
3447
|
+
// attempt, so without this one its target would refuse new requests `target-pending` for good. Two things
|
|
3448
|
+
// differ: the terminal line is `job_lost_lock_after_completion` with the outcome alone, and nothing is posted.
|
|
3449
|
+
// The page is the same as any terminal failure's, and a stall's error carries no reason, so the hook's argv reads
|
|
3450
|
+
// `infra`. The run's inbox and held replies close as the run's end says (below). What the host that ran the
|
|
3451
|
+
// attempt may already have done is safe to do again: a second close of an inbox or of held replies acts on
|
|
3452
|
+
// nothing, and the draft's refusal writes the same state.
|
|
3453
|
+
const approvalFinalizers = new Set();
|
|
3454
|
+
let approvalFinalizerCloser = false;
|
|
3455
|
+
const onFailed = (job, err, { settledElsewhere = false } = {}) => {
|
|
3456
|
+
// Exactly one terminal line either way (REQ-LOCAL-JOB-VISIBILITY).
|
|
3457
|
+
if (settledElsewhere) log("job_lost_lock_after_completion", { jobId: job?.id, outcome: "failed" });
|
|
3458
|
+
else log("job_failed", { jobId: job?.id, attempt: job?.attemptsMade, reason: String(err?.message ?? err).slice(0, 120) });
|
|
2439
3459
|
// The one reason whose sentence is the fix itself (issue #458): logged WHOLE, beside the cut line above.
|
|
2440
3460
|
if (err?.reason === NETNS_KEEPER_NOT_HOLDING || err?.reason === NETNS_KEEPER_CRASH_LOOP) log("job_failed_netns_keeper", { jobId: job?.id, attempt: job?.attemptsMade, reason: String(err?.message ?? "") });
|
|
2441
3461
|
// The TERMINAL failed attempt only (issue #288): `finishedOn` is set by BullMQ's own move on the
|
|
@@ -2446,7 +3466,37 @@ export async function startWorker(
|
|
|
2446
3466
|
// never sees: the stall-kill (maxStalledCount 0 fails a crashed worker's job at next pickup)
|
|
2447
3467
|
// and the wait-gate rethrow that escapes above the processor's catch.
|
|
2448
3468
|
if (job?.finishedOn) {
|
|
2449
|
-
|
|
3469
|
+
if (job.data?.answerDraft) void answerDraftRuntime.refused(job.data, { reason: "failed" }).catch(() => log("answer_terminal_unavailable", {}));
|
|
3470
|
+
// An exhausted retry has no future container to consume its claim. Close only this queue job's
|
|
3471
|
+
// authority, never another job's, and let a new authorized run request a fresh approval.
|
|
3472
|
+
if (job.data?.approvalRequestId) {
|
|
3473
|
+
const finishing = settleWithin(Promise.resolve().then(() => approvalRuntime.finish({ requestId: job.data.approvalRequestId, jobId: String(job.id) })), 2500, { error: "approval-finish-timeout" })
|
|
3474
|
+
.then((result) => { if (!result?.ok) log("approval_terminal_finish_failed", { jobId: String(job.id), reason: "approval-finish-unavailable" }); })
|
|
3475
|
+
.catch(() => log("approval_terminal_finish_failed", { jobId: String(job.id), reason: "approval-finish-unavailable" }))
|
|
3476
|
+
.finally(() => approvalFinalizers.delete(finishing));
|
|
3477
|
+
approvalFinalizers.add(finishing);
|
|
3478
|
+
if (!approvalFinalizerCloser) {
|
|
3479
|
+
approvalFinalizerCloser = true;
|
|
3480
|
+
extraClosers.push({ close: () => Promise.allSettled([...approvalFinalizers]) });
|
|
3481
|
+
}
|
|
3482
|
+
}
|
|
3483
|
+
// Issue #646, review finding F2: a TERMINAL failure the processor never saw (a stall-kill of a crashed worker's
|
|
3484
|
+
// job with no recovery enqueued, a rethrow above the processor's catch) is the run's final outcome too, so its
|
|
3485
|
+
// inbox is closed here and what waits becomes `late` instead of sitting `queued` until the key expires. A
|
|
3486
|
+
// terminal failure the processor did see was settled before its record: this second close returns the same entries
|
|
3487
|
+
// with `first: false` (steer-inbox.mjs closeInbox), so settleInbox answers nothing again (review round 1, F2).
|
|
3488
|
+
// Both closes are the RUN's, keyed by its run id, which a #645 recovery keeps. A run settled elsewhere may have
|
|
3489
|
+
// been handed to a recovery there (a `recovery-enqueued` this build cannot read), and that recovery must find
|
|
3490
|
+
// both open (inbox-settle.mjs `isFinalOutcome`): closing them would answer its waiting messages as late, on the
|
|
3491
|
+
// forge, about a run that goes on. So they close then only for a job no recovery can continue, whose run ended
|
|
3492
|
+
// with it.
|
|
3493
|
+
if (!settledElsewhere || !isRecoveryArmed(job.data)) {
|
|
3494
|
+
if (job.data?.steer === true) void settleInbox({ redis, job, error: { reason: typeof err?.reason === "string" ? err.reason : "infra" }, log, onLateForge: lateForge });
|
|
3495
|
+
// Issue #638: the same for an awaitInput run's held replies; a failure never parked, so they are dropped, and
|
|
3496
|
+
// a second close (the processor settled first) acts on nothing.
|
|
3497
|
+
if (job.data?.awaitInput === true) void heldSettle({ job, error: { reason: typeof err?.reason === "string" ? err.reason : "infra" } });
|
|
3498
|
+
}
|
|
3499
|
+
if (!settledElsewhere) void comment({ ...job.data, id: job.id }, (typeof err?.reason === "string" && Object.hasOwn(FAILED_COMMENT_BY_REASON, err.reason) ? FAILED_COMMENT_BY_REASON[err.reason] : null) ?? FAILED_COMMENT);
|
|
2450
3500
|
onFailure?.({ jobId: job?.id, outcome: "failed", reason: typeof err?.reason === "string" ? err.reason : "infra" });
|
|
2451
3501
|
}
|
|
2452
3502
|
};
|
|
@@ -2459,7 +3509,59 @@ export async function startWorker(
|
|
|
2459
3509
|
// the completed listener would have sent for that record (a paid policy stop) is sent here instead, because
|
|
2460
3510
|
// that listener never saw the first finish. No record, an earlier attempt's, a `failed` one, or a lookup fault
|
|
2461
3511
|
// keeps the failure path below: the comment and the page.
|
|
3512
|
+
// Issue #645 (durable runs): a terminal stall of an ARMED job whose run has a checkpoint pointer is not the end of
|
|
3513
|
+
// the run. The worker that ran it died (that is what a stall is), so the processor never got to take its infra-exit
|
|
3514
|
+
// path; this listener takes it instead: the recovery is enqueued (routed to the host holding the checkpoint), the
|
|
3515
|
+
// crashed attempt's record is written (`failed`, `recovery-enqueued`: nothing else would write it, its processor is
|
|
3516
|
+
// gone), and the failure comment and on_failure are SUPPRESSED, because the run is continuing. Anything short of an
|
|
3517
|
+
// enqueued recovery (no pointer, the cap, an enqueue that failed) answers false, and the caller runs today's path:
|
|
3518
|
+
// the comment and the page, loudly, never a run that silently stops.
|
|
3519
|
+
const recoverStall = async (job) => {
|
|
3520
|
+
if (!isRecoveryArmed(job?.data)) return false;
|
|
3521
|
+
const runId = runIdOf(job);
|
|
3522
|
+
const pointer = await readCheckpointPointer({ runId });
|
|
3523
|
+
if (!pointer) {
|
|
3524
|
+
log("job_recovery_skipped", { jobId: job.id, reason: "no-checkpoint" });
|
|
3525
|
+
return false;
|
|
3526
|
+
}
|
|
3527
|
+
// Issue #662: the crashed attempt's start and how far it got, off the run's progress key, read BEFORE the enqueue,
|
|
3528
|
+
// because the recovery's first progress write replaces that key with its own empty snapshot. NOT `job.processedOn`:
|
|
3529
|
+
// BullMQ's stall check does not fail a stalled job outright, it moves it back to wait with the deferred failure
|
|
3530
|
+
// marked (`moveStalledJobsToWait` sets `defa`), and the next pickup's `prepareJobForProcessing` stamps a fresh
|
|
3531
|
+
// `processedOn` before it fails the job, so `processedOn` here is the re-pick time (a three-minute attempt read
|
|
3532
|
+
// as 23 ms long in the #645 lab). The key's `startedAt` is the attempt's own, written by its writer, and it is
|
|
3533
|
+
// taken only when three checks pass: the key has not ended (a crash never writes the final snapshot), its start is
|
|
3534
|
+
// not after now, and its start is no earlier than this job's enqueue time (`job.timestamp`). The last one ties the
|
|
3535
|
+
// key to THIS job: the key is the RUN's, so a recovery that died before its own first write (or ran on a host with
|
|
3536
|
+
// PI_RUN_PROGRESS=0) would otherwise find the base attempt's unended key and record a start from before the
|
|
3537
|
+
// recovery existed, with the base's counts. A retry of the same job id cannot be told apart by this (its
|
|
3538
|
+
// timestamp is the job's first enqueue); its writer's first write replaces the key, so only a retry that died
|
|
3539
|
+
// before writing would show the earlier attempt's start. Anything else is null, an honest unknown. Bounded and
|
|
3540
|
+
// never throwing (`readProgressKey`); the key is display only, so it decides nothing here either.
|
|
3541
|
+
const at0 = now();
|
|
3542
|
+
const key = await Promise.resolve().then(() => readProgressKeyFn({ redis, jobId: runId })).catch(() => null);
|
|
3543
|
+
const snap = key?.snapshot;
|
|
3544
|
+
const own = snap !== null && typeof snap === "object" && snap.ended === false && Number.isSafeInteger(snap.startedAt) && snap.startedAt > 0 && snap.startedAt <= at0 && Number.isSafeInteger(job.timestamp) && snap.startedAt >= job.timestamp;
|
|
3545
|
+
const crashedProgress = own ? { turns: snap.turn, steps: snap.steps, failed: snap.failed, interrupted: snap.interrupted, replayed: snap.replayed } : null;
|
|
3546
|
+
const r = await enqueueRecovery({ job, cause: "stall", pointer });
|
|
3547
|
+
if (!r?.enqueued) return false;
|
|
3548
|
+
// `attemptsMade` already counts this attempt here (moveToFailed added one), and `buildRecord` adds one for a record
|
|
3549
|
+
// written while processing, so the record is handed the count from before the move: it then names THIS attempt.
|
|
3550
|
+
const at = now();
|
|
3551
|
+
recordRun({
|
|
3552
|
+
job: { id: job.id, name: job.name, data: job.data, opts: job.opts, attemptsMade: Math.max(0, (Number.isSafeInteger(job.attemptsMade) ? job.attemptsMade : 1) - 1), timestamp: job.timestamp },
|
|
3553
|
+
// The snapshot's integer counts, rebuilt by `buildRecord` (`recordedProgress`): a bad value is dropped, never copied.
|
|
3554
|
+
result: { outcome: "failed", reason: "recovery-enqueued", ...(crashedProgress !== null ? { progress: crashedProgress } : {}) },
|
|
3555
|
+
startedAt: own ? new Date(snap.startedAt).toISOString() : null,
|
|
3556
|
+
endedAt: new Date(at).toISOString(),
|
|
3557
|
+
});
|
|
3558
|
+
log("job_failed", { jobId: job.id, attempt: job.attemptsMade, reason: "recovery-enqueued" });
|
|
3559
|
+
return true;
|
|
3560
|
+
};
|
|
2462
3561
|
for (const w of allWorkers) w.on("failed", (job, err) => {
|
|
3562
|
+
// `finishedOn` is the queue's word that no retry follows (`onFailed` reads it the same way), so the expiry is set
|
|
3563
|
+
// here, before any path below returns: the terminal failure, the stall and the lost-lock checks alike.
|
|
3564
|
+
if (job?.finishedOn) finishAttempt(w, job);
|
|
2463
3565
|
if (!(job?.finishedOn && err?.message === STALLED_FAILED_REASON)) return onFailed(job, err);
|
|
2464
3566
|
void (async () => {
|
|
2465
3567
|
// `attemptsMade` is read AFTER BullMQ's moveToFailed added one, so it equals the attempt number the
|
|
@@ -2468,9 +3570,54 @@ export async function startWorker(
|
|
|
2468
3570
|
// can see why the record did not suppress it (a clock skew past the tolerance reads `older-than-job`).
|
|
2469
3571
|
const onReject = (reason, source) => log("job_lost_lock_record_rejected", { jobId: job.id, reason, source });
|
|
2470
3572
|
const record = await settledRecord(job.id, { attempt: job.attemptsMade, since: job.timestamp, onReject });
|
|
2471
|
-
if (!record)
|
|
2472
|
-
|
|
2473
|
-
|
|
3573
|
+
if (!record) {
|
|
3574
|
+
// Issue #645: a RETURNED failure of this attempt that then lost its lock. `settledRecord` refuses every
|
|
3575
|
+
// `failed` record (a failure is the failure path's to report), but two returned failures already said
|
|
3576
|
+
// everything: `recovery-failed` posted its own sentence (so only its page is owed, once), and
|
|
3577
|
+
// `recovery-enqueued` handed the run to a recovery (nothing is owed). Running the failure path for either
|
|
3578
|
+
// would post a second failure comment, the second one false for a run that continues.
|
|
3579
|
+
// Read BOTH ways (`previousRecord`), never this host's file alone: on a fleet this host may hold an earlier
|
|
3580
|
+
// attempt's file while the host that ran THIS attempt wrote its record to the mirror. And each side held to
|
|
3581
|
+
// the rule every lookup of a job's own record shares (`recordOfThisJob`: the id, this attempt, and not older
|
|
3582
|
+
// than the job) BEFORE the higher attempt is kept, because ids are reused: a failed record an older job left
|
|
3583
|
+
// under the same id must not stand in for this attempt's and take away its recovery or its comment, and,
|
|
3584
|
+
// at a higher attempt, must not hide this attempt's own record on the other side either.
|
|
3585
|
+
const keep = (record) => recordOfThisJob(record, { jobId: job.id, attempt: job.attemptsMade, since: job.timestamp });
|
|
3586
|
+
const own = await previousRecord(job.id, { keep }).catch(() => null);
|
|
3587
|
+
const ownFailure = own !== null && own.outcome === "failed";
|
|
3588
|
+
// Issue #761: this attempt's `failed` record, written by a newer pi-dispatch. Its id, attempt, times and
|
|
3589
|
+
// outcome are stable core fields, so it says that the attempt ended on that host and what became of it was
|
|
3590
|
+
// decided there. Its `reason` is not core, so this build cannot tell `recovery-enqueued` (the run continues,
|
|
3591
|
+
// nothing is owed) from `recovery-failed` (its sentence posted, its page owed) from an ordinary failure (the
|
|
3592
|
+
// comment and the page owed). The failure path below fits only the last, and breaks the first:
|
|
3593
|
+
// `recoverStall` would ask for the newer host's recovery again and write this build's record over the newer
|
|
3594
|
+
// one, and the comment would say "will not be retried further" of a run that continues. So the terminal
|
|
3595
|
+
// failure path runs without its comment (`settledElsewhere`): every duty that closes what the job held, and
|
|
3596
|
+
// one page, which is owed or harmless in all three cases, where a wrong comment is wrong on the forge, in
|
|
3597
|
+
// front of everyone. An ordinary failure's comment is what this costs, and only while the fleet runs two
|
|
3598
|
+
// releases.
|
|
3599
|
+
if (ownFailure && !recordReadable(own)) return onFailed(job, err, { settledElsewhere: true });
|
|
3600
|
+
if (ownFailure && (own.reason === "recovery-failed" || own.reason === "recovery-enqueued")) {
|
|
3601
|
+
log("job_lost_lock_after_completion", { jobId: job.id, outcome: own.outcome, reason: own.reason });
|
|
3602
|
+
if (pagesAsReturnedFailure(own)) onFailure({ jobId: job.id, outcome: "failed", reason: own.reason });
|
|
3603
|
+
return;
|
|
3604
|
+
}
|
|
3605
|
+
// No settled record: the attempt died running. A run that can continue from its checkpoint does; any
|
|
3606
|
+
// fault on that path falls through to the failure comment and the page.
|
|
3607
|
+
let recovered = false;
|
|
3608
|
+
try {
|
|
3609
|
+
recovered = await recoverStall(job);
|
|
3610
|
+
} catch (e) {
|
|
3611
|
+
log("job_recovery_enqueue_failed", { jobId: job.id, code: typeof e?.code === "string" ? e.code : "error" });
|
|
3612
|
+
}
|
|
3613
|
+
if (recovered) return;
|
|
3614
|
+
return onFailed(job, err);
|
|
3615
|
+
}
|
|
3616
|
+
// Issue #761: a record a newer pi-dispatch wrote is read for its stable core only; its reason may mean
|
|
3617
|
+
// something else there, so it is neither logged nor paged on.
|
|
3618
|
+
const readable = recordReadable(record);
|
|
3619
|
+
log("job_lost_lock_after_completion", { jobId: job.id, outcome: record.outcome, ...(readable && record.reason ? { reason: record.reason } : {}) });
|
|
3620
|
+
if (readable && pagesAsPolicy(record)) onFailure({ jobId: job.id, outcome: "policy", reason: record.reason });
|
|
2474
3621
|
})().catch(() => {}); // `settledRecord` never rejects; this only keeps a throwing log line from going unhandled
|
|
2475
3622
|
});
|
|
2476
3623
|
|
|
@@ -2511,7 +3658,7 @@ export async function startWorker(
|
|
|
2511
3658
|
const r = await reconcileGated(rq, served, { registry, log, tz: hostTz, authored: authoredCron(config) });
|
|
2512
3659
|
log("schedules_installed", { installed: r.installed, removed: r.removed, ...(unserved.length > 0 && { unserved: unserved.length }) });
|
|
2513
3660
|
} finally {
|
|
2514
|
-
await rq
|
|
3661
|
+
await closeQueueSettled(rq);
|
|
2515
3662
|
}
|
|
2516
3663
|
} else {
|
|
2517
3664
|
log("schedules_installed", { installed: 0, removed: 0, ...(unserved.length > 0 && { unserved: unserved.length }) });
|
|
@@ -2522,13 +3669,13 @@ export async function startWorker(
|
|
|
2522
3669
|
// Only when a triggers file is configured; best-effort; a bad edit keeps the running schedulers. The
|
|
2523
3670
|
// closer each of the three returns joins `extraClosers`, so the watch stops with the worker (issue #295).
|
|
2524
3671
|
if (config.triggersFile) {
|
|
2525
|
-
extraClosers.push(watchTriggersFile(config, cronQueue, log, schedules, registry, hostTz, config.workerNameDeclared, atBoot.triggers, checkEnvelopePlace));
|
|
3672
|
+
extraClosers.push(watchTriggersFile(config, cronQueue, log, schedules, registry, hostTz, config.workerNameDeclared, atBoot.triggers, checkEnvelopePlace, triggersState, watchFs));
|
|
2526
3673
|
}
|
|
2527
3674
|
|
|
2528
3675
|
// REQ-SCOPED-PAUSE-WINDOWS live edit: watch the pause-windows file and hot-swap the in-memory windows, so
|
|
2529
3676
|
// an operator's add/delete of a pause window takes effect without a worker restart. A bad edit is kept out.
|
|
2530
3677
|
if (config.pauseWindowsFile) {
|
|
2531
|
-
extraClosers.push(watchPauseWindowsFile(config, pauseWindows, log, atBoot.pauseWindows));
|
|
3678
|
+
extraClosers.push(watchPauseWindowsFile(config, pauseWindows, log, atBoot.pauseWindows, watchFs));
|
|
2532
3679
|
}
|
|
2533
3680
|
|
|
2534
3681
|
// Issue #499 part B: the two files a project row joins reload as a pair; the deployment cap rides along so a limits
|
|
@@ -2556,6 +3703,15 @@ export async function startWorker(
|
|
|
2556
3703
|
extraClosers.push(watchModelEndpointsFile(config, modelEndpoints, log, atBoot.modelEndpoints));
|
|
2557
3704
|
}
|
|
2558
3705
|
|
|
3706
|
+
// Arm expiry only for declarations this host serves, including after a live edit first publishes a request.
|
|
3707
|
+
let declaresApprovals = false;
|
|
3708
|
+
try { declaresApprovals = parseTriggers(readFileSync(onceTriggersFile, "utf8"), onceTriggersFile).some((entry) => entry.run?.gates); } catch {}
|
|
3709
|
+
const retainedApprovals = await settleWithin(approvalStore.hasPending(durableHost), 2500, { error: "store-unavailable" });
|
|
3710
|
+
if (!retainedApprovals.ok) log("approval_expiry_scan_unavailable", { reason: "store-unavailable" });
|
|
3711
|
+
// A revoked declaration still has unanswered requests to retire. On an unreadable index, arm the retrying
|
|
3712
|
+
// sweep and name the uncertainty; do not silently abandon a durable deadline.
|
|
3713
|
+
if (declaresApprovals || retainedApprovals.pending || !retainedApprovals.ok) startApprovalSweep();
|
|
3714
|
+
|
|
2559
3715
|
// issue #292 / OQ-007: re-run the three retention sweeps on a timer, because the supported deployment
|
|
2560
3716
|
// is a service that restarts only on failure, so the healthy worker was the one that never re-swept.
|
|
2561
3717
|
// Armed HERE, at the end of boot beside the watches: all three closures exist, boot's own sweeps have
|
|
@@ -2567,14 +3723,38 @@ export async function startWorker(
|
|
|
2567
3723
|
// reaper. It is registered in `extraClosers` for issue #295's finding that UNREF'D IS NOT CLEANED UP --
|
|
2568
3724
|
// this handle holds an `rmSync`, and a tick landing mid-drain could delete a retained workspace behind a
|
|
2569
3725
|
// worker that already reported a clean shutdown.
|
|
3726
|
+
// Issue #645: the two durable-run sweeps, run once here at boot and then on every tick. The host-lost sweep on every
|
|
3727
|
+
// host (a fleet's recoveries are judged by whichever host gets there first); the checkpoint store's retention only
|
|
3728
|
+
// where there is a store. A fault is a log line, never a refused boot: both are backstops over a TTL.
|
|
3729
|
+
// The store sweep is given NO `active` list, deliberately: it removes a run's store only when its newest manifest is
|
|
3730
|
+
// older than the store TTL (7 days), and a run running here has written to its store within the job timeout (30
|
|
3731
|
+
// minutes: primeBase at its start, a checkpoint at most a minute apart), so no running run can be judged stale and an
|
|
3732
|
+
// active list could change no verdict. A run waiting for its recovery is bounded the same way: the host-lost sweep
|
|
3733
|
+
// gives up on it at RECOVERY_MAX_WAIT_MS, which IS that TTL. (The processor's running list holds job ids, not run
|
|
3734
|
+
// ids, so passing it would also name a `.rN` job's store wrongly.)
|
|
3735
|
+
const reapForkSources = forkStore ? () => forkStore.sweep() : null;
|
|
3736
|
+
const reapCheckpoints = workspaceCheckpointer ? async () => void (await workspaceCheckpointer.sweep()) : null;
|
|
3737
|
+
const reapHostLost = async () => void (await sweepHostLost());
|
|
3738
|
+
// NOT awaited: the host-lost sweep's first command waits for Valkey however long it is away, and the boot must not.
|
|
3739
|
+
// Each settles into its own log line; neither can reject unhandled.
|
|
3740
|
+
for (const [name, reap] of [["recovery_host_lost", reapHostLost], ...(reapForkSources ? [["fork_source", reapForkSources]] : []), ...(reapCheckpoints ? [["checkpoint", reapCheckpoints]] : [])]) {
|
|
3741
|
+
void Promise.resolve()
|
|
3742
|
+
.then(reap)
|
|
3743
|
+
.catch((err) => log(`${name}_reaper_skipped`, { reason: scrubCredentials(err?.message) }));
|
|
3744
|
+
}
|
|
2570
3745
|
if (config.sweepIntervalHours > 0) {
|
|
2571
3746
|
const sweep = makeRetentionSweepFn({
|
|
2572
3747
|
reapers: [
|
|
2573
3748
|
{ name: "log", reap: reapLogs },
|
|
2574
3749
|
// Issue #504 part B: where boot runs it, right after the run history it sits beside.
|
|
2575
3750
|
{ name: "allocation_log", reap: reapAllocationLogs },
|
|
3751
|
+
{ name: "console_audit", reap: reapConsoleAudit },
|
|
3752
|
+
{ name: "touch_outbox", reap: () => touchOutbox.repair() },
|
|
2576
3753
|
{ name: "sandbox", reap: reapSandboxes },
|
|
2577
3754
|
{ name: "session", reap: () => sessionStore.reapSessions() },
|
|
3755
|
+
...(reapForkSources ? [{ name: "fork_source", reap: reapForkSources }] : []),
|
|
3756
|
+
{ name: "recovery_host_lost", reap: reapHostLost },
|
|
3757
|
+
...(reapCheckpoints ? [{ name: "checkpoint", reap: reapCheckpoints }] : []),
|
|
2578
3758
|
],
|
|
2579
3759
|
intervalMs: config.sweepIntervalHours * 3600000,
|
|
2580
3760
|
log,
|
|
@@ -2583,6 +3763,37 @@ export async function startWorker(
|
|
|
2583
3763
|
extraClosers.push(sweep);
|
|
2584
3764
|
}
|
|
2585
3765
|
|
|
3766
|
+
// Issue #712 (`DES-DOCTOR-RESULTS-PUBLISHED`): `pi-dispatch doctor --publish` as a child, shortly after boot and every
|
|
3767
|
+
// PI_DOCTOR_INTERVAL_HOURS, so every host publishes its checks whether or not anyone runs doctor there. After the
|
|
3768
|
+
// registry has started (above), so the first run's fleet lines see this host's own row. At 0 nothing is constructed,
|
|
3769
|
+
// the sweep's posture: no timer, no closer, no child is reachable.
|
|
3770
|
+
if (config.doctorIntervalHours > 0) {
|
|
3771
|
+
const doctorRuns = makeDoctorSpawnerFn({
|
|
3772
|
+
cliPath: fileURLToPath(new URL("./cli.mjs", import.meta.url)),
|
|
3773
|
+
intervalMs: config.doctorIntervalHours * 3600000,
|
|
3774
|
+
log,
|
|
3775
|
+
});
|
|
3776
|
+
doctorRuns.start();
|
|
3777
|
+
extraClosers.push(doctorRuns);
|
|
3778
|
+
}
|
|
3779
|
+
|
|
3780
|
+
// Issue #796 (`DES-PACKAGE-UPDATES-PREPARED-NOT-APPLIED`): the look, off until PI_UPDATES_LOOK=1, and each host applying what a person asked for. Children for
|
|
3781
|
+
// doctor's reason: neither belongs on the loop that renews paid job locks. The reconcile starts nothing until a person asked for something, and reaches a
|
|
3782
|
+
// registry only then, which is why it needs no switch of its own: until then it is one free Valkey read every two minutes.
|
|
3783
|
+
const cliPath = fileURLToPath(new URL("./cli.mjs", import.meta.url));
|
|
3784
|
+
if (config.updatesLook) {
|
|
3785
|
+
const look = makeUpdatesRunnerFn({ cliPath, intervalMs: UPDATES_LOOK_EVERY_HOURS * 3600000, log, args: ["updates", "look"], events: { overlapped: "updates_look_overlapped", spawnFailed: "updates_look_failed", finished: "updates_look_finished" } });
|
|
3786
|
+
look.start();
|
|
3787
|
+
extraClosers.push(look);
|
|
3788
|
+
}
|
|
3789
|
+
const reconcile = makeUpdatesRunnerFn({
|
|
3790
|
+
cliPath, intervalMs: UPDATES_RECONCILE_EVERY_MS, bootDelayMs: UPDATES_RECONCILE_EVERY_MS, timeoutMs: UPDATES_RECONCILE_TIMEOUT_MS, log, args: ["updates", "reconcile"],
|
|
3791
|
+
events: { overlapped: "updates_reconcile_overlapped", spawnFailed: "updates_reconcile_failed", finished: "updates_reconcile_finished" },
|
|
3792
|
+
shouldRun: () => hostHasPending(redis, config.workerName),
|
|
3793
|
+
});
|
|
3794
|
+
reconcile.start();
|
|
3795
|
+
extraClosers.push(reconcile);
|
|
3796
|
+
|
|
2586
3797
|
log("worker_started", {
|
|
2587
3798
|
queue: "pi-jobs",
|
|
2588
3799
|
// Issue #278: the service's OWN answer to which daemon its jobs go to. `doctor` reads its caller's
|
|
@@ -2606,6 +3817,7 @@ export async function startWorker(
|
|
|
2606
3817
|
imageDigest: bootImage.imageDigest ?? null, // two hosts on two builds of one tag used to emit byte-identical boot lines
|
|
2607
3818
|
concurrency: bootConcurrency, // the slot count the Worker is actually constructed with (overlay may raise/lower it)
|
|
2608
3819
|
sweepIntervalHours: config.sweepIntervalHours, // 0 = boot-only sweeps, this version's pre-#292 behaviour
|
|
3820
|
+
doctorIntervalHours: config.doctorIntervalHours, // issue #712: 0 = the worker runs no doctor of its own
|
|
2609
3821
|
dailyCap: config.dailyCap,
|
|
2610
3822
|
weeklyCap: config.weeklyCap, // null when the weekly window is disabled
|
|
2611
3823
|
monthlyCap: config.monthlyCap, // null when the monthly window is disabled
|
|
@@ -2626,7 +3838,13 @@ export async function startWorker(
|
|
|
2626
3838
|
settingsFile: config.settingsFile,
|
|
2627
3839
|
captureJobLogs: config.captureJobLogs,
|
|
2628
3840
|
logRetentionDays: config.logRetentionDays,
|
|
3841
|
+
consoleAuditRetentionDays: config.consoleAuditRetentionDays, // issue #761: the console audit trail's own; 0 = kept forever
|
|
3842
|
+
runMirrorMaxRuns: config.runMirrorMaxRuns, // issue #761: the shared run index's cap this host trims to
|
|
2629
3843
|
sandboxRetentionHours: config.sandboxRetentionHours, // 0 = retention off; a run's directory is deleted as before
|
|
3844
|
+
// Issue #645 (durable runs): where checkpoints go (null = none: an armed job is refused before it spends) and
|
|
3845
|
+
// whether this host takes them (the store AND the run checkpointer); doctor reads the same directory.
|
|
3846
|
+
checkpointsDir: config.checkpointsDir ?? null,
|
|
3847
|
+
durableRuns: Boolean(runCheckpointer),
|
|
2630
3848
|
});
|
|
2631
3849
|
return worker;
|
|
2632
3850
|
} catch (err) {
|