@edgehero/pi-dispatch 4.3.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +84 -3
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +190 -29
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1639 -196
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +96 -6
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-host.mjs +42 -5
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +33 -3
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +734 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1550 -173
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +264 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +780 -38
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/processor.mjs
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { cleanTouch, touchScopeOf } from "./touch-state.mjs";
|
|
2
|
+
import { SubscriptionWait } from "./subscription-wait.mjs";
|
|
3
|
+
import { readQuestionContext } from "./question-context.mjs";
|
|
4
|
+
import { stageModelRoleInputs } from "./agent-snapshot.mjs";
|
|
5
|
+
import { effectiveCostCapMicros } from "./money.mjs";
|
|
6
|
+
import { validateFork, forkBranch, forkArchiveFact, FORK_THINKING_LEVELS } from "./fork-job.mjs";
|
|
7
|
+
import { sessionKeyFor } from "./session-key.mjs";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import { ABSENT, OPENSHELL_BIND_LINGERING, backendFor, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, DOCKER_NEVER_STARTED_EXITS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_CONF_WIDENS_JOB, OPENSHELL_GATEWAY_LOCAL, PODMAN_SERVICE_LOCAL, RUNTIME_ADDS_NO_MOUNTS } from "./backends.mjs";
|
|
2
10
|
import { resolveBackendName } from "./backend-registry.mjs";
|
|
3
11
|
import { lstatSync } from "node:fs";
|
|
4
12
|
import { checkTokenCap, recordTokenSpend, releaseLedgers, reserveLedgers } from "./budget.mjs";
|
|
@@ -6,12 +14,18 @@ import { configError } from "./config.mjs";
|
|
|
6
14
|
import { unqualifiedScope } from "./pause-windows.mjs";
|
|
7
15
|
import { PROJECT_CAP_REASON } from "./scoped-limits.mjs";
|
|
8
16
|
import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
|
|
17
|
+
import { mergeBundleSecrets } from "./secret-bundles.mjs";
|
|
18
|
+
import { providerKeyCandidates } from "./env-allowlist.mjs";
|
|
9
19
|
import { RESERVED_ENV_NAMES } from "./triggers.mjs";
|
|
10
|
-
import {
|
|
20
|
+
import { imageFactsOf, loadedWithBase, loadedWithImage } from "./loaded-record.mjs";
|
|
21
|
+
import { mcpRefusalComment } from "./mcp-preflight.mjs";
|
|
11
22
|
import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
|
|
12
|
-
import {
|
|
23
|
+
import { CONTAINER_EXIT_UNKNOWN, CONTAINER_INFRA_EXIT } from "./record-reason.mjs";
|
|
24
|
+
import { COST_CAP_WHYS, EXIT_OOM_KILLED, RECORD_CLOCK_SKEW_MS, RUNNER_POLICY_REASONS, recoveryCounts, recoveryCountsFromProgress } from "./run-history.mjs";
|
|
25
|
+
import { baseHasExtensions as baseHasExtensionsDefault, resetTrustPaths as resetTrustPathsDefault } from "./trust-paths.mjs";
|
|
26
|
+
import { normalizeHost, removeCheckpointPointer as removePointerDefault } from "./run-checkpoint.mjs";
|
|
13
27
|
import { DEFAULT_EGRESS_PROXY } from "./egress.mjs";
|
|
14
|
-
import { CAPABILITY_GATES, EXIT_AUTH_CAPABILITY } from "./image-preflight.mjs";
|
|
28
|
+
import { CAPABILITY_GATES, EXIT_AUTH_CAPABILITY, STEER_STDIN_CAPABILITY, TRANSCRIPT_STDERR_CAPABILITY } from "./image-preflight.mjs";
|
|
15
29
|
import { modelListProblem, modelOnList, splitModelEntry } from "./model-ref.mjs";
|
|
16
30
|
import { ALLOCATION_CAP_REASON, ENVELOPE_MISMATCH_REASON } from "./allocation.mjs";
|
|
17
31
|
|
|
@@ -19,8 +33,15 @@ import { ALLOCATION_CAP_REASON, ENVELOPE_MISMATCH_REASON } from "./allocation.mj
|
|
|
19
33
|
export const LOCAL_FOLDER_PROJECT_CHANGED = "local-folder-project-changed";
|
|
20
34
|
// Issue #505: a portfolio cron job refused before it spends, because this host could not apply the plan it would write.
|
|
21
35
|
export const PORTFOLIO_NO_ENVELOPE = "portfolio-no-envelope";
|
|
22
|
-
import { DOLLAR_CAP_REASON, DOLLAR_KEY_PREFIX, dollarLedgers, dollarSettlement, dollarsRecord, holdPart, meteredMicros, modelDollarSettlement, releaseDollars, reserveDollars, settleDollars } from "./dollar-budget.mjs";
|
|
23
|
-
import {
|
|
36
|
+
import { DOLLAR_CAP_REASON, DOLLAR_KEY_PREFIX, deploymentWindowsOf, dollarLedgers, dollarSettlement, dollarsRecord, holdPart, meteredMicros, modelDollarSettlement, releaseDollars, reserveDollars, settleDollars } from "./dollar-budget.mjs";
|
|
37
|
+
import { callableModelRefs, zeroRatedFor } from "./reserve-estimate.mjs";
|
|
38
|
+
import { outputCapView } from "./output-cap.mjs";
|
|
39
|
+
import { answerTrailComment, consoleAnswerBy, consoleDraftOf } from "./console-answer.mjs";
|
|
40
|
+
import { forgeDeliveryJobId } from "./job-id.mjs";
|
|
41
|
+
import { FORGE_JOB_RETENTION_MS } from "./retention.mjs";
|
|
42
|
+
// The forge-comment tables (issue #710): every fixed sentence this file posts, in one leaf the reason catalog can import
|
|
43
|
+
// without the job path. Re-exported below under the names they always had here.
|
|
44
|
+
import { MODEL_AGENTS_INVALID_COMMENT, APPROVAL_COMMENTS, APPROVAL_PARK_NEWER_COMMENT, BUNDLE_COMMENTS, ENVELOPE_MISMATCH_COMMENTS, HELD_TAKEN_COMMENT, JOB_USER_COMMENTS, MODEL_NOT_ALLOWED_COMMENTS, MODEL_UNKNOWN_COMMENT, modelUnknownComment, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, OVERLAY_REFUSED_COMMENT, PARKED_COMMENT, PARKED_COMMENT_PR, PODMAN_CONF_COMMENTS, RECOVERY_COMMENTS, REFUSAL_COMMENTS, TERMINAL_COMMENTS, VENUE_BUNDLE_REFUSED_COMMENT, VENUE_REFUSED_COMMENT, parkedComment } from "./comment-tables.mjs";
|
|
24
45
|
|
|
25
46
|
/**
|
|
26
47
|
* The forge comment's reason for each observation a floor refusal missed (issues #278 and #345), keyed like
|
|
@@ -38,6 +59,9 @@ export const OBSERVATION_COMMENT = Object.freeze({
|
|
|
38
59
|
[PODMAN_BOUNDS_DELEGATED]: "the worker's rootless Podman is not observed applying a container's pid, memory and cpu bounds",
|
|
39
60
|
[PODMAN_ADDS_NO_MOUNTS]: "the worker's rootless Podman is not observed adding no mounts of its own to a job container",
|
|
40
61
|
[PODMAN_SERVICE_LOCAL]: "the podman CLI is not observed running containers on this host rather than through a remote service, so the job's credentials could cross a network the deployment does not own",
|
|
62
|
+
// Issue #683. Never reached through a floor alone: the openshell venue refuses a gateway it does not observe local
|
|
63
|
+
// whatever the floor says (`venueRefused`), and this sentence is the floor's half for a deployment that also asks.
|
|
64
|
+
[OPENSHELL_GATEWAY_LOCAL]: "the OpenShell gateway is not observed listening on this host, so the job's credentials could cross a network the deployment does not own",
|
|
41
65
|
});
|
|
42
66
|
|
|
43
67
|
/**
|
|
@@ -107,72 +131,9 @@ export function peakReachedLimit(memPeak, limit) {
|
|
|
107
131
|
return memPeak >= Math.ceil((limit * 9) / 10);
|
|
108
132
|
}
|
|
109
133
|
|
|
110
|
-
// The post-spend terminal comments (issue #288)
|
|
111
|
-
//
|
|
112
|
-
|
|
113
|
-
// failure comment" -- held only below the spend line). Keyed by the reason token so a new abort
|
|
114
|
-
// classification adds a ROW here, never a re-plumb; sentences are FIXED and path-free, because for a
|
|
115
|
-
// local job the adapter logs the full text into a persistent service log (the "this folder" discipline
|
|
116
|
-
// at the scope-cap refusal). The completed path stays silent on purpose: exit 0 is where the AGENT'S own
|
|
117
|
-
// status comment lives (the prompt contract instructs it, including for "I cannot fix this"), and exit 2
|
|
118
|
-
// by construction means the agent was cut off before that step.
|
|
119
|
-
// EXPORTED only so a test can hold every RUNNER_POLICY_REASONS member to a row here (issue #437 review).
|
|
120
|
-
export const TERMINAL_COMMENTS = {
|
|
121
|
-
"worker-abort": "Stopped: the worker ended this run before it finished (the 30-minute job limit, or a worker shutdown). Partial work may exist. Not retried.",
|
|
122
|
-
"operator-cancel": "Stopped: the operator cancelled this run. Partial work may exist. Not retried.",
|
|
123
|
-
"runner-policy": "Stopped: the run ended inside the container before finishing (a turn or token budget, or an in-container configuration refusal). Partial work may exist. Not retried.",
|
|
124
|
-
// Issue #437. Names the cause but never the provider's own message, which may echo a key fragment. "Or
|
|
125
|
-
// access" because a 403 is as often a key that works but may not use this model or route as a bad key.
|
|
126
|
-
"provider-auth-refused": "Stopped: the AI provider refused this worker's credentials or access (an authentication or permission error). The operator needs to check the provider key and what it is allowed to use. Not retried.",
|
|
127
|
-
// Issues #501, #502. The two stops name the policy, never the amount or the model: both are operator
|
|
128
|
-
// configuration, and the comment's reader may be an issue author who can act on neither.
|
|
129
|
-
"cost-cap": "Stopped: the next AI call could have taken this run past its cost limit, so it was not made. Partial work may exist. Not retried.",
|
|
130
|
-
"model-not-allowed": "Stopped: the run tried to call an AI model this trigger does not allow, or to change an AI request in a way it does not allow, so the call was not made. Partial work may exist. Not retried.",
|
|
131
|
-
"cost-cap-unenforceable": "Stopped: this run has a cost limit, and the job image could not enforce it before each AI call, so nothing was sent to the AI provider. The operator needs to update the job image. Not retried.",
|
|
132
|
-
"model-policy-unenforceable": "Stopped: this run is limited to certain AI models, and the job image could not enforce that before each AI call, so nothing was sent to the AI provider. The operator needs to update the job image. Not retried.",
|
|
133
|
-
// The runner's refusals of the job's MCP servers, each before the prompt. Never the server's name, its host or a
|
|
134
|
-
// variable: the exit line carries the name and the cause in its message, which the worker never parses (it reaches
|
|
135
|
-
// the worker's log as the container's output). Each says what the operator can check; the issue's author can act on
|
|
136
|
-
// none of it.
|
|
137
|
-
"mcp-unavailable": "Stopped: an MCP server this trigger names could not be used (it did not start, failed its handshake or refused the connection, an authentication error among them), so the run ended before the agent started. The operator needs to check that server and the job image. Not retried.",
|
|
138
|
-
"mcp-config-invalid": "Stopped: the list of MCP servers the worker wrote for this job is not one the job image accepts, so the run ended before the agent started. The operator needs to run a worker and a job image of the same release. Not retried.",
|
|
139
|
-
"mcp-config-mismatch": "Stopped: the MCP servers the worker wrote for this job are not the ones it named for it, so the run ended before the agent started. The operator needs to run a worker and a job image of the same release. Not retried.",
|
|
140
|
-
"mcp-secret-missing": "Stopped: a credential an MCP server this trigger names needs was not set inside the job, so the run ended before the agent started. The operator needs to check the secrets this trigger passes. Not retried.",
|
|
141
|
-
"mcp-command-missing": "Stopped: the job image does not have the program an MCP server this trigger names runs, so the run ended before the agent started. The operator needs to install that server in the job image or the overlay. Not retried.",
|
|
142
|
-
"mcp-shadowed": "Stopped: another extension in this job registers a tool or command that only the job's MCP servers may, so the run ended before the agent started. The operator needs to remove that extension or this trigger's MCP servers. Not retried.",
|
|
143
|
-
// Issue #625. The runner's two refusals of a job armed with run.awaitInput, each before the prompt.
|
|
144
|
-
"await-input-shadowed": "Stopped: another extension in this job registers the tool a run uses to stop and wait for an answer, so the run ended before the agent started. The operator needs to remove that extension or this trigger's run.awaitInput. Not retried.",
|
|
145
|
-
"await-input-without-session": "Stopped: this run may stop to wait for an answer but had nowhere to keep its work, so it ended before the agent started. The operator needs to run a worker and a job image of the same release. Not retried.",
|
|
146
|
-
// Issue #596. Never the size or the project: both are operator configuration, and the reader may be an issue author
|
|
147
|
-
// who can act on neither. The worker log and the run record carry them.
|
|
148
|
-
[EXIT_OOM_KILLED]: "Stopped: the job's container ran out of memory and was stopped. Partial work may exist. Not retried, because the same size would stop the same way. The operator can raise this job's memory size.",
|
|
149
|
-
};
|
|
150
|
-
|
|
151
|
-
// Issue #625: the one comment the COMPLETED path posts, on a run that ended awaiting-input. The agent posted its own
|
|
152
|
-
// question (the tool tells it to, before it ends); this says how the run continues. Fixed and path-free, the
|
|
153
|
-
// TERMINAL_COMMENTS rule, and it carries no model text: the question never crosses the exit line.
|
|
154
|
-
// On an issue any reply continues it (the receiver routes it on the marker); a pull request's reply keeps the phrase,
|
|
155
|
-
// since its session key needs a head branch that costs a token to read (parked.mjs).
|
|
156
|
-
export const PARKED_COMMENT = "This run stopped to wait for an answer to the question above. Reply here, and the next run continues from here.";
|
|
157
|
-
export const PARKED_COMMENT_PR = "This run stopped to wait for an answer to the question above. Reply in a comment that starts with this repository's pi-dispatch comment phrase, and the next run continues from here.";
|
|
158
|
-
export function parkedComment(target) {
|
|
159
|
-
return target?.type === "pull_request" ? PARKED_COMMENT_PR : PARKED_COMMENT;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// Issue #502: the `model-unknown` refusal's comment. Names no model: the reader may be an issue author.
|
|
163
|
-
export const MODEL_UNKNOWN_COMMENT = "Refused: this job names an AI model that this deployment does not know (it is in neither pi's model catalog nor the overlay models.json), so no container was started and nothing was spent. Ask the operator to check the trigger's model settings. Not run.";
|
|
164
|
-
|
|
165
|
-
// The `model-unknown` refusal whose `why` is `overlay-*` (PR #558, the end-of-round check of #501 and #502): the deployment's overlay
|
|
166
|
-
// models.json is the problem, not the job's model, so the generic text above would send the author to the wrong
|
|
167
|
-
// place. Fixed text naming no path and no model; the operator finds which file and why in the worker log.
|
|
168
|
-
export const OVERLAY_REFUSED_COMMENT = "Refused before starting: the deployment's model settings file (models.json in the overlay) cannot be used for this job, so no container was started and nothing was spent. The operator needs to fix that file. Not run.";
|
|
169
|
-
|
|
170
|
-
// Issue #341: the forge comments for a `job-user-unmappable` refusal, keyed by cause. Shorter than the operator
|
|
171
|
-
// texts in job-user.mjs on purpose: a comment's reader may be an issue author, who can act on none of it.
|
|
172
|
-
const JOB_USER_COMMENTS = Object.freeze({
|
|
173
|
-
default: "Refused: the worker host's container runtime cannot give this job a non-root user that can read and write its own files. Not run.",
|
|
174
|
-
"runtime-unreadable": "Refused: the worker host's container runtime answered in a form the worker cannot read, so which user this job may run as is unknown. Not run.",
|
|
175
|
-
});
|
|
134
|
+
// The post-spend terminal comments (TERMINAL_COMMENTS, issue #288), the park's, the model and job-user refusals', the
|
|
135
|
+
// durable-run refusals' and the bundle refusals' live in comment-tables.mjs since issue #710, with the rule each follows.
|
|
136
|
+
export { HELD_TAKEN_COMMENT, MODEL_UNKNOWN_COMMENT, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, OVERLAY_REFUSED_COMMENT, PARKED_COMMENT, PARKED_COMMENT_PR, RECOVERY_COMMENTS, TERMINAL_COMMENTS, parkedComment };
|
|
176
137
|
|
|
177
138
|
/**
|
|
178
139
|
* The runtime a venue's own preflights talk to, named in a retry's words (issue #354). The message is the job_failed log
|
|
@@ -225,20 +186,287 @@ function modelPrefixesOf(modelDollars) {
|
|
|
225
186
|
}
|
|
226
187
|
|
|
227
188
|
/**
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* zero-rated check reads as "not zero-rated" (fail closed).
|
|
189
|
+
* How long a console answer's trail claim lives: a forge job's retained data (`FORGE_JOB_RETENTION_MS`, retention.mjs),
|
|
190
|
+
* the longest the same id can return. Derived, so a change to the retention moves the claim with it.
|
|
231
191
|
*/
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
192
|
+
export const CONSOLE_TRAIL_CLAIM_MS = FORGE_JOB_RETENTION_MS;
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Claim the one trail comment of a console answer (issue #642): true when this attempt set `console:trail:<jobId>` and
|
|
196
|
+
* so must post, false when an earlier attempt of the same job already did, when the job's id cannot be derived, or when
|
|
197
|
+
* Valkey did not answer (each logged by its kind, never the text). The id is the queue's own (`forgeDeliveryJobId` over
|
|
198
|
+
* the job's kind, delivery id and replica), so a retry of the same job, which keeps its data, reaches the same key.
|
|
199
|
+
* Never throws.
|
|
200
|
+
*/
|
|
201
|
+
async function claimConsoleTrail(redis, job, log) {
|
|
202
|
+
let jobId;
|
|
203
|
+
try {
|
|
204
|
+
jobId = forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica);
|
|
205
|
+
} catch {
|
|
206
|
+
log("console_trail_skipped", { kind: job.kind ?? null, reason: "no-job-id" });
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
try {
|
|
210
|
+
const set = await redis.set(`console:trail:${jobId}`, "1", "PX", CONSOLE_TRAIL_CLAIM_MS, "NX");
|
|
211
|
+
if (set === "OK") return true;
|
|
212
|
+
log("console_trail_skipped", { kind: job.kind ?? null, reason: "already-posted" });
|
|
213
|
+
return false;
|
|
214
|
+
} catch (error) {
|
|
215
|
+
log("console_trail_skipped", { kind: job.kind ?? null, reason: "valkey-error", code: typeof error?.code === "string" ? error.code : "error" });
|
|
216
|
+
return false;
|
|
237
217
|
}
|
|
238
|
-
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// `callableModelRefs` (the models a job may call) moved to reserve-estimate.mjs (issue #642) with the zero-rated test
|
|
221
|
+
// it feeds, so the console's reserve estimate and this processor's reserve judge one rule (`zeroRatedFor`).
|
|
222
|
+
|
|
223
|
+
// ── Durable runs (issue #645) ───────────────────────────────────────────────────────────────────────────────────────
|
|
224
|
+
|
|
225
|
+
/** How many times one run may be continued from its checkpoint: a recovery job numbered past this is `recovery-cap`. */
|
|
226
|
+
export const RECOVERY_CAP = 2;
|
|
227
|
+
/** The reason an armed run's attempt RETURNS with when its continuation was queued instead of a retry thrown. */
|
|
228
|
+
export const RECOVERY_ENQUEUED = "recovery-enqueued";
|
|
229
|
+
/** A recovery job whose run already reached its final outcome elsewhere: refused silently, never commented. */
|
|
230
|
+
export const RECOVERY_SUPERSEDED = "recovery-superseded";
|
|
231
|
+
/** How long a recovery job waits before asking again whether its base run's container is still running here. */
|
|
232
|
+
export const RECOVERY_WAIT_MS = 30_000;
|
|
233
|
+
/** How long a recovery job waits before trying again after a never-started fault (it has no attempt to retry with). */
|
|
234
|
+
export const RECOVERY_RETRY_WAIT_MS = 60_000;
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* The run this attempt belongs to, decided from the job data alone (issue #645): `{ armed, runId, isRecovery, recovery,
|
|
238
|
+
* recoverSkipped, seq, recovered, checkpoints }`. The last three are filled in as the attempt goes, so the record can say
|
|
239
|
+
* them on every exit.
|
|
240
|
+
*
|
|
241
|
+
* `runId` is `job.recoveryOf` for a recovery job, else the queue's own id, DERIVED like the console trail's
|
|
242
|
+
* (`forgeDeliveryJobId` over kind, delivery id and replica: queue.mjs `enqueueForgeJob` names every forge job so). The
|
|
243
|
+
* processor is handed the effective job, which has no `.id`, and a derivation from data the queue itself keyed on is one
|
|
244
|
+
* answer for every attempt, retries and recoveries included; a new wiring argument would be a second. A recovery job's
|
|
245
|
+
* id (`<runId>.r<n>`) is never parsed: `recoveryOf` and `recovery` carry it.
|
|
246
|
+
*
|
|
247
|
+
* Not armed, with the reason the record names (fail open and say which): a local job (`local`, v1 recovers forge jobs
|
|
248
|
+
* only, so it runs exactly as it did before #645) and a forge job whose id cannot be derived (`no-run-id`).
|
|
249
|
+
*/
|
|
250
|
+
function durableOf(job) {
|
|
251
|
+
const d = { armed: false, runId: null, isRecovery: false, recovery: null, recoverSkipped: null, seq: null, recovered: null, checkpoints: null };
|
|
252
|
+
if (job?.recover !== true) return d;
|
|
253
|
+
if (job.kind === "local") {
|
|
254
|
+
d.recoverSkipped = "local";
|
|
255
|
+
return d;
|
|
256
|
+
}
|
|
257
|
+
if (typeof job.recoveryOf === "string" && job.recoveryOf !== "") {
|
|
258
|
+
d.runId = job.recoveryOf;
|
|
259
|
+
d.isRecovery = true;
|
|
260
|
+
d.recovery = job.recovery;
|
|
261
|
+
} else {
|
|
262
|
+
try {
|
|
263
|
+
d.runId = forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica);
|
|
264
|
+
} catch {
|
|
265
|
+
d.recoverSkipped = "no-run-id";
|
|
266
|
+
return d;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
d.armed = true;
|
|
270
|
+
return d;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** The durable-run facts the record carries, added to a result only when there are any (so every other is unchanged). */
|
|
274
|
+
function withDurable(result, d) {
|
|
275
|
+
if (result === null || typeof result !== "object") return result;
|
|
276
|
+
if (d.recoverSkipped === null && d.recovered === null && d.checkpoints === null) return result;
|
|
277
|
+
return { ...result, ...(d.recoverSkipped !== null ? { recoverSkipped: d.recoverSkipped } : {}), ...(d.recovered !== null ? { recovered: d.recovered } : {}), ...(d.checkpoints !== null ? { checkpoints: d.checkpoints } : {}) };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** The same on a throw, so a retried or failed attempt's record says them too (buildRecord reads the error). */
|
|
281
|
+
function tagDurable(e, d) {
|
|
282
|
+
if (e === null || typeof e !== "object") return;
|
|
283
|
+
try {
|
|
284
|
+
if (d.recoverSkipped !== null && e.recoverSkipped === undefined) e.recoverSkipped = d.recoverSkipped;
|
|
285
|
+
if (d.recovered !== null && e.recovered === undefined) e.recovered = d.recovered;
|
|
286
|
+
if (d.checkpoints !== null && e.checkpoints === undefined) e.checkpoints = d.checkpoints;
|
|
287
|
+
} catch {
|
|
288
|
+
// a frozen error keeps its own fields
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* The run reached its final outcome (a completed or policy result): remove its checkpoint store entry, its Valkey
|
|
294
|
+
* pointer and its budget ledger. Each best effort and logged by name, never thrown: every one has a TTL or a sweep
|
|
295
|
+
* behind it, and a cleanup fault must not turn a finished run into a failure.
|
|
296
|
+
*/
|
|
297
|
+
async function endDurableRun(d, deps) {
|
|
298
|
+
const log = typeof deps.log === "function" ? deps.log : () => {};
|
|
299
|
+
const quietly = async (what, fn) => {
|
|
300
|
+
try {
|
|
301
|
+
const r = await fn();
|
|
302
|
+
if (r && typeof r === "object" && (r.error || r.skipped)) log("durable_cleanup_failed", { what, code: String(r.error ?? r.skipped).slice(0, 40) });
|
|
303
|
+
} catch (error) {
|
|
304
|
+
log("durable_cleanup_failed", { what, code: typeof error?.code === "string" ? error.code : "error" });
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
if (deps.workspaceCheckpointer) await quietly("store", () => deps.workspaceCheckpointer.remove(d.runId));
|
|
308
|
+
// The wiring's seam when it hands one over; else run-checkpoint.mjs's own delete on this processor's client, so a bare
|
|
309
|
+
// wiring cannot leave a pointer that makes a later recovery think the run is still live. It answers false, never a
|
|
310
|
+
// throw, when Valkey did not answer: logged, and the pointer's 7-day expiry is the backstop.
|
|
311
|
+
await quietly("pointer", async () => {
|
|
312
|
+
const done = typeof deps.removeCheckpointPointer === "function" ? await deps.removeCheckpointPointer({ runId: d.runId }) : await removePointerDefault({ redis: deps.redis, runId: d.runId });
|
|
313
|
+
return done === false ? { error: "unanswered" } : null;
|
|
314
|
+
});
|
|
315
|
+
if (deps.runBudgetLedger) await quietly("ledger", () => deps.runBudgetLedger.remove(d.runId));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** A confirmed draft still belongs to the forge/account where it was captured. Check before even a
|
|
319
|
+
* refusal/approval comment can mint credentials; a restarted worker may serve another configured instance. */
|
|
320
|
+
export async function checkDraftAnswerSource(job, deps) {
|
|
321
|
+
const provenance = consoleDraftOf(job?.console);
|
|
322
|
+
if (!provenance) return null;
|
|
323
|
+
const refused = reason => ({ outcome: "policy", reason, budgetReserved: false });
|
|
324
|
+
if (provenance.invalid || job.parkedReply !== true || !deps?.answerDraftStore || typeof deps.answerDraftRuntime?.checkSource !== "function") return refused("answer-draft-refused");
|
|
325
|
+
let meta, input;
|
|
326
|
+
try {
|
|
327
|
+
meta = await deps.answerDraftStore.readMeta(provenance.draftSourceRunId);
|
|
328
|
+
input = await deps.answerDraftStore.privateInput(provenance.draftSourceRunId);
|
|
329
|
+
} catch { throw new InfraRetry("answer draft provenance could not be read", { reason: "container-never-started", budgetReserved: false }); }
|
|
330
|
+
if (!meta || !["drafted", "sent"].includes(meta.status) || meta.draftRunId !== provenance.draftRunId || meta.draftDigest !== provenance.draftDigest || input?.source?.kind !== job.kind || String(input.source.repo).toLowerCase() !== String(job.repo).toLowerCase() || input.source.target?.type !== job.target?.type || input.source.target?.number !== job.target?.number || (input.source.projectId ?? null) !== (job.projectId ?? null) || (input.source.azure?.projectId ?? null) !== (job.azure?.projectId ?? null)) return refused("answer-draft-refused");
|
|
331
|
+
try { return await deps.answerDraftRuntime.checkSource(input.source) === true ? null : refused("answer-draft-config-changed"); }
|
|
332
|
+
catch { throw new InfraRetry("answer draft current forge identity could not be read", { reason: "container-never-started", budgetReserved: false }); }
|
|
239
333
|
}
|
|
240
334
|
|
|
241
335
|
export async function runJob(job, deps) {
|
|
336
|
+
const sourceRefusal = await checkDraftAnswerSource(job, deps);
|
|
337
|
+
if (sourceRefusal) return sourceRefusal;
|
|
338
|
+
const touches = [];
|
|
339
|
+
let touchLineageSkipped = null;
|
|
340
|
+
// Issue #796: what the runner said of a canary's candidate, carried on every way out, since a job can load its candidate and still end any way at all.
|
|
341
|
+
// Issue #800: what the job loaded, the same way and for the same reason, a retried attempt's record included: a server
|
|
342
|
+
// that timed out exits 1 and retries (`mcp-unreachable`), and that record is the one a person opens to find out why.
|
|
343
|
+
const canaryFacts = { load: null, loaded: null };
|
|
344
|
+
const recordTouch = async (value) => {
|
|
345
|
+
const event = cleanTouch(value);
|
|
346
|
+
if (!event) return;
|
|
347
|
+
touches.push(event);
|
|
348
|
+
try {
|
|
349
|
+
if (deps.touchOutbox) { const intent = await deps.touchOutbox.begin(event); await deps.touchOutbox.applied(intent); }
|
|
350
|
+
else if (deps.touchStore) await deps.touchStore.write(event);
|
|
351
|
+
else await deps.recordTouch?.(event);
|
|
352
|
+
} catch { deps.log?.("touch_record_pending", {}); }
|
|
353
|
+
};
|
|
354
|
+
try {
|
|
355
|
+
const result = withCanaryLoad(await runJobInner(job, { ...deps, recordTouch, recordTouchGap: () => { touchLineageSkipped = "source-unavailable"; }, canaryFacts }), canaryFacts);
|
|
356
|
+
return result && typeof result === "object" && (touches.length || touchLineageSkipped) ? { ...result, ...(touches.length ? { touches } : {}), ...(touchLineageSkipped ? { touchLineageSkipped } : {}) } : result;
|
|
357
|
+
} catch (error) {
|
|
358
|
+
if (canaryFacts.load !== null && error && typeof error === "object") error.canaryLoad = canaryFacts.load;
|
|
359
|
+
if (error && typeof error === "object") {
|
|
360
|
+
const ran = error.loaded ?? null;
|
|
361
|
+
// A throw that already holds what the container said (`InfraRetry`'s `loaded`) came out of the run's own wrapper before
|
|
362
|
+
// the processor could note it, so what is noted here is the worker's own facts alone. The line stands, with them added.
|
|
363
|
+
if (ran !== null && canaryFacts.loaded !== null) error.loaded = loadedWithBase({ ...ran, ...imageFactsOf(canaryFacts.loaded) }, canaryFacts.loaded.baseSha);
|
|
364
|
+
else if (ran === null && canaryFacts.loaded !== null) error.loaded = canaryFacts.loaded;
|
|
365
|
+
}
|
|
366
|
+
if (touches.length && error && typeof error === "object") error.touches = touches;
|
|
367
|
+
if (touchLineageSkipped && error && typeof error === "object") error.touchLineageSkipped = touchLineageSkipped;
|
|
368
|
+
throw error;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function withCanaryLoad(result, facts) {
|
|
373
|
+
if (result === null || typeof result !== "object" || (facts.load === null && facts.loaded === null)) return result;
|
|
374
|
+
return { ...result, ...(facts.load !== null ? { canaryLoad: facts.load } : {}), ...(facts.loaded !== null ? { loaded: facts.loaded } : {}) };
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
async function runJobInner(job, deps) {
|
|
378
|
+
// Queue fields never confer approval authority. Resolve the host-owned decision before the mint or any reserve.
|
|
379
|
+
job = { ...job };
|
|
380
|
+
delete job.approvalDecision;
|
|
381
|
+
let approval = null;
|
|
382
|
+
if (job.gates !== undefined || job.approvalRequestId !== undefined || job.resume === true && deps.approvalRuntime) {
|
|
383
|
+
const id = deps.queueJobId ?? forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica);
|
|
384
|
+
if (!deps.approvalRuntime) return { outcome: "policy", reason: "approval-store-unavailable", budgetReserved: false, exitCode: null, turns: null, tokens: null };
|
|
385
|
+
approval = await deps.approvalRuntime.before({ ...job, id });
|
|
386
|
+
if (approval.error) throw new InfraRetry("approval state could not be read", { reason: "container-never-started", budgetReserved: false });
|
|
387
|
+
// Issue #761: the live triggers file was written by a newer pi-dispatch, so this build cannot judge the approval
|
|
388
|
+
// declaration. Before the mint and every reserve, and before any request is decided or claimed: the job waits for
|
|
389
|
+
// a worker that reads the file (`newer-release`), never refused, so neither the delivery nor the decision is lost.
|
|
390
|
+
if (approval.newer === true) throw new NewerReleaseHold("the triggers file");
|
|
391
|
+
if (!approval.ok) {
|
|
392
|
+
await deps.comment?.(job, `Refused: ${approval.refused}. Check the pending approval and current trigger declaration. Not run.`);
|
|
393
|
+
return { outcome: "policy", reason: approval.refused, ...(approval.audit ? { approval: approval.audit } : {}), budgetReserved: false, exitCode: null, turns: null, tokens: null };
|
|
394
|
+
}
|
|
395
|
+
if (approval.decision) {
|
|
396
|
+
job.approvalDecision = approval.decision;
|
|
397
|
+
// A phrase-prefixed /approve is a decision, not a new flow override. Continue the requesting flow.
|
|
398
|
+
if (typeof approval.request?.resumeJob?.flow === "string") job.flow = approval.request.resumeJob.flow;
|
|
399
|
+
if (deps.approvalSessionCheck) {
|
|
400
|
+
const session = deps.approvalSessionCheck(approval.request);
|
|
401
|
+
if (!session.ok) {
|
|
402
|
+
await deps.approvalRuntime.finish({ requestId: approval.request.requestId, jobId: id });
|
|
403
|
+
return { outcome: "policy", reason: "approval-session-unavailable", approval: approval.audit, budgetReserved: false, exitCode: null, turns: null, tokens: null };
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
if (approval.budgetCapUsd !== undefined) job.maxCostMicros = effectiveCostCapMicros(approval.budgetCapUsd, deps.deploymentMaxCostUsd ?? null);
|
|
408
|
+
deps = { ...deps, approvalRequest: approval.request ?? null, approvalJobId: id };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const durable = durableOf(job);
|
|
412
|
+
// Issue #704: what this attempt did with the project's secret bundle, for the record (`bundle` or `bundleSkipped`).
|
|
413
|
+
const bundleFacts = { bundle: null, bundleSkipped: null };
|
|
414
|
+
const archiveFacts = { value: null };
|
|
415
|
+
let result;
|
|
416
|
+
try {
|
|
417
|
+
result = await runAttempt(job, deps, durable, bundleFacts, archiveFacts);
|
|
418
|
+
if (approval?.audit && !result?.approval) result = { ...result, approval: approval.audit };
|
|
419
|
+
if (approval?.request && result?.outcome === "policy") {
|
|
420
|
+
const closed = await deps.approvalRuntime.finish({ requestId: approval.request.requestId, jobId: deps.approvalJobId });
|
|
421
|
+
if (!closed.ok) result = { ...result, reason: "approval-finish-unavailable" };
|
|
422
|
+
}
|
|
423
|
+
} catch (e) {
|
|
424
|
+
try { if (archiveFacts.value && e && typeof e === "object") e.forkSource = archiveFacts.value; } catch {}
|
|
425
|
+
if (approval?.audit && e && typeof e === "object") e.approval = approval.audit;
|
|
426
|
+
tagDurable(e, durable);
|
|
427
|
+
tagBundle(e, bundleFacts);
|
|
428
|
+
// A RECOVERY job has one attempt (recovery-queue.mjs), so a never-started retry would fail it for good over a
|
|
429
|
+
// blip (a pointer read, a ledger write, a restore that met a lock). Every such throw becomes a wait hold instead:
|
|
430
|
+
// `makeProcessor` delays it without spending the attempt, and the run's recovery deadline (`recoverBy`, the
|
|
431
|
+
// host-lost sweep) bounds how long it can wait. Nothing was started, so nothing is lost by asking again.
|
|
432
|
+
if (durable.isRecovery && e instanceof InfraRetry && e.reason === "container-never-started" && !(e instanceof RecoveryWaitHold)) {
|
|
433
|
+
const hold = new RecoveryWaitHold(e.message, { cause: e, reason: e.reason, provider: e.provider, model: e.model, budgetReserved: e.budgetReserved, dollars: e.dollars, resources: e.resources, exitCode: e.exitCode, turns: e.turns, tokens: e.tokens, session: e.session, usage: e.usage, waitMs: RECOVERY_RETRY_WAIT_MS });
|
|
434
|
+
tagDurable(hold, durable);
|
|
435
|
+
tagBundle(hold, bundleFacts);
|
|
436
|
+
deps?.log?.("recovery_retry_held", { delayMs: RECOVERY_RETRY_WAIT_MS });
|
|
437
|
+
throw hold;
|
|
438
|
+
}
|
|
439
|
+
throw e;
|
|
440
|
+
}
|
|
441
|
+
// The run's FINAL outcome cleans up after it. Not a queued recovery (its checkpoint is what the recovery restores),
|
|
442
|
+
// and not `recovery-superseded`: that one is decided on a pointer read that came back empty, and an empty read must
|
|
443
|
+
// never be the reason another run's store is deleted; the run that settled cleaned up after itself.
|
|
444
|
+
if (durable.armed && result?.reason !== RECOVERY_ENQUEUED && result?.reason !== RECOVERY_SUPERSEDED) await endDurableRun(durable, deps);
|
|
445
|
+
return withBundle(withDurable(archiveFacts.value ? { ...result, forkSource: archiveFacts.value } : result, durable), bundleFacts);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* The secret-bundle facts the record carries (issue #704), added to a result only when there are any, so every other
|
|
450
|
+
* result is unchanged. NAMES only: `bundle` is `{ id, variables, shadowed }`, never a value.
|
|
451
|
+
*/
|
|
452
|
+
function withBundle(result, b) {
|
|
453
|
+
if (result === null || typeof result !== "object") return result;
|
|
454
|
+
if (b.bundle === null && b.bundleSkipped === null) return result;
|
|
455
|
+
return { ...result, ...(b.bundle !== null ? { bundle: b.bundle } : {}), ...(b.bundleSkipped !== null ? { bundleSkipped: b.bundleSkipped } : {}) };
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/** The same on a throw, so a retried or failed attempt's record says what it held (buildRecord reads the error). */
|
|
459
|
+
function tagBundle(e, b) {
|
|
460
|
+
if (e === null || typeof e !== "object") return;
|
|
461
|
+
try {
|
|
462
|
+
if (b.bundle !== null && e.bundle === undefined) e.bundle = b.bundle;
|
|
463
|
+
if (b.bundleSkipped !== null && e.bundleSkipped === undefined) e.bundleSkipped = b.bundleSkipped;
|
|
464
|
+
} catch {
|
|
465
|
+
// a frozen error keeps its own fields
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
async function runAttempt(job, deps, durable, bundleFacts = { bundle: null, bundleSkipped: null }, archiveFacts = { value: null }) {
|
|
242
470
|
const {
|
|
243
471
|
redis,
|
|
244
472
|
caps, // { day, week, month }; week/month null when that window is disabled (REQ-SPEND-CAPS-MULTI-WINDOW)
|
|
@@ -256,7 +484,8 @@ export async function runJob(job, deps) {
|
|
|
256
484
|
// this job names is on this host (image-preflight.mjs). Default admits everything, so a wiring that
|
|
257
485
|
// omits it behaves exactly as before -- the container's own failure stays the backstop.
|
|
258
486
|
imagePreflight = async () => ({ ok: true }),
|
|
259
|
-
|
|
487
|
+
subscriptionAuthRequired = async () => false,
|
|
488
|
+
// (job) => { ok } | { refused, at, jobId } | { newer } (issue #761). The one-shot pre-spend check (issue #231,
|
|
260
489
|
// DES-ONE-SHOT-DISARM-IN-THE-FILE). Default admits everything -- an unwired processor behaves
|
|
261
490
|
// exactly as before, and the gate below only calls it for a job whose matched rule was a
|
|
262
491
|
// one-shot, so the default is never a probe running on every delivery.
|
|
@@ -271,6 +500,8 @@ export async function runJob(job, deps) {
|
|
|
271
500
|
// the real read happens where it always did, because threading a live credential through the processor
|
|
272
501
|
// would put it in scope for every log line and record between here and the container.
|
|
273
502
|
checkProviderCredential = () => ({ ok: true }),
|
|
503
|
+
prepareSubscriptionAuth = null,
|
|
504
|
+
validateSubscriptionAuth = null,
|
|
274
505
|
// (refs) => { ok } | { unknown: { provider, id }, why } | { unavailable: code } (issue #502). Is each of this
|
|
275
506
|
// job's models, its main one and every listed one, a model pi knows (model-catalog.mjs `checkModelsKnown`,
|
|
276
507
|
// the builtin catalog plus the overlay models.json)? Admit-everything by default, like the credential probe
|
|
@@ -306,6 +537,11 @@ export async function runJob(job, deps) {
|
|
|
306
537
|
// so the answer is not swallowed against it. Takes no job: the key belongs to the REAL BullMQ job, which only the
|
|
307
538
|
// wiring holds (index.mjs binds queue.mjs's `releaseSemanticWindow` to it). Never throws. The default does nothing.
|
|
308
539
|
releaseParkedWindow = async () => {},
|
|
540
|
+
// () => Promise<{ resumed } | null> (issue #638, DES-AWAIT-INPUT-ENDS-THE-JOB): take the replies held for this run
|
|
541
|
+
// while it was parking, after the window is released: the first recent one is enqueued as the resumed job, the
|
|
542
|
+
// rest answered once (held-settle.mjs `takeHeld`). Bound by the wiring to the REAL job, like the release. Never
|
|
543
|
+
// throws. The default takes nothing, so a wiring without it parks exactly as before.
|
|
544
|
+
takeHeldReplies = async () => null,
|
|
309
545
|
// The session store's root, or null when the feature is unavailable (REQ-RESUMABLE-SESSION). Read
|
|
310
546
|
// ONLY to answer the fail-closed gate below -- nothing here opens it, and it never reaches the
|
|
311
547
|
// container env (docker-run.mjs mounts a per-job COPY, never the store).
|
|
@@ -342,6 +578,29 @@ export async function runJob(job, deps) {
|
|
|
342
578
|
// would leave an INJECTED resolver running on every job, which is how a probe nobody wanted starts
|
|
343
579
|
// spawning a subprocess per delivery to learn nothing.
|
|
344
580
|
resolveSecrets = async (job) => ({ profileUnknown: job.secretsProfile ?? DEFAULT_SECRETS_PROFILE }),
|
|
581
|
+
// Issue #704, project secret bundles. `findSecretBundle(job, { project })` is the worker's own read of the triggers
|
|
582
|
+
// file's `secretBundles` (triggers-file.mjs `makeFindSecretBundle`); the default finds none, so a bare processor
|
|
583
|
+
// and every deployment without the key run exactly as before. `secretBundles` is the bundle seam
|
|
584
|
+
// (secret-bundles.mjs `makeBundleResolver`, its overlay and signal bound by the wiring): `lookup(profile)` with
|
|
585
|
+
// no spawn, `resolve(job, binding)` with one. Its default FAILS CLOSED, `resolveSecrets`' posture and reason:
|
|
586
|
+
// a wiring that finds a bundle and cannot resolve one must refuse, never start the job without the environment.
|
|
587
|
+
findSecretBundle = () => ({ none: true }),
|
|
588
|
+
secretBundles = { lookup: (profile) => ({ profileUnknown: profile }), resolve: async (_job, binding) => ({ profileUnknown: binding?.profile ?? null }) },
|
|
589
|
+
// `(job, token) => { headRef, headRepo }` for a pull-request target, the forge host's own (prepare-github.mjs reads
|
|
590
|
+
// the same seam for run.resume): whether the head lives in the base repository, answered by the FORGE, never the
|
|
591
|
+
// payload. Asked only for a job whose project has a bundle without `forkReviews`. Null on a bare wiring, which
|
|
592
|
+
// can confirm no head and so grants no bundle to a pull request (`bundleSkipped: "fork"`, the direction that
|
|
593
|
+
// grants nothing).
|
|
594
|
+
resolvePullRequestHead = null,
|
|
595
|
+
// Issue #704 review: whether a profile this host lacks DEFERS the job rather than refusing it: true only for a job
|
|
596
|
+
// on a multi-host worker's shared queue (index.mjs, the never-fits gate's predicate), where another host may serve
|
|
597
|
+
// it. False by default, so a single host refuses, as it always did.
|
|
598
|
+
bundleProfileDeferrable = false,
|
|
599
|
+
// Issue #704 part 3: the job's venue's judgement of its project secret bundle (the registry's `bundleGate`,
|
|
600
|
+
// backend-openshell.mjs's for both OpenShell venues), `(job, { endpoints, vars, shadowed })` => `{ ok, placeholders?,
|
|
601
|
+
// plain? }`, `{ venueRefused: { reason, comment } }` or `{ unavailable }`. The default admits, which is every venue
|
|
602
|
+
// that holds its env as values: there a bundle's variables simply join the env.
|
|
603
|
+
bundleGate = async () => ({ ok: true }),
|
|
345
604
|
// (job) => { ok, document } | { refused, name, why } | { unavailable: code } (REQ-MCP-OPERATOR-SERVERS,
|
|
346
605
|
// mcp-preflight.mjs). The default passes a job that names no server and refuses one that does, so a wiring that
|
|
347
606
|
// forgot the check can never run a job without the servers its trigger names.
|
|
@@ -381,7 +640,7 @@ export async function runJob(job, deps) {
|
|
|
381
640
|
// never sets it classifies every abort as worker-abort, exactly as before.
|
|
382
641
|
runContainer,
|
|
383
642
|
cleanup, // (dirs) => void
|
|
384
|
-
comment, // (job, text) => void (issue status; no-op for local jobs)
|
|
643
|
+
comment: rawComment, // (job, text) => void (issue status; no-op for local jobs)
|
|
385
644
|
log = () => {},
|
|
386
645
|
// The outbox chain collector (INT-OUTBOX-CONTRACT). No-op default so a job whose wiring omits it --
|
|
387
646
|
// or a github job with no /outbox -- chains nothing. It NEVER throws (outbox.mjs), so its counts are
|
|
@@ -392,6 +651,7 @@ export async function runJob(job, deps) {
|
|
|
392
651
|
// NEVER throws, so a refused plan is a recorded outcome of a completed job and never a retry. The default collects
|
|
393
652
|
// nothing (null: no plan), so a wiring that omits it records `plan: null`.
|
|
394
653
|
collectPlan = async () => null,
|
|
654
|
+
collectWatch = async () => {},
|
|
395
655
|
// Issue #501: the deployment's dollar windows `{ day, week, month }` in micro-dollars (each null when unset), or
|
|
396
656
|
// null when no window is set. Null is the default and the off switch: nothing is reserved or settled and no
|
|
397
657
|
// `budget:usd:*` key is written, so a deployment with no dollar setting is byte-identical.
|
|
@@ -414,6 +674,10 @@ export async function runJob(job, deps) {
|
|
|
414
674
|
otherDollars = null,
|
|
415
675
|
dollarCapSource = null,
|
|
416
676
|
envelopeMismatch = false,
|
|
677
|
+
// Issue #761: the endpoint tag of the Valkey this worker's dollar counters live in (`valkeyEndpointTag`), which the
|
|
678
|
+
// record's `dollars.reservedIn` carries so doctor sums only the records of the counter it reads. Null (an unwired
|
|
679
|
+
// processor) writes no `reservedIn`, so such a record keeps the shape it had.
|
|
680
|
+
valkeyTag = null,
|
|
417
681
|
// Issue #505: whether the LIVE triggers file still flags this job's cron trigger `run.portfolio: true`, as
|
|
418
682
|
// `(job) => boolean`, and this host's envelope `delegation` block (`{ enabled, writers }`), null with no envelope.
|
|
419
683
|
// The flag on the job data is what the trigger said when the job was queued; the file is what the operator says
|
|
@@ -438,8 +702,63 @@ export async function runJob(job, deps) {
|
|
|
438
702
|
// `builtinModel`). Read only by the zero-rated check; the default knows no builtin model, so an unwired
|
|
439
703
|
// processor judges overlay models alone and reserves for every other.
|
|
440
704
|
builtinModel = () => null,
|
|
705
|
+
// Issue #645, durable runs. Every one defaults to inert, and none is read unless the job armed `run.recover`, so an
|
|
706
|
+
// unwired processor runs every other job exactly as before. `workspaceCheckpointer` is workspace-checkpoint.mjs's
|
|
707
|
+
// (null when PI_CHECKPOINTS_DIR is unset), `runBudgetLedger` run-budget-ledger.mjs's, `readCheckpointPointer`
|
|
708
|
+
// `({ runId }) => pointer | null` and `enqueueRecovery` `({ job, cause, pointer }) => { enqueued, jobId, n } |
|
|
709
|
+
// { enqueued: false, reason }` (recovery-queue.mjs, bound to the real BullMQ job by the wiring). `stageRecovered`
|
|
710
|
+
// is the session store's (session-store.mjs). `resetTrustPaths` is trust-paths.mjs's. `runningJobs` is the
|
|
711
|
+
// processor's `{ list() }` of the jobs running on this host, and `hostName` this worker's name, as the checkpoint
|
|
712
|
+
// manifest records it.
|
|
713
|
+
workspaceCheckpointer = null,
|
|
714
|
+
forkStore = null,
|
|
715
|
+
checkForkPolicy = async () => ({ refused: "fork-policy-unavailable" }),
|
|
716
|
+
checkThinking = async () => false,
|
|
717
|
+
runBudgetLedger = null,
|
|
718
|
+
readCheckpointPointer = null,
|
|
719
|
+
enqueueRecovery = null,
|
|
720
|
+
stageRecovered = null,
|
|
721
|
+
resetTrustPaths = resetTrustPathsDefault,
|
|
722
|
+
runningJobs = null,
|
|
723
|
+
onDollarHold = () => {},
|
|
724
|
+
hostName = null,
|
|
725
|
+
// `({ runId, pointer, attempt }) => { seq } | { skipped }` (start.mjs): a salvage checkpoint from a crashed attempt's
|
|
726
|
+
// job directory still on this host, asked for a recovery job before its manifest is read. Null takes none.
|
|
727
|
+
salvageCheckpoint = null,
|
|
728
|
+
// `(jobId, { keep }) => record | null` (start.mjs `readRunRecord`): the settled-run check a recovery asks first, this
|
|
729
|
+
// host's file and then the fleet's copy, the first that passes `keep` when one is given. Null skips it, and the
|
|
730
|
+
// pointer and store logic below still decide.
|
|
731
|
+
readRecord = null,
|
|
732
|
+
// `({ workspace, baseSha }) => boolean` (trust-paths.mjs): whether the base commit holds anything under
|
|
733
|
+
// `.pi/extensions/`, which makes a run unrecoverable (see its use).
|
|
734
|
+
baseHasExtensions = baseHasExtensionsDefault,
|
|
441
735
|
now = new Date(),
|
|
442
736
|
} = deps;
|
|
737
|
+
// `armed` can still drop to false for a fresh attempt whose base holds `.pi/extensions/` (below), before anything durable.
|
|
738
|
+
let { armed } = durable;
|
|
739
|
+
const { runId, isRecovery } = durable;
|
|
740
|
+
// The checkpoint a recovery job restores, read and judged among the free gates below.
|
|
741
|
+
let manifest = null;
|
|
742
|
+
let forkSource = null;
|
|
743
|
+
let sourcePrimed = false;
|
|
744
|
+
let sourceRunId = deps.queueJobId ?? null;
|
|
745
|
+
if (sourceRunId === null) { try { sourceRunId = forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica); } catch {} }
|
|
746
|
+
const forkRefusal = async (reason) => {
|
|
747
|
+
await comment(job, `Refused: ${reason}. No fork container was started. Check the source run, current trigger policy and retained workspace points.`);
|
|
748
|
+
return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
|
|
749
|
+
};
|
|
750
|
+
let transcriptPath = null;
|
|
751
|
+
// Whether this attempt's dollar hold has a durable record (`recordHold`): only then do its settle and release go through
|
|
752
|
+
// the ledger's compare-and-set. A hold that failed to be recorded is given back directly, since no one else can see it.
|
|
753
|
+
let holdRecorded = false;
|
|
754
|
+
// Issue #645: the job-count slot an armed run carries from an earlier attempt (`claimSlot` answered "carried"). It is
|
|
755
|
+
// still counted, by the attempt that reserved it, so this attempt reserves none and refunds none.
|
|
756
|
+
let slotCarried = false;
|
|
757
|
+
// Whether an armed run's slot belongs to the RUN rather than to this attempt (any successful claim). Such a slot is not
|
|
758
|
+
// refunded by a never-started retry: the retry carries it (`claimSlot` answers "carried"), so refunding it here would
|
|
759
|
+
// let the retry run on a slot nothing counts. Only the run's final outcome gives a slot back (a refusal or
|
|
760
|
+
// config-refused, exactly as before).
|
|
761
|
+
let slotOwnedByRun = false;
|
|
443
762
|
|
|
444
763
|
// "Forge-backed" is the negation of local, not an enumeration of forges: a job that is not editing a
|
|
445
764
|
// folder on this host is working against a remote, and every gate below applies for the same reason
|
|
@@ -452,6 +771,11 @@ export async function runJob(job, deps) {
|
|
|
452
771
|
const wantsForgeToken = isForgeBacked || job.github === true;
|
|
453
772
|
let token = null;
|
|
454
773
|
let prepared = null;
|
|
774
|
+
const comment = job.answerDraft ? async () => {} : rawComment;
|
|
775
|
+
let watchCompleted = false;
|
|
776
|
+
let answerUsage = null;
|
|
777
|
+
let watchUsage = null;
|
|
778
|
+
let watchPolicyRefused = false;
|
|
455
779
|
// The job-count reservations still standing, in reserve order (issue #499 part B): what `reserveLedgers` took and
|
|
456
780
|
// no refund has given back yet. EVERY refund below is `releaseLedgers` over this one list, last first, so no path
|
|
457
781
|
// can give back one ledger and forget another, and none can give one back twice: a released ledger leaves the list.
|
|
@@ -461,7 +785,8 @@ export async function runJob(job, deps) {
|
|
|
461
785
|
// refund, global-only as INT-RUN-HISTORY-FILE-CONTRACT has it. A scoped or project slot a failed refund left behind
|
|
462
786
|
// is in the `budget_release_failed` log line, not in this field.
|
|
463
787
|
let globalLedger = null;
|
|
464
|
-
|
|
788
|
+
// A carried slot (issue #645) is held too: by the run, under the attempt that reserved it.
|
|
789
|
+
const globalHeld = () => (globalLedger !== null && held.includes(globalLedger)) || slotCarried;
|
|
465
790
|
// Give back every held job-count slot, NEVER throwing: a refund that did not land must not replace the caller's
|
|
466
791
|
// classification with a Redis message. Logged with `at`; returns whether the list is empty afterwards.
|
|
467
792
|
const refundLedgers = async (at) => {
|
|
@@ -489,9 +814,14 @@ export async function runJob(job, deps) {
|
|
|
489
814
|
// taken or once it has been settled or given back, so no path can settle or refund it twice. `dollars` is what
|
|
490
815
|
// the record says about it (INT-RUN-HISTORY-FILE-CONTRACT), null until the reservation step ran.
|
|
491
816
|
let dollarHold = null;
|
|
817
|
+
// A reporting failure must never change admission, settlement or the result of a paid run.
|
|
818
|
+
const publishDollarHold = (hold) => { try { onDollarHold(hold); } catch {} };
|
|
492
819
|
let dollars = null;
|
|
493
820
|
// Issue #596: what the container used (`resources` off its exit line), null until a container ran and reported it.
|
|
494
821
|
let resources = null;
|
|
822
|
+
// Issue #644: the run's progress summary (run-progress.mjs `summary`), null until a container ran with the reducer on.
|
|
823
|
+
// Display only: carried to the record like `resources`, and never read by any decision below.
|
|
824
|
+
let progress = null;
|
|
495
825
|
|
|
496
826
|
try {
|
|
497
827
|
// The one-shot pre-spend check (issue #231), FIRST on the ladder: one file read, cheaper than
|
|
@@ -504,6 +834,11 @@ export async function runJob(job, deps) {
|
|
|
504
834
|
// there, not optional hardening.
|
|
505
835
|
if (job.trigger?.matched?.once === true) {
|
|
506
836
|
const spent = await checkOnceSpent(job);
|
|
837
|
+
// Issue #761: the one answer that is not fail-open. A triggers file written by a newer pi-dispatch may hold this
|
|
838
|
+
// one-shot's disarm in a shape this build cannot read, and this build's own disarm of it would be refused, so
|
|
839
|
+
// running would fire the one-shot again on every delivery. Deferred (`newer-release`), never refused: a refusal
|
|
840
|
+
// would use up the delivery. Thrown here, first on the ladder, so nothing is minted, cloned or reserved.
|
|
841
|
+
if (spent.newer === true) throw new NewerReleaseHold("the triggers file");
|
|
507
842
|
if (spent.refused) {
|
|
508
843
|
// Commented like every sibling policy refusal: explainability is this refusal's whole
|
|
509
844
|
// purpose, and only a DISTINCT re-close reaches it past the GUID dedup, so the noise
|
|
@@ -523,8 +858,12 @@ export async function runJob(job, deps) {
|
|
|
523
858
|
// standing between a stale receiver and a paid job that ran when the operator wrote "wait".
|
|
524
859
|
{
|
|
525
860
|
const skew = await checkWaitSkew(job);
|
|
526
|
-
// Issue #
|
|
527
|
-
//
|
|
861
|
+
// Issue #761: a triggers file written by a newer pi-dispatch may author a narrowing field this build has no row
|
|
862
|
+
// for, which no job field can show missing, so neither "skewed" nor "fine" is an answer this build can give. The
|
|
863
|
+
// job waits for a worker that reads the file (`newer-release`), before any spend, the once check's posture.
|
|
864
|
+
if (skew.newer === true) throw new NewerReleaseHold("the triggers file");
|
|
865
|
+
// Issue #502: an authored NARROWING field the job arrived without, one of the rows of `AUTHORED_NARROWING_FIELDS`
|
|
866
|
+
// (triggers-file.mjs). The same two causes as a dropped wait, and the same refusal shape; without it the job would
|
|
528
867
|
// run on the deployment's list, or on none, while every record reads like a correct run. The FIELD is named,
|
|
529
868
|
// never its value: the comment's reader may be an issue author.
|
|
530
869
|
if (skew.skewed && typeof skew.field === "string") {
|
|
@@ -567,6 +906,16 @@ export async function runJob(job, deps) {
|
|
|
567
906
|
log("refused_backend_unblessed", { backend: job.backend, blessed: blessedBackends });
|
|
568
907
|
return { outcome: "policy", reason: "backend-unblessed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
569
908
|
}
|
|
909
|
+
// Issue #683: a folder job (a local run, a cron trigger's folder) on a venue that declares `localFolders` absent.
|
|
910
|
+
// The loader refuses a trigger that NAMES such a venue (`refusesLocalWorkspace`); this is the one it could not
|
|
911
|
+
// see, a trigger with no `run.backend` on a deployment whose DEFAULT is such a venue. A copy is edited elsewhere
|
|
912
|
+
// and written back afterwards, racing every edit the operator makes meanwhile (`DES-OPENSHELL-VENUE`).
|
|
913
|
+
const venueName = resolveBackendName(job, blessedBackends[0]);
|
|
914
|
+
if (job.kind === "local" && backendFor(venueName)?.declares?.localFolders === ABSENT) {
|
|
915
|
+
await comment(job, `Refused: this job runs against a folder on this machine, and its venue "${venueName}" copies a job's files instead of editing a folder in place. Run it on a venue that mounts folders (local or podman). Not run.`);
|
|
916
|
+
log("refused_venue_local_folders", { backend: venueName });
|
|
917
|
+
return { outcome: "policy", reason: "venue-local-folders-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
918
|
+
}
|
|
570
919
|
|
|
571
920
|
// Issue #278. PI_BACKEND_FLOOR can ask for a guarantee a backend declares but EARNS only while something
|
|
572
921
|
// about this host is observed -- `local`'s credentialTransit, which holds only while the docker CLI sends
|
|
@@ -614,6 +963,16 @@ export async function runJob(job, deps) {
|
|
|
614
963
|
budgetReserved: false,
|
|
615
964
|
};
|
|
616
965
|
}
|
|
966
|
+
// Issue #683: a venue's own refusal, made before spend whatever the floor says: the openshell venue's gateway is not
|
|
967
|
+
// on this host, has a gateway-wide policy in force, or the host has no Landlock (`CONST-VENUE-EGRESS-LAYER`'s third
|
|
968
|
+
// and fourth conditions). Determinate, so a RETURN (CONST-RETRY-INFRA-ONLY), ahead of the image preflight and the
|
|
969
|
+
// mint. The comment is the venue's fixed sentence for its reason, never anything the runtime said.
|
|
970
|
+
if (observed?.venueRefused && typeof observed.venueRefused.reason === "string") {
|
|
971
|
+
const { reason, comment: text } = observed.venueRefused;
|
|
972
|
+
await comment(job, typeof text === "string" && text !== "" ? text : VENUE_REFUSED_COMMENT);
|
|
973
|
+
log("refused_venue", { reason });
|
|
974
|
+
return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
975
|
+
}
|
|
617
976
|
if (observed?.unavailable) {
|
|
618
977
|
// The local venue's words stay what they were (they are its log line and BullMQ's failedReason); another venue
|
|
619
978
|
// is not docker's to name.
|
|
@@ -662,15 +1021,15 @@ export async function runJob(job, deps) {
|
|
|
662
1021
|
job,
|
|
663
1022
|
rootful
|
|
664
1023
|
? key
|
|
665
|
-
? "
|
|
666
|
-
: "
|
|
1024
|
+
? PODMAN_CONF_COMMENTS["rootful-widens"]
|
|
1025
|
+
: PODMAN_CONF_COMMENTS["rootful-unread"]
|
|
667
1026
|
: live
|
|
668
1027
|
? key
|
|
669
|
-
? "
|
|
670
|
-
: "
|
|
1028
|
+
? PODMAN_CONF_COMMENTS["live-widens"]
|
|
1029
|
+
: PODMAN_CONF_COMMENTS["live-unread"]
|
|
671
1030
|
: key
|
|
672
|
-
?
|
|
673
|
-
:
|
|
1031
|
+
? PODMAN_CONF_COMMENTS.widens
|
|
1032
|
+
: PODMAN_CONF_COMMENTS.unread,
|
|
674
1033
|
);
|
|
675
1034
|
log("refused_podman_conf_widens_job", { key: key ?? null, ...(live ? { live: true } : {}), ...(rootful ? { rootful: true } : {}), message: observed.podmanConfRefused.message });
|
|
676
1035
|
return { outcome: "policy", reason: PODMAN_CONF_WIDENS_JOB, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
@@ -681,12 +1040,22 @@ export async function runJob(job, deps) {
|
|
|
681
1040
|
// image refuses without minting a credential it will not use, cloning a repo it will not read, or
|
|
682
1041
|
// burning a cap slot. Jobs run with --pull=never (docker-run.mjs), so an absent image is never
|
|
683
1042
|
// fetched -- this refusal IS the whole diagnosis, not a race with a background pull.
|
|
684
|
-
const
|
|
1043
|
+
const needsSubscriptionAuth = await subscriptionAuthRequired(job);
|
|
1044
|
+
if (typeof needsSubscriptionAuth !== "boolean") throw new Error("the subscription image requirement is unavailable");
|
|
1045
|
+
const checkedImage = await imagePreflight(job, { subscriptionAuthRequired: needsSubscriptionAuth });
|
|
1046
|
+
// Enforce the same host-only requirement at the DI seam too: a preflight returning capabilities
|
|
1047
|
+
// cannot accidentally admit an older image before access preparation, mint or reservations.
|
|
1048
|
+
const img = checkedImage.ok && needsSubscriptionAuth && !(checkedImage.capabilities ?? []).includes("subscriptionAuth")
|
|
1049
|
+
? { subscriptionAuthUnsupported: checkedImage.image ?? job.image ?? "configured job image", declared: checkedImage.capabilities ?? [] }
|
|
1050
|
+
: checkedImage;
|
|
685
1051
|
// The image's declared pi version, read on the inspect the preflight already ran. Needed BEFORE the
|
|
686
1052
|
// container starts, because a transcript written by a different pi may hold tool-call arguments the
|
|
687
1053
|
// current schema no longer accepts -- so the resume has to be refused, not repaired mid-run. Null
|
|
688
1054
|
// when the image declares none, which downstream means "never resume": the safe direction.
|
|
689
1055
|
const piVersion = img.piVersion ?? null;
|
|
1056
|
+
// Issue #800: from here a refusal can say which image it would have run on. Before this, and on a preflight that found
|
|
1057
|
+
// no usable image, no value is set, so such a record is byte-identical to what it was.
|
|
1058
|
+
if (deps.canaryFacts && img.ok === true) deps.canaryFacts.loaded = loadedWithImage(null, img);
|
|
690
1059
|
if (img.missing) {
|
|
691
1060
|
await comment(job, `Refused: the job image "${img.missing}" is not present on the worker host. Not run.`);
|
|
692
1061
|
log("refused_image_missing", { image: img.missing });
|
|
@@ -716,6 +1085,22 @@ export async function runJob(job, deps) {
|
|
|
716
1085
|
log("refused_image_forge_unsupported", { image: img.forgeUnsupported, kind: img.kind, declared: img.declared });
|
|
717
1086
|
return { outcome: "policy", reason: "job-image-forge-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
718
1087
|
}
|
|
1088
|
+
if (deps.approvalRequest && deps.approvalSessionCheck) {
|
|
1089
|
+
const ready = deps.approvalSessionCheck(deps.approvalRequest, { piVersion: img.piVersion, venue: resolveBackendName(job, blessedBackends[0]), provider: job.provider, modelId: job.model });
|
|
1090
|
+
if (!ready.ok) return { outcome: "policy", reason: "approval-session-unavailable", exitCode: null, turns: null, tokens: null, budgetReserved: false };
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
if (job.fallbackModels !== undefined && !(img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY)) {
|
|
1094
|
+
const required = CAPABILITY_GATES.find((row) => row.token === "modelFailover");
|
|
1095
|
+
await comment(job, required.comment(img.image ?? job.image ?? "selected image", "lacks authenticated exit support"));
|
|
1096
|
+
return { outcome: "policy", reason: "job-image-model-failover-unsupported", exitCode: null, turns: null, tokens: null, budgetReserved: false };
|
|
1097
|
+
}
|
|
1098
|
+
if (job.answerDraft && !(img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY)) return { outcome: "policy", reason: "job-image-answer-draft-unsupported", budgetReserved: false };
|
|
1099
|
+
if (job.gates && !(img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY)) {
|
|
1100
|
+
await comment(job, APPROVAL_COMMENTS["job-image-approval-auth-unsupported"]);
|
|
1101
|
+
return { outcome: "policy", reason: "job-image-approval-auth-unsupported", exitCode: null, turns: null, tokens: null, budgetReserved: false };
|
|
1102
|
+
}
|
|
1103
|
+
|
|
719
1104
|
// The image capability gates (one table, CAPABILITY_GATES in image-preflight.mjs): the image is present and
|
|
720
1105
|
// does not declare a feature this job carries, so its runner would silently ignore it. Determinate, so a
|
|
721
1106
|
// refusal rather than a retry, and pre-spend, because no version of this gets better by running. Like the
|
|
@@ -790,7 +1175,7 @@ export async function runJob(job, deps) {
|
|
|
790
1175
|
if (known?.unknown) {
|
|
791
1176
|
// An `overlay-*` why is the deployment's file, not the job's model: its own comment.
|
|
792
1177
|
const why = typeof known.why === "string" ? known.why : null;
|
|
793
|
-
await comment(job, why
|
|
1178
|
+
await comment(job, modelUnknownComment(why));
|
|
794
1179
|
// `why` is a fixed token: `overlay-unparseable` tells the operator the file is the problem, not the id.
|
|
795
1180
|
log("refused_model_unknown", { provider: known.unknown.provider ?? null, model: known.unknown.id ?? null, why: known.why ?? null });
|
|
796
1181
|
return { outcome: "policy", reason: "model-unknown", why, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
@@ -800,7 +1185,7 @@ export async function runJob(job, deps) {
|
|
|
800
1185
|
// does not list `PI_MODEL`, or a `dispatch_set model` that moved the default off a trigger's list. Both
|
|
801
1186
|
// halves are compared, exact (`modelOnList`), the runner guard's rule.
|
|
802
1187
|
if (Array.isArray(job.models) && !modelOnList(job.models, job.provider, job.model)) {
|
|
803
|
-
await comment(job,
|
|
1188
|
+
await comment(job, MODEL_NOT_ALLOWED_COMMENTS.main);
|
|
804
1189
|
log("refused_model_not_allowed", { provider: job.provider ?? null, model: job.model ?? null });
|
|
805
1190
|
return { outcome: "policy", reason: "model-not-allowed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
806
1191
|
}
|
|
@@ -809,12 +1194,15 @@ export async function runJob(job, deps) {
|
|
|
809
1194
|
// the container started; on any other api the worker is stricter than the runner, by decision. Refused here, free. The log names the listed model, the operator's own configuration; the
|
|
810
1195
|
// comment names none.
|
|
811
1196
|
if (known?.fallbackUnlisted) {
|
|
812
|
-
await comment(job, "
|
|
1197
|
+
await comment(job, MODEL_NOT_ALLOWED_COMMENTS["fallback-unlisted"]);
|
|
813
1198
|
log("refused_model_not_allowed", { provider: known.fallbackUnlisted.provider ?? null, model: known.fallbackUnlisted.id ?? null, why: "fallback-unlisted" });
|
|
814
1199
|
return { outcome: "policy", reason: "model-not-allowed", why: "fallback-unlisted", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
815
1200
|
}
|
|
816
1201
|
}
|
|
817
1202
|
|
|
1203
|
+
if (job.watch?.project && job.watch.project !== pickupProject) return { outcome: "policy", reason: "watch-project-mismatch", budgetReserved: false };
|
|
1204
|
+
if (job.answerDraft?.project && job.answerDraft.project !== pickupProject) return { outcome: "policy", reason: "answer-draft-config-changed", budgetReserved: false };
|
|
1205
|
+
|
|
818
1206
|
// Is there a credential to run this job with at all? FREE, determinate and I/O-light: a pure function
|
|
819
1207
|
// of the job's provider, the worker env, and (only when the env has no key) one small readFileSync of
|
|
820
1208
|
// pi's auth.json. So it goes here, with the other free gates, which is further than issue #310 asked
|
|
@@ -853,8 +1241,8 @@ export async function runJob(job, deps) {
|
|
|
853
1241
|
await comment(
|
|
854
1242
|
job,
|
|
855
1243
|
absent
|
|
856
|
-
? "
|
|
857
|
-
:
|
|
1244
|
+
? ENVELOPE_MISMATCH_COMMENTS["no-envelope"]
|
|
1245
|
+
: ENVELOPE_MISMATCH_COMMENTS.mismatch,
|
|
858
1246
|
);
|
|
859
1247
|
log("refused_envelope_mismatch", absent ? { envelope: "none" } : {});
|
|
860
1248
|
return { outcome: "policy", reason: ENVELOPE_MISMATCH_REASON, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
@@ -884,28 +1272,46 @@ export async function runJob(job, deps) {
|
|
|
884
1272
|
}
|
|
885
1273
|
}
|
|
886
1274
|
|
|
887
|
-
const
|
|
888
|
-
if (
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
1275
|
+
const credentialJobs = [job];
|
|
1276
|
+
if (job.watch && job.watchDecisionModel && job.watchDecisionModel.provider !== job.provider) credentialJobs.push({ ...job, provider: job.watchDecisionModel.provider, model: job.watchDecisionModel.id });
|
|
1277
|
+
if (job.questionGrouping && job.questionGroupModel && job.questionGroupModel.provider !== job.provider) credentialJobs.push({ ...job, provider: job.questionGroupModel.provider, model: job.questionGroupModel.id });
|
|
1278
|
+
if (job.answerDraft && job.answerDecisionModel && job.answerDecisionModel.provider !== job.provider) credentialJobs.push({ ...job, provider: job.answerDecisionModel.provider, model: job.answerDecisionModel.id });
|
|
1279
|
+
for (const credentialJob of credentialJobs) {
|
|
1280
|
+
const credential = await checkProviderCredential(credentialJob, { modelEndpoints });
|
|
1281
|
+
if (credential.unavailable) {
|
|
1282
|
+
// Issue #503: the overlay models.json could not be read at this pickup for a transient reason, so the gate has no
|
|
1283
|
+
// verdict for a provider that may be keyless. Retried, never refused: a refusal is permanent and public, and the
|
|
1284
|
+
// next attempt may read the file. The code is a fixed errno token, never file text.
|
|
1285
|
+
log("provider_credential_unavailable", { provider: credentialJob.provider ?? null, reason: credential.unavailable });
|
|
1286
|
+
throw new InfraRetry("the provider credential could not be decided", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
1287
|
+
}
|
|
1288
|
+
if (!credential.ok) {
|
|
1289
|
+
// The PROVIDER is named and the message is not. The provider is operator-authored config, already
|
|
1290
|
+
// on the record and the mirror, and named freely by the sibling refusals (`backend-unblessed` names
|
|
1291
|
+
// the backend, `job-image-missing` names the image), so withholding it would tell the operator less
|
|
1292
|
+
// than is safe. The message is withheld from BOTH surfaces: `credentialFromPiAuth`'s refusals carry
|
|
1293
|
+
// `auth.json`'s absolute path, which is an OS account name, and the scoped-budget refusal below
|
|
1294
|
+
// keeps a host path out of its own log line citing no-pii-in-logs. `doctor` prints the variable and
|
|
1295
|
+
// the path, on the operator's terminal, which is where that belongs.
|
|
1296
|
+
await comment(job, `Refused: this deployment has no usable credential for the "${credentialJob.provider}" provider, so no container was started and nothing was spent. Ask the operator to run \`pi-dispatch doctor\`. Not run.`);
|
|
1297
|
+
log("refused_provider_unconfigured", { provider: credentialJob.provider ?? null });
|
|
1298
|
+
return { outcome: "policy", reason: "provider-unconfigured", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1299
|
+
}
|
|
906
1300
|
}
|
|
907
1301
|
|
|
908
1302
|
|
|
1303
|
+
// Access-only handoff is prepared after local determinate gates and before any paid work.
|
|
1304
|
+
let subscriptionAuth = [];
|
|
1305
|
+
if (prepareSubscriptionAuth) {
|
|
1306
|
+
const preparedAuth = await prepareSubscriptionAuth(job);
|
|
1307
|
+
if (!preparedAuth.ok) {
|
|
1308
|
+
if (preparedAuth.wait) throw new SubscriptionWait(preparedAuth);
|
|
1309
|
+
if (preparedAuth.unavailable) throw new InfraRetry("subscription authentication unavailable", { reason: "container-never-started", budgetReserved: false });
|
|
1310
|
+
return { outcome: "policy", reason: preparedAuth.reason ?? "subscription-quota", exitCode: null, budgetReserved: false };
|
|
1311
|
+
}
|
|
1312
|
+
subscriptionAuth = preparedAuth.snapshots;
|
|
1313
|
+
}
|
|
1314
|
+
|
|
909
1315
|
// REQ-EGRESS-ALLOWLIST. The egress policy this deployment claims must be able to serve this job
|
|
910
1316
|
// BEFORE the job costs anything. It is one `docker inspect` when the policy is armed and ZERO spawns
|
|
911
1317
|
// when it is not, so a deployment without one pays nothing at all.
|
|
@@ -990,13 +1396,39 @@ export async function runJob(job, deps) {
|
|
|
990
1396
|
// mark beside its transcript, and it is free to read, so it is asked here, before the mint and before any spend.
|
|
991
1397
|
// Without it the reply is ordinary conversation on an issue: refused, silently (no comment, since the person was
|
|
992
1398
|
// talking to people), and the stale marker is cleared so the next comment is not routed here again.
|
|
1399
|
+
if (job.fork !== undefined) {
|
|
1400
|
+
try { validateFork(job.fork); } catch { return await forkRefusal("fork-request-invalid"); }
|
|
1401
|
+
if (job.kind === "local" || job.target?.type !== "issue" || job.command || job.replica !== undefined || job.replicas !== undefined) return await forkRefusal("fork-target-unsupported");
|
|
1402
|
+
const venue = resolveBackendName(job, blessedBackends[0]);
|
|
1403
|
+
if (venue === "openshell") return await forkRefusal("openshell-fork-unsupported");
|
|
1404
|
+
if (job.awaitInput === true) return await forkRefusal("fork-await-input-unsupported");
|
|
1405
|
+
if (!isRecovery && !forkStore) return await forkRefusal("fork-source-store-unset");
|
|
1406
|
+
const policy = await checkForkPolicy(job);
|
|
1407
|
+
if (policy?.ok !== true) return await forkRefusal(policy?.refused ?? "fork-policy-unavailable");
|
|
1408
|
+
const expected = job.forkReserveMicros;
|
|
1409
|
+
if (expected !== null && (!Number.isSafeInteger(expected) || expected < 0)) return await forkRefusal("fork-request-invalid");
|
|
1410
|
+
if (expected !== null && !zeroRatedFor(job, { modelEndpoints, builtinModel }).zeroRated && (expected === 0 || !Number.isSafeInteger(job.maxCostMicros) || job.maxCostMicros > expected)) return await forkRefusal("fork-reservation-changed");
|
|
1411
|
+
const sourceKey = sessionKeyFor({ ...job, fork: undefined });
|
|
1412
|
+
if (sourceKey !== job.fork.rootKey) return await forkRefusal("fork-source-target-mismatch");
|
|
1413
|
+
// A recovery continues THIS fork's checkpoint; selecting the parent's original point again loses its work.
|
|
1414
|
+
if (!isRecovery) {
|
|
1415
|
+
forkSource = await forkStore.readPoint({ runId: job.fork.fromJob, sourceDigest: job.fork.sourceDigest, entryId: job.fork.fromEntry, position: job.fork.position });
|
|
1416
|
+
if (forkSource?.unavailable) throw new InfraRetry("the fork source could not be read just now", { reason: "container-never-started", budgetReserved: false });
|
|
1417
|
+
if (!forkSource?.ok) return await forkRefusal(forkSource?.refused ?? "fork-source-unavailable");
|
|
1418
|
+
if (forkSource.manifest.sessionKey !== sourceKey) return await forkRefusal("fork-source-target-mismatch");
|
|
1419
|
+
if (forkSource.manifest.piVersion !== piVersion) return await forkRefusal("fork-source-pi-version-mismatch");
|
|
1420
|
+
if (forkSource.manifest.venue !== venue) return await forkRefusal("fork-source-venue-mismatch");
|
|
1421
|
+
if (hostName && forkSource.manifest.host !== normalizeHost(hostName)) return await forkRefusal("fork-source-host-mismatch");
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
if (job.thinkingLevel !== undefined && (!FORK_THINKING_LEVELS.includes(job.thinkingLevel) || !(await checkThinking(job)))) return await forkRefusal("thinking-level-unsupported");
|
|
993
1425
|
if (job.parkedReply === true && !isParked(job)) {
|
|
994
1426
|
log("refused_not_parked", { kind: job.kind ?? null });
|
|
995
1427
|
await parkedLabel(job, false);
|
|
996
1428
|
return { outcome: "policy", reason: "not-parked", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
997
1429
|
}
|
|
998
1430
|
if (job.resume === true && !sessionsDir) {
|
|
999
|
-
await comment(job, "
|
|
1431
|
+
await comment(job, REFUSAL_COMMENTS["sessions-dir-unset"]);
|
|
1000
1432
|
// The variable NAME, never a value: there is no path to print here (its absence IS the refusal),
|
|
1001
1433
|
// and the store's path is the one setting SECURITY.md calls a PII store. `kind` is host-assigned,
|
|
1002
1434
|
// the same PII class as the `repo` on the branch refusal below.
|
|
@@ -1007,6 +1439,144 @@ export async function runJob(job, deps) {
|
|
|
1007
1439
|
return { outcome: "policy", reason: "sessions-dir-unset", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1008
1440
|
}
|
|
1009
1441
|
|
|
1442
|
+
// Issue #645, durable runs: `run.recover`'s free refusals, here beside run.resume's and for its reasons. Free (a
|
|
1443
|
+
// value in hand, a Valkey GET, a manifest read off this host's disk), determinate, and strictly before the mint,
|
|
1444
|
+
// the clone, the token-cap read and every reserve (CONST-BUDGET-BEFORE-TOKENS).
|
|
1445
|
+
//
|
|
1446
|
+
// The store's absence is run.resume's one fail-CLOSED case again: without a checkpoint store nothing is kept, a
|
|
1447
|
+
// lost run is retried from scratch, and the job still looks protected. A wiring that has a store but not the
|
|
1448
|
+
// rest of the machinery (a ledger, the pointer read, the enqueuer) is a defect of this build, not of the operator's
|
|
1449
|
+
// settings, so it is a config refusal: refunded, `doctor`-pointed, never run half-armed.
|
|
1450
|
+
if (armed && !workspaceCheckpointer) {
|
|
1451
|
+
await comment(job, RECOVERY_COMMENTS["recover-store-unset"]);
|
|
1452
|
+
log("refused_checkpoints_dir_unset", { kind: job.kind ?? null });
|
|
1453
|
+
return { outcome: "policy", reason: "recover-store-unset", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1454
|
+
}
|
|
1455
|
+
if (armed && (!runBudgetLedger || typeof readCheckpointPointer !== "function" || typeof enqueueRecovery !== "function")) {
|
|
1456
|
+
throw configError("run.recover is armed, and this worker was built without the run budget ledger, the checkpoint pointer read or the recovery enqueuer");
|
|
1457
|
+
}
|
|
1458
|
+
if (isRecovery) {
|
|
1459
|
+
const n = durable.recovery;
|
|
1460
|
+
// SETTLED, first of all: the run (its base attempt or an earlier recovery) already wrote a completed or policy
|
|
1461
|
+
// record, so it reached its final outcome and there is nothing to continue. Silent, and cleans nothing (that
|
|
1462
|
+
// outcome cleaned up after itself). Asked BEFORE the pointer and the store, because those can disagree after a
|
|
1463
|
+
// cleanup that half failed (a store removal that failed beside a pointer delete that landed), and that
|
|
1464
|
+
// disagreement must never put a failure comment under a run that completed.
|
|
1465
|
+
// Only a record of THIS run settles it (issue #811). Ids come back (a lost queue, a hand redelivery) while records
|
|
1466
|
+
// stay for the retention window, so a `completed` record an older job left under the run's id would end this
|
|
1467
|
+
// recovery as superseded, silently, and the lost run would be neither continued nor reported. The recovery's data
|
|
1468
|
+
// carries the base run's creation time (`recoveryOfCreatedAt`, recovery-queue.mjs `recoveryDataOf`), and a record
|
|
1469
|
+
// whose time (`startedAt`, or `endedAt` with no start) is older than that, less `RECORD_CLOCK_SKEW_MS`, is
|
|
1470
|
+
// another job's and settles nothing. Only a time that PARSES is judged so: `whyNotThisJob` would take a record
|
|
1471
|
+
// with no readable time for another job's, but here that would run a run that may have completed, and paying
|
|
1472
|
+
// twice for a settled run is the worse mistake, so such a record still settles, as before. A recovery without
|
|
1473
|
+
// the field (queued by a build from before it) keeps the check without an age. Both sides are read, this host's
|
|
1474
|
+
// file and the fleet's copy, each held to the rule (`keep`), so a stale record on this host cannot hide the
|
|
1475
|
+
// settled one in the mirror; the answer is judged again, so a reader that ignores `keep` cannot settle the run
|
|
1476
|
+
// on a record that fails it. Every field read is in the record's stable core, so a newer release's record is
|
|
1477
|
+
// judged the same way.
|
|
1478
|
+
const since = Number.isSafeInteger(job.recoveryOfCreatedAt) && job.recoveryOfCreatedAt >= 0 ? job.recoveryOfCreatedAt : null;
|
|
1479
|
+
const settlesRun = (rec, id) => {
|
|
1480
|
+
if (!rec || typeof rec !== "object" || rec.jobId !== id || (rec.outcome !== "completed" && rec.outcome !== "policy")) return false;
|
|
1481
|
+
if (since === null) return true;
|
|
1482
|
+
const at = Date.parse(rec.startedAt ?? rec.endedAt ?? "");
|
|
1483
|
+
return !(Number.isFinite(at) && at < since - RECORD_CLOCK_SKEW_MS);
|
|
1484
|
+
};
|
|
1485
|
+
if (typeof readRecord === "function") {
|
|
1486
|
+
const ids = [runId, ...Array.from({ length: Number.isSafeInteger(n) && n > 1 ? Math.min(n, RECOVERY_CAP + 1) - 1 : 0 }, (_, k) => `${runId}.r${k + 1}`)];
|
|
1487
|
+
for (const id of ids) {
|
|
1488
|
+
let rec = null;
|
|
1489
|
+
try {
|
|
1490
|
+
rec = await readRecord(id, { keep: (record) => settlesRun(record, id) });
|
|
1491
|
+
} catch {
|
|
1492
|
+
rec = null;
|
|
1493
|
+
}
|
|
1494
|
+
if (settlesRun(rec, id)) {
|
|
1495
|
+
log("recovery_superseded", { n: Number.isSafeInteger(n) ? n : null, by: "record" });
|
|
1496
|
+
return { outcome: "policy", reason: RECOVERY_SUPERSEDED, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
// THE CAP: a run continued twice is not continued a third time, whatever its checkpoint holds.
|
|
1501
|
+
if (!Number.isSafeInteger(n) || n < 1 || n > RECOVERY_CAP) {
|
|
1502
|
+
await comment(job, RECOVERY_COMMENTS["recovery-cap"]);
|
|
1503
|
+
log("refused_recovery_cap", { n: Number.isSafeInteger(n) ? n : null, cap: RECOVERY_CAP });
|
|
1504
|
+
return { outcome: "policy", reason: "recovery-cap", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1505
|
+
}
|
|
1506
|
+
// SUPERSEDED: the base run reached its final outcome after this recovery was queued (a completed or policy result
|
|
1507
|
+
// removes the pointer), so there is nothing left to continue. Silent: that outcome already said what it had to.
|
|
1508
|
+
// A read that THROWS is not an answer, so it is retried, never taken for "settled".
|
|
1509
|
+
let pointer;
|
|
1510
|
+
try {
|
|
1511
|
+
pointer = await readCheckpointPointer({ runId });
|
|
1512
|
+
} catch {
|
|
1513
|
+
throw new InfraRetry("the run's checkpoint pointer could not be read", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
1514
|
+
}
|
|
1515
|
+
// The pointer read answers null for "no pointer" AND for "Valkey did not answer in time" (run-checkpoint.mjs never
|
|
1516
|
+
// throws), so null alone must not decide: a silent refusal would drop a live run's continuation over a blip. A
|
|
1517
|
+
// run's final outcome removes its STORE ENTRY before its pointer (`endDurableRun`), so superseded is a missing
|
|
1518
|
+
// pointer AND a missing store. A missing pointer beside a store that is still there is ambiguous, and is
|
|
1519
|
+
// retried loudly rather than guessed either way (continuing could re-run a run that completed).
|
|
1520
|
+
if (pointer === null || pointer === undefined) {
|
|
1521
|
+
const still = await workspaceCheckpointer.readManifest(runId);
|
|
1522
|
+
if (still?.absent === true) {
|
|
1523
|
+
log("recovery_superseded", { n });
|
|
1524
|
+
return { outcome: "policy", reason: RECOVERY_SUPERSEDED, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1525
|
+
}
|
|
1526
|
+
// A store a newer pi-dispatch wrote is one this build cannot judge either way: waited on like the read below.
|
|
1527
|
+
if (still?.newer === true) throw new NewerReleaseHold("the run's checkpoint");
|
|
1528
|
+
log("recovery_pointer_unread", { n });
|
|
1529
|
+
throw new InfraRetry("the run's checkpoint pointer is missing while its checkpoint is still there", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
1530
|
+
}
|
|
1531
|
+
// THE BASE STILL RUNNING HERE: its container may still be alive and spending, so this recovery must neither
|
|
1532
|
+
// settle its hold as crashed nor start a second agent on the same branch. It waits, without an attempt
|
|
1533
|
+
// (`RecoveryWaitHold`), until that container is gone. Asked of this host only: the recovery is queued to the
|
|
1534
|
+
// host the checkpoint is on, which is the host the base ran on.
|
|
1535
|
+
// THIS job is in that list too (the processor admits a job before it runs it), under the id the recovery queue
|
|
1536
|
+
// gave it (`<runId>.r<n>`, recovery-queue.mjs), so its own entry is excluded: without that every recovery would
|
|
1537
|
+
// find itself running and wait for ever.
|
|
1538
|
+
const ownId = `${runId}.r${n}`;
|
|
1539
|
+
const live = typeof runningJobs?.list === "function" ? runningJobs.list() : [];
|
|
1540
|
+
if (live.some((e) => typeof e?.id === "string" && e.id !== ownId && (e.id === runId || e.id.startsWith(`${runId}.r`)))) {
|
|
1541
|
+
log("recovery_waits_for_base", { n, delayMs: RECOVERY_WAIT_MS });
|
|
1542
|
+
throw new RecoveryWaitHold("the run this recovery continues is still running on this host", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false, waitMs: RECOVERY_WAIT_MS });
|
|
1543
|
+
}
|
|
1544
|
+
// SALVAGE, before the manifest is read: a crashed attempt whose job directory is still on this host (the worker
|
|
1545
|
+
// died, the directory did not) gets one last checkpoint taken from it (`kind: "salvage"`), so the recovery
|
|
1546
|
+
// continues from where the run really got to rather than from its last live checkpoint. The wiring's
|
|
1547
|
+
// (start.mjs); it never throws by contract, and is guarded anyway: a failed salvage leaves the last committed
|
|
1548
|
+
// checkpoint, which is still a correct place to continue from.
|
|
1549
|
+
if (typeof salvageCheckpoint === "function") {
|
|
1550
|
+
const salvaged = await Promise.resolve()
|
|
1551
|
+
.then(() => salvageCheckpoint({ runId, pointer, attempt: pointer.attempt }))
|
|
1552
|
+
.catch(() => ({ skipped: "error" }));
|
|
1553
|
+
log("recovery_salvage", Number.isSafeInteger(salvaged?.seq) ? { seq: salvaged.seq } : { skipped: typeof salvaged?.skipped === "string" ? salvaged.skipped.slice(0, 40) : "unknown" });
|
|
1554
|
+
}
|
|
1555
|
+
// THE CHECKPOINT, judged before anything spends: valid (readManifest hashes the transcript against its
|
|
1556
|
+
// manifest), taken on THIS host, by THIS image's pi version, in THIS job's venue. A transcript from another pi
|
|
1557
|
+
// may hold tool-call arguments the current schema refuses (the run.resume rule), and a checkpoint from another
|
|
1558
|
+
// venue was quiesced under another runtime's pause.
|
|
1559
|
+
const read = await workspaceCheckpointer.readManifest(runId);
|
|
1560
|
+
// Issue #761: a checkpoint a newer pi-dispatch wrote is not an absent one. Refused here, the run's final outcome
|
|
1561
|
+
// would remove its store and its pointer (`endDurableRun`) and a host upgraded again could never continue it; so
|
|
1562
|
+
// the recovery WAITS (`newer-release`, index.mjs) for a worker that reads the checkpoint, without spending its one
|
|
1563
|
+
// attempt (the never-started conversion in `runJobInner` takes only `container-never-started`, which this is
|
|
1564
|
+
// not), with no comment and no record, and with the store, the pointer and the job directory left in place. What
|
|
1565
|
+
// bounds the wait is the host-lost sweep's hard maximum (recovery-queue.mjs, `RECOVERY_MAX_WAIT_MS`).
|
|
1566
|
+
if (read?.newer === true) throw new NewerReleaseHold("the run's checkpoint");
|
|
1567
|
+
const m = read?.ok === true ? read.manifest : null;
|
|
1568
|
+
const venue = resolveBackendName(job, blessedBackends[0]);
|
|
1569
|
+
const why = m === null ? (read?.absent ? "absent" : "invalid") : typeof hostName === "string" && hostName !== "" && m.host !== normalizeHost(hostName) ? "host" : piVersion === null || m.piVersion !== piVersion ? "pi-version" : m.venue !== venue ? "venue" : (m.provider !== job.provider || m.model !== job.model) ? "model" : null;
|
|
1570
|
+
if (why !== null) {
|
|
1571
|
+
await comment(job, RECOVERY_COMMENTS["recovery-checkpoint-absent"]);
|
|
1572
|
+
// `why` is a fixed token, and `invalid`'s detail is the store's own fixed reason (`field:seq`, `transcript-mismatch`).
|
|
1573
|
+
log("refused_recovery_checkpoint_absent", { n, why, ...(why === "invalid" && typeof read?.reason === "string" ? { detail: read.reason.slice(0, 40) } : {}) });
|
|
1574
|
+
return { outcome: "policy", reason: "recovery-checkpoint-absent", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1575
|
+
}
|
|
1576
|
+
manifest = m;
|
|
1577
|
+
transcriptPath = read.transcriptPath;
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1010
1580
|
// REQ-PER-TRIGGER-SKILLS. A trigger that named a skills directory the worker cannot see would run
|
|
1011
1581
|
// its flow WITHOUT the skills it was written against, produce a plausible report, and exit 0. Free
|
|
1012
1582
|
// and determinate -- one lstat, no credential needed to know the answer -- so it belongs among the
|
|
@@ -1016,7 +1586,7 @@ export async function runJob(job, deps) {
|
|
|
1016
1586
|
// image blocks every job on this host, an unset sessions dir blocks every armed trigger, a bad
|
|
1017
1587
|
// skillsDir blocks one trigger.
|
|
1018
1588
|
if (job.skillsDir && !isReadableDir(job.skillsDir)) {
|
|
1019
|
-
await comment(job, "
|
|
1589
|
+
await comment(job, REFUSAL_COMMENTS["skills-dir-missing"]);
|
|
1020
1590
|
// The FIELD name, never its value. `comment` posts publicly on the issue, so a host path here
|
|
1021
1591
|
// would publish the operator's filesystem layout to anyone reading the thread; the log line is
|
|
1022
1592
|
// the same restraint refused_sessions_dir_unset keeps.
|
|
@@ -1024,6 +1594,22 @@ export async function runJob(job, deps) {
|
|
|
1024
1594
|
return { outcome: "policy", reason: "skills-dir-missing", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1025
1595
|
}
|
|
1026
1596
|
|
|
1597
|
+
// Issue #796: a canary's refusals, free and determinate like the one above, so before the mint and everything after it. The registry read is free too; one that cannot
|
|
1598
|
+
// be asked is infrastructure and retries (CONST-RETRY-INFRA-ONLY), since the same job can succeed. The ladder's narrowest gate, so it asks the network last.
|
|
1599
|
+
let canaryPlan = null;
|
|
1600
|
+
if (job.packageCanary !== undefined) {
|
|
1601
|
+
if (!deps.packageCanary) return { outcome: "policy", reason: "package-canary-invalid", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
|
|
1602
|
+
try {
|
|
1603
|
+
canaryPlan = await deps.packageCanary.check(job);
|
|
1604
|
+
} catch {
|
|
1605
|
+
throw new InfraRetry("the registry could not be asked about the canary's version", { reason: "container-never-started", budgetReserved: false, provider: job.provider ?? null, model: job.model ?? null });
|
|
1606
|
+
}
|
|
1607
|
+
if (canaryPlan.refused) {
|
|
1608
|
+
log("refused_package_canary", { reason: canaryPlan.refused });
|
|
1609
|
+
return { outcome: "policy", reason: canaryPlan.refused, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1027
1613
|
// REQ-MCP-OPERATOR-SERVERS. A trigger that named MCP servers its job cannot use would run its flow without them,
|
|
1028
1614
|
// or hand a server the job's own credentials, or open a connection the egress proxy refuses at its first call:
|
|
1029
1615
|
// each a plausible report and a clean exit. One read of the overlay's mcp.json, no credential and no network, so
|
|
@@ -1038,11 +1624,84 @@ export async function runJob(job, deps) {
|
|
|
1038
1624
|
// The server NAME is the operator's own configuration, the same class as the image ref on the image refusal.
|
|
1039
1625
|
// `why` can name an internal host or a variable (never a value, a url's path or its query), so it goes to the
|
|
1040
1626
|
// worker's log and the public comment says only which rule refused.
|
|
1041
|
-
await comment(job,
|
|
1627
|
+
await comment(job, mcpRefusalComment(mcpChecked.refused, mcpChecked.name));
|
|
1042
1628
|
log("refused_mcp", { kind: job.kind ?? null, reason: mcpChecked.refused, server: mcpChecked.name ?? null, why: mcpChecked.why });
|
|
1043
1629
|
return { outcome: "policy", reason: mcpChecked.refused, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1044
1630
|
}
|
|
1045
1631
|
|
|
1632
|
+
// PROJECT SECRET BUNDLES, the free half (issue #704, REQ-TRIGGER-SECRETS, DES-PROJECT-SECRET-BUNDLES). WHICH jobs
|
|
1633
|
+
// get their project's bundle is decided here, before the mint, the clone and every reserve, from one cached read
|
|
1634
|
+
// of the triggers file and the host's declarations: no spawn, no network (CONST-BUDGET-BEFORE-TOKENS). The read
|
|
1635
|
+
// itself comes after the run.secrets resolver and, like it, before the mint, except for a pull request that needs
|
|
1636
|
+
// the fork check: that one needs the forge and the job's token, so its read follows the mint (below).
|
|
1637
|
+
//
|
|
1638
|
+
// TRIGGER-MADE JOBS ONLY, and the predicate is the portfolio gate's: a chained `/outbox` child and every manual
|
|
1639
|
+
// run (`pi-dispatch run`, `/dispatch run`, `dispatch_run`) hold none, each recorded as `bundleSkipped`. The
|
|
1640
|
+
// operator's grant was to the triggers they reviewed, not to what a job queues next or what a model asks for
|
|
1641
|
+
// (outbox.mjs keeps `run.secrets` off children for the same reason). Chained is asked first: a child may carry
|
|
1642
|
+
// its parent's trigger context, and it must never read as manual. A trigger's `run.secretBundle: false` is
|
|
1643
|
+
// `opted-out`, read from the live file rather than the job data (triggers-file.mjs says why).
|
|
1644
|
+
//
|
|
1645
|
+
// Every refusal RETURNS (CONST-RETRY-INFRA-ONLY): an undeclared or doubly declared profile, a resumed job
|
|
1646
|
+
// (`run.resume` would replay a transcript that may hold an echoed value into every later job on its key, the
|
|
1647
|
+
// rule that refuses `run.secrets` beside it at load, which cannot be applied at load here because the file
|
|
1648
|
+
// cannot know a job's project), and a trigger the file no longer holds. A file that has never been read and
|
|
1649
|
+
// validated in this process RETRIES a job with a project (the finder answers from its last good read otherwise),
|
|
1650
|
+
// and a profile this host lacks is DEFERRED on a multi-host worker's shared queue, the never-fits gate's rule,
|
|
1651
|
+
// since another host may serve it. A venue's refusal of the bundle itself comes with the read (`readBundle`).
|
|
1652
|
+
const bundleChained = job.parentJobId !== undefined || job.chainDepth !== undefined;
|
|
1653
|
+
const bundleManual = job.trigger === undefined || job.trigger === null;
|
|
1654
|
+
let bundleBinding = null;
|
|
1655
|
+
const bundleFound = await Promise.resolve()
|
|
1656
|
+
.then(() => findSecretBundle(job, { project: pickupProject }))
|
|
1657
|
+
.catch(() => ({ unreadable: "read-error", transient: true }));
|
|
1658
|
+
if (bundleFound?.unreadable !== undefined) {
|
|
1659
|
+
// Never a good read in this process, and the job has a project: which bundle it would hold is unknown. RETRIED,
|
|
1660
|
+
// never refused (a refused webhook delivery is gone, and the fix is the operator's next save) and never run
|
|
1661
|
+
// without (the silent no-op). A chained or manual job could hold none whatever the file says, so it runs.
|
|
1662
|
+
if (!bundleChained && !bundleManual) {
|
|
1663
|
+
// The errno or our own word (`invalid`), never a path or file text.
|
|
1664
|
+
log("secret_bundles_unreadable", { kind: job.kind ?? null, why: String(bundleFound.unreadable).slice(0, 40) });
|
|
1665
|
+
throw new InfraRetry("the triggers file's secretBundles could not be read, and has not been in this process", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
1666
|
+
}
|
|
1667
|
+
} else if (bundleFound && bundleFound.none !== true) {
|
|
1668
|
+
if (bundleChained) bundleFacts.bundleSkipped = "chained";
|
|
1669
|
+
else if (bundleManual) bundleFacts.bundleSkipped = "manual";
|
|
1670
|
+
else if (bundleFound.triggerChanged === true) {
|
|
1671
|
+
await comment(job, BUNDLE_COMMENTS["secret-bundle-trigger-changed"]);
|
|
1672
|
+
log("refused_secret_bundle_trigger_changed", { kind: job.kind ?? null, bundle: bundleFound.id });
|
|
1673
|
+
return { outcome: "policy", reason: "secret-bundle-trigger-changed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1674
|
+
} else if (bundleFound.optedOut === true) bundleFacts.bundleSkipped = "opted-out";
|
|
1675
|
+
else {
|
|
1676
|
+
if (job.resume === true) {
|
|
1677
|
+
await comment(job, BUNDLE_COMMENTS["secret-bundle-resume"]);
|
|
1678
|
+
log("refused_secret_bundle_resume", { kind: job.kind ?? null, bundle: bundleFound.id });
|
|
1679
|
+
return { outcome: "policy", reason: "secret-bundle-resume", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1680
|
+
}
|
|
1681
|
+
const looked = secretBundles.lookup(bundleFound.bundle.profile);
|
|
1682
|
+
if (looked?.ok !== true && bundleProfileDeferrable === true) {
|
|
1683
|
+
// A multi-host worker's SHARED queue: another host may declare the profile, and routing does not steer a
|
|
1684
|
+
// bundled job there (DES-PROJECT-SECRET-BUNDLES says why). So this host does not refuse a job it merely
|
|
1685
|
+
// cannot serve: it hands it back, the never-fits gate's deferral, without spending an attempt.
|
|
1686
|
+
log("secret_bundle_profile_deferred", { kind: job.kind ?? null, bundle: bundleFound.id, profile: bundleFound.bundle.profile, ambiguous: looked?.ambiguous !== undefined });
|
|
1687
|
+
throw new SecretBundleProfileHold("this host cannot serve the job's secret bundle profile; another host may", { reason: looked?.ambiguous !== undefined ? "secret-bundle-profile-ambiguous" : "secret-bundle-profile-unknown", provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false });
|
|
1688
|
+
}
|
|
1689
|
+
if (looked?.ok !== true) {
|
|
1690
|
+
// The profile LABEL in the log only, and the bundle id: both operator config. The comment names neither,
|
|
1691
|
+
// the run.secrets refusals' restraint (the declared names would publish the operator's vault topology).
|
|
1692
|
+
const reason = looked?.ambiguous !== undefined ? "secret-bundle-profile-ambiguous" : "secret-bundle-profile-unknown";
|
|
1693
|
+
await comment(job, BUNDLE_COMMENTS[reason]);
|
|
1694
|
+
log(reason === "secret-bundle-profile-ambiguous" ? "refused_secret_bundle_profile_ambiguous" : "refused_secret_bundle_profile_unknown", { kind: job.kind ?? null, bundle: bundleFound.id, profile: bundleFound.bundle.profile });
|
|
1695
|
+
return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1696
|
+
}
|
|
1697
|
+
// Issue #704 part 3: no venue refusal here any more. Part 2 refused a bundled job on both OpenShell venues at
|
|
1698
|
+
// this point, which was AHEAD of the fork check, so a fork's job that would have run without the bundle was
|
|
1699
|
+
// refused for it. The venue's own judgement of the bundle (`bundleGate`) now runs inside the read below,
|
|
1700
|
+
// which a fork's job never reaches.
|
|
1701
|
+
bundleBinding = { id: bundleFound.id, ...bundleFound.bundle };
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1046
1705
|
// REQ-TRIGGER-SECRETS. A trigger that named secret references the worker cannot resolve would run its
|
|
1047
1706
|
// flow with those variables UNSET, get a 401 from whatever it was meant to reach, write a plausible
|
|
1048
1707
|
// report about why the integration is down, and exit 0.
|
|
@@ -1070,10 +1729,11 @@ export async function runJob(job, deps) {
|
|
|
1070
1729
|
// carry `secrets` the current set would refuse, and buildContainerEnv would write every one of them.
|
|
1071
1730
|
// The same set the loader uses, imported, so the two cannot drift, and checked before the resolver so
|
|
1072
1731
|
// no wiring of it can skip the check.
|
|
1073
|
-
const
|
|
1074
|
-
const
|
|
1732
|
+
const secondaryCredentialNames = new Set(job.answerDraft && job.answerDecisionModel ? providerKeyCandidates(job.answerDecisionModel.provider) : job.questionGrouping && job.questionGroupModel ? providerKeyCandidates(job.questionGroupModel.provider) : job.watch && job.watchDecisionModel ? providerKeyCandidates(job.watchDecisionModel.provider) : []);
|
|
1733
|
+
const loadReserved = secretsArmed(job) ? Object.keys(job.secrets ?? {}).find((name) => RESERVED_ENV_NAMES.has(name) || secondaryCredentialNames.has(name)) : undefined;
|
|
1734
|
+
const resolved = loadReserved !== undefined ? { reserved: loadReserved, atLoad: RESERVED_ENV_NAMES.has(loadReserved) } : secretsArmed(job) ? await resolveSecrets(job) : { ok: true, secrets: {} };
|
|
1075
1735
|
if (resolved.profileUnknown) {
|
|
1076
|
-
await comment(job, "
|
|
1736
|
+
await comment(job, REFUSAL_COMMENTS["secret-profile-unknown"]);
|
|
1077
1737
|
// The operator's own profile LABEL, and never a path, a reference, or a byte the resolver printed.
|
|
1078
1738
|
// `comment` posts publicly on the issue: a resolver path there publishes the operator's filesystem
|
|
1079
1739
|
// layout, and the DECLARED profile names would publish their vault topology -- which is why the
|
|
@@ -1090,7 +1750,7 @@ export async function runJob(job, deps) {
|
|
|
1090
1750
|
// anywhere -- redirect a profile the operator wrote in .env. This project already refuses ambiguity
|
|
1091
1751
|
// rather than resolving it (PI_EGRESS: "a typo must never leave you believing you have a policy you
|
|
1092
1752
|
// do not"), and an operator who sees this fixes it in seconds.
|
|
1093
|
-
await comment(job, "
|
|
1753
|
+
await comment(job, REFUSAL_COMMENTS["secret-profile-ambiguous"]);
|
|
1094
1754
|
log("refused_secret_profile_ambiguous", { kind: job.kind ?? null, profile: resolved.ambiguous });
|
|
1095
1755
|
return { outcome: "policy", reason: "secret-profile-ambiguous", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1096
1756
|
}
|
|
@@ -1147,7 +1807,136 @@ export async function runJob(job, deps) {
|
|
|
1147
1807
|
log("secret_resolver_unreachable", { kind: job.kind ?? null, name: resolved.unreachable, failure: resolved.failure ?? null, code: resolved.code ?? null, stderrBytes: resolved.stderrBytes ?? 0 });
|
|
1148
1808
|
throw new InfraRetry(`secret resolver could not answer for ${resolved.unreachable}`, { reason: "secret-resolver-unreachable", provider: job.provider ?? null, model: job.model ?? null });
|
|
1149
1809
|
}
|
|
1150
|
-
|
|
1810
|
+
// `let`: a project bundle's variables join it after the mint (below), under the trigger's own values.
|
|
1811
|
+
let secrets = resolved.secrets ?? {};
|
|
1812
|
+
// Issue #704 part 3: the bundle's `{ endpoints, plain }` split, for the venue (null: no bundle reached this job).
|
|
1813
|
+
let bundleKeys = null;
|
|
1814
|
+
|
|
1815
|
+
// Issue #704 part 3: the venue's answer, the same handling for both of its asks below. A refusal RETURNS with the
|
|
1816
|
+
// venue's fixed sentence (its log names the variable and host, never the comment); a source it could not read
|
|
1817
|
+
// RETRIES, as the observation preflight's does.
|
|
1818
|
+
// The hosts this job's OWN credentials are sent to beyond the forge's (issue #704 part 3, review round 1, H2), which a
|
|
1819
|
+
// bundle endpoint may not claim on a venue that binds one provider per host: the composed `baseUrl` of every model
|
|
1820
|
+
// the job may call (the overlay's models.json over pi's catalog, `outputCapView`, the composition the output cap and
|
|
1821
|
+
// the zero-rated check read), every declared model endpoint's host (keyed or not: the simple rule), and every
|
|
1822
|
+
// HTTP MCP server's host (its headers may carry a run.secrets value). A model whose baseUrl cannot be composed adds
|
|
1823
|
+
// nothing, and is said so in DES-OPENSHELL-VENUE.
|
|
1824
|
+
const credentialHosts = () => {
|
|
1825
|
+
const hosts = new Set();
|
|
1826
|
+
const add = (url) => {
|
|
1827
|
+
try {
|
|
1828
|
+
if (typeof url === "string" && url !== "") hosts.add(new URL(url).hostname.toLowerCase());
|
|
1829
|
+
} catch {}
|
|
1830
|
+
};
|
|
1831
|
+
for (const ref of callableModelRefs(job)) {
|
|
1832
|
+
if (ref === null) continue;
|
|
1833
|
+
try {
|
|
1834
|
+
add(outputCapView({ models: modelEndpoints?.models ?? null, provider: ref.provider, modelId: ref.id, builtinModel })?.baseUrl);
|
|
1835
|
+
} catch {}
|
|
1836
|
+
}
|
|
1837
|
+
for (const ep of modelEndpoints?.endpoints ?? []) if (typeof ep?.host === "string") hosts.add(ep.host.toLowerCase());
|
|
1838
|
+
for (const server of Object.values(mcpChecked?.document?.servers ?? {})) add(server?.url);
|
|
1839
|
+
return [...hosts].sort();
|
|
1840
|
+
};
|
|
1841
|
+
const askBundleGate = async (ask) => {
|
|
1842
|
+
const judged = await bundleGate(job, { ...ask, credentialHosts: credentialHosts(), capabilities: img.capabilities ?? [] });
|
|
1843
|
+
if (judged?.venueRefused && typeof judged.venueRefused.reason === "string") {
|
|
1844
|
+
const { reason, comment: text } = judged.venueRefused;
|
|
1845
|
+
await comment(job, typeof text === "string" && text !== "" ? text : VENUE_BUNDLE_REFUSED_COMMENT);
|
|
1846
|
+
log("refused_venue", { reason, bundle: bundleBinding.id });
|
|
1847
|
+
return { refused: { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false } }; // return => not retried
|
|
1848
|
+
}
|
|
1849
|
+
if (judged?.unavailable) throw new InfraRetry("the job's venue could not judge its secret bundle", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
1850
|
+
return { judged };
|
|
1851
|
+
};
|
|
1852
|
+
// PROJECT SECRET BUNDLES, the read (issue #704). ONE resolver call, then the strict parse, the reserved names and
|
|
1853
|
+
// the merge. BEFORE the mint for every job that needs no fork check (CONST-BUDGET-BEFORE-TOKENS: its determinate
|
|
1854
|
+
// refusals spend nothing, not even a token), and right after the mint for a pull request whose head must be asked
|
|
1855
|
+
// of the forge first; either way before branch protection, the clone, the token-cap read and every reserve.
|
|
1856
|
+
// Returns null, or the result to return.
|
|
1857
|
+
const readBundle = async () => {
|
|
1858
|
+
if (bundleBinding === null) return null;
|
|
1859
|
+
// Issue #704 part 3: the hosts first, before the resolver runs, so a file naming a host this job's venue cannot
|
|
1860
|
+
// bind costs no read of the operator's manager. Here rather than among the free gates above (where part 2
|
|
1861
|
+
// refused every OpenShell bundle), because only here is the fork check behind it: a fork's job runs without
|
|
1862
|
+
// the bundle, so nothing about the bundle may refuse it.
|
|
1863
|
+
const endpoints = bundleBinding.endpoints ?? {};
|
|
1864
|
+
if (Object.keys(endpoints).length > 0) {
|
|
1865
|
+
const pre = await askBundleGate({ endpoints, vars: null });
|
|
1866
|
+
if (pre.refused) return pre.refused;
|
|
1867
|
+
}
|
|
1868
|
+
const bundleRead = await secretBundles.resolve(job, bundleBinding);
|
|
1869
|
+
if (bundleRead?.ok !== true) {
|
|
1870
|
+
if (bundleRead?.unreachable) {
|
|
1871
|
+
// INDETERMINATE (exit 1, an unknown exit, a spawn fault, a timeout): retried, the run.secrets rule. A
|
|
1872
|
+
// rate-limited manager read lands here, which is why its documented resolvers exit 1 for one.
|
|
1873
|
+
log("secret_bundle_resolver_unreachable", { kind: job.kind ?? null, bundle: bundleBinding.id, failure: bundleRead.failure ?? null, code: bundleRead.code ?? null, stderrBytes: bundleRead.stderrBytes ?? 0 });
|
|
1874
|
+
throw new InfraRetry(`the secret bundle's resolver could not answer for bundle ${bundleBinding.id}`, { reason: "secret-resolver-unreachable", provider: job.provider ?? null, model: job.model ?? null });
|
|
1875
|
+
}
|
|
1876
|
+
if (bundleRead?.reserved !== undefined) {
|
|
1877
|
+
// The variable NAME, in the log, so the operator can rename it in their manager. Not in the public
|
|
1878
|
+
// comment: a bundle's names are the contents of the operator's manager, not of the reviewed file.
|
|
1879
|
+
await comment(job, BUNDLE_COMMENTS["secret-name-reserved"]);
|
|
1880
|
+
log("refused_secret_bundle_name_reserved", { kind: job.kind ?? null, bundle: bundleBinding.id, name: bundleRead.reserved });
|
|
1881
|
+
return { outcome: "policy", reason: "secret-name-reserved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1882
|
+
}
|
|
1883
|
+
if (bundleRead?.profileUnknown !== undefined || bundleRead?.ambiguous !== undefined) {
|
|
1884
|
+
// Only if the declarations changed between the lookup above and this read: refused as there.
|
|
1885
|
+
const reason = bundleRead.ambiguous !== undefined ? "secret-bundle-profile-ambiguous" : "secret-bundle-profile-unknown";
|
|
1886
|
+
await comment(job, BUNDLE_COMMENTS[reason]);
|
|
1887
|
+
log(reason === "secret-bundle-profile-ambiguous" ? "refused_secret_bundle_profile_ambiguous" : "refused_secret_bundle_profile_unknown", { kind: job.kind ?? null, bundle: bundleBinding.id, profile: bundleBinding.profile });
|
|
1888
|
+
return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1889
|
+
}
|
|
1890
|
+
// DETERMINATE: exit 2, an empty, oversized or malformed output, a bad name or value. `name` and `line`
|
|
1891
|
+
// ride the log only, for the reason above; the comment says which rule, in our words.
|
|
1892
|
+
const why = BUNDLE_FAILURES[bundleRead?.failure] ?? "did not return a usable environment";
|
|
1893
|
+
await comment(job, `Refused: this job's project has a secret bundle, and its resolver ${why}. The job would have started without the project's environment, and an agent that gets a 401 writes a plausible report and exits 0. Run the bundle's resolver by hand against its reference to see why: this comment carries neither the reference, nor the resolver's path, nor a byte of what it printed. Not run.`);
|
|
1894
|
+
log("refused_secret_bundle_unresolved", { kind: job.kind ?? null, bundle: bundleBinding.id, failure: bundleRead?.failure ?? null, name: bundleRead?.name ?? null, line: bundleRead?.line ?? null, code: bundleRead?.code ?? null, stderrBytes: bundleRead?.stderrBytes ?? 0 });
|
|
1895
|
+
return { outcome: "policy", reason: "secret-unresolved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1896
|
+
}
|
|
1897
|
+
// THE MERGE: the trigger's own run.secrets win a clash (secret-bundles.mjs `mergeBundleSecrets`), and the
|
|
1898
|
+
// shadowed names are recorded. The values join `secrets`, so every path that already treats a run.secrets
|
|
1899
|
+
// value as a secret treats these alike: buildContainerEnv injects them, the redaction covers every
|
|
1900
|
+
// non-NON_SECRET_ENV_KEYS name (run-container.mjs), and a checkpoint refuses to store any of them.
|
|
1901
|
+
const merged = mergeBundleSecrets(bundleRead.vars, secrets);
|
|
1902
|
+
// Issue #704 part 3: which received variables the bundle binds to a host (`endpoints`) and which go in plain; a
|
|
1903
|
+
// variable a run.secrets value shadowed is in neither unless `endpoints` names it (below). Said to every
|
|
1904
|
+
// venue (`bundleKeys` on the call bag); only a venue whose layer holds credentials reads it, and it judges the
|
|
1905
|
+
// split first, now with the values (`bundleGate`): a value its gateway would refuse is a refusal here, before
|
|
1906
|
+
// spend, never a create that fails as infrastructure and is retried into the same answer.
|
|
1907
|
+
const received = Object.fromEntries(Object.entries(bundleRead.vars).filter(([name]) => !merged.shadowed.includes(name)));
|
|
1908
|
+
const post = await askBundleGate({ endpoints, vars: received, shadowed: merged.shadowed });
|
|
1909
|
+
if (post.refused) return post.refused;
|
|
1910
|
+
// Review round 1, M2: a declared endpoint name a run.secrets value shadowed stays on that host's provider, now with
|
|
1911
|
+
// the trigger's value (the one that wins), so it resolves at the host the reviewed file binds the name to.
|
|
1912
|
+
bundleKeys = {
|
|
1913
|
+
endpoints: Object.fromEntries([...Object.keys(received), ...merged.shadowed].filter((name) => Object.hasOwn(endpoints, name)).map((name) => [name, endpoints[name]])),
|
|
1914
|
+
plain: Object.keys(received).filter((name) => !Object.hasOwn(endpoints, name)).sort(),
|
|
1915
|
+
};
|
|
1916
|
+
secrets = merged.secrets;
|
|
1917
|
+
bundleFacts.bundle = { id: bundleBinding.id, variables: Object.keys(bundleRead.vars).sort(), shadowed: merged.shadowed };
|
|
1918
|
+
// On a venue whose own layer holds the credentials, the record says how many reached the agent as placeholders
|
|
1919
|
+
// and how many as values (INT-RUN-HISTORY-FILE-CONTRACT), the venue's own counts.
|
|
1920
|
+
if (backendFor(resolveBackendName(job, blessedBackends[0]))?.credentialCustody === "layer" && Number.isSafeInteger(post.judged?.placeholders) && Number.isSafeInteger(post.judged?.plain)) {
|
|
1921
|
+
bundleFacts.bundle.placeholders = post.judged.placeholders;
|
|
1922
|
+
bundleFacts.bundle.plain = post.judged.plain;
|
|
1923
|
+
bundleFacts.bundle.placeholderNames = post.judged.placeholderNames;
|
|
1924
|
+
}
|
|
1925
|
+
log("secret_bundle_applied", { kind: job.kind ?? null, bundle: bundleBinding.id, count: bundleFacts.bundle.variables.length, shadowed: merged.shadowed.length });
|
|
1926
|
+
return null;
|
|
1927
|
+
};
|
|
1928
|
+
const bundleHeadCheck = bundleBinding !== null && job.target?.type === "pull_request" && bundleBinding.forkReviews !== true;
|
|
1929
|
+
if (bundleBinding !== null && !bundleHeadCheck) {
|
|
1930
|
+
const refused = await readBundle();
|
|
1931
|
+
if (refused !== null) return refused;
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
|
|
1935
|
+
const secondaryCollision = Object.keys(secrets ?? {}).find(name => secondaryCredentialNames.has(name));
|
|
1936
|
+
if (secondaryCollision !== undefined) {
|
|
1937
|
+
log("refused_watch_provider_credential_override", { name: secondaryCollision });
|
|
1938
|
+
return { outcome: "policy", reason: "secret-name-reserved", budgetReserved: false, provider: job.provider ?? null, model: job.model ?? null };
|
|
1939
|
+
}
|
|
1151
1940
|
|
|
1152
1941
|
if (wantsForgeToken) {
|
|
1153
1942
|
token = await mintToken(job);
|
|
@@ -1161,12 +1950,59 @@ export async function runJob(job, deps) {
|
|
|
1161
1950
|
}
|
|
1162
1951
|
}
|
|
1163
1952
|
|
|
1953
|
+
// PROJECT SECRET BUNDLES, the fork check (issue #704). AFTER the mint, because it needs the job's own token, and
|
|
1954
|
+
// BEFORE branch protection, the clone, the token-cap read and every reserve, so a refusal of the read that follows
|
|
1955
|
+
// still spends nothing but the minted token (CONST-BUDGET-BEFORE-TOKENS: every free refusal stays above the mint,
|
|
1956
|
+
// and so does the read of every job that needs no fork check). A refusal past the mint leaves that token to
|
|
1957
|
+
// expire, exactly as the unprotected-branch refusal below does: it is scoped to this job's repository and
|
|
1958
|
+
// short-lived (CONST-TOKEN-SCOPED-PER-JOB), and no path in this project revokes one, so this adds no second rule.
|
|
1959
|
+
//
|
|
1960
|
+
// A job on a pull request whose head lives in another repository reads a stranger's text (CONST-TRIGGER-AUTHOR-GATE
|
|
1961
|
+
// admits a collaborator's review of a fork), and must not hold every project secret while it does, so it runs
|
|
1962
|
+
// WITHOUT the bundle (`bundleSkipped: "fork"`) unless the bundle opted in with `forkReviews: true`. Answered by the
|
|
1963
|
+
// forge's own `resolvePullRequestHead`, the seam run.resume's fork gate already uses: the head repository must
|
|
1964
|
+
// EQUAL the job's, case-insensitively on GitHub and Forgejo, whose repository names are (GitLab and Azure answer by
|
|
1965
|
+
// project or repository id and report the job's own name back, so their comparison is exact). A webhook's
|
|
1966
|
+
// `head.repo` is attacker-supplied, and an `issue_comment` on a pull request carries no head at all, so the
|
|
1967
|
+
// payload cannot answer. A DETERMINATE forge answer (401, 403, 404: the token cannot see the pull request, or it
|
|
1968
|
+
// is gone) cannot become a yes by retrying, so the job runs without the bundle (`bundleSkipped:
|
|
1969
|
+
// "head-unverified"`); a network fault, a 5xx or a timeout RETRIES (InfraRetry), so a blip neither grants a bundle
|
|
1970
|
+
// nor drops one.
|
|
1971
|
+
if (bundleHeadCheck) {
|
|
1972
|
+
let head = null;
|
|
1973
|
+
let unverified = false;
|
|
1974
|
+
if (typeof resolvePullRequestHead === "function") {
|
|
1975
|
+
try {
|
|
1976
|
+
head = (await resolvePullRequestHead(job, token)) ?? {};
|
|
1977
|
+
} catch (error) {
|
|
1978
|
+
const status = headReadStatus(error);
|
|
1979
|
+
// A GitHub rate limit answers 403 (or 429) with `x-ratelimit-remaining: 0` or a `retry-after`: not a
|
|
1980
|
+
// determinate "you cannot see this pull request", so it RETRIES like any other transient answer.
|
|
1981
|
+
const rateLimited = headReadRateLimited(error);
|
|
1982
|
+
log("secret_bundle_head_unreadable", { kind: job.kind ?? null, status, rateLimited });
|
|
1983
|
+
if (rateLimited || (status !== 401 && status !== 403 && status !== 404)) throw new InfraRetry("the pull request's head could not be read to decide whether it is a fork", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
1984
|
+
unverified = true;
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
const fold = job.kind === "github" || job.kind === "forgejo" ? (v) => v.toLowerCase() : (v) => v;
|
|
1988
|
+
const same = !unverified && typeof head?.headRepo === "string" && typeof job.repo === "string" && fold(head.headRepo) === fold(job.repo);
|
|
1989
|
+
if (!same) {
|
|
1990
|
+
bundleFacts.bundleSkipped = unverified ? "head-unverified" : "fork";
|
|
1991
|
+
log("secret_bundle_skipped", { kind: job.kind ?? null, bundle: bundleBinding.id, reason: bundleFacts.bundleSkipped });
|
|
1992
|
+
bundleBinding = null;
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
if (bundleHeadCheck) {
|
|
1996
|
+
const refused = await readBundle();
|
|
1997
|
+
if (refused !== null) return refused;
|
|
1998
|
+
}
|
|
1999
|
+
|
|
1164
2000
|
if (isForgeBacked) {
|
|
1165
2001
|
// REQ-BRANCH-PROTECTION-PRECONDITION. The agent's token can merge, so branch protection is the
|
|
1166
2002
|
// only technical barrier to a self-merge. Refuse before spending anything. Forge-backed jobs
|
|
1167
2003
|
// only: a local job has no remote branch to protect.
|
|
1168
2004
|
if (!(await isDefaultBranchProtected(job, token))) {
|
|
1169
|
-
await comment(job, "
|
|
2005
|
+
await comment(job, REFUSAL_COMMENTS["unprotected-branch"]);
|
|
1170
2006
|
log("refused_unprotected", { repo: job.repo });
|
|
1171
2007
|
// exitCode/turns/tokens null: refused pre-container, so no container exit, turn, or token count exists.
|
|
1172
2008
|
return { outcome: "policy", reason: "unprotected-branch", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
@@ -1176,7 +2012,10 @@ export async function runJob(job, deps) {
|
|
|
1176
2012
|
// `podmanStore` (issue #429) only where the venue's job user carried one: the podman store the container ran in.
|
|
1177
2013
|
// `portfolio` (issue #505) only for a job the gate above confirmed: prepare then writes /job/portfolio.json, after
|
|
1178
2014
|
// asking the live file once more. Absent otherwise, so every other job's prepare call is unchanged.
|
|
1179
|
-
|
|
2015
|
+
// The existing read-only event file carries accepted human text. Environment/exec metadata stays id-only,
|
|
2016
|
+
// including on gateways that log their exec argv. Keep the queued command for the operator's forge trail.
|
|
2017
|
+
const stagedJob = job.approvalDecision ? { ...job, trigger: { ...job.trigger, comment: { ...job.trigger?.comment, body: job.approvalDecision.text ?? "" } } } : job;
|
|
2018
|
+
prepared = await prepareWorkspace(stagedJob, token, { piVersion, jobUser: { user: jobUser?.user ?? null, home: jobUser?.home ?? null }, ...(typeof jobUser?.store === "string" ? { podmanStore: jobUser.store } : {}), ...(portfolio ? { portfolio: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(mcpChecked.document ? { mcp: mcpChecked.document } : {}), ...(job.watch ? { watchSecrets: secrets } : {}), ...(isRecovery ? { pin: { sha: manifest.workspace.baseSha, from: workspaceCheckpointer.baseSourceUrl(runId) } } : forkSource ? { pin: { sha: forkSource.manifest.baseSha, from: forkStore.baseSourceUrl(job.fork.fromJob) } } : {}) }); // resolves SHA, clones, materialises .pi/, writes prompt
|
|
1180
2019
|
|
|
1181
2020
|
// A determinate prepare refusal -- sha-gone (the default branch advanced past the resolved tip),
|
|
1182
2021
|
// or a `pi-*` materialiser cap breach (the repo's .pi/ is too large to place in /job, issue #60)
|
|
@@ -1189,6 +2028,27 @@ export async function runJob(job, deps) {
|
|
|
1189
2028
|
if (prepared?.outcome === "policy") {
|
|
1190
2029
|
return { ...prepared, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
|
|
1191
2030
|
}
|
|
2031
|
+
// Issue #796: the candidate goes under this job's own directory, before any reservation: a version that cannot be installed spends nothing. After the prepare, because that
|
|
2032
|
+
// is what makes the directory, and on a forge job that is after the mint. The `finally` below removes the directory whichever way this goes.
|
|
2033
|
+
if (canaryPlan) {
|
|
2034
|
+
let installed;
|
|
2035
|
+
try {
|
|
2036
|
+
installed = await deps.packageCanary.install(job, prepared.jobDir, canaryPlan);
|
|
2037
|
+
} catch {
|
|
2038
|
+
throw new InfraRetry("the canary's candidate could not be installed", { reason: "container-never-started", budgetReserved: false, provider: job.provider ?? null, model: job.model ?? null });
|
|
2039
|
+
}
|
|
2040
|
+
if (installed.refused) {
|
|
2041
|
+
log("refused_package_canary", { reason: installed.refused });
|
|
2042
|
+
return { outcome: "policy", reason: installed.refused, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
|
|
2043
|
+
}
|
|
2044
|
+
prepared = { ...prepared, packageCanary: installed };
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
|
|
2048
|
+
if (deps.approvalRequest && (prepared.session?.key !== deps.approvalRequest.sessionKey || prepared.session?.resume !== true)) {
|
|
2049
|
+
await comment(job, APPROVAL_COMMENTS["approval-session-unavailable"]);
|
|
2050
|
+
return { outcome: "policy", reason: "approval-session-unavailable", exitCode: null, turns: null, tokens: null, budgetReserved: false };
|
|
2051
|
+
}
|
|
1192
2052
|
|
|
1193
2053
|
// THE RESOLVED FOLDER'S PROJECT (issue #504 part B). The pickup decided the project, and so the share a job reserves
|
|
1194
2054
|
// against, from the folder AS NAMED; the container mounts the folder prepare RESOLVED. A link or a case variant
|
|
@@ -1196,15 +2056,115 @@ export async function runJob(job, deps) {
|
|
|
1196
2056
|
// Refused here, after prepare and before the token-cap read and every reserve: free, determinate, never retried. A
|
|
1197
2057
|
// resolved folder in no project is not refused, because the named spelling is the operator's own membership
|
|
1198
2058
|
// (a symlinked member is listed by the path the triggers use, docs/projects.md).
|
|
1199
|
-
if (job.kind === "local" && typeof folderProject === "function" && typeof prepared?.workspace === "string") {
|
|
2059
|
+
if (job.kind === "local" && !job.questionGrouping && typeof folderProject === "function" && typeof prepared?.workspace === "string") {
|
|
1200
2060
|
const resolvedProject = folderProject(prepared.workspace);
|
|
1201
2061
|
if (resolvedProject !== null && resolvedProject !== pickupProject) {
|
|
1202
|
-
await comment(job, "
|
|
2062
|
+
await comment(job, REFUSAL_COMMENTS["local-folder-project-changed"]);
|
|
1203
2063
|
log("refused_local_folder_project_changed", { pickup: pickupProject, resolved: resolvedProject });
|
|
1204
2064
|
return { outcome: "policy", reason: LOCAL_FOLDER_PROJECT_CHANGED, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1205
2065
|
}
|
|
1206
2066
|
}
|
|
1207
2067
|
|
|
2068
|
+
// The archive's base is copied before agent-written git state exists. Never mount the archive itself.
|
|
2069
|
+
if (forkStore && sourceRunId && (job.resume === true || job.fork !== undefined) && sessionKeyFor(job) && prepared.sha && !isRecovery) {
|
|
2070
|
+
let primed;
|
|
2071
|
+
try { primed = await forkStore.primeBase({ runId: sourceRunId, workspace: prepared.workspace, baseSha: prepared.sha }); } catch { primed = { refused: "fork-source-unavailable" }; }
|
|
2072
|
+
sourcePrimed = primed?.ok === true;
|
|
2073
|
+
archiveFacts.value = forkArchiveFact({ reason: sourcePrimed ? "not-completed" : primed?.refused ?? primed?.skipped });
|
|
2074
|
+
if (!sourcePrimed) log("fork_source_skipped", { reason: primed?.refused ?? primed?.skipped ?? "base-unavailable" });
|
|
2075
|
+
}
|
|
2076
|
+
// Issue #645: A BASE WITH `.pi/extensions/` IS NOT RECOVERABLE. A merge-gated extension may import a file outside
|
|
2077
|
+
// `.pi/` (`../lib/x.js`) or a bare package resolved from the workspace's `node_modules`, and in a recovered run
|
|
2078
|
+
// those come from the restored tree, which the agent wrote. The trust reset puts `.pi/` back but cannot enumerate
|
|
2079
|
+
// what an extension imports, so the only sound answer is not to recover such a run at all. Decided here, at arm
|
|
2080
|
+
// time, from the fresh clone's own `.git` (the worker's, before any container), so the run is simply not armed:
|
|
2081
|
+
// no ledger claim, no checkpoints, no pointer, and it runs exactly as it did before #645, the record saying
|
|
2082
|
+
// `recoverSkipped: "workspace-extensions"`. A base that could not be read is not armed either, fail closed, and
|
|
2083
|
+
// says `base-unprimed`. The recovery side asks again before it restores (defence in depth).
|
|
2084
|
+
if (armed && !isRecovery) {
|
|
2085
|
+
let unsafe;
|
|
2086
|
+
try {
|
|
2087
|
+
unsafe = await baseHasExtensions({ workspace: prepared.workspace, baseSha: prepared.sha });
|
|
2088
|
+
} catch {
|
|
2089
|
+
unsafe = null;
|
|
2090
|
+
}
|
|
2091
|
+
if (unsafe !== false) {
|
|
2092
|
+
armed = false;
|
|
2093
|
+
durable.armed = false;
|
|
2094
|
+
durable.recoverSkipped = unsafe === true ? "workspace-extensions" : "base-unprimed";
|
|
2095
|
+
log("recover_not_armed", { reason: durable.recoverSkipped });
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
// Issue #645: A RECOVERY JOB'S WORKSPACE. The preparer cloned the checkpoint's base commit from the checkpoint store
|
|
2100
|
+
// (`pin`), so the checkout is exactly the tree the checkpoint is a diff against; the restore applies the crashed
|
|
2101
|
+
// run's work on top. All of it local and free, before the token-cap read and every reserve.
|
|
2102
|
+
if (isRecovery) {
|
|
2103
|
+
const baseSha = manifest.workspace.baseSha;
|
|
2104
|
+
// The arm-time rule, again, on the base the store just handed over: a checkpoint of such a base can only exist
|
|
2105
|
+
// from a worker that armed it before this rule, and is refused rather than restored. Unreadable: retried.
|
|
2106
|
+
let unsafe;
|
|
2107
|
+
try {
|
|
2108
|
+
unsafe = await baseHasExtensions({ workspace: prepared.workspace, baseSha });
|
|
2109
|
+
} catch {
|
|
2110
|
+
unsafe = null;
|
|
2111
|
+
}
|
|
2112
|
+
if (unsafe === true) {
|
|
2113
|
+
await comment(job, RECOVERY_COMMENTS["recovery-unsafe-extensions"]);
|
|
2114
|
+
log("refused_recovery_unsafe_extensions", { n: durable.recovery });
|
|
2115
|
+
return { outcome: "policy", reason: "recovery-unsafe-extensions", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
2116
|
+
}
|
|
2117
|
+
if (unsafe !== false) throw new InfraRetry("whether the recovered base holds .pi/extensions could not be read", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2118
|
+
const restored = await workspaceCheckpointer.restore({ runId, manifest, intoWorkspace: prepared.workspace });
|
|
2119
|
+
if (restored?.refused) {
|
|
2120
|
+
// Determinate: the store refused its own checkpoint (tampered, another run's, a base that does not match).
|
|
2121
|
+
await comment(job, RECOVERY_COMMENTS["recovery-checkpoint-absent"]);
|
|
2122
|
+
log("refused_recovery_checkpoint_absent", { n: durable.recovery, why: "restore-refused", detail: String(restored.refused).slice(0, 40) });
|
|
2123
|
+
return { outcome: "policy", reason: "recovery-checkpoint-absent", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
2124
|
+
}
|
|
2125
|
+
if (!restored || restored.skipped || !Number.isSafeInteger(restored.written)) {
|
|
2126
|
+
log("recovery_restore_skipped", { reason: typeof restored?.skipped === "string" ? restored.skipped.slice(0, 40) : "error" });
|
|
2127
|
+
throw new InfraRetry("the checkpoint could not be restored", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2128
|
+
}
|
|
2129
|
+
// THE TRUST PATHS: what pi loads as the repository's own code and instructions (`.pi/`, `.agents/`, the root
|
|
2130
|
+
// AGENTS/CLAUDE files) goes back to the base commit's content, since the restored tree was written by the agent.
|
|
2131
|
+
// A reset that failed means the recovery would start on the agent's configuration, so it does not start.
|
|
2132
|
+
try {
|
|
2133
|
+
await resetTrustPaths({ workspace: prepared.workspace, baseSha });
|
|
2134
|
+
} catch (error) {
|
|
2135
|
+
log("recovery_trust_reset_failed", { code: typeof error?.code === "string" ? error.code : "error" });
|
|
2136
|
+
throw new InfraRetry("the recovered workspace's trust paths could not be reset", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2137
|
+
}
|
|
2138
|
+
// THE CONVERSATION: the checkpoint's transcript, staged where every job's transcript file is. A resume job's is a
|
|
2139
|
+
// session with its key (promoted on completion as before); any other job's is its live file.
|
|
2140
|
+
const staged = typeof stageRecovered === "function" ? await stageRecovered({ jobDir: prepared.jobDir, transcriptPath, piVersion, job, resolved: prepared.recoveryResolved ?? {} }) : null;
|
|
2141
|
+
if (!staged || typeof staged.hostDir !== "string") {
|
|
2142
|
+
throw new InfraRetry("the checkpoint's transcript could not be staged", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2143
|
+
}
|
|
2144
|
+
prepared = staged.key ? { ...prepared, session: staged } : { ...prepared, live: { hostDir: staged.hostDir } };
|
|
2145
|
+
durable.recovered = { from: manifest.kind, n: durable.recovery, fromAttempt: manifest.attempt, checkpointSeq: manifest.seq, orphans: null, replayed: null, interrupted: null, notStarted: null };
|
|
2146
|
+
}
|
|
2147
|
+
|
|
2148
|
+
if (forkSource) {
|
|
2149
|
+
const baseSha = forkSource.manifest.baseSha;
|
|
2150
|
+
const unsafe = await baseHasExtensions({ workspace: prepared.workspace, baseSha });
|
|
2151
|
+
if (unsafe === true) return await forkRefusal("fork-unsafe-extensions");
|
|
2152
|
+
if (unsafe !== false) throw new InfraRetry("the fork base could not be inspected", { reason: "container-never-started", budgetReserved: false });
|
|
2153
|
+
const restored = await forkStore.restorePoint({ runId: job.fork.fromJob, sourceDigest: job.fork.sourceDigest, entryId: job.fork.fromEntry, position: job.fork.position, intoWorkspace: prepared.workspace });
|
|
2154
|
+
if (restored?.unavailable || restored?.skipped) throw new InfraRetry("the fork workspace could not be restored just now", { reason: "container-never-started", budgetReserved: false });
|
|
2155
|
+
if (!restored?.ok) return await forkRefusal(restored?.refused ?? "fork-workspace-unavailable");
|
|
2156
|
+
await resetTrustPaths({ workspace: prepared.workspace, baseSha });
|
|
2157
|
+
const staged = await stageRecovered?.({ jobDir: prepared.jobDir, transcriptPath: forkSource.transcriptPath, piVersion, job: { ...job, resume: true } });
|
|
2158
|
+
if (!staged?.key) throw new InfraRetry("the fork transcript could not be staged", { reason: "container-never-started", budgetReserved: false });
|
|
2159
|
+
prepared = { ...prepared, session: staged };
|
|
2160
|
+
await comment(job, `Fork of run \`${job.fork.fromJob}\`, ${job.fork.position} entry \`${job.fork.fromEntry}\`, using \`${job.provider}/${job.model}\` on \`${forkBranch(job)}\`. The original attempt is preserved.`);
|
|
2161
|
+
}
|
|
2162
|
+
try { await (deps.stageModelRoleInputs ?? stageModelRoleInputs)(job, prepared); } catch (error) {
|
|
2163
|
+
if (!error?.piDispatchConfig) throw new InfraRetry("the trusted agent snapshot could not be staged", { reason: "container-never-started", provider: job.provider, model: job.model, budgetReserved: false });
|
|
2164
|
+
log("refused_model_agents_invalid", {});
|
|
2165
|
+
await comment(job, MODEL_AGENTS_INVALID_COMMENT);
|
|
2166
|
+
return { outcome: "policy", reason: "model-agents-invalid", exitCode: null, turns: null, tokens: null, provider: job.provider, model: job.model, budgetReserved: false };
|
|
2167
|
+
}
|
|
1208
2168
|
// Daily TOKEN cap (issue #25): the deliberate check-AFTER control. Token cost is only known
|
|
1209
2169
|
// post-run, so this cannot check-and-increment before the spend the way the job-count cap does
|
|
1210
2170
|
// (CONST-BUDGET-BEFORE-TOKENS). It is a read-only GET of prior jobs' recorded spend -- it consumes
|
|
@@ -1235,10 +2195,23 @@ export async function runJob(job, deps) {
|
|
|
1235
2195
|
// ledger's day counted, an EXPIRE that faults leaves its key without a TTL): `reserveBudget` does not say which of
|
|
1236
2196
|
// its windows landed, so giving them back could DECR a window that never rose. That is the pre-existing
|
|
1237
2197
|
// mid-reserve posture of one ledger, unchanged.
|
|
2198
|
+
// Issue #645: an armed run's job-count slot is the RUN's, claimed once in its budget ledger. "new": this attempt
|
|
2199
|
+
// reserves the ledgers below, exactly as every job did before. "carried": an earlier attempt (a retry's first try,
|
|
2200
|
+
// or the attempt a recovery continues) reserved it and is still counted in it, so nothing is reserved again. A
|
|
2201
|
+
// ledger that did not answer leaves the question open, so nothing is reserved and the attempt is retried.
|
|
2202
|
+
if (armed) {
|
|
2203
|
+
const claim = await runBudgetLedger.claimSlot(runId);
|
|
2204
|
+
if (claim !== "new" && claim !== "carried") {
|
|
2205
|
+
log("run_budget_claim_failed", { code: typeof claim?.error === "string" ? claim.error : "unknown" });
|
|
2206
|
+
throw new InfraRetry("the run's budget ledger could not be read", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2207
|
+
}
|
|
2208
|
+
slotCarried = claim === "carried";
|
|
2209
|
+
slotOwnedByRun = true;
|
|
2210
|
+
}
|
|
1238
2211
|
globalLedger = { scope: null, keyPrefix: null, caps, softHoldPct, reason: null };
|
|
1239
2212
|
let counted;
|
|
1240
2213
|
try {
|
|
1241
|
-
counted = await reserveLedgers(redis, [...(scopedLedgers ?? []), globalLedger], held, { now });
|
|
2214
|
+
counted = slotCarried ? { allowed: true } : await reserveLedgers(redis, [...(scopedLedgers ?? []), globalLedger], held, { now });
|
|
1242
2215
|
} catch (error) {
|
|
1243
2216
|
// Valkey failed mid-walk. No container can have started, and `held` lists exactly the reservations that
|
|
1244
2217
|
// landed, so they go back (last first, never throwing) before the error escapes; otherwise a repo and a
|
|
@@ -1283,6 +2256,29 @@ export async function runJob(job, deps) {
|
|
|
1283
2256
|
return { outcome: "policy", reason: counted.refusedBy.reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
|
|
1284
2257
|
}
|
|
1285
2258
|
|
|
2259
|
+
// Issue #645: THIS CONTAINER'S NUMBER in the run (`nextSeq`: attempt 1 is 1, its retry 2, a recovery's container the
|
|
2260
|
+
// next), which addresses its dollar hold in the run's ledger and names its checkpoints' attempt. Before the dollar
|
|
2261
|
+
// reserve, because a hold is recorded under it: without a number there is no record, and without a record there is
|
|
2262
|
+
// no hold. A recovery then settles AT THE FLOOR every hold of the run's earlier containers still `held`: those
|
|
2263
|
+
// containers are dead (the base-still-running gate above holds this job until none of the run's is alive here),
|
|
2264
|
+
// and their real spend is unknown, so the reservation already in every window is what they are charged
|
|
2265
|
+
// (`settleCrashed`, bookkeeping only, no window key moves). A settle that failed is logged and the run goes on: the
|
|
2266
|
+
// holds stay `held` and in the windows at the floor, which is the safe side, and nothing releases an earlier
|
|
2267
|
+
// container's hold but that container itself.
|
|
2268
|
+
if (armed) {
|
|
2269
|
+
const seq = await runBudgetLedger.nextSeq(runId);
|
|
2270
|
+
if (!Number.isSafeInteger(seq) || seq < 1) {
|
|
2271
|
+
log("run_budget_seq_failed", { code: typeof seq?.error === "string" ? seq.error : "unknown" });
|
|
2272
|
+
throw new InfraRetry("the run's container number could not be taken", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2273
|
+
}
|
|
2274
|
+
durable.seq = seq;
|
|
2275
|
+
if (isRecovery) {
|
|
2276
|
+
const crashed = await runBudgetLedger.settleCrashed(runId, seq);
|
|
2277
|
+
if (Array.isArray(crashed?.settled)) log("run_budget_settled_crashed", { holds: crashed.settled.length });
|
|
2278
|
+
else log("run_budget_settle_crashed_failed", { code: typeof crashed?.error === "string" ? crashed.error : "unknown" });
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
|
|
1286
2282
|
// DOLLAR windows (issue #501, part 3), after EVERY job-count reserve and before the container: the last gate
|
|
1287
2283
|
// before the spend line (CONST-BUDGET-BEFORE-TOKENS). Every free gate and every job-count ledger come first, so
|
|
1288
2284
|
// a refusal anywhere above never touches a dollar counter. The amount is the job's per-job cost cap, which the
|
|
@@ -1310,7 +2306,7 @@ export async function runJob(job, deps) {
|
|
|
1310
2306
|
// job carried a non-null cap), so an image that would ignore the 0 never gets here. A cap that is ALREADY 0
|
|
1311
2307
|
// (a malformed queued value reads as 0, `effectiveCostCapMicros`) has nothing to reserve either.
|
|
1312
2308
|
const refs = callableModelRefs(job);
|
|
1313
|
-
const zero = job
|
|
2309
|
+
const zero = zeroRatedFor(job, { modelEndpoints, builtinModel });
|
|
1314
2310
|
if (zero.zeroRated) {
|
|
1315
2311
|
containerJob = { ...job, maxCostMicros: 0 };
|
|
1316
2312
|
dollars = dollarsRecord({ reservedMicros: 0, settledMicros: 0, basis: "unreserved" });
|
|
@@ -1370,20 +2366,134 @@ export async function runJob(job, deps) {
|
|
|
1370
2366
|
return { outcome: "policy", reason: refusalReason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: globalHeld(), dollars: stranded ? dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: job.maxCostMicros, basis: "floor", modelBasis }) : dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: 0, basis: "refunded", modelBasis }) }; // return => not retried
|
|
1371
2367
|
}
|
|
1372
2368
|
dollarHold = reservation.hold;
|
|
1373
|
-
|
|
2369
|
+
publishDollarHold({ amountMicros: dollarHold.amountMicros, at: new Date(now).getTime() });
|
|
2370
|
+
// Issue #761: every key holds the whole reservation until a settle or a release moves it, so every deployment
|
|
2371
|
+
// window the hold reserved in holds what this record states (`deploymentWindowsOf`). Each later path below
|
|
2372
|
+
// replaces `dollars`, with the windows that hold what IT states.
|
|
2373
|
+
dollars = dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: job.maxCostMicros, basis: "floor", modelBasis: modelPrefixes.size > 0 ? "floor" : null, reservedIn: deploymentWindowsOf(dollarHold, { applied: 0, target: job.maxCostMicros, recorded: job.maxCostMicros, valkey: valkeyTag }) });
|
|
2374
|
+
// Issue #645: an armed run never runs a container without a DURABLE record of its hold (run-budget-ledger.mjs's
|
|
2375
|
+
// contract), so a recovery after a crash can see it and settle it exactly once. A record that did not land
|
|
2376
|
+
// throws a never-started retry before any container; the catch gives the hold back directly (`holdRecorded`
|
|
2377
|
+
// false: no one else can see it to settle or release it).
|
|
2378
|
+
if (armed) {
|
|
2379
|
+
const rec = await runBudgetLedger.recordHold(runId, durable.seq, dollarHold);
|
|
2380
|
+
if (rec?.ok !== true) {
|
|
2381
|
+
log("run_budget_hold_unrecorded", { code: typeof rec?.error === "string" ? rec.error : "unknown" });
|
|
2382
|
+
throw new InfraRetry("the run's dollar hold could not be recorded", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
|
|
2383
|
+
}
|
|
2384
|
+
holdRecorded = true;
|
|
2385
|
+
}
|
|
1374
2386
|
}
|
|
1375
2387
|
}
|
|
1376
2388
|
|
|
2389
|
+
// Issue #642: ONE trail comment says who answered from the operator console and quotes what (console-answer.mjs:
|
|
2390
|
+
// mentions neutralised, no control byte). Posted HERE, after every pre-spend gate and every reserve has passed and
|
|
2391
|
+
// just before the container starts, so the issue never reads "answered" above a refusal that then dropped the
|
|
2392
|
+
// answer (a `not-parked`, a missing session store, a cap); and still before the provider is called, so it spends
|
|
2393
|
+
// nothing (CONST-BUDGET-BEFORE-TOKENS). The `comment` adapter posts as the deployment's own identity, which the
|
|
2394
|
+
// receiver's self-guard drops as `self`, so the trail never starts a job. A `console` field whose name is not a
|
|
2395
|
+
// console token's (a hand-built queue entry; enqueueForgeJob refuses one) posts nothing.
|
|
2396
|
+
//
|
|
2397
|
+
// ONCE PER JOB, not once per attempt: a forge job has `attempts: 2`, so an InfraRetry after this point re-runs every
|
|
2398
|
+
// gate and would reach here again, and a console retry of a failed answer job is the same job id again. The claim
|
|
2399
|
+
// is `SET console:trail:<jobId> 1 NX PX CONSOLE_TRAIL_CLAIM_MS` (the retention of a forge job's data, so the claim
|
|
2400
|
+
// outlives every way the same id can come back), and only the attempt that set it posts. On a Valkey error the trail
|
|
2401
|
+
// is SKIPPED and logged, never posted blind: at most once is the promise, because a duplicate trail reads on the forge
|
|
2402
|
+
// as a second answer, while a missing one leaves the answer in the job and the audit log. Never throws: the trail is
|
|
2403
|
+
// provenance, and a failed claim is no reason to refuse a job every gate has admitted.
|
|
2404
|
+
if (job.parkedReply === true && job.console !== undefined) {
|
|
2405
|
+
const by = consoleAnswerBy(job.console);
|
|
2406
|
+
if (by === null) log("console_answer_by_invalid", { kind: job.kind ?? null });
|
|
2407
|
+
else if (await claimConsoleTrail(redis, job, log)) await comment(job, answerTrailComment(by, job.trigger?.comment?.body, job.console?.draftRunId));
|
|
2408
|
+
}
|
|
2409
|
+
|
|
1377
2410
|
// The user the gate above decided is the user that runs: one answer, never two call sites that agree.
|
|
1378
2411
|
// Issue #545: an image that declares `exitAuth` is handed a per-job key on stdin and signs its exit line with it, and
|
|
1379
2412
|
// then only a signed line is read (run-container.mjs, run-history.mjs `authenticExitLines`). An image that does not
|
|
1380
2413
|
// declare it is read as before, under the #542 trust rule below alone.
|
|
1381
2414
|
const exitAuth = (img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY);
|
|
1382
|
-
|
|
2415
|
+
// Issue #686: an image whose runner can mirror its transcript over stderr. Handed as a fact; runContainer arms it only
|
|
2416
|
+
// on a venue that copies (`transcript: "stdio"`) and only beside a key, so every other run is called as before.
|
|
2417
|
+
const transcriptStderr = (img.capabilities ?? []).includes(TRANSCRIPT_STDERR_CAPABILITY);
|
|
2418
|
+
// Issue #686, part 2: an image whose runner reads steer frames on stdin. Handed as a fact, like the one above;
|
|
2419
|
+
// runContainer arms it only for a steer job on a venue that copies, so every other run is called as before.
|
|
2420
|
+
const steerStdin = (img.capabilities ?? []).includes(STEER_STDIN_CAPABILITY);
|
|
2421
|
+
// Issue #645: a FRESH armed run seeds the checkpoint store from its clone, now, before the container: the one moment
|
|
2422
|
+
// the workspace's `.git` is still the worker's own (workspace-checkpoint.mjs `primeBase`). After every reserve, so a
|
|
2423
|
+
// refused job never writes to the store. A recovery's base is already there. A base that could not be primed means
|
|
2424
|
+
// no checkpoint of this run can ever be restored: the run goes on unprotected and the record says so.
|
|
2425
|
+
let checkpointArgs = null;
|
|
2426
|
+
const baseSha = isRecovery ? manifest.workspace.baseSha : prepared.sha;
|
|
2427
|
+
if (armed) {
|
|
2428
|
+
if (!isRecovery) {
|
|
2429
|
+
const objectFormat = typeof baseSha === "string" && baseSha.length === 64 ? "sha256" : "sha1";
|
|
2430
|
+
let primed = null;
|
|
2431
|
+
try {
|
|
2432
|
+
primed = await workspaceCheckpointer.primeBase({ runId, workspace: prepared.workspace, baseSha, objectFormat });
|
|
2433
|
+
} catch {
|
|
2434
|
+
primed = null;
|
|
2435
|
+
}
|
|
2436
|
+
if (primed?.ok !== true) {
|
|
2437
|
+
durable.recoverSkipped = "base-unprimed";
|
|
2438
|
+
log("recover_base_unprimed", { reason: typeof primed?.skipped === "string" ? primed.skipped.slice(0, 40) : "error" });
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
// The live checkpointer's open arguments (run-checkpoint.mjs): the run, this container's number, the base, and
|
|
2442
|
+
// the values a snapshot must refuse to store (the job's token and its resolved secrets). `attempt` IS the
|
|
2443
|
+
// container number: unique across retries and recoveries, which a BullMQ attempt count is not. `meta` is what the
|
|
2444
|
+
// manifest records and a recovery judges above: never a string the job chose.
|
|
2445
|
+
const secretValues = [token, ...Object.values(secrets ?? {})].filter((v) => typeof v === "string" && v !== "");
|
|
2446
|
+
checkpointArgs = { runId, attempt: durable.seq, baseSha, branch: null, secretValues, meta: { piVersion, imageDigest: img.imageDigest ?? null, venue: resolveBackendName(job, blessedBackends[0]), provider: job.provider ?? null, model: job.model ?? null } };
|
|
2447
|
+
}
|
|
2448
|
+
if (job.watch?.stage === "decision") await deps.watchRuntime?.started(job);
|
|
2449
|
+
if (subscriptionAuth.length > 0 && !(await validateSubscriptionAuth(subscriptionAuth))) throw new InfraRetry("subscription binding changed before container start", { reason: "container-never-started", budgetReserved: true });
|
|
2450
|
+
// Issue #800: the commit the repository's part of this run came from, beside the image's facts. Noted here, the last
|
|
2451
|
+
// thing before the container, and not at the prepare above: a refusal between the two loaded nothing, so it has nothing
|
|
2452
|
+
// to point at, and its record stays byte-identical to what it was. Every way out from here keeps it, a retried attempt's too.
|
|
2453
|
+
if (deps.canaryFacts) deps.canaryFacts.loaded = loadedWithBase(deps.canaryFacts.loaded, baseSha);
|
|
2454
|
+
const { canaryLoad: canaryLoadOfRun = null, loaded: loadedOfRun = null, code, aborted, abortReason, turns, tokens, session, usage, context, detached, exitReason, approval: approvalMetadata = null, exitWhy = null, exitLineCode = null, exitAuth: exitAuthResult = null, exitOomKilled = false, memoryLimit = null, resources: ranResources = null, progress: ranProgress = null, checkpoint: ranCheckpoint = null, recovery: ranRecovery = null, lingering = false } = await runContainer({ job: containerJob, token, prepared, secrets, ...(subscriptionAuth.length ? { subscriptionAuth } : {}), ...(bundleKeys !== null ? { bundleKeys } : {}), user: jobUser?.user ?? null, home: jobUser?.home ?? null, relabel: jobUser?.relabel === true, ...(modelEndpoints?.endpoints?.length > 0 ? { modelEndpoints } : {}), ...(exitAuth ? { exitAuth: true } : {}), ...(transcriptStderr ? { transcriptStderr: true } : {}), ...(steerStdin ? { steerStdin: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(Number.isSafeInteger(jobUser?.hostCpus) ? { hostCpus: jobUser.hostCpus } : {}), ...(Number.isSafeInteger(cpuBudgetCenti) ? { cpuBudgetCenti } : {}), ...(Array.isArray(jobUser?.unenforced) && jobUser.unenforced.length > 0 ? { unenforced: jobUser.unenforced } : {}), ...(checkpointArgs !== null ? { recover: true, recovering: isRecovery, checkpoint: checkpointArgs } : {}) });
|
|
2455
|
+
const answeredPark = prepared.session?.parkedSource;
|
|
2456
|
+
// A phrase-prefixed comment uses ordinary routing, but resuming this exact parked transcript still
|
|
2457
|
+
// answers its question. A held receipt also identifies an answer whose legacy source cannot be proved.
|
|
2458
|
+
if (!deps.approvalRequest && session?.resumed === true && (job.parkedReply === true || typeof job.trigger?.comment?.body === "string" && (answeredPark || Number.isSafeInteger(job.replyReceivedAt)))) {
|
|
2459
|
+
if (answeredPark) {
|
|
2460
|
+
const receivedAt = Number.isSafeInteger(deps.replyReceivedAt) ? deps.replyReceivedAt : now.getTime();
|
|
2461
|
+
await deps.recordTouch({ runId: answeredPark.runId, cause: "question-answer", at: Math.max(answeredPark.at, receivedAt), receivedAt: Math.max(answeredPark.at, receivedAt), answerSource: job.console?.draftRunId ? "draft" : job.console?.by === "panel" ? "panel" : job.console ? "console" : "forge", ...(job.console?.draftRunId ? { draftRunId: job.console.draftRunId } : {}), parkedAt: answeredPark.at, scopeId: touchScopeOf(job, hostName), project: pickupProject ?? undefined, nonce: sourceRunId });
|
|
2462
|
+
} else {
|
|
2463
|
+
deps.recordTouchGap();
|
|
2464
|
+
log("touch_lineage_unavailable", { reason: "source-unavailable" });
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
|
|
1383
2468
|
containerRan = true;
|
|
2469
|
+
watchPolicyRefused = code === EXIT_POLICY;
|
|
2470
|
+
watchUsage = Number.isFinite(tokens?.cost) && tokens.cost >= 0 ? { costUsd: tokens.cost } : null;
|
|
2471
|
+
answerUsage = { tokens };
|
|
2472
|
+
if (sourcePrimed && !lingering && !detached && !aborted && code === EXIT_COMPLETED) {
|
|
2473
|
+
const rootKey = sessionKeyFor(job).split(".")[0];
|
|
2474
|
+
const transcriptPath = join(prepared.session?.hostDir ?? prepared.live?.hostDir ?? join(prepared.jobDir, "session"), "current.jsonl");
|
|
2475
|
+
let pointSkipped = null;
|
|
2476
|
+
// Dead-container snapshots also make the completed boundary available without live recovery.
|
|
2477
|
+
if (typeof forkStore.capturePoint === "function") {
|
|
2478
|
+
try {
|
|
2479
|
+
const captured = await forkStore.capturePoint({ runId: sourceRunId, workspace: prepared.workspace, baseSha: prepared.sha, quiesced: "dead", kind: "exit", attempt: durable.seq ?? 0, transcriptPath, offset: lstatSync(transcriptPath).size, secretValues: [token, ...Object.values(secrets ?? {})].filter((v) => typeof v === "string" && v !== ""), meta: { piVersion, venue: resolveBackendName(job, blessedBackends[0]), provider: job.provider ?? null, model: job.model ?? null } });
|
|
2480
|
+
if (!captured?.ok) {
|
|
2481
|
+
pointSkipped = captured?.refused ?? captured?.skipped ?? "checkpoint-unavailable";
|
|
2482
|
+
log("fork_checkpoint_skipped", { reason: pointSkipped });
|
|
2483
|
+
}
|
|
2484
|
+
} catch { pointSkipped = "checkpoint-unavailable"; log("fork_checkpoint_skipped", { reason: "unavailable" }); }
|
|
2485
|
+
}
|
|
2486
|
+
let saved;
|
|
2487
|
+
try { saved = forkStore.publish({ runId: sourceRunId, sessionKey: rootKey, transcriptPath: join(prepared.session?.hostDir ?? prepared.live?.hostDir ?? join(prepared.jobDir, "session"), "current.jsonl"), baseSha: prepared.sha, venue: resolveBackendName(job, blessedBackends[0]), piVersion, attempt: durable.seq ?? 0 }); } catch { saved = { refused: "fork-source-unavailable" }; }
|
|
2488
|
+
archiveFacts.value = forkArchiveFact({ saved: saved?.ok === true, reason: saved?.refused, pointSkipped });
|
|
2489
|
+
log("fork_source_saved", saved?.ok ? { saved: true } : { saved: false, reason: saved?.refused ?? "unavailable" });
|
|
2490
|
+
}
|
|
1384
2491
|
// Issue #596: what the container used, off its exit line, rebuilt by the sink (null from a runContainer that predates
|
|
1385
2492
|
// the field). Every result and every throw below carries it, so a retried attempt's record says what it used too.
|
|
1386
2493
|
resources = ranResources ?? null;
|
|
2494
|
+
if (deps.canaryFacts && job.packageCanary !== undefined) deps.canaryFacts.load = canaryLoadOfRun ?? null;
|
|
2495
|
+
if (deps.canaryFacts && loadedOfRun !== null) deps.canaryFacts.loaded = loadedWithBase(loadedWithImage(loadedOfRun, img), baseSha);
|
|
2496
|
+
progress = ranProgress ?? null;
|
|
1387
2497
|
// Issue #596: CONFIRMED killed for memory. The image's supervisor (image/runner/src/supervise.mjs) outlives the runner,
|
|
1388
2498
|
// whose process tree it gives the highest OOM score, and when the runner dies of SIGKILL with the cgroup's
|
|
1389
2499
|
// `oom_kill` above 0 it writes the signed line `code: 137, reason: "oom-killed"` (parseExitOomKilled). All three
|
|
@@ -1394,6 +2504,79 @@ export async function runJob(job, deps) {
|
|
|
1394
2504
|
// `oom_kill` counts the HOST's OOM killer too (`peakReachedLimit`). A report below it stays infrastructure.
|
|
1395
2505
|
const oomReported = exitOomKilled === true && exitAuthResult === "verified" && code === EXIT_SIGKILL;
|
|
1396
2506
|
const oomKilled = oomReported && peakReachedLimit(resources?.memPeak ?? null, memoryLimit);
|
|
2507
|
+
// Issue #645: what the runner says it did with the crashed attempt's orphaned tool calls (the exit line's `recovery`).
|
|
2508
|
+
// Issue #662: a recovery whose own container was killed writes no exit line, so its counts come from the transcript
|
|
2509
|
+
// instead (the reducer's recovery marks, this attempt's lines only), labelled `countsFrom: "transcript"`. The marks
|
|
2510
|
+
// are in an agent-writable file, so the exit line's counts win whenever the exit line CARRIES a `recovery` block
|
|
2511
|
+
// (`recoveryCounts` not null); an exit line without one (a verified line whose recovery step threw, say) falls back
|
|
2512
|
+
// to the transcript like a missing line. Neither feeds any decision.
|
|
2513
|
+
if (durable.recovered !== null) {
|
|
2514
|
+
const counts = recoveryCounts(ranRecovery);
|
|
2515
|
+
const fromTranscript = counts === null ? recoveryCountsFromProgress(progress) : null;
|
|
2516
|
+
if (counts !== null) durable.recovered = { ...durable.recovered, ...counts, countsFrom: "exit-line" };
|
|
2517
|
+
else if (fromTranscript !== null) durable.recovered = { ...durable.recovered, ...fromTranscript, countsFrom: "transcript" };
|
|
2518
|
+
}
|
|
2519
|
+
// A container the checkpointer stopped because it could not unpause it (`checkpointStopped`, code 137, not aborted)
|
|
2520
|
+
// is an infrastructure exit like any other here: not the worker's abort, not an OOM (no supervisor report), so it
|
|
2521
|
+
// reaches the unknown-exit arm below and recovers from the checkpoint that was committed before the pause.
|
|
2522
|
+
// Issue #645: THE EXIT CHECKPOINT, then the checkpointer's close, before anything below can return or throw. Taken
|
|
2523
|
+
// only on an exit this attempt may recover from: infrastructure (exit 1, an unknown code, a detached container),
|
|
2524
|
+
// never a worker's abort, an OOM kill, a completed or policy exit (none of those recovers) nor a never-started one
|
|
2525
|
+
// (nothing ran, and today's retry is cheaper). The container is dead, so nothing is paused. B1's handle is read
|
|
2526
|
+
// defensively: absent, or without `exit`, means no exit checkpoint, never a failure. Neither call may throw out of here.
|
|
2527
|
+
const neverStartedExit = isNeverStartedExit({ code, aborted, detached }, neverStartedExits(job));
|
|
2528
|
+
const recoverable = armed && !aborted && !oomKilled && code !== EXIT_COMPLETED && code !== EXIT_POLICY && !neverStartedExit;
|
|
2529
|
+
// Issue #686 part 3, review round 2 of #714: a venue whose box was not confirmed gone after the run (`lingering`,
|
|
2530
|
+
// `openshell-bind`) takes NO exit checkpoint: the host tree may still be written, so a snapshot of it is not a state
|
|
2531
|
+
// the run reached. A recovery then continues from the last committed checkpoint.
|
|
2532
|
+
if (ranCheckpoint !== null && typeof ranCheckpoint === "object") {
|
|
2533
|
+
if (recoverable && lingering !== true && typeof ranCheckpoint.exit === "function") {
|
|
2534
|
+
const taken = await Promise.resolve().then(() => ranCheckpoint.exit({ kind: "exit" })).catch(() => ({ skipped: "error" }));
|
|
2535
|
+
log("checkpoint_exit", taken && Number.isSafeInteger(taken.seq) ? { seq: taken.seq } : { skipped: typeof taken?.skipped === "string" ? taken.skipped.slice(0, 40) : "unknown" });
|
|
2536
|
+
}
|
|
2537
|
+
const closed = typeof ranCheckpoint.close === "function" ? await Promise.resolve().then(() => ranCheckpoint.close()).catch(() => null) : null;
|
|
2538
|
+
// `close()` answers the counts; `stats()` (a function on B1's handle) is the fallback when close did not.
|
|
2539
|
+
const stats = closed ?? (typeof ranCheckpoint.stats === "function" ? await Promise.resolve().then(() => ranCheckpoint.stats()).catch(() => null) : null);
|
|
2540
|
+
const count = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
|
|
2541
|
+
if (stats && count(stats.written) !== null && count(stats.failed) !== null && count(stats.skipped) !== null) durable.checkpoints = { written: stats.written, failed: stats.failed, skipped: stats.skipped };
|
|
2542
|
+
}
|
|
2543
|
+
// An infrastructure exit of an armed run: queue its continuation from the checkpoint instead of throwing a retry
|
|
2544
|
+
// (CONST-RETRY-INFRA-ONLY holds: nothing is returned as success, and the queued recovery IS the retry, one that
|
|
2545
|
+
// keeps the work). RETURNED, so BullMQ spends no second attempt that would start from scratch beside it. Not
|
|
2546
|
+
// queued (no checkpoint, the cap, a claim already taken, Valkey) -> today's InfraRetry, unchanged.
|
|
2547
|
+
// A RECOVERY job has one attempt, so a throw here would fail it with the generic failure comment and nothing else:
|
|
2548
|
+
// when its container ran, ended infra and no further recovery was queued, it RETURNS `recovery-failed` with its own
|
|
2549
|
+
// comment instead (the run's final outcome, so its checkpoint, pointer and ledger are cleaned up).
|
|
2550
|
+
const notQueued = async (err) => {
|
|
2551
|
+
if (!isRecovery) throw err;
|
|
2552
|
+
await comment(job, RECOVERY_COMMENTS["recovery-failed"]);
|
|
2553
|
+
return { outcome: "failed", reason: "recovery-failed", exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
|
|
2554
|
+
};
|
|
2555
|
+
const recoverOrThrow = async (err) => {
|
|
2556
|
+
if (!recoverable) return notQueued(err);
|
|
2557
|
+
let pointer = null;
|
|
2558
|
+
try {
|
|
2559
|
+
pointer = await readCheckpointPointer({ runId });
|
|
2560
|
+
} catch {
|
|
2561
|
+
pointer = null;
|
|
2562
|
+
}
|
|
2563
|
+
if (pointer === null || pointer === undefined) {
|
|
2564
|
+
log("recovery_not_enqueued", { cause: "infra", reason: "no-checkpoint" });
|
|
2565
|
+
return notQueued(err);
|
|
2566
|
+
}
|
|
2567
|
+
let queued;
|
|
2568
|
+
try {
|
|
2569
|
+
queued = await enqueueRecovery({ job, cause: "infra", pointer });
|
|
2570
|
+
} catch {
|
|
2571
|
+
queued = { enqueued: false, reason: "error" };
|
|
2572
|
+
}
|
|
2573
|
+
if (queued?.enqueued !== true) {
|
|
2574
|
+
log("recovery_not_enqueued", { cause: "infra", reason: typeof queued?.reason === "string" && /^[a-z0-9-]{1,40}$/.test(queued.reason) ? queued.reason : "unknown" });
|
|
2575
|
+
return notQueued(err);
|
|
2576
|
+
}
|
|
2577
|
+
log("recovery_enqueued_after_exit", { cause: "infra", n: Number.isSafeInteger(queued.n) ? queued.n : null, exitCode: code });
|
|
2578
|
+
return { outcome: "failed", reason: RECOVERY_ENQUEUED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
|
|
2579
|
+
};
|
|
1397
2580
|
// Numbers only (bytes), never a path or a project: the operator's trace of a kill read as the host's. Never on a run
|
|
1398
2581
|
// the worker stopped itself (a timeout, a cancel, a shutdown): that 137 is the worker's own, the abort decides it
|
|
1399
2582
|
// below, and a line naming the host's OOM killer would send the operator after a kill that never happened.
|
|
@@ -1418,7 +2601,7 @@ export async function runJob(job, deps) {
|
|
|
1418
2601
|
// completed, runner policy, a worker abort or cancel, exit 1, an unknown exit, and a detached container. A
|
|
1419
2602
|
// never-started exit is not settled: the catch refunds it whole, and `isNeverStartedExit` is the one answer both
|
|
1420
2603
|
// ask. Before any classification below, so every return and every throw carries the same `dollars`.
|
|
1421
|
-
const neverStarted =
|
|
2604
|
+
const neverStarted = neverStartedExit;
|
|
1422
2605
|
if (dollarHold !== null && !neverStarted) {
|
|
1423
2606
|
// The exit line is trusted only when the container exited ON ITS OWN (the worker did not abort, cancel, time
|
|
1424
2607
|
// out or detach it, so the runner had the chance to write its genuine last line) AND that line's own `code` is
|
|
@@ -1428,10 +2611,19 @@ export async function runJob(job, deps) {
|
|
|
1428
2611
|
const { settledMicros, basis } = dollarSettlement({ tokens, usage: usage ?? null, reservedMicros, trusted });
|
|
1429
2612
|
// The deployment, the repo or folder and the project windows settle to the job's cost; each model window to its
|
|
1430
2613
|
// own row of the usage ledger (`modelDollarSettlement`), never to the job's total.
|
|
2614
|
+
// Issue #645: a recorded hold is settled through the run's ledger first, `held` -> `settled`, and only the call
|
|
2615
|
+
// that MOVED it adjusts a window key. Anything else ("not-held": someone already finished it; a ledger error)
|
|
2616
|
+
// leaves the reservation in every window, the floor, which is the safe side and what the record then says.
|
|
2617
|
+
let ledgerMoved = true;
|
|
2618
|
+
if (holdRecorded) {
|
|
2619
|
+
const moved = await runBudgetLedger.settle(runId, durable.seq);
|
|
2620
|
+
ledgerMoved = moved === "moved";
|
|
2621
|
+
if (!ledgerMoved) log("run_budget_settle_skipped", { result: typeof moved === "string" ? moved : typeof moved?.error === "string" ? moved.error : "unknown" });
|
|
2622
|
+
}
|
|
1431
2623
|
const jobPart = holdPart(dollarHold, (prefix) => jobCostPrefixes.has(prefix));
|
|
1432
|
-
const { applied, of } = await settleDollars(redis, jobPart, settledMicros, { log });
|
|
2624
|
+
const { applied, of } = ledgerMoved ? await settleDollars(redis, jobPart, settledMicros, { log }) : { applied: 0, of: 1 };
|
|
1433
2625
|
let modelBasis = null;
|
|
1434
|
-
for (const part of dollarHold.ledgers ?? []) {
|
|
2626
|
+
for (const part of ledgerMoved ? (dollarHold.ledgers ?? []) : []) {
|
|
1435
2627
|
const ref = modelPrefixes.get(part.keyPrefix);
|
|
1436
2628
|
if (ref === undefined) continue;
|
|
1437
2629
|
const m = modelDollarSettlement({ ref, basis, tokens, usage: usage ?? null, reservedMicros, trusted });
|
|
@@ -1445,8 +2637,14 @@ export async function runJob(job, deps) {
|
|
|
1445
2637
|
// A fault that adjusted no key leaves the whole reservation in every window, which is a floor whatever the
|
|
1446
2638
|
// basis would have been, and the record says so. A partial one is logged (dollar_settle_error) and recorded
|
|
1447
2639
|
// as computed: the keys that were adjusted hold the settled amount.
|
|
1448
|
-
|
|
2640
|
+
if (!ledgerMoved && modelPrefixes.size > 0) modelBasis = "floor";
|
|
2641
|
+
const kept = applied === 0 && of > 0 && (settledMicros !== reservedMicros || !ledgerMoved);
|
|
2642
|
+
// Issue #761: the deployment windows whose keys hold what this record states, along the keys the settle walked
|
|
2643
|
+
// (`jobPart`, the deployment's first): a key it moved holds the settled amount, one it did not the reservation.
|
|
2644
|
+
const reservedIn = deploymentWindowsOf(jobPart, { applied, target: settledMicros, recorded: kept ? reservedMicros : settledMicros, valkey: valkeyTag });
|
|
2645
|
+
dollars = kept ? dollarsRecord({ reservedMicros, settledMicros: reservedMicros, basis: "floor", modelBasis, reservedIn }) : dollarsRecord({ reservedMicros, settledMicros, basis, modelBasis, reservedIn });
|
|
1449
2646
|
dollarHold = null;
|
|
2647
|
+
publishDollarHold(null);
|
|
1450
2648
|
log("dollar_settled", { reservedMicros: dollars.reservedMicros, settledMicros: dollars.settledMicros, basis: dollars.basis, ...(modelBasis !== null ? { modelBasis } : {}) });
|
|
1451
2649
|
} else if (dollars?.basis === "unreserved") {
|
|
1452
2650
|
// Nothing was held, so nothing is written. A metered cost above 0 here would mean the runner's cap of 0 let a
|
|
@@ -1460,7 +2658,7 @@ export async function runJob(job, deps) {
|
|
|
1460
2658
|
// mid-job). It DID start, so this is never refunded as never-started: it keeps its slot and retries as infrastructure,
|
|
1461
2659
|
// BEFORE the exit-code switch, where the same code would read as a free never-started exit.
|
|
1462
2660
|
if (detached === true) {
|
|
1463
|
-
|
|
2661
|
+
return await recoverOrThrow(new InfraRetry(`the container outlived its docker run, exit ${code}`, { reason: "container-detached", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources }));
|
|
1464
2662
|
}
|
|
1465
2663
|
|
|
1466
2664
|
// A WORKER-initiated stop (30-min timeout via cancelJob, graceful-shutdown docker stop, or an
|
|
@@ -1475,11 +2673,11 @@ export async function runJob(job, deps) {
|
|
|
1475
2673
|
// `abortReason` itself never reaches the record -- buildRecord copies named fields only.
|
|
1476
2674
|
// exitCode/turns/tokens carry the container's own exit, turn count, and usage totals; budgetReserved true post-reserve.
|
|
1477
2675
|
if (aborted) {
|
|
1478
|
-
const reason =
|
|
2676
|
+
const reason = ["operator-cancel", "agent-cancel"].includes(abortReason) ? abortReason : "worker-abort";
|
|
1479
2677
|
// Awaited bare like every determinate refusal above: the adapter never throws by contract, and
|
|
1480
2678
|
// the one swallowed comment in this file (the catch's) justifies itself by its position.
|
|
1481
2679
|
await comment(job, TERMINAL_COMMENTS[reason]);
|
|
1482
|
-
return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}) };
|
|
2680
|
+
return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
|
|
1483
2681
|
}
|
|
1484
2682
|
|
|
1485
2683
|
// Issue #596: a runner the kernel killed for memory, CONFIRMED (`oomKilled` above). After the abort branch, so a
|
|
@@ -1495,21 +2693,34 @@ export async function runJob(job, deps) {
|
|
|
1495
2693
|
// The project and the size go to the log and the record, never to the comment (the TERMINAL_COMMENTS rule).
|
|
1496
2694
|
log("oom_killed", { jobId: job.id ?? null });
|
|
1497
2695
|
await comment(job, TERMINAL_COMMENTS[EXIT_OOM_KILLED]);
|
|
1498
|
-
return { outcome: "policy", reason: EXIT_OOM_KILLED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}) };
|
|
2696
|
+
return { outcome: "policy", reason: EXIT_OOM_KILLED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
|
|
2697
|
+
}
|
|
2698
|
+
// Issue #686 part 3, review round 2 of #714 (CONST-RETRY-INFRA-ONLY): a box not confirmed gone after the run. Exit 2
|
|
2699
|
+
// keeps its own reason below, and a worker's abort stays the abort. A COMPLETED run is not thrown away as a retry,
|
|
2700
|
+
// and a FOLDER job is not retried while the first sandbox may still write the folder: both RETURN the venue's named
|
|
2701
|
+
// result, never retried, spend kept and settled above. The completed run's work stays where the agent put it (a
|
|
2702
|
+
// pushed branch, a folder's edits), but nothing more is done with its result: no session promotion, no chained
|
|
2703
|
+
// child, no completion comment, since the tree it would read may still change. Any other exit of a forge job is
|
|
2704
|
+
// infrastructure as before (its retry gets a fresh clone; its recovery the last committed checkpoint).
|
|
2705
|
+
if (lingering === true && !aborted && code !== EXIT_POLICY && (code === EXIT_COMPLETED || job.kind === "local")) {
|
|
2706
|
+
log("venue_box_lingering", { exitCode: code, kind: job.kind ?? null });
|
|
2707
|
+
await comment(job, OPENSHELL_BIND_LINGERING.comment);
|
|
2708
|
+
return { outcome: "policy", reason: OPENSHELL_BIND_LINGERING.reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
|
|
1499
2709
|
}
|
|
1500
2710
|
|
|
1501
2711
|
switch (code) {
|
|
1502
2712
|
case EXIT_COMPLETED: {
|
|
2713
|
+
watchCompleted = true;
|
|
1503
2714
|
// The SOLE chain-collection point. Read the completed parent's /outbox and enqueue children
|
|
1504
2715
|
// BEFORE the `finally` deletes jobDir -- the await resolves inside this case, so the read
|
|
1505
2716
|
// finishes before control leaves to cleanup. NOT reached on any other branch (policy, abort,
|
|
1506
2717
|
// over-budget, infra): an InfraRetry job is retried, so chaining there would double-enqueue.
|
|
1507
2718
|
// collectChain never throws; chainEnqueued/chainRefused are additive telemetry only.
|
|
1508
|
-
const chain = await collectChain({ job, prepared });
|
|
2719
|
+
const chain = job.answerDraft ? { enqueued: 0, refused: 0 } : await collectChain({ job, prepared });
|
|
1509
2720
|
// Issue #505: the plan, after the chain and on this branch only, for the reason the chain is here: a policy or
|
|
1510
2721
|
// infra exit collects nothing, and a retried job must not apply what its failed attempt wrote. `portfolio` is
|
|
1511
2722
|
// the pickup's decision; the collector asks the live file again, and both must agree. Never throws.
|
|
1512
|
-
const plan = await collectPlan({ job, prepared, portfolio });
|
|
2723
|
+
const plan = job.answerDraft ? null : await collectPlan({ job, prepared, portfolio });
|
|
1513
2724
|
// COMPLETED-ONLY PROMOTION, and the exclusivity is the point rather than an optimisation.
|
|
1514
2725
|
// A policy or infra exit leaves the canonical transcript byte-identical to what it was
|
|
1515
2726
|
// before this run, so a retry starts from exactly what the first attempt did -- promote on
|
|
@@ -1522,22 +2733,69 @@ export async function runJob(job, deps) {
|
|
|
1522
2733
|
// exitReason is RUNNER_COMPLETED_REASONS-filtered and read only here, on code 0, so it cannot move a job
|
|
1523
2734
|
// between retry classes. Only a job armed with run.awaitInput may park: a line that says so on any other
|
|
1524
2735
|
// job is ignored, since that runner was never offered the tool.
|
|
1525
|
-
const
|
|
1526
|
-
const
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
2736
|
+
const approvalParked = exitReason === "awaiting-approval" && Array.isArray(job.gates) && job.gates.includes(approvalMetadata?.action);
|
|
2737
|
+
const parked = (exitReason === "awaiting-input" || approvalParked) && job.awaitInput === true;
|
|
2738
|
+
const promoted = prepared.session ? promoteSession(prepared.session, { piVersion, context, parked, ...(sourceRunId ? { parkedSource: { runId: sourceRunId } } : {}) }) : null;
|
|
2739
|
+
if (parked && !approvalParked && promoted?.promoted === true && deps.questionCaptureEnabled?.() === true && deps.questionGroupStore && sourceRunId) {
|
|
2740
|
+
try {
|
|
2741
|
+
const context = typeof prepared.questionRedact === "function"
|
|
2742
|
+
? (deps.readQuestionContext ?? readQuestionContext)({ path: join(prepared.session.hostDir, "current.jsonl"), redact: prepared.questionRedact })
|
|
2743
|
+
: { status: "source-unavailable" };
|
|
2744
|
+
const captured = await deps.questionGroupStore.capture({ runId: sourceRunId, scopeId: touchScopeOf(job, hostName), project: pickupProject ?? null, repo: job.repo ?? null, at: deps.touchNow?.() ?? Number(now), text: context.text ?? null, sourceStatus: context.status === "ready" ? null : context.status, contextTruncated: context.truncated === true, redact: (text) => text, ...(Number.isSafeInteger(job.trigger?.matched?.index) ? { triggerIndex: job.trigger.matched.index } : {}) });
|
|
2745
|
+
if (!captured?.ok || captured.status !== "unassigned") log("question_capture_partial", { status: captured?.status ?? "source-unavailable" });
|
|
2746
|
+
} catch { log("question_source_unavailable", {}); }
|
|
2747
|
+
}
|
|
2748
|
+
|
|
2749
|
+
if (deps.approvalRequest) {
|
|
2750
|
+
const ended = await deps.approvalRuntime.finish({ requestId: deps.approvalRequest.requestId, jobId: deps.approvalJobId });
|
|
2751
|
+
if (!ended.ok) return { outcome: "policy", reason: "approval-finish-unavailable", exitCode: code, turns, tokens, budgetReserved: true, session: mergeSession(prepared, session, promoted) };
|
|
2752
|
+
}
|
|
2753
|
+
let approvalFacts = null;
|
|
2754
|
+
if (approvalParked) {
|
|
2755
|
+
// A failed publication must not re-run a paid container. Keep the saved transcript and name the repair.
|
|
2756
|
+
const fingerprint = promoted?.promoted === true && deps.approvalSessionCheck?.fingerprint ? deps.approvalSessionCheck.fingerprint({ sessionKey: prepared.session.key, ...approvalMetadata }) : null;
|
|
2757
|
+
const approvalSession = fingerprint?.ok ? { ...prepared.session, digest: fingerprint.digest } : prepared.session;
|
|
2758
|
+
const published = promoted?.promoted === true && isParked(job) ? await deps.approvalRuntime.afterPark({ job: { ...job, id: deps.queueJobId ?? sourceRunId }, session: approvalSession, metadata: approvalMetadata, rawJob: deps.rawApprovalJob ?? job }) : { refused: "approval-session-not-saved" };
|
|
2759
|
+
if (!published.ok) {
|
|
2760
|
+
// Issue #761: `{ newer: true }`, the live triggers file was written by a newer pi-dispatch, so the declaration
|
|
2761
|
+
// this park would be armed under cannot be judged here. NOT deferred, unlike the free gates: the container has
|
|
2762
|
+
// run and been paid for, and a deferral would run it again. No request is published, so nothing can decide or
|
|
2763
|
+
// resume this park on this build; recorded as the declaration that could not be read, exactly as an
|
|
2764
|
+
// unreadable file is, never as a store fault, and said in its own sentence, which names the upgrade.
|
|
2765
|
+
if (published.newer === true) log("approval_park_newer_release", {});
|
|
2766
|
+
await comment(job, published.newer === true ? APPROVAL_PARK_NEWER_COMMENT : APPROVAL_COMMENTS["approval-state-unavailable"]);
|
|
2767
|
+
return { outcome: "policy", reason: published.newer === true ? "approval-policy-unavailable" : (published.refused ?? "approval-state-unavailable"), exitCode: code, turns, tokens, budgetReserved: true, session: mergeSession(prepared, session, promoted) };
|
|
2768
|
+
}
|
|
2769
|
+
approvalFacts = published.audit;
|
|
2770
|
+
}
|
|
2771
|
+
// THE PARK SEQUENCE (issue #638 reordered it): promote, marker ON, release the window, take the held replies,
|
|
2772
|
+
// then the comment. Every step closes a gap the one before it leaves open:
|
|
2773
|
+
// - the marker AFTER the promotion (a reply it routes must find the `parked` mark and the transcript) and
|
|
2774
|
+
// BEFORE the release: a phrase-free reply in between routes as a parked reply, meets the window this run
|
|
2775
|
+
// still holds, and is HELD (the keeper still registers the run), so no reply slips past both;
|
|
2776
|
+
// - the release before the take, so the taken reply is enqueued WITH the window (held-settle.mjs) and becomes
|
|
2777
|
+
// the one job every later answer is coalesced against, never a second resumed run beside it;
|
|
2778
|
+
// - the comment LAST, because it invites an answer: with a reply taken it says the run continues
|
|
2779
|
+
// (HELD_TAKEN_COMMENT) instead of asking for one.
|
|
2780
|
+
// The marker stays on when a reply was taken; the resumed run clears it when it ends without parking, as any
|
|
2781
|
+
// run on the key does. Every park takes this path, a resumed run that parks again included. Each seam never
|
|
2782
|
+
// throws and is guarded anyway: a fault must not cost the person the comment that says how to continue. The
|
|
2783
|
+
// marker: set on a park, cleared when a run on a key that could park ends without parking; it decides nothing,
|
|
2784
|
+
// the sidecar the promotion just wrote does.
|
|
2785
|
+
if (parked) await parkedLabel(job, true, approvalParked);
|
|
1537
2786
|
else if (job.awaitInput === true || job.parkedReply === true) await parkedLabel(job, false);
|
|
2787
|
+
if (parked) await Promise.resolve().then(releaseParkedWindow).catch(() => {});
|
|
2788
|
+
const taken = parked ? await Promise.resolve().then(takeHeldReplies).catch(() => null) : null;
|
|
2789
|
+
if (parked && !approvalParked && promoted?.promoted === true && deps.answerDraftRuntime && sourceRunId) {
|
|
2790
|
+
await deps.answerDraftRuntime.afterPark({ job, prepared, runId: sourceRunId, project: pickupProject, startedAt: now.getTime(), taken: Boolean(taken?.taken || Number(taken?.resumed) > 0) }).catch(() => log("answer_draft_enqueue_unavailable", {}));
|
|
2791
|
+
}
|
|
2792
|
+
// `taken` "other": a reply that got in between the release and the take already is the resumed job, so the
|
|
2793
|
+
// comment says the run continues there too, never "Reply here" (review round 2).
|
|
2794
|
+
if (parked) await comment(job, approvalParked ? APPROVAL_COMMENTS["awaiting-approval"] : taken?.taken === "here" || taken?.taken === "other" || Number(taken?.resumed) > 0 ? HELD_TAKEN_COMMENT : parkedComment(job.target));
|
|
1538
2795
|
return {
|
|
1539
2796
|
outcome: "completed",
|
|
1540
|
-
...(parked ? { reason: "awaiting-input" } : {}),
|
|
2797
|
+
...(parked ? { reason: approvalParked ? "awaiting-approval" : "awaiting-input" } : {}),
|
|
2798
|
+
...(approvalFacts ? { approval: approvalFacts } : {}),
|
|
1541
2799
|
exitCode: code,
|
|
1542
2800
|
turns,
|
|
1543
2801
|
tokens,
|
|
@@ -1556,6 +2814,7 @@ export async function runJob(job, deps) {
|
|
|
1556
2814
|
// record's `plan` is null otherwise, and every other result is unchanged.
|
|
1557
2815
|
...(plan ? { plan } : {}),
|
|
1558
2816
|
...(resources ? { resources } : {}),
|
|
2817
|
+
...(progress ? { progress } : {}),
|
|
1559
2818
|
};
|
|
1560
2819
|
}
|
|
1561
2820
|
case EXIT_POLICY: {
|
|
@@ -1582,7 +2841,7 @@ export async function runJob(job, deps) {
|
|
|
1582
2841
|
// `why`. parseExitWhy keeps only a member of the closed COST_CAP_WHYS off a `cost-cap` line that said code 2,
|
|
1583
2842
|
// and it rides only beside a `cost-cap` reason, so a forged line can at worst name the wrong rule of three.
|
|
1584
2843
|
const why = reason === "cost-cap" && COST_CAP_WHYS.includes(exitWhy) ? { why: exitWhy } : {};
|
|
1585
|
-
return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...why, ...(resources ? { resources } : {}) };
|
|
2844
|
+
return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...why, ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
|
|
1586
2845
|
}
|
|
1587
2846
|
case EXIT_INFRA:
|
|
1588
2847
|
// NO comment on any infra throw, here or in the catch: an InfraRetry may be retried and
|
|
@@ -1590,7 +2849,7 @@ export async function runJob(job, deps) {
|
|
|
1590
2849
|
// the whole infra class lives at the terminal seam -- start.mjs's failed listener, guarded on
|
|
1591
2850
|
// BullMQ's own finishedOn -- which also catches the stall-kill and wait-gate paths this
|
|
1592
2851
|
// function never sees (issue #288).
|
|
1593
|
-
|
|
2852
|
+
return await recoverOrThrow(new InfraRetry(`infra failure, container exit ${code}`, { reason: CONTAINER_INFRA_EXIT, exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources }));
|
|
1594
2853
|
default:
|
|
1595
2854
|
// THE RUNTIME NEVER HANDED CONTROL TO THE RUNNER, in whatever integers this venue spells that
|
|
1596
2855
|
// (issue #227). For docker it is 125 (`docker run` itself failed), 126 (the entrypoint exists
|
|
@@ -1608,7 +2867,7 @@ export async function runJob(job, deps) {
|
|
|
1608
2867
|
// No `dollars` here: the hold is still standing, and the catch refunds it whole with the job-count slots.
|
|
1609
2868
|
throw new InfraRetry(`the runtime could not start the container, exit ${code}`, { reason: "container-never-started", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), resources });
|
|
1610
2869
|
}
|
|
1611
|
-
|
|
2870
|
+
return await recoverOrThrow(new InfraRetry(`unknown container exit ${code}`, { reason: CONTAINER_EXIT_UNKNOWN, exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources }));
|
|
1612
2871
|
}
|
|
1613
2872
|
} catch (e) {
|
|
1614
2873
|
// A CONFIG-tagged throw is a determinate policy refusal wearing an exception, and issue #310 is the
|
|
@@ -1642,14 +2901,26 @@ export async function runJob(job, deps) {
|
|
|
1642
2901
|
const hold = dollarHold;
|
|
1643
2902
|
dollarHold = null;
|
|
1644
2903
|
if ((e?.piDispatchConfig === true && !containerRan) || isNeverStartedRetry(e)) {
|
|
1645
|
-
|
|
2904
|
+
// Issue #645: a RECORDED hold is given back only by the call that moves it `held` -> `released` in the run's
|
|
2905
|
+
// ledger; any other answer means someone else finished it or the ledger cannot say, and giving it back again
|
|
2906
|
+
// could refund money twice, so it stays at the floor. An unrecorded hold (its record failed) is given back
|
|
2907
|
+
// directly, as every hold was before: no one else can see it.
|
|
2908
|
+
const releasable = !holdRecorded || (await runBudgetLedger.release(runId, durable.seq)) === "moved";
|
|
2909
|
+
if (!releasable) log("run_budget_release_skipped", { seq: durable.seq });
|
|
2910
|
+
const { applied, of } = releasable ? await releaseDollars(redis, hold, { log }) : { applied: 0, of: 1 };
|
|
1646
2911
|
const heldModel = (hold.ledgers ?? []).some((l) => modelPrefixesOf(modelDollars).has(l.keyPrefix));
|
|
1647
|
-
|
|
2912
|
+
// Issue #761: along the whole hold, which the release walked: a key it gave back holds 0 of this job's money, so
|
|
2913
|
+
// a release that stopped part-way (the record then states the whole reservation) leaves those windows out.
|
|
2914
|
+
const reservedIn = deploymentWindowsOf(hold, { applied, target: 0, recorded: applied === of ? 0 : hold.amountMicros, valkey: valkeyTag });
|
|
2915
|
+
dollars = dollarsRecord({ reservedMicros: hold.amountMicros, settledMicros: applied === of ? 0 : hold.amountMicros, basis: applied === of ? "refunded" : "floor", modelBasis: heldModel ? (applied === of ? "refunded" : "floor") : null, reservedIn });
|
|
1648
2916
|
} else {
|
|
1649
2917
|
log("dollar_hold_unsettled", { reservedMicros: hold.amountMicros, error: e instanceof InfraRetry ? "infra-retry" : (e?.name ?? "error") });
|
|
1650
2918
|
const heldModel = (hold.ledgers ?? []).some((l) => modelPrefixesOf(modelDollars).has(l.keyPrefix));
|
|
1651
|
-
|
|
2919
|
+
// Issue #761: nothing moved a key, so each holds the whole reservation, which is what this record states.
|
|
2920
|
+
dollars = dollarsRecord({ reservedMicros: hold.amountMicros, settledMicros: hold.amountMicros, basis: "floor", modelBasis: heldModel ? "floor" : null, reservedIn: deploymentWindowsOf(hold, { applied: 0, target: hold.amountMicros, recorded: hold.amountMicros, valkey: valkeyTag }) });
|
|
1652
2921
|
}
|
|
2922
|
+
// A failed give-back becomes a settled floor, not a running job's reservation.
|
|
2923
|
+
publishDollarHold(null);
|
|
1653
2924
|
}
|
|
1654
2925
|
// The record reads `dollars` off the error on every throw path (buildRecord), so a retried or failed attempt says
|
|
1655
2926
|
// what its windows were charged.
|
|
@@ -1660,6 +2931,14 @@ export async function runJob(job, deps) {
|
|
|
1660
2931
|
// a frozen error keeps its own fields; the log lines above are the record of the hold
|
|
1661
2932
|
}
|
|
1662
2933
|
}
|
|
2934
|
+
// Issue #644: the same for the run's progress, so a retried or failed attempt's record says where it got to.
|
|
2935
|
+
if (progress !== null && e !== null && typeof e === "object" && e.progress === undefined) {
|
|
2936
|
+
try {
|
|
2937
|
+
e.progress = progress;
|
|
2938
|
+
} catch {
|
|
2939
|
+
// a frozen error keeps its own fields; progress is display only
|
|
2940
|
+
}
|
|
2941
|
+
}
|
|
1663
2942
|
|
|
1664
2943
|
if (e?.piDispatchConfig === true && !containerRan) {
|
|
1665
2944
|
// GUARDED, and the `held` list is the record of what actually happened. `releaseLedgers` is a loop of
|
|
@@ -1682,7 +2961,7 @@ export async function runJob(job, deps) {
|
|
|
1682
2961
|
// classification that has ALREADY released the budget, and the job would escape as whatever the
|
|
1683
2962
|
// comment threw, with the ledger refunded and the record saying something else entirely. The
|
|
1684
2963
|
// shipped adapter never throws; this makes that a property of the arm rather than of the wiring.
|
|
1685
|
-
await comment(job, "
|
|
2964
|
+
await comment(job, REFUSAL_COMMENTS["config-refused"]).catch(() => {});
|
|
1686
2965
|
log("refused_config", { kind: job.kind ?? null, refunded });
|
|
1687
2966
|
// budgetReserved reflects the LEDGER: whether the GLOBAL slot is still held after the refund (`globalHeld`).
|
|
1688
2967
|
return { outcome: "policy", reason: "config-refused", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: globalHeld(), ...(dollars ? { dollars } : {}) }; // return => not retried
|
|
@@ -1693,10 +2972,13 @@ export async function runJob(job, deps) {
|
|
|
1693
2972
|
// here (exit-1 infra, unknown exit) means the container ran and legitimately spent its slot,
|
|
1694
2973
|
// so `reason` gates the release to the never-started case only. It releases only what `held` still
|
|
1695
2974
|
// lists and run once per invocation; a BullMQ retry reserves afresh, so this cannot double-release.
|
|
1696
|
-
if (isNeverStartedRetry(e)) {
|
|
2975
|
+
if (isNeverStartedRetry(e) && !slotOwnedByRun) {
|
|
1697
2976
|
// All-or-none (issues #242 and #499 part B): a never-started container follows EVERY job-count reserve, so
|
|
1698
2977
|
// every held ledger refunds together, last first, through the one helper -- and a scoped or project refusal
|
|
1699
2978
|
// returned above with the ledgers before it already given back.
|
|
2979
|
+
// NOT for an armed run's slot (issue #645, `slotOwnedByRun`): the run's ledger says the slot is held, so the
|
|
2980
|
+
// retry carries it instead of reserving; refunding it here would let that retry run on a slot nothing counts.
|
|
2981
|
+
// The cost: a run whose every attempt never started keeps one slot counted, the safe side.
|
|
1700
2982
|
await refundLedgers("container-never-started");
|
|
1701
2983
|
}
|
|
1702
2984
|
// After the refund, so it says what the ledger holds: false when never-started gave the global slot back, true
|
|
@@ -1704,18 +2986,13 @@ export async function runJob(job, deps) {
|
|
|
1704
2986
|
if (e instanceof InfraRetry) e.budgetReserved = globalHeld();
|
|
1705
2987
|
throw e;
|
|
1706
2988
|
} finally {
|
|
2989
|
+
if (prepared && job.watch) await collectWatch({ job, prepared, completed: watchCompleted, policyRefused: watchPolicyRefused, usage: watchUsage }).catch(() => {});
|
|
2990
|
+
if (prepared && job.questionGrouping) await deps.questionGroupRuntime?.collect({ job, prepared, completed: watchCompleted }).catch(() => {});
|
|
2991
|
+
if (prepared && job.answerDraft) await deps.answerDraftRuntime?.collect({ job, prepared, completed: watchCompleted, result: answerUsage }).catch(() => {});
|
|
1707
2992
|
if (prepared) await cleanup(prepared).catch(() => {});
|
|
1708
2993
|
}
|
|
1709
2994
|
}
|
|
1710
2995
|
|
|
1711
|
-
/**
|
|
1712
|
-
* The reason a job retried for the podman venue's rootless network keeper carries (issue #458, PR #463 round 2): a
|
|
1713
|
-
* fixed token, as every run-record reason is, and the key the terminal comment is chosen by.
|
|
1714
|
-
*/
|
|
1715
|
-
export const NETNS_KEEPER_NOT_HOLDING = "netns-keeper-not-holding";
|
|
1716
|
-
/** Issue #476: a job held on a young keeper that kept restarting, or stayed young past the hold's bound. */
|
|
1717
|
-
export const NETNS_KEEPER_CRASH_LOOP = "netns-keeper-crash-loop";
|
|
1718
|
-
|
|
1719
2996
|
/** Thrown for the retryable (infra) class only. The BullMQ processor lets this propagate to retry. */
|
|
1720
2997
|
/**
|
|
1721
2998
|
* The one `session` object the run record carries, from the host's intent and the container's report
|
|
@@ -1736,6 +3013,13 @@ export const NETNS_KEEPER_CRASH_LOOP = "netns-keeper-crash-loop";
|
|
|
1736
3013
|
function mergeSession(prepared, fromRunner, promoted = null) {
|
|
1737
3014
|
const host = prepared?.session;
|
|
1738
3015
|
if (!host && !fromRunner) return null;
|
|
3016
|
+
// Issue #643: a job that is not a resume job now runs on a live transcript file (`prepared.live`), which the runner
|
|
3017
|
+
// opens empty and reports as `absent`. That is a fact about the live file, not about resuming, and this object
|
|
3018
|
+
// describes resuming: such a job's record says `disabled`, exactly what the runner said for it before the live file
|
|
3019
|
+
// existed, so its record is byte-identical to one written before the feature. Narrow on purpose, the live file and
|
|
3020
|
+
// the runner's token exactly `absent`: any other report is carried as it always was, and resume jobs (`host`
|
|
3021
|
+
// present) never reach this line.
|
|
3022
|
+
if (!host && prepared?.live && fromRunner?.resumed === false && fromRunner.reason === "absent") return { resumed: false, reason: "disabled", bytes: null };
|
|
1739
3023
|
// A HOST GATE THAT REFUSED OUTRANKS THE RUNNER'S `absent`, and without this rule it never reached a
|
|
1740
3024
|
// record at all. A refused read stages a 0-byte file rather than nothing (session-store.mjs, where the
|
|
1741
3025
|
// reasoning is pi's EEXIST race), the container is handed that file either way, and pi opens it and
|
|
@@ -1777,6 +3061,25 @@ const SECRET_FAILURES = {
|
|
|
1777
3061
|
nul: "printed a value containing a NUL byte, which cannot survive the container's argv",
|
|
1778
3062
|
};
|
|
1779
3063
|
|
|
3064
|
+
/**
|
|
3065
|
+
* The bundle resolver's failures (issue #704, secret-bundles.mjs), our words, the `SECRET_FAILURES` rule: an unknown
|
|
3066
|
+
* code degrades to the caller's generic sentence. None names a variable or a value: this text is posted publicly.
|
|
3067
|
+
*/
|
|
3068
|
+
const BUNDLE_FAILURES = {
|
|
3069
|
+
exit: "refused the bundle's reference",
|
|
3070
|
+
empty: "printed no variable",
|
|
3071
|
+
overflow: "printed more than the bundle size cap allows",
|
|
3072
|
+
nul: "printed output containing a NUL byte",
|
|
3073
|
+
malformed: "printed output that is not in the format its profile declares",
|
|
3074
|
+
quoted: "printed a quoted dotenv value, and the dotenv grammar takes no quotes (declare the profile as json instead)",
|
|
3075
|
+
"not-string": "printed a JSON value that is not a string",
|
|
3076
|
+
"bad-name": "printed a name that is not an environment variable name",
|
|
3077
|
+
duplicate: "printed one variable twice",
|
|
3078
|
+
"empty-value": "printed a variable with an empty value",
|
|
3079
|
+
"value-too-large": "printed a value larger than one secret may be",
|
|
3080
|
+
"too-many": "printed more variables than one bundle may carry",
|
|
3081
|
+
};
|
|
3082
|
+
|
|
1780
3083
|
/**
|
|
1781
3084
|
* Is this throw the "the runner never ran" retry (issue #227), whichever path raised it: a never-started exit
|
|
1782
3085
|
* (`isNeverStartedExit`), a spawn fault, or a pre-start gate? The catch's refunds (the job-count slots and, issue
|
|
@@ -1787,10 +3090,13 @@ function isNeverStartedRetry(e) {
|
|
|
1787
3090
|
}
|
|
1788
3091
|
|
|
1789
3092
|
export class InfraRetry extends Error {
|
|
1790
|
-
constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved, dollars, resources } = {}) {
|
|
3093
|
+
constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved, dollars, resources, loaded } = {}) {
|
|
1791
3094
|
super(message, cause ? { cause } : undefined);
|
|
1792
3095
|
this.name = "InfraRetry";
|
|
1793
3096
|
this.piDispatchRetry = true;
|
|
3097
|
+
// The message default is never recorded or passed on as text: the record's door (`recordReasonOf`) records a
|
|
3098
|
+
// reason that is no token as `worker-error`, and the failure hook flattens one to `infra`. A throw that knows its
|
|
3099
|
+
// cause passes a token.
|
|
1794
3100
|
this.reason = reason ?? message;
|
|
1795
3101
|
this.exitCode = exitCode ?? null;
|
|
1796
3102
|
this.turns = turns ?? null;
|
|
@@ -1811,6 +3117,9 @@ export class InfraRetry extends Error {
|
|
|
1811
3117
|
this.dollars = dollars ?? null;
|
|
1812
3118
|
// Issue #596: what the container used, off its exit line, or null; set on a throw after a container ran.
|
|
1813
3119
|
this.resources = resources ?? null;
|
|
3120
|
+
// Issue #800: what the run loaded, for a throw that REPLACES another (`runJob` sets it on the first), or that comes out
|
|
3121
|
+
// of the run's wrapper before the processor noted it. Absent otherwise, so an error that holds none says so.
|
|
3122
|
+
if (loaded !== undefined && loaded !== null) this.loaded = loaded;
|
|
1814
3123
|
}
|
|
1815
3124
|
}
|
|
1816
3125
|
|
|
@@ -1841,4 +3150,72 @@ export class NetnsKeeperYoungHold extends InfraRetry {
|
|
|
1841
3150
|
}
|
|
1842
3151
|
}
|
|
1843
3152
|
|
|
3153
|
+
/**
|
|
3154
|
+
* A recovery job held while the run it continues still has a container running on this host (issue #645). An
|
|
3155
|
+
* `InfraRetry` with `container-never-started`, so any path that does not know it still retries and refunds nothing it did
|
|
3156
|
+
* not take; `makeProcessor` knows it by `recoveryWaitMs`, and moves the job to the delayed set for that long without
|
|
3157
|
+
* spending an attempt (a recovery job has one), the move the two holds above make.
|
|
3158
|
+
*/
|
|
3159
|
+
/**
|
|
3160
|
+
* A job whose project's secret bundle names a profile this host does not serve, on a multi-host worker's SHARED queue
|
|
3161
|
+
* (issue #704 review). An `InfraRetry`, so any path that does not know it still retries; `makeProcessor` knows it by
|
|
3162
|
+
* `bundleProfileHold`, and moves the job to the delayed set for the never-fits gate's recheck without spending an
|
|
3163
|
+
* attempt, so a host that declares the profile can take it.
|
|
3164
|
+
*/
|
|
3165
|
+
export class SecretBundleProfileHold extends InfraRetry {
|
|
3166
|
+
constructor(message, options) {
|
|
3167
|
+
super(message, options);
|
|
3168
|
+
this.name = "SecretBundleProfileHold";
|
|
3169
|
+
this.bundleProfileHold = true;
|
|
3170
|
+
}
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3173
|
+
/**
|
|
3174
|
+
* The HTTP status of a failed pull-request head read, or null: octokit's `status`, else the `returned <code>` the
|
|
3175
|
+
* fetch-based forge hosts put in their InfraRetry message (status only, never the body). Read only to tell a
|
|
3176
|
+
* determinate 401/403/404 from a fault worth retrying.
|
|
3177
|
+
*/
|
|
3178
|
+
function headReadStatus(error) {
|
|
3179
|
+
if (Number.isInteger(error?.status)) return error.status;
|
|
3180
|
+
const m = /\breturned (\d{3})\b/.exec(String(error?.message ?? ""));
|
|
3181
|
+
return m ? Number(m[1]) : null;
|
|
3182
|
+
}
|
|
3183
|
+
|
|
3184
|
+
/**
|
|
3185
|
+
* Whether a failed head read is a RATE LIMIT (issue #704 review round 2): octokit's error carries the response headers,
|
|
3186
|
+
* and GitHub's primary limit answers 403 with `x-ratelimit-remaining: 0`, its secondary limit 403 or 429 with a
|
|
3187
|
+
* `retry-after`. A header is read by name only, never echoed. The fetch-based hosts surface no headers, and their
|
|
3188
|
+
* 429 is already outside the determinate set.
|
|
3189
|
+
*/
|
|
3190
|
+
function headReadRateLimited(error) {
|
|
3191
|
+
const headers = error?.response?.headers;
|
|
3192
|
+
if (headers === null || typeof headers !== "object") return false;
|
|
3193
|
+
const get = (name) => (typeof headers.get === "function" ? headers.get(name) : headers[name]);
|
|
3194
|
+
const remaining = get("x-ratelimit-remaining");
|
|
3195
|
+
return String(remaining ?? "") === "0" || (get("retry-after") !== undefined && get("retry-after") !== null);
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
/**
|
|
3199
|
+
* A job this worker must not run because something it reads was written by a newer pi-dispatch: the triggers file a
|
|
3200
|
+
* free gate reads live, or a checkpoint (`DES-STORE-VERSION-WHEN-NEEDED`). The processor's catch DEFERS it
|
|
3201
|
+
* (`newer-release`) rather than count an attempt, so the job waits for a worker that can read it, spending nothing.
|
|
3202
|
+
* `what` names the store, for the log line only.
|
|
3203
|
+
*/
|
|
3204
|
+
export class NewerReleaseHold extends InfraRetry {
|
|
3205
|
+
constructor(what) {
|
|
3206
|
+
super(`${what} was written by a newer pi-dispatch`, { budgetReserved: false });
|
|
3207
|
+
this.name = "NewerReleaseHold";
|
|
3208
|
+
this.newerRelease = true;
|
|
3209
|
+
this.what = what;
|
|
3210
|
+
}
|
|
3211
|
+
}
|
|
3212
|
+
|
|
3213
|
+
export class RecoveryWaitHold extends InfraRetry {
|
|
3214
|
+
constructor(message, { waitMs = RECOVERY_WAIT_MS, ...options } = {}) {
|
|
3215
|
+
super(message, options);
|
|
3216
|
+
this.name = "RecoveryWaitHold";
|
|
3217
|
+
this.recoveryWaitMs = waitMs;
|
|
3218
|
+
}
|
|
3219
|
+
}
|
|
3220
|
+
|
|
1844
3221
|
export { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY };
|