@tokenfactory/acc-runner 0.44.0 → 0.44.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -1
- package/dist/bin-resolve.d.ts +39 -3
- package/dist/bin-resolve.d.ts.map +1 -1
- package/dist/bin-resolve.js +176 -12
- package/dist/bin-resolve.js.map +1 -1
- package/dist/build-fingerprint.d.ts +66 -0
- package/dist/build-fingerprint.d.ts.map +1 -0
- package/dist/build-fingerprint.js +70 -0
- package/dist/build-fingerprint.js.map +1 -0
- package/dist/build-info.generated.d.ts +3 -0
- package/dist/build-info.generated.d.ts.map +1 -0
- package/dist/build-info.generated.js +3 -0
- package/dist/build-info.generated.js.map +1 -0
- package/dist/capacity/resume-controller.d.ts +5 -3
- package/dist/capacity/resume-controller.d.ts.map +1 -1
- package/dist/capacity/resume-controller.js +9 -0
- package/dist/capacity/resume-controller.js.map +1 -1
- package/dist/chat-delegation-guard.d.ts +23 -0
- package/dist/chat-delegation-guard.d.ts.map +1 -0
- package/dist/chat-delegation-guard.js +62 -0
- package/dist/chat-delegation-guard.js.map +1 -0
- package/dist/chat-lane.d.ts +17 -6
- package/dist/chat-lane.d.ts.map +1 -1
- package/dist/chat-lane.js +29 -8
- package/dist/chat-lane.js.map +1 -1
- package/dist/companion-identity.d.ts +37 -0
- package/dist/companion-identity.d.ts.map +1 -1
- package/dist/companion-identity.js +98 -29
- package/dist/companion-identity.js.map +1 -1
- package/dist/companion-run.d.ts +1 -50
- package/dist/companion-run.d.ts.map +1 -1
- package/dist/companion-run.js +48 -105
- package/dist/companion-run.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -3
- package/dist/config.js.map +1 -1
- package/dist/conformance.d.ts +49 -0
- package/dist/conformance.d.ts.map +1 -0
- package/dist/conformance.js +121 -0
- package/dist/conformance.js.map +1 -0
- package/dist/cost-pricing.d.ts +91 -0
- package/dist/cost-pricing.d.ts.map +1 -1
- package/dist/cost-pricing.js +115 -1
- package/dist/cost-pricing.js.map +1 -1
- package/dist/credential-identity.d.ts +70 -0
- package/dist/credential-identity.d.ts.map +1 -0
- package/dist/credential-identity.js +128 -0
- package/dist/credential-identity.js.map +1 -0
- package/dist/doctor.d.ts +37 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +300 -41
- package/dist/doctor.js.map +1 -1
- package/dist/engines/chat-stream.d.ts +71 -1
- package/dist/engines/chat-stream.d.ts.map +1 -1
- package/dist/engines/chat-stream.js +126 -8
- package/dist/engines/chat-stream.js.map +1 -1
- package/dist/engines/claude-code-chat.d.ts +11 -1
- package/dist/engines/claude-code-chat.d.ts.map +1 -1
- package/dist/engines/claude-code-chat.js +69 -5
- package/dist/engines/claude-code-chat.js.map +1 -1
- package/dist/engines/claude-code.d.ts.map +1 -1
- package/dist/engines/claude-code.js +30 -6
- package/dist/engines/claude-code.js.map +1 -1
- package/dist/engines/codex.d.ts.map +1 -1
- package/dist/engines/codex.js +18 -5
- package/dist/engines/codex.js.map +1 -1
- package/dist/engines/registry.d.ts +26 -0
- package/dist/engines/registry.d.ts.map +1 -1
- package/dist/engines/registry.js +37 -0
- package/dist/engines/registry.js.map +1 -1
- package/dist/engines/types.d.ts +5 -0
- package/dist/engines/types.d.ts.map +1 -1
- package/dist/failure-classifier.d.ts +1 -1
- package/dist/failure-classifier.d.ts.map +1 -1
- package/dist/failure-classifier.js +65 -12
- package/dist/failure-classifier.js.map +1 -1
- package/dist/gh.d.ts +15 -0
- package/dist/gh.d.ts.map +1 -1
- package/dist/gh.js +18 -0
- package/dist/gh.js.map +1 -1
- package/dist/git.d.ts +6 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +27 -1
- package/dist/git.js.map +1 -1
- package/dist/instance.d.ts +30 -0
- package/dist/instance.d.ts.map +1 -0
- package/dist/instance.js +51 -0
- package/dist/instance.js.map +1 -0
- package/dist/keychain.d.ts +20 -7
- package/dist/keychain.d.ts.map +1 -1
- package/dist/keychain.js +40 -4
- package/dist/keychain.js.map +1 -1
- package/dist/loc-advisory.d.ts +70 -0
- package/dist/loc-advisory.d.ts.map +1 -0
- package/dist/loc-advisory.js +196 -0
- package/dist/loc-advisory.js.map +1 -0
- package/dist/locks/claim-lock-policy.d.ts +67 -0
- package/dist/locks/claim-lock-policy.d.ts.map +1 -0
- package/dist/locks/claim-lock-policy.js +132 -0
- package/dist/locks/claim-lock-policy.js.map +1 -0
- package/dist/login.d.ts +63 -2
- package/dist/login.d.ts.map +1 -1
- package/dist/login.js +93 -8
- package/dist/login.js.map +1 -1
- package/dist/mcp-spawn.d.ts +54 -0
- package/dist/mcp-spawn.d.ts.map +1 -1
- package/dist/mcp-spawn.js +88 -4
- package/dist/mcp-spawn.js.map +1 -1
- package/dist/memory-context.d.ts +106 -0
- package/dist/memory-context.d.ts.map +1 -0
- package/dist/memory-context.js +220 -0
- package/dist/memory-context.js.map +1 -0
- package/dist/memory-curation.d.ts +172 -0
- package/dist/memory-curation.d.ts.map +1 -0
- package/dist/memory-curation.js +277 -0
- package/dist/memory-curation.js.map +1 -0
- package/dist/migration-seq-guard.d.ts +28 -1
- package/dist/migration-seq-guard.d.ts.map +1 -1
- package/dist/migration-seq-guard.js +111 -6
- package/dist/migration-seq-guard.js.map +1 -1
- package/dist/output-diagnostics.d.ts +15 -0
- package/dist/output-diagnostics.d.ts.map +1 -0
- package/dist/output-diagnostics.js +95 -0
- package/dist/output-diagnostics.js.map +1 -0
- package/dist/pr-open-guard.d.ts +67 -0
- package/dist/pr-open-guard.d.ts.map +1 -0
- package/dist/pr-open-guard.js +79 -0
- package/dist/pr-open-guard.js.map +1 -0
- package/dist/prepr-gate.d.ts +229 -1
- package/dist/prepr-gate.d.ts.map +1 -1
- package/dist/prepr-gate.js +406 -3
- package/dist/prepr-gate.js.map +1 -1
- package/dist/program.d.ts.map +1 -1
- package/dist/program.js +43 -5
- package/dist/program.js.map +1 -1
- package/dist/provider-auth.d.ts +27 -0
- package/dist/provider-auth.d.ts.map +1 -1
- package/dist/provider-auth.js +33 -0
- package/dist/provider-auth.js.map +1 -1
- package/dist/runtime/cache-dir.d.ts.map +1 -1
- package/dist/runtime/cache-dir.js +8 -3
- package/dist/runtime/cache-dir.js.map +1 -1
- package/dist/runtime/decision-provenance.d.ts +63 -0
- package/dist/runtime/decision-provenance.d.ts.map +1 -0
- package/dist/runtime/decision-provenance.js +107 -0
- package/dist/runtime/decision-provenance.js.map +1 -0
- package/dist/runtime/prompt-substitute.d.ts +100 -0
- package/dist/runtime/prompt-substitute.d.ts.map +1 -0
- package/dist/runtime/prompt-substitute.js +124 -0
- package/dist/runtime/prompt-substitute.js.map +1 -0
- package/dist/runtime/quarantine.d.ts +11 -0
- package/dist/runtime/quarantine.d.ts.map +1 -1
- package/dist/runtime/quarantine.js +16 -1
- package/dist/runtime/quarantine.js.map +1 -1
- package/dist/runtime/review-cost-cap.d.ts +167 -0
- package/dist/runtime/review-cost-cap.d.ts.map +1 -0
- package/dist/runtime/review-cost-cap.js +182 -0
- package/dist/runtime/review-cost-cap.js.map +1 -0
- package/dist/runtime/reviewer-abstain.d.ts +101 -0
- package/dist/runtime/reviewer-abstain.d.ts.map +1 -0
- package/dist/runtime/reviewer-abstain.js +138 -0
- package/dist/runtime/reviewer-abstain.js.map +1 -0
- package/dist/runtime/reviewer.d.ts +165 -2
- package/dist/runtime/reviewer.d.ts.map +1 -1
- package/dist/runtime/reviewer.js +752 -111
- package/dist/runtime/reviewer.js.map +1 -1
- package/dist/runtime/running-build.d.ts +24 -0
- package/dist/runtime/running-build.d.ts.map +1 -0
- package/dist/runtime/running-build.js +62 -0
- package/dist/runtime/running-build.js.map +1 -0
- package/dist/runtime/singleton.d.ts.map +1 -1
- package/dist/runtime/singleton.js +9 -1
- package/dist/runtime/singleton.js.map +1 -1
- package/dist/runtime/worktree.d.ts +45 -0
- package/dist/runtime/worktree.d.ts.map +1 -1
- package/dist/runtime/worktree.js +174 -5
- package/dist/runtime/worktree.js.map +1 -1
- package/dist/serving-auth/index.d.ts +79 -0
- package/dist/serving-auth/index.d.ts.map +1 -0
- package/dist/serving-auth/index.js +169 -0
- package/dist/serving-auth/index.js.map +1 -0
- package/dist/task-runner.d.ts +134 -5
- package/dist/task-runner.d.ts.map +1 -1
- package/dist/task-runner.js +1255 -129
- package/dist/task-runner.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watch-chat/wire.d.ts +80 -0
- package/dist/watch-chat/wire.d.ts.map +1 -1
- package/dist/watch-chat/wire.js +203 -43
- package/dist/watch-chat/wire.js.map +1 -1
- package/dist/watch.d.ts +70 -16
- package/dist/watch.d.ts.map +1 -1
- package/dist/watch.js +548 -108
- package/dist/watch.js.map +1 -1
- package/package.json +4 -3
package/dist/watch.js
CHANGED
|
@@ -32,8 +32,10 @@ import { applyBootstrapConfig, concurrencyEnvExplicit, loadConfig, resolveEffect
|
|
|
32
32
|
// existing tests that import them from `../src/watch.js` keep resolving.
|
|
33
33
|
export { resolveConcurrencyLimit, resolveReservedReviewSlots } from "./config.js";
|
|
34
34
|
import { getTokenProvider } from "./token-provider.js";
|
|
35
|
-
import { machineIdentity, detectInstalledModels, runnerCapsWithRepos } from "./login.js";
|
|
35
|
+
import { advertisedCapabilities, machineIdentity, detectInstalledModels, runnerCapsWithRepos, registerRunnerCaps, } from "./login.js";
|
|
36
36
|
import { detectCapableEngines } from "./doctor.js";
|
|
37
|
+
import { safeText } from "./machine-capabilities/redact.js";
|
|
38
|
+
import { resolveCredentialIdentity } from "./credential-identity.js";
|
|
37
39
|
import { createRunnerClient } from "./supabase.js";
|
|
38
40
|
import { releaseHeldLocks } from "./locks/index.js";
|
|
39
41
|
// SH-4 (PAUSE-CHECKPOINT-RESUME): the additive task_paused broadcast + the
|
|
@@ -42,7 +44,7 @@ import { TASK_PAUSED_EVENT, captureWorktreeCheckpoint, parseTaskPausedBroadcast,
|
|
|
42
44
|
import { worktreePath } from "./runtime/worktree.js";
|
|
43
45
|
import { runTask } from "./task-runner.js";
|
|
44
46
|
import { runReview, } from "./runtime/reviewer.js";
|
|
45
|
-
import { getQuarantine, setQuarantine, clearQuarantine, } from "./runtime/quarantine.js";
|
|
47
|
+
import { getQuarantine, publishedQuarantineState, setQuarantine, clearQuarantine, } from "./runtime/quarantine.js";
|
|
46
48
|
import { acquireSingletonLock, singletonRunnerId, SingletonLockHeldError, } from "./runtime/singleton.js";
|
|
47
49
|
import { getClaudeVersion, recordTaskClaudeVersion, } from "./runtime/version-drift.js";
|
|
48
50
|
import { autoUpgradeDisabled, maybeSelfUpgrade, maybeSelfUpdateOn426, } from "./runtime/self-upgrade.js";
|
|
@@ -62,7 +64,7 @@ import { AccountProbeScheduler } from "./engines/account-probe.js";
|
|
|
62
64
|
// AX-H1 (MACHINE-CAPABILITIES): the host-capability probe + reporter. Driven by
|
|
63
65
|
// the AS-1 account-probe cadence above, never by a loop of its own.
|
|
64
66
|
import { MachineCapabilityReporter, probeMachineCapabilities, } from "./machine-capabilities/index.js";
|
|
65
|
-
import {
|
|
67
|
+
import { getConfiguredChatEngine, getEngine } from "./engines/registry.js";
|
|
66
68
|
// FLEET-SERVE (wire): fleet-fallback chat serving runs ALONGSIDE the task pump on
|
|
67
69
|
// a `watch` runner — the runner serves its bound user's chat turns behind their
|
|
68
70
|
// companion. Glue lives in watch-chat/wire.ts to keep this file the task loop.
|
|
@@ -71,13 +73,20 @@ import { ClaimDeadman, readDispatchSignal, DEFAULT_DEADMAN_CHECK_MS, DEFAULT_FAL
|
|
|
71
73
|
import { unknownAccountIdentity } from "./engines/account-identity.js";
|
|
72
74
|
import { checkVersion, compareSemver } from "./version-check.js";
|
|
73
75
|
import { PACKAGE_VERSION } from "./pkg-version.js";
|
|
76
|
+
import { buildFingerprintCaps, publishBuildFingerprintCaps, LOADED_BUILD_FINGERPRINT, } from "./build-fingerprint.js";
|
|
77
|
+
import { recordRunningBuild } from "./runtime/running-build.js";
|
|
74
78
|
import { authPrecedenceHint, servingAuthKind, describeServingAuthKind, forwardsAmbientKey, anthropicApiKey, } from "./provider-auth.js";
|
|
79
|
+
// RUNNER-SERVING-AUTH: the org's subscription-vs-BYOK choice, resolved once
|
|
80
|
+
// per boot and expressed through the same ACC_RUNNER_SERVING_AUTH_KIND var the
|
|
81
|
+
// imports above already fold.
|
|
82
|
+
import { fetchServingAuth, installServingAuth, servingAuthKindIsPinned, describeServingAuthResolution, } from "./serving-auth/index.js";
|
|
75
83
|
// AUTH-REFRESH: the refresh + heartbeat-classification
|
|
76
84
|
// primitives now live in a shared module so the companion runner reuses them
|
|
77
85
|
// verbatim. Re-exported below to keep watch.js's existing test-facing surface.
|
|
78
86
|
import { RefreshReuseDetectedError, classifyHeartbeatError, expiresSoon, nextHeartbeatDelayMs, refreshAccessToken, } from "./runtime/auth-lifecycle.js";
|
|
79
87
|
// AU-7 RUNNER-FLAP-ROOT-CAUSE: bounded + write-then-read-verified heartbeat.
|
|
80
88
|
import { HEARTBEAT_RPC_TIMEOUT_MS, HeartbeatVerifier, nextBeatDelayMs, sendHeartbeat, withDeadline, } from "./heartbeat/beat.js";
|
|
89
|
+
import { buildEnv, requireExecutable } from "./bin-resolve.js";
|
|
81
90
|
export { RefreshReuseDetectedError, classifyHeartbeatError, nextHeartbeatDelayMs, };
|
|
82
91
|
const REFRESH_CHECK_MS = 30 * 60 * 1000;
|
|
83
92
|
// v0.12.0 (T-52-7): idle-TTL check cadence ceiling. The actual cadence is
|
|
@@ -222,6 +231,21 @@ export function resolveIdleTtlMs(env = process.env) {
|
|
|
222
231
|
}
|
|
223
232
|
return minutes * 60_000;
|
|
224
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* The DEFAULT reviewer factory. FAIL CLOSED: constrain every review to the engines
|
|
236
|
+
* THIS runner ALREADY PUBLISHED into `acc.runners.caps.engines` — the value the
|
|
237
|
+
* assignment was routed against — threaded as a FIXED `advertisedEngineIds`.
|
|
238
|
+
* `advertisedEngines` is resolved ONCE at watch startup from the same
|
|
239
|
+
* `detectCapableEngines()` source that feeds the register_runner caps payload, so
|
|
240
|
+
* it is a stable snapshot. It deliberately does NOT pass a live `detectEngines`
|
|
241
|
+
* probe: a per-review re-probe could drift from the published caps (an engine
|
|
242
|
+
* appearing/disappearing mid-session) and silently route a review to an engine the
|
|
243
|
+
* assignment never assumed. `run` is injectable for the wiring test; production
|
|
244
|
+
* uses `runReview`.
|
|
245
|
+
*/
|
|
246
|
+
export function capsConstrainedReviewerFactory(advertisedEngines, run = runReview) {
|
|
247
|
+
return (assignment, deps) => run(assignment, { ...deps, advertisedEngineIds: advertisedEngines });
|
|
248
|
+
}
|
|
225
249
|
/** v0.12.0 (T-52-7): record work-related activity for the idle TTL clock. */
|
|
226
250
|
function touchActivity(state) {
|
|
227
251
|
state.lastActivityMs = Date.now();
|
|
@@ -238,15 +262,27 @@ function failOnReuse(err) {
|
|
|
238
262
|
}
|
|
239
263
|
// Single source of truth for "this task is new — push it into the runner
|
|
240
264
|
// queue". Both the Realtime listener and the polling loop go through here.
|
|
241
|
-
function enqueue(state, taskId, factory) {
|
|
265
|
+
function enqueue(state, taskId, factory, engine) {
|
|
242
266
|
if (state.seen.has(taskId))
|
|
243
267
|
return false;
|
|
268
|
+
// CLAIM-LOCK-GLOB: a task that lost the claim to a live
|
|
269
|
+
// file-lock holder and spent its fast attempts is held out of the queue until
|
|
270
|
+
// its bounded yield elapses (AC3). Skip re-driving it until then; the periodic
|
|
271
|
+
// claim-scan / backstop re-lists it once the yield expires. Expired stamps are
|
|
272
|
+
// dropped so the very next pass re-drives the task.
|
|
273
|
+
const yieldUntil = state.claimYieldUntil.get(taskId);
|
|
274
|
+
if (yieldUntil != null) {
|
|
275
|
+
if (Date.now() < yieldUntil)
|
|
276
|
+
return false;
|
|
277
|
+
state.claimYieldUntil.delete(taskId);
|
|
278
|
+
}
|
|
244
279
|
// v0.10 T-49-2: check the running Map (replaces v0.9 state.current check).
|
|
245
280
|
if (state.running.has(taskId)) {
|
|
246
281
|
state.seen.add(taskId);
|
|
247
282
|
return false;
|
|
248
283
|
}
|
|
249
284
|
state.seen.add(taskId);
|
|
285
|
+
state.taskEngines.set(taskId, capacityEngine(engine));
|
|
250
286
|
state.queue.push(taskId);
|
|
251
287
|
touchActivity(state);
|
|
252
288
|
// CLAIM-DEADMAN: a claim was processed via SOME path
|
|
@@ -259,6 +295,33 @@ function enqueue(state, taskId, factory) {
|
|
|
259
295
|
void pump(state, factory);
|
|
260
296
|
return true;
|
|
261
297
|
}
|
|
298
|
+
async function lookupAssignedQueuedTaskEngine(state, taskId) {
|
|
299
|
+
const { data, error } = await state.supabase.rpc("list_assigned_queued_tasks", {
|
|
300
|
+
p_runner_id: state.session.runner_id,
|
|
301
|
+
p_since: "1970-01-01T00:00:00.000Z",
|
|
302
|
+
});
|
|
303
|
+
if (error) {
|
|
304
|
+
process.stderr.write(`[acc-runner] task_assigned engine lookup failed for ${taskId}: ${error.message}\n`);
|
|
305
|
+
return { found: false };
|
|
306
|
+
}
|
|
307
|
+
const row = (data ?? []).find((candidate) => candidate?.id === taskId);
|
|
308
|
+
return row ? { found: true, engine: row.engine } : { found: false };
|
|
309
|
+
}
|
|
310
|
+
async function enqueueAssignedTaskBroadcast(state, taskId, factory, payload) {
|
|
311
|
+
if (Object.prototype.hasOwnProperty.call(payload, "engine")) {
|
|
312
|
+
enqueue(state, taskId, factory, payload.engine);
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
const resolved = await lookupAssignedQueuedTaskEngine(state, taskId);
|
|
316
|
+
if (state.stopped || state.seen.has(taskId) || state.running.has(taskId))
|
|
317
|
+
return;
|
|
318
|
+
// Enqueue even when the lookup can't confirm the row (empty result or a
|
|
319
|
+
// transient error). Dropping it would leave the id un-`seen`, so a later
|
|
320
|
+
// delta-poll re-claims the same no-op and drags its watermark past newer
|
|
321
|
+
// tasks (the watch-claim-scan wedge). This mirrors the pre-engine broadcast
|
|
322
|
+
// path, which always enqueued; a still-unknown engine falls back to default.
|
|
323
|
+
enqueue(state, taskId, factory, resolved.found ? resolved.engine : undefined);
|
|
324
|
+
}
|
|
262
325
|
// Single source of truth for "this review is new — push it into the review
|
|
263
326
|
// queue". The Realtime listener AND the periodic review scan (v0.65 T-65-2)
|
|
264
327
|
// both go through here; the shared `seenReviews` guard dedupes a review that
|
|
@@ -336,6 +399,7 @@ async function handleTaskPaused(state, payload) {
|
|
|
336
399
|
}
|
|
337
400
|
// Let the resume redispatch of this SAME id through the dedupe set.
|
|
338
401
|
state.seen.delete(taskId);
|
|
402
|
+
state.taskEngines.delete(taskId);
|
|
339
403
|
}
|
|
340
404
|
function subscribeChannel(state, taskRunnerFactory) {
|
|
341
405
|
state.channel = state.supabase
|
|
@@ -344,8 +408,9 @@ function subscribeChannel(state, taskRunnerFactory) {
|
|
|
344
408
|
})
|
|
345
409
|
.on("broadcast", { event: "task_assigned" }, ({ payload }) => {
|
|
346
410
|
const taskId = payload?.task_id;
|
|
347
|
-
if (taskId)
|
|
348
|
-
|
|
411
|
+
if (taskId) {
|
|
412
|
+
void enqueueAssignedTaskBroadcast(state, taskId, taskRunnerFactory, payload);
|
|
413
|
+
}
|
|
349
414
|
})
|
|
350
415
|
.on("broadcast", { event: "task_cancelled" }, ({ payload }) => {
|
|
351
416
|
const taskId = payload?.task_id;
|
|
@@ -362,6 +427,7 @@ function subscribeChannel(state, taskRunnerFactory) {
|
|
|
362
427
|
// Drop from seen so a re-queue (cancel → re-start) is honored on
|
|
363
428
|
// the next broadcast or poll.
|
|
364
429
|
state.seen.delete(taskId);
|
|
430
|
+
state.taskEngines.delete(taskId);
|
|
365
431
|
})
|
|
366
432
|
.on("broadcast", { event: TASK_PAUSED_EVENT }, ({ payload }) => {
|
|
367
433
|
void handleTaskPaused(state, payload);
|
|
@@ -454,6 +520,15 @@ export async function reassertRunningClaims(state) {
|
|
|
454
520
|
}
|
|
455
521
|
}));
|
|
456
522
|
}
|
|
523
|
+
function capacityEngine(engine) {
|
|
524
|
+
return getEngine(engine ?? null).id;
|
|
525
|
+
}
|
|
526
|
+
function hasCapacityPause(state) {
|
|
527
|
+
return state.pausedCapacity.size > 0;
|
|
528
|
+
}
|
|
529
|
+
function isCapacityPausedFor(state, engine) {
|
|
530
|
+
return state.pausedCapacity.has(capacityEngine(engine));
|
|
531
|
+
}
|
|
457
532
|
// Single-flight poll. The watermark advances only after a clean response so
|
|
458
533
|
// a transient error doesn't permanently skip rows.
|
|
459
534
|
async function pollOnce(state, factory) {
|
|
@@ -485,7 +560,7 @@ async function pollOnce(state, factory) {
|
|
|
485
560
|
// keeps the watermark a conservative "newest task I claimed by polling"
|
|
486
561
|
// mark and stops the delta-poll losing still-queued work. The periodic
|
|
487
562
|
// full claim scan (claimScan) is the belt to this braces.
|
|
488
|
-
const claimed = enqueue(state, row.id, factory);
|
|
563
|
+
const claimed = enqueue(state, row.id, factory, row.engine);
|
|
489
564
|
if (claimed && row.updated_at && row.updated_at > maxSeen) {
|
|
490
565
|
maxSeen = row.updated_at;
|
|
491
566
|
}
|
|
@@ -532,7 +607,7 @@ async function claimScan(state, factory, context) {
|
|
|
532
607
|
for (const row of rows) {
|
|
533
608
|
if (!row?.id)
|
|
534
609
|
continue;
|
|
535
|
-
if (enqueue(state, row.id, factory))
|
|
610
|
+
if (enqueue(state, row.id, factory, row.engine))
|
|
536
611
|
enqueued += 1;
|
|
537
612
|
}
|
|
538
613
|
if (context === "startup") {
|
|
@@ -599,13 +674,8 @@ async function claimWatchdogScan(state, factory) {
|
|
|
599
674
|
state.claimWatchdogSince.clear();
|
|
600
675
|
return;
|
|
601
676
|
}
|
|
602
|
-
//
|
|
603
|
-
|
|
604
|
-
// leave its tasks queued; a resubscribe changes nothing and produced the
|
|
605
|
-
// resubscribe storm interleaved with QUARANTINED in the claim-livelock
|
|
606
|
-
// incident. Mirror the backstop's guard and drop the stale timers so an
|
|
607
|
-
// age never spans the pause and falsely trips the instant it lifts.
|
|
608
|
-
if (state.quarantined || state.pausedCapacity) {
|
|
677
|
+
// A quarantined runner is deliberately not claiming; resubscribe cannot help.
|
|
678
|
+
if (state.quarantined) {
|
|
609
679
|
state.claimWatchdogSince.clear();
|
|
610
680
|
return;
|
|
611
681
|
}
|
|
@@ -621,8 +691,12 @@ async function claimWatchdogScan(state, factory) {
|
|
|
621
691
|
const now = Date.now();
|
|
622
692
|
const currentIds = new Set();
|
|
623
693
|
for (const row of rows) {
|
|
624
|
-
if (row?.id)
|
|
625
|
-
|
|
694
|
+
if (!row?.id)
|
|
695
|
+
continue;
|
|
696
|
+
// Paused-engine rows are deliberately unclaimed; still watch other engines.
|
|
697
|
+
if (isCapacityPausedFor(state, row.engine))
|
|
698
|
+
continue;
|
|
699
|
+
currentIds.add(row.id);
|
|
626
700
|
}
|
|
627
701
|
// Reset: drop timers for tasks no longer assigned+queued (claimed/unassigned).
|
|
628
702
|
for (const id of [...state.claimWatchdogSince.keys()]) {
|
|
@@ -635,6 +709,17 @@ async function claimWatchdogScan(state, factory) {
|
|
|
635
709
|
if (!currentIds.has(id))
|
|
636
710
|
state.claimContentionSince.delete(id);
|
|
637
711
|
}
|
|
712
|
+
// CLAIM-LOCK-GLOB: same reset for the claim-loss back-off
|
|
713
|
+
// — a task that left the queue is no longer contending, so its attempt count
|
|
714
|
+
// and yield window are stale.
|
|
715
|
+
for (const id of [...state.claimLossState.keys()]) {
|
|
716
|
+
if (!currentIds.has(id))
|
|
717
|
+
state.claimLossState.delete(id);
|
|
718
|
+
}
|
|
719
|
+
for (const id of [...state.claimYieldUntil.keys()]) {
|
|
720
|
+
if (!currentIds.has(id))
|
|
721
|
+
state.claimYieldUntil.delete(id);
|
|
722
|
+
}
|
|
638
723
|
// Observe: stamp the first-seen instant for newly assigned-unclaimed tasks;
|
|
639
724
|
// collect any that have now been stale past the threshold, splitting off
|
|
640
725
|
// those that are merely waiting on file-lock contention (v0.41.4).
|
|
@@ -751,10 +836,8 @@ async function claimWatchdogScan(state, factory) {
|
|
|
751
836
|
async function claimBackstopScan(state, factory) {
|
|
752
837
|
if (state.stopped || state.backstopScanning)
|
|
753
838
|
return;
|
|
754
|
-
//
|
|
755
|
-
|
|
756
|
-
// recovers any genuine `seen`-trap once the runner is claiming again.
|
|
757
|
-
if (state.quarantined || state.pausedCapacity)
|
|
839
|
+
// Quarantine is a deliberate whole-runner stop; capacity is checked per row.
|
|
840
|
+
if (state.quarantined)
|
|
758
841
|
return;
|
|
759
842
|
state.backstopScanning = true;
|
|
760
843
|
try {
|
|
@@ -772,6 +855,9 @@ async function claimBackstopScan(state, factory) {
|
|
|
772
855
|
for (const row of rows) {
|
|
773
856
|
if (!row?.id)
|
|
774
857
|
continue;
|
|
858
|
+
// Leave paused-engine rows queued; recover rows for claimable engines.
|
|
859
|
+
if (isCapacityPausedFor(state, row.engine))
|
|
860
|
+
continue;
|
|
775
861
|
// The wedge signature: DB says assigned+queued, but the id is trapped in
|
|
776
862
|
// `seen` while live in neither `queue` nor `running`. Clear `seen` and let
|
|
777
863
|
// enqueue() re-drive it (enqueue re-adds to `seen` + pumps).
|
|
@@ -779,7 +865,7 @@ async function claimBackstopScan(state, factory) {
|
|
|
779
865
|
!state.running.has(row.id) &&
|
|
780
866
|
!inQueue.has(row.id)) {
|
|
781
867
|
state.seen.delete(row.id);
|
|
782
|
-
if (enqueue(state, row.id, factory))
|
|
868
|
+
if (enqueue(state, row.id, factory, row.engine))
|
|
783
869
|
recovered.push(row.id);
|
|
784
870
|
}
|
|
785
871
|
}
|
|
@@ -817,17 +903,39 @@ async function claimBackstopScan(state, factory) {
|
|
|
817
903
|
* incident). This pass re-probes claude while quarantined under `env_broken`
|
|
818
904
|
* and, when the probe passes, clears the quarantine and resumes claiming.
|
|
819
905
|
*
|
|
820
|
-
*
|
|
821
|
-
*
|
|
822
|
-
*
|
|
906
|
+
* The sentinel is authoritative for every cause: an operator clearing this
|
|
907
|
+
* instance's file releases a running watcher on the next pass. Automatic
|
|
908
|
+
* probing remains scoped to `env_broken`; a usage_limit with a known reset is
|
|
909
|
+
* also released by capacity resume, while a hard billing wall and auth_expired
|
|
910
|
+
* require explicit operator action. Single-flight; never throws out of the loop.
|
|
823
911
|
*/
|
|
824
912
|
async function quarantineRecoveryScan(state, factory) {
|
|
825
913
|
if (state.stopped || state.quarantineRecovering)
|
|
826
914
|
return;
|
|
827
|
-
if (!state.quarantined || state.quarantineCause !== "env_broken")
|
|
828
|
-
return;
|
|
829
915
|
state.quarantineRecovering = true;
|
|
830
916
|
try {
|
|
917
|
+
const sentinel = await getQuarantine();
|
|
918
|
+
if (!sentinel) {
|
|
919
|
+
if (!state.quarantined)
|
|
920
|
+
return;
|
|
921
|
+
const cause = state.quarantineCause;
|
|
922
|
+
state.quarantined = false;
|
|
923
|
+
state.quarantineCause = null;
|
|
924
|
+
process.stderr.write(chalk.green(`[acc-runner] quarantine cleared for this instance (${cause ?? "unknown"}); ` +
|
|
925
|
+
`resuming task claims without a restart.\n`));
|
|
926
|
+
if (cause === "usage_limit" && state.pausedCapacity) {
|
|
927
|
+
state.resumeController.resumeByOperator();
|
|
928
|
+
await state.resumeController.settle();
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
void pollOnce(state, factory);
|
|
932
|
+
void pump(state, factory);
|
|
933
|
+
return;
|
|
934
|
+
}
|
|
935
|
+
state.quarantined = true;
|
|
936
|
+
state.quarantineCause = sentinel.cause;
|
|
937
|
+
if (sentinel.cause !== "env_broken")
|
|
938
|
+
return;
|
|
831
939
|
const probe = await state.healthProbe();
|
|
832
940
|
if (!probe.ok)
|
|
833
941
|
return; // still broken — stay quarantined, try again next tick.
|
|
@@ -863,12 +971,11 @@ async function quarantineRecoveryScan(state, factory) {
|
|
|
863
971
|
* CLAIM-DEADMAN: a runner may have free capacity but be
|
|
864
972
|
* deliberately not claiming (quarantined, capacity-paused, review-claim-paused,
|
|
865
973
|
* or already at its effective task-claim ceiling). In those states pending work
|
|
866
|
-
* SHOULD sit queued, so it must NOT be read as a wedge.
|
|
867
|
-
*
|
|
868
|
-
* can claim another task.
|
|
974
|
+
* SHOULD sit queued, so it must NOT be read as a wedge. Returns true only when
|
|
975
|
+
* the runner both wants to and can claim another task.
|
|
869
976
|
*/
|
|
870
977
|
function deadmanCanClaim(state) {
|
|
871
|
-
if (state.quarantined || state
|
|
978
|
+
if (state.quarantined || hasCapacityPause(state) || state.capacityClaimPaused) {
|
|
872
979
|
return false;
|
|
873
980
|
}
|
|
874
981
|
const limit = computeTaskClaimLimit({
|
|
@@ -1108,7 +1215,7 @@ async function reviewScan(state, context) {
|
|
|
1108
1215
|
return;
|
|
1109
1216
|
// v0.56 (T-56-1) parity with pumpReviews: don't claim reviews while paused
|
|
1110
1217
|
// for capacity. The scheduled resume re-drives the review pump.
|
|
1111
|
-
if (state
|
|
1218
|
+
if (hasCapacityPause(state))
|
|
1112
1219
|
return;
|
|
1113
1220
|
state.reviewScanning = true;
|
|
1114
1221
|
try {
|
|
@@ -1358,6 +1465,7 @@ export async function watchCommand(options = {}) {
|
|
|
1358
1465
|
gitAuthProbeOnce: async () => false,
|
|
1359
1466
|
quarantineRecoveryOnce: async () => { },
|
|
1360
1467
|
runningCount: () => 0,
|
|
1468
|
+
pausedEnginesSnapshot: () => [],
|
|
1361
1469
|
};
|
|
1362
1470
|
}
|
|
1363
1471
|
// Park: release our flock so a manual restart / operator install isn't
|
|
@@ -1388,25 +1496,6 @@ export async function watchCommand(options = {}) {
|
|
|
1388
1496
|
`(auto-upgrade disabled via ACC_RUNNER_NO_AUTO_UPGRADE=1).\n`
|
|
1389
1497
|
: `Auto-upgrade will install npm latest when idle.\n`));
|
|
1390
1498
|
}
|
|
1391
|
-
// Surface how spawned claude sessions will authenticate under subscription-only
|
|
1392
|
-
// serving. All serving rides subscription capacity unless the serving identity
|
|
1393
|
-
// is BYOK api_key — a host ANTHROPIC_API_KEY is forwarded ONLY in that case,
|
|
1394
|
-
// and stripped otherwise so it can never silently bill the API. The serving
|
|
1395
|
-
// auth_kind comes from ACC_RUNNER_SERVING_AUTH_KIND (fail-safe: subscription
|
|
1396
|
-
// when unset).
|
|
1397
|
-
const servingKind = servingAuthKind();
|
|
1398
|
-
console.log(chalk.gray(`Serving auth: ${describeServingAuthKind(servingKind)}`));
|
|
1399
|
-
if (!forwardsAmbientKey() && anthropicApiKey()) {
|
|
1400
|
-
// Reassure the operator that a stray host key is inert, not billed.
|
|
1401
|
-
console.log(chalk.gray("Serving auth: a host ANTHROPIC_API_KEY is set but will be STRIPPED from spawns (subscription-only)."));
|
|
1402
|
-
}
|
|
1403
|
-
if (forwardsAmbientKey()) {
|
|
1404
|
-
// Only meaningful in BYOK api_key mode — surface the key-vs-claude.ai-login
|
|
1405
|
-
// precedence hint so a 429 is diagnosable from the banner.
|
|
1406
|
-
const authHint = authPrecedenceHint();
|
|
1407
|
-
if (authHint)
|
|
1408
|
-
console.log(chalk.gray(`Serving auth: ${authHint}`));
|
|
1409
|
-
}
|
|
1410
1499
|
// Startup refresh: if the access token expires in less than an hour,
|
|
1411
1500
|
// grab a fresh one before opening the realtime connection.
|
|
1412
1501
|
// v0.12.0 (T-52-7): skipped when the provider has no refresh token —
|
|
@@ -1459,22 +1548,41 @@ export async function watchCommand(options = {}) {
|
|
|
1459
1548
|
concurrency: concurrencyLimit,
|
|
1460
1549
|
reservedReviewSlots,
|
|
1461
1550
|
};
|
|
1462
|
-
//
|
|
1463
|
-
//
|
|
1464
|
-
//
|
|
1465
|
-
//
|
|
1466
|
-
//
|
|
1467
|
-
//
|
|
1468
|
-
|
|
1551
|
+
// AC2 (upstream WINDOWS-RUNNER-DISPATCH-1): a runner must never register/
|
|
1552
|
+
// heartbeat as healthy while it cannot resolve the default task engine —
|
|
1553
|
+
// otherwise it accepts claude-code work that can only fail at phase=claude_exit.
|
|
1554
|
+
// This guard runs on EVERY startup path (kept from upstream, ahead of the
|
|
1555
|
+
// unconditional register below); only an explicit options.claudePreflight
|
|
1556
|
+
// override skips it, so a test opts out visibly.
|
|
1557
|
+
const claudePreflight = options.claudePreflight ?? (() => requireExecutable("claude", { env: buildEnv() }));
|
|
1558
|
+
try {
|
|
1559
|
+
claudePreflight();
|
|
1560
|
+
}
|
|
1561
|
+
catch (err) {
|
|
1562
|
+
await singletonLock?.release();
|
|
1563
|
+
process.stderr.write(`[acc-runner] ${chalk.red(err.message)}\n`);
|
|
1564
|
+
exitFn(1);
|
|
1565
|
+
throw err;
|
|
1566
|
+
}
|
|
1567
|
+
// CAPS-ENGINES-STALE-KEYCHAIN-1 (AC1/AC2): caps.engines must be THIS watcher
|
|
1568
|
+
// process's live surface, so refresh it on every boot for EVERY token mode
|
|
1569
|
+
// (was gated to mode==="env"). Env-token runners also create/adopt their row here.
|
|
1570
|
+
{
|
|
1469
1571
|
const boot = createRunnerClient(cfg, session.access_token, session.realtime_config);
|
|
1470
1572
|
const models = await detectInstalledModels();
|
|
1471
|
-
//
|
|
1472
|
-
|
|
1473
|
-
//
|
|
1474
|
-
//
|
|
1475
|
-
|
|
1573
|
+
// AC11: advertise what THIS serving process resolves (seam defaults to real).
|
|
1574
|
+
const engines = await (options.detectEngines ?? detectCapableEngines)();
|
|
1575
|
+
// Record the SERVING model-credential account (from upstream's capacity-
|
|
1576
|
+
// attribution work) so a keychain OR env-token runner's account is
|
|
1577
|
+
// comparable in the runners table from its first heartbeat.
|
|
1578
|
+
const identity = await resolveCredentialIdentity();
|
|
1476
1579
|
const { data: resolvedId, error: regErr } = await boot.rpc("register_runner", {
|
|
1477
1580
|
p_id: session.runner_id,
|
|
1581
|
+
// finding-4: send p_name = runner_id on EVERY boot. For an env row that
|
|
1582
|
+
// IS its name; for a keychain row, register_runner (mig 0349) treats an
|
|
1583
|
+
// incoming name equal to the row id as "no rename" and PRESERVES the
|
|
1584
|
+
// existing (possibly operator-renamed) name — so a boot caps-refresh never
|
|
1585
|
+
// clobbers an operator-set runner name.
|
|
1478
1586
|
p_name: session.runner_id,
|
|
1479
1587
|
p_owner: session.email ?? session.user_id,
|
|
1480
1588
|
p_machine: machineIdentity(),
|
|
@@ -1486,18 +1594,80 @@ export async function watchCommand(options = {}) {
|
|
|
1486
1594
|
// CONC-1: pass the once-resolved concurrency so caps.concurrency /
|
|
1487
1595
|
// caps.reserved_review_slots match the watch loop's real capacity and
|
|
1488
1596
|
// survive register_runner's per-heartbeat caps replace (mig 0153).
|
|
1489
|
-
|
|
1597
|
+
// AC10: SAME shared builder login.ts uses — boot ≡ login caps (now rich:
|
|
1598
|
+
// process caps + serving identity + build fingerprint folded in).
|
|
1599
|
+
p_caps: registerRunnerCaps(cfg, engines, concurrencyCaps, identity, await buildFingerprintCaps()),
|
|
1490
1600
|
p_version: PACKAGE_VERSION,
|
|
1601
|
+
// REGISTER-CAPS-PRESERVE-1: same explicit send as the login path — an
|
|
1602
|
+
// env-token runner re-registers on EVERY boot, so this was the
|
|
1603
|
+
// highest-frequency door into the capability wipe. Mig 0349 now
|
|
1604
|
+
// union-merges capabilities (never shrinks), so this refresh is non-destructive.
|
|
1605
|
+
p_capabilities: advertisedCapabilities(cfg),
|
|
1491
1606
|
});
|
|
1492
1607
|
if (regErr) {
|
|
1493
|
-
|
|
1494
|
-
process.
|
|
1608
|
+
// AC3: env path keeps its exact "ephemeral" wording; keychain path is new.
|
|
1609
|
+
process.stderr.write(`[acc-runner] ${tokenProvider.mode === "env" ? "ephemeral" : "boot"} register_runner failed: ${safeText(regErr.message, 300) ?? "unknown error"}\n`);
|
|
1610
|
+
if (tokenProvider.mode === "env")
|
|
1611
|
+
process.exit(1);
|
|
1495
1612
|
}
|
|
1496
|
-
|
|
1497
|
-
|
|
1613
|
+
else {
|
|
1614
|
+
// AC-SAFETY: ONLY env-token runners (no login flow; id may be server-minted)
|
|
1615
|
+
// adopt a resolved id and persist it. A keychain runner has a valid
|
|
1616
|
+
// logged-in identity — refresh its caps only, NEVER rewrite its stored
|
|
1617
|
+
// session (an unguarded identity rewrite risks orphaning the credential).
|
|
1618
|
+
if (tokenProvider.mode === "env") {
|
|
1619
|
+
if (typeof resolvedId === "string" && resolvedId && resolvedId !== session.runner_id) {
|
|
1620
|
+
session = { ...session, runner_id: resolvedId };
|
|
1621
|
+
}
|
|
1622
|
+
await tokenProvider.save(session);
|
|
1623
|
+
}
|
|
1624
|
+
console.log(chalk.green(tokenProvider.mode === "env"
|
|
1625
|
+
? `✓ Registered ephemeral runner: ${session.runner_id}`
|
|
1626
|
+
: `✓ Refreshed runner caps: ${session.runner_id}`));
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
// RUNNER-SERVING-AUTH: ask the control plane which serving auth this ORG
|
|
1630
|
+
// chose in Settings, before the banner below reads it. This is what turns
|
|
1631
|
+
// "an operator hand-edits ACC_RUNNER_SERVING_AUTH_KIND on the host" into an
|
|
1632
|
+
// admin-gated setting — it writes the SAME env var and changes nothing else.
|
|
1633
|
+
// Never throws: any failure resolves to subscription (today's behaviour),
|
|
1634
|
+
// and an explicit launch-env value always wins, so container fleets that pin
|
|
1635
|
+
// the var are untouched.
|
|
1636
|
+
//
|
|
1637
|
+
// POSITION MATTERS, twice over. It runs AFTER the startup refresh + ephemeral
|
|
1638
|
+
// registration so the JWT it presents is the final one, not a token that was
|
|
1639
|
+
// about to expire. And it runs BEFORE the NS RUN-3 vault hydration below, so
|
|
1640
|
+
// an explicit ACC_RUNNER_ANTHROPIC_KEY_SOURCE=secret-store fleet still wins
|
|
1641
|
+
// the shared key cache — the per-fleet opt-in outranks the org default, same
|
|
1642
|
+
// precedence rule as the env var itself.
|
|
1643
|
+
if (!servingAuthKindIsPinned()) {
|
|
1644
|
+
const servingChoice = await fetchServingAuth({
|
|
1645
|
+
publicUrl: cfg.publicUrl,
|
|
1646
|
+
jwt: session.access_token,
|
|
1647
|
+
});
|
|
1648
|
+
installServingAuth(servingChoice);
|
|
1649
|
+
if (servingChoice.fallbackReason) {
|
|
1650
|
+
console.log(chalk.gray(`Serving auth: ${describeServingAuthResolution(servingChoice)}`));
|
|
1498
1651
|
}
|
|
1499
|
-
|
|
1500
|
-
|
|
1652
|
+
}
|
|
1653
|
+
// Surface how spawned claude sessions will authenticate under subscription-only
|
|
1654
|
+
// serving. All serving rides subscription capacity unless the serving identity
|
|
1655
|
+
// is BYOK api_key — a host ANTHROPIC_API_KEY is forwarded ONLY in that case,
|
|
1656
|
+
// and stripped otherwise so it can never silently bill the API. The serving
|
|
1657
|
+
// auth_kind comes from ACC_RUNNER_SERVING_AUTH_KIND (fail-safe: subscription
|
|
1658
|
+
// when unset), which the org setting above may have just supplied.
|
|
1659
|
+
const servingKind = servingAuthKind();
|
|
1660
|
+
console.log(chalk.gray(`Serving auth: ${describeServingAuthKind(servingKind)}`));
|
|
1661
|
+
if (!forwardsAmbientKey() && anthropicApiKey()) {
|
|
1662
|
+
// Reassure the operator that a stray host key is inert, not billed.
|
|
1663
|
+
console.log(chalk.gray("Serving auth: a host ANTHROPIC_API_KEY is set but will be STRIPPED from spawns (subscription-only)."));
|
|
1664
|
+
}
|
|
1665
|
+
if (forwardsAmbientKey()) {
|
|
1666
|
+
// Only meaningful in BYOK api_key mode — surface the key-vs-claude.ai-login
|
|
1667
|
+
// precedence hint so a 429 is diagnosable from the banner.
|
|
1668
|
+
const authHint = authPrecedenceHint();
|
|
1669
|
+
if (authHint)
|
|
1670
|
+
console.log(chalk.gray(`Serving auth: ${authHint}`));
|
|
1501
1671
|
}
|
|
1502
1672
|
// NS RUN-3: hydrate ANTHROPIC_API_KEY from the vault secret store, BEFORE
|
|
1503
1673
|
// any claude spawn can read it. Defaults to a no-op (source "env") so
|
|
@@ -1547,15 +1717,29 @@ export async function watchCommand(options = {}) {
|
|
|
1547
1717
|
const taskRunnerFactory = options.taskRunnerFactory ?? runTask;
|
|
1548
1718
|
const heartbeatMs = options.heartbeatMs ?? 4_000;
|
|
1549
1719
|
const pollMs = options.pollMs ?? POLL_INTERVAL_MS;
|
|
1550
|
-
|
|
1720
|
+
// CROSS-ENGINE-REVIEW-1 (AC3) — FAIL CLOSED: constrain reviewer-engine selection
|
|
1721
|
+
// to the engines THIS runner ALREADY PUBLISHED into `acc.runners.caps.engines` —
|
|
1722
|
+
// the value the assignment was routed against — NOT a fresh per-review probe.
|
|
1723
|
+
// Resolved ONCE here from the SAME `detectCapableEngines()` source that feeds the
|
|
1724
|
+
// register_runner caps payload, then threaded as a fixed `advertisedEngineIds`.
|
|
1725
|
+
// A per-review re-probe could drift from the published caps (an engine appearing
|
|
1726
|
+
// or disappearing mid-session), silently routing a review to an engine the
|
|
1727
|
+
// assignment never assumed; the fixed snapshot cannot. A claude-only runner is
|
|
1728
|
+
// never routed to codex, and an empty set fails closed downstream (no candidate →
|
|
1729
|
+
// no-route infra error) rather than falling back to the local registry. Tests
|
|
1730
|
+
// inject `options.reviewerFactory`.
|
|
1731
|
+
const advertisedEngines = await detectCapableEngines();
|
|
1732
|
+
const reviewerFactory = options.reviewerFactory ?? capsConstrainedReviewerFactory(advertisedEngines);
|
|
1551
1733
|
// v0.48: check for an existing quarantine file before opening the
|
|
1552
1734
|
// realtime connection. If quarantined, emit a loud warning — the
|
|
1553
1735
|
// runner will not claim tasks until `acc-runner quarantine clear` runs.
|
|
1554
1736
|
const startupQuarantine = await getQuarantine().catch(() => null);
|
|
1555
1737
|
if (startupQuarantine) {
|
|
1556
1738
|
process.stderr.write(chalk.red.bold(`[acc-runner] QUARANTINED (${startupQuarantine.cause}): ${startupQuarantine.detail}\n`));
|
|
1557
|
-
process.stderr.write(chalk.red(`[acc-runner] Runner will NOT claim tasks. Resolve the issue then run ` +
|
|
1558
|
-
`\`acc-runner quarantine clear
|
|
1739
|
+
process.stderr.write(chalk.red(`[acc-runner] Runner will NOT claim tasks. Resolve the issue, then run ` +
|
|
1740
|
+
`\`acc-runner quarantine clear\` for this named instance; repeat for each ` +
|
|
1741
|
+
`instance on the host. The running watcher resumes on its next recovery tick; ` +
|
|
1742
|
+
`no restart is required.\n`));
|
|
1559
1743
|
}
|
|
1560
1744
|
// v0.12.0 (T-52-7): idle TTL — option override wins (tests), then env.
|
|
1561
1745
|
const idleTtlMs = options.idleTtlMs !== undefined ? options.idleTtlMs : resolveIdleTtlMs();
|
|
@@ -1581,6 +1765,8 @@ export async function watchCommand(options = {}) {
|
|
|
1581
1765
|
claimWatchdogChecking: false,
|
|
1582
1766
|
// v0.41.4: file-lock contention tracker (see WatchState doc).
|
|
1583
1767
|
claimContentionSince: new Map(),
|
|
1768
|
+
claimLossState: new Map(),
|
|
1769
|
+
claimYieldUntil: new Map(),
|
|
1584
1770
|
// FIX-D: periodic DB-poll claim backstop.
|
|
1585
1771
|
claimBackstopTimer: undefined,
|
|
1586
1772
|
claimBackstopIntervalMs: options.claimBackstopIntervalMs ?? DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS,
|
|
@@ -1597,6 +1783,7 @@ export async function watchCommand(options = {}) {
|
|
|
1597
1783
|
wedgeThresholdMs: options.claimDeadmanWedgeMs ?? DEFAULT_CLAIM_DEADMAN_WEDGE_MS,
|
|
1598
1784
|
rebuildGraceMs: options.claimDeadmanRebuildGraceMs ?? DEFAULT_CLAIM_DEADMAN_REBUILD_GRACE_MS,
|
|
1599
1785
|
maxRebuilds: options.claimDeadmanMaxRebuilds ?? DEFAULT_CLAIM_DEADMAN_MAX_REBUILDS,
|
|
1786
|
+
now: options.claimDeadmanNow,
|
|
1600
1787
|
}),
|
|
1601
1788
|
claimDeadmanTimer: undefined,
|
|
1602
1789
|
claimDeadmanCheckMs: options.claimDeadmanCheckMs ?? DEFAULT_DEADMAN_CHECK_MS,
|
|
@@ -1638,6 +1825,7 @@ export async function watchCommand(options = {}) {
|
|
|
1638
1825
|
// RVU-2 (FU-RVU2R): claim-pause starts off; set under review capacity pressure.
|
|
1639
1826
|
capacityClaimPaused: false,
|
|
1640
1827
|
queue: [],
|
|
1828
|
+
droppedEngines: new Set(),
|
|
1641
1829
|
seen: new Set(),
|
|
1642
1830
|
pumping: false,
|
|
1643
1831
|
stopped: false,
|
|
@@ -1663,10 +1851,12 @@ export async function watchCommand(options = {}) {
|
|
|
1663
1851
|
selfUpgradeFn: options.selfUpgrade ?? ((input) => maybeSelfUpgrade(input)),
|
|
1664
1852
|
selfUpgrading: false,
|
|
1665
1853
|
taskRunnerFactory,
|
|
1666
|
-
pausedCapacity:
|
|
1854
|
+
pausedCapacity: new Set(),
|
|
1667
1855
|
pausedUntil: null,
|
|
1668
1856
|
capacityTriggerTaskId: null,
|
|
1669
1857
|
capacityPauseDetail: "",
|
|
1858
|
+
capacityPauseEngine: null,
|
|
1859
|
+
taskEngines: new Map(),
|
|
1670
1860
|
// AS-3 (RESUME-WIRE): assigned immediately below once `state` exists (the
|
|
1671
1861
|
// controller's seams close over `state`). `undefined as unknown as` mirrors
|
|
1672
1862
|
// the channel/timer bootstrap fields above.
|
|
@@ -1787,7 +1977,7 @@ export async function watchCommand(options = {}) {
|
|
|
1787
1977
|
state.claimBackstopTimer = setInterval(() => {
|
|
1788
1978
|
void claimBackstopScan(state, taskRunnerFactory);
|
|
1789
1979
|
}, state.claimBackstopIntervalMs);
|
|
1790
|
-
//
|
|
1980
|
+
// Reconcile operator clears for every cause and probe env_broken recovery.
|
|
1791
1981
|
state.quarantineRecoveryTimer = setInterval(() => {
|
|
1792
1982
|
void quarantineRecoveryScan(state, taskRunnerFactory);
|
|
1793
1983
|
}, state.quarantineRecoveryIntervalMs);
|
|
@@ -1820,10 +2010,20 @@ export async function watchCommand(options = {}) {
|
|
|
1820
2010
|
// requiring a full `acc-runner login` re-flow. Default-empty is the
|
|
1821
2011
|
// backward-compat path for 0.6.3 runners and matches any task with
|
|
1822
2012
|
// an empty required_capabilities array.
|
|
1823
|
-
|
|
2013
|
+
//
|
|
2014
|
+
// REGISTER-CAPS-PRESERVE-1: skip the publish when this runner declared NO
|
|
2015
|
+
// tags. set_runner_capabilities (0114) is an unconditional `capabilities =
|
|
2016
|
+
// coalesce(p_capabilities, '{}')`, so an empty array is a pure WIPE carrying
|
|
2017
|
+
// no information — and it is the OTHER door into the fleet-starvation
|
|
2018
|
+
// incidents (the 03:20 launchctl reload took this path, not register_runner:
|
|
2019
|
+
// a local-token runner never re-runs the boot register). Clearing tags is a
|
|
2020
|
+
// board action, not a side effect of an unset env var. A non-empty set is
|
|
2021
|
+
// still published verbatim, so narrowing from the env stays one restart.
|
|
2022
|
+
const advertised = advertisedCapabilities(cfg);
|
|
2023
|
+
if (advertised.length > 0) {
|
|
1824
2024
|
const { error: capsErr } = await state.supabase.rpc("set_runner_capabilities", {
|
|
1825
2025
|
p_id: state.session.runner_id,
|
|
1826
|
-
p_capabilities:
|
|
2026
|
+
p_capabilities: advertised,
|
|
1827
2027
|
});
|
|
1828
2028
|
if (capsErr) {
|
|
1829
2029
|
// Non-fatal: a pre-0114 server (or an Old DB the runner is
|
|
@@ -1864,11 +2064,25 @@ export async function watchCommand(options = {}) {
|
|
|
1864
2064
|
if (cfg.repos && cfg.repos.length > 0) {
|
|
1865
2065
|
console.log(chalk.gray(` serving repos: ${cfg.repos.join(", ")}`));
|
|
1866
2066
|
}
|
|
2067
|
+
// BUILD-FINGERPRINT-RUNNING-1: stamp the build THIS long-running process
|
|
2068
|
+
// loaded onto disk so a fresh `acc version` or `acc doctor` can later tell
|
|
2069
|
+
// whether an on-disk `npm i -g` has been picked up by this launchd process
|
|
2070
|
+
// yet — a question a fresh CLI invocation (loaded === installed) cannot
|
|
2071
|
+
// answer. Both `acc version` and `acc doctor` read this record.
|
|
2072
|
+
await recordRunningBuild(LOADED_BUILD_FINGERPRINT).catch((err) => {
|
|
2073
|
+
process.stderr.write(`[acc-runner] running-build record failed (continuing): ${err.message}\n`);
|
|
2074
|
+
});
|
|
1867
2075
|
// Initial heartbeat fires immediately so the runner page reflects "online".
|
|
1868
2076
|
await state.supabase.rpc("heartbeat_runner", {
|
|
1869
2077
|
p_id: state.session.runner_id,
|
|
1870
2078
|
p_version: PACKAGE_VERSION,
|
|
1871
2079
|
});
|
|
2080
|
+
await publishBuildFingerprintCapsDetail(state).catch((err) => {
|
|
2081
|
+
process.stderr.write(`[acc-runner] build fingerprint publish failed (continuing): ${err.message}\n`);
|
|
2082
|
+
});
|
|
2083
|
+
await publishQuarantineDetail(state).catch((err) => {
|
|
2084
|
+
process.stderr.write(`[acc-runner] quarantine detail publish failed (continuing): ${err.message}\n`);
|
|
2085
|
+
});
|
|
1872
2086
|
// RUNCFG-1: pull + apply any server-set concurrency at startup (and publish
|
|
1873
2087
|
// the effective ceiling + source into caps) so a runner that boots with an
|
|
1874
2088
|
// operator override already in place honours it from the first tick, not only
|
|
@@ -2029,8 +2243,49 @@ export async function watchCommand(options = {}) {
|
|
|
2029
2243
|
gitAuthProbeOnce: () => gitAuthProbeScan(state), // GIT-AUTH-ALERT
|
|
2030
2244
|
quarantineRecoveryOnce: () => quarantineRecoveryScan(state, taskRunnerFactory), // v0.41.4
|
|
2031
2245
|
runningCount: () => state.running.size,
|
|
2246
|
+
pausedEnginesSnapshot: () => [...state.pausedCapacity],
|
|
2032
2247
|
};
|
|
2033
2248
|
}
|
|
2249
|
+
async function publishQuarantineDetail(state) {
|
|
2250
|
+
const sentinel = await getQuarantine().catch(() => null);
|
|
2251
|
+
const published = publishedQuarantineState(sentinel);
|
|
2252
|
+
if (sentinel && !published) {
|
|
2253
|
+
await clearQuarantine().catch(() => undefined);
|
|
2254
|
+
state.quarantined = false;
|
|
2255
|
+
state.quarantineCause = null;
|
|
2256
|
+
}
|
|
2257
|
+
const { data, error: readErr } = await state.supabase
|
|
2258
|
+
.from("runners")
|
|
2259
|
+
.select("detail")
|
|
2260
|
+
.eq("id", state.session.runner_id)
|
|
2261
|
+
.maybeSingle();
|
|
2262
|
+
if (readErr)
|
|
2263
|
+
throw new Error(readErr.message);
|
|
2264
|
+
const current = data?.detail && typeof data.detail === "object" && !Array.isArray(data.detail)
|
|
2265
|
+
? { ...data.detail }
|
|
2266
|
+
: {};
|
|
2267
|
+
if (published)
|
|
2268
|
+
current.quarantine = published;
|
|
2269
|
+
else
|
|
2270
|
+
delete current.quarantine;
|
|
2271
|
+
const { error: writeErr } = await state.supabase
|
|
2272
|
+
.from("runners")
|
|
2273
|
+
.update({ detail: current })
|
|
2274
|
+
.eq("id", state.session.runner_id);
|
|
2275
|
+
if (writeErr)
|
|
2276
|
+
throw new Error(writeErr.message);
|
|
2277
|
+
}
|
|
2278
|
+
/**
|
|
2279
|
+
* BUILD-FINGERPRINT-ON-CAPS: refresh the build fingerprint on the caps
|
|
2280
|
+
* contract (acc.runners.caps.build) — the same field the boot register
|
|
2281
|
+
* populates — so the loaded build is published the same way for every auth
|
|
2282
|
+
* mode, not left to a best-effort `detail` side-channel. Id-scoped self-update
|
|
2283
|
+
* (never register_runner) so a named-instance keychain runner can't re-home
|
|
2284
|
+
* onto a sibling's row.
|
|
2285
|
+
*/
|
|
2286
|
+
async function publishBuildFingerprintCapsDetail(state) {
|
|
2287
|
+
await publishBuildFingerprintCaps(state.supabase, state.session.runner_id, await buildFingerprintCaps());
|
|
2288
|
+
}
|
|
2034
2289
|
/**
|
|
2035
2290
|
* RUNCFG-1: apply the operator's server-set concurrency to the LIVE watch-loop
|
|
2036
2291
|
* ceiling, precedence server-set > env > default. Idempotent: a no-change row
|
|
@@ -2113,6 +2368,58 @@ async function syncConcurrency(state, factory) {
|
|
|
2113
2368
|
const row = (Array.isArray(data) ? data[0] : data);
|
|
2114
2369
|
applyServerConcurrency(state, row, factory);
|
|
2115
2370
|
}
|
|
2371
|
+
export async function repairUnknownAdvertisedEngine(state, engineId, err, deps = {}) {
|
|
2372
|
+
const detail = safeText(err.message, 300) ?? "unknown engine";
|
|
2373
|
+
// AC6: redact the engine id too before it reaches the log line / event payload
|
|
2374
|
+
// (the DROP below keeps the RAW id — filtering must match caps.engines exactly).
|
|
2375
|
+
const safeEngineId = safeText(engineId, 80) ?? "<redacted>";
|
|
2376
|
+
// AC12: PERSIST the drop so a later republish can never re-advertise it.
|
|
2377
|
+
state.droppedEngines.add(engineId);
|
|
2378
|
+
process.stderr.write(`[acc-runner] runner.engine_caps_self_inconsistent runner=${state.session.runner_id} ` +
|
|
2379
|
+
`engine=${safeEngineId}: ${detail}\n`);
|
|
2380
|
+
try {
|
|
2381
|
+
await state.supabase.rpc("log_activity", {
|
|
2382
|
+
p_verb: "runner.engine_caps_self_inconsistent",
|
|
2383
|
+
p_target_id: state.session.runner_id,
|
|
2384
|
+
p_target_type: "runner",
|
|
2385
|
+
p_payload: {
|
|
2386
|
+
engine: safeEngineId,
|
|
2387
|
+
error: detail,
|
|
2388
|
+
action: "republish_caps",
|
|
2389
|
+
},
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
catch { /* best-effort signal */ }
|
|
2393
|
+
try {
|
|
2394
|
+
const engines = await (deps.detectEngines ?? detectCapableEngines)();
|
|
2395
|
+
const models = await (deps.detectModels ?? detectInstalledModels)();
|
|
2396
|
+
const { error } = await state.supabase.rpc("register_runner", {
|
|
2397
|
+
p_id: state.session.runner_id,
|
|
2398
|
+
// finding-4: p_name = runner_id preserves the operator-set name (mig 0349
|
|
2399
|
+
// treats incoming-name==id as "no rename"), for both env and keychain rows.
|
|
2400
|
+
p_name: state.session.runner_id,
|
|
2401
|
+
p_owner: state.session.email ?? state.session.user_id,
|
|
2402
|
+
p_machine: machineIdentity(),
|
|
2403
|
+
p_models: models,
|
|
2404
|
+
p_caps: runnerCapsWithRepos(state.cfg.repos,
|
|
2405
|
+
// AC12: filter EVERY persisted drop, not just this engineId — otherwise a
|
|
2406
|
+
// repair for one engine re-advertises another that was dropped earlier.
|
|
2407
|
+
engines.filter((id) => !state.droppedEngines.has(id)), {
|
|
2408
|
+
concurrency: state.concurrencyLimit,
|
|
2409
|
+
reservedReviewSlots: state.reservedReviewSlots,
|
|
2410
|
+
}, state.concurrencySource),
|
|
2411
|
+
p_version: PACKAGE_VERSION,
|
|
2412
|
+
p_capabilities: advertisedCapabilities(state.cfg),
|
|
2413
|
+
});
|
|
2414
|
+
if (error) {
|
|
2415
|
+
process.stderr.write(`[acc-runner] runner.engine_caps_self_inconsistent caps repair failed: ${safeText(error.message, 300) ?? "unknown error"}\n`);
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
catch (repairErr) {
|
|
2419
|
+
process.stderr.write(`[acc-runner] runner.engine_caps_self_inconsistent caps repair crashed: ` +
|
|
2420
|
+
`${safeText(repairErr.message, 300) ?? "unknown error"}\n`);
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2116
2423
|
/**
|
|
2117
2424
|
* AU-7: the transport `sendHeartbeat` drives. `readHeartbeatAt` is the audit
|
|
2118
2425
|
* half of the write-then-read verify — one indexed single-row read of the
|
|
@@ -2159,6 +2466,14 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs, lastRoundMs = 0) {
|
|
|
2159
2466
|
const error = beat.ok ? null : { message: `${beat.kind}: ${beat.message}` };
|
|
2160
2467
|
if (!error) {
|
|
2161
2468
|
state.heartbeatFailures = 0;
|
|
2469
|
+
// BUILD-FINGERPRINT: the fingerprint is published ONCE at boot (it cannot
|
|
2470
|
+
// change within a process). It is deliberately NOT re-published on the
|
|
2471
|
+
// heartbeat timer — heartbeat_runner does not touch caps, so the boot
|
|
2472
|
+
// value persists, and a per-beat read-merge-write would amplify the caps
|
|
2473
|
+
// clobber tracked in T-1788838000022 across every runner, every few seconds.
|
|
2474
|
+
await publishQuarantineDetail(state).catch((err) => {
|
|
2475
|
+
process.stderr.write(`[acc-runner] quarantine detail publish failed (continuing): ${err.message}\n`);
|
|
2476
|
+
});
|
|
2162
2477
|
// CLAIM-DEADMAN: capture the pending-dispatch count
|
|
2163
2478
|
// + suspect flag when the heartbeat RESPONSE carries them, so the runner
|
|
2164
2479
|
// learns it is being offered work even with a dead WS. Forward-compatible:
|
|
@@ -2243,7 +2558,8 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs, lastRoundMs = 0) {
|
|
|
2243
2558
|
async function maybeSelfUpgradeTick(state) {
|
|
2244
2559
|
if (state.stopped || state.idleExiting || state.selfUpgrading)
|
|
2245
2560
|
return;
|
|
2246
|
-
|
|
2561
|
+
// Process lifecycle actions wait until all capacity pauses have resumed.
|
|
2562
|
+
if (hasCapacityPause(state))
|
|
2247
2563
|
return;
|
|
2248
2564
|
if (state.running.size > 0 ||
|
|
2249
2565
|
state.queue.length > 0 ||
|
|
@@ -2286,10 +2602,8 @@ async function maybeSelfUpgradeTick(state) {
|
|
|
2286
2602
|
async function maybeIdleExit(state, stop, exitFn) {
|
|
2287
2603
|
if (state.stopped || state.idleExiting || state.idleTtlMs === null)
|
|
2288
2604
|
return;
|
|
2289
|
-
//
|
|
2290
|
-
|
|
2291
|
-
// it out from under the scheduled resume.
|
|
2292
|
-
if (state.pausedCapacity)
|
|
2605
|
+
// A capacity pause is not idleness; keep the process alive for resume.
|
|
2606
|
+
if (hasCapacityPause(state))
|
|
2293
2607
|
return;
|
|
2294
2608
|
if (state.running.size > 0 ||
|
|
2295
2609
|
state.queue.length > 0 ||
|
|
@@ -2317,6 +2631,9 @@ async function maybeIdleExit(state, stop, exitFn) {
|
|
|
2317
2631
|
p_id: state.session.runner_id,
|
|
2318
2632
|
p_version: PACKAGE_VERSION,
|
|
2319
2633
|
});
|
|
2634
|
+
// No fingerprint re-publish on idle exit: it was published at boot and the
|
|
2635
|
+
// process is shutting down, so a read-merge-write here is pure clobber risk.
|
|
2636
|
+
await publishQuarantineDetail(state);
|
|
2320
2637
|
}
|
|
2321
2638
|
catch { /* best-effort */ }
|
|
2322
2639
|
await stop();
|
|
@@ -2371,19 +2688,22 @@ async function pumpReviews(state) {
|
|
|
2371
2688
|
// v0.56 (T-56-1): don't claim reviews while paused for capacity, and hold
|
|
2372
2689
|
// reviews back during the single-probe window so the probe TASK confirms
|
|
2373
2690
|
// the session reopened before we spend more capacity on reviews.
|
|
2374
|
-
if (state
|
|
2691
|
+
if (hasCapacityPause(state) || state.capacityProbe)
|
|
2375
2692
|
return;
|
|
2376
2693
|
state.reviewPumping = true;
|
|
2377
2694
|
try {
|
|
2378
2695
|
while (!state.stopped &&
|
|
2379
|
-
!state
|
|
2696
|
+
!hasCapacityPause(state) &&
|
|
2380
2697
|
!state.capacityProbe &&
|
|
2381
2698
|
state.reviewQueue.length > 0) {
|
|
2382
2699
|
const next = state.reviewQueue.shift();
|
|
2383
2700
|
if (!next)
|
|
2384
2701
|
continue;
|
|
2385
2702
|
try {
|
|
2386
|
-
const outcome = await state.runReview(next, {
|
|
2703
|
+
const outcome = await state.runReview(next, {
|
|
2704
|
+
supabase: state.supabase,
|
|
2705
|
+
onUnknownAdvertisedEngine: (engineId, err) => repairUnknownAdvertisedEngine(state, engineId, err),
|
|
2706
|
+
});
|
|
2387
2707
|
const tag = outcome.decision === "reviewer_error" ? chalk.red : chalk.gray;
|
|
2388
2708
|
console.log(tag(`[acc-runner] review ${next.review_id} task=${next.task_id} pr=${next.pr_number} decision=${outcome.decision} confidence=${outcome.confidence.toFixed(2)}`));
|
|
2389
2709
|
// RVU-2 (FU-RVU2R): claim-pause before review-pause. A review that ran
|
|
@@ -2398,7 +2718,7 @@ async function pumpReviews(state) {
|
|
|
2398
2718
|
// reviewer_capacity row; here we pause the whole runner exactly as a
|
|
2399
2719
|
// capacity_exhausted task does.
|
|
2400
2720
|
if (outcome.decision === "reviewer_capacity") {
|
|
2401
|
-
enterCapacityPause(state, next.task_id, outcome.resume_at ?? null, `reviewer ${next.review_id} hit session/usage limit
|
|
2721
|
+
enterCapacityPause(state, next.task_id, outcome.resume_at ?? null, `reviewer ${next.review_id} hit session/usage limit`, outcome.engine);
|
|
2402
2722
|
return;
|
|
2403
2723
|
}
|
|
2404
2724
|
}
|
|
@@ -2417,7 +2737,7 @@ async function pumpReviews(state) {
|
|
|
2417
2737
|
// to protect, so release the claim-pause and let queued tasks resume claiming.
|
|
2418
2738
|
// (A whole-chain reviewer_capacity `return`s above and never reaches here;
|
|
2419
2739
|
// pausedCapacity keeps everything paused until the window reopens.)
|
|
2420
|
-
if (state.capacityClaimPaused && !state
|
|
2740
|
+
if (state.capacityClaimPaused && !hasCapacityPause(state)) {
|
|
2421
2741
|
state.capacityClaimPaused = false;
|
|
2422
2742
|
void pump(state, state.taskRunnerFactory);
|
|
2423
2743
|
}
|
|
@@ -2529,11 +2849,27 @@ function enterQuarantine(state, taskId, cause, detail,
|
|
|
2529
2849
|
// v0.53 T-53-4: how many consecutive instant-empty exits produced an
|
|
2530
2850
|
// env_broken cause (1 = definitive, 2 = heuristic confirmed by the
|
|
2531
2851
|
// retry). Persisted into quarantine.json's additive `consecutive` field.
|
|
2532
|
-
consecutive
|
|
2852
|
+
consecutive,
|
|
2853
|
+
// USAGE-LIMIT-INVISIBLE-1 (AC4): the fleet capacity reset this quarantine
|
|
2854
|
+
// was folded onto (see signalUsageLimitCapacity), when cause is
|
|
2855
|
+
// usage_limit — null for an indefinite cap. Threaded through so the
|
|
2856
|
+
// operator-facing bus message + audit event carry a concrete retry path
|
|
2857
|
+
// instead of a bare "quarantined" with no next step: the RUNNER auto-clears
|
|
2858
|
+
// on this reset (restartClaims), but the specific failed task does not
|
|
2859
|
+
// requeue itself and needs a manual retry_task / `/acc retry`.
|
|
2860
|
+
usageLimitResumeAtIso, engine) {
|
|
2533
2861
|
if (state.quarantined)
|
|
2534
2862
|
return;
|
|
2535
2863
|
state.quarantined = true;
|
|
2536
2864
|
state.quarantineCause = cause; // v0.41.4: gates env_broken auto-recovery.
|
|
2865
|
+
// Attribute an engine ONLY for a genuine per-engine capacity cap. `usage_limit`
|
|
2866
|
+
// is the sole cap cause that quarantines (capacity_exhausted PAUSES instead),
|
|
2867
|
+
// so it alone names the engine that hit the cap. Every other cause
|
|
2868
|
+
// (env_broken, auth_expired, claude_unavailable, …) is a whole-runner fault
|
|
2869
|
+
// with no provider to blame — labelling it with the task's engine would be a
|
|
2870
|
+
// false attribution in the audit field (capacityEngine() also defaults a
|
|
2871
|
+
// missing engine to claude-code). null = "no specific engine".
|
|
2872
|
+
const engineId = cause === "usage_limit" && engine ? capacityEngine(engine) : null;
|
|
2537
2873
|
const qState = {
|
|
2538
2874
|
cause,
|
|
2539
2875
|
classifiedAt: new Date().toISOString(),
|
|
@@ -2546,6 +2882,12 @@ consecutive) {
|
|
|
2546
2882
|
});
|
|
2547
2883
|
process.stderr.write(chalk.red.bold(`[acc-runner] QUARANTINE: runner ${state.session.runner_id} quarantined ` +
|
|
2548
2884
|
`(${cause}) after task ${taskId} failed.\n`));
|
|
2885
|
+
const retryHint = cause === "usage_limit"
|
|
2886
|
+
? `capacity-paused fleet-wide (see /api/health fleet_pause); resumes ` +
|
|
2887
|
+
`${usageLimitResumeAtIso ?? "at an unknown time (indefinite cap)"} — this ` +
|
|
2888
|
+
`runner un-quarantines automatically on resume, but task ${taskId} must be ` +
|
|
2889
|
+
`retried manually (retry_task RPC / \`/acc retry\`) once capacity returns.`
|
|
2890
|
+
: undefined;
|
|
2549
2891
|
void postRunnerStateMessage(state.supabase, {
|
|
2550
2892
|
task_id: taskId,
|
|
2551
2893
|
sender_id: state.session.runner_id,
|
|
@@ -2555,7 +2897,9 @@ consecutive) {
|
|
|
2555
2897
|
capacity_alert: true,
|
|
2556
2898
|
quarantine_cause: cause,
|
|
2557
2899
|
runner_id: state.session.runner_id,
|
|
2900
|
+
engine: engineId,
|
|
2558
2901
|
detail,
|
|
2902
|
+
...(retryHint ? { retry_hint: retryHint, capacity_resume_at: usageLimitResumeAtIso ?? null } : {}),
|
|
2559
2903
|
},
|
|
2560
2904
|
kind: "blocked",
|
|
2561
2905
|
}).catch((err) => {
|
|
@@ -2566,7 +2910,13 @@ consecutive) {
|
|
|
2566
2910
|
await state.supabase.rpc("log_activity", {
|
|
2567
2911
|
p_verb: "runner.quarantine_enter",
|
|
2568
2912
|
p_target_id: state.session.runner_id,
|
|
2569
|
-
p_payload: {
|
|
2913
|
+
p_payload: {
|
|
2914
|
+
cause,
|
|
2915
|
+
engine: engineId,
|
|
2916
|
+
task_id: taskId,
|
|
2917
|
+
detail,
|
|
2918
|
+
...(retryHint ? { retry_hint: retryHint, capacity_resume_at: usageLimitResumeAtIso ?? null } : {}),
|
|
2919
|
+
},
|
|
2570
2920
|
p_target_type: "runner",
|
|
2571
2921
|
});
|
|
2572
2922
|
}
|
|
@@ -2590,7 +2940,7 @@ consecutive) {
|
|
|
2590
2940
|
* Idempotent: a straggler cap while already paused only ever pushes the reset
|
|
2591
2941
|
* later (the window fold's extend-only rule), never shortens the wait.
|
|
2592
2942
|
*/
|
|
2593
|
-
function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail) {
|
|
2943
|
+
function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail, engine) {
|
|
2594
2944
|
const now = Date.now();
|
|
2595
2945
|
const parsed = resumeAtIso ? Date.parse(resumeAtIso) : NaN;
|
|
2596
2946
|
const baseResumeMs = Number.isFinite(parsed) && parsed > now ? parsed : now + state.capacityBackoffMs;
|
|
@@ -2604,17 +2954,53 @@ function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail) {
|
|
|
2604
2954
|
// canonical pause payload + bus message (the seam only sees {account, reset}).
|
|
2605
2955
|
state.capacityTriggerTaskId = triggerTaskId;
|
|
2606
2956
|
state.capacityPauseDetail = detail;
|
|
2957
|
+
state.capacityPauseEngine = capacityEngine(engine);
|
|
2607
2958
|
// Stop claiming SYNCHRONOUSLY (the controller's stopClaims seam runs on the
|
|
2608
2959
|
// serialized action tail a microtask later; pump()/pumpReviews() read this
|
|
2609
2960
|
// flag between their own awaits, so setting it here closes the gap and keeps
|
|
2610
2961
|
// the v0.56 "no new claim the instant a 429 lands" guarantee byte-identical).
|
|
2611
|
-
state.pausedCapacity
|
|
2962
|
+
state.pausedCapacity.add(state.capacityPauseEngine);
|
|
2612
2963
|
state.resumeController.reportCapacity({
|
|
2613
2964
|
accountKey: state.resumeAccountKey ?? state.session.runner_id,
|
|
2614
2965
|
resetAtMs: resumeMs,
|
|
2615
2966
|
detail,
|
|
2616
2967
|
});
|
|
2617
2968
|
}
|
|
2969
|
+
/**
|
|
2970
|
+
* USAGE-LIMIT-INVISIBLE-1: fold a usage_limit quarantine into the SAME
|
|
2971
|
+
* fleet-visible capacity signal `enterCapacityPause` emits for
|
|
2972
|
+
* capacity_exhausted, so `/api/health` fleet_pause and the FH-1b dispatch
|
|
2973
|
+
* governor see the account as out of capacity instead of reading a live
|
|
2974
|
+
* `paused_count: 0` while every task quietly dies (the 2026-08-30 incident:
|
|
2975
|
+
* four usage_limit exits, zero capacity signal anywhere).
|
|
2976
|
+
*
|
|
2977
|
+
* Deliberately NOT `enterCapacityPause`: that helper falls back to a fixed
|
|
2978
|
+
* backoff (`now + capacityBackoffMs`) when no reset was parsed, which is the
|
|
2979
|
+
* right call for a KNOWN-recoverable 429 but would fabricate a retry time for
|
|
2980
|
+
* a class the classifier does not promise resets on. Here a null resume
|
|
2981
|
+
* folds onto the account window as an INDEFINITE cap (account-window.ts
|
|
2982
|
+
* `isStillCapped`) — visible in fleet_pause as `resume_at: null` on a
|
|
2983
|
+
* `paused_count > 0` row (never silently indistinguishable from "no pause"),
|
|
2984
|
+
* and it only clears on an account swap, a positive probe ping, or the
|
|
2985
|
+
* runner's own quarantine being cleared and re-observing the account.
|
|
2986
|
+
*
|
|
2987
|
+
* Runner-local claiming is already halted by `state.quarantined` (set by the
|
|
2988
|
+
* caller's `enterQuarantine`); this only drives the fleet-visible signal +
|
|
2989
|
+
* the resume bookkeeping so OTHER runners sharing the account — and the
|
|
2990
|
+
* dispatcher — stop feeding it work too.
|
|
2991
|
+
*/
|
|
2992
|
+
function signalUsageLimitCapacity(state, triggerTaskId, resumeAtIso, detail, engine) {
|
|
2993
|
+
const parsed = resumeAtIso ? Date.parse(resumeAtIso) : NaN;
|
|
2994
|
+
const resetAtMs = Number.isFinite(parsed) ? parsed : null;
|
|
2995
|
+
state.capacityTriggerTaskId = triggerTaskId;
|
|
2996
|
+
state.capacityPauseDetail = detail;
|
|
2997
|
+
state.capacityPauseEngine = capacityEngine(engine);
|
|
2998
|
+
state.resumeController.reportCapacity({
|
|
2999
|
+
accountKey: state.resumeAccountKey ?? state.session.runner_id,
|
|
3000
|
+
resetAtMs,
|
|
3001
|
+
detail,
|
|
3002
|
+
});
|
|
3003
|
+
}
|
|
2618
3004
|
/**
|
|
2619
3005
|
* AS-3 (RESUME-WIRE): the ResumeController side-effect seams, backed by the REAL
|
|
2620
3006
|
* runner path (log_activity RPCs + pollOnce/pump). Exported so the account-switch
|
|
@@ -2643,6 +3029,7 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2643
3029
|
const resumeAtIso = resetAtMs !== null ? new Date(resetAtMs).toISOString() : null;
|
|
2644
3030
|
const detail = state.capacityPauseDetail;
|
|
2645
3031
|
const triggerTaskId = state.capacityTriggerTaskId;
|
|
3032
|
+
const engine = state.capacityPauseEngine;
|
|
2646
3033
|
state.accountProbe?.setPaused(true);
|
|
2647
3034
|
process.stderr.write(chalk.yellow(`[acc-runner] PAUSED (capacity) on ${account}: ${detail}. Not claiming ` +
|
|
2648
3035
|
`tasks or reviews; resuming at ${resumeAtIso ?? "reset / account change"}.\n`));
|
|
@@ -2654,6 +3041,7 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2654
3041
|
runner_id: state.session.runner_id,
|
|
2655
3042
|
resume_at: resumeAtIso,
|
|
2656
3043
|
reason: detail,
|
|
3044
|
+
engine,
|
|
2657
3045
|
account,
|
|
2658
3046
|
version: PACKAGE_VERSION,
|
|
2659
3047
|
},
|
|
@@ -2675,6 +3063,7 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2675
3063
|
capacity_paused: true,
|
|
2676
3064
|
runner_id: state.session.runner_id,
|
|
2677
3065
|
resume_at: resumeAtIso,
|
|
3066
|
+
engine,
|
|
2678
3067
|
detail,
|
|
2679
3068
|
},
|
|
2680
3069
|
kind: "blocked",
|
|
@@ -2684,7 +3073,8 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2684
3073
|
}
|
|
2685
3074
|
},
|
|
2686
3075
|
stopClaims: () => {
|
|
2687
|
-
state.
|
|
3076
|
+
if (state.capacityPauseEngine)
|
|
3077
|
+
state.pausedCapacity.add(state.capacityPauseEngine);
|
|
2688
3078
|
},
|
|
2689
3079
|
clearServerPause: async () => {
|
|
2690
3080
|
try {
|
|
@@ -2716,7 +3106,34 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2716
3106
|
restartClaims: async () => {
|
|
2717
3107
|
if (state.stopped)
|
|
2718
3108
|
return;
|
|
2719
|
-
|
|
3109
|
+
// USAGE-LIMIT-INVISIBLE-1 (AC4): a usage_limit quarantine anchored to
|
|
3110
|
+
// the SAME account window that just reopened is released automatically
|
|
3111
|
+
// — the account is no longer capped, so there is nothing left for an
|
|
3112
|
+
// operator to "fix" by hand. Scoped to `usage_limit` only: env_broken
|
|
3113
|
+
// has its own probe-gated auto-recovery (quarantineRecoveryScan) and
|
|
3114
|
+
// auth_expired needs a fresh token, not a capacity reset. The task this
|
|
3115
|
+
// runner already failed still needs a manual `retry_task` / `/acc
|
|
3116
|
+
// retry` (it is NOT re-queued here) — see the retry hint on the
|
|
3117
|
+
// quarantine's blocked/error_context message.
|
|
3118
|
+
if (state.quarantined && state.quarantineCause === "usage_limit") {
|
|
3119
|
+
await clearQuarantine().catch(() => { });
|
|
3120
|
+
state.quarantined = false;
|
|
3121
|
+
state.quarantineCause = null;
|
|
3122
|
+
process.stderr.write(chalk.green(`[acc-runner] quarantine auto-cleared: usage_limit capacity window reopened. ` +
|
|
3123
|
+
`Resuming task claims (the quarantined task itself still needs a manual retry).\n`));
|
|
3124
|
+
try {
|
|
3125
|
+
await state.supabase.rpc("log_activity", {
|
|
3126
|
+
p_verb: "runner.quarantine_auto_cleared",
|
|
3127
|
+
p_target_id: state.session.runner_id,
|
|
3128
|
+
p_payload: { cause: "usage_limit", version: PACKAGE_VERSION },
|
|
3129
|
+
p_target_type: "runner",
|
|
3130
|
+
});
|
|
3131
|
+
}
|
|
3132
|
+
catch { /* best-effort */ }
|
|
3133
|
+
}
|
|
3134
|
+
// Resume is account-wide, so clear all engine pauses in this window.
|
|
3135
|
+
state.pausedCapacity.clear();
|
|
3136
|
+
state.capacityPauseEngine = null;
|
|
2720
3137
|
state.pausedUntil = null;
|
|
2721
3138
|
// RVU-2 (FU-RVU2R): a full pause supersedes the soft claim-pause; clear it
|
|
2722
3139
|
// so the probe window (task limit pinned to 1) isn't zeroed out.
|
|
@@ -2755,7 +3172,7 @@ async function wireCapacityResume(state, factory, options) {
|
|
|
2755
3172
|
// accountIdentity() otherwise (an honest per-host synthetic account when no
|
|
2756
3173
|
// real login introspects). Never throws — a failed probe degrades to unknown
|
|
2757
3174
|
// here, and to a logged no-op on the scheduler's later cadence probes.
|
|
2758
|
-
const chatEngine =
|
|
3175
|
+
const chatEngine = getConfiguredChatEngine();
|
|
2759
3176
|
const probe = options.accountProbe ??
|
|
2760
3177
|
(async () => {
|
|
2761
3178
|
if (chatEngine.accountIdentity)
|
|
@@ -2851,12 +3268,9 @@ async function pump(state, factory) {
|
|
|
2851
3268
|
// v0.48: check quarantine before claiming any task.
|
|
2852
3269
|
if (state.quarantined) {
|
|
2853
3270
|
process.stderr.write(`[acc-runner] QUARANTINED: runner ${state.session.runner_id} is not claiming tasks.\n` +
|
|
2854
|
-
`[acc-runner] Fix the underlying issue then run \`acc-runner quarantine clear
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
// v0.56 (T-56-1): out of session/usage capacity — don't claim. Queued
|
|
2858
|
-
// tasks stay queued; the scheduled resume re-drives pump().
|
|
2859
|
-
if (state.pausedCapacity) {
|
|
3271
|
+
`[acc-runner] Fix the underlying issue, then run \`acc-runner quarantine clear\` ` +
|
|
3272
|
+
`for this named instance; repeat for each instance on the host. The running ` +
|
|
3273
|
+
`watcher resumes on its next recovery tick; no restart is required.\n`);
|
|
2860
3274
|
return;
|
|
2861
3275
|
}
|
|
2862
3276
|
// v0.56 (T-56-1): during the post-resume probe window, claim exactly one
|
|
@@ -2878,10 +3292,12 @@ async function pump(state, factory) {
|
|
|
2878
3292
|
// Fill available concurrency slots from the queue.
|
|
2879
3293
|
while (!state.stopped &&
|
|
2880
3294
|
!state.quarantined &&
|
|
2881
|
-
!state.pausedCapacity &&
|
|
2882
3295
|
state.queue.length > 0 &&
|
|
2883
3296
|
state.running.size < effectiveLimit) {
|
|
2884
|
-
const
|
|
3297
|
+
const nextIndex = state.queue.findIndex((id) => !isCapacityPausedFor(state, state.taskEngines.get(id)));
|
|
3298
|
+
if (nextIndex === -1)
|
|
3299
|
+
return;
|
|
3300
|
+
const [next] = state.queue.splice(nextIndex, 1);
|
|
2885
3301
|
if (!next)
|
|
2886
3302
|
continue;
|
|
2887
3303
|
const ctrl = factory(next, {
|
|
@@ -2892,6 +3308,11 @@ async function pump(state, factory) {
|
|
|
2892
3308
|
runnerId: state.session.runner_id,
|
|
2893
3309
|
},
|
|
2894
3310
|
publicUrl: state.cfg.publicUrl,
|
|
3311
|
+
onUnknownAdvertisedEngine: (engineId, err) => repairUnknownAdvertisedEngine(state, engineId, err),
|
|
3312
|
+
// CLAIM-LOCK-GLOB: hand the per-runner claim-loss map
|
|
3313
|
+
// to runTask so file-lock conflicts count attempts across re-dispatch
|
|
3314
|
+
// and back off after the fast attempts are spent (AC3).
|
|
3315
|
+
claimLossState: state.claimLossState,
|
|
2895
3316
|
});
|
|
2896
3317
|
state.running.set(next, ctrl);
|
|
2897
3318
|
// Detached completion handler. Runs AFTER pump() returns so the
|
|
@@ -2901,9 +3322,7 @@ async function pump(state, factory) {
|
|
|
2901
3322
|
.then((outcome) => {
|
|
2902
3323
|
state.running.delete(next);
|
|
2903
3324
|
touchActivity(state); // v0.12.0 (T-52-7): idle clock restarts at task end
|
|
2904
|
-
//
|
|
2905
|
-
// never a task_error, never quarantine. Do not re-fire pump for new
|
|
2906
|
-
// work; the scheduled resume does that with a single probe.
|
|
3325
|
+
// Capacity exhaustion pauses only the refused engine's task claims.
|
|
2907
3326
|
//
|
|
2908
3327
|
// FH-1a: runTask now CLEANLY REQUEUES the in-flight task
|
|
2909
3328
|
// (running→queued) instead of abandoning it to the stale-running
|
|
@@ -2917,7 +3336,7 @@ async function pump(state, factory) {
|
|
|
2917
3336
|
// seen.delete (cancel → re-start).
|
|
2918
3337
|
if (outcome.status === "capacity_paused" || outcome.capacity_exhausted) {
|
|
2919
3338
|
state.seen.delete(next);
|
|
2920
|
-
enterCapacityPause(state, next, outcome.resume_at ?? null, outcome.error || "task hit session/usage limit");
|
|
3339
|
+
enterCapacityPause(state, next, outcome.resume_at ?? null, outcome.error || "task hit session/usage limit", outcome.engine ?? state.taskEngines.get(next));
|
|
2921
3340
|
return;
|
|
2922
3341
|
}
|
|
2923
3342
|
// Repo-scoped ignore: the task's repo is outside this runner's
|
|
@@ -2961,6 +3380,17 @@ async function pump(state, factory) {
|
|
|
2961
3380
|
// does not answer it with a resubscribe storm.
|
|
2962
3381
|
if (outcome.phase === "claim_locks") {
|
|
2963
3382
|
state.claimContentionSince.set(next, Date.now());
|
|
3383
|
+
// CLAIM-LOCK-GLOB: once the fast attempts are
|
|
3384
|
+
// spent runTask returns a bounded yield instant — hold the task
|
|
3385
|
+
// out of the re-dispatch queue until then (AC3) so a losing task
|
|
3386
|
+
// actually backs off instead of re-claiming every backstop cycle.
|
|
3387
|
+
// A null yield leaves it eligible for the next fast retry.
|
|
3388
|
+
if (outcome.claim_lock_yield_until_ms != null) {
|
|
3389
|
+
state.claimYieldUntil.set(next, outcome.claim_lock_yield_until_ms);
|
|
3390
|
+
}
|
|
3391
|
+
else {
|
|
3392
|
+
state.claimYieldUntil.delete(next);
|
|
3393
|
+
}
|
|
2964
3394
|
}
|
|
2965
3395
|
// v0.48: machine-level failure → quarantine. Guard with
|
|
2966
3396
|
// !state.quarantined so concurrent tasks don't fire duplicate
|
|
@@ -3003,12 +3433,22 @@ async function pump(state, factory) {
|
|
|
3003
3433
|
void pump(state, factory);
|
|
3004
3434
|
}
|
|
3005
3435
|
else {
|
|
3006
|
-
enterQuarantine(state, next, cause, reason, outcome.quarantine_consecutive);
|
|
3436
|
+
enterQuarantine(state, next, cause, reason, outcome.quarantine_consecutive, undefined, outcome.engine ?? state.taskEngines.get(next));
|
|
3007
3437
|
}
|
|
3008
3438
|
});
|
|
3009
3439
|
return;
|
|
3010
3440
|
}
|
|
3011
|
-
|
|
3441
|
+
// USAGE-LIMIT-INVISIBLE-1: a usage_limit quarantine is ALSO a
|
|
3442
|
+
// fleet capacity signal — the account (not just this runner) is
|
|
3443
|
+
// out of room. Fold it into the same account-window / fleet_pause
|
|
3444
|
+
// machinery capacity_exhausted uses, carrying whatever reset the
|
|
3445
|
+
// provider stated (outcome.resume_at) or an explicit indefinite
|
|
3446
|
+
// cap (null) — BEFORE quarantining so a paused-fleet dispatcher
|
|
3447
|
+
// read never races an empty fleet_pause against the new quarantine.
|
|
3448
|
+
if (cause === "usage_limit") {
|
|
3449
|
+
signalUsageLimitCapacity(state, next, outcome.resume_at ?? null, reason, outcome.engine ?? state.taskEngines.get(next));
|
|
3450
|
+
}
|
|
3451
|
+
enterQuarantine(state, next, cause, reason, outcome.quarantine_consecutive, cause === "usage_limit" ? outcome.resume_at ?? null : undefined, outcome.engine ?? state.taskEngines.get(next));
|
|
3012
3452
|
// Quarantined — do not re-fire pump for new tasks.
|
|
3013
3453
|
// Other in-flight tasks (still in state.running) complete
|
|
3014
3454
|
// normally; we just stop accepting new work.
|