@tokenfactory/acc-runner 0.44.1 → 0.44.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/dist/bin-resolve.d.ts +39 -3
- package/dist/bin-resolve.d.ts.map +1 -1
- package/dist/bin-resolve.js +176 -12
- package/dist/bin-resolve.js.map +1 -1
- package/dist/build-fingerprint.d.ts +66 -0
- package/dist/build-fingerprint.d.ts.map +1 -0
- package/dist/build-fingerprint.js +70 -0
- package/dist/build-fingerprint.js.map +1 -0
- package/dist/build-info.generated.d.ts +3 -0
- package/dist/build-info.generated.d.ts.map +1 -0
- package/dist/build-info.generated.js +3 -0
- package/dist/build-info.generated.js.map +1 -0
- package/dist/capacity/resume-controller.d.ts +5 -3
- package/dist/capacity/resume-controller.d.ts.map +1 -1
- package/dist/capacity/resume-controller.js +9 -0
- package/dist/capacity/resume-controller.js.map +1 -1
- package/dist/chat-delegation-guard.d.ts +23 -0
- package/dist/chat-delegation-guard.d.ts.map +1 -0
- package/dist/chat-delegation-guard.js +62 -0
- package/dist/chat-delegation-guard.js.map +1 -0
- package/dist/chat-lane.d.ts +17 -6
- package/dist/chat-lane.d.ts.map +1 -1
- package/dist/chat-lane.js +29 -8
- package/dist/chat-lane.js.map +1 -1
- package/dist/companion-identity.d.ts +37 -0
- package/dist/companion-identity.d.ts.map +1 -1
- package/dist/companion-identity.js +98 -29
- package/dist/companion-identity.js.map +1 -1
- package/dist/companion-run.d.ts +1 -50
- package/dist/companion-run.d.ts.map +1 -1
- package/dist/companion-run.js +48 -105
- package/dist/companion-run.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -3
- package/dist/config.js.map +1 -1
- package/dist/conformance.d.ts +49 -0
- package/dist/conformance.d.ts.map +1 -0
- package/dist/conformance.js +121 -0
- package/dist/conformance.js.map +1 -0
- package/dist/cost-pricing.d.ts +46 -0
- package/dist/cost-pricing.d.ts.map +1 -1
- package/dist/cost-pricing.js +58 -1
- package/dist/cost-pricing.js.map +1 -1
- package/dist/credential-identity.d.ts +70 -0
- package/dist/credential-identity.d.ts.map +1 -0
- package/dist/credential-identity.js +128 -0
- package/dist/credential-identity.js.map +1 -0
- package/dist/doctor.d.ts +33 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +284 -44
- package/dist/doctor.js.map +1 -1
- package/dist/engines/chat-stream.d.ts +71 -1
- package/dist/engines/chat-stream.d.ts.map +1 -1
- package/dist/engines/chat-stream.js +126 -8
- package/dist/engines/chat-stream.js.map +1 -1
- package/dist/engines/claude-code-chat.d.ts +11 -1
- package/dist/engines/claude-code-chat.d.ts.map +1 -1
- package/dist/engines/claude-code-chat.js +69 -5
- package/dist/engines/claude-code-chat.js.map +1 -1
- package/dist/engines/claude-code.d.ts.map +1 -1
- package/dist/engines/claude-code.js +30 -6
- package/dist/engines/claude-code.js.map +1 -1
- package/dist/engines/codex.d.ts.map +1 -1
- package/dist/engines/codex.js +18 -5
- package/dist/engines/codex.js.map +1 -1
- package/dist/engines/registry.d.ts +26 -0
- package/dist/engines/registry.d.ts.map +1 -1
- package/dist/engines/registry.js +37 -0
- package/dist/engines/registry.js.map +1 -1
- package/dist/engines/types.d.ts +5 -0
- package/dist/engines/types.d.ts.map +1 -1
- package/dist/failure-classifier.d.ts +1 -1
- package/dist/failure-classifier.d.ts.map +1 -1
- package/dist/failure-classifier.js +65 -12
- package/dist/failure-classifier.js.map +1 -1
- package/dist/gh.d.ts +15 -0
- package/dist/gh.d.ts.map +1 -1
- package/dist/gh.js +18 -0
- package/dist/gh.js.map +1 -1
- package/dist/git.d.ts +6 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +27 -1
- package/dist/git.js.map +1 -1
- package/dist/keychain.d.ts +17 -0
- package/dist/keychain.d.ts.map +1 -1
- package/dist/keychain.js +30 -0
- package/dist/keychain.js.map +1 -1
- package/dist/loc-advisory.d.ts +70 -0
- package/dist/loc-advisory.d.ts.map +1 -0
- package/dist/loc-advisory.js +196 -0
- package/dist/loc-advisory.js.map +1 -0
- package/dist/locks/claim-lock-policy.d.ts +67 -0
- package/dist/locks/claim-lock-policy.d.ts.map +1 -0
- package/dist/locks/claim-lock-policy.js +132 -0
- package/dist/locks/claim-lock-policy.js.map +1 -0
- package/dist/login.d.ts +59 -2
- package/dist/login.d.ts.map +1 -1
- package/dist/login.js +87 -7
- package/dist/login.js.map +1 -1
- package/dist/mcp-spawn.d.ts +54 -0
- package/dist/mcp-spawn.d.ts.map +1 -1
- package/dist/mcp-spawn.js +88 -4
- package/dist/mcp-spawn.js.map +1 -1
- package/dist/migration-seq-guard.d.ts +28 -1
- package/dist/migration-seq-guard.d.ts.map +1 -1
- package/dist/migration-seq-guard.js +111 -6
- package/dist/migration-seq-guard.js.map +1 -1
- package/dist/output-diagnostics.d.ts +15 -0
- package/dist/output-diagnostics.d.ts.map +1 -0
- package/dist/output-diagnostics.js +95 -0
- package/dist/output-diagnostics.js.map +1 -0
- package/dist/pr-open-guard.d.ts +67 -0
- package/dist/pr-open-guard.d.ts.map +1 -0
- package/dist/pr-open-guard.js +79 -0
- package/dist/pr-open-guard.js.map +1 -0
- package/dist/prepr-gate.d.ts +229 -1
- package/dist/prepr-gate.d.ts.map +1 -1
- package/dist/prepr-gate.js +406 -3
- package/dist/prepr-gate.js.map +1 -1
- package/dist/program.d.ts.map +1 -1
- package/dist/program.js +30 -5
- package/dist/program.js.map +1 -1
- package/dist/provider-auth.d.ts +8 -0
- package/dist/provider-auth.d.ts.map +1 -1
- package/dist/provider-auth.js +10 -0
- package/dist/provider-auth.js.map +1 -1
- package/dist/runtime/quarantine.d.ts +11 -0
- package/dist/runtime/quarantine.d.ts.map +1 -1
- package/dist/runtime/quarantine.js +16 -1
- package/dist/runtime/quarantine.js.map +1 -1
- package/dist/runtime/reviewer.d.ts +90 -2
- package/dist/runtime/reviewer.d.ts.map +1 -1
- package/dist/runtime/reviewer.js +547 -46
- package/dist/runtime/reviewer.js.map +1 -1
- package/dist/runtime/running-build.d.ts +24 -0
- package/dist/runtime/running-build.d.ts.map +1 -0
- package/dist/runtime/running-build.js +62 -0
- package/dist/runtime/running-build.js.map +1 -0
- package/dist/runtime/worktree.d.ts +45 -0
- package/dist/runtime/worktree.d.ts.map +1 -1
- package/dist/runtime/worktree.js +174 -5
- package/dist/runtime/worktree.js.map +1 -1
- package/dist/task-runner.d.ts +126 -5
- package/dist/task-runner.d.ts.map +1 -1
- package/dist/task-runner.js +1219 -127
- package/dist/task-runner.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watch-chat/wire.d.ts +80 -0
- package/dist/watch-chat/wire.d.ts.map +1 -1
- package/dist/watch-chat/wire.js +203 -43
- package/dist/watch-chat/wire.js.map +1 -1
- package/dist/watch.d.ts +70 -16
- package/dist/watch.d.ts.map +1 -1
- package/dist/watch.js +501 -89
- package/dist/watch.js.map +1 -1
- package/package.json +2 -2
package/dist/watch.js
CHANGED
|
@@ -32,8 +32,10 @@ import { applyBootstrapConfig, concurrencyEnvExplicit, loadConfig, resolveEffect
|
|
|
32
32
|
// existing tests that import them from `../src/watch.js` keep resolving.
|
|
33
33
|
export { resolveConcurrencyLimit, resolveReservedReviewSlots } from "./config.js";
|
|
34
34
|
import { getTokenProvider } from "./token-provider.js";
|
|
35
|
-
import { machineIdentity, detectInstalledModels, runnerCapsWithRepos } from "./login.js";
|
|
35
|
+
import { advertisedCapabilities, machineIdentity, detectInstalledModels, runnerCapsWithRepos, registerRunnerCaps, } from "./login.js";
|
|
36
36
|
import { detectCapableEngines } from "./doctor.js";
|
|
37
|
+
import { safeText } from "./machine-capabilities/redact.js";
|
|
38
|
+
import { resolveCredentialIdentity } from "./credential-identity.js";
|
|
37
39
|
import { createRunnerClient } from "./supabase.js";
|
|
38
40
|
import { releaseHeldLocks } from "./locks/index.js";
|
|
39
41
|
// SH-4 (PAUSE-CHECKPOINT-RESUME): the additive task_paused broadcast + the
|
|
@@ -42,7 +44,7 @@ import { TASK_PAUSED_EVENT, captureWorktreeCheckpoint, parseTaskPausedBroadcast,
|
|
|
42
44
|
import { worktreePath } from "./runtime/worktree.js";
|
|
43
45
|
import { runTask } from "./task-runner.js";
|
|
44
46
|
import { runReview, } from "./runtime/reviewer.js";
|
|
45
|
-
import { getQuarantine, setQuarantine, clearQuarantine, } from "./runtime/quarantine.js";
|
|
47
|
+
import { getQuarantine, publishedQuarantineState, setQuarantine, clearQuarantine, } from "./runtime/quarantine.js";
|
|
46
48
|
import { acquireSingletonLock, singletonRunnerId, SingletonLockHeldError, } from "./runtime/singleton.js";
|
|
47
49
|
import { getClaudeVersion, recordTaskClaudeVersion, } from "./runtime/version-drift.js";
|
|
48
50
|
import { autoUpgradeDisabled, maybeSelfUpgrade, maybeSelfUpdateOn426, } from "./runtime/self-upgrade.js";
|
|
@@ -62,7 +64,7 @@ import { AccountProbeScheduler } from "./engines/account-probe.js";
|
|
|
62
64
|
// AX-H1 (MACHINE-CAPABILITIES): the host-capability probe + reporter. Driven by
|
|
63
65
|
// the AS-1 account-probe cadence above, never by a loop of its own.
|
|
64
66
|
import { MachineCapabilityReporter, probeMachineCapabilities, } from "./machine-capabilities/index.js";
|
|
65
|
-
import {
|
|
67
|
+
import { getConfiguredChatEngine, getEngine } from "./engines/registry.js";
|
|
66
68
|
// FLEET-SERVE (wire): fleet-fallback chat serving runs ALONGSIDE the task pump on
|
|
67
69
|
// a `watch` runner — the runner serves its bound user's chat turns behind their
|
|
68
70
|
// companion. Glue lives in watch-chat/wire.ts to keep this file the task loop.
|
|
@@ -71,6 +73,8 @@ import { ClaimDeadman, readDispatchSignal, DEFAULT_DEADMAN_CHECK_MS, DEFAULT_FAL
|
|
|
71
73
|
import { unknownAccountIdentity } from "./engines/account-identity.js";
|
|
72
74
|
import { checkVersion, compareSemver } from "./version-check.js";
|
|
73
75
|
import { PACKAGE_VERSION } from "./pkg-version.js";
|
|
76
|
+
import { buildFingerprintCaps, publishBuildFingerprintCaps, LOADED_BUILD_FINGERPRINT, } from "./build-fingerprint.js";
|
|
77
|
+
import { recordRunningBuild } from "./runtime/running-build.js";
|
|
74
78
|
import { authPrecedenceHint, servingAuthKind, describeServingAuthKind, forwardsAmbientKey, anthropicApiKey, } from "./provider-auth.js";
|
|
75
79
|
// RUNNER-SERVING-AUTH: the org's subscription-vs-BYOK choice, resolved once
|
|
76
80
|
// per boot and expressed through the same ACC_RUNNER_SERVING_AUTH_KIND var the
|
|
@@ -82,6 +86,7 @@ import { fetchServingAuth, installServingAuth, servingAuthKindIsPinned, describe
|
|
|
82
86
|
import { RefreshReuseDetectedError, classifyHeartbeatError, expiresSoon, nextHeartbeatDelayMs, refreshAccessToken, } from "./runtime/auth-lifecycle.js";
|
|
83
87
|
// AU-7 RUNNER-FLAP-ROOT-CAUSE: bounded + write-then-read-verified heartbeat.
|
|
84
88
|
import { HEARTBEAT_RPC_TIMEOUT_MS, HeartbeatVerifier, nextBeatDelayMs, sendHeartbeat, withDeadline, } from "./heartbeat/beat.js";
|
|
89
|
+
import { buildEnv, requireExecutable } from "./bin-resolve.js";
|
|
85
90
|
export { RefreshReuseDetectedError, classifyHeartbeatError, nextHeartbeatDelayMs, };
|
|
86
91
|
const REFRESH_CHECK_MS = 30 * 60 * 1000;
|
|
87
92
|
// v0.12.0 (T-52-7): idle-TTL check cadence ceiling. The actual cadence is
|
|
@@ -226,6 +231,21 @@ export function resolveIdleTtlMs(env = process.env) {
|
|
|
226
231
|
}
|
|
227
232
|
return minutes * 60_000;
|
|
228
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* The DEFAULT reviewer factory. FAIL CLOSED: constrain every review to the engines
|
|
236
|
+
* THIS runner ALREADY PUBLISHED into `acc.runners.caps.engines` — the value the
|
|
237
|
+
* assignment was routed against — threaded as a FIXED `advertisedEngineIds`.
|
|
238
|
+
* `advertisedEngines` is resolved ONCE at watch startup from the same
|
|
239
|
+
* `detectCapableEngines()` source that feeds the register_runner caps payload, so
|
|
240
|
+
* it is a stable snapshot. It deliberately does NOT pass a live `detectEngines`
|
|
241
|
+
* probe: a per-review re-probe could drift from the published caps (an engine
|
|
242
|
+
* appearing/disappearing mid-session) and silently route a review to an engine the
|
|
243
|
+
* assignment never assumed. `run` is injectable for the wiring test; production
|
|
244
|
+
* uses `runReview`.
|
|
245
|
+
*/
|
|
246
|
+
export function capsConstrainedReviewerFactory(advertisedEngines, run = runReview) {
|
|
247
|
+
return (assignment, deps) => run(assignment, { ...deps, advertisedEngineIds: advertisedEngines });
|
|
248
|
+
}
|
|
229
249
|
/** v0.12.0 (T-52-7): record work-related activity for the idle TTL clock. */
|
|
230
250
|
function touchActivity(state) {
|
|
231
251
|
state.lastActivityMs = Date.now();
|
|
@@ -242,15 +262,27 @@ function failOnReuse(err) {
|
|
|
242
262
|
}
|
|
243
263
|
// Single source of truth for "this task is new — push it into the runner
|
|
244
264
|
// queue". Both the Realtime listener and the polling loop go through here.
|
|
245
|
-
function enqueue(state, taskId, factory) {
|
|
265
|
+
function enqueue(state, taskId, factory, engine) {
|
|
246
266
|
if (state.seen.has(taskId))
|
|
247
267
|
return false;
|
|
268
|
+
// CLAIM-LOCK-GLOB: a task that lost the claim to a live
|
|
269
|
+
// file-lock holder and spent its fast attempts is held out of the queue until
|
|
270
|
+
// its bounded yield elapses (AC3). Skip re-driving it until then; the periodic
|
|
271
|
+
// claim-scan / backstop re-lists it once the yield expires. Expired stamps are
|
|
272
|
+
// dropped so the very next pass re-drives the task.
|
|
273
|
+
const yieldUntil = state.claimYieldUntil.get(taskId);
|
|
274
|
+
if (yieldUntil != null) {
|
|
275
|
+
if (Date.now() < yieldUntil)
|
|
276
|
+
return false;
|
|
277
|
+
state.claimYieldUntil.delete(taskId);
|
|
278
|
+
}
|
|
248
279
|
// v0.10 T-49-2: check the running Map (replaces v0.9 state.current check).
|
|
249
280
|
if (state.running.has(taskId)) {
|
|
250
281
|
state.seen.add(taskId);
|
|
251
282
|
return false;
|
|
252
283
|
}
|
|
253
284
|
state.seen.add(taskId);
|
|
285
|
+
state.taskEngines.set(taskId, capacityEngine(engine));
|
|
254
286
|
state.queue.push(taskId);
|
|
255
287
|
touchActivity(state);
|
|
256
288
|
// CLAIM-DEADMAN: a claim was processed via SOME path
|
|
@@ -263,6 +295,33 @@ function enqueue(state, taskId, factory) {
|
|
|
263
295
|
void pump(state, factory);
|
|
264
296
|
return true;
|
|
265
297
|
}
|
|
298
|
+
async function lookupAssignedQueuedTaskEngine(state, taskId) {
|
|
299
|
+
const { data, error } = await state.supabase.rpc("list_assigned_queued_tasks", {
|
|
300
|
+
p_runner_id: state.session.runner_id,
|
|
301
|
+
p_since: "1970-01-01T00:00:00.000Z",
|
|
302
|
+
});
|
|
303
|
+
if (error) {
|
|
304
|
+
process.stderr.write(`[acc-runner] task_assigned engine lookup failed for ${taskId}: ${error.message}\n`);
|
|
305
|
+
return { found: false };
|
|
306
|
+
}
|
|
307
|
+
const row = (data ?? []).find((candidate) => candidate?.id === taskId);
|
|
308
|
+
return row ? { found: true, engine: row.engine } : { found: false };
|
|
309
|
+
}
|
|
310
|
+
async function enqueueAssignedTaskBroadcast(state, taskId, factory, payload) {
|
|
311
|
+
if (Object.prototype.hasOwnProperty.call(payload, "engine")) {
|
|
312
|
+
enqueue(state, taskId, factory, payload.engine);
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
const resolved = await lookupAssignedQueuedTaskEngine(state, taskId);
|
|
316
|
+
if (state.stopped || state.seen.has(taskId) || state.running.has(taskId))
|
|
317
|
+
return;
|
|
318
|
+
// Enqueue even when the lookup can't confirm the row (empty result or a
|
|
319
|
+
// transient error). Dropping it would leave the id un-`seen`, so a later
|
|
320
|
+
// delta-poll re-claims the same no-op and drags its watermark past newer
|
|
321
|
+
// tasks (the watch-claim-scan wedge). This mirrors the pre-engine broadcast
|
|
322
|
+
// path, which always enqueued; a still-unknown engine falls back to default.
|
|
323
|
+
enqueue(state, taskId, factory, resolved.found ? resolved.engine : undefined);
|
|
324
|
+
}
|
|
266
325
|
// Single source of truth for "this review is new — push it into the review
|
|
267
326
|
// queue". The Realtime listener AND the periodic review scan (v0.65 T-65-2)
|
|
268
327
|
// both go through here; the shared `seenReviews` guard dedupes a review that
|
|
@@ -340,6 +399,7 @@ async function handleTaskPaused(state, payload) {
|
|
|
340
399
|
}
|
|
341
400
|
// Let the resume redispatch of this SAME id through the dedupe set.
|
|
342
401
|
state.seen.delete(taskId);
|
|
402
|
+
state.taskEngines.delete(taskId);
|
|
343
403
|
}
|
|
344
404
|
function subscribeChannel(state, taskRunnerFactory) {
|
|
345
405
|
state.channel = state.supabase
|
|
@@ -348,8 +408,9 @@ function subscribeChannel(state, taskRunnerFactory) {
|
|
|
348
408
|
})
|
|
349
409
|
.on("broadcast", { event: "task_assigned" }, ({ payload }) => {
|
|
350
410
|
const taskId = payload?.task_id;
|
|
351
|
-
if (taskId)
|
|
352
|
-
|
|
411
|
+
if (taskId) {
|
|
412
|
+
void enqueueAssignedTaskBroadcast(state, taskId, taskRunnerFactory, payload);
|
|
413
|
+
}
|
|
353
414
|
})
|
|
354
415
|
.on("broadcast", { event: "task_cancelled" }, ({ payload }) => {
|
|
355
416
|
const taskId = payload?.task_id;
|
|
@@ -366,6 +427,7 @@ function subscribeChannel(state, taskRunnerFactory) {
|
|
|
366
427
|
// Drop from seen so a re-queue (cancel → re-start) is honored on
|
|
367
428
|
// the next broadcast or poll.
|
|
368
429
|
state.seen.delete(taskId);
|
|
430
|
+
state.taskEngines.delete(taskId);
|
|
369
431
|
})
|
|
370
432
|
.on("broadcast", { event: TASK_PAUSED_EVENT }, ({ payload }) => {
|
|
371
433
|
void handleTaskPaused(state, payload);
|
|
@@ -458,6 +520,15 @@ export async function reassertRunningClaims(state) {
|
|
|
458
520
|
}
|
|
459
521
|
}));
|
|
460
522
|
}
|
|
523
|
+
function capacityEngine(engine) {
|
|
524
|
+
return getEngine(engine ?? null).id;
|
|
525
|
+
}
|
|
526
|
+
function hasCapacityPause(state) {
|
|
527
|
+
return state.pausedCapacity.size > 0;
|
|
528
|
+
}
|
|
529
|
+
function isCapacityPausedFor(state, engine) {
|
|
530
|
+
return state.pausedCapacity.has(capacityEngine(engine));
|
|
531
|
+
}
|
|
461
532
|
// Single-flight poll. The watermark advances only after a clean response so
|
|
462
533
|
// a transient error doesn't permanently skip rows.
|
|
463
534
|
async function pollOnce(state, factory) {
|
|
@@ -489,7 +560,7 @@ async function pollOnce(state, factory) {
|
|
|
489
560
|
// keeps the watermark a conservative "newest task I claimed by polling"
|
|
490
561
|
// mark and stops the delta-poll losing still-queued work. The periodic
|
|
491
562
|
// full claim scan (claimScan) is the belt to this braces.
|
|
492
|
-
const claimed = enqueue(state, row.id, factory);
|
|
563
|
+
const claimed = enqueue(state, row.id, factory, row.engine);
|
|
493
564
|
if (claimed && row.updated_at && row.updated_at > maxSeen) {
|
|
494
565
|
maxSeen = row.updated_at;
|
|
495
566
|
}
|
|
@@ -536,7 +607,7 @@ async function claimScan(state, factory, context) {
|
|
|
536
607
|
for (const row of rows) {
|
|
537
608
|
if (!row?.id)
|
|
538
609
|
continue;
|
|
539
|
-
if (enqueue(state, row.id, factory))
|
|
610
|
+
if (enqueue(state, row.id, factory, row.engine))
|
|
540
611
|
enqueued += 1;
|
|
541
612
|
}
|
|
542
613
|
if (context === "startup") {
|
|
@@ -603,13 +674,8 @@ async function claimWatchdogScan(state, factory) {
|
|
|
603
674
|
state.claimWatchdogSince.clear();
|
|
604
675
|
return;
|
|
605
676
|
}
|
|
606
|
-
//
|
|
607
|
-
|
|
608
|
-
// leave its tasks queued; a resubscribe changes nothing and produced the
|
|
609
|
-
// resubscribe storm interleaved with QUARANTINED in the claim-livelock
|
|
610
|
-
// incident. Mirror the backstop's guard and drop the stale timers so an
|
|
611
|
-
// age never spans the pause and falsely trips the instant it lifts.
|
|
612
|
-
if (state.quarantined || state.pausedCapacity) {
|
|
677
|
+
// A quarantined runner is deliberately not claiming; resubscribe cannot help.
|
|
678
|
+
if (state.quarantined) {
|
|
613
679
|
state.claimWatchdogSince.clear();
|
|
614
680
|
return;
|
|
615
681
|
}
|
|
@@ -625,8 +691,12 @@ async function claimWatchdogScan(state, factory) {
|
|
|
625
691
|
const now = Date.now();
|
|
626
692
|
const currentIds = new Set();
|
|
627
693
|
for (const row of rows) {
|
|
628
|
-
if (row?.id)
|
|
629
|
-
|
|
694
|
+
if (!row?.id)
|
|
695
|
+
continue;
|
|
696
|
+
// Paused-engine rows are deliberately unclaimed; still watch other engines.
|
|
697
|
+
if (isCapacityPausedFor(state, row.engine))
|
|
698
|
+
continue;
|
|
699
|
+
currentIds.add(row.id);
|
|
630
700
|
}
|
|
631
701
|
// Reset: drop timers for tasks no longer assigned+queued (claimed/unassigned).
|
|
632
702
|
for (const id of [...state.claimWatchdogSince.keys()]) {
|
|
@@ -639,6 +709,17 @@ async function claimWatchdogScan(state, factory) {
|
|
|
639
709
|
if (!currentIds.has(id))
|
|
640
710
|
state.claimContentionSince.delete(id);
|
|
641
711
|
}
|
|
712
|
+
// CLAIM-LOCK-GLOB: same reset for the claim-loss back-off
|
|
713
|
+
// — a task that left the queue is no longer contending, so its attempt count
|
|
714
|
+
// and yield window are stale.
|
|
715
|
+
for (const id of [...state.claimLossState.keys()]) {
|
|
716
|
+
if (!currentIds.has(id))
|
|
717
|
+
state.claimLossState.delete(id);
|
|
718
|
+
}
|
|
719
|
+
for (const id of [...state.claimYieldUntil.keys()]) {
|
|
720
|
+
if (!currentIds.has(id))
|
|
721
|
+
state.claimYieldUntil.delete(id);
|
|
722
|
+
}
|
|
642
723
|
// Observe: stamp the first-seen instant for newly assigned-unclaimed tasks;
|
|
643
724
|
// collect any that have now been stale past the threshold, splitting off
|
|
644
725
|
// those that are merely waiting on file-lock contention (v0.41.4).
|
|
@@ -755,10 +836,8 @@ async function claimWatchdogScan(state, factory) {
|
|
|
755
836
|
async function claimBackstopScan(state, factory) {
|
|
756
837
|
if (state.stopped || state.backstopScanning)
|
|
757
838
|
return;
|
|
758
|
-
//
|
|
759
|
-
|
|
760
|
-
// recovers any genuine `seen`-trap once the runner is claiming again.
|
|
761
|
-
if (state.quarantined || state.pausedCapacity)
|
|
839
|
+
// Quarantine is a deliberate whole-runner stop; capacity is checked per row.
|
|
840
|
+
if (state.quarantined)
|
|
762
841
|
return;
|
|
763
842
|
state.backstopScanning = true;
|
|
764
843
|
try {
|
|
@@ -776,6 +855,9 @@ async function claimBackstopScan(state, factory) {
|
|
|
776
855
|
for (const row of rows) {
|
|
777
856
|
if (!row?.id)
|
|
778
857
|
continue;
|
|
858
|
+
// Leave paused-engine rows queued; recover rows for claimable engines.
|
|
859
|
+
if (isCapacityPausedFor(state, row.engine))
|
|
860
|
+
continue;
|
|
779
861
|
// The wedge signature: DB says assigned+queued, but the id is trapped in
|
|
780
862
|
// `seen` while live in neither `queue` nor `running`. Clear `seen` and let
|
|
781
863
|
// enqueue() re-drive it (enqueue re-adds to `seen` + pumps).
|
|
@@ -783,7 +865,7 @@ async function claimBackstopScan(state, factory) {
|
|
|
783
865
|
!state.running.has(row.id) &&
|
|
784
866
|
!inQueue.has(row.id)) {
|
|
785
867
|
state.seen.delete(row.id);
|
|
786
|
-
if (enqueue(state, row.id, factory))
|
|
868
|
+
if (enqueue(state, row.id, factory, row.engine))
|
|
787
869
|
recovered.push(row.id);
|
|
788
870
|
}
|
|
789
871
|
}
|
|
@@ -821,17 +903,39 @@ async function claimBackstopScan(state, factory) {
|
|
|
821
903
|
* incident). This pass re-probes claude while quarantined under `env_broken`
|
|
822
904
|
* and, when the probe passes, clears the quarantine and resumes claiming.
|
|
823
905
|
*
|
|
824
|
-
*
|
|
825
|
-
*
|
|
826
|
-
*
|
|
906
|
+
* The sentinel is authoritative for every cause: an operator clearing this
|
|
907
|
+
* instance's file releases a running watcher on the next pass. Automatic
|
|
908
|
+
* probing remains scoped to `env_broken`; a usage_limit with a known reset is
|
|
909
|
+
* also released by capacity resume, while a hard billing wall and auth_expired
|
|
910
|
+
* require explicit operator action. Single-flight; never throws out of the loop.
|
|
827
911
|
*/
|
|
828
912
|
async function quarantineRecoveryScan(state, factory) {
|
|
829
913
|
if (state.stopped || state.quarantineRecovering)
|
|
830
914
|
return;
|
|
831
|
-
if (!state.quarantined || state.quarantineCause !== "env_broken")
|
|
832
|
-
return;
|
|
833
915
|
state.quarantineRecovering = true;
|
|
834
916
|
try {
|
|
917
|
+
const sentinel = await getQuarantine();
|
|
918
|
+
if (!sentinel) {
|
|
919
|
+
if (!state.quarantined)
|
|
920
|
+
return;
|
|
921
|
+
const cause = state.quarantineCause;
|
|
922
|
+
state.quarantined = false;
|
|
923
|
+
state.quarantineCause = null;
|
|
924
|
+
process.stderr.write(chalk.green(`[acc-runner] quarantine cleared for this instance (${cause ?? "unknown"}); ` +
|
|
925
|
+
`resuming task claims without a restart.\n`));
|
|
926
|
+
if (cause === "usage_limit" && state.pausedCapacity) {
|
|
927
|
+
state.resumeController.resumeByOperator();
|
|
928
|
+
await state.resumeController.settle();
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
void pollOnce(state, factory);
|
|
932
|
+
void pump(state, factory);
|
|
933
|
+
return;
|
|
934
|
+
}
|
|
935
|
+
state.quarantined = true;
|
|
936
|
+
state.quarantineCause = sentinel.cause;
|
|
937
|
+
if (sentinel.cause !== "env_broken")
|
|
938
|
+
return;
|
|
835
939
|
const probe = await state.healthProbe();
|
|
836
940
|
if (!probe.ok)
|
|
837
941
|
return; // still broken — stay quarantined, try again next tick.
|
|
@@ -867,12 +971,11 @@ async function quarantineRecoveryScan(state, factory) {
|
|
|
867
971
|
* CLAIM-DEADMAN: a runner may have free capacity but be
|
|
868
972
|
* deliberately not claiming (quarantined, capacity-paused, review-claim-paused,
|
|
869
973
|
* or already at its effective task-claim ceiling). In those states pending work
|
|
870
|
-
* SHOULD sit queued, so it must NOT be read as a wedge.
|
|
871
|
-
*
|
|
872
|
-
* can claim another task.
|
|
974
|
+
* SHOULD sit queued, so it must NOT be read as a wedge. Returns true only when
|
|
975
|
+
* the runner both wants to and can claim another task.
|
|
873
976
|
*/
|
|
874
977
|
function deadmanCanClaim(state) {
|
|
875
|
-
if (state.quarantined || state
|
|
978
|
+
if (state.quarantined || hasCapacityPause(state) || state.capacityClaimPaused) {
|
|
876
979
|
return false;
|
|
877
980
|
}
|
|
878
981
|
const limit = computeTaskClaimLimit({
|
|
@@ -1112,7 +1215,7 @@ async function reviewScan(state, context) {
|
|
|
1112
1215
|
return;
|
|
1113
1216
|
// v0.56 (T-56-1) parity with pumpReviews: don't claim reviews while paused
|
|
1114
1217
|
// for capacity. The scheduled resume re-drives the review pump.
|
|
1115
|
-
if (state
|
|
1218
|
+
if (hasCapacityPause(state))
|
|
1116
1219
|
return;
|
|
1117
1220
|
state.reviewScanning = true;
|
|
1118
1221
|
try {
|
|
@@ -1362,6 +1465,7 @@ export async function watchCommand(options = {}) {
|
|
|
1362
1465
|
gitAuthProbeOnce: async () => false,
|
|
1363
1466
|
quarantineRecoveryOnce: async () => { },
|
|
1364
1467
|
runningCount: () => 0,
|
|
1468
|
+
pausedEnginesSnapshot: () => [],
|
|
1365
1469
|
};
|
|
1366
1470
|
}
|
|
1367
1471
|
// Park: release our flock so a manual restart / operator install isn't
|
|
@@ -1444,22 +1548,41 @@ export async function watchCommand(options = {}) {
|
|
|
1444
1548
|
concurrency: concurrencyLimit,
|
|
1445
1549
|
reservedReviewSlots,
|
|
1446
1550
|
};
|
|
1447
|
-
//
|
|
1448
|
-
//
|
|
1449
|
-
//
|
|
1450
|
-
//
|
|
1451
|
-
//
|
|
1452
|
-
//
|
|
1453
|
-
|
|
1551
|
+
// AC2 (upstream WINDOWS-RUNNER-DISPATCH-1): a runner must never register/
|
|
1552
|
+
// heartbeat as healthy while it cannot resolve the default task engine —
|
|
1553
|
+
// otherwise it accepts claude-code work that can only fail at phase=claude_exit.
|
|
1554
|
+
// This guard runs on EVERY startup path (kept from upstream, ahead of the
|
|
1555
|
+
// unconditional register below); only an explicit options.claudePreflight
|
|
1556
|
+
// override skips it, so a test opts out visibly.
|
|
1557
|
+
const claudePreflight = options.claudePreflight ?? (() => requireExecutable("claude", { env: buildEnv() }));
|
|
1558
|
+
try {
|
|
1559
|
+
claudePreflight();
|
|
1560
|
+
}
|
|
1561
|
+
catch (err) {
|
|
1562
|
+
await singletonLock?.release();
|
|
1563
|
+
process.stderr.write(`[acc-runner] ${chalk.red(err.message)}\n`);
|
|
1564
|
+
exitFn(1);
|
|
1565
|
+
throw err;
|
|
1566
|
+
}
|
|
1567
|
+
// CAPS-ENGINES-STALE-KEYCHAIN-1 (AC1/AC2): caps.engines must be THIS watcher
|
|
1568
|
+
// process's live surface, so refresh it on every boot for EVERY token mode
|
|
1569
|
+
// (was gated to mode==="env"). Env-token runners also create/adopt their row here.
|
|
1570
|
+
{
|
|
1454
1571
|
const boot = createRunnerClient(cfg, session.access_token, session.realtime_config);
|
|
1455
1572
|
const models = await detectInstalledModels();
|
|
1456
|
-
//
|
|
1457
|
-
|
|
1458
|
-
//
|
|
1459
|
-
//
|
|
1460
|
-
|
|
1573
|
+
// AC11: advertise what THIS serving process resolves (seam defaults to real).
|
|
1574
|
+
const engines = await (options.detectEngines ?? detectCapableEngines)();
|
|
1575
|
+
// Record the SERVING model-credential account (from upstream's capacity-
|
|
1576
|
+
// attribution work) so a keychain OR env-token runner's account is
|
|
1577
|
+
// comparable in the runners table from its first heartbeat.
|
|
1578
|
+
const identity = await resolveCredentialIdentity();
|
|
1461
1579
|
const { data: resolvedId, error: regErr } = await boot.rpc("register_runner", {
|
|
1462
1580
|
p_id: session.runner_id,
|
|
1581
|
+
// finding-4: send p_name = runner_id on EVERY boot. For an env row that
|
|
1582
|
+
// IS its name; for a keychain row, register_runner (mig 0349) treats an
|
|
1583
|
+
// incoming name equal to the row id as "no rename" and PRESERVES the
|
|
1584
|
+
// existing (possibly operator-renamed) name — so a boot caps-refresh never
|
|
1585
|
+
// clobbers an operator-set runner name.
|
|
1463
1586
|
p_name: session.runner_id,
|
|
1464
1587
|
p_owner: session.email ?? session.user_id,
|
|
1465
1588
|
p_machine: machineIdentity(),
|
|
@@ -1471,18 +1594,37 @@ export async function watchCommand(options = {}) {
|
|
|
1471
1594
|
// CONC-1: pass the once-resolved concurrency so caps.concurrency /
|
|
1472
1595
|
// caps.reserved_review_slots match the watch loop's real capacity and
|
|
1473
1596
|
// survive register_runner's per-heartbeat caps replace (mig 0153).
|
|
1474
|
-
|
|
1597
|
+
// AC10: SAME shared builder login.ts uses — boot ≡ login caps (now rich:
|
|
1598
|
+
// process caps + serving identity + build fingerprint folded in).
|
|
1599
|
+
p_caps: registerRunnerCaps(cfg, engines, concurrencyCaps, identity, await buildFingerprintCaps()),
|
|
1475
1600
|
p_version: PACKAGE_VERSION,
|
|
1601
|
+
// REGISTER-CAPS-PRESERVE-1: same explicit send as the login path — an
|
|
1602
|
+
// env-token runner re-registers on EVERY boot, so this was the
|
|
1603
|
+
// highest-frequency door into the capability wipe. Mig 0349 now
|
|
1604
|
+
// union-merges capabilities (never shrinks), so this refresh is non-destructive.
|
|
1605
|
+
p_capabilities: advertisedCapabilities(cfg),
|
|
1476
1606
|
});
|
|
1477
1607
|
if (regErr) {
|
|
1478
|
-
|
|
1479
|
-
process.
|
|
1608
|
+
// AC3: env path keeps its exact "ephemeral" wording; keychain path is new.
|
|
1609
|
+
process.stderr.write(`[acc-runner] ${tokenProvider.mode === "env" ? "ephemeral" : "boot"} register_runner failed: ${safeText(regErr.message, 300) ?? "unknown error"}\n`);
|
|
1610
|
+
if (tokenProvider.mode === "env")
|
|
1611
|
+
process.exit(1);
|
|
1480
1612
|
}
|
|
1481
|
-
|
|
1482
|
-
|
|
1613
|
+
else {
|
|
1614
|
+
// AC-SAFETY: ONLY env-token runners (no login flow; id may be server-minted)
|
|
1615
|
+
// adopt a resolved id and persist it. A keychain runner has a valid
|
|
1616
|
+
// logged-in identity — refresh its caps only, NEVER rewrite its stored
|
|
1617
|
+
// session (an unguarded identity rewrite risks orphaning the credential).
|
|
1618
|
+
if (tokenProvider.mode === "env") {
|
|
1619
|
+
if (typeof resolvedId === "string" && resolvedId && resolvedId !== session.runner_id) {
|
|
1620
|
+
session = { ...session, runner_id: resolvedId };
|
|
1621
|
+
}
|
|
1622
|
+
await tokenProvider.save(session);
|
|
1623
|
+
}
|
|
1624
|
+
console.log(chalk.green(tokenProvider.mode === "env"
|
|
1625
|
+
? `✓ Registered ephemeral runner: ${session.runner_id}`
|
|
1626
|
+
: `✓ Refreshed runner caps: ${session.runner_id}`));
|
|
1483
1627
|
}
|
|
1484
|
-
await tokenProvider.save(session);
|
|
1485
|
-
console.log(chalk.green(`✓ Registered ephemeral runner: ${session.runner_id}`));
|
|
1486
1628
|
}
|
|
1487
1629
|
// RUNNER-SERVING-AUTH: ask the control plane which serving auth this ORG
|
|
1488
1630
|
// chose in Settings, before the banner below reads it. This is what turns
|
|
@@ -1575,15 +1717,29 @@ export async function watchCommand(options = {}) {
|
|
|
1575
1717
|
const taskRunnerFactory = options.taskRunnerFactory ?? runTask;
|
|
1576
1718
|
const heartbeatMs = options.heartbeatMs ?? 4_000;
|
|
1577
1719
|
const pollMs = options.pollMs ?? POLL_INTERVAL_MS;
|
|
1578
|
-
|
|
1720
|
+
// CROSS-ENGINE-REVIEW-1 (AC3) — FAIL CLOSED: constrain reviewer-engine selection
|
|
1721
|
+
// to the engines THIS runner ALREADY PUBLISHED into `acc.runners.caps.engines` —
|
|
1722
|
+
// the value the assignment was routed against — NOT a fresh per-review probe.
|
|
1723
|
+
// Resolved ONCE here from the SAME `detectCapableEngines()` source that feeds the
|
|
1724
|
+
// register_runner caps payload, then threaded as a fixed `advertisedEngineIds`.
|
|
1725
|
+
// A per-review re-probe could drift from the published caps (an engine appearing
|
|
1726
|
+
// or disappearing mid-session), silently routing a review to an engine the
|
|
1727
|
+
// assignment never assumed; the fixed snapshot cannot. A claude-only runner is
|
|
1728
|
+
// never routed to codex, and an empty set fails closed downstream (no candidate →
|
|
1729
|
+
// no-route infra error) rather than falling back to the local registry. Tests
|
|
1730
|
+
// inject `options.reviewerFactory`.
|
|
1731
|
+
const advertisedEngines = await detectCapableEngines();
|
|
1732
|
+
const reviewerFactory = options.reviewerFactory ?? capsConstrainedReviewerFactory(advertisedEngines);
|
|
1579
1733
|
// v0.48: check for an existing quarantine file before opening the
|
|
1580
1734
|
// realtime connection. If quarantined, emit a loud warning — the
|
|
1581
1735
|
// runner will not claim tasks until `acc-runner quarantine clear` runs.
|
|
1582
1736
|
const startupQuarantine = await getQuarantine().catch(() => null);
|
|
1583
1737
|
if (startupQuarantine) {
|
|
1584
1738
|
process.stderr.write(chalk.red.bold(`[acc-runner] QUARANTINED (${startupQuarantine.cause}): ${startupQuarantine.detail}\n`));
|
|
1585
|
-
process.stderr.write(chalk.red(`[acc-runner] Runner will NOT claim tasks. Resolve the issue then run ` +
|
|
1586
|
-
`\`acc-runner quarantine clear
|
|
1739
|
+
process.stderr.write(chalk.red(`[acc-runner] Runner will NOT claim tasks. Resolve the issue, then run ` +
|
|
1740
|
+
`\`acc-runner quarantine clear\` for this named instance; repeat for each ` +
|
|
1741
|
+
`instance on the host. The running watcher resumes on its next recovery tick; ` +
|
|
1742
|
+
`no restart is required.\n`));
|
|
1587
1743
|
}
|
|
1588
1744
|
// v0.12.0 (T-52-7): idle TTL — option override wins (tests), then env.
|
|
1589
1745
|
const idleTtlMs = options.idleTtlMs !== undefined ? options.idleTtlMs : resolveIdleTtlMs();
|
|
@@ -1609,6 +1765,8 @@ export async function watchCommand(options = {}) {
|
|
|
1609
1765
|
claimWatchdogChecking: false,
|
|
1610
1766
|
// v0.41.4: file-lock contention tracker (see WatchState doc).
|
|
1611
1767
|
claimContentionSince: new Map(),
|
|
1768
|
+
claimLossState: new Map(),
|
|
1769
|
+
claimYieldUntil: new Map(),
|
|
1612
1770
|
// FIX-D: periodic DB-poll claim backstop.
|
|
1613
1771
|
claimBackstopTimer: undefined,
|
|
1614
1772
|
claimBackstopIntervalMs: options.claimBackstopIntervalMs ?? DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS,
|
|
@@ -1625,6 +1783,7 @@ export async function watchCommand(options = {}) {
|
|
|
1625
1783
|
wedgeThresholdMs: options.claimDeadmanWedgeMs ?? DEFAULT_CLAIM_DEADMAN_WEDGE_MS,
|
|
1626
1784
|
rebuildGraceMs: options.claimDeadmanRebuildGraceMs ?? DEFAULT_CLAIM_DEADMAN_REBUILD_GRACE_MS,
|
|
1627
1785
|
maxRebuilds: options.claimDeadmanMaxRebuilds ?? DEFAULT_CLAIM_DEADMAN_MAX_REBUILDS,
|
|
1786
|
+
now: options.claimDeadmanNow,
|
|
1628
1787
|
}),
|
|
1629
1788
|
claimDeadmanTimer: undefined,
|
|
1630
1789
|
claimDeadmanCheckMs: options.claimDeadmanCheckMs ?? DEFAULT_DEADMAN_CHECK_MS,
|
|
@@ -1666,6 +1825,7 @@ export async function watchCommand(options = {}) {
|
|
|
1666
1825
|
// RVU-2 (FU-RVU2R): claim-pause starts off; set under review capacity pressure.
|
|
1667
1826
|
capacityClaimPaused: false,
|
|
1668
1827
|
queue: [],
|
|
1828
|
+
droppedEngines: new Set(),
|
|
1669
1829
|
seen: new Set(),
|
|
1670
1830
|
pumping: false,
|
|
1671
1831
|
stopped: false,
|
|
@@ -1691,10 +1851,12 @@ export async function watchCommand(options = {}) {
|
|
|
1691
1851
|
selfUpgradeFn: options.selfUpgrade ?? ((input) => maybeSelfUpgrade(input)),
|
|
1692
1852
|
selfUpgrading: false,
|
|
1693
1853
|
taskRunnerFactory,
|
|
1694
|
-
pausedCapacity:
|
|
1854
|
+
pausedCapacity: new Set(),
|
|
1695
1855
|
pausedUntil: null,
|
|
1696
1856
|
capacityTriggerTaskId: null,
|
|
1697
1857
|
capacityPauseDetail: "",
|
|
1858
|
+
capacityPauseEngine: null,
|
|
1859
|
+
taskEngines: new Map(),
|
|
1698
1860
|
// AS-3 (RESUME-WIRE): assigned immediately below once `state` exists (the
|
|
1699
1861
|
// controller's seams close over `state`). `undefined as unknown as` mirrors
|
|
1700
1862
|
// the channel/timer bootstrap fields above.
|
|
@@ -1815,7 +1977,7 @@ export async function watchCommand(options = {}) {
|
|
|
1815
1977
|
state.claimBackstopTimer = setInterval(() => {
|
|
1816
1978
|
void claimBackstopScan(state, taskRunnerFactory);
|
|
1817
1979
|
}, state.claimBackstopIntervalMs);
|
|
1818
|
-
//
|
|
1980
|
+
// Reconcile operator clears for every cause and probe env_broken recovery.
|
|
1819
1981
|
state.quarantineRecoveryTimer = setInterval(() => {
|
|
1820
1982
|
void quarantineRecoveryScan(state, taskRunnerFactory);
|
|
1821
1983
|
}, state.quarantineRecoveryIntervalMs);
|
|
@@ -1848,10 +2010,20 @@ export async function watchCommand(options = {}) {
|
|
|
1848
2010
|
// requiring a full `acc-runner login` re-flow. Default-empty is the
|
|
1849
2011
|
// backward-compat path for 0.6.3 runners and matches any task with
|
|
1850
2012
|
// an empty required_capabilities array.
|
|
1851
|
-
|
|
2013
|
+
//
|
|
2014
|
+
// REGISTER-CAPS-PRESERVE-1: skip the publish when this runner declared NO
|
|
2015
|
+
// tags. set_runner_capabilities (0114) is an unconditional `capabilities =
|
|
2016
|
+
// coalesce(p_capabilities, '{}')`, so an empty array is a pure WIPE carrying
|
|
2017
|
+
// no information — and it is the OTHER door into the fleet-starvation
|
|
2018
|
+
// incidents (the 03:20 launchctl reload took this path, not register_runner:
|
|
2019
|
+
// a local-token runner never re-runs the boot register). Clearing tags is a
|
|
2020
|
+
// board action, not a side effect of an unset env var. A non-empty set is
|
|
2021
|
+
// still published verbatim, so narrowing from the env stays one restart.
|
|
2022
|
+
const advertised = advertisedCapabilities(cfg);
|
|
2023
|
+
if (advertised.length > 0) {
|
|
1852
2024
|
const { error: capsErr } = await state.supabase.rpc("set_runner_capabilities", {
|
|
1853
2025
|
p_id: state.session.runner_id,
|
|
1854
|
-
p_capabilities:
|
|
2026
|
+
p_capabilities: advertised,
|
|
1855
2027
|
});
|
|
1856
2028
|
if (capsErr) {
|
|
1857
2029
|
// Non-fatal: a pre-0114 server (or an Old DB the runner is
|
|
@@ -1892,11 +2064,25 @@ export async function watchCommand(options = {}) {
|
|
|
1892
2064
|
if (cfg.repos && cfg.repos.length > 0) {
|
|
1893
2065
|
console.log(chalk.gray(` serving repos: ${cfg.repos.join(", ")}`));
|
|
1894
2066
|
}
|
|
2067
|
+
// BUILD-FINGERPRINT-RUNNING-1: stamp the build THIS long-running process
|
|
2068
|
+
// loaded onto disk so a fresh `acc version` or `acc doctor` can later tell
|
|
2069
|
+
// whether an on-disk `npm i -g` has been picked up by this launchd process
|
|
2070
|
+
// yet — a question a fresh CLI invocation (loaded === installed) cannot
|
|
2071
|
+
// answer. Both `acc version` and `acc doctor` read this record.
|
|
2072
|
+
await recordRunningBuild(LOADED_BUILD_FINGERPRINT).catch((err) => {
|
|
2073
|
+
process.stderr.write(`[acc-runner] running-build record failed (continuing): ${err.message}\n`);
|
|
2074
|
+
});
|
|
1895
2075
|
// Initial heartbeat fires immediately so the runner page reflects "online".
|
|
1896
2076
|
await state.supabase.rpc("heartbeat_runner", {
|
|
1897
2077
|
p_id: state.session.runner_id,
|
|
1898
2078
|
p_version: PACKAGE_VERSION,
|
|
1899
2079
|
});
|
|
2080
|
+
await publishBuildFingerprintCapsDetail(state).catch((err) => {
|
|
2081
|
+
process.stderr.write(`[acc-runner] build fingerprint publish failed (continuing): ${err.message}\n`);
|
|
2082
|
+
});
|
|
2083
|
+
await publishQuarantineDetail(state).catch((err) => {
|
|
2084
|
+
process.stderr.write(`[acc-runner] quarantine detail publish failed (continuing): ${err.message}\n`);
|
|
2085
|
+
});
|
|
1900
2086
|
// RUNCFG-1: pull + apply any server-set concurrency at startup (and publish
|
|
1901
2087
|
// the effective ceiling + source into caps) so a runner that boots with an
|
|
1902
2088
|
// operator override already in place honours it from the first tick, not only
|
|
@@ -2057,8 +2243,49 @@ export async function watchCommand(options = {}) {
|
|
|
2057
2243
|
gitAuthProbeOnce: () => gitAuthProbeScan(state), // GIT-AUTH-ALERT
|
|
2058
2244
|
quarantineRecoveryOnce: () => quarantineRecoveryScan(state, taskRunnerFactory), // v0.41.4
|
|
2059
2245
|
runningCount: () => state.running.size,
|
|
2246
|
+
pausedEnginesSnapshot: () => [...state.pausedCapacity],
|
|
2060
2247
|
};
|
|
2061
2248
|
}
|
|
2249
|
+
async function publishQuarantineDetail(state) {
|
|
2250
|
+
const sentinel = await getQuarantine().catch(() => null);
|
|
2251
|
+
const published = publishedQuarantineState(sentinel);
|
|
2252
|
+
if (sentinel && !published) {
|
|
2253
|
+
await clearQuarantine().catch(() => undefined);
|
|
2254
|
+
state.quarantined = false;
|
|
2255
|
+
state.quarantineCause = null;
|
|
2256
|
+
}
|
|
2257
|
+
const { data, error: readErr } = await state.supabase
|
|
2258
|
+
.from("runners")
|
|
2259
|
+
.select("detail")
|
|
2260
|
+
.eq("id", state.session.runner_id)
|
|
2261
|
+
.maybeSingle();
|
|
2262
|
+
if (readErr)
|
|
2263
|
+
throw new Error(readErr.message);
|
|
2264
|
+
const current = data?.detail && typeof data.detail === "object" && !Array.isArray(data.detail)
|
|
2265
|
+
? { ...data.detail }
|
|
2266
|
+
: {};
|
|
2267
|
+
if (published)
|
|
2268
|
+
current.quarantine = published;
|
|
2269
|
+
else
|
|
2270
|
+
delete current.quarantine;
|
|
2271
|
+
const { error: writeErr } = await state.supabase
|
|
2272
|
+
.from("runners")
|
|
2273
|
+
.update({ detail: current })
|
|
2274
|
+
.eq("id", state.session.runner_id);
|
|
2275
|
+
if (writeErr)
|
|
2276
|
+
throw new Error(writeErr.message);
|
|
2277
|
+
}
|
|
2278
|
+
/**
|
|
2279
|
+
* BUILD-FINGERPRINT-ON-CAPS: refresh the build fingerprint on the caps
|
|
2280
|
+
* contract (acc.runners.caps.build) — the same field the boot register
|
|
2281
|
+
* populates — so the loaded build is published the same way for every auth
|
|
2282
|
+
* mode, not left to a best-effort `detail` side-channel. Id-scoped self-update
|
|
2283
|
+
* (never register_runner) so a named-instance keychain runner can't re-home
|
|
2284
|
+
* onto a sibling's row.
|
|
2285
|
+
*/
|
|
2286
|
+
async function publishBuildFingerprintCapsDetail(state) {
|
|
2287
|
+
await publishBuildFingerprintCaps(state.supabase, state.session.runner_id, await buildFingerprintCaps());
|
|
2288
|
+
}
|
|
2062
2289
|
/**
|
|
2063
2290
|
* RUNCFG-1: apply the operator's server-set concurrency to the LIVE watch-loop
|
|
2064
2291
|
* ceiling, precedence server-set > env > default. Idempotent: a no-change row
|
|
@@ -2141,6 +2368,58 @@ async function syncConcurrency(state, factory) {
|
|
|
2141
2368
|
const row = (Array.isArray(data) ? data[0] : data);
|
|
2142
2369
|
applyServerConcurrency(state, row, factory);
|
|
2143
2370
|
}
|
|
2371
|
+
export async function repairUnknownAdvertisedEngine(state, engineId, err, deps = {}) {
|
|
2372
|
+
const detail = safeText(err.message, 300) ?? "unknown engine";
|
|
2373
|
+
// AC6: redact the engine id too before it reaches the log line / event payload
|
|
2374
|
+
// (the DROP below keeps the RAW id — filtering must match caps.engines exactly).
|
|
2375
|
+
const safeEngineId = safeText(engineId, 80) ?? "<redacted>";
|
|
2376
|
+
// AC12: PERSIST the drop so a later republish can never re-advertise it.
|
|
2377
|
+
state.droppedEngines.add(engineId);
|
|
2378
|
+
process.stderr.write(`[acc-runner] runner.engine_caps_self_inconsistent runner=${state.session.runner_id} ` +
|
|
2379
|
+
`engine=${safeEngineId}: ${detail}\n`);
|
|
2380
|
+
try {
|
|
2381
|
+
await state.supabase.rpc("log_activity", {
|
|
2382
|
+
p_verb: "runner.engine_caps_self_inconsistent",
|
|
2383
|
+
p_target_id: state.session.runner_id,
|
|
2384
|
+
p_target_type: "runner",
|
|
2385
|
+
p_payload: {
|
|
2386
|
+
engine: safeEngineId,
|
|
2387
|
+
error: detail,
|
|
2388
|
+
action: "republish_caps",
|
|
2389
|
+
},
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
catch { /* best-effort signal */ }
|
|
2393
|
+
try {
|
|
2394
|
+
const engines = await (deps.detectEngines ?? detectCapableEngines)();
|
|
2395
|
+
const models = await (deps.detectModels ?? detectInstalledModels)();
|
|
2396
|
+
const { error } = await state.supabase.rpc("register_runner", {
|
|
2397
|
+
p_id: state.session.runner_id,
|
|
2398
|
+
// finding-4: p_name = runner_id preserves the operator-set name (mig 0349
|
|
2399
|
+
// treats incoming-name==id as "no rename"), for both env and keychain rows.
|
|
2400
|
+
p_name: state.session.runner_id,
|
|
2401
|
+
p_owner: state.session.email ?? state.session.user_id,
|
|
2402
|
+
p_machine: machineIdentity(),
|
|
2403
|
+
p_models: models,
|
|
2404
|
+
p_caps: runnerCapsWithRepos(state.cfg.repos,
|
|
2405
|
+
// AC12: filter EVERY persisted drop, not just this engineId — otherwise a
|
|
2406
|
+
// repair for one engine re-advertises another that was dropped earlier.
|
|
2407
|
+
engines.filter((id) => !state.droppedEngines.has(id)), {
|
|
2408
|
+
concurrency: state.concurrencyLimit,
|
|
2409
|
+
reservedReviewSlots: state.reservedReviewSlots,
|
|
2410
|
+
}, state.concurrencySource),
|
|
2411
|
+
p_version: PACKAGE_VERSION,
|
|
2412
|
+
p_capabilities: advertisedCapabilities(state.cfg),
|
|
2413
|
+
});
|
|
2414
|
+
if (error) {
|
|
2415
|
+
process.stderr.write(`[acc-runner] runner.engine_caps_self_inconsistent caps repair failed: ${safeText(error.message, 300) ?? "unknown error"}\n`);
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
catch (repairErr) {
|
|
2419
|
+
process.stderr.write(`[acc-runner] runner.engine_caps_self_inconsistent caps repair crashed: ` +
|
|
2420
|
+
`${safeText(repairErr.message, 300) ?? "unknown error"}\n`);
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2144
2423
|
/**
|
|
2145
2424
|
* AU-7: the transport `sendHeartbeat` drives. `readHeartbeatAt` is the audit
|
|
2146
2425
|
* half of the write-then-read verify — one indexed single-row read of the
|
|
@@ -2187,6 +2466,14 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs, lastRoundMs = 0) {
|
|
|
2187
2466
|
const error = beat.ok ? null : { message: `${beat.kind}: ${beat.message}` };
|
|
2188
2467
|
if (!error) {
|
|
2189
2468
|
state.heartbeatFailures = 0;
|
|
2469
|
+
// BUILD-FINGERPRINT: the fingerprint is published ONCE at boot (it cannot
|
|
2470
|
+
// change within a process). It is deliberately NOT re-published on the
|
|
2471
|
+
// heartbeat timer — heartbeat_runner does not touch caps, so the boot
|
|
2472
|
+
// value persists, and a per-beat read-merge-write would amplify the caps
|
|
2473
|
+
// clobber tracked in T-1788838000022 across every runner, every few seconds.
|
|
2474
|
+
await publishQuarantineDetail(state).catch((err) => {
|
|
2475
|
+
process.stderr.write(`[acc-runner] quarantine detail publish failed (continuing): ${err.message}\n`);
|
|
2476
|
+
});
|
|
2190
2477
|
// CLAIM-DEADMAN: capture the pending-dispatch count
|
|
2191
2478
|
// + suspect flag when the heartbeat RESPONSE carries them, so the runner
|
|
2192
2479
|
// learns it is being offered work even with a dead WS. Forward-compatible:
|
|
@@ -2271,7 +2558,8 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs, lastRoundMs = 0) {
|
|
|
2271
2558
|
async function maybeSelfUpgradeTick(state) {
|
|
2272
2559
|
if (state.stopped || state.idleExiting || state.selfUpgrading)
|
|
2273
2560
|
return;
|
|
2274
|
-
|
|
2561
|
+
// Process lifecycle actions wait until all capacity pauses have resumed.
|
|
2562
|
+
if (hasCapacityPause(state))
|
|
2275
2563
|
return;
|
|
2276
2564
|
if (state.running.size > 0 ||
|
|
2277
2565
|
state.queue.length > 0 ||
|
|
@@ -2314,10 +2602,8 @@ async function maybeSelfUpgradeTick(state) {
|
|
|
2314
2602
|
async function maybeIdleExit(state, stop, exitFn) {
|
|
2315
2603
|
if (state.stopped || state.idleExiting || state.idleTtlMs === null)
|
|
2316
2604
|
return;
|
|
2317
|
-
//
|
|
2318
|
-
|
|
2319
|
-
// it out from under the scheduled resume.
|
|
2320
|
-
if (state.pausedCapacity)
|
|
2605
|
+
// A capacity pause is not idleness; keep the process alive for resume.
|
|
2606
|
+
if (hasCapacityPause(state))
|
|
2321
2607
|
return;
|
|
2322
2608
|
if (state.running.size > 0 ||
|
|
2323
2609
|
state.queue.length > 0 ||
|
|
@@ -2345,6 +2631,9 @@ async function maybeIdleExit(state, stop, exitFn) {
|
|
|
2345
2631
|
p_id: state.session.runner_id,
|
|
2346
2632
|
p_version: PACKAGE_VERSION,
|
|
2347
2633
|
});
|
|
2634
|
+
// No fingerprint re-publish on idle exit: it was published at boot and the
|
|
2635
|
+
// process is shutting down, so a read-merge-write here is pure clobber risk.
|
|
2636
|
+
await publishQuarantineDetail(state);
|
|
2348
2637
|
}
|
|
2349
2638
|
catch { /* best-effort */ }
|
|
2350
2639
|
await stop();
|
|
@@ -2399,19 +2688,22 @@ async function pumpReviews(state) {
|
|
|
2399
2688
|
// v0.56 (T-56-1): don't claim reviews while paused for capacity, and hold
|
|
2400
2689
|
// reviews back during the single-probe window so the probe TASK confirms
|
|
2401
2690
|
// the session reopened before we spend more capacity on reviews.
|
|
2402
|
-
if (state
|
|
2691
|
+
if (hasCapacityPause(state) || state.capacityProbe)
|
|
2403
2692
|
return;
|
|
2404
2693
|
state.reviewPumping = true;
|
|
2405
2694
|
try {
|
|
2406
2695
|
while (!state.stopped &&
|
|
2407
|
-
!state
|
|
2696
|
+
!hasCapacityPause(state) &&
|
|
2408
2697
|
!state.capacityProbe &&
|
|
2409
2698
|
state.reviewQueue.length > 0) {
|
|
2410
2699
|
const next = state.reviewQueue.shift();
|
|
2411
2700
|
if (!next)
|
|
2412
2701
|
continue;
|
|
2413
2702
|
try {
|
|
2414
|
-
const outcome = await state.runReview(next, {
|
|
2703
|
+
const outcome = await state.runReview(next, {
|
|
2704
|
+
supabase: state.supabase,
|
|
2705
|
+
onUnknownAdvertisedEngine: (engineId, err) => repairUnknownAdvertisedEngine(state, engineId, err),
|
|
2706
|
+
});
|
|
2415
2707
|
const tag = outcome.decision === "reviewer_error" ? chalk.red : chalk.gray;
|
|
2416
2708
|
console.log(tag(`[acc-runner] review ${next.review_id} task=${next.task_id} pr=${next.pr_number} decision=${outcome.decision} confidence=${outcome.confidence.toFixed(2)}`));
|
|
2417
2709
|
// RVU-2 (FU-RVU2R): claim-pause before review-pause. A review that ran
|
|
@@ -2426,7 +2718,7 @@ async function pumpReviews(state) {
|
|
|
2426
2718
|
// reviewer_capacity row; here we pause the whole runner exactly as a
|
|
2427
2719
|
// capacity_exhausted task does.
|
|
2428
2720
|
if (outcome.decision === "reviewer_capacity") {
|
|
2429
|
-
enterCapacityPause(state, next.task_id, outcome.resume_at ?? null, `reviewer ${next.review_id} hit session/usage limit
|
|
2721
|
+
enterCapacityPause(state, next.task_id, outcome.resume_at ?? null, `reviewer ${next.review_id} hit session/usage limit`, outcome.engine);
|
|
2430
2722
|
return;
|
|
2431
2723
|
}
|
|
2432
2724
|
}
|
|
@@ -2445,7 +2737,7 @@ async function pumpReviews(state) {
|
|
|
2445
2737
|
// to protect, so release the claim-pause and let queued tasks resume claiming.
|
|
2446
2738
|
// (A whole-chain reviewer_capacity `return`s above and never reaches here;
|
|
2447
2739
|
// pausedCapacity keeps everything paused until the window reopens.)
|
|
2448
|
-
if (state.capacityClaimPaused && !state
|
|
2740
|
+
if (state.capacityClaimPaused && !hasCapacityPause(state)) {
|
|
2449
2741
|
state.capacityClaimPaused = false;
|
|
2450
2742
|
void pump(state, state.taskRunnerFactory);
|
|
2451
2743
|
}
|
|
@@ -2557,11 +2849,27 @@ function enterQuarantine(state, taskId, cause, detail,
|
|
|
2557
2849
|
// v0.53 T-53-4: how many consecutive instant-empty exits produced an
|
|
2558
2850
|
// env_broken cause (1 = definitive, 2 = heuristic confirmed by the
|
|
2559
2851
|
// retry). Persisted into quarantine.json's additive `consecutive` field.
|
|
2560
|
-
consecutive
|
|
2852
|
+
consecutive,
|
|
2853
|
+
// USAGE-LIMIT-INVISIBLE-1 (AC4): the fleet capacity reset this quarantine
|
|
2854
|
+
// was folded onto (see signalUsageLimitCapacity), when cause is
|
|
2855
|
+
// usage_limit — null for an indefinite cap. Threaded through so the
|
|
2856
|
+
// operator-facing bus message + audit event carry a concrete retry path
|
|
2857
|
+
// instead of a bare "quarantined" with no next step: the RUNNER auto-clears
|
|
2858
|
+
// on this reset (restartClaims), but the specific failed task does not
|
|
2859
|
+
// requeue itself and needs a manual retry_task / `/acc retry`.
|
|
2860
|
+
usageLimitResumeAtIso, engine) {
|
|
2561
2861
|
if (state.quarantined)
|
|
2562
2862
|
return;
|
|
2563
2863
|
state.quarantined = true;
|
|
2564
2864
|
state.quarantineCause = cause; // v0.41.4: gates env_broken auto-recovery.
|
|
2865
|
+
// Attribute an engine ONLY for a genuine per-engine capacity cap. `usage_limit`
|
|
2866
|
+
// is the sole cap cause that quarantines (capacity_exhausted PAUSES instead),
|
|
2867
|
+
// so it alone names the engine that hit the cap. Every other cause
|
|
2868
|
+
// (env_broken, auth_expired, claude_unavailable, …) is a whole-runner fault
|
|
2869
|
+
// with no provider to blame — labelling it with the task's engine would be a
|
|
2870
|
+
// false attribution in the audit field (capacityEngine() also defaults a
|
|
2871
|
+
// missing engine to claude-code). null = "no specific engine".
|
|
2872
|
+
const engineId = cause === "usage_limit" && engine ? capacityEngine(engine) : null;
|
|
2565
2873
|
const qState = {
|
|
2566
2874
|
cause,
|
|
2567
2875
|
classifiedAt: new Date().toISOString(),
|
|
@@ -2574,6 +2882,12 @@ consecutive) {
|
|
|
2574
2882
|
});
|
|
2575
2883
|
process.stderr.write(chalk.red.bold(`[acc-runner] QUARANTINE: runner ${state.session.runner_id} quarantined ` +
|
|
2576
2884
|
`(${cause}) after task ${taskId} failed.\n`));
|
|
2885
|
+
const retryHint = cause === "usage_limit"
|
|
2886
|
+
? `capacity-paused fleet-wide (see /api/health fleet_pause); resumes ` +
|
|
2887
|
+
`${usageLimitResumeAtIso ?? "at an unknown time (indefinite cap)"} — this ` +
|
|
2888
|
+
`runner un-quarantines automatically on resume, but task ${taskId} must be ` +
|
|
2889
|
+
`retried manually (retry_task RPC / \`/acc retry\`) once capacity returns.`
|
|
2890
|
+
: undefined;
|
|
2577
2891
|
void postRunnerStateMessage(state.supabase, {
|
|
2578
2892
|
task_id: taskId,
|
|
2579
2893
|
sender_id: state.session.runner_id,
|
|
@@ -2583,7 +2897,9 @@ consecutive) {
|
|
|
2583
2897
|
capacity_alert: true,
|
|
2584
2898
|
quarantine_cause: cause,
|
|
2585
2899
|
runner_id: state.session.runner_id,
|
|
2900
|
+
engine: engineId,
|
|
2586
2901
|
detail,
|
|
2902
|
+
...(retryHint ? { retry_hint: retryHint, capacity_resume_at: usageLimitResumeAtIso ?? null } : {}),
|
|
2587
2903
|
},
|
|
2588
2904
|
kind: "blocked",
|
|
2589
2905
|
}).catch((err) => {
|
|
@@ -2594,7 +2910,13 @@ consecutive) {
|
|
|
2594
2910
|
await state.supabase.rpc("log_activity", {
|
|
2595
2911
|
p_verb: "runner.quarantine_enter",
|
|
2596
2912
|
p_target_id: state.session.runner_id,
|
|
2597
|
-
p_payload: {
|
|
2913
|
+
p_payload: {
|
|
2914
|
+
cause,
|
|
2915
|
+
engine: engineId,
|
|
2916
|
+
task_id: taskId,
|
|
2917
|
+
detail,
|
|
2918
|
+
...(retryHint ? { retry_hint: retryHint, capacity_resume_at: usageLimitResumeAtIso ?? null } : {}),
|
|
2919
|
+
},
|
|
2598
2920
|
p_target_type: "runner",
|
|
2599
2921
|
});
|
|
2600
2922
|
}
|
|
@@ -2618,7 +2940,7 @@ consecutive) {
|
|
|
2618
2940
|
* Idempotent: a straggler cap while already paused only ever pushes the reset
|
|
2619
2941
|
* later (the window fold's extend-only rule), never shortens the wait.
|
|
2620
2942
|
*/
|
|
2621
|
-
function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail) {
|
|
2943
|
+
function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail, engine) {
|
|
2622
2944
|
const now = Date.now();
|
|
2623
2945
|
const parsed = resumeAtIso ? Date.parse(resumeAtIso) : NaN;
|
|
2624
2946
|
const baseResumeMs = Number.isFinite(parsed) && parsed > now ? parsed : now + state.capacityBackoffMs;
|
|
@@ -2632,17 +2954,53 @@ function enterCapacityPause(state, triggerTaskId, resumeAtIso, detail) {
|
|
|
2632
2954
|
// canonical pause payload + bus message (the seam only sees {account, reset}).
|
|
2633
2955
|
state.capacityTriggerTaskId = triggerTaskId;
|
|
2634
2956
|
state.capacityPauseDetail = detail;
|
|
2957
|
+
state.capacityPauseEngine = capacityEngine(engine);
|
|
2635
2958
|
// Stop claiming SYNCHRONOUSLY (the controller's stopClaims seam runs on the
|
|
2636
2959
|
// serialized action tail a microtask later; pump()/pumpReviews() read this
|
|
2637
2960
|
// flag between their own awaits, so setting it here closes the gap and keeps
|
|
2638
2961
|
// the v0.56 "no new claim the instant a 429 lands" guarantee byte-identical).
|
|
2639
|
-
state.pausedCapacity
|
|
2962
|
+
state.pausedCapacity.add(state.capacityPauseEngine);
|
|
2640
2963
|
state.resumeController.reportCapacity({
|
|
2641
2964
|
accountKey: state.resumeAccountKey ?? state.session.runner_id,
|
|
2642
2965
|
resetAtMs: resumeMs,
|
|
2643
2966
|
detail,
|
|
2644
2967
|
});
|
|
2645
2968
|
}
|
|
2969
|
+
/**
|
|
2970
|
+
* USAGE-LIMIT-INVISIBLE-1: fold a usage_limit quarantine into the SAME
|
|
2971
|
+
* fleet-visible capacity signal `enterCapacityPause` emits for
|
|
2972
|
+
* capacity_exhausted, so `/api/health` fleet_pause and the FH-1b dispatch
|
|
2973
|
+
* governor see the account as out of capacity instead of reading a live
|
|
2974
|
+
* `paused_count: 0` while every task quietly dies (the 2026-08-30 incident:
|
|
2975
|
+
* four usage_limit exits, zero capacity signal anywhere).
|
|
2976
|
+
*
|
|
2977
|
+
* Deliberately NOT `enterCapacityPause`: that helper falls back to a fixed
|
|
2978
|
+
* backoff (`now + capacityBackoffMs`) when no reset was parsed, which is the
|
|
2979
|
+
* right call for a KNOWN-recoverable 429 but would fabricate a retry time for
|
|
2980
|
+
* a class the classifier does not promise resets on. Here a null resume
|
|
2981
|
+
* folds onto the account window as an INDEFINITE cap (account-window.ts
|
|
2982
|
+
* `isStillCapped`) — visible in fleet_pause as `resume_at: null` on a
|
|
2983
|
+
* `paused_count > 0` row (never silently indistinguishable from "no pause"),
|
|
2984
|
+
* and it only clears on an account swap, a positive probe ping, or the
|
|
2985
|
+
* runner's own quarantine being cleared and re-observing the account.
|
|
2986
|
+
*
|
|
2987
|
+
* Runner-local claiming is already halted by `state.quarantined` (set by the
|
|
2988
|
+
* caller's `enterQuarantine`); this only drives the fleet-visible signal +
|
|
2989
|
+
* the resume bookkeeping so OTHER runners sharing the account — and the
|
|
2990
|
+
* dispatcher — stop feeding it work too.
|
|
2991
|
+
*/
|
|
2992
|
+
function signalUsageLimitCapacity(state, triggerTaskId, resumeAtIso, detail, engine) {
|
|
2993
|
+
const parsed = resumeAtIso ? Date.parse(resumeAtIso) : NaN;
|
|
2994
|
+
const resetAtMs = Number.isFinite(parsed) ? parsed : null;
|
|
2995
|
+
state.capacityTriggerTaskId = triggerTaskId;
|
|
2996
|
+
state.capacityPauseDetail = detail;
|
|
2997
|
+
state.capacityPauseEngine = capacityEngine(engine);
|
|
2998
|
+
state.resumeController.reportCapacity({
|
|
2999
|
+
accountKey: state.resumeAccountKey ?? state.session.runner_id,
|
|
3000
|
+
resetAtMs,
|
|
3001
|
+
detail,
|
|
3002
|
+
});
|
|
3003
|
+
}
|
|
2646
3004
|
/**
|
|
2647
3005
|
* AS-3 (RESUME-WIRE): the ResumeController side-effect seams, backed by the REAL
|
|
2648
3006
|
* runner path (log_activity RPCs + pollOnce/pump). Exported so the account-switch
|
|
@@ -2671,6 +3029,7 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2671
3029
|
const resumeAtIso = resetAtMs !== null ? new Date(resetAtMs).toISOString() : null;
|
|
2672
3030
|
const detail = state.capacityPauseDetail;
|
|
2673
3031
|
const triggerTaskId = state.capacityTriggerTaskId;
|
|
3032
|
+
const engine = state.capacityPauseEngine;
|
|
2674
3033
|
state.accountProbe?.setPaused(true);
|
|
2675
3034
|
process.stderr.write(chalk.yellow(`[acc-runner] PAUSED (capacity) on ${account}: ${detail}. Not claiming ` +
|
|
2676
3035
|
`tasks or reviews; resuming at ${resumeAtIso ?? "reset / account change"}.\n`));
|
|
@@ -2682,6 +3041,7 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2682
3041
|
runner_id: state.session.runner_id,
|
|
2683
3042
|
resume_at: resumeAtIso,
|
|
2684
3043
|
reason: detail,
|
|
3044
|
+
engine,
|
|
2685
3045
|
account,
|
|
2686
3046
|
version: PACKAGE_VERSION,
|
|
2687
3047
|
},
|
|
@@ -2703,6 +3063,7 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2703
3063
|
capacity_paused: true,
|
|
2704
3064
|
runner_id: state.session.runner_id,
|
|
2705
3065
|
resume_at: resumeAtIso,
|
|
3066
|
+
engine,
|
|
2706
3067
|
detail,
|
|
2707
3068
|
},
|
|
2708
3069
|
kind: "blocked",
|
|
@@ -2712,7 +3073,8 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2712
3073
|
}
|
|
2713
3074
|
},
|
|
2714
3075
|
stopClaims: () => {
|
|
2715
|
-
state.
|
|
3076
|
+
if (state.capacityPauseEngine)
|
|
3077
|
+
state.pausedCapacity.add(state.capacityPauseEngine);
|
|
2716
3078
|
},
|
|
2717
3079
|
clearServerPause: async () => {
|
|
2718
3080
|
try {
|
|
@@ -2744,7 +3106,34 @@ export function buildResumeControllerDeps(state, factory) {
|
|
|
2744
3106
|
restartClaims: async () => {
|
|
2745
3107
|
if (state.stopped)
|
|
2746
3108
|
return;
|
|
2747
|
-
|
|
3109
|
+
// USAGE-LIMIT-INVISIBLE-1 (AC4): a usage_limit quarantine anchored to
|
|
3110
|
+
// the SAME account window that just reopened is released automatically
|
|
3111
|
+
// — the account is no longer capped, so there is nothing left for an
|
|
3112
|
+
// operator to "fix" by hand. Scoped to `usage_limit` only: env_broken
|
|
3113
|
+
// has its own probe-gated auto-recovery (quarantineRecoveryScan) and
|
|
3114
|
+
// auth_expired needs a fresh token, not a capacity reset. The task this
|
|
3115
|
+
// runner already failed still needs a manual `retry_task` / `/acc
|
|
3116
|
+
// retry` (it is NOT re-queued here) — see the retry hint on the
|
|
3117
|
+
// quarantine's blocked/error_context message.
|
|
3118
|
+
if (state.quarantined && state.quarantineCause === "usage_limit") {
|
|
3119
|
+
await clearQuarantine().catch(() => { });
|
|
3120
|
+
state.quarantined = false;
|
|
3121
|
+
state.quarantineCause = null;
|
|
3122
|
+
process.stderr.write(chalk.green(`[acc-runner] quarantine auto-cleared: usage_limit capacity window reopened. ` +
|
|
3123
|
+
`Resuming task claims (the quarantined task itself still needs a manual retry).\n`));
|
|
3124
|
+
try {
|
|
3125
|
+
await state.supabase.rpc("log_activity", {
|
|
3126
|
+
p_verb: "runner.quarantine_auto_cleared",
|
|
3127
|
+
p_target_id: state.session.runner_id,
|
|
3128
|
+
p_payload: { cause: "usage_limit", version: PACKAGE_VERSION },
|
|
3129
|
+
p_target_type: "runner",
|
|
3130
|
+
});
|
|
3131
|
+
}
|
|
3132
|
+
catch { /* best-effort */ }
|
|
3133
|
+
}
|
|
3134
|
+
// Resume is account-wide, so clear all engine pauses in this window.
|
|
3135
|
+
state.pausedCapacity.clear();
|
|
3136
|
+
state.capacityPauseEngine = null;
|
|
2748
3137
|
state.pausedUntil = null;
|
|
2749
3138
|
// RVU-2 (FU-RVU2R): a full pause supersedes the soft claim-pause; clear it
|
|
2750
3139
|
// so the probe window (task limit pinned to 1) isn't zeroed out.
|
|
@@ -2783,7 +3172,7 @@ async function wireCapacityResume(state, factory, options) {
|
|
|
2783
3172
|
// accountIdentity() otherwise (an honest per-host synthetic account when no
|
|
2784
3173
|
// real login introspects). Never throws — a failed probe degrades to unknown
|
|
2785
3174
|
// here, and to a logged no-op on the scheduler's later cadence probes.
|
|
2786
|
-
const chatEngine =
|
|
3175
|
+
const chatEngine = getConfiguredChatEngine();
|
|
2787
3176
|
const probe = options.accountProbe ??
|
|
2788
3177
|
(async () => {
|
|
2789
3178
|
if (chatEngine.accountIdentity)
|
|
@@ -2879,12 +3268,9 @@ async function pump(state, factory) {
|
|
|
2879
3268
|
// v0.48: check quarantine before claiming any task.
|
|
2880
3269
|
if (state.quarantined) {
|
|
2881
3270
|
process.stderr.write(`[acc-runner] QUARANTINED: runner ${state.session.runner_id} is not claiming tasks.\n` +
|
|
2882
|
-
`[acc-runner] Fix the underlying issue then run \`acc-runner quarantine clear
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
// v0.56 (T-56-1): out of session/usage capacity — don't claim. Queued
|
|
2886
|
-
// tasks stay queued; the scheduled resume re-drives pump().
|
|
2887
|
-
if (state.pausedCapacity) {
|
|
3271
|
+
`[acc-runner] Fix the underlying issue, then run \`acc-runner quarantine clear\` ` +
|
|
3272
|
+
`for this named instance; repeat for each instance on the host. The running ` +
|
|
3273
|
+
`watcher resumes on its next recovery tick; no restart is required.\n`);
|
|
2888
3274
|
return;
|
|
2889
3275
|
}
|
|
2890
3276
|
// v0.56 (T-56-1): during the post-resume probe window, claim exactly one
|
|
@@ -2906,10 +3292,12 @@ async function pump(state, factory) {
|
|
|
2906
3292
|
// Fill available concurrency slots from the queue.
|
|
2907
3293
|
while (!state.stopped &&
|
|
2908
3294
|
!state.quarantined &&
|
|
2909
|
-
!state.pausedCapacity &&
|
|
2910
3295
|
state.queue.length > 0 &&
|
|
2911
3296
|
state.running.size < effectiveLimit) {
|
|
2912
|
-
const
|
|
3297
|
+
const nextIndex = state.queue.findIndex((id) => !isCapacityPausedFor(state, state.taskEngines.get(id)));
|
|
3298
|
+
if (nextIndex === -1)
|
|
3299
|
+
return;
|
|
3300
|
+
const [next] = state.queue.splice(nextIndex, 1);
|
|
2913
3301
|
if (!next)
|
|
2914
3302
|
continue;
|
|
2915
3303
|
const ctrl = factory(next, {
|
|
@@ -2920,6 +3308,11 @@ async function pump(state, factory) {
|
|
|
2920
3308
|
runnerId: state.session.runner_id,
|
|
2921
3309
|
},
|
|
2922
3310
|
publicUrl: state.cfg.publicUrl,
|
|
3311
|
+
onUnknownAdvertisedEngine: (engineId, err) => repairUnknownAdvertisedEngine(state, engineId, err),
|
|
3312
|
+
// CLAIM-LOCK-GLOB: hand the per-runner claim-loss map
|
|
3313
|
+
// to runTask so file-lock conflicts count attempts across re-dispatch
|
|
3314
|
+
// and back off after the fast attempts are spent (AC3).
|
|
3315
|
+
claimLossState: state.claimLossState,
|
|
2923
3316
|
});
|
|
2924
3317
|
state.running.set(next, ctrl);
|
|
2925
3318
|
// Detached completion handler. Runs AFTER pump() returns so the
|
|
@@ -2929,9 +3322,7 @@ async function pump(state, factory) {
|
|
|
2929
3322
|
.then((outcome) => {
|
|
2930
3323
|
state.running.delete(next);
|
|
2931
3324
|
touchActivity(state); // v0.12.0 (T-52-7): idle clock restarts at task end
|
|
2932
|
-
//
|
|
2933
|
-
// never a task_error, never quarantine. Do not re-fire pump for new
|
|
2934
|
-
// work; the scheduled resume does that with a single probe.
|
|
3325
|
+
// Capacity exhaustion pauses only the refused engine's task claims.
|
|
2935
3326
|
//
|
|
2936
3327
|
// FH-1a: runTask now CLEANLY REQUEUES the in-flight task
|
|
2937
3328
|
// (running→queued) instead of abandoning it to the stale-running
|
|
@@ -2945,7 +3336,7 @@ async function pump(state, factory) {
|
|
|
2945
3336
|
// seen.delete (cancel → re-start).
|
|
2946
3337
|
if (outcome.status === "capacity_paused" || outcome.capacity_exhausted) {
|
|
2947
3338
|
state.seen.delete(next);
|
|
2948
|
-
enterCapacityPause(state, next, outcome.resume_at ?? null, outcome.error || "task hit session/usage limit");
|
|
3339
|
+
enterCapacityPause(state, next, outcome.resume_at ?? null, outcome.error || "task hit session/usage limit", outcome.engine ?? state.taskEngines.get(next));
|
|
2949
3340
|
return;
|
|
2950
3341
|
}
|
|
2951
3342
|
// Repo-scoped ignore: the task's repo is outside this runner's
|
|
@@ -2989,6 +3380,17 @@ async function pump(state, factory) {
|
|
|
2989
3380
|
// does not answer it with a resubscribe storm.
|
|
2990
3381
|
if (outcome.phase === "claim_locks") {
|
|
2991
3382
|
state.claimContentionSince.set(next, Date.now());
|
|
3383
|
+
// CLAIM-LOCK-GLOB: once the fast attempts are
|
|
3384
|
+
// spent runTask returns a bounded yield instant — hold the task
|
|
3385
|
+
// out of the re-dispatch queue until then (AC3) so a losing task
|
|
3386
|
+
// actually backs off instead of re-claiming every backstop cycle.
|
|
3387
|
+
// A null yield leaves it eligible for the next fast retry.
|
|
3388
|
+
if (outcome.claim_lock_yield_until_ms != null) {
|
|
3389
|
+
state.claimYieldUntil.set(next, outcome.claim_lock_yield_until_ms);
|
|
3390
|
+
}
|
|
3391
|
+
else {
|
|
3392
|
+
state.claimYieldUntil.delete(next);
|
|
3393
|
+
}
|
|
2992
3394
|
}
|
|
2993
3395
|
// v0.48: machine-level failure → quarantine. Guard with
|
|
2994
3396
|
// !state.quarantined so concurrent tasks don't fire duplicate
|
|
@@ -3031,12 +3433,22 @@ async function pump(state, factory) {
|
|
|
3031
3433
|
void pump(state, factory);
|
|
3032
3434
|
}
|
|
3033
3435
|
else {
|
|
3034
|
-
enterQuarantine(state, next, cause, reason, outcome.quarantine_consecutive);
|
|
3436
|
+
enterQuarantine(state, next, cause, reason, outcome.quarantine_consecutive, undefined, outcome.engine ?? state.taskEngines.get(next));
|
|
3035
3437
|
}
|
|
3036
3438
|
});
|
|
3037
3439
|
return;
|
|
3038
3440
|
}
|
|
3039
|
-
|
|
3441
|
+
// USAGE-LIMIT-INVISIBLE-1: a usage_limit quarantine is ALSO a
|
|
3442
|
+
// fleet capacity signal — the account (not just this runner) is
|
|
3443
|
+
// out of room. Fold it into the same account-window / fleet_pause
|
|
3444
|
+
// machinery capacity_exhausted uses, carrying whatever reset the
|
|
3445
|
+
// provider stated (outcome.resume_at) or an explicit indefinite
|
|
3446
|
+
// cap (null) — BEFORE quarantining so a paused-fleet dispatcher
|
|
3447
|
+
// read never races an empty fleet_pause against the new quarantine.
|
|
3448
|
+
if (cause === "usage_limit") {
|
|
3449
|
+
signalUsageLimitCapacity(state, next, outcome.resume_at ?? null, reason, outcome.engine ?? state.taskEngines.get(next));
|
|
3450
|
+
}
|
|
3451
|
+
enterQuarantine(state, next, cause, reason, outcome.quarantine_consecutive, cause === "usage_limit" ? outcome.resume_at ?? null : undefined, outcome.engine ?? state.taskEngines.get(next));
|
|
3040
3452
|
// Quarantined — do not re-fire pump for new tasks.
|
|
3041
3453
|
// Other in-flight tasks (still in state.running) complete
|
|
3042
3454
|
// normally; we just stop accepting new work.
|