claude-code-session-manager 0.87.0 → 0.88.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -53
- package/dist/assets/{AgentLibrary-DyLWzZDf.js → AgentLibrary-kQF8_wAw.js} +1 -1
- package/dist/assets/{DataModel--mISIJ6h.js → DataModel-CY_mMnsr.js} +1 -1
- package/dist/assets/{History-C2ahUXTg.js → History-Sji2brb0.js} +1 -1
- package/dist/assets/{Hooks-BiC6oyR2.js → Hooks-CtqpO-p5.js} +1 -1
- package/dist/assets/{HostBilko-BPleEOld.js → HostBilko-DN0ydOCr.js} +1 -1
- package/dist/assets/{Library-Dc8Qst1R.js → Library-CoiLZTKh.js} +1 -1
- package/dist/assets/{ListDetail-DIXh-OLX.js → ListDetail-CmqiQ_io.js} +1 -1
- package/dist/assets/{MarkdownEditor-C90bkLXK.js → MarkdownEditor-HPoNeKhT.js} +1 -1
- package/dist/assets/{McpServers-DqcbLOLZ.js → McpServers-Bj1X3_NT.js} +1 -1
- package/dist/assets/{Memory-CW62MXlh.js → Memory-1PKjsM3b.js} +1 -1
- package/dist/assets/{Panel-Bw1FhRuF.js → Panel-I1s6ghPo.js} +1 -1
- package/dist/assets/{Permissions-BcUC-5y8.js → Permissions-BhiiZjHQ.js} +1 -1
- package/dist/assets/{Plugins-BnKx9flD.js → Plugins-gv3_fkPX.js} +2 -2
- package/dist/assets/{ProvenanceBadge-Bw5vNVPT.js → ProvenanceBadge-BGcgBd8V.js} +1 -1
- package/dist/assets/{SaveBar-CWr0O_w-.js → SaveBar-BGr9eZ4g.js} +1 -1
- package/dist/assets/{Scheduler-DYdLuUqq.js → Scheduler-Bdo_awPq.js} +7 -7
- package/dist/assets/{ScopeSwitcher-CrBLbg8s.js → ScopeSwitcher-ukl0qWaA.js} +1 -1
- package/dist/assets/{Settings-DluB-vN1.js → Settings-DJkFj22G.js} +1 -1
- package/dist/assets/{SkillReferenceGraph-CHLSseay.js → SkillReferenceGraph-0t1Qblp6.js} +1 -1
- package/dist/assets/{Skills-gNdo_HNK.js → Skills-CeVdBGE-.js} +1 -1
- package/dist/assets/{SystemPrompt-Cru05-Ia.js → SystemPrompt-CfKvNHi3.js} +1 -1
- package/dist/assets/{TagLibrary-DNHY0xou.js → TagLibrary-DAzNaUPd.js} +1 -1
- package/dist/assets/{TiptapBody-I4lmbCgP.js → TiptapBody--S_rGXoA.js} +1 -1
- package/dist/assets/{Toggle-bWMHjmRh.js → Toggle-Dszm0xnp.js} +1 -1
- package/dist/assets/{index-DV3PorRY.css → index-BHTX4OTc.css} +1 -1
- package/dist/assets/{index-fc_JjdxL.js → index-Dp4Rc--q.js} +356 -356
- package/dist/assets/{settingsSchema-BfhtZnGD.js → settingsSchema-BLbssMYo.js} +1 -1
- package/dist/assets/{whisperWorker-Dbia1OpC.js → whisperWorker-C7ZGQwKg.js} +7 -7
- package/dist/index.html +2 -2
- package/dist/vad/ort-wasm-simd-threaded.asyncify.mjs +106 -110
- package/dist/vad/ort-wasm-simd-threaded.asyncify.wasm +0 -0
- package/dist/vad/ort-wasm-simd-threaded.jsep.mjs +98 -98
- package/dist/vad/ort-wasm-simd-threaded.jsep.wasm +0 -0
- package/dist/vad/ort-wasm-simd-threaded.jspi.mjs +99 -102
- package/dist/vad/ort-wasm-simd-threaded.jspi.wasm +0 -0
- package/dist/vad/ort-wasm-simd-threaded.mjs +46 -46
- package/dist/vad/ort-wasm-simd-threaded.wasm +0 -0
- package/package.json +10 -13
- package/scripts/README.md +59 -0
- package/scripts/audit-ops-hygiene.cjs +350 -0
- package/scripts/hooks/guard-destructive-git.cjs +10 -34
- package/scripts/hooks/guard-inline-implementation.cjs +14 -8
- package/scripts/hooks/guard-prd-writes.cjs +7 -36
- package/scripts/hooks/guard-self-schedule.cjs +175 -0
- package/scripts/ops-sweep.cjs +355 -0
- package/scripts/scheduler-mcp-server.cjs +28 -71
- package/src/main/__tests__/bilkoHost-integration.test.cjs +3 -3
- package/src/main/__tests__/chat-cancel-terminal.test.cjs +6 -9
- package/src/main/__tests__/chat-exit-close-race.test.cjs +3 -3
- package/src/main/__tests__/chat-mcp-consent-notice.test.cjs +4 -5
- package/src/main/__tests__/chat-queue.test.cjs +2 -2
- package/src/main/__tests__/chat-stop-signal.test.cjs +2 -2
- package/src/main/__tests__/dep-orphan-archive-health.test.cjs +77 -0
- package/src/main/__tests__/dod-batchkey.test.cjs +2 -2
- package/src/main/__tests__/dod-drain-hook.test.cjs +2 -2
- package/src/main/__tests__/dod-report.test.cjs +2 -2
- package/src/main/__tests__/dod-reverify.test.cjs +2 -2
- package/src/main/__tests__/epicMint.test.cjs +2 -2
- package/src/main/__tests__/exchanges.test.cjs +2 -2
- package/src/main/__tests__/extractJson.test.cjs +2 -2
- package/src/main/__tests__/files-reject-credentials.test.cjs +1 -1
- package/src/main/__tests__/fixtures/1218-fo-01-move-scripts-lib-into-src-main-lib.log +556 -0
- package/src/main/__tests__/health-build-freshness.test.cjs +39 -0
- package/src/main/__tests__/health-delegation-chain.test.cjs +15 -1
- package/src/main/__tests__/health-queue-dispatch.test.cjs +58 -7
- package/src/main/__tests__/health-tick-liveness.test.cjs +27 -3
- package/src/main/__tests__/health-usage-poller.test.cjs +70 -23
- package/src/main/__tests__/historyRollup.test.cjs +2 -2
- package/src/main/__tests__/kg-augment.test.cjs +2 -2
- package/src/main/__tests__/mcpStatus.test.cjs +2 -2
- package/src/main/__tests__/memoryAggregate.test.cjs +1 -1
- package/src/main/__tests__/memoryStale.test.cjs +1 -1
- package/src/main/__tests__/opsErrorLogTelemetryTap.test.cjs +25 -1
- package/src/main/__tests__/pollLoop-dispatch-on-failure.test.cjs +33 -3
- package/src/main/__tests__/prd-group-allocator.test.cjs +2 -2
- package/src/main/__tests__/prdAdminRouteParity.test.cjs +2 -0
- package/src/main/__tests__/prdAdminRoutes.test.cjs +14 -2
- package/src/main/__tests__/prdAuthoringSeed.test.cjs +39 -0
- package/src/main/__tests__/prdLocationsArchived.test.cjs +20 -13
- package/src/main/__tests__/proc-role-env.test.cjs +125 -0
- package/src/main/__tests__/procname-claude-spawn-sites.test.cjs +304 -0
- package/src/main/__tests__/procname-sm-processes.test.cjs +127 -0
- package/src/main/__tests__/projectHomeAdminRoutes.test.cjs +81 -401
- package/src/main/__tests__/projectPages.test.cjs +63 -149
- package/src/main/__tests__/queue-health-verdict.test.cjs +9 -9
- package/src/main/__tests__/queue-starvation-dispatch-driver.test.cjs +117 -20
- package/src/main/__tests__/queueHistory.test.cjs +2 -2
- package/src/main/__tests__/rateLimitPollerStreak.test.cjs +38 -3
- package/src/main/__tests__/runVerify-landed-commit-outranks.test.cjs +181 -0
- package/src/main/__tests__/runVerify.test.cjs +5 -5
- package/src/main/__tests__/scheduleJobStatusDrift.test.cjs +3 -3
- package/src/main/__tests__/scheduleJobTransitions.test.cjs +2 -2
- package/src/main/__tests__/scheduler-adopted-run-supervision.test.cjs +143 -0
- package/src/main/__tests__/scheduler-autofix-select.test.cjs +2 -2
- package/src/main/__tests__/scheduler-autopromote.test.cjs +2 -2
- package/src/main/__tests__/scheduler-bash-timeout-env.test.cjs +3 -5
- package/src/main/__tests__/scheduler-boot-orphans.test.cjs +78 -97
- package/src/main/__tests__/scheduler-default-eligible-heal.test.cjs +161 -0
- package/src/main/__tests__/scheduler-dispatch-loop.test.cjs +58 -0
- package/src/main/__tests__/scheduler-epic-digest.test.cjs +3 -5
- package/src/main/__tests__/scheduler-force-tick-outcome.test.cjs +2 -2
- package/src/main/__tests__/scheduler-gate-shadow.test.cjs +119 -0
- package/src/main/__tests__/scheduler-guard-verdict-autoresolve.test.cjs +46 -0
- package/src/main/__tests__/scheduler-heartbeat-payload.test.cjs +80 -0
- package/src/main/__tests__/scheduler-inplace-salvage.test.cjs +25 -18
- package/src/main/__tests__/scheduler-investigation-prompt.test.cjs +4 -4
- package/src/main/__tests__/scheduler-launch-failure.test.cjs +3 -5
- package/src/main/__tests__/scheduler-looks-done.test.cjs +26 -7
- package/src/main/__tests__/scheduler-manual-pause.test.cjs +118 -0
- package/src/main/__tests__/scheduler-meta-code-sha.test.cjs +26 -3
- package/src/main/__tests__/scheduler-prd-missing-skip.test.cjs +17 -2
- package/src/main/__tests__/scheduler-prd-persona-spawn.test.cjs +3 -5
- package/src/main/__tests__/scheduler-quiet-machine-lease.test.cjs +41 -6
- package/src/main/__tests__/scheduler-rate-limit-pause.test.cjs +62 -6
- package/src/main/__tests__/scheduler-rate-limit-spin-guard.test.cjs +3 -5
- package/src/main/__tests__/scheduler-reap-dead-running-jobs.test.cjs +23 -4
- package/src/main/__tests__/scheduler-reconcile-cwd-preserve.test.cjs +100 -0
- package/src/main/__tests__/scheduler-reconcile-invalid-repair.test.cjs +3 -3
- package/src/main/__tests__/scheduler-reconcile-quarantine.test.cjs +4 -4
- package/src/main/__tests__/scheduler-shard-quarantine.test.cjs +110 -0
- package/src/main/__tests__/scheduler-shared-tree-guard.test.cjs +81 -1
- package/src/main/__tests__/scheduler-stall-per-project.test.cjs +14 -0
- package/src/main/__tests__/scheduler-starve-escalation.test.cjs +4 -4
- package/src/main/__tests__/scheduler-stranded-autofix-park.test.cjs +245 -0
- package/src/main/__tests__/scheduler-supervisor-record.test.cjs +81 -0
- package/src/main/__tests__/scheduler-tick-cancel-token.test.cjs +2 -2
- package/src/main/__tests__/scheduler-tick-wedge.test.cjs +172 -0
- package/src/main/__tests__/scheduler-transient-failure.test.cjs +2 -2
- package/src/main/__tests__/scheduler-worktree-cap-defer.test.cjs +37 -11
- package/src/main/__tests__/scheduler-worktree-exec-cwd.test.cjs +3 -5
- package/src/main/__tests__/usageSingleFlight.test.cjs +157 -0
- package/src/main/__tests__/workTypeLibrary.test.cjs +1 -1
- package/src/main/bilkoHost.cjs +16 -7
- package/src/main/build-info.json +8 -0
- package/src/main/chatRunner.cjs +4 -2
- package/src/main/config.cjs +2 -3
- package/src/main/docEdit.cjs +4 -2
- package/src/main/health.cjs +276 -52
- package/src/main/heapSnapshot.cjs +2 -2
- package/src/main/historyAggregator.cjs +1 -1
- package/src/main/index.cjs +61 -10
- package/src/main/ipcSchemas.cjs +6 -17
- package/src/main/lib/__tests__/auditLog.test.cjs +38 -0
- package/src/main/lib/__tests__/buildIdentity.test.cjs +121 -0
- package/src/main/lib/__tests__/cwdClassify.test.cjs +111 -0
- package/src/main/lib/__tests__/definitionOfDoneSequence.test.cjs +95 -0
- package/src/main/lib/__tests__/delegationReadiness.test.cjs +1 -1
- package/src/main/lib/__tests__/dispatchLoop.test.cjs +63 -0
- package/src/main/lib/__tests__/gateFixtures.json +20 -0
- package/src/main/lib/__tests__/gitWorktree.test.cjs +8 -1
- package/src/main/lib/__tests__/instanceLock.test.cjs +93 -7
- package/src/main/lib/__tests__/jobSupervisorRecord.test.cjs +78 -0
- package/src/main/lib/__tests__/jobWorktreeBootLive.test.cjs +11 -0
- package/src/main/lib/__tests__/localAdminHttp.test.cjs +1 -1
- package/src/main/lib/__tests__/mcpToolCatalog.test.cjs +6 -1
- package/src/main/lib/__tests__/procIdentity.test.cjs +119 -0
- package/src/main/lib/__tests__/procName.test.cjs +92 -0
- package/src/main/lib/__tests__/queueStoreMachineStateRecovery.test.cjs +67 -0
- package/src/main/lib/__tests__/schedulerMcpServerHelp.test.cjs +6 -7
- package/src/main/lib/__tests__/schedulerMcpServerProjectHome.test.cjs +37 -204
- package/src/main/lib/__tests__/schedulerPaths.test.cjs +226 -0
- package/src/main/lib/__tests__/schedulerPathsWorktree.test.cjs +94 -0
- package/src/main/lib/__tests__/schedulerRuntimeState.test.cjs +56 -0
- package/src/main/lib/__tests__/sessionSlots.test.cjs +45 -0
- package/src/main/lib/__tests__/telemetryBacklog.test.cjs +1 -1
- package/src/main/lib/__tests__/upgradeDrain.test.cjs +130 -0
- package/src/main/lib/__tests__/usageCircuit.test.cjs +61 -0
- package/src/main/lib/__tests__/watchdog-helpers.test.cjs +63 -0
- package/src/main/lib/__tests__/watchdog-relaunch.test.cjs +73 -0
- package/src/main/lib/activeIndexRebuild.cjs +1 -3
- package/src/main/lib/activeSessions.cjs +20 -88
- package/src/main/lib/adoptedRunSupervisor.cjs +136 -0
- package/src/main/lib/agentModelResolve.cjs +33 -1
- package/src/main/lib/agentPersonaSchema.cjs +2 -2
- package/src/main/lib/auditLog.cjs +30 -5
- package/src/main/lib/buildIdentity.cjs +113 -0
- package/src/main/lib/classifyPromptTicket.cjs +3 -2
- package/src/main/lib/claudeBin.cjs +37 -2
- package/src/main/lib/cleanEnv.cjs +27 -1
- package/src/main/lib/credentials.cjs +4 -2
- package/src/main/lib/cwdClassify.cjs +185 -0
- package/src/main/lib/definitionOfDone.cjs +296 -52
- package/src/main/lib/dispatchLoop.cjs +40 -0
- package/src/main/lib/effectiveModelInfo.cjs +9 -10
- package/src/main/lib/ephemeralCwd.cjs +7 -28
- package/src/main/lib/gitWorktree.cjs +95 -8
- package/src/main/lib/guardShims.cjs +3 -3
- package/src/main/lib/historyRollup.cjs +6 -7
- package/src/main/lib/instanceLock.cjs +32 -5
- package/src/main/lib/jobSupervisorRecord.cjs +147 -0
- package/src/main/lib/jobWorktreeBootLive.cjs +9 -5
- package/src/main/lib/localAdminHttp.cjs +9 -27
- package/src/main/lib/mcpToolCatalog.cjs +29 -77
- package/src/main/lib/opsOwnership.cjs +27 -19
- package/src/main/lib/prdAuthoringSeed.cjs +36 -0
- package/src/main/lib/prdLocations.cjs +48 -1
- package/src/main/lib/procIdentity.cjs +126 -0
- package/src/main/lib/procName.cjs +98 -0
- package/src/main/lib/projectHomeAdminRoutes.cjs +56 -327
- package/src/main/lib/queueHistory.cjs +8 -7
- package/src/main/lib/queueStore.cjs +57 -37
- package/src/main/lib/quietMachineLease.cjs +19 -2
- package/src/main/lib/reservationExpiry.cjs +30 -0
- package/src/main/lib/runClaudeP.cjs +4 -2
- package/src/main/lib/runLogRetention.cjs +3 -2
- package/src/main/lib/scheduleJobSchema.cjs +2 -2
- package/src/main/lib/scheduleJobTransitions.cjs +29 -2
- package/src/main/lib/schedulerBatch.cjs +27 -2
- package/src/main/lib/schedulerPaths.cjs +175 -0
- package/src/main/lib/schedulerRuntimeState.cjs +59 -0
- package/src/main/lib/sessionSlots.cjs +47 -9
- package/src/main/lib/smProcNames.cjs +51 -0
- package/src/main/lib/upgradeDrain.cjs +188 -0
- package/src/main/lib/usageCircuit.cjs +53 -6
- package/src/main/lib/watchdogHelpers.cjs +70 -37
- package/src/main/lib/withTimeout.cjs +33 -0
- package/src/main/mcpStatus.cjs +4 -2
- package/src/main/pluginInstall.cjs +6 -2
- package/src/main/projectPages.cjs +41 -145
- package/src/main/pty.cjs +8 -1
- package/src/main/queueOps.cjs +10 -10
- package/src/main/runVerify.cjs +69 -3
- package/src/main/scheduler.cjs +1607 -383
- package/src/main/seedAgentPersonas.cjs +1 -1
- package/src/main/seedDevPlugin.cjs +1 -1
- package/src/main/seedSchedulerMcp.cjs +7 -6
- package/src/main/seedStatus.cjs +1 -1
- package/src/main/supervisor.cjs +5 -3
- package/src/main/usage.cjs +126 -62
- package/src/preload/api.d.ts +18 -64
- package/src/preload/index.cjs +2 -4
- package/src/seed/agents/project-home-builder.md +31 -48
- package/screenshots/.gitkeep +0 -0
- package/screenshots/README-screenshots.md +0 -13
- package/src/main/lib/projectPageSummarySchema.cjs +0 -181
- package/src/main/teams.cjs +0 -95
- package/src/main/templates/project-pages-catalog.json +0 -741
- package/src/main/templates/project-pages-default-home.html +0 -123
- package/src/main/templates/project-pages-pipeline.md +0 -417
- package/src/seed/prompts/code-review/ac-coverage-check.md +0 -8
- package/src/seed/prompts/code-review/correctness-only.md +0 -8
- package/src/seed/prompts/code-review/full-spectrum-high.md +0 -8
- package/src/seed/prompts/code-review/hallucination-check.md +0 -8
- package/src/seed/prompts/code-review/public-api-compat.md +0 -8
- package/src/seed/prompts/code-review/readability-naming.md +0 -8
- package/src/seed/prompts/debugging/bug-as-failing-test.md +0 -8
- package/src/seed/prompts/debugging/git-bisect-regression.md +0 -8
- package/src/seed/prompts/debugging/instrument-intermittent-bug.md +0 -8
- package/src/seed/prompts/debugging/localize-pipeline-failure.md +0 -8
- package/src/seed/prompts/debugging/reproduce-then-diagnose.md +0 -8
- package/src/seed/prompts/documentation/adr-from-change.md +0 -8
- package/src/seed/prompts/documentation/module-readme.md +0 -8
- package/src/seed/prompts/documentation/onboarding-plan.md +0 -8
- package/src/seed/prompts/documentation/refresh-claude-md.md +0 -8
- package/src/seed/prompts/documentation/tsdoc-public-exports.md +0 -8
- package/src/seed/prompts/git-pr/conventional-commit.md +0 -8
- package/src/seed/prompts/git-pr/draft-pr-title-body.md +0 -8
- package/src/seed/prompts/git-pr/pre-commit-safety-sweep.md +0 -8
- package/src/seed/prompts/git-pr/release-notes-block.md +0 -8
- package/src/seed/prompts/git-pr/split-large-pr.md +0 -8
- package/src/seed/prompts/performance/bundle-startup-audit.md +0 -8
- package/src/seed/prompts/performance/complexity-audit.md +0 -8
- package/src/seed/prompts/performance/cpu-profile-hot-path.md +0 -8
- package/src/seed/prompts/performance/db-query-plan-review.md +0 -8
- package/src/seed/prompts/performance/memory-leak-hunt.md +0 -8
- package/src/seed/prompts/qa/api-contract-tests.md +0 -8
- package/src/seed/prompts/qa/e2e-critical-path.md +0 -8
- package/src/seed/prompts/qa/failing-test-for-bug.md +0 -8
- package/src/seed/prompts/qa/find-missing-test-coverage.md +0 -8
- package/src/seed/prompts/qa/stabilize-flaky-test.md +0 -8
- package/src/seed/prompts/qa/tdd-red-first.md +0 -8
- package/src/seed/prompts/qa/visual-regression-review.md +0 -8
- package/src/seed/prompts/qa/wcag-axe-scan.md +0 -8
- package/src/seed/prompts/refactoring/dead-code-sweep.md +0 -8
- package/src/seed/prompts/refactoring/extract-duplicated-pattern.md +0 -8
- package/src/seed/prompts/refactoring/modernize-legacy-file.md +0 -8
- package/src/seed/prompts/refactoring/reduce-cyclomatic-complexity.md +0 -8
- package/src/seed/prompts/refactoring/tighten-module-boundaries.md +0 -8
- package/src/seed/prompts/security/authz-audit.md +0 -8
- package/src/seed/prompts/security/crypto-correctness.md +0 -8
- package/src/seed/prompts/security/cwe-top-25-hunt.md +0 -8
- package/src/seed/prompts/security/dependency-audit.md +0 -8
- package/src/seed/prompts/security/ipc-boundary-hardening.md +0 -8
- package/src/seed/prompts/security/owasp-top-10-staged-diff.md +0 -8
- package/src/seed/prompts/security/secret-credential-scan.md +0 -8
- package/web/README.md +0 -41
- package/web/project-pages/logic/dist/logic.cjs +0 -4709
- package/web/project-pages/render.cjs +0 -70
- package/web/project-pages/renderer/dist/renderer.cjs +0 -18900
- package/web/project-pages/validate-summary.cjs +0 -62
package/src/main/health.cjs
CHANGED
|
@@ -11,14 +11,18 @@ const os = require('node:os');
|
|
|
11
11
|
const { execFileSync } = require('node:child_process');
|
|
12
12
|
const { POLL_INTERVAL_MS, loadGateThreshold } = require('./lib/schedulerConfig.cjs');
|
|
13
13
|
const { opsPath } = require('./lib/opsOwnership.cjs');
|
|
14
|
+
const { effectivePaused } = require('./lib/upgradeDrain.cjs');
|
|
14
15
|
const { checkPersonaImports } = require('./lib/personaImportHealth.cjs');
|
|
15
16
|
const { checkDelegationReadiness } = require('./lib/delegationReadiness.cjs');
|
|
16
17
|
const { resolvePrdsDirs } = require('./lib/prdLocations.cjs');
|
|
17
18
|
const { migratePrds } = require('./lib/prdMigration.cjs');
|
|
18
19
|
const queueStore = require('./lib/queueStore.cjs');
|
|
19
|
-
const
|
|
20
|
-
const {
|
|
21
|
-
const {
|
|
20
|
+
const schedulerPaths = require('./lib/schedulerPaths.cjs');
|
|
21
|
+
const { evaluateDispatchLiveness } = require('./lib/watchdogHelpers.cjs');
|
|
22
|
+
const { computeStallSummary, computeDepHistorySatisfaction, FAILURE_STREAK_ESCALATION_MS, classifyQueueStarvation, launchBlockedSlugs, STARVE_ESCALATION_MS, REVERIFY_INTERVAL_MS } = require('./scheduler.cjs');
|
|
23
|
+
const { findStarvedProjects, findUnresolvableDepRoots, DEFAULT_PROJECT_CWD, DEP_HISTORY_FAIL_OPEN } = require('./lib/schedulerBatch.cjs');
|
|
24
|
+
const { auditLogPath, readTail } = require('./lib/auditLog.cjs');
|
|
25
|
+
const { resolveBuildIdentity } = require('./lib/buildIdentity.cjs');
|
|
22
26
|
const { DEFAULT_RUNS_DIR, computeReport, isRetentionEnabled, liveKeysFromJobs } = require('./lib/runLogRetention.cjs');
|
|
23
27
|
const { allProjectCwds } = require('./lib/activeSessions.cjs');
|
|
24
28
|
|
|
@@ -40,6 +44,7 @@ const TICK_STALL_THRESHOLD_MS = TICK_STALL_MULTIPLIER * POLL_INTERVAL_MS;
|
|
|
40
44
|
// scheduler-heartbeat.log is only appended to while Electron is running, so
|
|
41
45
|
// a stale line here is silent-on-purpose, not evidence of anything.
|
|
42
46
|
const HEARTBEAT_STALE_MS = 5 * 60_000;
|
|
47
|
+
const AUDIT_TAIL_BYTES = 1024 * 1024;
|
|
43
48
|
|
|
44
49
|
function runCheck(cmd, cwd = PROJECT_ROOT) {
|
|
45
50
|
try {
|
|
@@ -72,6 +77,7 @@ function readFreshHeartbeat(heartbeatPath) {
|
|
|
72
77
|
} catch {
|
|
73
78
|
return null;
|
|
74
79
|
}
|
|
80
|
+
if (entry.degraded === true) return null; // a subsystem threw — utilization was not read
|
|
75
81
|
if (typeof entry.ts !== 'number' || Date.now() - entry.ts > HEARTBEAT_STALE_MS) return null;
|
|
76
82
|
return entry;
|
|
77
83
|
}
|
|
@@ -82,7 +88,15 @@ function readFreshHeartbeat(heartbeatPath) {
|
|
|
82
88
|
// now) so it's testable without touching the filesystem.
|
|
83
89
|
function evaluateTickLiveness(queueState, heartbeat, now, runningCount) {
|
|
84
90
|
const jobs = queueState.jobs || [];
|
|
85
|
-
|
|
91
|
+
// A pending row whose dependsOn names a job that has not completed cannot be
|
|
92
|
+
// dispatched yet — it is waiting on that job, not on a tick. Counting it made
|
|
93
|
+
// a healthy dependsOn chain (one running, the rest waiting) read as a stall.
|
|
94
|
+
const statusBySlug = new Map(jobs.map((j) => [j.slug, j.status]));
|
|
95
|
+
const depsSatisfied = (j) => (j.dependsOn || []).every((d) => {
|
|
96
|
+
const st = statusBySlug.get(d);
|
|
97
|
+
return st === undefined || st === 'completed' || st === 'skipped';
|
|
98
|
+
});
|
|
99
|
+
const pending = jobs.filter((j) => j.status === 'pending' && depsSatisfied(j));
|
|
86
100
|
const running = runningCount ?? jobs.filter((j) => j.status === 'running').length;
|
|
87
101
|
const config = queueState.config || {};
|
|
88
102
|
// The scheduler's private concurrencyCap is retired — the machine-wide
|
|
@@ -92,7 +106,7 @@ function evaluateTickLiveness(queueState, heartbeat, now, runningCount) {
|
|
|
92
106
|
?? (() => { try { return require('./lib/sessionSlots.cjs').totalSlots(); } catch { return Infinity; } })();
|
|
93
107
|
|
|
94
108
|
if (pending.length === 0) return { stalled: false, reason: 'no-pending-jobs' };
|
|
95
|
-
if (queueState
|
|
109
|
+
if (effectivePaused(queueState)) return { stalled: false, reason: queueState.paused ? 'paused' : 'draining' };
|
|
96
110
|
if (config.enabled === false) return { stalled: false, reason: 'disabled' };
|
|
97
111
|
if (running >= concurrencyCap) return { stalled: false, reason: 'at-capacity' };
|
|
98
112
|
// 'manual' means the operator fires batches by hand — the scheduler is not
|
|
@@ -375,27 +389,50 @@ function evaluateEpicIndexHealth(cwds) {
|
|
|
375
389
|
// not a failure, matches evaluatePrdMigrationHealth's fail-open-on-absence
|
|
376
390
|
// spirit. Kept separate from the fs-touching check() call site, same pattern
|
|
377
391
|
// as every other evaluate* helper in this file.
|
|
378
|
-
|
|
392
|
+
//
|
|
393
|
+
// Ladder is driven by the shared usageCircuit's OWN open/closed state and
|
|
394
|
+
// how long it's been open (usageCircuitState/usageCircuitOpenedAt, persisted
|
|
395
|
+
// by scheduler.cjs's persistSchedulerState — this process never holds the
|
|
396
|
+
// live in-memory circuit itself) — not a binary consecutiveFailures<5 check,
|
|
397
|
+
// which used to read GREEN right up until the exact failure that also fired
|
|
398
|
+
// the one-time WARN, then stayed flatly non-GREEN forever after with no way
|
|
399
|
+
// to tell "just tripped" from "down for hours" apart:
|
|
400
|
+
// GREEN — circuit closed (or half_open probing after a fresh backoff).
|
|
401
|
+
// YELLOW — circuit open, less than FAILURE_STREAK_ESCALATION_MS (30 min).
|
|
402
|
+
// Still `ok: true` — the circuit's OWN open threshold (3
|
|
403
|
+
// consecutive failures, usageCircuit.cjs) is lower than the old
|
|
404
|
+
// 5-failure WARN, so treating YELLOW as `ok: false` here would
|
|
405
|
+
// silently tighten the overall `npm run health` rollup's failure
|
|
406
|
+
// bar from 5 down to 3. Graceful degradation (the whole point of
|
|
407
|
+
// the breaker) should not itself read as a critical failure.
|
|
408
|
+
// RED — circuit open for FAILURE_STREAK_ESCALATION_MS or longer —
|
|
409
|
+
// `ok: false`, matching the same 30-minute threshold the
|
|
410
|
+
// escalation ladder (warnFailureStreakIfNeeded) re-alerts on.
|
|
411
|
+
function evaluateUsagePollerHealth(state, redThresholdMs = FAILURE_STREAK_ESCALATION_MS) {
|
|
379
412
|
if (!state || typeof state !== 'object') return { ok: true, applicable: false };
|
|
380
413
|
const consecutiveFailures = typeof state.consecutiveFailures === 'number' ? state.consecutiveFailures : 0;
|
|
381
414
|
const backoffMs = typeof state.backoffMs === 'number' ? state.backoffMs : null;
|
|
382
415
|
const lastPollAt = typeof state.lastPollAt === 'number' ? state.lastPollAt : null;
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
416
|
+
const circuitState = typeof state.usageCircuitState === 'string' ? state.usageCircuitState : 'closed';
|
|
417
|
+
const openedAt = typeof state.usageCircuitOpenedAt === 'number' ? state.usageCircuitOpenedAt : null;
|
|
418
|
+
|
|
419
|
+
if (circuitState !== 'open' && circuitState !== 'half_open') {
|
|
420
|
+
return { ok: true, applicable: true, color: 'GREEN', consecutiveFailures, backoffMs, lastPollAt };
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
const openMs = openedAt ? Math.max(0, Date.now() - openedAt) : 0;
|
|
424
|
+
const color = openMs >= redThresholdMs ? 'RED' : 'YELLOW';
|
|
388
425
|
return {
|
|
389
|
-
ok,
|
|
426
|
+
ok: color !== 'RED',
|
|
390
427
|
applicable: true,
|
|
428
|
+
color,
|
|
391
429
|
consecutiveFailures,
|
|
392
430
|
backoffMs,
|
|
393
431
|
lastPollAt,
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
}),
|
|
432
|
+
openedAt,
|
|
433
|
+
openMs,
|
|
434
|
+
message: `usage/rate-limit poller circuit is ${circuitState}, open for ${Math.round(openMs / 60_000)}m (${color}) — `
|
|
435
|
+
+ `${consecutiveFailures} consecutive failures, backoffMs=${backoffMs} — see logs scope=scheduler`,
|
|
399
436
|
};
|
|
400
437
|
}
|
|
401
438
|
|
|
@@ -416,6 +453,10 @@ function evaluateUsagePollerHealth(state, threshold = FAILURE_STREAK_WARN_THRESH
|
|
|
416
453
|
// OWN forcing threshold; this is the "even the watchdog isn't helping
|
|
417
454
|
// anymore" signal a human needs to see.
|
|
418
455
|
const DISPATCH_STALL_THRESHOLD_MS = 2 * 60 * 60_000;
|
|
456
|
+
// Early-warning tier off the SAME launch-keyed clock (lastRunAt — a batch
|
|
457
|
+
// actually launched): still ok:true, but flagged so a slow leak surfaces
|
|
458
|
+
// well before the 2-hour cold-report threshold.
|
|
459
|
+
const DISPATCH_WARN_THRESHOLD_MS = 15 * 60_000;
|
|
419
460
|
|
|
420
461
|
/**
|
|
421
462
|
* evaluateQueueDispatchHealth(queueState, runningCount, now, thresholdMs) →
|
|
@@ -430,14 +471,18 @@ const DISPATCH_STALL_THRESHOLD_MS = 2 * 60 * 60_000;
|
|
|
430
471
|
* `blocked` verdict therefore stays `ok: true` (never trips the health gate)
|
|
431
472
|
* but is still reported by name so it isn't silently invisible either.
|
|
432
473
|
*/
|
|
433
|
-
function evaluateQueueDispatchHealth(queueState, runningCount, now, thresholdMs = DISPATCH_STALL_THRESHOLD_MS) {
|
|
474
|
+
function evaluateQueueDispatchHealth(queueState, runningCount, now, thresholdMs = DISPATCH_STALL_THRESHOLD_MS, warnMs = DISPATCH_WARN_THRESHOLD_MS) {
|
|
475
|
+
// Clock = lastRunAt (launches), never lastDispatchAttemptAt (loop-alive
|
|
476
|
+
// heartbeat, refreshed by ticks that launch nothing). Cold process: no
|
|
477
|
+
// in-memory pause/boot stamps to fold in. Breaker-held rows count blocked.
|
|
434
478
|
const verdict = classifyQueueStarvation({
|
|
435
479
|
jobs: queueState?.jobs,
|
|
436
|
-
paused: queueState
|
|
480
|
+
paused: effectivePaused(queueState),
|
|
437
481
|
runningCount,
|
|
438
|
-
lastRunAtMs: Date.parse(queueState?.
|
|
482
|
+
lastRunAtMs: Date.parse(queueState?.lastRunAt ?? ''),
|
|
483
|
+
heldSlugs: launchBlockedSlugs(queueState?.jobs, queueState?.launchBlocks),
|
|
439
484
|
now,
|
|
440
|
-
thresholdMs,
|
|
485
|
+
thresholdMs: Math.min(warnMs, thresholdMs),
|
|
441
486
|
});
|
|
442
487
|
if (!verdict) return { ok: true };
|
|
443
488
|
if (verdict.kind === 'blocked') {
|
|
@@ -445,18 +490,79 @@ function evaluateQueueDispatchHealth(queueState, runningCount, now, thresholdMs
|
|
|
445
490
|
ok: true,
|
|
446
491
|
blocked: true,
|
|
447
492
|
pending: verdict.pending,
|
|
448
|
-
message: `${verdict.pending} pending job(s) behind a blocked dependsOn chain — dispatch cannot help, needs a human`,
|
|
493
|
+
message: `${verdict.pending} pending job(s) behind a blocked dependsOn chain or open launch breaker — dispatch cannot help, needs a human`,
|
|
449
494
|
};
|
|
450
495
|
}
|
|
451
496
|
const ageMin = Math.round(verdict.idleMs / 60_000);
|
|
497
|
+
const critical = verdict.idleMs >= thresholdMs;
|
|
452
498
|
return {
|
|
453
|
-
ok:
|
|
454
|
-
starved: true,
|
|
499
|
+
ok: !critical,
|
|
500
|
+
...(critical ? { starved: true } : { warn: true }),
|
|
455
501
|
pending: verdict.pending,
|
|
456
502
|
dispatchable: verdict.dispatchable,
|
|
457
503
|
idleMs: verdict.idleMs,
|
|
458
|
-
message: `${verdict.dispatchable} dispatchable pending job(s), 0 running, no
|
|
459
|
-
+
|
|
504
|
+
message: `${verdict.dispatchable} dispatchable pending job(s), 0 running, no launch in ~${ageMin}m `
|
|
505
|
+
+ (critical
|
|
506
|
+
? `(threshold ${Math.round(thresholdMs / 60_000)}m) — dispatch appears stuck`
|
|
507
|
+
: `(warn at ${Math.round(warnMs / 60_000)}m, fail at ${Math.round(thresholdMs / 60_000)}m) — dispatch may be stalling`),
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* evaluateUnresolvableDepHealth(jobs, satisfiedSlugsByCwd) →
|
|
513
|
+
* { ok, projects?: [{ cwd, roots, pending }], message? }
|
|
514
|
+
*
|
|
515
|
+
* Pure. A project whose EVERY pending row is held (directly or transitively)
|
|
516
|
+
* behind a dependsOn slug that names no live row and no history/archive
|
|
517
|
+
* record is fully blocked with a healthy engine — the 2026-09-18 fo-01 stall
|
|
518
|
+
* (21 pending rows, 0 running, one archived root, every other component
|
|
519
|
+
* GREEN). classifyQueueStarvation's 'blocked' kind cannot see this shape (it
|
|
520
|
+
* only walks failed/skipped ROWS), and evaluateQueueDispatchHealth keeps
|
|
521
|
+
* 'blocked' ok:true by design, so this is the component that fails it, naming
|
|
522
|
+
* the root slug(s). Reuses the picker's own resolver (findUnresolvableDepRoots)
|
|
523
|
+
* so health can never disagree with what dispatch actually holds. A project
|
|
524
|
+
* with anything running, or any dispatchable pending row, is not fully blocked.
|
|
525
|
+
* O(jobs + deps) per project; a fail-open project (history unreadable) is skipped.
|
|
526
|
+
*/
|
|
527
|
+
function evaluateUnresolvableDepHealth(jobs, satisfiedSlugsByCwd) {
|
|
528
|
+
const byCwd = new Map();
|
|
529
|
+
for (const j of Array.isArray(jobs) ? jobs : []) {
|
|
530
|
+
if (!j) continue;
|
|
531
|
+
const cwd = j.cwd || DEFAULT_PROJECT_CWD;
|
|
532
|
+
if (!byCwd.has(cwd)) byCwd.set(cwd, []);
|
|
533
|
+
byCwd.get(cwd).push(j);
|
|
534
|
+
}
|
|
535
|
+
const projects = [];
|
|
536
|
+
for (const [cwd, rows] of byCwd) {
|
|
537
|
+
const satisfied = satisfiedSlugsByCwd?.get?.(cwd) ?? new Set();
|
|
538
|
+
if (satisfied === DEP_HISTORY_FAIL_OPEN) continue;
|
|
539
|
+
const pending = rows.filter((j) => j.status === 'pending');
|
|
540
|
+
if (pending.length === 0 || rows.some((j) => j.status === 'running')) continue;
|
|
541
|
+
const roots = findUnresolvableDepRoots(rows, satisfied);
|
|
542
|
+
if (roots.length === 0) continue;
|
|
543
|
+
const rootSet = new Set(roots);
|
|
544
|
+
const bare = (x) => String(x ?? '').replace(/^\d+-/, '');
|
|
545
|
+
const held = new Set();
|
|
546
|
+
let grew = true;
|
|
547
|
+
while (grew) {
|
|
548
|
+
grew = false;
|
|
549
|
+
for (const j of pending) {
|
|
550
|
+
if (held.has(j.slug)) continue;
|
|
551
|
+
const isHeld = (j.dependsOn ?? []).some((d) => rootSet.has(d)
|
|
552
|
+
|| pending.some((p) => held.has(p.slug) && (p.slug === d || bare(p.slug) === bare(d))));
|
|
553
|
+
if (isHeld) { held.add(j.slug); grew = true; }
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
if (held.size === pending.length) projects.push({ cwd, roots, pending: pending.length });
|
|
557
|
+
}
|
|
558
|
+
if (projects.length === 0) return { ok: true };
|
|
559
|
+
return {
|
|
560
|
+
ok: false,
|
|
561
|
+
projects,
|
|
562
|
+
message: projects
|
|
563
|
+
.map((p) => `${p.cwd}: all ${p.pending} pending job(s) held behind unresolvable dependsOn root ${p.roots.join(', ')} `
|
|
564
|
+
+ '(no live row, no history/archive record) — restore its completion record or fix the dependsOn')
|
|
565
|
+
.join('; '),
|
|
460
566
|
};
|
|
461
567
|
}
|
|
462
568
|
|
|
@@ -472,12 +578,9 @@ function evaluateQueueDispatchHealth(queueState, runningCount, now, thresholdMs
|
|
|
472
578
|
* Missing/unreadable log → {} (no reasons known yet), never a throw.
|
|
473
579
|
*/
|
|
474
580
|
function latestStarveEscalationReasons(auditLogPath) {
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
} catch {
|
|
479
|
-
return {};
|
|
480
|
-
}
|
|
581
|
+
// Bounded reverse tail — the log is never rotated (28 MB+), and the latest
|
|
582
|
+
// escalation per cwd is by definition recent.
|
|
583
|
+
const lines = readTail(AUDIT_TAIL_BYTES, auditLogPath);
|
|
481
584
|
const byCwd = {};
|
|
482
585
|
for (const line of lines) {
|
|
483
586
|
let rec;
|
|
@@ -491,6 +594,24 @@ function latestStarveEscalationReasons(auditLogPath) {
|
|
|
491
594
|
return byCwd;
|
|
492
595
|
}
|
|
493
596
|
|
|
597
|
+
/**
|
|
598
|
+
* evaluateBuildFreshness({running, installedOnDisk, repoHead}) → component
|
|
599
|
+
*
|
|
600
|
+
* Pure; each input is a short git sha or null. restartNeeded: the on-disk
|
|
601
|
+
* install differs from what the live heartbeat says is running. publishNeeded:
|
|
602
|
+
* repo HEAD differs from the installed build. Both are reported, neither is
|
|
603
|
+
* critical (never flips status.ok).
|
|
604
|
+
*/
|
|
605
|
+
function evaluateBuildFreshness({ running = null, installedOnDisk = null, repoHead = null } = {}) {
|
|
606
|
+
const differs = (a, b) => !!a && !!b && !(a.startsWith(b) || b.startsWith(a));
|
|
607
|
+
const restartNeeded = differs(running, installedOnDisk);
|
|
608
|
+
const publishNeeded = differs(repoHead, installedOnDisk);
|
|
609
|
+
const notes = [];
|
|
610
|
+
if (restartNeeded) notes.push(`restart needed: running ${running}, installed ${installedOnDisk}`);
|
|
611
|
+
if (publishNeeded) notes.push(`publish needed: repo HEAD ${repoHead}, installed ${installedOnDisk}`);
|
|
612
|
+
return { ok: true, running, installedOnDisk, repoHead, restartNeeded, publishNeeded, ...(notes.length ? { note: notes.join('; ') } : {}) };
|
|
613
|
+
}
|
|
614
|
+
|
|
494
615
|
/**
|
|
495
616
|
* evaluateStarveEscalationHealth(jobs, now, thresholdMs, escalationReasons)
|
|
496
617
|
* → { ok, projects?, message? }
|
|
@@ -532,6 +653,69 @@ function loadUsagePollerState(statePath) {
|
|
|
532
653
|
}
|
|
533
654
|
}
|
|
534
655
|
|
|
656
|
+
/**
|
|
657
|
+
* evaluateBlockingParkHealth(jobs, now, thresholdMs = REVERIFY_INTERVAL_MS) →
|
|
658
|
+
* { ok, parks?: [{ cwd, slug, verdict, ageMs, dependents }], message? }
|
|
659
|
+
*
|
|
660
|
+
* Pure. A `needs_review` row parked longer than one full periodic-reverify
|
|
661
|
+
* interval that STILL holds >= 1 pending dependent (transitively, through
|
|
662
|
+
* pending rows, via dependsOn — exact or bare slug, same matching as the
|
|
663
|
+
* picker) is a silently-blocking park: the self-heal ladder had a full pass
|
|
664
|
+
* and did not release it. 2026-09-18: 1229-fo-03 held 19 of 20 pending rows
|
|
665
|
+
* while every other component read GREEN. Parked-since is the row's last
|
|
666
|
+
* `to: needs_review` statusHistory entry (fallback finishedAt); a row with no
|
|
667
|
+
* recoverable timestamp is skipped rather than guessed at. O(jobs + deps).
|
|
668
|
+
*/
|
|
669
|
+
function evaluateBlockingParkHealth(jobs, now, thresholdMs = REVERIFY_INTERVAL_MS) {
|
|
670
|
+
const rows = (Array.isArray(jobs) ? jobs : []).filter(Boolean);
|
|
671
|
+
const bare = (x) => String(x ?? '').replace(/^\d+-/, '');
|
|
672
|
+
const byCwd = new Map();
|
|
673
|
+
for (const j of rows) {
|
|
674
|
+
const cwd = j.cwd || DEFAULT_PROJECT_CWD;
|
|
675
|
+
if (!byCwd.has(cwd)) byCwd.set(cwd, []);
|
|
676
|
+
byCwd.get(cwd).push(j);
|
|
677
|
+
}
|
|
678
|
+
const parks = [];
|
|
679
|
+
for (const [cwd, group] of byCwd) {
|
|
680
|
+
const pending = group.filter((j) => j.status === 'pending');
|
|
681
|
+
if (pending.length === 0) continue;
|
|
682
|
+
for (const park of group) {
|
|
683
|
+
if (park.status !== 'needs_review') continue;
|
|
684
|
+
const history = Array.isArray(park.statusHistory) ? park.statusHistory : [];
|
|
685
|
+
let at = null;
|
|
686
|
+
for (let i = history.length - 1; i >= 0; i--) {
|
|
687
|
+
if (history[i]?.to === 'needs_review') { at = history[i].at; break; }
|
|
688
|
+
}
|
|
689
|
+
const since = Date.parse(at ?? park.finishedAt ?? '');
|
|
690
|
+
if (Number.isNaN(since) || now - since <= thresholdMs) continue;
|
|
691
|
+
const held = new Set();
|
|
692
|
+
const heldBy = (d, root) => root.slug === d || bare(root.slug) === bare(d);
|
|
693
|
+
let grew = true;
|
|
694
|
+
while (grew) {
|
|
695
|
+
grew = false;
|
|
696
|
+
for (const j of pending) {
|
|
697
|
+
if (held.has(j.slug)) continue;
|
|
698
|
+
if ((j.dependsOn ?? []).some((d) => heldBy(d, park) || pending.some((p) => held.has(p.slug) && heldBy(d, p)))) {
|
|
699
|
+
held.add(j.slug);
|
|
700
|
+
grew = true;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
if (held.size > 0) parks.push({ cwd, slug: park.slug, verdict: park.verifierVerdict ?? null, ageMs: now - since, dependents: held.size });
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
if (parks.length === 0) return { ok: true };
|
|
708
|
+
return {
|
|
709
|
+
ok: false,
|
|
710
|
+
parks,
|
|
711
|
+
message: parks
|
|
712
|
+
.map((p) => `${p.slug} (${p.cwd}) parked needs_review${p.verdict ? `/${p.verdict}` : ''} for ${Math.round(p.ageMs / 60_000)}m `
|
|
713
|
+
+ `(> one ${Math.round(REVERIFY_INTERVAL_MS / 60_000)}m reverify interval) is blocking ${p.dependents} pending dependent(s) — `
|
|
714
|
+
+ 'the self-heal ladder did not release it; resolve it or scheduler_reset_job')
|
|
715
|
+
.join('; '),
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
|
|
535
719
|
async function check() {
|
|
536
720
|
const start = Date.now();
|
|
537
721
|
const status = {
|
|
@@ -577,7 +761,7 @@ async function check() {
|
|
|
577
761
|
};
|
|
578
762
|
|
|
579
763
|
// 2. Check config directory exists and is writable.
|
|
580
|
-
const configDir =
|
|
764
|
+
const configDir = schedulerPaths.claudeHome();
|
|
581
765
|
try {
|
|
582
766
|
await fsp.access(configDir, fs.constants.R_OK | fs.constants.W_OK);
|
|
583
767
|
const stat = await fsp.stat(configDir);
|
|
@@ -599,11 +783,14 @@ async function check() {
|
|
|
599
783
|
// 3. Check scheduler state (federated, 2026-07-31): machine runtime file +
|
|
600
784
|
// per-project job shards merged via queueStore — the retired global
|
|
601
785
|
// queue.json is no longer consulted.
|
|
602
|
-
const queuePath =
|
|
786
|
+
const queuePath = schedulerPaths.machineStatePath();
|
|
603
787
|
let queueState = null;
|
|
604
788
|
try {
|
|
605
789
|
queueState = queueStore.readMergedSync();
|
|
606
790
|
if (queueState.unreadable) throw new Error(queueState.unreadable);
|
|
791
|
+
for (const u of queueState.unreadableCwds ?? []) {
|
|
792
|
+
status.issues.push(`Scheduler queue shard quarantined for ${u.cwd}: ${u.error} — that project is skipped until it reads clean`);
|
|
793
|
+
}
|
|
607
794
|
// A torn scheduler-machine.json is now recovered rather than halting
|
|
608
795
|
// dispatch (queueStore.cjs's findLongestValidJsonPrefix) — but recovery
|
|
609
796
|
// happening at all means the file failed to parse moments ago, which is
|
|
@@ -626,11 +813,7 @@ async function check() {
|
|
|
626
813
|
const quarantinedCount = Object.values(queueState.jobs || {}).filter(
|
|
627
814
|
(j) => j.status === 'quarantined'
|
|
628
815
|
).length;
|
|
629
|
-
const
|
|
630
|
-
os.homedir(),
|
|
631
|
-
'.claude/session-manager/scheduler-heartbeat.log'
|
|
632
|
-
);
|
|
633
|
-
const heartbeat = readFreshHeartbeat(heartbeatPath);
|
|
816
|
+
const heartbeat = readFreshHeartbeat(schedulerPaths.heartbeatPath());
|
|
634
817
|
const liveness = evaluateTickLiveness(queueState, heartbeat, Date.now(), runningCount);
|
|
635
818
|
|
|
636
819
|
// Per-project rollup (PRD: monitoring must not collapse per-project
|
|
@@ -647,10 +830,13 @@ async function check() {
|
|
|
647
830
|
.map(([cwd]) => cwd);
|
|
648
831
|
|
|
649
832
|
status.components.scheduler_queue = {
|
|
650
|
-
ok: !liveness.stalled && projectsPastThreshold.length === 0 && !queueState.machineStateRecovered
|
|
833
|
+
ok: !liveness.stalled && projectsPastThreshold.length === 0 && !queueState.machineStateRecovered
|
|
834
|
+
&& (queueState.unreadableCwds ?? []).length === 0,
|
|
651
835
|
path: queuePath,
|
|
652
836
|
jobs: Object.keys(queueState.jobs || {}).length,
|
|
653
837
|
running: runningCount,
|
|
838
|
+
// Deliberate restart drain (lib/upgradeDrain.cjs) — reported so it is never read as a stall.
|
|
839
|
+
drain: queueState.drain?.active ? { since: queueState.drain.since ?? null, requestedAt: queueState.drain.requestedAt ?? null } : null,
|
|
654
840
|
failed: failedCount,
|
|
655
841
|
needsReview: needsReviewCount,
|
|
656
842
|
quarantined: quarantinedCount,
|
|
@@ -658,6 +844,7 @@ async function check() {
|
|
|
658
844
|
perProjectStall,
|
|
659
845
|
tickLiveness: liveness.reason,
|
|
660
846
|
machineStateRecovered: queueState.machineStateRecovered ?? false,
|
|
847
|
+
quarantinedCwds: (queueState.unreadableCwds ?? []).map((u) => u.cwd),
|
|
661
848
|
machineStateRecoveryMode: queueState.machineStateRecoveryMode ?? null,
|
|
662
849
|
// Informational only (PRD 1085): current 1-min loadavg per core vs the
|
|
663
850
|
// launch-gate threshold, so a "nothing is launching" report can be
|
|
@@ -694,10 +881,31 @@ async function check() {
|
|
|
694
881
|
);
|
|
695
882
|
}
|
|
696
883
|
}
|
|
884
|
+
// Same classification the external watchdog logs (observability only).
|
|
885
|
+
const dispatchLiveness = evaluateDispatchLiveness(heartbeat, now);
|
|
886
|
+
status.components.scheduler_queue.dispatchLiveness = dispatchLiveness;
|
|
887
|
+
if (dispatchLiveness.dead) {
|
|
888
|
+
status.issues.push('Scheduler dispatch appears dead: heartbeat is fresh but nothing has launched in 30m+ with dispatchable work pending and nothing running');
|
|
889
|
+
}
|
|
697
890
|
status.components.queue_dispatch = evaluateQueueDispatchHealth(queueState, runningCount, now);
|
|
698
891
|
if (!status.components.queue_dispatch.ok || status.components.queue_dispatch.blocked) {
|
|
699
892
|
status.issues.push(`Queue dispatch: ${status.components.queue_dispatch.message}`);
|
|
700
893
|
}
|
|
894
|
+
try {
|
|
895
|
+
status.components.queue_dep_roots = evaluateUnresolvableDepHealth(
|
|
896
|
+
queueState.jobs, await computeDepHistorySatisfaction(queueState),
|
|
897
|
+
);
|
|
898
|
+
} catch (e) {
|
|
899
|
+
status.components.queue_dep_roots = { ok: true, error: `dep-root check failed: ${e?.message}` };
|
|
900
|
+
}
|
|
901
|
+
if (!status.components.queue_dep_roots.ok) {
|
|
902
|
+
status.issues.push(`Queue dependsOn: ${status.components.queue_dep_roots.message}`);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
status.components.blocking_parks = evaluateBlockingParkHealth(queueState.jobs, now);
|
|
906
|
+
if (!status.components.blocking_parks.ok) {
|
|
907
|
+
status.issues.push(`Blocking park: ${status.components.blocking_parks.message}`);
|
|
908
|
+
}
|
|
701
909
|
|
|
702
910
|
// Per-project starve escalation (bounded consequence for project_starved
|
|
703
911
|
// — see runStarveEscalationSweep in scheduler.cjs). Distinct from
|
|
@@ -705,7 +913,7 @@ async function check() {
|
|
|
705
913
|
// "has any ONE project been starved past the LATER escalation threshold",
|
|
706
914
|
// the exact condition the 2026-09-12 19h Bilko starve went unreported by.
|
|
707
915
|
status.components.project_starve_escalation = evaluateStarveEscalationHealth(
|
|
708
|
-
queueState.jobs, now, STARVE_ESCALATION_MS, latestStarveEscalationReasons(
|
|
916
|
+
queueState.jobs, now, STARVE_ESCALATION_MS, latestStarveEscalationReasons(auditLogPath()),
|
|
709
917
|
);
|
|
710
918
|
if (!status.components.project_starve_escalation.ok) {
|
|
711
919
|
status.issues.push(status.components.project_starve_escalation.message);
|
|
@@ -746,7 +954,7 @@ async function check() {
|
|
|
746
954
|
// failure. A CORRUPT file (the torn-write class fixed for the sibling
|
|
747
955
|
// scheduler-machine.json in f56bdc0) is NOT the same as missing — surfaced
|
|
748
956
|
// as non-GREEN rather than fail-open, via loadUsagePollerState below.
|
|
749
|
-
const loaded = loadUsagePollerState(
|
|
957
|
+
const loaded = loadUsagePollerState(schedulerPaths.schedulerStatePath());
|
|
750
958
|
if (loaded.missing) {
|
|
751
959
|
status.components.usage_poller = { ok: true, applicable: false };
|
|
752
960
|
} else if (loaded.errorMessage) {
|
|
@@ -790,10 +998,7 @@ async function check() {
|
|
|
790
998
|
// of a stale installed build vs. the git repo — it fires regardless of
|
|
791
999
|
// *why* the build is stale (the 2026-07-31 burrow-project ENOENT + the
|
|
792
1000
|
// 223-file/189-resurrected-job incident both trace back to this).
|
|
793
|
-
const legacyPrdsDir =
|
|
794
|
-
os.homedir(),
|
|
795
|
-
'.claude/session-manager/scheduled-plans/prds'
|
|
796
|
-
);
|
|
1001
|
+
const legacyPrdsDir = schedulerPaths.prdsRoot();
|
|
797
1002
|
try {
|
|
798
1003
|
const migrationResult = await migratePrds(legacyPrdsDir);
|
|
799
1004
|
status.components.prd_migration = evaluatePrdMigrationHealth(migrationResult, legacyPrdsDir);
|
|
@@ -809,7 +1014,7 @@ async function check() {
|
|
|
809
1014
|
}
|
|
810
1015
|
|
|
811
1016
|
// 5. Check transcripts directory (where live session logs are tailed).
|
|
812
|
-
const projectsDir =
|
|
1017
|
+
const projectsDir = schedulerPaths.claudeProjectsDir();
|
|
813
1018
|
try {
|
|
814
1019
|
await fsp.access(projectsDir, fs.constants.R_OK);
|
|
815
1020
|
status.components.transcripts_dir = {
|
|
@@ -826,10 +1031,7 @@ async function check() {
|
|
|
826
1031
|
}
|
|
827
1032
|
|
|
828
1033
|
// 6. Check session-manager's own logs (informational, not blocking).
|
|
829
|
-
const smLogsDir =
|
|
830
|
-
os.homedir(),
|
|
831
|
-
'.claude/session-manager/logs'
|
|
832
|
-
);
|
|
1034
|
+
const smLogsDir = schedulerPaths.watchdogLogsDir();
|
|
833
1035
|
let logAge = null;
|
|
834
1036
|
try {
|
|
835
1037
|
const files = await fsp.readdir(smLogsDir);
|
|
@@ -944,11 +1146,29 @@ async function check() {
|
|
|
944
1146
|
status.issues.push(`Epic index health check failed: ${e.message}`);
|
|
945
1147
|
}
|
|
946
1148
|
|
|
1149
|
+
// 6.7. Build freshness (informational): running heartbeat vs installed
|
|
1150
|
+
// build-info vs repo HEAD.
|
|
1151
|
+
try {
|
|
1152
|
+
let repoHead = null;
|
|
1153
|
+
try {
|
|
1154
|
+
repoHead = execFileSync('git', ['-C', PROJECT_ROOT, 'rev-parse', '--short', 'HEAD'], {
|
|
1155
|
+
timeout: 5000, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],
|
|
1156
|
+
}).trim() || null;
|
|
1157
|
+
} catch { /* not a git checkout */ }
|
|
1158
|
+
status.components.build = evaluateBuildFreshness({
|
|
1159
|
+
running: readFreshHeartbeat(schedulerPaths.heartbeatPath())?.build?.codeSha ?? null,
|
|
1160
|
+
installedOnDisk: resolveBuildIdentity().codeSha ?? null,
|
|
1161
|
+
repoHead,
|
|
1162
|
+
});
|
|
1163
|
+
} catch (e) {
|
|
1164
|
+
status.components.build = { ok: true, error: e.message };
|
|
1165
|
+
}
|
|
1166
|
+
|
|
947
1167
|
// 7. Summary scoring: ok if all critical components pass.
|
|
948
1168
|
// Critical: nodejs, config dir, typescript, build artifact, test infrastructure.
|
|
949
1169
|
// Non-fatal: scheduler/transcripts dirs may not exist on fresh install.
|
|
950
1170
|
// Informational: app log age (shows if app is running, but not blocking).
|
|
951
|
-
const criticalComponents = ['nodejs', 'config_dir', 'typescript', 'build_artifact', 'test_infrastructure', 'scheduler_queue', 'queue_dispatch', 'project_starve_escalation', 'usage_poller', 'prd_migration', 'claude_md_budget', 'delegation_chain'];
|
|
1171
|
+
const criticalComponents = ['nodejs', 'config_dir', 'typescript', 'build_artifact', 'test_infrastructure', 'scheduler_queue', 'queue_dispatch', 'queue_dep_roots', 'blocking_parks', 'project_starve_escalation', 'usage_poller', 'prd_migration', 'claude_md_budget', 'delegation_chain'];
|
|
952
1172
|
status.ok = criticalComponents.every((c) => status.components[c]?.ok !== false);
|
|
953
1173
|
|
|
954
1174
|
status.elapsedMs = Date.now() - start;
|
|
@@ -980,9 +1200,13 @@ module.exports = {
|
|
|
980
1200
|
evaluateUsagePollerHealth,
|
|
981
1201
|
loadUsagePollerState,
|
|
982
1202
|
evaluateQueueDispatchHealth,
|
|
1203
|
+
evaluateUnresolvableDepHealth,
|
|
1204
|
+
evaluateBlockingParkHealth,
|
|
983
1205
|
evaluateStarveEscalationHealth,
|
|
1206
|
+
evaluateBuildFreshness,
|
|
984
1207
|
latestStarveEscalationReasons,
|
|
985
1208
|
DISPATCH_STALL_THRESHOLD_MS,
|
|
1209
|
+
DISPATCH_WARN_THRESHOLD_MS,
|
|
986
1210
|
TICK_STALL_THRESHOLD_MS,
|
|
987
1211
|
HEARTBEAT_STALE_MS,
|
|
988
1212
|
};
|
|
@@ -23,8 +23,8 @@
|
|
|
23
23
|
'use strict';
|
|
24
24
|
|
|
25
25
|
const path = require('node:path');
|
|
26
|
-
const os = require('node:os');
|
|
27
26
|
const fsp = require('node:fs/promises');
|
|
27
|
+
const { heapSnapshotDir } = require('./lib/schedulerPaths.cjs');
|
|
28
28
|
|
|
29
29
|
const CHANNEL = 'diagnostics:heap-snapshot';
|
|
30
30
|
const DEFAULT_TIMEOUT_MS = 3 * 60 * 1000; // 3 min — snapshots can be multi-hundred-MB
|
|
@@ -34,7 +34,7 @@ function isEnabled() {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
function snapshotDir() {
|
|
37
|
-
return
|
|
37
|
+
return heapSnapshotDir();
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
function timestampedFilename(now = new Date()) {
|
|
@@ -643,7 +643,7 @@ function resetIntradayRegistryForTests() {
|
|
|
643
643
|
/**
|
|
644
644
|
* Pure walk-and-parse step (no rollup write) — split out from
|
|
645
645
|
* refreshIntradayToday() so it can be benchmarked read-only against the real
|
|
646
|
-
* ~/.claude/projects
|
|
646
|
+
* ~/.claude/projects without touching
|
|
647
647
|
* the rollup file on disk.
|
|
648
648
|
*/
|
|
649
649
|
async function computeIntradayBuckets() {
|