@moretea-labs/forge 1.5.1 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/CHANGELOG.md +46 -0
- package/CLAUDE.md +2 -0
- package/DESIGN.md +7 -2
- package/README.md +4 -4
- package/README.zh-CN.md +2 -2
- package/SKILL.md +2 -2
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/agents/openai.yaml +1 -1
- package/assets/hooks/hook-input.sh +11 -8
- package/assets/hooks/lib/session-state.sh +10 -7
- package/assets/hooks/lib/workflow-state.sh +13 -177
- package/assets/hooks/post-tool-observer.sh +8 -2
- package/assets/hooks/pre-edit-guard.sh +8 -39
- package/assets/hooks/prompt-guard.sh +7 -83
- package/assets/hooks/session-start-context.sh +56 -1
- package/assets/partials/05-workflow.partial.md +1 -1
- package/assets/partials-agents/03-orchestration.partial.md +1 -1
- package/assets/plugin-registry.v1.json +54 -3
- package/assets/reference-configs/document-generation.md +1 -1
- package/assets/reference-configs/harness-overview.md +1 -1
- package/assets/reference-configs/hook-operations.md +1 -1
- package/assets/reference-configs/sprint-contracts.md +3 -3
- package/assets/skill-version.json +2 -2
- package/assets/templates/contract.template.md +2 -7
- package/assets/templates/helpers/capture-plan.sh +2 -2
- package/assets/templates/helpers/check-task-workflow.sh +4 -4
- package/assets/templates/helpers/contract-worktree.sh +1 -21
- package/assets/templates/helpers/ensure-task-workflow.sh +13 -38
- package/assets/templates/helpers/migrate-project-template.sh +22 -45
- package/assets/templates/helpers/new-plan.sh +1 -1
- package/assets/templates/helpers/plan-to-todo.sh +12 -37
- package/assets/templates/helpers/ship-worktrees.sh +2 -29
- package/assets/templates/helpers/sprint-backlog.sh +6 -6
- package/assets/templates/helpers/verify-contract.sh +1 -111
- package/assets/templates/helpers/verify-sprint.sh +3 -180
- package/assets/templates/plan.template.md +1 -1
- package/assets/templates/review.template.md +9 -34
- package/assets/workflow-contract.v1.json +10 -19
- package/docs/README.md +10 -10
- package/docs/architecture/CURRENT.md +170 -0
- package/docs/architecture/EVOLUTION.md +42 -0
- package/docs/architecture/current/README.md +4 -156
- package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +2 -2
- package/docs/architecture/decisions/20260823-immediate-controller-relay.md +53 -0
- package/docs/architecture/decisions/20260826-browser-runtime-v2.md +73 -0
- package/docs/architecture/history/README.md +7 -0
- package/docs/architecture/{chatgpt-handoff-facade.md → history/chatgpt-handoff-facade.md} +4 -0
- package/docs/architecture/{global-hook-runtime.md → history/global-hook-runtime.md} +7 -3
- package/docs/architecture/{ios-semantic-automation-provider-v2.md → history/ios-semantic-automation-provider-v2.md} +6 -2
- package/docs/architecture/history.md +3 -34
- package/docs/architecture/index.md +15 -98
- package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
- package/docs/architecture/reviews/2026-08-22-semantic-harness-convergence-acceptance.md +39 -0
- package/docs/architecture/versions/1.6.md +38 -0
- package/docs/architecture/versions/1.7.md +41 -0
- package/docs/forge-chatgpt-browser-engine.md +2 -2
- package/docs/forge-chatgpt-mcp-setup.md +4 -28
- package/docs/forge-chatgpt-supervised-automation.md +3 -3
- package/docs/forge-github-issue-launcher.md +1 -1
- package/docs/forge-runtime-self-healing-loop.md +1 -1
- package/docs/operations/controller-browser-plugin.md +38 -22
- package/docs/operations/controller-performance-and-502.md +124 -0
- package/docs/operations/controller-reliability-runbook.md +1 -1
- package/docs/operations/features.md +3 -3
- package/docs/operations/github-repository.md +1 -1
- package/docs/operations/homebrew.md +2 -2
- package/docs/operations/mcp-tool-exposure.md +5 -3
- package/docs/tutorials/01-install-and-start.md +1 -1
- package/docs/tutorials/01-install-and-start.zh-CN.md +1 -1
- package/docs/wiki/Architecture.md +6 -6
- package/docs/wiki/Implementation.md +2 -2
- package/docs/wiki/Installation.md +1 -1
- package/docs/wiki/Integrations.md +2 -2
- package/docs/wiki/Quick-Start.md +1 -1
- package/docs/wiki/Work-Lifecycle.md +1 -1
- package/package.json +29 -10
- package/scripts/activate-source-baseline.ts +17 -13
- package/scripts/benchmark-background-check-overlap.ts +228 -0
- package/scripts/benchmark-mcp-host.sh +117 -0
- package/scripts/benchmark-mcp-transport.ts +170 -0
- package/scripts/benchmark-quality-harness.ts +146 -0
- package/scripts/benchmark-route-session-concurrency.ts +207 -6
- package/scripts/benchmark-thin-harness-mcp-e2e.ts +7 -1
- package/scripts/benchmark-typescript-navigation.ts +186 -0
- package/scripts/benchmark-work-efficiency.ts +334 -0
- package/scripts/capture-plan.sh +3 -3
- package/scripts/check-agent-tooling.sh +8 -35
- package/scripts/check-architecture-sync.sh +17 -86
- package/scripts/check-mcp-compatibility.ts +14 -0
- package/scripts/check-public-docs.sh +14 -1
- package/scripts/check-runtime-architecture.mjs +107 -64
- package/scripts/check-stable-baseline.ts +92 -0
- package/scripts/check-task-workflow.sh +4 -4
- package/scripts/contract-worktree.sh +1 -21
- package/scripts/ensure-task-workflow.sh +22 -55
- package/scripts/host-temporary-cloud-mcp-e2e.sh +265 -0
- package/scripts/install-standalone-recovery.ts +24 -0
- package/scripts/lib/project-init-lib.sh +15 -44
- package/scripts/migrate-chatgpt-automation-schedules.ts +64 -0
- package/scripts/migrate-project-template.sh +23 -1179
- package/scripts/new-plan.sh +1 -1
- package/scripts/plan-to-todo.sh +11 -36
- package/scripts/run-governed-gate.ts +2 -1
- package/scripts/ship-worktrees.sh +2 -29
- package/scripts/smoke-ios-physical-device.ts +4 -2
- package/scripts/smoke-runtime-recovery.ts +1 -35
- package/scripts/smoke-schedule-engine.ts +1 -20
- package/scripts/sprint-backlog.sh +1 -1
- package/scripts/verify-contract.sh +1 -111
- package/scripts/verify-forge-runtime.sh +0 -1
- package/scripts/verify-sprint.sh +3 -180
- package/scripts/workflow-contract.ts +1 -5
- package/src/cli/commands/chatgpt.ts +31 -2
- package/src/cli/commands/init-hook.ts +3 -3
- package/src/cli/commands/mcp.ts +9 -5
- package/src/cli/commands/plugin.ts +37 -10
- package/src/cli/commands/recovery.ts +25 -3
- package/src/cli/commands/runtime.ts +55 -3
- package/src/cli/commands/security.ts +12 -1
- package/src/cli/commands/setup-profile.ts +31 -21
- package/src/cli/controller/check-runner.ts +106 -3
- package/src/cli/controller/composite-result.ts +1 -1
- package/src/cli/controller/context/focus.ts +51 -0
- package/src/cli/controller/context/instruction-context.ts +114 -0
- package/src/cli/controller/context/known-paths.ts +76 -0
- package/src/cli/controller/context/query-planning.ts +110 -0
- package/src/cli/controller/context/rule-resolution.ts +532 -0
- package/src/cli/controller/context/source-materializer.ts +278 -0
- package/src/cli/controller/context/structural.ts +16 -0
- package/src/cli/controller/context/types.ts +286 -0
- package/src/cli/controller/context-pack.ts +898 -430
- package/src/cli/controller/launch-agents.ts +77 -2
- package/src/cli/controller/operational-plan.ts +2 -2
- package/src/cli/controller/work-mode.ts +6 -4
- package/src/cli/editing/edit-session.ts +60 -7
- package/src/cli/github/github.ts +133 -0
- package/src/cli/hook/prompt-guard-decision.ts +17 -18
- package/src/cli/hook/prompt-intents.ts +1 -1
- package/src/cli/installer/managed-entries.ts +15 -1
- package/src/cli/local-bridge/console-automations.ts +225 -67
- package/src/cli/local-bridge/console-view-models.ts +3 -35
- package/src/cli/local-bridge/facade-api.ts +2 -217
- package/src/cli/local-bridge/frontend/src/App.tsx +3 -3
- package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +153 -3
- package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +37 -4
- package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +183 -3
- package/src/cli/local-bridge/frontend/src/features/repositories/RepositoriesView.tsx +1 -1
- package/src/cli/local-bridge/frontend/src/features/work/WorkView.tsx +4 -3
- package/src/cli/local-bridge/frontend/src/lib/api.ts +18 -11
- package/src/cli/local-bridge/frontend/src/lib/routes.ts +1 -1
- package/src/cli/local-bridge/frontend/src/shell/CommandBar.tsx +1 -1
- package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +4 -3
- package/src/cli/local-bridge/frontend/src/styles.css +27 -11
- package/src/cli/local-bridge/frontend/src/types.ts +8 -7
- package/src/cli/local-bridge/job-store.ts +1 -490
- package/src/cli/local-bridge/server.ts +29 -532
- package/src/cli/local-bridge/ui-dist/app.css +1 -1
- package/src/cli/local-bridge/ui-dist/app.js +5101 -4967
- package/src/cli/mcp/auth.ts +5 -5
- package/src/cli/mcp/instructions.ts +3 -2
- package/src/cli/mcp/legacy-tool-service.ts +0 -189
- package/src/cli/mcp/multi-repository.ts +16 -5
- package/src/cli/mcp/oauth.ts +175 -26
- package/src/cli/mcp/paths.ts +8 -2
- package/src/cli/mcp/repository-tools.ts +356 -101
- package/src/cli/mcp/server.ts +625 -54
- package/src/cli/mcp/tool-capability-cache.ts +1 -1
- package/src/cli/mcp/toolset-selection.ts +1 -1
- package/src/cli/mcp/toolset.ts +42 -5
- package/src/cli/mcp/transports/http.ts +125 -15
- package/src/cli/mcp/transports/session-registry.ts +67 -5
- package/src/cli/mcp/types.ts +3 -3
- package/src/cli/repositories/command-classifier.ts +209 -14
- package/src/cli/repositories/command-executor.ts +55 -521
- package/src/cli/repositories/command-process.ts +108 -0
- package/src/cli/repositories/command-scope.ts +99 -41
- package/src/cli/repositories/controller-home.ts +7 -8
- package/src/cli/repositories/git-command-executor.ts +2 -0
- package/src/cli/repositories/locks.ts +53 -16
- package/src/cli/repositories/repository-snapshot.ts +121 -0
- package/src/cli/repositories/safe-patch.ts +6 -4
- package/src/cli/repositories/selected-path-actions.ts +17 -9
- package/src/cli/repositories/structured-git.ts +110 -10
- package/src/cli/repository/inspector.ts +565 -62
- package/src/cli/repository/session-cache.ts +58 -17
- package/src/cli/v81-entry.ts +1 -4
- package/src/effects/fs-transaction.ts +85 -17
- package/src/runtime/context/codegraph-read-provider.ts +233 -2
- package/src/runtime/context/codegraph-sidecar.cjs +74 -40
- package/src/runtime/context/generic-lsp-provider.ts +355 -0
- package/src/runtime/context/lsp-client.ts +283 -0
- package/src/runtime/context/semantic-navigation.ts +275 -0
- package/src/runtime/context/swift-navigation.ts +565 -0
- package/src/runtime/context/typescript-navigation.ts +226 -0
- package/src/runtime/control-plane/dispatch-priority.ts +28 -0
- package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +196 -2
- package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +86 -24
- package/src/runtime/control-plane/execution/execution-identity.ts +65 -14
- package/src/runtime/control-plane/execution/session-store.ts +18 -0
- package/src/runtime/control-plane/execution/validation.ts +15 -2
- package/src/runtime/control-plane/execution/verification-evidence.ts +58 -3
- package/src/runtime/control-plane/execution/work-handle-store.ts +27 -2
- package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +258 -0
- package/src/runtime/control-plane/execution/work-remote-delivery.ts +215 -0
- package/src/runtime/control-plane/execution/work-task-receipt.ts +54 -11
- package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +175 -8
- package/src/runtime/control-plane/execution/work-validation-reconciler.ts +28 -21
- package/src/runtime/control-plane/execution/work-verification-snapshot.ts +87 -9
- package/src/runtime/control-plane/facade/capability-registry.ts +129 -25
- package/src/runtime/control-plane/facade/controller-round-relay.ts +991 -0
- package/src/runtime/control-plane/facade/controller-session-store.ts +144 -14
- package/src/runtime/control-plane/facade/goal-workloop-access.ts +9 -2
- package/src/runtime/control-plane/facade/goal-workloop.ts +885 -105
- package/src/runtime/control-plane/facade/index.ts +1 -0
- package/src/runtime/control-plane/facade/plan-contract-store.ts +211 -26
- package/src/runtime/control-plane/facade/requirement-board.ts +24 -8
- package/src/runtime/control-plane/facade/semantic-admission.ts +205 -0
- package/src/runtime/control-plane/facade/suggested-actions.ts +5 -1
- package/src/runtime/control-plane/facade/types.ts +55 -11
- package/src/runtime/control-plane/facade/work-contract-store.ts +204 -73
- package/src/runtime/control-plane/generated-cache-retention.ts +271 -0
- package/src/runtime/control-plane/global-scheduler/config.ts +44 -0
- package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts +80 -0
- package/src/runtime/control-plane/global-scheduler/durable-admission.ts +46 -0
- package/src/runtime/control-plane/global-scheduler/maintenance.ts +145 -0
- package/src/runtime/control-plane/global-scheduler/projection-refresh.ts +84 -0
- package/src/runtime/control-plane/global-scheduler/reconciliation.ts +3 -3
- package/src/runtime/control-plane/global-scheduler/scheduler.ts +273 -542
- package/src/runtime/control-plane/global-scheduler/source-scan.ts +59 -0
- package/src/runtime/control-plane/global-scheduler/state.ts +88 -0
- package/src/runtime/control-plane/global-scheduler/worker-attachment.ts +57 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts +38 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts +175 -0
- package/src/runtime/control-plane/global-scheduler/worker-launch.ts +111 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts +64 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts +150 -0
- package/src/runtime/control-plane/global-scheduler/worker-process.ts +119 -0
- package/src/runtime/control-plane/global-scheduler/worker-stderr.ts +60 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +50 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +642 -51
- package/src/runtime/control-plane/launcher/launch-reservation-store.ts +40 -0
- package/src/runtime/control-plane/launcher/thin-launcher.ts +194 -10
- package/src/runtime/control-plane/persistence/controller-home-migration.ts +487 -0
- package/src/runtime/control-plane/persistence/requirement-store.ts +15 -11
- package/src/runtime/control-plane/persistence/sqlite-store.ts +33 -2
- package/src/runtime/control-plane/release-retention.ts +62 -2
- package/src/runtime/control-plane/routing/route-policy.ts +7 -16
- package/src/runtime/control-plane/runtime-status-client.ts +0 -2
- package/src/runtime/diagnostics/mcp-timing.ts +19 -0
- package/src/runtime/diagnostics/mcp-transport-benchmark.ts +129 -0
- package/src/runtime/diagnostics/performance.ts +24 -1
- package/src/runtime/evidence/event-ledger.ts +1 -1
- package/src/runtime/execution/jobs/store.ts +58 -4
- package/src/runtime/execution/jobs/types.ts +1 -1
- package/src/runtime/execution/managed-workspace.ts +203 -1
- package/src/runtime/execution/process-runtime/check-facade.ts +52 -14
- package/src/runtime/execution/process-runtime/check-result.ts +63 -2
- package/src/runtime/execution/process-runtime/check-scheduling.ts +114 -0
- package/src/runtime/execution/process-runtime/command-facade.ts +346 -40
- package/src/runtime/execution/process-runtime/gc.ts +98 -12
- package/src/runtime/execution/process-runtime/lightweight-managed.ts +1184 -0
- package/src/runtime/execution/process-runtime/persisted-check.ts +57 -6
- package/src/runtime/execution/process-runtime/resource-claims.ts +97 -8
- package/src/runtime/execution/process-runtime/runtime.ts +59 -4
- package/src/runtime/execution/process-runtime/store.ts +7 -0
- package/src/runtime/execution/process-runtime/types.ts +2 -0
- package/src/runtime/execution/thin-harness/async-process.ts +12 -2
- package/src/runtime/execution/thin-harness/search-worker.ts +54 -5
- package/src/runtime/execution/workers/executor.ts +0 -48
- package/src/runtime/gateway/mcp/execution-tools.ts +1219 -126
- package/src/runtime/gateway/mcp/process-tools.ts +83 -37
- package/src/runtime/gateway/mcp/router.ts +185 -83
- package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +546 -0
- package/src/runtime/gateway/mcp/runtime-tools.ts +2555 -1616
- package/src/runtime/model-clients/deepseek-adapter.ts +0 -32
- package/src/runtime/plugins/app-store-connect-adapter.ts +123 -19
- package/src/runtime/plugins/browser-adapter.ts +2925 -318
- package/src/runtime/plugins/browser-automation-service.ts +10 -0
- package/src/runtime/plugins/browser-handoff-host.ts +6 -31
- package/src/runtime/plugins/browser-handoff.ts +92 -5
- package/src/runtime/plugins/browser-macos-bridge.ts +586 -45
- package/src/runtime/plugins/browser-node-bridge.ts +6 -34
- package/src/runtime/plugins/browser-provider-registry.ts +109 -0
- package/src/runtime/plugins/browser-runtime-contract.ts +152 -0
- package/src/runtime/plugins/browser-runtime.ts +183 -0
- package/src/runtime/plugins/browser-session-authority.ts +332 -0
- package/src/runtime/plugins/capability-authorization-grants.ts +354 -0
- package/src/runtime/plugins/desktop-operator-registration.ts +15 -12
- package/src/runtime/plugins/external-adapter.ts +324 -3
- package/src/runtime/plugins/external-registration.ts +84 -5
- package/src/runtime/plugins/external-unix-socket.ts +7 -1
- package/src/runtime/plugins/first-party-registry.ts +4 -2
- package/src/runtime/plugins/github-adapter.ts +125 -0
- package/src/runtime/plugins/gmail-adapter.ts +28 -2
- package/src/runtime/plugins/ios/agent-device-provider.ts +1 -1
- package/src/runtime/plugins/ios/agent-device-typed-provider.ts +52 -9
- package/src/runtime/plugins/ios/remote-xpc-hid.ts +1171 -0
- package/src/runtime/plugins/ios-adapter.ts +106 -16
- package/src/runtime/plugins/ios-agent-device.ts +234 -64
- package/src/runtime/plugins/ios-physical-device.ts +1143 -392
- package/src/runtime/plugins/lightweight-action.ts +153 -0
- package/src/runtime/plugins/local-system-adapter.ts +125 -8
- package/src/runtime/plugins/macos-capability-broker.ts +55 -9
- package/src/runtime/plugins/plugin-action-sidecar.ts +59 -0
- package/src/runtime/plugins/plugin-management-adapter.ts +181 -0
- package/src/runtime/plugins/resend-adapter.ts +1 -1
- package/src/runtime/plugins/store.ts +366 -24
- package/src/runtime/plugins/types.ts +39 -2
- package/src/runtime/plugins/xiaohongshu-publish.ts +515 -0
- package/src/runtime/projections/git-status-sampler.ts +16 -16
- package/src/runtime/projections/materialized-view.ts +28 -10
- package/src/runtime/recovery/actions.ts +0 -10
- package/src/runtime/recovery/classifier.ts +0 -9
- package/src/runtime/recovery/maintenance-executor.ts +415 -18
- package/src/runtime/recovery/probe.ts +0 -1
- package/src/runtime/recovery/types.ts +0 -1
- package/src/runtime/root/mcp-transport.ts +180 -10
- package/src/runtime/root/package-connector-service.ts +186 -8
- package/src/runtime/root/package-runtime-release.ts +143 -17
- package/src/runtime/root/package-runtime-service.ts +430 -22
- package/src/runtime/root/release-manifest.ts +7 -2
- package/src/runtime/root/release-materialize.ts +162 -19
- package/src/runtime/root/release-store.ts +48 -11
- package/src/runtime/root/runtime.ts +23 -5
- package/src/runtime/root/service.ts +100 -27
- package/src/runtime/root/types.ts +2 -0
- package/src/runtime/safe-tooling/index.ts +0 -1
- package/src/runtime/safe-tooling/ios-development.ts +321 -63
- package/src/runtime/safe-tooling/types.ts +0 -30
- package/src/runtime/shared/trusted-executable.ts +62 -0
- package/src/runtime/shared/trusted-node-executable.ts +48 -0
- package/src/runtime/standalone-recovery/core.ts +702 -94
- package/src/runtime/standalone-recovery/entry.ts +99 -23
- package/src/runtime/standalone-recovery/installer.ts +19 -3
- package/src/runtime/standalone-recovery/service-labels.ts +2 -0
- package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +160 -0
- package/src/runtime/workflow/schedules/browser-probe.ts +258 -0
- package/src/runtime/workflow/schedules/chatgpt-automation-migration.ts +40 -0
- package/src/runtime/workflow/schedules/engine.ts +509 -80
- package/src/runtime/workflow/schedules/github-issue-watch.ts +158 -0
- package/src/runtime/workflow/schedules/settlement.ts +48 -4
- package/src/runtime/workflow/schedules/store.ts +131 -10
- package/src/runtime/workflow/schedules/types.ts +6 -3
- package/src/runtime/workflow/schedules/work-continuation.ts +120 -34
- package/src/testing/test-governance.ts +8 -40
- package/assets/reference-configs/heartbeat-triage.md +0 -57
- package/assets/templates/helpers/harness-trace-grade.sh +0 -150
- package/assets/templates/helpers/heartbeat-triage.sh +0 -361
- package/assets/templates/helpers/maintenance-triage.sh +0 -167
- package/bin/forge-desktop-helper.mjs +0 -204
- package/docs/architecture/current/approved-target-architecture.zh-CN.md +0 -1501
- package/docs/architecture/current/architecture-invariants.md +0 -331
- package/docs/architecture/current/automation-and-schedule-engine.md +0 -453
- package/docs/architecture/current/codegraph-context-provider.md +0 -226
- package/docs/architecture/current/control-plane-authority-inventory.md +0 -94
- package/docs/architecture/current/controller-repository-decoupling.md +0 -148
- package/docs/architecture/current/dispatch-and-agent-strategy.md +0 -354
- package/docs/architecture/current/entity-model.md +0 -592
- package/docs/architecture/current/external-plugin-governance.md +0 -396
- package/docs/architecture/current/failure-recovery.md +0 -447
- package/docs/architecture/current/governance.md +0 -213
- package/docs/architecture/current/human-interaction-plane.md +0 -87
- package/docs/architecture/current/implementation-status.md +0 -128
- package/docs/architecture/current/job-and-run-lifecycle.md +0 -413
- package/docs/architecture/current/legacy-governance.md +0 -58
- package/docs/architecture/current/local-workspace-target-grants.md +0 -70
- package/docs/architecture/current/migration-roadmap.md +0 -105
- package/docs/architecture/current/multi-repository-execution.md +0 -371
- package/docs/architecture/current/operations-runbook.md +0 -45
- package/docs/architecture/current/peer-supercontroller-runtime.md +0 -32
- package/docs/architecture/current/personal-assistant-plugin-baseline.md +0 -206
- package/docs/architecture/current/repository-access-modes.md +0 -102
- package/docs/architecture/current/route-policy.md +0 -15
- package/docs/architecture/current/route-session-concurrency.md +0 -85
- package/docs/architecture/current/runtime-architecture-simplification.md +0 -214
- package/docs/architecture/current/runtime-directory-map.md +0 -49
- package/docs/architecture/current/runtime-health-and-resource-lifecycle.md +0 -85
- package/docs/architecture/current/scheduler-and-resource-claims.md +0 -422
- package/docs/architecture/current/session-aware-execution-and-authorization.md +0 -77
- package/docs/architecture/current/system-overview.md +0 -144
- package/docs/architecture/current/target-requirements-traceability.md +0 -202
- package/docs/architecture/current/thin-harness-governance-policy.md +0 -32
- package/docs/architecture/current/thin-harness-v1.md +0 -311
- package/docs/architecture/current/verification-and-release-gates.md +0 -391
- package/docs/forge-autonomous-goal-loop.md +0 -9
- package/docs/forge-provider-configuration.md +0 -7
- package/docs/operations/chatgpt-personal-assistant-runtime.md +0 -83
- package/docs/operations/controller-desktop-plugin.md +0 -112
- package/docs/operations/gmail-assistant-routines.md +0 -186
- package/docs/operations/personal-assistant-operator-guide.md +0 -61
- package/docs/operations/provider-configuration.md +0 -31
- package/scripts/harness-trace-grade.sh +0 -150
- package/scripts/heartbeat-triage.sh +0 -361
- package/scripts/maintenance-triage.sh +0 -167
- package/src/cli/local-bridge/frontend/src/features/settings/SettingsView.tsx +0 -2
- package/src/cli/repositories/goal-registry.ts +0 -239
- package/src/runtime/assistant/action-proposals.ts +0 -279
- package/src/runtime/assistant/intent.ts +0 -395
- package/src/runtime/assistant/model-provider.ts +0 -460
- package/src/runtime/assistant/openapi.ts +0 -223
- package/src/runtime/assistant/policy.ts +0 -89
- package/src/runtime/assistant/readiness.ts +0 -258
- package/src/runtime/assistant/routine-runtime.ts +0 -477
- package/src/runtime/assistant/schedule-binding.ts +0 -171
- package/src/runtime/assistant/standing-grants.ts +0 -375
- package/src/runtime/assistant/store.ts +0 -243
- package/src/runtime/assistant/types.ts +0 -135
- package/src/runtime/control-plane/goal-loop/config-facade.ts +0 -700
- package/src/runtime/control-plane/goal-loop/config-store.ts +0 -570
- package/src/runtime/control-plane/goal-loop/config-types.ts +0 -213
- package/src/runtime/control-plane/goal-loop/executor-router.ts +0 -109
- package/src/runtime/control-plane/goal-loop/failure-taxonomy.ts +0 -149
- package/src/runtime/control-plane/goal-loop/goal-contract-store.ts +0 -304
- package/src/runtime/control-plane/goal-loop/goal-loop-engine.ts +0 -1001
- package/src/runtime/control-plane/goal-loop/handoff-packet.ts +0 -160
- package/src/runtime/control-plane/goal-loop/index.ts +0 -13
- package/src/runtime/control-plane/goal-loop/local-tools.ts +0 -290
- package/src/runtime/control-plane/goal-loop/policy-gates.ts +0 -203
- package/src/runtime/control-plane/goal-loop/provider-dispatch.ts +0 -93
- package/src/runtime/control-plane/goal-loop/provider-registry.ts +0 -545
- package/src/runtime/control-plane/goal-loop/types.ts +0 -378
- package/src/runtime/control-plane/startup-recovery.ts +0 -198
- package/src/runtime/gateway/mcp/resource-policy.ts +0 -186
- package/src/runtime/personal-assistant/gmail-triage-manager.ts +0 -231
- package/src/runtime/personal-assistant/reporting-runtime.ts +0 -344
- package/src/runtime/personal-assistant/triage-runtime.ts +0 -748
- package/src/runtime/plugins/desktop-adapter.ts +0 -374
- package/src/runtime/safe-tooling/browser-safe.ts +0 -133
- package/src/runtime/workflow/findings/store.ts +0 -204
- package/src/runtime/workflow/findings/types.ts +0 -58
- package/src/runtime/workflow/portfolio/engine.ts +0 -20
- package/src/runtime/workflow/portfolio/store.ts +0 -185
- package/src/runtime/workflow/portfolio/types.ts +0 -29
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
import { execFile,
|
|
2
|
-
import {
|
|
3
|
-
import { fileURLToPath } from 'url';
|
|
4
|
-
import { dirname, join, resolve } from 'path';
|
|
1
|
+
import { execFile, type ChildProcess } from 'child_process';
|
|
2
|
+
import { resolve } from 'path';
|
|
5
3
|
import { cpus, freemem, loadavg } from 'os';
|
|
6
4
|
import { listRepositories } from '../../../cli/repositories/registry';
|
|
7
|
-
import { resolveControllerHome } from '../../../cli/repositories/controller-home';
|
|
8
5
|
import { writeAgentExecutableReadinessSnapshot } from '../../../cli/agent-jobs/executable-resolver';
|
|
9
6
|
import { withControllerLock } from '../../../cli/repositories/locks';
|
|
10
7
|
import {
|
|
@@ -13,25 +10,13 @@ import {
|
|
|
13
10
|
runtimeWriteClaimEnvironment,
|
|
14
11
|
} from '../../root/write-fence';
|
|
15
12
|
import {
|
|
16
|
-
attachExecutionWorker,
|
|
17
|
-
executionJobRoot,
|
|
18
13
|
getExecutionJob,
|
|
19
14
|
listActiveExecutionJobs,
|
|
20
|
-
markExecutionJobSchedulerObserved,
|
|
21
|
-
transitionExecutionJob,
|
|
22
|
-
updateExecutionJob,
|
|
23
15
|
} from '../../execution/jobs/store';
|
|
24
|
-
import type
|
|
25
|
-
import { tryRecoverJobFromWorkerReceipt } from '../../execution/jobs/receipt-recovery';
|
|
26
|
-
import { releaseExecutionLeases } from '../../resources/leases/store';
|
|
16
|
+
import { type ExecutionWorkerLifecycle } from '../../execution/jobs/types';
|
|
27
17
|
import { RepoActorRegistry } from '../repo-actor/registry';
|
|
28
18
|
import { reconcileExecutionJobsAsync } from './reconciliation';
|
|
29
|
-
import {
|
|
30
|
-
import { tickPortfolioWorkflows } from '../../workflow/portfolio/engine';
|
|
31
|
-
import { tickGoalLoopsForController } from '../goal-loop';
|
|
32
|
-
import { readJsonFile, writeJsonAtomic } from '../../shared/json-files';
|
|
33
|
-
import { resolveBunExecutable } from '../../shared/process-environment';
|
|
34
|
-
import { isProcessAlive, terminateProcessTree } from '../../shared/process-tree';
|
|
19
|
+
import { isProcessAlive } from '../../shared/process-tree';
|
|
35
20
|
import { readSchedulerWakeSignal, waitForSchedulerWakeSignal } from './wake-signal';
|
|
36
21
|
import { cleanupControllerRuntimeState } from '../runtime-cleanup';
|
|
37
22
|
import { reconcileTerminalWorkCleanups } from '../execution/work-terminal-cleanup';
|
|
@@ -39,39 +24,73 @@ import { gcTerminalProcesses } from '../../execution/process-runtime/gc';
|
|
|
39
24
|
import { reconcilePendingWorkValidations } from '../execution/work-validation-reconciler';
|
|
40
25
|
import { reconcilePendingEditValidations } from '../execution/edit-validation-coordinator';
|
|
41
26
|
import { schedulerDispatchAllowed } from '../facade/work-admission-policy';
|
|
42
|
-
import { rebuildRepositoryProjection, refreshRepositoryProjectionForRepository } from '../../projections/materialized-view';
|
|
43
|
-
import { readRepositoryGitStatusSample, sampleRepositoryGitStatusForRepositories } from '../../projections/git-status-sampler';
|
|
44
27
|
import {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
} from '
|
|
28
|
+
runSchedulerControllerRoundRecovery,
|
|
29
|
+
runSchedulerPeriodicCleanup,
|
|
30
|
+
runSchedulerValidationReconciliation,
|
|
31
|
+
} from './maintenance';
|
|
32
|
+
import { planSchedulerSourceSampling } from './source-scan';
|
|
33
|
+
import { runSchedulerDurableAdmission } from './durable-admission';
|
|
34
|
+
import { sampleRepositoryGitStatusForRepositories } from '../../projections/git-status-sampler';
|
|
35
|
+
import { selectExecutionJobDispatchRepositories } from '../dispatch-priority';
|
|
36
|
+
import {
|
|
37
|
+
buildSchedulerWorkerLaunchDescriptor,
|
|
38
|
+
resolveSchedulerWorkerCommand,
|
|
39
|
+
selectSchedulerWorkerEnvironment,
|
|
40
|
+
} from './worker-launch';
|
|
41
|
+
import {
|
|
42
|
+
consumeSchedulerDispatchCapacity,
|
|
43
|
+
createSchedulerDispatchCapacity,
|
|
44
|
+
schedulerDispatchCapacityAllows,
|
|
45
|
+
} from './dispatch-capacity';
|
|
46
|
+
import {
|
|
47
|
+
refreshSchedulerRepositoryProjections,
|
|
48
|
+
selectSchedulerDirtyProjectionRepositories,
|
|
49
|
+
} from './projection-refresh';
|
|
50
|
+
import { createSchedulerWorkerStderrCapture } from './worker-stderr';
|
|
51
|
+
import { persistSchedulerWorkerAttachment } from './worker-attachment';
|
|
52
|
+
import {
|
|
53
|
+
cleanupSchedulerWorkerProcesses,
|
|
54
|
+
registerSchedulerWorkerProcess,
|
|
55
|
+
spawnSchedulerWorkerProcess,
|
|
56
|
+
wireSchedulerWorkerProcess,
|
|
57
|
+
} from './worker-process';
|
|
58
|
+
import { reconcileSchedulerWorkerExit } from './worker-exit-reconciler';
|
|
59
|
+
import { persistSchedulerSpawnedWorkerLifecycle } from './worker-lifecycle-store';
|
|
60
|
+
import {
|
|
61
|
+
buildSchedulerWorkerExitedLifecycle,
|
|
62
|
+
buildSchedulerWorkerSpawnFailureLifecycle,
|
|
63
|
+
buildSchedulerWorkerSpawnedLifecycle,
|
|
64
|
+
} from './worker-lifecycle';
|
|
65
|
+
import { normalizeSchedulerConfig, type SchedulerConfig } from './config';
|
|
66
|
+
import {
|
|
67
|
+
readSchedulerHealthSnapshot,
|
|
68
|
+
restoreSchedulerState,
|
|
69
|
+
writeSchedulerHealthSnapshot,
|
|
70
|
+
} from './state';
|
|
71
|
+
export { normalizeSchedulerConfig } from './config';
|
|
72
|
+
export type { SchedulerConfig } from './config';
|
|
73
|
+
export {
|
|
74
|
+
buildSchedulerHealthSnapshot,
|
|
75
|
+
readSchedulerHealthSnapshot,
|
|
76
|
+
restoreSchedulerState,
|
|
77
|
+
} from './state';
|
|
78
|
+
export type {
|
|
79
|
+
SchedulerHealthSnapshot,
|
|
80
|
+
SchedulerRestoredState,
|
|
81
|
+
SchedulerStateSnapshotInput,
|
|
82
|
+
} from './state';
|
|
83
|
+
export {
|
|
84
|
+
buildSchedulerWorkerLaunchDescriptor,
|
|
85
|
+
resolveSchedulerWorkerCommand,
|
|
86
|
+
resolveSchedulerWorkerExecutable,
|
|
87
|
+
selectSchedulerWorkerEnvironment,
|
|
88
|
+
} from './worker-launch';
|
|
89
|
+
export type { SchedulerWorkerCommand, SchedulerWorkerLaunchDescriptor } from './worker-launch';
|
|
90
|
+
export { selectSchedulerSourceScanRepositories } from './source-scan';
|
|
49
91
|
|
|
50
92
|
const DARWIN_MEMORY_SAMPLE_TTL_MS = 5_000;
|
|
51
|
-
const MAX_WORKER_STDERR_BYTES = 16 * 1024;
|
|
52
|
-
const WORKER_ENVIRONMENT_KEYS = [
|
|
53
|
-
'PATH',
|
|
54
|
-
'HOME',
|
|
55
|
-
'BUN_INSTALL',
|
|
56
|
-
'NODE_OPTIONS',
|
|
57
|
-
'FORGE_CONTROLLER_HOME',
|
|
58
|
-
'FORGE_CONTROLLER_RUNTIME_SOURCE_ROOT',
|
|
59
|
-
'FORGE_EXECUTION_WORKER',
|
|
60
|
-
'FORGE_RUNTIME_INSTANCE_ID',
|
|
61
|
-
'FORGE_RUNTIME_OWNER_PID',
|
|
62
|
-
'FORGE_RELEASE_AUTHORITY_REVISION',
|
|
63
|
-
// The release fencing token is passed to the actual child environment below,
|
|
64
|
-
// but deliberately excluded from the persisted lifecycle diagnostic.
|
|
65
|
-
'FORGE_RELEASE_ID',
|
|
66
|
-
'FORGE_ARTIFACT_IDENTITY',
|
|
67
|
-
'FORGE_WORKER_PROTOCOL_VERSION',
|
|
68
|
-
] as const;
|
|
69
93
|
const RUNTIME_CLEANUP_INTERVAL_MS = Math.max(30_000, Number(process.env.FORGE_RUNTIME_CLEANUP_INTERVAL_MS ?? 60_000));
|
|
70
|
-
const GIT_STATUS_SAMPLE_INTERVAL_MS = Math.max(1_000, Number(process.env.FORGE_GIT_STATUS_SAMPLE_INTERVAL_MS ?? 5_000));
|
|
71
|
-
const IDLE_REPOSITORY_SCAN_INTERVAL_MS = Math.max(
|
|
72
|
-
GIT_STATUS_SAMPLE_INTERVAL_MS,
|
|
73
|
-
Number(process.env.FORGE_IDLE_REPOSITORY_SCAN_INTERVAL_MS ?? 60_000),
|
|
74
|
-
);
|
|
75
94
|
const DARWIN_RECLAIMABLE_PAGE_LABELS = new Set([
|
|
76
95
|
'Pages free',
|
|
77
96
|
'Pages inactive',
|
|
@@ -132,64 +151,6 @@ export function sampleDarwinAvailableMemoryMb(
|
|
|
132
151
|
});
|
|
133
152
|
}
|
|
134
153
|
|
|
135
|
-
interface SchedulerState {
|
|
136
|
-
schemaVersion: 1;
|
|
137
|
-
updatedAt: string;
|
|
138
|
-
loopStartedAt?: string;
|
|
139
|
-
lastHeartbeatAt?: string;
|
|
140
|
-
heartbeatTimeoutMs?: number;
|
|
141
|
-
lastTickAt?: string;
|
|
142
|
-
lastDispatchAt?: string;
|
|
143
|
-
lastReconcileAt?: string;
|
|
144
|
-
lastSourceScanAt?: string;
|
|
145
|
-
lastSourceScanRepoCount?: number;
|
|
146
|
-
sourceScansAvoided?: number;
|
|
147
|
-
lastRepoDispatch: Record<string, number>;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
function schedulerStatePath(controllerHome: string): string {
|
|
151
|
-
return join(resolveControllerHome(controllerHome), 'scheduler', 'state.json');
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
export interface SchedulerHealthSnapshot {
|
|
155
|
-
schemaVersion: 1;
|
|
156
|
-
updatedAt: string;
|
|
157
|
-
loopStartedAt?: string;
|
|
158
|
-
lastHeartbeatAt?: string;
|
|
159
|
-
heartbeatTimeoutMs?: number;
|
|
160
|
-
lastTickAt?: string;
|
|
161
|
-
lastDispatchAt?: string;
|
|
162
|
-
lastReconcileAt?: string;
|
|
163
|
-
lastSourceScanAt?: string;
|
|
164
|
-
lastSourceScanRepoCount?: number;
|
|
165
|
-
sourceScansAvoided?: number;
|
|
166
|
-
lastRepoDispatch: Record<string, number>;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
export function readSchedulerHealthSnapshot(controllerHome: string): SchedulerHealthSnapshot {
|
|
170
|
-
return readJsonFile<SchedulerHealthSnapshot>(schedulerStatePath(controllerHome), {
|
|
171
|
-
schemaVersion: 1,
|
|
172
|
-
updatedAt: new Date().toISOString(),
|
|
173
|
-
lastRepoDispatch: {},
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
export interface SchedulerConfig {
|
|
178
|
-
maxWorkers: number;
|
|
179
|
-
maxConcurrentRepositories: number;
|
|
180
|
-
pollIntervalMs: number;
|
|
181
|
-
idleBackoffMaxMs: number;
|
|
182
|
-
heartbeatIntervalMs: number;
|
|
183
|
-
heartbeatTimeoutMs: number;
|
|
184
|
-
maxHeavyChecks: number;
|
|
185
|
-
maxAgentProcesses: number;
|
|
186
|
-
maxCodexProcesses: number;
|
|
187
|
-
maxClaudeProcesses: number;
|
|
188
|
-
maxGitHubProcesses: number;
|
|
189
|
-
minFreeMemoryMb: number;
|
|
190
|
-
maxLoadPerCpu: number;
|
|
191
|
-
}
|
|
192
|
-
|
|
193
154
|
export interface SchedulerRuntimeBinding {
|
|
194
155
|
controllerPid?: number;
|
|
195
156
|
runtimeSourceRoot?: string;
|
|
@@ -198,14 +159,6 @@ export interface SchedulerRuntimeBinding {
|
|
|
198
159
|
fatalOnTickError?: boolean;
|
|
199
160
|
}
|
|
200
161
|
|
|
201
|
-
export function resolveSchedulerWorkerExecutable(
|
|
202
|
-
isBun: boolean = Boolean(process.versions.bun),
|
|
203
|
-
execPath: string = process.execPath,
|
|
204
|
-
env: NodeJS.ProcessEnv = process.env,
|
|
205
|
-
): string {
|
|
206
|
-
return isBun ? resolveBunExecutable(execPath, env) : execPath;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
162
|
export class GlobalScheduler {
|
|
210
163
|
private readonly controllerHome: string;
|
|
211
164
|
private readonly actors: RepoActorRegistry;
|
|
@@ -216,8 +169,6 @@ export class GlobalScheduler {
|
|
|
216
169
|
private readonly workerEntrypoint?: string;
|
|
217
170
|
private readonly fatalOnTickError: boolean;
|
|
218
171
|
private lastScheduleTick = 0;
|
|
219
|
-
private lastPortfolioTick = 0;
|
|
220
|
-
private lastGoalLoopTick = 0;
|
|
221
172
|
private lastReconcile = 0;
|
|
222
173
|
private lastPersistedAt = 0;
|
|
223
174
|
private readonly lastRepoDispatch = new Map<string, number>();
|
|
@@ -249,98 +200,39 @@ export class GlobalScheduler {
|
|
|
249
200
|
this.controllerHome = controllerHome;
|
|
250
201
|
this.controllerPid = runtime.controllerPid ?? process.pid;
|
|
251
202
|
this.actors = new RepoActorRegistry(controllerHome);
|
|
252
|
-
|
|
253
|
-
const idleBackoffMaxMs = Math.max(250, config.idleBackoffMaxMs ?? Number(process.env.FORGE_IDLE_BACKOFF_MAX_MS ?? 2_000));
|
|
254
|
-
const heartbeatIntervalMs = Math.max(25, config.heartbeatIntervalMs ?? Number(process.env.FORGE_SCHEDULER_HEARTBEAT_INTERVAL_MS ?? 1_000));
|
|
255
|
-
const heartbeatTimeoutMs = Math.max(
|
|
256
|
-
heartbeatIntervalMs * 4,
|
|
257
|
-
idleBackoffMaxMs * 4,
|
|
258
|
-
config.heartbeatTimeoutMs ?? Number(process.env.FORGE_SCHEDULER_HEARTBEAT_TIMEOUT_MS ?? 60_000),
|
|
259
|
-
);
|
|
260
|
-
this.config = {
|
|
261
|
-
maxWorkers: Math.max(1, config.maxWorkers ?? Number(process.env.FORGE_MAX_WORKERS ?? 4)),
|
|
262
|
-
maxConcurrentRepositories: Math.max(1, config.maxConcurrentRepositories ?? Number(process.env.FORGE_MAX_ACTIVE_REPOS ?? 4)),
|
|
263
|
-
pollIntervalMs,
|
|
264
|
-
idleBackoffMaxMs,
|
|
265
|
-
heartbeatIntervalMs,
|
|
266
|
-
heartbeatTimeoutMs,
|
|
267
|
-
maxHeavyChecks: Math.max(1, config.maxHeavyChecks ?? Number(process.env.FORGE_MAX_HEAVY_CHECKS ?? 2)),
|
|
268
|
-
maxAgentProcesses: Math.max(1, config.maxAgentProcesses ?? Number(process.env.FORGE_MAX_AGENT_PROCESSES ?? 4)),
|
|
269
|
-
maxCodexProcesses: Math.max(1, config.maxCodexProcesses ?? Number(process.env.FORGE_MAX_CODEX_PROCESSES ?? 3)),
|
|
270
|
-
maxClaudeProcesses: Math.max(1, config.maxClaudeProcesses ?? Number(process.env.FORGE_MAX_CLAUDE_PROCESSES ?? 2)),
|
|
271
|
-
maxGitHubProcesses: Math.max(1, config.maxGitHubProcesses ?? Number(process.env.FORGE_MAX_GITHUB_PROCESSES ?? 2)),
|
|
272
|
-
minFreeMemoryMb: Math.max(64, config.minFreeMemoryMb ?? Number(process.env.FORGE_MIN_FREE_MEMORY_MB ?? 512)),
|
|
273
|
-
maxLoadPerCpu: Math.max(0.25, config.maxLoadPerCpu ?? Number(process.env.FORGE_MAX_LOAD_PER_CPU ?? 1.5)),
|
|
274
|
-
};
|
|
203
|
+
this.config = normalizeSchedulerConfig(config);
|
|
275
204
|
this.runtimeSourceRoot = runtime.runtimeSourceRoot ? resolve(runtime.runtimeSourceRoot) : undefined;
|
|
276
205
|
this.workerEntrypoint = runtime.workerEntrypoint ? resolve(runtime.workerEntrypoint) : undefined;
|
|
277
206
|
this.fatalOnTickError = runtime.fatalOnTickError === true;
|
|
278
|
-
const
|
|
279
|
-
this.lastSourceScanAt =
|
|
280
|
-
this.lastSourceScanRepoCount =
|
|
281
|
-
this.sourceScansAvoided =
|
|
282
|
-
for (const [repoId, timestamp] of
|
|
283
|
-
if (Number.isFinite(timestamp)) this.lastRepoDispatch.set(repoId, timestamp);
|
|
284
|
-
}
|
|
207
|
+
const restoredState = restoreSchedulerState(readSchedulerHealthSnapshot(controllerHome));
|
|
208
|
+
this.lastSourceScanAt = restoredState.lastSourceScanAt;
|
|
209
|
+
this.lastSourceScanRepoCount = restoredState.lastSourceScanRepoCount;
|
|
210
|
+
this.sourceScansAvoided = restoredState.sourceScansAvoided;
|
|
211
|
+
for (const [repoId, timestamp] of restoredState.lastRepoDispatch) this.lastRepoDispatch.set(repoId, timestamp);
|
|
285
212
|
}
|
|
286
213
|
|
|
287
214
|
private persistState(force = false): void {
|
|
288
215
|
const now = Date.now();
|
|
289
216
|
if (!force && now - this.lastPersistedAt < 1_000) return;
|
|
290
217
|
this.lastPersistedAt = now;
|
|
291
|
-
|
|
292
|
-
schemaVersion: 1,
|
|
293
|
-
updatedAt: new Date().toISOString(),
|
|
218
|
+
writeSchedulerHealthSnapshot(this.controllerHome, {
|
|
294
219
|
loopStartedAt: this.loopStartedAt,
|
|
295
220
|
lastHeartbeatAt: this.lastHeartbeatAt,
|
|
296
221
|
heartbeatTimeoutMs: this.config.heartbeatTimeoutMs,
|
|
297
222
|
lastTickAt: this.lastTickAt,
|
|
298
223
|
lastDispatchAt: this.lastDispatchAt,
|
|
299
224
|
lastReconcileAt: this.lastReconcileAt,
|
|
300
|
-
lastSourceScanAt: this.lastSourceScanAt
|
|
225
|
+
lastSourceScanAt: this.lastSourceScanAt,
|
|
301
226
|
lastSourceScanRepoCount: this.lastSourceScanRepoCount,
|
|
302
227
|
sourceScansAvoided: this.sourceScansAvoided,
|
|
303
|
-
lastRepoDispatch:
|
|
304
|
-
}
|
|
228
|
+
lastRepoDispatch: this.lastRepoDispatch,
|
|
229
|
+
}, now);
|
|
305
230
|
}
|
|
306
231
|
|
|
307
232
|
private pidAlive(pid: number | undefined): boolean {
|
|
308
233
|
return isProcessAlive(pid);
|
|
309
234
|
}
|
|
310
235
|
|
|
311
|
-
private async cleanupSpawnedWorkers(): Promise<void> {
|
|
312
|
-
const workers = [...this.children.values()];
|
|
313
|
-
this.children.clear();
|
|
314
|
-
await Promise.all(workers.map((child) => terminateProcessTree(child.pid)));
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
private workerCommand(): { entry: string; loader: string; cwd: string } {
|
|
318
|
-
const sourceEntry = this.runtimeSourceRoot
|
|
319
|
-
? join(this.runtimeSourceRoot, 'src', 'runtime', 'execution', 'workers', 'worker-entry.ts')
|
|
320
|
-
: fileURLToPath(new URL('../../execution/workers/worker-entry.ts', import.meta.url));
|
|
321
|
-
const loader = this.runtimeSourceRoot
|
|
322
|
-
? join(this.runtimeSourceRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs')
|
|
323
|
-
: fileURLToPath(new URL('../../shared/node-ts-loader.mjs', import.meta.url));
|
|
324
|
-
const entry = this.workerEntrypoint ?? sourceEntry;
|
|
325
|
-
const cwd = this.runtimeSourceRoot ?? process.cwd();
|
|
326
|
-
if (!existsSync(entry)) throw new Error(`WORKER_ENTRYPOINT_MISSING: ${entry}`);
|
|
327
|
-
if (!process.versions.bun && !existsSync(loader)) throw new Error(`WORKER_LOADER_MISSING: ${loader}`);
|
|
328
|
-
return { entry, loader, cwd };
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
private workerEnvironment(): Record<string, string | undefined> {
|
|
332
|
-
return Object.fromEntries(WORKER_ENVIRONMENT_KEYS.map((key) => [key, process.env[key]]));
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
private persistSpawnedWorker(repoId: string, jobId: string, lifecycle: ExecutionWorkerLifecycle): void {
|
|
336
|
-
try {
|
|
337
|
-
updateExecutionJob(this.controllerHome, repoId, jobId, (current) => {
|
|
338
|
-
if (!['dispatched', 'running'].includes(current.status) || current.workerPid !== undefined) return current;
|
|
339
|
-
return { ...current, workerLifecycle: lifecycle };
|
|
340
|
-
});
|
|
341
|
-
} catch { /* the Job may have been superseded or made terminal */ }
|
|
342
|
-
}
|
|
343
|
-
|
|
344
236
|
private async recordWorkerExit(
|
|
345
237
|
repoId: string,
|
|
346
238
|
jobId: string,
|
|
@@ -353,82 +245,29 @@ export class GlobalScheduler {
|
|
|
353
245
|
stderrTruncated: boolean,
|
|
354
246
|
startupError?: string,
|
|
355
247
|
): Promise<void> {
|
|
356
|
-
const diagnosticLifecycle
|
|
357
|
-
|
|
358
|
-
|
|
248
|
+
const diagnosticLifecycle = buildSchedulerWorkerExitedLifecycle({
|
|
249
|
+
lifecycle,
|
|
250
|
+
childPid: child?.pid,
|
|
359
251
|
exitCode,
|
|
360
252
|
signal,
|
|
361
|
-
workerPid: child?.pid ?? lifecycle.workerPid,
|
|
362
|
-
processGroupId: lifecycle.processGroupId ?? (process.platform !== 'win32' ? child?.pid : undefined),
|
|
363
253
|
stderr,
|
|
364
254
|
stderrTruncated,
|
|
365
|
-
|
|
366
|
-
};
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
// lease ownership, finalize from that receipt and never emit WORKER_EXITED.
|
|
382
|
-
const recovered = tryRecoverJobFromWorkerReceipt(this.controllerHome, current);
|
|
383
|
-
if (recovered) {
|
|
384
|
-
updateExecutionJob(this.controllerHome, repoId, jobId, (latest) => ({ ...latest, workerLifecycle: mergedLifecycle }));
|
|
385
|
-
try { rebuildRepositoryProjection(this.controllerHome, repoId); } catch { /* the next scheduler/status read can retry */ }
|
|
386
|
-
return;
|
|
387
|
-
}
|
|
388
|
-
const rechecked = getExecutionJob(this.controllerHome, repoId, jobId);
|
|
389
|
-
if (rechecked.attempt !== attempt) return;
|
|
390
|
-
const recheckedLifecycle = rechecked.workerLifecycle ?? lifecycle;
|
|
391
|
-
const recheckedMerged = { ...recheckedLifecycle, ...diagnosticLifecycle };
|
|
392
|
-
if (['succeeded', 'failed', 'timed_out', 'cancelled', 'orphaned', 'stale', 'human_attention_required'].includes(rechecked.status)) {
|
|
393
|
-
updateExecutionJob(this.controllerHome, repoId, jobId, (latest) => ({ ...latest, workerLifecycle: recheckedMerged }));
|
|
394
|
-
try { rebuildRepositoryProjection(this.controllerHome, repoId); } catch { /* the next scheduler/status read can retry */ }
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
const details: Record<string, unknown> = {
|
|
399
|
-
workerLostReason: startupError ? 'spawn_failed' : 'process_exit',
|
|
400
|
-
executable: recheckedMerged.executable,
|
|
401
|
-
cwd: recheckedMerged.cwd,
|
|
402
|
-
exitCode,
|
|
403
|
-
signal,
|
|
404
|
-
stderr,
|
|
405
|
-
stderrTruncated,
|
|
406
|
-
stderrPath: recheckedMerged.stderrPath,
|
|
407
|
-
processGroupId: recheckedMerged.processGroupId,
|
|
408
|
-
ownerPid: recheckedMerged.ownerPid,
|
|
409
|
-
runtimeInstanceId: recheckedMerged.runtimeInstanceId,
|
|
410
|
-
releaseAuthorityRevision: recheckedMerged.releaseAuthorityRevision,
|
|
411
|
-
releaseId: recheckedMerged.releaseId,
|
|
412
|
-
artifactIdentity: recheckedMerged.artifactIdentity,
|
|
413
|
-
workerProtocolVersion: recheckedMerged.workerProtocolVersion,
|
|
414
|
-
attempt: rechecked.attempt,
|
|
415
|
-
maxAttempts: rechecked.maxAttempts,
|
|
416
|
-
...(startupError ? { startupError } : {}),
|
|
417
|
-
};
|
|
418
|
-
const stderrSummary = stderr.trim() ? ` Worker stderr: ${stderr.trim()}` : '';
|
|
419
|
-
const startupSummary = startupError ? ` Startup error: ${startupError}.` : '';
|
|
420
|
-
const message = `Execution Worker ${recheckedMerged.executable} exited before completion (cwd ${recheckedMerged.cwd}, exit code ${exitCode ?? 'unknown'}${signal ? `, signal ${signal}` : ''}).${startupSummary}${stderrSummary}`;
|
|
421
|
-
releaseExecutionLeases(this.controllerHome, repoId, jobId, rechecked.leaseRefs);
|
|
422
|
-
const retryable = rechecked.attempt < rechecked.maxAttempts;
|
|
423
|
-
transitionExecutionJob(this.controllerHome, repoId, jobId, retryable ? 'queued' : 'failed', {
|
|
424
|
-
workerPid: undefined,
|
|
425
|
-
heartbeatAt: undefined,
|
|
426
|
-
leaseRefs: [],
|
|
427
|
-
workerLifecycle: recheckedMerged,
|
|
428
|
-
error: { code: startupError ? 'WORKER_START_FAILED' : 'WORKER_EXITED', message, retryable, details },
|
|
429
|
-
});
|
|
430
|
-
try { rebuildRepositoryProjection(this.controllerHome, repoId); } catch { /* the next scheduler/status read can retry */ }
|
|
431
|
-
} catch { /* the Job may have been finalized by the Worker or reconciliation */ }
|
|
255
|
+
startupError,
|
|
256
|
+
});
|
|
257
|
+
reconcileSchedulerWorkerExit({
|
|
258
|
+
controllerHome: this.controllerHome,
|
|
259
|
+
repoId,
|
|
260
|
+
jobId,
|
|
261
|
+
attempt,
|
|
262
|
+
childPid: child?.pid,
|
|
263
|
+
lifecycle,
|
|
264
|
+
diagnosticLifecycle,
|
|
265
|
+
exitCode,
|
|
266
|
+
signal,
|
|
267
|
+
stderr,
|
|
268
|
+
stderrTruncated,
|
|
269
|
+
startupError,
|
|
270
|
+
});
|
|
432
271
|
}
|
|
433
272
|
|
|
434
273
|
private spawnWorker(repoId: string, jobId: string): boolean {
|
|
@@ -441,129 +280,119 @@ export class GlobalScheduler {
|
|
|
441
280
|
if (!['dispatched', 'running'].includes(current.status)) return false;
|
|
442
281
|
if (current.workerPid && this.pidAlive(current.workerPid)) return false;
|
|
443
282
|
const command = (() => {
|
|
444
|
-
try {
|
|
283
|
+
try {
|
|
284
|
+
return resolveSchedulerWorkerCommand({
|
|
285
|
+
runtimeSourceRoot: this.runtimeSourceRoot,
|
|
286
|
+
workerEntrypoint: this.workerEntrypoint,
|
|
287
|
+
});
|
|
288
|
+
} catch (error) {
|
|
445
289
|
const message = error instanceof Error ? error.message : String(error);
|
|
446
|
-
const lifecycle
|
|
290
|
+
const lifecycle = buildSchedulerWorkerSpawnFailureLifecycle({
|
|
447
291
|
executable: process.execPath,
|
|
448
|
-
args: [],
|
|
449
292
|
cwd: this.runtimeSourceRoot ?? process.cwd(),
|
|
450
|
-
environment:
|
|
293
|
+
environment: selectSchedulerWorkerEnvironment(process.env),
|
|
451
294
|
ownerPid: this.controllerPid,
|
|
452
295
|
attempt: current.attempt,
|
|
453
296
|
maxAttempts: current.maxAttempts,
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
297
|
+
});
|
|
298
|
+
persistSchedulerSpawnedWorkerLifecycle({
|
|
299
|
+
controllerHome: this.controllerHome,
|
|
300
|
+
repoId,
|
|
301
|
+
jobId,
|
|
302
|
+
lifecycle,
|
|
303
|
+
});
|
|
458
304
|
void this.recordWorkerExit(repoId, jobId, current.attempt, undefined, lifecycle, null, null, '', false, message);
|
|
459
305
|
return undefined;
|
|
460
306
|
}
|
|
461
307
|
})();
|
|
462
308
|
if (!command) return false;
|
|
463
|
-
const bun = Boolean(process.versions.bun);
|
|
464
|
-
const workerExecutable = resolveSchedulerWorkerExecutable(bun);
|
|
465
309
|
// Pass the parent's captured Runtime owner and whole-release claim. A Worker
|
|
466
310
|
// never re-adopts current authority after spawn; owner/release rotation fences it.
|
|
467
311
|
const writeClaim = getRuntimeWriteClaim();
|
|
468
|
-
const
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
const stderrPath = join(executionJobRoot(this.controllerHome, repoId), 'worker-stderr', `${jobId}-attempt-${current.attempt}.log`);
|
|
486
|
-
mkdirSync(dirname(stderrPath), { recursive: true });
|
|
487
|
-
writeFileSync(stderrPath, '', 'utf8');
|
|
488
|
-
const lifecycle: ExecutionWorkerLifecycle = {
|
|
489
|
-
executable: workerExecutable,
|
|
490
|
-
args,
|
|
491
|
-
cwd: command.cwd,
|
|
492
|
-
environment: Object.fromEntries(WORKER_ENVIRONMENT_KEYS.map((key) => [key, environment[key]])),
|
|
312
|
+
const launch = buildSchedulerWorkerLaunchDescriptor({
|
|
313
|
+
command,
|
|
314
|
+
controllerHome: this.controllerHome,
|
|
315
|
+
repoId,
|
|
316
|
+
jobId,
|
|
317
|
+
controllerPid: this.controllerPid,
|
|
318
|
+
runtimeSourceRoot: this.runtimeSourceRoot,
|
|
319
|
+
writeClaimEnvironment: writeClaim ? runtimeWriteClaimEnvironment(writeClaim) : {},
|
|
320
|
+
});
|
|
321
|
+
const stderrCapture = createSchedulerWorkerStderrCapture({
|
|
322
|
+
controllerHome: this.controllerHome,
|
|
323
|
+
repoId,
|
|
324
|
+
jobId,
|
|
325
|
+
attempt: current.attempt,
|
|
326
|
+
});
|
|
327
|
+
const lifecycle = buildSchedulerWorkerSpawnedLifecycle({
|
|
328
|
+
launch,
|
|
493
329
|
ownerPid: this.controllerPid,
|
|
494
|
-
|
|
330
|
+
releaseIdentity: writeClaim ? {
|
|
495
331
|
runtimeInstanceId: writeClaim.runtimeInstanceId,
|
|
496
332
|
releaseAuthorityRevision: writeClaim.releaseAuthorityRevision,
|
|
497
333
|
releaseId: writeClaim.releaseId,
|
|
498
334
|
artifactIdentity: writeClaim.artifactIdentity,
|
|
499
335
|
workerProtocolVersion: writeClaim.workerProtocolVersion,
|
|
500
|
-
} :
|
|
336
|
+
} : undefined,
|
|
501
337
|
attempt: current.attempt,
|
|
502
338
|
maxAttempts: current.maxAttempts,
|
|
503
|
-
|
|
504
|
-
stderrPath,
|
|
505
|
-
startupState: 'spawned',
|
|
506
|
-
};
|
|
507
|
-
this.persistSpawnedWorker(repoId, jobId, lifecycle);
|
|
508
|
-
let child: ChildProcess;
|
|
509
|
-
try {
|
|
510
|
-
child = spawn(workerExecutable, args, {
|
|
511
|
-
cwd: command.cwd,
|
|
512
|
-
stdio: ['ignore', 'ignore', 'pipe'],
|
|
513
|
-
detached: process.platform !== 'win32',
|
|
514
|
-
env: environment,
|
|
515
|
-
});
|
|
516
|
-
} catch (error) {
|
|
517
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
518
|
-
void this.recordWorkerExit(repoId, jobId, current.attempt, undefined, lifecycle, null, null, '', false, message);
|
|
519
|
-
return false;
|
|
520
|
-
}
|
|
521
|
-
let stderr = '';
|
|
522
|
-
let stderrBytes = 0;
|
|
523
|
-
let stderrTruncated = false;
|
|
524
|
-
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
525
|
-
const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
|
526
|
-
const bytes = Buffer.byteLength(text);
|
|
527
|
-
const remaining = MAX_WORKER_STDERR_BYTES - stderrBytes;
|
|
528
|
-
const accepted = remaining > 0 ? Buffer.from(text).subarray(0, remaining).toString('utf8') : '';
|
|
529
|
-
if (accepted) {
|
|
530
|
-
stderr += accepted;
|
|
531
|
-
stderrBytes += Buffer.byteLength(accepted);
|
|
532
|
-
try { appendFileSync(stderrPath, accepted, 'utf8'); } catch { stderrTruncated = true; }
|
|
533
|
-
}
|
|
534
|
-
if (bytes > Math.max(0, remaining)) stderrTruncated = true;
|
|
339
|
+
stderrPath: stderrCapture.path,
|
|
535
340
|
});
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
341
|
+
persistSchedulerSpawnedWorkerLifecycle({
|
|
342
|
+
controllerHome: this.controllerHome,
|
|
343
|
+
repoId,
|
|
344
|
+
jobId,
|
|
345
|
+
lifecycle,
|
|
346
|
+
});
|
|
347
|
+
const spawned = spawnSchedulerWorkerProcess(launch);
|
|
348
|
+
if (!spawned.ok) {
|
|
349
|
+
void this.recordWorkerExit(
|
|
350
|
+
repoId,
|
|
351
|
+
jobId,
|
|
352
|
+
current.attempt,
|
|
353
|
+
undefined,
|
|
354
|
+
lifecycle,
|
|
355
|
+
null,
|
|
356
|
+
null,
|
|
357
|
+
'',
|
|
358
|
+
false,
|
|
359
|
+
spawned.startupError,
|
|
360
|
+
);
|
|
555
361
|
return false;
|
|
556
362
|
}
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
})
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
363
|
+
const child = spawned.child;
|
|
364
|
+
wireSchedulerWorkerProcess({
|
|
365
|
+
jobId,
|
|
366
|
+
child,
|
|
367
|
+
children: this.children,
|
|
368
|
+
stderrCapture,
|
|
369
|
+
onExit: ({ exitCode, signal, stderr, stderrTruncated, startupError }) => {
|
|
370
|
+
void this.recordWorkerExit(
|
|
371
|
+
repoId,
|
|
372
|
+
jobId,
|
|
373
|
+
current.attempt,
|
|
374
|
+
child,
|
|
375
|
+
lifecycle,
|
|
376
|
+
exitCode,
|
|
377
|
+
signal,
|
|
378
|
+
stderr,
|
|
379
|
+
stderrTruncated,
|
|
380
|
+
startupError,
|
|
381
|
+
);
|
|
382
|
+
},
|
|
383
|
+
});
|
|
384
|
+
return registerSchedulerWorkerProcess({
|
|
385
|
+
jobId,
|
|
386
|
+
child,
|
|
387
|
+
children: this.children,
|
|
388
|
+
attach: (workerPid) => persistSchedulerWorkerAttachment({
|
|
389
|
+
controllerHome: this.controllerHome,
|
|
390
|
+
repoId,
|
|
391
|
+
jobId,
|
|
392
|
+
workerPid,
|
|
393
|
+
lifecycle,
|
|
394
|
+
}),
|
|
395
|
+
});
|
|
567
396
|
}
|
|
568
397
|
|
|
569
398
|
private refreshDarwinAvailableMemoryMb(fallback: number): void {
|
|
@@ -606,102 +435,104 @@ export class GlobalScheduler {
|
|
|
606
435
|
};
|
|
607
436
|
}
|
|
608
437
|
|
|
609
|
-
private agentProvider(job: { payload: { arguments?: Record<string, unknown> } }): 'codex' | 'claude' | 'github-copilot' {
|
|
610
|
-
const agent = job.payload.arguments?.agent;
|
|
611
|
-
if (agent === 'claude' || agent === 'github-copilot') return agent;
|
|
612
|
-
return 'codex';
|
|
613
|
-
}
|
|
614
|
-
|
|
615
438
|
async tick(): Promise<{ activeJobs: number }> {
|
|
616
439
|
const now = Date.now();
|
|
617
440
|
this.lastHeartbeatAt = new Date(now).toISOString();
|
|
618
441
|
this.lastTickAt = this.lastHeartbeatAt;
|
|
619
442
|
this.persistState();
|
|
620
443
|
const repositories = this.repositoryList(this.controllerHome).filter((repo) => repo.enabled && !repo.removedAt);
|
|
444
|
+
let periodicCleanupRan = false;
|
|
621
445
|
if (now - this.lastCleanupAt >= RUNTIME_CLEANUP_INTERVAL_MS) {
|
|
622
446
|
// Advance the interval before cleanup so a failing pass cannot create a
|
|
623
447
|
// tight retry loop on every scheduler tick.
|
|
624
448
|
this.lastCleanupAt = now;
|
|
625
|
-
|
|
626
|
-
this.
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
} catch (error) {
|
|
637
|
-
console.error('[forge cleanup] terminal Work cleanup failed:', error);
|
|
638
|
-
}
|
|
639
|
-
if (repositories.length > 0) {
|
|
640
|
-
const slot = Math.floor(now / RUNTIME_CLEANUP_INTERVAL_MS) % repositories.length;
|
|
641
|
-
const repo = repositories[slot]!;
|
|
642
|
-
const result = this.processGc({ controllerHome: this.controllerHome, repoId: repo.repoId });
|
|
643
|
-
if (!result.ok) console.error('[forge cleanup] Process GC failed:', result.error ?? 'unknown error');
|
|
644
|
-
}
|
|
449
|
+
await runSchedulerPeriodicCleanup({
|
|
450
|
+
controllerHome: this.controllerHome,
|
|
451
|
+
controllerPid: this.controllerPid,
|
|
452
|
+
nowMs: now,
|
|
453
|
+
cleanupIntervalMs: RUNTIME_CLEANUP_INTERVAL_MS,
|
|
454
|
+
repositories,
|
|
455
|
+
runtimeCleanup: this.runtimeCleanup,
|
|
456
|
+
terminalWorkCleanup: this.terminalWorkCleanup,
|
|
457
|
+
processGc: this.processGc,
|
|
458
|
+
});
|
|
459
|
+
periodicCleanupRan = true;
|
|
645
460
|
}
|
|
646
461
|
if (now - this.lastReconcile >= 5_000) {
|
|
647
462
|
await reconcileExecutionJobsAsync(this.controllerHome);
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
}
|
|
655
|
-
const editValidation = await this.editValidationReconcile(this.controllerHome, repository, 200);
|
|
656
|
-
if (editValidation.errors.length > 0) {
|
|
657
|
-
console.error(
|
|
658
|
-
`[forge validation] background EditSession reconciliation reported ${editValidation.errors.length} error(s) for ${repository.repoId}`,
|
|
659
|
-
);
|
|
660
|
-
}
|
|
661
|
-
}
|
|
463
|
+
await runSchedulerValidationReconciliation({
|
|
464
|
+
controllerHome: this.controllerHome,
|
|
465
|
+
repositories,
|
|
466
|
+
workValidationReconcile: this.workValidationReconcile,
|
|
467
|
+
editValidationReconcile: this.editValidationReconcile,
|
|
468
|
+
});
|
|
662
469
|
this.lastReconcile = now;
|
|
663
470
|
this.lastReconcileAt = new Date(now).toISOString();
|
|
664
471
|
}
|
|
665
472
|
const activeJobSnapshot = listActiveExecutionJobs(this.controllerHome);
|
|
666
|
-
const
|
|
667
|
-
const
|
|
668
|
-
|
|
473
|
+
const activeSourceRepoIds = new Set(activeJobSnapshot.map((job) => job.repoId));
|
|
474
|
+
const sourceSampling = planSchedulerSourceSampling({
|
|
475
|
+
repositories,
|
|
476
|
+
activeRepoIds: activeSourceRepoIds,
|
|
477
|
+
nowMs: now,
|
|
478
|
+
lastSourceScanAt: this.lastSourceScanAt,
|
|
479
|
+
lastGitStatusSampleAt: this.lastGitStatusSampleAt,
|
|
480
|
+
});
|
|
481
|
+
const sourceScanRepositories = sourceSampling.sourceScanRepositories;
|
|
482
|
+
const dirtyProjectionRepositories = selectSchedulerDirtyProjectionRepositories(this.controllerHome, repositories);
|
|
483
|
+
if (sourceSampling.shouldSample) {
|
|
484
|
+
// Advance source-sampling state before the filesystem probe so a failing
|
|
485
|
+
// sample cannot create a tight retry loop on every scheduler tick.
|
|
669
486
|
this.lastGitStatusSampleAt = now;
|
|
670
487
|
this.lastSourceScanAt = now;
|
|
671
|
-
this.lastSourceScanRepoCount =
|
|
672
|
-
sampleRepositoryGitStatusForRepositories(this.controllerHome,
|
|
673
|
-
} else if (!sourceScanDue) {
|
|
674
|
-
// fs.watch wakeups cover active mutations; a bounded safety rescan keeps
|
|
675
|
-
// idle repositories fresh without re-running a full source scan per tick.
|
|
676
|
-
this.sourceScansAvoided += repositories.length;
|
|
488
|
+
this.lastSourceScanRepoCount = sourceScanRepositories.length;
|
|
489
|
+
sampleRepositoryGitStatusForRepositories(this.controllerHome, sourceScanRepositories);
|
|
677
490
|
}
|
|
491
|
+
// Dirty projection markers are authoritative maintenance requests. Do not
|
|
492
|
+
// make them wait behind the idle one-repository-per-minute safety scan: only
|
|
493
|
+
// the dirty repositories receive this extra fresh source sample.
|
|
494
|
+
const alreadySampledRepoIds = new Set(
|
|
495
|
+
sourceSampling.shouldSample ? sourceScanRepositories.map((repository) => repository.repoId) : [],
|
|
496
|
+
);
|
|
497
|
+
const dirtyRepositoriesToSample = dirtyProjectionRepositories
|
|
498
|
+
.filter((repository) => !alreadySampledRepoIds.has(repository.repoId));
|
|
499
|
+
if (dirtyRepositoriesToSample.length > 0) {
|
|
500
|
+
sampleRepositoryGitStatusForRepositories(this.controllerHome, dirtyRepositoriesToSample);
|
|
501
|
+
}
|
|
502
|
+
const projectionMaintenanceRepositories = Array.from(new Map(
|
|
503
|
+
[...sourceScanRepositories, ...dirtyProjectionRepositories]
|
|
504
|
+
.map((repository) => [repository.repoId, repository] as const),
|
|
505
|
+
).values());
|
|
506
|
+
this.sourceScansAvoided += sourceSampling.avoidedRepositoryCount;
|
|
678
507
|
// Phase 0 reuses one durable Work admission policy. Cleanup, stale-state
|
|
679
|
-
// reconciliation,
|
|
680
|
-
// ordinary schedule/workflow advancement and Worker dispatch stop here.
|
|
508
|
+
// reconciliation, read-only source sampling, and projection maintenance remain
|
|
509
|
+
// available, but ordinary schedule/workflow advancement and Worker dispatch stop here.
|
|
681
510
|
if (!schedulerDispatchAllowed(this.controllerHome)) {
|
|
511
|
+
refreshSchedulerRepositoryProjections({
|
|
512
|
+
controllerHome: this.controllerHome,
|
|
513
|
+
repositories,
|
|
514
|
+
sourceScanRepositories: projectionMaintenanceRepositories,
|
|
515
|
+
projectionRefreshRepoIds: [],
|
|
516
|
+
controllerPid: this.controllerPid,
|
|
517
|
+
});
|
|
682
518
|
this.lastHeartbeatAt = new Date().toISOString();
|
|
683
519
|
this.persistState(true);
|
|
684
520
|
return { activeJobs: activeJobSnapshot.length };
|
|
685
521
|
}
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
markExecutionJobSchedulerObserved(this.controllerHome, job.repoId, job.jobId);
|
|
693
|
-
} catch {
|
|
694
|
-
// Another scheduler or a terminal transition won the Job-local race.
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
if (now - this.lastScheduleTick >= 30_000) {
|
|
698
|
-
await tickSchedules(this.controllerHome, repositories.map((repo) => repo.repoId));
|
|
699
|
-
this.lastScheduleTick = now;
|
|
700
|
-
}
|
|
701
|
-
if (now - this.lastPortfolioTick >= 1_000) {
|
|
702
|
-
tickPortfolioWorkflows(this.controllerHome);
|
|
703
|
-
this.lastPortfolioTick = now;
|
|
522
|
+
if (periodicCleanupRan) {
|
|
523
|
+
await runSchedulerControllerRoundRecovery({
|
|
524
|
+
controllerHome: this.controllerHome,
|
|
525
|
+
nowMs: now,
|
|
526
|
+
repositories,
|
|
527
|
+
});
|
|
704
528
|
}
|
|
529
|
+
const durableAdmission = await runSchedulerDurableAdmission({
|
|
530
|
+
controllerHome: this.controllerHome,
|
|
531
|
+
repositoryIds: repositories.map((repo) => repo.repoId),
|
|
532
|
+
nowMs: now,
|
|
533
|
+
lastScheduleTickAt: this.lastScheduleTick,
|
|
534
|
+
});
|
|
535
|
+
if (durableAdmission.scheduleTicked) this.lastScheduleTick = now;
|
|
705
536
|
let activeJobs = 0;
|
|
706
537
|
const pendingSpawns: Array<{ repoId: string; jobId: string }> = [];
|
|
707
538
|
const projectionRefreshRepos = new Set<string>();
|
|
@@ -717,67 +548,16 @@ export class GlobalScheduler {
|
|
|
717
548
|
`global-scheduler:${this.controllerPid}`,
|
|
718
549
|
() => {
|
|
719
550
|
const active = listActiveExecutionJobs(this.controllerHome);
|
|
720
|
-
const
|
|
721
|
-
|
|
722
|
-
if (capacity <= 0) return active.length;
|
|
723
|
-
|
|
724
|
-
let heavyCapacity = this.config.maxHeavyChecks
|
|
725
|
-
- reserved.filter((job) => job.type === 'check' || job.type === 'verify-edit').length;
|
|
726
|
-
const reservedAgents = reserved.filter((job) => job.type === 'agent-run' || job.type === 'dispatch-task');
|
|
727
|
-
let agentCapacity = this.config.maxAgentProcesses - reservedAgents.length;
|
|
728
|
-
const providerCapacity = new Map([
|
|
729
|
-
['codex', this.config.maxCodexProcesses - reservedAgents.filter((job) => this.agentProvider(job) === 'codex').length],
|
|
730
|
-
['claude', this.config.maxClaudeProcesses - reservedAgents.filter((job) => this.agentProvider(job) === 'claude').length],
|
|
731
|
-
['github-copilot', this.config.maxGitHubProcesses - reservedAgents.filter((job) => this.agentProvider(job) === 'github-copilot').length],
|
|
732
|
-
] as const);
|
|
733
|
-
if (pressure.pressured) {
|
|
734
|
-
// Under host pressure, keep one recovery slot available so queued read-only
|
|
735
|
-
// or bounded repository work does not stall forever behind a global stop.
|
|
736
|
-
capacity = Math.min(capacity, 1);
|
|
737
|
-
heavyCapacity = Math.min(heavyCapacity, 1);
|
|
738
|
-
agentCapacity = 0;
|
|
739
|
-
providerCapacity.set('codex', 0);
|
|
740
|
-
providerCapacity.set('claude', 0);
|
|
741
|
-
providerCapacity.set('github-copilot', 0);
|
|
742
|
-
}
|
|
743
|
-
if (capacity <= 0) return active.length;
|
|
551
|
+
const capacity = createSchedulerDispatchCapacity(active, this.config, pressure.pressured);
|
|
552
|
+
if (capacity.workers <= 0) return active.length;
|
|
744
553
|
|
|
745
554
|
const scheduleNow = Date.now();
|
|
746
|
-
const
|
|
747
|
-
const
|
|
748
|
-
waiting.map((job) => [job.jobId, rankExecutionJobForDispatch(job, scheduleNow)] as const),
|
|
749
|
-
);
|
|
750
|
-
const compareWaiting = (left: (typeof active)[number], right: (typeof active)[number]): number =>
|
|
751
|
-
compareExecutionJobDispatchRanks(rankByJobId.get(left.jobId)!, rankByJobId.get(right.jobId)!);
|
|
752
|
-
const topByRepo = new Map<string, (typeof active)[number]>();
|
|
753
|
-
for (const job of waiting.slice().sort(compareWaiting)) {
|
|
754
|
-
if (!topByRepo.has(job.repoId)) topByRepo.set(job.repoId, job);
|
|
755
|
-
}
|
|
756
|
-
const repoIds = [...topByRepo.keys()].sort((left, right) => {
|
|
757
|
-
const leftTop = topByRepo.get(left)!;
|
|
758
|
-
const rightTop = topByRepo.get(right)!;
|
|
759
|
-
const leftRank = rankByJobId.get(leftTop.jobId)!;
|
|
760
|
-
const rightRank = rankByJobId.get(rightTop.jobId)!;
|
|
761
|
-
const priority = leftRank.effectivePriority - rightRank.effectivePriority;
|
|
762
|
-
if (priority !== 0) return priority;
|
|
763
|
-
const fairness = (this.lastRepoDispatch.get(left) ?? 0) - (this.lastRepoDispatch.get(right) ?? 0);
|
|
764
|
-
return fairness
|
|
765
|
-
|| leftRank.queuedAtMs - rightRank.queuedAtMs
|
|
766
|
-
|| leftRank.jobId.localeCompare(rightRank.jobId)
|
|
767
|
-
|| left.localeCompare(right);
|
|
768
|
-
});
|
|
769
|
-
const reservedRepos = new Set(reserved.map((job) => job.repoId));
|
|
555
|
+
const repoIds = selectExecutionJobDispatchRepositories(active, scheduleNow, this.lastRepoDispatch);
|
|
556
|
+
const reservedRepos = new Set(capacity.reservedJobs.map((job) => job.repoId));
|
|
770
557
|
let dispatchStateChanged = false;
|
|
771
|
-
const canDispatch = (job: (typeof active)[number]): boolean =>
|
|
772
|
-
if ((job.type === 'check' || job.type === 'verify-edit') && heavyCapacity <= 0) return false;
|
|
773
|
-
if (job.type === 'agent-run' || job.type === 'dispatch-task') {
|
|
774
|
-
if (agentCapacity <= 0) return false;
|
|
775
|
-
if ((providerCapacity.get(this.agentProvider(job)) ?? 0) <= 0) return false;
|
|
776
|
-
}
|
|
777
|
-
return true;
|
|
778
|
-
};
|
|
558
|
+
const canDispatch = (job: (typeof active)[number]): boolean => schedulerDispatchCapacityAllows(capacity, job);
|
|
779
559
|
for (const repoId of repoIds) {
|
|
780
|
-
if (capacity <= 0) break;
|
|
560
|
+
if (capacity.workers <= 0) break;
|
|
781
561
|
if (!reservedRepos.has(repoId) && reservedRepos.size >= this.config.maxConcurrentRepositories) continue;
|
|
782
562
|
const actor = this.actors.get(repoId);
|
|
783
563
|
let dispatch: ReturnType<typeof actor.tryClaimNext>;
|
|
@@ -798,14 +578,8 @@ export class GlobalScheduler {
|
|
|
798
578
|
// A successful claim is the capacity reservation. Count it immediately,
|
|
799
579
|
// before the worker PID is spawned or attached, so concurrent schedulers
|
|
800
580
|
// cannot over-dispatch through the dispatched -> running window.
|
|
801
|
-
capacity
|
|
581
|
+
consumeSchedulerDispatchCapacity(capacity, dispatch.job);
|
|
802
582
|
reservedRepos.add(repoId);
|
|
803
|
-
if (dispatch.job.type === 'check' || dispatch.job.type === 'verify-edit') heavyCapacity -= 1;
|
|
804
|
-
if (dispatch.job.type === 'agent-run' || dispatch.job.type === 'dispatch-task') {
|
|
805
|
-
agentCapacity -= 1;
|
|
806
|
-
const provider = this.agentProvider(dispatch.job);
|
|
807
|
-
providerCapacity.set(provider, (providerCapacity.get(provider) ?? 0) - 1);
|
|
808
|
-
}
|
|
809
583
|
const dispatchedAt = Date.now();
|
|
810
584
|
this.lastRepoDispatch.set(repoId, dispatchedAt);
|
|
811
585
|
this.lastDispatchAt = new Date(dispatchedAt).toISOString();
|
|
@@ -823,65 +597,22 @@ export class GlobalScheduler {
|
|
|
823
597
|
// leave all jobs queued for the next wake/tick rather than risking overrun.
|
|
824
598
|
activeJobs = listActiveExecutionJobs(this.controllerHome).length;
|
|
825
599
|
}
|
|
826
|
-
// Repo Actor mutations leave projection dirty markers.
|
|
827
|
-
//
|
|
828
|
-
// after the
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
}
|
|
836
|
-
for (const repoId of projectionRefreshRepos) {
|
|
837
|
-
if (projectionRefreshCandidates.has(repoId)) continue;
|
|
838
|
-
try {
|
|
839
|
-
rebuildRepositoryProjection(this.controllerHome, repoId);
|
|
840
|
-
} catch (error) {
|
|
841
|
-
console.error('[forge scheduler] projection refresh failed:', error);
|
|
842
|
-
}
|
|
843
|
-
}
|
|
844
|
-
for (const repository of projectionRefreshCandidates.values()) {
|
|
845
|
-
try {
|
|
846
|
-
const sample = readRepositoryGitStatusSample(
|
|
847
|
-
this.controllerHome,
|
|
848
|
-
repository.repoId,
|
|
849
|
-
repository.activeCheckoutId,
|
|
850
|
-
);
|
|
851
|
-
const runtimeInstanceId = getRuntimeWriteClaim()?.runtimeInstanceId;
|
|
852
|
-
refreshRepositoryProjectionForRepository(this.controllerHome, repository, {
|
|
853
|
-
sourceRevision: sample?.head ?? undefined,
|
|
854
|
-
reason: 'scheduler-source-scan',
|
|
855
|
-
owner: {
|
|
856
|
-
pid: this.controllerPid,
|
|
857
|
-
...(runtimeInstanceId ? { runtimeInstanceId } : {}),
|
|
858
|
-
},
|
|
859
|
-
});
|
|
860
|
-
} catch (error) {
|
|
861
|
-
console.error('[forge scheduler] projection refresh failed:', error);
|
|
862
|
-
}
|
|
863
|
-
}
|
|
600
|
+
// Repo Actor mutations leave projection dirty markers. Refresh those repos
|
|
601
|
+
// immediately; the independent source safety scan contributes only its bounded
|
|
602
|
+
// active/round-robin candidates after the dispatch reservation lock is free.
|
|
603
|
+
refreshSchedulerRepositoryProjections({
|
|
604
|
+
controllerHome: this.controllerHome,
|
|
605
|
+
repositories,
|
|
606
|
+
sourceScanRepositories: projectionMaintenanceRepositories,
|
|
607
|
+
projectionRefreshRepoIds: projectionRefreshRepos,
|
|
608
|
+
controllerPid: this.controllerPid,
|
|
609
|
+
});
|
|
864
610
|
// Process creation, lifecycle file writes, and Worker attachment are all
|
|
865
611
|
// deliberately outside the global dispatch reservation lock. The durable
|
|
866
612
|
// dispatched state is the capacity reservation while spawn proceeds.
|
|
867
613
|
for (const pending of pendingSpawns) {
|
|
868
614
|
this.spawnWorker(pending.repoId, pending.jobId);
|
|
869
615
|
}
|
|
870
|
-
// Dispatch is latency-sensitive. Run repository workflow maintenance only
|
|
871
|
-
// after queued Jobs have had a chance to claim capacity. Goal
|
|
872
|
-
// state transitions still wake the scheduler immediately when they enqueue work.
|
|
873
|
-
|
|
874
|
-
if (now - this.lastGoalLoopTick >= 5_000) {
|
|
875
|
-
try {
|
|
876
|
-
tickGoalLoopsForController(
|
|
877
|
-
this.controllerHome,
|
|
878
|
-
repositories.map((repo) => repo.repoId),
|
|
879
|
-
);
|
|
880
|
-
} catch (error) {
|
|
881
|
-
console.error('[forge goal-loop] tick failed:', error);
|
|
882
|
-
}
|
|
883
|
-
this.lastGoalLoopTick = now;
|
|
884
|
-
}
|
|
885
616
|
this.lastHeartbeatAt = new Date().toISOString();
|
|
886
617
|
this.persistState();
|
|
887
618
|
return { activeJobs };
|
|
@@ -936,7 +667,7 @@ export class GlobalScheduler {
|
|
|
936
667
|
}
|
|
937
668
|
} finally {
|
|
938
669
|
clearInterval(heartbeatTimer);
|
|
939
|
-
await this.
|
|
670
|
+
await cleanupSchedulerWorkerProcesses(this.children);
|
|
940
671
|
}
|
|
941
672
|
}
|
|
942
673
|
}
|