@moretea-labs/forge 1.6.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/CHANGELOG.md +39 -0
- package/CLAUDE.md +4 -0
- package/README.md +2 -2
- package/README.zh-CN.md +1 -1
- package/SKILL.md +2 -2
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/agents/openai.yaml +1 -1
- package/assets/hooks/hook-input.sh +11 -8
- package/assets/hooks/lib/session-state.sh +10 -7
- package/assets/hooks/lib/workflow-state.sh +13 -177
- package/assets/hooks/post-tool-observer.sh +8 -2
- package/assets/hooks/prompt-guard.sh +5 -81
- package/assets/hooks/session-start-context.sh +56 -1
- package/assets/partials/05-workflow.partial.md +1 -1
- package/assets/partials-agents/02-operating-mode.partial.md +2 -0
- package/assets/partials-agents/03-orchestration.partial.md +1 -1
- package/assets/plugin-registry.v1.json +3 -3
- package/assets/reference-configs/document-generation.md +1 -1
- package/assets/reference-configs/harness-overview.md +1 -1
- package/assets/reference-configs/sprint-contracts.md +3 -3
- package/assets/skill-version.json +2 -2
- package/assets/templates/contract.template.md +2 -7
- package/assets/templates/helpers/capture-plan.sh +2 -2
- package/assets/templates/helpers/check-architecture-sync.sh +17 -0
- package/assets/templates/helpers/check-task-workflow.sh +4 -4
- package/assets/templates/helpers/contract-worktree.sh +1 -21
- package/assets/templates/helpers/ensure-task-workflow.sh +11 -36
- package/assets/templates/helpers/inspect-project-state.ts +8 -6
- package/assets/templates/helpers/migrate-project-template.sh +22 -45
- package/assets/templates/helpers/migrate-workflow-docs.ts +9 -7
- package/assets/templates/helpers/new-plan.sh +1 -1
- package/assets/templates/helpers/plan-to-todo.sh +12 -37
- package/assets/templates/helpers/ship-worktrees.sh +2 -29
- package/assets/templates/helpers/verify-contract.sh +1 -111
- package/assets/templates/helpers/verify-sprint.sh +3 -180
- package/assets/templates/helpers/workflow-contract.ts +5 -1
- package/assets/templates/plan.template.md +1 -1
- package/assets/templates/review.template.md +9 -34
- package/assets/workflow-contract.v1.json +10 -19
- package/docs/README.md +10 -10
- package/docs/architecture/CURRENT.md +178 -0
- package/docs/architecture/EVOLUTION.md +42 -0
- package/docs/architecture/current/README.md +4 -156
- package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +2 -2
- package/docs/architecture/decisions/20260823-immediate-controller-relay.md +53 -0
- package/docs/architecture/decisions/20260826-browser-runtime-v2.md +73 -0
- package/docs/architecture/history/README.md +7 -0
- package/docs/architecture/{chatgpt-handoff-facade.md → history/chatgpt-handoff-facade.md} +4 -0
- package/docs/architecture/{global-hook-runtime.md → history/global-hook-runtime.md} +7 -3
- package/docs/architecture/{ios-semantic-automation-provider-v2.md → history/ios-semantic-automation-provider-v2.md} +6 -2
- package/docs/architecture/history.md +3 -34
- package/docs/architecture/index.md +15 -98
- package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
- package/docs/architecture/reviews/2026-08-22-semantic-harness-convergence-acceptance.md +39 -0
- package/docs/architecture/versions/1.6.md +38 -0
- package/docs/architecture/versions/1.7.md +41 -0
- package/docs/forge-chatgpt-browser-engine.md +2 -2
- package/docs/forge-chatgpt-mcp-setup.md +4 -28
- package/docs/forge-chatgpt-supervised-automation.md +3 -3
- package/docs/forge-github-issue-launcher.md +1 -1
- package/docs/forge-plugin-management.md +1 -1
- package/docs/forge-runtime-self-healing-loop.md +1 -1
- package/docs/operations/controller-browser-plugin.md +26 -6
- package/docs/operations/controller-performance-and-502.md +143 -0
- package/docs/operations/controller-reliability-runbook.md +1 -1
- package/docs/operations/features.md +2 -2
- package/docs/operations/github-repository.md +1 -1
- package/docs/operations/homebrew.md +2 -2
- package/docs/operations/mcp-tool-exposure.md +5 -3
- package/docs/wiki/Architecture.md +6 -6
- package/docs/wiki/Implementation.md +2 -2
- package/docs/wiki/Integrations.md +2 -2
- package/docs/wiki/Work-Lifecycle.md +1 -1
- package/package.json +33 -11
- package/scripts/activate-source-baseline.ts +17 -13
- package/scripts/architecture-event.ts +2 -820
- package/scripts/architecture-queue.sh +2 -645
- package/scripts/archive-architecture-request.sh +2 -242
- package/scripts/archive-workflow.sh +1 -291
- package/scripts/benchmark-background-check-overlap.ts +228 -0
- package/scripts/benchmark-mcp-host.sh +117 -0
- package/scripts/benchmark-mcp-transport.ts +170 -0
- package/scripts/benchmark-quality-harness.ts +146 -0
- package/scripts/benchmark-route-session-concurrency.ts +207 -6
- package/scripts/benchmark-thin-harness-mcp-e2e.ts +7 -1
- package/scripts/benchmark-typescript-navigation.ts +186 -0
- package/scripts/benchmark-work-efficiency.ts +334 -0
- package/scripts/capability-config.ts +2 -392
- package/scripts/capability-resolver.ts +2 -503
- package/scripts/capture-plan.sh +1 -332
- package/scripts/check-agent-tooling.sh +2 -1558
- package/scripts/check-architecture-sync.sh +2 -371
- package/scripts/check-brain-manifest.sh +2 -253
- package/scripts/check-context-files.sh +2 -47
- package/scripts/check-deploy-sql-order.sh +2 -115
- package/scripts/check-mcp-compatibility.ts +14 -0
- package/scripts/check-runtime-architecture.mjs +349 -79
- package/scripts/check-skill-version.ts +3 -191
- package/scripts/check-source-duplication.mjs +61 -0
- package/scripts/check-stable-baseline.ts +92 -0
- package/scripts/check-task-sync.sh +2 -63
- package/scripts/check-task-workflow.sh +2 -1000
- package/scripts/codex-handoff-resume.sh +2 -267
- package/scripts/context-contract-sync.sh +2 -514
- package/scripts/contract-run.ts +2 -463
- package/scripts/contract-worktree.sh +1 -722
- package/scripts/ensure-task-workflow.sh +1 -1225
- package/scripts/host-temporary-cloud-mcp-e2e.sh +265 -0
- package/scripts/inspect-project-state.ts +4 -279
- package/scripts/install-standalone-recovery.ts +24 -0
- package/scripts/lib/project-init-lib.sh +22 -46
- package/scripts/lib/run-forge-helper.sh +19 -0
- package/scripts/lib/run-forge-helper.ts +26 -0
- package/scripts/migrate-project-template.sh +1 -1189
- package/scripts/migrate-workflow-docs.ts +4 -412
- package/scripts/new-plan.sh +1 -180
- package/scripts/new-spec.sh +1 -47
- package/scripts/new-sprint.sh +1 -46
- package/scripts/plan-to-todo.sh +1 -796
- package/scripts/prepare-codex-handoff.sh +2 -201
- package/scripts/prepare-handoff.sh +1 -78
- package/scripts/refresh-current-status.sh +2 -467
- package/scripts/run-governed-gate.ts +5 -1
- package/scripts/select-agent-context-blocks.sh +2 -64
- package/scripts/ship-worktrees.sh +1 -558
- package/scripts/smoke-ios-physical-device.ts +4 -2
- package/scripts/smoke-runtime-recovery.ts +1 -35
- package/scripts/smoke-schedule-engine.ts +1 -20
- package/scripts/sprint-backlog.sh +1 -834
- package/scripts/stage-runtime-release.ts +2 -1
- package/scripts/summarize-failures.sh +2 -123
- package/scripts/switch-plan.sh +1 -150
- package/scripts/sync-brain-docs.sh +2 -379
- package/scripts/sync-generated-authority.mjs +175 -0
- package/scripts/verify-contract.sh +2 -703
- package/scripts/verify-forge-runtime.sh +0 -1
- package/scripts/verify-sprint.sh +1 -677
- package/scripts/workflow-contract.ts +4 -205
- package/scripts/workstream-sync.sh +2 -261
- package/src/cli/agent-jobs/executor-health.ts +1 -1
- package/src/cli/commands/init-hook.ts +3 -3
- package/src/cli/commands/mcp.ts +9 -5
- package/src/cli/commands/plugin.ts +12 -3
- package/src/cli/commands/recovery.ts +25 -3
- package/src/cli/commands/runtime.ts +55 -3
- package/src/cli/commands/security.ts +12 -1
- package/src/cli/commands/setup-profile.ts +31 -21
- package/src/cli/controller/check-runner.ts +113 -5
- package/src/cli/controller/composite-result.ts +1 -1
- package/src/cli/controller/context/focus.ts +51 -0
- package/src/cli/controller/context/instruction-context.ts +114 -0
- package/src/cli/controller/context/known-paths.ts +76 -0
- package/src/cli/controller/context/query-planning.ts +110 -0
- package/src/cli/controller/context/rule-resolution.ts +532 -0
- package/src/cli/controller/context/source-materializer.ts +278 -0
- package/src/cli/controller/context/structural.ts +16 -0
- package/src/cli/controller/context/types.ts +286 -0
- package/src/cli/controller/context-pack.ts +828 -487
- package/src/cli/controller/launch-agents.ts +77 -2
- package/src/cli/controller/operational-plan.ts +2 -2
- package/src/cli/controller/work-mode.ts +6 -4
- package/src/cli/editing/edit-session.ts +60 -7
- package/src/cli/github/contracts.ts +16 -0
- package/src/cli/github/github.ts +135 -16
- package/src/cli/github/session.ts +2 -1
- package/src/cli/installer/managed-entries.ts +15 -1
- package/src/cli/local-bridge/console-automations.ts +54 -78
- package/src/cli/local-bridge/console-view-models.ts +1 -35
- package/src/cli/local-bridge/facade-api.ts +37 -264
- package/src/cli/local-bridge/frontend/src/App.tsx +3 -3
- package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +23 -13
- package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +37 -4
- package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +6 -4
- package/src/cli/local-bridge/frontend/src/lib/api.ts +1 -6
- package/src/cli/local-bridge/frontend/src/lib/routes.ts +1 -1
- package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +2 -2
- package/src/cli/local-bridge/frontend/src/styles.css +1 -1
- package/src/cli/local-bridge/frontend/src/types.ts +5 -8
- package/src/cli/local-bridge/job-store.ts +1 -490
- package/src/cli/local-bridge/server.ts +2 -515
- package/src/cli/local-bridge/ui-dist/app.css +1 -1
- package/src/cli/local-bridge/ui-dist/app.js +3703 -3757
- package/src/cli/mcp/auth.ts +5 -5
- package/src/cli/mcp/instructions.ts +3 -2
- package/src/cli/mcp/legacy-tool-service.ts +10 -222
- package/src/cli/mcp/multi-repository.ts +16 -5
- package/src/cli/mcp/oauth.ts +175 -26
- package/src/cli/mcp/post-finalize-work-attribution.ts +150 -0
- package/src/cli/mcp/repository-tools.ts +282 -146
- package/src/cli/mcp/server.ts +586 -116
- package/src/cli/mcp/tool-capability-cache.ts +1 -1
- package/src/cli/mcp/tool-contract.ts +14 -0
- package/src/cli/mcp/toolset-selection.ts +1 -1
- package/src/cli/mcp/toolset.ts +44 -7
- package/src/cli/mcp/transports/http.ts +124 -14
- package/src/cli/mcp/transports/session-registry.ts +43 -5
- package/src/cli/mcp/types.ts +3 -3
- package/src/cli/repositories/command-classifier.ts +103 -21
- package/src/cli/repositories/command-executor.ts +78 -526
- package/src/cli/repositories/command-process.ts +108 -0
- package/src/cli/repositories/command-scope.ts +99 -41
- package/src/cli/repositories/controller-home.ts +7 -8
- package/src/cli/repositories/git-command-executor.ts +2 -0
- package/src/cli/repositories/locks.ts +53 -16
- package/src/cli/repositories/repository-snapshot.ts +124 -0
- package/src/cli/repositories/selected-path-actions.ts +17 -9
- package/src/cli/repositories/structured-git.ts +118 -6
- package/src/cli/repository/inspector.ts +563 -65
- package/src/cli/repository/session-cache.ts +58 -17
- package/src/cli/v81-entry.ts +1 -4
- package/src/effects/fs-transaction.ts +85 -17
- package/src/runtime/context/codegraph-read-provider.ts +233 -2
- package/src/runtime/context/codegraph-sidecar.cjs +74 -40
- package/src/runtime/context/generic-lsp-provider.ts +355 -0
- package/src/runtime/context/lsp-client.ts +283 -0
- package/src/runtime/context/semantic-navigation-contract.ts +61 -0
- package/src/runtime/context/semantic-navigation.ts +223 -0
- package/src/runtime/context/swift-navigation.ts +565 -0
- package/src/runtime/context/typescript-navigation.ts +226 -0
- package/src/runtime/control-plane/dispatch-priority.ts +28 -0
- package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +48 -16
- package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +86 -24
- package/src/runtime/control-plane/execution/repository-work-attribution.ts +106 -0
- package/src/runtime/control-plane/execution/retained-work-resume.ts +203 -0
- package/src/runtime/control-plane/execution/session-store.ts +18 -0
- package/src/runtime/control-plane/execution/verification-evidence.ts +58 -3
- package/src/runtime/control-plane/execution/work-completion-authority.ts +58 -0
- package/src/runtime/control-plane/execution/work-evidence-policy.ts +200 -0
- package/src/runtime/control-plane/execution/work-execution-support.ts +386 -0
- package/src/runtime/control-plane/execution/work-finalization-service.ts +1913 -0
- package/src/runtime/control-plane/execution/work-handle-authority.ts +96 -0
- package/src/runtime/control-plane/execution/work-handle-store.ts +27 -2
- package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +258 -0
- package/src/runtime/control-plane/execution/work-operation-service.ts +468 -0
- package/src/runtime/control-plane/execution/work-path-scope.ts +37 -0
- package/src/runtime/control-plane/execution/work-preparation-service.ts +479 -0
- package/src/runtime/control-plane/execution/work-process-evidence.ts +129 -0
- package/src/runtime/control-plane/execution/work-remote-delivery.ts +215 -0
- package/src/runtime/control-plane/execution/work-task-receipt.ts +22 -18
- package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +156 -11
- package/src/runtime/control-plane/execution/work-validation-reconciler.ts +22 -14
- package/src/runtime/control-plane/execution/work-verification-context.ts +56 -0
- package/src/runtime/control-plane/execution/work-verification-service.ts +388 -0
- package/src/runtime/control-plane/execution/work-verification-snapshot.ts +90 -9
- package/src/runtime/control-plane/facade/capability-registry.ts +129 -25
- package/src/runtime/control-plane/facade/controller-round-relay.ts +1118 -0
- package/src/runtime/control-plane/facade/controller-session-store.ts +302 -12
- package/src/runtime/control-plane/facade/goal-workloop-access.ts +17 -1
- package/src/runtime/control-plane/facade/goal-workloop.ts +928 -190
- package/src/runtime/control-plane/facade/index.ts +3 -0
- package/src/runtime/control-plane/facade/plan-contract-store.ts +275 -42
- package/src/runtime/control-plane/facade/repository-work-admission.ts +147 -0
- package/src/runtime/control-plane/facade/requirement-authority.ts +83 -0
- package/src/runtime/control-plane/facade/semantic-admission.ts +205 -0
- package/src/runtime/control-plane/facade/suggested-actions.ts +5 -1
- package/src/runtime/control-plane/facade/types.ts +90 -10
- package/src/runtime/control-plane/facade/work-contract-store.ts +157 -261
- package/src/runtime/control-plane/facade/work-state-machine.ts +250 -0
- package/src/runtime/control-plane/generated-cache-retention.ts +271 -0
- package/src/runtime/control-plane/global-scheduler/config.ts +44 -0
- package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts +80 -0
- package/src/runtime/control-plane/global-scheduler/durable-admission.ts +46 -0
- package/src/runtime/control-plane/global-scheduler/maintenance.ts +152 -0
- package/src/runtime/control-plane/global-scheduler/projection-refresh.ts +84 -0
- package/src/runtime/control-plane/global-scheduler/reconciliation.ts +3 -3
- package/src/runtime/control-plane/global-scheduler/scheduler.ts +278 -543
- package/src/runtime/control-plane/global-scheduler/source-scan.ts +59 -0
- package/src/runtime/control-plane/global-scheduler/state.ts +88 -0
- package/src/runtime/control-plane/global-scheduler/worker-attachment.ts +57 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts +38 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts +175 -0
- package/src/runtime/control-plane/global-scheduler/worker-launch.ts +111 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts +64 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts +150 -0
- package/src/runtime/control-plane/global-scheduler/worker-process.ts +119 -0
- package/src/runtime/control-plane/global-scheduler/worker-stderr.ts +60 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +63 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +365 -117
- package/src/runtime/control-plane/launcher/launch-reservation-store.ts +89 -0
- package/src/runtime/control-plane/launcher/thin-launcher.ts +244 -11
- package/src/runtime/control-plane/persistence/controller-home-migration.ts +487 -0
- package/src/runtime/control-plane/persistence/requirement-store.ts +0 -12
- package/src/runtime/control-plane/persistence/sqlite-store.ts +33 -2
- package/src/runtime/control-plane/release-retention.ts +62 -2
- package/src/runtime/control-plane/repo-actor/registry.ts +4 -2
- package/src/runtime/control-plane/routing/route-policy.ts +24 -21
- package/src/runtime/control-plane/routing/workspace-admission.ts +70 -0
- package/src/runtime/control-plane/runtime-status-client.ts +0 -2
- package/src/runtime/diagnostics/mcp-timing.ts +13 -0
- package/src/runtime/diagnostics/mcp-transport-benchmark.ts +129 -0
- package/src/runtime/diagnostics/performance.ts +24 -1
- package/src/runtime/evidence/event-ledger.ts +1 -1
- package/src/runtime/evidence/process-check-execution.ts +18 -0
- package/src/runtime/execution/jobs/child-reference.ts +6 -2
- package/src/runtime/execution/jobs/store.ts +58 -4
- package/src/runtime/execution/jobs/types.ts +1 -1
- package/src/runtime/execution/managed-workspace.ts +203 -1
- package/src/runtime/execution/process-runtime/check-facade.ts +51 -4
- package/src/runtime/execution/process-runtime/check-result.ts +88 -0
- package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +7 -1
- package/src/runtime/execution/process-runtime/check-scheduling.ts +114 -0
- package/src/runtime/execution/process-runtime/command-facade.ts +367 -40
- package/src/runtime/execution/process-runtime/gc.ts +38 -6
- package/src/runtime/execution/process-runtime/lightweight-managed.ts +1201 -0
- package/src/runtime/execution/process-runtime/persisted-check.ts +51 -5
- package/src/runtime/execution/process-runtime/resource-claims.ts +10 -6
- package/src/runtime/execution/process-runtime/runtime.ts +53 -3
- package/src/runtime/execution/process-runtime/store.ts +1 -0
- package/src/runtime/execution/process-runtime/types.ts +4 -12
- package/src/runtime/execution/thin-harness/async-process.ts +12 -2
- package/src/runtime/execution/thin-harness/search-worker.ts +54 -5
- package/src/runtime/execution/work-activity.ts +21 -0
- package/src/runtime/execution/workers/executor.ts +18 -61
- package/src/runtime/gateway/mcp/execution-tools.ts +35 -1852
- package/src/runtime/gateway/mcp/legacy-ios-tool-adapter.ts +61 -0
- package/src/runtime/gateway/mcp/process-tools.ts +83 -37
- package/src/runtime/gateway/mcp/router.ts +186 -84
- package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +565 -0
- package/src/runtime/gateway/mcp/runtime-tools.ts +2378 -1947
- package/src/runtime/plugins/app-store-connect-adapter.ts +216 -47
- package/src/runtime/plugins/browser-adapter.ts +2662 -829
- package/src/runtime/plugins/browser-automation-service.ts +10 -0
- package/src/runtime/plugins/browser-handoff-host.ts +2 -25
- package/src/runtime/plugins/browser-handoff.ts +110 -5
- package/src/runtime/plugins/browser-macos-bridge.ts +701 -63
- package/src/runtime/plugins/browser-manifest-surface.ts +603 -0
- package/src/runtime/plugins/browser-native-session-service.ts +222 -0
- package/src/runtime/plugins/browser-node-bridge.ts +6 -34
- package/src/runtime/plugins/browser-provider-registry.ts +109 -0
- package/src/runtime/plugins/browser-runtime-contract.ts +154 -0
- package/src/runtime/plugins/browser-runtime.ts +193 -0
- package/src/runtime/plugins/browser-session-authority.ts +332 -0
- package/src/runtime/plugins/browser-session-store.ts +121 -0
- package/src/runtime/plugins/browser-session-types.ts +98 -0
- package/src/runtime/plugins/capability-authorization-grants.ts +354 -0
- package/src/runtime/plugins/desktop-operator-registration.ts +10 -4
- package/src/runtime/plugins/external-adapter.ts +358 -3
- package/src/runtime/plugins/external-unix-socket.ts +7 -1
- package/src/runtime/plugins/first-party-registry.ts +2 -0
- package/src/runtime/plugins/github-adapter.ts +125 -0
- package/src/runtime/plugins/ios/agent-device-provider.ts +1 -1
- package/src/runtime/plugins/ios/agent-device-typed-provider.ts +52 -9
- package/src/runtime/plugins/ios/remote-xpc-hid.ts +1171 -0
- package/src/runtime/plugins/ios-adapter.ts +106 -16
- package/src/runtime/plugins/ios-agent-device.ts +234 -64
- package/src/runtime/plugins/ios-physical-device.ts +1143 -392
- package/src/runtime/plugins/lightweight-action.ts +153 -0
- package/src/runtime/plugins/local-system-adapter.ts +123 -6
- package/src/runtime/plugins/macos-capability-broker.ts +47 -9
- package/src/runtime/plugins/plugin-action-sidecar.ts +59 -0
- package/src/runtime/plugins/plugin-management-adapter.ts +49 -8
- package/src/runtime/plugins/resend-adapter.ts +1 -1
- package/src/runtime/plugins/store.ts +330 -33
- package/src/runtime/plugins/types.ts +38 -3
- package/src/runtime/plugins/xiaohongshu-publish.ts +515 -0
- package/src/runtime/projections/git-status-sampler.ts +16 -16
- package/src/runtime/projections/materialized-view.ts +28 -10
- package/src/runtime/recovery/maintenance-executor.ts +426 -25
- package/src/runtime/root/mcp-transport.ts +180 -10
- package/src/runtime/root/package-connector-service.ts +230 -10
- package/src/runtime/root/package-runtime-release.ts +143 -17
- package/src/runtime/root/package-runtime-service.ts +461 -22
- package/src/runtime/root/release-manifest.ts +144 -18
- package/src/runtime/root/release-materialize.ts +125 -32
- package/src/runtime/root/release-store.ts +48 -11
- package/src/runtime/root/runtime.ts +23 -5
- package/src/runtime/root/service.ts +100 -29
- package/src/runtime/root/types.ts +12 -0
- package/src/runtime/safe-tooling/ios-development.ts +321 -63
- package/src/runtime/shared/trusted-executable.ts +62 -0
- package/src/runtime/shared/trusted-node-executable.ts +48 -0
- package/src/runtime/standalone-recovery/core.ts +1079 -240
- package/src/runtime/standalone-recovery/entry.ts +100 -24
- package/src/runtime/standalone-recovery/installer.ts +20 -4
- package/src/runtime/standalone-recovery/service-labels.ts +2 -0
- package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +160 -0
- package/src/runtime/workflow/schedules/browser-probe.ts +81 -3
- package/src/runtime/workflow/schedules/engine.ts +364 -202
- package/src/runtime/workflow/schedules/github-issue-watch.ts +158 -0
- package/src/runtime/workflow/schedules/settlement.ts +64 -17
- package/src/runtime/workflow/schedules/store.ts +209 -19
- package/src/runtime/workflow/schedules/types.ts +2 -4
- package/src/runtime/workflow/schedules/work-continuation.ts +26 -13
- package/src/testing/test-governance.ts +8 -40
- package/assets/reference-configs/heartbeat-triage.md +0 -57
- package/assets/templates/helpers/harness-trace-grade.sh +0 -150
- package/assets/templates/helpers/heartbeat-triage.sh +0 -361
- package/assets/templates/helpers/maintenance-triage.sh +0 -167
- package/docs/architecture/current/approved-target-architecture.zh-CN.md +0 -1501
- package/docs/architecture/current/architecture-invariants.md +0 -331
- package/docs/architecture/current/automation-and-schedule-engine.md +0 -459
- package/docs/architecture/current/codegraph-context-provider.md +0 -238
- package/docs/architecture/current/control-plane-authority-inventory.md +0 -94
- package/docs/architecture/current/controller-repository-decoupling.md +0 -148
- package/docs/architecture/current/dispatch-and-agent-strategy.md +0 -373
- package/docs/architecture/current/entity-model.md +0 -612
- package/docs/architecture/current/external-plugin-governance.md +0 -396
- package/docs/architecture/current/failure-recovery.md +0 -447
- package/docs/architecture/current/governance.md +0 -213
- package/docs/architecture/current/human-interaction-plane.md +0 -87
- package/docs/architecture/current/implementation-status.md +0 -128
- package/docs/architecture/current/job-and-run-lifecycle.md +0 -413
- package/docs/architecture/current/legacy-governance.md +0 -58
- package/docs/architecture/current/local-workspace-target-grants.md +0 -70
- package/docs/architecture/current/migration-roadmap.md +0 -105
- package/docs/architecture/current/multi-repository-execution.md +0 -371
- package/docs/architecture/current/operations-runbook.md +0 -45
- package/docs/architecture/current/peer-supercontroller-runtime.md +0 -32
- package/docs/architecture/current/personal-assistant-plugin-baseline.md +0 -206
- package/docs/architecture/current/repository-access-modes.md +0 -102
- package/docs/architecture/current/route-policy.md +0 -15
- package/docs/architecture/current/route-session-concurrency.md +0 -85
- package/docs/architecture/current/runtime-architecture-simplification.md +0 -214
- package/docs/architecture/current/runtime-directory-map.md +0 -49
- package/docs/architecture/current/runtime-health-and-resource-lifecycle.md +0 -85
- package/docs/architecture/current/scheduler-and-resource-claims.md +0 -422
- package/docs/architecture/current/session-aware-execution-and-authorization.md +0 -77
- package/docs/architecture/current/system-overview.md +0 -169
- package/docs/architecture/current/target-requirements-traceability.md +0 -202
- package/docs/architecture/current/thin-harness-governance-policy.md +0 -32
- package/docs/architecture/current/thin-harness-v1.md +0 -321
- package/docs/architecture/current/verification-and-release-gates.md +0 -391
- package/docs/forge-autonomous-goal-loop.md +0 -9
- package/docs/forge-provider-configuration.md +0 -7
- package/docs/operations/chatgpt-personal-assistant-runtime.md +0 -93
- package/docs/operations/gmail-assistant-routines.md +0 -186
- package/docs/operations/personal-assistant-operator-guide.md +0 -61
- package/docs/operations/provider-configuration.md +0 -31
- package/scripts/harness-trace-grade.sh +0 -150
- package/scripts/heartbeat-triage.sh +0 -361
- package/scripts/maintenance-triage.sh +0 -167
- package/src/cli/local-bridge/frontend/src/features/settings/SettingsView.tsx +0 -2
- package/src/cli/repositories/goal-registry.ts +0 -239
- package/src/runtime/assistant/action-proposals.ts +0 -279
- package/src/runtime/assistant/intent.ts +0 -395
- package/src/runtime/assistant/model-provider.ts +0 -460
- package/src/runtime/assistant/openapi.ts +0 -223
- package/src/runtime/assistant/policy.ts +0 -89
- package/src/runtime/assistant/readiness.ts +0 -258
- package/src/runtime/assistant/routine-runtime.ts +0 -477
- package/src/runtime/assistant/schedule-binding.ts +0 -171
- package/src/runtime/assistant/standing-grants.ts +0 -375
- package/src/runtime/assistant/store.ts +0 -243
- package/src/runtime/assistant/types.ts +0 -135
- package/src/runtime/control-plane/goal-loop/config-facade.ts +0 -700
- package/src/runtime/control-plane/goal-loop/config-store.ts +0 -570
- package/src/runtime/control-plane/goal-loop/config-types.ts +0 -213
- package/src/runtime/control-plane/goal-loop/executor-router.ts +0 -109
- package/src/runtime/control-plane/goal-loop/failure-taxonomy.ts +0 -149
- package/src/runtime/control-plane/goal-loop/goal-contract-store.ts +0 -304
- package/src/runtime/control-plane/goal-loop/goal-loop-engine.ts +0 -1001
- package/src/runtime/control-plane/goal-loop/handoff-packet.ts +0 -160
- package/src/runtime/control-plane/goal-loop/index.ts +0 -13
- package/src/runtime/control-plane/goal-loop/local-tools.ts +0 -290
- package/src/runtime/control-plane/goal-loop/policy-gates.ts +0 -203
- package/src/runtime/control-plane/goal-loop/provider-dispatch.ts +0 -93
- package/src/runtime/control-plane/goal-loop/provider-registry.ts +0 -545
- package/src/runtime/control-plane/goal-loop/types.ts +0 -378
- package/src/runtime/control-plane/startup-recovery.ts +0 -198
- package/src/runtime/gateway/mcp/resource-policy.ts +0 -186
- package/src/runtime/personal-assistant/gmail-triage-manager.ts +0 -231
- package/src/runtime/personal-assistant/reporting-runtime.ts +0 -344
- package/src/runtime/personal-assistant/triage-runtime.ts +0 -748
- package/src/runtime/workflow/findings/store.ts +0 -204
- package/src/runtime/workflow/findings/types.ts +0 -58
- package/src/runtime/workflow/portfolio/engine.ts +0 -20
- package/src/runtime/workflow/portfolio/store.ts +0 -185
- package/src/runtime/workflow/portfolio/types.ts +0 -29
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
import { execFile,
|
|
2
|
-
import {
|
|
3
|
-
import { fileURLToPath } from 'url';
|
|
4
|
-
import { dirname, join, resolve } from 'path';
|
|
1
|
+
import { execFile, type ChildProcess } from 'child_process';
|
|
2
|
+
import { resolve } from 'path';
|
|
5
3
|
import { cpus, freemem, loadavg } from 'os';
|
|
6
4
|
import { listRepositories } from '../../../cli/repositories/registry';
|
|
7
|
-
import { resolveControllerHome } from '../../../cli/repositories/controller-home';
|
|
8
5
|
import { writeAgentExecutableReadinessSnapshot } from '../../../cli/agent-jobs/executable-resolver';
|
|
9
6
|
import { withControllerLock } from '../../../cli/repositories/locks';
|
|
10
7
|
import {
|
|
@@ -13,25 +10,13 @@ import {
|
|
|
13
10
|
runtimeWriteClaimEnvironment,
|
|
14
11
|
} from '../../root/write-fence';
|
|
15
12
|
import {
|
|
16
|
-
attachExecutionWorker,
|
|
17
|
-
executionJobRoot,
|
|
18
13
|
getExecutionJob,
|
|
19
14
|
listActiveExecutionJobs,
|
|
20
|
-
markExecutionJobSchedulerObserved,
|
|
21
|
-
transitionExecutionJob,
|
|
22
|
-
updateExecutionJob,
|
|
23
15
|
} from '../../execution/jobs/store';
|
|
24
|
-
import type
|
|
25
|
-
import { tryRecoverJobFromWorkerReceipt } from '../../execution/jobs/receipt-recovery';
|
|
26
|
-
import { releaseExecutionLeases } from '../../resources/leases/store';
|
|
16
|
+
import { type ExecutionWorkerLifecycle } from '../../execution/jobs/types';
|
|
27
17
|
import { RepoActorRegistry } from '../repo-actor/registry';
|
|
28
18
|
import { reconcileExecutionJobsAsync } from './reconciliation';
|
|
29
|
-
import {
|
|
30
|
-
import { tickPortfolioWorkflows } from '../../workflow/portfolio/engine';
|
|
31
|
-
import { tickGoalLoopsForController } from '../goal-loop';
|
|
32
|
-
import { readJsonFile, writeJsonAtomic } from '../../shared/json-files';
|
|
33
|
-
import { resolveBunExecutable } from '../../shared/process-environment';
|
|
34
|
-
import { isProcessAlive, terminateProcessTree } from '../../shared/process-tree';
|
|
19
|
+
import { isProcessAlive } from '../../shared/process-tree';
|
|
35
20
|
import { readSchedulerWakeSignal, waitForSchedulerWakeSignal } from './wake-signal';
|
|
36
21
|
import { cleanupControllerRuntimeState } from '../runtime-cleanup';
|
|
37
22
|
import { reconcileTerminalWorkCleanups } from '../execution/work-terminal-cleanup';
|
|
@@ -39,39 +24,73 @@ import { gcTerminalProcesses } from '../../execution/process-runtime/gc';
|
|
|
39
24
|
import { reconcilePendingWorkValidations } from '../execution/work-validation-reconciler';
|
|
40
25
|
import { reconcilePendingEditValidations } from '../execution/edit-validation-coordinator';
|
|
41
26
|
import { schedulerDispatchAllowed } from '../facade/work-admission-policy';
|
|
42
|
-
import { rebuildRepositoryProjection, refreshRepositoryProjectionForRepository } from '../../projections/materialized-view';
|
|
43
|
-
import { readRepositoryGitStatusSample, sampleRepositoryGitStatusForRepositories } from '../../projections/git-status-sampler';
|
|
44
27
|
import {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
} from '
|
|
28
|
+
runSchedulerControllerRoundRecovery,
|
|
29
|
+
runSchedulerPeriodicCleanup,
|
|
30
|
+
runSchedulerValidationReconciliation,
|
|
31
|
+
} from './maintenance';
|
|
32
|
+
import { planSchedulerSourceSampling } from './source-scan';
|
|
33
|
+
import { runSchedulerDurableAdmission } from './durable-admission';
|
|
34
|
+
import { sampleRepositoryGitStatusForRepositories } from '../../projections/git-status-sampler';
|
|
35
|
+
import { selectExecutionJobDispatchRepositories } from '../dispatch-priority';
|
|
36
|
+
import {
|
|
37
|
+
buildSchedulerWorkerLaunchDescriptor,
|
|
38
|
+
resolveSchedulerWorkerCommand,
|
|
39
|
+
selectSchedulerWorkerEnvironment,
|
|
40
|
+
} from './worker-launch';
|
|
41
|
+
import {
|
|
42
|
+
consumeSchedulerDispatchCapacity,
|
|
43
|
+
createSchedulerDispatchCapacity,
|
|
44
|
+
schedulerDispatchCapacityAllows,
|
|
45
|
+
} from './dispatch-capacity';
|
|
46
|
+
import {
|
|
47
|
+
refreshSchedulerRepositoryProjections,
|
|
48
|
+
selectSchedulerDirtyProjectionRepositories,
|
|
49
|
+
} from './projection-refresh';
|
|
50
|
+
import { createSchedulerWorkerStderrCapture } from './worker-stderr';
|
|
51
|
+
import { persistSchedulerWorkerAttachment } from './worker-attachment';
|
|
52
|
+
import {
|
|
53
|
+
cleanupSchedulerWorkerProcesses,
|
|
54
|
+
registerSchedulerWorkerProcess,
|
|
55
|
+
spawnSchedulerWorkerProcess,
|
|
56
|
+
wireSchedulerWorkerProcess,
|
|
57
|
+
} from './worker-process';
|
|
58
|
+
import { reconcileSchedulerWorkerExit } from './worker-exit-reconciler';
|
|
59
|
+
import { persistSchedulerSpawnedWorkerLifecycle } from './worker-lifecycle-store';
|
|
60
|
+
import {
|
|
61
|
+
buildSchedulerWorkerExitedLifecycle,
|
|
62
|
+
buildSchedulerWorkerSpawnFailureLifecycle,
|
|
63
|
+
buildSchedulerWorkerSpawnedLifecycle,
|
|
64
|
+
} from './worker-lifecycle';
|
|
65
|
+
import { normalizeSchedulerConfig, type SchedulerConfig } from './config';
|
|
66
|
+
import {
|
|
67
|
+
readSchedulerHealthSnapshot,
|
|
68
|
+
restoreSchedulerState,
|
|
69
|
+
writeSchedulerHealthSnapshot,
|
|
70
|
+
} from './state';
|
|
71
|
+
export { normalizeSchedulerConfig } from './config';
|
|
72
|
+
export type { SchedulerConfig } from './config';
|
|
73
|
+
export {
|
|
74
|
+
buildSchedulerHealthSnapshot,
|
|
75
|
+
readSchedulerHealthSnapshot,
|
|
76
|
+
restoreSchedulerState,
|
|
77
|
+
} from './state';
|
|
78
|
+
export type {
|
|
79
|
+
SchedulerHealthSnapshot,
|
|
80
|
+
SchedulerRestoredState,
|
|
81
|
+
SchedulerStateSnapshotInput,
|
|
82
|
+
} from './state';
|
|
83
|
+
export {
|
|
84
|
+
buildSchedulerWorkerLaunchDescriptor,
|
|
85
|
+
resolveSchedulerWorkerCommand,
|
|
86
|
+
resolveSchedulerWorkerExecutable,
|
|
87
|
+
selectSchedulerWorkerEnvironment,
|
|
88
|
+
} from './worker-launch';
|
|
89
|
+
export type { SchedulerWorkerCommand, SchedulerWorkerLaunchDescriptor } from './worker-launch';
|
|
90
|
+
export { selectSchedulerSourceScanRepositories } from './source-scan';
|
|
49
91
|
|
|
50
92
|
const DARWIN_MEMORY_SAMPLE_TTL_MS = 5_000;
|
|
51
|
-
const MAX_WORKER_STDERR_BYTES = 16 * 1024;
|
|
52
|
-
const WORKER_ENVIRONMENT_KEYS = [
|
|
53
|
-
'PATH',
|
|
54
|
-
'HOME',
|
|
55
|
-
'BUN_INSTALL',
|
|
56
|
-
'NODE_OPTIONS',
|
|
57
|
-
'FORGE_CONTROLLER_HOME',
|
|
58
|
-
'FORGE_CONTROLLER_RUNTIME_SOURCE_ROOT',
|
|
59
|
-
'FORGE_EXECUTION_WORKER',
|
|
60
|
-
'FORGE_RUNTIME_INSTANCE_ID',
|
|
61
|
-
'FORGE_RUNTIME_OWNER_PID',
|
|
62
|
-
'FORGE_RELEASE_AUTHORITY_REVISION',
|
|
63
|
-
// The release fencing token is passed to the actual child environment below,
|
|
64
|
-
// but deliberately excluded from the persisted lifecycle diagnostic.
|
|
65
|
-
'FORGE_RELEASE_ID',
|
|
66
|
-
'FORGE_ARTIFACT_IDENTITY',
|
|
67
|
-
'FORGE_WORKER_PROTOCOL_VERSION',
|
|
68
|
-
] as const;
|
|
69
93
|
const RUNTIME_CLEANUP_INTERVAL_MS = Math.max(30_000, Number(process.env.FORGE_RUNTIME_CLEANUP_INTERVAL_MS ?? 60_000));
|
|
70
|
-
const GIT_STATUS_SAMPLE_INTERVAL_MS = Math.max(1_000, Number(process.env.FORGE_GIT_STATUS_SAMPLE_INTERVAL_MS ?? 5_000));
|
|
71
|
-
const IDLE_REPOSITORY_SCAN_INTERVAL_MS = Math.max(
|
|
72
|
-
GIT_STATUS_SAMPLE_INTERVAL_MS,
|
|
73
|
-
Number(process.env.FORGE_IDLE_REPOSITORY_SCAN_INTERVAL_MS ?? 60_000),
|
|
74
|
-
);
|
|
75
94
|
const DARWIN_RECLAIMABLE_PAGE_LABELS = new Set([
|
|
76
95
|
'Pages free',
|
|
77
96
|
'Pages inactive',
|
|
@@ -132,64 +151,6 @@ export function sampleDarwinAvailableMemoryMb(
|
|
|
132
151
|
});
|
|
133
152
|
}
|
|
134
153
|
|
|
135
|
-
interface SchedulerState {
|
|
136
|
-
schemaVersion: 1;
|
|
137
|
-
updatedAt: string;
|
|
138
|
-
loopStartedAt?: string;
|
|
139
|
-
lastHeartbeatAt?: string;
|
|
140
|
-
heartbeatTimeoutMs?: number;
|
|
141
|
-
lastTickAt?: string;
|
|
142
|
-
lastDispatchAt?: string;
|
|
143
|
-
lastReconcileAt?: string;
|
|
144
|
-
lastSourceScanAt?: string;
|
|
145
|
-
lastSourceScanRepoCount?: number;
|
|
146
|
-
sourceScansAvoided?: number;
|
|
147
|
-
lastRepoDispatch: Record<string, number>;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
function schedulerStatePath(controllerHome: string): string {
|
|
151
|
-
return join(resolveControllerHome(controllerHome), 'scheduler', 'state.json');
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
export interface SchedulerHealthSnapshot {
|
|
155
|
-
schemaVersion: 1;
|
|
156
|
-
updatedAt: string;
|
|
157
|
-
loopStartedAt?: string;
|
|
158
|
-
lastHeartbeatAt?: string;
|
|
159
|
-
heartbeatTimeoutMs?: number;
|
|
160
|
-
lastTickAt?: string;
|
|
161
|
-
lastDispatchAt?: string;
|
|
162
|
-
lastReconcileAt?: string;
|
|
163
|
-
lastSourceScanAt?: string;
|
|
164
|
-
lastSourceScanRepoCount?: number;
|
|
165
|
-
sourceScansAvoided?: number;
|
|
166
|
-
lastRepoDispatch: Record<string, number>;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
export function readSchedulerHealthSnapshot(controllerHome: string): SchedulerHealthSnapshot {
|
|
170
|
-
return readJsonFile<SchedulerHealthSnapshot>(schedulerStatePath(controllerHome), {
|
|
171
|
-
schemaVersion: 1,
|
|
172
|
-
updatedAt: new Date().toISOString(),
|
|
173
|
-
lastRepoDispatch: {},
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
export interface SchedulerConfig {
|
|
178
|
-
maxWorkers: number;
|
|
179
|
-
maxConcurrentRepositories: number;
|
|
180
|
-
pollIntervalMs: number;
|
|
181
|
-
idleBackoffMaxMs: number;
|
|
182
|
-
heartbeatIntervalMs: number;
|
|
183
|
-
heartbeatTimeoutMs: number;
|
|
184
|
-
maxHeavyChecks: number;
|
|
185
|
-
maxAgentProcesses: number;
|
|
186
|
-
maxCodexProcesses: number;
|
|
187
|
-
maxClaudeProcesses: number;
|
|
188
|
-
maxGitHubProcesses: number;
|
|
189
|
-
minFreeMemoryMb: number;
|
|
190
|
-
maxLoadPerCpu: number;
|
|
191
|
-
}
|
|
192
|
-
|
|
193
154
|
export interface SchedulerRuntimeBinding {
|
|
194
155
|
controllerPid?: number;
|
|
195
156
|
runtimeSourceRoot?: string;
|
|
@@ -198,14 +159,6 @@ export interface SchedulerRuntimeBinding {
|
|
|
198
159
|
fatalOnTickError?: boolean;
|
|
199
160
|
}
|
|
200
161
|
|
|
201
|
-
export function resolveSchedulerWorkerExecutable(
|
|
202
|
-
isBun: boolean = Boolean(process.versions.bun),
|
|
203
|
-
execPath: string = process.execPath,
|
|
204
|
-
env: NodeJS.ProcessEnv = process.env,
|
|
205
|
-
): string {
|
|
206
|
-
return isBun ? resolveBunExecutable(execPath, env) : execPath;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
162
|
export class GlobalScheduler {
|
|
210
163
|
private readonly controllerHome: string;
|
|
211
164
|
private readonly actors: RepoActorRegistry;
|
|
@@ -216,8 +169,6 @@ export class GlobalScheduler {
|
|
|
216
169
|
private readonly workerEntrypoint?: string;
|
|
217
170
|
private readonly fatalOnTickError: boolean;
|
|
218
171
|
private lastScheduleTick = 0;
|
|
219
|
-
private lastPortfolioTick = 0;
|
|
220
|
-
private lastGoalLoopTick = 0;
|
|
221
172
|
private lastReconcile = 0;
|
|
222
173
|
private lastPersistedAt = 0;
|
|
223
174
|
private readonly lastRepoDispatch = new Map<string, number>();
|
|
@@ -248,99 +199,44 @@ export class GlobalScheduler {
|
|
|
248
199
|
) {
|
|
249
200
|
this.controllerHome = controllerHome;
|
|
250
201
|
this.controllerPid = runtime.controllerPid ?? process.pid;
|
|
251
|
-
this.
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
idleBackoffMaxMs * 4,
|
|
258
|
-
config.heartbeatTimeoutMs ?? Number(process.env.FORGE_SCHEDULER_HEARTBEAT_TIMEOUT_MS ?? 60_000),
|
|
259
|
-
);
|
|
260
|
-
this.config = {
|
|
261
|
-
maxWorkers: Math.max(1, config.maxWorkers ?? Number(process.env.FORGE_MAX_WORKERS ?? 4)),
|
|
262
|
-
maxConcurrentRepositories: Math.max(1, config.maxConcurrentRepositories ?? Number(process.env.FORGE_MAX_ACTIVE_REPOS ?? 4)),
|
|
263
|
-
pollIntervalMs,
|
|
264
|
-
idleBackoffMaxMs,
|
|
265
|
-
heartbeatIntervalMs,
|
|
266
|
-
heartbeatTimeoutMs,
|
|
267
|
-
maxHeavyChecks: Math.max(1, config.maxHeavyChecks ?? Number(process.env.FORGE_MAX_HEAVY_CHECKS ?? 2)),
|
|
268
|
-
maxAgentProcesses: Math.max(1, config.maxAgentProcesses ?? Number(process.env.FORGE_MAX_AGENT_PROCESSES ?? 4)),
|
|
269
|
-
maxCodexProcesses: Math.max(1, config.maxCodexProcesses ?? Number(process.env.FORGE_MAX_CODEX_PROCESSES ?? 3)),
|
|
270
|
-
maxClaudeProcesses: Math.max(1, config.maxClaudeProcesses ?? Number(process.env.FORGE_MAX_CLAUDE_PROCESSES ?? 2)),
|
|
271
|
-
maxGitHubProcesses: Math.max(1, config.maxGitHubProcesses ?? Number(process.env.FORGE_MAX_GITHUB_PROCESSES ?? 2)),
|
|
272
|
-
minFreeMemoryMb: Math.max(64, config.minFreeMemoryMb ?? Number(process.env.FORGE_MIN_FREE_MEMORY_MB ?? 512)),
|
|
273
|
-
maxLoadPerCpu: Math.max(0.25, config.maxLoadPerCpu ?? Number(process.env.FORGE_MAX_LOAD_PER_CPU ?? 1.5)),
|
|
274
|
-
};
|
|
202
|
+
this.config = normalizeSchedulerConfig(config);
|
|
203
|
+
// The per-repository mailbox must not impose a hidden concurrency ceiling
|
|
204
|
+
// below the scheduler's global worker budget. Resource leases still serialize
|
|
205
|
+
// conflicting workspace/path/integration claims, while independent jobs in a
|
|
206
|
+
// busy single repository may use the available global worker capacity.
|
|
207
|
+
this.actors = new RepoActorRegistry(controllerHome, { maxConcurrentWorkers: this.config.maxWorkers });
|
|
275
208
|
this.runtimeSourceRoot = runtime.runtimeSourceRoot ? resolve(runtime.runtimeSourceRoot) : undefined;
|
|
276
209
|
this.workerEntrypoint = runtime.workerEntrypoint ? resolve(runtime.workerEntrypoint) : undefined;
|
|
277
210
|
this.fatalOnTickError = runtime.fatalOnTickError === true;
|
|
278
|
-
const
|
|
279
|
-
this.lastSourceScanAt =
|
|
280
|
-
this.lastSourceScanRepoCount =
|
|
281
|
-
this.sourceScansAvoided =
|
|
282
|
-
for (const [repoId, timestamp] of
|
|
283
|
-
if (Number.isFinite(timestamp)) this.lastRepoDispatch.set(repoId, timestamp);
|
|
284
|
-
}
|
|
211
|
+
const restoredState = restoreSchedulerState(readSchedulerHealthSnapshot(controllerHome));
|
|
212
|
+
this.lastSourceScanAt = restoredState.lastSourceScanAt;
|
|
213
|
+
this.lastSourceScanRepoCount = restoredState.lastSourceScanRepoCount;
|
|
214
|
+
this.sourceScansAvoided = restoredState.sourceScansAvoided;
|
|
215
|
+
for (const [repoId, timestamp] of restoredState.lastRepoDispatch) this.lastRepoDispatch.set(repoId, timestamp);
|
|
285
216
|
}
|
|
286
217
|
|
|
287
218
|
private persistState(force = false): void {
|
|
288
219
|
const now = Date.now();
|
|
289
220
|
if (!force && now - this.lastPersistedAt < 1_000) return;
|
|
290
221
|
this.lastPersistedAt = now;
|
|
291
|
-
|
|
292
|
-
schemaVersion: 1,
|
|
293
|
-
updatedAt: new Date().toISOString(),
|
|
222
|
+
writeSchedulerHealthSnapshot(this.controllerHome, {
|
|
294
223
|
loopStartedAt: this.loopStartedAt,
|
|
295
224
|
lastHeartbeatAt: this.lastHeartbeatAt,
|
|
296
225
|
heartbeatTimeoutMs: this.config.heartbeatTimeoutMs,
|
|
297
226
|
lastTickAt: this.lastTickAt,
|
|
298
227
|
lastDispatchAt: this.lastDispatchAt,
|
|
299
228
|
lastReconcileAt: this.lastReconcileAt,
|
|
300
|
-
lastSourceScanAt: this.lastSourceScanAt
|
|
229
|
+
lastSourceScanAt: this.lastSourceScanAt,
|
|
301
230
|
lastSourceScanRepoCount: this.lastSourceScanRepoCount,
|
|
302
231
|
sourceScansAvoided: this.sourceScansAvoided,
|
|
303
|
-
lastRepoDispatch:
|
|
304
|
-
}
|
|
232
|
+
lastRepoDispatch: this.lastRepoDispatch,
|
|
233
|
+
}, now);
|
|
305
234
|
}
|
|
306
235
|
|
|
307
236
|
private pidAlive(pid: number | undefined): boolean {
|
|
308
237
|
return isProcessAlive(pid);
|
|
309
238
|
}
|
|
310
239
|
|
|
311
|
-
private async cleanupSpawnedWorkers(): Promise<void> {
|
|
312
|
-
const workers = [...this.children.values()];
|
|
313
|
-
this.children.clear();
|
|
314
|
-
await Promise.all(workers.map((child) => terminateProcessTree(child.pid)));
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
private workerCommand(): { entry: string; loader: string; cwd: string } {
|
|
318
|
-
const sourceEntry = this.runtimeSourceRoot
|
|
319
|
-
? join(this.runtimeSourceRoot, 'src', 'runtime', 'execution', 'workers', 'worker-entry.ts')
|
|
320
|
-
: fileURLToPath(new URL('../../execution/workers/worker-entry.ts', import.meta.url));
|
|
321
|
-
const loader = this.runtimeSourceRoot
|
|
322
|
-
? join(this.runtimeSourceRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs')
|
|
323
|
-
: fileURLToPath(new URL('../../shared/node-ts-loader.mjs', import.meta.url));
|
|
324
|
-
const entry = this.workerEntrypoint ?? sourceEntry;
|
|
325
|
-
const cwd = this.runtimeSourceRoot ?? process.cwd();
|
|
326
|
-
if (!existsSync(entry)) throw new Error(`WORKER_ENTRYPOINT_MISSING: ${entry}`);
|
|
327
|
-
if (!process.versions.bun && !existsSync(loader)) throw new Error(`WORKER_LOADER_MISSING: ${loader}`);
|
|
328
|
-
return { entry, loader, cwd };
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
private workerEnvironment(): Record<string, string | undefined> {
|
|
332
|
-
return Object.fromEntries(WORKER_ENVIRONMENT_KEYS.map((key) => [key, process.env[key]]));
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
private persistSpawnedWorker(repoId: string, jobId: string, lifecycle: ExecutionWorkerLifecycle): void {
|
|
336
|
-
try {
|
|
337
|
-
updateExecutionJob(this.controllerHome, repoId, jobId, (current) => {
|
|
338
|
-
if (!['dispatched', 'running'].includes(current.status) || current.workerPid !== undefined) return current;
|
|
339
|
-
return { ...current, workerLifecycle: lifecycle };
|
|
340
|
-
});
|
|
341
|
-
} catch { /* the Job may have been superseded or made terminal */ }
|
|
342
|
-
}
|
|
343
|
-
|
|
344
240
|
private async recordWorkerExit(
|
|
345
241
|
repoId: string,
|
|
346
242
|
jobId: string,
|
|
@@ -353,82 +249,29 @@ export class GlobalScheduler {
|
|
|
353
249
|
stderrTruncated: boolean,
|
|
354
250
|
startupError?: string,
|
|
355
251
|
): Promise<void> {
|
|
356
|
-
const diagnosticLifecycle
|
|
357
|
-
|
|
358
|
-
|
|
252
|
+
const diagnosticLifecycle = buildSchedulerWorkerExitedLifecycle({
|
|
253
|
+
lifecycle,
|
|
254
|
+
childPid: child?.pid,
|
|
359
255
|
exitCode,
|
|
360
256
|
signal,
|
|
361
|
-
workerPid: child?.pid ?? lifecycle.workerPid,
|
|
362
|
-
processGroupId: lifecycle.processGroupId ?? (process.platform !== 'win32' ? child?.pid : undefined),
|
|
363
257
|
stderr,
|
|
364
258
|
stderrTruncated,
|
|
365
|
-
|
|
366
|
-
};
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
// lease ownership, finalize from that receipt and never emit WORKER_EXITED.
|
|
382
|
-
const recovered = tryRecoverJobFromWorkerReceipt(this.controllerHome, current);
|
|
383
|
-
if (recovered) {
|
|
384
|
-
updateExecutionJob(this.controllerHome, repoId, jobId, (latest) => ({ ...latest, workerLifecycle: mergedLifecycle }));
|
|
385
|
-
try { rebuildRepositoryProjection(this.controllerHome, repoId); } catch { /* the next scheduler/status read can retry */ }
|
|
386
|
-
return;
|
|
387
|
-
}
|
|
388
|
-
const rechecked = getExecutionJob(this.controllerHome, repoId, jobId);
|
|
389
|
-
if (rechecked.attempt !== attempt) return;
|
|
390
|
-
const recheckedLifecycle = rechecked.workerLifecycle ?? lifecycle;
|
|
391
|
-
const recheckedMerged = { ...recheckedLifecycle, ...diagnosticLifecycle };
|
|
392
|
-
if (['succeeded', 'failed', 'timed_out', 'cancelled', 'orphaned', 'stale', 'human_attention_required'].includes(rechecked.status)) {
|
|
393
|
-
updateExecutionJob(this.controllerHome, repoId, jobId, (latest) => ({ ...latest, workerLifecycle: recheckedMerged }));
|
|
394
|
-
try { rebuildRepositoryProjection(this.controllerHome, repoId); } catch { /* the next scheduler/status read can retry */ }
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
const details: Record<string, unknown> = {
|
|
399
|
-
workerLostReason: startupError ? 'spawn_failed' : 'process_exit',
|
|
400
|
-
executable: recheckedMerged.executable,
|
|
401
|
-
cwd: recheckedMerged.cwd,
|
|
402
|
-
exitCode,
|
|
403
|
-
signal,
|
|
404
|
-
stderr,
|
|
405
|
-
stderrTruncated,
|
|
406
|
-
stderrPath: recheckedMerged.stderrPath,
|
|
407
|
-
processGroupId: recheckedMerged.processGroupId,
|
|
408
|
-
ownerPid: recheckedMerged.ownerPid,
|
|
409
|
-
runtimeInstanceId: recheckedMerged.runtimeInstanceId,
|
|
410
|
-
releaseAuthorityRevision: recheckedMerged.releaseAuthorityRevision,
|
|
411
|
-
releaseId: recheckedMerged.releaseId,
|
|
412
|
-
artifactIdentity: recheckedMerged.artifactIdentity,
|
|
413
|
-
workerProtocolVersion: recheckedMerged.workerProtocolVersion,
|
|
414
|
-
attempt: rechecked.attempt,
|
|
415
|
-
maxAttempts: rechecked.maxAttempts,
|
|
416
|
-
...(startupError ? { startupError } : {}),
|
|
417
|
-
};
|
|
418
|
-
const stderrSummary = stderr.trim() ? ` Worker stderr: ${stderr.trim()}` : '';
|
|
419
|
-
const startupSummary = startupError ? ` Startup error: ${startupError}.` : '';
|
|
420
|
-
const message = `Execution Worker ${recheckedMerged.executable} exited before completion (cwd ${recheckedMerged.cwd}, exit code ${exitCode ?? 'unknown'}${signal ? `, signal ${signal}` : ''}).${startupSummary}${stderrSummary}`;
|
|
421
|
-
releaseExecutionLeases(this.controllerHome, repoId, jobId, rechecked.leaseRefs);
|
|
422
|
-
const retryable = rechecked.attempt < rechecked.maxAttempts;
|
|
423
|
-
transitionExecutionJob(this.controllerHome, repoId, jobId, retryable ? 'queued' : 'failed', {
|
|
424
|
-
workerPid: undefined,
|
|
425
|
-
heartbeatAt: undefined,
|
|
426
|
-
leaseRefs: [],
|
|
427
|
-
workerLifecycle: recheckedMerged,
|
|
428
|
-
error: { code: startupError ? 'WORKER_START_FAILED' : 'WORKER_EXITED', message, retryable, details },
|
|
429
|
-
});
|
|
430
|
-
try { rebuildRepositoryProjection(this.controllerHome, repoId); } catch { /* the next scheduler/status read can retry */ }
|
|
431
|
-
} catch { /* the Job may have been finalized by the Worker or reconciliation */ }
|
|
259
|
+
startupError,
|
|
260
|
+
});
|
|
261
|
+
reconcileSchedulerWorkerExit({
|
|
262
|
+
controllerHome: this.controllerHome,
|
|
263
|
+
repoId,
|
|
264
|
+
jobId,
|
|
265
|
+
attempt,
|
|
266
|
+
childPid: child?.pid,
|
|
267
|
+
lifecycle,
|
|
268
|
+
diagnosticLifecycle,
|
|
269
|
+
exitCode,
|
|
270
|
+
signal,
|
|
271
|
+
stderr,
|
|
272
|
+
stderrTruncated,
|
|
273
|
+
startupError,
|
|
274
|
+
});
|
|
432
275
|
}
|
|
433
276
|
|
|
434
277
|
private spawnWorker(repoId: string, jobId: string): boolean {
|
|
@@ -441,129 +284,119 @@ export class GlobalScheduler {
|
|
|
441
284
|
if (!['dispatched', 'running'].includes(current.status)) return false;
|
|
442
285
|
if (current.workerPid && this.pidAlive(current.workerPid)) return false;
|
|
443
286
|
const command = (() => {
|
|
444
|
-
try {
|
|
287
|
+
try {
|
|
288
|
+
return resolveSchedulerWorkerCommand({
|
|
289
|
+
runtimeSourceRoot: this.runtimeSourceRoot,
|
|
290
|
+
workerEntrypoint: this.workerEntrypoint,
|
|
291
|
+
});
|
|
292
|
+
} catch (error) {
|
|
445
293
|
const message = error instanceof Error ? error.message : String(error);
|
|
446
|
-
const lifecycle
|
|
294
|
+
const lifecycle = buildSchedulerWorkerSpawnFailureLifecycle({
|
|
447
295
|
executable: process.execPath,
|
|
448
|
-
args: [],
|
|
449
296
|
cwd: this.runtimeSourceRoot ?? process.cwd(),
|
|
450
|
-
environment:
|
|
297
|
+
environment: selectSchedulerWorkerEnvironment(process.env),
|
|
451
298
|
ownerPid: this.controllerPid,
|
|
452
299
|
attempt: current.attempt,
|
|
453
300
|
maxAttempts: current.maxAttempts,
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
301
|
+
});
|
|
302
|
+
persistSchedulerSpawnedWorkerLifecycle({
|
|
303
|
+
controllerHome: this.controllerHome,
|
|
304
|
+
repoId,
|
|
305
|
+
jobId,
|
|
306
|
+
lifecycle,
|
|
307
|
+
});
|
|
458
308
|
void this.recordWorkerExit(repoId, jobId, current.attempt, undefined, lifecycle, null, null, '', false, message);
|
|
459
309
|
return undefined;
|
|
460
310
|
}
|
|
461
311
|
})();
|
|
462
312
|
if (!command) return false;
|
|
463
|
-
const bun = Boolean(process.versions.bun);
|
|
464
|
-
const workerExecutable = resolveSchedulerWorkerExecutable(bun);
|
|
465
313
|
// Pass the parent's captured Runtime owner and whole-release claim. A Worker
|
|
466
314
|
// never re-adopts current authority after spawn; owner/release rotation fences it.
|
|
467
315
|
const writeClaim = getRuntimeWriteClaim();
|
|
468
|
-
const
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
const stderrPath = join(executionJobRoot(this.controllerHome, repoId), 'worker-stderr', `${jobId}-attempt-${current.attempt}.log`);
|
|
486
|
-
mkdirSync(dirname(stderrPath), { recursive: true });
|
|
487
|
-
writeFileSync(stderrPath, '', 'utf8');
|
|
488
|
-
const lifecycle: ExecutionWorkerLifecycle = {
|
|
489
|
-
executable: workerExecutable,
|
|
490
|
-
args,
|
|
491
|
-
cwd: command.cwd,
|
|
492
|
-
environment: Object.fromEntries(WORKER_ENVIRONMENT_KEYS.map((key) => [key, environment[key]])),
|
|
316
|
+
const launch = buildSchedulerWorkerLaunchDescriptor({
|
|
317
|
+
command,
|
|
318
|
+
controllerHome: this.controllerHome,
|
|
319
|
+
repoId,
|
|
320
|
+
jobId,
|
|
321
|
+
controllerPid: this.controllerPid,
|
|
322
|
+
runtimeSourceRoot: this.runtimeSourceRoot,
|
|
323
|
+
writeClaimEnvironment: writeClaim ? runtimeWriteClaimEnvironment(writeClaim) : {},
|
|
324
|
+
});
|
|
325
|
+
const stderrCapture = createSchedulerWorkerStderrCapture({
|
|
326
|
+
controllerHome: this.controllerHome,
|
|
327
|
+
repoId,
|
|
328
|
+
jobId,
|
|
329
|
+
attempt: current.attempt,
|
|
330
|
+
});
|
|
331
|
+
const lifecycle = buildSchedulerWorkerSpawnedLifecycle({
|
|
332
|
+
launch,
|
|
493
333
|
ownerPid: this.controllerPid,
|
|
494
|
-
|
|
334
|
+
releaseIdentity: writeClaim ? {
|
|
495
335
|
runtimeInstanceId: writeClaim.runtimeInstanceId,
|
|
496
336
|
releaseAuthorityRevision: writeClaim.releaseAuthorityRevision,
|
|
497
337
|
releaseId: writeClaim.releaseId,
|
|
498
338
|
artifactIdentity: writeClaim.artifactIdentity,
|
|
499
339
|
workerProtocolVersion: writeClaim.workerProtocolVersion,
|
|
500
|
-
} :
|
|
340
|
+
} : undefined,
|
|
501
341
|
attempt: current.attempt,
|
|
502
342
|
maxAttempts: current.maxAttempts,
|
|
503
|
-
|
|
504
|
-
stderrPath,
|
|
505
|
-
startupState: 'spawned',
|
|
506
|
-
};
|
|
507
|
-
this.persistSpawnedWorker(repoId, jobId, lifecycle);
|
|
508
|
-
let child: ChildProcess;
|
|
509
|
-
try {
|
|
510
|
-
child = spawn(workerExecutable, args, {
|
|
511
|
-
cwd: command.cwd,
|
|
512
|
-
stdio: ['ignore', 'ignore', 'pipe'],
|
|
513
|
-
detached: process.platform !== 'win32',
|
|
514
|
-
env: environment,
|
|
515
|
-
});
|
|
516
|
-
} catch (error) {
|
|
517
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
518
|
-
void this.recordWorkerExit(repoId, jobId, current.attempt, undefined, lifecycle, null, null, '', false, message);
|
|
519
|
-
return false;
|
|
520
|
-
}
|
|
521
|
-
let stderr = '';
|
|
522
|
-
let stderrBytes = 0;
|
|
523
|
-
let stderrTruncated = false;
|
|
524
|
-
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
525
|
-
const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
|
526
|
-
const bytes = Buffer.byteLength(text);
|
|
527
|
-
const remaining = MAX_WORKER_STDERR_BYTES - stderrBytes;
|
|
528
|
-
const accepted = remaining > 0 ? Buffer.from(text).subarray(0, remaining).toString('utf8') : '';
|
|
529
|
-
if (accepted) {
|
|
530
|
-
stderr += accepted;
|
|
531
|
-
stderrBytes += Buffer.byteLength(accepted);
|
|
532
|
-
try { appendFileSync(stderrPath, accepted, 'utf8'); } catch { stderrTruncated = true; }
|
|
533
|
-
}
|
|
534
|
-
if (bytes > Math.max(0, remaining)) stderrTruncated = true;
|
|
343
|
+
stderrPath: stderrCapture.path,
|
|
535
344
|
});
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
345
|
+
persistSchedulerSpawnedWorkerLifecycle({
|
|
346
|
+
controllerHome: this.controllerHome,
|
|
347
|
+
repoId,
|
|
348
|
+
jobId,
|
|
349
|
+
lifecycle,
|
|
350
|
+
});
|
|
351
|
+
const spawned = spawnSchedulerWorkerProcess(launch);
|
|
352
|
+
if (!spawned.ok) {
|
|
353
|
+
void this.recordWorkerExit(
|
|
354
|
+
repoId,
|
|
355
|
+
jobId,
|
|
356
|
+
current.attempt,
|
|
357
|
+
undefined,
|
|
358
|
+
lifecycle,
|
|
359
|
+
null,
|
|
360
|
+
null,
|
|
361
|
+
'',
|
|
362
|
+
false,
|
|
363
|
+
spawned.startupError,
|
|
364
|
+
);
|
|
555
365
|
return false;
|
|
556
366
|
}
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
})
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
367
|
+
const child = spawned.child;
|
|
368
|
+
wireSchedulerWorkerProcess({
|
|
369
|
+
jobId,
|
|
370
|
+
child,
|
|
371
|
+
children: this.children,
|
|
372
|
+
stderrCapture,
|
|
373
|
+
onExit: ({ exitCode, signal, stderr, stderrTruncated, startupError }) => {
|
|
374
|
+
void this.recordWorkerExit(
|
|
375
|
+
repoId,
|
|
376
|
+
jobId,
|
|
377
|
+
current.attempt,
|
|
378
|
+
child,
|
|
379
|
+
lifecycle,
|
|
380
|
+
exitCode,
|
|
381
|
+
signal,
|
|
382
|
+
stderr,
|
|
383
|
+
stderrTruncated,
|
|
384
|
+
startupError,
|
|
385
|
+
);
|
|
386
|
+
},
|
|
387
|
+
});
|
|
388
|
+
return registerSchedulerWorkerProcess({
|
|
389
|
+
jobId,
|
|
390
|
+
child,
|
|
391
|
+
children: this.children,
|
|
392
|
+
attach: (workerPid) => persistSchedulerWorkerAttachment({
|
|
393
|
+
controllerHome: this.controllerHome,
|
|
394
|
+
repoId,
|
|
395
|
+
jobId,
|
|
396
|
+
workerPid,
|
|
397
|
+
lifecycle,
|
|
398
|
+
}),
|
|
399
|
+
});
|
|
567
400
|
}
|
|
568
401
|
|
|
569
402
|
private refreshDarwinAvailableMemoryMb(fallback: number): void {
|
|
@@ -606,102 +439,104 @@ export class GlobalScheduler {
|
|
|
606
439
|
};
|
|
607
440
|
}
|
|
608
441
|
|
|
609
|
-
private agentProvider(job: { payload: { arguments?: Record<string, unknown> } }): 'codex' | 'claude' | 'github-copilot' {
|
|
610
|
-
const agent = job.payload.arguments?.agent;
|
|
611
|
-
if (agent === 'claude' || agent === 'github-copilot') return agent;
|
|
612
|
-
return 'codex';
|
|
613
|
-
}
|
|
614
|
-
|
|
615
442
|
async tick(): Promise<{ activeJobs: number }> {
|
|
616
443
|
const now = Date.now();
|
|
617
444
|
this.lastHeartbeatAt = new Date(now).toISOString();
|
|
618
445
|
this.lastTickAt = this.lastHeartbeatAt;
|
|
619
446
|
this.persistState();
|
|
620
447
|
const repositories = this.repositoryList(this.controllerHome).filter((repo) => repo.enabled && !repo.removedAt);
|
|
448
|
+
let periodicCleanupRan = false;
|
|
621
449
|
if (now - this.lastCleanupAt >= RUNTIME_CLEANUP_INTERVAL_MS) {
|
|
622
450
|
// Advance the interval before cleanup so a failing pass cannot create a
|
|
623
451
|
// tight retry loop on every scheduler tick.
|
|
624
452
|
this.lastCleanupAt = now;
|
|
625
|
-
|
|
626
|
-
this.
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
} catch (error) {
|
|
637
|
-
console.error('[forge cleanup] terminal Work cleanup failed:', error);
|
|
638
|
-
}
|
|
639
|
-
if (repositories.length > 0) {
|
|
640
|
-
const slot = Math.floor(now / RUNTIME_CLEANUP_INTERVAL_MS) % repositories.length;
|
|
641
|
-
const repo = repositories[slot]!;
|
|
642
|
-
const result = this.processGc({ controllerHome: this.controllerHome, repoId: repo.repoId });
|
|
643
|
-
if (!result.ok) console.error('[forge cleanup] Process GC failed:', result.error ?? 'unknown error');
|
|
644
|
-
}
|
|
453
|
+
await runSchedulerPeriodicCleanup({
|
|
454
|
+
controllerHome: this.controllerHome,
|
|
455
|
+
controllerPid: this.controllerPid,
|
|
456
|
+
nowMs: now,
|
|
457
|
+
cleanupIntervalMs: RUNTIME_CLEANUP_INTERVAL_MS,
|
|
458
|
+
repositories,
|
|
459
|
+
runtimeCleanup: this.runtimeCleanup,
|
|
460
|
+
terminalWorkCleanup: this.terminalWorkCleanup,
|
|
461
|
+
processGc: this.processGc,
|
|
462
|
+
});
|
|
463
|
+
periodicCleanupRan = true;
|
|
645
464
|
}
|
|
646
465
|
if (now - this.lastReconcile >= 5_000) {
|
|
647
466
|
await reconcileExecutionJobsAsync(this.controllerHome);
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
}
|
|
655
|
-
const editValidation = await this.editValidationReconcile(this.controllerHome, repository, 200);
|
|
656
|
-
if (editValidation.errors.length > 0) {
|
|
657
|
-
console.error(
|
|
658
|
-
`[forge validation] background EditSession reconciliation reported ${editValidation.errors.length} error(s) for ${repository.repoId}`,
|
|
659
|
-
);
|
|
660
|
-
}
|
|
661
|
-
}
|
|
467
|
+
await runSchedulerValidationReconciliation({
|
|
468
|
+
controllerHome: this.controllerHome,
|
|
469
|
+
repositories,
|
|
470
|
+
workValidationReconcile: this.workValidationReconcile,
|
|
471
|
+
editValidationReconcile: this.editValidationReconcile,
|
|
472
|
+
});
|
|
662
473
|
this.lastReconcile = now;
|
|
663
474
|
this.lastReconcileAt = new Date(now).toISOString();
|
|
664
475
|
}
|
|
665
476
|
const activeJobSnapshot = listActiveExecutionJobs(this.controllerHome);
|
|
666
|
-
const
|
|
667
|
-
const
|
|
668
|
-
|
|
477
|
+
const activeSourceRepoIds = new Set(activeJobSnapshot.map((job) => job.repoId));
|
|
478
|
+
const sourceSampling = planSchedulerSourceSampling({
|
|
479
|
+
repositories,
|
|
480
|
+
activeRepoIds: activeSourceRepoIds,
|
|
481
|
+
nowMs: now,
|
|
482
|
+
lastSourceScanAt: this.lastSourceScanAt,
|
|
483
|
+
lastGitStatusSampleAt: this.lastGitStatusSampleAt,
|
|
484
|
+
});
|
|
485
|
+
const sourceScanRepositories = sourceSampling.sourceScanRepositories;
|
|
486
|
+
const dirtyProjectionRepositories = selectSchedulerDirtyProjectionRepositories(this.controllerHome, repositories);
|
|
487
|
+
if (sourceSampling.shouldSample) {
|
|
488
|
+
// Advance source-sampling state before the filesystem probe so a failing
|
|
489
|
+
// sample cannot create a tight retry loop on every scheduler tick.
|
|
669
490
|
this.lastGitStatusSampleAt = now;
|
|
670
491
|
this.lastSourceScanAt = now;
|
|
671
|
-
this.lastSourceScanRepoCount =
|
|
672
|
-
sampleRepositoryGitStatusForRepositories(this.controllerHome,
|
|
673
|
-
} else if (!sourceScanDue) {
|
|
674
|
-
// fs.watch wakeups cover active mutations; a bounded safety rescan keeps
|
|
675
|
-
// idle repositories fresh without re-running a full source scan per tick.
|
|
676
|
-
this.sourceScansAvoided += repositories.length;
|
|
492
|
+
this.lastSourceScanRepoCount = sourceScanRepositories.length;
|
|
493
|
+
sampleRepositoryGitStatusForRepositories(this.controllerHome, sourceScanRepositories);
|
|
677
494
|
}
|
|
495
|
+
// Dirty projection markers are authoritative maintenance requests. Do not
|
|
496
|
+
// make them wait behind the idle one-repository-per-minute safety scan: only
|
|
497
|
+
// the dirty repositories receive this extra fresh source sample.
|
|
498
|
+
const alreadySampledRepoIds = new Set(
|
|
499
|
+
sourceSampling.shouldSample ? sourceScanRepositories.map((repository) => repository.repoId) : [],
|
|
500
|
+
);
|
|
501
|
+
const dirtyRepositoriesToSample = dirtyProjectionRepositories
|
|
502
|
+
.filter((repository) => !alreadySampledRepoIds.has(repository.repoId));
|
|
503
|
+
if (dirtyRepositoriesToSample.length > 0) {
|
|
504
|
+
sampleRepositoryGitStatusForRepositories(this.controllerHome, dirtyRepositoriesToSample);
|
|
505
|
+
}
|
|
506
|
+
const projectionMaintenanceRepositories = Array.from(new Map(
|
|
507
|
+
[...sourceScanRepositories, ...dirtyProjectionRepositories]
|
|
508
|
+
.map((repository) => [repository.repoId, repository] as const),
|
|
509
|
+
).values());
|
|
510
|
+
this.sourceScansAvoided += sourceSampling.avoidedRepositoryCount;
|
|
678
511
|
// Phase 0 reuses one durable Work admission policy. Cleanup, stale-state
|
|
679
|
-
// reconciliation,
|
|
680
|
-
// ordinary schedule/workflow advancement and Worker dispatch stop here.
|
|
512
|
+
// reconciliation, read-only source sampling, and projection maintenance remain
|
|
513
|
+
// available, but ordinary schedule/workflow advancement and Worker dispatch stop here.
|
|
681
514
|
if (!schedulerDispatchAllowed(this.controllerHome)) {
|
|
515
|
+
refreshSchedulerRepositoryProjections({
|
|
516
|
+
controllerHome: this.controllerHome,
|
|
517
|
+
repositories,
|
|
518
|
+
sourceScanRepositories: projectionMaintenanceRepositories,
|
|
519
|
+
projectionRefreshRepoIds: [],
|
|
520
|
+
controllerPid: this.controllerPid,
|
|
521
|
+
});
|
|
682
522
|
this.lastHeartbeatAt = new Date().toISOString();
|
|
683
523
|
this.persistState(true);
|
|
684
524
|
return { activeJobs: activeJobSnapshot.length };
|
|
685
525
|
}
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
markExecutionJobSchedulerObserved(this.controllerHome, job.repoId, job.jobId);
|
|
693
|
-
} catch {
|
|
694
|
-
// Another scheduler or a terminal transition won the Job-local race.
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
if (now - this.lastScheduleTick >= 30_000) {
|
|
698
|
-
await tickSchedules(this.controllerHome, repositories.map((repo) => repo.repoId));
|
|
699
|
-
this.lastScheduleTick = now;
|
|
700
|
-
}
|
|
701
|
-
if (now - this.lastPortfolioTick >= 1_000) {
|
|
702
|
-
tickPortfolioWorkflows(this.controllerHome);
|
|
703
|
-
this.lastPortfolioTick = now;
|
|
526
|
+
if (periodicCleanupRan) {
|
|
527
|
+
await runSchedulerControllerRoundRecovery({
|
|
528
|
+
controllerHome: this.controllerHome,
|
|
529
|
+
nowMs: now,
|
|
530
|
+
repositories,
|
|
531
|
+
});
|
|
704
532
|
}
|
|
533
|
+
const durableAdmission = await runSchedulerDurableAdmission({
|
|
534
|
+
controllerHome: this.controllerHome,
|
|
535
|
+
repositoryIds: repositories.map((repo) => repo.repoId),
|
|
536
|
+
nowMs: now,
|
|
537
|
+
lastScheduleTickAt: this.lastScheduleTick,
|
|
538
|
+
});
|
|
539
|
+
if (durableAdmission.scheduleTicked) this.lastScheduleTick = now;
|
|
705
540
|
let activeJobs = 0;
|
|
706
541
|
const pendingSpawns: Array<{ repoId: string; jobId: string }> = [];
|
|
707
542
|
const projectionRefreshRepos = new Set<string>();
|
|
@@ -717,67 +552,16 @@ export class GlobalScheduler {
|
|
|
717
552
|
`global-scheduler:${this.controllerPid}`,
|
|
718
553
|
() => {
|
|
719
554
|
const active = listActiveExecutionJobs(this.controllerHome);
|
|
720
|
-
const
|
|
721
|
-
|
|
722
|
-
if (capacity <= 0) return active.length;
|
|
723
|
-
|
|
724
|
-
let heavyCapacity = this.config.maxHeavyChecks
|
|
725
|
-
- reserved.filter((job) => job.type === 'check' || job.type === 'verify-edit').length;
|
|
726
|
-
const reservedAgents = reserved.filter((job) => job.type === 'agent-run' || job.type === 'dispatch-task');
|
|
727
|
-
let agentCapacity = this.config.maxAgentProcesses - reservedAgents.length;
|
|
728
|
-
const providerCapacity = new Map([
|
|
729
|
-
['codex', this.config.maxCodexProcesses - reservedAgents.filter((job) => this.agentProvider(job) === 'codex').length],
|
|
730
|
-
['claude', this.config.maxClaudeProcesses - reservedAgents.filter((job) => this.agentProvider(job) === 'claude').length],
|
|
731
|
-
['github-copilot', this.config.maxGitHubProcesses - reservedAgents.filter((job) => this.agentProvider(job) === 'github-copilot').length],
|
|
732
|
-
] as const);
|
|
733
|
-
if (pressure.pressured) {
|
|
734
|
-
// Under host pressure, keep one recovery slot available so queued read-only
|
|
735
|
-
// or bounded repository work does not stall forever behind a global stop.
|
|
736
|
-
capacity = Math.min(capacity, 1);
|
|
737
|
-
heavyCapacity = Math.min(heavyCapacity, 1);
|
|
738
|
-
agentCapacity = 0;
|
|
739
|
-
providerCapacity.set('codex', 0);
|
|
740
|
-
providerCapacity.set('claude', 0);
|
|
741
|
-
providerCapacity.set('github-copilot', 0);
|
|
742
|
-
}
|
|
743
|
-
if (capacity <= 0) return active.length;
|
|
555
|
+
const capacity = createSchedulerDispatchCapacity(active, this.config, pressure.pressured);
|
|
556
|
+
if (capacity.workers <= 0) return active.length;
|
|
744
557
|
|
|
745
558
|
const scheduleNow = Date.now();
|
|
746
|
-
const
|
|
747
|
-
const
|
|
748
|
-
waiting.map((job) => [job.jobId, rankExecutionJobForDispatch(job, scheduleNow)] as const),
|
|
749
|
-
);
|
|
750
|
-
const compareWaiting = (left: (typeof active)[number], right: (typeof active)[number]): number =>
|
|
751
|
-
compareExecutionJobDispatchRanks(rankByJobId.get(left.jobId)!, rankByJobId.get(right.jobId)!);
|
|
752
|
-
const topByRepo = new Map<string, (typeof active)[number]>();
|
|
753
|
-
for (const job of waiting.slice().sort(compareWaiting)) {
|
|
754
|
-
if (!topByRepo.has(job.repoId)) topByRepo.set(job.repoId, job);
|
|
755
|
-
}
|
|
756
|
-
const repoIds = [...topByRepo.keys()].sort((left, right) => {
|
|
757
|
-
const leftTop = topByRepo.get(left)!;
|
|
758
|
-
const rightTop = topByRepo.get(right)!;
|
|
759
|
-
const leftRank = rankByJobId.get(leftTop.jobId)!;
|
|
760
|
-
const rightRank = rankByJobId.get(rightTop.jobId)!;
|
|
761
|
-
const priority = leftRank.effectivePriority - rightRank.effectivePriority;
|
|
762
|
-
if (priority !== 0) return priority;
|
|
763
|
-
const fairness = (this.lastRepoDispatch.get(left) ?? 0) - (this.lastRepoDispatch.get(right) ?? 0);
|
|
764
|
-
return fairness
|
|
765
|
-
|| leftRank.queuedAtMs - rightRank.queuedAtMs
|
|
766
|
-
|| leftRank.jobId.localeCompare(rightRank.jobId)
|
|
767
|
-
|| left.localeCompare(right);
|
|
768
|
-
});
|
|
769
|
-
const reservedRepos = new Set(reserved.map((job) => job.repoId));
|
|
559
|
+
const repoIds = selectExecutionJobDispatchRepositories(active, scheduleNow, this.lastRepoDispatch);
|
|
560
|
+
const reservedRepos = new Set(capacity.reservedJobs.map((job) => job.repoId));
|
|
770
561
|
let dispatchStateChanged = false;
|
|
771
|
-
const canDispatch = (job: (typeof active)[number]): boolean =>
|
|
772
|
-
if ((job.type === 'check' || job.type === 'verify-edit') && heavyCapacity <= 0) return false;
|
|
773
|
-
if (job.type === 'agent-run' || job.type === 'dispatch-task') {
|
|
774
|
-
if (agentCapacity <= 0) return false;
|
|
775
|
-
if ((providerCapacity.get(this.agentProvider(job)) ?? 0) <= 0) return false;
|
|
776
|
-
}
|
|
777
|
-
return true;
|
|
778
|
-
};
|
|
562
|
+
const canDispatch = (job: (typeof active)[number]): boolean => schedulerDispatchCapacityAllows(capacity, job);
|
|
779
563
|
for (const repoId of repoIds) {
|
|
780
|
-
if (capacity <= 0) break;
|
|
564
|
+
if (capacity.workers <= 0) break;
|
|
781
565
|
if (!reservedRepos.has(repoId) && reservedRepos.size >= this.config.maxConcurrentRepositories) continue;
|
|
782
566
|
const actor = this.actors.get(repoId);
|
|
783
567
|
let dispatch: ReturnType<typeof actor.tryClaimNext>;
|
|
@@ -798,14 +582,8 @@ export class GlobalScheduler {
|
|
|
798
582
|
// A successful claim is the capacity reservation. Count it immediately,
|
|
799
583
|
// before the worker PID is spawned or attached, so concurrent schedulers
|
|
800
584
|
// cannot over-dispatch through the dispatched -> running window.
|
|
801
|
-
capacity
|
|
585
|
+
consumeSchedulerDispatchCapacity(capacity, dispatch.job);
|
|
802
586
|
reservedRepos.add(repoId);
|
|
803
|
-
if (dispatch.job.type === 'check' || dispatch.job.type === 'verify-edit') heavyCapacity -= 1;
|
|
804
|
-
if (dispatch.job.type === 'agent-run' || dispatch.job.type === 'dispatch-task') {
|
|
805
|
-
agentCapacity -= 1;
|
|
806
|
-
const provider = this.agentProvider(dispatch.job);
|
|
807
|
-
providerCapacity.set(provider, (providerCapacity.get(provider) ?? 0) - 1);
|
|
808
|
-
}
|
|
809
587
|
const dispatchedAt = Date.now();
|
|
810
588
|
this.lastRepoDispatch.set(repoId, dispatchedAt);
|
|
811
589
|
this.lastDispatchAt = new Date(dispatchedAt).toISOString();
|
|
@@ -823,65 +601,22 @@ export class GlobalScheduler {
|
|
|
823
601
|
// leave all jobs queued for the next wake/tick rather than risking overrun.
|
|
824
602
|
activeJobs = listActiveExecutionJobs(this.controllerHome).length;
|
|
825
603
|
}
|
|
826
|
-
// Repo Actor mutations leave projection dirty markers.
|
|
827
|
-
//
|
|
828
|
-
// after the
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
}
|
|
836
|
-
for (const repoId of projectionRefreshRepos) {
|
|
837
|
-
if (projectionRefreshCandidates.has(repoId)) continue;
|
|
838
|
-
try {
|
|
839
|
-
rebuildRepositoryProjection(this.controllerHome, repoId);
|
|
840
|
-
} catch (error) {
|
|
841
|
-
console.error('[forge scheduler] projection refresh failed:', error);
|
|
842
|
-
}
|
|
843
|
-
}
|
|
844
|
-
for (const repository of projectionRefreshCandidates.values()) {
|
|
845
|
-
try {
|
|
846
|
-
const sample = readRepositoryGitStatusSample(
|
|
847
|
-
this.controllerHome,
|
|
848
|
-
repository.repoId,
|
|
849
|
-
repository.activeCheckoutId,
|
|
850
|
-
);
|
|
851
|
-
const runtimeInstanceId = getRuntimeWriteClaim()?.runtimeInstanceId;
|
|
852
|
-
refreshRepositoryProjectionForRepository(this.controllerHome, repository, {
|
|
853
|
-
sourceRevision: sample?.head ?? undefined,
|
|
854
|
-
reason: 'scheduler-source-scan',
|
|
855
|
-
owner: {
|
|
856
|
-
pid: this.controllerPid,
|
|
857
|
-
...(runtimeInstanceId ? { runtimeInstanceId } : {}),
|
|
858
|
-
},
|
|
859
|
-
});
|
|
860
|
-
} catch (error) {
|
|
861
|
-
console.error('[forge scheduler] projection refresh failed:', error);
|
|
862
|
-
}
|
|
863
|
-
}
|
|
604
|
+
// Repo Actor mutations leave projection dirty markers. Refresh those repos
|
|
605
|
+
// immediately; the independent source safety scan contributes only its bounded
|
|
606
|
+
// active/round-robin candidates after the dispatch reservation lock is free.
|
|
607
|
+
refreshSchedulerRepositoryProjections({
|
|
608
|
+
controllerHome: this.controllerHome,
|
|
609
|
+
repositories,
|
|
610
|
+
sourceScanRepositories: projectionMaintenanceRepositories,
|
|
611
|
+
projectionRefreshRepoIds: projectionRefreshRepos,
|
|
612
|
+
controllerPid: this.controllerPid,
|
|
613
|
+
});
|
|
864
614
|
// Process creation, lifecycle file writes, and Worker attachment are all
|
|
865
615
|
// deliberately outside the global dispatch reservation lock. The durable
|
|
866
616
|
// dispatched state is the capacity reservation while spawn proceeds.
|
|
867
617
|
for (const pending of pendingSpawns) {
|
|
868
618
|
this.spawnWorker(pending.repoId, pending.jobId);
|
|
869
619
|
}
|
|
870
|
-
// Dispatch is latency-sensitive. Run repository workflow maintenance only
|
|
871
|
-
// after queued Jobs have had a chance to claim capacity. Goal
|
|
872
|
-
// state transitions still wake the scheduler immediately when they enqueue work.
|
|
873
|
-
|
|
874
|
-
if (now - this.lastGoalLoopTick >= 5_000) {
|
|
875
|
-
try {
|
|
876
|
-
tickGoalLoopsForController(
|
|
877
|
-
this.controllerHome,
|
|
878
|
-
repositories.map((repo) => repo.repoId),
|
|
879
|
-
);
|
|
880
|
-
} catch (error) {
|
|
881
|
-
console.error('[forge goal-loop] tick failed:', error);
|
|
882
|
-
}
|
|
883
|
-
this.lastGoalLoopTick = now;
|
|
884
|
-
}
|
|
885
620
|
this.lastHeartbeatAt = new Date().toISOString();
|
|
886
621
|
this.persistState();
|
|
887
622
|
return { activeJobs };
|
|
@@ -936,7 +671,7 @@ export class GlobalScheduler {
|
|
|
936
671
|
}
|
|
937
672
|
} finally {
|
|
938
673
|
clearInterval(heartbeatTimer);
|
|
939
|
-
await this.
|
|
674
|
+
await cleanupSchedulerWorkerProcesses(this.children);
|
|
940
675
|
}
|
|
941
676
|
}
|
|
942
677
|
}
|