@kbediako/codex-orchestrator 0.1.37 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/README.md +73 -291
- package/bin/codex-orchestrator.js +161 -0
- package/codex.orchestrator.json +149 -13
- package/dist/bin/codex-orchestrator.js +795 -1154
- package/dist/orchestrator/src/cli/adapters/CommandPlanner.js +22 -4
- package/dist/orchestrator/src/cli/adapters/CommandReviewer.js +3 -3
- package/dist/orchestrator/src/cli/adapters/CommandTester.js +2 -2
- package/dist/orchestrator/src/cli/adapters/cloudFailureDiagnostics.js +183 -11
- package/dist/orchestrator/src/cli/coStatusAttachCliShell.js +402 -0
- package/dist/orchestrator/src/cli/coStatusCliShell.js +429 -0
- package/dist/orchestrator/src/cli/coStatusOperatorAutopilotCliShell.js +120 -0
- package/dist/orchestrator/src/cli/codexCliShell.js +72 -0
- package/dist/orchestrator/src/cli/codexDefaultsSetup.js +49 -11
- package/dist/orchestrator/src/cli/config/delegationConfig.js +317 -5
- package/dist/orchestrator/src/cli/config/repoConfigPolicy.js +2 -3
- package/dist/orchestrator/src/cli/config/userConfig.js +28 -13
- package/dist/orchestrator/src/cli/control/authenticatedControlRouteGate.js +69 -0
- package/dist/orchestrator/src/cli/control/authenticatedRouteComposition.js +267 -0
- package/dist/orchestrator/src/cli/control/authenticatedRouteController.js +5 -0
- package/dist/orchestrator/src/cli/control/authenticatedRouteDispatcher.js +41 -0
- package/dist/orchestrator/src/cli/control/compatibilityIssuePresenter.js +1035 -0
- package/dist/orchestrator/src/cli/control/confirmationApproveController.js +62 -0
- package/dist/orchestrator/src/cli/control/confirmationCreateController.js +69 -0
- package/dist/orchestrator/src/cli/control/confirmationIssueConsumeController.js +43 -0
- package/dist/orchestrator/src/cli/control/confirmationListController.js +22 -0
- package/dist/orchestrator/src/cli/control/confirmationValidateController.js +58 -0
- package/dist/orchestrator/src/cli/control/confirmations.js +25 -3
- package/dist/orchestrator/src/cli/control/controlActionCancelConfirmation.js +65 -0
- package/dist/orchestrator/src/cli/control/controlActionController.js +77 -0
- package/dist/orchestrator/src/cli/control/controlActionControllerSequencing.js +161 -0
- package/dist/orchestrator/src/cli/control/controlActionExecution.js +142 -0
- package/dist/orchestrator/src/cli/control/controlActionFinalization.js +43 -0
- package/dist/orchestrator/src/cli/control/controlActionOutcome.js +60 -0
- package/dist/orchestrator/src/cli/control/controlActionPreflight.js +476 -0
- package/dist/orchestrator/src/cli/control/controlAuthenticatedRouteHandoff.js +57 -0
- package/dist/orchestrator/src/cli/control/controlBootstrapAssembly.js +39 -0
- package/dist/orchestrator/src/cli/control/controlBootstrapMetadataPersistence.js +16 -0
- package/dist/orchestrator/src/cli/control/controlEventTransport.js +49 -0
- package/dist/orchestrator/src/cli/control/controlExpiryLifecycle.js +102 -0
- package/dist/orchestrator/src/cli/control/controlHostOwnership.js +480 -0
- package/dist/orchestrator/src/cli/control/controlHostSupervision.js +608 -0
- package/dist/orchestrator/src/cli/control/controlOversightFacade.js +8 -0
- package/dist/orchestrator/src/cli/control/controlOversightReadContract.js +1 -0
- package/dist/orchestrator/src/cli/control/controlOversightReadService.js +16 -0
- package/dist/orchestrator/src/cli/control/controlOversightUpdateContract.js +1 -0
- package/dist/orchestrator/src/cli/control/controlPersistenceFiles.js +6 -0
- package/dist/orchestrator/src/cli/control/controlQuestionChildResolution.js +18 -0
- package/dist/orchestrator/src/cli/control/controlRequestContext.js +42 -0
- package/dist/orchestrator/src/cli/control/controlRequestController.js +9 -0
- package/dist/orchestrator/src/cli/control/controlRequestPredispatch.js +17 -0
- package/dist/orchestrator/src/cli/control/controlRequestRouteDispatch.js +44 -0
- package/dist/orchestrator/src/cli/control/controlRuntime.js +992 -0
- package/dist/orchestrator/src/cli/control/controlServer.js +23 -1456
- package/dist/orchestrator/src/cli/control/controlServerAuditAndErrorHelpers.js +115 -0
- package/dist/orchestrator/src/cli/control/controlServerAuthenticatedRouteBranch.js +29 -0
- package/dist/orchestrator/src/cli/control/controlServerBootstrapLifecycle.js +30 -0
- package/dist/orchestrator/src/cli/control/controlServerBootstrapStartSequence.js +21 -0
- package/dist/orchestrator/src/cli/control/controlServerOwnedRuntimeLifecycle.js +67 -0
- package/dist/orchestrator/src/cli/control/controlServerPublicLifecycle.js +756 -0
- package/dist/orchestrator/src/cli/control/controlServerPublicRouteHelpers.js +86 -0
- package/dist/orchestrator/src/cli/control/controlServerReadyInstanceLifecycle.js +25 -0
- package/dist/orchestrator/src/cli/control/controlServerReadyInstanceStartup.js +18 -0
- package/dist/orchestrator/src/cli/control/controlServerRequestBodyHelpers.js +37 -0
- package/dist/orchestrator/src/cli/control/controlServerRequestShell.js +40 -0
- package/dist/orchestrator/src/cli/control/controlServerRequestShellBinding.js +17 -0
- package/dist/orchestrator/src/cli/control/controlServerSeedLoading.js +27 -0
- package/dist/orchestrator/src/cli/control/controlServerSeededRuntimeAssembly.js +186 -0
- package/dist/orchestrator/src/cli/control/controlServerStartupInputPreparation.js +31 -0
- package/dist/orchestrator/src/cli/control/controlServerStartupSequence.js +49 -0
- package/dist/orchestrator/src/cli/control/controlState.js +233 -2
- package/dist/orchestrator/src/cli/control/controlStatusDashboard.js +1899 -0
- package/dist/orchestrator/src/cli/control/controlTelegramBridgeBootstrapLifecycle.js +22 -0
- package/dist/orchestrator/src/cli/control/controlTelegramBridgeLifecycle.js +67 -0
- package/dist/orchestrator/src/cli/control/controlTelegramBridgeOversightFacadeFactory.js +8 -0
- package/dist/orchestrator/src/cli/control/controlTelegramCommandController.js +49 -0
- package/dist/orchestrator/src/cli/control/controlTelegramDispatchRead.js +40 -0
- package/dist/orchestrator/src/cli/control/controlTelegramPollingController.js +89 -0
- package/dist/orchestrator/src/cli/control/controlTelegramProjectionNotificationController.js +29 -0
- package/dist/orchestrator/src/cli/control/controlTelegramPushState.js +63 -0
- package/dist/orchestrator/src/cli/control/controlTelegramQuestionRead.js +13 -0
- package/dist/orchestrator/src/cli/control/controlTelegramReadController.js +216 -0
- package/dist/orchestrator/src/cli/control/controlTelegramUpdateHandler.js +63 -0
- package/dist/orchestrator/src/cli/control/controlWatcher.js +73 -5
- package/dist/orchestrator/src/cli/control/delegationRegisterController.js +35 -0
- package/dist/orchestrator/src/cli/control/dynamicToolBridgePolicy.js +139 -0
- package/dist/orchestrator/src/cli/control/eventsSseController.js +12 -0
- package/dist/orchestrator/src/cli/control/linearBudgetState.js +1789 -0
- package/dist/orchestrator/src/cli/control/linearDispatchSource.js +1137 -0
- package/dist/orchestrator/src/cli/control/linearGraphqlClient.js +150 -0
- package/dist/orchestrator/src/cli/control/linearRateLimit.js +102 -0
- package/dist/orchestrator/src/cli/control/linearWebhookController.js +499 -0
- package/dist/orchestrator/src/cli/control/liveLinearAdvisoryRuntime.js +70 -0
- package/dist/orchestrator/src/cli/control/observabilityApiController.js +173 -0
- package/dist/orchestrator/src/cli/control/observabilityReadModel.js +500 -0
- package/dist/orchestrator/src/cli/control/observabilitySurface.js +284 -0
- package/dist/orchestrator/src/cli/control/observabilityUpdateNotifier.js +22 -0
- package/dist/orchestrator/src/cli/control/operatorDashboardPresenter.js +252 -0
- package/dist/orchestrator/src/cli/control/providerAgentCapacity.js +70 -0
- package/dist/orchestrator/src/cli/control/providerControlHostFreshnessGauge.js +1068 -0
- package/dist/orchestrator/src/cli/control/providerIntakeState.js +473 -0
- package/dist/orchestrator/src/cli/control/providerIssueHandoff.js +6811 -0
- package/dist/orchestrator/src/cli/control/providerIssueObservability.js +1348 -0
- package/dist/orchestrator/src/cli/control/providerIssueRetryQueue.js +84 -0
- package/dist/orchestrator/src/cli/control/providerLinearRuntimeProof.js +588 -0
- package/dist/orchestrator/src/cli/control/providerLinearScreenshotProof.js +473 -0
- package/dist/orchestrator/src/cli/control/providerLinearWorkerTruth.js +383 -0
- package/dist/orchestrator/src/cli/control/providerLinearWorkflowAudit.js +254 -0
- package/dist/orchestrator/src/cli/control/providerLinearWorkflowFacade.js +5573 -0
- package/dist/orchestrator/src/cli/control/providerLinearWorkflowStates.js +115 -0
- package/dist/orchestrator/src/cli/control/providerMergeCloseout.js +1868 -0
- package/dist/orchestrator/src/cli/control/providerOperatorAutopilot.js +1580 -0
- package/dist/orchestrator/src/cli/control/providerOperatorAutopilotLifecycle.js +154 -0
- package/dist/orchestrator/src/cli/control/providerOperatorAutopilotLocalRolloutExecution.js +1006 -0
- package/dist/orchestrator/src/cli/control/providerPollingHealth.js +435 -0
- package/dist/orchestrator/src/cli/control/providerTerminalCleanup.js +516 -0
- package/dist/orchestrator/src/cli/control/providerWorkerHosts.js +191 -0
- package/dist/orchestrator/src/cli/control/providerWorkflowConfigStore.js +515 -0
- package/dist/orchestrator/src/cli/control/questionChildResolutionAdapter.js +361 -0
- package/dist/orchestrator/src/cli/control/questionQueueController.js +181 -0
- package/dist/orchestrator/src/cli/control/questionReadRetryDeduplication.js +9 -0
- package/dist/orchestrator/src/cli/control/questionReadSequence.js +10 -0
- package/dist/orchestrator/src/cli/control/securityViolationController.js +27 -0
- package/dist/orchestrator/src/cli/control/selectedRunProjection.js +1838 -0
- package/dist/orchestrator/src/cli/control/telegramOversightApiClient.js +48 -0
- package/dist/orchestrator/src/cli/control/telegramOversightBridge.js +180 -0
- package/dist/orchestrator/src/cli/control/telegramOversightBridgeProjectionDeliveryQueue.js +25 -0
- package/dist/orchestrator/src/cli/control/telegramOversightBridgeRuntimeLifecycle.js +45 -0
- package/dist/orchestrator/src/cli/control/telegramOversightBridgeStateStore.js +77 -0
- package/dist/orchestrator/src/cli/control/telegramOversightControlActionApiClient.js +45 -0
- package/dist/orchestrator/src/cli/control/trackerDispatchPilot.js +439 -0
- package/dist/orchestrator/src/cli/control/uiDataController.js +34 -0
- package/dist/orchestrator/src/cli/control/uiSessionController.js +100 -0
- package/dist/orchestrator/src/cli/controlHostCliShell.js +860 -0
- package/dist/orchestrator/src/cli/controlHostFreshnessGaugeCliShell.js +129 -0
- package/dist/orchestrator/src/cli/controlHostSupervisionCliShell.js +2127 -0
- package/dist/orchestrator/src/cli/delegationCliShell.js +62 -0
- package/dist/orchestrator/src/cli/delegationServer.js +567 -678
- package/dist/orchestrator/src/cli/delegationServerCliShell.js +52 -0
- package/dist/orchestrator/src/cli/delegationServerQuestionFlowShell.js +228 -0
- package/dist/orchestrator/src/cli/delegationServerToolDispatchShell.js +411 -0
- package/dist/orchestrator/src/cli/delegationServerTransport.js +274 -0
- package/dist/orchestrator/src/cli/delegationSetup.js +51 -171
- package/dist/orchestrator/src/cli/devtoolsCliShell.js +34 -0
- package/dist/orchestrator/src/cli/doctor.js +542 -122
- package/dist/orchestrator/src/cli/doctorCliRequestShell.js +72 -0
- package/dist/orchestrator/src/cli/doctorCliShell.js +138 -0
- package/dist/orchestrator/src/cli/doctorUsage.js +136 -16
- package/dist/orchestrator/src/cli/exec/experience.js +16 -2
- package/dist/orchestrator/src/cli/exec/summary.js +3 -0
- package/dist/orchestrator/src/cli/execCliShell.js +51 -0
- package/dist/orchestrator/src/cli/flowCliRequestShell.js +44 -0
- package/dist/orchestrator/src/cli/flowCliShell.js +239 -0
- package/dist/orchestrator/src/cli/frontendTestCliRequestShell.js +80 -0
- package/dist/orchestrator/src/cli/frontendTestCliShell.js +41 -0
- package/dist/orchestrator/src/cli/init.js +1 -0
- package/dist/orchestrator/src/cli/initCliShell.js +50 -0
- package/dist/orchestrator/src/cli/linearCliShell.js +1200 -0
- package/dist/orchestrator/src/cli/mcpEnableCliShell.js +132 -0
- package/dist/orchestrator/src/cli/metrics/metricsAggregator.js +3 -2
- package/dist/orchestrator/src/cli/metrics/metricsRecorder.js +56 -0
- package/dist/orchestrator/src/cli/orchestrator.js +66 -1376
- package/dist/orchestrator/src/cli/planCliShell.js +19 -0
- package/dist/orchestrator/src/cli/prCliShell.js +41 -0
- package/dist/orchestrator/src/cli/providerLinearChildLanePhaseContract.js +204 -0
- package/dist/orchestrator/src/cli/providerLinearChildLaneRunner.js +1772 -0
- package/dist/orchestrator/src/cli/providerLinearChildLaneShell.js +2420 -0
- package/dist/orchestrator/src/cli/providerLinearChildStreamShell.js +385 -0
- package/dist/orchestrator/src/cli/providerLinearWorkerRunner.js +5738 -0
- package/dist/orchestrator/src/cli/resumeCliShell.js +14 -0
- package/dist/orchestrator/src/cli/reviewCliLaunchShell.js +72 -0
- package/dist/orchestrator/src/cli/rlm/alignment.js +3 -3
- package/dist/orchestrator/src/cli/rlm/context.js +94 -7
- package/dist/orchestrator/src/cli/rlm/rlmCodexRuntimeShell.js +546 -0
- package/dist/orchestrator/src/cli/rlm/symbolic.js +4 -2
- package/dist/orchestrator/src/cli/rlmCliRequestShell.js +42 -0
- package/dist/orchestrator/src/cli/rlmCompletionCliShell.js +46 -0
- package/dist/orchestrator/src/cli/rlmLaunchCliShell.js +51 -0
- package/dist/orchestrator/src/cli/rlmRunner.js +83 -523
- package/dist/orchestrator/src/cli/run/blockMemory.js +500 -0
- package/dist/orchestrator/src/cli/run/manifest.js +410 -73
- package/dist/orchestrator/src/cli/run/manifestPersister.js +45 -14
- package/dist/orchestrator/src/cli/run/runMemoryController.js +216 -0
- package/dist/orchestrator/src/cli/run/source0.js +690 -0
- package/dist/orchestrator/src/cli/run/workspacePath.js +101 -0
- package/dist/orchestrator/src/cli/runtime/mode.js +2 -1
- package/dist/orchestrator/src/cli/runtime/provider.js +39 -2
- package/dist/orchestrator/src/cli/selfCheckCliShell.js +12 -0
- package/dist/orchestrator/src/cli/services/commandRunner.js +668 -18
- package/dist/orchestrator/src/cli/services/execRuntime.js +66 -1
- package/dist/orchestrator/src/cli/services/orchestratorAutoScoutEvidenceRecorder.js +71 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudBranchResolution.js +8 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudEnvironmentResolution.js +22 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudExecutionLifecycleShell.js +39 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudPromptBuilder.js +37 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudRouteFallbackContract.js +45 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudRouteShell.js +36 -0
- package/dist/orchestrator/src/cli/services/orchestratorCloudTargetExecutor.js +277 -0
- package/dist/orchestrator/src/cli/services/orchestratorControlPlaneLifecycle.js +98 -0
- package/dist/orchestrator/src/cli/services/orchestratorControlPlaneLifecycleShell.js +54 -0
- package/dist/orchestrator/src/cli/services/orchestratorExecutionLifecycle.js +112 -0
- package/dist/orchestrator/src/cli/services/orchestratorExecutionModePolicy.js +27 -0
- package/dist/orchestrator/src/cli/services/orchestratorExecutionRouteAdapterShell.js +59 -0
- package/dist/orchestrator/src/cli/services/orchestratorExecutionRouteDecisionShell.js +57 -0
- package/dist/orchestrator/src/cli/services/orchestratorExecutionRouteState.js +21 -0
- package/dist/orchestrator/src/cli/services/orchestratorExecutionRouter.js +2 -0
- package/dist/orchestrator/src/cli/services/orchestratorLocalPipelineExecutor.js +149 -0
- package/dist/orchestrator/src/cli/services/orchestratorLocalRouteShell.js +63 -0
- package/dist/orchestrator/src/cli/services/orchestratorPlanShell.js +54 -0
- package/dist/orchestrator/src/cli/services/orchestratorPlanTargetTracker.js +16 -0
- package/dist/orchestrator/src/cli/services/orchestratorResumePreparationShell.js +84 -0
- package/dist/orchestrator/src/cli/services/orchestratorResumeTokenValidation.js +15 -0
- package/dist/orchestrator/src/cli/services/orchestratorRunLifecycleCompletion.js +31 -0
- package/dist/orchestrator/src/cli/services/orchestratorRunLifecycleExecutionRegistration.js +37 -0
- package/dist/orchestrator/src/cli/services/orchestratorRunLifecycleOrchestrationShell.js +83 -0
- package/dist/orchestrator/src/cli/services/orchestratorRunLifecycleTaskManagerShell.js +37 -0
- package/dist/orchestrator/src/cli/services/orchestratorRuntimeManifestMutation.js +20 -0
- package/dist/orchestrator/src/cli/services/orchestratorStartPreparationShell.js +56 -0
- package/dist/orchestrator/src/cli/services/orchestratorStatusShell.js +70 -0
- package/dist/orchestrator/src/cli/services/pipelineResolver.js +7 -3
- package/dist/orchestrator/src/cli/services/plannerMemory.js +119 -0
- package/dist/orchestrator/src/cli/services/runPreparation.js +7 -3
- package/dist/orchestrator/src/cli/services/runSummaryWriter.js +9 -0
- package/dist/orchestrator/src/cli/setupBootstrapShell.js +114 -0
- package/dist/orchestrator/src/cli/setupCliShell.js +51 -0
- package/dist/orchestrator/src/cli/skillsCliShell.js +56 -0
- package/dist/orchestrator/src/cli/startCliRequestShell.js +53 -0
- package/dist/orchestrator/src/cli/startCliShell.js +68 -0
- package/dist/orchestrator/src/cli/statusCliShell.js +22 -0
- package/dist/orchestrator/src/cli/utils/authProvenanceFingerprint.js +27 -0
- package/dist/orchestrator/src/cli/utils/cloudPreflight.js +83 -1
- package/dist/orchestrator/src/cli/utils/delegationConfigParser.js +250 -0
- package/dist/orchestrator/src/cli/utils/delegationMcpHealth.js +1382 -0
- package/dist/orchestrator/src/cli/utils/devtools.js +2 -54
- package/dist/orchestrator/src/cli/utils/mcpServerEntry.js +53 -0
- package/dist/orchestrator/src/cli/utils/packageProgramResolver.js +151 -0
- package/dist/orchestrator/src/cli/utils/providerOverrideEnv.js +71 -0
- package/dist/orchestrator/src/cli/utils/trailingJsonObject.js +59 -0
- package/dist/orchestrator/src/learning/crystalizer.js +2 -2
- package/dist/orchestrator/src/persistence/ExperienceStore.js +233 -49
- package/dist/orchestrator/src/persistence/TaskStateStore.js +6 -6
- package/dist/orchestrator/src/persistence/lockFile.js +70 -4
- package/dist/orchestrator/src/persistence/sanitizeIdentifier.js +39 -0
- package/dist/orchestrator/src/sync/createCloudSyncWorker.js +3 -2
- package/dist/orchestrator/src/utils/atomicWrite.js +17 -2
- package/dist/packages/orchestrator/src/exec/unified-exec.js +99 -6
- package/dist/packages/orchestrator/src/instructions/promptPacks.js +150 -19
- package/dist/packages/sdk-node/src/orchestrator.js +137 -13
- package/dist/packages/shared/config/designConfig.js +8 -1
- package/dist/packages/shared/streams/stdio.js +1 -1
- package/dist/scripts/design/pipeline/permit.js +15 -0
- package/dist/scripts/lib/docs-catalog.js +365 -0
- package/dist/scripts/lib/docs-helpers.js +87 -5
- package/dist/scripts/lib/pr-watch-merge.js +1088 -80
- package/dist/scripts/lib/provider-run-contract.js +26 -0
- package/dist/scripts/lib/review-command-intent-classification.js +532 -0
- package/dist/scripts/lib/review-command-probe-classification.js +385 -0
- package/dist/scripts/lib/review-execution-boundary-preflight.js +279 -0
- package/dist/scripts/lib/review-execution-runtime.js +753 -0
- package/dist/scripts/lib/review-execution-state.js +1144 -0
- package/dist/scripts/lib/review-execution-telemetry.js +215 -0
- package/dist/scripts/lib/review-inspection-target-parsing.js +78 -0
- package/dist/scripts/lib/review-launch-attempt.js +601 -0
- package/dist/scripts/lib/review-meta-surface-boundary-analysis.js +300 -0
- package/dist/scripts/lib/review-meta-surface-normalization.js +746 -0
- package/dist/scripts/lib/review-non-interactive-handoff.js +61 -0
- package/dist/scripts/lib/review-prompt-context.js +376 -0
- package/dist/scripts/lib/review-scope-advisory.js +286 -0
- package/dist/scripts/lib/review-scope-paths.js +123 -0
- package/dist/scripts/lib/review-shell-command-parser.js +389 -0
- package/dist/scripts/lib/review-shell-env-interpreter.js +340 -0
- package/dist/scripts/lib/run-manifests.js +192 -36
- package/dist/scripts/lib/spark-policy-classifier.js +593 -0
- package/dist/scripts/run-review.js +507 -1777
- package/docs/public/downstream-setup.md +106 -0
- package/docs/public/provider-onboarding.md +173 -0
- package/package.json +30 -11
- package/plugins/codex-orchestrator/.codex-plugin/plugin.json +30 -0
- package/plugins/codex-orchestrator/.mcp.json +13 -0
- package/plugins/codex-orchestrator/launcher.mjs +359 -0
- package/schemas/manifest.json +395 -0
- package/skills/chrome-devtools/SKILL.md +1 -1
- package/skills/codex-orchestrator/SKILL.md +83 -0
- package/skills/collab-subagents-first/SKILL.md +2 -1
- package/skills/delegation-usage/DELEGATION_GUIDE.md +24 -11
- package/skills/delegation-usage/SKILL.md +20 -13
- package/skills/land/SKILL.md +77 -0
- package/skills/linear/SKILL.md +255 -0
- package/skills/release/SKILL.md +47 -3
- package/skills/standalone-review/SKILL.md +6 -1
- package/templates/README.md +4 -2
- package/templates/codex/.codex/agents/awaiter-high.toml +2 -2
- package/templates/codex/.codex/agents/explorer-fast.toml +1 -0
- package/templates/codex/.codex/agents/worker-complex.toml +1 -1
- package/templates/codex/.codex/config.toml +3 -4
- package/templates/codex/.codex/providers/README.md +13 -0
- package/templates/codex/.codex/providers/control.example.json +18 -0
- package/templates/codex/.codex/providers/provider.env.example +15 -0
- package/templates/codex/AGENTS.md +12 -7
- package/templates/codex/mcp-client.json +5 -1
- package/docs/README.md +0 -307
- package/docs/assets/setup.gif +0 -0
|
@@ -0,0 +1,2127 @@
|
|
|
1
|
+
/* eslint-disable patterns/prefer-logger-over-console */
|
|
2
|
+
import { execFile, spawn } from 'node:child_process';
|
|
3
|
+
import { constants as fsConstants } from 'node:fs';
|
|
4
|
+
import { access, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
5
|
+
import { dirname, join, resolve } from 'node:path';
|
|
6
|
+
import process from 'node:process';
|
|
7
|
+
import { promisify } from 'node:util';
|
|
8
|
+
import { CONTROL_HOST_SUPERVISION_RESTART_HISTORY_LIMIT, CONTROL_HOST_SUPERVISION_MAX_NODE_TIMER_SECONDS, DEFAULT_CONTROL_HOST_SUPERVISION_LABEL, DEFAULT_CONTROL_HOST_SUPERVISION_KILL_TIMEOUT_SECONDS, DEFAULT_CONTROL_HOST_SUPERVISION_RESTART_EXIT_CODE, buildControlHostSupervisionConfig, buildControlHostSupervisionPlist, buildInitialControlHostSupervisionState, evaluateControlHostSupervisionHealthPayload, evaluateControlHostSupervisionProbeTimeoutDiagnostic, readControlHostSupervisionHealthDiagnostic, parseControlHostSupervisionCsv, resolveControlHostSupervisionPaths, resolveDefaultControlHostSupervisionEntrypoint, resolveDefaultControlHostSupervisionEnvFiles } from './control/controlHostSupervision.js';
|
|
9
|
+
import { PROVIDER_INTAKE_STATE_FILE } from './control/controlPersistenceFiles.js';
|
|
10
|
+
import { normalizeProviderIntakeState } from './control/providerIntakeState.js';
|
|
11
|
+
import { evaluateProviderControlHostFreshnessGauge } from './control/providerControlHostFreshnessGauge.js';
|
|
12
|
+
import { DEFAULT_ATTACH_REQUEST_TIMEOUT_MS } from './coStatusAttachCliShell.js';
|
|
13
|
+
import { findPackageRoot } from './utils/packageInfo.js';
|
|
14
|
+
import { sanitizeProviderOverrideEnv } from './utils/providerOverrideEnv.js';
|
|
15
|
+
import { sanitizeRunId } from '../persistence/sanitizeRunId.js';
|
|
16
|
+
import { sanitizeTaskId } from '../persistence/sanitizeTaskId.js';
|
|
17
|
+
const REQUIRED_CONTROL_HOST_SUPERVISION_STRING_FIELDS = [
|
|
18
|
+
'label',
|
|
19
|
+
'repoRoot',
|
|
20
|
+
'nodePath',
|
|
21
|
+
'cliEntrypoint',
|
|
22
|
+
'taskId',
|
|
23
|
+
'runId',
|
|
24
|
+
'pipelineId',
|
|
25
|
+
'shellPath',
|
|
26
|
+
'homeDir'
|
|
27
|
+
];
|
|
28
|
+
const REQUIRED_CONTROL_HOST_SUPERVISION_INTEGER_FIELDS = [
|
|
29
|
+
'version',
|
|
30
|
+
'healthIntervalSeconds',
|
|
31
|
+
'unhealthyThreshold',
|
|
32
|
+
'launchdThrottleSeconds',
|
|
33
|
+
'killTimeoutSeconds'
|
|
34
|
+
];
|
|
35
|
+
const REQUIRED_CONTROL_HOST_SUPERVISION_PATH_FIELDS = [
|
|
36
|
+
'supportDir',
|
|
37
|
+
'configPath',
|
|
38
|
+
'statePath',
|
|
39
|
+
'plistPath',
|
|
40
|
+
'logsDir',
|
|
41
|
+
'stdoutLogPath',
|
|
42
|
+
'stderrLogPath'
|
|
43
|
+
];
|
|
44
|
+
const execFileAsync = promisify(execFile);
|
|
45
|
+
const COMMAND_BUFFER_MAX_BYTES = 16 * 1024 * 1024;
|
|
46
|
+
const CONTROL_HOST_SUPERVISION_PROBE_TIMEOUT_CAP_MS = 45_000;
|
|
47
|
+
const CONTROL_HOST_SUPERVISION_PROBE_TIMEOUT_HEADROOM_MS = 5_000;
|
|
48
|
+
const CONTROL_HOST_SUPERVISION_PROBE_ENDPOINT_READ_ATTEMPTS = 2;
|
|
49
|
+
const CONTROL_HOST_SUPERVISION_PROBE_TIMEOUT_FLOOR_MS = 1_000;
|
|
50
|
+
const CONTROL_HOST_SUPERVISION_LAUNCHCTL_BOOTSTRAP_RETRY_ATTEMPTS = 5;
|
|
51
|
+
const CONTROL_HOST_SUPERVISION_LAUNCHCTL_BOOTSTRAP_RETRY_DELAY_MS = 1_000;
|
|
52
|
+
export async function runControlHostSupervisionCliShell(params) {
|
|
53
|
+
const { positionals, flags } = params;
|
|
54
|
+
if (flags.help !== undefined || positionals[0] === undefined || positionals[0] === 'help') {
|
|
55
|
+
params.printHelp();
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
const [subcommand, ...rest] = positionals;
|
|
59
|
+
if (rest.length > 0) {
|
|
60
|
+
throw new Error(`Unknown control-host supervise argument(s): ${rest.join(' ')}`);
|
|
61
|
+
}
|
|
62
|
+
switch (subcommand) {
|
|
63
|
+
case 'install':
|
|
64
|
+
await installControlHostSupervision(flags);
|
|
65
|
+
return;
|
|
66
|
+
case 'status':
|
|
67
|
+
await printControlHostSupervisionStatus(flags);
|
|
68
|
+
return;
|
|
69
|
+
case 'restart':
|
|
70
|
+
await restartControlHostSupervision(flags);
|
|
71
|
+
return;
|
|
72
|
+
case 'uninstall':
|
|
73
|
+
await uninstallControlHostSupervision(flags);
|
|
74
|
+
return;
|
|
75
|
+
case 'run':
|
|
76
|
+
await runControlHostSupervision(flags);
|
|
77
|
+
return;
|
|
78
|
+
default:
|
|
79
|
+
throw new Error(`Unknown control-host supervise subcommand: ${subcommand}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
async function installControlHostSupervision(flags) {
|
|
83
|
+
const format = readFormatFlag(flags);
|
|
84
|
+
const install = resolveInstallConfig(flags);
|
|
85
|
+
const serviceTarget = resolveControlHostSupervisionServiceTarget(install.config.label);
|
|
86
|
+
const priorInstall = await captureExistingControlHostSupervisionInstall(install.config.paths);
|
|
87
|
+
await assertControlHostSupervisionInstallPaths(install.config);
|
|
88
|
+
await mkdir(dirname(install.config.paths.plistPath), { recursive: true });
|
|
89
|
+
await mkdir(install.config.paths.supportDir, { recursive: true });
|
|
90
|
+
await mkdir(install.config.paths.logsDir, { recursive: true });
|
|
91
|
+
try {
|
|
92
|
+
await writeJsonFile(install.config.paths.configPath, install.config);
|
|
93
|
+
await writeJsonFile(install.config.paths.statePath, buildInitialControlHostSupervisionState({
|
|
94
|
+
config: install.config,
|
|
95
|
+
serviceTarget,
|
|
96
|
+
status: 'installed',
|
|
97
|
+
updatedAt: new Date().toISOString(),
|
|
98
|
+
message: 'LaunchAgent installed; waiting for launchd supervision.'
|
|
99
|
+
}));
|
|
100
|
+
await writeFile(install.config.paths.plistPath, buildControlHostSupervisionPlist(install.config), 'utf8');
|
|
101
|
+
await bootoutLaunchctlServiceTarget(serviceTarget);
|
|
102
|
+
await bootstrapLaunchctlPlist(install.config.paths.plistPath);
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
const detail = error.message;
|
|
106
|
+
try {
|
|
107
|
+
if (priorInstall) {
|
|
108
|
+
await restoreExistingControlHostSupervisionInstall(priorInstall, serviceTarget);
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
await rollbackFailedControlHostSupervisionInstall(install.config.paths, serviceTarget);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (rollbackError) {
|
|
115
|
+
throw new Error(`${detail} (rollback failed: ${rollbackError.message})`);
|
|
116
|
+
}
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
const payload = {
|
|
120
|
+
status: 'installed',
|
|
121
|
+
label: install.config.label,
|
|
122
|
+
service_target: serviceTarget,
|
|
123
|
+
config_path: install.config.paths.configPath,
|
|
124
|
+
state_path: install.config.paths.statePath,
|
|
125
|
+
plist_path: install.config.paths.plistPath,
|
|
126
|
+
logs: {
|
|
127
|
+
directory: install.config.paths.logsDir,
|
|
128
|
+
stdout_path: install.config.paths.stdoutLogPath,
|
|
129
|
+
stderr_path: install.config.paths.stderrLogPath
|
|
130
|
+
},
|
|
131
|
+
repo_root: install.config.repoRoot,
|
|
132
|
+
task_id: install.config.taskId,
|
|
133
|
+
run_id: install.config.runId,
|
|
134
|
+
pipeline_id: install.config.pipelineId,
|
|
135
|
+
health_interval_seconds: install.config.healthIntervalSeconds,
|
|
136
|
+
unhealthy_threshold: install.config.unhealthyThreshold,
|
|
137
|
+
env_files: install.config.envFiles
|
|
138
|
+
};
|
|
139
|
+
emitOutput(format, payload, [
|
|
140
|
+
`Installed control-host supervision for ${install.config.label}.`,
|
|
141
|
+
`Service target: ${serviceTarget}`,
|
|
142
|
+
`Config: ${install.config.paths.configPath}`,
|
|
143
|
+
`Plist: ${install.config.paths.plistPath}`,
|
|
144
|
+
`Logs: ${install.config.paths.stdoutLogPath} | ${install.config.paths.stderrLogPath}`
|
|
145
|
+
].join('\n'));
|
|
146
|
+
}
|
|
147
|
+
async function printControlHostSupervisionStatus(flags) {
|
|
148
|
+
const format = readFormatFlag(flags);
|
|
149
|
+
const resolved = await resolveStoredControlHostSupervision(flags, false);
|
|
150
|
+
const serviceTarget = resolveControlHostSupervisionServiceTarget(resolved.label);
|
|
151
|
+
const launchctl = await runLaunchctl(['print', serviceTarget], { allowFailure: true });
|
|
152
|
+
const state = await readJsonFileIfExists(resolved.paths.statePath);
|
|
153
|
+
const plistContents = await readTextFileIfExists(resolved.paths.plistPath);
|
|
154
|
+
const launchAgent = inspectControlHostSupervisionLaunchAgent(plistContents, resolved.config);
|
|
155
|
+
const liveHost = resolved.config && launchAgent.classification === 'managed_supervision'
|
|
156
|
+
? await inspectControlHostSupervisionLiveHealth(resolved.config, state)
|
|
157
|
+
: null;
|
|
158
|
+
const payload = buildControlHostSupervisionStatusPayload({
|
|
159
|
+
resolved,
|
|
160
|
+
serviceTarget,
|
|
161
|
+
state,
|
|
162
|
+
launchctl,
|
|
163
|
+
launchAgent,
|
|
164
|
+
liveHost
|
|
165
|
+
});
|
|
166
|
+
emitOutput(format, payload, formatControlHostSupervisionStatus(payload));
|
|
167
|
+
}
|
|
168
|
+
async function restartControlHostSupervision(flags) {
|
|
169
|
+
const format = readFormatFlag(flags);
|
|
170
|
+
const resolved = await resolveStoredControlHostSupervision(flags, true);
|
|
171
|
+
if (!resolved.config) {
|
|
172
|
+
throw new Error('control-host supervision restart requires an installed config.');
|
|
173
|
+
}
|
|
174
|
+
const config = resolved.config;
|
|
175
|
+
const serviceTarget = resolveControlHostSupervisionServiceTarget(resolved.label);
|
|
176
|
+
const restart = await restartExistingControlHostSupervision({
|
|
177
|
+
...resolved,
|
|
178
|
+
config
|
|
179
|
+
}, serviceTarget);
|
|
180
|
+
const payload = {
|
|
181
|
+
status: 'restarted',
|
|
182
|
+
label: resolved.label,
|
|
183
|
+
service_target: serviceTarget,
|
|
184
|
+
config_path: resolved.paths.configPath,
|
|
185
|
+
plist_path: resolved.paths.plistPath,
|
|
186
|
+
previous_child_pid: restart.previousChildPid,
|
|
187
|
+
child_pid: restart.childPid,
|
|
188
|
+
cleanup: {
|
|
189
|
+
result: restart.cleanup.result,
|
|
190
|
+
orphaned_process_group_pids: restart.cleanup.orphanedProcessGroupPids,
|
|
191
|
+
orphaned_descendant_pids: restart.cleanup.orphanedDescendantPids
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
const restartDetail = restart.cleanup.result === 'no_prior_child'
|
|
195
|
+
? 'No previously tracked supervised child pid was recorded.'
|
|
196
|
+
: restart.cleanup.result === 'exited_after_kickstart'
|
|
197
|
+
? `Previous supervised child pid ${restart.previousChildPid} exited before restart completed.`
|
|
198
|
+
: `Force-cleaned previous supervised child pid ${restart.previousChildPid}; orphaned group pids=${restart.cleanup.orphanedProcessGroupPids.join(',') || 'none'} descendants=${restart.cleanup.orphanedDescendantPids.join(',') || 'none'}.`;
|
|
199
|
+
emitOutput(format, payload, `Restarted control-host supervision for ${resolved.label} via ${serviceTarget}. ${restartDetail}`);
|
|
200
|
+
}
|
|
201
|
+
async function uninstallControlHostSupervision(flags) {
|
|
202
|
+
const format = readFormatFlag(flags);
|
|
203
|
+
const resolved = await resolveStoredControlHostSupervision(flags, true);
|
|
204
|
+
const removedPaths = await removeInstalledControlHostSupervisionArtifacts(resolved);
|
|
205
|
+
const serviceTarget = resolveControlHostSupervisionServiceTarget(resolved.label);
|
|
206
|
+
const payload = {
|
|
207
|
+
status: 'uninstalled',
|
|
208
|
+
label: resolved.label,
|
|
209
|
+
service_target: serviceTarget,
|
|
210
|
+
config_path: removedPaths.configPath,
|
|
211
|
+
plist_path: removedPaths.plistPath,
|
|
212
|
+
logs_dir: removedPaths.logsDir
|
|
213
|
+
};
|
|
214
|
+
emitOutput(format, payload, `Uninstalled control-host supervision for ${resolved.label}.`);
|
|
215
|
+
}
|
|
216
|
+
async function runControlHostSupervision(flags) {
|
|
217
|
+
const resolved = await resolveStoredControlHostSupervision(flags, true);
|
|
218
|
+
if (!resolved.config) {
|
|
219
|
+
throw new Error('control-host supervise run requires an installed config.');
|
|
220
|
+
}
|
|
221
|
+
const config = resolved.config;
|
|
222
|
+
const serviceTarget = resolveControlHostSupervisionServiceTarget(config.label);
|
|
223
|
+
await assertPathExists(config.nodePath, 'Node executable');
|
|
224
|
+
await assertPathExists(config.cliEntrypoint, 'Control-host supervision entrypoint');
|
|
225
|
+
const priorState = (await readJsonFileIfExists(config.paths.statePath)) ??
|
|
226
|
+
buildInitialControlHostSupervisionState({
|
|
227
|
+
config,
|
|
228
|
+
serviceTarget,
|
|
229
|
+
updatedAt: new Date().toISOString()
|
|
230
|
+
});
|
|
231
|
+
const writeState = async (update) => {
|
|
232
|
+
const nextState = buildNextControlHostSupervisionState({
|
|
233
|
+
priorState,
|
|
234
|
+
update,
|
|
235
|
+
config,
|
|
236
|
+
serviceTarget
|
|
237
|
+
});
|
|
238
|
+
await writeJsonFile(config.paths.statePath, nextState);
|
|
239
|
+
Object.assign(priorState, nextState);
|
|
240
|
+
return nextState;
|
|
241
|
+
};
|
|
242
|
+
let childEnv;
|
|
243
|
+
try {
|
|
244
|
+
childEnv = await loadBootstrapEnvironment(config);
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
const failedAt = new Date().toISOString();
|
|
248
|
+
await writeState({
|
|
249
|
+
status: 'bootstrap_failed',
|
|
250
|
+
updated_at: failedAt,
|
|
251
|
+
message: error.message
|
|
252
|
+
});
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
const controlHostArgs = [
|
|
256
|
+
config.cliEntrypoint,
|
|
257
|
+
'control-host',
|
|
258
|
+
'--task',
|
|
259
|
+
config.taskId,
|
|
260
|
+
'--run',
|
|
261
|
+
config.runId,
|
|
262
|
+
'--pipeline',
|
|
263
|
+
config.pipelineId,
|
|
264
|
+
'--format',
|
|
265
|
+
'json'
|
|
266
|
+
];
|
|
267
|
+
const child = spawn(config.nodePath, controlHostArgs, {
|
|
268
|
+
cwd: config.repoRoot,
|
|
269
|
+
env: childEnv,
|
|
270
|
+
// Give the supervised host its own process group so timeout cleanup can
|
|
271
|
+
// kill the whole wrapper->runner tree even if the wrapper exits first.
|
|
272
|
+
detached: true,
|
|
273
|
+
stdio: 'inherit'
|
|
274
|
+
});
|
|
275
|
+
const { childExitPromise, childErrorPromise } = createControlHostSupervisionChildEventPromises(child);
|
|
276
|
+
// Fail closed if state persistence breaks after spawn; otherwise launchd can
|
|
277
|
+
// restart while an orphaned control-host keeps running outside supervision.
|
|
278
|
+
const writeRuntimeState = async (update) => writeRuntimeStateWithCleanup(child, config.killTimeoutSeconds, () => writeState(update));
|
|
279
|
+
const startedAt = new Date().toISOString();
|
|
280
|
+
await writeRuntimeState({
|
|
281
|
+
status: 'running',
|
|
282
|
+
updated_at: startedAt,
|
|
283
|
+
child_pid: child.pid ?? null,
|
|
284
|
+
last_started_at: startedAt,
|
|
285
|
+
message: 'control-host supervision runner started.'
|
|
286
|
+
});
|
|
287
|
+
const stopWaiter = createStopSignalWaiter();
|
|
288
|
+
let consecutiveUnhealthySamples = 0;
|
|
289
|
+
const restartCount = priorState.restart_count ?? 0;
|
|
290
|
+
try {
|
|
291
|
+
for (;;) {
|
|
292
|
+
const tickWaiter = createSleepWaiter(config.healthIntervalSeconds * 1_000);
|
|
293
|
+
const event = await Promise.race([
|
|
294
|
+
childExitPromise,
|
|
295
|
+
childErrorPromise,
|
|
296
|
+
stopWaiter.promise,
|
|
297
|
+
tickWaiter.promise
|
|
298
|
+
]);
|
|
299
|
+
tickWaiter.dispose();
|
|
300
|
+
if (event.type === 'tick') {
|
|
301
|
+
const probe = await probeControlHostHealth(config, childEnv, {
|
|
302
|
+
minPollingUpdatedAt: startedAt,
|
|
303
|
+
restartHistory: priorState.restart_history ?? null
|
|
304
|
+
});
|
|
305
|
+
const checkedAt = new Date().toISOString();
|
|
306
|
+
if (probe.healthy) {
|
|
307
|
+
if (isControlHostSupervisionQuarantineProbe(probe)) {
|
|
308
|
+
consecutiveUnhealthySamples =
|
|
309
|
+
resolveControlHostSupervisionQuarantineUnhealthySamples({
|
|
310
|
+
currentConsecutiveUnhealthySamples: consecutiveUnhealthySamples,
|
|
311
|
+
priorState,
|
|
312
|
+
config
|
|
313
|
+
});
|
|
314
|
+
await writeRuntimeState({
|
|
315
|
+
status: 'quarantined',
|
|
316
|
+
updated_at: checkedAt,
|
|
317
|
+
last_health_check_at: checkedAt,
|
|
318
|
+
last_health_status: probe.reason,
|
|
319
|
+
last_probe_duration_ms: probe.probeDurationMs,
|
|
320
|
+
consecutive_unhealthy_samples: consecutiveUnhealthySamples,
|
|
321
|
+
message: probe.message
|
|
322
|
+
});
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
consecutiveUnhealthySamples = 0;
|
|
326
|
+
await writeRuntimeState({
|
|
327
|
+
status: 'healthy',
|
|
328
|
+
updated_at: checkedAt,
|
|
329
|
+
last_health_check_at: checkedAt,
|
|
330
|
+
last_health_status: probe.reason,
|
|
331
|
+
last_probe_duration_ms: probe.probeDurationMs,
|
|
332
|
+
consecutive_unhealthy_samples: 0,
|
|
333
|
+
message: probe.message
|
|
334
|
+
});
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
consecutiveUnhealthySamples += 1;
|
|
338
|
+
await writeRuntimeState({
|
|
339
|
+
status: 'unhealthy',
|
|
340
|
+
updated_at: checkedAt,
|
|
341
|
+
last_health_check_at: checkedAt,
|
|
342
|
+
last_health_status: probe.reason,
|
|
343
|
+
last_probe_duration_ms: probe.probeDurationMs,
|
|
344
|
+
consecutive_unhealthy_samples: consecutiveUnhealthySamples,
|
|
345
|
+
message: probe.message
|
|
346
|
+
});
|
|
347
|
+
if (consecutiveUnhealthySamples < config.unhealthyThreshold) {
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
const restartRequestedAt = new Date().toISOString();
|
|
351
|
+
const restartMessage = `${probe.message} launchd restart requested after ${consecutiveUnhealthySamples} consecutive unhealthy samples.`;
|
|
352
|
+
await writeRuntimeState({
|
|
353
|
+
status: 'restart_required',
|
|
354
|
+
updated_at: restartRequestedAt,
|
|
355
|
+
last_health_check_at: restartRequestedAt,
|
|
356
|
+
last_health_status: probe.reason,
|
|
357
|
+
last_probe_duration_ms: probe.probeDurationMs,
|
|
358
|
+
consecutive_unhealthy_samples: consecutiveUnhealthySamples,
|
|
359
|
+
restart_count: restartCount + 1,
|
|
360
|
+
last_restart_reason: probe.reason,
|
|
361
|
+
last_restart_requested_at: restartRequestedAt,
|
|
362
|
+
restart_history: appendControlHostSupervisionRestartRecord(priorState.restart_history ?? null, buildControlHostSupervisionRestartRecord({
|
|
363
|
+
requestedAt: restartRequestedAt,
|
|
364
|
+
reason: probe.reason,
|
|
365
|
+
message: restartMessage,
|
|
366
|
+
consecutiveUnhealthySamples,
|
|
367
|
+
childPid: child.pid ?? null,
|
|
368
|
+
probeDurationMs: probe.probeDurationMs,
|
|
369
|
+
diagnostic: probe.diagnostic
|
|
370
|
+
})),
|
|
371
|
+
message: restartMessage
|
|
372
|
+
});
|
|
373
|
+
console.error(`${restartRequestedAt} control-host unhealthy for ${consecutiveUnhealthySamples} checks; exiting for launchd restart (${probe.reason}).`);
|
|
374
|
+
await terminateChildProcess(child, config.killTimeoutSeconds);
|
|
375
|
+
process.exitCode = DEFAULT_CONTROL_HOST_SUPERVISION_RESTART_EXIT_CODE;
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (event.type === 'stop') {
|
|
379
|
+
const stoppedAt = new Date().toISOString();
|
|
380
|
+
await writeRuntimeState({
|
|
381
|
+
status: 'stopping',
|
|
382
|
+
updated_at: stoppedAt,
|
|
383
|
+
message: `Supervisor received ${event.signal}; stopping child process.`
|
|
384
|
+
});
|
|
385
|
+
await terminateChildProcess(child, config.killTimeoutSeconds);
|
|
386
|
+
const exitResult = await childExitPromise;
|
|
387
|
+
const finishedAt = new Date().toISOString();
|
|
388
|
+
await writeRuntimeState({
|
|
389
|
+
status: 'stopped',
|
|
390
|
+
updated_at: finishedAt,
|
|
391
|
+
child_pid: null,
|
|
392
|
+
last_exit_at: finishedAt,
|
|
393
|
+
last_exit_code: exitResult.code,
|
|
394
|
+
last_signal: exitResult.signal,
|
|
395
|
+
message: `Supervisor stopped after ${event.signal}.`
|
|
396
|
+
});
|
|
397
|
+
process.exitCode = 0;
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
if (event.type === 'child_error') {
|
|
401
|
+
const failedAt = new Date().toISOString();
|
|
402
|
+
await writeRuntimeState({
|
|
403
|
+
status: 'child_error',
|
|
404
|
+
updated_at: failedAt,
|
|
405
|
+
child_pid: null,
|
|
406
|
+
last_exit_at: failedAt,
|
|
407
|
+
message: event.error.message
|
|
408
|
+
});
|
|
409
|
+
throw event.error;
|
|
410
|
+
}
|
|
411
|
+
const exitedAt = new Date().toISOString();
|
|
412
|
+
await writeRuntimeState({
|
|
413
|
+
status: 'child_exited',
|
|
414
|
+
updated_at: exitedAt,
|
|
415
|
+
child_pid: null,
|
|
416
|
+
last_exit_at: exitedAt,
|
|
417
|
+
last_exit_code: event.code,
|
|
418
|
+
last_signal: event.signal,
|
|
419
|
+
message: event.code === null
|
|
420
|
+
? `control-host exited due to signal ${event.signal ?? 'unknown'}.`
|
|
421
|
+
: `control-host exited with code ${event.code}.`
|
|
422
|
+
});
|
|
423
|
+
process.exitCode = event.code ?? 0;
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
finally {
|
|
428
|
+
stopWaiter.dispose();
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
function resolveInstallConfig(flags) {
|
|
432
|
+
const cwd = process.cwd();
|
|
433
|
+
const homeDir = resolve(process.env.HOME ?? process.cwd());
|
|
434
|
+
const label = readStringFlag(flags, 'label') ?? undefined;
|
|
435
|
+
const packageRoot = findPackageRoot();
|
|
436
|
+
const envFilesFlag = parseControlHostSupervisionCsv(readStringFlag(flags, 'env-files'));
|
|
437
|
+
const config = buildControlHostSupervisionConfig({
|
|
438
|
+
homeDir,
|
|
439
|
+
cwd,
|
|
440
|
+
label,
|
|
441
|
+
repoRoot: readStringFlag(flags, 'repo-root') ?? cwd,
|
|
442
|
+
nodePath: readStringFlag(flags, 'node') ?? process.execPath,
|
|
443
|
+
cliEntrypoint: readStringFlag(flags, 'cli-entrypoint') ??
|
|
444
|
+
resolveDefaultControlHostSupervisionEntrypoint(process.argv[1] ?? null, packageRoot),
|
|
445
|
+
taskId: readStringFlag(flags, 'task') ?? undefined,
|
|
446
|
+
runId: readStringFlag(flags, 'run') ?? undefined,
|
|
447
|
+
pipelineId: readStringFlag(flags, 'pipeline') ?? undefined,
|
|
448
|
+
healthIntervalSeconds: readIntegerFlag(flags, 'health-interval'),
|
|
449
|
+
unhealthyThreshold: readIntegerFlag(flags, 'unhealthy-threshold'),
|
|
450
|
+
launchdThrottleSeconds: readIntegerFlag(flags, 'launchd-throttle'),
|
|
451
|
+
killTimeoutSeconds: readIntegerFlag(flags, 'kill-timeout') ??
|
|
452
|
+
DEFAULT_CONTROL_HOST_SUPERVISION_KILL_TIMEOUT_SECONDS,
|
|
453
|
+
envFiles: envFilesFlag ?? resolveDefaultControlHostSupervisionEnvFiles(homeDir),
|
|
454
|
+
shellPath: readStringFlag(flags, 'shell') ?? undefined
|
|
455
|
+
});
|
|
456
|
+
return {
|
|
457
|
+
label: config.label,
|
|
458
|
+
paths: config.paths,
|
|
459
|
+
config
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
async function resolveStoredControlHostSupervision(flags, requireConfig) {
|
|
463
|
+
const explicitConfigPath = readStringFlag(flags, 'config');
|
|
464
|
+
if (explicitConfigPath) {
|
|
465
|
+
const configPath = resolve(explicitConfigPath);
|
|
466
|
+
const config = await readJsonFileIfExists(configPath);
|
|
467
|
+
if (!config) {
|
|
468
|
+
throw new Error(`Control-host supervision config not found: ${configPath}`);
|
|
469
|
+
}
|
|
470
|
+
assertStoredControlHostSupervisionConfig(configPath, config);
|
|
471
|
+
return {
|
|
472
|
+
label: config.label,
|
|
473
|
+
paths: config.paths,
|
|
474
|
+
config
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
const homeDir = resolve(process.env.HOME ?? process.cwd());
|
|
478
|
+
const label = readStringFlag(flags, 'label') ?? undefined;
|
|
479
|
+
const paths = resolveControlHostSupervisionPaths({ homeDir, label });
|
|
480
|
+
const config = await readJsonFileIfExists(paths.configPath);
|
|
481
|
+
if (config) {
|
|
482
|
+
assertStoredControlHostSupervisionConfig(paths.configPath, config);
|
|
483
|
+
}
|
|
484
|
+
if (requireConfig && !config) {
|
|
485
|
+
throw new Error(`Control-host supervision is not installed for ${label ?? 'the default label'} (missing ${paths.configPath}).`);
|
|
486
|
+
}
|
|
487
|
+
return {
|
|
488
|
+
label: config?.label ?? label ?? DEFAULT_CONTROL_HOST_SUPERVISION_LABEL,
|
|
489
|
+
paths: config?.paths ?? paths,
|
|
490
|
+
config
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
async function loadBootstrapEnvironment(config, commandRunner = runCommandBuffer) {
|
|
494
|
+
const baseEnv = {
|
|
495
|
+
...process.env,
|
|
496
|
+
HOME: config.homeDir
|
|
497
|
+
};
|
|
498
|
+
if (config.envFiles.length === 0) {
|
|
499
|
+
return baseEnv;
|
|
500
|
+
}
|
|
501
|
+
const existingEnvFiles = [];
|
|
502
|
+
for (const envFile of config.envFiles) {
|
|
503
|
+
if (await pathExists(envFile)) {
|
|
504
|
+
existingEnvFiles.push(envFile);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
if (existingEnvFiles.length === 0) {
|
|
508
|
+
return baseEnv;
|
|
509
|
+
}
|
|
510
|
+
const shellScript = [
|
|
511
|
+
'set -a',
|
|
512
|
+
...existingEnvFiles.map((envFile) => `if [ -f '${escapeShellSingleQuotes(envFile)}' ]; then . '${escapeShellSingleQuotes(envFile)}'; fi`),
|
|
513
|
+
'env -0'
|
|
514
|
+
].join('; ');
|
|
515
|
+
const bootstrapTimeoutMs = resolveControlHostSupervisionProbeTimeoutMs(config.healthIntervalSeconds);
|
|
516
|
+
const result = await commandRunner(config.shellPath, ['-lc', shellScript], {
|
|
517
|
+
cwd: config.repoRoot,
|
|
518
|
+
env: baseEnv,
|
|
519
|
+
timeoutMs: bootstrapTimeoutMs
|
|
520
|
+
});
|
|
521
|
+
if (result.timedOut === true) {
|
|
522
|
+
throw new Error(`Timed out while sourcing control-host supervision env/bootstrap files after ${Math.round(bootstrapTimeoutMs / 1_000)}s.`);
|
|
523
|
+
}
|
|
524
|
+
if (result.exitCode !== 0) {
|
|
525
|
+
const stderr = result.stderr.toString('utf8').trim();
|
|
526
|
+
throw new Error(`Failed to source control-host supervision env/bootstrap files: ${stderr || 'shell returned a non-zero exit code.'}`);
|
|
527
|
+
}
|
|
528
|
+
return parseNulDelimitedEnv(result.stdout);
|
|
529
|
+
}
|
|
530
|
+
async function probeControlHostHealth(config, env, options = {}, commandRunner = runCommand) {
|
|
531
|
+
const probeTimeoutMs = resolveControlHostSupervisionProbeTimeoutMs(config.healthIntervalSeconds);
|
|
532
|
+
const probeStartedAt = Date.now();
|
|
533
|
+
const result = await commandRunner(config.nodePath, [
|
|
534
|
+
config.cliEntrypoint,
|
|
535
|
+
'co-status',
|
|
536
|
+
'--task',
|
|
537
|
+
config.taskId,
|
|
538
|
+
'--run',
|
|
539
|
+
config.runId,
|
|
540
|
+
'--format',
|
|
541
|
+
'json'
|
|
542
|
+
], {
|
|
543
|
+
cwd: config.repoRoot,
|
|
544
|
+
env,
|
|
545
|
+
timeoutMs: probeTimeoutMs
|
|
546
|
+
});
|
|
547
|
+
const probeDurationMs = Math.max(0, Date.now() - probeStartedAt);
|
|
548
|
+
if (result.timedOut === true) {
|
|
549
|
+
const diagnostic = await readControlHostSupervisionProbeTimeoutDiagnostic(config, env);
|
|
550
|
+
const timeoutQuarantine = evaluateControlHostSupervisionProbeTimeoutDiagnostic(diagnostic, {
|
|
551
|
+
minPollingUpdatedAt: options.minPollingUpdatedAt ?? null,
|
|
552
|
+
restartHistory: options.restartHistory ?? null
|
|
553
|
+
});
|
|
554
|
+
if (timeoutQuarantine) {
|
|
555
|
+
return {
|
|
556
|
+
healthy: timeoutQuarantine.healthy,
|
|
557
|
+
reason: timeoutQuarantine.reason,
|
|
558
|
+
message: timeoutQuarantine.message,
|
|
559
|
+
probeDurationMs,
|
|
560
|
+
diagnostic
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
return {
|
|
564
|
+
healthy: false,
|
|
565
|
+
reason: 'probe_timeout',
|
|
566
|
+
message: `co-status probe timed out after ${Math.round(probeTimeoutMs / 1_000)}s.`,
|
|
567
|
+
probeDurationMs,
|
|
568
|
+
diagnostic
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
if (result.exitCode !== 0) {
|
|
572
|
+
const detail = result.stderr.trim() || result.stdout.trim() || 'co-status command failed.';
|
|
573
|
+
return {
|
|
574
|
+
healthy: false,
|
|
575
|
+
reason: 'probe_failed',
|
|
576
|
+
message: `co-status probe failed: ${detail}`,
|
|
577
|
+
probeDurationMs,
|
|
578
|
+
diagnostic: null
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
let payload;
|
|
582
|
+
try {
|
|
583
|
+
payload = JSON.parse(result.stdout);
|
|
584
|
+
}
|
|
585
|
+
catch (error) {
|
|
586
|
+
return {
|
|
587
|
+
healthy: false,
|
|
588
|
+
reason: 'invalid_payload',
|
|
589
|
+
message: `co-status probe returned invalid JSON: ${error.message}`,
|
|
590
|
+
probeDurationMs,
|
|
591
|
+
diagnostic: null
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
const diagnostic = readControlHostSupervisionHealthDiagnostic(payload);
|
|
595
|
+
const evaluation = evaluateControlHostSupervisionHealthPayload(payload, {
|
|
596
|
+
minPollingUpdatedAt: options.minPollingUpdatedAt ?? null,
|
|
597
|
+
staleRestartRequiredGraceMs: config.healthIntervalSeconds * config.unhealthyThreshold * 1_000,
|
|
598
|
+
restartHistory: options.restartHistory ?? null
|
|
599
|
+
});
|
|
600
|
+
return {
|
|
601
|
+
healthy: evaluation.healthy,
|
|
602
|
+
reason: evaluation.reason,
|
|
603
|
+
message: evaluation.message,
|
|
604
|
+
probeDurationMs,
|
|
605
|
+
diagnostic
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
async function readControlHostSupervisionProbeTimeoutDiagnostic(config, env) {
|
|
609
|
+
try {
|
|
610
|
+
const statePath = resolveControlHostSupervisionProviderIntakeStatePath(config, env);
|
|
611
|
+
const persistedState = await readJsonFileIfExists(statePath);
|
|
612
|
+
if (!persistedState) {
|
|
613
|
+
return null;
|
|
614
|
+
}
|
|
615
|
+
const state = normalizeProviderIntakeState(persistedState);
|
|
616
|
+
const runningClaims = state.claims.filter(isRunningProviderIntakeClaim);
|
|
617
|
+
return readControlHostSupervisionHealthDiagnostic({
|
|
618
|
+
counts: {
|
|
619
|
+
running: runningClaims.length,
|
|
620
|
+
retrying: null,
|
|
621
|
+
max_allowed: null
|
|
622
|
+
},
|
|
623
|
+
polling: state.polling ?? null,
|
|
624
|
+
running: runningClaims.map(buildControlHostSupervisionRunningClaimSnapshot)
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
catch {
|
|
628
|
+
return null;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
function resolveControlHostSupervisionProviderIntakeStatePath(config, env) {
|
|
632
|
+
const effectiveRepoRoot = resolveControlHostSupervisionEffectiveRepoRoot(config, env);
|
|
633
|
+
const configuredRunsDir = env.CODEX_ORCHESTRATOR_RUNS_DIR?.trim();
|
|
634
|
+
const runsRoot = configuredRunsDir && configuredRunsDir.length > 0
|
|
635
|
+
? resolve(effectiveRepoRoot, configuredRunsDir)
|
|
636
|
+
: join(effectiveRepoRoot, '.runs');
|
|
637
|
+
return join(runsRoot, sanitizeTaskId(config.taskId), 'cli', sanitizeRunId(config.runId), PROVIDER_INTAKE_STATE_FILE);
|
|
638
|
+
}
|
|
639
|
+
function resolveControlHostSupervisionEffectiveRepoRoot(config, env) {
|
|
640
|
+
const envRepoRoot = env.CODEX_ORCHESTRATOR_ROOT?.trim();
|
|
641
|
+
return envRepoRoot && envRepoRoot.length > 0
|
|
642
|
+
? resolve(config.repoRoot, envRepoRoot)
|
|
643
|
+
: config.repoRoot;
|
|
644
|
+
}
|
|
645
|
+
function isRunningProviderIntakeClaim(claim) {
|
|
646
|
+
return claim.state === 'running';
|
|
647
|
+
}
|
|
648
|
+
function buildControlHostSupervisionRunningClaimSnapshot(claim) {
|
|
649
|
+
return {
|
|
650
|
+
issue_id: claim.issue_id,
|
|
651
|
+
issue_identifier: claim.issue_identifier,
|
|
652
|
+
state: claim.state,
|
|
653
|
+
display_state: claim.issue_state,
|
|
654
|
+
pid: null,
|
|
655
|
+
worker_host: claim.worker_host ?? null,
|
|
656
|
+
session_id: claim.run_id,
|
|
657
|
+
started_at: claim.launch_started_at ?? claim.updated_at,
|
|
658
|
+
last_event_at: claim.updated_at
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
async function inspectControlHostSupervisionLiveHealth(config, state, dependencies = {}) {
|
|
662
|
+
const loadBootstrapEnvironmentImpl = dependencies.loadBootstrapEnvironment ?? loadBootstrapEnvironment;
|
|
663
|
+
const probeControlHostHealthImpl = dependencies.probeControlHostHealth ?? probeControlHostHealth;
|
|
664
|
+
const evaluateFreshnessGaugeImpl = dependencies.evaluateFreshnessGauge ?? evaluateProviderControlHostFreshnessGauge;
|
|
665
|
+
const checkedAt = new Date().toISOString();
|
|
666
|
+
let bootstrappedEnv = {};
|
|
667
|
+
let coStatus = null;
|
|
668
|
+
try {
|
|
669
|
+
bootstrappedEnv = sanitizeProviderOverrideEnv(await loadBootstrapEnvironmentImpl(config), {
|
|
670
|
+
stripWorkspaceArtifactEnv: true
|
|
671
|
+
});
|
|
672
|
+
const probe = await probeControlHostHealthImpl(config, bootstrappedEnv, {
|
|
673
|
+
minPollingUpdatedAt: state?.last_started_at ?? null,
|
|
674
|
+
restartHistory: state?.restart_history ?? null
|
|
675
|
+
});
|
|
676
|
+
coStatus = {
|
|
677
|
+
healthy: probe.healthy,
|
|
678
|
+
reason: probe.reason,
|
|
679
|
+
message: probe.message,
|
|
680
|
+
diagnostic: probe.diagnostic
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
catch (error) {
|
|
684
|
+
coStatus = {
|
|
685
|
+
healthy: false,
|
|
686
|
+
reason: 'probe_failed',
|
|
687
|
+
message: error.message,
|
|
688
|
+
diagnostic: null
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
let freshnessGauge = null;
|
|
692
|
+
try {
|
|
693
|
+
const artifactRoot = dirname(resolveControlHostSupervisionProviderIntakeStatePath(config, bootstrappedEnv));
|
|
694
|
+
const freshnessReport = await evaluateFreshnessGaugeImpl({
|
|
695
|
+
artifactRoot
|
|
696
|
+
});
|
|
697
|
+
freshnessGauge = {
|
|
698
|
+
artifact_root: artifactRoot,
|
|
699
|
+
verdict: freshnessReport.verdict,
|
|
700
|
+
supporting_metrics_healthy: hasHealthyLiveProviderControlHostFreshness(freshnessReport)
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
catch {
|
|
704
|
+
freshnessGauge = null;
|
|
705
|
+
}
|
|
706
|
+
if (coStatus?.healthy) {
|
|
707
|
+
return {
|
|
708
|
+
checked_at: checkedAt,
|
|
709
|
+
healthy: true,
|
|
710
|
+
source: freshnessGauge ? 'co_status+freshness_gauge' : 'co_status',
|
|
711
|
+
reason: coStatus.reason,
|
|
712
|
+
message: coStatus.message,
|
|
713
|
+
stale_launchctl_metadata: false,
|
|
714
|
+
stale_persisted_state: false,
|
|
715
|
+
co_status: coStatus,
|
|
716
|
+
freshness_gauge: freshnessGauge
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
if (freshnessGauge?.supporting_metrics_healthy === true &&
|
|
720
|
+
(coStatus === null ||
|
|
721
|
+
coStatus.reason === 'probe_failed' ||
|
|
722
|
+
coStatus.reason === 'invalid_payload')) {
|
|
723
|
+
return {
|
|
724
|
+
checked_at: checkedAt,
|
|
725
|
+
healthy: true,
|
|
726
|
+
source: coStatus ? 'co_status+freshness_gauge' : 'freshness_gauge',
|
|
727
|
+
reason: 'fresh_artifacts',
|
|
728
|
+
message: 'Provider/control-host freshness artifacts remain current and advancing, so live host recovery is healthier than the stale launchd or persisted supervision metadata.',
|
|
729
|
+
stale_launchctl_metadata: false,
|
|
730
|
+
stale_persisted_state: false,
|
|
731
|
+
co_status: coStatus,
|
|
732
|
+
freshness_gauge: freshnessGauge
|
|
733
|
+
};
|
|
734
|
+
}
|
|
735
|
+
if (coStatus !== null) {
|
|
736
|
+
return {
|
|
737
|
+
checked_at: checkedAt,
|
|
738
|
+
healthy: coStatus.healthy,
|
|
739
|
+
source: 'co_status',
|
|
740
|
+
reason: coStatus.reason,
|
|
741
|
+
message: coStatus.message,
|
|
742
|
+
stale_launchctl_metadata: false,
|
|
743
|
+
stale_persisted_state: false,
|
|
744
|
+
co_status: coStatus,
|
|
745
|
+
freshness_gauge: freshnessGauge
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
if (freshnessGauge !== null) {
|
|
749
|
+
return {
|
|
750
|
+
checked_at: checkedAt,
|
|
751
|
+
healthy: freshnessGauge.supporting_metrics_healthy,
|
|
752
|
+
source: 'freshness_gauge',
|
|
753
|
+
reason: freshnessGauge.supporting_metrics_healthy
|
|
754
|
+
? 'fresh_artifacts'
|
|
755
|
+
: 'freshness_unhealthy',
|
|
756
|
+
message: freshnessGauge.supporting_metrics_healthy
|
|
757
|
+
? 'Provider/control-host freshness artifacts remain current and advancing.'
|
|
758
|
+
: 'Provider/control-host freshness artifacts are not current enough to override stored supervision state.',
|
|
759
|
+
stale_launchctl_metadata: false,
|
|
760
|
+
stale_persisted_state: false,
|
|
761
|
+
co_status: null,
|
|
762
|
+
freshness_gauge: freshnessGauge
|
|
763
|
+
};
|
|
764
|
+
}
|
|
765
|
+
return null;
|
|
766
|
+
}
|
|
767
|
+
function hasHealthyLiveProviderControlHostFreshness(report) {
|
|
768
|
+
return (report.metrics.last_successful_refresh_age_ms.verdict === 'healthy' &&
|
|
769
|
+
report.metrics.active_heartbeat_age_ms.verdict === 'healthy' &&
|
|
770
|
+
report.metrics.polling_health.verdict === 'healthy' &&
|
|
771
|
+
report.metrics.polling_health.value === 'ok');
|
|
772
|
+
}
|
|
773
|
+
function buildControlHostSupervisionStatusPayload(input) {
|
|
774
|
+
const launchctlLoaded = input.launchctl.exitCode === 0;
|
|
775
|
+
const serviceLoaded = launchctlLoaded ||
|
|
776
|
+
(input.launchAgent.classification === 'managed_supervision' &&
|
|
777
|
+
input.liveHost?.healthy === true);
|
|
778
|
+
const effectiveState = resolveEffectiveControlHostSupervisionState(input.state, input.liveHost ?? null);
|
|
779
|
+
const stalePersistedState = hasControlHostSupervisionStateDrift(input.state, effectiveState);
|
|
780
|
+
const staleLaunchctlMetadata = serviceLoaded && !launchctlLoaded;
|
|
781
|
+
const summarySource = input.launchctl.stdout.trim() || input.launchctl.stderr.trim() || null;
|
|
782
|
+
const liveHost = input.liveHost === undefined || input.liveHost === null
|
|
783
|
+
? null
|
|
784
|
+
: {
|
|
785
|
+
...input.liveHost,
|
|
786
|
+
stale_launchctl_metadata: staleLaunchctlMetadata,
|
|
787
|
+
stale_persisted_state: stalePersistedState
|
|
788
|
+
};
|
|
789
|
+
return {
|
|
790
|
+
installed: input.resolved.config !== null,
|
|
791
|
+
label: input.resolved.label,
|
|
792
|
+
service_target: input.serviceTarget,
|
|
793
|
+
config_path: input.resolved.paths.configPath,
|
|
794
|
+
state_path: input.resolved.paths.statePath,
|
|
795
|
+
plist_path: input.resolved.paths.plistPath,
|
|
796
|
+
logs: {
|
|
797
|
+
directory: input.resolved.paths.logsDir,
|
|
798
|
+
stdout_path: input.resolved.paths.stdoutLogPath,
|
|
799
|
+
stderr_path: input.resolved.paths.stderrLogPath
|
|
800
|
+
},
|
|
801
|
+
config: input.resolved.config,
|
|
802
|
+
state: effectiveState,
|
|
803
|
+
persisted_state: input.state,
|
|
804
|
+
launch_agent: input.launchAgent,
|
|
805
|
+
live_host: liveHost,
|
|
806
|
+
rollout: classifyControlHostSupervisionRollout({
|
|
807
|
+
config: input.resolved.config,
|
|
808
|
+
launchAgent: input.launchAgent,
|
|
809
|
+
serviceLoaded,
|
|
810
|
+
launchctlLoaded
|
|
811
|
+
}),
|
|
812
|
+
service: {
|
|
813
|
+
loaded: serviceLoaded,
|
|
814
|
+
loaded_source: serviceLoaded === launchctlLoaded ? 'launchctl' : 'live_host',
|
|
815
|
+
launchctl_loaded: launchctlLoaded,
|
|
816
|
+
stale_launchctl_metadata: staleLaunchctlMetadata,
|
|
817
|
+
exit_code: input.launchctl.exitCode,
|
|
818
|
+
summary: summarySource ? firstNonEmptyLine(summarySource) : null,
|
|
819
|
+
stderr: input.launchctl.stderr.trim().length > 0 ? input.launchctl.stderr.trim() : null
|
|
820
|
+
}
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
function formatControlHostSupervisionStatus(payload) {
|
|
824
|
+
const lines = [
|
|
825
|
+
`Control-host supervision: ${payload.installed ? 'installed' : 'not installed'}`,
|
|
826
|
+
`Rollout: ${payload.rollout.mode}`,
|
|
827
|
+
`Migration required: ${payload.rollout.migration_required ? 'yes' : 'no'}`,
|
|
828
|
+
`Label: ${payload.label}`,
|
|
829
|
+
`Service target: ${payload.service_target}`,
|
|
830
|
+
`Service loaded: ${payload.service.loaded ? 'yes' : 'no'}`,
|
|
831
|
+
`launchctl loaded: ${payload.service.launchctl_loaded ? 'yes' : 'no'}${payload.service.stale_launchctl_metadata
|
|
832
|
+
? ' (stale metadata; live host evidence is healthier)'
|
|
833
|
+
: ''}`,
|
|
834
|
+
`Config: ${payload.config_path}`,
|
|
835
|
+
`Plist: ${payload.plist_path}`,
|
|
836
|
+
`State: ${payload.state_path}`,
|
|
837
|
+
`Logs: ${payload.logs.stdout_path} | ${payload.logs.stderr_path}`
|
|
838
|
+
];
|
|
839
|
+
lines.push(`Rollout summary: ${payload.rollout.summary}`);
|
|
840
|
+
if (payload.launch_agent.detected_program) {
|
|
841
|
+
lines.push(`LaunchAgent program: ${payload.launch_agent.detected_program}`);
|
|
842
|
+
}
|
|
843
|
+
if (payload.config) {
|
|
844
|
+
lines.push(`Repo root: ${payload.config.repoRoot}`);
|
|
845
|
+
lines.push(`CLI entrypoint: ${payload.config.cliEntrypoint}`);
|
|
846
|
+
lines.push(`Task/run/pipeline: ${payload.config.taskId} / ${payload.config.runId} / ${payload.config.pipelineId}`);
|
|
847
|
+
lines.push(`Health: interval=${payload.config.healthIntervalSeconds}s threshold=${payload.config.unhealthyThreshold}`);
|
|
848
|
+
}
|
|
849
|
+
if (payload.live_host) {
|
|
850
|
+
lines.push(`Live host: ${payload.live_host.healthy === null
|
|
851
|
+
? 'unknown'
|
|
852
|
+
: payload.live_host.healthy
|
|
853
|
+
? 'healthy'
|
|
854
|
+
: 'unhealthy'} via ${formatControlHostSupervisionLiveHealthSource(payload.live_host.source)}`);
|
|
855
|
+
if (payload.live_host.reason) {
|
|
856
|
+
lines.push(`Live host reason: ${payload.live_host.reason}`);
|
|
857
|
+
}
|
|
858
|
+
if (payload.live_host.message) {
|
|
859
|
+
lines.push(`Live host detail: ${payload.live_host.message}`);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
if (payload.state) {
|
|
863
|
+
lines.push(`State status: ${payload.state.status}`);
|
|
864
|
+
lines.push(`Supervised child pid: ${payload.state.child_pid === null ? 'none recorded' : payload.state.child_pid}`);
|
|
865
|
+
if (payload.state.last_health_status) {
|
|
866
|
+
lines.push(`Last health: ${payload.state.last_health_status} (${payload.state.consecutive_unhealthy_samples}/${payload.state.unhealthy_threshold})`);
|
|
867
|
+
}
|
|
868
|
+
if (payload.state.last_restart_reason) {
|
|
869
|
+
lines.push(`Last restart reason: ${payload.state.last_restart_reason}`);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
if (payload.persisted_state && payload.live_host?.stale_persisted_state === true) {
|
|
873
|
+
lines.push(`Persisted state status: ${payload.persisted_state.status}`);
|
|
874
|
+
if (payload.persisted_state.last_health_status) {
|
|
875
|
+
lines.push(`Persisted last health: ${payload.persisted_state.last_health_status} (${payload.persisted_state.consecutive_unhealthy_samples}/${payload.persisted_state.unhealthy_threshold})`);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
if (payload.service.summary) {
|
|
879
|
+
lines.push(`launchctl: ${payload.service.summary}`);
|
|
880
|
+
}
|
|
881
|
+
return lines.join('\n');
|
|
882
|
+
}
|
|
883
|
+
function formatControlHostSupervisionLiveHealthSource(source) {
|
|
884
|
+
switch (source) {
|
|
885
|
+
case 'co_status':
|
|
886
|
+
return 'co-status';
|
|
887
|
+
case 'freshness_gauge':
|
|
888
|
+
return 'provider freshness gauge';
|
|
889
|
+
case 'co_status+freshness_gauge':
|
|
890
|
+
return 'co-status plus provider freshness gauge';
|
|
891
|
+
default:
|
|
892
|
+
return 'no live host evidence';
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
function resolveEffectiveControlHostSupervisionState(persistedState, liveHost) {
|
|
896
|
+
if (!persistedState || liveHost?.healthy !== true) {
|
|
897
|
+
return persistedState;
|
|
898
|
+
}
|
|
899
|
+
const effectiveStatus = liveHost.reason === 'active_worker_restart_quarantine' ||
|
|
900
|
+
liveHost.reason === 'active_worker_probe_timeout_quarantine'
|
|
901
|
+
? 'quarantined'
|
|
902
|
+
: 'healthy';
|
|
903
|
+
const effectiveLastHealthStatus = liveHost.reason ?? persistedState.last_health_status;
|
|
904
|
+
const effectiveConsecutiveUnhealthySamples = effectiveStatus === 'healthy' ? 0 : persistedState.consecutive_unhealthy_samples;
|
|
905
|
+
const effectiveMessage = liveHost.message ?? persistedState.message;
|
|
906
|
+
if (persistedState.status === effectiveStatus &&
|
|
907
|
+
persistedState.last_health_status === effectiveLastHealthStatus &&
|
|
908
|
+
persistedState.consecutive_unhealthy_samples === effectiveConsecutiveUnhealthySamples &&
|
|
909
|
+
persistedState.message === effectiveMessage) {
|
|
910
|
+
return persistedState;
|
|
911
|
+
}
|
|
912
|
+
return {
|
|
913
|
+
...persistedState,
|
|
914
|
+
status: effectiveStatus,
|
|
915
|
+
updated_at: liveHost.checked_at ?? persistedState.updated_at,
|
|
916
|
+
last_health_check_at: liveHost.checked_at ?? persistedState.last_health_check_at,
|
|
917
|
+
last_health_status: effectiveLastHealthStatus,
|
|
918
|
+
consecutive_unhealthy_samples: effectiveConsecutiveUnhealthySamples,
|
|
919
|
+
message: effectiveMessage
|
|
920
|
+
};
|
|
921
|
+
}
|
|
922
|
+
function hasControlHostSupervisionStateDrift(persistedState, effectiveState) {
|
|
923
|
+
if (!persistedState || !effectiveState) {
|
|
924
|
+
return false;
|
|
925
|
+
}
|
|
926
|
+
return (persistedState.status !== effectiveState.status ||
|
|
927
|
+
persistedState.updated_at !== effectiveState.updated_at ||
|
|
928
|
+
persistedState.last_health_check_at !== effectiveState.last_health_check_at ||
|
|
929
|
+
persistedState.last_health_status !== effectiveState.last_health_status ||
|
|
930
|
+
persistedState.consecutive_unhealthy_samples !== effectiveState.consecutive_unhealthy_samples ||
|
|
931
|
+
persistedState.message !== effectiveState.message);
|
|
932
|
+
}
|
|
933
|
+
function inspectControlHostSupervisionLaunchAgent(plistContents, config) {
|
|
934
|
+
if (plistContents === null) {
|
|
935
|
+
return {
|
|
936
|
+
exists: false,
|
|
937
|
+
program_arguments: [],
|
|
938
|
+
working_directory: null,
|
|
939
|
+
detected_program: null,
|
|
940
|
+
classification: 'missing'
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
const programArguments = extractPlistStringArray(plistContents, 'ProgramArguments');
|
|
944
|
+
const workingDirectory = extractPlistStringValue(plistContents, 'WorkingDirectory');
|
|
945
|
+
const detectedProgram = programArguments[0] ?? null;
|
|
946
|
+
return {
|
|
947
|
+
exists: true,
|
|
948
|
+
program_arguments: programArguments,
|
|
949
|
+
working_directory: workingDirectory,
|
|
950
|
+
detected_program: detectedProgram,
|
|
951
|
+
classification: classifyControlHostSupervisionLaunchAgent(programArguments, config)
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
function classifyControlHostSupervisionLaunchAgent(programArguments, config) {
|
|
955
|
+
const detectedProgram = programArguments[0] ?? null;
|
|
956
|
+
if (detectedProgram === null) {
|
|
957
|
+
return 'unknown';
|
|
958
|
+
}
|
|
959
|
+
if (detectedProgram.endsWith('/co-control-host-supervisor.sh')) {
|
|
960
|
+
return 'legacy_shim';
|
|
961
|
+
}
|
|
962
|
+
if (config &&
|
|
963
|
+
arraysEqual(programArguments, buildExpectedControlHostSupervisionProgramArguments(config))) {
|
|
964
|
+
return 'managed_supervision';
|
|
965
|
+
}
|
|
966
|
+
return 'unknown';
|
|
967
|
+
}
|
|
968
|
+
function buildExpectedControlHostSupervisionProgramArguments(config) {
|
|
969
|
+
return [
|
|
970
|
+
config.nodePath,
|
|
971
|
+
config.cliEntrypoint,
|
|
972
|
+
'control-host',
|
|
973
|
+
'supervise',
|
|
974
|
+
'run',
|
|
975
|
+
'--config',
|
|
976
|
+
config.paths.configPath
|
|
977
|
+
];
|
|
978
|
+
}
|
|
979
|
+
function classifyControlHostSupervisionRollout(input) {
|
|
980
|
+
if (input.config &&
|
|
981
|
+
input.launchAgent.exists &&
|
|
982
|
+
input.launchAgent.classification === 'managed_supervision') {
|
|
983
|
+
if (input.serviceLoaded && input.launchctlLoaded === false) {
|
|
984
|
+
return {
|
|
985
|
+
mode: 'managed_supervision',
|
|
986
|
+
migration_required: false,
|
|
987
|
+
summary: 'LaunchAgent matches the stored managed supervision config; launchctl metadata appears stale because live host evidence remains healthy.'
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
if (!input.serviceLoaded) {
|
|
991
|
+
return {
|
|
992
|
+
mode: 'mixed',
|
|
993
|
+
migration_required: true,
|
|
994
|
+
summary: 'Managed LaunchAgent plist exists, but launchctl does not report the managed service target as loaded.'
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
return {
|
|
998
|
+
mode: 'managed_supervision',
|
|
999
|
+
migration_required: false,
|
|
1000
|
+
summary: 'LaunchAgent matches the stored managed supervision config.'
|
|
1001
|
+
};
|
|
1002
|
+
}
|
|
1003
|
+
if (input.launchAgent.classification === 'legacy_shim') {
|
|
1004
|
+
return {
|
|
1005
|
+
mode: input.config ? 'mixed' : 'legacy_shim',
|
|
1006
|
+
migration_required: true,
|
|
1007
|
+
summary: input.config
|
|
1008
|
+
? 'Stored managed config exists, but the active LaunchAgent still targets the legacy shim.'
|
|
1009
|
+
: 'LaunchAgent still targets the legacy shim wrapper.'
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
if (!input.config && !input.launchAgent.exists) {
|
|
1013
|
+
return {
|
|
1014
|
+
mode: 'not_installed',
|
|
1015
|
+
migration_required: false,
|
|
1016
|
+
summary: 'No managed config or LaunchAgent plist is installed.'
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
if (input.config && !input.launchAgent.exists) {
|
|
1020
|
+
return {
|
|
1021
|
+
mode: 'mixed',
|
|
1022
|
+
migration_required: true,
|
|
1023
|
+
summary: 'Managed config exists, but the LaunchAgent plist is missing.'
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
if (!input.config && input.launchAgent.exists) {
|
|
1027
|
+
return {
|
|
1028
|
+
mode: 'mixed',
|
|
1029
|
+
migration_required: true,
|
|
1030
|
+
summary: 'LaunchAgent exists without a matching managed config; inspect the plist before rollout.'
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
return {
|
|
1034
|
+
mode: 'mixed',
|
|
1035
|
+
migration_required: true,
|
|
1036
|
+
summary: 'Managed config exists, but the LaunchAgent program arguments do not match the packaged supervision runner.'
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
function emitOutput(format, payload, text) {
|
|
1040
|
+
if (format === 'json') {
|
|
1041
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
1042
|
+
return;
|
|
1043
|
+
}
|
|
1044
|
+
console.log(text);
|
|
1045
|
+
}
|
|
1046
|
+
function readFormatFlag(flags) {
|
|
1047
|
+
const format = readStringFlag(flags, 'format');
|
|
1048
|
+
if (format === undefined || format === 'text') {
|
|
1049
|
+
return 'text';
|
|
1050
|
+
}
|
|
1051
|
+
if (format === 'json') {
|
|
1052
|
+
return 'json';
|
|
1053
|
+
}
|
|
1054
|
+
throw new Error('--format must be either "text" or "json".');
|
|
1055
|
+
}
|
|
1056
|
+
function readStringFlag(flags, key) {
|
|
1057
|
+
const value = flags[key];
|
|
1058
|
+
if (value === true) {
|
|
1059
|
+
throw new Error(`--${key} requires a value.`);
|
|
1060
|
+
}
|
|
1061
|
+
if (typeof value !== 'string') {
|
|
1062
|
+
return undefined;
|
|
1063
|
+
}
|
|
1064
|
+
const trimmed = value.trim();
|
|
1065
|
+
if (trimmed.length === 0) {
|
|
1066
|
+
throw new Error(`--${key} requires a value.`);
|
|
1067
|
+
}
|
|
1068
|
+
return trimmed;
|
|
1069
|
+
}
|
|
1070
|
+
function readIntegerFlag(flags, key) {
|
|
1071
|
+
const value = readStringFlag(flags, key);
|
|
1072
|
+
if (!value) {
|
|
1073
|
+
return undefined;
|
|
1074
|
+
}
|
|
1075
|
+
if (!/^[+-]?\d+$/u.test(value)) {
|
|
1076
|
+
throw new Error(`--${key} must be an integer.`);
|
|
1077
|
+
}
|
|
1078
|
+
const parsed = Number(value);
|
|
1079
|
+
if (!Number.isInteger(parsed)) {
|
|
1080
|
+
throw new Error(`--${key} must be an integer.`);
|
|
1081
|
+
}
|
|
1082
|
+
return parsed;
|
|
1083
|
+
}
|
|
1084
|
+
async function runLaunchctl(args, options) {
|
|
1085
|
+
const result = await runCommand('launchctl', args);
|
|
1086
|
+
if (result.exitCode !== 0 && !options?.allowFailure) {
|
|
1087
|
+
const detail = result.stderr.trim() || result.stdout.trim() || 'launchctl failed.';
|
|
1088
|
+
throw new Error(`launchctl ${args.join(' ')} failed: ${detail}`);
|
|
1089
|
+
}
|
|
1090
|
+
return result;
|
|
1091
|
+
}
|
|
1092
|
+
function buildNextControlHostSupervisionState(input) {
|
|
1093
|
+
const resetForRunning = input.update.status === 'running'
|
|
1094
|
+
? {
|
|
1095
|
+
last_exit_at: null,
|
|
1096
|
+
last_exit_code: null,
|
|
1097
|
+
last_signal: null,
|
|
1098
|
+
last_health_check_at: null,
|
|
1099
|
+
last_health_status: null,
|
|
1100
|
+
last_probe_duration_ms: null,
|
|
1101
|
+
consecutive_unhealthy_samples: 0
|
|
1102
|
+
}
|
|
1103
|
+
: {};
|
|
1104
|
+
return {
|
|
1105
|
+
...input.priorState,
|
|
1106
|
+
...resetForRunning,
|
|
1107
|
+
...input.update,
|
|
1108
|
+
label: input.config.label,
|
|
1109
|
+
repo_root: input.config.repoRoot,
|
|
1110
|
+
service_target: input.serviceTarget,
|
|
1111
|
+
unhealthy_threshold: input.config.unhealthyThreshold,
|
|
1112
|
+
health_interval_seconds: input.config.healthIntervalSeconds
|
|
1113
|
+
};
|
|
1114
|
+
}
|
|
1115
|
+
function buildControlHostSupervisionRestartRecord(input) {
|
|
1116
|
+
return {
|
|
1117
|
+
requested_at: input.requestedAt,
|
|
1118
|
+
reason: input.reason,
|
|
1119
|
+
message: input.message,
|
|
1120
|
+
consecutive_unhealthy_samples: input.consecutiveUnhealthySamples,
|
|
1121
|
+
child_pid: input.childPid,
|
|
1122
|
+
probe_duration_ms: normalizeProbeDurationMs(input.probeDurationMs),
|
|
1123
|
+
diagnostic: input.diagnostic
|
|
1124
|
+
};
|
|
1125
|
+
}
|
|
1126
|
+
function normalizeProbeDurationMs(value) {
|
|
1127
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
1128
|
+
return null;
|
|
1129
|
+
}
|
|
1130
|
+
return Math.max(0, Math.round(value));
|
|
1131
|
+
}
|
|
1132
|
+
function appendControlHostSupervisionRestartRecord(history, record) {
|
|
1133
|
+
return [...(history ?? []), record].slice(-CONTROL_HOST_SUPERVISION_RESTART_HISTORY_LIMIT);
|
|
1134
|
+
}
|
|
1135
|
+
function isControlHostSupervisionQuarantineProbe(probe) {
|
|
1136
|
+
return (probe.reason === 'active_worker_restart_quarantine' ||
|
|
1137
|
+
probe.reason === 'active_worker_probe_timeout_quarantine');
|
|
1138
|
+
}
|
|
1139
|
+
function resolveControlHostSupervisionQuarantineUnhealthySamples(input) {
|
|
1140
|
+
const latestRestartRecord = input.priorState.restart_history && input.priorState.restart_history.length > 0
|
|
1141
|
+
? input.priorState.restart_history[input.priorState.restart_history.length - 1]
|
|
1142
|
+
: null;
|
|
1143
|
+
const candidates = [
|
|
1144
|
+
input.currentConsecutiveUnhealthySamples,
|
|
1145
|
+
input.priorState.consecutive_unhealthy_samples,
|
|
1146
|
+
latestRestartRecord?.consecutive_unhealthy_samples ?? 0
|
|
1147
|
+
].filter((value) => Number.isFinite(value) && value > 0);
|
|
1148
|
+
const priorMaximum = candidates.length > 0 ? Math.max(...candidates) : 0;
|
|
1149
|
+
if (priorMaximum > 0) {
|
|
1150
|
+
return priorMaximum;
|
|
1151
|
+
}
|
|
1152
|
+
return Math.max(1, input.config.unhealthyThreshold);
|
|
1153
|
+
}
|
|
1154
|
+
async function bootoutLaunchctlServiceTarget(serviceTarget) {
|
|
1155
|
+
const result = await runLaunchctl(['bootout', serviceTarget], { allowFailure: true });
|
|
1156
|
+
if (result.exitCode === 0 || isIgnorableLaunchctlBootoutFailure(result)) {
|
|
1157
|
+
return;
|
|
1158
|
+
}
|
|
1159
|
+
const detail = result.stderr.trim() || result.stdout.trim() || 'launchctl bootout failed.';
|
|
1160
|
+
throw new Error(`launchctl bootout ${serviceTarget} failed: ${detail}`);
|
|
1161
|
+
}
|
|
1162
|
+
async function rollbackFailedControlHostSupervisionInstall(paths, serviceTarget, options) {
|
|
1163
|
+
const bootout = options?.bootout ?? bootoutLaunchctlServiceTarget;
|
|
1164
|
+
const remove = options?.remove ?? rm;
|
|
1165
|
+
await bootout(serviceTarget);
|
|
1166
|
+
await remove(paths.plistPath, { force: true });
|
|
1167
|
+
await remove(paths.supportDir, { recursive: true, force: true });
|
|
1168
|
+
await remove(paths.logsDir, { recursive: true, force: true });
|
|
1169
|
+
}
|
|
1170
|
+
async function bootstrapLaunchctlPlist(plistPath, options) {
|
|
1171
|
+
const bootstrap = options?.bootstrap ??
|
|
1172
|
+
(async (args) => {
|
|
1173
|
+
await runLaunchctl(args);
|
|
1174
|
+
});
|
|
1175
|
+
const retryAttempts = options?.retryAttempts ?? CONTROL_HOST_SUPERVISION_LAUNCHCTL_BOOTSTRAP_RETRY_ATTEMPTS;
|
|
1176
|
+
const retryDelayMs = options?.retryDelayMs ?? CONTROL_HOST_SUPERVISION_LAUNCHCTL_BOOTSTRAP_RETRY_DELAY_MS;
|
|
1177
|
+
const sleep = options?.sleep ??
|
|
1178
|
+
(async (ms) => {
|
|
1179
|
+
await new Promise((resolve) => {
|
|
1180
|
+
setTimeout(resolve, ms);
|
|
1181
|
+
});
|
|
1182
|
+
});
|
|
1183
|
+
let attemptsRemaining = retryAttempts;
|
|
1184
|
+
for (;;) {
|
|
1185
|
+
try {
|
|
1186
|
+
await bootstrap(['bootstrap', resolveLaunchdDomain(), plistPath]);
|
|
1187
|
+
return;
|
|
1188
|
+
}
|
|
1189
|
+
catch (error) {
|
|
1190
|
+
attemptsRemaining -= 1;
|
|
1191
|
+
if (attemptsRemaining <= 0 || !isRetryableLaunchctlBootstrapError(error)) {
|
|
1192
|
+
throw error;
|
|
1193
|
+
}
|
|
1194
|
+
await sleep(retryDelayMs);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
async function captureExistingControlHostSupervisionInstall(paths) {
|
|
1199
|
+
const [configContents, stateContents, plistContents] = await Promise.all([
|
|
1200
|
+
readTextFileIfExists(paths.configPath),
|
|
1201
|
+
readTextFileIfExists(paths.statePath),
|
|
1202
|
+
readTextFileIfExists(paths.plistPath)
|
|
1203
|
+
]);
|
|
1204
|
+
if (configContents === null && stateContents === null && plistContents === null) {
|
|
1205
|
+
return null;
|
|
1206
|
+
}
|
|
1207
|
+
return {
|
|
1208
|
+
paths,
|
|
1209
|
+
configContents,
|
|
1210
|
+
stateContents,
|
|
1211
|
+
plistContents
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
async function restoreExistingControlHostSupervisionInstall(snapshot, serviceTarget, options) {
|
|
1215
|
+
const bootout = options?.bootout ?? bootoutLaunchctlServiceTarget;
|
|
1216
|
+
const remove = options?.remove ?? rm;
|
|
1217
|
+
const write = options?.write ?? writeFile;
|
|
1218
|
+
await bootout(serviceTarget);
|
|
1219
|
+
await mkdir(snapshot.paths.supportDir, { recursive: true });
|
|
1220
|
+
await mkdir(dirname(snapshot.paths.plistPath), { recursive: true });
|
|
1221
|
+
await restoreTextFile(snapshot.paths.configPath, snapshot.configContents, {
|
|
1222
|
+
write,
|
|
1223
|
+
remove
|
|
1224
|
+
});
|
|
1225
|
+
await restoreTextFile(snapshot.paths.statePath, snapshot.stateContents, {
|
|
1226
|
+
write,
|
|
1227
|
+
remove
|
|
1228
|
+
});
|
|
1229
|
+
await restoreTextFile(snapshot.paths.plistPath, snapshot.plistContents, {
|
|
1230
|
+
write,
|
|
1231
|
+
remove
|
|
1232
|
+
});
|
|
1233
|
+
if (snapshot.plistContents !== null) {
|
|
1234
|
+
await bootstrapLaunchctlPlist(snapshot.paths.plistPath, {
|
|
1235
|
+
bootstrap: options?.bootstrap
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
async function removeInstalledControlHostSupervisionArtifacts(resolved, options) {
|
|
1240
|
+
await rollbackFailedControlHostSupervisionInstall(resolved.paths, resolveControlHostSupervisionServiceTarget(resolved.label), options);
|
|
1241
|
+
return resolved.paths;
|
|
1242
|
+
}
|
|
1243
|
+
function createControlHostSupervisionChildEventPromises(child) {
|
|
1244
|
+
return {
|
|
1245
|
+
childExitPromise: new Promise((resolve) => {
|
|
1246
|
+
child.once('exit', (code, signal) => {
|
|
1247
|
+
resolve({
|
|
1248
|
+
type: 'child_exit',
|
|
1249
|
+
code: typeof code === 'number' ? code : null,
|
|
1250
|
+
signal: typeof signal === 'string' ? signal : null
|
|
1251
|
+
});
|
|
1252
|
+
});
|
|
1253
|
+
}),
|
|
1254
|
+
childErrorPromise: new Promise((resolve) => {
|
|
1255
|
+
child.once('error', (error) => {
|
|
1256
|
+
resolve({
|
|
1257
|
+
type: 'child_error',
|
|
1258
|
+
error
|
|
1259
|
+
});
|
|
1260
|
+
});
|
|
1261
|
+
})
|
|
1262
|
+
};
|
|
1263
|
+
}
|
|
1264
|
+
function isIgnorableLaunchctlBootoutFailure(result) {
|
|
1265
|
+
const detail = `${result.stdout}\n${result.stderr}`.toLowerCase();
|
|
1266
|
+
return /could not find service|service.*not found|no such process|not loaded/u.test(detail);
|
|
1267
|
+
}
|
|
1268
|
+
function isRetryableLaunchctlBootstrapError(error) {
|
|
1269
|
+
const detail = error instanceof Error
|
|
1270
|
+
? error.message
|
|
1271
|
+
: typeof error === 'string'
|
|
1272
|
+
? error
|
|
1273
|
+
: JSON.stringify(error);
|
|
1274
|
+
return /bootstrap failed:\s*5:\s*input\/output error/ui.test(detail);
|
|
1275
|
+
}
|
|
1276
|
+
function resolveControlHostSupervisionServiceTarget(label) {
|
|
1277
|
+
return `${resolveLaunchdDomain()}/${label}`;
|
|
1278
|
+
}
|
|
1279
|
+
function resolveLaunchdDomain() {
|
|
1280
|
+
const uid = process.getuid?.();
|
|
1281
|
+
if (!Number.isInteger(uid)) {
|
|
1282
|
+
throw new Error('control-host supervision currently requires a POSIX user id.');
|
|
1283
|
+
}
|
|
1284
|
+
return `gui/${uid}`;
|
|
1285
|
+
}
|
|
1286
|
+
async function terminateChildProcess(child, killTimeoutSeconds, options) {
|
|
1287
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1288
|
+
return;
|
|
1289
|
+
}
|
|
1290
|
+
const rootPid = normalizeTrackedPid(child.pid);
|
|
1291
|
+
const exitPromise = new Promise((resolve) => {
|
|
1292
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
1293
|
+
resolve();
|
|
1294
|
+
return;
|
|
1295
|
+
}
|
|
1296
|
+
child.once('exit', () => resolve());
|
|
1297
|
+
});
|
|
1298
|
+
child.kill('SIGTERM');
|
|
1299
|
+
const processGroupExitController = new AbortController();
|
|
1300
|
+
const processGroupExitPromise = rootPid === null
|
|
1301
|
+
? Promise.resolve()
|
|
1302
|
+
: waitForProcessGroupToExit(rootPid, options?.listProcessGroupPids, processGroupExitController.signal);
|
|
1303
|
+
const killWaiter = createSleepWaiter(killTimeoutSeconds * 1_000);
|
|
1304
|
+
const timedOut = await Promise.race([
|
|
1305
|
+
Promise.all([exitPromise, processGroupExitPromise]).then(() => false),
|
|
1306
|
+
killWaiter.promise.then(() => true)
|
|
1307
|
+
]).finally(() => {
|
|
1308
|
+
killWaiter.dispose();
|
|
1309
|
+
});
|
|
1310
|
+
if (!timedOut) {
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
processGroupExitController.abort();
|
|
1314
|
+
if (rootPid !== null) {
|
|
1315
|
+
killTrackedProcessGroup(rootPid, 'SIGKILL', options?.killProcessGroup);
|
|
1316
|
+
}
|
|
1317
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
1318
|
+
if (rootPid !== null) {
|
|
1319
|
+
await (options?.listDescendantPids ?? listDescendantProcessIds)(rootPid).catch(() => []);
|
|
1320
|
+
}
|
|
1321
|
+
// Generic timeout cleanup is process-group-scoped. Detached provider-worker
|
|
1322
|
+
// issue runs can remain descendants until reparenting and must stay
|
|
1323
|
+
// diagnostic-only here instead of becoming additional kill targets.
|
|
1324
|
+
child.kill('SIGKILL');
|
|
1325
|
+
await exitPromise.catch(() => undefined);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
function normalizeTrackedPid(pid) {
|
|
1329
|
+
return typeof pid === 'number' && Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
1330
|
+
}
|
|
1331
|
+
async function waitForProcessGroupToExit(rootPid, listProcessGroupPids = listProcessGroupProcessIds, signal) {
|
|
1332
|
+
for (;;) {
|
|
1333
|
+
if (signal?.aborted) {
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
const processGroupPids = await listProcessGroupPids(rootPid).catch(() => null);
|
|
1337
|
+
if (processGroupPids !== null && processGroupPids.length === 0) {
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
await waitForAbortableSleep(25, signal);
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
async function waitForAbortableSleep(ms, signal) {
|
|
1344
|
+
if (signal?.aborted) {
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
const waiter = createSleepWaiter(ms);
|
|
1348
|
+
let abortListener = null;
|
|
1349
|
+
try {
|
|
1350
|
+
await Promise.race([
|
|
1351
|
+
waiter.promise,
|
|
1352
|
+
new Promise((resolve) => {
|
|
1353
|
+
if (signal === undefined) {
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
if (signal.aborted) {
|
|
1357
|
+
waiter.dispose();
|
|
1358
|
+
resolve();
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
abortListener = () => {
|
|
1362
|
+
waiter.dispose();
|
|
1363
|
+
resolve();
|
|
1364
|
+
};
|
|
1365
|
+
signal.addEventListener('abort', abortListener, { once: true });
|
|
1366
|
+
})
|
|
1367
|
+
]);
|
|
1368
|
+
}
|
|
1369
|
+
finally {
|
|
1370
|
+
if (signal !== undefined && abortListener !== null) {
|
|
1371
|
+
signal.removeEventListener('abort', abortListener);
|
|
1372
|
+
}
|
|
1373
|
+
waiter.dispose();
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
async function listDescendantProcessIds(rootPid) {
|
|
1377
|
+
const snapshot = await runCommand('ps', ['-ax', '-o', 'pid=,ppid=']);
|
|
1378
|
+
if (snapshot.exitCode !== 0) {
|
|
1379
|
+
throw new Error(snapshot.stderr || `ps exited with code ${snapshot.exitCode}`);
|
|
1380
|
+
}
|
|
1381
|
+
const childrenByParent = new Map();
|
|
1382
|
+
for (const line of snapshot.stdout.split('\n')) {
|
|
1383
|
+
const trimmed = line.trim();
|
|
1384
|
+
if (trimmed.length === 0) {
|
|
1385
|
+
continue;
|
|
1386
|
+
}
|
|
1387
|
+
const [pidToken, parentPidToken] = trimmed.split(/\s+/u, 2);
|
|
1388
|
+
const pid = Number.parseInt(pidToken ?? '', 10);
|
|
1389
|
+
const parentPid = Number.parseInt(parentPidToken ?? '', 10);
|
|
1390
|
+
if (!Number.isInteger(pid) || !Number.isInteger(parentPid)) {
|
|
1391
|
+
continue;
|
|
1392
|
+
}
|
|
1393
|
+
const children = childrenByParent.get(parentPid) ?? [];
|
|
1394
|
+
children.push(pid);
|
|
1395
|
+
childrenByParent.set(parentPid, children);
|
|
1396
|
+
}
|
|
1397
|
+
const descendants = [];
|
|
1398
|
+
const visit = (parentPid) => {
|
|
1399
|
+
for (const childPid of childrenByParent.get(parentPid) ?? []) {
|
|
1400
|
+
visit(childPid);
|
|
1401
|
+
descendants.push(childPid);
|
|
1402
|
+
}
|
|
1403
|
+
};
|
|
1404
|
+
visit(rootPid);
|
|
1405
|
+
return descendants;
|
|
1406
|
+
}
|
|
1407
|
+
async function listProcessGroupProcessIds(rootPid) {
|
|
1408
|
+
const snapshot = await runCommand('ps', ['-ax', '-o', 'pid=,pgid=']);
|
|
1409
|
+
if (snapshot.exitCode !== 0) {
|
|
1410
|
+
throw new Error(snapshot.stderr || `ps exited with code ${snapshot.exitCode}`);
|
|
1411
|
+
}
|
|
1412
|
+
const processGroupPids = [];
|
|
1413
|
+
for (const line of snapshot.stdout.split('\n')) {
|
|
1414
|
+
const trimmed = line.trim();
|
|
1415
|
+
if (trimmed.length === 0) {
|
|
1416
|
+
continue;
|
|
1417
|
+
}
|
|
1418
|
+
const [pidToken, processGroupToken] = trimmed.split(/\s+/u, 2);
|
|
1419
|
+
const pid = Number.parseInt(pidToken ?? '', 10);
|
|
1420
|
+
const processGroupId = Number.parseInt(processGroupToken ?? '', 10);
|
|
1421
|
+
if (!Number.isInteger(pid) || !Number.isInteger(processGroupId)) {
|
|
1422
|
+
continue;
|
|
1423
|
+
}
|
|
1424
|
+
if (processGroupId === rootPid) {
|
|
1425
|
+
processGroupPids.push(pid);
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
return processGroupPids;
|
|
1429
|
+
}
|
|
1430
|
+
function killTrackedProcessGroup(pid, signal, killProcessGroup = (groupPid, nextSignal) => process.kill(-groupPid, nextSignal)) {
|
|
1431
|
+
try {
|
|
1432
|
+
killProcessGroup(pid, signal);
|
|
1433
|
+
}
|
|
1434
|
+
catch (error) {
|
|
1435
|
+
if (!isMissingProcessError(error)) {
|
|
1436
|
+
throw error;
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
async function waitForProcessGroupToExitWithinTimeout(rootPid, timeoutMs, options) {
|
|
1441
|
+
const exitController = new AbortController();
|
|
1442
|
+
const killWaiter = createSleepWaiter(timeoutMs);
|
|
1443
|
+
const completed = await Promise.race([
|
|
1444
|
+
waitForProcessGroupToExit(rootPid, options?.listProcessGroupPids, exitController.signal).then(() => true),
|
|
1445
|
+
killWaiter.promise.then(() => false)
|
|
1446
|
+
]).finally(() => {
|
|
1447
|
+
exitController.abort();
|
|
1448
|
+
killWaiter.dispose();
|
|
1449
|
+
});
|
|
1450
|
+
return completed;
|
|
1451
|
+
}
|
|
1452
|
+
async function ensureTrackedProcessTreeExited(rootPid, killTimeoutSeconds, options) {
|
|
1453
|
+
const timeoutMs = Math.max(0, killTimeoutSeconds * 1_000);
|
|
1454
|
+
const exitedAfterKickstart = await waitForProcessGroupToExitWithinTimeout(rootPid, timeoutMs, {
|
|
1455
|
+
listProcessGroupPids: options?.listProcessGroupPids
|
|
1456
|
+
});
|
|
1457
|
+
if (exitedAfterKickstart) {
|
|
1458
|
+
return {
|
|
1459
|
+
result: 'exited_after_kickstart',
|
|
1460
|
+
orphanedProcessGroupPids: [],
|
|
1461
|
+
orphanedDescendantPids: []
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
const shouldForceKill = await (options?.shouldForceKillTrackedProcessGroup ??
|
|
1465
|
+
(async () => true))(rootPid);
|
|
1466
|
+
if (!shouldForceKill) {
|
|
1467
|
+
const remainingProcessGroupPids = await (options?.listProcessGroupPids ?? listProcessGroupProcessIds)(rootPid);
|
|
1468
|
+
if (remainingProcessGroupPids.length > 0) {
|
|
1469
|
+
throw new Error(`Previous supervised control-host child pid ${rootPid} is still alive, but force cleanup was skipped because identity verification failed.`);
|
|
1470
|
+
}
|
|
1471
|
+
return {
|
|
1472
|
+
result: 'exited_after_kickstart',
|
|
1473
|
+
orphanedProcessGroupPids: [],
|
|
1474
|
+
orphanedDescendantPids: []
|
|
1475
|
+
};
|
|
1476
|
+
}
|
|
1477
|
+
const initialOrphanedProcessGroupPids = await (options?.listProcessGroupPids ?? listProcessGroupProcessIds)(rootPid);
|
|
1478
|
+
if (initialOrphanedProcessGroupPids.length === 0) {
|
|
1479
|
+
return {
|
|
1480
|
+
result: 'exited_after_kickstart',
|
|
1481
|
+
orphanedProcessGroupPids: [],
|
|
1482
|
+
orphanedDescendantPids: []
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
const shouldStillForceKill = await (options?.shouldForceKillTrackedProcessGroup ??
|
|
1486
|
+
(async () => true))(rootPid);
|
|
1487
|
+
if (!shouldStillForceKill) {
|
|
1488
|
+
const remainingProcessGroupPids = await (options?.listProcessGroupPids ?? listProcessGroupProcessIds)(rootPid);
|
|
1489
|
+
if (remainingProcessGroupPids.length > 0) {
|
|
1490
|
+
throw new Error(`Previous supervised control-host child pid ${rootPid} is still alive, but force cleanup was skipped because identity verification failed.`);
|
|
1491
|
+
}
|
|
1492
|
+
return {
|
|
1493
|
+
result: 'exited_after_kickstart',
|
|
1494
|
+
orphanedProcessGroupPids: [],
|
|
1495
|
+
orphanedDescendantPids: []
|
|
1496
|
+
};
|
|
1497
|
+
}
|
|
1498
|
+
const orphanedProcessGroupPids = await (options?.listProcessGroupPids ?? listProcessGroupProcessIds)(rootPid);
|
|
1499
|
+
if (orphanedProcessGroupPids.length === 0) {
|
|
1500
|
+
return {
|
|
1501
|
+
result: 'exited_after_kickstart',
|
|
1502
|
+
orphanedProcessGroupPids: [],
|
|
1503
|
+
orphanedDescendantPids: []
|
|
1504
|
+
};
|
|
1505
|
+
}
|
|
1506
|
+
const orphanedDescendantPids = await (options?.listDescendantPids ?? listDescendantProcessIds)(rootPid).catch(() => []);
|
|
1507
|
+
// Force cleanup is scoped to the stale supervised control-host process group.
|
|
1508
|
+
// Detached provider-worker issue runs can still appear as descendants and must
|
|
1509
|
+
// be preserved; we record them for diagnostics instead of killing them.
|
|
1510
|
+
killTrackedProcessGroup(rootPid, 'SIGKILL', options?.killProcessGroup);
|
|
1511
|
+
const exitedAfterForceKill = await waitForProcessGroupToExitWithinTimeout(rootPid, timeoutMs, {
|
|
1512
|
+
listProcessGroupPids: options?.listProcessGroupPids
|
|
1513
|
+
});
|
|
1514
|
+
if (!exitedAfterForceKill) {
|
|
1515
|
+
throw new Error(`Previous supervised control-host child pid ${rootPid} remained alive after forced cleanup.`);
|
|
1516
|
+
}
|
|
1517
|
+
return {
|
|
1518
|
+
result: 'force_killed',
|
|
1519
|
+
orphanedProcessGroupPids,
|
|
1520
|
+
orphanedDescendantPids
|
|
1521
|
+
};
|
|
1522
|
+
}
|
|
1523
|
+
async function readProcessCommand(pid) {
|
|
1524
|
+
const snapshot = await runCommand('ps', ['-p', String(pid), '-o', 'args=']);
|
|
1525
|
+
if (snapshot.exitCode !== 0) {
|
|
1526
|
+
if (snapshot.stdout.trim().length === 0) {
|
|
1527
|
+
return null;
|
|
1528
|
+
}
|
|
1529
|
+
throw new Error(snapshot.stderr || `ps exited with code ${snapshot.exitCode}`);
|
|
1530
|
+
}
|
|
1531
|
+
const command = snapshot.stdout.trim();
|
|
1532
|
+
return command.length > 0 ? command : null;
|
|
1533
|
+
}
|
|
1534
|
+
function parseShellStyleArguments(command) {
|
|
1535
|
+
const args = [];
|
|
1536
|
+
let current = '';
|
|
1537
|
+
let quote = null;
|
|
1538
|
+
let escaped = false;
|
|
1539
|
+
const flushCurrent = () => {
|
|
1540
|
+
if (current.length === 0) {
|
|
1541
|
+
return;
|
|
1542
|
+
}
|
|
1543
|
+
args.push(current);
|
|
1544
|
+
current = '';
|
|
1545
|
+
};
|
|
1546
|
+
for (let index = 0; index < command.length; index += 1) {
|
|
1547
|
+
const character = command[index];
|
|
1548
|
+
if (escaped) {
|
|
1549
|
+
current += character;
|
|
1550
|
+
escaped = false;
|
|
1551
|
+
continue;
|
|
1552
|
+
}
|
|
1553
|
+
if (quote === "'") {
|
|
1554
|
+
if (character === "'") {
|
|
1555
|
+
quote = null;
|
|
1556
|
+
}
|
|
1557
|
+
else {
|
|
1558
|
+
current += character;
|
|
1559
|
+
}
|
|
1560
|
+
continue;
|
|
1561
|
+
}
|
|
1562
|
+
if (quote === '"') {
|
|
1563
|
+
if (character === '"') {
|
|
1564
|
+
quote = null;
|
|
1565
|
+
continue;
|
|
1566
|
+
}
|
|
1567
|
+
if (character === '\\') {
|
|
1568
|
+
const nextCharacter = command[index + 1];
|
|
1569
|
+
if (nextCharacter && ['\\', '"', '$', '`'].includes(nextCharacter)) {
|
|
1570
|
+
current += nextCharacter;
|
|
1571
|
+
index += 1;
|
|
1572
|
+
continue;
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
current += character;
|
|
1576
|
+
continue;
|
|
1577
|
+
}
|
|
1578
|
+
if (/\s/.test(character)) {
|
|
1579
|
+
flushCurrent();
|
|
1580
|
+
continue;
|
|
1581
|
+
}
|
|
1582
|
+
if (character === "'" || character === '"') {
|
|
1583
|
+
quote = character;
|
|
1584
|
+
continue;
|
|
1585
|
+
}
|
|
1586
|
+
if (character === '\\') {
|
|
1587
|
+
escaped = true;
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
current += character;
|
|
1591
|
+
}
|
|
1592
|
+
if (escaped) {
|
|
1593
|
+
current += '\\';
|
|
1594
|
+
}
|
|
1595
|
+
flushCurrent();
|
|
1596
|
+
return args;
|
|
1597
|
+
}
|
|
1598
|
+
function readFlagValueFromArgs(args, flag) {
|
|
1599
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
1600
|
+
const argument = args[index];
|
|
1601
|
+
if (!argument) {
|
|
1602
|
+
continue;
|
|
1603
|
+
}
|
|
1604
|
+
if (argument === flag) {
|
|
1605
|
+
return args[index + 1] ?? null;
|
|
1606
|
+
}
|
|
1607
|
+
if (argument.startsWith(`${flag}=`)) {
|
|
1608
|
+
return argument.slice(flag.length + 1);
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
return null;
|
|
1612
|
+
}
|
|
1613
|
+
function matchesExpectedSupervisedControlHostCommand(command, config) {
|
|
1614
|
+
const args = parseShellStyleArguments(command);
|
|
1615
|
+
return (command.includes(config.cliEntrypoint) &&
|
|
1616
|
+
args.includes('control-host') &&
|
|
1617
|
+
readFlagValueFromArgs(args, '--task') === config.taskId &&
|
|
1618
|
+
readFlagValueFromArgs(args, '--run') === config.runId &&
|
|
1619
|
+
readFlagValueFromArgs(args, '--pipeline') === config.pipelineId);
|
|
1620
|
+
}
|
|
1621
|
+
async function isTrackedSupervisedProcessRoot(pid, config, options) {
|
|
1622
|
+
const command = await (options?.readProcessCommand ?? readProcessCommand)(pid);
|
|
1623
|
+
return command !== null && matchesExpectedSupervisedControlHostCommand(command, config);
|
|
1624
|
+
}
|
|
1625
|
+
async function isTrackedSupervisedProcessGroup(rootPid, config, options) {
|
|
1626
|
+
const readTrackedProcessCommand = options?.readProcessCommand ?? readProcessCommand;
|
|
1627
|
+
if (await isTrackedSupervisedProcessRoot(rootPid, config, {
|
|
1628
|
+
readProcessCommand: readTrackedProcessCommand
|
|
1629
|
+
})) {
|
|
1630
|
+
return true;
|
|
1631
|
+
}
|
|
1632
|
+
const processGroupPids = await (options?.listProcessGroupPids ?? listProcessGroupProcessIds)(rootPid).catch(() => []);
|
|
1633
|
+
for (const pid of processGroupPids) {
|
|
1634
|
+
if (pid === rootPid) {
|
|
1635
|
+
continue;
|
|
1636
|
+
}
|
|
1637
|
+
const command = await readTrackedProcessCommand(pid);
|
|
1638
|
+
if (command !== null && matchesExpectedSupervisedControlHostCommand(command, config)) {
|
|
1639
|
+
return true;
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
return false;
|
|
1643
|
+
}
|
|
1644
|
+
function parseIsoTimestampToMs(value) {
|
|
1645
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
1646
|
+
return null;
|
|
1647
|
+
}
|
|
1648
|
+
const parsed = Date.parse(value);
|
|
1649
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
1650
|
+
}
|
|
1651
|
+
async function resolveReportedSupervisedChildPid(nextState, previousState, config, options) {
|
|
1652
|
+
const nextChildPid = normalizeTrackedPid(nextState?.child_pid ?? undefined);
|
|
1653
|
+
if (nextChildPid !== null) {
|
|
1654
|
+
const nextUpdatedAtMs = parseIsoTimestampToMs(nextState?.updated_at);
|
|
1655
|
+
const previousUpdatedAtMs = parseIsoTimestampToMs(previousState?.updated_at);
|
|
1656
|
+
if (nextUpdatedAtMs === null ||
|
|
1657
|
+
(previousUpdatedAtMs !== null && nextUpdatedAtMs <= previousUpdatedAtMs)) {
|
|
1658
|
+
return null;
|
|
1659
|
+
}
|
|
1660
|
+
return (await isTrackedSupervisedProcessRoot(nextChildPid, config, options))
|
|
1661
|
+
? nextChildPid
|
|
1662
|
+
: null;
|
|
1663
|
+
}
|
|
1664
|
+
const fallbackChildPid = normalizeTrackedPid(options?.fallbackChildPid ?? undefined);
|
|
1665
|
+
if (fallbackChildPid === null ||
|
|
1666
|
+
fallbackChildPid === normalizeTrackedPid(options?.previousTrackedChildPid ?? undefined)) {
|
|
1667
|
+
return null;
|
|
1668
|
+
}
|
|
1669
|
+
return (await isTrackedSupervisedProcessRoot(fallbackChildPid, config, options))
|
|
1670
|
+
? fallbackChildPid
|
|
1671
|
+
: null;
|
|
1672
|
+
}
|
|
1673
|
+
function extractLaunchctlServicePid(output) {
|
|
1674
|
+
const pidMatch = /^\s*pid = (\d+)\s*$/mu.exec(output);
|
|
1675
|
+
return normalizeTrackedPid(pidMatch ? Number.parseInt(pidMatch[1] ?? '', 10) : undefined);
|
|
1676
|
+
}
|
|
1677
|
+
async function readTrackedChildSnapshotForRestart(statePath, serviceTarget, options) {
|
|
1678
|
+
try {
|
|
1679
|
+
const state = await (options?.readState ??
|
|
1680
|
+
(async (path) => await readJsonFileIfExists(path)))(statePath);
|
|
1681
|
+
return {
|
|
1682
|
+
state,
|
|
1683
|
+
trackedChildPid: normalizeTrackedPid(state?.child_pid ?? undefined)
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
catch {
|
|
1687
|
+
const launchctl = await (options?.readLaunchctlPrint ??
|
|
1688
|
+
(async (nextServiceTarget) => await runLaunchctl(['print', nextServiceTarget], { allowFailure: true })))(serviceTarget);
|
|
1689
|
+
return {
|
|
1690
|
+
state: null,
|
|
1691
|
+
trackedChildPid: launchctl.exitCode === 0 ? extractLaunchctlServicePid(launchctl.stdout) : null
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
async function restartExistingControlHostSupervision(resolved, serviceTarget, options) {
|
|
1696
|
+
const previousSnapshot = await readTrackedChildSnapshotForRestart(resolved.paths.statePath, serviceTarget, {
|
|
1697
|
+
readState: options?.readState,
|
|
1698
|
+
readLaunchctlPrint: options?.readLaunchctlPrint
|
|
1699
|
+
});
|
|
1700
|
+
const previousChildPid = previousSnapshot.trackedChildPid;
|
|
1701
|
+
await (options?.kickstart ??
|
|
1702
|
+
(async (nextServiceTarget) => {
|
|
1703
|
+
await runLaunchctl(['kickstart', '-k', nextServiceTarget]);
|
|
1704
|
+
}))(serviceTarget);
|
|
1705
|
+
const cleanup = previousChildPid === null
|
|
1706
|
+
? {
|
|
1707
|
+
result: 'no_prior_child',
|
|
1708
|
+
orphanedProcessGroupPids: [],
|
|
1709
|
+
orphanedDescendantPids: []
|
|
1710
|
+
}
|
|
1711
|
+
: await (options?.ensureTrackedProcessTreeExited ?? ensureTrackedProcessTreeExited)(previousChildPid, resolved.config.killTimeoutSeconds, {
|
|
1712
|
+
shouldForceKillTrackedProcessGroup: options?.shouldForceKillTrackedProcessGroup ??
|
|
1713
|
+
(async (rootPid) => await isTrackedSupervisedProcessGroup(rootPid, resolved.config, {
|
|
1714
|
+
readProcessCommand: options?.readProcessCommand
|
|
1715
|
+
}))
|
|
1716
|
+
});
|
|
1717
|
+
const nextSnapshot = await readTrackedChildSnapshotForRestart(resolved.paths.statePath, serviceTarget, {
|
|
1718
|
+
readState: options?.readState,
|
|
1719
|
+
readLaunchctlPrint: options?.readLaunchctlPrint
|
|
1720
|
+
});
|
|
1721
|
+
return {
|
|
1722
|
+
previousChildPid,
|
|
1723
|
+
childPid: await resolveReportedSupervisedChildPid(nextSnapshot.state, previousSnapshot.state, resolved.config, {
|
|
1724
|
+
readProcessCommand: options?.readProcessCommand,
|
|
1725
|
+
fallbackChildPid: nextSnapshot.trackedChildPid,
|
|
1726
|
+
previousTrackedChildPid: previousSnapshot.trackedChildPid
|
|
1727
|
+
}),
|
|
1728
|
+
cleanup
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
function isMissingProcessError(error) {
|
|
1732
|
+
return (error instanceof Error &&
|
|
1733
|
+
'code' in error &&
|
|
1734
|
+
error.code === 'ESRCH');
|
|
1735
|
+
}
|
|
1736
|
+
async function writeRuntimeStateWithCleanup(child, killTimeoutSeconds, persist) {
|
|
1737
|
+
try {
|
|
1738
|
+
return await persist();
|
|
1739
|
+
}
|
|
1740
|
+
catch (error) {
|
|
1741
|
+
await terminateChildProcess(child, killTimeoutSeconds).catch((cleanupError) => {
|
|
1742
|
+
console.error(`Failed to stop control-host after supervision state write failure: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`);
|
|
1743
|
+
});
|
|
1744
|
+
throw error;
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
function createStopSignalWaiter() {
|
|
1748
|
+
let dispose = null;
|
|
1749
|
+
const promise = new Promise((resolve) => {
|
|
1750
|
+
const handle = (signal) => {
|
|
1751
|
+
dispose?.();
|
|
1752
|
+
resolve({ type: 'stop', signal });
|
|
1753
|
+
};
|
|
1754
|
+
const onSigint = () => handle('SIGINT');
|
|
1755
|
+
const onSigterm = () => handle('SIGTERM');
|
|
1756
|
+
process.on('SIGINT', onSigint);
|
|
1757
|
+
process.on('SIGTERM', onSigterm);
|
|
1758
|
+
dispose = () => {
|
|
1759
|
+
process.off('SIGINT', onSigint);
|
|
1760
|
+
process.off('SIGTERM', onSigterm);
|
|
1761
|
+
};
|
|
1762
|
+
});
|
|
1763
|
+
return {
|
|
1764
|
+
promise,
|
|
1765
|
+
dispose: () => dispose?.()
|
|
1766
|
+
};
|
|
1767
|
+
}
|
|
1768
|
+
async function runCommand(command, args, options) {
|
|
1769
|
+
try {
|
|
1770
|
+
const { stdout, stderr } = await execFileAsync(command, args, {
|
|
1771
|
+
cwd: options?.cwd,
|
|
1772
|
+
env: options?.env,
|
|
1773
|
+
...(typeof options?.timeoutMs === 'number' ? { timeout: options.timeoutMs } : {}),
|
|
1774
|
+
maxBuffer: COMMAND_BUFFER_MAX_BYTES
|
|
1775
|
+
});
|
|
1776
|
+
return {
|
|
1777
|
+
exitCode: 0,
|
|
1778
|
+
stdout: String(stdout),
|
|
1779
|
+
stderr: String(stderr)
|
|
1780
|
+
};
|
|
1781
|
+
}
|
|
1782
|
+
catch (error) {
|
|
1783
|
+
const execError = error;
|
|
1784
|
+
const exitCode = typeof execError.code === 'number' ? execError.code : execError.code === 'ENOENT' ? 127 : 1;
|
|
1785
|
+
const timedOut = typeof options?.timeoutMs === 'number' &&
|
|
1786
|
+
execError.killed === true &&
|
|
1787
|
+
execError.signal === 'SIGTERM';
|
|
1788
|
+
return {
|
|
1789
|
+
exitCode,
|
|
1790
|
+
stdout: bufferLikeToString(execError.stdout),
|
|
1791
|
+
stderr: bufferLikeToString(execError.stderr) ||
|
|
1792
|
+
(timedOut ? `command timed out after ${options.timeoutMs}ms` : execError.message),
|
|
1793
|
+
timedOut
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
async function runCommandBuffer(command, args, options) {
|
|
1798
|
+
try {
|
|
1799
|
+
const { stdout, stderr } = await execFileAsync(command, args, {
|
|
1800
|
+
cwd: options?.cwd,
|
|
1801
|
+
env: options?.env,
|
|
1802
|
+
encoding: 'buffer',
|
|
1803
|
+
...(typeof options?.timeoutMs === 'number' ? { timeout: options.timeoutMs } : {}),
|
|
1804
|
+
maxBuffer: COMMAND_BUFFER_MAX_BYTES
|
|
1805
|
+
});
|
|
1806
|
+
return {
|
|
1807
|
+
exitCode: 0,
|
|
1808
|
+
stdout: bufferLikeToBuffer(stdout),
|
|
1809
|
+
stderr: bufferLikeToBuffer(stderr)
|
|
1810
|
+
};
|
|
1811
|
+
}
|
|
1812
|
+
catch (error) {
|
|
1813
|
+
const execError = error;
|
|
1814
|
+
const exitCode = typeof execError.code === 'number' ? execError.code : execError.code === 'ENOENT' ? 127 : 1;
|
|
1815
|
+
const timedOut = typeof options?.timeoutMs === 'number' &&
|
|
1816
|
+
execError.killed === true &&
|
|
1817
|
+
execError.signal === 'SIGTERM';
|
|
1818
|
+
return {
|
|
1819
|
+
exitCode,
|
|
1820
|
+
stdout: bufferLikeToBuffer(execError.stdout),
|
|
1821
|
+
stderr: bufferLikeToBuffer(execError.stderr ?? (timedOut ? `command timed out after ${options.timeoutMs}ms` : execError.message)),
|
|
1822
|
+
timedOut
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
function parseNulDelimitedEnv(raw) {
|
|
1827
|
+
const nextEnv = {};
|
|
1828
|
+
for (const entry of raw.toString('utf8').split('\u0000')) {
|
|
1829
|
+
if (!entry) {
|
|
1830
|
+
continue;
|
|
1831
|
+
}
|
|
1832
|
+
const separatorIndex = entry.indexOf('=');
|
|
1833
|
+
if (separatorIndex <= 0) {
|
|
1834
|
+
continue;
|
|
1835
|
+
}
|
|
1836
|
+
const key = entry.slice(0, separatorIndex);
|
|
1837
|
+
const value = entry.slice(separatorIndex + 1);
|
|
1838
|
+
nextEnv[key] = value;
|
|
1839
|
+
}
|
|
1840
|
+
return nextEnv;
|
|
1841
|
+
}
|
|
1842
|
+
function firstNonEmptyLine(value) {
|
|
1843
|
+
for (const line of value.split('\n')) {
|
|
1844
|
+
const trimmed = line.trim();
|
|
1845
|
+
if (trimmed.length > 0) {
|
|
1846
|
+
return trimmed;
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
return null;
|
|
1850
|
+
}
|
|
1851
|
+
function extractPlistStringArray(plistContents, key) {
|
|
1852
|
+
const block = new RegExp(`<key>${escapeRegExp(key)}</key>\\s*<array>([\\s\\S]*?)</array>`, 'u').exec(plistContents)?.[1];
|
|
1853
|
+
if (!block) {
|
|
1854
|
+
return [];
|
|
1855
|
+
}
|
|
1856
|
+
return [...block.matchAll(/<string>([\s\S]*?)<\/string>/gu)].map((match) => decodePlistString(match[1] ?? ''));
|
|
1857
|
+
}
|
|
1858
|
+
function extractPlistStringValue(plistContents, key) {
|
|
1859
|
+
const value = new RegExp(`<key>${escapeRegExp(key)}</key>\\s*<string>([\\s\\S]*?)</string>`, 'u').exec(plistContents)?.[1];
|
|
1860
|
+
return typeof value === 'string' ? decodePlistString(value) : null;
|
|
1861
|
+
}
|
|
1862
|
+
function decodePlistString(value) {
|
|
1863
|
+
return value
|
|
1864
|
+
.replaceAll('<', '<')
|
|
1865
|
+
.replaceAll('>', '>')
|
|
1866
|
+
.replaceAll('"', '"')
|
|
1867
|
+
.replaceAll(''', "'")
|
|
1868
|
+
.replaceAll('&', '&');
|
|
1869
|
+
}
|
|
1870
|
+
function escapeRegExp(value) {
|
|
1871
|
+
return value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
|
|
1872
|
+
}
|
|
1873
|
+
function arraysEqual(left, right) {
|
|
1874
|
+
if (left.length !== right.length) {
|
|
1875
|
+
return false;
|
|
1876
|
+
}
|
|
1877
|
+
for (const [index, value] of left.entries()) {
|
|
1878
|
+
if (value !== right[index]) {
|
|
1879
|
+
return false;
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
return true;
|
|
1883
|
+
}
|
|
1884
|
+
function isNonEmptyString(value) {
|
|
1885
|
+
return typeof value === 'string' && value.trim().length > 0;
|
|
1886
|
+
}
|
|
1887
|
+
async function assertPathExists(path, label, exists = pathExists) {
|
|
1888
|
+
if (!(await exists(path))) {
|
|
1889
|
+
throw new Error(`${label} not found: ${path}`);
|
|
1890
|
+
}
|
|
1891
|
+
}
|
|
1892
|
+
async function assertExecutablePath(path, label, exists = pathExists, isExecutable = pathIsExecutable, isFile = pathIsFile) {
|
|
1893
|
+
await assertFilePath(path, label, exists, isFile);
|
|
1894
|
+
if (!(await isExecutable(path))) {
|
|
1895
|
+
throw new Error(`${label} is not executable: ${path}`);
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
async function assertFilePath(path, label, exists = pathExists, isFile = pathIsFile) {
|
|
1899
|
+
await assertPathExists(path, label, exists);
|
|
1900
|
+
if (!(await isFile(path))) {
|
|
1901
|
+
throw new Error(`${label} is not a regular file: ${path}`);
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
async function assertDirectoryPath(path, label, exists = pathExists, isDirectory = pathIsDirectory) {
|
|
1905
|
+
await assertPathExists(path, label, exists);
|
|
1906
|
+
if (!(await isDirectory(path))) {
|
|
1907
|
+
throw new Error(`${label} is not a directory: ${path}`);
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
async function assertControlHostSupervisionInstallPaths(config, exists = pathExists, isExecutable = pathIsExecutable, isDirectory = pathIsDirectory, isFile = pathIsFile) {
|
|
1911
|
+
await assertDirectoryPath(config.repoRoot, 'Control-host supervision repo root', exists, isDirectory);
|
|
1912
|
+
await assertExecutablePath(config.nodePath, 'Node executable', exists, isExecutable, isFile);
|
|
1913
|
+
await assertFilePath(config.cliEntrypoint, 'Control-host supervision entrypoint', exists, isFile);
|
|
1914
|
+
await assertExecutablePath(config.shellPath, 'Shell executable', exists, isExecutable, isFile);
|
|
1915
|
+
}
|
|
1916
|
+
function assertStoredControlHostSupervisionConfig(configPath, config) {
|
|
1917
|
+
if (typeof config !== 'object' || config === null) {
|
|
1918
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: expected an object.`);
|
|
1919
|
+
}
|
|
1920
|
+
const record = config;
|
|
1921
|
+
for (const key of REQUIRED_CONTROL_HOST_SUPERVISION_STRING_FIELDS) {
|
|
1922
|
+
if (!isNonEmptyString(record[key])) {
|
|
1923
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: missing ${key}.`);
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
for (const key of REQUIRED_CONTROL_HOST_SUPERVISION_INTEGER_FIELDS) {
|
|
1927
|
+
if (!Number.isInteger(record[key])) {
|
|
1928
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: invalid ${key}.`);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
assertStoredTimerField(configPath, record.healthIntervalSeconds, 'healthIntervalSeconds');
|
|
1932
|
+
assertStoredTimerField(configPath, record.killTimeoutSeconds, 'killTimeoutSeconds');
|
|
1933
|
+
assertStoredPositiveIntegerField(configPath, record.unhealthyThreshold, 'unhealthyThreshold');
|
|
1934
|
+
if (!Array.isArray(record.envFiles) || record.envFiles.some((entry) => !isNonEmptyString(entry))) {
|
|
1935
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: invalid envFiles.`);
|
|
1936
|
+
}
|
|
1937
|
+
if (typeof record.paths !== 'object' || record.paths === null) {
|
|
1938
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: missing paths.`);
|
|
1939
|
+
}
|
|
1940
|
+
const paths = record.paths;
|
|
1941
|
+
for (const key of REQUIRED_CONTROL_HOST_SUPERVISION_PATH_FIELDS) {
|
|
1942
|
+
if (!isNonEmptyString(paths[key])) {
|
|
1943
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: missing paths.${key}.`);
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
const expectedPaths = resolveControlHostSupervisionPaths({
|
|
1947
|
+
homeDir: record.homeDir,
|
|
1948
|
+
label: record.label
|
|
1949
|
+
});
|
|
1950
|
+
const resolvedConfigPath = resolve(configPath);
|
|
1951
|
+
if (resolvedConfigPath !== expectedPaths.configPath) {
|
|
1952
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: config path must match the managed path ${expectedPaths.configPath}.`);
|
|
1953
|
+
}
|
|
1954
|
+
for (const key of REQUIRED_CONTROL_HOST_SUPERVISION_PATH_FIELDS) {
|
|
1955
|
+
if (paths[key] !== expectedPaths[key]) {
|
|
1956
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: paths.${key} must match the managed path ${expectedPaths[key]}.`);
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
}
|
|
1960
|
+
function assertStoredTimerField(configPath, value, key) {
|
|
1961
|
+
const timerSeconds = typeof value === 'number' ? value : Number.NaN;
|
|
1962
|
+
if (!Number.isInteger(timerSeconds) || timerSeconds <= 0) {
|
|
1963
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: invalid ${key}.`);
|
|
1964
|
+
}
|
|
1965
|
+
if (timerSeconds > CONTROL_HOST_SUPERVISION_MAX_NODE_TIMER_SECONDS) {
|
|
1966
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: ${key} must be <= ${CONTROL_HOST_SUPERVISION_MAX_NODE_TIMER_SECONDS}.`);
|
|
1967
|
+
}
|
|
1968
|
+
}
|
|
1969
|
+
function assertStoredPositiveIntegerField(configPath, value, key) {
|
|
1970
|
+
const integerValue = typeof value === 'number' ? value : Number.NaN;
|
|
1971
|
+
if (!Number.isInteger(integerValue) || integerValue <= 0) {
|
|
1972
|
+
throw new Error(`Invalid control-host supervision config at ${configPath}: invalid ${key}.`);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
async function pathExists(path) {
|
|
1976
|
+
try {
|
|
1977
|
+
await stat(path);
|
|
1978
|
+
return true;
|
|
1979
|
+
}
|
|
1980
|
+
catch {
|
|
1981
|
+
return false;
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
async function pathIsExecutable(path) {
|
|
1985
|
+
try {
|
|
1986
|
+
await access(path, fsConstants.X_OK);
|
|
1987
|
+
return true;
|
|
1988
|
+
}
|
|
1989
|
+
catch {
|
|
1990
|
+
return false;
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
async function pathIsDirectory(path) {
|
|
1994
|
+
try {
|
|
1995
|
+
return (await stat(path)).isDirectory();
|
|
1996
|
+
}
|
|
1997
|
+
catch {
|
|
1998
|
+
return false;
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
async function pathIsFile(path) {
|
|
2002
|
+
try {
|
|
2003
|
+
return (await stat(path)).isFile();
|
|
2004
|
+
}
|
|
2005
|
+
catch {
|
|
2006
|
+
return false;
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
async function readJsonFileIfExists(path) {
|
|
2010
|
+
try {
|
|
2011
|
+
const raw = await readFile(path, 'utf8');
|
|
2012
|
+
return JSON.parse(raw);
|
|
2013
|
+
}
|
|
2014
|
+
catch (error) {
|
|
2015
|
+
if (error.code === 'ENOENT') {
|
|
2016
|
+
return null;
|
|
2017
|
+
}
|
|
2018
|
+
throw error;
|
|
2019
|
+
}
|
|
2020
|
+
}
|
|
2021
|
+
async function readTextFileIfExists(path) {
|
|
2022
|
+
try {
|
|
2023
|
+
return await readFile(path, 'utf8');
|
|
2024
|
+
}
|
|
2025
|
+
catch (error) {
|
|
2026
|
+
if (error.code === 'ENOENT') {
|
|
2027
|
+
return null;
|
|
2028
|
+
}
|
|
2029
|
+
throw error;
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
async function restoreTextFile(path, contents, options) {
|
|
2033
|
+
const write = options?.write ?? writeFile;
|
|
2034
|
+
const remove = options?.remove ?? rm;
|
|
2035
|
+
if (contents === null) {
|
|
2036
|
+
await remove(path, { force: true });
|
|
2037
|
+
return;
|
|
2038
|
+
}
|
|
2039
|
+
await write(path, contents, 'utf8');
|
|
2040
|
+
}
|
|
2041
|
+
async function writeJsonFile(path, value) {
|
|
2042
|
+
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
|
|
2043
|
+
}
|
|
2044
|
+
function bufferLikeToString(value) {
|
|
2045
|
+
if (typeof value === 'string') {
|
|
2046
|
+
return value;
|
|
2047
|
+
}
|
|
2048
|
+
if (Buffer.isBuffer(value)) {
|
|
2049
|
+
return value.toString('utf8');
|
|
2050
|
+
}
|
|
2051
|
+
return '';
|
|
2052
|
+
}
|
|
2053
|
+
function bufferLikeToBuffer(value) {
|
|
2054
|
+
if (Buffer.isBuffer(value)) {
|
|
2055
|
+
return value;
|
|
2056
|
+
}
|
|
2057
|
+
if (typeof value === 'string') {
|
|
2058
|
+
return Buffer.from(value, 'utf8');
|
|
2059
|
+
}
|
|
2060
|
+
return Buffer.alloc(0);
|
|
2061
|
+
}
|
|
2062
|
+
function escapeShellSingleQuotes(value) {
|
|
2063
|
+
return value.replaceAll("'", "'\\''");
|
|
2064
|
+
}
|
|
2065
|
+
function resolveControlHostSupervisionProbeTimeoutMs(healthIntervalSeconds) {
|
|
2066
|
+
const minimumStatusReadBudgetMs = DEFAULT_ATTACH_REQUEST_TIMEOUT_MS * CONTROL_HOST_SUPERVISION_PROBE_ENDPOINT_READ_ATTEMPTS +
|
|
2067
|
+
CONTROL_HOST_SUPERVISION_PROBE_TIMEOUT_HEADROOM_MS;
|
|
2068
|
+
return Math.max(CONTROL_HOST_SUPERVISION_PROBE_TIMEOUT_FLOOR_MS, Math.min(Math.max(healthIntervalSeconds * 1_000, minimumStatusReadBudgetMs), CONTROL_HOST_SUPERVISION_PROBE_TIMEOUT_CAP_MS));
|
|
2069
|
+
}
|
|
2070
|
+
function createSleepWaiter(ms) {
|
|
2071
|
+
let timer = null;
|
|
2072
|
+
const promise = new Promise((resolve) => {
|
|
2073
|
+
timer = setTimeout(() => {
|
|
2074
|
+
timer = null;
|
|
2075
|
+
resolve({ type: 'tick' });
|
|
2076
|
+
}, ms);
|
|
2077
|
+
});
|
|
2078
|
+
return {
|
|
2079
|
+
promise,
|
|
2080
|
+
dispose: () => {
|
|
2081
|
+
if (timer !== null) {
|
|
2082
|
+
clearTimeout(timer);
|
|
2083
|
+
timer = null;
|
|
2084
|
+
}
|
|
2085
|
+
}
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2088
|
+
export const __test__ = {
|
|
2089
|
+
assertControlHostSupervisionInstallPaths,
|
|
2090
|
+
assertStoredControlHostSupervisionConfig,
|
|
2091
|
+
bootstrapLaunchctlPlist,
|
|
2092
|
+
buildNextControlHostSupervisionState,
|
|
2093
|
+
buildControlHostSupervisionRestartRecord,
|
|
2094
|
+
buildControlHostSupervisionStatusPayload,
|
|
2095
|
+
classifyControlHostSupervisionRollout,
|
|
2096
|
+
captureExistingControlHostSupervisionInstall,
|
|
2097
|
+
createSleepWaiter,
|
|
2098
|
+
createControlHostSupervisionChildEventPromises,
|
|
2099
|
+
hasHealthyLiveProviderControlHostFreshness,
|
|
2100
|
+
formatControlHostSupervisionStatus,
|
|
2101
|
+
inspectControlHostSupervisionLiveHealth,
|
|
2102
|
+
inspectControlHostSupervisionLaunchAgent,
|
|
2103
|
+
isIgnorableLaunchctlBootoutFailure,
|
|
2104
|
+
isRetryableLaunchctlBootstrapError,
|
|
2105
|
+
loadBootstrapEnvironment,
|
|
2106
|
+
parseNulDelimitedEnv,
|
|
2107
|
+
probeControlHostHealth,
|
|
2108
|
+
readFormatFlag,
|
|
2109
|
+
readStringFlag,
|
|
2110
|
+
resolveEffectiveControlHostSupervisionState,
|
|
2111
|
+
resolveControlHostSupervisionProviderIntakeStatePath,
|
|
2112
|
+
resolveControlHostSupervisionQuarantineUnhealthySamples,
|
|
2113
|
+
readIntegerFlag,
|
|
2114
|
+
removeInstalledControlHostSupervisionArtifacts,
|
|
2115
|
+
restoreExistingControlHostSupervisionInstall,
|
|
2116
|
+
resolveReportedSupervisedChildPid,
|
|
2117
|
+
rollbackFailedControlHostSupervisionInstall,
|
|
2118
|
+
restartExistingControlHostSupervision,
|
|
2119
|
+
isTrackedSupervisedProcessGroup,
|
|
2120
|
+
resolveControlHostSupervisionProbeTimeoutMs,
|
|
2121
|
+
resolveControlHostSupervisionServiceTarget,
|
|
2122
|
+
extractLaunchctlServicePid,
|
|
2123
|
+
ensureTrackedProcessTreeExited,
|
|
2124
|
+
terminateChildProcess,
|
|
2125
|
+
waitForProcessGroupToExitWithinTimeout,
|
|
2126
|
+
writeRuntimeStateWithCleanup
|
|
2127
|
+
};
|