dsh-compound-engineering 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +160 -2
- package/README.zh.md +161 -0
- package/cordis.patch.yml +6 -0
- package/lib/frontmatter.js +167 -0
- package/lib/index.js +46 -0
- package/lib/provider.js +201 -0
- package/package.json +62 -4
- package/skills/CODING_STANDARDS.md +68 -0
- package/skills/ce-babysit-pr/SKILL.md +59 -0
- package/skills/ce-babysit-pr/references/branch-currency.md +22 -0
- package/skills/ce-babysit-pr/references/envelope.md +44 -0
- package/skills/ce-babysit-pr/references/pipeline.md +15 -0
- package/skills/ce-babysit-pr/references/report.md +20 -0
- package/skills/ce-babysit-pr/references/settle.md +37 -0
- package/skills/ce-babysit-pr/references/setup.md +33 -0
- package/skills/ce-babysit-pr/references/stack-commands.md +39 -0
- package/skills/ce-babysit-pr/references/stack.md +43 -0
- package/skills/ce-babysit-pr/references/tick.md +102 -0
- package/skills/ce-babysit-pr/references/watch-loop.md +222 -0
- package/skills/ce-babysit-pr/scripts/pr-snapshot +3656 -0
- package/skills/ce-bakeoff/SKILL.md +48 -0
- package/skills/ce-bakeoff/references/candidates.md +37 -0
- package/skills/ce-bakeoff/references/judging.md +19 -0
- package/skills/ce-bakeoff/references/output.md +11 -0
- package/skills/ce-bakeoff/references/verification.md +15 -0
- package/skills/ce-brainstorm/SKILL.md +69 -0
- package/skills/ce-brainstorm/references/agents/slack-researcher.md +127 -0
- package/skills/ce-brainstorm/references/approaches.md +57 -0
- package/skills/ce-brainstorm/references/bakeoff.md +9 -0
- package/skills/ce-brainstorm/references/blindspot-pass.md +70 -0
- package/skills/ce-brainstorm/references/brainstorm-sections.md +458 -0
- package/skills/ce-brainstorm/references/dialogue.md +89 -0
- package/skills/ce-brainstorm/references/handoff.md +187 -0
- package/skills/ce-brainstorm/references/html-rendering.md +638 -0
- package/skills/ce-brainstorm/references/interaction-rules.md +33 -0
- package/skills/ce-brainstorm/references/markdown-rendering.md +242 -0
- package/skills/ce-brainstorm/references/model-tiers.md +13 -0
- package/skills/ce-brainstorm/references/output-mode.md +26 -0
- package/skills/ce-brainstorm/references/phase-0.md +104 -0
- package/skills/ce-brainstorm/references/plan-write.md +32 -0
- package/skills/ce-brainstorm/references/product-pressure-test.md +42 -0
- package/skills/ce-brainstorm/references/reasoning-elevation.md +124 -0
- package/skills/ce-brainstorm/references/settled-decisions.md +45 -0
- package/skills/ce-brainstorm/references/synthesis-summary.md +149 -0
- package/skills/ce-brainstorm/references/universal-brainstorming.md +72 -0
- package/skills/ce-brainstorm/references/verdict-routing.md +26 -0
- package/skills/ce-brainstorm/references/visual-probes.md +164 -0
- package/skills/ce-brainstorm/scripts/elevation-dispatch.sh +288 -0
- package/skills/ce-brainstorm/scripts/light-webserver.js +1275 -0
- package/skills/ce-brainstorm/scripts/packs-resolve.py +737 -0
- package/skills/ce-brainstorm/scripts/peer-job-runner.py +2594 -0
- package/skills/ce-code-review/SKILL.md +40 -0
- package/skills/ce-code-review/references/action-class-rubric.md +67 -0
- package/skills/ce-code-review/references/cross-model-recovery.md +38 -0
- package/skills/ce-code-review/references/cross-model-review.md +191 -0
- package/skills/ce-code-review/references/depth-paths.md +32 -0
- package/skills/ce-code-review/references/diff-scope.md +59 -0
- package/skills/ce-code-review/references/dispatch-reviewers.md +135 -0
- package/skills/ce-code-review/references/findings-schema.json +125 -0
- package/skills/ce-code-review/references/finish-input.md +118 -0
- package/skills/ce-code-review/references/finish-review.md +202 -0
- package/skills/ce-code-review/references/intent-and-plan.md +73 -0
- package/skills/ce-code-review/references/modes-and-output.md +156 -0
- package/skills/ce-code-review/references/persona-catalog.md +66 -0
- package/skills/ce-code-review/references/personas/adversarial-reviewer.md +111 -0
- package/skills/ce-code-review/references/personas/agent-native-reviewer.md +173 -0
- package/skills/ce-code-review/references/personas/api-contract-reviewer.md +44 -0
- package/skills/ce-code-review/references/personas/correctness-reviewer.md +48 -0
- package/skills/ce-code-review/references/personas/data-migration-reviewer.md +111 -0
- package/skills/ce-code-review/references/personas/deployment-verification-agent.md +157 -0
- package/skills/ce-code-review/references/personas/julik-frontend-races-reviewer.md +45 -0
- package/skills/ce-code-review/references/personas/learnings-researcher.md +260 -0
- package/skills/ce-code-review/references/personas/maintainability-reviewer.md +85 -0
- package/skills/ce-code-review/references/personas/performance-reviewer.md +45 -0
- package/skills/ce-code-review/references/personas/previous-comments-reviewer.md +59 -0
- package/skills/ce-code-review/references/personas/project-standards-reviewer.md +77 -0
- package/skills/ce-code-review/references/personas/reliability-reviewer.md +49 -0
- package/skills/ce-code-review/references/personas/security-reviewer.md +50 -0
- package/skills/ce-code-review/references/personas/swift-ios-reviewer.md +99 -0
- package/skills/ce-code-review/references/personas/testing-reviewer.md +51 -0
- package/skills/ce-code-review/references/review-output-template.md +172 -0
- package/skills/ce-code-review/references/scope.md +176 -0
- package/skills/ce-code-review/references/select-and-route.md +93 -0
- package/skills/ce-code-review/references/subagent-template.md +206 -0
- package/skills/ce-code-review/references/validator-batch-template.md +67 -0
- package/skills/ce-code-review/scripts/cross-model-adversarial-review.sh +1275 -0
- package/skills/ce-code-review/scripts/findings-mechanics.py +274 -0
- package/skills/ce-code-review/scripts/packs-resolve.py +737 -0
- package/skills/ce-code-review/scripts/peer-job-runner.py +2594 -0
- package/skills/ce-code-review/scripts/review-scope.py +419 -0
- package/skills/ce-code-review/scripts/run-log.py +280 -0
- package/skills/ce-commit/SKILL.md +60 -0
- package/skills/ce-commit-push-pr/SKILL.md +69 -0
- package/skills/ce-commit-push-pr/references/apply-and-handoff.md +79 -0
- package/skills/ce-commit-push-pr/references/branch-creation.md +47 -0
- package/skills/ce-commit-push-pr/references/commit-and-push.md +26 -0
- package/skills/ce-commit-push-pr/references/compose.md +29 -0
- package/skills/ce-commit-push-pr/references/context.md +42 -0
- package/skills/ce-commit-push-pr/references/gh-stack-cli.md +83 -0
- package/skills/ce-commit-push-pr/references/pr-description-writing.md +208 -0
- package/skills/ce-commit-push-pr/references/stack-submit.md +81 -0
- package/skills/ce-compound/SKILL.md +79 -0
- package/skills/ce-compound/assets/resolution-template.md +96 -0
- package/skills/ce-compound/references/agents/best-practices-researcher.md +115 -0
- package/skills/ce-compound/references/agents/data-integrity-guardian.md +68 -0
- package/skills/ce-compound/references/agents/framework-docs-researcher.md +93 -0
- package/skills/ce-compound/references/agents/pattern-recognition-specialist.md +55 -0
- package/skills/ce-compound/references/agents/performance-oracle.md +108 -0
- package/skills/ce-compound/references/agents/security-sentinel.md +91 -0
- package/skills/ce-compound/references/agents/session-historian.md +82 -0
- package/skills/ce-compound/references/assembly.md +127 -0
- package/skills/ce-compound/references/concepts-vocabulary.md +98 -0
- package/skills/ce-compound/references/enhancement.md +42 -0
- package/skills/ce-compound/references/grounding-validation.md +89 -0
- package/skills/ce-compound/references/lightweight.md +56 -0
- package/skills/ce-compound/references/modes.md +13 -0
- package/skills/ce-compound/references/refresh-and-discoverability.md +96 -0
- package/skills/ce-compound/references/report.md +115 -0
- package/skills/ce-compound/references/research.md +144 -0
- package/skills/ce-compound/references/schema.yaml +236 -0
- package/skills/ce-compound/references/session-history.md +73 -0
- package/skills/ce-compound/references/yaml-schema.md +130 -0
- package/skills/ce-compound/scripts/packs-resolve.py +737 -0
- package/skills/ce-compound/scripts/session-history/__pycache__/dsh_session.cpython-314.pyc +0 -0
- package/skills/ce-compound/scripts/session-history/discover-sessions.sh +290 -0
- package/skills/ce-compound/scripts/session-history/discover_dsh.py +123 -0
- package/skills/ce-compound/scripts/session-history/dsh_session.py +225 -0
- package/skills/ce-compound/scripts/session-history/extract-errors.py +178 -0
- package/skills/ce-compound/scripts/session-history/extract-metadata.py +301 -0
- package/skills/ce-compound/scripts/session-history/extract-skeleton.py +346 -0
- package/skills/ce-compound/scripts/validate-doc-claims.py +497 -0
- package/skills/ce-compound/scripts/validate-frontmatter.py +137 -0
- package/skills/ce-compound-refresh/SKILL.md +88 -0
- package/skills/ce-compound-refresh/assets/resolution-template.md +96 -0
- package/skills/ce-compound-refresh/references/classify.md +42 -0
- package/skills/ce-compound-refresh/references/commit.md +7 -0
- package/skills/ce-compound-refresh/references/concepts-vocabulary.md +116 -0
- package/skills/ce-compound-refresh/references/discoverability.md +15 -0
- package/skills/ce-compound-refresh/references/investigate.md +23 -0
- package/skills/ce-compound-refresh/references/modes.md +27 -0
- package/skills/ce-compound-refresh/references/per-action-flows.md +142 -0
- package/skills/ce-compound-refresh/references/report.md +23 -0
- package/skills/ce-compound-refresh/references/schema.yaml +236 -0
- package/skills/ce-compound-refresh/references/scope.md +18 -0
- package/skills/ce-compound-refresh/references/worth-audit.md +35 -0
- package/skills/ce-compound-refresh/references/yaml-schema.md +130 -0
- package/skills/ce-compound-refresh/scripts/validate-doc-claims.py +497 -0
- package/skills/ce-compound-refresh/scripts/validate-frontmatter.py +137 -0
- package/skills/ce-debug/SKILL.md +122 -0
- package/skills/ce-debug/references/anti-patterns.md +91 -0
- package/skills/ce-debug/references/defense-in-depth.md +39 -0
- package/skills/ce-debug/references/fix.md +21 -0
- package/skills/ce-debug/references/investigate.md +114 -0
- package/skills/ce-debug/references/investigation-techniques.md +384 -0
- package/skills/ce-debug/references/pipeline-mode.md +78 -0
- package/skills/ce-debug/references/post-fix-handoff.md +60 -0
- package/skills/ce-debug/references/return-to-caller.md +57 -0
- package/skills/ce-doc-review/SKILL.md +75 -0
- package/skills/ce-doc-review/references/bulk-preview.md +146 -0
- package/skills/ce-doc-review/references/cross-model-review.md +207 -0
- package/skills/ce-doc-review/references/decision-primer.md +47 -0
- package/skills/ce-doc-review/references/dispatch.md +56 -0
- package/skills/ce-doc-review/references/document-intake.md +44 -0
- package/skills/ce-doc-review/references/findings-schema.json +85 -0
- package/skills/ce-doc-review/references/modes.md +20 -0
- package/skills/ce-doc-review/references/open-questions-defer.md +158 -0
- package/skills/ce-doc-review/references/persona-selection.md +41 -0
- package/skills/ce-doc-review/references/personas/adversarial-document-reviewer.md +109 -0
- package/skills/ce-doc-review/references/personas/coherence-reviewer.md +69 -0
- package/skills/ce-doc-review/references/personas/design-lens-reviewer.md +48 -0
- package/skills/ce-doc-review/references/personas/feasibility-reviewer.md +50 -0
- package/skills/ce-doc-review/references/personas/product-lens-reviewer.md +85 -0
- package/skills/ce-doc-review/references/personas/scope-guardian-reviewer.md +83 -0
- package/skills/ce-doc-review/references/personas/security-lens-reviewer.md +41 -0
- package/skills/ce-doc-review/references/personas/whole-doc-reviewer.md +25 -0
- package/skills/ce-doc-review/references/rendering-floor.md +175 -0
- package/skills/ce-doc-review/references/review-output-template.md +123 -0
- package/skills/ce-doc-review/references/subagent-template.md +188 -0
- package/skills/ce-doc-review/references/synthesis-and-presentation.md +350 -0
- package/skills/ce-doc-review/references/walkthrough.md +347 -0
- package/skills/ce-doc-review/scripts/cross-model-doc-review.sh +1270 -0
- package/skills/ce-doc-review/scripts/packs-resolve.py +737 -0
- package/skills/ce-doc-review/scripts/peer-job-runner.py +2594 -0
- package/skills/ce-dogfood/SKILL.md +66 -0
- package/skills/ce-dogfood/agents/openai.yaml +2 -0
- package/skills/ce-dogfood/references/dogfood-report-template.md +97 -0
- package/skills/ce-dogfood/references/phases.md +167 -0
- package/skills/ce-dogfood/references/test-matrix-taxonomy.md +62 -0
- package/skills/ce-dogfood/scripts/packs-resolve.py +737 -0
- package/skills/ce-explain/SKILL.md +81 -0
- package/skills/ce-explain/references/agents/behavior-trace-scout.md +19 -0
- package/skills/ce-explain/references/agents/work-recap-scout.md +23 -0
- package/skills/ce-explain/references/check-in.md +18 -0
- package/skills/ce-explain/references/destinations.md +35 -0
- package/skills/ce-explain/references/explainer-html.md +29 -0
- package/skills/ce-explain/references/explainer-markdown.md +24 -0
- package/skills/ce-explain/references/intake.md +50 -0
- package/skills/ce-explain/references/orchestration.md +46 -0
- package/skills/ce-handoff/SKILL.md +54 -0
- package/skills/ce-handoff/references/create.md +88 -0
- package/skills/ce-handoff/references/resume.md +46 -0
- package/skills/ce-ideate/SKILL.md +75 -0
- package/skills/ce-ideate/references/agents/issue-intelligence-analyst.md +166 -0
- package/skills/ce-ideate/references/agents/learnings-researcher.md +247 -0
- package/skills/ce-ideate/references/agents/slack-researcher.md +127 -0
- package/skills/ce-ideate/references/agents/web-researcher.md +121 -0
- package/skills/ce-ideate/references/decomposition.md +39 -0
- package/skills/ce-ideate/references/divergent-ideation.md +112 -0
- package/skills/ce-ideate/references/grounding.md +138 -0
- package/skills/ce-ideate/references/html-rendering.md +638 -0
- package/skills/ce-ideate/references/ideation-sections.md +191 -0
- package/skills/ce-ideate/references/issue-intelligence.md +44 -0
- package/skills/ce-ideate/references/markdown-rendering.md +242 -0
- package/skills/ce-ideate/references/output-mode.md +34 -0
- package/skills/ce-ideate/references/post-ideation-workflow.md +169 -0
- package/skills/ce-ideate/references/scope-gates.md +148 -0
- package/skills/ce-ideate/references/universal-ideation.md +119 -0
- package/skills/ce-ideate/references/user-research-artifacts.md +28 -0
- package/skills/ce-ideate/references/web-research-cache.md +56 -0
- package/skills/ce-noslop/SKILL.md +45 -0
- package/skills/ce-noslop/references/patterns.md +61 -0
- package/skills/ce-noslop/references/terminology.md +3 -0
- package/skills/ce-optimize/SKILL.md +64 -0
- package/skills/ce-optimize/references/agents/learnings-researcher.md +247 -0
- package/skills/ce-optimize/references/agents/repo-research-analyst.md +258 -0
- package/skills/ce-optimize/references/example-expensive-benchmark-spec.yaml +88 -0
- package/skills/ce-optimize/references/example-hard-spec.yaml +64 -0
- package/skills/ce-optimize/references/example-judge-spec.yaml +83 -0
- package/skills/ce-optimize/references/example-text-target-spec.yaml +80 -0
- package/skills/ce-optimize/references/experiment-log-schema.yaml +563 -0
- package/skills/ce-optimize/references/experiment-prompt-template.md +143 -0
- package/skills/ce-optimize/references/judge-prompt-template.md +117 -0
- package/skills/ce-optimize/references/loop.md +286 -0
- package/skills/ce-optimize/references/measurement.md +168 -0
- package/skills/ce-optimize/references/optimize-spec-schema.yaml +570 -0
- package/skills/ce-optimize/references/persistence.md +93 -0
- package/skills/ce-optimize/references/spec.md +107 -0
- package/skills/ce-optimize/references/text-targets.md +51 -0
- package/skills/ce-optimize/references/usage-guide.md +147 -0
- package/skills/ce-optimize/references/wrap-up.md +54 -0
- package/skills/ce-optimize/scripts/decide.mjs +676 -0
- package/skills/ce-optimize/scripts/experiment-worktree.sh +330 -0
- package/skills/ce-optimize/scripts/measure.sh +140 -0
- package/skills/ce-optimize/scripts/parallel-probe.sh +134 -0
- package/skills/ce-plan/SKILL.md +60 -0
- package/skills/ce-plan/references/agents/agent-native-planning-strategist.md +66 -0
- package/skills/ce-plan/references/agents/architecture-strategist.md +46 -0
- package/skills/ce-plan/references/agents/best-practices-researcher.md +118 -0
- package/skills/ce-plan/references/agents/data-integrity-guardian.md +68 -0
- package/skills/ce-plan/references/agents/framework-docs-researcher.md +97 -0
- package/skills/ce-plan/references/agents/git-history-analyzer.md +40 -0
- package/skills/ce-plan/references/agents/learnings-researcher.md +268 -0
- package/skills/ce-plan/references/agents/pattern-recognition-specialist.md +55 -0
- package/skills/ce-plan/references/agents/performance-oracle.md +108 -0
- package/skills/ce-plan/references/agents/repo-research-analyst.md +262 -0
- package/skills/ce-plan/references/agents/security-sentinel.md +91 -0
- package/skills/ce-plan/references/agents/slack-researcher.md +131 -0
- package/skills/ce-plan/references/agents/spec-flow-analyzer.md +84 -0
- package/skills/ce-plan/references/agents/web-researcher.md +125 -0
- package/skills/ce-plan/references/approach-altitude.md +55 -0
- package/skills/ce-plan/references/bakeoff.md +9 -0
- package/skills/ce-plan/references/deepening-workflow.md +263 -0
- package/skills/ce-plan/references/final-review.md +134 -0
- package/skills/ce-plan/references/html-rendering.md +638 -0
- package/skills/ce-plan/references/intake.md +158 -0
- package/skills/ce-plan/references/markdown-rendering.md +242 -0
- package/skills/ce-plan/references/output-contracts.md +29 -0
- package/skills/ce-plan/references/output-mode.md +45 -0
- package/skills/ce-plan/references/plan-handoff.md +116 -0
- package/skills/ce-plan/references/plan-sections.md +489 -0
- package/skills/ce-plan/references/reasoning-elevation.md +124 -0
- package/skills/ce-plan/references/research.md +211 -0
- package/skills/ce-plan/references/resume.md +61 -0
- package/skills/ce-plan/references/settled-decisions.md +45 -0
- package/skills/ce-plan/references/structure.md +207 -0
- package/skills/ce-plan/references/synthesis-summary.md +260 -0
- package/skills/ce-plan/references/universal-planning.md +170 -0
- package/skills/ce-plan/scripts/elevation-dispatch.sh +288 -0
- package/skills/ce-plan/scripts/packs-resolve.py +737 -0
- package/skills/ce-plan/scripts/peer-job-runner.py +2594 -0
- package/skills/ce-polish/SKILL.md +22 -0
- package/skills/ce-polish/agents/openai.yaml +2 -0
- package/skills/ce-polish/references/dev-server-astro.md +58 -0
- package/skills/ce-polish/references/dev-server-detection.md +40 -0
- package/skills/ce-polish/references/dev-server-next.md +61 -0
- package/skills/ce-polish/references/dev-server-nuxt.md +58 -0
- package/skills/ce-polish/references/dev-server-procfile.md +59 -0
- package/skills/ce-polish/references/dev-server-rails.md +50 -0
- package/skills/ce-polish/references/dev-server-remix.md +58 -0
- package/skills/ce-polish/references/dev-server-sveltekit.md +58 -0
- package/skills/ce-polish/references/dev-server-vite.md +48 -0
- package/skills/ce-polish/references/install-riffrec.md +35 -0
- package/skills/ce-polish/references/launch-json-schema.md +177 -0
- package/skills/ce-polish/references/live-loop.md +98 -0
- package/skills/ce-polish/references/live-remote.md +35 -0
- package/skills/ce-polish/references/live-start.md +93 -0
- package/skills/ce-polish/references/live-stream-contract.md +253 -0
- package/skills/ce-polish/references/run.md +67 -0
- package/skills/ce-polish/scripts/detect-project-type.sh +277 -0
- package/skills/ce-polish/scripts/detect-riffrec.sh +225 -0
- package/skills/ce-polish/scripts/live-endpoint.js +2841 -0
- package/skills/ce-polish/scripts/read-launch-json.sh +117 -0
- package/skills/ce-polish/scripts/resolve-package-manager.sh +95 -0
- package/skills/ce-polish/scripts/resolve-port.sh +307 -0
- package/skills/ce-pov/SKILL.md +64 -0
- package/skills/ce-pov/references/agents/external-evidence-researcher.md +26 -0
- package/skills/ce-pov/references/agents/pov-peer.md +43 -0
- package/skills/ce-pov/references/agents/precedent-activity-scout.md +21 -0
- package/skills/ce-pov/references/agents/project-grounding-scout.md +32 -0
- package/skills/ce-pov/references/boundaries.md +36 -0
- package/skills/ce-pov/references/cross-model-panel.md +596 -0
- package/skills/ce-pov/references/followup.md +9 -0
- package/skills/ce-pov/references/grounding.md +47 -0
- package/skills/ce-pov/references/intake.md +61 -0
- package/skills/ce-pov/references/invocation.md +47 -0
- package/skills/ce-pov/references/method.md +68 -0
- package/skills/ce-pov/references/pov-schema.json +56 -0
- package/skills/ce-pov/references/report.md +7 -0
- package/skills/ce-pov/scripts/cross-model-pov.sh +1110 -0
- package/skills/ce-pov/scripts/peer-job-runner.py +2594 -0
- package/skills/ce-product-pulse/SKILL.md +81 -0
- package/skills/ce-product-pulse/agents/openai.yaml +2 -0
- package/skills/ce-product-pulse/references/config.md +20 -0
- package/skills/ce-product-pulse/references/interview.md +260 -0
- package/skills/ce-product-pulse/references/report-template.md +89 -0
- package/skills/ce-product-pulse/references/run.md +46 -0
- package/skills/ce-product-pulse/references/setup.md +43 -0
- package/skills/ce-promote/SKILL.md +90 -0
- package/skills/ce-promote/agents/openai.yaml +2 -0
- package/skills/ce-promote/references/spiral-cli.md +149 -0
- package/skills/ce-proof/SKILL.md +65 -0
- package/skills/ce-proof/references/api.md +176 -0
- package/skills/ce-proof/references/workflows.md +124 -0
- package/skills/ce-prototype/SKILL.md +59 -0
- package/skills/ce-prototype/assets/annotate.css +206 -0
- package/skills/ce-prototype/assets/annotate.js +837 -0
- package/skills/ce-prototype/references/annotation-loop.md +32 -0
- package/skills/ce-prototype/references/build.md +39 -0
- package/skills/ce-prototype/references/craft-floor.md +43 -0
- package/skills/ce-prototype/references/preview.md +118 -0
- package/skills/ce-prototype/references/scoping.md +38 -0
- package/skills/ce-prototype/references/write-back.md +37 -0
- package/skills/ce-prototype/scripts/light-webserver.js +1275 -0
- package/skills/ce-resolve-pr-feedback/SKILL.md +60 -0
- package/skills/ce-resolve-pr-feedback/references/agents/pr-comment-resolver.md +59 -0
- package/skills/ce-resolve-pr-feedback/references/evaluation-rubric.md +141 -0
- package/skills/ce-resolve-pr-feedback/references/full-mode.md +336 -0
- package/skills/ce-resolve-pr-feedback/references/pipeline-mode.md +23 -0
- package/skills/ce-resolve-pr-feedback/references/resume.md +60 -0
- package/skills/ce-resolve-pr-feedback/references/return-to-caller.md +111 -0
- package/skills/ce-resolve-pr-feedback/references/targeted-mode.md +48 -0
- package/skills/ce-resolve-pr-feedback/scripts/get-pr-comments +176 -0
- package/skills/ce-resolve-pr-feedback/scripts/get-thread-for-comment +84 -0
- package/skills/ce-resolve-pr-feedback/scripts/pending-feedback.py +287 -0
- package/skills/ce-resolve-pr-feedback/scripts/reply-to-pr-thread +57 -0
- package/skills/ce-resolve-pr-feedback/scripts/resolve-pr-thread +23 -0
- package/skills/ce-retune/SKILL.md +42 -0
- package/skills/ce-retune/agents/openai.yaml +2 -0
- package/skills/ce-retune/references/baseline-mining.md +122 -0
- package/skills/ce-retune/references/corpus-audit.md +128 -0
- package/skills/ce-retune/references/cut-passes.md +129 -0
- package/skills/ce-retune/references/halt-taxonomy.md +123 -0
- package/skills/ce-retune/references/noise-floor.md +131 -0
- package/skills/ce-retune/references/workflow-shapes.md +73 -0
- package/skills/ce-riffrec-feedback-analysis/SKILL.md +33 -0
- package/skills/ce-riffrec-feedback-analysis/references/analyzer.md +18 -0
- package/skills/ce-riffrec-feedback-analysis/references/compound-engineering-feedback-format.md +116 -0
- package/skills/ce-riffrec-feedback-analysis/references/extensive-analysis.md +98 -0
- package/skills/ce-riffrec-feedback-analysis/references/install-riffrec.md +27 -0
- package/skills/ce-riffrec-feedback-analysis/references/quick-bug-report.md +38 -0
- package/skills/ce-riffrec-feedback-analysis/scripts/analyze_riffrec_zip.py +1306 -0
- package/skills/ce-setup/SKILL.md +103 -0
- package/skills/ce-setup/agents/openai.yaml +2 -0
- package/skills/ce-setup/assets/compounding-directive.md +11 -0
- package/skills/ce-setup/assets/noslop-directive.md +7 -0
- package/skills/ce-setup/assets/pack-rule-template.md +11 -0
- package/skills/ce-setup/references/acpx-cache-warm.md +14 -0
- package/skills/ce-setup/references/config-template.yaml +210 -0
- package/skills/ce-setup/references/legacy-codex-tool-map.md +15 -0
- package/skills/ce-setup/references/pack-scaffold.md +45 -0
- package/skills/ce-setup/references/repo-fixes.md +121 -0
- package/skills/ce-setup/scripts/check-health +929 -0
- package/skills/ce-setup/scripts/packs-resolve.py +737 -0
- package/skills/ce-simplify-code/SKILL.md +76 -0
- package/skills/ce-simplify-code/references/personas/code-quality-reviewer.md +17 -0
- package/skills/ce-simplify-code/references/personas/code-reuse-reviewer.md +7 -0
- package/skills/ce-simplify-code/references/personas/comment-reviewer.md +24 -0
- package/skills/ce-simplify-code/references/personas/efficiency-reviewer.md +11 -0
- package/skills/ce-simplify-code/references/personas/test-value-reviewer.md +23 -0
- package/skills/ce-simplify-code/references/reviewer-dispatch.md +11 -0
- package/skills/ce-simplify-code/references/standalone-report.md +25 -0
- package/skills/ce-strategy/SKILL.md +61 -0
- package/skills/ce-strategy/references/grounding.md +22 -0
- package/skills/ce-strategy/references/interview.md +179 -0
- package/skills/ce-strategy/references/strategy-template.md +94 -0
- package/skills/ce-strategy/references/update-run.md +33 -0
- package/skills/ce-sweep/SKILL.md +93 -0
- package/skills/ce-sweep/agents/openai.yaml +2 -0
- package/skills/ce-sweep/references/agents/media-analyzer.md +54 -0
- package/skills/ce-sweep/references/interview.md +208 -0
- package/skills/ce-sweep/references/model-tiers.md +13 -0
- package/skills/ce-sweep/references/plan-template.md +62 -0
- package/skills/ce-sweep/references/run.md +126 -0
- package/skills/ce-sweep/references/sources/email.md +57 -0
- package/skills/ce-sweep/references/sources/github-issues.md +59 -0
- package/skills/ce-sweep/references/sources/slack.md +61 -0
- package/skills/ce-sweep/references/state-schema.md +157 -0
- package/skills/ce-sweep/references/subagent-template.md +60 -0
- package/skills/ce-sweep/scripts/analyze_riffrec_zip.py +1306 -0
- package/skills/ce-sweep/scripts/sweep-state.py +793 -0
- package/skills/ce-test-browser/SKILL.md +50 -0
- package/skills/ce-test-browser/references/agent-browser-driver.md +47 -0
- package/skills/ce-test-browser/references/pipeline-orchestration.md +45 -0
- package/skills/ce-test-browser/references/route-and-report.md +129 -0
- package/skills/ce-test-browser/scripts/resolve-port.sh +41 -0
- package/skills/ce-test-xcode/SKILL.md +26 -0
- package/skills/ce-test-xcode/agents/openai.yaml +2 -0
- package/skills/ce-test-xcode/references/setup-and-build.md +32 -0
- package/skills/ce-test-xcode/references/test-and-report.md +64 -0
- package/skills/ce-work/SKILL.md +62 -0
- package/skills/ce-work/references/agents/figma-design-sync.md +27 -0
- package/skills/ce-work/references/agents/implementation-worker.md +17 -0
- package/skills/ce-work/references/cross-model-execution.md +191 -0
- package/skills/ce-work/references/execution-engines.md +64 -0
- package/skills/ce-work/references/execution-strategy.md +73 -0
- package/skills/ce-work/references/implementation-loop.md +107 -0
- package/skills/ce-work/references/implementation-result-schema.json +52 -0
- package/skills/ce-work/references/input-triage.md +50 -0
- package/skills/ce-work/references/non-code-execution.md +23 -0
- package/skills/ce-work/references/return-to-caller.md +33 -0
- package/skills/ce-work/references/review-findings-followup.md +82 -0
- package/skills/ce-work/references/shipping-workflow.md +89 -0
- package/skills/ce-work/references/tracker-defer.md +150 -0
- package/skills/ce-work/references/work-intake.md +51 -0
- package/skills/ce-work/references/workspace-setup.md +27 -0
- package/skills/ce-work/scripts/__pycache__/unit_workspace_jobs.cpython-314.pyc +0 -0
- package/skills/ce-work/scripts/__pycache__/unit_workspace_state.cpython-314.pyc +0 -0
- package/skills/ce-work/scripts/cross-model-work.sh +1385 -0
- package/skills/ce-work/scripts/peer-job-runner.py +2594 -0
- package/skills/ce-work/scripts/unit-workspace.py +215 -0
- package/skills/ce-work/scripts/unit_workspace_ignored.py +93 -0
- package/skills/ce-work/scripts/unit_workspace_integration.py +677 -0
- package/skills/ce-work/scripts/unit_workspace_jobs.py +1216 -0
- package/skills/ce-work/scripts/unit_workspace_lifecycle.py +981 -0
- package/skills/ce-work/scripts/unit_workspace_state.py +1379 -0
- package/skills/ce-work/scripts/unit_workspace_transaction.py +612 -0
- package/skills/ce-worktree/SKILL.md +54 -0
- package/skills/lfg/SKILL.md +59 -0
- package/skills/lfg/references/debug-return.md +25 -0
- package/skills/lfg/references/intake.md +30 -0
- package/skills/lfg/references/next-work-handoff.md +96 -0
- package/skills/lfg/references/plan-brief.md +36 -0
- package/skills/lfg/references/review-followup.md +78 -0
- package/skills/lfg/references/shipping.md +65 -0
- package/skills/lfg/references/stage-routing.md +49 -0
- package/skills/lfg/references/task-visibility.md +11 -0
- package/skills/lfg/references/tracker-defer.md +150 -0
- package/skills/lfg/references/work-return.md +27 -0
- package/skills/wtf/SKILL.md +33 -0
- package/skills/wtf/agents/openai.yaml +2 -0
|
@@ -0,0 +1,2594 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Detached peer-job runner: start/status/wait/result/reap for delegated work.
|
|
3
|
+
|
|
4
|
+
Some harnesses enforce a hard ceiling on a single shell tool call and kill the
|
|
5
|
+
supervising shell mid-run, so no tool call may span a peer worker's runtime.
|
|
6
|
+
This runner splits the lifecycle so every call is short and all durable state
|
|
7
|
+
lives on disk:
|
|
8
|
+
|
|
9
|
+
start claim a job dir, preflight the worker, detach it into its own
|
|
10
|
+
session (double fork with os.setsid between the forks), print ONLY
|
|
11
|
+
the job id, return fast. The detached process supervises the worker
|
|
12
|
+
and writes ONE atomic terminal record. Also sweeps sibling run roots
|
|
13
|
+
older than 24 hours (best-effort, owner-checked).
|
|
14
|
+
status print each job's state word without blocking.
|
|
15
|
+
wait bounded poll (~1s cadence, never longer than --max-secs) that
|
|
16
|
+
returns early once every watched job has settled. With
|
|
17
|
+
--deadline-secs it also ends at the aggregate deadline and closes
|
|
18
|
+
with one line naming what to do next.
|
|
19
|
+
result ownership-checked bounded read of a done job's published artifact.
|
|
20
|
+
reap ask each job's detached supervisor to terminate it now; returns fast.
|
|
21
|
+
If the supervisor itself is gone, reap kills the worker tree and
|
|
22
|
+
writes the terminal record itself. Reaping a terminal job is a
|
|
23
|
+
safe no-op.
|
|
24
|
+
|
|
25
|
+
Job directory (durable state, the source of truth):
|
|
26
|
+
<root>/<skill>/<run-id>/jobs/<job-id>/
|
|
27
|
+
meta.json identity: skill, run id, label, input digest, start time,
|
|
28
|
+
worker argv, result path (written at start, before detach)
|
|
29
|
+
pid supervisor pid + worker pid (written by the supervisor before
|
|
30
|
+
start returns; its presence marks "detached"). Platform-
|
|
31
|
+
conditional fields — consumers must use .get(): POSIX adds
|
|
32
|
+
supervisor_pgid; Windows adds job_name (its job object) and
|
|
33
|
+
supervisor_identity / worker_identity (GetProcessTimes guards
|
|
34
|
+
so a recycled pid is not treated as the original process).
|
|
35
|
+
out.log worker's combined stdout+stderr (byte growth = liveness)
|
|
36
|
+
reason terminal detail, written before the status rename so the
|
|
37
|
+
status file is always the LAST record to land
|
|
38
|
+
status exactly one word, published atomically (tmp + os.replace):
|
|
39
|
+
done | failed | timeout | died-without-result
|
|
40
|
+
|
|
41
|
+
States reported by status/wait:
|
|
42
|
+
running detached, no terminal record yet
|
|
43
|
+
done worker exited 0 (and, when --result-path was declared,
|
|
44
|
+
the result file exists non-empty)
|
|
45
|
+
failed nonzero exit, byte-cap kill, or exit 0 without the
|
|
46
|
+
declared result
|
|
47
|
+
timeout supervisor idle/hard window fired, or a requested reap
|
|
48
|
+
died-without-result worker killed by an external signal with no result
|
|
49
|
+
evidence (or vanished together with its supervisor)
|
|
50
|
+
never-started meta exists but nothing was ever detached (preflight
|
|
51
|
+
failure)
|
|
52
|
+
unreadable an ownership or sanity check failed; content withheld
|
|
53
|
+
|
|
54
|
+
Supervision (runs inside the detached session, never in a tool call): poll
|
|
55
|
+
~2s; liveness is out.log byte growth; idle window with no growth reaps the
|
|
56
|
+
worker tree; a hard cap reaps it regardless; byte caps on out.log and the
|
|
57
|
+
published result classify as failed with a recorded reason. Reaping is TERM
|
|
58
|
+
to the worker's own process group (the worker is started as a session/group
|
|
59
|
+
leader), a grace period, then KILL — with a deepest-first tree walk as the
|
|
60
|
+
fallback when the group kill is unavailable. The supervisor classifies the
|
|
61
|
+
outcome exactly once; when both the worker's internal cap and the
|
|
62
|
+
supervisor's window fire, the supervisor's record wins.
|
|
63
|
+
|
|
64
|
+
Environment overrides (defaults in parentheses):
|
|
65
|
+
CE_PEER_JOBS_ROOT base dir (/tmp/compound-engineering-<effective-uid>,
|
|
66
|
+
or $TMPDIR/compound-engineering-<effective-uid> when
|
|
67
|
+
/tmp cannot host a writable private root, e.g. under
|
|
68
|
+
a sandbox that only allowlists $TMPDIR)
|
|
69
|
+
CE_WORK_RUNS_ROOT parent CE Work dir containing all <run-id>/ dirs
|
|
70
|
+
CE_PEER_IDLE_SECS idle window, no out.log growth (240)
|
|
71
|
+
CE_PEER_HARD_SECS hard cap on worker wall clock
|
|
72
|
+
(default: max(1230, CROSS_MODEL_HARD_SECS+30);
|
|
73
|
+
an explicit value always wins)
|
|
74
|
+
CROSS_MODEL_HARD_SECS when CE_PEER_HARD_SECS is unset, widens the
|
|
75
|
+
supervisor hard window (see above)
|
|
76
|
+
CE_PEER_LOG_MAX_BYTES out.log byte cap (10485760)
|
|
77
|
+
CE_PEER_RESULT_MAX_BYTES result byte cap, supervise + read (5242880)
|
|
78
|
+
CE_PEER_POLL_SECS supervisor poll interval (2)
|
|
79
|
+
CE_PEER_GRACE_SECS TERM-to-KILL grace during reap (5)
|
|
80
|
+
CE_PEER_BASH Windows: absolute bash.exe for peer workers
|
|
81
|
+
(preferred over PATH / WSL System32 bash)
|
|
82
|
+
CLAUDE_CODE_GIT_BASH_PATH Claude Code Git Bash path; used on Windows when
|
|
83
|
+
CE_PEER_BASH is unset (#1268)
|
|
84
|
+
|
|
85
|
+
Security posture: the job root is a predictable, owner-private directory under
|
|
86
|
+
world-shared /tmp. Every read of job state opens the file first (no-follow) and
|
|
87
|
+
verifies the descriptor's owner (os.fstat st_uid == os.geteuid, guarded where
|
|
88
|
+
geteuid is unavailable) before any content is emitted; a mismatch reports
|
|
89
|
+
"unreadable", never content. Reads are bounded by size caps — out.log is never
|
|
90
|
+
slurped. Directory/file creation uses 0700/0600 modes, exclusive no-follow
|
|
91
|
+
creation, owner/type verification on path components, exact 0700 verification
|
|
92
|
+
on the top-level root, and atomic rename for every publish. The worker argv is
|
|
93
|
+
exec'd directly (argv list, never a shell); job
|
|
94
|
+
ids are minted internally; --skill/--run-id/--label are restricted to
|
|
95
|
+
[A-Za-z0-9._-]. Nothing here ever prompts: headless/CI-safe by design.
|
|
96
|
+
|
|
97
|
+
Platform (#1243): the mechanisms above describe POSIX. Native Windows Python
|
|
98
|
+
has no fork/setsid, uid, mode bits, or process groups, so the same contract is
|
|
99
|
+
met by win32 equivalents, all behind `sys.platform == "win32"` branches so the
|
|
100
|
+
POSIX path is behaviorally unchanged:
|
|
101
|
+
detach re-invoke this script as a DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP
|
|
102
|
+
child (CREATE_BREAKAWAY_FROM_JOB where the job allows) running the
|
|
103
|
+
internal `__supervise` entrypoint; the pid file is the ack.
|
|
104
|
+
reap cmd_reap drops a `.reap` marker the supervisor polls for (no directed
|
|
105
|
+
signal to a detached, console-less process).
|
|
106
|
+
teardown the worker tree lives in a named Job Object -- the real killpg
|
|
107
|
+
analog, since it reaches descendants of an already-exited leader,
|
|
108
|
+
which taskkill /T cannot (it walks parent->child from a LIVE
|
|
109
|
+
parent). Windows releases a named object's name once the last
|
|
110
|
+
handle closes, so a cmd_reap running after the supervisor died
|
|
111
|
+
falls back to a Toolhelp32 snapshot walk; that works because
|
|
112
|
+
Windows never reparents orphans, so a dead pid still appears as
|
|
113
|
+
th32ParentProcessID on its live children. A recycled pid that is
|
|
114
|
+
now this process (or whose GetProcessTimes identity does not
|
|
115
|
+
match the pid file) is not the original leader: sweep stale-PPID
|
|
116
|
+
descendants, do not TerminateProcess the live reused process.
|
|
117
|
+
ownership st_uid == geteuid becomes: the object's owner SID is one this token
|
|
118
|
+
creates objects as (user or default-owner SID), checked on the opened
|
|
119
|
+
handle (GetSecurityInfo) exactly like the POSIX fstat-by-fd check.
|
|
120
|
+
privacy 0700/0600 modes become a hardened ACL (icacls: break inheritance,
|
|
121
|
+
grant only the user + SYSTEM + Administrators — the root-equivalents).
|
|
122
|
+
jobs root defaults under %LOCALAPPDATA%\\compound-engineering-jobs (then the
|
|
123
|
+
user temp dir), owner-private, since there is no shared /tmp.
|
|
124
|
+
|
|
125
|
+
Pure stdlib. No third-party dependencies.
|
|
126
|
+
"""
|
|
127
|
+
import argparse
|
|
128
|
+
import calendar
|
|
129
|
+
import glob
|
|
130
|
+
import json
|
|
131
|
+
import math
|
|
132
|
+
import os
|
|
133
|
+
import re
|
|
134
|
+
import shutil
|
|
135
|
+
import signal
|
|
136
|
+
import stat
|
|
137
|
+
import subprocess
|
|
138
|
+
import sys
|
|
139
|
+
import tempfile
|
|
140
|
+
import time
|
|
141
|
+
|
|
142
|
+
# Identifier charset for --skill/--run-id/--label and bare job refs. The dot is
|
|
143
|
+
# allowed (model/date tokens use it) but an all-dot value (".", "..") would be a
|
|
144
|
+
# path component that escapes the jobs root, so it is rejected separately below.
|
|
145
|
+
SAFE_RE = re.compile(r"^[A-Za-z0-9._-]+$")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _is_safe_token(value: str) -> bool:
|
|
149
|
+
return bool(SAFE_RE.match(value)) and value.strip(".") != ""
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
TERMINAL_STATES = ("done", "failed", "timeout", "died-without-result")
|
|
153
|
+
IS_WINDOWS = sys.platform == "win32"
|
|
154
|
+
_uid_getter = getattr(os, "geteuid", None) or getattr(os, "getuid", None)
|
|
155
|
+
_EFFECTIVE_UID = _uid_getter() if _uid_getter is not None else None
|
|
156
|
+
if IS_WINDOWS:
|
|
157
|
+
# No geteuid on Windows; the current-user SID is the ownership identity
|
|
158
|
+
# (see the Windows security section below), and the per-user jobs root lives
|
|
159
|
+
# under LOCALAPPDATA (falling back to the user temp dir) with a hardened ACL
|
|
160
|
+
# so R6 has a working default rather than a required override.
|
|
161
|
+
_WIN_ROOT_BASE = os.environ.get("LOCALAPPDATA") or tempfile.gettempdir()
|
|
162
|
+
DEFAULT_ROOT = os.path.join(_WIN_ROOT_BASE, "compound-engineering-jobs")
|
|
163
|
+
elif _EFFECTIVE_UID is not None:
|
|
164
|
+
DEFAULT_ROOT = os.path.join("/tmp", f"compound-engineering-{_EFFECTIVE_UID}")
|
|
165
|
+
else:
|
|
166
|
+
DEFAULT_ROOT = None
|
|
167
|
+
O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
|
|
168
|
+
# Windows CPython opens os.open() descriptors in CRT *text* mode by default:
|
|
169
|
+
# writes expand \n -> \r\n and reads stop at the first 0x1A (Ctrl-Z EOF), which
|
|
170
|
+
# would silently corrupt and truncate a peer's result artifact and desync the
|
|
171
|
+
# out.log byte caps from st_size. O_BINARY is 0 on POSIX, so this is a no-op
|
|
172
|
+
# there and every os.open below stays byte-exact on both platforms.
|
|
173
|
+
O_BINARY = getattr(os, "O_BINARY", 0)
|
|
174
|
+
SWEEP_AGE_SECS = 24 * 3600
|
|
175
|
+
CLAIM_ATTEMPTS = 16
|
|
176
|
+
STATUS_READ_CAP = 256
|
|
177
|
+
REASON_READ_CAP = 1024
|
|
178
|
+
META_READ_CAP = 64 * 1024
|
|
179
|
+
STARTED_AT_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
|
180
|
+
REAP_SETTLE_SECS = 15
|
|
181
|
+
|
|
182
|
+
EXIT_CODES_DOC = """\
|
|
183
|
+
exit codes:
|
|
184
|
+
0 the command itself succeeded. For status/wait this means the query ran;
|
|
185
|
+
it says nothing about job outcomes — parse stdout (or --json) for states.
|
|
186
|
+
For `result` it means a done job's artifact (or a --path file) was emitted;
|
|
187
|
+
for reap it includes the safe no-op on an already-terminal job.
|
|
188
|
+
1 runtime error (preflight failure, unknown job, detach failure); for a
|
|
189
|
+
multi-job reap, also the code when its jobs failed in different ways
|
|
190
|
+
2 usage error; for `result`: the job is still running
|
|
191
|
+
3 for `result`: job settled but not done (failed / timeout /
|
|
192
|
+
died-without-result / never-started), or the result file is missing
|
|
193
|
+
4 the read was refused, so content is never emitted: the ownership check
|
|
194
|
+
failed (job state or result not owned by the current user), or the path
|
|
195
|
+
is there but unreadable (a symlink rejected by O_NOFOLLOW, a non-regular
|
|
196
|
+
file, a byte-cap overrun). Only a genuinely absent file is 3.
|
|
197
|
+
|
|
198
|
+
environment overrides: CE_PEER_JOBS_ROOT, CE_WORK_RUNS_ROOT, CE_PEER_IDLE_SECS,
|
|
199
|
+
CE_PEER_HARD_SECS, CROSS_MODEL_HARD_SECS, CE_PEER_LOG_MAX_BYTES,
|
|
200
|
+
CE_PEER_RESULT_MAX_BYTES, CE_PEER_POLL_SECS, CE_PEER_GRACE_SECS (defaults in
|
|
201
|
+
the module docstring).
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class RunnerError(Exception):
|
|
206
|
+
"""Actionable operational error: message to stderr, exit 1."""
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class Unreadable(Exception):
|
|
210
|
+
"""Job state failed an ownership or sanity check; content withheld."""
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
# --- configuration -----------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
# Supervisor hard-window floor: clears the highest cross-model worker default
|
|
216
|
+
# (review skills use CROSS_MODEL_HARD_SECS:-1200) so an unset knob still nests
|
|
217
|
+
# worker < deadline < runner without orchestrator arithmetic. Grace matches the
|
|
218
|
+
# historical prose +30s so a raised knob widens the runner the same way.
|
|
219
|
+
_RUNNER_HARD_FLOOR = 1230.0
|
|
220
|
+
_RUNNER_HARD_GRACE = 30.0
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _private_root_usable(path: str) -> bool:
|
|
224
|
+
"""True when `path` is (or can now be) a directory we own and can write into.
|
|
225
|
+
|
|
226
|
+
Creation is the probe: a sandbox that denies writes under /tmp refuses the
|
|
227
|
+
mkdir, and one that lets a pre-existing root stand still fails the access
|
|
228
|
+
check, so both land on the fallback instead of failing at the first job.
|
|
229
|
+
"""
|
|
230
|
+
try:
|
|
231
|
+
os.mkdir(path, 0o700)
|
|
232
|
+
except FileExistsError:
|
|
233
|
+
pass
|
|
234
|
+
except OSError:
|
|
235
|
+
return False
|
|
236
|
+
try:
|
|
237
|
+
_check_owned_dir(path)
|
|
238
|
+
except (OSError, RunnerError):
|
|
239
|
+
return False
|
|
240
|
+
return os.access(path, os.W_OK)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _fallback_root() -> str:
|
|
244
|
+
return os.path.join(os.environ.get("TMPDIR") or "/tmp", f"compound-engineering-{_EFFECTIVE_UID}")
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def jobs_root_base() -> str:
|
|
248
|
+
configured = os.environ.get("CE_PEER_JOBS_ROOT")
|
|
249
|
+
if configured:
|
|
250
|
+
return os.path.abspath(configured)
|
|
251
|
+
if DEFAULT_ROOT is None:
|
|
252
|
+
raise RunnerError("effective user ID is unavailable; cannot derive the jobs root")
|
|
253
|
+
if IS_WINDOWS or _private_root_usable(DEFAULT_ROOT):
|
|
254
|
+
return os.path.abspath(DEFAULT_ROOT)
|
|
255
|
+
# Same order and candidates as the skills' shell preamble, so a job started
|
|
256
|
+
# there is found here.
|
|
257
|
+
return os.path.abspath(_fallback_root())
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def candidate_jobs_root_bases() -> list:
|
|
261
|
+
"""Every root an existing job may live under: the configured root alone, or
|
|
262
|
+
both the /tmp root and the $TMPDIR fallback (deduplicated, primary first).
|
|
263
|
+
|
|
264
|
+
Creation uses jobs_root_base(); lookup of an already-started job must not
|
|
265
|
+
depend on which root *this* invocation would create under, because a
|
|
266
|
+
sandboxed session and a later unsandboxed one resolve different roots.
|
|
267
|
+
"""
|
|
268
|
+
configured = os.environ.get("CE_PEER_JOBS_ROOT")
|
|
269
|
+
if configured:
|
|
270
|
+
return [os.path.abspath(configured)]
|
|
271
|
+
if DEFAULT_ROOT is None:
|
|
272
|
+
raise RunnerError("effective user ID is unavailable; cannot derive the jobs root")
|
|
273
|
+
bases = [os.path.abspath(DEFAULT_ROOT)]
|
|
274
|
+
if not IS_WINDOWS:
|
|
275
|
+
fallback = os.path.abspath(_fallback_root())
|
|
276
|
+
if fallback not in bases:
|
|
277
|
+
bases.append(fallback)
|
|
278
|
+
return bases
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def skill_runs_root(skill: str) -> str:
|
|
282
|
+
if skill == "ce-work" and os.environ.get("CE_WORK_RUNS_ROOT"):
|
|
283
|
+
return os.path.abspath(os.environ["CE_WORK_RUNS_ROOT"])
|
|
284
|
+
return os.path.join(jobs_root_base(), skill)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def candidate_skill_runs_roots(skill: str) -> list:
|
|
288
|
+
if skill == "ce-work" and os.environ.get("CE_WORK_RUNS_ROOT"):
|
|
289
|
+
return [os.path.abspath(os.environ["CE_WORK_RUNS_ROOT"])]
|
|
290
|
+
return [os.path.join(base, skill) for base in candidate_jobs_root_bases()]
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _env_num(name: str, default: float, conv, *, allow_zero: bool = False):
|
|
294
|
+
raw = os.environ.get(name)
|
|
295
|
+
if not raw:
|
|
296
|
+
return default
|
|
297
|
+
try:
|
|
298
|
+
val = conv(raw)
|
|
299
|
+
except ValueError:
|
|
300
|
+
return default
|
|
301
|
+
if allow_zero and val == 0:
|
|
302
|
+
return None
|
|
303
|
+
return val if val > 0 else default
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _derived_hard_default() -> float:
|
|
307
|
+
"""Outermost supervisor hard window when CE_PEER_HARD_SECS is unset.
|
|
308
|
+
|
|
309
|
+
Reads ambient CROSS_MODEL_HARD_SECS (the runner already forwards os.environ
|
|
310
|
+
to the worker, so a user-set knob is present here). Explicit CE_PEER_HARD_SECS
|
|
311
|
+
still wins via cfg() — ce-work and elevation paths keep their own windows.
|
|
312
|
+
"""
|
|
313
|
+
cross = _env_num("CROSS_MODEL_HARD_SECS", 0.0, float)
|
|
314
|
+
return max(_RUNNER_HARD_FLOOR, cross + _RUNNER_HARD_GRACE)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def cfg(skill=None) -> dict:
|
|
318
|
+
return {
|
|
319
|
+
"idle": _env_num("CE_PEER_IDLE_SECS", 240.0, float, allow_zero=skill == "ce-work"),
|
|
320
|
+
"hard": _env_num("CE_PEER_HARD_SECS", _derived_hard_default(), float),
|
|
321
|
+
"log_max": int(_env_num("CE_PEER_LOG_MAX_BYTES", 10 * 1024 * 1024, int)),
|
|
322
|
+
"result_max": int(_env_num("CE_PEER_RESULT_MAX_BYTES", 5 * 1024 * 1024, int)),
|
|
323
|
+
"poll": _env_num("CE_PEER_POLL_SECS", 2.0, float),
|
|
324
|
+
"grace": _env_num("CE_PEER_GRACE_SECS", 5.0, float),
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
# --- Windows security + process primitives ------------------------------------
|
|
329
|
+
#
|
|
330
|
+
# POSIX ownership is `fstat().st_uid == geteuid()` plus mode 0700/0600. Windows
|
|
331
|
+
# has neither uids nor mode bits, so the equivalent identity is the current
|
|
332
|
+
# user's SID: a job dir/file is "ours" when its owner SID is one this process's
|
|
333
|
+
# token creates objects as (the user SID or the token's default owner SID -- an
|
|
334
|
+
# elevated process defaults new objects to Administrators). A foreign user's
|
|
335
|
+
# planted dir carries neither SID and is rejected, exactly as a uid mismatch is
|
|
336
|
+
# on POSIX. The DACL is hardened to user+SYSTEM+Administrators (root-equivalents,
|
|
337
|
+
# mirroring how root still reaches a 0700 dir) with inheritance broken, so no
|
|
338
|
+
# world/Users grant survives. Pure stdlib via ctypes -- no pywin32.
|
|
339
|
+
|
|
340
|
+
if IS_WINDOWS:
|
|
341
|
+
import ctypes
|
|
342
|
+
import msvcrt
|
|
343
|
+
from ctypes import wintypes
|
|
344
|
+
|
|
345
|
+
_advapi32 = ctypes.WinDLL("advapi32", use_last_error=True)
|
|
346
|
+
_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
347
|
+
|
|
348
|
+
_SE_FILE_OBJECT = 1
|
|
349
|
+
_OWNER_SECURITY_INFORMATION = 0x00000001
|
|
350
|
+
_TOKEN_QUERY = 0x0008
|
|
351
|
+
_TOKEN_USER_CLASS = 1
|
|
352
|
+
_TOKEN_OWNER_CLASS = 4
|
|
353
|
+
_STILL_ACTIVE = 259
|
|
354
|
+
_WAIT_TIMEOUT = 0x00000102
|
|
355
|
+
_PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
|
|
356
|
+
_SYNCHRONIZE = 0x00100000
|
|
357
|
+
# A detached, console-less parent still gives its children a NEW console
|
|
358
|
+
# unless this is set, so every job would flash a window on the user's
|
|
359
|
+
# desktop. Applied to the worker and to every helper tool we shell out to.
|
|
360
|
+
_WIN_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
|
|
361
|
+
|
|
362
|
+
def _win_tool(name: str) -> str:
|
|
363
|
+
"""Absolute path to a System32 tool. CreateProcess searches the
|
|
364
|
+
application and current directories before System32, so invoking
|
|
365
|
+
`icacls`/`taskkill` by bare name from an untrusted CWD is a binary-
|
|
366
|
+
hijack surface. Falls back to the bare name only if System32 is
|
|
367
|
+
unresolvable, which is strictly better than never running."""
|
|
368
|
+
root = os.environ.get("SystemRoot") or r"C:\Windows"
|
|
369
|
+
candidate = os.path.join(root, "System32", name + ".exe")
|
|
370
|
+
return candidate if os.path.isfile(candidate) else name
|
|
371
|
+
|
|
372
|
+
_advapi32.ConvertSidToStringSidW.argtypes = [
|
|
373
|
+
ctypes.c_void_p, ctypes.POINTER(ctypes.c_wchar_p)]
|
|
374
|
+
_advapi32.ConvertSidToStringSidW.restype = wintypes.BOOL
|
|
375
|
+
_advapi32.OpenProcessToken.argtypes = [
|
|
376
|
+
wintypes.HANDLE, wintypes.DWORD, ctypes.POINTER(wintypes.HANDLE)]
|
|
377
|
+
_advapi32.OpenProcessToken.restype = wintypes.BOOL
|
|
378
|
+
_advapi32.GetTokenInformation.argtypes = [
|
|
379
|
+
wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD,
|
|
380
|
+
ctypes.POINTER(wintypes.DWORD)]
|
|
381
|
+
_advapi32.GetTokenInformation.restype = wintypes.BOOL
|
|
382
|
+
_advapi32.GetSecurityInfo.argtypes = [
|
|
383
|
+
wintypes.HANDLE, ctypes.c_int, wintypes.DWORD,
|
|
384
|
+
ctypes.POINTER(ctypes.c_void_p), ctypes.c_void_p, ctypes.c_void_p,
|
|
385
|
+
ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
|
|
386
|
+
_advapi32.GetSecurityInfo.restype = wintypes.DWORD
|
|
387
|
+
_advapi32.GetNamedSecurityInfoW.argtypes = [
|
|
388
|
+
wintypes.LPCWSTR, ctypes.c_int, wintypes.DWORD,
|
|
389
|
+
ctypes.POINTER(ctypes.c_void_p), ctypes.c_void_p, ctypes.c_void_p,
|
|
390
|
+
ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
|
|
391
|
+
_advapi32.GetNamedSecurityInfoW.restype = wintypes.DWORD
|
|
392
|
+
_kernel32.GetCurrentProcess.restype = wintypes.HANDLE
|
|
393
|
+
_kernel32.OpenProcess.argtypes = [
|
|
394
|
+
wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
395
|
+
_kernel32.OpenProcess.restype = wintypes.HANDLE
|
|
396
|
+
_kernel32.GetProcessTimes.argtypes = [
|
|
397
|
+
wintypes.HANDLE,
|
|
398
|
+
ctypes.POINTER(wintypes.FILETIME),
|
|
399
|
+
ctypes.POINTER(wintypes.FILETIME),
|
|
400
|
+
ctypes.POINTER(wintypes.FILETIME),
|
|
401
|
+
ctypes.POINTER(wintypes.FILETIME),
|
|
402
|
+
]
|
|
403
|
+
_kernel32.GetProcessTimes.restype = wintypes.BOOL
|
|
404
|
+
_kernel32.QueryFullProcessImageNameW.argtypes = [
|
|
405
|
+
wintypes.HANDLE, wintypes.DWORD, wintypes.LPWSTR,
|
|
406
|
+
ctypes.POINTER(wintypes.DWORD)]
|
|
407
|
+
_kernel32.QueryFullProcessImageNameW.restype = wintypes.BOOL
|
|
408
|
+
_kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD]
|
|
409
|
+
_kernel32.WaitForSingleObject.restype = wintypes.DWORD
|
|
410
|
+
_kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
411
|
+
_kernel32.CloseHandle.restype = wintypes.BOOL
|
|
412
|
+
_kernel32.LocalFree.argtypes = [wintypes.HGLOBAL]
|
|
413
|
+
_kernel32.LocalFree.restype = wintypes.HGLOBAL
|
|
414
|
+
_kernel32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
|
|
415
|
+
_kernel32.CreateJobObjectW.restype = wintypes.HANDLE
|
|
416
|
+
_kernel32.OpenJobObjectW.argtypes = [
|
|
417
|
+
wintypes.DWORD, wintypes.BOOL, wintypes.LPCWSTR]
|
|
418
|
+
_kernel32.OpenJobObjectW.restype = wintypes.HANDLE
|
|
419
|
+
_kernel32.AssignProcessToJobObject.argtypes = [
|
|
420
|
+
wintypes.HANDLE, wintypes.HANDLE]
|
|
421
|
+
_kernel32.AssignProcessToJobObject.restype = wintypes.BOOL
|
|
422
|
+
_kernel32.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT]
|
|
423
|
+
_kernel32.TerminateJobObject.restype = wintypes.BOOL
|
|
424
|
+
|
|
425
|
+
_JOB_OBJECT_TERMINATE = 0x0008
|
|
426
|
+
_PROCESS_TERMINATE = 0x0001
|
|
427
|
+
_PROCESS_SET_QUOTA = 0x0100
|
|
428
|
+
_TH32CS_SNAPPROCESS = 0x00000002
|
|
429
|
+
_TH32CS_SNAPTHREAD = 0x00000004
|
|
430
|
+
_THREAD_SUSPEND_RESUME = 0x0002
|
|
431
|
+
# CreateProcess CREATE_SUSPENDED: primary thread starts frozen so we can
|
|
432
|
+
# AssignProcessToJobObject before any user code (or child spawn) runs.
|
|
433
|
+
_CREATE_SUSPENDED = 0x00000004
|
|
434
|
+
|
|
435
|
+
class _PROCESSENTRY32W(ctypes.Structure):
|
|
436
|
+
_fields_ = [
|
|
437
|
+
("dwSize", wintypes.DWORD),
|
|
438
|
+
("cntUsage", wintypes.DWORD),
|
|
439
|
+
("th32ProcessID", wintypes.DWORD),
|
|
440
|
+
("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)),
|
|
441
|
+
("th32ModuleID", wintypes.DWORD),
|
|
442
|
+
("cntThreads", wintypes.DWORD),
|
|
443
|
+
("th32ParentProcessID", wintypes.DWORD),
|
|
444
|
+
("pcPriClassBase", ctypes.c_long),
|
|
445
|
+
("dwFlags", wintypes.DWORD),
|
|
446
|
+
("szExeFile", ctypes.c_wchar * 260),
|
|
447
|
+
]
|
|
448
|
+
|
|
449
|
+
class _THREADENTRY32(ctypes.Structure):
|
|
450
|
+
_fields_ = [
|
|
451
|
+
("dwSize", wintypes.DWORD),
|
|
452
|
+
("cntUsage", wintypes.DWORD),
|
|
453
|
+
("th32ThreadID", wintypes.DWORD),
|
|
454
|
+
("th32OwnerProcessID", wintypes.DWORD),
|
|
455
|
+
("tpBasePri", ctypes.c_long),
|
|
456
|
+
("tpDeltaPri", ctypes.c_long),
|
|
457
|
+
("dwFlags", wintypes.DWORD),
|
|
458
|
+
]
|
|
459
|
+
|
|
460
|
+
_kernel32.CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD]
|
|
461
|
+
_kernel32.CreateToolhelp32Snapshot.restype = ctypes.c_void_p
|
|
462
|
+
_kernel32.Process32FirstW.argtypes = [
|
|
463
|
+
ctypes.c_void_p, ctypes.POINTER(_PROCESSENTRY32W)]
|
|
464
|
+
_kernel32.Process32FirstW.restype = wintypes.BOOL
|
|
465
|
+
_kernel32.Process32NextW.argtypes = [
|
|
466
|
+
ctypes.c_void_p, ctypes.POINTER(_PROCESSENTRY32W)]
|
|
467
|
+
_kernel32.Process32NextW.restype = wintypes.BOOL
|
|
468
|
+
_kernel32.Thread32First.argtypes = [
|
|
469
|
+
ctypes.c_void_p, ctypes.POINTER(_THREADENTRY32)]
|
|
470
|
+
_kernel32.Thread32First.restype = wintypes.BOOL
|
|
471
|
+
_kernel32.Thread32Next.argtypes = [
|
|
472
|
+
ctypes.c_void_p, ctypes.POINTER(_THREADENTRY32)]
|
|
473
|
+
_kernel32.Thread32Next.restype = wintypes.BOOL
|
|
474
|
+
_kernel32.OpenThread.argtypes = [
|
|
475
|
+
wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
476
|
+
_kernel32.OpenThread.restype = wintypes.HANDLE
|
|
477
|
+
_kernel32.ResumeThread.argtypes = [wintypes.HANDLE]
|
|
478
|
+
_kernel32.ResumeThread.restype = wintypes.DWORD
|
|
479
|
+
_kernel32.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT]
|
|
480
|
+
_kernel32.TerminateProcess.restype = wintypes.BOOL
|
|
481
|
+
|
|
482
|
+
def _win_process_children_map():
|
|
483
|
+
"""th32ParentProcessID -> [child pids] from one Toolhelp snapshot."""
|
|
484
|
+
snap = _kernel32.CreateToolhelp32Snapshot(_TH32CS_SNAPPROCESS, 0)
|
|
485
|
+
if not snap or snap == ctypes.c_void_p(-1).value:
|
|
486
|
+
return {}
|
|
487
|
+
children = {}
|
|
488
|
+
try:
|
|
489
|
+
entry = _PROCESSENTRY32W()
|
|
490
|
+
entry.dwSize = ctypes.sizeof(_PROCESSENTRY32W)
|
|
491
|
+
more = _kernel32.Process32FirstW(snap, ctypes.byref(entry))
|
|
492
|
+
while more:
|
|
493
|
+
children.setdefault(entry.th32ParentProcessID, []).append(
|
|
494
|
+
entry.th32ProcessID)
|
|
495
|
+
more = _kernel32.Process32NextW(snap, ctypes.byref(entry))
|
|
496
|
+
finally:
|
|
497
|
+
_kernel32.CloseHandle(ctypes.c_void_p(snap))
|
|
498
|
+
return children
|
|
499
|
+
|
|
500
|
+
def _win_descendants_deepest_first(root_pid: int, children=None):
|
|
501
|
+
"""Children before parents, via a process snapshot. This is the direct
|
|
502
|
+
analog of the POSIX `ps`-based walk and carries the same pid-reuse
|
|
503
|
+
exposure. It works on an EXITED leader because Windows never reparents
|
|
504
|
+
orphans: a dead pid still appears as th32ParentProcessID on its live
|
|
505
|
+
children (unlike POSIX, where orphans are reparented to init)."""
|
|
506
|
+
if children is None:
|
|
507
|
+
children = _win_process_children_map()
|
|
508
|
+
order, queue = [], [root_pid]
|
|
509
|
+
while queue:
|
|
510
|
+
for child in children.get(queue.pop(0), []):
|
|
511
|
+
order.append(child)
|
|
512
|
+
queue.append(child)
|
|
513
|
+
return list(reversed(order))
|
|
514
|
+
|
|
515
|
+
def _win_process_identity(pid: int):
|
|
516
|
+
"""Creation time plus image path — analog of `ps -o lstart= -o command=`.
|
|
517
|
+
|
|
518
|
+
Creation time is the PID-reuse guard: Windows recycles PIDs aggressively,
|
|
519
|
+
and a recycled pid always carries a later FILETIME than the worker we
|
|
520
|
+
recorded. None means unproven (gone or unopenable)."""
|
|
521
|
+
handle = _kernel32.OpenProcess(
|
|
522
|
+
_PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
|
523
|
+
if not handle:
|
|
524
|
+
return None
|
|
525
|
+
try:
|
|
526
|
+
created, exited, kernel, user = (wintypes.FILETIME() for _ in range(4))
|
|
527
|
+
if not _kernel32.GetProcessTimes(
|
|
528
|
+
handle, ctypes.byref(created), ctypes.byref(exited),
|
|
529
|
+
ctypes.byref(kernel), ctypes.byref(user)):
|
|
530
|
+
return None
|
|
531
|
+
started = (created.dwHighDateTime << 32) | created.dwLowDateTime
|
|
532
|
+
if not started:
|
|
533
|
+
return None
|
|
534
|
+
size = wintypes.DWORD(32768)
|
|
535
|
+
buf = ctypes.create_unicode_buffer(size.value)
|
|
536
|
+
image = (buf.value if _kernel32.QueryFullProcessImageNameW(
|
|
537
|
+
handle, 0, buf, ctypes.byref(size)) else "")
|
|
538
|
+
finally:
|
|
539
|
+
_kernel32.CloseHandle(handle)
|
|
540
|
+
return "{} {}".format(started, image)
|
|
541
|
+
|
|
542
|
+
def _win_process_start_time(pid: int):
|
|
543
|
+
handle = _kernel32.OpenProcess(
|
|
544
|
+
_PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
|
545
|
+
if not handle:
|
|
546
|
+
return None
|
|
547
|
+
try:
|
|
548
|
+
created, exited, kernel, user = (wintypes.FILETIME() for _ in range(4))
|
|
549
|
+
if not _kernel32.GetProcessTimes(
|
|
550
|
+
handle, ctypes.byref(created), ctypes.byref(exited),
|
|
551
|
+
ctypes.byref(kernel), ctypes.byref(user)):
|
|
552
|
+
return None
|
|
553
|
+
started = (created.dwHighDateTime << 32) | created.dwLowDateTime
|
|
554
|
+
return started or None
|
|
555
|
+
finally:
|
|
556
|
+
_kernel32.CloseHandle(handle)
|
|
557
|
+
|
|
558
|
+
def _win_process_identity_matches(pid: int, recorded) -> bool:
|
|
559
|
+
if not recorded:
|
|
560
|
+
return True
|
|
561
|
+
current = _win_process_identity(pid)
|
|
562
|
+
return current is not None and current == recorded
|
|
563
|
+
|
|
564
|
+
def _win_terminate_pid(pid: int) -> bool:
|
|
565
|
+
if pid <= 0 or pid == os.getpid():
|
|
566
|
+
return False
|
|
567
|
+
handle = _kernel32.OpenProcess(_PROCESS_TERMINATE, False, pid)
|
|
568
|
+
if not handle:
|
|
569
|
+
return False
|
|
570
|
+
try:
|
|
571
|
+
return bool(_kernel32.TerminateProcess(handle, 1))
|
|
572
|
+
finally:
|
|
573
|
+
_kernel32.CloseHandle(handle)
|
|
574
|
+
|
|
575
|
+
def _win_job_name(job_dir: str) -> str:
|
|
576
|
+
"""A per-job named kernel object. Naming it is what makes this a real
|
|
577
|
+
pgid analog: a DIFFERENT process (cmd_reap, after the supervisor is
|
|
578
|
+
gone) can reopen it by name and terminate the whole tree."""
|
|
579
|
+
return "Local\\ce-peer-job-" + os.path.basename(job_dir.rstrip("\\/"))
|
|
580
|
+
|
|
581
|
+
def _win_create_job(name: str):
|
|
582
|
+
"""Create the job the worker tree will live in. Deliberately WITHOUT
|
|
583
|
+
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: like a POSIX process group, the job
|
|
584
|
+
must outlive the supervisor so a dead-supervisor reap still finds a
|
|
585
|
+
live tree to classify and sweep (matching the POSIX lifecycle tests)."""
|
|
586
|
+
handle = _kernel32.CreateJobObjectW(None, name)
|
|
587
|
+
return handle or None
|
|
588
|
+
|
|
589
|
+
def _win_assign_to_job(job_handle, pid: int) -> bool:
|
|
590
|
+
proc = _kernel32.OpenProcess(
|
|
591
|
+
_PROCESS_SET_QUOTA | _PROCESS_TERMINATE, False, pid)
|
|
592
|
+
if not proc:
|
|
593
|
+
return False
|
|
594
|
+
try:
|
|
595
|
+
return bool(_kernel32.AssignProcessToJobObject(job_handle, proc))
|
|
596
|
+
finally:
|
|
597
|
+
_kernel32.CloseHandle(proc)
|
|
598
|
+
|
|
599
|
+
def _win_resume_process(pid: int) -> bool:
|
|
600
|
+
"""Resume every thread of a CREATE_SUSPENDED process. subprocess.Popen
|
|
601
|
+
does not expose hThread from PROCESS_INFORMATION, so walk the thread
|
|
602
|
+
snapshot. CREATE_SUSPENDED only freezes the primary thread; resuming
|
|
603
|
+
all owned threads is still correct and idempotent for running ones."""
|
|
604
|
+
snap = _kernel32.CreateToolhelp32Snapshot(_TH32CS_SNAPTHREAD, 0)
|
|
605
|
+
if not snap or snap == ctypes.c_void_p(-1).value:
|
|
606
|
+
return False
|
|
607
|
+
resumed = False
|
|
608
|
+
try:
|
|
609
|
+
entry = _THREADENTRY32()
|
|
610
|
+
entry.dwSize = ctypes.sizeof(_THREADENTRY32)
|
|
611
|
+
more = _kernel32.Thread32First(snap, ctypes.byref(entry))
|
|
612
|
+
while more:
|
|
613
|
+
if entry.th32OwnerProcessID == pid:
|
|
614
|
+
handle = _kernel32.OpenThread(
|
|
615
|
+
_THREAD_SUSPEND_RESUME, False, entry.th32ThreadID)
|
|
616
|
+
if handle:
|
|
617
|
+
try:
|
|
618
|
+
# (DWORD)-1 == failure; 0xFFFFFFFF as unsigned.
|
|
619
|
+
if _kernel32.ResumeThread(handle) != 0xFFFFFFFF:
|
|
620
|
+
resumed = True
|
|
621
|
+
finally:
|
|
622
|
+
_kernel32.CloseHandle(handle)
|
|
623
|
+
more = _kernel32.Thread32Next(snap, ctypes.byref(entry))
|
|
624
|
+
finally:
|
|
625
|
+
_kernel32.CloseHandle(ctypes.c_void_p(snap))
|
|
626
|
+
return resumed
|
|
627
|
+
|
|
628
|
+
def _win_terminate_job(name: str) -> bool:
|
|
629
|
+
"""Terminate every process in the named job, whatever the tree shape.
|
|
630
|
+
This is the piece taskkill /T cannot do: it reaches descendants whose
|
|
631
|
+
parent has already exited, because job membership is inherited and
|
|
632
|
+
does not depend on a live parent to walk from."""
|
|
633
|
+
handle = _kernel32.OpenJobObjectW(_JOB_OBJECT_TERMINATE, False, name)
|
|
634
|
+
if not handle:
|
|
635
|
+
return False
|
|
636
|
+
try:
|
|
637
|
+
return bool(_kernel32.TerminateJobObject(handle, 1))
|
|
638
|
+
finally:
|
|
639
|
+
_kernel32.CloseHandle(handle)
|
|
640
|
+
|
|
641
|
+
_WIN_IDENTITY_SIDS = None
|
|
642
|
+
|
|
643
|
+
def _win_sid_to_string(psid) -> str:
|
|
644
|
+
strp = ctypes.c_wchar_p()
|
|
645
|
+
if not _advapi32.ConvertSidToStringSidW(psid, ctypes.byref(strp)):
|
|
646
|
+
raise OSError(f"ConvertSidToStringSid failed: {ctypes.get_last_error()}")
|
|
647
|
+
try:
|
|
648
|
+
return strp.value
|
|
649
|
+
finally:
|
|
650
|
+
_kernel32.LocalFree(ctypes.cast(strp, wintypes.HGLOBAL))
|
|
651
|
+
|
|
652
|
+
def _win_token_sid(token, info_class) -> str:
|
|
653
|
+
size = wintypes.DWORD(0)
|
|
654
|
+
_advapi32.GetTokenInformation(token, info_class, None, 0, ctypes.byref(size))
|
|
655
|
+
buf = (ctypes.c_byte * size.value)()
|
|
656
|
+
if not _advapi32.GetTokenInformation(
|
|
657
|
+
token, info_class, buf, size, ctypes.byref(size)
|
|
658
|
+
):
|
|
659
|
+
raise OSError(f"GetTokenInformation failed: {ctypes.get_last_error()}")
|
|
660
|
+
# TOKEN_USER / TOKEN_OWNER both begin with a PSID at offset 0.
|
|
661
|
+
sid_ptr = ctypes.cast(buf, ctypes.POINTER(ctypes.c_void_p))[0]
|
|
662
|
+
return _win_sid_to_string(ctypes.c_void_p(sid_ptr))
|
|
663
|
+
|
|
664
|
+
def _win_identity_sids() -> frozenset:
|
|
665
|
+
"""The SID strings this process's token creates objects as: the user SID
|
|
666
|
+
and the default-owner SID (they differ for an elevated process). Cached;
|
|
667
|
+
an object owned by any of these is treated as ours."""
|
|
668
|
+
global _WIN_IDENTITY_SIDS
|
|
669
|
+
if _WIN_IDENTITY_SIDS is not None:
|
|
670
|
+
return _WIN_IDENTITY_SIDS
|
|
671
|
+
token = wintypes.HANDLE()
|
|
672
|
+
if not _advapi32.OpenProcessToken(
|
|
673
|
+
_kernel32.GetCurrentProcess(), _TOKEN_QUERY, ctypes.byref(token)
|
|
674
|
+
):
|
|
675
|
+
raise OSError(f"OpenProcessToken failed: {ctypes.get_last_error()}")
|
|
676
|
+
try:
|
|
677
|
+
sids = {
|
|
678
|
+
_win_token_sid(token, _TOKEN_USER_CLASS),
|
|
679
|
+
_win_token_sid(token, _TOKEN_OWNER_CLASS),
|
|
680
|
+
}
|
|
681
|
+
finally:
|
|
682
|
+
_kernel32.CloseHandle(token)
|
|
683
|
+
_WIN_IDENTITY_SIDS = frozenset(s for s in sids if s)
|
|
684
|
+
return _WIN_IDENTITY_SIDS
|
|
685
|
+
|
|
686
|
+
def _win_owner_sid(api, target) -> str:
|
|
687
|
+
"""Shared GetSecurityInfo / GetNamedSecurityInfoW shape: read the OWNER
|
|
688
|
+
SID into a freshly allocated security descriptor and stringify it. The
|
|
689
|
+
SID points INSIDE that descriptor, so freeing the descriptor is the only
|
|
690
|
+
(and required) cleanup -- never free the SID separately."""
|
|
691
|
+
psid = ctypes.c_void_p()
|
|
692
|
+
psd = ctypes.c_void_p()
|
|
693
|
+
err = api(target, _SE_FILE_OBJECT, _OWNER_SECURITY_INFORMATION,
|
|
694
|
+
ctypes.byref(psid), None, None, None, ctypes.byref(psd))
|
|
695
|
+
if err != 0:
|
|
696
|
+
raise OSError(f"{api.__name__} failed: {err}")
|
|
697
|
+
try:
|
|
698
|
+
return _win_sid_to_string(psid)
|
|
699
|
+
finally:
|
|
700
|
+
_kernel32.LocalFree(ctypes.cast(psd, wintypes.HGLOBAL))
|
|
701
|
+
|
|
702
|
+
def _win_owner_sid_from_handle(handle: int) -> str:
|
|
703
|
+
return _win_owner_sid(_advapi32.GetSecurityInfo, wintypes.HANDLE(handle))
|
|
704
|
+
|
|
705
|
+
def _win_owner_sid_from_path(path: str) -> str:
|
|
706
|
+
return _win_owner_sid(_advapi32.GetNamedSecurityInfoW, path)
|
|
707
|
+
|
|
708
|
+
def _win_owns_path(path: str) -> bool:
|
|
709
|
+
return _win_owner_sid_from_path(path) in _win_identity_sids()
|
|
710
|
+
|
|
711
|
+
def _win_owns_handle(handle: int) -> bool:
|
|
712
|
+
return _win_owner_sid_from_handle(handle) in _win_identity_sids()
|
|
713
|
+
|
|
714
|
+
def _win_run_quiet(cmd) -> bool:
|
|
715
|
+
"""Fire-and-forget a Windows tool (icacls/taskkill): output suppressed,
|
|
716
|
+
exit status returned but never raised. check=False suppresses a NONZERO
|
|
717
|
+
exit, NOT a missing executable -- Popen still raises FileNotFoundError
|
|
718
|
+
when the tool is absent from PATH, which would otherwise escape the
|
|
719
|
+
supervisor's teardown and turn an already-classified `done` job into
|
|
720
|
+
`failed`. Returns True only when the tool ran and exited 0."""
|
|
721
|
+
try:
|
|
722
|
+
return subprocess.run(
|
|
723
|
+
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
724
|
+
check=False, creationflags=_WIN_NO_WINDOW).returncode == 0
|
|
725
|
+
except OSError:
|
|
726
|
+
return False
|
|
727
|
+
|
|
728
|
+
def _win_harden_acl(path: str) -> None:
|
|
729
|
+
"""Break inheritance and grant only the current user plus the
|
|
730
|
+
root-equivalents (SYSTEM, Administrators), so no world/Users grant
|
|
731
|
+
survives -- the 0700 analog. Best-effort: the owner check and the
|
|
732
|
+
O_EXCL/O_CREAT claim remain the hard gates if icacls is unavailable."""
|
|
733
|
+
sids = _win_identity_sids()
|
|
734
|
+
if not sids:
|
|
735
|
+
return False
|
|
736
|
+
inherit = "(OI)(CI)" if os.path.isdir(path) else ""
|
|
737
|
+
# Grant EVERY identity SID, not an arbitrary one from the set: an
|
|
738
|
+
# elevated token carries two (user + default owner), and picking one
|
|
739
|
+
# nondeterministically could grant the wrong principal.
|
|
740
|
+
grants = []
|
|
741
|
+
for sid in sorted(sids) + ["S-1-5-18", "S-1-5-32-544"]:
|
|
742
|
+
grants += ["/grant:r", f"*{sid}:{inherit}F"]
|
|
743
|
+
return _win_run_quiet([_win_tool("icacls"), path, "/inheritance:r"] + grants)
|
|
744
|
+
|
|
745
|
+
def _win_pid_alive(pid: int) -> bool:
|
|
746
|
+
handle = _kernel32.OpenProcess(
|
|
747
|
+
_PROCESS_QUERY_LIMITED_INFORMATION | _SYNCHRONIZE, False, pid)
|
|
748
|
+
if not handle:
|
|
749
|
+
return False
|
|
750
|
+
try:
|
|
751
|
+
return _kernel32.WaitForSingleObject(handle, 0) == _WAIT_TIMEOUT
|
|
752
|
+
finally:
|
|
753
|
+
_kernel32.CloseHandle(handle)
|
|
754
|
+
|
|
755
|
+
def _win_kill_tree(
|
|
756
|
+
root_pid: int, grace: float, job_name=None, expected_identity=None,
|
|
757
|
+
) -> bool:
|
|
758
|
+
"""Terminate the worker tree (KTD3). Returns whether the LEADER was
|
|
759
|
+
alive when the kill began -- the reap classification signal -- which is
|
|
760
|
+
independent of how much of the tree we then sweep.
|
|
761
|
+
|
|
762
|
+
The job object is the primary mechanism and the true killpg analog: it
|
|
763
|
+
reaches descendants even after the leader has exited. taskkill /T can
|
|
764
|
+
NOT -- it walks parent->child from a live parent, so against an exited
|
|
765
|
+
pid it returns "process not found" and silently leaves grandchildren
|
|
766
|
+
running forever. That is why the sweep is attempted whenever a job name
|
|
767
|
+
exists, regardless of leader liveness.
|
|
768
|
+
|
|
769
|
+
No graceful phase: a console-less worker cannot receive taskkill's
|
|
770
|
+
WM_CLOSE (it reports "can only be terminated forcefully"), so the grace
|
|
771
|
+
window was pure latency that also widened the cmd_reap race.
|
|
772
|
+
|
|
773
|
+
The job is only reachable by the process that created it: Windows
|
|
774
|
+
releases a named object's NAME once the last handle closes, even while
|
|
775
|
+
member processes keep the object alive (verified: OpenJobObject then
|
|
776
|
+
fails with ERROR_FILE_NOT_FOUND). So a cmd_reap running after the
|
|
777
|
+
supervisor died cannot use it, and falls back to the snapshot walk --
|
|
778
|
+
which is exactly the dead-leader case, hence deepest-first descendants
|
|
779
|
+
BEFORE the leader, and never gated on leader liveness.
|
|
780
|
+
|
|
781
|
+
A live pid at root_pid is not automatically the original leader:
|
|
782
|
+
Windows recycles PIDs, and cmd_reap is often the next python.exe after
|
|
783
|
+
the worker exits (the orphan-grandchild smoke). Never terminate this
|
|
784
|
+
process, and never terminate a live pid whose GetProcessTimes identity
|
|
785
|
+
does not match the recorded worker. Stale-PPID orphans still show the
|
|
786
|
+
dead leader as parent. When the pid was reused, the start-time cutoff
|
|
787
|
+
applies only to *direct* children of that pid (the new process's own
|
|
788
|
+
children vs stale-PPID orphans). A pre-reuse child's full subtree is
|
|
789
|
+
still original-tree work, including descendants spawned after reuse."""
|
|
790
|
+
self_pid = os.getpid()
|
|
791
|
+
is_self = root_pid == self_pid
|
|
792
|
+
alive = (not is_self) and _win_pid_alive(root_pid)
|
|
793
|
+
recorded_leader = alive and _win_process_identity_matches(
|
|
794
|
+
root_pid, expected_identity)
|
|
795
|
+
if job_name:
|
|
796
|
+
_win_terminate_job(job_name)
|
|
797
|
+
# Always Toolhelp-sweep after (or without) the job terminate: children
|
|
798
|
+
# that raced outside the job before AssignProcessToJobObject completed
|
|
799
|
+
# are not members, and TerminateJobObject alone would leave them.
|
|
800
|
+
# CREATE_SUSPENDED closes that spawn race; this remains the belt.
|
|
801
|
+
children_map = _win_process_children_map()
|
|
802
|
+
reuse_cutoff = None
|
|
803
|
+
if not recorded_leader and (is_self or alive):
|
|
804
|
+
reuse_cutoff = _win_process_start_time(root_pid)
|
|
805
|
+
if reuse_cutoff is not None:
|
|
806
|
+
def _predates_reuse(pid):
|
|
807
|
+
started = _win_process_start_time(pid)
|
|
808
|
+
return started is None or started < reuse_cutoff
|
|
809
|
+
kill_set = _pre_reuse_descendant_pids(
|
|
810
|
+
root_pid, children_map, _predates_reuse, self_pid)
|
|
811
|
+
else:
|
|
812
|
+
kill_set = None
|
|
813
|
+
for pid in _win_descendants_deepest_first(root_pid, children_map):
|
|
814
|
+
if pid == self_pid:
|
|
815
|
+
continue
|
|
816
|
+
if kill_set is not None and pid not in kill_set:
|
|
817
|
+
continue
|
|
818
|
+
_win_terminate_pid(pid)
|
|
819
|
+
if recorded_leader:
|
|
820
|
+
_win_terminate_pid(root_pid)
|
|
821
|
+
return recorded_leader
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
# --- hardened I/O primitives --------------------------------------------------
|
|
825
|
+
|
|
826
|
+
def _euid():
|
|
827
|
+
return _EFFECTIVE_UID
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _check_owned_dir(path: str, require_private: bool = False) -> None:
|
|
831
|
+
st = os.lstat(path)
|
|
832
|
+
if not stat.S_ISDIR(st.st_mode):
|
|
833
|
+
raise RunnerError(f"{path}: not a real directory (symlink or file planted?)")
|
|
834
|
+
if IS_WINDOWS:
|
|
835
|
+
# SID ownership stands in for st_uid; the hardened ACL (not a mode bit)
|
|
836
|
+
# provides privacy, so there is no separate require_private gate.
|
|
837
|
+
if not _win_owns_path(path):
|
|
838
|
+
raise RunnerError(f"{path}: not owned by the current user")
|
|
839
|
+
return
|
|
840
|
+
euid = _euid()
|
|
841
|
+
if euid is not None and st.st_uid != euid:
|
|
842
|
+
raise RunnerError(f"{path}: not owned by the current user")
|
|
843
|
+
if require_private:
|
|
844
|
+
mode = stat.S_IMODE(st.st_mode)
|
|
845
|
+
if mode != 0o700:
|
|
846
|
+
raise RunnerError(f"{path}: must have mode 0700, found {mode:04o}")
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def ensure_owned_dirs(base: str, path: str) -> None:
|
|
850
|
+
"""mkdir -p `path` (mode 0700) verifying owner and type on every component
|
|
851
|
+
from `base` down — a planted symlink or foreign dir aborts, never traversed."""
|
|
852
|
+
rel = os.path.relpath(path, base)
|
|
853
|
+
comps = [] if rel == "." else rel.split(os.sep)
|
|
854
|
+
cur = base
|
|
855
|
+
created_base = True
|
|
856
|
+
try:
|
|
857
|
+
os.mkdir(cur, 0o700)
|
|
858
|
+
except FileExistsError:
|
|
859
|
+
created_base = False
|
|
860
|
+
_check_owned_dir(cur)
|
|
861
|
+
if IS_WINDOWS:
|
|
862
|
+
# `icacls /inheritance:r` is destructive and irreversible in a way
|
|
863
|
+
# POSIX's chmod 0700 is not: it permanently drops inherited ACEs. So
|
|
864
|
+
# only re-ACL a root this runner owns -- one we just created, or the
|
|
865
|
+
# managed default (repairing a default left non-private, which is what
|
|
866
|
+
# the POSIX unconditional chmod is for). A pre-existing user-supplied
|
|
867
|
+
# CE_PEER_JOBS_ROOT keeps its ACLs and rests on the owner check.
|
|
868
|
+
default_root = os.path.abspath(DEFAULT_ROOT) if DEFAULT_ROOT else None
|
|
869
|
+
ours = created_base or (
|
|
870
|
+
default_root is not None
|
|
871
|
+
and os.path.normcase(cur) == os.path.normcase(default_root))
|
|
872
|
+
if ours and not _win_harden_acl(cur):
|
|
873
|
+
# Never proceed as if hardened: an unverified root is the one case
|
|
874
|
+
# where the privacy half of the model would silently be missing.
|
|
875
|
+
raise RunnerError(
|
|
876
|
+
f"{cur}: could not harden the jobs-root ACL (icacls failed or "
|
|
877
|
+
"is unavailable); refusing to use a root whose privacy is "
|
|
878
|
+
"unverified"
|
|
879
|
+
)
|
|
880
|
+
else:
|
|
881
|
+
os.chmod(cur, 0o700)
|
|
882
|
+
_check_owned_dir(cur, require_private=True)
|
|
883
|
+
for comp in comps:
|
|
884
|
+
cur = os.path.join(cur, comp)
|
|
885
|
+
created = False
|
|
886
|
+
try:
|
|
887
|
+
os.mkdir(cur, 0o700)
|
|
888
|
+
created = True
|
|
889
|
+
except FileExistsError:
|
|
890
|
+
pass
|
|
891
|
+
if created:
|
|
892
|
+
if IS_WINDOWS:
|
|
893
|
+
_win_harden_acl(cur)
|
|
894
|
+
else:
|
|
895
|
+
os.chmod(cur, 0o700)
|
|
896
|
+
_check_owned_dir(cur)
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def read_owned(path: str, cap: int) -> bytes:
|
|
900
|
+
"""Open no-follow, verify the OPENED descriptor's owner via fstat, enforce
|
|
901
|
+
the size cap, and return content. Raises Unreadable on any trust failure."""
|
|
902
|
+
fd = os.open(path, os.O_RDONLY | O_NOFOLLOW | O_BINARY)
|
|
903
|
+
try:
|
|
904
|
+
st = os.fstat(fd)
|
|
905
|
+
if IS_WINDOWS:
|
|
906
|
+
# Verify the OPENED handle's owner SID (TOCTOU-safe, like the POSIX
|
|
907
|
+
# fstat-by-fd check) before emitting a byte.
|
|
908
|
+
if not _win_owns_handle(msvcrt.get_osfhandle(fd)):
|
|
909
|
+
raise Unreadable(f"{path}: not owned by the current user; refusing to read")
|
|
910
|
+
else:
|
|
911
|
+
euid = _euid()
|
|
912
|
+
if euid is not None and st.st_uid != euid:
|
|
913
|
+
raise Unreadable(f"{path}: not owned by the current user; refusing to read")
|
|
914
|
+
if not stat.S_ISREG(st.st_mode):
|
|
915
|
+
raise Unreadable(f"{path}: not a regular file")
|
|
916
|
+
if st.st_size > cap:
|
|
917
|
+
raise Unreadable(f"{path}: {st.st_size} bytes exceeds the {cap}-byte read cap")
|
|
918
|
+
chunks = []
|
|
919
|
+
got = 0
|
|
920
|
+
while got <= cap:
|
|
921
|
+
chunk = os.read(fd, 65536)
|
|
922
|
+
if not chunk:
|
|
923
|
+
break
|
|
924
|
+
chunks.append(chunk)
|
|
925
|
+
got += len(chunk)
|
|
926
|
+
if got > cap:
|
|
927
|
+
raise Unreadable(f"{path}: grew past the {cap}-byte read cap during read")
|
|
928
|
+
return b"".join(chunks)
|
|
929
|
+
finally:
|
|
930
|
+
os.close(fd)
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def create_exclusive(path: str, data: bytes = b"", mode: int = 0o600) -> None:
|
|
934
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_NOFOLLOW | O_BINARY, mode)
|
|
935
|
+
try:
|
|
936
|
+
if data:
|
|
937
|
+
os.write(fd, data)
|
|
938
|
+
finally:
|
|
939
|
+
os.close(fd)
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def write_atomic(path: str, data: bytes) -> None:
|
|
943
|
+
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".tmp-")
|
|
944
|
+
try:
|
|
945
|
+
with os.fdopen(fd, "wb") as f:
|
|
946
|
+
f.write(data)
|
|
947
|
+
os.replace(tmp, path)
|
|
948
|
+
except BaseException:
|
|
949
|
+
try:
|
|
950
|
+
os.unlink(tmp)
|
|
951
|
+
except OSError:
|
|
952
|
+
pass
|
|
953
|
+
raise
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def write_terminal(job_dir: str, state: str, reason: str, overwrite: bool = True) -> None:
|
|
957
|
+
"""Publish the single terminal record. The reason detail lands FIRST so the
|
|
958
|
+
atomic status rename is always the last record; a reason write failure never
|
|
959
|
+
blocks the status."""
|
|
960
|
+
status_path = os.path.join(job_dir, "status")
|
|
961
|
+
if not overwrite and os.path.lexists(status_path):
|
|
962
|
+
return
|
|
963
|
+
try:
|
|
964
|
+
write_atomic(os.path.join(job_dir, "reason"), (reason.rstrip("\n") + "\n").encode())
|
|
965
|
+
except OSError:
|
|
966
|
+
pass
|
|
967
|
+
write_atomic(status_path, (state + "\n").encode())
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
# --- job identity and resolution ----------------------------------------------
|
|
971
|
+
|
|
972
|
+
def mint_job_id() -> str:
|
|
973
|
+
return f"{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}-{os.urandom(4).hex()}"
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def claim_job_dir(jobs_root: str):
|
|
977
|
+
"""Atomically claim a fresh job dir: os.mkdir (no -p) fails on collision,
|
|
978
|
+
so the id is regenerated rather than a dir ever being shared."""
|
|
979
|
+
for _ in range(CLAIM_ATTEMPTS):
|
|
980
|
+
job_id = mint_job_id()
|
|
981
|
+
job_dir = os.path.join(jobs_root, job_id)
|
|
982
|
+
try:
|
|
983
|
+
os.mkdir(job_dir, 0o700)
|
|
984
|
+
return job_id, job_dir
|
|
985
|
+
except FileExistsError:
|
|
986
|
+
continue
|
|
987
|
+
raise RunnerError(f"could not claim a unique job dir after {CLAIM_ATTEMPTS} attempts")
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def resolve_job_dir(ref: str, skill=None) -> str:
|
|
991
|
+
if os.sep in ref:
|
|
992
|
+
p = os.path.abspath(ref)
|
|
993
|
+
if os.path.isdir(p):
|
|
994
|
+
return p
|
|
995
|
+
raise RunnerError(f"no such job dir: {ref}")
|
|
996
|
+
if not _is_safe_token(ref):
|
|
997
|
+
raise RunnerError(f"invalid job ref: {ref!r}")
|
|
998
|
+
if skill is not None:
|
|
999
|
+
if not _is_safe_token(skill):
|
|
1000
|
+
raise RunnerError(f"invalid skill: {skill!r}")
|
|
1001
|
+
search_roots = candidate_skill_runs_roots(skill)
|
|
1002
|
+
patterns = [os.path.join(root, "*", "jobs", ref) for root in search_roots]
|
|
1003
|
+
else:
|
|
1004
|
+
search_roots = candidate_jobs_root_bases()
|
|
1005
|
+
patterns = [os.path.join(root, "*", "*", "jobs", ref) for root in search_roots]
|
|
1006
|
+
matches = sorted({match for pattern in patterns for match in glob.glob(pattern)})
|
|
1007
|
+
if not matches:
|
|
1008
|
+
raise RunnerError(f"job not found under {', '.join(search_roots)}: {ref}")
|
|
1009
|
+
if len(matches) > 1:
|
|
1010
|
+
raise RunnerError(f"ambiguous job id {ref}: {len(matches)} matches; pass the job dir path")
|
|
1011
|
+
return matches[0]
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def job_state(job_dir: str) -> str:
|
|
1015
|
+
try:
|
|
1016
|
+
_check_owned_dir(job_dir)
|
|
1017
|
+
except (RunnerError, OSError):
|
|
1018
|
+
return "unreadable"
|
|
1019
|
+
try:
|
|
1020
|
+
word = read_owned(os.path.join(job_dir, "status"), STATUS_READ_CAP)
|
|
1021
|
+
word = word.decode("utf-8", "replace").strip()
|
|
1022
|
+
return word if word in TERMINAL_STATES else "unreadable"
|
|
1023
|
+
except FileNotFoundError:
|
|
1024
|
+
pass
|
|
1025
|
+
except (Unreadable, OSError):
|
|
1026
|
+
return "unreadable"
|
|
1027
|
+
if os.path.lexists(os.path.join(job_dir, "pid")):
|
|
1028
|
+
return "running"
|
|
1029
|
+
return "never-started"
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def job_reason(job_dir: str) -> str:
|
|
1033
|
+
"""The terminal record's detail line, or "" when unavailable. Decorative
|
|
1034
|
+
context for a message; never load-bearing, so every failure reads as ""."""
|
|
1035
|
+
try:
|
|
1036
|
+
raw = read_owned(os.path.join(job_dir, "reason"), REASON_READ_CAP)
|
|
1037
|
+
except (Unreadable, OSError):
|
|
1038
|
+
return ""
|
|
1039
|
+
return raw.decode("utf-8", "replace").strip()
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
# --- process-tree control -----------------------------------------------------
|
|
1043
|
+
|
|
1044
|
+
def _pid_alive(pid: int) -> bool:
|
|
1045
|
+
if IS_WINDOWS:
|
|
1046
|
+
return _win_pid_alive(pid)
|
|
1047
|
+
try:
|
|
1048
|
+
os.kill(pid, 0)
|
|
1049
|
+
return True
|
|
1050
|
+
except ProcessLookupError:
|
|
1051
|
+
return False
|
|
1052
|
+
except OSError:
|
|
1053
|
+
return True
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _pid_running(pid: int) -> bool:
|
|
1057
|
+
"""True only for a live process, NOT a <defunct> zombie. os.kill(pid, 0)
|
|
1058
|
+
succeeds for a zombie (the process exited but has not been reaped), which
|
|
1059
|
+
must not count as a live worker when classifying a reap: a zombie leader
|
|
1060
|
+
means the worker is gone (died-without-result), not still running (timeout).
|
|
1061
|
+
Falls back to the kill -0 result when process state is unavailable."""
|
|
1062
|
+
if IS_WINDOWS:
|
|
1063
|
+
# Windows has no <defunct> zombie state -- a terminated process's handle
|
|
1064
|
+
# is signaled and OpenProcess-based liveness already reports it dead.
|
|
1065
|
+
return _win_pid_alive(pid)
|
|
1066
|
+
if not _pid_alive(pid):
|
|
1067
|
+
return False
|
|
1068
|
+
try:
|
|
1069
|
+
out = subprocess.run(
|
|
1070
|
+
["ps", "-o", "state=", "-p", str(pid)],
|
|
1071
|
+
capture_output=True, text=True, check=False,
|
|
1072
|
+
).stdout.strip()
|
|
1073
|
+
except OSError:
|
|
1074
|
+
return True
|
|
1075
|
+
if not out:
|
|
1076
|
+
return False
|
|
1077
|
+
return not out.startswith("Z")
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _kill_quiet(pid: int, sig: int) -> bool:
|
|
1081
|
+
try:
|
|
1082
|
+
os.kill(pid, sig)
|
|
1083
|
+
return True
|
|
1084
|
+
except OSError:
|
|
1085
|
+
return False
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def _killpg_quiet(pgid: int, sig: int) -> bool:
|
|
1089
|
+
try:
|
|
1090
|
+
os.killpg(pgid, sig)
|
|
1091
|
+
return True
|
|
1092
|
+
except OSError:
|
|
1093
|
+
return False
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
def _descendants_deepest_first(root_pid: int):
|
|
1097
|
+
"""Fallback tree enumeration via ps when a process-group kill is not
|
|
1098
|
+
available: children die before their parents can respawn or orphan them."""
|
|
1099
|
+
try:
|
|
1100
|
+
out = subprocess.run(
|
|
1101
|
+
["ps", "-eo", "pid=,ppid="], capture_output=True, text=True, check=False
|
|
1102
|
+
).stdout
|
|
1103
|
+
except OSError:
|
|
1104
|
+
return []
|
|
1105
|
+
children = {}
|
|
1106
|
+
for line in out.splitlines():
|
|
1107
|
+
parts = line.split()
|
|
1108
|
+
if len(parts) != 2:
|
|
1109
|
+
continue
|
|
1110
|
+
try:
|
|
1111
|
+
pid, ppid = int(parts[0]), int(parts[1])
|
|
1112
|
+
except ValueError:
|
|
1113
|
+
continue
|
|
1114
|
+
children.setdefault(ppid, []).append(pid)
|
|
1115
|
+
order, queue = [], [root_pid]
|
|
1116
|
+
while queue:
|
|
1117
|
+
for child in children.get(queue.pop(0), []):
|
|
1118
|
+
order.append(child)
|
|
1119
|
+
queue.append(child)
|
|
1120
|
+
return list(reversed(order))
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def _signal_group_or_tree(pid: int, sig: int) -> None:
|
|
1124
|
+
"""Signal the pid's process group, falling back to a deepest-first tree
|
|
1125
|
+
walk when the group kill is unavailable."""
|
|
1126
|
+
if not _killpg_quiet(pid, sig):
|
|
1127
|
+
for descendant in _descendants_deepest_first(pid):
|
|
1128
|
+
_kill_quiet(descendant, sig)
|
|
1129
|
+
_kill_quiet(pid, sig)
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def _pre_reuse_descendant_pids(root_pid, children, predates_reuse, skip_pid=None):
|
|
1133
|
+
"""Direct children that predate a recycled leader pid, plus each of those
|
|
1134
|
+
children's full subtree.
|
|
1135
|
+
|
|
1136
|
+
Toolhelp still lists the original tree under a dead pid as parent, mixed
|
|
1137
|
+
with the new process's own children. The start-time cutoff applies only to
|
|
1138
|
+
direct children. A pre-reuse child's later descendants stay original-tree
|
|
1139
|
+
work even if they started after the reuse.
|
|
1140
|
+
"""
|
|
1141
|
+
keep = set()
|
|
1142
|
+
queue = []
|
|
1143
|
+
for child in children.get(root_pid, []):
|
|
1144
|
+
if skip_pid is not None and child == skip_pid:
|
|
1145
|
+
continue
|
|
1146
|
+
if not predates_reuse(child):
|
|
1147
|
+
continue
|
|
1148
|
+
queue.append(child)
|
|
1149
|
+
while queue:
|
|
1150
|
+
pid = queue.pop(0)
|
|
1151
|
+
if skip_pid is not None and pid == skip_pid:
|
|
1152
|
+
continue
|
|
1153
|
+
if pid in keep:
|
|
1154
|
+
continue
|
|
1155
|
+
keep.add(pid)
|
|
1156
|
+
queue.extend(children.get(pid, []))
|
|
1157
|
+
return keep
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
def kill_tree(root_pid: int, grace: float, job_name=None, expected_identity=None) -> bool:
|
|
1161
|
+
"""TERM the pid's process group (workers are started as group leaders),
|
|
1162
|
+
falling back to a deepest-first tree walk; grace, then KILL survivors.
|
|
1163
|
+
|
|
1164
|
+
`job_name` is Windows-only (the worker's job object, the pgid analog) and
|
|
1165
|
+
is ignored on POSIX, where the pgid is derived from the pid itself.
|
|
1166
|
+
`expected_identity` is Windows-only (GetProcessTimes identity recorded at
|
|
1167
|
+
start) and is ignored on POSIX."""
|
|
1168
|
+
if IS_WINDOWS:
|
|
1169
|
+
return _win_kill_tree(root_pid, grace, job_name, expected_identity)
|
|
1170
|
+
# Do NOT early-return just because the leader pid is dead: killpg targets
|
|
1171
|
+
# the pgid, which persists while any group member lives even after the
|
|
1172
|
+
# leader exits, so a dead leader can still front a live group we must sweep.
|
|
1173
|
+
# Use _pid_running (zombie-aware), not _pid_alive: a just-exited leader is
|
|
1174
|
+
# briefly a <defunct> zombie for which kill -0 still succeeds, and counting
|
|
1175
|
+
# that as alive would misclassify the reap as timeout instead of
|
|
1176
|
+
# died-without-result (and make the dead-leader sweep test timing-dependent).
|
|
1177
|
+
leader_alive = _pid_running(root_pid)
|
|
1178
|
+
# Snapshot the descendant set BEFORE any KILL: once the group leader is
|
|
1179
|
+
# reaped its children reparent to init and drop out of the tree, so a set
|
|
1180
|
+
# enumerated after the kill would miss them and leak orphans.
|
|
1181
|
+
survivors = _descendants_deepest_first(root_pid)
|
|
1182
|
+
_signal_group_or_tree(root_pid, signal.SIGTERM)
|
|
1183
|
+
deadline = time.monotonic() + grace
|
|
1184
|
+
while time.monotonic() < deadline:
|
|
1185
|
+
if leader_alive and not _pid_alive(root_pid):
|
|
1186
|
+
break
|
|
1187
|
+
time.sleep(0.1)
|
|
1188
|
+
_killpg_quiet(root_pid, signal.SIGKILL)
|
|
1189
|
+
for pid in survivors:
|
|
1190
|
+
_kill_quiet(pid, signal.SIGKILL)
|
|
1191
|
+
_kill_quiet(root_pid, signal.SIGKILL)
|
|
1192
|
+
return leader_alive
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
# --- the supervisor (runs inside the detached session) -------------------------
|
|
1196
|
+
|
|
1197
|
+
def classify_exit(rc: int, result_path, conf: dict):
|
|
1198
|
+
result_size = None
|
|
1199
|
+
if result_path:
|
|
1200
|
+
try:
|
|
1201
|
+
st = os.lstat(result_path)
|
|
1202
|
+
if stat.S_ISREG(st.st_mode) and st.st_size > 0:
|
|
1203
|
+
result_size = st.st_size
|
|
1204
|
+
except OSError:
|
|
1205
|
+
pass
|
|
1206
|
+
if result_size is not None and result_size > conf["result_max"]:
|
|
1207
|
+
return "failed", (
|
|
1208
|
+
f"result exceeded byte cap ({result_size} > {conf['result_max']} bytes)"
|
|
1209
|
+
)
|
|
1210
|
+
if rc == 0:
|
|
1211
|
+
if result_path is None or result_size is not None:
|
|
1212
|
+
return "done", "worker exited 0"
|
|
1213
|
+
return "failed", "worker exited 0 without publishing a non-empty result"
|
|
1214
|
+
if rc < 0:
|
|
1215
|
+
if result_size is not None:
|
|
1216
|
+
return "done", f"worker killed by signal {-rc} after publishing its result"
|
|
1217
|
+
return "died-without-result", (
|
|
1218
|
+
f"worker killed by signal {-rc} with no result evidence"
|
|
1219
|
+
)
|
|
1220
|
+
return "failed", f"worker exited {rc}"
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def classify_exit_with_pending_reap(rc: int, result_path, conf: dict, reap_pending: bool):
|
|
1224
|
+
"""Classify a worker that already exited, optionally under a pending reap.
|
|
1225
|
+
|
|
1226
|
+
When reap is pending (Windows `.reap` or POSIX SIGTERM flag) and the worker
|
|
1227
|
+
was killed by the fallback path, classify_exit would record "failed" for a
|
|
1228
|
+
non-zero kill exit — prefer timeout. When the worker already completed
|
|
1229
|
+
successfully (done + result), keep that: a late reap must not rewrite a
|
|
1230
|
+
finished peer run.
|
|
1231
|
+
"""
|
|
1232
|
+
state, reason = classify_exit(rc, result_path, conf)
|
|
1233
|
+
if reap_pending and state != "done":
|
|
1234
|
+
return "timeout", "reaped on request before completion"
|
|
1235
|
+
return state, reason
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
def _reap_worker(proc, conf: dict, job_name=None) -> None:
|
|
1239
|
+
# Deliberately parallel to kill_tree but driven by proc.poll(): an unreaped
|
|
1240
|
+
# Popen child is a zombie that os.kill(pid, 0) still reports alive, so the
|
|
1241
|
+
# pid-based liveness check would burn the whole grace window.
|
|
1242
|
+
if proc.poll() is not None:
|
|
1243
|
+
return
|
|
1244
|
+
if IS_WINDOWS:
|
|
1245
|
+
_win_kill_tree(proc.pid, conf["grace"], job_name)
|
|
1246
|
+
try:
|
|
1247
|
+
proc.wait(timeout=5)
|
|
1248
|
+
except Exception:
|
|
1249
|
+
pass
|
|
1250
|
+
return
|
|
1251
|
+
_signal_group_or_tree(proc.pid, signal.SIGTERM)
|
|
1252
|
+
deadline = time.monotonic() + conf["grace"]
|
|
1253
|
+
while time.monotonic() < deadline:
|
|
1254
|
+
if proc.poll() is not None:
|
|
1255
|
+
break
|
|
1256
|
+
time.sleep(0.1)
|
|
1257
|
+
if proc.poll() is None:
|
|
1258
|
+
_killpg_quiet(proc.pid, signal.SIGKILL)
|
|
1259
|
+
for pid in _descendants_deepest_first(proc.pid):
|
|
1260
|
+
_kill_quiet(pid, signal.SIGKILL)
|
|
1261
|
+
try:
|
|
1262
|
+
proc.wait(timeout=5)
|
|
1263
|
+
except Exception:
|
|
1264
|
+
pass
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
def _reap_requested(flag: dict, job_dir: str) -> bool:
|
|
1268
|
+
"""POSIX delivers the reap as SIGTERM (sets flag). Windows has no reliable
|
|
1269
|
+
directed-signal path to a detached, console-less supervisor, so cmd_reap
|
|
1270
|
+
drops a `.reap` marker in the job dir and the loop polls for it."""
|
|
1271
|
+
if flag["reap"]:
|
|
1272
|
+
return True
|
|
1273
|
+
if IS_WINDOWS and os.path.lexists(os.path.join(job_dir, ".reap")):
|
|
1274
|
+
return True
|
|
1275
|
+
return False
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def _interruptible_sleep(secs: float, flag: dict, job_dir: str) -> None:
|
|
1279
|
+
end = time.monotonic() + secs
|
|
1280
|
+
while time.monotonic() < end:
|
|
1281
|
+
# On POSIX _reap_requested reduces to flag["reap"] (the IS_WINDOWS
|
|
1282
|
+
# branch never fires), so this is the original signal-driven behavior.
|
|
1283
|
+
if _reap_requested(flag, job_dir):
|
|
1284
|
+
return
|
|
1285
|
+
time.sleep(min(0.1, max(0.01, end - time.monotonic())))
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
def _is_system32_wsl_bash(path: str) -> bool:
|
|
1289
|
+
"""True for Windows System32 WSL launchers, including Sysnative aliases."""
|
|
1290
|
+
if not path:
|
|
1291
|
+
return False
|
|
1292
|
+
base = os.path.basename(path).lower()
|
|
1293
|
+
if base not in ("bash", "bash.exe", "sh", "sh.exe"):
|
|
1294
|
+
return False
|
|
1295
|
+
system_root = os.environ.get("SystemRoot") or r"C:\Windows"
|
|
1296
|
+
windows_root = os.path.abspath(system_root)
|
|
1297
|
+
blocked_parents = {
|
|
1298
|
+
os.path.normcase(os.path.join(windows_root, name))
|
|
1299
|
+
for name in ("System32", "Sysnative")
|
|
1300
|
+
}
|
|
1301
|
+
parent = os.path.normcase(os.path.dirname(os.path.abspath(path)))
|
|
1302
|
+
return parent in blocked_parents
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
def _git_bash_well_known_paths():
|
|
1306
|
+
"""Standard Git for Windows bash.exe locations."""
|
|
1307
|
+
pf64 = os.environ.get("ProgramW6432") or ""
|
|
1308
|
+
pf = os.environ.get("ProgramFiles") or r"C:\Program Files"
|
|
1309
|
+
pf86 = os.environ.get("ProgramFiles(x86)") or r"C:\Program Files (x86)"
|
|
1310
|
+
local = os.environ.get("LOCALAPPDATA") or ""
|
|
1311
|
+
roots = []
|
|
1312
|
+
seen = set()
|
|
1313
|
+
for root in (pf64, pf, pf86):
|
|
1314
|
+
if not root:
|
|
1315
|
+
continue
|
|
1316
|
+
key = os.path.normcase(os.path.abspath(root))
|
|
1317
|
+
if key in seen:
|
|
1318
|
+
continue
|
|
1319
|
+
seen.add(key)
|
|
1320
|
+
roots.append(root)
|
|
1321
|
+
paths = []
|
|
1322
|
+
for root in roots:
|
|
1323
|
+
paths.extend([
|
|
1324
|
+
os.path.join(root, "Git", "bin", "bash.exe"),
|
|
1325
|
+
os.path.join(root, "Git", "usr", "bin", "bash.exe"),
|
|
1326
|
+
])
|
|
1327
|
+
if local:
|
|
1328
|
+
paths.extend([
|
|
1329
|
+
os.path.join(local, "Programs", "Git", "bin", "bash.exe"),
|
|
1330
|
+
os.path.join(local, "Programs", "Git", "usr", "bin", "bash.exe"),
|
|
1331
|
+
])
|
|
1332
|
+
return paths
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
def _windows_path_shell_candidates():
|
|
1336
|
+
"""Every bash/sh on PATH in PATH order (not only shutil.which's first hit)."""
|
|
1337
|
+
path_env = os.environ.get("PATH") or ""
|
|
1338
|
+
names = ("bash.exe", "bash", "sh.exe", "sh")
|
|
1339
|
+
found = []
|
|
1340
|
+
seen = set()
|
|
1341
|
+
for directory in path_env.split(os.pathsep):
|
|
1342
|
+
if not directory:
|
|
1343
|
+
continue
|
|
1344
|
+
for name in names:
|
|
1345
|
+
candidate = os.path.join(directory, name)
|
|
1346
|
+
try:
|
|
1347
|
+
if not os.path.isfile(candidate):
|
|
1348
|
+
continue
|
|
1349
|
+
except OSError:
|
|
1350
|
+
continue
|
|
1351
|
+
key = os.path.normcase(os.path.abspath(candidate))
|
|
1352
|
+
if key in seen:
|
|
1353
|
+
continue
|
|
1354
|
+
seen.add(key)
|
|
1355
|
+
found.append(candidate)
|
|
1356
|
+
return found
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
def _env_assignment_token(token: str, allow_option_like: bool = False) -> bool:
|
|
1360
|
+
"""True for env(1) NAME=value operands (not options or the command)."""
|
|
1361
|
+
if (
|
|
1362
|
+
not token
|
|
1363
|
+
or (token.startswith("-") and not allow_option_like)
|
|
1364
|
+
or "=" not in token
|
|
1365
|
+
):
|
|
1366
|
+
return False
|
|
1367
|
+
return True
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
def _env_option_advance(tok: str) -> int:
|
|
1371
|
+
"""How many argv slots an env(1) option occupies (incl. the option itself).
|
|
1372
|
+
|
|
1373
|
+
GNU env options that take a separate operand: -u/--unset, -C/--chdir.
|
|
1374
|
+
Attached `--name=value` forms are a single slot.
|
|
1375
|
+
Short options may be clustered. No-operand flags (-i/-v and their exact
|
|
1376
|
+
long aliases and signal-handling options) advance one slot; -u/-C consume
|
|
1377
|
+
the rest of the token as an
|
|
1378
|
+
attached operand or the next argv slot. Unsupported options fail closed
|
|
1379
|
+
before worker detach.
|
|
1380
|
+
(#1292 Codex P2)
|
|
1381
|
+
"""
|
|
1382
|
+
if tok in ("-u", "--unset", "-C", "--chdir"):
|
|
1383
|
+
return 2
|
|
1384
|
+
if tok.startswith(("--unset=", "--chdir=")):
|
|
1385
|
+
return 1
|
|
1386
|
+
if tok in ("--ignore-environment", "--debug"):
|
|
1387
|
+
return 1
|
|
1388
|
+
if tok == "--list-signal-handling" or tok in (
|
|
1389
|
+
"--block-signal",
|
|
1390
|
+
"--default-signal",
|
|
1391
|
+
"--ignore-signal",
|
|
1392
|
+
) or tok.startswith((
|
|
1393
|
+
"--block-signal=",
|
|
1394
|
+
"--default-signal=",
|
|
1395
|
+
"--ignore-signal=",
|
|
1396
|
+
)):
|
|
1397
|
+
return 1
|
|
1398
|
+
if tok == "--null":
|
|
1399
|
+
raise RunnerError(
|
|
1400
|
+
"env -0/--null cannot be used with a command by native Windows "
|
|
1401
|
+
"peer workers; remove the null-output option"
|
|
1402
|
+
)
|
|
1403
|
+
if not tok.startswith("-"):
|
|
1404
|
+
return 1
|
|
1405
|
+
if tok.startswith("--"):
|
|
1406
|
+
raise RunnerError(
|
|
1407
|
+
f"unsupported env long option {tok!r} for native Windows peer "
|
|
1408
|
+
"workers; use an exact supported option or pass -- before "
|
|
1409
|
+
"option-like assignments"
|
|
1410
|
+
)
|
|
1411
|
+
|
|
1412
|
+
cluster = tok[1:]
|
|
1413
|
+
for index, option in enumerate(cluster):
|
|
1414
|
+
if option in "iv":
|
|
1415
|
+
continue
|
|
1416
|
+
if option == "0":
|
|
1417
|
+
raise RunnerError(
|
|
1418
|
+
"env -0/--null cannot be used with a command by native "
|
|
1419
|
+
"Windows peer workers; remove the null-output option"
|
|
1420
|
+
)
|
|
1421
|
+
if option == "S":
|
|
1422
|
+
raise RunnerError(
|
|
1423
|
+
"env -S/--split-string is unsupported for native Windows "
|
|
1424
|
+
"peer workers; pass env assignments and the command as "
|
|
1425
|
+
"separate arguments"
|
|
1426
|
+
)
|
|
1427
|
+
if option in "uC":
|
|
1428
|
+
return 1 if index + 1 < len(cluster) else 2
|
|
1429
|
+
raise RunnerError(
|
|
1430
|
+
f"unsupported env short-option cluster {tok!r} for native "
|
|
1431
|
+
"Windows peer workers; pass env options separately"
|
|
1432
|
+
)
|
|
1433
|
+
return 1
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def _env_bash_index(argv):
|
|
1437
|
+
"""Locate the env(1)-launched bash/sh command, for #1268/#1292 rewriting.
|
|
1438
|
+
|
|
1439
|
+
Matches the production cross-model shape `env VAR=… bash script.sh …`
|
|
1440
|
+
(#1268). Operand-taking options (-u/-C and long forms) consume their
|
|
1441
|
+
arguments before the command token is sought (#1292). Split-string forms
|
|
1442
|
+
fail closed because Python shlex does not match Git env.exe semantics.
|
|
1443
|
+
|
|
1444
|
+
Returns (argv_index, None), or (-1, None) when no bash/sh command is
|
|
1445
|
+
present.
|
|
1446
|
+
"""
|
|
1447
|
+
if not argv:
|
|
1448
|
+
return -1, None
|
|
1449
|
+
if os.path.basename(argv[0]).lower() not in ("env", "env.exe"):
|
|
1450
|
+
return -1, None
|
|
1451
|
+
i = 1
|
|
1452
|
+
options_done = False
|
|
1453
|
+
while i < len(argv):
|
|
1454
|
+
tok = argv[i]
|
|
1455
|
+
if tok in ("-", "--") and not options_done:
|
|
1456
|
+
options_done = True
|
|
1457
|
+
i += 1
|
|
1458
|
+
continue
|
|
1459
|
+
if _env_assignment_token(tok, allow_option_like=options_done):
|
|
1460
|
+
options_done = True
|
|
1461
|
+
i += 1
|
|
1462
|
+
continue
|
|
1463
|
+
if not options_done and (
|
|
1464
|
+
tok in ("-S", "--split-string") or tok.startswith(
|
|
1465
|
+
("-S", "--split-string=")
|
|
1466
|
+
)
|
|
1467
|
+
):
|
|
1468
|
+
raise RunnerError(
|
|
1469
|
+
"env -S/--split-string is unsupported for native Windows "
|
|
1470
|
+
"peer workers; pass env assignments and the command as "
|
|
1471
|
+
"separate arguments"
|
|
1472
|
+
)
|
|
1473
|
+
if not options_done and tok.startswith("-"):
|
|
1474
|
+
span = _env_option_advance(tok)
|
|
1475
|
+
if span > 1 and i + 1 >= len(argv):
|
|
1476
|
+
return -1, None
|
|
1477
|
+
i += span
|
|
1478
|
+
continue
|
|
1479
|
+
base = os.path.basename(tok).lower()
|
|
1480
|
+
if base in ("bash", "bash.exe", "sh", "sh.exe"):
|
|
1481
|
+
return i, None
|
|
1482
|
+
return -1, None
|
|
1483
|
+
return -1, None
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def _windows_path_is_absolute(path: str) -> bool:
|
|
1487
|
+
"""True for Windows absolute paths (drive letter or path separator)."""
|
|
1488
|
+
return os.sep in path or (len(path) >= 2 and path[1] == ":")
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
def _prefer_windows_posix_shell(token: str) -> str:
|
|
1492
|
+
"""Absolute non-WSL bash/sh kept; bare names and System32 go through resolve.
|
|
1493
|
+
|
|
1494
|
+
Explicit absolute paths (portable Git, custom installs) must not be
|
|
1495
|
+
substituted by the preferred resolver (#1292 Codex P2). Bare `bash`/`sh`
|
|
1496
|
+
and System32 WSL launchers still use `_resolve_windows_posix_shell()`.
|
|
1497
|
+
"""
|
|
1498
|
+
if _windows_path_is_absolute(token):
|
|
1499
|
+
path = os.path.abspath(token)
|
|
1500
|
+
if not os.path.isfile(path):
|
|
1501
|
+
raise RunnerError(
|
|
1502
|
+
f"peer worker shell does not exist or is not a regular file: {token}"
|
|
1503
|
+
)
|
|
1504
|
+
if _is_system32_wsl_bash(path):
|
|
1505
|
+
return _resolve_windows_posix_shell()
|
|
1506
|
+
return path
|
|
1507
|
+
return _resolve_windows_posix_shell()
|
|
1508
|
+
|
|
1509
|
+
|
|
1510
|
+
def _rewrite_windows_env_bash_argv(argv):
|
|
1511
|
+
"""Rewrite bare bash/sh inside an env-prefixed argv.
|
|
1512
|
+
|
|
1513
|
+
Returns (argv, resolved_shell_or_None). Raises RunnerError when a bash/sh
|
|
1514
|
+
token is present but no usable non-WSL shell can be resolved. Absolute
|
|
1515
|
+
non-WSL bash tokens are kept unchanged (#1292 P2). Split-string options
|
|
1516
|
+
are rejected before detach because their parser semantics are not safely
|
|
1517
|
+
reproduced here (#1292).
|
|
1518
|
+
"""
|
|
1519
|
+
idx, split_prefix = _env_bash_index(argv)
|
|
1520
|
+
if idx < 0:
|
|
1521
|
+
return list(argv), None
|
|
1522
|
+
out = list(argv)
|
|
1523
|
+
assert split_prefix is None
|
|
1524
|
+
shell = _prefer_windows_posix_shell(out[idx])
|
|
1525
|
+
if os.path.normcase(os.path.abspath(out[idx])) != os.path.normcase(shell):
|
|
1526
|
+
out[idx] = shell
|
|
1527
|
+
return out, shell
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
def _resolve_windows_posix_shell() -> str:
|
|
1531
|
+
"""Absolute path to a non-WSL POSIX shell for native Windows peer workers.
|
|
1532
|
+
|
|
1533
|
+
Order: CE_PEER_BASH, CLAUDE_CODE_GIT_BASH_PATH, well-known Git Bash
|
|
1534
|
+
installs, then every PATH bash/sh excluding System32 WSL. Fail closed when
|
|
1535
|
+
nothing usable remains — never select System32\\bash.exe (#1268).
|
|
1536
|
+
"""
|
|
1537
|
+
candidates = []
|
|
1538
|
+
for key in ("CE_PEER_BASH", "CLAUDE_CODE_GIT_BASH_PATH"):
|
|
1539
|
+
val = (os.environ.get(key) or "").strip()
|
|
1540
|
+
if val:
|
|
1541
|
+
candidates.append(val)
|
|
1542
|
+
candidates.extend(_git_bash_well_known_paths())
|
|
1543
|
+
candidates.extend(_windows_path_shell_candidates())
|
|
1544
|
+
|
|
1545
|
+
seen = set()
|
|
1546
|
+
for raw in candidates:
|
|
1547
|
+
path = os.path.abspath(raw)
|
|
1548
|
+
key = os.path.normcase(path)
|
|
1549
|
+
if key in seen:
|
|
1550
|
+
continue
|
|
1551
|
+
seen.add(key)
|
|
1552
|
+
if not os.path.isfile(path):
|
|
1553
|
+
continue
|
|
1554
|
+
if _is_system32_wsl_bash(path):
|
|
1555
|
+
continue
|
|
1556
|
+
return path
|
|
1557
|
+
|
|
1558
|
+
raise RunnerError(
|
|
1559
|
+
"no usable Git Bash (or other non-WSL POSIX shell) for native Windows "
|
|
1560
|
+
"peer workers; install Git for Windows or set CE_PEER_BASH / "
|
|
1561
|
+
"CLAUDE_CODE_GIT_BASH_PATH to an absolute bash.exe path "
|
|
1562
|
+
"(System32\\bash.exe / WSL is not used)"
|
|
1563
|
+
)
|
|
1564
|
+
|
|
1565
|
+
|
|
1566
|
+
def _popen_argv(argv):
|
|
1567
|
+
"""Argv for subprocess.Popen.
|
|
1568
|
+
|
|
1569
|
+
On Windows, CreateProcess does not honor shebang, so a bare *.sh / *.bash
|
|
1570
|
+
worker must be launched through bash/sh. Prefer Git Bash over System32
|
|
1571
|
+
WSL bash (#1268). Bare `bash`/`sh` prefixes (review skills) and bare
|
|
1572
|
+
`bash`/`sh` tokens after `env VAR=…` (cross-model) are rewritten to that
|
|
1573
|
+
absolute path. Explicit absolute non-WSL bash/sh paths are kept (#1292 P2).
|
|
1574
|
+
meta.json still records the caller argv for authorize-dispatch contracts
|
|
1575
|
+
that forbid a shell prefix on ce-work.
|
|
1576
|
+
"""
|
|
1577
|
+
if not IS_WINDOWS or not argv:
|
|
1578
|
+
return list(argv)
|
|
1579
|
+
head = argv[0]
|
|
1580
|
+
base = os.path.basename(head).lower()
|
|
1581
|
+
if base in ("env", "env.exe"):
|
|
1582
|
+
rewritten, _shell = _rewrite_windows_env_bash_argv(argv)
|
|
1583
|
+
return rewritten
|
|
1584
|
+
if base in ("bash", "bash.exe", "sh", "sh.exe"):
|
|
1585
|
+
shell = _prefer_windows_posix_shell(head)
|
|
1586
|
+
if os.path.normcase(os.path.abspath(head)) == os.path.normcase(shell):
|
|
1587
|
+
return list(argv)
|
|
1588
|
+
return [shell] + list(argv[1:])
|
|
1589
|
+
lower = head.lower()
|
|
1590
|
+
if not (lower.endswith(".sh") or lower.endswith(".bash")):
|
|
1591
|
+
return list(argv)
|
|
1592
|
+
shell = _resolve_windows_posix_shell()
|
|
1593
|
+
return [shell, head] + list(argv[1:])
|
|
1594
|
+
|
|
1595
|
+
|
|
1596
|
+
def supervise(job_dir: str, argv, result_path, conf: dict, ack_fd: int) -> None:
|
|
1597
|
+
"""The watchdog around the worker child. Owns liveness (out.log growth),
|
|
1598
|
+
the idle/hard windows, byte caps, reap-on-request, and the single terminal
|
|
1599
|
+
classification."""
|
|
1600
|
+
flag = {"reap": False}
|
|
1601
|
+
|
|
1602
|
+
def on_term(signum, frame):
|
|
1603
|
+
flag["reap"] = True
|
|
1604
|
+
|
|
1605
|
+
if not IS_WINDOWS:
|
|
1606
|
+
signal.signal(signal.SIGTERM, on_term)
|
|
1607
|
+
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
|
1608
|
+
# On Windows there is no SIGHUP and no reliable directed SIGTERM to a
|
|
1609
|
+
# detached supervisor; reap arrives via the `.reap` marker polled below.
|
|
1610
|
+
|
|
1611
|
+
acked = False
|
|
1612
|
+
|
|
1613
|
+
def ack():
|
|
1614
|
+
nonlocal acked
|
|
1615
|
+
if acked:
|
|
1616
|
+
return
|
|
1617
|
+
acked = True
|
|
1618
|
+
# Windows detach has no ack pipe (ack_fd is None): the pid file written
|
|
1619
|
+
# just above IS the ack, and the parent polls for it.
|
|
1620
|
+
if ack_fd is None:
|
|
1621
|
+
return
|
|
1622
|
+
try:
|
|
1623
|
+
os.write(ack_fd, b"ok")
|
|
1624
|
+
os.close(ack_fd)
|
|
1625
|
+
except OSError:
|
|
1626
|
+
pass
|
|
1627
|
+
|
|
1628
|
+
log_fd = None
|
|
1629
|
+
job_name = None
|
|
1630
|
+
job_handle = None
|
|
1631
|
+
try:
|
|
1632
|
+
log_fd = os.open(
|
|
1633
|
+
os.path.join(job_dir, "out.log"),
|
|
1634
|
+
os.O_WRONLY | os.O_APPEND | O_NOFOLLOW | O_BINARY)
|
|
1635
|
+
if IS_WINDOWS:
|
|
1636
|
+
# Created BEFORE the worker so the tree can never start outside it.
|
|
1637
|
+
# The handle is held for the supervisor's lifetime; the job is what
|
|
1638
|
+
# makes teardown reach descendants of an exited leader.
|
|
1639
|
+
job_name = _win_job_name(job_dir)
|
|
1640
|
+
job_handle = _win_create_job(job_name)
|
|
1641
|
+
devnull = os.open(os.devnull, os.O_RDONLY)
|
|
1642
|
+
try:
|
|
1643
|
+
# Export the interpreter running this supervisor so Windows workers
|
|
1644
|
+
# (and any adapter that honors it) do not re-resolve to the Store
|
|
1645
|
+
# python3 stub — see resolve-python convention / #1247.
|
|
1646
|
+
worker_env = {
|
|
1647
|
+
**os.environ,
|
|
1648
|
+
"CE_PEER_JOB_ID": os.path.basename(job_dir),
|
|
1649
|
+
"CE_PEER_PYTHON": sys.executable,
|
|
1650
|
+
}
|
|
1651
|
+
popen_kwargs = dict(
|
|
1652
|
+
stdin=devnull,
|
|
1653
|
+
stdout=log_fd,
|
|
1654
|
+
stderr=log_fd,
|
|
1655
|
+
env=worker_env,
|
|
1656
|
+
close_fds=True,
|
|
1657
|
+
)
|
|
1658
|
+
if IS_WINDOWS:
|
|
1659
|
+
# New process group so the worker's own tree is isolated; reap =
|
|
1660
|
+
# job terminate + Toolhelp walk (there is no killpg on Windows).
|
|
1661
|
+
# CREATE_NO_WINDOW because the supervisor is console-less, so
|
|
1662
|
+
# without it Windows allocates a NEW console per worker and the
|
|
1663
|
+
# user sees a window flash for every job. (It is mutually
|
|
1664
|
+
# exclusive with DETACHED_PROCESS, which is why the supervisor
|
|
1665
|
+
# itself uses DETACHED_PROCESS and only the worker uses this.)
|
|
1666
|
+
# CREATE_SUSPENDED: assign to the Job Object before any worker
|
|
1667
|
+
# code runs, so early child spawns inherit membership.
|
|
1668
|
+
popen_kwargs["creationflags"] = (
|
|
1669
|
+
subprocess.CREATE_NEW_PROCESS_GROUP
|
|
1670
|
+
| _WIN_NO_WINDOW
|
|
1671
|
+
| _CREATE_SUSPENDED)
|
|
1672
|
+
else:
|
|
1673
|
+
popen_kwargs["start_new_session"] = True # worker leads its own group
|
|
1674
|
+
# Wrap bare *.sh on Windows at spawn time only — meta still has the
|
|
1675
|
+
# caller argv (see _popen_argv).
|
|
1676
|
+
proc = subprocess.Popen(_popen_argv(argv), **popen_kwargs)
|
|
1677
|
+
finally:
|
|
1678
|
+
os.close(devnull)
|
|
1679
|
+
pid_doc = {
|
|
1680
|
+
"supervisor_pid": os.getpid(),
|
|
1681
|
+
"worker_pid": proc.pid,
|
|
1682
|
+
}
|
|
1683
|
+
if IS_WINDOWS:
|
|
1684
|
+
sup_ident = _win_process_identity(os.getpid())
|
|
1685
|
+
worker_ident = _win_process_identity(proc.pid)
|
|
1686
|
+
if sup_ident:
|
|
1687
|
+
pid_doc["supervisor_identity"] = sup_ident
|
|
1688
|
+
if worker_ident:
|
|
1689
|
+
pid_doc["worker_identity"] = worker_ident
|
|
1690
|
+
# Assign while still suspended, then resume. Record the job only
|
|
1691
|
+
# once the worker is actually a member, so a later reap never trusts
|
|
1692
|
+
# a name that owns nothing. If assignment fails (job creation denied,
|
|
1693
|
+
# or a nested-job restriction), leave it unset and let teardown fall
|
|
1694
|
+
# back to the Toolhelp walk.
|
|
1695
|
+
if job_handle is not None and _win_assign_to_job(job_handle, proc.pid):
|
|
1696
|
+
pid_doc["job_name"] = job_name
|
|
1697
|
+
else:
|
|
1698
|
+
job_name = None
|
|
1699
|
+
if not _win_resume_process(proc.pid):
|
|
1700
|
+
raise RunnerError(
|
|
1701
|
+
f"could not resume suspended Windows worker pid {proc.pid}"
|
|
1702
|
+
)
|
|
1703
|
+
else:
|
|
1704
|
+
# pgid drives POSIX group kills; Windows reaps by job object.
|
|
1705
|
+
pid_doc["supervisor_pgid"] = os.getpgid(0)
|
|
1706
|
+
# The pid file lands before the parent is acked, so a returned `start`
|
|
1707
|
+
# guarantees the detach marker exists (status never mis-reads a fresh
|
|
1708
|
+
# job as never-started).
|
|
1709
|
+
write_atomic(os.path.join(job_dir, "pid"), (json.dumps(pid_doc) + "\n").encode())
|
|
1710
|
+
except Exception as exc:
|
|
1711
|
+
write_terminal(job_dir, "failed", f"could not launch worker: {exc}")
|
|
1712
|
+
ack()
|
|
1713
|
+
return
|
|
1714
|
+
ack()
|
|
1715
|
+
|
|
1716
|
+
start_t = time.monotonic()
|
|
1717
|
+
last_growth = start_t
|
|
1718
|
+
last_size = 0
|
|
1719
|
+
while True:
|
|
1720
|
+
rc = proc.poll()
|
|
1721
|
+
if rc is not None:
|
|
1722
|
+
state, reason = classify_exit_with_pending_reap(
|
|
1723
|
+
rc, result_path, conf, _reap_requested(flag, job_dir),
|
|
1724
|
+
)
|
|
1725
|
+
break
|
|
1726
|
+
if _reap_requested(flag, job_dir):
|
|
1727
|
+
# Classification is fixed BEFORE the kill: even if the worker
|
|
1728
|
+
# publishes and exits 0 during the grace window, the supervisor's
|
|
1729
|
+
# record wins (R3).
|
|
1730
|
+
_reap_worker(proc, conf, job_name)
|
|
1731
|
+
state, reason = "timeout", "reaped on request before completion"
|
|
1732
|
+
break
|
|
1733
|
+
try:
|
|
1734
|
+
size = os.fstat(log_fd).st_size
|
|
1735
|
+
except OSError:
|
|
1736
|
+
size = last_size
|
|
1737
|
+
now = time.monotonic()
|
|
1738
|
+
if size > last_size:
|
|
1739
|
+
last_size, last_growth = size, now
|
|
1740
|
+
if size > conf["log_max"]:
|
|
1741
|
+
_reap_worker(proc, conf, job_name)
|
|
1742
|
+
state, reason = "failed", (
|
|
1743
|
+
f"out.log exceeded byte cap ({size} > {conf['log_max']} bytes)"
|
|
1744
|
+
)
|
|
1745
|
+
break
|
|
1746
|
+
if conf["idle"] is not None and now - last_growth >= conf["idle"]:
|
|
1747
|
+
_reap_worker(proc, conf, job_name)
|
|
1748
|
+
state, reason = "timeout", f"no output for {conf['idle']:g}s (idle window)"
|
|
1749
|
+
break
|
|
1750
|
+
if now - start_t >= conf["hard"]:
|
|
1751
|
+
_reap_worker(proc, conf, job_name)
|
|
1752
|
+
state, reason = "timeout", f"hard cap {conf['hard']:g}s exceeded"
|
|
1753
|
+
break
|
|
1754
|
+
_interruptible_sleep(conf["poll"], flag, job_dir)
|
|
1755
|
+
# An externally killed worker can leave group members behind (its shell's
|
|
1756
|
+
# children); sweep the group before publishing so no orphan outlives the
|
|
1757
|
+
# terminal record. A pgid cannot be recycled while members remain.
|
|
1758
|
+
if IS_WINDOWS:
|
|
1759
|
+
# Job-object sweep: unlike taskkill this still reaches descendants when
|
|
1760
|
+
# the worker leader has already exited, which is the orphan case the
|
|
1761
|
+
# POSIX killpg pair below covers.
|
|
1762
|
+
_win_kill_tree(proc.pid, min(conf["grace"], 1.0), job_name)
|
|
1763
|
+
else:
|
|
1764
|
+
_killpg_quiet(proc.pid, signal.SIGTERM)
|
|
1765
|
+
_killpg_quiet(proc.pid, signal.SIGKILL)
|
|
1766
|
+
write_terminal(job_dir, state, reason)
|
|
1767
|
+
|
|
1768
|
+
|
|
1769
|
+
def detach_supervisor(job_dir: str, argv, result_path, conf: dict) -> bool:
|
|
1770
|
+
"""setsid double-fork. The grandchild (new session, stdio on /dev/null,
|
|
1771
|
+
reparented to init) runs the supervisor; the parent returns once the
|
|
1772
|
+
supervisor acks that the pid file exists."""
|
|
1773
|
+
if IS_WINDOWS:
|
|
1774
|
+
return detach_supervisor_windows(job_dir, argv, result_path, conf)
|
|
1775
|
+
sys.stdout.flush()
|
|
1776
|
+
sys.stderr.flush()
|
|
1777
|
+
read_fd, write_fd = os.pipe()
|
|
1778
|
+
pid1 = os.fork()
|
|
1779
|
+
if pid1 == 0:
|
|
1780
|
+
os.close(read_fd)
|
|
1781
|
+
os.setsid()
|
|
1782
|
+
if os.fork() > 0:
|
|
1783
|
+
os._exit(0)
|
|
1784
|
+
rc = 0
|
|
1785
|
+
try:
|
|
1786
|
+
devnull = os.open(os.devnull, os.O_RDWR)
|
|
1787
|
+
os.dup2(devnull, 0)
|
|
1788
|
+
os.dup2(devnull, 1)
|
|
1789
|
+
os.dup2(devnull, 2)
|
|
1790
|
+
if devnull > 2:
|
|
1791
|
+
os.close(devnull)
|
|
1792
|
+
supervise(job_dir, argv, result_path, conf, write_fd)
|
|
1793
|
+
except BaseException:
|
|
1794
|
+
rc = 1
|
|
1795
|
+
try:
|
|
1796
|
+
write_terminal(
|
|
1797
|
+
job_dir, "failed", "supervisor crashed before classification",
|
|
1798
|
+
overwrite=False,
|
|
1799
|
+
)
|
|
1800
|
+
except BaseException:
|
|
1801
|
+
pass
|
|
1802
|
+
os._exit(rc)
|
|
1803
|
+
os.close(write_fd)
|
|
1804
|
+
os.waitpid(pid1, 0)
|
|
1805
|
+
ack = b""
|
|
1806
|
+
try:
|
|
1807
|
+
while len(ack) < 2:
|
|
1808
|
+
chunk = os.read(read_fd, 2 - len(ack))
|
|
1809
|
+
if not chunk:
|
|
1810
|
+
break
|
|
1811
|
+
ack += chunk
|
|
1812
|
+
finally:
|
|
1813
|
+
os.close(read_fd)
|
|
1814
|
+
return ack == b"ok"
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def detach_supervisor_windows(job_dir: str, argv, result_path, conf: dict) -> bool:
|
|
1818
|
+
"""Windows detach: there is no fork/setsid, so re-invoke this script as a
|
|
1819
|
+
fresh DETACHED_PROCESS running the internal `__supervise` entrypoint. The
|
|
1820
|
+
spawn spec travels through an owner-private file in the job dir; the parent
|
|
1821
|
+
returns once the supervisor has left its ack marker (the pid file, or a
|
|
1822
|
+
terminal status if the worker could not launch). CREATE_BREAKAWAY_FROM_JOB
|
|
1823
|
+
is the analog of setsid's reparent-to-init: it lets the supervisor outlive a
|
|
1824
|
+
launching harness that runs inside a kill-on-close Job Object, falling back
|
|
1825
|
+
when the job forbids breakaway."""
|
|
1826
|
+
spec = {"argv": list(argv), "result_path": result_path, "conf": conf}
|
|
1827
|
+
create_exclusive(
|
|
1828
|
+
os.path.join(job_dir, ".spawn.json"),
|
|
1829
|
+
(json.dumps(spec) + "\n").encode(),
|
|
1830
|
+
)
|
|
1831
|
+
cmd = [sys.executable, os.path.abspath(__file__), "__supervise", job_dir]
|
|
1832
|
+
base_flags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
|
|
1833
|
+
devnull = os.open(os.devnull, os.O_RDWR)
|
|
1834
|
+
proc = None
|
|
1835
|
+
try:
|
|
1836
|
+
for flags in (base_flags | subprocess.CREATE_BREAKAWAY_FROM_JOB, base_flags):
|
|
1837
|
+
try:
|
|
1838
|
+
proc = subprocess.Popen(
|
|
1839
|
+
cmd,
|
|
1840
|
+
stdin=devnull, stdout=devnull, stderr=devnull,
|
|
1841
|
+
close_fds=True, creationflags=flags,
|
|
1842
|
+
)
|
|
1843
|
+
break
|
|
1844
|
+
except OSError:
|
|
1845
|
+
proc = None
|
|
1846
|
+
if proc is None:
|
|
1847
|
+
return False
|
|
1848
|
+
finally:
|
|
1849
|
+
os.close(devnull)
|
|
1850
|
+
|
|
1851
|
+
pid_path = os.path.join(job_dir, "pid")
|
|
1852
|
+
status_path = os.path.join(job_dir, "status")
|
|
1853
|
+
deadline = time.monotonic() + 15.0
|
|
1854
|
+
while time.monotonic() < deadline:
|
|
1855
|
+
if os.path.lexists(pid_path) or os.path.lexists(status_path):
|
|
1856
|
+
return True
|
|
1857
|
+
if proc.poll() is not None:
|
|
1858
|
+
# Supervisor process exited without leaving a marker: detach failed.
|
|
1859
|
+
return os.path.lexists(pid_path) or os.path.lexists(status_path)
|
|
1860
|
+
time.sleep(0.05)
|
|
1861
|
+
# Deadline with the supervisor still running. Do not abandon it: reporting
|
|
1862
|
+
# a detach failure while leaving a live, unreachable supervisor/worker pair
|
|
1863
|
+
# behind is exactly the orphan this runner exists to prevent. The snapshot
|
|
1864
|
+
# walk reaches the worker as the supervisor's child.
|
|
1865
|
+
try:
|
|
1866
|
+
if proc.poll() is None:
|
|
1867
|
+
_win_kill_tree(proc.pid, 0.0)
|
|
1868
|
+
except Exception:
|
|
1869
|
+
pass
|
|
1870
|
+
return False
|
|
1871
|
+
|
|
1872
|
+
|
|
1873
|
+
def _win_supervise_from_spec(job_dir: str) -> int:
|
|
1874
|
+
"""Internal `__supervise` entrypoint: the detached Windows supervisor. Drops
|
|
1875
|
+
its console-less std handles onto NUL, reads the owner-checked spawn spec,
|
|
1876
|
+
and runs the shared supervisor loop with a file-based (not fd) ack."""
|
|
1877
|
+
rc = 0
|
|
1878
|
+
# Redundant with detach_supervisor_windows, which already binds this
|
|
1879
|
+
# process's stdio to NUL via Popen -- kept deliberately so the entrypoint is
|
|
1880
|
+
# self-contained: a supervisor is long-lived, and any future/manual
|
|
1881
|
+
# invocation that inherited a real pipe could block forever once it filled.
|
|
1882
|
+
try:
|
|
1883
|
+
devnull = os.open(os.devnull, os.O_RDWR)
|
|
1884
|
+
for fd in (0, 1, 2):
|
|
1885
|
+
try:
|
|
1886
|
+
os.dup2(devnull, fd)
|
|
1887
|
+
except OSError:
|
|
1888
|
+
pass
|
|
1889
|
+
if devnull > 2:
|
|
1890
|
+
os.close(devnull)
|
|
1891
|
+
except OSError:
|
|
1892
|
+
pass
|
|
1893
|
+
try:
|
|
1894
|
+
_check_owned_dir(job_dir)
|
|
1895
|
+
spec = json.loads(read_owned(os.path.join(job_dir, ".spawn.json"), META_READ_CAP))
|
|
1896
|
+
argv = spec["argv"]
|
|
1897
|
+
result_path = spec.get("result_path")
|
|
1898
|
+
conf = spec["conf"]
|
|
1899
|
+
try:
|
|
1900
|
+
os.unlink(os.path.join(job_dir, ".spawn.json"))
|
|
1901
|
+
except OSError:
|
|
1902
|
+
pass
|
|
1903
|
+
supervise(job_dir, argv, result_path, conf, None)
|
|
1904
|
+
except BaseException:
|
|
1905
|
+
rc = 1
|
|
1906
|
+
try:
|
|
1907
|
+
write_terminal(
|
|
1908
|
+
job_dir, "failed", "supervisor crashed before classification",
|
|
1909
|
+
overwrite=False,
|
|
1910
|
+
)
|
|
1911
|
+
except BaseException:
|
|
1912
|
+
pass
|
|
1913
|
+
return rc
|
|
1914
|
+
|
|
1915
|
+
|
|
1916
|
+
# --- subcommands ---------------------------------------------------------------
|
|
1917
|
+
|
|
1918
|
+
def sweep_stale_runs(skill_dir: str, keep: str) -> None:
|
|
1919
|
+
"""Best-effort retention (R14): remove sibling run roots older than 24h.
|
|
1920
|
+
Owner-checked via lstat; never raises, never touches the current run."""
|
|
1921
|
+
try:
|
|
1922
|
+
entries = list(os.scandir(skill_dir))
|
|
1923
|
+
except OSError:
|
|
1924
|
+
return
|
|
1925
|
+
now = time.time()
|
|
1926
|
+
euid = _euid()
|
|
1927
|
+
keep_abs = os.path.abspath(keep)
|
|
1928
|
+
for entry in entries:
|
|
1929
|
+
if os.path.abspath(entry.path) == keep_abs:
|
|
1930
|
+
continue
|
|
1931
|
+
try:
|
|
1932
|
+
st = entry.stat(follow_symlinks=False)
|
|
1933
|
+
except OSError:
|
|
1934
|
+
continue
|
|
1935
|
+
if not stat.S_ISDIR(st.st_mode):
|
|
1936
|
+
continue
|
|
1937
|
+
if IS_WINDOWS:
|
|
1938
|
+
try:
|
|
1939
|
+
if not _win_owns_path(entry.path):
|
|
1940
|
+
continue
|
|
1941
|
+
except OSError:
|
|
1942
|
+
continue
|
|
1943
|
+
elif euid is not None and st.st_uid != euid:
|
|
1944
|
+
continue
|
|
1945
|
+
if now - st.st_mtime <= SWEEP_AGE_SECS:
|
|
1946
|
+
continue
|
|
1947
|
+
shutil.rmtree(entry.path, ignore_errors=True)
|
|
1948
|
+
|
|
1949
|
+
|
|
1950
|
+
def _require_detach_support() -> None:
|
|
1951
|
+
"""Detached peer jobs need a supported detach path: os.fork/os.setsid on
|
|
1952
|
+
POSIX, or the native Windows DETACHED_PROCESS path (#1243). Checked first,
|
|
1953
|
+
before jobs_root_base()/geteuid, so an unsupported host fails with this clear
|
|
1954
|
+
message instead of jobs_root_base()'s unrelated "effective user ID is
|
|
1955
|
+
unavailable" error or an AttributeError mid-detach. Native Windows is now
|
|
1956
|
+
supported; only a non-win32 Python missing fork/setsid (some embedded
|
|
1957
|
+
builds) is rejected here."""
|
|
1958
|
+
if IS_WINDOWS:
|
|
1959
|
+
return
|
|
1960
|
+
if not hasattr(os, "fork") or not hasattr(os, "setsid"):
|
|
1961
|
+
raise RunnerError(
|
|
1962
|
+
"detached peer jobs require os.fork/os.setsid on this platform; no "
|
|
1963
|
+
"job was started. Run under a POSIX Python, or on native Windows use "
|
|
1964
|
+
"a Windows Python 3 build (see "
|
|
1965
|
+
"EveryInc/compound-engineering-plugin#1243)."
|
|
1966
|
+
)
|
|
1967
|
+
|
|
1968
|
+
|
|
1969
|
+
def cmd_start(args, worker_argv) -> int:
|
|
1970
|
+
_require_detach_support()
|
|
1971
|
+
for flag, value in (("--skill", args.skill), ("--run-id", args.run_id)):
|
|
1972
|
+
if not _is_safe_token(value):
|
|
1973
|
+
raise RunnerError(f"{flag} must match [A-Za-z0-9._-]+ and not be all dots (got {value!r})")
|
|
1974
|
+
if args.label is not None and not _is_safe_token(args.label):
|
|
1975
|
+
raise RunnerError(f"--label must match [A-Za-z0-9._-]+ and not be all dots (got {args.label!r})")
|
|
1976
|
+
if not worker_argv:
|
|
1977
|
+
raise RunnerError("no worker argv; place it after `--`")
|
|
1978
|
+
|
|
1979
|
+
base = jobs_root_base()
|
|
1980
|
+
skill_dir = skill_runs_root(args.skill)
|
|
1981
|
+
run_dir = os.path.join(skill_dir, args.run_id)
|
|
1982
|
+
jobs_root = os.path.join(run_dir, "jobs")
|
|
1983
|
+
ensure_owned_dirs(skill_dir if skill_dir != os.path.join(base, args.skill) else base, jobs_root)
|
|
1984
|
+
if not args.no_sweep:
|
|
1985
|
+
sweep_stale_runs(skill_dir, keep=run_dir)
|
|
1986
|
+
|
|
1987
|
+
job_id, job_dir = claim_job_dir(jobs_root)
|
|
1988
|
+
result_path = os.path.abspath(args.result_path) if args.result_path else None
|
|
1989
|
+
|
|
1990
|
+
argv0 = worker_argv[0]
|
|
1991
|
+
problem = None
|
|
1992
|
+
windows_posix_shell = None
|
|
1993
|
+
base0 = os.path.basename(argv0).lower()
|
|
1994
|
+
if IS_WINDOWS and base0 in ("bash", "bash.exe", "sh", "sh.exe"):
|
|
1995
|
+
# Prefer Git Bash over PATH/System32 WSL before meta + detach (#1268).
|
|
1996
|
+
# Keep an explicit absolute non-WSL bash (portable Git) (#1292 P2).
|
|
1997
|
+
try:
|
|
1998
|
+
resolved = _prefer_windows_posix_shell(argv0)
|
|
1999
|
+
windows_posix_shell = resolved
|
|
2000
|
+
except RunnerError as exc:
|
|
2001
|
+
problem = str(exc)
|
|
2002
|
+
resolved = argv0
|
|
2003
|
+
elif IS_WINDOWS and base0 in ("env", "env.exe"):
|
|
2004
|
+
# Production cross-model: env VAR=… bash script.sh — rewrite bash
|
|
2005
|
+
# before detach so env cannot PATH-resolve System32 WSL (#1268).
|
|
2006
|
+
if os.sep in argv0 or (len(argv0) >= 2 and argv0[1] == ":"):
|
|
2007
|
+
resolved = os.path.abspath(argv0)
|
|
2008
|
+
if not os.path.isfile(resolved):
|
|
2009
|
+
problem = "does not exist or is not a regular file"
|
|
2010
|
+
else:
|
|
2011
|
+
resolved = shutil.which(argv0)
|
|
2012
|
+
if resolved is None:
|
|
2013
|
+
problem = "was not found on PATH"
|
|
2014
|
+
resolved = argv0
|
|
2015
|
+
try:
|
|
2016
|
+
rewritten, shell = _rewrite_windows_env_bash_argv(list(worker_argv))
|
|
2017
|
+
if shell is not None:
|
|
2018
|
+
windows_posix_shell = shell
|
|
2019
|
+
worker_argv = rewritten
|
|
2020
|
+
except RunnerError as exc:
|
|
2021
|
+
problem = str(exc) if problem is None else f"{problem}; {exc}"
|
|
2022
|
+
elif os.sep in argv0 or (IS_WINDOWS and len(argv0) >= 2 and argv0[1] == ":"):
|
|
2023
|
+
resolved = os.path.abspath(argv0)
|
|
2024
|
+
if not os.path.isfile(resolved):
|
|
2025
|
+
problem = "does not exist or is not a regular file"
|
|
2026
|
+
elif IS_WINDOWS and resolved.lower().endswith((".sh", ".bash")):
|
|
2027
|
+
# CreateProcess cannot run shebang scripts; _popen_argv wraps with
|
|
2028
|
+
# Git Bash. Require that shell now so start fails closed, not after
|
|
2029
|
+
# detach. Skip the X_OK check — Windows often marks .sh non-exec.
|
|
2030
|
+
try:
|
|
2031
|
+
windows_posix_shell = _resolve_windows_posix_shell()
|
|
2032
|
+
except RunnerError as exc:
|
|
2033
|
+
problem = str(exc)
|
|
2034
|
+
elif not os.access(resolved, os.X_OK):
|
|
2035
|
+
problem = "is not executable"
|
|
2036
|
+
else:
|
|
2037
|
+
resolved = shutil.which(argv0)
|
|
2038
|
+
if resolved is None:
|
|
2039
|
+
problem = "was not found on PATH"
|
|
2040
|
+
resolved = argv0
|
|
2041
|
+
elif IS_WINDOWS and resolved.lower().endswith((".sh", ".bash")):
|
|
2042
|
+
try:
|
|
2043
|
+
windows_posix_shell = _resolve_windows_posix_shell()
|
|
2044
|
+
except RunnerError as exc:
|
|
2045
|
+
problem = str(exc)
|
|
2046
|
+
elif IS_WINDOWS and os.path.basename(resolved).lower() in (
|
|
2047
|
+
"bash", "bash.exe", "sh", "sh.exe",
|
|
2048
|
+
):
|
|
2049
|
+
# which() may have returned System32 WSL — rewrite now.
|
|
2050
|
+
try:
|
|
2051
|
+
resolved = _resolve_windows_posix_shell()
|
|
2052
|
+
windows_posix_shell = resolved
|
|
2053
|
+
except RunnerError as exc:
|
|
2054
|
+
problem = str(exc)
|
|
2055
|
+
argv = [resolved] + list(worker_argv[1:])
|
|
2056
|
+
|
|
2057
|
+
conf = cfg(args.skill)
|
|
2058
|
+
meta = {
|
|
2059
|
+
"job_id": job_id,
|
|
2060
|
+
"skill": args.skill,
|
|
2061
|
+
"run_id": args.run_id,
|
|
2062
|
+
"label": args.label,
|
|
2063
|
+
"input_digest": args.input_digest,
|
|
2064
|
+
"started_at": time.strftime(STARTED_AT_FORMAT, time.gmtime()),
|
|
2065
|
+
"worker_argv": argv,
|
|
2066
|
+
"result_path": result_path,
|
|
2067
|
+
"sweep_enabled": not args.no_sweep,
|
|
2068
|
+
"supervision": conf,
|
|
2069
|
+
}
|
|
2070
|
+
if windows_posix_shell:
|
|
2071
|
+
meta["windows_posix_shell"] = windows_posix_shell
|
|
2072
|
+
try:
|
|
2073
|
+
create_exclusive(
|
|
2074
|
+
os.path.join(job_dir, "meta.json"),
|
|
2075
|
+
(json.dumps(meta, indent=2) + "\n").encode(),
|
|
2076
|
+
)
|
|
2077
|
+
except OSError as exc:
|
|
2078
|
+
raise RunnerError(f"cannot write job metadata for {job_id}: {exc}")
|
|
2079
|
+
|
|
2080
|
+
if problem is not None:
|
|
2081
|
+
raise RunnerError(
|
|
2082
|
+
f"preflight failed for job {job_id}: worker {argv0!r} {problem}; "
|
|
2083
|
+
f"nothing was detached (job left never-started at {job_dir})"
|
|
2084
|
+
)
|
|
2085
|
+
try:
|
|
2086
|
+
create_exclusive(os.path.join(job_dir, "out.log"))
|
|
2087
|
+
except OSError as exc:
|
|
2088
|
+
raise RunnerError(
|
|
2089
|
+
f"preflight failed for job {job_id}: job dir not writable ({exc}); "
|
|
2090
|
+
"nothing was detached"
|
|
2091
|
+
)
|
|
2092
|
+
|
|
2093
|
+
if not detach_supervisor(job_dir, argv, result_path, conf):
|
|
2094
|
+
raise RunnerError(
|
|
2095
|
+
f"detach failed for job {job_id}: supervisor did not acknowledge; "
|
|
2096
|
+
f"inspect {job_dir}"
|
|
2097
|
+
)
|
|
2098
|
+
print(job_id)
|
|
2099
|
+
return 0
|
|
2100
|
+
|
|
2101
|
+
|
|
2102
|
+
def _emit_states(rows, as_json: bool) -> None:
|
|
2103
|
+
if as_json:
|
|
2104
|
+
print(json.dumps(
|
|
2105
|
+
[{"ref": r, "job_dir": d, "state": s} for r, d, s in rows]
|
|
2106
|
+
))
|
|
2107
|
+
elif len(rows) == 1:
|
|
2108
|
+
print(rows[0][2])
|
|
2109
|
+
else:
|
|
2110
|
+
for ref, _, state in rows:
|
|
2111
|
+
print(f"{ref}\t{state}")
|
|
2112
|
+
|
|
2113
|
+
|
|
2114
|
+
def cmd_status(args) -> int:
|
|
2115
|
+
rows = []
|
|
2116
|
+
for ref in args.jobs:
|
|
2117
|
+
job_dir = resolve_job_dir(ref, args.skill)
|
|
2118
|
+
rows.append((ref, job_dir, job_state(job_dir)))
|
|
2119
|
+
_emit_states(rows, args.json)
|
|
2120
|
+
return 0
|
|
2121
|
+
|
|
2122
|
+
|
|
2123
|
+
def _finite_secs(raw: str) -> float:
|
|
2124
|
+
try:
|
|
2125
|
+
value = float(raw)
|
|
2126
|
+
except ValueError:
|
|
2127
|
+
raise argparse.ArgumentTypeError(f"not a number of seconds: {raw!r}")
|
|
2128
|
+
if not math.isfinite(value) or value < 0:
|
|
2129
|
+
raise argparse.ArgumentTypeError(f"must be a finite, nonnegative number of seconds: {raw!r}")
|
|
2130
|
+
return value
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
def _job_meta(job_dir: str) -> dict:
|
|
2134
|
+
try:
|
|
2135
|
+
meta = json.loads(read_owned(os.path.join(job_dir, "meta.json"), META_READ_CAP))
|
|
2136
|
+
except (Unreadable, OSError, ValueError):
|
|
2137
|
+
return {}
|
|
2138
|
+
return meta if isinstance(meta, dict) else {}
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def _started_epoch(job_dir: str, meta: dict):
|
|
2142
|
+
try:
|
|
2143
|
+
return calendar.timegm(time.strptime(meta["started_at"], STARTED_AT_FORMAT))
|
|
2144
|
+
except (KeyError, TypeError, ValueError):
|
|
2145
|
+
pass
|
|
2146
|
+
try:
|
|
2147
|
+
return os.lstat(os.path.join(job_dir, "meta.json")).st_mtime
|
|
2148
|
+
except OSError:
|
|
2149
|
+
return None
|
|
2150
|
+
|
|
2151
|
+
|
|
2152
|
+
def _summary_reason(job_dir: str, state: str) -> str:
|
|
2153
|
+
if state == "unreadable":
|
|
2154
|
+
# Never read `reason` from a directory that failed its ownership check.
|
|
2155
|
+
return "state unreadable (ownership or corruption); no result"
|
|
2156
|
+
if state == "never-started":
|
|
2157
|
+
return job_reason(job_dir) or "never started; no result"
|
|
2158
|
+
if state in TERMINAL_STATES and state != "done":
|
|
2159
|
+
return job_reason(job_dir)
|
|
2160
|
+
return ""
|
|
2161
|
+
|
|
2162
|
+
|
|
2163
|
+
def _emit_deadline_summary(rows, metas, deadline_at, as_json: bool) -> None:
|
|
2164
|
+
now = time.time()
|
|
2165
|
+
jobs = []
|
|
2166
|
+
for ref, d, state in rows:
|
|
2167
|
+
meta = metas[ref]
|
|
2168
|
+
started = _started_epoch(d, meta)
|
|
2169
|
+
jobs.append({
|
|
2170
|
+
"ref": ref,
|
|
2171
|
+
"job_dir": d,
|
|
2172
|
+
"label": meta.get("label") or "-",
|
|
2173
|
+
"state": state,
|
|
2174
|
+
"elapsed_secs": int(now - started) if started is not None else None,
|
|
2175
|
+
"reason": _summary_reason(d, state),
|
|
2176
|
+
})
|
|
2177
|
+
running = [j["ref"] for j in jobs if j["state"] == "running"]
|
|
2178
|
+
secs_left = max(0, math.ceil(deadline_at - now))
|
|
2179
|
+
deadline_passed = bool(running) and now >= deadline_at
|
|
2180
|
+
if as_json:
|
|
2181
|
+
print(json.dumps({
|
|
2182
|
+
"jobs": jobs,
|
|
2183
|
+
"all_settled": not running,
|
|
2184
|
+
"deadline_passed": deadline_passed,
|
|
2185
|
+
"secs_left": secs_left,
|
|
2186
|
+
}))
|
|
2187
|
+
return
|
|
2188
|
+
for j in jobs:
|
|
2189
|
+
elapsed = "?" if j["elapsed_secs"] is None else f"{j['elapsed_secs']}s"
|
|
2190
|
+
print(f"{j['label']}\t{j['ref']}\t{j['state']}\t{elapsed}" + (f"\t{j['reason']}" if j["reason"] else ""))
|
|
2191
|
+
if not running:
|
|
2192
|
+
print("all settled")
|
|
2193
|
+
elif deadline_passed:
|
|
2194
|
+
print(
|
|
2195
|
+
f"deadline passed: reap {' '.join(running)}, then wait once more; "
|
|
2196
|
+
"a job still running after that counts as timed out"
|
|
2197
|
+
)
|
|
2198
|
+
else:
|
|
2199
|
+
print(f"waiting: {len(running)} of {len(jobs)} still running; {secs_left}s left before the deadline")
|
|
2200
|
+
|
|
2201
|
+
|
|
2202
|
+
def cmd_wait(args) -> int:
|
|
2203
|
+
dirs = [(ref, resolve_job_dir(ref, args.skill)) for ref in args.jobs]
|
|
2204
|
+
deadline = time.monotonic() + max(0.0, args.max_secs)
|
|
2205
|
+
metas = {}
|
|
2206
|
+
deadline_at = None
|
|
2207
|
+
if args.deadline_secs is not None:
|
|
2208
|
+
metas = {ref: _job_meta(d) for ref, d in dirs}
|
|
2209
|
+
starts = [s for s in (_started_epoch(d, metas[ref]) for ref, d in dirs) if s is not None]
|
|
2210
|
+
# No start time anywhere means no way to measure the window: let it pass
|
|
2211
|
+
# now rather than restart it on every call.
|
|
2212
|
+
window = args.deadline_secs + args.deadline_grace_secs
|
|
2213
|
+
last_start = max(starts) if starts else time.time() - window
|
|
2214
|
+
deadline_at = last_start + window
|
|
2215
|
+
until_deadline = deadline_at - time.time()
|
|
2216
|
+
if until_deadline > 0:
|
|
2217
|
+
deadline = min(deadline, time.monotonic() + until_deadline)
|
|
2218
|
+
else:
|
|
2219
|
+
# Past the deadline, poll briefly so jobs just reaped can record
|
|
2220
|
+
# their terminal state before the summary prints.
|
|
2221
|
+
deadline = min(deadline, time.monotonic() + REAP_SETTLE_SECS)
|
|
2222
|
+
rows = [(ref, d, "running") for ref, d in dirs]
|
|
2223
|
+
while True:
|
|
2224
|
+
# Settled states are final; only still-running jobs get re-read.
|
|
2225
|
+
rows = [
|
|
2226
|
+
(ref, d, state if state != "running" else job_state(d))
|
|
2227
|
+
for ref, d, state in rows
|
|
2228
|
+
]
|
|
2229
|
+
if all(state != "running" for _, _, state in rows):
|
|
2230
|
+
break
|
|
2231
|
+
remaining = deadline - time.monotonic()
|
|
2232
|
+
if remaining <= 0:
|
|
2233
|
+
break
|
|
2234
|
+
time.sleep(min(1.0, remaining))
|
|
2235
|
+
if args.deadline_secs is not None:
|
|
2236
|
+
_emit_deadline_summary(rows, metas, deadline_at, args.json)
|
|
2237
|
+
else:
|
|
2238
|
+
_emit_states(rows, args.json)
|
|
2239
|
+
return 0
|
|
2240
|
+
|
|
2241
|
+
|
|
2242
|
+
def _emit_bytes(data: bytes) -> None:
|
|
2243
|
+
buffer = getattr(sys.stdout, "buffer", None)
|
|
2244
|
+
if buffer is not None:
|
|
2245
|
+
buffer.write(data)
|
|
2246
|
+
buffer.flush()
|
|
2247
|
+
else:
|
|
2248
|
+
sys.stdout.write(data.decode("utf-8", "replace"))
|
|
2249
|
+
|
|
2250
|
+
|
|
2251
|
+
def _report_absent_artifact(target: str, args) -> int:
|
|
2252
|
+
"""An absent --path artifact is an outcome, not a read error: a peer that
|
|
2253
|
+
skipped its gate exits 0 and writes nothing, so the file is legitimately
|
|
2254
|
+
missing on the most common fold-in path. Name that outcome, and when the
|
|
2255
|
+
caller also passed the job id, name the job's state -- otherwise "still
|
|
2256
|
+
running" and "ran, produced nothing" arrive as one errno the caller cannot
|
|
2257
|
+
act on. Each outcome keeps the exit code the job-result contract already
|
|
2258
|
+
assigns it, so a trust or lookup failure never reads as the routine skip:
|
|
2259
|
+
2 running, 4 ownership, 1 unknown job, 3 settled with no artifact."""
|
|
2260
|
+
sys.stderr.write(f"peer-job-runner: no artifact at {target}\n")
|
|
2261
|
+
if not args.job:
|
|
2262
|
+
return 3
|
|
2263
|
+
try:
|
|
2264
|
+
job_dir = resolve_job_dir(args.job, args.skill)
|
|
2265
|
+
except RunnerError as exc:
|
|
2266
|
+
sys.stderr.write(f"peer-job-runner: {exc}\n")
|
|
2267
|
+
return 1
|
|
2268
|
+
state = job_state(job_dir)
|
|
2269
|
+
if state == "unreadable":
|
|
2270
|
+
# Do not read `reason` here: job_dir already failed its owner check, and
|
|
2271
|
+
# O_NOFOLLOW guards only the final component, so a swapped directory
|
|
2272
|
+
# could redirect that read.
|
|
2273
|
+
sys.stderr.write(
|
|
2274
|
+
f"peer-job-runner: job state unreadable (ownership or corruption): {job_dir}\n"
|
|
2275
|
+
)
|
|
2276
|
+
return 4
|
|
2277
|
+
if state == "running":
|
|
2278
|
+
sys.stderr.write(f"peer-job-runner: job {args.job} is still running\n")
|
|
2279
|
+
return 2
|
|
2280
|
+
reason = job_reason(job_dir)
|
|
2281
|
+
sys.stderr.write(
|
|
2282
|
+
f"peer-job-runner: job {args.job}: {state}" + (f" ({reason})" if reason else "") + "\n"
|
|
2283
|
+
)
|
|
2284
|
+
return 3
|
|
2285
|
+
|
|
2286
|
+
|
|
2287
|
+
def cmd_result(args) -> int:
|
|
2288
|
+
if not getattr(args, "path", None) and not args.job:
|
|
2289
|
+
sys.stderr.write("peer-job-runner: result needs a job id or --path FILE\n")
|
|
2290
|
+
return 2
|
|
2291
|
+
if getattr(args, "path", None):
|
|
2292
|
+
# Verified read of an arbitrary artifact: same fd-ownership check and
|
|
2293
|
+
# bounded read as job results. Exists because fold-in filenames can embed
|
|
2294
|
+
# values unknown at start time (so no --result-path was declared), yet the
|
|
2295
|
+
# consumer must never read a predictable /tmp path unchecked.
|
|
2296
|
+
target = os.path.abspath(args.path)
|
|
2297
|
+
try:
|
|
2298
|
+
data = read_owned(target, cfg()["result_max"])
|
|
2299
|
+
except Unreadable as exc:
|
|
2300
|
+
sys.stderr.write(f"peer-job-runner: unreadable: {exc}\n")
|
|
2301
|
+
return 4
|
|
2302
|
+
except FileNotFoundError:
|
|
2303
|
+
return _report_absent_artifact(target, args)
|
|
2304
|
+
except OSError as exc:
|
|
2305
|
+
# Only a genuine ENOENT is "the peer produced nothing". Every other
|
|
2306
|
+
# read failure means the path is there but was refused -- a planted
|
|
2307
|
+
# symlink rejected by O_NOFOLLOW is the case this guard exists for --
|
|
2308
|
+
# so it takes the trust-failure code, never the routine one.
|
|
2309
|
+
sys.stderr.write(f"peer-job-runner: refused to read {target}: {exc}\n")
|
|
2310
|
+
return 4
|
|
2311
|
+
_emit_bytes(data)
|
|
2312
|
+
return 0
|
|
2313
|
+
job_dir = resolve_job_dir(args.job, args.skill)
|
|
2314
|
+
state = job_state(job_dir)
|
|
2315
|
+
if state == "unreadable":
|
|
2316
|
+
sys.stderr.write(
|
|
2317
|
+
f"peer-job-runner: job state unreadable (ownership or corruption): {job_dir}\n"
|
|
2318
|
+
)
|
|
2319
|
+
return 4
|
|
2320
|
+
if state == "running":
|
|
2321
|
+
sys.stderr.write("peer-job-runner: running\n")
|
|
2322
|
+
return 2
|
|
2323
|
+
if state != "done":
|
|
2324
|
+
sys.stderr.write(f"peer-job-runner: {state}\n")
|
|
2325
|
+
return 3
|
|
2326
|
+
conf = cfg()
|
|
2327
|
+
try:
|
|
2328
|
+
meta = json.loads(read_owned(os.path.join(job_dir, "meta.json"), META_READ_CAP))
|
|
2329
|
+
except Unreadable as exc:
|
|
2330
|
+
sys.stderr.write(f"peer-job-runner: unreadable: {exc}\n")
|
|
2331
|
+
return 4
|
|
2332
|
+
except (OSError, ValueError) as exc:
|
|
2333
|
+
sys.stderr.write(f"peer-job-runner: cannot read job metadata: {exc}\n")
|
|
2334
|
+
return 4
|
|
2335
|
+
result_path = meta.get("result_path") if isinstance(meta, dict) else None
|
|
2336
|
+
if not result_path:
|
|
2337
|
+
sys.stderr.write("peer-job-runner: job declared no result path; nothing to emit\n")
|
|
2338
|
+
return 0
|
|
2339
|
+
try:
|
|
2340
|
+
data = read_owned(result_path, conf["result_max"])
|
|
2341
|
+
except Unreadable as exc:
|
|
2342
|
+
sys.stderr.write(f"peer-job-runner: unreadable: {exc}\n")
|
|
2343
|
+
return 4
|
|
2344
|
+
except OSError as exc:
|
|
2345
|
+
sys.stderr.write(f"peer-job-runner: result missing or unreadable: {exc}\n")
|
|
2346
|
+
return 3
|
|
2347
|
+
_emit_bytes(data)
|
|
2348
|
+
return 0
|
|
2349
|
+
|
|
2350
|
+
|
|
2351
|
+
def cmd_reap(args) -> int:
|
|
2352
|
+
codes = set()
|
|
2353
|
+
for ref in args.jobs:
|
|
2354
|
+
try:
|
|
2355
|
+
rc = _reap_one(ref, args.skill)
|
|
2356
|
+
except RunnerError as exc:
|
|
2357
|
+
sys.stderr.write(f"peer-job-runner: {exc}\n")
|
|
2358
|
+
rc = 1
|
|
2359
|
+
if rc:
|
|
2360
|
+
sys.stderr.write(f"peer-job-runner: reap {ref}: exit {rc}\n")
|
|
2361
|
+
codes.add(rc)
|
|
2362
|
+
# Exit codes are categories, so a mixed set reports the generic failure and
|
|
2363
|
+
# leaves the per-job detail on stderr.
|
|
2364
|
+
return codes.pop() if len(codes) == 1 else (1 if codes else 0)
|
|
2365
|
+
|
|
2366
|
+
|
|
2367
|
+
def _reap_one(ref: str, skill) -> int:
|
|
2368
|
+
job_dir = resolve_job_dir(ref, skill)
|
|
2369
|
+
state = job_state(job_dir)
|
|
2370
|
+
if state in TERMINAL_STATES or state == "never-started":
|
|
2371
|
+
return 0
|
|
2372
|
+
if state == "unreadable":
|
|
2373
|
+
sys.stderr.write(
|
|
2374
|
+
f"peer-job-runner: job state unreadable (ownership or corruption): {job_dir}\n"
|
|
2375
|
+
)
|
|
2376
|
+
return 4
|
|
2377
|
+
conf = cfg()
|
|
2378
|
+
pid_doc = None
|
|
2379
|
+
try:
|
|
2380
|
+
pid_doc = json.loads(read_owned(os.path.join(job_dir, "pid"), META_READ_CAP))
|
|
2381
|
+
except (Unreadable, OSError, ValueError):
|
|
2382
|
+
pid_doc = None
|
|
2383
|
+
if not isinstance(pid_doc, dict):
|
|
2384
|
+
pid_doc = {}
|
|
2385
|
+
sup_pid = pid_doc.get("supervisor_pid")
|
|
2386
|
+
sup_pgid = pid_doc.get("supervisor_pgid")
|
|
2387
|
+
worker_pid = pid_doc.get("worker_pid")
|
|
2388
|
+
# Windows-only: the worker tree's job object. Named precisely so this
|
|
2389
|
+
# process -- which never held the supervisor's handle -- can reopen and
|
|
2390
|
+
# terminate the tree even after the worker leader has exited.
|
|
2391
|
+
job_name = pid_doc.get("job_name")
|
|
2392
|
+
worker_identity = pid_doc.get("worker_identity")
|
|
2393
|
+
supervisor_identity = pid_doc.get("supervisor_identity")
|
|
2394
|
+
|
|
2395
|
+
supervisor_ours = (
|
|
2396
|
+
isinstance(sup_pid, int)
|
|
2397
|
+
and not (IS_WINDOWS and sup_pid == os.getpid())
|
|
2398
|
+
and _pid_alive(sup_pid)
|
|
2399
|
+
and (
|
|
2400
|
+
not IS_WINDOWS
|
|
2401
|
+
or _win_process_identity_matches(sup_pid, supervisor_identity)
|
|
2402
|
+
)
|
|
2403
|
+
)
|
|
2404
|
+
if supervisor_ours:
|
|
2405
|
+
# The supervisor owns TERM-grace-KILL and the terminal classification.
|
|
2406
|
+
# POSIX signals it (SIGTERM to the group or pid); Windows drops the
|
|
2407
|
+
# `.reap` marker the supervisor's loop polls for.
|
|
2408
|
+
if IS_WINDOWS:
|
|
2409
|
+
try:
|
|
2410
|
+
with open(os.path.join(job_dir, ".reap"), "w", encoding="utf-8") as f:
|
|
2411
|
+
f.write("reap\n")
|
|
2412
|
+
signaled = True
|
|
2413
|
+
except OSError:
|
|
2414
|
+
signaled = False
|
|
2415
|
+
else:
|
|
2416
|
+
signaled = (isinstance(sup_pgid, int) and _killpg_quiet(sup_pgid, signal.SIGTERM)) \
|
|
2417
|
+
or _kill_quiet(sup_pid, signal.SIGTERM)
|
|
2418
|
+
if signaled:
|
|
2419
|
+
# kill -0 is true for a zombie, so confirm the classification landed
|
|
2420
|
+
# rather than trusting the signal; fall through to self-cleanup if not.
|
|
2421
|
+
# Windows: the supervisor only notices `.reap` on its next poll tick
|
|
2422
|
+
# (default 2s), so min(grace, 1.0) alone is shorter than one poll and
|
|
2423
|
+
# races into the fallback self-classify path.
|
|
2424
|
+
wait_budget = min(conf["grace"], 1.0)
|
|
2425
|
+
if IS_WINDOWS:
|
|
2426
|
+
wait_budget = max(wait_budget, conf["poll"] + 0.25)
|
|
2427
|
+
deadline = time.monotonic() + wait_budget
|
|
2428
|
+
while time.monotonic() < deadline:
|
|
2429
|
+
if job_state(job_dir) in TERMINAL_STATES:
|
|
2430
|
+
return 0
|
|
2431
|
+
time.sleep(0.05)
|
|
2432
|
+
|
|
2433
|
+
# Supervisor gone: perform the tree kill and classification ourselves,
|
|
2434
|
+
# with a short grace so reap still returns quickly. Sweep whenever we have a
|
|
2435
|
+
# worker pid, NOT only when its leader is still alive: a child can survive in
|
|
2436
|
+
# the worker's process group after the leader exits, and kill_tree targets
|
|
2437
|
+
# the pgid precisely so that orphan is swept instead of leaked. Guarding this
|
|
2438
|
+
# on _pid_alive would re-defeat kill_tree's dead-leader-safe path. kill_tree
|
|
2439
|
+
# returns whether the leader was alive, which is the reap classification.
|
|
2440
|
+
worker_leader_alive = False
|
|
2441
|
+
if isinstance(worker_pid, int):
|
|
2442
|
+
worker_leader_alive = kill_tree(
|
|
2443
|
+
worker_pid, min(conf["grace"], 1.0), job_name, worker_identity)
|
|
2444
|
+
# A worker can publish its declared result and exit before this fallback runs
|
|
2445
|
+
# (e.g. the supervisor died mid-run, then the worker completed cleanly). Honor
|
|
2446
|
+
# that result instead of discarding it as died-without-result: read the
|
|
2447
|
+
# declared result_path and classify from the artifact, mirroring
|
|
2448
|
+
# classify_exit. Only with no usable result do we fall back to timeout (leader
|
|
2449
|
+
# was alive) / died-without-result (leader gone).
|
|
2450
|
+
result_path = _job_meta(job_dir).get("result_path")
|
|
2451
|
+
result_size = None
|
|
2452
|
+
if result_path:
|
|
2453
|
+
try:
|
|
2454
|
+
st = os.lstat(result_path)
|
|
2455
|
+
if stat.S_ISREG(st.st_mode) and st.st_size > 0:
|
|
2456
|
+
result_size = st.st_size
|
|
2457
|
+
except OSError:
|
|
2458
|
+
pass
|
|
2459
|
+
if result_size is not None and result_size > conf["result_max"]:
|
|
2460
|
+
word, reason = "failed", (
|
|
2461
|
+
f"result exceeded byte cap ({result_size} > {conf['result_max']} bytes)"
|
|
2462
|
+
)
|
|
2463
|
+
elif result_size is not None:
|
|
2464
|
+
word, reason = "done", "worker published its result before reap (supervisor was gone)"
|
|
2465
|
+
elif worker_leader_alive:
|
|
2466
|
+
word, reason = "timeout", (
|
|
2467
|
+
"reaped by request; supervisor was gone, worker tree killed by reap"
|
|
2468
|
+
)
|
|
2469
|
+
else:
|
|
2470
|
+
word, reason = "died-without-result", (
|
|
2471
|
+
"supervisor and worker both gone without a terminal record"
|
|
2472
|
+
)
|
|
2473
|
+
write_terminal(job_dir, word, reason, overwrite=False)
|
|
2474
|
+
return 0
|
|
2475
|
+
|
|
2476
|
+
|
|
2477
|
+
# --- CLI -----------------------------------------------------------------------
|
|
2478
|
+
|
|
2479
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
2480
|
+
parser = argparse.ArgumentParser(
|
|
2481
|
+
prog="peer-job-runner.py",
|
|
2482
|
+
description=(
|
|
2483
|
+
"Detached, supervised job lifecycle for delegated peer work: "
|
|
2484
|
+
"no call here ever spans the worker's runtime."
|
|
2485
|
+
),
|
|
2486
|
+
epilog=EXIT_CODES_DOC,
|
|
2487
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
2488
|
+
)
|
|
2489
|
+
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
2490
|
+
|
|
2491
|
+
p_start = sub.add_parser(
|
|
2492
|
+
"start",
|
|
2493
|
+
help="claim a job, preflight, detach the worker, print the job id",
|
|
2494
|
+
)
|
|
2495
|
+
p_start.add_argument("--skill", required=True)
|
|
2496
|
+
p_start.add_argument("--run-id", required=True, dest="run_id")
|
|
2497
|
+
p_start.add_argument("--label", default=None)
|
|
2498
|
+
p_start.add_argument("--input-digest", default=None, dest="input_digest")
|
|
2499
|
+
p_start.add_argument(
|
|
2500
|
+
"--result-path", default=None, dest="result_path",
|
|
2501
|
+
help="worker's expected result file; done then requires it non-empty",
|
|
2502
|
+
)
|
|
2503
|
+
p_start.add_argument(
|
|
2504
|
+
"--no-sweep", action="store_true",
|
|
2505
|
+
help="retain old sibling run roots (ce-work durable recovery)",
|
|
2506
|
+
)
|
|
2507
|
+
|
|
2508
|
+
p_status = sub.add_parser("status", help="print each job's state word")
|
|
2509
|
+
p_status.add_argument("--skill", default=None, help="limit job-id lookup to this skill")
|
|
2510
|
+
p_status.add_argument("--json", action="store_true")
|
|
2511
|
+
p_status.add_argument("jobs", nargs="+", help="job ids or job dir paths")
|
|
2512
|
+
|
|
2513
|
+
p_wait = sub.add_parser(
|
|
2514
|
+
"wait", help="bounded poll until all watched jobs settle (or the cap)"
|
|
2515
|
+
)
|
|
2516
|
+
p_wait.add_argument("--skill", default=None, help="limit job-id lookup to this skill")
|
|
2517
|
+
p_wait.add_argument("--max-secs", type=float, default=30.0, dest="max_secs")
|
|
2518
|
+
p_wait.add_argument(
|
|
2519
|
+
"--deadline-secs", type=_finite_secs, default=None, dest="deadline_secs",
|
|
2520
|
+
help=(
|
|
2521
|
+
"aggregate deadline, counted from the last job's start; prints each "
|
|
2522
|
+
"job's label, state and elapsed time, then one line: 'all settled', "
|
|
2523
|
+
"'waiting: ...', or 'deadline passed: reap ...'"
|
|
2524
|
+
),
|
|
2525
|
+
)
|
|
2526
|
+
p_wait.add_argument(
|
|
2527
|
+
"--deadline-grace-secs", type=_finite_secs, default=0.0, dest="deadline_grace_secs",
|
|
2528
|
+
help="added to --deadline-secs, so a caller can pass a worker cap unchanged",
|
|
2529
|
+
)
|
|
2530
|
+
p_wait.add_argument("--json", action="store_true")
|
|
2531
|
+
p_wait.add_argument("jobs", nargs="+", help="job ids or job dir paths")
|
|
2532
|
+
|
|
2533
|
+
p_result = sub.add_parser(
|
|
2534
|
+
"result",
|
|
2535
|
+
help="emit a done job's artifact (exit: 0 done, 2 running, 3 other, 4 unreadable)",
|
|
2536
|
+
)
|
|
2537
|
+
p_result.add_argument("--skill", default=None, help="limit job-id lookup to this skill")
|
|
2538
|
+
p_result.add_argument("job", nargs="?", default=None)
|
|
2539
|
+
p_result.add_argument(
|
|
2540
|
+
"--path",
|
|
2541
|
+
default=None,
|
|
2542
|
+
help=(
|
|
2543
|
+
"ownership-checked bounded read of this file instead of a job's "
|
|
2544
|
+
"declared result; pass the job id too so an absent file reports "
|
|
2545
|
+
"that job's state"
|
|
2546
|
+
),
|
|
2547
|
+
)
|
|
2548
|
+
|
|
2549
|
+
p_reap = sub.add_parser(
|
|
2550
|
+
"reap", help="terminate running jobs now; no-op for any already terminal"
|
|
2551
|
+
)
|
|
2552
|
+
p_reap.add_argument("--skill", default=None, help="limit job-id lookup to this skill")
|
|
2553
|
+
p_reap.add_argument("jobs", nargs="+", help="job ids or job dir paths")
|
|
2554
|
+
return parser
|
|
2555
|
+
|
|
2556
|
+
|
|
2557
|
+
def main(argv) -> int:
|
|
2558
|
+
# Internal Windows detach re-invocation (not a user-facing subcommand): the
|
|
2559
|
+
# detached supervisor process runs `__supervise <job_dir>`. Gated on
|
|
2560
|
+
# IS_WINDOWS so POSIX keeps its previous behavior exactly (argparse usage
|
|
2561
|
+
# error), and so a non-win32 Python without geteuid -- where the ownership
|
|
2562
|
+
# checks degrade -- can never be steered into exec'ing argv from a planted
|
|
2563
|
+
# .spawn.json. Only the Windows detach path ever emits this argv.
|
|
2564
|
+
if IS_WINDOWS and argv and argv[0] == "__supervise":
|
|
2565
|
+
if len(argv) < 2:
|
|
2566
|
+
return 2
|
|
2567
|
+
return _win_supervise_from_spec(argv[1])
|
|
2568
|
+
worker_argv = []
|
|
2569
|
+
if "--" in argv:
|
|
2570
|
+
split = argv.index("--")
|
|
2571
|
+
argv, worker_argv = argv[:split], argv[split + 1:]
|
|
2572
|
+
args = build_parser().parse_args(argv)
|
|
2573
|
+
try:
|
|
2574
|
+
if args.cmd == "start":
|
|
2575
|
+
return cmd_start(args, worker_argv)
|
|
2576
|
+
if args.cmd == "status":
|
|
2577
|
+
return cmd_status(args)
|
|
2578
|
+
if args.cmd == "wait":
|
|
2579
|
+
return cmd_wait(args)
|
|
2580
|
+
if args.cmd == "result":
|
|
2581
|
+
return cmd_result(args)
|
|
2582
|
+
if args.cmd == "reap":
|
|
2583
|
+
return cmd_reap(args)
|
|
2584
|
+
return 2
|
|
2585
|
+
except RunnerError as exc:
|
|
2586
|
+
sys.stderr.write(f"peer-job-runner: {exc}\n")
|
|
2587
|
+
return 1
|
|
2588
|
+
except Unreadable as exc:
|
|
2589
|
+
sys.stderr.write(f"peer-job-runner: unreadable: {exc}\n")
|
|
2590
|
+
return 4
|
|
2591
|
+
|
|
2592
|
+
|
|
2593
|
+
if __name__ == "__main__":
|
|
2594
|
+
sys.exit(main(sys.argv[1:]))
|