okstra 0.229.0 → 0.230.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -5
- package/dist/cli-registry.mjs +0 -18
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/lifecycle/doctor.mjs +0 -15
- package/dist/commands/lifecycle/doctor.mjs.map +1 -1
- package/dist/lib/skill-catalog.mjs +4 -4
- package/dist/lib/skill-catalog.mjs.map +1 -1
- package/docs/architecture/storage-model.md +105 -16
- package/docs/architecture.md +13 -14
- package/docs/contributor-change-matrix.md +5 -5
- package/docs/project-structure-overview.md +11 -18
- package/package.json +1 -2
- package/runtime/BUILD.json +2 -2
- package/runtime/bin/lib/okstra/interactive.sh +2 -1
- package/runtime/prompts/launch.template.md +5 -5
- package/runtime/prompts/lead/convergence.md +4 -4
- package/runtime/prompts/lead/okstra-lead-contract.md +14 -13
- package/runtime/prompts/lead/phase-routing.md +1 -1
- package/runtime/prompts/lead/report-writer.md +3 -3
- package/runtime/prompts/lead/team-contract.md +3 -3
- package/runtime/python/okstra_ctl/agent/invocation.py +142 -5
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +31 -1
- package/runtime/python/okstra_ctl/cmux.py +4 -2
- package/runtime/python/okstra_ctl/code_review_census.py +194 -0
- package/runtime/python/okstra_ctl/code_review_coverage.py +3 -1
- package/runtime/python/okstra_ctl/code_review_scope.py +9 -6
- package/runtime/python/okstra_ctl/code_review_target.py +38 -2
- package/runtime/python/okstra_ctl/conformance.py +30 -0
- package/runtime/python/okstra_ctl/conformance_evidence.py +235 -0
- package/runtime/python/okstra_ctl/correction_round.py +19 -13
- package/runtime/python/okstra_ctl/dispatch_core.py +1 -1
- package/runtime/python/okstra_ctl/dispatch_state.py +1 -0
- package/runtime/python/okstra_ctl/generated_input.py +21 -0
- package/runtime/python/okstra_ctl/ids.py +1 -41
- package/runtime/python/okstra_ctl/initial_prompt_materialization.py +30 -0
- package/runtime/python/okstra_ctl/interactive_cli.py +5 -0
- package/runtime/python/okstra_ctl/locks.py +28 -1
- package/runtime/python/okstra_ctl/model_io/renderers.py +7 -0
- package/runtime/python/okstra_ctl/paths.py +75 -33
- package/runtime/python/okstra_ctl/phases/catalog.py +3 -0
- package/runtime/python/okstra_ctl/phases/change_impact_analysis/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/discussion/spec.md +63 -63
- package/runtime/python/okstra_ctl/phases/discussion/wizard.py +2 -11
- package/runtime/python/okstra_ctl/phases/error_analysis/report_assets/error-analysis-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/error_analysis/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/feature_analysis/spec.md +1 -1
- package/runtime/python/okstra_ctl/phases/final_verification/profile.md +4 -4
- package/runtime/python/okstra_ctl/phases/final_verification/report_assets/final-verification-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/final_verification/spec.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation/host-rules.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +2 -2
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-executor.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation/spec.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation_option_selection/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/implementation_planning/instructions/plan-body-verification.md +11 -11
- package/runtime/python/okstra_ctl/phases/implementation_planning/plan_body.py +14 -0
- package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation_planning/report_assets/implementation-planning-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation_planning/spec.md +1 -1
- package/runtime/python/okstra_ctl/phases/project_analysis/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/release_handoff/entry.py +7 -3
- package/runtime/python/okstra_ctl/phases/release_handoff/report_assets/release-handoff-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/release_handoff/spec.md +4 -4
- package/runtime/python/okstra_ctl/phases/technical_verification/spec.md +1 -1
- package/runtime/python/okstra_ctl/process_group.py +26 -3
- package/runtime/python/okstra_ctl/qa_commands.py +5 -1
- package/runtime/python/okstra_ctl/recap.py +1 -2
- package/runtime/python/okstra_ctl/render.py +10 -2
- package/runtime/python/okstra_ctl/report_finalize.py +3 -3
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +5 -0
- package/runtime/python/okstra_ctl/run.py +24 -20
- package/runtime/python/okstra_ctl/stage_command_commits.py +143 -0
- package/runtime/python/okstra_ctl/verification_capture.py +291 -0
- package/runtime/python/okstra_ctl/verification_conformance.py +28 -9
- package/runtime/python/okstra_ctl/verification_dispatch.py +130 -0
- package/runtime/python/okstra_ctl/verification_evidence.py +652 -0
- package/runtime/python/okstra_ctl/verification_generation.py +22 -12
- package/runtime/python/okstra_ctl/verification_input_ref.py +9 -0
- package/runtime/python/okstra_ctl/verification_input_types.py +2 -0
- package/runtime/python/okstra_ctl/verification_inputs.py +73 -2
- package/runtime/python/okstra_ctl/verification_invocation.py +139 -0
- package/runtime/python/okstra_ctl/verification_target.py +10 -3
- package/runtime/python/okstra_ctl/wizard/registry.py +2 -1
- package/runtime/python/okstra_ctl/wizard/render.py +2 -1
- package/runtime/python/okstra_ctl/wizard/sources.py +14 -3
- package/runtime/python/okstra_ctl/wizard/steps_identity.py +2 -1
- package/runtime/python/okstra_ctl/worker_runner.py +1 -1
- package/runtime/python/okstra_ctl/worktree_lock_cli.py +70 -17
- package/runtime/python/okstra_ctl/worktree_registry.py +4 -0
- package/runtime/skills/okstra-code-review/SKILL.md +7 -5
- package/runtime/skills/okstra-code-review/references/census-rules.md +2 -2
- package/runtime/skills/okstra-explain-flow/SKILL.md +1 -1
- package/runtime/skills/okstra-run/SKILL.md +6 -6
- package/runtime/skills/okstra-schedule-gen/SKILL.md +3 -3
- package/runtime/skills/okstra-setup/SKILL.md +2 -3
- package/runtime/validators/validate-run.py +110 -19
- package/docs/cli.md +0 -1060
- package/docs/container.md +0 -100
- package/docs/follow-ups/2026-07-10-final-report-option-3.md +0 -51
- package/docs/performance-improvement-plan-v2.md +0 -369
- package/runtime/python/okstra_ctl/container.py +0 -702
- package/runtime/python/okstra_ctl/rollup.py +0 -241
- package/runtime/python/okstra_ctl/usage_report.py +0 -362
- package/runtime/skills/okstra-container-build/SKILL.md +0 -139
- package/runtime/skills/okstra-inspect/SKILL.md +0 -100
- package/runtime/skills/okstra-inspect/facets/cost.md +0 -84
- package/runtime/skills/okstra-inspect/facets/error-zip.md +0 -41
- package/runtime/skills/okstra-inspect/facets/errors.md +0 -69
- package/runtime/skills/okstra-inspect/facets/history.md +0 -86
- package/runtime/skills/okstra-inspect/facets/logs.md +0 -89
- package/runtime/skills/okstra-inspect/facets/recap.md +0 -167
- package/runtime/skills/okstra-inspect/facets/report.md +0 -57
- package/runtime/skills/okstra-inspect/facets/run-audit.md +0 -34
- package/runtime/skills/okstra-inspect/facets/status.md +0 -163
- package/runtime/skills/okstra-inspect/facets/time.md +0 -57
- package/runtime/skills/okstra-rollup/SKILL.md +0 -75
- package/runtime/skills/okstra-usage/SKILL.md +0 -71
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Reject a stage-gating validation command naming a commit the stage's worktree lacks.
|
|
2
|
+
|
|
3
|
+
A stage's worktree holds the implementation anchor plus the work of the stages in its
|
|
4
|
+
`dependsOn` closure, nothing else. A command that compares against another branch's
|
|
5
|
+
head fails there before any edit, so the stage can never pass it.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from functools import cache
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Mapping
|
|
13
|
+
|
|
14
|
+
from .incremental_scope import parse_stage_graph
|
|
15
|
+
from .stage_targets import order_stage_closure
|
|
16
|
+
from .worktree import _resolve_commit_sha, is_ancestor
|
|
17
|
+
|
|
18
|
+
# Any 7-40 character hex word may be an abbreviated commit, digits-only included;
|
|
19
|
+
# git resolution decides which ones name a commit.
|
|
20
|
+
_COMMIT_TOKEN = re.compile(r"(?<![0-9A-Za-z])[0-9a-f]{7,40}(?![0-9A-Za-z])")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def stage_command_commit_errors(
|
|
24
|
+
planning: Mapping[str, Any],
|
|
25
|
+
*,
|
|
26
|
+
repo_dir: Path,
|
|
27
|
+
anchor_base: str,
|
|
28
|
+
done_heads: Mapping[int, str],
|
|
29
|
+
) -> list[str]:
|
|
30
|
+
stage_map = planning.get("stageMap") or ()
|
|
31
|
+
if not isinstance(stage_map, list) or not all(
|
|
32
|
+
isinstance(row, Mapping) for row in stage_map
|
|
33
|
+
):
|
|
34
|
+
return ["stageMap must be a list of stage rows to check stage command commits"]
|
|
35
|
+
try:
|
|
36
|
+
graph = parse_stage_graph({"implementationPlanning": planning})
|
|
37
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
38
|
+
return [f"stageMap cannot be read to check stage command commits: {exc!r}"]
|
|
39
|
+
resolve = cache(lambda ref: _resolve_commit_sha(repo_dir, ref))
|
|
40
|
+
contains = cache(lambda tip, commit: is_ancestor(repo_dir, commit, tip))
|
|
41
|
+
failures: list[str] = []
|
|
42
|
+
for row in planning.get("validationChecklist") or ():
|
|
43
|
+
if not isinstance(row, Mapping):
|
|
44
|
+
continue
|
|
45
|
+
stages = [
|
|
46
|
+
stage for stage in row.get("stageRefs") or ()
|
|
47
|
+
if isinstance(stage, int) and not isinstance(stage, bool)
|
|
48
|
+
and stage not in done_heads
|
|
49
|
+
]
|
|
50
|
+
if not stages:
|
|
51
|
+
continue
|
|
52
|
+
command = str(row.get("commandOrObservation") or "")
|
|
53
|
+
commits = {
|
|
54
|
+
token: commit
|
|
55
|
+
for token in dict.fromkeys(_COMMIT_TOKEN.findall(command))
|
|
56
|
+
if (commit := resolve(token))
|
|
57
|
+
}
|
|
58
|
+
if not commits:
|
|
59
|
+
continue
|
|
60
|
+
for stage in stages:
|
|
61
|
+
tips = _worktree_tips(stage, graph, anchor_base, done_heads)
|
|
62
|
+
if not tips:
|
|
63
|
+
continue
|
|
64
|
+
unresolved = [tip for tip in tips if not resolve(tip)]
|
|
65
|
+
if unresolved:
|
|
66
|
+
failures.append(
|
|
67
|
+
f"validationChecklist {row.get('id')} gates stage {stage}, whose "
|
|
68
|
+
f"worktree base commit(s) {unresolved} do not resolve in this "
|
|
69
|
+
"repository; reconcile the implementation anchor or the consumers "
|
|
70
|
+
"ledger (`okstra git-reconcile`) before judging the command"
|
|
71
|
+
)
|
|
72
|
+
continue
|
|
73
|
+
missing = [
|
|
74
|
+
token for token, commit in commits.items()
|
|
75
|
+
if not any(contains(tip, commit) for tip in tips)
|
|
76
|
+
]
|
|
77
|
+
if missing:
|
|
78
|
+
failures.append(
|
|
79
|
+
f"validationChecklist {row.get('id')} gates stage {stage} but "
|
|
80
|
+
f"names commit(s) {missing} that stage {stage}'s worktree does not "
|
|
81
|
+
"contain: it starts from the implementation anchor and its "
|
|
82
|
+
f"dependsOn closure ({', '.join(tip[:12] for tip in tips)}); "
|
|
83
|
+
"compare against a commit in that chain or remove the stage "
|
|
84
|
+
"from stageRefs"
|
|
85
|
+
)
|
|
86
|
+
return failures
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _worktree_tips(
|
|
90
|
+
stage: int,
|
|
91
|
+
graph: list[tuple[int, list[int]]],
|
|
92
|
+
anchor_base: str,
|
|
93
|
+
done_heads: Mapping[int, str],
|
|
94
|
+
) -> list[str]:
|
|
95
|
+
closure = order_stage_closure(graph, {stage}, set())
|
|
96
|
+
tips = [anchor_base] if anchor_base else []
|
|
97
|
+
tips.extend(done_heads[number] for number in closure if number in done_heads)
|
|
98
|
+
return tips
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def stage_command_commit_errors_from_task_state(
|
|
102
|
+
data_path: Path, data: Mapping[str, Any],
|
|
103
|
+
) -> list[str]:
|
|
104
|
+
"""Judge the plan record at ``data_path`` against the task's recorded stage bases.
|
|
105
|
+
|
|
106
|
+
The anchor comes from the worktree registry and the done heads from the consumers
|
|
107
|
+
ledger of the planning run that owns the record. Returns failure lines; a plan
|
|
108
|
+
with no task key yields none, since no recorded base can be looked up for it.
|
|
109
|
+
"""
|
|
110
|
+
from okstra_project.state import StateError, parse_task_key
|
|
111
|
+
|
|
112
|
+
from .consumers import read_stage_consumer_state
|
|
113
|
+
from .worktree_registry import get_implementation_base
|
|
114
|
+
|
|
115
|
+
header = data.get("header")
|
|
116
|
+
task_key = header.get("taskKey") if isinstance(header, Mapping) else None
|
|
117
|
+
try:
|
|
118
|
+
project_id, task_group, task_id = parse_task_key(str(task_key or ""))
|
|
119
|
+
except StateError:
|
|
120
|
+
# expected-miss: the report schema checks the header; without a task key there
|
|
121
|
+
# is no recorded anchor or ledger to judge the plan against.
|
|
122
|
+
return []
|
|
123
|
+
reports_dir = data_path.parent
|
|
124
|
+
plan_run_root = reports_dir.parent
|
|
125
|
+
try:
|
|
126
|
+
state = read_stage_consumer_state(plan_run_root)
|
|
127
|
+
except (OSError, UnicodeError) as exc:
|
|
128
|
+
return [
|
|
129
|
+
"stage command commit check could not read the consumers ledger under "
|
|
130
|
+
f"{plan_run_root}: {exc}"
|
|
131
|
+
]
|
|
132
|
+
done_heads = {
|
|
133
|
+
stage: str(row.get("head_commit") or "")
|
|
134
|
+
for stage, row in state.done_by_stage.items()
|
|
135
|
+
if row.get("head_commit")
|
|
136
|
+
}
|
|
137
|
+
planning = data.get("implementationPlanning")
|
|
138
|
+
return stage_command_commit_errors(
|
|
139
|
+
planning if isinstance(planning, Mapping) else {},
|
|
140
|
+
repo_dir=reports_dir,
|
|
141
|
+
anchor_base=get_implementation_base(project_id, task_group, task_id) or "",
|
|
142
|
+
done_heads=done_heads,
|
|
143
|
+
)
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""Byte-exact capture of one verification command's output and process group."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
import select
|
|
7
|
+
import selectors
|
|
8
|
+
import signal
|
|
9
|
+
import stat
|
|
10
|
+
import subprocess
|
|
11
|
+
import time
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Callable, Sequence
|
|
15
|
+
|
|
16
|
+
from .process_group import kill_group, live_group_members
|
|
17
|
+
|
|
18
|
+
DRAIN_AFTER_EXIT_SECONDS = 2.0
|
|
19
|
+
CLEANUP_GRACE_SECONDS = 5.0
|
|
20
|
+
_SELECT_SECONDS = 0.1
|
|
21
|
+
_READ_SIZE = 65536
|
|
22
|
+
# A forwarding destination that falls this far behind is not reading; its
|
|
23
|
+
# copy is dropped so the saved evidence and the deadlines keep going.
|
|
24
|
+
_FORWARD_BACKLOG_BYTES = 4 * 1024 * 1024
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class StreamCapture:
|
|
29
|
+
"""`bytes` and `digest` describe what the pipe delivered; the file at
|
|
30
|
+
`path` holds exactly those bytes only when `save_error` is None."""
|
|
31
|
+
|
|
32
|
+
path: Path
|
|
33
|
+
bytes: int
|
|
34
|
+
digest: str
|
|
35
|
+
save_error: str | None
|
|
36
|
+
forward_error: str | None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class CommandCapture:
|
|
41
|
+
returncode: int
|
|
42
|
+
drain_complete: bool
|
|
43
|
+
cleanup: str
|
|
44
|
+
interrupted: bool
|
|
45
|
+
stdout: StreamCapture
|
|
46
|
+
stderr: StreamCapture
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class _Stream:
|
|
50
|
+
def __init__(self, path: Path, forward_fd: int) -> None:
|
|
51
|
+
self.path = path
|
|
52
|
+
self.hash = hashlib.sha256()
|
|
53
|
+
self.size = 0
|
|
54
|
+
self.save_error: str | None = None
|
|
55
|
+
self.forward_error: str | None = None
|
|
56
|
+
self.pending = bytearray()
|
|
57
|
+
self.save_fd: int | None = None
|
|
58
|
+
self.forward_fd: int | None = None
|
|
59
|
+
try:
|
|
60
|
+
self.save_fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
61
|
+
except OSError as exc:
|
|
62
|
+
self.save_error = str(exc)
|
|
63
|
+
mode = 0
|
|
64
|
+
try:
|
|
65
|
+
# A private descriptor, so two streams forwarded to one pipe still
|
|
66
|
+
# register separately and closing it never closes the caller's.
|
|
67
|
+
self.forward_fd = os.dup(forward_fd)
|
|
68
|
+
mode = os.fstat(self.forward_fd).st_mode
|
|
69
|
+
except OSError as exc:
|
|
70
|
+
self.drop_forward(str(exc))
|
|
71
|
+
# A write larger than PIPE_BUF to a writable pipe can still block.
|
|
72
|
+
self.forward_chunk = None if stat.S_ISREG(mode) else select.PIPE_BUF
|
|
73
|
+
|
|
74
|
+
def feed(self, chunk: bytes) -> None:
|
|
75
|
+
self.hash.update(chunk)
|
|
76
|
+
self.size += len(chunk)
|
|
77
|
+
if self.save_fd is not None:
|
|
78
|
+
try:
|
|
79
|
+
_write_all(self.save_fd, chunk)
|
|
80
|
+
except OSError as exc:
|
|
81
|
+
# Draining continues so the child never blocks on a full pipe;
|
|
82
|
+
# save_error marks the saved file as short.
|
|
83
|
+
self.save_error = str(exc)
|
|
84
|
+
os.close(self.save_fd)
|
|
85
|
+
self.save_fd = None
|
|
86
|
+
if self.forward_fd is None:
|
|
87
|
+
return
|
|
88
|
+
self.pending += chunk
|
|
89
|
+
if self.forward_chunk is None:
|
|
90
|
+
self.forward()
|
|
91
|
+
elif len(self.pending) > _FORWARD_BACKLOG_BYTES:
|
|
92
|
+
self.drop_forward("the forwarding destination stopped reading")
|
|
93
|
+
|
|
94
|
+
def forward(self) -> None:
|
|
95
|
+
"""Write what the destination accepts now; called when it is writable."""
|
|
96
|
+
assert self.forward_fd is not None
|
|
97
|
+
try:
|
|
98
|
+
if self.forward_chunk is None:
|
|
99
|
+
_write_all(self.forward_fd, self.pending)
|
|
100
|
+
self.pending.clear()
|
|
101
|
+
else:
|
|
102
|
+
written = os.write(self.forward_fd, self.pending[:self.forward_chunk])
|
|
103
|
+
del self.pending[:written]
|
|
104
|
+
except OSError as exc:
|
|
105
|
+
self.drop_forward(str(exc))
|
|
106
|
+
|
|
107
|
+
def drop_forward(self, reason: str) -> None:
|
|
108
|
+
self.forward_error = self.forward_error or reason
|
|
109
|
+
self.pending.clear()
|
|
110
|
+
if self.forward_fd is not None:
|
|
111
|
+
os.close(self.forward_fd)
|
|
112
|
+
self.forward_fd = None
|
|
113
|
+
|
|
114
|
+
def close(self) -> None:
|
|
115
|
+
for fd in (self.save_fd, self.forward_fd):
|
|
116
|
+
if fd is not None:
|
|
117
|
+
os.close(fd)
|
|
118
|
+
self.save_fd = self.forward_fd = None
|
|
119
|
+
|
|
120
|
+
def result(self) -> StreamCapture:
|
|
121
|
+
return StreamCapture(
|
|
122
|
+
self.path, self.size, "sha256:" + self.hash.hexdigest(),
|
|
123
|
+
self.save_error, self.forward_error,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass
|
|
128
|
+
class _Stop:
|
|
129
|
+
sent_at: float | None = None
|
|
130
|
+
killed: bool = False
|
|
131
|
+
drained: bool = False
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _write_all(fd: int, chunk: bytes | bytearray) -> None:
|
|
135
|
+
view = memoryview(chunk)
|
|
136
|
+
while view:
|
|
137
|
+
view = view[os.write(fd, view):]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def run_captured(
|
|
141
|
+
argv: Sequence[str], cwd: Path, *, stdout_path: Path, stderr_path: Path,
|
|
142
|
+
stop_requested: Callable[[], bool], forward: tuple[int, int] = (1, 2),
|
|
143
|
+
drain_seconds: float = DRAIN_AFTER_EXIT_SECONDS,
|
|
144
|
+
grace_seconds: float = CLEANUP_GRACE_SECONDS,
|
|
145
|
+
) -> CommandCapture:
|
|
146
|
+
"""Run argv in its own process group, save and forward both streams.
|
|
147
|
+
|
|
148
|
+
Reads and forwarding writes share one select loop, so neither a full pipe
|
|
149
|
+
nor a destination that stops reading stalls the other stream, the
|
|
150
|
+
interruption or the drain deadline. That deadline is set once when the
|
|
151
|
+
main process exits; grandchild output never extends it.
|
|
152
|
+
"""
|
|
153
|
+
streams = (_Stream(stdout_path, forward[0]), _Stream(stderr_path, forward[1]))
|
|
154
|
+
try:
|
|
155
|
+
process = subprocess.Popen(
|
|
156
|
+
list(argv), cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
157
|
+
stderr=subprocess.PIPE, start_new_session=True,
|
|
158
|
+
)
|
|
159
|
+
except BaseException:
|
|
160
|
+
for stream in streams:
|
|
161
|
+
stream.close()
|
|
162
|
+
raise
|
|
163
|
+
stop = _Stop()
|
|
164
|
+
try:
|
|
165
|
+
_drain(process, streams, stop, stop_requested, drain_seconds, grace_seconds)
|
|
166
|
+
finally:
|
|
167
|
+
returncode, cleanup = _end_group(process, stop, grace_seconds)
|
|
168
|
+
for stream in streams:
|
|
169
|
+
stream.close()
|
|
170
|
+
return CommandCapture(
|
|
171
|
+
returncode, stop.drained, cleanup, stop.sent_at is not None,
|
|
172
|
+
streams[0].result(), streams[1].result(),
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _drain(
|
|
177
|
+
process: subprocess.Popen[bytes], streams: tuple[_Stream, _Stream],
|
|
178
|
+
stop: _Stop, stop_requested: Callable[[], bool], drain_seconds: float,
|
|
179
|
+
grace_seconds: float,
|
|
180
|
+
) -> None:
|
|
181
|
+
assert process.stdout is not None and process.stderr is not None
|
|
182
|
+
with selectors.DefaultSelector() as selector:
|
|
183
|
+
for pipe, stream in zip((process.stdout, process.stderr), streams):
|
|
184
|
+
selector.register(pipe, selectors.EVENT_READ, stream)
|
|
185
|
+
deadline: float | None = None
|
|
186
|
+
while (_reading(selector) or any(s.pending for s in streams)
|
|
187
|
+
or not _exited(process)):
|
|
188
|
+
_escalate(process, stop, stop_requested, grace_seconds)
|
|
189
|
+
_watch_forwarding(selector, streams)
|
|
190
|
+
_pump_once(selector)
|
|
191
|
+
if deadline is None and _exited(process):
|
|
192
|
+
deadline = time.monotonic() + drain_seconds
|
|
193
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
194
|
+
break
|
|
195
|
+
stop.drained = not _reading(selector)
|
|
196
|
+
for stream in streams:
|
|
197
|
+
if stream.pending:
|
|
198
|
+
stream.drop_forward("the forwarded copy was cut at the drain deadline")
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _reading(selector: selectors.BaseSelector) -> bool:
|
|
202
|
+
return any(key.events & selectors.EVENT_READ
|
|
203
|
+
for key in selector.get_map().values())
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _escalate(
|
|
207
|
+
process: subprocess.Popen[bytes], stop: _Stop,
|
|
208
|
+
stop_requested: Callable[[], bool], grace_seconds: float,
|
|
209
|
+
) -> None:
|
|
210
|
+
if stop.sent_at is None and stop_requested():
|
|
211
|
+
stop.sent_at = time.monotonic()
|
|
212
|
+
kill_group(process.pid, signal.SIGTERM)
|
|
213
|
+
if (stop.sent_at is not None and not stop.killed
|
|
214
|
+
and time.monotonic() - stop.sent_at >= grace_seconds):
|
|
215
|
+
stop.killed = True
|
|
216
|
+
kill_group(process.pid)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _watch_forwarding(
|
|
220
|
+
selector: selectors.BaseSelector, streams: tuple[_Stream, _Stream],
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Watch a destination for writability only while it has pending bytes.
|
|
223
|
+
|
|
224
|
+
A destination dropped since the last pass has a closed descriptor, so its
|
|
225
|
+
registration is removed before the next select.
|
|
226
|
+
"""
|
|
227
|
+
for key in list(selector.get_map().values()):
|
|
228
|
+
if key.events & selectors.EVENT_WRITE and (
|
|
229
|
+
key.data.forward_fd != key.fd or not key.data.pending):
|
|
230
|
+
selector.unregister(key.fileobj)
|
|
231
|
+
registered = selector.get_map()
|
|
232
|
+
for stream in streams:
|
|
233
|
+
fd = stream.forward_fd
|
|
234
|
+
if stream.pending and fd is not None and fd not in registered:
|
|
235
|
+
selector.register(fd, selectors.EVENT_WRITE, stream)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _pump_once(selector: selectors.BaseSelector) -> None:
|
|
239
|
+
if not selector.get_map():
|
|
240
|
+
time.sleep(_SELECT_SECONDS)
|
|
241
|
+
return
|
|
242
|
+
for key, events in selector.select(timeout=_SELECT_SECONDS):
|
|
243
|
+
stream: _Stream = key.data
|
|
244
|
+
if events & selectors.EVENT_WRITE:
|
|
245
|
+
if stream.forward_fd == key.fd:
|
|
246
|
+
stream.forward()
|
|
247
|
+
continue
|
|
248
|
+
chunk = os.read(key.fd, _READ_SIZE)
|
|
249
|
+
if chunk:
|
|
250
|
+
stream.feed(chunk)
|
|
251
|
+
else:
|
|
252
|
+
selector.unregister(key.fileobj)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _end_group(
|
|
256
|
+
process: subprocess.Popen[bytes], stop: _Stop, grace_seconds: float,
|
|
257
|
+
) -> tuple[int, str]:
|
|
258
|
+
"""Kill the group, reap the main process, and confirm the group gone.
|
|
259
|
+
|
|
260
|
+
The main process stays unreaped until the kill, so its pid, which is the
|
|
261
|
+
group id, cannot be reused by an unrelated group first. An interrupted
|
|
262
|
+
group keeps the rest of its grace even when its leader exited early.
|
|
263
|
+
"""
|
|
264
|
+
if stop.sent_at is not None and not stop.killed:
|
|
265
|
+
_await_no_live_member(process.pid, stop.sent_at + grace_seconds)
|
|
266
|
+
kill_group(process.pid)
|
|
267
|
+
returncode = process.wait()
|
|
268
|
+
for pipe in (process.stdout, process.stderr):
|
|
269
|
+
if pipe is not None:
|
|
270
|
+
pipe.close()
|
|
271
|
+
gone = _await_no_live_member(process.pid, time.monotonic() + grace_seconds)
|
|
272
|
+
return returncode, "confirmed" if gone else "unconfirmed"
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _exited(process: subprocess.Popen[bytes]) -> bool:
|
|
276
|
+
"""Whether the main process exited, observed without reaping it."""
|
|
277
|
+
try:
|
|
278
|
+
return os.waitid(
|
|
279
|
+
os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT,
|
|
280
|
+
) is not None
|
|
281
|
+
except ChildProcessError:
|
|
282
|
+
return process.poll() is not None
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _await_no_live_member(pgid: int, limit: float) -> bool:
|
|
286
|
+
while True:
|
|
287
|
+
if live_group_members(pgid) == 0:
|
|
288
|
+
return True
|
|
289
|
+
if time.monotonic() >= limit:
|
|
290
|
+
return False
|
|
291
|
+
time.sleep(_SELECT_SECONDS)
|
|
@@ -45,17 +45,19 @@ def _project_script_path(project: Path, task_root: Path, script: str) -> str:
|
|
|
45
45
|
return safe.relative_to(project).as_posix()
|
|
46
46
|
|
|
47
47
|
|
|
48
|
+
def conformance_stage(stage_key: str) -> int:
|
|
49
|
+
match = re.fullmatch(r".+-stage-([1-9][0-9]*)", stage_key)
|
|
50
|
+
if match is None:
|
|
51
|
+
raise ValueError("conformance stageKey must end in -stage-<positive integer>")
|
|
52
|
+
return int(match[1])
|
|
53
|
+
|
|
54
|
+
|
|
48
55
|
def _entries_by_stage(
|
|
49
56
|
manifest: Mapping[str, Any], declared_stages: set[int],
|
|
50
57
|
) -> dict[int, dict[str, Any]]:
|
|
51
58
|
entries = {}
|
|
52
59
|
for entry in manifest["entries"]:
|
|
53
|
-
|
|
54
|
-
if match is None:
|
|
55
|
-
raise ValueError(
|
|
56
|
-
"conformance stageKey must end in -stage-<positive integer>"
|
|
57
|
-
)
|
|
58
|
-
stage = int(match[1])
|
|
60
|
+
stage = conformance_stage(entry["stageKey"])
|
|
59
61
|
if stage in entries:
|
|
60
62
|
raise ValueError(f"conformance stage {stage} has multiple entries")
|
|
61
63
|
if stage not in declared_stages:
|
|
@@ -118,6 +120,25 @@ def _read_conformance_manifest(
|
|
|
118
120
|
)
|
|
119
121
|
|
|
120
122
|
|
|
123
|
+
def conformance_manifest_path(project: Path, task_root: Path) -> str:
|
|
124
|
+
return (task_root / "qa/conformance-manifest.json").relative_to(
|
|
125
|
+
project,
|
|
126
|
+
).as_posix()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def capture_conformance_baseline(
|
|
130
|
+
project: Path, task_root: Path, plan: FrozenVerificationPlan,
|
|
131
|
+
) -> dict[str, str]:
|
|
132
|
+
entries, _ = _read_conformance_manifest(
|
|
133
|
+
project, conformance_manifest_path(project, task_root),
|
|
134
|
+
{row["stage"] for row in plan["conformance"]},
|
|
135
|
+
)
|
|
136
|
+
return {
|
|
137
|
+
str(stage): _project_script_path(project, task_root, entry["script"])
|
|
138
|
+
for stage, entry in sorted(entries.items())
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
121
142
|
def capture_generation_conformance(
|
|
122
143
|
project: Path, ref: RunRef, plan: FrozenVerificationPlan,
|
|
123
144
|
changed_artifact_paths: frozenset[str], conformance_baseline: Mapping[int, str],
|
|
@@ -134,9 +155,7 @@ def capture_generation_conformance(
|
|
|
134
155
|
stage: _project_script_path(project, ref.task_root, row["script"])
|
|
135
156
|
for stage, row in declarations.items() if "script" in row
|
|
136
157
|
}
|
|
137
|
-
relative_manifest = (ref.task_root
|
|
138
|
-
project,
|
|
139
|
-
).as_posix()
|
|
158
|
+
relative_manifest = conformance_manifest_path(project, ref.task_root)
|
|
140
159
|
entries, sealed_manifest = _read_conformance_manifest(
|
|
141
160
|
project, relative_manifest, set(declarations),
|
|
142
161
|
)
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import subprocess
|
|
5
|
+
from typing import Any, Mapping
|
|
6
|
+
|
|
7
|
+
from .correction_round import completed_round_attempts, correction_round
|
|
8
|
+
from .execution_manifest import read_execution_manifest
|
|
9
|
+
from .paths import RunRef
|
|
10
|
+
from .qa_procedures import safe_qa_path
|
|
11
|
+
from .verification_generation import prepare_verification_generation
|
|
12
|
+
from .verification_input_ref import VerificationInputReference
|
|
13
|
+
from .verification_input_types import VerificationInputs
|
|
14
|
+
from .verification_inputs import (
|
|
15
|
+
frozen_conformance_baseline, read_verification_inputs,
|
|
16
|
+
)
|
|
17
|
+
from .verification_invocation import (
|
|
18
|
+
VerificationGenerationReference, verification_generation_required,
|
|
19
|
+
)
|
|
20
|
+
from .verification_target import read_worktree_head
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class VerificationDispatchError(ValueError):
|
|
24
|
+
"""A verifier dispatch whose generation cannot be sealed."""
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self, message: str, *, missing_executor_result: bool = False,
|
|
28
|
+
) -> None:
|
|
29
|
+
super().__init__(message)
|
|
30
|
+
self.missing_executor_result = missing_executor_result
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def seal_dispatch_generation(
|
|
34
|
+
project: Path, manifest_path: Path, manifest: Mapping[str, Any],
|
|
35
|
+
active_context: Mapping[str, Any], *, audience: str, dispatch_kind: str,
|
|
36
|
+
) -> VerificationGenerationReference | None:
|
|
37
|
+
try:
|
|
38
|
+
return _seal(
|
|
39
|
+
project, manifest_path, manifest, active_context,
|
|
40
|
+
audience=audience, dispatch_kind=dispatch_kind,
|
|
41
|
+
)
|
|
42
|
+
except VerificationDispatchError:
|
|
43
|
+
raise
|
|
44
|
+
except KeyError as exc:
|
|
45
|
+
raise VerificationDispatchError(
|
|
46
|
+
f"required field is missing: {exc.args[0]}",
|
|
47
|
+
) from exc
|
|
48
|
+
except (OSError, ValueError, subprocess.SubprocessError) as exc:
|
|
49
|
+
raise VerificationDispatchError(str(exc)) from exc
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def verification_generation_number(dispatch_kind: str) -> int:
|
|
53
|
+
"""Correction round N seals generation N + 1; every other dispatch seals 1."""
|
|
54
|
+
return 1 + (correction_round(dispatch_kind) or 0)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _seal(
|
|
58
|
+
project: Path, manifest_path: Path, manifest: Mapping[str, Any],
|
|
59
|
+
active_context: Mapping[str, Any], *, audience: str, dispatch_kind: str,
|
|
60
|
+
) -> VerificationGenerationReference | None:
|
|
61
|
+
if not verification_generation_required(manifest, audience):
|
|
62
|
+
return None
|
|
63
|
+
project = project.resolve()
|
|
64
|
+
reference = VerificationInputReference.required_from_artifact_fields(manifest)
|
|
65
|
+
round_number = correction_round(dispatch_kind)
|
|
66
|
+
implementation = manifest["taskType"] == "implementation"
|
|
67
|
+
if implementation:
|
|
68
|
+
changed = _executor_changes(manifest_path, dispatch_kind, round_number)
|
|
69
|
+
else:
|
|
70
|
+
changed = _final_verification_changes(round_number)
|
|
71
|
+
inputs = read_verification_inputs(
|
|
72
|
+
safe_qa_path(project, reference.relative_path), reference.digest,
|
|
73
|
+
)
|
|
74
|
+
head = (
|
|
75
|
+
read_worktree_head(Path(inputs["sourceRoot"])) if implementation
|
|
76
|
+
else str(active_context["verificationTarget"]["headRef"])
|
|
77
|
+
)
|
|
78
|
+
path, digest = prepare_verification_generation(
|
|
79
|
+
project, RunRef.from_manifest(manifest_path, manifest), reference,
|
|
80
|
+
generation=verification_generation_number(dispatch_kind),
|
|
81
|
+
expected_head=head,
|
|
82
|
+
source_inputs=_present_declared_sources(inputs),
|
|
83
|
+
generated_roots=frozenset(), artifact_inputs=frozenset(),
|
|
84
|
+
changed_artifact_paths=changed,
|
|
85
|
+
conformance_baseline=frozen_conformance_baseline(
|
|
86
|
+
inputs, manifest["taskType"],
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
return VerificationGenerationReference(
|
|
90
|
+
path.relative_to(project).as_posix(), digest,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _present_declared_sources(inputs: VerificationInputs) -> frozenset[str]:
|
|
95
|
+
# A related source the executor deleted is part of the change under review;
|
|
96
|
+
# only the files present at the seal can be protected from the verifiers.
|
|
97
|
+
root = Path(inputs["sourceRoot"])
|
|
98
|
+
return frozenset(
|
|
99
|
+
relative
|
|
100
|
+
for entries in inputs["qaCommands"].values() for entry in entries
|
|
101
|
+
for relative in entry.get("sourcePaths", ())
|
|
102
|
+
if (root / relative).exists() or (root / relative).is_symlink()
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _executor_changes(
|
|
107
|
+
manifest_path: Path, dispatch_kind: str, round_number: int | None,
|
|
108
|
+
) -> frozenset[str]:
|
|
109
|
+
attempts = completed_round_attempts(
|
|
110
|
+
read_execution_manifest(manifest_path), round_number, "implementer",
|
|
111
|
+
)
|
|
112
|
+
if not attempts:
|
|
113
|
+
raise VerificationDispatchError(
|
|
114
|
+
"the implementation executor has no ok result in dispatch "
|
|
115
|
+
f"{dispatch_kind}; dispatch the executor alone and await it before "
|
|
116
|
+
"its verifiers",
|
|
117
|
+
missing_executor_result=True,
|
|
118
|
+
)
|
|
119
|
+
return frozenset(
|
|
120
|
+
path for attempt in attempts
|
|
121
|
+
for path in attempt.change_summary.get("changedArtifactPaths", ())
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _final_verification_changes(round_number: int | None) -> frozenset[str]:
|
|
126
|
+
if round_number is not None:
|
|
127
|
+
raise ValueError(
|
|
128
|
+
"final-verification seals only its initial verifier dispatch"
|
|
129
|
+
)
|
|
130
|
+
return frozenset()
|