okstra 0.228.2 → 0.230.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -5
- package/dist/cli-registry.mjs +0 -18
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/lifecycle/doctor.mjs +0 -15
- package/dist/commands/lifecycle/doctor.mjs.map +1 -1
- package/dist/lib/skill-catalog.mjs +4 -4
- package/dist/lib/skill-catalog.mjs.map +1 -1
- package/docs/architecture/storage-model.md +179 -13
- package/docs/architecture.md +13 -14
- package/docs/contributor-change-matrix.md +5 -5
- package/docs/project-structure-overview.md +16 -21
- package/package.json +1 -2
- package/runtime/BUILD.json +2 -2
- package/runtime/bin/lib/okstra/interactive.sh +2 -1
- package/runtime/bin/okstra-provider-exec.py +53 -4
- package/runtime/prompts/duties/code-reviewer.json +5 -2
- package/runtime/prompts/launch.template.md +5 -5
- package/runtime/prompts/lead/convergence.md +4 -4
- package/runtime/prompts/lead/okstra-lead-contract.md +14 -13
- package/runtime/prompts/lead/phase-routing.md +1 -1
- package/runtime/prompts/lead/report-writer.md +3 -3
- package/runtime/prompts/lead/team-contract.md +3 -3
- package/runtime/python/okstra_ctl/agent/invocation.py +142 -5
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +31 -1
- package/runtime/python/okstra_ctl/cmux.py +4 -2
- package/runtime/python/okstra_ctl/code_review_census.py +194 -0
- package/runtime/python/okstra_ctl/code_review_coverage.py +18 -5
- package/runtime/python/okstra_ctl/code_review_scope.py +11 -7
- package/runtime/python/okstra_ctl/code_review_target.py +79 -25
- package/runtime/python/okstra_ctl/conformance.py +61 -0
- package/runtime/python/okstra_ctl/conformance_evidence.py +235 -0
- package/runtime/python/okstra_ctl/correction_round.py +19 -13
- package/runtime/python/okstra_ctl/dispatch_core.py +1 -1
- package/runtime/python/okstra_ctl/dispatch_state.py +1 -0
- package/runtime/python/okstra_ctl/execution_mutation_audit.py +100 -2
- package/runtime/python/okstra_ctl/generated_input.py +21 -0
- package/runtime/python/okstra_ctl/ids.py +1 -41
- package/runtime/python/okstra_ctl/initial_prompt_materialization.py +30 -0
- package/runtime/python/okstra_ctl/interactive_cli.py +5 -0
- package/runtime/python/okstra_ctl/locks.py +28 -1
- package/runtime/python/okstra_ctl/model_io/renderers.py +17 -1
- package/runtime/python/okstra_ctl/paths.py +84 -31
- package/runtime/python/okstra_ctl/phases/catalog.py +3 -0
- package/runtime/python/okstra_ctl/phases/change_impact_analysis/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/discussion/spec.md +63 -63
- package/runtime/python/okstra_ctl/phases/discussion/wizard.py +2 -11
- package/runtime/python/okstra_ctl/phases/error_analysis/report_assets/error-analysis-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/error_analysis/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/feature_analysis/spec.md +1 -1
- package/runtime/python/okstra_ctl/phases/final_verification/profile.md +4 -4
- package/runtime/python/okstra_ctl/phases/final_verification/report_assets/final-verification-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/final_verification/spec.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation/host-rules.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-deliverable.md +2 -2
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-executor.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation/instructions/_implementation-verifier.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation/report_assets/implementation-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation/spec.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation_option_selection/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/implementation_planning/instructions/plan-body-verification.md +11 -11
- package/runtime/python/okstra_ctl/phases/implementation_planning/plan_body.py +14 -0
- package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +4 -4
- package/runtime/python/okstra_ctl/phases/implementation_planning/report_assets/implementation-planning-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation_planning/spec.md +1 -1
- package/runtime/python/okstra_ctl/phases/project_analysis/spec.md +2 -2
- package/runtime/python/okstra_ctl/phases/release_handoff/entry.py +7 -3
- package/runtime/python/okstra_ctl/phases/release_handoff/report_assets/release-handoff-input.template.md +1 -1
- package/runtime/python/okstra_ctl/phases/release_handoff/spec.md +4 -4
- package/runtime/python/okstra_ctl/phases/technical_verification/spec.md +1 -1
- package/runtime/python/okstra_ctl/process_group.py +26 -3
- package/runtime/python/okstra_ctl/qa_commands.py +7 -2
- package/runtime/python/okstra_ctl/qa_procedures.py +60 -17
- package/runtime/python/okstra_ctl/recap.py +1 -2
- package/runtime/python/okstra_ctl/render.py +10 -2
- package/runtime/python/okstra_ctl/report_finalize.py +3 -3
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +5 -0
- package/runtime/python/okstra_ctl/run.py +24 -20
- package/runtime/python/okstra_ctl/stage_command_commits.py +143 -0
- package/runtime/python/okstra_ctl/verification_capture.py +291 -0
- package/runtime/python/okstra_ctl/verification_conformance.py +182 -0
- package/runtime/python/okstra_ctl/verification_dispatch.py +130 -0
- package/runtime/python/okstra_ctl/verification_evidence.py +652 -0
- package/runtime/python/okstra_ctl/verification_generation.py +248 -0
- package/runtime/python/okstra_ctl/verification_input_ref.py +9 -0
- package/runtime/python/okstra_ctl/verification_input_types.py +23 -2
- package/runtime/python/okstra_ctl/verification_inputs.py +161 -23
- package/runtime/python/okstra_ctl/verification_invocation.py +139 -0
- package/runtime/python/okstra_ctl/verification_target.py +20 -5
- package/runtime/python/okstra_ctl/wizard/registry.py +2 -1
- package/runtime/python/okstra_ctl/wizard/render.py +2 -1
- package/runtime/python/okstra_ctl/wizard/sources.py +14 -3
- package/runtime/python/okstra_ctl/wizard/steps_identity.py +2 -1
- package/runtime/python/okstra_ctl/worker_runner.py +1 -1
- package/runtime/python/okstra_ctl/worktree_lock_cli.py +70 -17
- package/runtime/python/okstra_ctl/worktree_registry.py +4 -0
- package/runtime/skills/okstra-code-review/SKILL.md +84 -48
- package/runtime/skills/okstra-code-review/references/census-rules.md +10 -4
- package/runtime/skills/okstra-code-review/references/review-calibration.md +7 -4
- package/runtime/skills/okstra-explain-flow/SKILL.md +1 -1
- package/runtime/skills/okstra-run/SKILL.md +6 -6
- package/runtime/skills/okstra-schedule-gen/SKILL.md +3 -3
- package/runtime/skills/okstra-setup/SKILL.md +2 -3
- package/runtime/validators/validate-run.py +118 -52
- package/docs/cli.md +0 -1060
- package/docs/container.md +0 -100
- package/docs/follow-ups/2026-07-10-final-report-option-3.md +0 -51
- package/docs/performance-improvement-plan-v2.md +0 -369
- package/runtime/python/okstra_ctl/container.py +0 -702
- package/runtime/python/okstra_ctl/rollup.py +0 -241
- package/runtime/python/okstra_ctl/usage_report.py +0 -362
- package/runtime/skills/okstra-container-build/SKILL.md +0 -139
- package/runtime/skills/okstra-inspect/SKILL.md +0 -100
- package/runtime/skills/okstra-inspect/facets/cost.md +0 -84
- package/runtime/skills/okstra-inspect/facets/error-zip.md +0 -41
- package/runtime/skills/okstra-inspect/facets/errors.md +0 -69
- package/runtime/skills/okstra-inspect/facets/history.md +0 -86
- package/runtime/skills/okstra-inspect/facets/logs.md +0 -89
- package/runtime/skills/okstra-inspect/facets/recap.md +0 -167
- package/runtime/skills/okstra-inspect/facets/report.md +0 -57
- package/runtime/skills/okstra-inspect/facets/run-audit.md +0 -34
- package/runtime/skills/okstra-inspect/facets/status.md +0 -163
- package/runtime/skills/okstra-inspect/facets/time.md +0 -57
- package/runtime/skills/okstra-rollup/SKILL.md +0 -75
- package/runtime/skills/okstra-usage/SKILL.md +0 -71
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""Byte-exact capture of one verification command's output and process group."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
import select
|
|
7
|
+
import selectors
|
|
8
|
+
import signal
|
|
9
|
+
import stat
|
|
10
|
+
import subprocess
|
|
11
|
+
import time
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Callable, Sequence
|
|
15
|
+
|
|
16
|
+
from .process_group import kill_group, live_group_members
|
|
17
|
+
|
|
18
|
+
DRAIN_AFTER_EXIT_SECONDS = 2.0
|
|
19
|
+
CLEANUP_GRACE_SECONDS = 5.0
|
|
20
|
+
_SELECT_SECONDS = 0.1
|
|
21
|
+
_READ_SIZE = 65536
|
|
22
|
+
# A forwarding destination that falls this far behind is not reading; its
|
|
23
|
+
# copy is dropped so the saved evidence and the deadlines keep going.
|
|
24
|
+
_FORWARD_BACKLOG_BYTES = 4 * 1024 * 1024
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class StreamCapture:
|
|
29
|
+
"""`bytes` and `digest` describe what the pipe delivered; the file at
|
|
30
|
+
`path` holds exactly those bytes only when `save_error` is None."""
|
|
31
|
+
|
|
32
|
+
path: Path
|
|
33
|
+
bytes: int
|
|
34
|
+
digest: str
|
|
35
|
+
save_error: str | None
|
|
36
|
+
forward_error: str | None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class CommandCapture:
|
|
41
|
+
returncode: int
|
|
42
|
+
drain_complete: bool
|
|
43
|
+
cleanup: str
|
|
44
|
+
interrupted: bool
|
|
45
|
+
stdout: StreamCapture
|
|
46
|
+
stderr: StreamCapture
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class _Stream:
|
|
50
|
+
def __init__(self, path: Path, forward_fd: int) -> None:
|
|
51
|
+
self.path = path
|
|
52
|
+
self.hash = hashlib.sha256()
|
|
53
|
+
self.size = 0
|
|
54
|
+
self.save_error: str | None = None
|
|
55
|
+
self.forward_error: str | None = None
|
|
56
|
+
self.pending = bytearray()
|
|
57
|
+
self.save_fd: int | None = None
|
|
58
|
+
self.forward_fd: int | None = None
|
|
59
|
+
try:
|
|
60
|
+
self.save_fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
61
|
+
except OSError as exc:
|
|
62
|
+
self.save_error = str(exc)
|
|
63
|
+
mode = 0
|
|
64
|
+
try:
|
|
65
|
+
# A private descriptor, so two streams forwarded to one pipe still
|
|
66
|
+
# register separately and closing it never closes the caller's.
|
|
67
|
+
self.forward_fd = os.dup(forward_fd)
|
|
68
|
+
mode = os.fstat(self.forward_fd).st_mode
|
|
69
|
+
except OSError as exc:
|
|
70
|
+
self.drop_forward(str(exc))
|
|
71
|
+
# A write larger than PIPE_BUF to a writable pipe can still block.
|
|
72
|
+
self.forward_chunk = None if stat.S_ISREG(mode) else select.PIPE_BUF
|
|
73
|
+
|
|
74
|
+
def feed(self, chunk: bytes) -> None:
|
|
75
|
+
self.hash.update(chunk)
|
|
76
|
+
self.size += len(chunk)
|
|
77
|
+
if self.save_fd is not None:
|
|
78
|
+
try:
|
|
79
|
+
_write_all(self.save_fd, chunk)
|
|
80
|
+
except OSError as exc:
|
|
81
|
+
# Draining continues so the child never blocks on a full pipe;
|
|
82
|
+
# save_error marks the saved file as short.
|
|
83
|
+
self.save_error = str(exc)
|
|
84
|
+
os.close(self.save_fd)
|
|
85
|
+
self.save_fd = None
|
|
86
|
+
if self.forward_fd is None:
|
|
87
|
+
return
|
|
88
|
+
self.pending += chunk
|
|
89
|
+
if self.forward_chunk is None:
|
|
90
|
+
self.forward()
|
|
91
|
+
elif len(self.pending) > _FORWARD_BACKLOG_BYTES:
|
|
92
|
+
self.drop_forward("the forwarding destination stopped reading")
|
|
93
|
+
|
|
94
|
+
def forward(self) -> None:
|
|
95
|
+
"""Write what the destination accepts now; called when it is writable."""
|
|
96
|
+
assert self.forward_fd is not None
|
|
97
|
+
try:
|
|
98
|
+
if self.forward_chunk is None:
|
|
99
|
+
_write_all(self.forward_fd, self.pending)
|
|
100
|
+
self.pending.clear()
|
|
101
|
+
else:
|
|
102
|
+
written = os.write(self.forward_fd, self.pending[:self.forward_chunk])
|
|
103
|
+
del self.pending[:written]
|
|
104
|
+
except OSError as exc:
|
|
105
|
+
self.drop_forward(str(exc))
|
|
106
|
+
|
|
107
|
+
def drop_forward(self, reason: str) -> None:
|
|
108
|
+
self.forward_error = self.forward_error or reason
|
|
109
|
+
self.pending.clear()
|
|
110
|
+
if self.forward_fd is not None:
|
|
111
|
+
os.close(self.forward_fd)
|
|
112
|
+
self.forward_fd = None
|
|
113
|
+
|
|
114
|
+
def close(self) -> None:
|
|
115
|
+
for fd in (self.save_fd, self.forward_fd):
|
|
116
|
+
if fd is not None:
|
|
117
|
+
os.close(fd)
|
|
118
|
+
self.save_fd = self.forward_fd = None
|
|
119
|
+
|
|
120
|
+
def result(self) -> StreamCapture:
|
|
121
|
+
return StreamCapture(
|
|
122
|
+
self.path, self.size, "sha256:" + self.hash.hexdigest(),
|
|
123
|
+
self.save_error, self.forward_error,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass
|
|
128
|
+
class _Stop:
|
|
129
|
+
sent_at: float | None = None
|
|
130
|
+
killed: bool = False
|
|
131
|
+
drained: bool = False
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _write_all(fd: int, chunk: bytes | bytearray) -> None:
|
|
135
|
+
view = memoryview(chunk)
|
|
136
|
+
while view:
|
|
137
|
+
view = view[os.write(fd, view):]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def run_captured(
|
|
141
|
+
argv: Sequence[str], cwd: Path, *, stdout_path: Path, stderr_path: Path,
|
|
142
|
+
stop_requested: Callable[[], bool], forward: tuple[int, int] = (1, 2),
|
|
143
|
+
drain_seconds: float = DRAIN_AFTER_EXIT_SECONDS,
|
|
144
|
+
grace_seconds: float = CLEANUP_GRACE_SECONDS,
|
|
145
|
+
) -> CommandCapture:
|
|
146
|
+
"""Run argv in its own process group, save and forward both streams.
|
|
147
|
+
|
|
148
|
+
Reads and forwarding writes share one select loop, so neither a full pipe
|
|
149
|
+
nor a destination that stops reading stalls the other stream, the
|
|
150
|
+
interruption or the drain deadline. That deadline is set once when the
|
|
151
|
+
main process exits; grandchild output never extends it.
|
|
152
|
+
"""
|
|
153
|
+
streams = (_Stream(stdout_path, forward[0]), _Stream(stderr_path, forward[1]))
|
|
154
|
+
try:
|
|
155
|
+
process = subprocess.Popen(
|
|
156
|
+
list(argv), cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
157
|
+
stderr=subprocess.PIPE, start_new_session=True,
|
|
158
|
+
)
|
|
159
|
+
except BaseException:
|
|
160
|
+
for stream in streams:
|
|
161
|
+
stream.close()
|
|
162
|
+
raise
|
|
163
|
+
stop = _Stop()
|
|
164
|
+
try:
|
|
165
|
+
_drain(process, streams, stop, stop_requested, drain_seconds, grace_seconds)
|
|
166
|
+
finally:
|
|
167
|
+
returncode, cleanup = _end_group(process, stop, grace_seconds)
|
|
168
|
+
for stream in streams:
|
|
169
|
+
stream.close()
|
|
170
|
+
return CommandCapture(
|
|
171
|
+
returncode, stop.drained, cleanup, stop.sent_at is not None,
|
|
172
|
+
streams[0].result(), streams[1].result(),
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _drain(
|
|
177
|
+
process: subprocess.Popen[bytes], streams: tuple[_Stream, _Stream],
|
|
178
|
+
stop: _Stop, stop_requested: Callable[[], bool], drain_seconds: float,
|
|
179
|
+
grace_seconds: float,
|
|
180
|
+
) -> None:
|
|
181
|
+
assert process.stdout is not None and process.stderr is not None
|
|
182
|
+
with selectors.DefaultSelector() as selector:
|
|
183
|
+
for pipe, stream in zip((process.stdout, process.stderr), streams):
|
|
184
|
+
selector.register(pipe, selectors.EVENT_READ, stream)
|
|
185
|
+
deadline: float | None = None
|
|
186
|
+
while (_reading(selector) or any(s.pending for s in streams)
|
|
187
|
+
or not _exited(process)):
|
|
188
|
+
_escalate(process, stop, stop_requested, grace_seconds)
|
|
189
|
+
_watch_forwarding(selector, streams)
|
|
190
|
+
_pump_once(selector)
|
|
191
|
+
if deadline is None and _exited(process):
|
|
192
|
+
deadline = time.monotonic() + drain_seconds
|
|
193
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
194
|
+
break
|
|
195
|
+
stop.drained = not _reading(selector)
|
|
196
|
+
for stream in streams:
|
|
197
|
+
if stream.pending:
|
|
198
|
+
stream.drop_forward("the forwarded copy was cut at the drain deadline")
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _reading(selector: selectors.BaseSelector) -> bool:
|
|
202
|
+
return any(key.events & selectors.EVENT_READ
|
|
203
|
+
for key in selector.get_map().values())
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _escalate(
|
|
207
|
+
process: subprocess.Popen[bytes], stop: _Stop,
|
|
208
|
+
stop_requested: Callable[[], bool], grace_seconds: float,
|
|
209
|
+
) -> None:
|
|
210
|
+
if stop.sent_at is None and stop_requested():
|
|
211
|
+
stop.sent_at = time.monotonic()
|
|
212
|
+
kill_group(process.pid, signal.SIGTERM)
|
|
213
|
+
if (stop.sent_at is not None and not stop.killed
|
|
214
|
+
and time.monotonic() - stop.sent_at >= grace_seconds):
|
|
215
|
+
stop.killed = True
|
|
216
|
+
kill_group(process.pid)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _watch_forwarding(
|
|
220
|
+
selector: selectors.BaseSelector, streams: tuple[_Stream, _Stream],
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Watch a destination for writability only while it has pending bytes.
|
|
223
|
+
|
|
224
|
+
A destination dropped since the last pass has a closed descriptor, so its
|
|
225
|
+
registration is removed before the next select.
|
|
226
|
+
"""
|
|
227
|
+
for key in list(selector.get_map().values()):
|
|
228
|
+
if key.events & selectors.EVENT_WRITE and (
|
|
229
|
+
key.data.forward_fd != key.fd or not key.data.pending):
|
|
230
|
+
selector.unregister(key.fileobj)
|
|
231
|
+
registered = selector.get_map()
|
|
232
|
+
for stream in streams:
|
|
233
|
+
fd = stream.forward_fd
|
|
234
|
+
if stream.pending and fd is not None and fd not in registered:
|
|
235
|
+
selector.register(fd, selectors.EVENT_WRITE, stream)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _pump_once(selector: selectors.BaseSelector) -> None:
|
|
239
|
+
if not selector.get_map():
|
|
240
|
+
time.sleep(_SELECT_SECONDS)
|
|
241
|
+
return
|
|
242
|
+
for key, events in selector.select(timeout=_SELECT_SECONDS):
|
|
243
|
+
stream: _Stream = key.data
|
|
244
|
+
if events & selectors.EVENT_WRITE:
|
|
245
|
+
if stream.forward_fd == key.fd:
|
|
246
|
+
stream.forward()
|
|
247
|
+
continue
|
|
248
|
+
chunk = os.read(key.fd, _READ_SIZE)
|
|
249
|
+
if chunk:
|
|
250
|
+
stream.feed(chunk)
|
|
251
|
+
else:
|
|
252
|
+
selector.unregister(key.fileobj)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _end_group(
|
|
256
|
+
process: subprocess.Popen[bytes], stop: _Stop, grace_seconds: float,
|
|
257
|
+
) -> tuple[int, str]:
|
|
258
|
+
"""Kill the group, reap the main process, and confirm the group gone.
|
|
259
|
+
|
|
260
|
+
The main process stays unreaped until the kill, so its pid, which is the
|
|
261
|
+
group id, cannot be reused by an unrelated group first. An interrupted
|
|
262
|
+
group keeps the rest of its grace even when its leader exited early.
|
|
263
|
+
"""
|
|
264
|
+
if stop.sent_at is not None and not stop.killed:
|
|
265
|
+
_await_no_live_member(process.pid, stop.sent_at + grace_seconds)
|
|
266
|
+
kill_group(process.pid)
|
|
267
|
+
returncode = process.wait()
|
|
268
|
+
for pipe in (process.stdout, process.stderr):
|
|
269
|
+
if pipe is not None:
|
|
270
|
+
pipe.close()
|
|
271
|
+
gone = _await_no_live_member(process.pid, time.monotonic() + grace_seconds)
|
|
272
|
+
return returncode, "confirmed" if gone else "unconfirmed"
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _exited(process: subprocess.Popen[bytes]) -> bool:
|
|
276
|
+
"""Whether the main process exited, observed without reaping it."""
|
|
277
|
+
try:
|
|
278
|
+
return os.waitid(
|
|
279
|
+
os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT,
|
|
280
|
+
) is not None
|
|
281
|
+
except ChildProcessError:
|
|
282
|
+
return process.poll() is not None
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _await_no_live_member(pgid: int, limit: float) -> bool:
|
|
286
|
+
while True:
|
|
287
|
+
if live_group_members(pgid) == 0:
|
|
288
|
+
return True
|
|
289
|
+
if time.monotonic() >= limit:
|
|
290
|
+
return False
|
|
291
|
+
time.sleep(_SELECT_SECONDS)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Select and validate actual conformance inputs for an immutable generation."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Mapping
|
|
8
|
+
|
|
9
|
+
from .conformance import (
|
|
10
|
+
conformance_entry_mismatches, normalize_conformance_script,
|
|
11
|
+
validate_conformance_manifest,
|
|
12
|
+
)
|
|
13
|
+
from .json_boundary import load_owned_object_snapshot
|
|
14
|
+
from .paths import RunRef
|
|
15
|
+
from .qa_procedures import content_digest, existing_qa_file, safe_qa_path
|
|
16
|
+
from .verification_input_types import FrozenVerificationPlan
|
|
17
|
+
|
|
18
|
+
_MAX_SCRIPT_PATH_LENGTH = 4096
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class GenerationConformance:
|
|
23
|
+
stages: tuple[int, ...]
|
|
24
|
+
artifact_paths: frozenset[str]
|
|
25
|
+
manifest: tuple[str, str] | None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _project_script_path(project: Path, task_root: Path, script: str) -> str:
|
|
29
|
+
script = normalize_conformance_script(script)
|
|
30
|
+
path = Path(script)
|
|
31
|
+
task_relative = task_root.relative_to(project)
|
|
32
|
+
if path.is_absolute():
|
|
33
|
+
if not path.is_relative_to(task_root):
|
|
34
|
+
raise ValueError(f"conformance script escapes task root: {script}")
|
|
35
|
+
path = path.relative_to(task_root)
|
|
36
|
+
elif path.is_relative_to(task_relative):
|
|
37
|
+
path = path.relative_to(task_relative)
|
|
38
|
+
relative = path.as_posix()
|
|
39
|
+
if (len(relative) > _MAX_SCRIPT_PATH_LENGTH
|
|
40
|
+
or not relative.startswith("qa/scripts/")):
|
|
41
|
+
raise ValueError(
|
|
42
|
+
f"conformance script must remain under task qa/scripts: {script}"
|
|
43
|
+
)
|
|
44
|
+
safe = safe_qa_path(task_root, relative)
|
|
45
|
+
return safe.relative_to(project).as_posix()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def conformance_stage(stage_key: str) -> int:
|
|
49
|
+
match = re.fullmatch(r".+-stage-([1-9][0-9]*)", stage_key)
|
|
50
|
+
if match is None:
|
|
51
|
+
raise ValueError("conformance stageKey must end in -stage-<positive integer>")
|
|
52
|
+
return int(match[1])
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _entries_by_stage(
|
|
56
|
+
manifest: Mapping[str, Any], declared_stages: set[int],
|
|
57
|
+
) -> dict[int, dict[str, Any]]:
|
|
58
|
+
entries = {}
|
|
59
|
+
for entry in manifest["entries"]:
|
|
60
|
+
stage = conformance_stage(entry["stageKey"])
|
|
61
|
+
if stage in entries:
|
|
62
|
+
raise ValueError(f"conformance stage {stage} has multiple entries")
|
|
63
|
+
if stage not in declared_stages:
|
|
64
|
+
raise ValueError(f"conformance stage {stage} is undeclared")
|
|
65
|
+
entries[stage] = entry
|
|
66
|
+
return entries
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _selected_stages(
|
|
70
|
+
ref: RunRef, declarations: Mapping[int, dict[str, Any]],
|
|
71
|
+
declared_paths: Mapping[int, str], actual_paths: Mapping[int, str],
|
|
72
|
+
baseline_paths: Mapping[int, str], changed_paths: frozenset[str],
|
|
73
|
+
) -> tuple[int, ...]:
|
|
74
|
+
if ref.task_type == "final-verification":
|
|
75
|
+
return tuple(sorted(declarations))
|
|
76
|
+
stage = ref.stage
|
|
77
|
+
if stage not in declarations or isinstance(stage, bool):
|
|
78
|
+
raise ValueError(f"conformance stage {stage} has no approved declaration")
|
|
79
|
+
selected = {stage}
|
|
80
|
+
for number in declarations:
|
|
81
|
+
if number < stage and any(
|
|
82
|
+
paths.get(number) in changed_paths
|
|
83
|
+
for paths in (declared_paths, actual_paths, baseline_paths)
|
|
84
|
+
):
|
|
85
|
+
selected.add(number)
|
|
86
|
+
return tuple(sorted(selected))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _check_selected_entry(
|
|
90
|
+
stage: int, declaration: Mapping[str, Any], entry: Mapping[str, Any] | None,
|
|
91
|
+
declared_paths: Mapping[int, str], actual_paths: Mapping[int, str],
|
|
92
|
+
) -> None:
|
|
93
|
+
if "script" not in declaration:
|
|
94
|
+
return
|
|
95
|
+
if entry is None:
|
|
96
|
+
raise ValueError(f"conformance stage {stage} has no matching manifest entry")
|
|
97
|
+
errors = conformance_entry_mismatches(
|
|
98
|
+
actual_script=actual_paths[stage], declared_script=declared_paths[stage],
|
|
99
|
+
actual_requires=entry.get("requires"),
|
|
100
|
+
declared_requires=frozenset(declaration["requires"]),
|
|
101
|
+
)
|
|
102
|
+
if errors:
|
|
103
|
+
raise ValueError(f"conformance stage {stage} {errors[0]}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _read_conformance_manifest(
|
|
107
|
+
project: Path, relative: str, declared_stages: set[int],
|
|
108
|
+
) -> tuple[dict[int, dict[str, Any]], tuple[str, str] | None]:
|
|
109
|
+
path = safe_qa_path(project, relative)
|
|
110
|
+
if not path.exists():
|
|
111
|
+
return {}, None
|
|
112
|
+
stored = load_owned_object_snapshot(
|
|
113
|
+
existing_qa_file(project, relative),
|
|
114
|
+
artifact="verification conformance manifest",
|
|
115
|
+
validate_cross_fields=validate_conformance_manifest,
|
|
116
|
+
)
|
|
117
|
+
return (
|
|
118
|
+
_entries_by_stage(stored.value, declared_stages),
|
|
119
|
+
(relative, content_digest(stored.raw_bytes)),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def conformance_manifest_path(project: Path, task_root: Path) -> str:
|
|
124
|
+
return (task_root / "qa/conformance-manifest.json").relative_to(
|
|
125
|
+
project,
|
|
126
|
+
).as_posix()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def capture_conformance_baseline(
|
|
130
|
+
project: Path, task_root: Path, plan: FrozenVerificationPlan,
|
|
131
|
+
) -> dict[str, str]:
|
|
132
|
+
entries, _ = _read_conformance_manifest(
|
|
133
|
+
project, conformance_manifest_path(project, task_root),
|
|
134
|
+
{row["stage"] for row in plan["conformance"]},
|
|
135
|
+
)
|
|
136
|
+
return {
|
|
137
|
+
str(stage): _project_script_path(project, task_root, entry["script"])
|
|
138
|
+
for stage, entry in sorted(entries.items())
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def capture_generation_conformance(
|
|
143
|
+
project: Path, ref: RunRef, plan: FrozenVerificationPlan,
|
|
144
|
+
changed_artifact_paths: frozenset[str], conformance_baseline: Mapping[int, str],
|
|
145
|
+
) -> GenerationConformance:
|
|
146
|
+
declarations = {row["stage"]: row for row in plan["conformance"]}
|
|
147
|
+
if any(type(stage) is not int or stage not in declarations
|
|
148
|
+
for stage in conformance_baseline):
|
|
149
|
+
raise ValueError("conformance baseline has an invalid or undeclared stage")
|
|
150
|
+
baseline_paths = {
|
|
151
|
+
stage: _project_script_path(project, ref.task_root, script)
|
|
152
|
+
for stage, script in conformance_baseline.items()
|
|
153
|
+
}
|
|
154
|
+
declared_paths = {
|
|
155
|
+
stage: _project_script_path(project, ref.task_root, row["script"])
|
|
156
|
+
for stage, row in declarations.items() if "script" in row
|
|
157
|
+
}
|
|
158
|
+
relative_manifest = conformance_manifest_path(project, ref.task_root)
|
|
159
|
+
entries, sealed_manifest = _read_conformance_manifest(
|
|
160
|
+
project, relative_manifest, set(declarations),
|
|
161
|
+
)
|
|
162
|
+
actual_paths = {
|
|
163
|
+
stage: _project_script_path(project, ref.task_root, row["script"])
|
|
164
|
+
for stage, row in entries.items()
|
|
165
|
+
}
|
|
166
|
+
stages = _selected_stages(
|
|
167
|
+
ref, declarations, declared_paths, actual_paths, baseline_paths,
|
|
168
|
+
changed_artifact_paths,
|
|
169
|
+
)
|
|
170
|
+
artifacts = {relative_manifest} if sealed_manifest is not None else set()
|
|
171
|
+
for stage in stages:
|
|
172
|
+
if (baseline_paths.get(stage) in changed_artifact_paths
|
|
173
|
+
and stage not in entries):
|
|
174
|
+
raise ValueError(f"changed conformance stage {stage} has no actual entry")
|
|
175
|
+
_check_selected_entry(
|
|
176
|
+
stage, declarations[stage], entries.get(stage),
|
|
177
|
+
declared_paths, actual_paths,
|
|
178
|
+
)
|
|
179
|
+
if stage in actual_paths:
|
|
180
|
+
existing_qa_file(project, actual_paths[stage])
|
|
181
|
+
artifacts.add(actual_paths[stage])
|
|
182
|
+
return GenerationConformance(stages, frozenset(artifacts), sealed_manifest)
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import subprocess
|
|
5
|
+
from typing import Any, Mapping
|
|
6
|
+
|
|
7
|
+
from .correction_round import completed_round_attempts, correction_round
|
|
8
|
+
from .execution_manifest import read_execution_manifest
|
|
9
|
+
from .paths import RunRef
|
|
10
|
+
from .qa_procedures import safe_qa_path
|
|
11
|
+
from .verification_generation import prepare_verification_generation
|
|
12
|
+
from .verification_input_ref import VerificationInputReference
|
|
13
|
+
from .verification_input_types import VerificationInputs
|
|
14
|
+
from .verification_inputs import (
|
|
15
|
+
frozen_conformance_baseline, read_verification_inputs,
|
|
16
|
+
)
|
|
17
|
+
from .verification_invocation import (
|
|
18
|
+
VerificationGenerationReference, verification_generation_required,
|
|
19
|
+
)
|
|
20
|
+
from .verification_target import read_worktree_head
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class VerificationDispatchError(ValueError):
|
|
24
|
+
"""A verifier dispatch whose generation cannot be sealed."""
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self, message: str, *, missing_executor_result: bool = False,
|
|
28
|
+
) -> None:
|
|
29
|
+
super().__init__(message)
|
|
30
|
+
self.missing_executor_result = missing_executor_result
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def seal_dispatch_generation(
|
|
34
|
+
project: Path, manifest_path: Path, manifest: Mapping[str, Any],
|
|
35
|
+
active_context: Mapping[str, Any], *, audience: str, dispatch_kind: str,
|
|
36
|
+
) -> VerificationGenerationReference | None:
|
|
37
|
+
try:
|
|
38
|
+
return _seal(
|
|
39
|
+
project, manifest_path, manifest, active_context,
|
|
40
|
+
audience=audience, dispatch_kind=dispatch_kind,
|
|
41
|
+
)
|
|
42
|
+
except VerificationDispatchError:
|
|
43
|
+
raise
|
|
44
|
+
except KeyError as exc:
|
|
45
|
+
raise VerificationDispatchError(
|
|
46
|
+
f"required field is missing: {exc.args[0]}",
|
|
47
|
+
) from exc
|
|
48
|
+
except (OSError, ValueError, subprocess.SubprocessError) as exc:
|
|
49
|
+
raise VerificationDispatchError(str(exc)) from exc
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def verification_generation_number(dispatch_kind: str) -> int:
|
|
53
|
+
"""Correction round N seals generation N + 1; every other dispatch seals 1."""
|
|
54
|
+
return 1 + (correction_round(dispatch_kind) or 0)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _seal(
|
|
58
|
+
project: Path, manifest_path: Path, manifest: Mapping[str, Any],
|
|
59
|
+
active_context: Mapping[str, Any], *, audience: str, dispatch_kind: str,
|
|
60
|
+
) -> VerificationGenerationReference | None:
|
|
61
|
+
if not verification_generation_required(manifest, audience):
|
|
62
|
+
return None
|
|
63
|
+
project = project.resolve()
|
|
64
|
+
reference = VerificationInputReference.required_from_artifact_fields(manifest)
|
|
65
|
+
round_number = correction_round(dispatch_kind)
|
|
66
|
+
implementation = manifest["taskType"] == "implementation"
|
|
67
|
+
if implementation:
|
|
68
|
+
changed = _executor_changes(manifest_path, dispatch_kind, round_number)
|
|
69
|
+
else:
|
|
70
|
+
changed = _final_verification_changes(round_number)
|
|
71
|
+
inputs = read_verification_inputs(
|
|
72
|
+
safe_qa_path(project, reference.relative_path), reference.digest,
|
|
73
|
+
)
|
|
74
|
+
head = (
|
|
75
|
+
read_worktree_head(Path(inputs["sourceRoot"])) if implementation
|
|
76
|
+
else str(active_context["verificationTarget"]["headRef"])
|
|
77
|
+
)
|
|
78
|
+
path, digest = prepare_verification_generation(
|
|
79
|
+
project, RunRef.from_manifest(manifest_path, manifest), reference,
|
|
80
|
+
generation=verification_generation_number(dispatch_kind),
|
|
81
|
+
expected_head=head,
|
|
82
|
+
source_inputs=_present_declared_sources(inputs),
|
|
83
|
+
generated_roots=frozenset(), artifact_inputs=frozenset(),
|
|
84
|
+
changed_artifact_paths=changed,
|
|
85
|
+
conformance_baseline=frozen_conformance_baseline(
|
|
86
|
+
inputs, manifest["taskType"],
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
return VerificationGenerationReference(
|
|
90
|
+
path.relative_to(project).as_posix(), digest,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _present_declared_sources(inputs: VerificationInputs) -> frozenset[str]:
|
|
95
|
+
# A related source the executor deleted is part of the change under review;
|
|
96
|
+
# only the files present at the seal can be protected from the verifiers.
|
|
97
|
+
root = Path(inputs["sourceRoot"])
|
|
98
|
+
return frozenset(
|
|
99
|
+
relative
|
|
100
|
+
for entries in inputs["qaCommands"].values() for entry in entries
|
|
101
|
+
for relative in entry.get("sourcePaths", ())
|
|
102
|
+
if (root / relative).exists() or (root / relative).is_symlink()
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _executor_changes(
|
|
107
|
+
manifest_path: Path, dispatch_kind: str, round_number: int | None,
|
|
108
|
+
) -> frozenset[str]:
|
|
109
|
+
attempts = completed_round_attempts(
|
|
110
|
+
read_execution_manifest(manifest_path), round_number, "implementer",
|
|
111
|
+
)
|
|
112
|
+
if not attempts:
|
|
113
|
+
raise VerificationDispatchError(
|
|
114
|
+
"the implementation executor has no ok result in dispatch "
|
|
115
|
+
f"{dispatch_kind}; dispatch the executor alone and await it before "
|
|
116
|
+
"its verifiers",
|
|
117
|
+
missing_executor_result=True,
|
|
118
|
+
)
|
|
119
|
+
return frozenset(
|
|
120
|
+
path for attempt in attempts
|
|
121
|
+
for path in attempt.change_summary.get("changedArtifactPaths", ())
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _final_verification_changes(round_number: int | None) -> frozenset[str]:
|
|
126
|
+
if round_number is not None:
|
|
127
|
+
raise ValueError(
|
|
128
|
+
"final-verification seals only its initial verifier dispatch"
|
|
129
|
+
)
|
|
130
|
+
return frozenset()
|