okstra 0.234.1 → 0.236.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-registry.mjs +7 -0
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/implementation-report/implementation-report.d.mts +1 -0
- package/dist/commands/implementation-report/implementation-report.mjs +73 -0
- package/dist/commands/implementation-report/implementation-report.mjs.map +1 -0
- package/dist/commands/trace/comparison.d.mts +39 -0
- package/dist/commands/trace/comparison.mjs +138 -0
- package/dist/commands/trace/comparison.mjs.map +1 -0
- package/dist/commands/trace/document.d.mts +8 -0
- package/dist/commands/trace/document.mjs +37 -0
- package/dist/commands/trace/document.mjs.map +1 -0
- package/dist/commands/trace/line-counts.d.mts +15 -0
- package/dist/commands/trace/line-counts.mjs +107 -0
- package/dist/commands/trace/line-counts.mjs.map +1 -0
- package/dist/commands/trace/model.d.mts +2 -1
- package/dist/commands/trace/model.mjs +13 -1
- package/dist/commands/trace/model.mjs.map +1 -1
- package/dist/commands/trace/progression.mjs +7 -3
- package/dist/commands/trace/progression.mjs.map +1 -1
- package/dist/commands/trace/task-link.d.mts +3 -0
- package/dist/commands/trace/task-link.mjs.map +1 -1
- package/dist/commands/trace/trace.mjs +64 -44
- package/dist/commands/trace/trace.mjs.map +1 -1
- package/dist/lib/host-registry-client.d.mts +9 -2
- package/dist/lib/host-registry-client.mjs +29 -3
- package/dist/lib/host-registry-client.mjs.map +1 -1
- package/dist/lib/python-helper.d.mts +1 -1
- package/dist/lib/python-helper.mjs +11 -5
- package/dist/lib/python-helper.mjs.map +1 -1
- package/dist/lib/runtime-payload.mjs +1 -0
- package/dist/lib/runtime-payload.mjs.map +1 -1
- package/dist/lib/types.d.mts +1 -0
- package/docs/architecture/storage-model.md +50 -0
- package/docs/architecture.md +1 -1
- package/docs/cli.md +37 -15
- package/docs/project-structure-overview.md +6 -1
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/agents/operations/trace-tests-review.json +6 -0
- package/runtime/bin/lib/okstra/usage.sh +8 -8
- package/runtime/bin/okstra-qwen-exec.sh +5 -0
- package/runtime/prompts/duties/test-reviewer.json +32 -0
- package/runtime/prompts/lead/convergence.md +4 -1
- package/runtime/prompts/lead/okstra-lead-contract.md +3 -3
- package/runtime/prompts/lead/report-writer.md +13 -1
- package/runtime/prompts/profiles/_common-contract.md +10 -1
- package/runtime/prompts/wizard/prompts.ko.json +16 -4
- package/runtime/python/okstra_ctl/adapters/providers/antigravity/adapter.py +10 -2
- package/runtime/python/okstra_ctl/adapters/providers/claude/adapter.py +16 -1
- package/runtime/python/okstra_ctl/adapters/providers/codex/adapter.py +20 -5
- package/runtime/python/okstra_ctl/adapters/providers/kimi/adapter.py +17 -9
- package/runtime/python/okstra_ctl/adapters/providers/qwen/adapter.py +88 -0
- package/runtime/python/okstra_ctl/adapters/providers/qwen/manifest.json +1 -0
- package/runtime/python/okstra_ctl/adapters/providers/qwen/settings.json +12 -0
- package/runtime/python/okstra_ctl/adapters/providers/zai/adapter.py +8 -2
- package/runtime/python/okstra_ctl/agent/invocation.py +109 -56
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +17 -0
- package/runtime/python/okstra_ctl/attempt_evidence.py +5 -2
- package/runtime/python/okstra_ctl/business_flow/contracts.py +11 -0
- package/runtime/python/okstra_ctl/business_flow/engine.py +90 -32
- package/runtime/python/okstra_ctl/business_flow/hooks.py +4 -2
- package/runtime/python/okstra_ctl/business_flow/source.py +39 -7
- package/runtime/python/okstra_ctl/business_flow/store.py +94 -28
- package/runtime/python/okstra_ctl/code_review_census.py +38 -0
- package/runtime/python/okstra_ctl/code_review_coverage.py +15 -4
- package/runtime/python/okstra_ctl/code_review_prepare.py +15 -3
- package/runtime/python/okstra_ctl/code_review_scope.py +3 -2
- package/runtime/python/okstra_ctl/code_review_target.py +16 -6
- package/runtime/python/okstra_ctl/conformance_evidence.py +44 -21
- package/runtime/python/okstra_ctl/consumers.py +6 -1
- package/runtime/python/okstra_ctl/convergence.py +95 -5
- package/runtime/python/okstra_ctl/convergence_critic_verify_prompt.py +11 -10
- package/runtime/python/okstra_ctl/convergence_engine.py +30 -15
- package/runtime/python/okstra_ctl/convergence_reverify_prompt.py +9 -2
- package/runtime/python/okstra_ctl/coverage_census.py +75 -12
- package/runtime/python/okstra_ctl/design_review.py +6 -26
- package/runtime/python/okstra_ctl/design_review_evidence.py +6 -26
- package/runtime/python/okstra_ctl/dispatch_core.py +9 -2
- package/runtime/python/okstra_ctl/dispatch_state.py +5 -0
- package/runtime/python/okstra_ctl/domain/role.py +1 -0
- package/runtime/python/okstra_ctl/entrypoints/hosts.py +1 -1
- package/runtime/python/okstra_ctl/execution_manifest.py +8 -0
- package/runtime/python/okstra_ctl/implementation_comparison/__init__.py +12 -0
- package/runtime/python/okstra_ctl/implementation_comparison/__main__.py +129 -0
- package/runtime/python/okstra_ctl/implementation_comparison/assembly.py +340 -0
- package/runtime/python/okstra_ctl/implementation_comparison/baseline.py +254 -0
- package/runtime/python/okstra_ctl/implementation_comparison/business_reuse.py +118 -0
- package/runtime/python/okstra_ctl/implementation_comparison/commit_source.py +113 -0
- package/runtime/python/okstra_ctl/implementation_comparison/html.py +49 -0
- package/runtime/python/okstra_ctl/implementation_comparison/inputs.py +348 -0
- package/runtime/python/okstra_ctl/implementation_comparison/model.py +47 -0
- package/runtime/python/okstra_ctl/implementation_comparison/preparation.py +195 -0
- package/runtime/python/okstra_ctl/implementation_comparison/publication.py +317 -0
- package/runtime/python/okstra_ctl/implementation_comparison/run_evidence.py +208 -0
- package/runtime/python/okstra_ctl/implementation_comparison/run_integration.py +198 -0
- package/runtime/python/okstra_ctl/implementation_comparison/source.py +342 -0
- package/runtime/python/okstra_ctl/implementation_comparison/target.py +144 -0
- package/runtime/python/okstra_ctl/implementation_comparison/task_assembly.py +333 -0
- package/runtime/python/okstra_ctl/implementation_comparison/task_selection.py +309 -0
- package/runtime/python/okstra_ctl/implementation_generation_evidence.py +134 -0
- package/runtime/python/okstra_ctl/implementation_outcome.py +2 -2
- package/runtime/python/okstra_ctl/initial_prompt_materialization.py +1 -0
- package/runtime/python/okstra_ctl/json_boundary.py +28 -8
- package/runtime/python/okstra_ctl/lead_progress.py +1 -1
- package/runtime/python/okstra_ctl/model_discovery.py +1 -1
- package/runtime/python/okstra_ctl/models.py +2 -2
- package/runtime/python/okstra_ctl/paths.py +7 -4
- package/runtime/python/okstra_ctl/phases/discussion/profile.md +2 -2
- package/runtime/python/okstra_ctl/phases/error_analysis/profile.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation_option_selection/profile.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation_planning/authoring.py +2 -1
- package/runtime/python/okstra_ctl/phases/implementation_planning/instructions/plan-body-verification.md +1 -1
- package/runtime/python/okstra_ctl/phases/implementation_planning/profile.md +3 -3
- package/runtime/python/okstra_ctl/phases/requirements_discovery/profile.md +5 -5
- package/runtime/python/okstra_ctl/plan_run_root.py +8 -4
- package/runtime/python/okstra_ctl/qa_procedures.py +33 -2
- package/runtime/python/okstra_ctl/registry/provider_registry.py +1 -1
- package/runtime/python/okstra_ctl/render.py +4 -0
- package/runtime/python/okstra_ctl/render_final_report.py +1 -1
- package/runtime/python/okstra_ctl/report_execution_evidence.py +88 -15
- package/runtime/python/okstra_ctl/report_finalize.py +18 -4
- package/runtime/python/okstra_ctl/report_html/render.py +46 -7
- package/runtime/python/okstra_ctl/report_html/validation.py +32 -0
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +37 -3
- package/runtime/python/okstra_ctl/report_view_artifacts.py +12 -0
- package/runtime/python/okstra_ctl/review_verdicts.py +9 -0
- package/runtime/python/okstra_ctl/run.py +17 -1
- package/runtime/python/okstra_ctl/run_context.py +18 -3
- package/runtime/python/okstra_ctl/schema_excerpt.py +14 -9
- package/runtime/python/okstra_ctl/stage_done.py +11 -3
- package/runtime/python/okstra_ctl/stage_ledger.py +2 -2
- package/runtime/python/okstra_ctl/stage_map.py +3 -3
- package/runtime/python/okstra_ctl/trace_review.py +1 -0
- package/runtime/python/okstra_ctl/trace_tasks.py +22 -7
- package/runtime/python/okstra_ctl/verdict_blocks.py +7 -2
- package/runtime/python/okstra_ctl/verification_dispatch.py +32 -0
- package/runtime/python/okstra_ctl/verification_evidence.py +23 -3
- package/runtime/python/okstra_ctl/verification_generation.py +3 -2
- package/runtime/python/okstra_ctl/verification_inputs.py +3 -2
- package/runtime/python/okstra_ctl/verification_invocation.py +7 -3
- package/runtime/python/okstra_ctl/wizard/engine.py +28 -1
- package/runtime/python/okstra_ctl/wizard/ids.py +8 -0
- package/runtime/python/okstra_ctl/wizard/picker_navigation.py +64 -2
- package/runtime/python/okstra_ctl/wizard/roles.py +194 -45
- package/runtime/python/okstra_ctl/wizard/sources.py +10 -1
- package/runtime/python/okstra_ctl/wizard/state.py +67 -20
- package/runtime/python/okstra_ctl/wizard/steps_roles.py +6 -3
- package/runtime/python/okstra_ctl/wizard/worker_settings.py +129 -25
- package/runtime/python/okstra_ctl/worker_audit_ledger.py +44 -8
- package/runtime/python/okstra_ctl/worker_prompt_body.py +26 -2
- package/runtime/python/okstra_ctl/worker_prompt_headers.py +12 -9
- package/runtime/python/okstra_token_usage/claude.py +5 -1
- package/runtime/python/okstra_token_usage/collect.py +3 -3
- package/runtime/python/okstra_token_usage/pricing.py +1 -0
- package/runtime/schemas/report-synthesis-packet-v1.0.schema.json +17 -0
- package/runtime/skills/_fragments/wizard-relay-protocol.md +1 -1
- package/runtime/skills/okstra-code-review/SKILL.md +1 -1
- package/runtime/skills/okstra-code-review/references/census-rules.md +1 -1
- package/runtime/skills/okstra-code-review/references/review-calibration.md +1 -1
- package/runtime/skills/okstra-discussion/SKILL.md +1 -1
- package/runtime/skills/okstra-run/SKILL.md +33 -2
- package/runtime/skills/okstra-tdd/SKILL.md +74 -13
- package/runtime/templates/reports/html/assets/implementation-comparison.css +39 -0
- package/runtime/templates/reports/html/assets/implementation-comparison.js +55 -0
- package/runtime/templates/reports/html/i18n/en.json +113 -0
- package/runtime/templates/reports/html/i18n/ko.json +113 -0
- package/runtime/templates/reports/html/implementation-comparison.template.html +77 -0
- package/runtime/templates/reports/implementation-comparison-notes.md +72 -0
- package/runtime/templates/trace/trace.template.md +2 -0
- package/runtime/templates/worker-prompt-preamble.md +12 -2
- package/runtime/validators/validate-report-views.py +4 -36
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Bounded, read-only Git and working-source collection."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
import selectors
|
|
7
|
+
import signal
|
|
8
|
+
import stat
|
|
9
|
+
import subprocess
|
|
10
|
+
import time
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Any, Iterator
|
|
15
|
+
|
|
16
|
+
from ..json_boundary import PageBudgetExhausted, ReadBudget
|
|
17
|
+
from ..process_group import kill_group
|
|
18
|
+
|
|
19
|
+
MAX_PATHS = 10_000
|
|
20
|
+
MAX_FILE_READ_BYTES = 1024 * 1024
|
|
21
|
+
MAX_TOTAL_READ_BYTES = 64 * 1024 * 1024
|
|
22
|
+
MAX_GIT_COMMAND_BYTES = 4 * 1024 * 1024
|
|
23
|
+
MAX_GIT_TOTAL_BYTES = 8 * 1024 * 1024
|
|
24
|
+
GIT_TIMEOUT_SECONDS = 10
|
|
25
|
+
READ_CHUNK_BYTES = 64 * 1024
|
|
26
|
+
SOURCE_CHANGED_DURING_READ = "Source changed while being read"
|
|
27
|
+
EXCLUDED_PARTS = frozenset({
|
|
28
|
+
".git", ".okstra", ".project-docs", ".worktrees", "node_modules", "vendor",
|
|
29
|
+
".venv", "venv", "__pycache__", ".pytest_cache", ".cache", ".mypy_cache",
|
|
30
|
+
".ruff_cache", "dist", "build", "runtime", "target", "coverage", ".next",
|
|
31
|
+
"graphify-out",
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def digest_bytes(body: bytes) -> str:
|
|
36
|
+
return hashlib.sha256(body).hexdigest()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class GitOutput:
|
|
41
|
+
stdout: bytes
|
|
42
|
+
stderr: bytes
|
|
43
|
+
returncode: int | None
|
|
44
|
+
complete: bool
|
|
45
|
+
reason: str = ""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class GitCollector:
|
|
49
|
+
def __init__(
|
|
50
|
+
self, directory: Path | None = None, *, shared_budget: ReadBudget | None = None,
|
|
51
|
+
deadline: float | None = None,
|
|
52
|
+
) -> None:
|
|
53
|
+
self.directory = directory
|
|
54
|
+
self.budget = ReadBudget(MAX_GIT_TOTAL_BYTES)
|
|
55
|
+
self.shared_budget = shared_budget
|
|
56
|
+
self.deadline = deadline
|
|
57
|
+
self.outputs: dict[str, dict[str, Any]] = {}
|
|
58
|
+
|
|
59
|
+
def run(self, root: Path, label: str, *args: str) -> GitOutput:
|
|
60
|
+
if self.shared_budget is not None and self.shared_budget.bytes_read >= self.shared_budget.max_bytes:
|
|
61
|
+
raise PageBudgetExhausted("Comparison source byte budget exhausted")
|
|
62
|
+
if self.budget.bytes_read >= self.budget.max_bytes:
|
|
63
|
+
result = GitOutput(b"", b"", None, False, "Git output budget exhausted")
|
|
64
|
+
elif self.deadline is not None and time.monotonic() >= self.deadline:
|
|
65
|
+
result = GitOutput(b"", b"", None, False, "Git request deadline exhausted")
|
|
66
|
+
else:
|
|
67
|
+
result = self._read_command(root, args)
|
|
68
|
+
files: dict[str, Any] = {}
|
|
69
|
+
for name, body in (("stdout", result.stdout), ("stderr", result.stderr)):
|
|
70
|
+
relative = f"git/{label}.{name}"
|
|
71
|
+
if self.directory is not None:
|
|
72
|
+
(self.directory / relative).write_bytes(body)
|
|
73
|
+
files[name] = {"path": relative, "bytes": len(body), "storedDigest": digest_bytes(body)}
|
|
74
|
+
self.outputs[label] = {
|
|
75
|
+
"argv": list(args), "returncode": result.returncode,
|
|
76
|
+
"complete": result.complete, "reason": result.reason, **files,
|
|
77
|
+
}
|
|
78
|
+
return result
|
|
79
|
+
|
|
80
|
+
def _remaining(self, command_bytes: int) -> int:
|
|
81
|
+
remaining = min(MAX_GIT_COMMAND_BYTES - command_bytes,
|
|
82
|
+
self.budget.max_bytes - self.budget.bytes_read)
|
|
83
|
+
if self.shared_budget is not None:
|
|
84
|
+
remaining = min(remaining, self.shared_budget.max_bytes - self.shared_budget.bytes_read)
|
|
85
|
+
return remaining
|
|
86
|
+
|
|
87
|
+
def _read_command(self, root: Path, args: tuple[str, ...]) -> GitOutput:
|
|
88
|
+
env = {key: value for key, value in os.environ.items() if key not in {
|
|
89
|
+
"GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX",
|
|
90
|
+
"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
|
91
|
+
}}
|
|
92
|
+
command = ["git", "--no-pager", "-c", "core.fsmonitor=false", "-C", str(root), *args]
|
|
93
|
+
process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
94
|
+
stderr=subprocess.PIPE, env=env, start_new_session=True)
|
|
95
|
+
bodies = [bytearray(), bytearray()]
|
|
96
|
+
reason = ""
|
|
97
|
+
deadline = time.monotonic() + GIT_TIMEOUT_SECONDS
|
|
98
|
+
if self.deadline is not None:
|
|
99
|
+
deadline = min(deadline, self.deadline)
|
|
100
|
+
try:
|
|
101
|
+
with selectors.DefaultSelector() as selector:
|
|
102
|
+
for index, pipe in enumerate((process.stdout, process.stderr)):
|
|
103
|
+
selector.register(pipe, selectors.EVENT_READ, index)
|
|
104
|
+
while selector.get_map():
|
|
105
|
+
remaining = self._remaining(sum(map(len, bodies)))
|
|
106
|
+
if remaining <= 0 or time.monotonic() >= deadline:
|
|
107
|
+
reason = "Git output limit reached" if remaining <= 0 else "Git command timed out"
|
|
108
|
+
break
|
|
109
|
+
for key, _ in selector.select(timeout=min(0.1, max(0, deadline - time.monotonic()))):
|
|
110
|
+
remaining = self._remaining(sum(map(len, bodies)))
|
|
111
|
+
if remaining <= 0:
|
|
112
|
+
break
|
|
113
|
+
chunk = os.read(key.fd, min(READ_CHUNK_BYTES, remaining))
|
|
114
|
+
self.budget.bytes_read += len(chunk)
|
|
115
|
+
if self.shared_budget is not None and self.shared_budget is not self.budget:
|
|
116
|
+
self.shared_budget.bytes_read += len(chunk)
|
|
117
|
+
bodies[key.data].extend(chunk)
|
|
118
|
+
remaining -= len(chunk)
|
|
119
|
+
if not chunk:
|
|
120
|
+
selector.unregister(key.fileobj)
|
|
121
|
+
if reason:
|
|
122
|
+
kill_group(process.pid, signal.SIGKILL)
|
|
123
|
+
try:
|
|
124
|
+
returncode = process.wait(timeout=max(0, deadline - time.monotonic()))
|
|
125
|
+
except subprocess.TimeoutExpired:
|
|
126
|
+
reason = reason or "Git command timed out"
|
|
127
|
+
kill_group(process.pid, signal.SIGKILL)
|
|
128
|
+
returncode = process.wait()
|
|
129
|
+
finally:
|
|
130
|
+
if process.poll() is None:
|
|
131
|
+
kill_group(process.pid, signal.SIGKILL)
|
|
132
|
+
process.wait()
|
|
133
|
+
for pipe in (process.stdout, process.stderr):
|
|
134
|
+
if pipe is not None:
|
|
135
|
+
pipe.close()
|
|
136
|
+
return GitOutput(bytes(bodies[0]), bytes(bodies[1]), returncode, not reason, reason)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@contextmanager
|
|
140
|
+
def _source_parent(root: Path, relative: str) -> Iterator[tuple[int, str]]:
|
|
141
|
+
parts = PurePosixPath(relative).parts
|
|
142
|
+
if not parts or PurePosixPath(relative).is_absolute() or any(p in {".", ".."} for p in parts):
|
|
143
|
+
raise ValueError("Source path is not relative to its root")
|
|
144
|
+
directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
|
145
|
+
try:
|
|
146
|
+
for part in parts[:-1]:
|
|
147
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory)
|
|
148
|
+
os.close(directory)
|
|
149
|
+
directory = child
|
|
150
|
+
yield directory, parts[-1]
|
|
151
|
+
finally:
|
|
152
|
+
os.close(directory)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def open_source_file(root: Path, relative: str) -> int:
|
|
156
|
+
with _source_parent(root, relative) as (directory, name):
|
|
157
|
+
return os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _source_info(root: Path, relative: str) -> tuple[os.stat_result, str | None]:
|
|
161
|
+
with _source_parent(root, relative) as (directory, name):
|
|
162
|
+
info = os.stat(name, dir_fd=directory, follow_symlinks=False)
|
|
163
|
+
return info, os.readlink(name, dir_fd=directory) if stat.S_ISLNK(info.st_mode) else None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def read_source_bytes(
|
|
167
|
+
root: Path, relative: str, budget: ReadBudget, *, file_limit: int | None = None,
|
|
168
|
+
shared_budget: ReadBudget | None = None,
|
|
169
|
+
) -> tuple[bytes | None, os.stat_result, str]:
|
|
170
|
+
descriptor = open_source_file(root, relative)
|
|
171
|
+
try:
|
|
172
|
+
before = os.fstat(descriptor)
|
|
173
|
+
if not stat.S_ISREG(before.st_mode):
|
|
174
|
+
return None, before, "Source is not a regular file"
|
|
175
|
+
maximum = min(MAX_FILE_READ_BYTES if file_limit is None else file_limit,
|
|
176
|
+
budget.max_bytes - budget.bytes_read)
|
|
177
|
+
if shared_budget is not None and before.st_size > shared_budget.max_bytes - shared_budget.bytes_read:
|
|
178
|
+
raise PageBudgetExhausted("Comparison source byte budget exhausted")
|
|
179
|
+
if before.st_size > maximum:
|
|
180
|
+
return None, before, "Source exceeds the remaining file read budget"
|
|
181
|
+
body = bytearray()
|
|
182
|
+
while len(body) < before.st_size:
|
|
183
|
+
chunk = os.read(descriptor, min(READ_CHUNK_BYTES, before.st_size - len(body)))
|
|
184
|
+
budget.bytes_read += len(chunk)
|
|
185
|
+
if shared_budget is not None and shared_budget is not budget:
|
|
186
|
+
shared_budget.bytes_read += len(chunk)
|
|
187
|
+
if not chunk:
|
|
188
|
+
break
|
|
189
|
+
body.extend(chunk)
|
|
190
|
+
after = os.fstat(descriptor)
|
|
191
|
+
if file_state(before) != file_state(after) or len(body) != before.st_size:
|
|
192
|
+
return None, after, SOURCE_CHANGED_DURING_READ
|
|
193
|
+
return bytes(body), after, ""
|
|
194
|
+
finally:
|
|
195
|
+
os.close(descriptor)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def file_state(info: os.stat_result) -> tuple[int, int, int, int, int]:
|
|
199
|
+
return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _names(output: GitOutput) -> list[bytes]:
|
|
203
|
+
return output.stdout.split(b"\0") if output.complete and output.returncode == 0 else []
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _pathspecs() -> list[str]:
|
|
207
|
+
return ["--", ".", *[f":(glob,exclude)**/{part}/**" for part in sorted(EXCLUDED_PARTS)]]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def collect_source(project_root: Path, source_root: Path, requested_ref: str, directory: Path) -> dict[str, Any]:
|
|
211
|
+
(directory / "git").mkdir()
|
|
212
|
+
(directory / "contents").mkdir()
|
|
213
|
+
git = GitCollector(directory)
|
|
214
|
+
budget = ReadBudget(MAX_TOTAL_READ_BYTES)
|
|
215
|
+
result: dict[str, Any] = {
|
|
216
|
+
"head": None, "baselineCommit": None, "files": [], "limitations": [],
|
|
217
|
+
"excludedParts": sorted(EXCLUDED_PARTS), "completeInventory": False,
|
|
218
|
+
"scope": {"workingTreeText": "full-within-read-limits",
|
|
219
|
+
"index": "binary-patch-and-stage-object-identifiers",
|
|
220
|
+
"commitTree": "commit-identity-only"},
|
|
221
|
+
}
|
|
222
|
+
project_git = git.run(project_root, "project-repository", "rev-parse", "--git-common-dir")
|
|
223
|
+
source_git = project_git if project_root == source_root else git.run(
|
|
224
|
+
source_root, "source-repository", "rev-parse", "--git-common-dir",
|
|
225
|
+
)
|
|
226
|
+
if any(not item.complete or item.returncode != 0 for item in (project_git, source_git)):
|
|
227
|
+
result["limitations"].append("Git repository is unavailable; non-Git source enumeration was not attempted")
|
|
228
|
+
return _finish_source(result, git, budget, 0)
|
|
229
|
+
project_common = (project_root / os.fsdecode(project_git.stdout).strip()).resolve()
|
|
230
|
+
source_common = (source_root / os.fsdecode(source_git.stdout).strip()).resolve()
|
|
231
|
+
if project_common != source_common:
|
|
232
|
+
raise ValueError("Source root belongs to a different Git repository")
|
|
233
|
+
head = git.run(source_root, "head-before", "rev-parse", "--verify", "HEAD^{commit}")
|
|
234
|
+
baseline = head if not requested_ref else git.run(
|
|
235
|
+
source_root, "baseline-commit", "rev-parse", "--verify", "--end-of-options", f"{requested_ref}^{{commit}}",
|
|
236
|
+
)
|
|
237
|
+
for key, value in (("head", head), ("baselineCommit", baseline)):
|
|
238
|
+
if value.complete and value.returncode == 0:
|
|
239
|
+
result[key] = value.stdout.decode("ascii").strip()
|
|
240
|
+
else:
|
|
241
|
+
result["limitations"].append(f"{key} could not be resolved")
|
|
242
|
+
before = _git_state(git, source_root, "before")
|
|
243
|
+
entries: dict[str, dict[str, Any]] = {}
|
|
244
|
+
paths_omitted = False
|
|
245
|
+
for raw in _names(before["tracked"]):
|
|
246
|
+
if not raw:
|
|
247
|
+
continue
|
|
248
|
+
entry, name = raw.split(b"\t", 1)
|
|
249
|
+
path = os.fsdecode(name)
|
|
250
|
+
if any(part in EXCLUDED_PARTS for part in PurePosixPath(path).parts):
|
|
251
|
+
continue
|
|
252
|
+
if path not in entries and len(entries) >= MAX_PATHS:
|
|
253
|
+
paths_omitted = True
|
|
254
|
+
continue
|
|
255
|
+
mode, object_id, stage = entry.decode("ascii").split()
|
|
256
|
+
row = entries.setdefault(path, {"path": path, "tracked": True, "indexEntries": []})
|
|
257
|
+
row["indexEntries"].append({"mode": mode, "object": object_id, "stage": stage})
|
|
258
|
+
for raw in _names(before["untracked"]):
|
|
259
|
+
if not raw:
|
|
260
|
+
continue
|
|
261
|
+
path = os.fsdecode(raw)
|
|
262
|
+
if any(part in EXCLUDED_PARTS for part in PurePosixPath(path).parts):
|
|
263
|
+
continue
|
|
264
|
+
if path not in entries and len(entries) >= MAX_PATHS:
|
|
265
|
+
paths_omitted = True
|
|
266
|
+
continue
|
|
267
|
+
entries.setdefault(path, {"path": path, "tracked": False})
|
|
268
|
+
complete_inventory = all(value.complete and value.returncode == 0 for value in before.values())
|
|
269
|
+
if paths_omitted:
|
|
270
|
+
complete_inventory = False
|
|
271
|
+
result["limitations"].append("Source path limit reached; inventory may be incomplete")
|
|
272
|
+
if not complete_inventory:
|
|
273
|
+
result["limitations"].append("Git inventory, status, or index capture is incomplete")
|
|
274
|
+
result["completeInventory"] = complete_inventory
|
|
275
|
+
saved_bytes = 0
|
|
276
|
+
states: dict[str, tuple[int, int, int, int, int]] = {}
|
|
277
|
+
for path, entry in entries.items():
|
|
278
|
+
row = {**entry, "digest": None, "contentPath": None}
|
|
279
|
+
try:
|
|
280
|
+
info, link = _source_info(source_root, path)
|
|
281
|
+
if link is not None:
|
|
282
|
+
body, problem = None, "Symbolic link source was not followed"
|
|
283
|
+
row.update({"kind": "symlink", "linkTarget": link})
|
|
284
|
+
else:
|
|
285
|
+
body, info, problem = read_source_bytes(source_root, path, budget)
|
|
286
|
+
row.update({"mode": info.st_mode, "size": info.st_size})
|
|
287
|
+
states[path] = file_state(info)
|
|
288
|
+
if problem:
|
|
289
|
+
row.update({"status": "partial", "reason": problem})
|
|
290
|
+
else:
|
|
291
|
+
row.update({"status": "complete", "digest": digest_bytes(body)})
|
|
292
|
+
try:
|
|
293
|
+
text = body.decode("utf-8")
|
|
294
|
+
is_text = "\0" not in text
|
|
295
|
+
except UnicodeDecodeError:
|
|
296
|
+
is_text = False
|
|
297
|
+
row["kind"] = "text" if is_text else "binary"
|
|
298
|
+
if is_text:
|
|
299
|
+
relative = f"contents/{row['digest']}.txt"
|
|
300
|
+
(directory / relative).write_bytes(body)
|
|
301
|
+
row["contentPath"] = relative
|
|
302
|
+
saved_bytes += len(body)
|
|
303
|
+
except FileNotFoundError:
|
|
304
|
+
row.update({"status": "missing", "reason": "Source was absent during capture"})
|
|
305
|
+
except (OSError, ValueError) as exc:
|
|
306
|
+
row.update({"status": "partial", "reason": f"Source was not followed or could not be read: {exc}"})
|
|
307
|
+
result["files"].append(row)
|
|
308
|
+
if row["status"] not in {"complete", "missing"}:
|
|
309
|
+
result["limitations"].append(f"{path}: {row['reason']}")
|
|
310
|
+
after = _git_state(git, source_root, "after")
|
|
311
|
+
after_head = git.run(source_root, "head-after", "rev-parse", "--verify", "HEAD^{commit}")
|
|
312
|
+
if after_head != head:
|
|
313
|
+
result["limitations"].append("HEAD changed or could not be rechecked during capture")
|
|
314
|
+
for key, previous in before.items():
|
|
315
|
+
current = after[key]
|
|
316
|
+
if not current.complete or current.returncode != 0 or current.stdout != previous.stdout:
|
|
317
|
+
result["limitations"].append(f"Git {key} changed or could not be rechecked during capture")
|
|
318
|
+
for path, previous in states.items():
|
|
319
|
+
try:
|
|
320
|
+
if file_state(_source_info(source_root, path)[0]) != previous:
|
|
321
|
+
result["limitations"].append(f"{path}: source changed after its content was captured")
|
|
322
|
+
except OSError:
|
|
323
|
+
result["limitations"].append(f"{path}: source disappeared after its content was captured")
|
|
324
|
+
return _finish_source(result, git, budget, saved_bytes)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _git_state(git: GitCollector, root: Path, suffix: str) -> dict[str, GitOutput]:
|
|
328
|
+
return {
|
|
329
|
+
"tracked": git.run(root, f"tracked-{suffix}", "ls-files", "--stage", "-z", *_pathspecs()),
|
|
330
|
+
"untracked": git.run(root, f"untracked-{suffix}", "ls-files", "--others", "--exclude-standard", "-z", *_pathspecs()),
|
|
331
|
+
"status": git.run(root, f"status-{suffix}", "status", "--porcelain=v1", "-z", "--untracked-files=all", *_pathspecs()),
|
|
332
|
+
"index": git.run(root, f"index-{suffix}", "diff", "--no-ext-diff", "--no-textconv", "--cached", "--binary", *_pathspecs()),
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _finish_source(result: dict[str, Any], git: GitCollector, budget: ReadBudget, saved: int) -> dict[str, Any]:
|
|
337
|
+
result.update({"git": git.outputs, "bytesRead": budget.bytes_read,
|
|
338
|
+
"bytesPreserved": saved, "gitBytesRead": git.budget.bytes_read})
|
|
339
|
+
result["status"] = "partial" if result["limitations"] else "complete"
|
|
340
|
+
if not result["completeInventory"] and not result["files"]:
|
|
341
|
+
result["status"] = "unavailable"
|
|
342
|
+
return result
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Generation-bound immutable implementation targets."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..json_boundary import ReadBudget, load_owned_object_snapshot, write_owned_object_atomic
|
|
12
|
+
from ..run_context import dir_flock
|
|
13
|
+
from ..trace_tasks import read_trace_source, trace_integration_completed
|
|
14
|
+
from .baseline import (
|
|
15
|
+
MAX_BASELINE_RESPONSE_BYTES, ComparisonError, ComparisonOwner, _capture_source_record, _owner_and_directory,
|
|
16
|
+
_read_capture, _response,
|
|
17
|
+
)
|
|
18
|
+
from .source import digest_bytes
|
|
19
|
+
from .inputs import _validate_source
|
|
20
|
+
from .model import MAX_COMPARISON_READ_BYTES
|
|
21
|
+
|
|
22
|
+
GENERATION_PATTERN = re.compile(r"[a-f0-9]{64}\Z")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _target_directory(directory: Path, generation: str) -> Path:
|
|
26
|
+
if not isinstance(generation, str) or not GENERATION_PATTERN.fullmatch(generation):
|
|
27
|
+
raise ComparisonError("Target generation must be a settled input SHA-256")
|
|
28
|
+
path = directory
|
|
29
|
+
for part in ("targets", generation):
|
|
30
|
+
path /= part
|
|
31
|
+
if path.is_symlink():
|
|
32
|
+
raise ComparisonError("Target paths must not follow symbolic links")
|
|
33
|
+
return path
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _read_target_pointer(directory: Path, owner: dict[str, Any], *, budget: ReadBudget | None = None) -> tuple[dict[str, Any] | None, str | None]:
|
|
37
|
+
path = directory / "target-current.json"
|
|
38
|
+
if not path.exists() and not path.is_symlink():
|
|
39
|
+
return None, None
|
|
40
|
+
if path.is_symlink():
|
|
41
|
+
raise ComparisonError("Target pointer must not follow symbolic links")
|
|
42
|
+
snapshot = load_owned_object_snapshot(path, artifact="implementation target pointer", max_bytes=16 * 1024, budget=budget)
|
|
43
|
+
pointer = snapshot.value
|
|
44
|
+
if pointer.get("owner") != owner or pointer.get("schemaVersion") != "1.0":
|
|
45
|
+
raise ComparisonError("Target pointer owner is invalid")
|
|
46
|
+
_target_directory(directory, pointer.get("generation"))
|
|
47
|
+
if not isinstance(pointer.get("sourceRoot"), str):
|
|
48
|
+
raise ComparisonError("Target pointer source is invalid")
|
|
49
|
+
return pointer, digest_bytes(snapshot.raw_bytes)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _target_result(root: Path, directory: Path, generation: str | None, response: dict[str, Any]) -> dict[str, Any]:
|
|
53
|
+
result = {
|
|
54
|
+
**response, "directory": directory.relative_to(root).as_posix(),
|
|
55
|
+
"notesPath": (directory / "notes.json").relative_to(root).as_posix(),
|
|
56
|
+
"generation": generation,
|
|
57
|
+
"targetRef": response.get("targetRef") if response["status"] != "unavailable" else None,
|
|
58
|
+
}
|
|
59
|
+
while len(json.dumps(result, ensure_ascii=True).encode()) + 1 > MAX_BASELINE_RESPONSE_BYTES:
|
|
60
|
+
if not result["limitations"]:
|
|
61
|
+
raise ComparisonError("Target response metadata exceeds its byte limit")
|
|
62
|
+
result["limitations"].pop()
|
|
63
|
+
result["limitationsOmitted"] = True
|
|
64
|
+
return result
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _current_target(root: Path, directory: Path, owner: dict[str, Any], *, budget: ReadBudget | None = None) -> tuple[dict[str, Any], dict[str, Any] | None, dict[str, str | None]]:
|
|
68
|
+
pointer, pointer_digest = _read_target_pointer(directory, owner, budget=budget)
|
|
69
|
+
fingerprints = {(directory / "target-current.json").relative_to(root).as_posix(): pointer_digest}
|
|
70
|
+
if pointer is None:
|
|
71
|
+
return _target_result(root, directory, None, _response(
|
|
72
|
+
root, directory, None, None, "Target was not captured", record_name="target.json",
|
|
73
|
+
)), None, fingerprints
|
|
74
|
+
generation = pointer["generation"]
|
|
75
|
+
response, record = _read_capture(root, _target_directory(directory, generation), owner, record_name="target.json", budget=budget)
|
|
76
|
+
if response.get("targetRef"):
|
|
77
|
+
fingerprints[response["targetRef"]["path"]] = response["targetRef"]["digest"]
|
|
78
|
+
if record is not None and (record.get("generation") != generation or record.get("sourceRoot") != pointer["sourceRoot"]):
|
|
79
|
+
raise ComparisonError("Target generation conflicts with its original capture")
|
|
80
|
+
return _target_result(root, directory, generation, response), record, fingerprints
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def capture_target(
|
|
84
|
+
project_root: Path, source_root: Path, comparison_dir: Path | None,
|
|
85
|
+
owner: ComparisonOwner, *, generation: str,
|
|
86
|
+
) -> dict[str, Any]:
|
|
87
|
+
root, directory, identity = _owner_and_directory(project_root, comparison_dir, owner)
|
|
88
|
+
if owner["kind"] not in {"run", "trace"}:
|
|
89
|
+
raise ComparisonError("Target capture requires a run or trace owner")
|
|
90
|
+
target_dir = _target_directory(directory, generation)
|
|
91
|
+
source = Path(source_root).resolve()
|
|
92
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
93
|
+
with dir_flock(directory, ".comparison.lock"):
|
|
94
|
+
pointer, _ = _read_target_pointer(directory, identity)
|
|
95
|
+
if pointer is None and owner["kind"] == "trace" and (directory / "targets").exists():
|
|
96
|
+
with os.scandir(directory / "targets") as reservations:
|
|
97
|
+
if any(entry.name != generation for entry in reservations):
|
|
98
|
+
raise ComparisonError("Trace target generation was already reserved")
|
|
99
|
+
if pointer is not None and owner["kind"] == "trace" and pointer["generation"] != generation:
|
|
100
|
+
raise ComparisonError("Trace target generation is already fixed")
|
|
101
|
+
existing = (target_dir / "target.json").exists() or (target_dir / "target.json").is_symlink()
|
|
102
|
+
if existing and pointer is None:
|
|
103
|
+
raise ComparisonError("Target generation has no current pointer")
|
|
104
|
+
if target_dir.exists() and not existing:
|
|
105
|
+
if pointer is not None and pointer["generation"] == generation and pointer["sourceRoot"] != str(source):
|
|
106
|
+
raise ComparisonError("Target source conflicts with its original capture")
|
|
107
|
+
response, _ = _read_capture(root, target_dir, identity, record_name="target.json")
|
|
108
|
+
return _target_result(root, directory, generation, response)
|
|
109
|
+
if not existing:
|
|
110
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
write_owned_object_atomic(directory / "target-current.json", {
|
|
112
|
+
"schemaVersion": "1.0", "owner": identity,
|
|
113
|
+
"generation": generation, "sourceRoot": str(source),
|
|
114
|
+
}, artifact="implementation target pointer")
|
|
115
|
+
response = _capture_source_record(
|
|
116
|
+
root, source, target_dir, identity, "", record_name="target.json",
|
|
117
|
+
metadata={"generation": generation},
|
|
118
|
+
)
|
|
119
|
+
return _target_result(root, directory, generation, response)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def capture_trace_target(
|
|
123
|
+
project_root: Path, source_root: Path, comparison_dir: Path | None,
|
|
124
|
+
owner: ComparisonOwner, source: Mapping[str, Any], *, generation: str,
|
|
125
|
+
) -> dict[str, Any]:
|
|
126
|
+
root, directory, identity = _owner_and_directory(project_root, comparison_dir, owner)
|
|
127
|
+
if identity["kind"] != "trace":
|
|
128
|
+
raise ComparisonError("Trace target capture requires a trace owner")
|
|
129
|
+
if not _validate_source(root, identity, source, {}, ReadBudget(MAX_COMPARISON_READ_BYTES)):
|
|
130
|
+
raise ComparisonError("Comparison source no longer matches its saved original")
|
|
131
|
+
trace = read_trace_source(root, identity["path"])
|
|
132
|
+
if trace.digest != source["digest"] or trace.digest != generation:
|
|
133
|
+
raise ComparisonError("Trace target generation must match its saved original digest")
|
|
134
|
+
if not trace_integration_completed(trace.frontmatter):
|
|
135
|
+
raise ComparisonError("Trace target requires its saved completed integration")
|
|
136
|
+
return capture_target(root, source_root, directory, identity, generation=generation)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def read_target_status(project_root: Path, comparison_dir: Path | None, owner: ComparisonOwner) -> dict[str, Any]:
|
|
140
|
+
root, directory, identity = _owner_and_directory(project_root, comparison_dir, owner)
|
|
141
|
+
if not (directory / "target-current.json").exists() and not (directory / "target-current.json").is_symlink():
|
|
142
|
+
return _current_target(root, directory, identity)[0]
|
|
143
|
+
with dir_flock(directory, ".comparison.lock"):
|
|
144
|
+
return _current_target(root, directory, identity)[0]
|