okstra 0.234.0 → 0.235.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-registry.mjs +7 -0
- package/dist/cli-registry.mjs.map +1 -1
- package/dist/commands/implementation-report/implementation-report.d.mts +1 -0
- package/dist/commands/implementation-report/implementation-report.mjs +73 -0
- package/dist/commands/implementation-report/implementation-report.mjs.map +1 -0
- package/dist/commands/trace/comparison.d.mts +39 -0
- package/dist/commands/trace/comparison.mjs +138 -0
- package/dist/commands/trace/comparison.mjs.map +1 -0
- package/dist/commands/trace/document.d.mts +8 -0
- package/dist/commands/trace/document.mjs +37 -0
- package/dist/commands/trace/document.mjs.map +1 -0
- package/dist/commands/trace/line-counts.d.mts +15 -0
- package/dist/commands/trace/line-counts.mjs +107 -0
- package/dist/commands/trace/line-counts.mjs.map +1 -0
- package/dist/commands/trace/model.d.mts +2 -1
- package/dist/commands/trace/model.mjs +13 -1
- package/dist/commands/trace/model.mjs.map +1 -1
- package/dist/commands/trace/progression.mjs +12 -7
- package/dist/commands/trace/progression.mjs.map +1 -1
- package/dist/commands/trace/task-link.d.mts +3 -0
- package/dist/commands/trace/task-link.mjs.map +1 -1
- package/dist/commands/trace/trace.mjs +66 -46
- package/dist/commands/trace/trace.mjs.map +1 -1
- package/dist/lib/python-helper.d.mts +1 -1
- package/dist/lib/python-helper.mjs +11 -5
- package/dist/lib/python-helper.mjs.map +1 -1
- package/dist/lib/types.d.mts +1 -0
- package/docs/architecture/storage-model.md +50 -0
- package/docs/cli.md +31 -15
- package/docs/project-structure-overview.md +6 -1
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/agents/operations/trace-tests-review.json +6 -0
- package/runtime/prompts/duties/test-reviewer.json +32 -0
- package/runtime/prompts/lead/report-writer.md +13 -1
- package/runtime/python/okstra_ctl/agent/invocation.py +109 -56
- package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +17 -0
- package/runtime/python/okstra_ctl/attempt_evidence.py +5 -2
- package/runtime/python/okstra_ctl/code_review_census.py +38 -0
- package/runtime/python/okstra_ctl/code_review_coverage.py +15 -4
- package/runtime/python/okstra_ctl/code_review_prepare.py +15 -3
- package/runtime/python/okstra_ctl/code_review_scope.py +3 -2
- package/runtime/python/okstra_ctl/code_review_target.py +16 -6
- package/runtime/python/okstra_ctl/conformance_evidence.py +44 -21
- package/runtime/python/okstra_ctl/consumers.py +6 -1
- package/runtime/python/okstra_ctl/design_review.py +6 -26
- package/runtime/python/okstra_ctl/design_review_evidence.py +6 -26
- package/runtime/python/okstra_ctl/dispatch_core.py +9 -2
- package/runtime/python/okstra_ctl/dispatch_state.py +5 -0
- package/runtime/python/okstra_ctl/domain/role.py +1 -0
- package/runtime/python/okstra_ctl/execution_manifest.py +8 -0
- package/runtime/python/okstra_ctl/implementation_comparison/__init__.py +12 -0
- package/runtime/python/okstra_ctl/implementation_comparison/__main__.py +129 -0
- package/runtime/python/okstra_ctl/implementation_comparison/assembly.py +340 -0
- package/runtime/python/okstra_ctl/implementation_comparison/baseline.py +254 -0
- package/runtime/python/okstra_ctl/implementation_comparison/business_reuse.py +118 -0
- package/runtime/python/okstra_ctl/implementation_comparison/commit_source.py +113 -0
- package/runtime/python/okstra_ctl/implementation_comparison/html.py +49 -0
- package/runtime/python/okstra_ctl/implementation_comparison/inputs.py +348 -0
- package/runtime/python/okstra_ctl/implementation_comparison/model.py +47 -0
- package/runtime/python/okstra_ctl/implementation_comparison/preparation.py +195 -0
- package/runtime/python/okstra_ctl/implementation_comparison/publication.py +317 -0
- package/runtime/python/okstra_ctl/implementation_comparison/run_evidence.py +208 -0
- package/runtime/python/okstra_ctl/implementation_comparison/run_integration.py +198 -0
- package/runtime/python/okstra_ctl/implementation_comparison/source.py +342 -0
- package/runtime/python/okstra_ctl/implementation_comparison/target.py +144 -0
- package/runtime/python/okstra_ctl/implementation_comparison/task_assembly.py +333 -0
- package/runtime/python/okstra_ctl/implementation_comparison/task_selection.py +309 -0
- package/runtime/python/okstra_ctl/implementation_generation_evidence.py +134 -0
- package/runtime/python/okstra_ctl/implementation_outcome.py +2 -2
- package/runtime/python/okstra_ctl/initial_prompt_materialization.py +1 -0
- package/runtime/python/okstra_ctl/json_boundary.py +28 -8
- package/runtime/python/okstra_ctl/paths.py +7 -4
- package/runtime/python/okstra_ctl/plan_run_root.py +8 -4
- package/runtime/python/okstra_ctl/qa_procedures.py +33 -2
- package/runtime/python/okstra_ctl/render.py +4 -0
- package/runtime/python/okstra_ctl/report_execution_evidence.py +88 -15
- package/runtime/python/okstra_ctl/report_finalize.py +18 -4
- package/runtime/python/okstra_ctl/report_html/render.py +46 -7
- package/runtime/python/okstra_ctl/report_html/validation.py +32 -0
- package/runtime/python/okstra_ctl/report_synthesis_packet.py +37 -3
- package/runtime/python/okstra_ctl/report_view_artifacts.py +12 -0
- package/runtime/python/okstra_ctl/review_verdicts.py +9 -0
- package/runtime/python/okstra_ctl/run.py +16 -0
- package/runtime/python/okstra_ctl/stage_done.py +11 -3
- package/runtime/python/okstra_ctl/stage_ledger.py +2 -2
- package/runtime/python/okstra_ctl/stage_map.py +3 -3
- package/runtime/python/okstra_ctl/trace_review.py +1 -0
- package/runtime/python/okstra_ctl/trace_tasks.py +22 -7
- package/runtime/python/okstra_ctl/verification_dispatch.py +32 -0
- package/runtime/python/okstra_ctl/verification_evidence.py +23 -3
- package/runtime/python/okstra_ctl/verification_generation.py +3 -2
- package/runtime/python/okstra_ctl/verification_inputs.py +3 -2
- package/runtime/python/okstra_ctl/verification_invocation.py +7 -3
- package/runtime/schemas/report-synthesis-packet-v1.0.schema.json +17 -0
- package/runtime/skills/okstra-brief-gen/SKILL.md +52 -6
- package/runtime/skills/okstra-code-review/SKILL.md +1 -1
- package/runtime/skills/okstra-code-review/references/census-rules.md +1 -1
- package/runtime/skills/okstra-code-review/references/review-calibration.md +1 -1
- package/runtime/skills/okstra-run/SKILL.md +31 -0
- package/runtime/skills/okstra-tdd/SKILL.md +75 -11
- package/runtime/templates/reports/html/assets/implementation-comparison.css +39 -0
- package/runtime/templates/reports/html/assets/implementation-comparison.js +55 -0
- package/runtime/templates/reports/html/i18n/en.json +113 -0
- package/runtime/templates/reports/html/i18n/ko.json +113 -0
- package/runtime/templates/reports/html/implementation-comparison.template.html +77 -0
- package/runtime/templates/reports/implementation-comparison-notes.md +72 -0
- package/runtime/templates/trace/trace.template.md +2 -0
- package/runtime/validators/validate-report-views.py +4 -36
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Connect optional run comparison facts to report writing and closeout."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import subprocess
|
|
7
|
+
from typing import Any, Mapping
|
|
8
|
+
|
|
9
|
+
from okstra_vendor.jinja2 import TemplateError
|
|
10
|
+
|
|
11
|
+
from ..i18n import I18nError
|
|
12
|
+
from ..json_boundary import PageBudgetExhausted, load_owned_object_snapshot, read_bounded_bytes
|
|
13
|
+
from ..paths import find_asset_root
|
|
14
|
+
from ..qa_procedures import project_artifact_relative_path, safe_qa_path
|
|
15
|
+
from .baseline import (
|
|
16
|
+
MAX_BASELINE_BYTES, MAX_LIMITATION_SUMMARY_CHARS, ComparisonOwner,
|
|
17
|
+
read_baseline_status,
|
|
18
|
+
)
|
|
19
|
+
from .inputs import run_comparison_scope, task_comparison_owner
|
|
20
|
+
from .model import MAX_COMPARISON_INPUT_BYTES
|
|
21
|
+
from .run_evidence import read_run_evidence
|
|
22
|
+
from .target import _current_target, capture_target
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _run_scope(
|
|
26
|
+
root: Path, manifest_path: Path, manifest: Mapping[str, Any],
|
|
27
|
+
) -> tuple[ComparisonOwner, Path] | None:
|
|
28
|
+
if manifest.get("taskType") != "implementation" or "implementationComparison" not in manifest:
|
|
29
|
+
return None
|
|
30
|
+
if manifest["implementationComparison"] is None:
|
|
31
|
+
raise ValueError("Run comparison declaration is invalid")
|
|
32
|
+
owner, directory, _ = run_comparison_scope(root, manifest_path, manifest)
|
|
33
|
+
safe_qa_path(root, (directory / "notes.json").relative_to(root).as_posix())
|
|
34
|
+
return owner, directory
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _saved_scope(root: Path, manifest_path: Path) -> tuple[ComparisonOwner, Path] | None:
|
|
38
|
+
path = safe_qa_path(root, project_artifact_relative_path(root, manifest_path))
|
|
39
|
+
manifest = load_owned_object_snapshot(
|
|
40
|
+
path, artifact="comparison run manifest", max_bytes=MAX_BASELINE_BYTES,
|
|
41
|
+
).value
|
|
42
|
+
return _run_scope(root, path, manifest)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _unavailable(reason: str) -> dict[str, Any]:
|
|
46
|
+
return {
|
|
47
|
+
"schemaVersion": "1.0", "owner": None, "baselineRef": None,
|
|
48
|
+
"targetRef": None, "notesPath": None, "generation": None,
|
|
49
|
+
"recordIds": [], "evidence": [], "notesGuide": None,
|
|
50
|
+
"limitations": [reason[:MAX_LIMITATION_SUMMARY_CHARS]],
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _notes_guide() -> tuple[str | None, list[str]]:
|
|
55
|
+
relative = ("templates", "reports", "implementation-comparison-notes.md")
|
|
56
|
+
try:
|
|
57
|
+
assets = find_asset_root(relative, is_present=Path.is_file)
|
|
58
|
+
if assets is None:
|
|
59
|
+
raise ValueError("Packaged comparison notes guide is missing")
|
|
60
|
+
return read_bounded_bytes(
|
|
61
|
+
assets.joinpath(*relative), max_bytes=MAX_COMPARISON_INPUT_BYTES,
|
|
62
|
+
artifact="implementation comparison notes guide",
|
|
63
|
+
).decode("utf-8"), []
|
|
64
|
+
except (OSError, ValueError) as exc:
|
|
65
|
+
return None, [f"Comparison notes guide was not supplied: {exc}"]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _writer_comparison(
|
|
69
|
+
root: Path, directory: Path, owner: ComparisonOwner,
|
|
70
|
+
target: Mapping[str, Any], record: Mapping[str, Any] | None,
|
|
71
|
+
) -> dict[str, Any]:
|
|
72
|
+
baseline = read_baseline_status(root, directory, owner)
|
|
73
|
+
evidence = read_run_evidence(root, owner, record)
|
|
74
|
+
guide, guide_limits = _notes_guide()
|
|
75
|
+
metadata = {
|
|
76
|
+
"schemaVersion": "1.0", "owner": owner,
|
|
77
|
+
"baselineRef": baseline["baselineRef"],
|
|
78
|
+
"targetRef": target["targetRef"] if target["generation"] == evidence.generation else None,
|
|
79
|
+
"notesPath": (directory / "notes.json").relative_to(root).as_posix(),
|
|
80
|
+
"generation": evidence.generation, "recordIds": [], "evidence": [], "limitations": [],
|
|
81
|
+
"notesGuide": guide,
|
|
82
|
+
}
|
|
83
|
+
omitted = "Additional comparison context was omitted at the byte limit"
|
|
84
|
+
remaining = MAX_COMPARISON_INPUT_BYTES - len(json.dumps(metadata, ensure_ascii=True).encode()) - len(json.dumps(omitted).encode()) - 2
|
|
85
|
+
if remaining < 0:
|
|
86
|
+
metadata["notesGuide"] = None
|
|
87
|
+
guide_limits.append("Comparison notes guide exceeds the available metadata byte limit")
|
|
88
|
+
remaining = MAX_COMPARISON_INPUT_BYTES - len(json.dumps(metadata, ensure_ascii=True).encode()) - len(json.dumps(omitted).encode()) - 2
|
|
89
|
+
limits = [*guide_limits, *baseline["limitations"], *target["limitations"], *evidence.limitations]
|
|
90
|
+
if target["generation"] != evidence.generation:
|
|
91
|
+
limits.append("The preserved target does not identify the current completed generation")
|
|
92
|
+
for limitation in limits:
|
|
93
|
+
summary = limitation[:MAX_LIMITATION_SUMMARY_CHARS]
|
|
94
|
+
cost = len(json.dumps(summary, ensure_ascii=True).encode()) + 2
|
|
95
|
+
if cost > remaining:
|
|
96
|
+
metadata["limitations"].append(omitted)
|
|
97
|
+
return metadata
|
|
98
|
+
metadata["limitations"].append(summary)
|
|
99
|
+
remaining -= cost
|
|
100
|
+
for row in evidence.rows:
|
|
101
|
+
cost = len(json.dumps(row, ensure_ascii=True).encode()) + len(json.dumps(row["recordId"], ensure_ascii=True).encode()) + 4
|
|
102
|
+
if cost > remaining:
|
|
103
|
+
metadata["limitations"].append(omitted)
|
|
104
|
+
break
|
|
105
|
+
metadata["evidence"].append(row)
|
|
106
|
+
metadata["recordIds"].append(row["recordId"])
|
|
107
|
+
remaining -= cost
|
|
108
|
+
return metadata
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def prepare_run_comparison(project_root: Path, manifest_path: Path) -> dict[str, Any] | None:
|
|
112
|
+
root = Path(project_root).resolve()
|
|
113
|
+
try:
|
|
114
|
+
scope = _saved_scope(root, manifest_path)
|
|
115
|
+
if scope is None:
|
|
116
|
+
return None
|
|
117
|
+
owner, directory = scope
|
|
118
|
+
evidence = read_run_evidence(root, owner)
|
|
119
|
+
target, record, _ = _current_target(root, directory, owner)
|
|
120
|
+
if evidence.generation is not None and evidence.source_root is not None and target["generation"] != evidence.generation:
|
|
121
|
+
capture_target(root, Path(evidence.source_root), directory, owner, generation=evidence.generation)
|
|
122
|
+
target, record, _ = _current_target(root, directory, owner)
|
|
123
|
+
return _writer_comparison(root, directory, owner, target, record)
|
|
124
|
+
except (OSError, ValueError, PageBudgetExhausted, subprocess.SubprocessError) as exc:
|
|
125
|
+
return _unavailable(f"Run comparison preparation failed: {exc}")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def read_run_comparison(project_root: Path, manifest_path: Path) -> dict[str, Any] | None:
|
|
129
|
+
root = Path(project_root).resolve()
|
|
130
|
+
try:
|
|
131
|
+
scope = _saved_scope(root, manifest_path)
|
|
132
|
+
if scope is None:
|
|
133
|
+
return None
|
|
134
|
+
owner, directory = scope
|
|
135
|
+
target, record, _ = _current_target(root, directory, owner)
|
|
136
|
+
return _writer_comparison(root, directory, owner, target, record)
|
|
137
|
+
except (OSError, ValueError, PageBudgetExhausted) as exc:
|
|
138
|
+
return _unavailable(f"Run comparison input could not be read: {exc}")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def run_comparison_notes_paths(
|
|
142
|
+
project_root: Path, manifest_path: Path, manifest: Mapping[str, Any], *, role: str,
|
|
143
|
+
) -> tuple[Path, ...]:
|
|
144
|
+
if role != "report-writer":
|
|
145
|
+
return ()
|
|
146
|
+
root = Path(project_root).resolve()
|
|
147
|
+
try:
|
|
148
|
+
scope = _run_scope(root, manifest_path, manifest)
|
|
149
|
+
except (OSError, ValueError):
|
|
150
|
+
return ()
|
|
151
|
+
return (scope[1] / "notes.json",) if scope is not None else ()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def finalize_run_comparison(project_root: Path, manifest_path: Path) -> dict[str, Any] | None:
|
|
155
|
+
from .publication import publish_comparison, read_comparison_status, update_implementation_comparison_link
|
|
156
|
+
from .run_evidence import check_live_run_target
|
|
157
|
+
|
|
158
|
+
root = Path(project_root).resolve()
|
|
159
|
+
warnings = []
|
|
160
|
+
try:
|
|
161
|
+
scope = _saved_scope(root, manifest_path)
|
|
162
|
+
if scope is None:
|
|
163
|
+
return None
|
|
164
|
+
owner, directory = scope
|
|
165
|
+
target, record, _ = _current_target(root, directory, owner)
|
|
166
|
+
limitations = check_live_run_target(root, owner, record) if record is not None and target["targetRef"] is not None else ()
|
|
167
|
+
if limitations:
|
|
168
|
+
comparison = {**read_comparison_status(root, directory, owner), "status": "stale"}
|
|
169
|
+
return {
|
|
170
|
+
"comparison": comparison, "linkStatus": "not-updated",
|
|
171
|
+
"warnings": [{"stage": "target", "message": message} for message in limitations],
|
|
172
|
+
}
|
|
173
|
+
comparison = publish_comparison(root, directory, owner)
|
|
174
|
+
except (OSError, ValueError, PageBudgetExhausted, subprocess.SubprocessError, TemplateError) as exc:
|
|
175
|
+
warnings.append({"stage": "publication", "message": str(exc)})
|
|
176
|
+
return {"comparison": {"status": "failed"}, "linkStatus": "not-updated", "warnings": warnings}
|
|
177
|
+
link_status = "not-updated"
|
|
178
|
+
if comparison["status"] in {"published", "reused"}:
|
|
179
|
+
try:
|
|
180
|
+
update_implementation_comparison_link(root, directory, owner)
|
|
181
|
+
except (OSError, ValueError, PageBudgetExhausted) as exc:
|
|
182
|
+
link_status = "failed"
|
|
183
|
+
warnings.append({"stage": "link", "message": str(exc)})
|
|
184
|
+
else:
|
|
185
|
+
link_status = "updated"
|
|
186
|
+
else:
|
|
187
|
+
warnings.extend({"stage": "publication", "message": message} for message in comparison.get("warnings", []))
|
|
188
|
+
return {"comparison": comparison, "linkStatus": link_status, "warnings": warnings}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def finalize_task_comparison(project_root: Path, task_root: Path, task_key: str) -> dict[str, Any]:
|
|
192
|
+
from .publication import publish_comparison
|
|
193
|
+
|
|
194
|
+
try:
|
|
195
|
+
owner = task_comparison_owner(project_root, task_root, task_key)
|
|
196
|
+
return publish_comparison(project_root, None, owner)
|
|
197
|
+
except (OSError, ValueError, KeyError, PageBudgetExhausted, subprocess.SubprocessError, TemplateError, RecursionError, I18nError) as exc:
|
|
198
|
+
return {"status": "failed", "htmlPath": None, "warnings": [f"Task comparison could not be generated: {exc}"]}
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Bounded, read-only Git and working-source collection."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
import selectors
|
|
7
|
+
import signal
|
|
8
|
+
import stat
|
|
9
|
+
import subprocess
|
|
10
|
+
import time
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Any, Iterator
|
|
15
|
+
|
|
16
|
+
from ..json_boundary import PageBudgetExhausted, ReadBudget
|
|
17
|
+
from ..process_group import kill_group
|
|
18
|
+
|
|
19
|
+
MAX_PATHS = 10_000
|
|
20
|
+
MAX_FILE_READ_BYTES = 1024 * 1024
|
|
21
|
+
MAX_TOTAL_READ_BYTES = 64 * 1024 * 1024
|
|
22
|
+
MAX_GIT_COMMAND_BYTES = 4 * 1024 * 1024
|
|
23
|
+
MAX_GIT_TOTAL_BYTES = 8 * 1024 * 1024
|
|
24
|
+
GIT_TIMEOUT_SECONDS = 10
|
|
25
|
+
READ_CHUNK_BYTES = 64 * 1024
|
|
26
|
+
SOURCE_CHANGED_DURING_READ = "Source changed while being read"
|
|
27
|
+
EXCLUDED_PARTS = frozenset({
|
|
28
|
+
".git", ".okstra", ".project-docs", ".worktrees", "node_modules", "vendor",
|
|
29
|
+
".venv", "venv", "__pycache__", ".pytest_cache", ".cache", ".mypy_cache",
|
|
30
|
+
".ruff_cache", "dist", "build", "runtime", "target", "coverage", ".next",
|
|
31
|
+
"graphify-out",
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def digest_bytes(body: bytes) -> str:
|
|
36
|
+
return hashlib.sha256(body).hexdigest()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class GitOutput:
|
|
41
|
+
stdout: bytes
|
|
42
|
+
stderr: bytes
|
|
43
|
+
returncode: int | None
|
|
44
|
+
complete: bool
|
|
45
|
+
reason: str = ""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class GitCollector:
|
|
49
|
+
def __init__(
|
|
50
|
+
self, directory: Path | None = None, *, shared_budget: ReadBudget | None = None,
|
|
51
|
+
deadline: float | None = None,
|
|
52
|
+
) -> None:
|
|
53
|
+
self.directory = directory
|
|
54
|
+
self.budget = ReadBudget(MAX_GIT_TOTAL_BYTES)
|
|
55
|
+
self.shared_budget = shared_budget
|
|
56
|
+
self.deadline = deadline
|
|
57
|
+
self.outputs: dict[str, dict[str, Any]] = {}
|
|
58
|
+
|
|
59
|
+
def run(self, root: Path, label: str, *args: str) -> GitOutput:
|
|
60
|
+
if self.shared_budget is not None and self.shared_budget.bytes_read >= self.shared_budget.max_bytes:
|
|
61
|
+
raise PageBudgetExhausted("Comparison source byte budget exhausted")
|
|
62
|
+
if self.budget.bytes_read >= self.budget.max_bytes:
|
|
63
|
+
result = GitOutput(b"", b"", None, False, "Git output budget exhausted")
|
|
64
|
+
elif self.deadline is not None and time.monotonic() >= self.deadline:
|
|
65
|
+
result = GitOutput(b"", b"", None, False, "Git request deadline exhausted")
|
|
66
|
+
else:
|
|
67
|
+
result = self._read_command(root, args)
|
|
68
|
+
files: dict[str, Any] = {}
|
|
69
|
+
for name, body in (("stdout", result.stdout), ("stderr", result.stderr)):
|
|
70
|
+
relative = f"git/{label}.{name}"
|
|
71
|
+
if self.directory is not None:
|
|
72
|
+
(self.directory / relative).write_bytes(body)
|
|
73
|
+
files[name] = {"path": relative, "bytes": len(body), "storedDigest": digest_bytes(body)}
|
|
74
|
+
self.outputs[label] = {
|
|
75
|
+
"argv": list(args), "returncode": result.returncode,
|
|
76
|
+
"complete": result.complete, "reason": result.reason, **files,
|
|
77
|
+
}
|
|
78
|
+
return result
|
|
79
|
+
|
|
80
|
+
def _remaining(self, command_bytes: int) -> int:
|
|
81
|
+
remaining = min(MAX_GIT_COMMAND_BYTES - command_bytes,
|
|
82
|
+
self.budget.max_bytes - self.budget.bytes_read)
|
|
83
|
+
if self.shared_budget is not None:
|
|
84
|
+
remaining = min(remaining, self.shared_budget.max_bytes - self.shared_budget.bytes_read)
|
|
85
|
+
return remaining
|
|
86
|
+
|
|
87
|
+
def _read_command(self, root: Path, args: tuple[str, ...]) -> GitOutput:
|
|
88
|
+
env = {key: value for key, value in os.environ.items() if key not in {
|
|
89
|
+
"GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX",
|
|
90
|
+
"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
|
91
|
+
}}
|
|
92
|
+
command = ["git", "--no-pager", "-c", "core.fsmonitor=false", "-C", str(root), *args]
|
|
93
|
+
process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
94
|
+
stderr=subprocess.PIPE, env=env, start_new_session=True)
|
|
95
|
+
bodies = [bytearray(), bytearray()]
|
|
96
|
+
reason = ""
|
|
97
|
+
deadline = time.monotonic() + GIT_TIMEOUT_SECONDS
|
|
98
|
+
if self.deadline is not None:
|
|
99
|
+
deadline = min(deadline, self.deadline)
|
|
100
|
+
try:
|
|
101
|
+
with selectors.DefaultSelector() as selector:
|
|
102
|
+
for index, pipe in enumerate((process.stdout, process.stderr)):
|
|
103
|
+
selector.register(pipe, selectors.EVENT_READ, index)
|
|
104
|
+
while selector.get_map():
|
|
105
|
+
remaining = self._remaining(sum(map(len, bodies)))
|
|
106
|
+
if remaining <= 0 or time.monotonic() >= deadline:
|
|
107
|
+
reason = "Git output limit reached" if remaining <= 0 else "Git command timed out"
|
|
108
|
+
break
|
|
109
|
+
for key, _ in selector.select(timeout=min(0.1, max(0, deadline - time.monotonic()))):
|
|
110
|
+
remaining = self._remaining(sum(map(len, bodies)))
|
|
111
|
+
if remaining <= 0:
|
|
112
|
+
break
|
|
113
|
+
chunk = os.read(key.fd, min(READ_CHUNK_BYTES, remaining))
|
|
114
|
+
self.budget.bytes_read += len(chunk)
|
|
115
|
+
if self.shared_budget is not None and self.shared_budget is not self.budget:
|
|
116
|
+
self.shared_budget.bytes_read += len(chunk)
|
|
117
|
+
bodies[key.data].extend(chunk)
|
|
118
|
+
remaining -= len(chunk)
|
|
119
|
+
if not chunk:
|
|
120
|
+
selector.unregister(key.fileobj)
|
|
121
|
+
if reason:
|
|
122
|
+
kill_group(process.pid, signal.SIGKILL)
|
|
123
|
+
try:
|
|
124
|
+
returncode = process.wait(timeout=max(0, deadline - time.monotonic()))
|
|
125
|
+
except subprocess.TimeoutExpired:
|
|
126
|
+
reason = reason or "Git command timed out"
|
|
127
|
+
kill_group(process.pid, signal.SIGKILL)
|
|
128
|
+
returncode = process.wait()
|
|
129
|
+
finally:
|
|
130
|
+
if process.poll() is None:
|
|
131
|
+
kill_group(process.pid, signal.SIGKILL)
|
|
132
|
+
process.wait()
|
|
133
|
+
for pipe in (process.stdout, process.stderr):
|
|
134
|
+
if pipe is not None:
|
|
135
|
+
pipe.close()
|
|
136
|
+
return GitOutput(bytes(bodies[0]), bytes(bodies[1]), returncode, not reason, reason)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@contextmanager
|
|
140
|
+
def _source_parent(root: Path, relative: str) -> Iterator[tuple[int, str]]:
|
|
141
|
+
parts = PurePosixPath(relative).parts
|
|
142
|
+
if not parts or PurePosixPath(relative).is_absolute() or any(p in {".", ".."} for p in parts):
|
|
143
|
+
raise ValueError("Source path is not relative to its root")
|
|
144
|
+
directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
|
145
|
+
try:
|
|
146
|
+
for part in parts[:-1]:
|
|
147
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory)
|
|
148
|
+
os.close(directory)
|
|
149
|
+
directory = child
|
|
150
|
+
yield directory, parts[-1]
|
|
151
|
+
finally:
|
|
152
|
+
os.close(directory)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def open_source_file(root: Path, relative: str) -> int:
|
|
156
|
+
with _source_parent(root, relative) as (directory, name):
|
|
157
|
+
return os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _source_info(root: Path, relative: str) -> tuple[os.stat_result, str | None]:
|
|
161
|
+
with _source_parent(root, relative) as (directory, name):
|
|
162
|
+
info = os.stat(name, dir_fd=directory, follow_symlinks=False)
|
|
163
|
+
return info, os.readlink(name, dir_fd=directory) if stat.S_ISLNK(info.st_mode) else None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def read_source_bytes(
|
|
167
|
+
root: Path, relative: str, budget: ReadBudget, *, file_limit: int | None = None,
|
|
168
|
+
shared_budget: ReadBudget | None = None,
|
|
169
|
+
) -> tuple[bytes | None, os.stat_result, str]:
|
|
170
|
+
descriptor = open_source_file(root, relative)
|
|
171
|
+
try:
|
|
172
|
+
before = os.fstat(descriptor)
|
|
173
|
+
if not stat.S_ISREG(before.st_mode):
|
|
174
|
+
return None, before, "Source is not a regular file"
|
|
175
|
+
maximum = min(MAX_FILE_READ_BYTES if file_limit is None else file_limit,
|
|
176
|
+
budget.max_bytes - budget.bytes_read)
|
|
177
|
+
if shared_budget is not None and before.st_size > shared_budget.max_bytes - shared_budget.bytes_read:
|
|
178
|
+
raise PageBudgetExhausted("Comparison source byte budget exhausted")
|
|
179
|
+
if before.st_size > maximum:
|
|
180
|
+
return None, before, "Source exceeds the remaining file read budget"
|
|
181
|
+
body = bytearray()
|
|
182
|
+
while len(body) < before.st_size:
|
|
183
|
+
chunk = os.read(descriptor, min(READ_CHUNK_BYTES, before.st_size - len(body)))
|
|
184
|
+
budget.bytes_read += len(chunk)
|
|
185
|
+
if shared_budget is not None and shared_budget is not budget:
|
|
186
|
+
shared_budget.bytes_read += len(chunk)
|
|
187
|
+
if not chunk:
|
|
188
|
+
break
|
|
189
|
+
body.extend(chunk)
|
|
190
|
+
after = os.fstat(descriptor)
|
|
191
|
+
if file_state(before) != file_state(after) or len(body) != before.st_size:
|
|
192
|
+
return None, after, SOURCE_CHANGED_DURING_READ
|
|
193
|
+
return bytes(body), after, ""
|
|
194
|
+
finally:
|
|
195
|
+
os.close(descriptor)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def file_state(info: os.stat_result) -> tuple[int, int, int, int, int]:
|
|
199
|
+
return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _names(output: GitOutput) -> list[bytes]:
|
|
203
|
+
return output.stdout.split(b"\0") if output.complete and output.returncode == 0 else []
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _pathspecs() -> list[str]:
|
|
207
|
+
return ["--", ".", *[f":(glob,exclude)**/{part}/**" for part in sorted(EXCLUDED_PARTS)]]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def collect_source(project_root: Path, source_root: Path, requested_ref: str, directory: Path) -> dict[str, Any]:
|
|
211
|
+
(directory / "git").mkdir()
|
|
212
|
+
(directory / "contents").mkdir()
|
|
213
|
+
git = GitCollector(directory)
|
|
214
|
+
budget = ReadBudget(MAX_TOTAL_READ_BYTES)
|
|
215
|
+
result: dict[str, Any] = {
|
|
216
|
+
"head": None, "baselineCommit": None, "files": [], "limitations": [],
|
|
217
|
+
"excludedParts": sorted(EXCLUDED_PARTS), "completeInventory": False,
|
|
218
|
+
"scope": {"workingTreeText": "full-within-read-limits",
|
|
219
|
+
"index": "binary-patch-and-stage-object-identifiers",
|
|
220
|
+
"commitTree": "commit-identity-only"},
|
|
221
|
+
}
|
|
222
|
+
project_git = git.run(project_root, "project-repository", "rev-parse", "--git-common-dir")
|
|
223
|
+
source_git = project_git if project_root == source_root else git.run(
|
|
224
|
+
source_root, "source-repository", "rev-parse", "--git-common-dir",
|
|
225
|
+
)
|
|
226
|
+
if any(not item.complete or item.returncode != 0 for item in (project_git, source_git)):
|
|
227
|
+
result["limitations"].append("Git repository is unavailable; non-Git source enumeration was not attempted")
|
|
228
|
+
return _finish_source(result, git, budget, 0)
|
|
229
|
+
project_common = (project_root / os.fsdecode(project_git.stdout).strip()).resolve()
|
|
230
|
+
source_common = (source_root / os.fsdecode(source_git.stdout).strip()).resolve()
|
|
231
|
+
if project_common != source_common:
|
|
232
|
+
raise ValueError("Source root belongs to a different Git repository")
|
|
233
|
+
head = git.run(source_root, "head-before", "rev-parse", "--verify", "HEAD^{commit}")
|
|
234
|
+
baseline = head if not requested_ref else git.run(
|
|
235
|
+
source_root, "baseline-commit", "rev-parse", "--verify", "--end-of-options", f"{requested_ref}^{{commit}}",
|
|
236
|
+
)
|
|
237
|
+
for key, value in (("head", head), ("baselineCommit", baseline)):
|
|
238
|
+
if value.complete and value.returncode == 0:
|
|
239
|
+
result[key] = value.stdout.decode("ascii").strip()
|
|
240
|
+
else:
|
|
241
|
+
result["limitations"].append(f"{key} could not be resolved")
|
|
242
|
+
before = _git_state(git, source_root, "before")
|
|
243
|
+
entries: dict[str, dict[str, Any]] = {}
|
|
244
|
+
paths_omitted = False
|
|
245
|
+
for raw in _names(before["tracked"]):
|
|
246
|
+
if not raw:
|
|
247
|
+
continue
|
|
248
|
+
entry, name = raw.split(b"\t", 1)
|
|
249
|
+
path = os.fsdecode(name)
|
|
250
|
+
if any(part in EXCLUDED_PARTS for part in PurePosixPath(path).parts):
|
|
251
|
+
continue
|
|
252
|
+
if path not in entries and len(entries) >= MAX_PATHS:
|
|
253
|
+
paths_omitted = True
|
|
254
|
+
continue
|
|
255
|
+
mode, object_id, stage = entry.decode("ascii").split()
|
|
256
|
+
row = entries.setdefault(path, {"path": path, "tracked": True, "indexEntries": []})
|
|
257
|
+
row["indexEntries"].append({"mode": mode, "object": object_id, "stage": stage})
|
|
258
|
+
for raw in _names(before["untracked"]):
|
|
259
|
+
if not raw:
|
|
260
|
+
continue
|
|
261
|
+
path = os.fsdecode(raw)
|
|
262
|
+
if any(part in EXCLUDED_PARTS for part in PurePosixPath(path).parts):
|
|
263
|
+
continue
|
|
264
|
+
if path not in entries and len(entries) >= MAX_PATHS:
|
|
265
|
+
paths_omitted = True
|
|
266
|
+
continue
|
|
267
|
+
entries.setdefault(path, {"path": path, "tracked": False})
|
|
268
|
+
complete_inventory = all(value.complete and value.returncode == 0 for value in before.values())
|
|
269
|
+
if paths_omitted:
|
|
270
|
+
complete_inventory = False
|
|
271
|
+
result["limitations"].append("Source path limit reached; inventory may be incomplete")
|
|
272
|
+
if not complete_inventory:
|
|
273
|
+
result["limitations"].append("Git inventory, status, or index capture is incomplete")
|
|
274
|
+
result["completeInventory"] = complete_inventory
|
|
275
|
+
saved_bytes = 0
|
|
276
|
+
states: dict[str, tuple[int, int, int, int, int]] = {}
|
|
277
|
+
for path, entry in entries.items():
|
|
278
|
+
row = {**entry, "digest": None, "contentPath": None}
|
|
279
|
+
try:
|
|
280
|
+
info, link = _source_info(source_root, path)
|
|
281
|
+
if link is not None:
|
|
282
|
+
body, problem = None, "Symbolic link source was not followed"
|
|
283
|
+
row.update({"kind": "symlink", "linkTarget": link})
|
|
284
|
+
else:
|
|
285
|
+
body, info, problem = read_source_bytes(source_root, path, budget)
|
|
286
|
+
row.update({"mode": info.st_mode, "size": info.st_size})
|
|
287
|
+
states[path] = file_state(info)
|
|
288
|
+
if problem:
|
|
289
|
+
row.update({"status": "partial", "reason": problem})
|
|
290
|
+
else:
|
|
291
|
+
row.update({"status": "complete", "digest": digest_bytes(body)})
|
|
292
|
+
try:
|
|
293
|
+
text = body.decode("utf-8")
|
|
294
|
+
is_text = "\0" not in text
|
|
295
|
+
except UnicodeDecodeError:
|
|
296
|
+
is_text = False
|
|
297
|
+
row["kind"] = "text" if is_text else "binary"
|
|
298
|
+
if is_text:
|
|
299
|
+
relative = f"contents/{row['digest']}.txt"
|
|
300
|
+
(directory / relative).write_bytes(body)
|
|
301
|
+
row["contentPath"] = relative
|
|
302
|
+
saved_bytes += len(body)
|
|
303
|
+
except FileNotFoundError:
|
|
304
|
+
row.update({"status": "missing", "reason": "Source was absent during capture"})
|
|
305
|
+
except (OSError, ValueError) as exc:
|
|
306
|
+
row.update({"status": "partial", "reason": f"Source was not followed or could not be read: {exc}"})
|
|
307
|
+
result["files"].append(row)
|
|
308
|
+
if row["status"] not in {"complete", "missing"}:
|
|
309
|
+
result["limitations"].append(f"{path}: {row['reason']}")
|
|
310
|
+
after = _git_state(git, source_root, "after")
|
|
311
|
+
after_head = git.run(source_root, "head-after", "rev-parse", "--verify", "HEAD^{commit}")
|
|
312
|
+
if after_head != head:
|
|
313
|
+
result["limitations"].append("HEAD changed or could not be rechecked during capture")
|
|
314
|
+
for key, previous in before.items():
|
|
315
|
+
current = after[key]
|
|
316
|
+
if not current.complete or current.returncode != 0 or current.stdout != previous.stdout:
|
|
317
|
+
result["limitations"].append(f"Git {key} changed or could not be rechecked during capture")
|
|
318
|
+
for path, previous in states.items():
|
|
319
|
+
try:
|
|
320
|
+
if file_state(_source_info(source_root, path)[0]) != previous:
|
|
321
|
+
result["limitations"].append(f"{path}: source changed after its content was captured")
|
|
322
|
+
except OSError:
|
|
323
|
+
result["limitations"].append(f"{path}: source disappeared after its content was captured")
|
|
324
|
+
return _finish_source(result, git, budget, saved_bytes)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _git_state(git: GitCollector, root: Path, suffix: str) -> dict[str, GitOutput]:
|
|
328
|
+
return {
|
|
329
|
+
"tracked": git.run(root, f"tracked-{suffix}", "ls-files", "--stage", "-z", *_pathspecs()),
|
|
330
|
+
"untracked": git.run(root, f"untracked-{suffix}", "ls-files", "--others", "--exclude-standard", "-z", *_pathspecs()),
|
|
331
|
+
"status": git.run(root, f"status-{suffix}", "status", "--porcelain=v1", "-z", "--untracked-files=all", *_pathspecs()),
|
|
332
|
+
"index": git.run(root, f"index-{suffix}", "diff", "--no-ext-diff", "--no-textconv", "--cached", "--binary", *_pathspecs()),
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _finish_source(result: dict[str, Any], git: GitCollector, budget: ReadBudget, saved: int) -> dict[str, Any]:
|
|
337
|
+
result.update({"git": git.outputs, "bytesRead": budget.bytes_read,
|
|
338
|
+
"bytesPreserved": saved, "gitBytesRead": git.budget.bytes_read})
|
|
339
|
+
result["status"] = "partial" if result["limitations"] else "complete"
|
|
340
|
+
if not result["completeInventory"] and not result["files"]:
|
|
341
|
+
result["status"] = "unavailable"
|
|
342
|
+
return result
|