okstra 0.234.0 → 0.235.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/dist/cli-registry.mjs +7 -0
  2. package/dist/cli-registry.mjs.map +1 -1
  3. package/dist/commands/implementation-report/implementation-report.d.mts +1 -0
  4. package/dist/commands/implementation-report/implementation-report.mjs +73 -0
  5. package/dist/commands/implementation-report/implementation-report.mjs.map +1 -0
  6. package/dist/commands/trace/comparison.d.mts +39 -0
  7. package/dist/commands/trace/comparison.mjs +138 -0
  8. package/dist/commands/trace/comparison.mjs.map +1 -0
  9. package/dist/commands/trace/document.d.mts +8 -0
  10. package/dist/commands/trace/document.mjs +37 -0
  11. package/dist/commands/trace/document.mjs.map +1 -0
  12. package/dist/commands/trace/line-counts.d.mts +15 -0
  13. package/dist/commands/trace/line-counts.mjs +107 -0
  14. package/dist/commands/trace/line-counts.mjs.map +1 -0
  15. package/dist/commands/trace/model.d.mts +2 -1
  16. package/dist/commands/trace/model.mjs +13 -1
  17. package/dist/commands/trace/model.mjs.map +1 -1
  18. package/dist/commands/trace/progression.mjs +12 -7
  19. package/dist/commands/trace/progression.mjs.map +1 -1
  20. package/dist/commands/trace/task-link.d.mts +3 -0
  21. package/dist/commands/trace/task-link.mjs.map +1 -1
  22. package/dist/commands/trace/trace.mjs +66 -46
  23. package/dist/commands/trace/trace.mjs.map +1 -1
  24. package/dist/lib/python-helper.d.mts +1 -1
  25. package/dist/lib/python-helper.mjs +11 -5
  26. package/dist/lib/python-helper.mjs.map +1 -1
  27. package/dist/lib/types.d.mts +1 -0
  28. package/docs/architecture/storage-model.md +50 -0
  29. package/docs/cli.md +31 -15
  30. package/docs/project-structure-overview.md +6 -1
  31. package/package.json +1 -1
  32. package/runtime/BUILD.json +2 -2
  33. package/runtime/agents/operations/trace-tests-review.json +6 -0
  34. package/runtime/prompts/duties/test-reviewer.json +32 -0
  35. package/runtime/prompts/lead/report-writer.md +13 -1
  36. package/runtime/python/okstra_ctl/agent/invocation.py +109 -56
  37. package/runtime/python/okstra_ctl/agent/prompt_cli/materialize.py +17 -0
  38. package/runtime/python/okstra_ctl/attempt_evidence.py +5 -2
  39. package/runtime/python/okstra_ctl/code_review_census.py +38 -0
  40. package/runtime/python/okstra_ctl/code_review_coverage.py +15 -4
  41. package/runtime/python/okstra_ctl/code_review_prepare.py +15 -3
  42. package/runtime/python/okstra_ctl/code_review_scope.py +3 -2
  43. package/runtime/python/okstra_ctl/code_review_target.py +16 -6
  44. package/runtime/python/okstra_ctl/conformance_evidence.py +44 -21
  45. package/runtime/python/okstra_ctl/consumers.py +6 -1
  46. package/runtime/python/okstra_ctl/design_review.py +6 -26
  47. package/runtime/python/okstra_ctl/design_review_evidence.py +6 -26
  48. package/runtime/python/okstra_ctl/dispatch_core.py +9 -2
  49. package/runtime/python/okstra_ctl/dispatch_state.py +5 -0
  50. package/runtime/python/okstra_ctl/domain/role.py +1 -0
  51. package/runtime/python/okstra_ctl/execution_manifest.py +8 -0
  52. package/runtime/python/okstra_ctl/implementation_comparison/__init__.py +12 -0
  53. package/runtime/python/okstra_ctl/implementation_comparison/__main__.py +129 -0
  54. package/runtime/python/okstra_ctl/implementation_comparison/assembly.py +340 -0
  55. package/runtime/python/okstra_ctl/implementation_comparison/baseline.py +254 -0
  56. package/runtime/python/okstra_ctl/implementation_comparison/business_reuse.py +118 -0
  57. package/runtime/python/okstra_ctl/implementation_comparison/commit_source.py +113 -0
  58. package/runtime/python/okstra_ctl/implementation_comparison/html.py +49 -0
  59. package/runtime/python/okstra_ctl/implementation_comparison/inputs.py +348 -0
  60. package/runtime/python/okstra_ctl/implementation_comparison/model.py +47 -0
  61. package/runtime/python/okstra_ctl/implementation_comparison/preparation.py +195 -0
  62. package/runtime/python/okstra_ctl/implementation_comparison/publication.py +317 -0
  63. package/runtime/python/okstra_ctl/implementation_comparison/run_evidence.py +208 -0
  64. package/runtime/python/okstra_ctl/implementation_comparison/run_integration.py +198 -0
  65. package/runtime/python/okstra_ctl/implementation_comparison/source.py +342 -0
  66. package/runtime/python/okstra_ctl/implementation_comparison/target.py +144 -0
  67. package/runtime/python/okstra_ctl/implementation_comparison/task_assembly.py +333 -0
  68. package/runtime/python/okstra_ctl/implementation_comparison/task_selection.py +309 -0
  69. package/runtime/python/okstra_ctl/implementation_generation_evidence.py +134 -0
  70. package/runtime/python/okstra_ctl/implementation_outcome.py +2 -2
  71. package/runtime/python/okstra_ctl/initial_prompt_materialization.py +1 -0
  72. package/runtime/python/okstra_ctl/json_boundary.py +28 -8
  73. package/runtime/python/okstra_ctl/paths.py +7 -4
  74. package/runtime/python/okstra_ctl/plan_run_root.py +8 -4
  75. package/runtime/python/okstra_ctl/qa_procedures.py +33 -2
  76. package/runtime/python/okstra_ctl/render.py +4 -0
  77. package/runtime/python/okstra_ctl/report_execution_evidence.py +88 -15
  78. package/runtime/python/okstra_ctl/report_finalize.py +18 -4
  79. package/runtime/python/okstra_ctl/report_html/render.py +46 -7
  80. package/runtime/python/okstra_ctl/report_html/validation.py +32 -0
  81. package/runtime/python/okstra_ctl/report_synthesis_packet.py +37 -3
  82. package/runtime/python/okstra_ctl/report_view_artifacts.py +12 -0
  83. package/runtime/python/okstra_ctl/review_verdicts.py +9 -0
  84. package/runtime/python/okstra_ctl/run.py +16 -0
  85. package/runtime/python/okstra_ctl/stage_done.py +11 -3
  86. package/runtime/python/okstra_ctl/stage_ledger.py +2 -2
  87. package/runtime/python/okstra_ctl/stage_map.py +3 -3
  88. package/runtime/python/okstra_ctl/trace_review.py +1 -0
  89. package/runtime/python/okstra_ctl/trace_tasks.py +22 -7
  90. package/runtime/python/okstra_ctl/verification_dispatch.py +32 -0
  91. package/runtime/python/okstra_ctl/verification_evidence.py +23 -3
  92. package/runtime/python/okstra_ctl/verification_generation.py +3 -2
  93. package/runtime/python/okstra_ctl/verification_inputs.py +3 -2
  94. package/runtime/python/okstra_ctl/verification_invocation.py +7 -3
  95. package/runtime/schemas/report-synthesis-packet-v1.0.schema.json +17 -0
  96. package/runtime/skills/okstra-brief-gen/SKILL.md +52 -6
  97. package/runtime/skills/okstra-code-review/SKILL.md +1 -1
  98. package/runtime/skills/okstra-code-review/references/census-rules.md +1 -1
  99. package/runtime/skills/okstra-code-review/references/review-calibration.md +1 -1
  100. package/runtime/skills/okstra-run/SKILL.md +31 -0
  101. package/runtime/skills/okstra-tdd/SKILL.md +75 -11
  102. package/runtime/templates/reports/html/assets/implementation-comparison.css +39 -0
  103. package/runtime/templates/reports/html/assets/implementation-comparison.js +55 -0
  104. package/runtime/templates/reports/html/i18n/en.json +113 -0
  105. package/runtime/templates/reports/html/i18n/ko.json +113 -0
  106. package/runtime/templates/reports/html/implementation-comparison.template.html +77 -0
  107. package/runtime/templates/reports/implementation-comparison-notes.md +72 -0
  108. package/runtime/templates/trace/trace.template.md +2 -0
  109. package/runtime/validators/validate-report-views.py +4 -36
@@ -0,0 +1,198 @@
1
+ """Connect optional run comparison facts to report writing and closeout."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ from pathlib import Path
6
+ import subprocess
7
+ from typing import Any, Mapping
8
+
9
+ from okstra_vendor.jinja2 import TemplateError
10
+
11
+ from ..i18n import I18nError
12
+ from ..json_boundary import PageBudgetExhausted, load_owned_object_snapshot, read_bounded_bytes
13
+ from ..paths import find_asset_root
14
+ from ..qa_procedures import project_artifact_relative_path, safe_qa_path
15
+ from .baseline import (
16
+ MAX_BASELINE_BYTES, MAX_LIMITATION_SUMMARY_CHARS, ComparisonOwner,
17
+ read_baseline_status,
18
+ )
19
+ from .inputs import run_comparison_scope, task_comparison_owner
20
+ from .model import MAX_COMPARISON_INPUT_BYTES
21
+ from .run_evidence import read_run_evidence
22
+ from .target import _current_target, capture_target
23
+
24
+
25
+ def _run_scope(
26
+ root: Path, manifest_path: Path, manifest: Mapping[str, Any],
27
+ ) -> tuple[ComparisonOwner, Path] | None:
28
+ if manifest.get("taskType") != "implementation" or "implementationComparison" not in manifest:
29
+ return None
30
+ if manifest["implementationComparison"] is None:
31
+ raise ValueError("Run comparison declaration is invalid")
32
+ owner, directory, _ = run_comparison_scope(root, manifest_path, manifest)
33
+ safe_qa_path(root, (directory / "notes.json").relative_to(root).as_posix())
34
+ return owner, directory
35
+
36
+
37
+ def _saved_scope(root: Path, manifest_path: Path) -> tuple[ComparisonOwner, Path] | None:
38
+ path = safe_qa_path(root, project_artifact_relative_path(root, manifest_path))
39
+ manifest = load_owned_object_snapshot(
40
+ path, artifact="comparison run manifest", max_bytes=MAX_BASELINE_BYTES,
41
+ ).value
42
+ return _run_scope(root, path, manifest)
43
+
44
+
45
+ def _unavailable(reason: str) -> dict[str, Any]:
46
+ return {
47
+ "schemaVersion": "1.0", "owner": None, "baselineRef": None,
48
+ "targetRef": None, "notesPath": None, "generation": None,
49
+ "recordIds": [], "evidence": [], "notesGuide": None,
50
+ "limitations": [reason[:MAX_LIMITATION_SUMMARY_CHARS]],
51
+ }
52
+
53
+
54
+ def _notes_guide() -> tuple[str | None, list[str]]:
55
+ relative = ("templates", "reports", "implementation-comparison-notes.md")
56
+ try:
57
+ assets = find_asset_root(relative, is_present=Path.is_file)
58
+ if assets is None:
59
+ raise ValueError("Packaged comparison notes guide is missing")
60
+ return read_bounded_bytes(
61
+ assets.joinpath(*relative), max_bytes=MAX_COMPARISON_INPUT_BYTES,
62
+ artifact="implementation comparison notes guide",
63
+ ).decode("utf-8"), []
64
+ except (OSError, ValueError) as exc:
65
+ return None, [f"Comparison notes guide was not supplied: {exc}"]
66
+
67
+
68
+ def _writer_comparison(
69
+ root: Path, directory: Path, owner: ComparisonOwner,
70
+ target: Mapping[str, Any], record: Mapping[str, Any] | None,
71
+ ) -> dict[str, Any]:
72
+ baseline = read_baseline_status(root, directory, owner)
73
+ evidence = read_run_evidence(root, owner, record)
74
+ guide, guide_limits = _notes_guide()
75
+ metadata = {
76
+ "schemaVersion": "1.0", "owner": owner,
77
+ "baselineRef": baseline["baselineRef"],
78
+ "targetRef": target["targetRef"] if target["generation"] == evidence.generation else None,
79
+ "notesPath": (directory / "notes.json").relative_to(root).as_posix(),
80
+ "generation": evidence.generation, "recordIds": [], "evidence": [], "limitations": [],
81
+ "notesGuide": guide,
82
+ }
83
+ omitted = "Additional comparison context was omitted at the byte limit"
84
+ remaining = MAX_COMPARISON_INPUT_BYTES - len(json.dumps(metadata, ensure_ascii=True).encode()) - len(json.dumps(omitted).encode()) - 2
85
+ if remaining < 0:
86
+ metadata["notesGuide"] = None
87
+ guide_limits.append("Comparison notes guide exceeds the available metadata byte limit")
88
+ remaining = MAX_COMPARISON_INPUT_BYTES - len(json.dumps(metadata, ensure_ascii=True).encode()) - len(json.dumps(omitted).encode()) - 2
89
+ limits = [*guide_limits, *baseline["limitations"], *target["limitations"], *evidence.limitations]
90
+ if target["generation"] != evidence.generation:
91
+ limits.append("The preserved target does not identify the current completed generation")
92
+ for limitation in limits:
93
+ summary = limitation[:MAX_LIMITATION_SUMMARY_CHARS]
94
+ cost = len(json.dumps(summary, ensure_ascii=True).encode()) + 2
95
+ if cost > remaining:
96
+ metadata["limitations"].append(omitted)
97
+ return metadata
98
+ metadata["limitations"].append(summary)
99
+ remaining -= cost
100
+ for row in evidence.rows:
101
+ cost = len(json.dumps(row, ensure_ascii=True).encode()) + len(json.dumps(row["recordId"], ensure_ascii=True).encode()) + 4
102
+ if cost > remaining:
103
+ metadata["limitations"].append(omitted)
104
+ break
105
+ metadata["evidence"].append(row)
106
+ metadata["recordIds"].append(row["recordId"])
107
+ remaining -= cost
108
+ return metadata
109
+
110
+
111
+ def prepare_run_comparison(project_root: Path, manifest_path: Path) -> dict[str, Any] | None:
112
+ root = Path(project_root).resolve()
113
+ try:
114
+ scope = _saved_scope(root, manifest_path)
115
+ if scope is None:
116
+ return None
117
+ owner, directory = scope
118
+ evidence = read_run_evidence(root, owner)
119
+ target, record, _ = _current_target(root, directory, owner)
120
+ if evidence.generation is not None and evidence.source_root is not None and target["generation"] != evidence.generation:
121
+ capture_target(root, Path(evidence.source_root), directory, owner, generation=evidence.generation)
122
+ target, record, _ = _current_target(root, directory, owner)
123
+ return _writer_comparison(root, directory, owner, target, record)
124
+ except (OSError, ValueError, PageBudgetExhausted, subprocess.SubprocessError) as exc:
125
+ return _unavailable(f"Run comparison preparation failed: {exc}")
126
+
127
+
128
+ def read_run_comparison(project_root: Path, manifest_path: Path) -> dict[str, Any] | None:
129
+ root = Path(project_root).resolve()
130
+ try:
131
+ scope = _saved_scope(root, manifest_path)
132
+ if scope is None:
133
+ return None
134
+ owner, directory = scope
135
+ target, record, _ = _current_target(root, directory, owner)
136
+ return _writer_comparison(root, directory, owner, target, record)
137
+ except (OSError, ValueError, PageBudgetExhausted) as exc:
138
+ return _unavailable(f"Run comparison input could not be read: {exc}")
139
+
140
+
141
+ def run_comparison_notes_paths(
142
+ project_root: Path, manifest_path: Path, manifest: Mapping[str, Any], *, role: str,
143
+ ) -> tuple[Path, ...]:
144
+ if role != "report-writer":
145
+ return ()
146
+ root = Path(project_root).resolve()
147
+ try:
148
+ scope = _run_scope(root, manifest_path, manifest)
149
+ except (OSError, ValueError):
150
+ return ()
151
+ return (scope[1] / "notes.json",) if scope is not None else ()
152
+
153
+
154
+ def finalize_run_comparison(project_root: Path, manifest_path: Path) -> dict[str, Any] | None:
155
+ from .publication import publish_comparison, read_comparison_status, update_implementation_comparison_link
156
+ from .run_evidence import check_live_run_target
157
+
158
+ root = Path(project_root).resolve()
159
+ warnings = []
160
+ try:
161
+ scope = _saved_scope(root, manifest_path)
162
+ if scope is None:
163
+ return None
164
+ owner, directory = scope
165
+ target, record, _ = _current_target(root, directory, owner)
166
+ limitations = check_live_run_target(root, owner, record) if record is not None and target["targetRef"] is not None else ()
167
+ if limitations:
168
+ comparison = {**read_comparison_status(root, directory, owner), "status": "stale"}
169
+ return {
170
+ "comparison": comparison, "linkStatus": "not-updated",
171
+ "warnings": [{"stage": "target", "message": message} for message in limitations],
172
+ }
173
+ comparison = publish_comparison(root, directory, owner)
174
+ except (OSError, ValueError, PageBudgetExhausted, subprocess.SubprocessError, TemplateError) as exc:
175
+ warnings.append({"stage": "publication", "message": str(exc)})
176
+ return {"comparison": {"status": "failed"}, "linkStatus": "not-updated", "warnings": warnings}
177
+ link_status = "not-updated"
178
+ if comparison["status"] in {"published", "reused"}:
179
+ try:
180
+ update_implementation_comparison_link(root, directory, owner)
181
+ except (OSError, ValueError, PageBudgetExhausted) as exc:
182
+ link_status = "failed"
183
+ warnings.append({"stage": "link", "message": str(exc)})
184
+ else:
185
+ link_status = "updated"
186
+ else:
187
+ warnings.extend({"stage": "publication", "message": message} for message in comparison.get("warnings", []))
188
+ return {"comparison": comparison, "linkStatus": link_status, "warnings": warnings}
189
+
190
+
191
+ def finalize_task_comparison(project_root: Path, task_root: Path, task_key: str) -> dict[str, Any]:
192
+ from .publication import publish_comparison
193
+
194
+ try:
195
+ owner = task_comparison_owner(project_root, task_root, task_key)
196
+ return publish_comparison(project_root, None, owner)
197
+ except (OSError, ValueError, KeyError, PageBudgetExhausted, subprocess.SubprocessError, TemplateError, RecursionError, I18nError) as exc:
198
+ return {"status": "failed", "htmlPath": None, "warnings": [f"Task comparison could not be generated: {exc}"]}
@@ -0,0 +1,342 @@
1
+ """Bounded, read-only Git and working-source collection."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import os
6
+ import selectors
7
+ import signal
8
+ import stat
9
+ import subprocess
10
+ import time
11
+ from dataclasses import dataclass
12
+ from contextlib import contextmanager
13
+ from pathlib import Path, PurePosixPath
14
+ from typing import Any, Iterator
15
+
16
+ from ..json_boundary import PageBudgetExhausted, ReadBudget
17
+ from ..process_group import kill_group
18
+
19
+ MAX_PATHS = 10_000
20
+ MAX_FILE_READ_BYTES = 1024 * 1024
21
+ MAX_TOTAL_READ_BYTES = 64 * 1024 * 1024
22
+ MAX_GIT_COMMAND_BYTES = 4 * 1024 * 1024
23
+ MAX_GIT_TOTAL_BYTES = 8 * 1024 * 1024
24
+ GIT_TIMEOUT_SECONDS = 10
25
+ READ_CHUNK_BYTES = 64 * 1024
26
+ SOURCE_CHANGED_DURING_READ = "Source changed while being read"
27
+ EXCLUDED_PARTS = frozenset({
28
+ ".git", ".okstra", ".project-docs", ".worktrees", "node_modules", "vendor",
29
+ ".venv", "venv", "__pycache__", ".pytest_cache", ".cache", ".mypy_cache",
30
+ ".ruff_cache", "dist", "build", "runtime", "target", "coverage", ".next",
31
+ "graphify-out",
32
+ })
33
+
34
+
35
+ def digest_bytes(body: bytes) -> str:
36
+ return hashlib.sha256(body).hexdigest()
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class GitOutput:
41
+ stdout: bytes
42
+ stderr: bytes
43
+ returncode: int | None
44
+ complete: bool
45
+ reason: str = ""
46
+
47
+
48
+ class GitCollector:
49
+ def __init__(
50
+ self, directory: Path | None = None, *, shared_budget: ReadBudget | None = None,
51
+ deadline: float | None = None,
52
+ ) -> None:
53
+ self.directory = directory
54
+ self.budget = ReadBudget(MAX_GIT_TOTAL_BYTES)
55
+ self.shared_budget = shared_budget
56
+ self.deadline = deadline
57
+ self.outputs: dict[str, dict[str, Any]] = {}
58
+
59
+ def run(self, root: Path, label: str, *args: str) -> GitOutput:
60
+ if self.shared_budget is not None and self.shared_budget.bytes_read >= self.shared_budget.max_bytes:
61
+ raise PageBudgetExhausted("Comparison source byte budget exhausted")
62
+ if self.budget.bytes_read >= self.budget.max_bytes:
63
+ result = GitOutput(b"", b"", None, False, "Git output budget exhausted")
64
+ elif self.deadline is not None and time.monotonic() >= self.deadline:
65
+ result = GitOutput(b"", b"", None, False, "Git request deadline exhausted")
66
+ else:
67
+ result = self._read_command(root, args)
68
+ files: dict[str, Any] = {}
69
+ for name, body in (("stdout", result.stdout), ("stderr", result.stderr)):
70
+ relative = f"git/{label}.{name}"
71
+ if self.directory is not None:
72
+ (self.directory / relative).write_bytes(body)
73
+ files[name] = {"path": relative, "bytes": len(body), "storedDigest": digest_bytes(body)}
74
+ self.outputs[label] = {
75
+ "argv": list(args), "returncode": result.returncode,
76
+ "complete": result.complete, "reason": result.reason, **files,
77
+ }
78
+ return result
79
+
80
+ def _remaining(self, command_bytes: int) -> int:
81
+ remaining = min(MAX_GIT_COMMAND_BYTES - command_bytes,
82
+ self.budget.max_bytes - self.budget.bytes_read)
83
+ if self.shared_budget is not None:
84
+ remaining = min(remaining, self.shared_budget.max_bytes - self.shared_budget.bytes_read)
85
+ return remaining
86
+
87
+ def _read_command(self, root: Path, args: tuple[str, ...]) -> GitOutput:
88
+ env = {key: value for key, value in os.environ.items() if key not in {
89
+ "GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX",
90
+ "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES",
91
+ }}
92
+ command = ["git", "--no-pager", "-c", "core.fsmonitor=false", "-C", str(root), *args]
93
+ process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
94
+ stderr=subprocess.PIPE, env=env, start_new_session=True)
95
+ bodies = [bytearray(), bytearray()]
96
+ reason = ""
97
+ deadline = time.monotonic() + GIT_TIMEOUT_SECONDS
98
+ if self.deadline is not None:
99
+ deadline = min(deadline, self.deadline)
100
+ try:
101
+ with selectors.DefaultSelector() as selector:
102
+ for index, pipe in enumerate((process.stdout, process.stderr)):
103
+ selector.register(pipe, selectors.EVENT_READ, index)
104
+ while selector.get_map():
105
+ remaining = self._remaining(sum(map(len, bodies)))
106
+ if remaining <= 0 or time.monotonic() >= deadline:
107
+ reason = "Git output limit reached" if remaining <= 0 else "Git command timed out"
108
+ break
109
+ for key, _ in selector.select(timeout=min(0.1, max(0, deadline - time.monotonic()))):
110
+ remaining = self._remaining(sum(map(len, bodies)))
111
+ if remaining <= 0:
112
+ break
113
+ chunk = os.read(key.fd, min(READ_CHUNK_BYTES, remaining))
114
+ self.budget.bytes_read += len(chunk)
115
+ if self.shared_budget is not None and self.shared_budget is not self.budget:
116
+ self.shared_budget.bytes_read += len(chunk)
117
+ bodies[key.data].extend(chunk)
118
+ remaining -= len(chunk)
119
+ if not chunk:
120
+ selector.unregister(key.fileobj)
121
+ if reason:
122
+ kill_group(process.pid, signal.SIGKILL)
123
+ try:
124
+ returncode = process.wait(timeout=max(0, deadline - time.monotonic()))
125
+ except subprocess.TimeoutExpired:
126
+ reason = reason or "Git command timed out"
127
+ kill_group(process.pid, signal.SIGKILL)
128
+ returncode = process.wait()
129
+ finally:
130
+ if process.poll() is None:
131
+ kill_group(process.pid, signal.SIGKILL)
132
+ process.wait()
133
+ for pipe in (process.stdout, process.stderr):
134
+ if pipe is not None:
135
+ pipe.close()
136
+ return GitOutput(bytes(bodies[0]), bytes(bodies[1]), returncode, not reason, reason)
137
+
138
+
139
+ @contextmanager
140
+ def _source_parent(root: Path, relative: str) -> Iterator[tuple[int, str]]:
141
+ parts = PurePosixPath(relative).parts
142
+ if not parts or PurePosixPath(relative).is_absolute() or any(p in {".", ".."} for p in parts):
143
+ raise ValueError("Source path is not relative to its root")
144
+ directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
145
+ try:
146
+ for part in parts[:-1]:
147
+ child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory)
148
+ os.close(directory)
149
+ directory = child
150
+ yield directory, parts[-1]
151
+ finally:
152
+ os.close(directory)
153
+
154
+
155
+ def open_source_file(root: Path, relative: str) -> int:
156
+ with _source_parent(root, relative) as (directory, name):
157
+ return os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory)
158
+
159
+
160
+ def _source_info(root: Path, relative: str) -> tuple[os.stat_result, str | None]:
161
+ with _source_parent(root, relative) as (directory, name):
162
+ info = os.stat(name, dir_fd=directory, follow_symlinks=False)
163
+ return info, os.readlink(name, dir_fd=directory) if stat.S_ISLNK(info.st_mode) else None
164
+
165
+
166
+ def read_source_bytes(
167
+ root: Path, relative: str, budget: ReadBudget, *, file_limit: int | None = None,
168
+ shared_budget: ReadBudget | None = None,
169
+ ) -> tuple[bytes | None, os.stat_result, str]:
170
+ descriptor = open_source_file(root, relative)
171
+ try:
172
+ before = os.fstat(descriptor)
173
+ if not stat.S_ISREG(before.st_mode):
174
+ return None, before, "Source is not a regular file"
175
+ maximum = min(MAX_FILE_READ_BYTES if file_limit is None else file_limit,
176
+ budget.max_bytes - budget.bytes_read)
177
+ if shared_budget is not None and before.st_size > shared_budget.max_bytes - shared_budget.bytes_read:
178
+ raise PageBudgetExhausted("Comparison source byte budget exhausted")
179
+ if before.st_size > maximum:
180
+ return None, before, "Source exceeds the remaining file read budget"
181
+ body = bytearray()
182
+ while len(body) < before.st_size:
183
+ chunk = os.read(descriptor, min(READ_CHUNK_BYTES, before.st_size - len(body)))
184
+ budget.bytes_read += len(chunk)
185
+ if shared_budget is not None and shared_budget is not budget:
186
+ shared_budget.bytes_read += len(chunk)
187
+ if not chunk:
188
+ break
189
+ body.extend(chunk)
190
+ after = os.fstat(descriptor)
191
+ if file_state(before) != file_state(after) or len(body) != before.st_size:
192
+ return None, after, SOURCE_CHANGED_DURING_READ
193
+ return bytes(body), after, ""
194
+ finally:
195
+ os.close(descriptor)
196
+
197
+
198
+ def file_state(info: os.stat_result) -> tuple[int, int, int, int, int]:
199
+ return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns
200
+
201
+
202
+ def _names(output: GitOutput) -> list[bytes]:
203
+ return output.stdout.split(b"\0") if output.complete and output.returncode == 0 else []
204
+
205
+
206
+ def _pathspecs() -> list[str]:
207
+ return ["--", ".", *[f":(glob,exclude)**/{part}/**" for part in sorted(EXCLUDED_PARTS)]]
208
+
209
+
210
+ def collect_source(project_root: Path, source_root: Path, requested_ref: str, directory: Path) -> dict[str, Any]:
211
+ (directory / "git").mkdir()
212
+ (directory / "contents").mkdir()
213
+ git = GitCollector(directory)
214
+ budget = ReadBudget(MAX_TOTAL_READ_BYTES)
215
+ result: dict[str, Any] = {
216
+ "head": None, "baselineCommit": None, "files": [], "limitations": [],
217
+ "excludedParts": sorted(EXCLUDED_PARTS), "completeInventory": False,
218
+ "scope": {"workingTreeText": "full-within-read-limits",
219
+ "index": "binary-patch-and-stage-object-identifiers",
220
+ "commitTree": "commit-identity-only"},
221
+ }
222
+ project_git = git.run(project_root, "project-repository", "rev-parse", "--git-common-dir")
223
+ source_git = project_git if project_root == source_root else git.run(
224
+ source_root, "source-repository", "rev-parse", "--git-common-dir",
225
+ )
226
+ if any(not item.complete or item.returncode != 0 for item in (project_git, source_git)):
227
+ result["limitations"].append("Git repository is unavailable; non-Git source enumeration was not attempted")
228
+ return _finish_source(result, git, budget, 0)
229
+ project_common = (project_root / os.fsdecode(project_git.stdout).strip()).resolve()
230
+ source_common = (source_root / os.fsdecode(source_git.stdout).strip()).resolve()
231
+ if project_common != source_common:
232
+ raise ValueError("Source root belongs to a different Git repository")
233
+ head = git.run(source_root, "head-before", "rev-parse", "--verify", "HEAD^{commit}")
234
+ baseline = head if not requested_ref else git.run(
235
+ source_root, "baseline-commit", "rev-parse", "--verify", "--end-of-options", f"{requested_ref}^{{commit}}",
236
+ )
237
+ for key, value in (("head", head), ("baselineCommit", baseline)):
238
+ if value.complete and value.returncode == 0:
239
+ result[key] = value.stdout.decode("ascii").strip()
240
+ else:
241
+ result["limitations"].append(f"{key} could not be resolved")
242
+ before = _git_state(git, source_root, "before")
243
+ entries: dict[str, dict[str, Any]] = {}
244
+ paths_omitted = False
245
+ for raw in _names(before["tracked"]):
246
+ if not raw:
247
+ continue
248
+ entry, name = raw.split(b"\t", 1)
249
+ path = os.fsdecode(name)
250
+ if any(part in EXCLUDED_PARTS for part in PurePosixPath(path).parts):
251
+ continue
252
+ if path not in entries and len(entries) >= MAX_PATHS:
253
+ paths_omitted = True
254
+ continue
255
+ mode, object_id, stage = entry.decode("ascii").split()
256
+ row = entries.setdefault(path, {"path": path, "tracked": True, "indexEntries": []})
257
+ row["indexEntries"].append({"mode": mode, "object": object_id, "stage": stage})
258
+ for raw in _names(before["untracked"]):
259
+ if not raw:
260
+ continue
261
+ path = os.fsdecode(raw)
262
+ if any(part in EXCLUDED_PARTS for part in PurePosixPath(path).parts):
263
+ continue
264
+ if path not in entries and len(entries) >= MAX_PATHS:
265
+ paths_omitted = True
266
+ continue
267
+ entries.setdefault(path, {"path": path, "tracked": False})
268
+ complete_inventory = all(value.complete and value.returncode == 0 for value in before.values())
269
+ if paths_omitted:
270
+ complete_inventory = False
271
+ result["limitations"].append("Source path limit reached; inventory may be incomplete")
272
+ if not complete_inventory:
273
+ result["limitations"].append("Git inventory, status, or index capture is incomplete")
274
+ result["completeInventory"] = complete_inventory
275
+ saved_bytes = 0
276
+ states: dict[str, tuple[int, int, int, int, int]] = {}
277
+ for path, entry in entries.items():
278
+ row = {**entry, "digest": None, "contentPath": None}
279
+ try:
280
+ info, link = _source_info(source_root, path)
281
+ if link is not None:
282
+ body, problem = None, "Symbolic link source was not followed"
283
+ row.update({"kind": "symlink", "linkTarget": link})
284
+ else:
285
+ body, info, problem = read_source_bytes(source_root, path, budget)
286
+ row.update({"mode": info.st_mode, "size": info.st_size})
287
+ states[path] = file_state(info)
288
+ if problem:
289
+ row.update({"status": "partial", "reason": problem})
290
+ else:
291
+ row.update({"status": "complete", "digest": digest_bytes(body)})
292
+ try:
293
+ text = body.decode("utf-8")
294
+ is_text = "\0" not in text
295
+ except UnicodeDecodeError:
296
+ is_text = False
297
+ row["kind"] = "text" if is_text else "binary"
298
+ if is_text:
299
+ relative = f"contents/{row['digest']}.txt"
300
+ (directory / relative).write_bytes(body)
301
+ row["contentPath"] = relative
302
+ saved_bytes += len(body)
303
+ except FileNotFoundError:
304
+ row.update({"status": "missing", "reason": "Source was absent during capture"})
305
+ except (OSError, ValueError) as exc:
306
+ row.update({"status": "partial", "reason": f"Source was not followed or could not be read: {exc}"})
307
+ result["files"].append(row)
308
+ if row["status"] not in {"complete", "missing"}:
309
+ result["limitations"].append(f"{path}: {row['reason']}")
310
+ after = _git_state(git, source_root, "after")
311
+ after_head = git.run(source_root, "head-after", "rev-parse", "--verify", "HEAD^{commit}")
312
+ if after_head != head:
313
+ result["limitations"].append("HEAD changed or could not be rechecked during capture")
314
+ for key, previous in before.items():
315
+ current = after[key]
316
+ if not current.complete or current.returncode != 0 or current.stdout != previous.stdout:
317
+ result["limitations"].append(f"Git {key} changed or could not be rechecked during capture")
318
+ for path, previous in states.items():
319
+ try:
320
+ if file_state(_source_info(source_root, path)[0]) != previous:
321
+ result["limitations"].append(f"{path}: source changed after its content was captured")
322
+ except OSError:
323
+ result["limitations"].append(f"{path}: source disappeared after its content was captured")
324
+ return _finish_source(result, git, budget, saved_bytes)
325
+
326
+
327
+ def _git_state(git: GitCollector, root: Path, suffix: str) -> dict[str, GitOutput]:
328
+ return {
329
+ "tracked": git.run(root, f"tracked-{suffix}", "ls-files", "--stage", "-z", *_pathspecs()),
330
+ "untracked": git.run(root, f"untracked-{suffix}", "ls-files", "--others", "--exclude-standard", "-z", *_pathspecs()),
331
+ "status": git.run(root, f"status-{suffix}", "status", "--porcelain=v1", "-z", "--untracked-files=all", *_pathspecs()),
332
+ "index": git.run(root, f"index-{suffix}", "diff", "--no-ext-diff", "--no-textconv", "--cached", "--binary", *_pathspecs()),
333
+ }
334
+
335
+
336
+ def _finish_source(result: dict[str, Any], git: GitCollector, budget: ReadBudget, saved: int) -> dict[str, Any]:
337
+ result.update({"git": git.outputs, "bytesRead": budget.bytes_read,
338
+ "bytesPreserved": saved, "gitBytesRead": git.budget.bytes_read})
339
+ result["status"] = "partial" if result["limitations"] else "complete"
340
+ if not result["completeInventory"] and not result["files"]:
341
+ result["status"] = "unavailable"
342
+ return result