okstra 0.228.2 → 0.229.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/architecture/storage-model.md +77 -0
- package/docs/cli.md +1 -1
- package/docs/project-structure-overview.md +5 -3
- package/package.json +1 -1
- package/runtime/BUILD.json +2 -2
- package/runtime/bin/okstra-provider-exec.py +53 -4
- package/runtime/prompts/duties/code-reviewer.json +5 -2
- package/runtime/python/okstra_ctl/code_review_coverage.py +15 -4
- package/runtime/python/okstra_ctl/code_review_scope.py +2 -1
- package/runtime/python/okstra_ctl/code_review_target.py +41 -23
- package/runtime/python/okstra_ctl/conformance.py +31 -0
- package/runtime/python/okstra_ctl/execution_mutation_audit.py +100 -2
- package/runtime/python/okstra_ctl/model_io/renderers.py +10 -1
- package/runtime/python/okstra_ctl/paths.py +12 -1
- package/runtime/python/okstra_ctl/qa_commands.py +2 -1
- package/runtime/python/okstra_ctl/qa_procedures.py +60 -17
- package/runtime/python/okstra_ctl/verification_conformance.py +163 -0
- package/runtime/python/okstra_ctl/verification_generation.py +238 -0
- package/runtime/python/okstra_ctl/verification_input_types.py +21 -2
- package/runtime/python/okstra_ctl/verification_inputs.py +90 -23
- package/runtime/python/okstra_ctl/verification_target.py +13 -5
- package/runtime/skills/okstra-code-review/SKILL.md +80 -46
- package/runtime/skills/okstra-code-review/references/census-rules.md +8 -2
- package/runtime/skills/okstra-code-review/references/review-calibration.md +7 -4
- package/runtime/validators/validate-run.py +8 -33
|
@@ -161,9 +161,18 @@ def render_verification_input(
|
|
|
161
161
|
"when reading commands and procedure bodies.\n",
|
|
162
162
|
"\n## QA Commands\n\n",
|
|
163
163
|
*quoted_value_lines("QA commands", inputs["qaCommands"]),
|
|
164
|
+
]
|
|
165
|
+
rows.append("\n## QA Source Observations\n\n")
|
|
166
|
+
if "qaSourceObservations" in inputs:
|
|
167
|
+
rows.extend(quoted_value_lines(
|
|
168
|
+
"QA source observations", inputs["qaSourceObservations"],
|
|
169
|
+
))
|
|
170
|
+
else:
|
|
171
|
+
rows.append("Source observations were not captured in this input.\n")
|
|
172
|
+
rows.extend([
|
|
164
173
|
"\n## Procedures\n\n",
|
|
165
174
|
*quoted_value_lines("Procedures", inputs["procedures"]),
|
|
166
|
-
]
|
|
175
|
+
])
|
|
167
176
|
if "approvedPlan" in inputs:
|
|
168
177
|
rows.extend([
|
|
169
178
|
"\n## Approved Plan\n\n",
|
|
@@ -219,9 +219,20 @@ class RunRef:
|
|
|
219
219
|
def manifest(self) -> Path:
|
|
220
220
|
return self.run_dir / "manifests" / f"run-manifest{self._suffix}.json"
|
|
221
221
|
|
|
222
|
+
@property
|
|
223
|
+
def verification_dir(self) -> Path:
|
|
224
|
+
return self.run_dir / "verification"
|
|
225
|
+
|
|
222
226
|
@property
|
|
223
227
|
def verification_inputs(self) -> Path:
|
|
224
|
-
return self.
|
|
228
|
+
return self.verification_dir / f"inputs{self._suffix}.json"
|
|
229
|
+
|
|
230
|
+
def verification_generation(self, generation: int) -> Path:
|
|
231
|
+
if (isinstance(generation, bool) or not isinstance(generation, int)
|
|
232
|
+
or generation < 1):
|
|
233
|
+
raise ValueError("verification generation must be a positive integer")
|
|
234
|
+
return (self.verification_dir / f"generations{self._suffix}"
|
|
235
|
+
/ f"generation-{generation}.json")
|
|
225
236
|
|
|
226
237
|
@property
|
|
227
238
|
def convergence_state(self) -> Path:
|
|
@@ -28,6 +28,7 @@ from typing import Iterable, Mapping
|
|
|
28
28
|
# diff 가 DB 를 건드릴 때 이 카테고리(또는 plan validation 의 db 스텝)를 요구한다.
|
|
29
29
|
ALLOWED_CATEGORIES: tuple[str, ...] = ("lint", "format", "typecheck", "test", "db-test")
|
|
30
30
|
QA_COMMAND_EXECUTING_TASK_TYPES = ("implementation", "final-verification")
|
|
31
|
+
QA_SOURCE_DIGEST_PATTERN = re.compile(r"sha256:[0-9a-f]{64}")
|
|
31
32
|
QA_RELATIVE_PATH_DESCRIPTION = (
|
|
32
33
|
"non-blank relative POSIX path with at least one component after normalization, "
|
|
33
34
|
"without traversal or backslashes"
|
|
@@ -286,7 +287,7 @@ def qa_metadata_errors(entry: Mapping[str, object], seen_ids: set[str]) -> list[
|
|
|
286
287
|
errors.append("resourceKeys must be non-secret resource identifiers")
|
|
287
288
|
digest = entry.get("sourceBaselineDigest")
|
|
288
289
|
if digest is not None and (
|
|
289
|
-
not isinstance(digest, str) or not
|
|
290
|
+
not isinstance(digest, str) or not QA_SOURCE_DIGEST_PATTERN.fullmatch(digest)
|
|
290
291
|
):
|
|
291
292
|
errors.append("sourceBaselineDigest must be a sha256 digest")
|
|
292
293
|
return errors
|
|
@@ -5,8 +5,9 @@ import hashlib
|
|
|
5
5
|
import json
|
|
6
6
|
import os
|
|
7
7
|
import tempfile
|
|
8
|
+
from functools import cache
|
|
8
9
|
from pathlib import Path
|
|
9
|
-
from typing import Any, Literal, Mapping
|
|
10
|
+
from typing import Any, Callable, Literal, Mapping
|
|
10
11
|
|
|
11
12
|
from okstra_project import project_json_path
|
|
12
13
|
from okstra_project.dirs import OKSTRA_DIR_NAME, json_lock_path, registry_lock
|
|
@@ -16,13 +17,15 @@ from .json_boundary import load_owned_object, write_owned_object_atomic
|
|
|
16
17
|
from .qa_commands import (
|
|
17
18
|
QA_RELATIVE_PATH_DESCRIPTION, qa_relative_path_valid, validate_qa_commands,
|
|
18
19
|
)
|
|
20
|
+
from .verification_input_types import QaSourceObservation, QaSourceStatus
|
|
19
21
|
|
|
20
22
|
PROCEDURE_SECTIONS = (
|
|
21
23
|
"Execution", "Environment", "User actions", "Success observations", "Cleanup",
|
|
22
24
|
)
|
|
25
|
+
QA_SOURCE_PATH_LIMIT = 1024
|
|
23
26
|
|
|
24
27
|
|
|
25
|
-
class QaInspectionRow(
|
|
28
|
+
class QaInspectionRow(QaSourceObservation, total=False):
|
|
26
29
|
category: str
|
|
27
30
|
index: int
|
|
28
31
|
id: str
|
|
@@ -31,11 +34,7 @@ class QaInspectionRow(TypedDict, total=False):
|
|
|
31
34
|
procedurePath: str
|
|
32
35
|
entryDigest: str
|
|
33
36
|
procedureDigest: str
|
|
34
|
-
sourceStatus: Literal[
|
|
35
|
-
"not-declared", "not-queried", "no-baseline", "unchanged", "changed",
|
|
36
|
-
]
|
|
37
37
|
evidenceStatus: Literal["not-queried"]
|
|
38
|
-
error: str
|
|
39
38
|
|
|
40
39
|
|
|
41
40
|
def content_digest(value: bytes) -> str:
|
|
@@ -86,13 +85,62 @@ def read_validated_procedure(path: Path) -> bytes:
|
|
|
86
85
|
return raw
|
|
87
86
|
|
|
88
87
|
|
|
89
|
-
def
|
|
88
|
+
def new_qa_source_digest_reader() -> Callable[[Path], str]:
|
|
89
|
+
@cache
|
|
90
|
+
def read_digest(path: Path) -> str:
|
|
91
|
+
return content_digest(path.read_bytes())
|
|
92
|
+
|
|
93
|
+
return read_digest
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def source_digest(
|
|
97
|
+
source: Path, relatives: list[str], *,
|
|
98
|
+
read_digest: Callable[[Path], str] | None = None,
|
|
99
|
+
) -> str:
|
|
100
|
+
if len(relatives) > QA_SOURCE_PATH_LIMIT:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"related sources must contain at most {QA_SOURCE_PATH_LIMIT} paths"
|
|
103
|
+
)
|
|
104
|
+
reader = read_digest if read_digest is not None else new_qa_source_digest_reader()
|
|
90
105
|
return canonical_json_digest({
|
|
91
|
-
relative:
|
|
106
|
+
relative: reader(existing_qa_file(source, relative))
|
|
92
107
|
for relative in sorted(relatives)
|
|
93
108
|
})
|
|
94
109
|
|
|
95
110
|
|
|
111
|
+
def qa_source_status(
|
|
112
|
+
entry: Mapping[str, Any], observed_digest: str | None,
|
|
113
|
+
) -> QaSourceStatus:
|
|
114
|
+
if not entry.get("sourcePaths"):
|
|
115
|
+
return "not-declared"
|
|
116
|
+
if observed_digest is None:
|
|
117
|
+
return "not-queried"
|
|
118
|
+
baseline = entry.get("sourceBaselineDigest")
|
|
119
|
+
if not baseline:
|
|
120
|
+
return "no-baseline"
|
|
121
|
+
return "unchanged" if observed_digest == baseline else "changed"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def observe_qa_sources(
|
|
125
|
+
source: Path, entry: Mapping[str, Any], read_digest: Callable[[Path], str],
|
|
126
|
+
) -> QaSourceObservation:
|
|
127
|
+
observation: QaSourceObservation = {
|
|
128
|
+
"sourceStatus": qa_source_status(entry, None),
|
|
129
|
+
}
|
|
130
|
+
if not entry.get("sourcePaths"):
|
|
131
|
+
return observation
|
|
132
|
+
try:
|
|
133
|
+
observed = source_digest(source, entry["sourcePaths"], read_digest=read_digest)
|
|
134
|
+
except (OSError, ValueError) as exc:
|
|
135
|
+
observation["error"] = str(exc)
|
|
136
|
+
else:
|
|
137
|
+
observation.update(
|
|
138
|
+
sourceStatus=qa_source_status(entry, observed),
|
|
139
|
+
observedSourceDigest=observed,
|
|
140
|
+
)
|
|
141
|
+
return observation
|
|
142
|
+
|
|
143
|
+
|
|
96
144
|
def load_qa_config(project: Path) -> dict[str, Any]:
|
|
97
145
|
config = load_owned_object(project_json_path(project), artifact="project config")
|
|
98
146
|
errors = validate_qa_commands(config.get("qaCommands"))
|
|
@@ -103,29 +151,24 @@ def load_qa_config(project: Path) -> dict[str, Any]:
|
|
|
103
151
|
|
|
104
152
|
def inspect_commands(project: Path, source: Path) -> list[QaInspectionRow]:
|
|
105
153
|
config = load_qa_config(project)
|
|
154
|
+
read_digest = new_qa_source_digest_reader()
|
|
106
155
|
rows: list[QaInspectionRow] = []
|
|
107
156
|
for category, entries in (config.get("qaCommands") or {}).items():
|
|
108
157
|
for index, entry in enumerate(entries):
|
|
109
158
|
row: QaInspectionRow = {
|
|
110
159
|
**entry, "category": category, "index": index,
|
|
111
160
|
"entryDigest": canonical_json_digest(entry),
|
|
112
|
-
"sourceStatus": (
|
|
113
|
-
"not-queried" if entry.get("sourcePaths") else "not-declared"
|
|
114
|
-
),
|
|
161
|
+
"sourceStatus": qa_source_status(entry, None),
|
|
115
162
|
"evidenceStatus": "not-queried",
|
|
116
163
|
}
|
|
164
|
+
row.pop("observedSourceDigest", None)
|
|
117
165
|
try:
|
|
118
166
|
relative = entry.get("procedurePath")
|
|
119
167
|
if relative:
|
|
120
168
|
root = procedure_root(project, source, relative)
|
|
121
169
|
raw = read_validated_procedure(existing_qa_file(root, relative))
|
|
122
170
|
row["procedureDigest"] = content_digest(raw)
|
|
123
|
-
|
|
124
|
-
observed = source_digest(source, entry["sourcePaths"])
|
|
125
|
-
baseline = entry.get("sourceBaselineDigest")
|
|
126
|
-
row["sourceStatus"] = "no-baseline" if not baseline else (
|
|
127
|
-
"unchanged" if observed == baseline else "changed"
|
|
128
|
-
)
|
|
171
|
+
row.update(observe_qa_sources(source, entry, read_digest))
|
|
129
172
|
except (OSError, ValueError) as exc:
|
|
130
173
|
row["error"] = str(exc)
|
|
131
174
|
rows.append(row)
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Select and validate actual conformance inputs for an immutable generation."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Mapping
|
|
8
|
+
|
|
9
|
+
from .conformance import (
|
|
10
|
+
conformance_entry_mismatches, normalize_conformance_script,
|
|
11
|
+
validate_conformance_manifest,
|
|
12
|
+
)
|
|
13
|
+
from .json_boundary import load_owned_object_snapshot
|
|
14
|
+
from .paths import RunRef
|
|
15
|
+
from .qa_procedures import content_digest, existing_qa_file, safe_qa_path
|
|
16
|
+
from .verification_input_types import FrozenVerificationPlan
|
|
17
|
+
|
|
18
|
+
_MAX_SCRIPT_PATH_LENGTH = 4096
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class GenerationConformance:
|
|
23
|
+
stages: tuple[int, ...]
|
|
24
|
+
artifact_paths: frozenset[str]
|
|
25
|
+
manifest: tuple[str, str] | None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _project_script_path(project: Path, task_root: Path, script: str) -> str:
|
|
29
|
+
script = normalize_conformance_script(script)
|
|
30
|
+
path = Path(script)
|
|
31
|
+
task_relative = task_root.relative_to(project)
|
|
32
|
+
if path.is_absolute():
|
|
33
|
+
if not path.is_relative_to(task_root):
|
|
34
|
+
raise ValueError(f"conformance script escapes task root: {script}")
|
|
35
|
+
path = path.relative_to(task_root)
|
|
36
|
+
elif path.is_relative_to(task_relative):
|
|
37
|
+
path = path.relative_to(task_relative)
|
|
38
|
+
relative = path.as_posix()
|
|
39
|
+
if (len(relative) > _MAX_SCRIPT_PATH_LENGTH
|
|
40
|
+
or not relative.startswith("qa/scripts/")):
|
|
41
|
+
raise ValueError(
|
|
42
|
+
f"conformance script must remain under task qa/scripts: {script}"
|
|
43
|
+
)
|
|
44
|
+
safe = safe_qa_path(task_root, relative)
|
|
45
|
+
return safe.relative_to(project).as_posix()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _entries_by_stage(
|
|
49
|
+
manifest: Mapping[str, Any], declared_stages: set[int],
|
|
50
|
+
) -> dict[int, dict[str, Any]]:
|
|
51
|
+
entries = {}
|
|
52
|
+
for entry in manifest["entries"]:
|
|
53
|
+
match = re.fullmatch(r".+-stage-([1-9][0-9]*)", entry["stageKey"])
|
|
54
|
+
if match is None:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"conformance stageKey must end in -stage-<positive integer>"
|
|
57
|
+
)
|
|
58
|
+
stage = int(match[1])
|
|
59
|
+
if stage in entries:
|
|
60
|
+
raise ValueError(f"conformance stage {stage} has multiple entries")
|
|
61
|
+
if stage not in declared_stages:
|
|
62
|
+
raise ValueError(f"conformance stage {stage} is undeclared")
|
|
63
|
+
entries[stage] = entry
|
|
64
|
+
return entries
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _selected_stages(
|
|
68
|
+
ref: RunRef, declarations: Mapping[int, dict[str, Any]],
|
|
69
|
+
declared_paths: Mapping[int, str], actual_paths: Mapping[int, str],
|
|
70
|
+
baseline_paths: Mapping[int, str], changed_paths: frozenset[str],
|
|
71
|
+
) -> tuple[int, ...]:
|
|
72
|
+
if ref.task_type == "final-verification":
|
|
73
|
+
return tuple(sorted(declarations))
|
|
74
|
+
stage = ref.stage
|
|
75
|
+
if stage not in declarations or isinstance(stage, bool):
|
|
76
|
+
raise ValueError(f"conformance stage {stage} has no approved declaration")
|
|
77
|
+
selected = {stage}
|
|
78
|
+
for number in declarations:
|
|
79
|
+
if number < stage and any(
|
|
80
|
+
paths.get(number) in changed_paths
|
|
81
|
+
for paths in (declared_paths, actual_paths, baseline_paths)
|
|
82
|
+
):
|
|
83
|
+
selected.add(number)
|
|
84
|
+
return tuple(sorted(selected))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _check_selected_entry(
|
|
88
|
+
stage: int, declaration: Mapping[str, Any], entry: Mapping[str, Any] | None,
|
|
89
|
+
declared_paths: Mapping[int, str], actual_paths: Mapping[int, str],
|
|
90
|
+
) -> None:
|
|
91
|
+
if "script" not in declaration:
|
|
92
|
+
return
|
|
93
|
+
if entry is None:
|
|
94
|
+
raise ValueError(f"conformance stage {stage} has no matching manifest entry")
|
|
95
|
+
errors = conformance_entry_mismatches(
|
|
96
|
+
actual_script=actual_paths[stage], declared_script=declared_paths[stage],
|
|
97
|
+
actual_requires=entry.get("requires"),
|
|
98
|
+
declared_requires=frozenset(declaration["requires"]),
|
|
99
|
+
)
|
|
100
|
+
if errors:
|
|
101
|
+
raise ValueError(f"conformance stage {stage} {errors[0]}")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _read_conformance_manifest(
|
|
105
|
+
project: Path, relative: str, declared_stages: set[int],
|
|
106
|
+
) -> tuple[dict[int, dict[str, Any]], tuple[str, str] | None]:
|
|
107
|
+
path = safe_qa_path(project, relative)
|
|
108
|
+
if not path.exists():
|
|
109
|
+
return {}, None
|
|
110
|
+
stored = load_owned_object_snapshot(
|
|
111
|
+
existing_qa_file(project, relative),
|
|
112
|
+
artifact="verification conformance manifest",
|
|
113
|
+
validate_cross_fields=validate_conformance_manifest,
|
|
114
|
+
)
|
|
115
|
+
return (
|
|
116
|
+
_entries_by_stage(stored.value, declared_stages),
|
|
117
|
+
(relative, content_digest(stored.raw_bytes)),
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def capture_generation_conformance(
|
|
122
|
+
project: Path, ref: RunRef, plan: FrozenVerificationPlan,
|
|
123
|
+
changed_artifact_paths: frozenset[str], conformance_baseline: Mapping[int, str],
|
|
124
|
+
) -> GenerationConformance:
|
|
125
|
+
declarations = {row["stage"]: row for row in plan["conformance"]}
|
|
126
|
+
if any(type(stage) is not int or stage not in declarations
|
|
127
|
+
for stage in conformance_baseline):
|
|
128
|
+
raise ValueError("conformance baseline has an invalid or undeclared stage")
|
|
129
|
+
baseline_paths = {
|
|
130
|
+
stage: _project_script_path(project, ref.task_root, script)
|
|
131
|
+
for stage, script in conformance_baseline.items()
|
|
132
|
+
}
|
|
133
|
+
declared_paths = {
|
|
134
|
+
stage: _project_script_path(project, ref.task_root, row["script"])
|
|
135
|
+
for stage, row in declarations.items() if "script" in row
|
|
136
|
+
}
|
|
137
|
+
relative_manifest = (ref.task_root / "qa/conformance-manifest.json").relative_to(
|
|
138
|
+
project,
|
|
139
|
+
).as_posix()
|
|
140
|
+
entries, sealed_manifest = _read_conformance_manifest(
|
|
141
|
+
project, relative_manifest, set(declarations),
|
|
142
|
+
)
|
|
143
|
+
actual_paths = {
|
|
144
|
+
stage: _project_script_path(project, ref.task_root, row["script"])
|
|
145
|
+
for stage, row in entries.items()
|
|
146
|
+
}
|
|
147
|
+
stages = _selected_stages(
|
|
148
|
+
ref, declarations, declared_paths, actual_paths, baseline_paths,
|
|
149
|
+
changed_artifact_paths,
|
|
150
|
+
)
|
|
151
|
+
artifacts = {relative_manifest} if sealed_manifest is not None else set()
|
|
152
|
+
for stage in stages:
|
|
153
|
+
if (baseline_paths.get(stage) in changed_artifact_paths
|
|
154
|
+
and stage not in entries):
|
|
155
|
+
raise ValueError(f"changed conformance stage {stage} has no actual entry")
|
|
156
|
+
_check_selected_entry(
|
|
157
|
+
stage, declarations[stage], entries.get(stage),
|
|
158
|
+
declared_paths, actual_paths,
|
|
159
|
+
)
|
|
160
|
+
if stage in actual_paths:
|
|
161
|
+
existing_qa_file(project, actual_paths[stage])
|
|
162
|
+
artifacts.add(actual_paths[stage])
|
|
163
|
+
return GenerationConformance(stages, frozenset(artifacts), sealed_manifest)
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Immutable source and inspection-file seals for verification generations."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import replace
|
|
5
|
+
from functools import partial
|
|
6
|
+
from itertools import pairwise
|
|
7
|
+
from pathlib import Path, PurePosixPath
|
|
8
|
+
from typing import Any, Literal, Mapping, TypedDict, cast
|
|
9
|
+
|
|
10
|
+
from okstra_project.dirs import OKSTRA_DIR_NAME
|
|
11
|
+
|
|
12
|
+
from .execution_mutation_audit import (
|
|
13
|
+
VerificationSourceSnapshot, capture_verification_source,
|
|
14
|
+
is_generated_verification_path,
|
|
15
|
+
)
|
|
16
|
+
from .json_boundary import load_owned_object_snapshot
|
|
17
|
+
from .paths import RunRef
|
|
18
|
+
from .qa_commands import (
|
|
19
|
+
QA_COMMAND_EXECUTING_TASK_TYPES, QA_SOURCE_DIGEST_PATTERN, qa_relative_path_valid,
|
|
20
|
+
)
|
|
21
|
+
from .qa_procedures import content_digest, existing_qa_file, safe_qa_path
|
|
22
|
+
from .verification_input_ref import VerificationInputReference
|
|
23
|
+
from .verification_conformance import capture_generation_conformance
|
|
24
|
+
from .verification_inputs import publish_verification_snapshot, read_verification_inputs
|
|
25
|
+
from .verification_target import read_verification_head
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class FrozenVerificationSource(TypedDict):
|
|
29
|
+
trackedPaths: list[str]
|
|
30
|
+
generatedRoots: list[str]
|
|
31
|
+
inputPaths: list[str]
|
|
32
|
+
protectedFiles: dict[str, str]
|
|
33
|
+
generatedFiles: dict[str, str]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _OptionalGenerationConformance(TypedDict, total=False):
|
|
37
|
+
conformanceStages: list[int]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class VerificationGenerationRecord(_OptionalGenerationConformance):
|
|
41
|
+
schemaVersion: Literal["1.0"]
|
|
42
|
+
kind: Literal["verification-generation"]
|
|
43
|
+
generation: int
|
|
44
|
+
taskType: str
|
|
45
|
+
runManifestPath: str
|
|
46
|
+
verificationInputsPath: str
|
|
47
|
+
verificationInputsDigest: str
|
|
48
|
+
head: str
|
|
49
|
+
sourceRoot: str
|
|
50
|
+
source: FrozenVerificationSource
|
|
51
|
+
artifactInputs: dict[str, str]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
_MAX_GENERATION_PATH_LENGTH = 4096
|
|
55
|
+
_DIGEST = {"type": "string", "pattern": f"^{QA_SOURCE_DIGEST_PATTERN.pattern}$"}
|
|
56
|
+
_PATH = {"type": "string", "minLength": 1, "maxLength": _MAX_GENERATION_PATH_LENGTH}
|
|
57
|
+
_PATHS = {"type": "array", "uniqueItems": True, "items": _PATH}
|
|
58
|
+
_FILES = {"type": "object", "additionalProperties": _DIGEST}
|
|
59
|
+
_SOURCE = {
|
|
60
|
+
"type": "object", "additionalProperties": False,
|
|
61
|
+
"required": ["trackedPaths", "generatedRoots", "inputPaths",
|
|
62
|
+
"protectedFiles", "generatedFiles"],
|
|
63
|
+
"properties": {
|
|
64
|
+
"trackedPaths": _PATHS, "generatedRoots": _PATHS, "inputPaths": _PATHS,
|
|
65
|
+
"protectedFiles": _FILES, "generatedFiles": _FILES,
|
|
66
|
+
},
|
|
67
|
+
}
|
|
68
|
+
_SCHEMA = {
|
|
69
|
+
"type": "object", "additionalProperties": False,
|
|
70
|
+
"required": list(VerificationGenerationRecord.__required_keys__),
|
|
71
|
+
"properties": {
|
|
72
|
+
"schemaVersion": {"const": "1.0"},
|
|
73
|
+
"kind": {"const": "verification-generation"},
|
|
74
|
+
"generation": {"type": "integer", "minimum": 1},
|
|
75
|
+
"taskType": {"enum": list(QA_COMMAND_EXECUTING_TASK_TYPES)},
|
|
76
|
+
"runManifestPath": _PATH, "verificationInputsPath": _PATH,
|
|
77
|
+
"verificationInputsDigest": _DIGEST,
|
|
78
|
+
"head": {"type": "string", "pattern": "^[0-9a-f]{40}([0-9a-f]{24})?$"},
|
|
79
|
+
"sourceRoot": _PATH, "source": _SOURCE, "artifactInputs": _FILES,
|
|
80
|
+
"conformanceStages": {"type": "array",
|
|
81
|
+
"items": {"type": "integer", "minimum": 1}},
|
|
82
|
+
},
|
|
83
|
+
}
|
|
84
|
+
_ARTIFACT = "verification generation"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _generation_errors(payload: dict[str, Any]) -> list[str]:
|
|
88
|
+
stages = payload.get("conformanceStages", [])
|
|
89
|
+
if any(left >= right for left, right in pairwise(stages)):
|
|
90
|
+
return ["generation conformance stages must be strictly increasing"]
|
|
91
|
+
if not Path(payload["sourceRoot"]).is_absolute():
|
|
92
|
+
return ["generation source root must be absolute"]
|
|
93
|
+
source = payload["source"]
|
|
94
|
+
declarations = [*source["generatedRoots"], *source["inputPaths"],
|
|
95
|
+
*payload["artifactInputs"], payload["runManifestPath"],
|
|
96
|
+
payload["verificationInputsPath"]]
|
|
97
|
+
if any(not qa_relative_path_valid(path) for path in declarations):
|
|
98
|
+
return ["generation declarations require safe relative POSIX paths"]
|
|
99
|
+
for field in ("runManifestPath", "verificationInputsPath"):
|
|
100
|
+
if PurePosixPath(payload[field]).parts[0] != OKSTRA_DIR_NAME:
|
|
101
|
+
return ["generation references must remain in project artifacts"]
|
|
102
|
+
protected, generated = source["protectedFiles"], source["generatedFiles"]
|
|
103
|
+
paths = {*source["trackedPaths"], *protected, *generated, *declarations}
|
|
104
|
+
for relative in paths:
|
|
105
|
+
if len(relative) > _MAX_GENERATION_PATH_LENGTH:
|
|
106
|
+
return ["generation source path is too long"]
|
|
107
|
+
path = PurePosixPath(relative)
|
|
108
|
+
if (not path.parts or path.is_absolute() or ".." in path.parts
|
|
109
|
+
or path.as_posix() != relative):
|
|
110
|
+
return ["generation source paths must be canonical and relative"]
|
|
111
|
+
if not set(source["inputPaths"]).issubset(protected):
|
|
112
|
+
return ["generation declared inputs must remain protected"]
|
|
113
|
+
roots = frozenset(source["generatedRoots"])
|
|
114
|
+
required = frozenset(source["trackedPaths"]) | frozenset(source["inputPaths"])
|
|
115
|
+
if any(is_generated_verification_path(path, roots, required) for path in protected):
|
|
116
|
+
return ["generation protected path conflicts with classification"]
|
|
117
|
+
if any(not is_generated_verification_path(path, roots, required)
|
|
118
|
+
for path in generated):
|
|
119
|
+
return ["generation output path conflicts with classification"]
|
|
120
|
+
return []
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _capture_generation_source(
|
|
124
|
+
project: Path, source_root: Path, *, source_inputs: frozenset[str],
|
|
125
|
+
generated_roots: frozenset[str], artifact_inputs: frozenset[str],
|
|
126
|
+
) -> tuple[VerificationSourceSnapshot, dict[str, str]]:
|
|
127
|
+
files = {existing_qa_file(project, path) for path in artifact_inputs}
|
|
128
|
+
if source_root == project:
|
|
129
|
+
source_inputs |= frozenset(
|
|
130
|
+
path.relative_to(project).as_posix() for path in files
|
|
131
|
+
)
|
|
132
|
+
snapshot = capture_verification_source(
|
|
133
|
+
source_root, generated_roots=generated_roots, input_paths=source_inputs,
|
|
134
|
+
)
|
|
135
|
+
artifacts = {}
|
|
136
|
+
for path in files:
|
|
137
|
+
relative = path.relative_to(project).as_posix()
|
|
138
|
+
artifacts[relative] = (
|
|
139
|
+
snapshot.protected_files[relative] if source_root == project
|
|
140
|
+
else content_digest(path.read_bytes())
|
|
141
|
+
)
|
|
142
|
+
return snapshot, artifacts
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _capture_generation(
|
|
146
|
+
project: Path, ref: RunRef, reference: VerificationInputReference, *,
|
|
147
|
+
generation: int, expected_head: str, source_inputs: frozenset[str],
|
|
148
|
+
generated_roots: frozenset[str], artifact_inputs: frozenset[str],
|
|
149
|
+
changed_artifact_paths: frozenset[str], conformance_baseline: Mapping[int, str],
|
|
150
|
+
) -> VerificationGenerationRecord:
|
|
151
|
+
inputs = read_verification_inputs(
|
|
152
|
+
safe_qa_path(project, reference.relative_path), reference.digest,
|
|
153
|
+
)
|
|
154
|
+
source_root = Path(inputs["sourceRoot"])
|
|
155
|
+
head = read_verification_head(source_root, expected_head)
|
|
156
|
+
conformance = None
|
|
157
|
+
if "approvedPlan" in inputs:
|
|
158
|
+
conformance = capture_generation_conformance(
|
|
159
|
+
project, ref, inputs["approvedPlan"], changed_artifact_paths,
|
|
160
|
+
conformance_baseline,
|
|
161
|
+
)
|
|
162
|
+
artifact_inputs |= conformance.artifact_paths
|
|
163
|
+
snapshot, artifacts = _capture_generation_source(
|
|
164
|
+
project, source_root, generated_roots=generated_roots,
|
|
165
|
+
source_inputs=source_inputs,
|
|
166
|
+
artifact_inputs=artifact_inputs,
|
|
167
|
+
)
|
|
168
|
+
if conformance is not None and conformance.manifest is not None:
|
|
169
|
+
relative, digest = conformance.manifest
|
|
170
|
+
if artifacts[relative] != digest:
|
|
171
|
+
raise ValueError("verification conformance manifest changed during capture")
|
|
172
|
+
read_verification_head(source_root, head)
|
|
173
|
+
return {
|
|
174
|
+
"schemaVersion": "1.0", "kind": "verification-generation",
|
|
175
|
+
"generation": generation, "taskType": ref.task_type,
|
|
176
|
+
"runManifestPath": ref.manifest.relative_to(project).as_posix(),
|
|
177
|
+
**reference.to_artifact_fields(), "head": head,
|
|
178
|
+
"sourceRoot": str(snapshot.root),
|
|
179
|
+
"source": {
|
|
180
|
+
"trackedPaths": sorted(snapshot.tracked_paths),
|
|
181
|
+
"generatedRoots": sorted(snapshot.generated_roots),
|
|
182
|
+
"inputPaths": sorted(snapshot.input_paths),
|
|
183
|
+
"protectedFiles": dict(snapshot.protected_files),
|
|
184
|
+
"generatedFiles": dict(snapshot.generated_files),
|
|
185
|
+
},
|
|
186
|
+
"artifactInputs": artifacts,
|
|
187
|
+
**({"conformanceStages": list(conformance.stages)}
|
|
188
|
+
if conformance is not None else {}),
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def prepare_verification_generation(
|
|
193
|
+
project: Path, ref: RunRef, reference: VerificationInputReference, *,
|
|
194
|
+
generation: int, expected_head: str, source_inputs: frozenset[str],
|
|
195
|
+
generated_roots: frozenset[str], artifact_inputs: frozenset[str],
|
|
196
|
+
changed_artifact_paths: frozenset[str], conformance_baseline: Mapping[int, str],
|
|
197
|
+
) -> tuple[Path, str]:
|
|
198
|
+
if ref.task_type not in QA_COMMAND_EXECUTING_TASK_TYPES:
|
|
199
|
+
raise ValueError("verification generations require a QA-executing phase")
|
|
200
|
+
reference_root = ref.project_root or project
|
|
201
|
+
project = project.resolve()
|
|
202
|
+
if reference_root.resolve() != project:
|
|
203
|
+
raise ValueError("verification generation project root mismatch")
|
|
204
|
+
relative_task = ref.task_root.absolute().relative_to(reference_root.absolute())
|
|
205
|
+
ref = replace(
|
|
206
|
+
ref, task_root_override=safe_qa_path(project, relative_task.as_posix()),
|
|
207
|
+
)
|
|
208
|
+
input_path = ref.verification_inputs.relative_to(project).as_posix()
|
|
209
|
+
if reference.relative_path != input_path:
|
|
210
|
+
raise ValueError("generation input reference belongs to a different run")
|
|
211
|
+
return publish_verification_snapshot(
|
|
212
|
+
project, ref.verification_generation(generation),
|
|
213
|
+
capture=partial(
|
|
214
|
+
_capture_generation, project, ref, reference, generation=generation,
|
|
215
|
+
expected_head=expected_head, source_inputs=source_inputs,
|
|
216
|
+
generated_roots=generated_roots, artifact_inputs=artifact_inputs,
|
|
217
|
+
changed_artifact_paths=changed_artifact_paths,
|
|
218
|
+
conformance_baseline=conformance_baseline,
|
|
219
|
+
),
|
|
220
|
+
artifact=_ARTIFACT, schema=_SCHEMA,
|
|
221
|
+
validate_cross_fields=_generation_errors, comparison_key=_generation_identity,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _generation_identity(payload: Mapping[str, Any]) -> dict[str, Any]:
|
|
226
|
+
return {**payload, "source": {**payload["source"], "generatedFiles": {}}}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def read_verification_generation(
|
|
230
|
+
path: Path, expected_digest: str,
|
|
231
|
+
) -> VerificationGenerationRecord:
|
|
232
|
+
stored = load_owned_object_snapshot(
|
|
233
|
+
path, artifact=_ARTIFACT, schema=_SCHEMA,
|
|
234
|
+
validate_cross_fields=_generation_errors,
|
|
235
|
+
)
|
|
236
|
+
if content_digest(stored.raw_bytes) != expected_digest:
|
|
237
|
+
raise ValueError("verification generation digest mismatch")
|
|
238
|
+
return cast(VerificationGenerationRecord, stored.value)
|
|
@@ -12,6 +12,24 @@ class FrozenProcedure(TypedDict):
|
|
|
12
12
|
digest: str
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
QaSourceStatus = Literal[
|
|
16
|
+
"not-declared", "not-queried", "no-baseline", "unchanged", "changed",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _OptionalQaSourceObservation(TypedDict, total=False):
|
|
21
|
+
observedSourceDigest: str
|
|
22
|
+
error: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class QaSourceObservation(_OptionalQaSourceObservation):
|
|
26
|
+
sourceStatus: QaSourceStatus
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class FrozenQaSourceObservation(QaSourceObservation):
|
|
30
|
+
entryDigest: str
|
|
31
|
+
|
|
32
|
+
|
|
15
33
|
class FrozenVerificationPlan(TypedDict):
|
|
16
34
|
path: str
|
|
17
35
|
digest: str
|
|
@@ -20,11 +38,12 @@ class FrozenVerificationPlan(TypedDict):
|
|
|
20
38
|
conformance: list[dict[str, Any]]
|
|
21
39
|
|
|
22
40
|
|
|
23
|
-
class
|
|
41
|
+
class _OptionalVerificationInputs(TypedDict, total=False):
|
|
24
42
|
approvedPlan: FrozenVerificationPlan
|
|
43
|
+
qaSourceObservations: dict[str, list[FrozenQaSourceObservation]]
|
|
25
44
|
|
|
26
45
|
|
|
27
|
-
class VerificationInputs(
|
|
46
|
+
class VerificationInputs(_OptionalVerificationInputs):
|
|
28
47
|
schemaVersion: Literal["1.0", "2.0"]
|
|
29
48
|
kind: Literal["verification-inputs"]
|
|
30
49
|
sourceRoot: str
|