devlyn-cli 3.1.1 → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -6
- package/CLAUDE.md +7 -7
- package/README.md +20 -5
- package/bin/devlyn.js +10 -52
- package/bin/instruction-templates.json +52 -0
- package/bin/instructions.js +137 -0
- package/config/skills/_shared/adapters/claude.md +1 -5
- package/config/skills/_shared/adapters/codex.md +1 -5
- package/config/skills/_shared/adapters/omp.md +0 -4
- package/config/skills/_shared/expected.schema.json +2 -2
- package/config/skills/_shared/platform-support.py +4 -1
- package/config/skills/_shared/process-evidence.py +16 -1
- package/config/skills/_shared/resolve-bootstrap.py +51 -6
- package/config/skills/_shared/spec-verify-check.py +525 -89
- package/config/skills/_shared/state-phase-write.py +202 -32
- package/config/skills/_shared/task-complete.py +260 -16
- package/config/skills/devlyn:ideate/SKILL.md +1 -1
- package/config/skills/devlyn:ideate/references/from-spec-mode.md +1 -1
- package/config/skills/devlyn:ideate/references/spec-template.md +3 -3
- package/config/skills/devlyn:resolve/SKILL.md +4 -4
- package/config/skills/devlyn:resolve/references/free-form-mode.md +2 -2
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +4 -3
- package/config/skills/devlyn:resolve/references/phases/verify.md +9 -7
- package/config/skills/devlyn:resolve/references/state-schema.md +2 -2
- package/config/skills/devlyn:resolve/references/task-completion.md +27 -0
- package/package.json +1 -1
- package/scripts/lint-skills.sh +2 -2
|
@@ -308,6 +308,36 @@ def bind_process_evidence(
|
|
|
308
308
|
raise runner.EvidenceError(
|
|
309
309
|
"spec-verify.results.json commands must be an array"
|
|
310
310
|
)
|
|
311
|
+
preflight_failure = "preflight_failure" in results
|
|
312
|
+
if preflight_failure:
|
|
313
|
+
failure = results["preflight_failure"]
|
|
314
|
+
if (
|
|
315
|
+
not isinstance(failure, dict)
|
|
316
|
+
or set(failure) != {"run_id", "phase", "round", "findings"}
|
|
317
|
+
or failure["run_id"] != state.get("run_id")
|
|
318
|
+
or failure["phase"] != phase
|
|
319
|
+
or type(failure["round"]) is not int
|
|
320
|
+
or failure["round"] != runner.phase_round(state, phase)
|
|
321
|
+
):
|
|
322
|
+
raise runner.EvidenceError("BUILD_GATE preflight failure identity is invalid")
|
|
323
|
+
record = failure["findings"]
|
|
324
|
+
if not isinstance(record, dict) or set(record) != {"path", "sha256"}:
|
|
325
|
+
raise runner.EvidenceError("BUILD_GATE preflight failure findings binding is invalid")
|
|
326
|
+
path = runner._checked_file(work, record["path"], "preflight failure findings")
|
|
327
|
+
raw = path.read_bytes()
|
|
328
|
+
if hashlib.sha256(raw).hexdigest() != record["sha256"]:
|
|
329
|
+
raise runner.EvidenceError("BUILD_GATE preflight failure findings digest mismatch")
|
|
330
|
+
findings = [loads_strict_json(line) for line in raw.decode("utf-8").splitlines() if line.strip()]
|
|
331
|
+
if len(findings) != 1 or not isinstance(findings[0], dict) or (
|
|
332
|
+
findings[0].get("phase") != phase
|
|
333
|
+
or findings[0].get("severity") != "CRITICAL"
|
|
334
|
+
or findings[0].get("rule_id") not in {
|
|
335
|
+
"correctness.spec-verify-malformed", "correctness.risk-probe-integrity",
|
|
336
|
+
}
|
|
337
|
+
):
|
|
338
|
+
raise runner.EvidenceError("BUILD_GATE preflight failure requires its CRITICAL finding")
|
|
339
|
+
if verdict not in {"FAIL", "BLOCKED"}:
|
|
340
|
+
raise runner.EvidenceError("BUILD_GATE preflight failure requires FAIL or BLOCKED verdict")
|
|
311
341
|
carrier = results.get("process_evidence")
|
|
312
342
|
if carrier is None:
|
|
313
343
|
if commands:
|
|
@@ -315,43 +345,42 @@ def bind_process_evidence(
|
|
|
315
345
|
"BUILD_GATE commands exist without a process-evidence carrier"
|
|
316
346
|
)
|
|
317
347
|
manifest_path = work / runner.manifest_relative_path(state, phase)
|
|
318
|
-
if
|
|
348
|
+
if os.path.lexists(manifest_path):
|
|
319
349
|
raise runner.EvidenceError(
|
|
320
350
|
"BUILD_GATE manifest exists without a process-evidence carrier"
|
|
321
351
|
)
|
|
322
|
-
if runner.mechanical_evidence_required(work, state):
|
|
352
|
+
if not preflight_failure and runner.mechanical_evidence_required(work, state):
|
|
323
353
|
raise runner.EvidenceError(
|
|
324
354
|
"required BUILD_GATE evidence has no process-evidence carrier"
|
|
325
355
|
)
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
)
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
)
|
|
356
|
+
else:
|
|
357
|
+
round_ = runner.phase_round(state, phase)
|
|
358
|
+
manifest = carrier.get("manifest") if isinstance(carrier, dict) else None
|
|
359
|
+
if (
|
|
360
|
+
not isinstance(carrier, dict)
|
|
361
|
+
or carrier.get("phase") != phase
|
|
362
|
+
or carrier.get("round") != round_
|
|
363
|
+
or not isinstance(manifest, dict)
|
|
364
|
+
or manifest.get("path") != runner.manifest_relative_path(state, phase)
|
|
365
|
+
):
|
|
366
|
+
raise runner.EvidenceError(
|
|
367
|
+
"BUILD_GATE process-evidence carrier does not match the active run/round"
|
|
368
|
+
)
|
|
369
|
+
outcome = runner.validate_summary_commands(work, commands, carrier)
|
|
370
|
+
if outcome["verdict"] == "BLOCKED" and verdict != "BLOCKED":
|
|
371
|
+
denial = outcome["capability_denials"][0]
|
|
372
|
+
raise runner.EvidenceError(
|
|
373
|
+
"BUILD_GATE capability denial requires BLOCKED verdict: "
|
|
374
|
+
f"{denial['id']}:{denial['operation']}"
|
|
375
|
+
)
|
|
376
|
+
if (
|
|
377
|
+
outcome["verdict"] == "NEEDS_WORK"
|
|
378
|
+
and verdict in {"PASS", "PASS_WITH_ISSUES"}
|
|
379
|
+
):
|
|
380
|
+
raise runner.EvidenceError(
|
|
381
|
+
"BUILD_GATE process evidence mismatch cannot complete as "
|
|
382
|
+
f"{verdict}: {','.join(outcome['failed_ids'])}"
|
|
383
|
+
)
|
|
355
384
|
except (runner.EvidenceError, OSError, UnicodeError, ValueError) as exc:
|
|
356
385
|
raise SystemExit(f"BLOCKED:process-evidence-invalid: {exc}") from exc
|
|
357
386
|
existing = state.get("process_evidence")
|
|
@@ -362,8 +391,11 @@ def bind_process_evidence(
|
|
|
362
391
|
try:
|
|
363
392
|
for prior in existing:
|
|
364
393
|
runner.validate_bound_carrier(work, prior)
|
|
365
|
-
except runner.EvidenceError as exc:
|
|
394
|
+
except (runner.EvidenceError, OSError, UnicodeError, ValueError) as exc:
|
|
366
395
|
raise SystemExit(f"BLOCKED:process-evidence-invalid: {exc}") from exc
|
|
396
|
+
if carrier is None:
|
|
397
|
+
state.setdefault("process_evidence", None)
|
|
398
|
+
return
|
|
367
399
|
if any(
|
|
368
400
|
isinstance(item, dict)
|
|
369
401
|
and item.get("phase") == carrier["phase"]
|
|
@@ -2996,6 +3028,144 @@ def self_test() -> int:
|
|
|
2996
3028
|
|
|
2997
3029
|
test_interrupted_build_gate()
|
|
2998
3030
|
|
|
3031
|
+
def test_preflight_build_gate() -> None:
|
|
3032
|
+
work = (devlyn / "preflight-build-gate").resolve()
|
|
3033
|
+
active = work / ".devlyn"
|
|
3034
|
+
active.mkdir(parents=True)
|
|
3035
|
+
state_file = active / "pipeline.state.json"
|
|
3036
|
+
results_file = active / "spec-verify.results.json"
|
|
3037
|
+
checker = pathlib.Path(__file__).with_name("spec-verify-check.py")
|
|
3038
|
+
env = {key: value for key, value in os.environ.items() if key not in {
|
|
3039
|
+
"BENCH_WORKDIR", "SPEC_VERIFY_PHASE", "SPEC_VERIFY_FINDINGS_FILE",
|
|
3040
|
+
"SPEC_VERIFY_FINDING_PREFIX",
|
|
3041
|
+
} and not key.startswith("DEVLYN_INVOCATION_")}
|
|
3042
|
+
runner = process_evidence_module()
|
|
3043
|
+
|
|
3044
|
+
def complete(verdict: str) -> subprocess.CompletedProcess[str]:
|
|
3045
|
+
return subprocess.run(
|
|
3046
|
+
[sys.executable, str(pathlib.Path(__file__).resolve()),
|
|
3047
|
+
"--devlyn-dir", ".devlyn", "--phase", "build_gate", "complete",
|
|
3048
|
+
"--verdict", verdict, "--findings-file", ".devlyn/spec-verify-findings.jsonl"],
|
|
3049
|
+
cwd=work, env=env, capture_output=True, text=True, encoding="utf-8",
|
|
3050
|
+
)
|
|
3051
|
+
|
|
3052
|
+
history = []
|
|
3053
|
+
for round_, scenario in enumerate(("fresh", "language", "stale", "denied")):
|
|
3054
|
+
fixture = {
|
|
3055
|
+
"version": "3.0", "run_id": "rs-preflight-build-gate",
|
|
3056
|
+
"source": {"type": "generated", "criteria_path": ".devlyn/criteria.generated.md"},
|
|
3057
|
+
"phases": {"build_gate": None}, "process_evidence": copy.deepcopy(history) or None,
|
|
3058
|
+
}
|
|
3059
|
+
do_spawn(fixture, "build_gate", round_, None, None, "claude", None)
|
|
3060
|
+
write_state(state_file, fixture)
|
|
3061
|
+
original_state = state_file.read_bytes()
|
|
3062
|
+
results_file.unlink(missing_ok=True)
|
|
3063
|
+
relative = runner.manifest_relative_path(fixture, "build_gate")
|
|
3064
|
+
manifest = work / relative
|
|
3065
|
+
obligation = {"id": "type-check", "phase": "build_gate",
|
|
3066
|
+
"argv": [sys.executable, "-c", "raise SystemExit(7)"]}
|
|
3067
|
+
if scenario == "language":
|
|
3068
|
+
runner.capture_process(work, manifest, fixture["run_id"], "build_gate", round_, obligation)
|
|
3069
|
+
elif scenario == "denied":
|
|
3070
|
+
runner.record_capability_denial(
|
|
3071
|
+
work, manifest, fixture["run_id"], "build_gate", round_, obligation,
|
|
3072
|
+
"subprocess", b"route denied subprocess",
|
|
3073
|
+
)
|
|
3074
|
+
elif scenario == "stale":
|
|
3075
|
+
results_file.write_bytes(prior_results)
|
|
3076
|
+
observed = {path: path.read_bytes() for path in manifest.parent.iterdir()} if manifest.exists() else {}
|
|
3077
|
+
rejected = subprocess.run(
|
|
3078
|
+
[sys.executable, str(checker), "--include-risk-probes"],
|
|
3079
|
+
cwd=work, env=env, capture_output=True, text=True, encoding="utf-8",
|
|
3080
|
+
)
|
|
3081
|
+
assert rejected.returncode == 1 and "source.criteria_path" in rejected.stderr, rejected.stderr
|
|
3082
|
+
results = loads_strict_json(results_file.read_text(encoding="utf-8"))
|
|
3083
|
+
assert results["preflight_failure"]["round"] == round_
|
|
3084
|
+
prior_results = results_file.read_bytes()
|
|
3085
|
+
finding = active / "spec-verify-findings.jsonl"
|
|
3086
|
+
finding_bytes = finding.read_bytes()
|
|
3087
|
+
for verdict in ("PASS", "PASS_WITH_ISSUES", "NEEDS_WORK"):
|
|
3088
|
+
denied = complete(verdict)
|
|
3089
|
+
assert denied.returncode != 0 and "preflight failure requires" in denied.stderr, denied.stderr
|
|
3090
|
+
assert state_file.read_bytes() == original_state
|
|
3091
|
+
if scenario == "denied":
|
|
3092
|
+
denied = complete("FAIL")
|
|
3093
|
+
assert denied.returncode != 0 and "capability denial requires BLOCKED" in denied.stderr
|
|
3094
|
+
assert state_file.read_bytes() == original_state
|
|
3095
|
+
completed = complete("BLOCKED" if scenario == "denied" else "FAIL")
|
|
3096
|
+
assert completed.returncode == 0, completed.stderr
|
|
3097
|
+
bound = read_state(state_file)
|
|
3098
|
+
assert bound["phases"]["build_gate"]["verdict"] == ("BLOCKED" if scenario == "denied" else "FAIL")
|
|
3099
|
+
assert finding.read_bytes() == finding_bytes and results_file.read_bytes() == prior_results
|
|
3100
|
+
assert {path: path.read_bytes() for path in observed} == observed
|
|
3101
|
+
if observed:
|
|
3102
|
+
assert bound["process_evidence"] == [*history, results["process_evidence"]]
|
|
3103
|
+
else:
|
|
3104
|
+
assert bound["process_evidence"] == (history or None) and results["commands"] == []
|
|
3105
|
+
history = bound["process_evidence"] or []
|
|
3106
|
+
for mutation in ("run", "phase", "round", "bool-round", "digest", "escape", "finding", "null"):
|
|
3107
|
+
state_file.write_bytes(original_state)
|
|
3108
|
+
altered = copy.deepcopy(results)
|
|
3109
|
+
failure = altered["preflight_failure"]
|
|
3110
|
+
if mutation in {"run", "phase", "round", "bool-round"}:
|
|
3111
|
+
field, value = {"run": ("run_id", "rs-other"), "phase": ("phase", "verify"),
|
|
3112
|
+
"round": ("round", round_ + 1), "bool-round": ("round", True)}[mutation]
|
|
3113
|
+
failure[field] = value
|
|
3114
|
+
elif mutation == "null":
|
|
3115
|
+
altered["preflight_failure"] = None
|
|
3116
|
+
elif mutation == "finding":
|
|
3117
|
+
finding.write_bytes(b"{}\n")
|
|
3118
|
+
failure["findings"]["sha256"] = hashlib.sha256(finding.read_bytes()).hexdigest()
|
|
3119
|
+
else:
|
|
3120
|
+
failure["findings"]["path" if mutation == "escape" else "sha256"] = (
|
|
3121
|
+
"../outside.jsonl" if mutation == "escape" else "0" * 64
|
|
3122
|
+
)
|
|
3123
|
+
results_file.write_text(json.dumps(altered), encoding="utf-8")
|
|
3124
|
+
result = complete("FAIL")
|
|
3125
|
+
assert result.returncode != 0 and "BLOCKED:process-evidence-invalid" in result.stderr, (mutation, result.stderr)
|
|
3126
|
+
assert state_file.read_bytes() == original_state
|
|
3127
|
+
finding.write_bytes(finding_bytes)
|
|
3128
|
+
results_file.write_bytes(prior_results)
|
|
3129
|
+
if scenario in {"fresh", "stale"}:
|
|
3130
|
+
state_file.write_bytes(original_state)
|
|
3131
|
+
malformed_prior = read_state(state_file)
|
|
3132
|
+
malformed_prior["process_evidence"] = {}
|
|
3133
|
+
write_state(state_file, malformed_prior)
|
|
3134
|
+
before = state_file.read_bytes()
|
|
3135
|
+
invalid = complete("FAIL")
|
|
3136
|
+
assert invalid.returncode != 0 and "null or an array" in invalid.stderr, invalid.stderr
|
|
3137
|
+
assert state_file.read_bytes() == before
|
|
3138
|
+
state_file.write_bytes(original_state)
|
|
3139
|
+
if scenario == "stale":
|
|
3140
|
+
prior_stream = work / history[0]["streams"][0]["stdout"]["path"]
|
|
3141
|
+
saved = prior_stream.read_bytes()
|
|
3142
|
+
prior_stream.write_bytes(b"altered prior observation")
|
|
3143
|
+
invalid = complete("FAIL")
|
|
3144
|
+
assert invalid.returncode != 0 and "mismatch" in invalid.stderr, invalid.stderr
|
|
3145
|
+
assert state_file.read_bytes() == original_state
|
|
3146
|
+
prior_stream.write_bytes(saved)
|
|
3147
|
+
from unittest.mock import patch
|
|
3148
|
+
read_bytes = pathlib.Path.read_bytes
|
|
3149
|
+
|
|
3150
|
+
def unreadable_prior(path):
|
|
3151
|
+
if path == prior_stream:
|
|
3152
|
+
raise PermissionError("prior stream unreadable")
|
|
3153
|
+
return read_bytes(path)
|
|
3154
|
+
|
|
3155
|
+
fixture = read_state(state_file)
|
|
3156
|
+
unchanged = copy.deepcopy(fixture)
|
|
3157
|
+
with patch.object(pathlib.Path, "read_bytes", unreadable_prior):
|
|
3158
|
+
try:
|
|
3159
|
+
bind_process_evidence(fixture, "build_gate", "FAIL", active, work)
|
|
3160
|
+
except SystemExit as exc:
|
|
3161
|
+
assert str(exc) == "BLOCKED:process-evidence-invalid: prior stream unreadable", exc
|
|
3162
|
+
else:
|
|
3163
|
+
raise AssertionError("unreadable prior evidence completed")
|
|
3164
|
+
assert fixture == unchanged and state_file.read_bytes() == original_state
|
|
3165
|
+
print("PASS preflight rejection: actual checker-to-completion, success floors, identity/digest controls")
|
|
3166
|
+
|
|
3167
|
+
test_preflight_build_gate()
|
|
3168
|
+
|
|
2999
3169
|
# complete() before spawn() must fail loudly, not silently invent data.
|
|
3000
3170
|
write_state(state_path, {"phases": {}})
|
|
3001
3171
|
state = read_state(state_path)
|
|
@@ -204,6 +204,9 @@ def allocate(args):
|
|
|
204
204
|
path = directory / "receipt.json"
|
|
205
205
|
receipt["id"] = key
|
|
206
206
|
receipt["recovery_ref"] = "refs/devlyn/completed/" + key
|
|
207
|
+
scratch = directory / "scratch"
|
|
208
|
+
scratch.mkdir()
|
|
209
|
+
receipt["scratch_identity"] = workspace_identity(scratch, directory)
|
|
207
210
|
# Persist prospective intent BEFORE native creation. An interrupted allocation
|
|
208
211
|
# stays visibly incomplete; a later call may not adopt whatever now exists.
|
|
209
212
|
atomic_json(path, receipt)
|
|
@@ -215,7 +218,7 @@ def allocate(args):
|
|
|
215
218
|
receipt["worktree_identity"] = workspace_identity(target, Path(receipt["worktree_gitdir"]))
|
|
216
219
|
receipt["allocation"] = "owned"
|
|
217
220
|
atomic_json(path, receipt)
|
|
218
|
-
return {"status": "ALLOCATED", "receipt": str(path), "worktree": str(target)}
|
|
221
|
+
return {"status": "ALLOCATED", "receipt": str(path), "worktree": str(target), "scratch": str(scratch)}
|
|
219
222
|
|
|
220
223
|
|
|
221
224
|
def pipeline_acceptance(work, acceptance, files, directory):
|
|
@@ -269,7 +272,7 @@ def pipeline_acceptance(work, acceptance, files, directory):
|
|
|
269
272
|
require(evidence_module["bound_carrier_outcome"](reconstructed, carrier)["verdict"] == "PASS", "required process evidence failed")
|
|
270
273
|
source = state.get("source") or {}
|
|
271
274
|
if source.get("spec_path"):
|
|
272
|
-
expected = Path(source["spec_path"]).
|
|
275
|
+
expected = Path(source["spec_path"]).with_name("spec.expected.json")
|
|
273
276
|
original = safe_path(work, str(expected))
|
|
274
277
|
if original.exists():
|
|
275
278
|
dest = safe_path(reconstructed, str(expected))
|
|
@@ -342,7 +345,8 @@ def bind_acceptance(receipt, path, supplied):
|
|
|
342
345
|
require(committed.returncode == 0 and committed.stdout == source_file.read_bytes(), "source contract changed since accepted commit")
|
|
343
346
|
bound_digest = source.get(key.replace("_path", "_sha256"))
|
|
344
347
|
require(file_record(source_file)["sha256"] == bound_digest, "source contract differs from run binding")
|
|
345
|
-
|
|
348
|
+
source_path = Path(source[key])
|
|
349
|
+
expected = str(source_path.with_name("spec.expected.json") if key == "spec_path" else source_path.with_suffix(".expected.json"))
|
|
346
350
|
if safe_path(work, expected).exists():
|
|
347
351
|
paths.append(expected)
|
|
348
352
|
else:
|
|
@@ -437,6 +441,75 @@ def stopped_writers(work, *, linked):
|
|
|
437
441
|
raise CompletionError("writer observation unsupported on this platform; retain workspace")
|
|
438
442
|
|
|
439
443
|
|
|
444
|
+
def scratch_mounts():
|
|
445
|
+
if sys.platform.startswith("linux"):
|
|
446
|
+
rows = Path("/proc/self/mountinfo").read_text(encoding="utf-8").splitlines()
|
|
447
|
+
paths = []
|
|
448
|
+
for line in rows:
|
|
449
|
+
fields = line.split()
|
|
450
|
+
require(len(fields) >= 6, "cannot parse mount table; retain scratch")
|
|
451
|
+
paths.append(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4]))
|
|
452
|
+
elif sys.platform == "darwin":
|
|
453
|
+
paths = []
|
|
454
|
+
for line in command(["mount"]).splitlines():
|
|
455
|
+
description, separator, options = line.rpartition(" (")
|
|
456
|
+
candidates = [description[match.end():] for match in re.finditer(" on ", description)]
|
|
457
|
+
require(separator and options.endswith(")") and candidates, "cannot parse mount table; retain scratch")
|
|
458
|
+
# Darwin prints literal paths. Consider every possible separator so
|
|
459
|
+
# ' on ' in a device or directory name can only cause retention.
|
|
460
|
+
paths.extend(candidates)
|
|
461
|
+
else:
|
|
462
|
+
raise CompletionError("mount observation unsupported; retain scratch")
|
|
463
|
+
return [Path(value).resolve() for value in paths]
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def raise_walk_error(error):
|
|
467
|
+
raise error
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def clean_scratch(receipt, path, writers_stopped):
|
|
471
|
+
"""Empty the owned build directory; keep its identity across resume/reuse."""
|
|
472
|
+
scratch = path.parent / "scratch"
|
|
473
|
+
if "scratch_identity" not in receipt:
|
|
474
|
+
return {"status": "NOT_OWNED"}
|
|
475
|
+
require(writers_stopped, "stop/yield scratch writers, then use --writers-stopped")
|
|
476
|
+
require(scratch == scratch.resolve() and scratch.is_dir(), "owned scratch missing or redirected; retain and inspect")
|
|
477
|
+
identity = receipt["scratch_identity"]
|
|
478
|
+
require(workspace_identity(scratch, path.parent) == identity, "scratch directory was replaced; retain")
|
|
479
|
+
require(shutil.rmtree.avoids_symlink_attacks, "safe scratch removal unsupported on this platform; retain")
|
|
480
|
+
require(not any(mount.is_relative_to(scratch) for mount in scratch_mounts()), "scratch contains a mounted filesystem; retain")
|
|
481
|
+
stopped_writers(scratch, linked=True)
|
|
482
|
+
size = 0
|
|
483
|
+
for directory, dirs, files in os.walk(scratch, followlinks=False, onerror=raise_walk_error):
|
|
484
|
+
require(not any(name.casefold() == ".git" for name in dirs + files), "scratch contains Git recovery data; move it to retained custody before cleanup")
|
|
485
|
+
for name in dirs + files:
|
|
486
|
+
item = Path(directory) / name
|
|
487
|
+
info = item.lstat()
|
|
488
|
+
require(info.st_dev == identity[0][0], "scratch contains a mounted filesystem; retain")
|
|
489
|
+
size += info.st_size if stat.S_ISREG(info.st_mode) else 0
|
|
490
|
+
stopped_writers(scratch, linked=True)
|
|
491
|
+
require(workspace_identity(scratch, path.parent) == identity, "scratch changed before removal; retain")
|
|
492
|
+
require(not any(mount.is_relative_to(scratch) for mount in scratch_mounts()), "scratch mount appeared before removal; retain")
|
|
493
|
+
for item in scratch.iterdir():
|
|
494
|
+
if item.is_dir() and not item.is_symlink():
|
|
495
|
+
shutil.rmtree(item)
|
|
496
|
+
else:
|
|
497
|
+
item.unlink()
|
|
498
|
+
require(workspace_identity(scratch, path.parent) == identity and not any(scratch.iterdir()), "scratch changed during cleanup; retain and inspect")
|
|
499
|
+
result = {"status": "CLEAN", "logical_bytes_removed": size}
|
|
500
|
+
receipt["scratch_cleanup"] = result
|
|
501
|
+
atomic_json(path, receipt)
|
|
502
|
+
return result
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def clean_scratch_command(args):
|
|
506
|
+
path = Path(args.receipt).absolute()
|
|
507
|
+
with locked_receipt(path) as receipt:
|
|
508
|
+
result = clean_scratch(receipt, path, args.writers_stopped)
|
|
509
|
+
return {"status": "SCRATCH_CLEAN" if result["status"] == "CLEAN" else "SCRATCH_NOT_OWNED",
|
|
510
|
+
"receipt": str(path), "scratch_cleanup": result, "product_verdict_unchanged": True}
|
|
511
|
+
|
|
512
|
+
|
|
440
513
|
def cleanup(receipt, path, pr, writers_stopped):
|
|
441
514
|
require(pr["state"] == "MERGED" and pr.get("mergedAt") and pr.get("mergeCommit", {}).get("oid"), "actual matching merge evidence is required")
|
|
442
515
|
require(writers_stopped, "wait actual children, stop/yield known writers, then resume with --writers-stopped")
|
|
@@ -522,11 +595,27 @@ def locked_receipt(path):
|
|
|
522
595
|
|
|
523
596
|
def complete(args):
|
|
524
597
|
path = Path(args.receipt).absolute()
|
|
525
|
-
resume = shlex.join([sys.executable, str(Path(__file__).resolve()), "complete", "--receipt", str(path)]
|
|
598
|
+
resume = shlex.join([sys.executable, str(Path(__file__).resolve()), "complete", "--receipt", str(path)] +
|
|
599
|
+
(["--writers-stopped"] if args.writers_stopped else []))
|
|
526
600
|
with locked_receipt(path) as receipt:
|
|
527
601
|
def result(status):
|
|
528
|
-
|
|
529
|
-
|
|
602
|
+
scratch = {"status": "NOT_OWNED"}
|
|
603
|
+
if "scratch_identity" in receipt:
|
|
604
|
+
try:
|
|
605
|
+
scratch = clean_scratch(receipt, path, args.writers_stopped)
|
|
606
|
+
except (CompletionError, OSError, ValueError, KeyError, TypeError, IndexError) as error:
|
|
607
|
+
scratch = {"status": "RETAINED", "reason": str(error), "resume": shlex.join([
|
|
608
|
+
sys.executable, str(Path(__file__).resolve()), "clean-scratch", "--receipt", str(path), "--writers-stopped"])}
|
|
609
|
+
receipt["scratch_cleanup"] = scratch
|
|
610
|
+
try:
|
|
611
|
+
atomic_json(path, receipt)
|
|
612
|
+
except OSError as record_error:
|
|
613
|
+
scratch["record_error"] = str(record_error)
|
|
614
|
+
return {"status": "CLEANUP_PENDING" if status == "COMPLETE" and scratch["status"] == "RETAINED" else status,
|
|
615
|
+
"delivery_status": status, "receipt": str(path), "pr": receipt.get("pr_url"), "resume": resume,
|
|
616
|
+
"acceptance": (receipt.get("acceptance") or {}).get("kind"), "product_verdict_unchanged": True,
|
|
617
|
+
"scratch_path": str(path.parent / "scratch") if "scratch_identity" in receipt else None,
|
|
618
|
+
"scratch_cleanup": scratch}
|
|
530
619
|
if args.local_only or receipt.get("local_only"):
|
|
531
620
|
receipt["local_only"] = True
|
|
532
621
|
atomic_json(path, receipt)
|
|
@@ -610,20 +699,24 @@ def main():
|
|
|
610
699
|
completion.add_argument("--mode", choices=("auto", "pr"))
|
|
611
700
|
completion.add_argument("--local-only", "--no-push", action="store_true")
|
|
612
701
|
completion.add_argument("--writers-stopped", action="store_true")
|
|
702
|
+
scratch_cleanup = actions.add_parser("clean-scratch")
|
|
703
|
+
scratch_cleanup.add_argument("--receipt", required=True)
|
|
704
|
+
scratch_cleanup.add_argument("--writers-stopped", action="store_true")
|
|
613
705
|
args = parser.parse_args()
|
|
614
706
|
if args.self_test:
|
|
615
707
|
return self_test()
|
|
616
708
|
if args.action is None:
|
|
617
|
-
parser.error("allocate or
|
|
709
|
+
parser.error("allocate, complete or clean-scratch is required")
|
|
618
710
|
try:
|
|
619
|
-
result = allocate(args) if args.action == "allocate" else
|
|
711
|
+
result = (allocate(args) if args.action == "allocate" else
|
|
712
|
+
clean_scratch_command(args) if args.action == "clean-scratch" else complete(args))
|
|
620
713
|
print(json.dumps(result, sort_keys=True))
|
|
621
714
|
return 0
|
|
622
|
-
except (CompletionError, OSError, ValueError, KeyError, TypeError, SystemExit) as exc:
|
|
715
|
+
except (CompletionError, OSError, ValueError, KeyError, TypeError, IndexError, SystemExit) as exc:
|
|
623
716
|
result = {"status": "BLOCKED", "reason": str(exc)}
|
|
624
|
-
if args.action
|
|
717
|
+
if args.action in {"complete", "clean-scratch"}:
|
|
625
718
|
result["receipt"] = str(Path(args.receipt).absolute())
|
|
626
|
-
result["resume"] = shlex.join([sys.executable, str(Path(__file__).resolve()),
|
|
719
|
+
result["resume"] = shlex.join([sys.executable, str(Path(__file__).resolve()), args.action, "--receipt", result["receipt"], *(["--writers-stopped"] if args.writers_stopped else [])])
|
|
627
720
|
print(json.dumps(result, sort_keys=True))
|
|
628
721
|
return 1
|
|
629
722
|
|
|
@@ -686,6 +779,14 @@ else:
|
|
|
686
779
|
raise SystemExit('unexpected gh arguments: '+str(a))
|
|
687
780
|
'''
|
|
688
781
|
|
|
782
|
+
GIT_DISPATCH = r'''#!/bin/sh
|
|
783
|
+
for arg do
|
|
784
|
+
case "$arg" in push|fetch|ls-remote|remove) exec python3 "$0.py" "$@";; esac
|
|
785
|
+
done
|
|
786
|
+
exec "$REAL_GIT" "$@"
|
|
787
|
+
'''
|
|
788
|
+
|
|
789
|
+
|
|
689
790
|
GIT_WRAPPER = r'''#!/usr/bin/env python3
|
|
690
791
|
import json, os, pathlib, subprocess, sys
|
|
691
792
|
a = sys.argv[1:]
|
|
@@ -729,7 +830,7 @@ class CompletionTests(unittest.TestCase):
|
|
|
729
830
|
for key in list(self.env):
|
|
730
831
|
if key in {"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"}:
|
|
731
832
|
del self.env[key]
|
|
732
|
-
for name, body in (("gh", FAKE_GH), ("git", GIT_WRAPPER)):
|
|
833
|
+
for name, body in (("gh", FAKE_GH), ("git.py", GIT_WRAPPER), ("git", GIT_DISPATCH)):
|
|
733
834
|
path = self.bin / name
|
|
734
835
|
path.write_text(body, encoding="utf-8")
|
|
735
836
|
path.chmod(0o755)
|
|
@@ -770,11 +871,138 @@ class CompletionTests(unittest.TestCase):
|
|
|
770
871
|
self.task = Path(result["worktree"])
|
|
771
872
|
return result
|
|
772
873
|
|
|
773
|
-
|
|
874
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
875
|
+
def test_disposable_scratch_cleans_for_local_only_and_preserves_source(self):
|
|
876
|
+
result = self.allocate()
|
|
877
|
+
scratch = Path(result["scratch"])
|
|
878
|
+
(scratch / "target").mkdir()
|
|
879
|
+
(scratch / "target" / "object").write_bytes(b"rebuildable")
|
|
880
|
+
(self.task / "uncommitted-work").write_text("preserve")
|
|
881
|
+
result, _ = self.cli("complete", "--receipt", self.receipt, "--local-only", "--writers-stopped")
|
|
882
|
+
self.assertEqual(result["status"], "LOCAL_ONLY")
|
|
883
|
+
self.assertFalse(any(scratch.iterdir()))
|
|
884
|
+
self.assertEqual((self.task / "uncommitted-work").read_text(), "preserve")
|
|
885
|
+
self.assertEqual(result["scratch_cleanup"]["logical_bytes_removed"], 11)
|
|
886
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped")
|
|
887
|
+
self.assertEqual(result["status"], "SCRATCH_CLEAN")
|
|
888
|
+
|
|
889
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
890
|
+
def test_scratch_rejects_active_writer_and_requires_owner_assertion(self):
|
|
891
|
+
scratch = Path(self.allocate()["scratch"])
|
|
892
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, success=False)
|
|
893
|
+
self.assertIn("--writers-stopped", result["reason"])
|
|
894
|
+
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], cwd=scratch)
|
|
895
|
+
try:
|
|
896
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped", success=False)
|
|
897
|
+
self.assertIn("active process", result["reason"])
|
|
898
|
+
self.assertTrue(scratch.is_dir())
|
|
899
|
+
finally:
|
|
900
|
+
child.terminate()
|
|
901
|
+
child.wait(timeout=5)
|
|
902
|
+
self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped")
|
|
903
|
+
|
|
904
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
905
|
+
def test_scratch_rejects_redirect_and_git_data_but_does_not_follow_child_links(self):
|
|
906
|
+
scratch = Path(self.allocate()["scratch"])
|
|
907
|
+
outside = self.root / "outside"
|
|
908
|
+
outside.mkdir()
|
|
909
|
+
(outside / "keep").write_text("source")
|
|
910
|
+
retained = scratch.with_name("retained")
|
|
911
|
+
scratch.rename(retained)
|
|
912
|
+
scratch.symlink_to(outside, target_is_directory=True)
|
|
913
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped", success=False)
|
|
914
|
+
self.assertIn("redirected", result["reason"])
|
|
915
|
+
scratch.unlink()
|
|
916
|
+
retained.rename(scratch)
|
|
917
|
+
(scratch / ".GiT").write_text("gitdir: elsewhere")
|
|
918
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped", success=False)
|
|
919
|
+
self.assertIn("Git recovery data", result["reason"])
|
|
920
|
+
(scratch / ".GiT").unlink()
|
|
921
|
+
(scratch / "external-link").symlink_to(outside, target_is_directory=True)
|
|
922
|
+
self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped")
|
|
923
|
+
self.assertEqual((outside / "keep").read_text(), "source")
|
|
924
|
+
|
|
925
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
926
|
+
def test_scratch_reuse_and_partial_cleanup_keep_original_identity(self):
|
|
927
|
+
scratch = Path(self.allocate()["scratch"])
|
|
928
|
+
(scratch / "first-build").write_bytes(b"first")
|
|
929
|
+
self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped")
|
|
930
|
+
(scratch / "resumed-build").write_bytes(b"next")
|
|
931
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped")
|
|
932
|
+
self.assertEqual(result["scratch_cleanup"]["logical_bytes_removed"], 4)
|
|
933
|
+
self.assertFalse(any(scratch.iterdir()))
|
|
934
|
+
scratch.rename(scratch.with_name("original"))
|
|
935
|
+
scratch.mkdir()
|
|
936
|
+
(scratch / "foreign").write_text("preserve")
|
|
937
|
+
result, _ = self.cli("clean-scratch", "--receipt", self.receipt, "--writers-stopped", success=False)
|
|
938
|
+
self.assertIn("replaced", result["reason"])
|
|
939
|
+
self.assertIn("--writers-stopped", result["resume"])
|
|
940
|
+
self.assertEqual((scratch / "foreign").read_text(), "preserve")
|
|
941
|
+
|
|
942
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
943
|
+
def test_scratch_refusal_does_not_block_local_only_delivery(self):
|
|
944
|
+
scratch = Path(self.allocate()["scratch"])
|
|
945
|
+
(scratch / ".git").mkdir()
|
|
946
|
+
result, _ = self.cli("complete", "--receipt", self.receipt, "--local-only", "--writers-stopped")
|
|
947
|
+
self.assertEqual(result["status"], "LOCAL_ONLY")
|
|
948
|
+
self.assertEqual(result["scratch_cleanup"]["status"], "RETAINED")
|
|
949
|
+
self.assertIn("Git recovery data", result["scratch_cleanup"]["reason"])
|
|
950
|
+
self.assertEqual(json.loads(self.receipt.read_text())["scratch_cleanup"]["status"], "RETAINED")
|
|
951
|
+
self.assertTrue((scratch / ".git").is_dir())
|
|
952
|
+
|
|
953
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
954
|
+
def test_scratch_refusal_reports_delivery_separately(self):
|
|
955
|
+
scratch = Path(self.allocate()["scratch"])
|
|
956
|
+
self.accept()
|
|
957
|
+
(scratch / ".git").mkdir()
|
|
958
|
+
result, _ = self.cli("complete", "--receipt", self.receipt, "--acceptance", self.acceptance, "--writers-stopped")
|
|
959
|
+
self.assertEqual(result["status"], "CLEANUP_PENDING")
|
|
960
|
+
self.assertEqual(result["delivery_status"], "COMPLETE")
|
|
961
|
+
self.assertEqual(result["scratch_cleanup"]["status"], "RETAINED")
|
|
962
|
+
self.assertIn("--writers-stopped", result["scratch_cleanup"]["resume"])
|
|
963
|
+
self.assertIn("--writers-stopped", result["resume"])
|
|
964
|
+
(scratch / ".git").rmdir()
|
|
965
|
+
result, _ = self.cli("complete", "--receipt", self.receipt, "--writers-stopped")
|
|
966
|
+
self.assertEqual(result["status"], "COMPLETE")
|
|
967
|
+
|
|
968
|
+
@unittest.skipUnless(sys.platform == "darwin" or sys.platform.startswith("linux"), "writer observation requires POSIX")
|
|
969
|
+
def test_scratch_mount_and_scan_failure_preserve_contents(self):
|
|
970
|
+
from unittest.mock import patch
|
|
971
|
+
scratch = Path(self.allocate()["scratch"])
|
|
972
|
+
(scratch / "keep").write_text("retained")
|
|
973
|
+
receipt = json.loads(self.receipt.read_text())
|
|
974
|
+
with patch.dict(clean_scratch.__globals__, {"scratch_mounts": lambda: [scratch / "same-device-bind"]}):
|
|
975
|
+
with self.assertRaisesRegex(CompletionError, "mounted filesystem"):
|
|
976
|
+
clean_scratch(receipt, self.receipt, True)
|
|
977
|
+
def failed_walk(*args, **kwargs):
|
|
978
|
+
kwargs["onerror"](PermissionError("unreadable subtree"))
|
|
979
|
+
with patch.object(os, "walk", failed_walk):
|
|
980
|
+
with self.assertRaisesRegex(PermissionError, "unreadable subtree"):
|
|
981
|
+
clean_scratch(receipt, self.receipt, True)
|
|
982
|
+
self.assertEqual((scratch / "keep").read_text(), "retained")
|
|
983
|
+
|
|
984
|
+
def test_scratch_mount_table_parsing_preserves_ambiguous_literal_paths(self):
|
|
985
|
+
from unittest.mock import patch
|
|
986
|
+
target = "/task/scratch/name on inside\\040literal"
|
|
987
|
+
with patch.object(sys, "platform", "darwin"), patch.dict(scratch_mounts.__globals__, {
|
|
988
|
+
"command": lambda args: "device on source on " + target + " (local, journaled)",
|
|
989
|
+
}):
|
|
990
|
+
self.assertIn(Path(target).resolve(), scratch_mounts())
|
|
991
|
+
alias = self.root / "scratch-alias"
|
|
992
|
+
alias.symlink_to(self.root, target_is_directory=True)
|
|
993
|
+
with patch.object(sys, "platform", "darwin"), patch.dict(scratch_mounts.__globals__, {
|
|
994
|
+
"command": lambda args: "device on " + str(alias / "mounted") + " (local)",
|
|
995
|
+
}):
|
|
996
|
+
self.assertEqual(scratch_mounts(), [(self.root / "mounted").resolve()])
|
|
997
|
+
with patch.object(sys, "platform", "linux"), patch.object(Path, "read_text", return_value="truncated row"):
|
|
998
|
+
with self.assertRaisesRegex(CompletionError, "cannot parse mount table"):
|
|
999
|
+
scratch_mounts()
|
|
1000
|
+
|
|
1001
|
+
def accept(self, pipeline=False, queue=False, spec_expected=None, spec_name="spec.md"):
|
|
774
1002
|
if spec_expected is not None:
|
|
775
|
-
(self.task /
|
|
1003
|
+
(self.task / spec_name).write_text("# Fixture\nProduct contains accepted bytes.\n", encoding="utf-8")
|
|
776
1004
|
(self.task / "spec.expected.json").write_text(json.dumps(spec_expected), encoding="utf-8")
|
|
777
|
-
self.g("add",
|
|
1005
|
+
self.g("add", spec_name, "spec.expected.json", work=self.task)
|
|
778
1006
|
(self.task / "product").write_text("accepted\n", encoding="utf-8")
|
|
779
1007
|
self.g("add", "product", work=self.task)
|
|
780
1008
|
self.g("commit", "-m", "scoped task", work=self.task)
|
|
@@ -802,7 +1030,7 @@ class CompletionTests(unittest.TestCase):
|
|
|
802
1030
|
self.state = {"run_id": a["run_id"], "mode": "free-form", "source": source, "phases": phases, "process_evidence": None}
|
|
803
1031
|
if spec_expected is not None:
|
|
804
1032
|
self.state["mode"] = "spec"
|
|
805
|
-
self.state["source"] = {"type": "spec", "spec_path":
|
|
1033
|
+
self.state["source"] = {"type": "spec", "spec_path": spec_name, "spec_sha256": hashlib.sha256((self.task / spec_name).read_bytes()).hexdigest()}
|
|
806
1034
|
(archive / "pipeline.state.json").write_text(json.dumps(self.state), encoding="utf-8")
|
|
807
1035
|
(archive / "finish-gate.summary.json").write_text(json.dumps({"mode": self.state["mode"], "exit": 0, "offenders": 0, "checked": 1}), encoding="utf-8")
|
|
808
1036
|
(archive / "verify-merge.summary.json").write_text(json.dumps({"verdict": "PASS"}), encoding="utf-8")
|
|
@@ -1167,6 +1395,22 @@ class CompletionTests(unittest.TestCase):
|
|
|
1167
1395
|
self.assertIn("implement process evidence", json.loads(r.stdout)["reason"])
|
|
1168
1396
|
self.assertEqual(json.loads(self.data.read_text(encoding="utf-8")).get("pushs",0), 0)
|
|
1169
1397
|
|
|
1398
|
+
def test_named_spec_custody(self):
|
|
1399
|
+
self.allocate(); self.accept(pipeline=True, spec_expected={"pure_design": True}, spec_name="X.md")
|
|
1400
|
+
result, _ = self.complete("--mode", "pr")
|
|
1401
|
+
self.assertEqual(result["status"], "PR")
|
|
1402
|
+
receipt = json.loads(self.receipt.read_text(encoding="utf-8"))
|
|
1403
|
+
self.assertIn("spec.expected.json", receipt["files"])
|
|
1404
|
+
self.assertEqual((self.receipt.parent / "custody/spec.expected.json").read_bytes(), (self.task / "spec.expected.json").read_bytes())
|
|
1405
|
+
|
|
1406
|
+
def test_named_spec_missing_required_process_evidence(self):
|
|
1407
|
+
expected = {"process_evidence": [{"id": "red-first", "phase": "implement", "argv": [sys.executable, "-c", "print('red')"], "exit_code": 0, "timeout_sec": 10}]}
|
|
1408
|
+
self.allocate(); self.accept(pipeline=True, spec_expected=expected, spec_name="X.md")
|
|
1409
|
+
_, r = self.complete(success=False)
|
|
1410
|
+
self.assertNotEqual(r.returncode, 0)
|
|
1411
|
+
self.assertIn("implement process evidence", json.loads(r.stdout)["reason"])
|
|
1412
|
+
self.assertEqual(json.loads(self.data.read_text(encoding="utf-8")).get("pushs",0), 0)
|
|
1413
|
+
|
|
1170
1414
|
|
|
1171
1415
|
if __name__ == "__main__":
|
|
1172
1416
|
runpy.run_path(str(Path(__file__).with_name("platform-support.py")))["configure_utf8"]()
|