devlyn-cli 4.3.24 → 4.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -27
- package/bin/devlyn.js +42 -94
- package/bin/engines.js +14 -12
- package/bin/harness-activation.js +4 -2
- package/bin/harness-context.js +395 -0
- package/bin/harness-run.js +148 -0
- package/bin/instructions.js +71 -159
- package/bin/skill-history.json +7 -0
- package/config/skills/_shared/storage-checkout.py +1 -0
- package/config/skills/_shared/storage-client.py +349 -14
- package/config/skills/_shared/task-complete.py +99 -24
- package/config/skills/_shared/task-completion.md +7 -3
- package/config/skills/devlyn-ideate/SKILL.md +2 -2
- package/config/skills/devlyn-ideate/references/loop.md +2 -2
- package/config/skills/devlyn-ideate/scripts/queue.py +10 -3
- package/native/claude/.claude-plugin/plugin.json +1 -1
- package/native/claude/skills/_shared/storage-checkout.py +1 -0
- package/native/claude/skills/_shared/storage-client.py +349 -14
- package/native/claude/skills/_shared/task-complete.py +99 -24
- package/native/claude/skills/_shared/task-completion.md +7 -3
- package/native/claude/skills/devlyn-ideate/SKILL.md +2 -2
- package/native/claude/skills/devlyn-ideate/references/loop.md +2 -2
- package/native/claude/skills/devlyn-ideate/scripts/queue.py +10 -3
- package/native/codex/.agents/plugins/marketplace.json +1 -1
- package/native/codex/plugins/devlyn/.codex-plugin/plugin.json +1 -1
- package/native/codex/plugins/devlyn/skills/_shared/storage-checkout.py +1 -0
- package/native/codex/plugins/devlyn/skills/_shared/storage-client.py +349 -14
- package/native/codex/plugins/devlyn/skills/_shared/task-complete.py +99 -24
- package/native/codex/plugins/devlyn/skills/_shared/task-completion.md +7 -3
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/SKILL.md +2 -2
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/loop.md +2 -2
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/queue.py +10 -3
- package/package.json +2 -2
- package/release.json +43 -33
|
@@ -26,11 +26,15 @@ import math
|
|
|
26
26
|
import os
|
|
27
27
|
from pathlib import Path
|
|
28
28
|
import re
|
|
29
|
+
import runpy
|
|
30
|
+
import shlex
|
|
29
31
|
import selectors
|
|
30
32
|
import socket
|
|
31
33
|
import stat
|
|
32
34
|
import struct
|
|
35
|
+
import subprocess
|
|
33
36
|
import sys
|
|
37
|
+
import tempfile
|
|
34
38
|
import threading
|
|
35
39
|
import time
|
|
36
40
|
import uuid
|
|
@@ -857,7 +861,7 @@ def regular_digest(path):
|
|
|
857
861
|
return digest.hexdigest()
|
|
858
862
|
|
|
859
863
|
|
|
860
|
-
def package_digest(root):
|
|
864
|
+
def package_digest(root, *, excluded=()):
|
|
861
865
|
digest = hashlib.sha256()
|
|
862
866
|
root = Path(root)
|
|
863
867
|
def visit(directory):
|
|
@@ -866,6 +870,8 @@ def package_digest(root):
|
|
|
866
870
|
for path in sorted(directory.iterdir(), key=lambda p: p.name.encode("utf-8")):
|
|
867
871
|
info = _evidence(path)
|
|
868
872
|
require(info is not None, "retained package entry disappeared")
|
|
873
|
+
if stat.S_ISREG(info.st_mode) and path.relative_to(root).as_posix() in excluded:
|
|
874
|
+
continue
|
|
869
875
|
name = str(path.relative_to(root)).encode()
|
|
870
876
|
digest.update(struct.pack("<Q", len(name)))
|
|
871
877
|
digest.update(name)
|
|
@@ -887,10 +893,11 @@ ENROLLMENT_SHAPE = {"schema": "u32", "control_directory": "str", "control_identi
|
|
|
887
893
|
"retained_helper": ARTIFACT_SHAPE, "retained_recovery": ARTIFACT_SHAPE}
|
|
888
894
|
|
|
889
895
|
|
|
890
|
-
def verify_artifact(artifact, *, package=False):
|
|
896
|
+
def verify_artifact(artifact, *, package=False, identity=True):
|
|
891
897
|
path = Path(artifact['path'])
|
|
892
898
|
require(path.is_absolute() and str(path.resolve(strict=True)) == artifact['path'], 'noncanonical retained path')
|
|
893
|
-
|
|
899
|
+
if identity:
|
|
900
|
+
require(persistent_identity(artifact['path']) == artifact['identity'], 'retained identity changed')
|
|
894
901
|
digest = package_digest(artifact['path']) if package else regular_digest(artifact['path'])
|
|
895
902
|
require(digest == artifact['sha256'], 'retained digest changed')
|
|
896
903
|
|
|
@@ -902,13 +909,25 @@ def verify_registration(record):
|
|
|
902
909
|
raise StorageRefusal('storage_unavailable', f'registration inspection failed: {exc}', phase='enrollment') from exc
|
|
903
910
|
|
|
904
911
|
|
|
905
|
-
def
|
|
912
|
+
def validate_registration_shape(record):
|
|
913
|
+
"""Rust evidence::validate_shape without reading any copied identity."""
|
|
906
914
|
validate(record, ENROLLMENT_SHAPE, exact=True)
|
|
907
915
|
helper, recovery = record["helper_release"], record["recovery_release"]
|
|
908
916
|
require(record["schema"] == 1 and re.fullmatch(
|
|
909
917
|
r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?", helper["version"])
|
|
910
|
-
and 0 < len(helper["target"]) <= 128 and recovery["package"] == "devlyn-cli"
|
|
918
|
+
and 0 < len(helper["target"].encode()) <= 128 and recovery["package"] == "devlyn-cli"
|
|
911
919
|
and recovery["version"] and recovery["digest"], "unsupported or incomplete enrollment", "storage_unsupported")
|
|
920
|
+
require(record["retained_helper"]["sha256"] == helper["sha256"]
|
|
921
|
+
and record["retained_recovery"]["path"] == recovery["root"], "retained release binding changed")
|
|
922
|
+
for value in (record['control_directory'], record['helper_path'], recovery['root'],
|
|
923
|
+
record['retained_helper']['path'], record['retained_recovery']['path']):
|
|
924
|
+
path = Path(value)
|
|
925
|
+
require(path.is_absolute() and str(path) == value and '..' not in path.parts, 'noncanonical enrollment path')
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _verify_registration(record):
|
|
929
|
+
validate_registration_shape(record)
|
|
930
|
+
helper = record['helper_release']
|
|
912
931
|
for value in (record['control_directory'], record['helper_path']):
|
|
913
932
|
path = Path(value)
|
|
914
933
|
require(path.is_absolute() and str(path.resolve(strict=True)) == value, "noncanonical enrollment path")
|
|
@@ -916,24 +935,259 @@ def _verify_registration(record):
|
|
|
916
935
|
require(regular_digest(record["helper_path"]) == helper["sha256"], "helper digest changed")
|
|
917
936
|
verify_artifact(record['retained_helper'])
|
|
918
937
|
verify_artifact(record['retained_recovery'], package=True)
|
|
919
|
-
require(record["retained_helper"]["sha256"] == helper["sha256"]
|
|
920
|
-
and record["retained_recovery"]["path"] == recovery["root"], "retained release binding changed")
|
|
921
938
|
return record
|
|
922
939
|
|
|
923
940
|
|
|
924
|
-
def read_evidence_json(path, observed):
|
|
941
|
+
def read_evidence_json(path, observed, *, limit=MAX_REQUEST):
|
|
925
942
|
info = _evidence(path)
|
|
926
943
|
require(info is not None and stat.S_ISREG(info.st_mode), f'missing/invalid {path}')
|
|
927
944
|
fd = os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_NONBLOCK', 0)
|
|
928
945
|
| getattr(os, 'O_BINARY', 0))
|
|
929
946
|
with os.fdopen(fd, 'rb') as stream:
|
|
930
947
|
require(_same_file(info, os.fstat(stream.fileno())), f'{path} changed while opening')
|
|
931
|
-
raw = stream.read(
|
|
932
|
-
require(len(raw) <=
|
|
948
|
+
raw = stream.read() if limit is None else stream.read(limit + 1)
|
|
949
|
+
require(limit is None or len(raw) <= limit, f'oversized {path}')
|
|
933
950
|
observed.append((path, info))
|
|
934
951
|
return loads(raw)
|
|
935
952
|
|
|
936
953
|
|
|
954
|
+
def lifecycle_evidence(root, path, observed):
|
|
955
|
+
"""Optional no-follow evidence, including every ancestor under the root."""
|
|
956
|
+
current = root
|
|
957
|
+
for part in path.relative_to(root).parts:
|
|
958
|
+
current = current / part
|
|
959
|
+
info = _evidence(current)
|
|
960
|
+
observed.append((current, info))
|
|
961
|
+
if info is None:
|
|
962
|
+
return None
|
|
963
|
+
require(stat.S_ISREG(info.st_mode) if current == path else stat.S_ISDIR(info.st_mode),
|
|
964
|
+
f'invalid lifecycle evidence kind: {current}')
|
|
965
|
+
return read_evidence_json(path, observed)
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def retirement_evidence(root, observed):
|
|
969
|
+
shape = {'schema': 'u32', 'operation_id': 'str', 'root_identity': 'identity', 'lock_identity': 'identity'}
|
|
970
|
+
fence = lifecycle_evidence(root, root / 'storage-client-fence.json', observed)
|
|
971
|
+
journal = lifecycle_evidence(root, root / 'storage-client-unenroll.json', observed)
|
|
972
|
+
if fence is not None:
|
|
973
|
+
validate(fence, shape, exact=True)
|
|
974
|
+
require(fence['schema'] == 1, 'unsupported retirement fence')
|
|
975
|
+
canonical_operation(fence['operation_id'])
|
|
976
|
+
if journal is not None:
|
|
977
|
+
validate(journal, {**shape, 'retired': 'bool', 'archives': [
|
|
978
|
+
{'source': 'str', 'destination': 'str', 'identity': 'identity', 'sha256': 'str'}]}, exact=True)
|
|
979
|
+
require(fence is not None and all(journal[key] == fence[key] for key in shape),
|
|
980
|
+
'retirement journal/fence differs')
|
|
981
|
+
sources = ('storage', 'control', 'harness/state.json', 'storage-client.json')
|
|
982
|
+
names = ('storage', 'control', 'harness-state.json', 'enrollment.json')
|
|
983
|
+
previous = -1
|
|
984
|
+
for archive in journal['archives']:
|
|
985
|
+
require(archive['source'] in [str(root / name) for name in sources], 'invalid retirement source')
|
|
986
|
+
index = sources.index(str(Path(archive['source']).relative_to(root)))
|
|
987
|
+
require(index > previous and Path(archive['destination']) == root / 'storage-client-retired'
|
|
988
|
+
/ fence['operation_id'] / names[index], 'invalid retirement archive path/order')
|
|
989
|
+
previous = index
|
|
990
|
+
if not journal['retired']:
|
|
991
|
+
for name in sources:
|
|
992
|
+
if str(root / name) not in [entry['source'] for entry in journal['archives']]:
|
|
993
|
+
require(_evidence(root / name) is None, 'uninventoried retirement evidence')
|
|
994
|
+
return fence, journal
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _retirement_move(root, pins, observed):
|
|
998
|
+
"""Establish a copied retirement before allowing digest-only bootstrap.
|
|
999
|
+
|
|
1000
|
+
Compare all identities in the evidence this client reads, never merely a
|
|
1001
|
+
failed retained-artifact identity. Same-volume replacements cannot qualify.
|
|
1002
|
+
"""
|
|
1003
|
+
fence, journal = retirement_evidence(root, observed)
|
|
1004
|
+
if fence is None:
|
|
1005
|
+
return False
|
|
1006
|
+
identities = []
|
|
1007
|
+
def record_identities(record):
|
|
1008
|
+
validate_registration_shape(record)
|
|
1009
|
+
identities.extend([record['control_identity'], record['retained_helper']['identity'],
|
|
1010
|
+
record['retained_recovery']['identity']])
|
|
1011
|
+
record_identities(pins['enrollment'])
|
|
1012
|
+
identities.append(pins['runtime']['identity'])
|
|
1013
|
+
record = lifecycle_evidence(root, root / 'storage-client.json', observed)
|
|
1014
|
+
if record is not None:
|
|
1015
|
+
record_identities(record)
|
|
1016
|
+
require(all(record[key] == pins['enrollment'][key] for key in ('control_directory', 'control_identity')),
|
|
1017
|
+
'move record/pins binding differs')
|
|
1018
|
+
identities.extend([fence['root_identity'], fence['lock_identity']])
|
|
1019
|
+
if journal is not None:
|
|
1020
|
+
identities.extend([journal['root_identity'], journal['lock_identity']])
|
|
1021
|
+
for archive in journal['archives']:
|
|
1022
|
+
identities.append(archive['identity'])
|
|
1023
|
+
paths = [Path(archive[key]) for key in ('source', 'destination')]
|
|
1024
|
+
existing = []
|
|
1025
|
+
for path in paths:
|
|
1026
|
+
# Check every ancestor without interpreting a directory as JSON.
|
|
1027
|
+
current = root
|
|
1028
|
+
for part in path.relative_to(root).parts:
|
|
1029
|
+
current /= part
|
|
1030
|
+
info = _evidence(current)
|
|
1031
|
+
observed.append((current, info))
|
|
1032
|
+
if info is None:
|
|
1033
|
+
break
|
|
1034
|
+
require(current == path or stat.S_ISDIR(info.st_mode), 'invalid archive ancestor')
|
|
1035
|
+
else:
|
|
1036
|
+
existing.append(path)
|
|
1037
|
+
require(len(existing) == 1, 'incomplete/ambiguous retirement archive')
|
|
1038
|
+
path = existing[0]
|
|
1039
|
+
digest = (package_digest(path, excluded=('service/serve.log', 'service/serve.log.1')
|
|
1040
|
+
if path.name == 'control' else ()) if path.is_dir() else regular_digest(path))
|
|
1041
|
+
require(digest == archive['sha256'], 'retirement archive digest changed')
|
|
1042
|
+
if archive['source'] == str(root / 'storage-client.json'):
|
|
1043
|
+
record_identities(read_evidence_json(path, observed))
|
|
1044
|
+
old = identities[0]['volume']
|
|
1045
|
+
require(old and all(identity['volume'] == old and identity['file'] for identity in identities),
|
|
1046
|
+
'mixed or malformed moved identities')
|
|
1047
|
+
current_volume = persistent_identity(root)['volume']
|
|
1048
|
+
if current_volume == old:
|
|
1049
|
+
return False
|
|
1050
|
+
for path, directory in ((root / 'storage-client.lock', False), (root / 'control', True)):
|
|
1051
|
+
info = _evidence(path)
|
|
1052
|
+
if info is not None:
|
|
1053
|
+
require((stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode))
|
|
1054
|
+
and persistent_identity(path)['volume'] == current_volume, 'mixed current move identities')
|
|
1055
|
+
return True
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def retirement_move(root, pins, observed):
|
|
1059
|
+
try:
|
|
1060
|
+
return _retirement_move(root, pins, observed)
|
|
1061
|
+
except StorageRefusal:
|
|
1062
|
+
# Rust Established::NotMoved: malformed/mixed evidence cannot authorize
|
|
1063
|
+
# bootstrap. The caller must still pass every ordinary identity check.
|
|
1064
|
+
return False
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def moved_operation(saved):
|
|
1068
|
+
"""A committed manifest or established retirement, never registry membership."""
|
|
1069
|
+
home = user_home()
|
|
1070
|
+
if home is None:
|
|
1071
|
+
return None
|
|
1072
|
+
root = (home / '.devlyn').resolve(strict=False)
|
|
1073
|
+
observed = []
|
|
1074
|
+
marker = root / 'storage-client-moved'
|
|
1075
|
+
info = _evidence(marker)
|
|
1076
|
+
journal_info = _evidence(root / 'storage-client-unenroll.json')
|
|
1077
|
+
if info is None and journal_info is None:
|
|
1078
|
+
return None
|
|
1079
|
+
require(type(saved) is dict, 'invalid saved authority binding')
|
|
1080
|
+
current = lifecycle_evidence(root, root / 'storage-client.json', observed)
|
|
1081
|
+
require(current is None or type(current) is dict, 'invalid current enrollment')
|
|
1082
|
+
if current is not None and all(key in current and key in saved and current[key] == saved[key]
|
|
1083
|
+
for key in ('control_directory', 'control_identity')):
|
|
1084
|
+
control = Path(current['control_directory'])
|
|
1085
|
+
if _evidence(control) is not None and persistent_identity(control) == current['control_identity']:
|
|
1086
|
+
return None
|
|
1087
|
+
if info is not None:
|
|
1088
|
+
require(stat.S_ISDIR(info.st_mode), 'redirected move archive')
|
|
1089
|
+
for directory in sorted(marker.iterdir()):
|
|
1090
|
+
# Like Rust, non-operation archive names are not manifests.
|
|
1091
|
+
try:
|
|
1092
|
+
canonical_operation(directory.name)
|
|
1093
|
+
except StorageRefusal:
|
|
1094
|
+
continue
|
|
1095
|
+
manifest = lifecycle_evidence(root, directory / 'storage-client-move.json', observed)
|
|
1096
|
+
if manifest is None:
|
|
1097
|
+
continue
|
|
1098
|
+
require(type(manifest) is dict, 'invalid move manifest')
|
|
1099
|
+
validate(manifest, {'schema': 'u32', 'operation_id': 'str', 'from_volume': 'str', 'to_volume': 'str',
|
|
1100
|
+
'old_binding': {'control_directory': 'str', 'control_identity': 'identity'} if manifest.get('old_binding') is not None else (None,)}, exact=True)
|
|
1101
|
+
require(manifest['schema'] == 1 and manifest['operation_id'] == directory.name
|
|
1102
|
+
and manifest['from_volume'] and manifest['to_volume']
|
|
1103
|
+
and manifest['from_volume'] != manifest['to_volume'], 'invalid move manifest')
|
|
1104
|
+
binding = manifest['old_binding']
|
|
1105
|
+
if binding is not None:
|
|
1106
|
+
path = Path(binding['control_directory'])
|
|
1107
|
+
require(path.is_absolute() and str(path) == binding['control_directory'] and '..' not in path.parts
|
|
1108
|
+
and binding['control_identity']['volume'] == manifest['from_volume']
|
|
1109
|
+
and binding['control_identity']['file'], 'invalid moved binding')
|
|
1110
|
+
if binding is not None and all(saved.get(key) == binding[key] for key in binding):
|
|
1111
|
+
return manifest['operation_id']
|
|
1112
|
+
if _evidence(root / 'storage-client-unenroll.json') is not None:
|
|
1113
|
+
pins = lifecycle_evidence(root, root / 'storage-client-recovery/pins.json', observed)
|
|
1114
|
+
if pins is None:
|
|
1115
|
+
return None # No retained binding can establish a moved receipt.
|
|
1116
|
+
validate(pins, {'schema': 'u32', 'enrollment': ENROLLMENT_SHAPE, 'runtime': ARTIFACT_SHAPE, 'node': 'str'}, exact=True)
|
|
1117
|
+
require(pins['schema'] == pins['enrollment']['schema'] == 1, 'unsupported recovery pins')
|
|
1118
|
+
if (all(saved.get(key) == pins['enrollment'][key] for key in ('control_directory', 'control_identity'))
|
|
1119
|
+
and retirement_move(root, pins, observed)):
|
|
1120
|
+
return read_evidence_json(root / 'storage-client-unenroll.json', observed)['operation_id']
|
|
1121
|
+
return None
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def unenroll_resume(record, operation, *, cancel=False):
|
|
1125
|
+
request = json.dumps({'adapter_protocol': 1, 'operation_id': operation}, separators=(',', ':'))
|
|
1126
|
+
return (shlex.join(['printf', '%s\\n', request]) + ' | '
|
|
1127
|
+
+ shlex.join([record['retained_helper']['path'], 'storage-client', 'unenroll',
|
|
1128
|
+
'--cancel' if cancel else '--reconcile', '--json']))
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
def replay_retirement(record):
|
|
1132
|
+
root = (user_home() / '.devlyn').resolve(strict=False)
|
|
1133
|
+
fence, journal = retirement_evidence(root, [])
|
|
1134
|
+
if journal is None or journal['retired']:
|
|
1135
|
+
return
|
|
1136
|
+
helper = record['retained_helper']['path']
|
|
1137
|
+
require(regular_digest(helper) == record['helper_release']['sha256'], 'retained helper changed')
|
|
1138
|
+
request = {'adapter_protocol': 1, 'operation_id': fence['operation_id']}
|
|
1139
|
+
adapter = runpy.run_path(str(Path(__file__).with_name('storage-checkout.py')))
|
|
1140
|
+
try:
|
|
1141
|
+
with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as diagnostic:
|
|
1142
|
+
process = subprocess.run([helper, 'storage-client', 'unenroll', '--reconcile', '--json'],
|
|
1143
|
+
input=json.dumps(request).encode()+b'\n', stdout=output, stderr=diagnostic)
|
|
1144
|
+
output.seek(0)
|
|
1145
|
+
reply = output.read(MAX_REPLY+1)
|
|
1146
|
+
result = adapter['decode_reply'](reply, 'unenroll', process.returncode)
|
|
1147
|
+
require(type(result) is dict and result.get('operation_id') == fence['operation_id']
|
|
1148
|
+
and result.get('retired') is True, 'retirement replay not complete')
|
|
1149
|
+
_, journal = retirement_evidence(root, [])
|
|
1150
|
+
require(journal is not None and journal['retired'], 'retirement replay did not commit')
|
|
1151
|
+
except (OSError, StorageRefusal, adapter['StorageRefusal']) as exc:
|
|
1152
|
+
resume = unenroll_resume(record, fence['operation_id'])
|
|
1153
|
+
detail = exc.diagnostic if isinstance(exc, StorageRefusal) else str(exc)
|
|
1154
|
+
raise StorageRefusal('storage_unavailable', detail, phase='retirement',
|
|
1155
|
+
message='Finish this storage retirement; resume ' + resume, resume=resume) from exc
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def retirement_refusal(root=None):
|
|
1159
|
+
"""An interrupted retirement has a headless exit through its retained helper."""
|
|
1160
|
+
if root is None:
|
|
1161
|
+
home = user_home()
|
|
1162
|
+
if home is None:
|
|
1163
|
+
return
|
|
1164
|
+
root = (home / '.devlyn').resolve(strict=False)
|
|
1165
|
+
if _evidence(root / 'storage-client-fence.json') is None and _evidence(root / 'storage-client-unenroll.json') is None:
|
|
1166
|
+
return
|
|
1167
|
+
try:
|
|
1168
|
+
# Completed retirement keeps operation_enrollment's existing validation,
|
|
1169
|
+
# including stale same-disk archive paths. Only an unfinished one exits here.
|
|
1170
|
+
journal = lifecycle_evidence(root, root / 'storage-client-unenroll.json', [])
|
|
1171
|
+
if type(journal) is dict and journal.get('retired') is True:
|
|
1172
|
+
return
|
|
1173
|
+
fence, journal = retirement_evidence(root, [])
|
|
1174
|
+
require(fence is not None, 'retirement fence missing')
|
|
1175
|
+
record = recovery_registration()
|
|
1176
|
+
except (StorageRefusal, OSError, ValueError, RuntimeError) as exc:
|
|
1177
|
+
detail = exc.diagnostic if isinstance(exc, StorageRefusal) else str(exc)
|
|
1178
|
+
repair = 'Open Devlyn to repair its storage registration'
|
|
1179
|
+
raise StorageRefusal('storage_unavailable', detail, phase='enrollment', message=repair, resume=repair) from exc
|
|
1180
|
+
resume = unenroll_resume(record, fence['operation_id'], cancel=journal is None)
|
|
1181
|
+
alternative = ''
|
|
1182
|
+
if journal is None:
|
|
1183
|
+
pins = read_evidence_json(root / 'storage-client-recovery/pins.json', [])
|
|
1184
|
+
if not retirement_move(root, pins, []):
|
|
1185
|
+
alternative = '; or cancel before retirement starts: ' + resume
|
|
1186
|
+
resume = unenroll_resume(record, fence['operation_id'])
|
|
1187
|
+
raise StorageRefusal('storage_unavailable', 'retirement is not complete', phase='enrollment',
|
|
1188
|
+
message='Finish this storage retirement; resume ' + resume + alternative, resume=resume)
|
|
1189
|
+
|
|
1190
|
+
|
|
937
1191
|
def recovery_registration():
|
|
938
1192
|
"""Current retained pins survive replacement of the app and unenrollment."""
|
|
939
1193
|
try:
|
|
@@ -953,11 +1207,13 @@ def recovery_registration():
|
|
|
953
1207
|
and record['recovery_release']['root'] == record['retained_recovery']['path']
|
|
954
1208
|
and node.is_absolute() and str(node.resolve(strict=True)) == pins['node']
|
|
955
1209
|
and node.is_relative_to(runtime), 'invalid recovery pin paths')
|
|
956
|
-
|
|
957
|
-
verify_artifact(record['
|
|
958
|
-
verify_artifact(
|
|
1210
|
+
moved = retirement_move(root, pins, observed)
|
|
1211
|
+
verify_artifact(record['retained_helper'], identity=not moved)
|
|
1212
|
+
verify_artifact(record['retained_recovery'], package=True, identity=not moved)
|
|
1213
|
+
verify_artifact(pins['runtime'], package=True, identity=not moved)
|
|
959
1214
|
for path, before in observed:
|
|
960
|
-
|
|
1215
|
+
after = _evidence(path)
|
|
1216
|
+
require(after is None if before is None else _same_file(before, after), f'recovery pins changed: {path}')
|
|
961
1217
|
return record
|
|
962
1218
|
except (StorageRefusal, OSError, ValueError, RuntimeError) as exc:
|
|
963
1219
|
detail = exc.diagnostic if isinstance(exc, StorageRefusal) else str(exc)
|
|
@@ -1031,6 +1287,7 @@ def enrollment(*, receipt=None, verify_registration=verify_registration):
|
|
|
1031
1287
|
root = named_root
|
|
1032
1288
|
else:
|
|
1033
1289
|
root = named_root.resolve(strict=True)
|
|
1290
|
+
retirement_refusal(root)
|
|
1034
1291
|
evidence = []
|
|
1035
1292
|
observations = []
|
|
1036
1293
|
|
|
@@ -1160,6 +1417,84 @@ def operation_enrollment():
|
|
|
1160
1417
|
message=repair, resume=repair) from exc
|
|
1161
1418
|
|
|
1162
1419
|
|
|
1420
|
+
def queue_handoff(common, binding):
|
|
1421
|
+
"""Follow Devlyn's pre-contract hold without changing enrollment or bindings."""
|
|
1422
|
+
try:
|
|
1423
|
+
decision = operation_enrollment()
|
|
1424
|
+
managed = decision['mode'] == 'managed'
|
|
1425
|
+
if not managed and not recovery_marker():
|
|
1426
|
+
return None
|
|
1427
|
+
except StorageRefusal as exc:
|
|
1428
|
+
return str(exc)
|
|
1429
|
+
reference = binding['release']
|
|
1430
|
+
listed = False
|
|
1431
|
+
try:
|
|
1432
|
+
manifest = read_evidence_json(Path(reference['root']) / 'release.json', [], limit=4*1024*1024)
|
|
1433
|
+
listed = (type(manifest) is dict and type(manifest.get('schemaVersion')) is int
|
|
1434
|
+
and manifest['schemaVersion'] == 1
|
|
1435
|
+
and all(manifest.get(key) == reference[key] for key in ('package', 'version'))
|
|
1436
|
+
and type(manifest.get('digest')) is dict
|
|
1437
|
+
and manifest['digest'].get('algorithm') == 'sha256'
|
|
1438
|
+
and manifest['digest'].get('value') == reference['digest']
|
|
1439
|
+
and type(manifest.get('files')) is list
|
|
1440
|
+
and sum(type(entry) is dict and entry.get('path') == 'config/storage-client-contract.json'
|
|
1441
|
+
for entry in manifest['files']) == 1)
|
|
1442
|
+
except (StorageRefusal, OSError, ValueError):
|
|
1443
|
+
listed = False # An unreadable manifest is pre-contract, never positive absence.
|
|
1444
|
+
drain = 'Devlyn runs this queue on this computer: start it with Drain in Devlyn'
|
|
1445
|
+
if listed:
|
|
1446
|
+
return drain if managed else None
|
|
1447
|
+
holders = []
|
|
1448
|
+
def unreadable(path):
|
|
1449
|
+
holders.append(f'{path} cannot be read; repair it so this release can read it')
|
|
1450
|
+
root = Path(common) / 'devlyn-completion'
|
|
1451
|
+
try:
|
|
1452
|
+
info = _evidence(root)
|
|
1453
|
+
if info is not None:
|
|
1454
|
+
require(stat.S_ISDIR(info.st_mode), 'invalid receipt root')
|
|
1455
|
+
for entry in sorted(root.iterdir()):
|
|
1456
|
+
receipt = entry / 'receipt.json'
|
|
1457
|
+
adapter_marker = False
|
|
1458
|
+
info = None
|
|
1459
|
+
try:
|
|
1460
|
+
info = _evidence(entry)
|
|
1461
|
+
require(info is not None, 'receipt entry disappeared')
|
|
1462
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
1463
|
+
continue
|
|
1464
|
+
try:
|
|
1465
|
+
(entry / 'storage-allocation.json').lstat()
|
|
1466
|
+
adapter_marker = True # Any Rust marker kind is positive presence.
|
|
1467
|
+
except FileNotFoundError:
|
|
1468
|
+
adapter_marker = False
|
|
1469
|
+
if _evidence(receipt) is None:
|
|
1470
|
+
if adapter_marker:
|
|
1471
|
+
holders.append(str(receipt))
|
|
1472
|
+
continue
|
|
1473
|
+
saved = read_evidence_json(receipt, [], limit=None)
|
|
1474
|
+
if adapter_marker or managed_receipt(saved):
|
|
1475
|
+
text = str(receipt)
|
|
1476
|
+
registration = saved.get('storage', {}).get('decision', {}).get('registration')
|
|
1477
|
+
if registration and moved_operation(registration):
|
|
1478
|
+
canonical_operation(saved['storage']['request']['operation_id'])
|
|
1479
|
+
# Share the operation-specific resume builder with task completion.
|
|
1480
|
+
helper = runpy.run_path(str(Path(__file__).with_name('task-complete.py')))
|
|
1481
|
+
text += ' resume ' + helper['receipt_resume']('recover-storage', receipt, saved)
|
|
1482
|
+
holders.append(text)
|
|
1483
|
+
except (StorageRefusal, OSError, ValueError, TypeError, AttributeError, KeyError):
|
|
1484
|
+
if adapter_marker:
|
|
1485
|
+
holders.append(str(receipt))
|
|
1486
|
+
continue
|
|
1487
|
+
unreadable(receipt if info is not None and stat.S_ISDIR(info.st_mode) else entry)
|
|
1488
|
+
except (StorageRefusal, OSError, ValueError):
|
|
1489
|
+
unreadable(root)
|
|
1490
|
+
if not holders:
|
|
1491
|
+
return drain if managed else None
|
|
1492
|
+
return ('this queue waits for the tasks in progress here: ' + '; '.join(holders)
|
|
1493
|
+
+ ". Finish each with this release, and follow any resume a refusal names; a moved task's resume releases it and keeps its files; "
|
|
1494
|
+
+ '`recover-storage --for-removal` removes one only when the owner says so (task-completion.md); '
|
|
1495
|
+
+ ('then start it with Drain in Devlyn' if managed else 'then run this again'))
|
|
1496
|
+
|
|
1497
|
+
|
|
1163
1498
|
def sync_directory(path):
|
|
1164
1499
|
"""Native durability barrier; failure is visible, never treated as success."""
|
|
1165
1500
|
path = Path(path)
|
|
@@ -316,15 +316,21 @@ def ignore_task_records(work):
|
|
|
316
316
|
f"{directory} records are not ignored or .devlyn is dirty; repair the starting commit's .devlyn paths and ignore rules before allocating")
|
|
317
317
|
|
|
318
318
|
|
|
319
|
-
def storage_call(name, *args, adapter=False, receipt_path=None, **kwargs):
|
|
319
|
+
def storage_call(name, *args, adapter=False, receipt_path=None, resume=None, **kwargs):
|
|
320
320
|
module = shared("storage-checkout" if adapter else "storage-client")
|
|
321
321
|
try:
|
|
322
322
|
return module[name](*args, **kwargs)
|
|
323
323
|
except module["StorageRefusal"] as exc:
|
|
324
324
|
request = args[2] if adapter and len(args) == 3 else kwargs.get("receipt", {}).get("storage", {}).get("request", {})
|
|
325
325
|
path = receipt_path or request.get("external_receipt")
|
|
326
|
-
resume = exc.resume or (storage_resume(path, for_removal=name in ('enrollment', 'recovery_registration')) if path else None)
|
|
327
|
-
|
|
326
|
+
resume = exc.resume or resume or (storage_resume(path, for_removal=name in ('enrollment', 'recovery_registration')) if path else None)
|
|
327
|
+
message = str(exc)
|
|
328
|
+
if resume:
|
|
329
|
+
if resume not in message:
|
|
330
|
+
message += '; resume ' + resume
|
|
331
|
+
else:
|
|
332
|
+
message += '; repair Devlyn enrollment and retry'
|
|
333
|
+
error = CompletionError(message)
|
|
328
334
|
error.storage_error = exc.result(receipt=path, resume=resume)
|
|
329
335
|
raise error from exc
|
|
330
336
|
|
|
@@ -339,10 +345,15 @@ def managed_registration(receipt, *, recovery=False):
|
|
|
339
345
|
require('storage' in receipt, 'managed receipt binding missing; retain and repair enrollment')
|
|
340
346
|
saved = receipt['storage']['decision']['registration']
|
|
341
347
|
if recovery:
|
|
348
|
+
storage_call('retirement_refusal')
|
|
342
349
|
current = storage_call('recovery_registration', receipt_path=receipt['storage'].get('request', {}).get('external_receipt'))
|
|
350
|
+
if authority_binding(current) != authority_binding(saved):
|
|
351
|
+
refuse_moved(receipt)
|
|
343
352
|
require(authority_binding(current) == authority_binding(saved),
|
|
344
353
|
'recovery pins authority binding changed; retain and repair enrollment')
|
|
345
354
|
return {**current, 'helper_path': current['retained_helper']['path']}
|
|
355
|
+
if decision['mode'] != 'managed' or authority_binding(decision['registration']) != authority_binding(saved):
|
|
356
|
+
refuse_moved(receipt)
|
|
346
357
|
require(decision['mode'] == 'managed', 'managed enrollment missing')
|
|
347
358
|
require(authority_binding(decision['registration']) == authority_binding(saved),
|
|
348
359
|
'enrollment authority binding changed; retain original recovery binding')
|
|
@@ -487,11 +498,61 @@ def storage_rpc(registration, request, path=None):
|
|
|
487
498
|
raise error from exc
|
|
488
499
|
|
|
489
500
|
|
|
501
|
+
def refuse_moved(receipt, path=None):
|
|
502
|
+
if receipt['storage'].get('moved') or storage_call('moved_operation', receipt['storage']['decision']['registration']):
|
|
503
|
+
path = Path(receipt['storage']['request']['external_receipt']) if path is None else path
|
|
504
|
+
resume = receipt_resume('recover-storage', path, receipt)
|
|
505
|
+
error = CompletionError('This task moved with Devlyn; release it and keep its files: ' + resume)
|
|
506
|
+
error.storage_error = {'receipt': str(path), 'resume': resume}
|
|
507
|
+
raise error
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def release_moved(receipt, path, operation):
|
|
511
|
+
state = receipt['storage']
|
|
512
|
+
if 'moved' not in state:
|
|
513
|
+
registration = storage_call('recovery_registration', receipt_path=str(path),
|
|
514
|
+
resume=receipt_resume('recover-storage', path, receipt))
|
|
515
|
+
registration = {**registration, 'helper_path': registration['retained_helper']['path']}
|
|
516
|
+
storage_call('replay_retirement', registration, receipt_path=str(path))
|
|
517
|
+
result = None
|
|
518
|
+
if path_present(path.with_name('storage-allocation.json')):
|
|
519
|
+
if 'removal_request' not in state:
|
|
520
|
+
work = Path(receipt['worktree'])
|
|
521
|
+
state['removal_request'] = {'adapter_protocol': 1,
|
|
522
|
+
'allocation_operation_id': state['request']['operation_id'], 'external_receipt': str(path),
|
|
523
|
+
'removal_operation_id': storage_call('operation_id'),
|
|
524
|
+
'worktree_identity': storage_call('persistent_identity', work) if path_present(work) else None}
|
|
525
|
+
atomic_json(path, receipt)
|
|
526
|
+
result = storage_call('call', registration, 'reconcile-for-removal', state['removal_request'], adapter=True)
|
|
527
|
+
require(result['operation_id'] == state['removal_request']['removal_operation_id'], 'removal fence ID changed')
|
|
528
|
+
require(result['closure_result']['established_by'] != 'legacy_unreserved', 'moved release lacks positive closure')
|
|
529
|
+
state['moved'] = {'operation_id': operation, 'removal_result': result}
|
|
530
|
+
atomic_json(path, receipt)
|
|
531
|
+
root = Path(receipt['common_gitdir']) / 'devlyn-completion'
|
|
532
|
+
moved = root / 'moved'
|
|
533
|
+
moved.mkdir(exist_ok=True)
|
|
534
|
+
require(moved.resolve() == moved and moved.is_dir(), 'moved receipt directory redirected; retain')
|
|
535
|
+
destination = moved / (receipt['id'] + '.' + state['request']['operation_id'])
|
|
536
|
+
if path.parent != destination:
|
|
537
|
+
require(not path_present(destination), 'moved receipt destination already exists; retain both')
|
|
538
|
+
path.parent.rename(destination)
|
|
539
|
+
# Also retry the retired source barrier after a rename whose reply was lost.
|
|
540
|
+
if path_present(root / 'retired'):
|
|
541
|
+
storage_call('sync_directory', root / 'retired')
|
|
542
|
+
storage_call('sync_directory', moved)
|
|
543
|
+
storage_call('sync_directory', root)
|
|
544
|
+
return {'status': 'STORAGE_MOVED', 'receipt': str(destination / 'receipt.json'), **state['moved']}
|
|
545
|
+
|
|
546
|
+
|
|
490
547
|
def recover_storage(args):
|
|
491
548
|
path = resolve_receipt(Path(args.receipt).absolute(), operation_id=getattr(args, "operation_id", None))
|
|
492
|
-
with locked_receipt(path, operation_id=getattr(args, "operation_id", None), resume=True) as receipt:
|
|
549
|
+
with locked_receipt(path, operation_id=getattr(args, "operation_id", None), resume=True, allow_moved=True) as receipt:
|
|
493
550
|
args.operation_id = receipt.get("storage", {}).get("request", {}).get("operation_id")
|
|
494
551
|
require(storage_call("managed_receipt", receipt), "receipt is not managed")
|
|
552
|
+
operation = receipt['storage'].get('moved', {}).get('operation_id') or storage_call(
|
|
553
|
+
'moved_operation', receipt['storage']['decision']['registration'])
|
|
554
|
+
if operation:
|
|
555
|
+
return release_moved(receipt, path, operation)
|
|
495
556
|
if args.for_removal:
|
|
496
557
|
accepted = receipt.get('acceptance') or receipt.get('pushed') or receipt.get('pr_number')
|
|
497
558
|
if accepted and receipt.get('pr_number') and receipt.get('delivery') not in {'COMPLETE', 'CLOSED'}:
|
|
@@ -1457,32 +1518,41 @@ def resolve_receipt(path, *, operation_id=None):
|
|
|
1457
1518
|
if operation_id is None or read_json(path).get('storage', {}).get('request', {}).get('operation_id') == operation_id:
|
|
1458
1519
|
return path
|
|
1459
1520
|
require(path.parent.parent.name == 'devlyn-completion', 'receipt operation ID differs from saved resume')
|
|
1521
|
+
kinds = ('retired', 'moved')
|
|
1522
|
+
if path.parent.parent.name == 'retired' and path.parent.parent.parent.name == 'devlyn-completion':
|
|
1523
|
+
key, separator, archived_operation = path.parent.name.partition('.')
|
|
1524
|
+
require(separator and (operation_id is None or operation_id == archived_operation),
|
|
1525
|
+
'receipt operation ID differs from saved resume')
|
|
1526
|
+
operation_id = archived_operation
|
|
1527
|
+
path = path.parent.parent.parent / key / 'receipt.json'
|
|
1528
|
+
kinds = ('moved',)
|
|
1460
1529
|
require(path.parent.parent.name == 'devlyn-completion', 'receipt locator is missing')
|
|
1461
1530
|
candidates = []
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1531
|
+
for kind in kinds:
|
|
1532
|
+
retired = path.parent.parent / kind
|
|
1533
|
+
if path_present(retired):
|
|
1534
|
+
require(retired == retired.resolve() and retired.is_dir(), f'{kind} receipt directory redirected')
|
|
1535
|
+
for candidate in retired.glob(path.parent.name + '.*/receipt.json'):
|
|
1536
|
+
if operation_id is not None and candidate.parent.name != path.parent.name + '.' + operation_id:
|
|
1537
|
+
continue
|
|
1538
|
+
saved = read_json(candidate)
|
|
1539
|
+
operation = saved.get('storage', {}).get('request', {}).get('operation_id')
|
|
1540
|
+
require(candidate == candidate.resolve() and saved.get('id') == path.parent.name
|
|
1541
|
+
and Path(saved['common_gitdir']) / 'devlyn-completion' / saved['id'] / 'receipt.json' == path
|
|
1542
|
+
and saved.get('storage', {}).get('request', {}).get('external_receipt') == str(path)
|
|
1543
|
+
and candidate.parent.name == saved['id'] + '.' + str(operation)
|
|
1544
|
+
and (managed_terminal(saved) if kind == 'retired' else bool(saved.get('storage', {}).get('moved'))),
|
|
1545
|
+
f'{kind} receipt locator/operation binding changed; retain')
|
|
1546
|
+
if operation_id is None or operation == operation_id:
|
|
1547
|
+
candidates.append(candidate)
|
|
1548
|
+
require(len(candidates) == 1, 'receipt operation/locator missing or ambiguous; use the operation-specific archived receipt: '
|
|
1479
1549
|
+ ', '.join(map(str, candidates)))
|
|
1480
1550
|
return candidates[0]
|
|
1481
1551
|
|
|
1482
1552
|
|
|
1483
1553
|
@contextlib.contextmanager
|
|
1484
|
-
def locked_receipt(path, *, blocking=True, operation_id=None, resume=False):
|
|
1485
|
-
require(path.name == "receipt.json" and (path.parent.parent.name == "devlyn-completion" or path.parent.parent.name
|
|
1554
|
+
def locked_receipt(path, *, blocking=True, operation_id=None, resume=False, allow_moved=False):
|
|
1555
|
+
require(path.name == "receipt.json" and (path.parent.parent.name == "devlyn-completion" or path.parent.parent.name in {"retired", "moved"} and path.parent.parent.parent.name == "devlyn-completion") and path == path.resolve(), "receipt must be its original external nonsymlink path")
|
|
1486
1556
|
preview = read_json(path)
|
|
1487
1557
|
if storage_call('managed_receipt', preview):
|
|
1488
1558
|
root = path.parent.parent if path.parent.parent.name == 'devlyn-completion' else path.parent.parent.parent
|
|
@@ -1504,7 +1574,8 @@ def locked_receipt(path, *, blocking=True, operation_id=None, resume=False):
|
|
|
1504
1574
|
require(operation_id is None or actual == operation_id, 'receipt operation ID differs from saved resume; retain both operations')
|
|
1505
1575
|
original = Path(receipt['common_gitdir']) / 'devlyn-completion' / receipt['id'] / 'receipt.json'
|
|
1506
1576
|
archived = original.parent.parent / 'retired' / (receipt['id'] + '.' + receipt.get('storage', {}).get('request', {}).get('operation_id', '')) / 'receipt.json'
|
|
1507
|
-
|
|
1577
|
+
moved = archived.parent.parent.parent / 'moved' / archived.parent.name / 'receipt.json'
|
|
1578
|
+
require(path in (original, archived, moved), 'receipt/common Gitdir binding mismatch')
|
|
1508
1579
|
require(receipt["id"] == hashlib.sha256(receipt["branch"].encode()).hexdigest()[:24] and receipt["branch"] not in {receipt["base"], "main", "master"}, "receipt branch ownership changed")
|
|
1509
1580
|
if storage_call("managed_receipt", receipt):
|
|
1510
1581
|
require(receipt.get("storage", {}).get("decision", {}).get("registration") and receipt.get("storage", {}).get("request"), "Open Devlyn and repair this managed receipt binding before resuming")
|
|
@@ -1513,6 +1584,10 @@ def locked_receipt(path, *, blocking=True, operation_id=None, resume=False):
|
|
|
1513
1584
|
require(receipt["allocation"] == "owned", "allocation was interrupted; no historical adoption is permitted")
|
|
1514
1585
|
if path == archived:
|
|
1515
1586
|
require(managed_terminal(receipt), 'retired receipt is not terminal')
|
|
1587
|
+
if path == moved:
|
|
1588
|
+
require(receipt.get('storage', {}).get('moved'), 'moved receipt lacks disposition')
|
|
1589
|
+
if storage_call('managed_receipt', receipt) and not allow_moved:
|
|
1590
|
+
refuse_moved(receipt, path)
|
|
1516
1591
|
yield receipt
|
|
1517
1592
|
|
|
1518
1593
|
|