devlyn-cli 4.3.24 → 4.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -27
- package/bin/devlyn.js +42 -94
- package/bin/engines.js +14 -12
- package/bin/harness-activation.js +4 -2
- package/bin/harness-context.js +395 -0
- package/bin/harness-run.js +148 -0
- package/bin/instructions.js +71 -159
- package/bin/skill-history.json +7 -0
- package/config/skills/_shared/storage-checkout.py +1 -0
- package/config/skills/_shared/storage-client.py +349 -14
- package/config/skills/_shared/task-complete.py +99 -24
- package/config/skills/_shared/task-completion.md +7 -3
- package/config/skills/devlyn-ideate/SKILL.md +2 -2
- package/config/skills/devlyn-ideate/references/loop.md +2 -2
- package/config/skills/devlyn-ideate/scripts/queue.py +10 -3
- package/native/claude/.claude-plugin/plugin.json +1 -1
- package/native/claude/skills/_shared/storage-checkout.py +1 -0
- package/native/claude/skills/_shared/storage-client.py +349 -14
- package/native/claude/skills/_shared/task-complete.py +99 -24
- package/native/claude/skills/_shared/task-completion.md +7 -3
- package/native/claude/skills/devlyn-ideate/SKILL.md +2 -2
- package/native/claude/skills/devlyn-ideate/references/loop.md +2 -2
- package/native/claude/skills/devlyn-ideate/scripts/queue.py +10 -3
- package/native/codex/.agents/plugins/marketplace.json +1 -1
- package/native/codex/plugins/devlyn/.codex-plugin/plugin.json +1 -1
- package/native/codex/plugins/devlyn/skills/_shared/storage-checkout.py +1 -0
- package/native/codex/plugins/devlyn/skills/_shared/storage-client.py +349 -14
- package/native/codex/plugins/devlyn/skills/_shared/task-complete.py +99 -24
- package/native/codex/plugins/devlyn/skills/_shared/task-completion.md +7 -3
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/SKILL.md +2 -2
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/loop.md +2 -2
- package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/queue.py +10 -3
- package/package.json +2 -2
- package/release.json +43 -33
|
@@ -106,8 +106,10 @@ New allocation and warming modes come from enrollment state alone:
|
|
|
106
106
|
| absent | present | verified `retired: true` | standalone (unenrollment complete) |
|
|
107
107
|
| absent | present | absent, unreadable or incomplete | refuse; Open Devlyn to repair its storage registration |
|
|
108
108
|
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
An unfinished retirement takes precedence over enrollment; follow its
|
|
110
|
+
operation-specific retained-helper resume to reconcile or cancel as named.
|
|
111
|
+
The journal `storage-client-unenroll.json` is read no-follow for retirement
|
|
112
|
+
recovery and the last two rows. Schema 1, canonical UUIDv7 operation, permanent `storage-client-fence.json`,
|
|
111
113
|
matching operation/root/lock identities and Rust's archive ordering/path
|
|
112
114
|
validation are required. If native root/lock identities or the home path changed,
|
|
113
115
|
completed retirement permits standalone only when `storage`, `control`,
|
|
@@ -119,7 +121,9 @@ completed unenrollment; a leftover managed key retains its own recovery command.
|
|
|
119
121
|
Existing operations keep their managed identity. Receipts pin the control
|
|
120
122
|
identity; current enrollment verifies the helper before allocation. Recovery uses
|
|
121
123
|
the current verified `storage-client-recovery/pins.json`, including after uninstall
|
|
122
|
-
or unenrollment, and requires its authority binding to match the receipt
|
|
124
|
+
or unenrollment, and requires its authority binding to match the receipt, except
|
|
125
|
+
for a proven moved task: use its unflagged, operation-specific `recover-storage`
|
|
126
|
+
resume to release it, retain its files, and archive its receipt under `moved/`. Missing
|
|
123
127
|
or invalid pins retain resources and require repair; saved artifacts are never a
|
|
124
128
|
fallback. Enrolled checkout preserves
|
|
125
129
|
`managed_pending`/`managed_owned` and Rust's `storage-allocation.json` subreceipt.
|
|
@@ -3,7 +3,7 @@ name: devlyn-ideate
|
|
|
3
3
|
description: Loop designer and intent queue. Turns an intent or a document into a validated loop package (a meta-prompt plus self-contained task contracts), queues its tasks, reports queue status, and drains the queue serially with evidence-derived acceptance, recovery and delivery. Operations — plan, add, status, drain; a bare intent is planned and, when the request authorizes implementation, continues through add and drain; no arguments shows status. Use when the user wants an idea, goal or document planned into tasks, wants to stack work ("queue this", "큐에 넣어줘"), asks what is queued, or wants unattended execution ("drain the queue", "큐 드레인 시작", "밤새 돌려줘").
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
Ideate owns planning and durable serial execution. Each task runs under the installed methodology, the
|
|
6
|
+
Ideate owns planning and durable serial execution. Each task runs under the installed methodology, the Devlyn defaults and project instructions; ideate adds no phase graph, reviewer quota, engine router or restart cycle.
|
|
7
7
|
|
|
8
8
|
<ideate_args>
|
|
9
9
|
$ARGUMENTS
|
|
@@ -94,7 +94,7 @@ A removed flag stops with its instruction and selects no other behavior.
|
|
|
94
94
|
|
|
95
95
|
An absolute path to a package's `meta.md` is added as it is; anything else is planned first. Add with `python3 "$DEVLYN_SKILL_DIR/scripts/queue.py" add '<absolute meta.md>'`, the only writer of a loop's queue file `docs/specs/<loop-id>/queue.md`; never edit one by hand. In every delivery mode add makes no commit on the checked-out branch and leaves the branch, the index and tracked files as they are: it holds the package and its rows at `refs/devlyn/captures/<loop-id>`, records the add and removes the package copies equal to that capture. Report the capture ref and its `git show` command, any `cleanup` entry, and that a revision needs a new loop id. A retry of the same add reports it as added, also once the copies are gone. Tasks start from committed state, a local loop from the branch it was added on and `auto`/`pr` from the remote base, so commit, and for `auto`/`pr` push, whatever the plan depends on. A pending legacy raw-intent row of `docs/specs/queue.md` is replaced in place by a package whose `## Intent` reproduces the row verbatim, added with `--materialize <line>`.
|
|
96
96
|
|
|
97
|
-
Release-backed queues retain their verified release in `<git-common-dir>/devlyn-loops/harness.json` until all tasks, recovery and delivery settle. A different release refuses
|
|
97
|
+
Release-backed queues retain their verified release in `<git-common-dir>/devlyn-loops/harness.json` until all tasks, recovery and delivery settle. A different release refuses: use the exact recovery argv when it gives one; otherwise do what it says (Drain in Devlyn, the tasks to finish first, or a repair). Never build the bound queue's path yourself. Existing unfinished unbound queues and standalone installations remain legacy.
|
|
98
98
|
|
|
99
99
|
## status
|
|
100
100
|
|
|
@@ -55,9 +55,9 @@ Package cleanup, parent pruning and report failures appear as diagnostics under
|
|
|
55
55
|
|
|
56
56
|
## Release binding and read-only consumers
|
|
57
57
|
|
|
58
|
-
`<git-common-dir>/devlyn-loops/harness.json` is authoritative across linked worktrees: `{schemaVersion: 1, release: {package, version, digest, root}, defaults: {path, sha256}}`. Paths are absolute and hashes cover exact bytes. The queue lock covers binding before the first capture. An interrupted first add, unfinished tasks, recovery conflicts and pending delivery retain it; settlement removes it. A different release refuses
|
|
58
|
+
`<git-common-dir>/devlyn-loops/harness.json` is authoritative across linked worktrees: `{schemaVersion: 1, release: {package, version, digest, root}, defaults: {path, sha256}}`. Paths are absolute and hashes cover exact bytes. The queue lock covers binding before the first capture. An interrupted first add, unfinished tasks, recovery conflicts and pending delivery retain it; settlement removes it. A different release refuses: use the exact recovery argv when it gives one (a JSON argv array preserving the caller’s arguments); otherwise do what it says (Drain in Devlyn, the tasks to finish first, or a repair). Never build the bound queue’s path yourself. No caller substitutes another implementation.
|
|
59
59
|
|
|
60
|
-
`queue.py inspect --repo <dir> --json` reads without cleanup, locks or writes. It returns the status summary plus `binding` (null for legacy) and `compatibility`: `compatible`, `legacy`, `reasons`, `recoveryArgv`, `taskPacket` ranges and `queueBinding` ranges. Incompatible or unreadable state is reported, never migrated. Apps should use this reader instead of `status`, which still reconciles cleanup. For a mismatch, `recoveryArgv` names the bound queue’s inspect invocation
|
|
60
|
+
`queue.py inspect --repo <dir> --json` reads without cleanup, locks or writes. It returns the status summary plus `binding` (null for legacy) and `compatibility`: `compatible`, `legacy`, `reasons`, `recoveryArgv`, `taskPacket` ranges and `queueBinding` ranges. Incompatible or unreadable state is reported, never migrated. Apps should use this reader instead of `status`, which still reconciles cleanup. For a mismatch, `recoveryArgv` names the bound queue’s inspect invocation when handoff is allowed, and is null when the refusal names Drain in Devlyn, tasks to finish first, or a repair. Use an exact recovery argv when provided; otherwise follow the refusal.
|
|
61
61
|
|
|
62
62
|
Codex projection writers place `.devlyn-harness-projection.json` at the copied plugin root, beside `skills` and `.codex-plugin`: `{schemaVersion: 1, release: <ReleaseRef>, source: "native/codex/plugins/devlyn"}`. The reader verifies the named release and every projected file’s bytes/mode against that subtree (excluding only the marker). Symlinked skill invocations resolve through their real path. Standalone installs and existing unfinished unbound queues retain legacy behavior; new legacy packets use v2 with a null release, and their `defaults` name the invoking installation's own `_shared/runtime-principles.md` (`source: "skill"`). No engine applicability is inferred from instruction filenames: bound queues use verified release defaults with native instructions taking precedence, except a Devlyn managed block, which the defaults replace. Worker and runner paths point to the bound release, independent of a projection cache’s lifetime.
|
|
63
63
|
|
|
@@ -188,8 +188,11 @@ def enforce_binding(common, anchor=None):
|
|
|
188
188
|
if binding and anchor is not None and release_settled_binding(anchor, common, interrupted_add=True):
|
|
189
189
|
return enforce_binding(common)
|
|
190
190
|
if binding:
|
|
191
|
-
|
|
192
|
-
|
|
191
|
+
line = shared("storage-client")["queue_handoff"](common, binding)
|
|
192
|
+
argv = harness_call("recovery_argv", binding, sys.argv[1:]) if line is None else None
|
|
193
|
+
error = LoopError(f"{exc}; {line}" if line is not None else f"{exc}; recovery argv: {json.dumps(argv, ensure_ascii=False)}")
|
|
194
|
+
error.handoff = line
|
|
195
|
+
raise error from exc
|
|
193
196
|
raise
|
|
194
197
|
return binding
|
|
195
198
|
|
|
@@ -1661,7 +1664,11 @@ def inspect(args):
|
|
|
1661
1664
|
compatibility.update(compatible=False, reasons=[str(exc)])
|
|
1662
1665
|
try:
|
|
1663
1666
|
valid = harness_call("read_binding", common)
|
|
1664
|
-
if
|
|
1667
|
+
if hasattr(exc, "handoff"):
|
|
1668
|
+
line = exc.handoff
|
|
1669
|
+
else:
|
|
1670
|
+
line = shared("storage-client")["queue_handoff"](common, valid) if valid else None
|
|
1671
|
+
if valid and line is None:
|
|
1665
1672
|
compatibility["recoveryArgv"] = harness_call("recovery_argv", valid, ["inspect", "--repo", str(anchor), "--json"])
|
|
1666
1673
|
except (LoopError, OSError):
|
|
1667
1674
|
pass # The diagnostic above describes the malformed binding; no safe argv can be inferred.
|
|
@@ -52,6 +52,7 @@ def validate_closure(value):
|
|
|
52
52
|
require(type(value) is dict, 'positive reservation closure missing')
|
|
53
53
|
tag = value.get('established_by')
|
|
54
54
|
shapes = {'released': {'established_by': ('released',), 'reservation': 'reservation'},
|
|
55
|
+
'authority_migrated': {'established_by': ('authority_migrated',), 'operation_id': 'str'},
|
|
55
56
|
'authority_expired': {'established_by': ('authority_expired',), 'operation_id': 'str'},
|
|
56
57
|
'never_admitted': {'established_by': ('never_admitted',), 'operation_id': 'str'},
|
|
57
58
|
'legacy_unreserved': {'established_by': ('legacy_unreserved',), 'receipt_version': 'u32'}}
|
|
@@ -26,11 +26,15 @@ import math
|
|
|
26
26
|
import os
|
|
27
27
|
from pathlib import Path
|
|
28
28
|
import re
|
|
29
|
+
import runpy
|
|
30
|
+
import shlex
|
|
29
31
|
import selectors
|
|
30
32
|
import socket
|
|
31
33
|
import stat
|
|
32
34
|
import struct
|
|
35
|
+
import subprocess
|
|
33
36
|
import sys
|
|
37
|
+
import tempfile
|
|
34
38
|
import threading
|
|
35
39
|
import time
|
|
36
40
|
import uuid
|
|
@@ -857,7 +861,7 @@ def regular_digest(path):
|
|
|
857
861
|
return digest.hexdigest()
|
|
858
862
|
|
|
859
863
|
|
|
860
|
-
def package_digest(root):
|
|
864
|
+
def package_digest(root, *, excluded=()):
|
|
861
865
|
digest = hashlib.sha256()
|
|
862
866
|
root = Path(root)
|
|
863
867
|
def visit(directory):
|
|
@@ -866,6 +870,8 @@ def package_digest(root):
|
|
|
866
870
|
for path in sorted(directory.iterdir(), key=lambda p: p.name.encode("utf-8")):
|
|
867
871
|
info = _evidence(path)
|
|
868
872
|
require(info is not None, "retained package entry disappeared")
|
|
873
|
+
if stat.S_ISREG(info.st_mode) and path.relative_to(root).as_posix() in excluded:
|
|
874
|
+
continue
|
|
869
875
|
name = str(path.relative_to(root)).encode()
|
|
870
876
|
digest.update(struct.pack("<Q", len(name)))
|
|
871
877
|
digest.update(name)
|
|
@@ -887,10 +893,11 @@ ENROLLMENT_SHAPE = {"schema": "u32", "control_directory": "str", "control_identi
|
|
|
887
893
|
"retained_helper": ARTIFACT_SHAPE, "retained_recovery": ARTIFACT_SHAPE}
|
|
888
894
|
|
|
889
895
|
|
|
890
|
-
def verify_artifact(artifact, *, package=False):
|
|
896
|
+
def verify_artifact(artifact, *, package=False, identity=True):
|
|
891
897
|
path = Path(artifact['path'])
|
|
892
898
|
require(path.is_absolute() and str(path.resolve(strict=True)) == artifact['path'], 'noncanonical retained path')
|
|
893
|
-
|
|
899
|
+
if identity:
|
|
900
|
+
require(persistent_identity(artifact['path']) == artifact['identity'], 'retained identity changed')
|
|
894
901
|
digest = package_digest(artifact['path']) if package else regular_digest(artifact['path'])
|
|
895
902
|
require(digest == artifact['sha256'], 'retained digest changed')
|
|
896
903
|
|
|
@@ -902,13 +909,25 @@ def verify_registration(record):
|
|
|
902
909
|
raise StorageRefusal('storage_unavailable', f'registration inspection failed: {exc}', phase='enrollment') from exc
|
|
903
910
|
|
|
904
911
|
|
|
905
|
-
def
|
|
912
|
+
def validate_registration_shape(record):
|
|
913
|
+
"""Rust evidence::validate_shape without reading any copied identity."""
|
|
906
914
|
validate(record, ENROLLMENT_SHAPE, exact=True)
|
|
907
915
|
helper, recovery = record["helper_release"], record["recovery_release"]
|
|
908
916
|
require(record["schema"] == 1 and re.fullmatch(
|
|
909
917
|
r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?", helper["version"])
|
|
910
|
-
and 0 < len(helper["target"]) <= 128 and recovery["package"] == "devlyn-cli"
|
|
918
|
+
and 0 < len(helper["target"].encode()) <= 128 and recovery["package"] == "devlyn-cli"
|
|
911
919
|
and recovery["version"] and recovery["digest"], "unsupported or incomplete enrollment", "storage_unsupported")
|
|
920
|
+
require(record["retained_helper"]["sha256"] == helper["sha256"]
|
|
921
|
+
and record["retained_recovery"]["path"] == recovery["root"], "retained release binding changed")
|
|
922
|
+
for value in (record['control_directory'], record['helper_path'], recovery['root'],
|
|
923
|
+
record['retained_helper']['path'], record['retained_recovery']['path']):
|
|
924
|
+
path = Path(value)
|
|
925
|
+
require(path.is_absolute() and str(path) == value and '..' not in path.parts, 'noncanonical enrollment path')
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _verify_registration(record):
|
|
929
|
+
validate_registration_shape(record)
|
|
930
|
+
helper = record['helper_release']
|
|
912
931
|
for value in (record['control_directory'], record['helper_path']):
|
|
913
932
|
path = Path(value)
|
|
914
933
|
require(path.is_absolute() and str(path.resolve(strict=True)) == value, "noncanonical enrollment path")
|
|
@@ -916,24 +935,259 @@ def _verify_registration(record):
|
|
|
916
935
|
require(regular_digest(record["helper_path"]) == helper["sha256"], "helper digest changed")
|
|
917
936
|
verify_artifact(record['retained_helper'])
|
|
918
937
|
verify_artifact(record['retained_recovery'], package=True)
|
|
919
|
-
require(record["retained_helper"]["sha256"] == helper["sha256"]
|
|
920
|
-
and record["retained_recovery"]["path"] == recovery["root"], "retained release binding changed")
|
|
921
938
|
return record
|
|
922
939
|
|
|
923
940
|
|
|
924
|
-
def read_evidence_json(path, observed):
|
|
941
|
+
def read_evidence_json(path, observed, *, limit=MAX_REQUEST):
|
|
925
942
|
info = _evidence(path)
|
|
926
943
|
require(info is not None and stat.S_ISREG(info.st_mode), f'missing/invalid {path}')
|
|
927
944
|
fd = os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_NONBLOCK', 0)
|
|
928
945
|
| getattr(os, 'O_BINARY', 0))
|
|
929
946
|
with os.fdopen(fd, 'rb') as stream:
|
|
930
947
|
require(_same_file(info, os.fstat(stream.fileno())), f'{path} changed while opening')
|
|
931
|
-
raw = stream.read(
|
|
932
|
-
require(len(raw) <=
|
|
948
|
+
raw = stream.read() if limit is None else stream.read(limit + 1)
|
|
949
|
+
require(limit is None or len(raw) <= limit, f'oversized {path}')
|
|
933
950
|
observed.append((path, info))
|
|
934
951
|
return loads(raw)
|
|
935
952
|
|
|
936
953
|
|
|
954
|
+
def lifecycle_evidence(root, path, observed):
|
|
955
|
+
"""Optional no-follow evidence, including every ancestor under the root."""
|
|
956
|
+
current = root
|
|
957
|
+
for part in path.relative_to(root).parts:
|
|
958
|
+
current = current / part
|
|
959
|
+
info = _evidence(current)
|
|
960
|
+
observed.append((current, info))
|
|
961
|
+
if info is None:
|
|
962
|
+
return None
|
|
963
|
+
require(stat.S_ISREG(info.st_mode) if current == path else stat.S_ISDIR(info.st_mode),
|
|
964
|
+
f'invalid lifecycle evidence kind: {current}')
|
|
965
|
+
return read_evidence_json(path, observed)
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def retirement_evidence(root, observed):
|
|
969
|
+
shape = {'schema': 'u32', 'operation_id': 'str', 'root_identity': 'identity', 'lock_identity': 'identity'}
|
|
970
|
+
fence = lifecycle_evidence(root, root / 'storage-client-fence.json', observed)
|
|
971
|
+
journal = lifecycle_evidence(root, root / 'storage-client-unenroll.json', observed)
|
|
972
|
+
if fence is not None:
|
|
973
|
+
validate(fence, shape, exact=True)
|
|
974
|
+
require(fence['schema'] == 1, 'unsupported retirement fence')
|
|
975
|
+
canonical_operation(fence['operation_id'])
|
|
976
|
+
if journal is not None:
|
|
977
|
+
validate(journal, {**shape, 'retired': 'bool', 'archives': [
|
|
978
|
+
{'source': 'str', 'destination': 'str', 'identity': 'identity', 'sha256': 'str'}]}, exact=True)
|
|
979
|
+
require(fence is not None and all(journal[key] == fence[key] for key in shape),
|
|
980
|
+
'retirement journal/fence differs')
|
|
981
|
+
sources = ('storage', 'control', 'harness/state.json', 'storage-client.json')
|
|
982
|
+
names = ('storage', 'control', 'harness-state.json', 'enrollment.json')
|
|
983
|
+
previous = -1
|
|
984
|
+
for archive in journal['archives']:
|
|
985
|
+
require(archive['source'] in [str(root / name) for name in sources], 'invalid retirement source')
|
|
986
|
+
index = sources.index(str(Path(archive['source']).relative_to(root)))
|
|
987
|
+
require(index > previous and Path(archive['destination']) == root / 'storage-client-retired'
|
|
988
|
+
/ fence['operation_id'] / names[index], 'invalid retirement archive path/order')
|
|
989
|
+
previous = index
|
|
990
|
+
if not journal['retired']:
|
|
991
|
+
for name in sources:
|
|
992
|
+
if str(root / name) not in [entry['source'] for entry in journal['archives']]:
|
|
993
|
+
require(_evidence(root / name) is None, 'uninventoried retirement evidence')
|
|
994
|
+
return fence, journal
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _retirement_move(root, pins, observed):
|
|
998
|
+
"""Establish a copied retirement before allowing digest-only bootstrap.
|
|
999
|
+
|
|
1000
|
+
Compare all identities in the evidence this client reads, never merely a
|
|
1001
|
+
failed retained-artifact identity. Same-volume replacements cannot qualify.
|
|
1002
|
+
"""
|
|
1003
|
+
fence, journal = retirement_evidence(root, observed)
|
|
1004
|
+
if fence is None:
|
|
1005
|
+
return False
|
|
1006
|
+
identities = []
|
|
1007
|
+
def record_identities(record):
|
|
1008
|
+
validate_registration_shape(record)
|
|
1009
|
+
identities.extend([record['control_identity'], record['retained_helper']['identity'],
|
|
1010
|
+
record['retained_recovery']['identity']])
|
|
1011
|
+
record_identities(pins['enrollment'])
|
|
1012
|
+
identities.append(pins['runtime']['identity'])
|
|
1013
|
+
record = lifecycle_evidence(root, root / 'storage-client.json', observed)
|
|
1014
|
+
if record is not None:
|
|
1015
|
+
record_identities(record)
|
|
1016
|
+
require(all(record[key] == pins['enrollment'][key] for key in ('control_directory', 'control_identity')),
|
|
1017
|
+
'move record/pins binding differs')
|
|
1018
|
+
identities.extend([fence['root_identity'], fence['lock_identity']])
|
|
1019
|
+
if journal is not None:
|
|
1020
|
+
identities.extend([journal['root_identity'], journal['lock_identity']])
|
|
1021
|
+
for archive in journal['archives']:
|
|
1022
|
+
identities.append(archive['identity'])
|
|
1023
|
+
paths = [Path(archive[key]) for key in ('source', 'destination')]
|
|
1024
|
+
existing = []
|
|
1025
|
+
for path in paths:
|
|
1026
|
+
# Check every ancestor without interpreting a directory as JSON.
|
|
1027
|
+
current = root
|
|
1028
|
+
for part in path.relative_to(root).parts:
|
|
1029
|
+
current /= part
|
|
1030
|
+
info = _evidence(current)
|
|
1031
|
+
observed.append((current, info))
|
|
1032
|
+
if info is None:
|
|
1033
|
+
break
|
|
1034
|
+
require(current == path or stat.S_ISDIR(info.st_mode), 'invalid archive ancestor')
|
|
1035
|
+
else:
|
|
1036
|
+
existing.append(path)
|
|
1037
|
+
require(len(existing) == 1, 'incomplete/ambiguous retirement archive')
|
|
1038
|
+
path = existing[0]
|
|
1039
|
+
digest = (package_digest(path, excluded=('service/serve.log', 'service/serve.log.1')
|
|
1040
|
+
if path.name == 'control' else ()) if path.is_dir() else regular_digest(path))
|
|
1041
|
+
require(digest == archive['sha256'], 'retirement archive digest changed')
|
|
1042
|
+
if archive['source'] == str(root / 'storage-client.json'):
|
|
1043
|
+
record_identities(read_evidence_json(path, observed))
|
|
1044
|
+
old = identities[0]['volume']
|
|
1045
|
+
require(old and all(identity['volume'] == old and identity['file'] for identity in identities),
|
|
1046
|
+
'mixed or malformed moved identities')
|
|
1047
|
+
current_volume = persistent_identity(root)['volume']
|
|
1048
|
+
if current_volume == old:
|
|
1049
|
+
return False
|
|
1050
|
+
for path, directory in ((root / 'storage-client.lock', False), (root / 'control', True)):
|
|
1051
|
+
info = _evidence(path)
|
|
1052
|
+
if info is not None:
|
|
1053
|
+
require((stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode))
|
|
1054
|
+
and persistent_identity(path)['volume'] == current_volume, 'mixed current move identities')
|
|
1055
|
+
return True
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def retirement_move(root, pins, observed):
|
|
1059
|
+
try:
|
|
1060
|
+
return _retirement_move(root, pins, observed)
|
|
1061
|
+
except StorageRefusal:
|
|
1062
|
+
# Rust Established::NotMoved: malformed/mixed evidence cannot authorize
|
|
1063
|
+
# bootstrap. The caller must still pass every ordinary identity check.
|
|
1064
|
+
return False
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def moved_operation(saved):
|
|
1068
|
+
"""A committed manifest or established retirement, never registry membership."""
|
|
1069
|
+
home = user_home()
|
|
1070
|
+
if home is None:
|
|
1071
|
+
return None
|
|
1072
|
+
root = (home / '.devlyn').resolve(strict=False)
|
|
1073
|
+
observed = []
|
|
1074
|
+
marker = root / 'storage-client-moved'
|
|
1075
|
+
info = _evidence(marker)
|
|
1076
|
+
journal_info = _evidence(root / 'storage-client-unenroll.json')
|
|
1077
|
+
if info is None and journal_info is None:
|
|
1078
|
+
return None
|
|
1079
|
+
require(type(saved) is dict, 'invalid saved authority binding')
|
|
1080
|
+
current = lifecycle_evidence(root, root / 'storage-client.json', observed)
|
|
1081
|
+
require(current is None or type(current) is dict, 'invalid current enrollment')
|
|
1082
|
+
if current is not None and all(key in current and key in saved and current[key] == saved[key]
|
|
1083
|
+
for key in ('control_directory', 'control_identity')):
|
|
1084
|
+
control = Path(current['control_directory'])
|
|
1085
|
+
if _evidence(control) is not None and persistent_identity(control) == current['control_identity']:
|
|
1086
|
+
return None
|
|
1087
|
+
if info is not None:
|
|
1088
|
+
require(stat.S_ISDIR(info.st_mode), 'redirected move archive')
|
|
1089
|
+
for directory in sorted(marker.iterdir()):
|
|
1090
|
+
# Like Rust, non-operation archive names are not manifests.
|
|
1091
|
+
try:
|
|
1092
|
+
canonical_operation(directory.name)
|
|
1093
|
+
except StorageRefusal:
|
|
1094
|
+
continue
|
|
1095
|
+
manifest = lifecycle_evidence(root, directory / 'storage-client-move.json', observed)
|
|
1096
|
+
if manifest is None:
|
|
1097
|
+
continue
|
|
1098
|
+
require(type(manifest) is dict, 'invalid move manifest')
|
|
1099
|
+
validate(manifest, {'schema': 'u32', 'operation_id': 'str', 'from_volume': 'str', 'to_volume': 'str',
|
|
1100
|
+
'old_binding': {'control_directory': 'str', 'control_identity': 'identity'} if manifest.get('old_binding') is not None else (None,)}, exact=True)
|
|
1101
|
+
require(manifest['schema'] == 1 and manifest['operation_id'] == directory.name
|
|
1102
|
+
and manifest['from_volume'] and manifest['to_volume']
|
|
1103
|
+
and manifest['from_volume'] != manifest['to_volume'], 'invalid move manifest')
|
|
1104
|
+
binding = manifest['old_binding']
|
|
1105
|
+
if binding is not None:
|
|
1106
|
+
path = Path(binding['control_directory'])
|
|
1107
|
+
require(path.is_absolute() and str(path) == binding['control_directory'] and '..' not in path.parts
|
|
1108
|
+
and binding['control_identity']['volume'] == manifest['from_volume']
|
|
1109
|
+
and binding['control_identity']['file'], 'invalid moved binding')
|
|
1110
|
+
if binding is not None and all(saved.get(key) == binding[key] for key in binding):
|
|
1111
|
+
return manifest['operation_id']
|
|
1112
|
+
if _evidence(root / 'storage-client-unenroll.json') is not None:
|
|
1113
|
+
pins = lifecycle_evidence(root, root / 'storage-client-recovery/pins.json', observed)
|
|
1114
|
+
if pins is None:
|
|
1115
|
+
return None # No retained binding can establish a moved receipt.
|
|
1116
|
+
validate(pins, {'schema': 'u32', 'enrollment': ENROLLMENT_SHAPE, 'runtime': ARTIFACT_SHAPE, 'node': 'str'}, exact=True)
|
|
1117
|
+
require(pins['schema'] == pins['enrollment']['schema'] == 1, 'unsupported recovery pins')
|
|
1118
|
+
if (all(saved.get(key) == pins['enrollment'][key] for key in ('control_directory', 'control_identity'))
|
|
1119
|
+
and retirement_move(root, pins, observed)):
|
|
1120
|
+
return read_evidence_json(root / 'storage-client-unenroll.json', observed)['operation_id']
|
|
1121
|
+
return None
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def unenroll_resume(record, operation, *, cancel=False):
|
|
1125
|
+
request = json.dumps({'adapter_protocol': 1, 'operation_id': operation}, separators=(',', ':'))
|
|
1126
|
+
return (shlex.join(['printf', '%s\\n', request]) + ' | '
|
|
1127
|
+
+ shlex.join([record['retained_helper']['path'], 'storage-client', 'unenroll',
|
|
1128
|
+
'--cancel' if cancel else '--reconcile', '--json']))
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
def replay_retirement(record):
|
|
1132
|
+
root = (user_home() / '.devlyn').resolve(strict=False)
|
|
1133
|
+
fence, journal = retirement_evidence(root, [])
|
|
1134
|
+
if journal is None or journal['retired']:
|
|
1135
|
+
return
|
|
1136
|
+
helper = record['retained_helper']['path']
|
|
1137
|
+
require(regular_digest(helper) == record['helper_release']['sha256'], 'retained helper changed')
|
|
1138
|
+
request = {'adapter_protocol': 1, 'operation_id': fence['operation_id']}
|
|
1139
|
+
adapter = runpy.run_path(str(Path(__file__).with_name('storage-checkout.py')))
|
|
1140
|
+
try:
|
|
1141
|
+
with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as diagnostic:
|
|
1142
|
+
process = subprocess.run([helper, 'storage-client', 'unenroll', '--reconcile', '--json'],
|
|
1143
|
+
input=json.dumps(request).encode()+b'\n', stdout=output, stderr=diagnostic)
|
|
1144
|
+
output.seek(0)
|
|
1145
|
+
reply = output.read(MAX_REPLY+1)
|
|
1146
|
+
result = adapter['decode_reply'](reply, 'unenroll', process.returncode)
|
|
1147
|
+
require(type(result) is dict and result.get('operation_id') == fence['operation_id']
|
|
1148
|
+
and result.get('retired') is True, 'retirement replay not complete')
|
|
1149
|
+
_, journal = retirement_evidence(root, [])
|
|
1150
|
+
require(journal is not None and journal['retired'], 'retirement replay did not commit')
|
|
1151
|
+
except (OSError, StorageRefusal, adapter['StorageRefusal']) as exc:
|
|
1152
|
+
resume = unenroll_resume(record, fence['operation_id'])
|
|
1153
|
+
detail = exc.diagnostic if isinstance(exc, StorageRefusal) else str(exc)
|
|
1154
|
+
raise StorageRefusal('storage_unavailable', detail, phase='retirement',
|
|
1155
|
+
message='Finish this storage retirement; resume ' + resume, resume=resume) from exc
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def retirement_refusal(root=None):
|
|
1159
|
+
"""An interrupted retirement has a headless exit through its retained helper."""
|
|
1160
|
+
if root is None:
|
|
1161
|
+
home = user_home()
|
|
1162
|
+
if home is None:
|
|
1163
|
+
return
|
|
1164
|
+
root = (home / '.devlyn').resolve(strict=False)
|
|
1165
|
+
if _evidence(root / 'storage-client-fence.json') is None and _evidence(root / 'storage-client-unenroll.json') is None:
|
|
1166
|
+
return
|
|
1167
|
+
try:
|
|
1168
|
+
# Completed retirement keeps operation_enrollment's existing validation,
|
|
1169
|
+
# including stale same-disk archive paths. Only an unfinished one exits here.
|
|
1170
|
+
journal = lifecycle_evidence(root, root / 'storage-client-unenroll.json', [])
|
|
1171
|
+
if type(journal) is dict and journal.get('retired') is True:
|
|
1172
|
+
return
|
|
1173
|
+
fence, journal = retirement_evidence(root, [])
|
|
1174
|
+
require(fence is not None, 'retirement fence missing')
|
|
1175
|
+
record = recovery_registration()
|
|
1176
|
+
except (StorageRefusal, OSError, ValueError, RuntimeError) as exc:
|
|
1177
|
+
detail = exc.diagnostic if isinstance(exc, StorageRefusal) else str(exc)
|
|
1178
|
+
repair = 'Open Devlyn to repair its storage registration'
|
|
1179
|
+
raise StorageRefusal('storage_unavailable', detail, phase='enrollment', message=repair, resume=repair) from exc
|
|
1180
|
+
resume = unenroll_resume(record, fence['operation_id'], cancel=journal is None)
|
|
1181
|
+
alternative = ''
|
|
1182
|
+
if journal is None:
|
|
1183
|
+
pins = read_evidence_json(root / 'storage-client-recovery/pins.json', [])
|
|
1184
|
+
if not retirement_move(root, pins, []):
|
|
1185
|
+
alternative = '; or cancel before retirement starts: ' + resume
|
|
1186
|
+
resume = unenroll_resume(record, fence['operation_id'])
|
|
1187
|
+
raise StorageRefusal('storage_unavailable', 'retirement is not complete', phase='enrollment',
|
|
1188
|
+
message='Finish this storage retirement; resume ' + resume + alternative, resume=resume)
|
|
1189
|
+
|
|
1190
|
+
|
|
937
1191
|
def recovery_registration():
|
|
938
1192
|
"""Current retained pins survive replacement of the app and unenrollment."""
|
|
939
1193
|
try:
|
|
@@ -953,11 +1207,13 @@ def recovery_registration():
|
|
|
953
1207
|
and record['recovery_release']['root'] == record['retained_recovery']['path']
|
|
954
1208
|
and node.is_absolute() and str(node.resolve(strict=True)) == pins['node']
|
|
955
1209
|
and node.is_relative_to(runtime), 'invalid recovery pin paths')
|
|
956
|
-
|
|
957
|
-
verify_artifact(record['
|
|
958
|
-
verify_artifact(
|
|
1210
|
+
moved = retirement_move(root, pins, observed)
|
|
1211
|
+
verify_artifact(record['retained_helper'], identity=not moved)
|
|
1212
|
+
verify_artifact(record['retained_recovery'], package=True, identity=not moved)
|
|
1213
|
+
verify_artifact(pins['runtime'], package=True, identity=not moved)
|
|
959
1214
|
for path, before in observed:
|
|
960
|
-
|
|
1215
|
+
after = _evidence(path)
|
|
1216
|
+
require(after is None if before is None else _same_file(before, after), f'recovery pins changed: {path}')
|
|
961
1217
|
return record
|
|
962
1218
|
except (StorageRefusal, OSError, ValueError, RuntimeError) as exc:
|
|
963
1219
|
detail = exc.diagnostic if isinstance(exc, StorageRefusal) else str(exc)
|
|
@@ -1031,6 +1287,7 @@ def enrollment(*, receipt=None, verify_registration=verify_registration):
|
|
|
1031
1287
|
root = named_root
|
|
1032
1288
|
else:
|
|
1033
1289
|
root = named_root.resolve(strict=True)
|
|
1290
|
+
retirement_refusal(root)
|
|
1034
1291
|
evidence = []
|
|
1035
1292
|
observations = []
|
|
1036
1293
|
|
|
@@ -1160,6 +1417,84 @@ def operation_enrollment():
|
|
|
1160
1417
|
message=repair, resume=repair) from exc
|
|
1161
1418
|
|
|
1162
1419
|
|
|
1420
|
+
def queue_handoff(common, binding):
|
|
1421
|
+
"""Follow Devlyn's pre-contract hold without changing enrollment or bindings."""
|
|
1422
|
+
try:
|
|
1423
|
+
decision = operation_enrollment()
|
|
1424
|
+
managed = decision['mode'] == 'managed'
|
|
1425
|
+
if not managed and not recovery_marker():
|
|
1426
|
+
return None
|
|
1427
|
+
except StorageRefusal as exc:
|
|
1428
|
+
return str(exc)
|
|
1429
|
+
reference = binding['release']
|
|
1430
|
+
listed = False
|
|
1431
|
+
try:
|
|
1432
|
+
manifest = read_evidence_json(Path(reference['root']) / 'release.json', [], limit=4*1024*1024)
|
|
1433
|
+
listed = (type(manifest) is dict and type(manifest.get('schemaVersion')) is int
|
|
1434
|
+
and manifest['schemaVersion'] == 1
|
|
1435
|
+
and all(manifest.get(key) == reference[key] for key in ('package', 'version'))
|
|
1436
|
+
and type(manifest.get('digest')) is dict
|
|
1437
|
+
and manifest['digest'].get('algorithm') == 'sha256'
|
|
1438
|
+
and manifest['digest'].get('value') == reference['digest']
|
|
1439
|
+
and type(manifest.get('files')) is list
|
|
1440
|
+
and sum(type(entry) is dict and entry.get('path') == 'config/storage-client-contract.json'
|
|
1441
|
+
for entry in manifest['files']) == 1)
|
|
1442
|
+
except (StorageRefusal, OSError, ValueError):
|
|
1443
|
+
listed = False # An unreadable manifest is pre-contract, never positive absence.
|
|
1444
|
+
drain = 'Devlyn runs this queue on this computer: start it with Drain in Devlyn'
|
|
1445
|
+
if listed:
|
|
1446
|
+
return drain if managed else None
|
|
1447
|
+
holders = []
|
|
1448
|
+
def unreadable(path):
|
|
1449
|
+
holders.append(f'{path} cannot be read; repair it so this release can read it')
|
|
1450
|
+
root = Path(common) / 'devlyn-completion'
|
|
1451
|
+
try:
|
|
1452
|
+
info = _evidence(root)
|
|
1453
|
+
if info is not None:
|
|
1454
|
+
require(stat.S_ISDIR(info.st_mode), 'invalid receipt root')
|
|
1455
|
+
for entry in sorted(root.iterdir()):
|
|
1456
|
+
receipt = entry / 'receipt.json'
|
|
1457
|
+
adapter_marker = False
|
|
1458
|
+
info = None
|
|
1459
|
+
try:
|
|
1460
|
+
info = _evidence(entry)
|
|
1461
|
+
require(info is not None, 'receipt entry disappeared')
|
|
1462
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
1463
|
+
continue
|
|
1464
|
+
try:
|
|
1465
|
+
(entry / 'storage-allocation.json').lstat()
|
|
1466
|
+
adapter_marker = True # Any Rust marker kind is positive presence.
|
|
1467
|
+
except FileNotFoundError:
|
|
1468
|
+
adapter_marker = False
|
|
1469
|
+
if _evidence(receipt) is None:
|
|
1470
|
+
if adapter_marker:
|
|
1471
|
+
holders.append(str(receipt))
|
|
1472
|
+
continue
|
|
1473
|
+
saved = read_evidence_json(receipt, [], limit=None)
|
|
1474
|
+
if adapter_marker or managed_receipt(saved):
|
|
1475
|
+
text = str(receipt)
|
|
1476
|
+
registration = saved.get('storage', {}).get('decision', {}).get('registration')
|
|
1477
|
+
if registration and moved_operation(registration):
|
|
1478
|
+
canonical_operation(saved['storage']['request']['operation_id'])
|
|
1479
|
+
# Share the operation-specific resume builder with task completion.
|
|
1480
|
+
helper = runpy.run_path(str(Path(__file__).with_name('task-complete.py')))
|
|
1481
|
+
text += ' resume ' + helper['receipt_resume']('recover-storage', receipt, saved)
|
|
1482
|
+
holders.append(text)
|
|
1483
|
+
except (StorageRefusal, OSError, ValueError, TypeError, AttributeError, KeyError):
|
|
1484
|
+
if adapter_marker:
|
|
1485
|
+
holders.append(str(receipt))
|
|
1486
|
+
continue
|
|
1487
|
+
unreadable(receipt if info is not None and stat.S_ISDIR(info.st_mode) else entry)
|
|
1488
|
+
except (StorageRefusal, OSError, ValueError):
|
|
1489
|
+
unreadable(root)
|
|
1490
|
+
if not holders:
|
|
1491
|
+
return drain if managed else None
|
|
1492
|
+
return ('this queue waits for the tasks in progress here: ' + '; '.join(holders)
|
|
1493
|
+
+ ". Finish each with this release, and follow any resume a refusal names; a moved task's resume releases it and keeps its files; "
|
|
1494
|
+
+ '`recover-storage --for-removal` removes one only when the owner says so (task-completion.md); '
|
|
1495
|
+
+ ('then start it with Drain in Devlyn' if managed else 'then run this again'))
|
|
1496
|
+
|
|
1497
|
+
|
|
1163
1498
|
def sync_directory(path):
|
|
1164
1499
|
"""Native durability barrier; failure is visible, never treated as success."""
|
|
1165
1500
|
path = Path(path)
|