@microck/raft 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -14
- package/deploy/deploy.py +112 -19
- package/deploy/install-incus.sh +10 -0
- package/deploy/raft-expire.py +109 -31
- package/deploy/verify-controls.py +34 -48
- package/deploy/verify-features.py +592 -3
- package/deploy/verify-fresh-host.py +64 -1
- package/deploy/verify-limits.py +327 -72
- package/deploy/verify-live.py +56 -31
- package/deploy/verify-self-update.py +372 -0
- package/docs/boat-capabilities.md +8 -7
- package/docs/boot-performance.mdx +1 -1
- package/docs/contract.md +33 -23
- package/docs/first-box.mdx +1 -1
- package/docs/guides/backup-recovery.mdx +2 -2
- package/docs/guides/config-policy.mdx +253 -0
- package/docs/guides/docker.mdx +9 -1
- package/docs/guides/long-running-services.mdx +10 -4
- package/docs/guides/meta.json +1 -0
- package/docs/guides/project-setup.mdx +4 -3
- package/docs/guides/scripting-ci.mdx +27 -10
- package/docs/guides/snapshots-forks.mdx +9 -3
- package/docs/guides/troubleshooting.mdx +37 -8
- package/docs/guides/upgrades.mdx +17 -5
- package/docs/index.mdx +1 -0
- package/docs/operations.mdx +16 -14
- package/docs/reference/capacity.mdx +198 -0
- package/docs/reference/cli.md +178 -35
- package/docs/reference/configuration.mdx +40 -2
- package/docs/reference/meta.json +1 -1
- package/docs/releases/0.5.0.md +19 -0
- package/docs/releases/0.6.0.md +16 -0
- package/docs/requirements.mdx +2 -2
- package/docs/verification.mdx +14 -8
- package/package.json +1 -1
- package/raft.py +1181 -128
- package/skills/raft-cli/SKILL.md +79 -13
- package/docs/reference/capacity-limits.mdx +0 -92
package/README.md
CHANGED
|
@@ -30,13 +30,16 @@ npm install -g @microck/raft
|
|
|
30
30
|
raft --version
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
+
`raft self-update` upgrades the CLI later. Raft prints a one-line stderr notice
|
|
34
|
+
when a new release exists; set `RAFT_NO_UPDATE_CHECK=1` to turn it off.
|
|
35
|
+
|
|
33
36
|
Python wheels and source archives are also available in
|
|
34
37
|
[releases](https://github.com/Microck/raft/releases). npm installation does not set
|
|
35
38
|
up hosts or download workspace images.
|
|
36
39
|
|
|
37
40
|
| requirement | minimum |
|
|
38
41
|
| --- | --- |
|
|
39
|
-
| host | dedicated ARM64 or AMD64 Ubuntu host, 2 CPUs, 8 GiB RAM, 80 GiB free disk |
|
|
42
|
+
| host | dedicated ARM64 or AMD64 Ubuntu host, 2 CPUs, 8 GiB RAM, 80 GiB free disk, plus the pool size if you enable own box disks |
|
|
40
43
|
| host access | SSH and passwordless sudo |
|
|
41
44
|
| controller | Linux, Python 3.11.8+, SSH and SCP; npm or uv for installation |
|
|
42
45
|
|
|
@@ -61,12 +64,25 @@ raft stop "$box"
|
|
|
61
64
|
`raft stop` and expiration keep persistent boxes on disk. `new --disposable`
|
|
62
65
|
deletes a temporary box when it stops or expires. `raft resume` starts
|
|
63
66
|
the box with a new lifetime. `raft destroy` deletes the box and its snapshots.
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
size
|
|
67
|
+
`raft policy lab --delete-stopped-after 48` makes the host delete boxes left
|
|
68
|
+
stopped for 48 hours, and logs each deletion in its journal. disposable
|
|
69
|
+
creation, prune and retention require the current host worker; redeploy from
|
|
70
|
+
the current checkout before using them.
|
|
71
|
+
|
|
72
|
+
size boxes for the job: `--cpu`, `--memory` and `--disk` take any whole number
|
|
73
|
+
up to the host. on hosts deployed with `--disk-pool SIZE`, `--disk 20GiB` gives a
|
|
74
|
+
box its own disk that it, and Docker inside it, cannot grow past; forks and
|
|
75
|
+
templates of it are copy-on-write. raft does not cap how many boxes you keep.
|
|
76
|
+
`raft capacity` prints an advisory estimate of what fits; it never blocks a box.
|
|
77
|
+
see [box sizing and capacity](docs/reference/capacity.mdx).
|
|
78
|
+
|
|
79
|
+
`raft config set` stores your defaults for `new`, such as location, TTL, size
|
|
80
|
+
and `disposable true`; `new --persistent` keeps one box anyway. size defaults
|
|
81
|
+
apply only to boxes from the image: `new --from` and `fork` keep the source's
|
|
82
|
+
size. `raft policy lab --require purpose,owner` makes the host reject boxes whose
|
|
83
|
+
note lacks those fields, with an error that names the missing flags. policies
|
|
84
|
+
are guardrails for agents: anyone with sudo on the host can change them.
|
|
85
|
+
see [defaults and host policies](docs/guides/config-policy.mdx).
|
|
70
86
|
|
|
71
87
|
## image and workspace control
|
|
72
88
|
|
|
@@ -131,8 +147,8 @@ records differences, sources and test coverage.
|
|
|
131
147
|
| Create/list/inspect/delete | ✅ | ✅ | Single operator |
|
|
132
148
|
| Persistent stop/resume | ✅ | ✅ | Shared host kernel |
|
|
133
149
|
| TTL and lifetime extension | ✅ | ✅ | Persistent boxes retain disk; disposable boxes delete |
|
|
134
|
-
| CPU/memory sizing and usage | ✅ | ✅ |
|
|
135
|
-
| Host capacity
|
|
150
|
+
| CPU/memory sizing and usage | ✅ | ✅ | Any whole CPU count and GiB size up to the host. Usage covers the shared pool or the box's own disk |
|
|
151
|
+
| Host capacity estimates | ❌ | ✅ | Advisory only; Raft has no box count limit. Boat reports plan limits |
|
|
136
152
|
| Root terminal and command execution | ✅ | ✅ | Host SSH transport |
|
|
137
153
|
| Background jobs, logs and cancellation | ✅ | ✅ | No process checkpoints |
|
|
138
154
|
| File upload/download | ✅ | ✅ | Files and recursive trees; new directory destinations only |
|
|
@@ -144,17 +160,17 @@ records differences, sources and test coverage.
|
|
|
144
160
|
| Desktop access | ✅ | ✅ | noVNC; see verification limits |
|
|
145
161
|
| Reverse forwarding | ✅ | ❌ | Not implemented |
|
|
146
162
|
| Public workspace hosting and browser-only streaming | ✅ | ❌ | Not implemented |
|
|
147
|
-
| Resize on resume/fork | ✅ | ✅ | Optional CPU/RAM overrides
|
|
148
|
-
| CLI-wide JSON mode | ✅ | ❌ | Boat JSON/JSONL; Raft list/info/usage/
|
|
163
|
+
| Resize on resume/fork | ✅ | ✅ | Optional CPU/RAM overrides; disks keep their size |
|
|
164
|
+
| CLI-wide JSON mode | ✅ | ❌ | Boat JSON/JSONL; Raft list/info/usage/capacity/config plus prune JSONL |
|
|
149
165
|
| Snapshot file browsing/download | ✅ | ❌ | Whole-box backup only |
|
|
150
|
-
| Strict per-box disk quotas | ✅ |
|
|
166
|
+
| Strict per-box disk quotas | ✅ | ✅ | On hosts deployed with `--disk-pool`, `new --disk SIZE` gives a box its own fixed-size, copy-on-write disk; the default shared pool has none |
|
|
151
167
|
| Separate-kernel VM isolation | ✅ | ❌ | Container runtime |
|
|
152
168
|
| Named environments and managed secrets | ✅ | ❌ | Manual configuration |
|
|
153
169
|
| Organizations and webhooks | ✅ | ❌ | Raft is single-operator; no event API |
|
|
154
170
|
| Dashboard, remote API and SDK | ✅ | ❌ | CLI only |
|
|
155
171
|
| Managed agent conversations | ✅ | ❌ | Not implemented |
|
|
156
|
-
| Automatic deletion and retention | ✅ | ✅ | Disposable deletion;
|
|
157
|
-
| CLI self-update | ✅ |
|
|
172
|
+
| Automatic deletion and retention | ✅ | ✅ | Disposable deletion, optionally by default; per-host retention deletes boxes stopped for N hours; age-based prune with preview |
|
|
173
|
+
| CLI self-update | ✅ | ✅ | Explicit `raft self-update` for npm and uv installs plus a daily stderr notice. Boat also updates itself after commands; Raft only notifies |
|
|
158
174
|
|
|
159
175
|
## verification and development
|
|
160
176
|
|
package/deploy/deploy.py
CHANGED
|
@@ -15,7 +15,7 @@ import sys
|
|
|
15
15
|
|
|
16
16
|
ROOT = Path(__file__).resolve().parent.parent
|
|
17
17
|
sys.path.insert(0, str(ROOT))
|
|
18
|
-
from raft import configuration # noqa: E402
|
|
18
|
+
from raft import BOX_DISK_POOL, configuration, gib, gib_size # noqa: E402
|
|
19
19
|
|
|
20
20
|
RELEASES = "https://github.com/Microck/raft/releases/download/"
|
|
21
21
|
# Matches the user agent install-incus.sh sends for its own downloads.
|
|
@@ -134,7 +134,86 @@ def import_archive(host, archive, architecture):
|
|
|
134
134
|
return checked_image(host, fingerprint, architecture)
|
|
135
135
|
|
|
136
136
|
|
|
137
|
-
def
|
|
137
|
+
def free_disk(host):
|
|
138
|
+
return int(
|
|
139
|
+
remote(
|
|
140
|
+
host,
|
|
141
|
+
shlex.join(
|
|
142
|
+
[
|
|
143
|
+
"python3",
|
|
144
|
+
"-c",
|
|
145
|
+
"import shutil; print(shutil.disk_usage('/var/lib/incus').free)",
|
|
146
|
+
]
|
|
147
|
+
),
|
|
148
|
+
)
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def box_disk_pool(host, pools, size):
|
|
153
|
+
"""Create or verify the thin LVM pool that holds own box disks.
|
|
154
|
+
|
|
155
|
+
Own box disks are opt-in: without --disk-pool a host has no raft-disks pool,
|
|
156
|
+
which is a normal state, and --disk-pool on a later redeploy adds it. Thin
|
|
157
|
+
volumes make forks, templates and snapshots of own-disk boxes snapshots that
|
|
158
|
+
share blocks. Raft never overcommits the pool: every write path that adds a
|
|
159
|
+
thin volume checks that all virtual sizes still fit its data space (see
|
|
160
|
+
BOX_DISK_ACCOUNTING in raft.py), so the pool cannot fill.
|
|
161
|
+
"""
|
|
162
|
+
pool = next((item for item in pools if item["name"] == BOX_DISK_POOL), None)
|
|
163
|
+
if pool is None:
|
|
164
|
+
if size is None:
|
|
165
|
+
return
|
|
166
|
+
# Same 20 GiB margin as the fresh-host check for the 60 GiB shared pool.
|
|
167
|
+
if free_disk(host) < (gib(size) + 20) * 1024**3:
|
|
168
|
+
raise RuntimeError(
|
|
169
|
+
f"Creating the {size} {BOX_DISK_POOL} pool needs {gib(size) + 20} GiB free disk; "
|
|
170
|
+
"pass a smaller --disk-pool or free host disk"
|
|
171
|
+
)
|
|
172
|
+
remote(
|
|
173
|
+
host,
|
|
174
|
+
f"sudo -n incus storage create {BOX_DISK_POOL} lvm size={size} lvm.use_thinpool=true",
|
|
175
|
+
)
|
|
176
|
+
pool = json.loads(remote(host, f"sudo -n incus query /1.0/storage-pools/{BOX_DISK_POOL}"))
|
|
177
|
+
config = pool["config"]
|
|
178
|
+
if pool["driver"] != "lvm" or config.get("lvm.use_thinpool") != "true":
|
|
179
|
+
raise RuntimeError(
|
|
180
|
+
f"{BOX_DISK_POOL} must be a thin LVM pool (lvm.use_thinpool=true); inspect it and "
|
|
181
|
+
"back up its boxes before explicit storage recovery"
|
|
182
|
+
)
|
|
183
|
+
if size is not None and config.get("size") != size:
|
|
184
|
+
raise RuntimeError(
|
|
185
|
+
f"{BOX_DISK_POOL} has size={config.get('size')}, not {size}. "
|
|
186
|
+
"Deployment does not resize existing storage; omit --disk-pool to keep it"
|
|
187
|
+
)
|
|
188
|
+
# The thin pool frees blocks that boxes discard, but must not pass discards on
|
|
189
|
+
# to its loop file: that would punch holes in the file, and a later write into
|
|
190
|
+
# a hole could find the host disk full and fail every own-disk box at once.
|
|
191
|
+
# Changing passdown to nopassdown is allowed while the pool is active.
|
|
192
|
+
thinpool = f"{config['lvm.vg_name']}/{config['lvm.thinpool_name']}"
|
|
193
|
+
discards = remote(host, f"sudo -n lvs --noheadings -o discards {thinpool}").strip()
|
|
194
|
+
if discards != "nopassdown":
|
|
195
|
+
remote(host, f"sudo -n lvchange --discards nopassdown {thinpool}")
|
|
196
|
+
# Incus creates the pool's loop file sparse. Allocating its full length makes
|
|
197
|
+
# the host disk hold the whole pool; fallocate only fills holes and never
|
|
198
|
+
# changes the file's length or data, so repeating it is a no-op. A pool an
|
|
199
|
+
# operator placed on a dedicated block device has no file to allocate.
|
|
200
|
+
remote(
|
|
201
|
+
host,
|
|
202
|
+
shlex.join(
|
|
203
|
+
[
|
|
204
|
+
"sudo",
|
|
205
|
+
"-n",
|
|
206
|
+
"sh",
|
|
207
|
+
"-c",
|
|
208
|
+
'test ! -f "$1" || fallocate -l "$(stat -c %s "$1")" "$1"',
|
|
209
|
+
"raft",
|
|
210
|
+
config["source"],
|
|
211
|
+
]
|
|
212
|
+
),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def deploy(name, host, disk_pool=None):
|
|
138
217
|
host_architecture(host)
|
|
139
218
|
copy(host, ROOT / "deploy/install-incus.sh", "/tmp/raft-install-incus.sh")
|
|
140
219
|
print(remote(host, "bash /tmp/raft-install-incus.sh </dev/null"), flush=True)
|
|
@@ -149,24 +228,29 @@ def deploy(name, host):
|
|
|
149
228
|
if project.get("description") != "Raft managed workspaces":
|
|
150
229
|
raise RuntimeError("Existing raft project needs ownership verification")
|
|
151
230
|
else:
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
[
|
|
161
|
-
"python3",
|
|
162
|
-
"-c",
|
|
163
|
-
"import shutil; print(shutil.disk_usage('/var/lib/incus').free)",
|
|
164
|
-
]
|
|
231
|
+
leftovers = [
|
|
232
|
+
name
|
|
233
|
+
for name, found in [
|
|
234
|
+
("storage pool raft-data", pool is not None),
|
|
235
|
+
("network rfbr0", network is not None),
|
|
236
|
+
(
|
|
237
|
+
f"storage pool {BOX_DISK_POOL}",
|
|
238
|
+
any(item["name"] == BOX_DISK_POOL for item in pools),
|
|
165
239
|
),
|
|
240
|
+
]
|
|
241
|
+
if found
|
|
242
|
+
]
|
|
243
|
+
if leftovers:
|
|
244
|
+
raise RuntimeError(
|
|
245
|
+
"Raft resource names already exist without an owned project: "
|
|
246
|
+
+ ", ".join(leftovers)
|
|
247
|
+
+ "; review them explicitly"
|
|
166
248
|
)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
249
|
+
# 60 GiB shared pool, 20 GiB for images and the host, and the own-disk
|
|
250
|
+
# pool when --disk-pool enables it.
|
|
251
|
+
needed = 80 + (gib(disk_pool) if disk_pool else 0)
|
|
252
|
+
if free_disk(host) < needed * 1024**3:
|
|
253
|
+
raise RuntimeError(f"Fresh deployment needs at least {needed} GiB free disk")
|
|
170
254
|
remote(
|
|
171
255
|
host,
|
|
172
256
|
"sudo -n incus project create raft -c features.images=true -c features.profiles=true </dev/null",
|
|
@@ -207,6 +291,7 @@ def deploy(name, host):
|
|
|
207
291
|
"Raft pool must have configured size=60GiB; inspect raft-data and back up boxes "
|
|
208
292
|
"before explicit storage recovery. Deployment does not resize existing storage."
|
|
209
293
|
)
|
|
294
|
+
box_disk_pool(host, pools, disk_pool)
|
|
210
295
|
|
|
211
296
|
devices = profile["devices"]
|
|
212
297
|
if (
|
|
@@ -261,9 +346,17 @@ def main():
|
|
|
261
346
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
262
347
|
configured = hosts()
|
|
263
348
|
parser.add_argument("--location", choices=configured)
|
|
349
|
+
parser.add_argument(
|
|
350
|
+
"--disk-pool",
|
|
351
|
+
type=gib_size,
|
|
352
|
+
metavar="SIZE",
|
|
353
|
+
help=f"Enable own box disks: create the {BOX_DISK_POOL} pool with SIZE of host disk "
|
|
354
|
+
"if it is missing (default: leave own box disks as they are); an existing pool must "
|
|
355
|
+
"already have this size",
|
|
356
|
+
)
|
|
264
357
|
args = parser.parse_args()
|
|
265
358
|
for name in [args.location] if args.location else configured:
|
|
266
|
-
deploy(name, configured[name])
|
|
359
|
+
deploy(name, configured[name], args.disk_pool)
|
|
267
360
|
|
|
268
361
|
|
|
269
362
|
if __name__ == "__main__":
|
package/deploy/install-incus.sh
CHANGED
|
@@ -15,6 +15,16 @@ if ! command -v incus >/dev/null; then
|
|
|
15
15
|
sudo apt-get update -qq
|
|
16
16
|
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends incus btrfs-progs
|
|
17
17
|
fi
|
|
18
|
+
# raft-disks, the opt-in pool for own box disks, uses LVM thin provisioning.
|
|
19
|
+
if ! command -v lvcreate >/dev/null; then
|
|
20
|
+
sudo apt-get update -qq
|
|
21
|
+
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends lvm2
|
|
22
|
+
fi
|
|
23
|
+
# raft recover reads the backup index, which Incus writes as YAML.
|
|
24
|
+
if ! python3 -c 'import yaml' 2>/dev/null; then
|
|
25
|
+
sudo apt-get update -qq
|
|
26
|
+
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends python3-yaml
|
|
27
|
+
fi
|
|
18
28
|
if ! command -v nft >/dev/null; then
|
|
19
29
|
sudo apt-get update -qq
|
|
20
30
|
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends nftables
|
package/deploy/raft-expire.py
CHANGED
|
@@ -1,61 +1,139 @@
|
|
|
1
1
|
#!/usr/bin/python3
|
|
2
|
-
"""Enforce running deadlines and preview or delete
|
|
2
|
+
"""Enforce running deadlines and stopped-box retention, and preview or delete
|
|
3
|
+
explicitly selected old boxes."""
|
|
3
4
|
|
|
4
5
|
import argparse
|
|
5
6
|
import json
|
|
7
|
+
import re
|
|
6
8
|
import subprocess
|
|
9
|
+
import sys
|
|
7
10
|
import time
|
|
8
11
|
|
|
9
|
-
|
|
12
|
+
# Checked by CHECK_WORKER in raft.py; bump both together. 2 added retention.
|
|
13
|
+
PROTOCOL = 2
|
|
10
14
|
|
|
11
15
|
PREFIX = ["incus", "--project", "raft"]
|
|
16
|
+
# Stopped-box retention set by `raft policy LOCATION --delete-stopped-after
|
|
17
|
+
# HOURS` on the Incus project, and when it was set. Mirrors RETENTION_KEY in
|
|
18
|
+
# raft.py, whose delete_at() reports the same deadline this worker enforces.
|
|
19
|
+
RETENTION_KEY = "user.raft.delete-stopped-after"
|
|
12
20
|
|
|
13
21
|
|
|
14
22
|
def run(*argv):
|
|
15
23
|
return subprocess.run([*PREFIX, *argv], check=True, capture_output=True, text=True).stdout
|
|
16
24
|
|
|
17
25
|
|
|
18
|
-
def
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
continue
|
|
41
|
-
if args.yes:
|
|
42
|
-
run("delete", name)
|
|
43
|
-
print(json.dumps({"box": name, "action": "deleted" if args.yes else "would-delete"}))
|
|
26
|
+
def retention():
|
|
27
|
+
"""Return the project's (hours, since) retention policy, or None when it is unset."""
|
|
28
|
+
projects = json.loads(run("project", "list", "--format", "json"))
|
|
29
|
+
config = next(item for item in projects if item["name"] == "raft")["config"]
|
|
30
|
+
hours = config.get(RETENTION_KEY, "")
|
|
31
|
+
since = config.get(RETENTION_KEY + ".since", "")
|
|
32
|
+
if not hours:
|
|
33
|
+
return None
|
|
34
|
+
# Raft writes whole numbers; anything else came from a native Incus edit.
|
|
35
|
+
if not re.fullmatch(r"[1-9][0-9]*", hours) or not re.fullmatch(r"[0-9]+", since):
|
|
36
|
+
raise ValueError(
|
|
37
|
+
f"Invalid retention {RETENTION_KEY}={hours!r} .since={since!r}; deleted no stopped "
|
|
38
|
+
"boxes. Fix it with raft policy LOCATION --delete-stopped-after HOURS or never"
|
|
39
|
+
)
|
|
40
|
+
return int(hours), int(since)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def prune(boxes, days, apply, now):
|
|
44
|
+
"""Preview or delete stopped boxes whose recorded stop is older than days."""
|
|
45
|
+
for box in boxes:
|
|
46
|
+
stopped_at = box["config"].get("user.raft.stopped-at")
|
|
47
|
+
if box["status"] != "Stopped" or not stopped_at:
|
|
44
48
|
continue
|
|
45
|
-
|
|
46
|
-
if not deadline or int(deadline) > now:
|
|
49
|
+
if int(stopped_at) > now - days * 86400:
|
|
47
50
|
continue
|
|
51
|
+
if apply:
|
|
52
|
+
run("delete", box["name"])
|
|
53
|
+
print(json.dumps({"box": box["name"], "action": "deleted" if apply else "would-delete"}))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def enforce(box, policy, now):
|
|
57
|
+
"""Apply the running deadline, then the stopped-box retention, to one box."""
|
|
58
|
+
config = box["config"]
|
|
59
|
+
name = box["name"]
|
|
60
|
+
deadline = config.get("user.raft.expires")
|
|
61
|
+
if deadline and int(deadline) <= now:
|
|
48
62
|
if box["status"] not in ["Running", "Stopped"]:
|
|
49
63
|
raise RuntimeError(f"Unexpected state for expired box {name}")
|
|
50
64
|
if config.get("user.raft.disposable") == "true":
|
|
51
65
|
run("delete", name, "--force")
|
|
52
66
|
print(f"Deleted expired disposable box {name}", flush=True)
|
|
53
|
-
|
|
67
|
+
return
|
|
54
68
|
if box["status"] == "Running":
|
|
55
69
|
run("stop", name, "--timeout", "30")
|
|
56
70
|
run("config", "set", name, f"user.raft.stopped-at={now}")
|
|
57
71
|
run("config", "unset", name, "user.raft.expires")
|
|
58
72
|
print(f"Stopped expired box {name}; disk retained", flush=True)
|
|
73
|
+
return
|
|
74
|
+
# Retention never touches a running box. A native incus start keeps the old
|
|
75
|
+
# stop time, so clear it here and the next stop starts a fresh clock. A start
|
|
76
|
+
# and stop between two worker runs still goes unnoticed.
|
|
77
|
+
if box["status"] != "Stopped":
|
|
78
|
+
if box["status"] == "Running" and config.get("user.raft.stopped-at"):
|
|
79
|
+
run("config", "unset", name, "user.raft.stopped-at")
|
|
80
|
+
return
|
|
81
|
+
stopped_at = config.get("user.raft.stopped-at")
|
|
82
|
+
if not stopped_at:
|
|
83
|
+
# Raft records the stop time on stop and expiry. A box stopped any other
|
|
84
|
+
# way (a guest poweroff, a native incus stop, a recovered backup or a
|
|
85
|
+
# failed start) starts its retention clock now. Its real stop time is
|
|
86
|
+
# unknown, and guessing an earlier one could delete it early.
|
|
87
|
+
run("config", "set", name, f"user.raft.stopped-at={now}")
|
|
88
|
+
print(f"Recorded stop time of box {name}, stopped outside Raft", flush=True)
|
|
89
|
+
return
|
|
90
|
+
if policy is None:
|
|
91
|
+
return
|
|
92
|
+
hours, since = policy
|
|
93
|
+
# The clock starts at the later of the stop and the policy change, so a new or
|
|
94
|
+
# shorter policy never deletes a box sooner than its hours after the change.
|
|
95
|
+
if max(int(stopped_at), since) + hours * 3600 > now:
|
|
96
|
+
return
|
|
97
|
+
run("delete", name)
|
|
98
|
+
stopped = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(stopped_at)))
|
|
99
|
+
message = f"Deleted box {name}, stopped since {stopped}, under the {hours}-hour retention"
|
|
100
|
+
# Retention deletes data without a command, so every deletion goes to the host
|
|
101
|
+
# journal beside the policy change that caused it (journalctl -t raft), whether
|
|
102
|
+
# the timer or raft gc ran the worker.
|
|
103
|
+
subprocess.run(["logger", "-t", "raft", message], check=True)
|
|
104
|
+
print(message, flush=True)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main():
|
|
108
|
+
cli = argparse.ArgumentParser(description=__doc__)
|
|
109
|
+
cli.add_argument("--prune-days", type=int)
|
|
110
|
+
cli.add_argument("--yes", action="store_true")
|
|
111
|
+
args = cli.parse_args()
|
|
112
|
+
if args.prune_days is not None and args.prune_days < 1:
|
|
113
|
+
cli.error("Retention must be at least one day")
|
|
114
|
+
if args.yes and args.prune_days is None:
|
|
115
|
+
cli.error("--yes requires --prune-days")
|
|
116
|
+
now = int(time.time())
|
|
117
|
+
# The service, gc and prune all hold the same lifecycle lock. Never select
|
|
118
|
+
# unrelated Incus instances or use creation time as a guessed stop time.
|
|
119
|
+
boxes = [
|
|
120
|
+
box
|
|
121
|
+
for box in json.loads(run("list", "--fast", "--format", "json"))
|
|
122
|
+
if box["config"].get("user.raft") == "true"
|
|
123
|
+
]
|
|
124
|
+
if args.prune_days is not None:
|
|
125
|
+
prune(boxes, args.prune_days, args.yes, now)
|
|
126
|
+
return
|
|
127
|
+
# An invalid policy must not stop deadline enforcement: apply deadlines, skip
|
|
128
|
+
# retention, and fail afterwards so the service run reports the problem.
|
|
129
|
+
try:
|
|
130
|
+
policy, problem = retention(), None
|
|
131
|
+
except ValueError as error:
|
|
132
|
+
policy, problem = None, str(error)
|
|
133
|
+
for box in boxes:
|
|
134
|
+
enforce(box, policy, now)
|
|
135
|
+
if problem:
|
|
136
|
+
sys.exit(problem)
|
|
59
137
|
|
|
60
138
|
|
|
61
139
|
if __name__ == "__main__":
|
|
@@ -9,7 +9,7 @@ import time
|
|
|
9
9
|
import uuid
|
|
10
10
|
|
|
11
11
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
12
|
-
from raft import
|
|
12
|
+
from raft import address, incus, inventory, parse_handle, transaction # noqa: E402
|
|
13
13
|
import importlib.util # noqa: E402
|
|
14
14
|
|
|
15
15
|
spec = importlib.util.spec_from_file_location(
|
|
@@ -32,61 +32,21 @@ def wait_address(box):
|
|
|
32
32
|
raise RuntimeError("Resumed workspace has no managed IPv4 address after 30 seconds")
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def verify_admission(location, directory):
|
|
36
|
-
fixtures = []
|
|
37
|
-
saved = len(inventory(location))
|
|
38
|
-
if saved >= 4:
|
|
39
|
-
raise RuntimeError("Admission test needs at least one free slot on its target")
|
|
40
|
-
try:
|
|
41
|
-
for _ in range(4 - saved):
|
|
42
|
-
box = live.raft(
|
|
43
|
-
"new", "--location", location, "--memory", "1GiB", "--ttl", "600"
|
|
44
|
-
).stdout.strip()
|
|
45
|
-
fixtures.append(box)
|
|
46
|
-
live.wait_running(box)
|
|
47
|
-
live.raft("stop", box)
|
|
48
|
-
rejected = live.raft("new", "--location", location, check=False)
|
|
49
|
-
if rejected.returncode == 0:
|
|
50
|
-
fixtures.append(rejected.stdout.strip())
|
|
51
|
-
assert rejected.returncode != 0 and "already holds four boxes" in rejected.stderr
|
|
52
|
-
rejected = live.raft("fork", fixtures[0], "--ttl", "600", check=False)
|
|
53
|
-
if rejected.returncode == 0:
|
|
54
|
-
fixtures.append(rejected.stdout.strip())
|
|
55
|
-
assert rejected.returncode != 0 and "already holds four boxes" in rejected.stderr
|
|
56
|
-
with tempfile.TemporaryDirectory(dir=directory, prefix="raft-admission-") as work:
|
|
57
|
-
# Admission must reject before native archive parsing, without creating a box.
|
|
58
|
-
archive = Path(work) / "rejected.tar.gz"
|
|
59
|
-
archive.write_bytes(b"not-an-incus-archive")
|
|
60
|
-
rejected = live.raft("recover", str(archive), "--location", location, check=False)
|
|
61
|
-
if rejected.returncode == 0:
|
|
62
|
-
fixtures.append(rejected.stdout.strip())
|
|
63
|
-
assert rejected.returncode != 0 and "already holds four boxes" in rejected.stderr
|
|
64
|
-
assert len(inventory(location)) == 4
|
|
65
|
-
report = json.loads(live.raft("limits", "--location", location, "--json").stdout)
|
|
66
|
-
assert report["savedBoxes"] == 4 and report["savedSlots"] == 0
|
|
67
|
-
assert all(size["newBoxes"] == 0 for size in report["recommendations"])
|
|
68
|
-
print(
|
|
69
|
-
f"{location}: stopped boxes count toward admission; new/fork/recover reject a fifth box",
|
|
70
|
-
flush=True,
|
|
71
|
-
)
|
|
72
|
-
finally:
|
|
73
|
-
for box in reversed(fixtures):
|
|
74
|
-
live.raft("destroy", box)
|
|
75
|
-
|
|
76
|
-
|
|
77
35
|
def verify(source, target, directory, development_image=False):
|
|
78
36
|
fixtures = []
|
|
79
37
|
name = "rf-" + uuid.uuid4().hex[:16]
|
|
80
38
|
box = source + ":" + name
|
|
81
39
|
try:
|
|
82
40
|
if development_image:
|
|
83
|
-
box = live.raft(
|
|
41
|
+
box = live.raft(
|
|
42
|
+
"new", "--location", source, "--ttl", "1800", "--cpu", "1", "--disk", "shared"
|
|
43
|
+
).stdout.strip()
|
|
84
44
|
fixtures.append(box)
|
|
85
45
|
live.wait_running(box)
|
|
86
46
|
else:
|
|
87
47
|
transaction(
|
|
88
48
|
source,
|
|
89
|
-
|
|
49
|
+
'incus --project raft launch images:debian/13 "$1" '
|
|
90
50
|
"-c user.raft=true -c limits.cpu=1 -c limits.memory=1GiB; "
|
|
91
51
|
'incus --project raft config set "$1" user.raft.expires="$(( $(date +%s) + 600 ))"',
|
|
92
52
|
name,
|
|
@@ -172,9 +132,36 @@ def verify(source, target, directory, development_image=False):
|
|
|
172
132
|
print(f"{source}: archive completed ({size} bytes)", flush=True)
|
|
173
133
|
assert live.raft("backup", box, str(archive), check=False).returncode != 0
|
|
174
134
|
assert archive.stat().st_size == size
|
|
175
|
-
|
|
176
|
-
|
|
135
|
+
# The archived box has no note, so a target policy rejects it after import
|
|
136
|
+
# and deletes the imported copy in the same locked transaction.
|
|
137
|
+
live.raft("policy", target, "--require", "purpose,owner")
|
|
138
|
+
try:
|
|
139
|
+
saved = len(inventory(target))
|
|
140
|
+
rejected = live.raft("recover", str(archive), "--location", target, check=False)
|
|
141
|
+
if rejected.returncode == 0:
|
|
142
|
+
fixtures.append(rejected.stdout.strip())
|
|
143
|
+
assert rejected.returncode == 1, rejected
|
|
144
|
+
assert "requires box note fields purpose, owner" in rejected.stderr, rejected
|
|
145
|
+
assert len(inventory(target)) == saved, "Rejected recovery kept a box"
|
|
146
|
+
restored = live.raft(
|
|
147
|
+
"recover",
|
|
148
|
+
str(archive),
|
|
149
|
+
"--location",
|
|
150
|
+
target,
|
|
151
|
+
"--purpose",
|
|
152
|
+
"Recovery verification",
|
|
153
|
+
"--owner",
|
|
154
|
+
"verify-controls",
|
|
155
|
+
).stdout.strip()
|
|
156
|
+
fixtures.append(restored)
|
|
157
|
+
finally:
|
|
158
|
+
live.raft("policy", target, "--require", "none")
|
|
177
159
|
print(f"{target}: recovered stopped workspace", flush=True)
|
|
160
|
+
note = json.loads(live.raft("info", restored).stdout)["config"]["user.raft.note"]
|
|
161
|
+
assert json.loads(note) == {
|
|
162
|
+
"purpose": "Recovery verification",
|
|
163
|
+
"owner": "verify-controls",
|
|
164
|
+
}
|
|
178
165
|
metadata = json.loads(live.raft("info", restored).stdout)
|
|
179
166
|
assert metadata["status"] == "Stopped"
|
|
180
167
|
assert not metadata["config"].get("user.raft.expires")
|
|
@@ -237,5 +224,4 @@ if __name__ == "__main__":
|
|
|
237
224
|
help="Export the complete configured development image rather than a small fixture",
|
|
238
225
|
)
|
|
239
226
|
args = parser.parse_args()
|
|
240
|
-
verify_admission(args.target, args.directory)
|
|
241
227
|
verify(args.source, args.target, args.directory, args.development_image)
|