@agentsdock/server 1.0.7-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +19 -0
- package/README.md +60 -0
- package/npm/cli.cjs +289 -0
- package/package.json +32 -0
- package/server/LICENSE +202 -0
- package/server/NOTICE +19 -0
- package/server/VERSION +1 -0
- package/server/activation_transaction.py +2458 -0
- package/server/agent_server.py +93226 -0
- package/server/agentsdock_chats.py +1237 -0
- package/server/agentsdock_emergency.py +338 -0
- package/server/agentsdock_jobs.py +640 -0
- package/server/agentsdock_mail.py +325 -0
- package/server/agentsdock_publish.py +404 -0
- package/server/agentsdock_team.py +793 -0
- package/server/agentsdock_team_hub/__init__.py +33 -0
- package/server/agentsdock_team_hub/auth.py +710 -0
- package/server/agentsdock_team_hub/cli.py +647 -0
- package/server/agentsdock_team_hub/database.py +310 -0
- package/server/agentsdock_team_hub/mail_hint_streams.py +176 -0
- package/server/agentsdock_team_hub/mail_hints.py +201 -0
- package/server/agentsdock_team_hub/migrations/0001_identity_auth.sql +690 -0
- package/server/agentsdock_team_hub/migrations/0002_teamspace_ledger.sql +1336 -0
- package/server/agentsdock_team_hub/migrations/0003_service_runtime.sql +361 -0
- package/server/agentsdock_team_hub/migrations/0004_managed_host_binding.sql +22 -0
- package/server/agentsdock_team_hub/migrations/0005_tailnet_bootstrap_delegations.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0006_team_network_mailbox.sql +423 -0
- package/server/agentsdock_team_hub/migrations/0007_local_agent_mail.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0008_managed_server_session.sql +73 -0
- package/server/agentsdock_team_hub/migrations/0009_team_messages.sql +324 -0
- package/server/agentsdock_team_hub/migrations/0010_team_attachment_orphan_reclamation.sql +52 -0
- package/server/agentsdock_team_hub/migrations/0011_human_admin_paging.sql +65 -0
- package/server/agentsdock_team_hub/migrations/0012_network_content_deletions.sql +69 -0
- package/server/agentsdock_team_hub/migrations/0013_team_message_revisions.sql +48 -0
- package/server/agentsdock_team_hub/migrations/0014_managed_network_owner.sql +50 -0
- package/server/agentsdock_team_hub/migrations/0015_team_message_inbox_dismissals.sql +3 -0
- package/server/agentsdock_team_hub/migrations/0016_team_message_all_servers.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0017_skill_announcement_deletions.sql +16 -0
- package/server/agentsdock_team_hub/migrations/0018_team_mail_subjects.sql +8 -0
- package/server/agentsdock_team_hub/migrations/0019_team_mailbox_state.sql +5 -0
- package/server/agentsdock_team_hub/migrations/0020_team_mail_arrivals.sql +44 -0
- package/server/agentsdock_team_hub/migrations/0021_team_mail_threads.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0022_team_bulletin_changes.sql +49 -0
- package/server/agentsdock_team_hub/migrations/0023_team_message_search.sql +93 -0
- package/server/agentsdock_team_hub/migrations/__init__.py +1 -0
- package/server/agentsdock_team_hub/notification_hints.py +306 -0
- package/server/agentsdock_team_hub/secure_peer.py +11215 -0
- package/server/agentsdock_team_hub/secure_peer_hub.py +1372 -0
- package/server/agentsdock_team_hub/security.py +318 -0
- package/server/agentsdock_team_hub/service.py +1737 -0
- package/server/agentsdock_team_hub/store.py +16276 -0
- package/server/chat_mailbox.py +448 -0
- package/server/claude_background_reconciliation.py +92 -0
- package/server/claude_goals.py +245 -0
- package/server/claude_history_provenance.py +137 -0
- package/server/claude_history_repair.py +1601 -0
- package/server/claude_sdk_client.py +3731 -0
- package/server/claude_side_question.py +212 -0
- package/server/codex_app_server.py +3030 -0
- package/server/codex_auth.py +93 -0
- package/server/codex_history_repair.py +1247 -0
- package/server/codex_provider.py +1120 -0
- package/server/codex_side_question.py +301 -0
- package/server/cursor_agent_client.py +654 -0
- package/server/cursor_process_guard.py +347 -0
- package/server/execution_activation.py +610 -0
- package/server/execution_control.py +233 -0
- package/server/execution_durability.py +184 -0
- package/server/execution_http.py +256 -0
- package/server/execution_install.py +792 -0
- package/server/execution_legacy_runner.py +406 -0
- package/server/execution_maintenance.py +137 -0
- package/server/execution_manage.py +615 -0
- package/server/execution_ownership.py +83 -0
- package/server/execution_preparation.py +362 -0
- package/server/execution_recovery.py +845 -0
- package/server/execution_recovery_status.py +195 -0
- package/server/execution_service.py +387 -0
- package/server/execution_transport.py +622 -0
- package/server/execution_uninstall.py +406 -0
- package/server/execution_update_status.py +49 -0
- package/server/install.sh +7074 -0
- package/server/interactive_chat_controls.py +153 -0
- package/server/interactive_chat_native.py +66 -0
- package/server/interactive_chat_projection.py +152 -0
- package/server/interactive_chat_runtime.py +132 -0
- package/server/interactive_chat_share_routes.py +534 -0
- package/server/interactive_chat_share_web.py +12 -0
- package/server/interactive_chat_shares.py +238 -0
- package/server/provider_commands.py +425 -0
- package/server/public_chat_share_routes.py +426 -0
- package/server/public_chat_shares.py +759 -0
- package/server/public_chat_transcript.py +357 -0
- package/server/pyproject.toml +29 -0
- package/server/release-public-key.pem +3 -0
- package/server/secure_peer_delivery.py +1236 -0
- package/server/secure_peer_runtime.py +7694 -0
- package/server/shared_chat_video_stream.py +262 -0
- package/server/shared_chat_videos.py +155 -0
- package/server/side_questions.py +594 -0
- package/server/team_hub_host.py +1354 -0
- package/server/team_mail_grants.py +136 -0
- package/server/team_mail_runtime.py +302 -0
- package/server/team_mail_websocket.py +211 -0
- package/server/title_generation.py +333 -0
- package/server/uninstall.sh +359 -0
- package/server/update_handoff.py +237 -0
- package/server/update_preparation.py +264 -0
- package/server/update_recovery.py +140 -0
- package/server/update_runner.py +1819 -0
- package/server/uv.lock +1101 -0
- package/server/workspace_git.py +545 -0
|
@@ -0,0 +1,610 @@
|
|
|
1
|
+
"""Split-service operations owned by install.sh's existing activation journal.
|
|
2
|
+
|
|
3
|
+
There is deliberately no independent commit, rollback, state snapshot, or lock.
|
|
4
|
+
The outer installer retains its authenticated health/Hub/secure-peer boundaries.
|
|
5
|
+
Every mutating operation requires its owned outer transaction and install lock.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
import fcntl
|
|
12
|
+
import hashlib
|
|
13
|
+
import ast
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
import re
|
|
18
|
+
import shlex
|
|
19
|
+
import subprocess
|
|
20
|
+
import stat
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import activation_transaction as activation
|
|
24
|
+
import execution_install as files
|
|
25
|
+
from execution_manage import NativeServices, WorkerControl
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def environment_file(path: Path) -> dict[str, str]:
|
|
29
|
+
data = activation._read_private(path, maximum=activation.MAX_CONFIG_BYTES)
|
|
30
|
+
result = {}
|
|
31
|
+
for line in data.decode().splitlines():
|
|
32
|
+
if not line.strip() or line.lstrip().startswith("#"):
|
|
33
|
+
continue
|
|
34
|
+
key, separator, value = line.partition("=")
|
|
35
|
+
if not separator or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None:
|
|
36
|
+
raise ValueError("runtime environment is not a supported assignment file")
|
|
37
|
+
# Generated paths are historically unquoted, including spaces. Only
|
|
38
|
+
# decode explicit quoting; never evaluate shell substitutions or code.
|
|
39
|
+
if value.startswith(('"', "'")):
|
|
40
|
+
parsed = shlex.split(value)
|
|
41
|
+
if len(parsed) != 1:
|
|
42
|
+
raise ValueError("runtime environment quoting is invalid")
|
|
43
|
+
value = parsed[0]
|
|
44
|
+
result[key] = value
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def command_layout(args: argparse.Namespace, *, environment: bool = False) -> files.ExecutionLayout:
|
|
49
|
+
return files.ExecutionLayout(
|
|
50
|
+
Path(args.root), Path(args.config_root), Path(args.state_root), Path(args.home), args.platform,
|
|
51
|
+
Path(args.release_dir), Path(args.release_dir), args.bind, args.port,
|
|
52
|
+
environment_file(Path(args.config_root) / "env") if environment else {})
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def transaction(args: argparse.Namespace) -> dict[str, Any]:
|
|
56
|
+
root = Path(args.root)
|
|
57
|
+
value = activation.execution_context(root)
|
|
58
|
+
if value is None:
|
|
59
|
+
raise RuntimeError("outer transaction does not authorize split service changes")
|
|
60
|
+
layout = command_layout(args)
|
|
61
|
+
if (value["service_path"] != str(layout.service_path("worker"))
|
|
62
|
+
or value["gateway_path"] != str(layout.service_path("gateway"))
|
|
63
|
+
or value["env_path"] != str(layout.config_root / "env")
|
|
64
|
+
or value["execution"]["runtime_dir"] != str(layout.runtime_dir)
|
|
65
|
+
or value["release_dir"] != str(layout.worker_release)):
|
|
66
|
+
raise RuntimeError("split service invocation differs from the outer transaction")
|
|
67
|
+
assert_install_lock(root)
|
|
68
|
+
return value
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def assert_install_lock(root: Path) -> None:
|
|
72
|
+
lock = root / ".install-lock"
|
|
73
|
+
files._owned_directory(lock, private=True)
|
|
74
|
+
pid_data = activation._read_owned_regular(lock / "pid", maximum=32)
|
|
75
|
+
if re.fullmatch(rb"[1-9][0-9]*\n?", pid_data) is None:
|
|
76
|
+
raise RuntimeError("installer lock identity is invalid")
|
|
77
|
+
owner = int(pid_data)
|
|
78
|
+
parent = os.getppid()
|
|
79
|
+
for _ in range(8):
|
|
80
|
+
if parent == owner or parent <= 1:
|
|
81
|
+
break
|
|
82
|
+
parent = int(subprocess.check_output(
|
|
83
|
+
["/bin/ps", "-p", str(parent), "-o", "ppid="], text=True).strip())
|
|
84
|
+
if parent != owner:
|
|
85
|
+
# run_without_server_secrets may introduce a short-lived shell, but
|
|
86
|
+
# the helper must still descend from the exact lock-owning installer.
|
|
87
|
+
raise RuntimeError("split operation is not a child of the owning installer")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def owned_args(value: dict[str, Any], root: Path) -> argparse.Namespace:
|
|
91
|
+
return argparse.Namespace(root=str(root), current=str(root / "current"), previous=str(root / "previous"),
|
|
92
|
+
env=value["env_path"], service=value["service_path"], release_dir=value["release_dir"],
|
|
93
|
+
release_version=value["release_version"], transaction_id=value["transaction_id"])
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def recovery_state(args: argparse.Namespace, value: dict[str, Any], *, native=None) -> str:
|
|
97
|
+
"""Prove an activation failed before any native takeover or recovery owner."""
|
|
98
|
+
phase = value.get("rollback_from") or value["phase"]
|
|
99
|
+
if phase not in {"prepared", "guarded"}:
|
|
100
|
+
return "required"
|
|
101
|
+
root = files._path(args.root)
|
|
102
|
+
files._owned_directory(root, private=True)
|
|
103
|
+
# The generic rolling-back validator deliberately permits mixed links and
|
|
104
|
+
# observed configurations. This exemption requires the stricter original
|
|
105
|
+
# pre-takeover view even while that rollback is being resumed.
|
|
106
|
+
activation._validate_invocation(owned_args(value, root), {**value, "phase": phase},
|
|
107
|
+
allow_missing_candidate=True)
|
|
108
|
+
from execution_recovery import DIRECTORY, RecoveryService, _service_path
|
|
109
|
+
parent = root / DIRECTORY
|
|
110
|
+
if parent.exists() or parent.is_symlink():
|
|
111
|
+
files._path(parent)
|
|
112
|
+
files._owned_directory(parent, private=True)
|
|
113
|
+
owner = parent / value["transaction_id"]
|
|
114
|
+
if owner.exists() or owner.is_symlink():
|
|
115
|
+
return "required"
|
|
116
|
+
registration = {"transaction_id": value["transaction_id"], "platform": args.platform, "home": args.home}
|
|
117
|
+
path = files._path(_service_path(registration))
|
|
118
|
+
if path.exists() or path.is_symlink():
|
|
119
|
+
return "required"
|
|
120
|
+
if args.platform == "Linux":
|
|
121
|
+
wants = path.parent / "default.target.wants"
|
|
122
|
+
if wants.exists() or wants.is_symlink():
|
|
123
|
+
files._path(wants)
|
|
124
|
+
files._owned_directory(wants)
|
|
125
|
+
enabled = wants / path.name
|
|
126
|
+
if enabled.exists() or enabled.is_symlink():
|
|
127
|
+
return "required"
|
|
128
|
+
(native or RecoveryService(registration)).assert_absent(path)
|
|
129
|
+
return "unarmed"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def publish(args: argparse.Namespace, value: dict[str, Any]) -> None:
|
|
133
|
+
layout = command_layout(args, environment=True)
|
|
134
|
+
layout.validate()
|
|
135
|
+
files.prepare_runtime(layout)
|
|
136
|
+
bodies = {"service": files.render_service(layout, "worker"),
|
|
137
|
+
"gateway": files.render_service(layout, "gateway"),
|
|
138
|
+
"execution_layout": files._json_bytes(files.layout_manifest(layout))}
|
|
139
|
+
for kind, payload in bodies.items():
|
|
140
|
+
# Re-read after each publication; do not reuse stale desired coordinates.
|
|
141
|
+
_directory, current = activation._read_manifest(layout.install_root)
|
|
142
|
+
destination = Path(current[f"{kind}_path"])
|
|
143
|
+
files._ensure_directory(destination.parent)
|
|
144
|
+
source, _secured = activation._config_staging_paths(destination, current["transaction_id"], kind)
|
|
145
|
+
mode = 0o644 if layout.platform == "Linux" and kind != "execution_layout" else 0o600
|
|
146
|
+
desired = current[f"desired_{kind}"]
|
|
147
|
+
if desired and activation._config_path_matches_desired(destination, desired):
|
|
148
|
+
if activation._read_owned_regular(destination, maximum=activation.MAX_CONFIG_BYTES) != payload:
|
|
149
|
+
raise RuntimeError("split candidate configuration changed during publication")
|
|
150
|
+
continue
|
|
151
|
+
if not source.exists() and not source.is_symlink() and desired is None:
|
|
152
|
+
activation._write_new_private(source, payload)
|
|
153
|
+
elif source.exists():
|
|
154
|
+
if activation._read_private(source, maximum=activation.MAX_CONFIG_BYTES) != payload:
|
|
155
|
+
raise RuntimeError("split candidate configuration staging changed")
|
|
156
|
+
call = owned_args(current, layout.install_root)
|
|
157
|
+
call.kind, call.source, call.mode = kind, str(source), oct(mode)[2:]
|
|
158
|
+
activation.replace_config(call)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def verify_components(args: argparse.Namespace, value: dict[str, Any], services: NativeServices,
|
|
162
|
+
control: WorkerControl) -> None:
|
|
163
|
+
# The outer installer obtained this response over its native-PID-pinned
|
|
164
|
+
# connection, before transmitting the preserved administrator credential.
|
|
165
|
+
health = json.loads(activation._read_private(Path(args.health_file), maximum=1024 * 1024))
|
|
166
|
+
layout = command_layout(args)
|
|
167
|
+
expected_worker_version = expected_gateway_version = value["release_version"]
|
|
168
|
+
expected_worker_root = value["release_dir"]
|
|
169
|
+
if args.command == "verify-restored":
|
|
170
|
+
if not value["execution_layout"]["existed"]:
|
|
171
|
+
return # authenticated legacy health remains the outer boundary
|
|
172
|
+
old_worker = activation._locate_release(value["execution"]["old_worker_release"])
|
|
173
|
+
old_gateway = activation._locate_release(value["old_release"])
|
|
174
|
+
if old_worker is None or old_gateway is None:
|
|
175
|
+
raise RuntimeError("restored execution generations are not retained")
|
|
176
|
+
expected_worker_root = str(old_worker)
|
|
177
|
+
expected_worker_version = activation._read_owned_regular(old_worker / "VERSION").decode().strip()
|
|
178
|
+
expected_gateway_version = activation._read_owned_regular(old_gateway / "VERSION").decode().strip()
|
|
179
|
+
record = control.worker_record(layout)
|
|
180
|
+
native = services.snapshot()
|
|
181
|
+
gateway, worker = health.get("gateway"), health.get("execution_service")
|
|
182
|
+
if (not isinstance(gateway, dict) or not isinstance(worker, dict)
|
|
183
|
+
or gateway.get("version") != expected_gateway_version
|
|
184
|
+
or worker.get("version") != expected_worker_version
|
|
185
|
+
or gateway.get("protocol") != files.PROTOCOL_VERSION
|
|
186
|
+
or worker.get("protocol") != files.PROTOCOL_VERSION
|
|
187
|
+
or record.get("release_root") != expected_worker_root
|
|
188
|
+
or record["pid"] != worker.get("pid")
|
|
189
|
+
or record["instance_id"] != worker.get("instance_id")
|
|
190
|
+
or native["worker"].get("pid") != worker.get("pid")
|
|
191
|
+
or native["gateway"].get("pid") != gateway.get("pid")):
|
|
192
|
+
raise RuntimeError("candidate gateway and worker do not match the paired release")
|
|
193
|
+
_record, status = control.status(layout)
|
|
194
|
+
lease = status.get("lease")
|
|
195
|
+
if value["phase"] != "committed" and (
|
|
196
|
+
not isinstance(lease, dict) or not lease.get("sealed")
|
|
197
|
+
or lease.get("operation_id") != value["execution"]["operation_id"]
|
|
198
|
+
):
|
|
199
|
+
raise RuntimeError("candidate worker is not held by this activation")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def candidate_api_contract(candidate: Path) -> int:
|
|
203
|
+
source = activation._read_owned_regular(candidate / "agent_server.py", maximum=16 * 1024 * 1024)
|
|
204
|
+
values = [node.value.value for node in ast.parse(source).body
|
|
205
|
+
if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name)
|
|
206
|
+
and target.id == "API_CONTRACT_VERSION" for target in node.targets)
|
|
207
|
+
and isinstance(node.value, ast.Constant)]
|
|
208
|
+
if len(values) != 1 or type(values[0]) is not int or values[0] < 1:
|
|
209
|
+
raise RuntimeError("candidate API contract is not one positive literal integer")
|
|
210
|
+
return values[0]
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def admitted_update_id(args: argparse.Namespace, status: dict[str, Any]) -> str:
|
|
214
|
+
identifier = status.get("update_id")
|
|
215
|
+
if args.managed_update_id:
|
|
216
|
+
if identifier != args.managed_update_id:
|
|
217
|
+
raise RuntimeError("installer update identity differs from admitted status")
|
|
218
|
+
return identifier
|
|
219
|
+
# Old macOS detached runners did not forward this ID for ordinary updates.
|
|
220
|
+
# Derive it only when this installer descends from the exact admitted runner.
|
|
221
|
+
runner = status.get("runner_pid")
|
|
222
|
+
if (args.platform == "Darwin" and "runner_pid" not in status
|
|
223
|
+
and re.fullmatch(r"[0-9a-f]{32}", str(identifier)) is not None):
|
|
224
|
+
# Pre-PID legacy runners require their full native process/session proof.
|
|
225
|
+
# Never downgrade a supplied but invalid runner identity to this path.
|
|
226
|
+
from execution_legacy_runner import verify_legacy_runner
|
|
227
|
+
return verify_legacy_runner(args, status)
|
|
228
|
+
if type(runner) is not int or runner <= 1 or re.fullmatch(r"[0-9a-f]{32}", str(identifier)) is None:
|
|
229
|
+
raise RuntimeError("legacy installer has no admitted runner authority")
|
|
230
|
+
parent = os.getppid()
|
|
231
|
+
for _ in range(10):
|
|
232
|
+
if parent == runner:
|
|
233
|
+
return identifier
|
|
234
|
+
if parent <= 1:
|
|
235
|
+
break
|
|
236
|
+
parent = int(subprocess.check_output(["/bin/ps", "-p", str(parent), "-o", "ppid="], text=True).strip())
|
|
237
|
+
raise RuntimeError("legacy installer is not descended from the admitted updater")
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def require_retired_legacy_intent(layout: files.ExecutionLayout, previous: Any, intent: dict) -> None:
|
|
241
|
+
"""Permit an old updater's carried-over intent only after proven retirement.
|
|
242
|
+
|
|
243
|
+
This reads historical evidence without loading or executing its bootstrap.
|
|
244
|
+
A retired owner's payload may predate additional bootstrap modules.
|
|
245
|
+
"""
|
|
246
|
+
import execution_recovery as recovery
|
|
247
|
+
from execution_preparation import _check_binding
|
|
248
|
+
from execution_recovery_status import INTENT_KEYS
|
|
249
|
+
|
|
250
|
+
root = layout.install_root
|
|
251
|
+
def no_journal():
|
|
252
|
+
for name in (".activation-transaction", ".execution-transaction", ".execution-uninstall.json"):
|
|
253
|
+
if (root / name).exists() or (root / name).is_symlink():
|
|
254
|
+
raise RuntimeError("a lifecycle journal still owns the previous recovery intent")
|
|
255
|
+
no_journal()
|
|
256
|
+
if (not isinstance(previous, dict) or set(previous) != INTENT_KEYS
|
|
257
|
+
or type(previous["format"]) is not int or previous["format"] != 1
|
|
258
|
+
or previous["root"] != str(root) or previous["server_identity"] != intent["server_identity"]
|
|
259
|
+
or not isinstance(previous["version"], str) or not previous["version"]
|
|
260
|
+
or type(previous["api_contract"]) is not int or previous["api_contract"] < 1
|
|
261
|
+
or re.fullmatch(r"[0-9a-f]{32}", str(previous["update_id"])) is None
|
|
262
|
+
or previous["update_id"] == intent["update_id"]):
|
|
263
|
+
raise RuntimeError("previous recovery intent is not a different admitted update")
|
|
264
|
+
_check_binding(root, previous["root_binding"])
|
|
265
|
+
parent = root / recovery.DIRECTORY
|
|
266
|
+
files._path(parent)
|
|
267
|
+
if not parent.exists() and not parent.is_symlink():
|
|
268
|
+
no_journal()
|
|
269
|
+
return # Pre-arm failure: no recovery owner was ever published.
|
|
270
|
+
files._owned_directory(parent, private=True)
|
|
271
|
+
directories = list(parent.iterdir())
|
|
272
|
+
if len(directories) > 4096:
|
|
273
|
+
raise RuntimeError("too many historical recovery owners")
|
|
274
|
+
digest = hashlib.sha256(files._json_bytes(previous)).hexdigest()
|
|
275
|
+
binding = {"format": 1, "path": str(layout.state_root / "admin/server-update.json"),
|
|
276
|
+
"update_id": previous["update_id"], "target_version": previous["version"],
|
|
277
|
+
"intent_sha256": digest}
|
|
278
|
+
matched = 0
|
|
279
|
+
for entry in directories:
|
|
280
|
+
directory = recovery._directory(root, entry.name)
|
|
281
|
+
owner = recovery._json(directory / "owner.json")
|
|
282
|
+
if (set(owner) != recovery.OWNER_KEYS or type(owner["format"]) is not int or owner["format"] != 1
|
|
283
|
+
or owner["root"] != str(root) or owner["transaction_id"] != entry.name
|
|
284
|
+
or owner["expected_server_identity"] != previous["server_identity"]
|
|
285
|
+
or owner["home"] != str(layout.home) or owner["platform"] != layout.platform
|
|
286
|
+
or owner["state_root"] != str(layout.state_root) or owner["config_root"] != str(layout.config_root)):
|
|
287
|
+
raise RuntimeError("historical recovery owner belongs to another installation")
|
|
288
|
+
_check_binding(root, owner["root_binding"])
|
|
289
|
+
previous_owner = (owner["managed_update_id"] == previous["update_id"]
|
|
290
|
+
or (isinstance(owner["status_binding"], dict)
|
|
291
|
+
and owner["status_binding"].get("update_id") == previous["update_id"]))
|
|
292
|
+
if previous_owner and (owner["status_binding"] != binding
|
|
293
|
+
or owner["root_binding"] != previous["root_binding"]
|
|
294
|
+
or owner["source_binding"] != previous["candidate_binding"]
|
|
295
|
+
or owner["managed_update_id"] != previous["update_id"]
|
|
296
|
+
or owner["version"] != previous["version"] or owner["api_contract"] != previous["api_contract"]):
|
|
297
|
+
raise RuntimeError("retired recovery owner differs from its admitted intent")
|
|
298
|
+
payload = owner["payload"]
|
|
299
|
+
required = set(recovery.PAYLOAD) - {"execution_http.py"}
|
|
300
|
+
if not isinstance(payload, dict) or not required <= set(payload) <= set(recovery.PAYLOAD):
|
|
301
|
+
raise RuntimeError("historical recovery bootstrap inventory is invalid")
|
|
302
|
+
for name, expected in payload.items():
|
|
303
|
+
if recovery._digest(directory / name) != expected:
|
|
304
|
+
raise RuntimeError("historical recovery bootstrap changed")
|
|
305
|
+
terminal = recovery._json(directory / "terminal.json")
|
|
306
|
+
snapshot = terminal.get("snapshot")
|
|
307
|
+
health = snapshot.get("health") if isinstance(snapshot, dict) else None
|
|
308
|
+
final = {"format": 1, "transaction_id": entry.name,
|
|
309
|
+
"terminal_sha256": recovery._digest(directory / "terminal.json")}
|
|
310
|
+
if (recovery._json(directory / "finalized.json") != final
|
|
311
|
+
or recovery._json(directory / "retired.json") != final
|
|
312
|
+
or set(terminal) != {"format", "transaction_id", "phase", "snapshot"}
|
|
313
|
+
or type(terminal["format"]) is not int or terminal["format"] != 1
|
|
314
|
+
or terminal["transaction_id"] != entry.name or terminal["phase"] != "rollback-healthy"
|
|
315
|
+
or not isinstance(health, dict) or health.get("server_identity") != previous["server_identity"]):
|
|
316
|
+
raise RuntimeError("previous recovery intent lacks finalized rollback proof")
|
|
317
|
+
service = recovery._service_path(owner)
|
|
318
|
+
if service.exists() or service.is_symlink() or recovery.RecoveryService(owner).running(service):
|
|
319
|
+
raise RuntimeError("previous recovery owner is still registered or running")
|
|
320
|
+
matched += int(previous_owner)
|
|
321
|
+
if matched > 1:
|
|
322
|
+
raise RuntimeError("previous recovery intent has no unique retired owner")
|
|
323
|
+
# No matching owner is an interrupted pre-arm attempt. Every historical
|
|
324
|
+
# owner above must nevertheless be terminal and absent from native jobs;
|
|
325
|
+
# partial directories, unretired owners and any journal fail closed.
|
|
326
|
+
no_journal()
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
@contextmanager
|
|
330
|
+
def legacy_process(args: argparse.Namespace, layout: files.ExecutionLayout,
|
|
331
|
+
services: NativeServices, control: WorkerControl):
|
|
332
|
+
"""Classify the incumbent without treating abandoned metadata as a process.
|
|
333
|
+
|
|
334
|
+
A rolled-back monolith does not manage split-process receipts. Only a dead
|
|
335
|
+
same-install receipt, an unheld worker lease, and native-authenticated legacy
|
|
336
|
+
health allow that receipt to be ignored. Nothing here removes it or contacts
|
|
337
|
+
its callback. The caller must still prove the complete legacy admission.
|
|
338
|
+
"""
|
|
339
|
+
if files.active_worker_release(layout.install_root) is not None:
|
|
340
|
+
yield False
|
|
341
|
+
return
|
|
342
|
+
path = layout.runtime_dir / "worker.json"
|
|
343
|
+
before = services.snapshot()["worker"]
|
|
344
|
+
if not path.exists() and not path.is_symlink():
|
|
345
|
+
yield True
|
|
346
|
+
if path.exists() or path.is_symlink() or files.active_worker_release(layout.install_root) is not None:
|
|
347
|
+
raise RuntimeError("execution ownership changed during legacy admission")
|
|
348
|
+
return
|
|
349
|
+
if not args.health_file:
|
|
350
|
+
raise RuntimeError("stale worker receipt requires authenticated legacy health")
|
|
351
|
+
health_bytes = activation._read_private(Path(args.health_file), maximum=1024 * 1024)
|
|
352
|
+
health = json.loads(health_bytes)
|
|
353
|
+
if (not isinstance(health, dict) or any(health.get(key) is not None for key in ("execution_service", "gateway"))
|
|
354
|
+
or args.expected_native_pid != before.get("pid")
|
|
355
|
+
or not args.expected_server_identity or health.get("server_identity") != args.expected_server_identity):
|
|
356
|
+
raise RuntimeError("worker receipt is not associated with authenticated legacy health")
|
|
357
|
+
files._path(layout.runtime_dir)
|
|
358
|
+
files._owned_directory(layout.runtime_dir, private=True)
|
|
359
|
+
directory = layout.runtime_dir.stat()
|
|
360
|
+
record = control.worker_record(layout)
|
|
361
|
+
if (type(record.get("schema")) is not int or record["schema"] != 1
|
|
362
|
+
or type(record.get("protocol")) is not int or record["protocol"] != files.PROTOCOL_VERSION
|
|
363
|
+
or type(record.get("pid")) is not int or record["pid"] <= 1
|
|
364
|
+
or re.fullmatch(r"[0-9a-f]{32}", str(record.get("instance_id"))) is None
|
|
365
|
+
or not isinstance(record.get("version"), str) or not record["version"]
|
|
366
|
+
or record.get("public_bind") != layout.bind
|
|
367
|
+
or type(record.get("public_port")) is not int or record["public_port"] != layout.port):
|
|
368
|
+
raise RuntimeError("abandoned worker receipt is invalid")
|
|
369
|
+
receipt_bytes, _mode = files._read_file(path, private=True)
|
|
370
|
+
receipt = path.lstat()
|
|
371
|
+
lock_path = layout.runtime_dir / "worker.lock"
|
|
372
|
+
fd = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
373
|
+
try:
|
|
374
|
+
lock = os.fstat(fd)
|
|
375
|
+
if (not stat.S_ISREG(lock.st_mode) or lock.st_uid != os.getuid() or lock.st_nlink != 1
|
|
376
|
+
or stat.S_IMODE(lock.st_mode) != 0o600 or lock.st_size != 0):
|
|
377
|
+
raise RuntimeError("abandoned worker lock is not an owned private lease")
|
|
378
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
379
|
+
|
|
380
|
+
def recheck():
|
|
381
|
+
files._path(layout.runtime_dir)
|
|
382
|
+
files._owned_directory(layout.runtime_dir, private=True)
|
|
383
|
+
for target, expected in ((layout.runtime_dir, directory), (path, receipt), (lock_path, lock)):
|
|
384
|
+
current = target.lstat()
|
|
385
|
+
if (current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_nlink, current.st_size) != (
|
|
386
|
+
expected.st_dev, expected.st_ino, expected.st_mode, expected.st_uid, expected.st_nlink, expected.st_size):
|
|
387
|
+
raise RuntimeError("abandoned worker ownership changed during admission")
|
|
388
|
+
if (files._read_file(path, private=True)[0] != receipt_bytes
|
|
389
|
+
or control.worker_record(layout) != record
|
|
390
|
+
or activation._read_private(Path(args.health_file), maximum=1024 * 1024) != health_bytes
|
|
391
|
+
or files.active_worker_release(layout.install_root) is not None
|
|
392
|
+
or services.snapshot()["worker"] != before):
|
|
393
|
+
raise RuntimeError("legacy worker classification changed during admission")
|
|
394
|
+
try:
|
|
395
|
+
os.kill(record["pid"], 0)
|
|
396
|
+
except ProcessLookupError:
|
|
397
|
+
pass
|
|
398
|
+
else:
|
|
399
|
+
raise RuntimeError("abandoned worker PID is still alive")
|
|
400
|
+
|
|
401
|
+
recheck()
|
|
402
|
+
yield True
|
|
403
|
+
recheck()
|
|
404
|
+
finally:
|
|
405
|
+
os.close(fd)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def seed_legacy_recovery(args: argparse.Namespace, services: NativeServices) -> None:
|
|
409
|
+
layout = command_layout(args)
|
|
410
|
+
if services.snapshot()["worker"]["state"] != "running":
|
|
411
|
+
return
|
|
412
|
+
with legacy_process(args, layout, services, WorkerControl()) as legacy:
|
|
413
|
+
if legacy:
|
|
414
|
+
_seed_legacy_recovery(args, layout)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _seed_legacy_recovery(args: argparse.Namespace, layout: files.ExecutionLayout) -> None:
|
|
418
|
+
from update_recovery import activation_intent
|
|
419
|
+
import update_runner as updates
|
|
420
|
+
status_path = layout.state_root / "admin/server-update.json"
|
|
421
|
+
with updates.server_update_status_lock(status_path):
|
|
422
|
+
status = json.loads(activation._read_private(status_path))
|
|
423
|
+
update_id = admitted_update_id(args, status)
|
|
424
|
+
admitted = json.loads(activation._read_private(Path(args.update_file)))
|
|
425
|
+
if (status.get("phase") not in updates.RUNNER_OWNED_ACTIVE_PHASES
|
|
426
|
+
or admitted.get("update_id") != update_id
|
|
427
|
+
or status.get("target_version") != args.release_version):
|
|
428
|
+
raise RuntimeError("legacy admission changed before recovery intent publication")
|
|
429
|
+
intent = activation_intent(root=layout.install_root, candidate=Path(args.candidate_source),
|
|
430
|
+
version=args.release_version, api_contract=args.api_contract, update_id=update_id,
|
|
431
|
+
server_identity=args.expected_server_identity)
|
|
432
|
+
previous = status.get("_activation_recovery")
|
|
433
|
+
if previous is not None and previous != intent:
|
|
434
|
+
require_retired_legacy_intent(layout, previous, intent)
|
|
435
|
+
updates._update_status_unlocked(status_path, status, _activation_recovery=intent)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def verify_stop(args: argparse.Namespace, value: dict[str, Any], services: NativeServices,
|
|
439
|
+
control: WorkerControl) -> None:
|
|
440
|
+
before = services.snapshot()["worker"]
|
|
441
|
+
if before["state"] != "running":
|
|
442
|
+
return
|
|
443
|
+
layout = command_layout(args)
|
|
444
|
+
with legacy_process(args, layout, services, control) as legacy:
|
|
445
|
+
_verify_stop(args, value, services, control, layout, before, legacy)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _verify_stop(args: argparse.Namespace, value: dict[str, Any], services: NativeServices,
|
|
449
|
+
control: WorkerControl, layout: files.ExecutionLayout, before: dict, legacy: bool) -> None:
|
|
450
|
+
if not legacy:
|
|
451
|
+
record, status = control.status(layout)
|
|
452
|
+
lease = status.get("lease")
|
|
453
|
+
if (record["pid"] != before.get("pid") or not isinstance(lease, dict)
|
|
454
|
+
or lease.get("operation_id") != value["execution"]["operation_id"]
|
|
455
|
+
or lease.get("sealed") is not True):
|
|
456
|
+
raise RuntimeError("running worker has no exact sealed installer handoff")
|
|
457
|
+
handoff = value["execution"]["handoff"]
|
|
458
|
+
# The old epoch must match the server-admitted handoff. A candidate
|
|
459
|
+
# epoch gets its own sealed startup lease from this exact journal.
|
|
460
|
+
if handoff and record.get("release_root") != value["release_dir"] and (
|
|
461
|
+
record["instance_id"] != handoff["worker_instance_id"]
|
|
462
|
+
or lease.get("lease_id") != handoff["lease_id"]
|
|
463
|
+
):
|
|
464
|
+
raise RuntimeError("retiring worker identity changed after admission")
|
|
465
|
+
else:
|
|
466
|
+
if not args.health_file:
|
|
467
|
+
raise RuntimeError("running monolith requires authenticated managed idle handoff")
|
|
468
|
+
health = json.loads(activation._read_private(Path(args.health_file), maximum=1024 * 1024))
|
|
469
|
+
status = json.loads(activation._read_private(layout.state_root / "admin/server-update.json"))
|
|
470
|
+
if not args.update_file:
|
|
471
|
+
raise RuntimeError("legacy admission requires native authenticated update status")
|
|
472
|
+
admitted = json.loads(activation._read_private(Path(args.update_file), maximum=1024 * 1024))
|
|
473
|
+
projected = health.get("server_update")
|
|
474
|
+
active_phases = {"starting", "checking", "downloading", "verifying", "installing", "restarting"}
|
|
475
|
+
queued = health.get("update_blocking_queued_count")
|
|
476
|
+
if queued is None:
|
|
477
|
+
queued = sum(health.get("queued", {}).values())
|
|
478
|
+
if (status.get("phase") not in active_phases or not isinstance(admitted, dict)
|
|
479
|
+
or admitted.get("phase") not in active_phases
|
|
480
|
+
or admitted.get("update_id") != status.get("update_id")
|
|
481
|
+
or admitted.get("server_identity") != health.get("server_identity")
|
|
482
|
+
or (health.get("server_instance_id") and admitted.get("server_instance_id") != health["server_instance_id"])
|
|
483
|
+
or (projected is not None and (not isinstance(projected, dict)
|
|
484
|
+
or projected.get("update_id") != status.get("update_id")))
|
|
485
|
+
or admitted_update_id(args, status) != status.get("update_id")
|
|
486
|
+
or type(health.get("active_count")) is not int or health.get("active_count") != 0
|
|
487
|
+
or type(queued) is not int or queued != 0
|
|
488
|
+
or args.expected_native_pid != before.get("pid")
|
|
489
|
+
or (args.expected_server_identity and health.get("server_identity") != args.expected_server_identity)):
|
|
490
|
+
raise RuntimeError("running monolith is not in the exact admitted idle update")
|
|
491
|
+
cgroup = health.get("update_service_cgroup", {})
|
|
492
|
+
if (cgroup.get("safe") is not True or type(cgroup.get("unknown_descendant_count")) is not int
|
|
493
|
+
or cgroup.get("unknown_descendant_count") != 0):
|
|
494
|
+
raise RuntimeError("monolith process ownership is not safe for handoff")
|
|
495
|
+
if services.snapshot()["worker"].get("pid") != before.get("pid"):
|
|
496
|
+
raise RuntimeError("service process changed before installer stop")
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def main(argv: list[str] | None = None) -> int:
|
|
500
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
501
|
+
parser.add_argument("command", choices=("detect", "snapshot", "candidate-api-contract", "preflight", "durability", "publish", "stop", "start", "suppress", "restore", "verify", "verify-restored", "previous-worker", "release", "recovery-state"))
|
|
502
|
+
for name in ("root", "config-root", "state-root", "home", "platform", "release-dir", "bind"):
|
|
503
|
+
parser.add_argument("--" + name, required=name == "root")
|
|
504
|
+
parser.add_argument("--port", type=int)
|
|
505
|
+
parser.add_argument("--candidate-source", default="")
|
|
506
|
+
parser.add_argument("--release-version", default="")
|
|
507
|
+
parser.add_argument("--api-contract", type=int, default=0)
|
|
508
|
+
parser.add_argument("--health-file", default="")
|
|
509
|
+
parser.add_argument("--update-file", default="")
|
|
510
|
+
parser.add_argument("--handoff-file", default="")
|
|
511
|
+
parser.add_argument("--managed-update-id", default="")
|
|
512
|
+
parser.add_argument("--expected-native-pid", type=int, default=0)
|
|
513
|
+
parser.add_argument("--expected-server-identity", default="")
|
|
514
|
+
args = parser.parse_args(argv)
|
|
515
|
+
if args.command == "candidate-api-contract":
|
|
516
|
+
print(candidate_api_contract(Path(args.candidate_source)))
|
|
517
|
+
return 0
|
|
518
|
+
if args.command == "detect":
|
|
519
|
+
root = files._path(Path(args.root))
|
|
520
|
+
generic = root / files.TRANSACTION_NAME
|
|
521
|
+
if generic.exists() or generic.is_symlink():
|
|
522
|
+
raise RuntimeError("independent execution activation must be recovered first")
|
|
523
|
+
outer = root / ".activation-transaction"
|
|
524
|
+
if outer.exists() or outer.is_symlink():
|
|
525
|
+
# Recovery must follow the journal even if layout publication had
|
|
526
|
+
# not happened, or rollback has removed the candidate manifest.
|
|
527
|
+
_directory, value = activation._read_manifest(root)
|
|
528
|
+
print("split" if value.get("execution") else "legacy")
|
|
529
|
+
elif (root / files.LAYOUT_NAME).exists() or (root / files.LAYOUT_NAME).is_symlink():
|
|
530
|
+
files.active_worker_release(root)
|
|
531
|
+
print("split")
|
|
532
|
+
else:
|
|
533
|
+
print("legacy")
|
|
534
|
+
return 0
|
|
535
|
+
layout = command_layout(args)
|
|
536
|
+
services, control = NativeServices(layout), WorkerControl()
|
|
537
|
+
if args.command == "snapshot":
|
|
538
|
+
observed = services.snapshot()["gateway"]
|
|
539
|
+
print(observed["state"] + "|" + ("true" if observed["enabled"] else "false"))
|
|
540
|
+
return 0
|
|
541
|
+
if args.command == "preflight":
|
|
542
|
+
assert_install_lock(layout.install_root)
|
|
543
|
+
handoff = None
|
|
544
|
+
if args.handoff_file:
|
|
545
|
+
from update_handoff import _read_handoff
|
|
546
|
+
handoff = _read_handoff(layout.install_root, Path(args.handoff_file))
|
|
547
|
+
if handoff["expected_server_identity"] != args.expected_server_identity:
|
|
548
|
+
raise RuntimeError("preflight handoff server identity changed")
|
|
549
|
+
# This is read-only admission before any outer snapshot/publication.
|
|
550
|
+
# Stop repeats the proof from durable journal metadata immediately
|
|
551
|
+
# before it touches either native job.
|
|
552
|
+
value = {"release_dir":str(layout.worker_release), "execution":{
|
|
553
|
+
"operation_id":handoff["operation_id"] if handoff else "",
|
|
554
|
+
"handoff":handoff}}
|
|
555
|
+
verify_stop(args, value, services, control)
|
|
556
|
+
seed_legacy_recovery(args, services)
|
|
557
|
+
return 0
|
|
558
|
+
value = transaction(args)
|
|
559
|
+
if args.command == "recovery-state":
|
|
560
|
+
print(recovery_state(args, value))
|
|
561
|
+
elif args.command == "durability":
|
|
562
|
+
from execution_durability import flush_activation
|
|
563
|
+
flush_activation(layout.install_root, value)
|
|
564
|
+
if transaction(args) != value:
|
|
565
|
+
raise RuntimeError("activation journal changed during runtime durability barrier")
|
|
566
|
+
elif args.command == "publish":
|
|
567
|
+
publish(args, value)
|
|
568
|
+
elif args.command in {"verify", "verify-restored"}:
|
|
569
|
+
verify_components(args, value, services, control)
|
|
570
|
+
elif args.command == "previous-worker":
|
|
571
|
+
if value["execution_layout"]["existed"]:
|
|
572
|
+
previous = activation._locate_release(value["execution"]["old_worker_release"])
|
|
573
|
+
if previous is None:
|
|
574
|
+
raise RuntimeError("previous worker generation is unavailable")
|
|
575
|
+
print(previous)
|
|
576
|
+
elif args.command == "stop":
|
|
577
|
+
verify_stop(args, value, services, control)
|
|
578
|
+
services.stop("gateway")
|
|
579
|
+
services.stop("worker")
|
|
580
|
+
elif args.command == "suppress":
|
|
581
|
+
for role in ("gateway", "worker"):
|
|
582
|
+
if services.snapshot()[role]["state"] != "absent":
|
|
583
|
+
services.set_enabled(role, False)
|
|
584
|
+
elif args.command == "start":
|
|
585
|
+
services.reload()
|
|
586
|
+
for role in ("worker", "gateway"):
|
|
587
|
+
if services._observe(role)["state"] not in {"running", "transitioning"}:
|
|
588
|
+
services.start(role)
|
|
589
|
+
elif args.command == "restore":
|
|
590
|
+
services.reload()
|
|
591
|
+
for role, prior in (
|
|
592
|
+
("worker", {"state": value["service_state"], "enabled": value["service_enabled"]}),
|
|
593
|
+
("gateway", {"state": value["execution"]["gateway_state"], "enabled": value["execution"]["gateway_enabled"]}),
|
|
594
|
+
):
|
|
595
|
+
if services._observe(role)["state"] != "running":
|
|
596
|
+
services.restore(role, prior)
|
|
597
|
+
elif prior["state"] != "absent":
|
|
598
|
+
services.set_enabled(role, prior["enabled"])
|
|
599
|
+
elif args.command == "release":
|
|
600
|
+
if value["phase"] not in {"committed", "rollback-healthy"}:
|
|
601
|
+
raise RuntimeError("admission cannot reopen before verified transaction settlement")
|
|
602
|
+
if value["phase"] == "rollback-healthy" and not value["execution_layout"]["existed"]:
|
|
603
|
+
return 0 # legacy monolith has no private execution lease
|
|
604
|
+
if services.snapshot()["worker"]["state"] == "running":
|
|
605
|
+
control.release(layout, value["execution"]["operation_id"])
|
|
606
|
+
return 0
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
if __name__ == "__main__":
|
|
610
|
+
raise SystemExit(main())
|