@agentsdock/server 1.0.7-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +19 -0
- package/README.md +60 -0
- package/npm/cli.cjs +289 -0
- package/package.json +32 -0
- package/server/LICENSE +202 -0
- package/server/NOTICE +19 -0
- package/server/VERSION +1 -0
- package/server/activation_transaction.py +2458 -0
- package/server/agent_server.py +93226 -0
- package/server/agentsdock_chats.py +1237 -0
- package/server/agentsdock_emergency.py +338 -0
- package/server/agentsdock_jobs.py +640 -0
- package/server/agentsdock_mail.py +325 -0
- package/server/agentsdock_publish.py +404 -0
- package/server/agentsdock_team.py +793 -0
- package/server/agentsdock_team_hub/__init__.py +33 -0
- package/server/agentsdock_team_hub/auth.py +710 -0
- package/server/agentsdock_team_hub/cli.py +647 -0
- package/server/agentsdock_team_hub/database.py +310 -0
- package/server/agentsdock_team_hub/mail_hint_streams.py +176 -0
- package/server/agentsdock_team_hub/mail_hints.py +201 -0
- package/server/agentsdock_team_hub/migrations/0001_identity_auth.sql +690 -0
- package/server/agentsdock_team_hub/migrations/0002_teamspace_ledger.sql +1336 -0
- package/server/agentsdock_team_hub/migrations/0003_service_runtime.sql +361 -0
- package/server/agentsdock_team_hub/migrations/0004_managed_host_binding.sql +22 -0
- package/server/agentsdock_team_hub/migrations/0005_tailnet_bootstrap_delegations.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0006_team_network_mailbox.sql +423 -0
- package/server/agentsdock_team_hub/migrations/0007_local_agent_mail.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0008_managed_server_session.sql +73 -0
- package/server/agentsdock_team_hub/migrations/0009_team_messages.sql +324 -0
- package/server/agentsdock_team_hub/migrations/0010_team_attachment_orphan_reclamation.sql +52 -0
- package/server/agentsdock_team_hub/migrations/0011_human_admin_paging.sql +65 -0
- package/server/agentsdock_team_hub/migrations/0012_network_content_deletions.sql +69 -0
- package/server/agentsdock_team_hub/migrations/0013_team_message_revisions.sql +48 -0
- package/server/agentsdock_team_hub/migrations/0014_managed_network_owner.sql +50 -0
- package/server/agentsdock_team_hub/migrations/0015_team_message_inbox_dismissals.sql +3 -0
- package/server/agentsdock_team_hub/migrations/0016_team_message_all_servers.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0017_skill_announcement_deletions.sql +16 -0
- package/server/agentsdock_team_hub/migrations/0018_team_mail_subjects.sql +8 -0
- package/server/agentsdock_team_hub/migrations/0019_team_mailbox_state.sql +5 -0
- package/server/agentsdock_team_hub/migrations/0020_team_mail_arrivals.sql +44 -0
- package/server/agentsdock_team_hub/migrations/0021_team_mail_threads.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0022_team_bulletin_changes.sql +49 -0
- package/server/agentsdock_team_hub/migrations/0023_team_message_search.sql +93 -0
- package/server/agentsdock_team_hub/migrations/__init__.py +1 -0
- package/server/agentsdock_team_hub/notification_hints.py +306 -0
- package/server/agentsdock_team_hub/secure_peer.py +11215 -0
- package/server/agentsdock_team_hub/secure_peer_hub.py +1372 -0
- package/server/agentsdock_team_hub/security.py +318 -0
- package/server/agentsdock_team_hub/service.py +1737 -0
- package/server/agentsdock_team_hub/store.py +16276 -0
- package/server/chat_mailbox.py +448 -0
- package/server/claude_background_reconciliation.py +92 -0
- package/server/claude_goals.py +245 -0
- package/server/claude_history_provenance.py +137 -0
- package/server/claude_history_repair.py +1601 -0
- package/server/claude_sdk_client.py +3731 -0
- package/server/claude_side_question.py +212 -0
- package/server/codex_app_server.py +3030 -0
- package/server/codex_auth.py +93 -0
- package/server/codex_history_repair.py +1247 -0
- package/server/codex_provider.py +1120 -0
- package/server/codex_side_question.py +301 -0
- package/server/cursor_agent_client.py +654 -0
- package/server/cursor_process_guard.py +347 -0
- package/server/execution_activation.py +610 -0
- package/server/execution_control.py +233 -0
- package/server/execution_durability.py +184 -0
- package/server/execution_http.py +256 -0
- package/server/execution_install.py +792 -0
- package/server/execution_legacy_runner.py +406 -0
- package/server/execution_maintenance.py +137 -0
- package/server/execution_manage.py +615 -0
- package/server/execution_ownership.py +83 -0
- package/server/execution_preparation.py +362 -0
- package/server/execution_recovery.py +845 -0
- package/server/execution_recovery_status.py +195 -0
- package/server/execution_service.py +387 -0
- package/server/execution_transport.py +622 -0
- package/server/execution_uninstall.py +406 -0
- package/server/execution_update_status.py +49 -0
- package/server/install.sh +7074 -0
- package/server/interactive_chat_controls.py +153 -0
- package/server/interactive_chat_native.py +66 -0
- package/server/interactive_chat_projection.py +152 -0
- package/server/interactive_chat_runtime.py +132 -0
- package/server/interactive_chat_share_routes.py +534 -0
- package/server/interactive_chat_share_web.py +12 -0
- package/server/interactive_chat_shares.py +238 -0
- package/server/provider_commands.py +425 -0
- package/server/public_chat_share_routes.py +426 -0
- package/server/public_chat_shares.py +759 -0
- package/server/public_chat_transcript.py +357 -0
- package/server/pyproject.toml +29 -0
- package/server/release-public-key.pem +3 -0
- package/server/secure_peer_delivery.py +1236 -0
- package/server/secure_peer_runtime.py +7694 -0
- package/server/shared_chat_video_stream.py +262 -0
- package/server/shared_chat_videos.py +155 -0
- package/server/side_questions.py +594 -0
- package/server/team_hub_host.py +1354 -0
- package/server/team_mail_grants.py +136 -0
- package/server/team_mail_runtime.py +302 -0
- package/server/team_mail_websocket.py +211 -0
- package/server/title_generation.py +333 -0
- package/server/uninstall.sh +359 -0
- package/server/update_handoff.py +237 -0
- package/server/update_preparation.py +264 -0
- package/server/update_recovery.py +140 -0
- package/server/update_runner.py +1819 -0
- package/server/uv.lock +1101 -0
- package/server/workspace_git.py +545 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""Callback-listener-only controls for the persistent execution owner.
|
|
2
|
+
|
|
3
|
+
Do not wrap the application exposed over execution IPC with this middleware.
|
|
4
|
+
The private control token is independent of all client/provider credentials.
|
|
5
|
+
Lease admission and persistence remain the execution owner's responsibility.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import hmac
|
|
12
|
+
import ipaddress
|
|
13
|
+
import json
|
|
14
|
+
import re
|
|
15
|
+
import uuid
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
CONTROL_PREFIX = "/api/admin/execution"
|
|
20
|
+
STATUS_PATH = CONTROL_PREFIX + "/status"
|
|
21
|
+
MAINTENANCE_PATH = CONTROL_PREFIX + "/maintenance"
|
|
22
|
+
MAX_BODY_BYTES = 4096
|
|
23
|
+
MAX_RESPONSE_BYTES = 64 * 1024
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ExecutionControlError(RuntimeError):
|
|
27
|
+
"""An expected control refusal raised by an execution-owner callback."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, status_code: int, detail: str) -> None:
|
|
30
|
+
if type(status_code) is not int or not 400 <= status_code <= 599:
|
|
31
|
+
raise ValueError("Invalid execution control error status")
|
|
32
|
+
super().__init__(detail)
|
|
33
|
+
self.status_code = status_code
|
|
34
|
+
self.detail = detail
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
38
|
+
result: dict[str, Any] = {}
|
|
39
|
+
for name, value in pairs:
|
|
40
|
+
if name in result:
|
|
41
|
+
raise ExecutionControlError(400, "Duplicate JSON field")
|
|
42
|
+
result[name] = value
|
|
43
|
+
return result
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _constant(_value: str) -> None:
|
|
47
|
+
raise ExecutionControlError(400, "Invalid JSON number")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _loopback(scope: dict[str, Any]) -> bool:
|
|
51
|
+
client = scope.get("client")
|
|
52
|
+
if not isinstance(client, (tuple, list)) or len(client) != 2:
|
|
53
|
+
return False
|
|
54
|
+
try:
|
|
55
|
+
address = ipaddress.ip_address(client[0])
|
|
56
|
+
except (ValueError, TypeError):
|
|
57
|
+
return False
|
|
58
|
+
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
|
|
59
|
+
address = address.ipv4_mapped
|
|
60
|
+
return address.is_loopback
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _headers(scope: dict[str, Any]) -> list[tuple[bytes, bytes]]:
|
|
64
|
+
headers = scope.get("headers", [])
|
|
65
|
+
if not isinstance(headers, (list, tuple)) or len(headers) > 128:
|
|
66
|
+
raise ExecutionControlError(400, "Invalid control headers")
|
|
67
|
+
result = []
|
|
68
|
+
size = 0
|
|
69
|
+
for item in headers:
|
|
70
|
+
if (not isinstance(item, (list, tuple)) or len(item) != 2
|
|
71
|
+
or not all(isinstance(part, bytes) for part in item)):
|
|
72
|
+
raise ExecutionControlError(400, "Invalid control headers")
|
|
73
|
+
size += len(item[0]) + len(item[1])
|
|
74
|
+
if size > 16384:
|
|
75
|
+
raise ExecutionControlError(400, "Control headers exceed their limit")
|
|
76
|
+
result.append((item[0].lower(), item[1]))
|
|
77
|
+
return result
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _header_values(headers: list[tuple[bytes, bytes]], name: bytes) -> list[bytes]:
|
|
81
|
+
return [value for key, value in headers if key == name]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _content_length(headers: list[tuple[bytes, bytes]]) -> int | None:
|
|
85
|
+
if _header_values(headers, b"transfer-encoding"):
|
|
86
|
+
raise ExecutionControlError(400, "Control requests do not accept transfer encoding")
|
|
87
|
+
values = _header_values(headers, b"content-length")
|
|
88
|
+
if not values:
|
|
89
|
+
return None
|
|
90
|
+
if len(values) != 1 or re.fullmatch(rb"0|[1-9][0-9]{0,9}", values[0]) is None:
|
|
91
|
+
raise ExecutionControlError(400, "Invalid control content length")
|
|
92
|
+
length = int(values[0])
|
|
93
|
+
if length > MAX_BODY_BYTES:
|
|
94
|
+
raise ExecutionControlError(413, "Control request exceeds its size limit")
|
|
95
|
+
return length
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def _body(receive: Any, declared: int | None) -> bytes:
|
|
99
|
+
async def read() -> bytes:
|
|
100
|
+
body = bytearray()
|
|
101
|
+
while True:
|
|
102
|
+
message = await receive()
|
|
103
|
+
if message.get("type") != "http.request" or not isinstance(message.get("body", b""), bytes):
|
|
104
|
+
raise ExecutionControlError(400, "Control request body was interrupted")
|
|
105
|
+
chunk = message.get("body", b"")
|
|
106
|
+
if len(chunk) > MAX_BODY_BYTES - len(body):
|
|
107
|
+
raise ExecutionControlError(413, "Control request exceeds its size limit")
|
|
108
|
+
body.extend(chunk)
|
|
109
|
+
if declared is not None and len(body) > declared:
|
|
110
|
+
raise ExecutionControlError(400, "Control content length does not match its body")
|
|
111
|
+
more = message.get("more_body", False)
|
|
112
|
+
if type(more) is not bool:
|
|
113
|
+
raise ExecutionControlError(400, "Invalid control body framing")
|
|
114
|
+
if not more:
|
|
115
|
+
if declared is not None and declared != len(body):
|
|
116
|
+
raise ExecutionControlError(400, "Control content length does not match its body")
|
|
117
|
+
return bytes(body)
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
return await asyncio.wait_for(read(), timeout=5.0)
|
|
121
|
+
except asyncio.TimeoutError:
|
|
122
|
+
raise ExecutionControlError(408, "Control request body timed out") from None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ExecutionControl:
|
|
126
|
+
"""Guard private status/maintenance requests before invoking owner callbacks."""
|
|
127
|
+
|
|
128
|
+
def __init__(self, app: Any, *, token: str, worker_instance_id: str,
|
|
129
|
+
status_callback: Any, maintenance_callback: Any) -> None:
|
|
130
|
+
if not isinstance(token, str) or re.fullmatch(r"[0-9a-f]{64}", token) is None:
|
|
131
|
+
raise ValueError("Invalid execution control token")
|
|
132
|
+
if (not isinstance(worker_instance_id, str) or not worker_instance_id
|
|
133
|
+
or len(worker_instance_id) > 128 or not worker_instance_id.isascii()
|
|
134
|
+
or any(character.isspace() for character in worker_instance_id)):
|
|
135
|
+
raise ValueError("Invalid worker instance identity")
|
|
136
|
+
self.app = app
|
|
137
|
+
self._token = token
|
|
138
|
+
self.worker_instance_id = worker_instance_id
|
|
139
|
+
self.status_callback = status_callback
|
|
140
|
+
self.maintenance_callback = maintenance_callback
|
|
141
|
+
|
|
142
|
+
async def _respond(self, send: Any, status: int, value: dict[str, Any]) -> None:
|
|
143
|
+
try:
|
|
144
|
+
content = json.dumps(value, allow_nan=False, ensure_ascii=True,
|
|
145
|
+
separators=(",", ":")).encode("ascii")
|
|
146
|
+
if len(content) > MAX_RESPONSE_BYTES or self._token.encode("ascii") in content:
|
|
147
|
+
raise ValueError("Invalid private control response")
|
|
148
|
+
except (ValueError, TypeError, RecursionError):
|
|
149
|
+
status = 500
|
|
150
|
+
content = b'{"detail":"Execution control response unavailable"}'
|
|
151
|
+
await send({"type": "http.response.start", "status": status, "headers": [
|
|
152
|
+
(b"content-type", b"application/json"), (b"content-length", str(len(content)).encode("ascii")),
|
|
153
|
+
(b"cache-control", b"no-store"), (b"x-content-type-options", b"nosniff"),
|
|
154
|
+
]})
|
|
155
|
+
await send({"type": "http.response.body", "body": content})
|
|
156
|
+
|
|
157
|
+
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
|
|
158
|
+
path = str(scope.get("path", ""))
|
|
159
|
+
if not path.startswith(CONTROL_PREFIX):
|
|
160
|
+
await self.app(scope, receive, send)
|
|
161
|
+
return
|
|
162
|
+
if scope.get("type") == "websocket":
|
|
163
|
+
await send({"type": "websocket.close", "code": 4403, "reason": "Private HTTP control required"})
|
|
164
|
+
return
|
|
165
|
+
if scope.get("type") != "http":
|
|
166
|
+
return
|
|
167
|
+
try:
|
|
168
|
+
headers = _headers(scope)
|
|
169
|
+
forbidden = {b"cookie", b"origin", b"forwarded", b"via", b"x-real-ip",
|
|
170
|
+
b"x-agentsdock-token", b"x-zenithdock-token"}
|
|
171
|
+
if (not _loopback(scope) or scope.get("query_string", b"")
|
|
172
|
+
or any(name in forbidden or name.startswith((b"sec-fetch-", b"x-forwarded-", b"tailscale-user-"))
|
|
173
|
+
for name, _ in headers)):
|
|
174
|
+
raise ExecutionControlError(403, "Private control transport is forbidden")
|
|
175
|
+
authorization = _header_values(headers, b"authorization")
|
|
176
|
+
expected = b"Bearer " + self._token.encode("ascii")
|
|
177
|
+
if len(authorization) != 1 or not hmac.compare_digest(authorization[0], expected):
|
|
178
|
+
raise ExecutionControlError(401, "Execution control authentication required")
|
|
179
|
+
if path not in {STATUS_PATH, MAINTENANCE_PATH}:
|
|
180
|
+
raise ExecutionControlError(404, "Unknown execution control")
|
|
181
|
+
method = scope.get("method")
|
|
182
|
+
if method != ("GET" if path == STATUS_PATH else "POST"):
|
|
183
|
+
raise ExecutionControlError(405, "Execution control method is not allowed")
|
|
184
|
+
length = _content_length(headers)
|
|
185
|
+
if path == STATUS_PATH:
|
|
186
|
+
if length not in {None, 0}:
|
|
187
|
+
raise ExecutionControlError(400, "Status requests do not accept a body")
|
|
188
|
+
result = await self.status_callback()
|
|
189
|
+
else:
|
|
190
|
+
content_types = _header_values(headers, b"content-type")
|
|
191
|
+
if (len(content_types) != 1 or content_types[0].lower()
|
|
192
|
+
not in {b"application/json", b"application/json; charset=utf-8"}):
|
|
193
|
+
raise ExecutionControlError(415, "Execution control requires JSON")
|
|
194
|
+
raw = await _body(receive, length)
|
|
195
|
+
try:
|
|
196
|
+
value = json.loads(raw.decode("utf-8"), object_pairs_hook=_pairs, parse_constant=_constant)
|
|
197
|
+
except (ValueError, UnicodeError, RecursionError):
|
|
198
|
+
raise ExecutionControlError(400, "Invalid execution control JSON") from None
|
|
199
|
+
required = {"action", "expected_worker_instance_id", "operation_id"}
|
|
200
|
+
if not isinstance(value, dict) or not required <= value.keys() or value.keys() - required - {"lease_id", "lifetime_seconds"}:
|
|
201
|
+
raise ExecutionControlError(422, "Invalid execution control fields")
|
|
202
|
+
if value["expected_worker_instance_id"] != self.worker_instance_id:
|
|
203
|
+
raise ExecutionControlError(409, "Execution worker instance changed")
|
|
204
|
+
action = value["action"]
|
|
205
|
+
if not isinstance(action, str) or action not in {"acquire", "renew", "seal", "release"}:
|
|
206
|
+
raise ExecutionControlError(422, "Invalid maintenance action")
|
|
207
|
+
operation_id = value["operation_id"]
|
|
208
|
+
try:
|
|
209
|
+
if not isinstance(operation_id, str) or str(uuid.UUID(operation_id)) != operation_id:
|
|
210
|
+
raise ValueError("noncanonical operation")
|
|
211
|
+
except (ValueError, AttributeError):
|
|
212
|
+
raise ExecutionControlError(422, "Operation ID must be a canonical UUID") from None
|
|
213
|
+
lease_id = value.get("lease_id")
|
|
214
|
+
if action == "acquire":
|
|
215
|
+
if "lease_id" in value:
|
|
216
|
+
raise ExecutionControlError(422, "Acquire must not supply a lease ID")
|
|
217
|
+
elif (not isinstance(lease_id, str) or not 1 <= len(lease_id) <= 128
|
|
218
|
+
or re.fullmatch(r"[A-Za-z0-9_-]+", lease_id) is None):
|
|
219
|
+
raise ExecutionControlError(422, "Maintenance requires a valid lease ID")
|
|
220
|
+
lifetime = value.get("lifetime_seconds", 120)
|
|
221
|
+
if type(lifetime) is not int or not 30 <= lifetime <= 300:
|
|
222
|
+
raise ExecutionControlError(422, "Maintenance lifetime must be 30 to 300 seconds")
|
|
223
|
+
result = await self.maintenance_callback(action, operation_id, lease_id, lifetime)
|
|
224
|
+
if not isinstance(result, dict):
|
|
225
|
+
raise ExecutionControlError(500, "Execution control response unavailable")
|
|
226
|
+
response_status, response_value = 200, result
|
|
227
|
+
except ExecutionControlError as exc:
|
|
228
|
+
response_status, response_value = exc.status_code, {"detail": exc.detail}
|
|
229
|
+
except Exception:
|
|
230
|
+
response_status, response_value = 500, {"detail": "Execution control unavailable"}
|
|
231
|
+
# Response transport loss must not invoke a handler again or attempt a
|
|
232
|
+
# second response after an already committed lease operation.
|
|
233
|
+
await self._respond(send, response_status, response_value)
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Flush retained executable generations before an admitted native service stop.
|
|
2
|
+
|
|
3
|
+
This is a durability barrier, not an alternate integrity or admission proof.
|
|
4
|
+
The caller owns the install lock and supplies an already validated outer journal.
|
|
5
|
+
No bytes, modes, links, configuration or service state are changed here.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import fcntl
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import re
|
|
13
|
+
import stat
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
import activation_transaction as activation
|
|
17
|
+
|
|
18
|
+
MAX_ENTRIES = 250_000
|
|
19
|
+
MAX_BYTES = 16 * 1024 * 1024 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _identity(info):
|
|
23
|
+
return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_size,
|
|
24
|
+
info.st_mtime_ns, info.st_ctime_ns, info.st_nlink)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _file_barrier(descriptor: int) -> None:
|
|
28
|
+
os.fsync(descriptor)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _full_barrier(path: Path) -> None:
|
|
32
|
+
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
33
|
+
try:
|
|
34
|
+
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
|
|
35
|
+
raise RuntimeError("durability anchor is not a regular file")
|
|
36
|
+
os.fsync(descriptor)
|
|
37
|
+
if sys.platform == "darwin":
|
|
38
|
+
# fsync alone need not drain a device's volatile write cache on macOS.
|
|
39
|
+
fcntl.fcntl(descriptor, getattr(fcntl, "F_FULLFSYNC", 51))
|
|
40
|
+
finally:
|
|
41
|
+
os.close(descriptor)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def flush_tree(root: Path, *, dependency_prefix: bool = False) -> dict[str, int]:
|
|
45
|
+
"""Flush one owned tree without following directory links or special files."""
|
|
46
|
+
root = Path(root)
|
|
47
|
+
if not root.is_absolute() or root.resolve(strict=True) != root:
|
|
48
|
+
raise ValueError("durability tree must have no linked ancestors")
|
|
49
|
+
root_info = root.lstat()
|
|
50
|
+
if (not stat.S_ISDIR(root_info.st_mode) or root_info.st_uid != os.getuid()
|
|
51
|
+
or root_info.st_mode & 0o022):
|
|
52
|
+
raise PermissionError("durability tree root is unsafe")
|
|
53
|
+
total = {"entries": 0, "bytes": 0}
|
|
54
|
+
external: set[Path] = set()
|
|
55
|
+
observed = {}
|
|
56
|
+
|
|
57
|
+
def visit(directory: Path, parent_fd: int | None = None) -> None:
|
|
58
|
+
before = directory.lstat()
|
|
59
|
+
descriptor = os.open(directory if parent_fd is None else directory.name,
|
|
60
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd)
|
|
61
|
+
try:
|
|
62
|
+
if (_identity(before) != _identity(os.fstat(descriptor))
|
|
63
|
+
or before.st_uid != os.getuid() or before.st_dev != root_info.st_dev):
|
|
64
|
+
raise PermissionError("durability directory ownership or device changed")
|
|
65
|
+
names = sorted(os.listdir(descriptor))
|
|
66
|
+
for name in names:
|
|
67
|
+
if any(ord(character) < 32 for character in name):
|
|
68
|
+
raise ValueError("durability member contains control characters")
|
|
69
|
+
child = directory / name
|
|
70
|
+
info = os.stat(name, dir_fd=descriptor, follow_symlinks=False)
|
|
71
|
+
if info.st_uid != os.getuid() or info.st_dev != root_info.st_dev:
|
|
72
|
+
raise PermissionError("durability member ownership or device changed")
|
|
73
|
+
if dependency_prefix and not stat.S_ISLNK(info.st_mode) and info.st_mode & 0o022:
|
|
74
|
+
raise PermissionError("standalone Python durability member is writable by others")
|
|
75
|
+
total["entries"] += 1
|
|
76
|
+
if total["entries"] > MAX_ENTRIES:
|
|
77
|
+
raise ValueError("durability tree exceeds entry limit")
|
|
78
|
+
if stat.S_ISDIR(info.st_mode):
|
|
79
|
+
visit(child, descriptor)
|
|
80
|
+
elif stat.S_ISREG(info.st_mode):
|
|
81
|
+
total["bytes"] += info.st_size
|
|
82
|
+
if total["bytes"] > MAX_BYTES:
|
|
83
|
+
raise ValueError("durability tree exceeds byte limit")
|
|
84
|
+
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=descriptor)
|
|
85
|
+
try:
|
|
86
|
+
if _identity(info) != _identity(os.fstat(fd)):
|
|
87
|
+
raise RuntimeError("durability file changed before flush")
|
|
88
|
+
_file_barrier(fd)
|
|
89
|
+
if _identity(info) != _identity(os.fstat(fd)):
|
|
90
|
+
raise RuntimeError("durability file changed while flushing")
|
|
91
|
+
finally:
|
|
92
|
+
os.close(fd)
|
|
93
|
+
elif stat.S_ISLNK(info.st_mode):
|
|
94
|
+
target = os.readlink(name, dir_fd=descriptor)
|
|
95
|
+
resolved = child.resolve(strict=True)
|
|
96
|
+
if root not in resolved.parents:
|
|
97
|
+
relative = child.relative_to(root).parts
|
|
98
|
+
if (dependency_prefix or len(relative) != 3 or relative[:2] != (".venv", "bin")
|
|
99
|
+
or not re.fullmatch(r"python(?:[0-9]+(?:\.[0-9]+)*)?", relative[2])
|
|
100
|
+
or not re.fullmatch(r"python(?:[0-9]+(?:\.[0-9]+)*)?", resolved.name)):
|
|
101
|
+
raise ValueError("external durability link is not a Python interpreter")
|
|
102
|
+
external.add(resolved)
|
|
103
|
+
if target != os.readlink(name, dir_fd=descriptor):
|
|
104
|
+
raise RuntimeError("durability link changed while flushing")
|
|
105
|
+
else:
|
|
106
|
+
raise ValueError("durability tree contains a special file")
|
|
107
|
+
if _identity(info) != _identity(os.stat(name, dir_fd=descriptor, follow_symlinks=False)):
|
|
108
|
+
raise RuntimeError("durability member changed while flushing")
|
|
109
|
+
observed[child] = _identity(info)
|
|
110
|
+
if names != sorted(os.listdir(descriptor)) or _identity(before) != _identity(directory.lstat()):
|
|
111
|
+
raise RuntimeError("durability directory changed while flushing")
|
|
112
|
+
os.fsync(descriptor)
|
|
113
|
+
finally:
|
|
114
|
+
os.close(descriptor)
|
|
115
|
+
|
|
116
|
+
visit(root)
|
|
117
|
+
for interpreter in sorted(external):
|
|
118
|
+
info = interpreter.lstat()
|
|
119
|
+
if (not stat.S_ISREG(info.st_mode) or info.st_uid not in {0, os.getuid()}
|
|
120
|
+
or info.st_mode & 0o022 or not info.st_mode & 0o111):
|
|
121
|
+
raise PermissionError("external durability interpreter is unsafe")
|
|
122
|
+
if info.st_uid == 0:
|
|
123
|
+
# System interpreters are provisioned outside this updater. Never
|
|
124
|
+
# traverse /usr or another OS-managed Python installation.
|
|
125
|
+
continue
|
|
126
|
+
prefix = interpreter.parent.parent
|
|
127
|
+
match = re.fullmatch(r"python([0-9]+\.[0-9]+)", interpreter.name)
|
|
128
|
+
standalone = re.fullmatch(r"cpython-[0-9]+\.[0-9]+\.[0-9]+(?:[a-z0-9.]*)?-(?:macos|linux)-[a-z0-9_]+-[a-z0-9_]+", prefix.name)
|
|
129
|
+
if standalone:
|
|
130
|
+
if (interpreter.parent.name != "bin" or match is None
|
|
131
|
+
or not (prefix / "lib" / ("python" + match[1])).is_dir()
|
|
132
|
+
or not (prefix / "BUILD").is_file() or (prefix / "BUILD").is_symlink()):
|
|
133
|
+
raise ValueError("owned uv Python has no bounded standalone runtime")
|
|
134
|
+
flush_tree(prefix, dependency_prefix=True)
|
|
135
|
+
parent_fd = os.open(prefix.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
136
|
+
try:
|
|
137
|
+
os.fsync(parent_fd)
|
|
138
|
+
finally:
|
|
139
|
+
os.close(parent_fd)
|
|
140
|
+
# Other selected interpreters (for example Homebrew) are preexisting
|
|
141
|
+
# prerequisites. Do not infer and recursively scan an arbitrary prefix.
|
|
142
|
+
_full_barrier(interpreter)
|
|
143
|
+
if _identity(info) != _identity(interpreter.lstat()):
|
|
144
|
+
raise RuntimeError("external durability interpreter changed")
|
|
145
|
+
for path, identity in observed.items():
|
|
146
|
+
if _identity(path.lstat()) != identity:
|
|
147
|
+
raise RuntimeError("durability tree changed after flush")
|
|
148
|
+
if _identity(root.lstat()) != _identity(root_info):
|
|
149
|
+
raise RuntimeError("durability root changed after flush")
|
|
150
|
+
return total
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def flush_activation(root: Path, value: dict) -> None:
|
|
154
|
+
root_info = root.lstat()
|
|
155
|
+
if (not stat.S_ISDIR(root_info.st_mode) or root_info.st_uid != os.getuid()
|
|
156
|
+
or stat.S_IMODE(root_info.st_mode) != 0o700):
|
|
157
|
+
raise PermissionError("activation durability requires a private install root")
|
|
158
|
+
if value.get("phase") not in {"prepared", "guarded"}:
|
|
159
|
+
raise RuntimeError("runtime durability must precede native quiescing")
|
|
160
|
+
releases = [value["candidate_release"], value["old_release"], value["execution"]["old_worker_release"]]
|
|
161
|
+
seen: set[Path] = set()
|
|
162
|
+
for release in releases:
|
|
163
|
+
if not release:
|
|
164
|
+
continue
|
|
165
|
+
path = activation._locate_release(release)
|
|
166
|
+
if path is None:
|
|
167
|
+
raise RuntimeError("durability runtime is not retained by the journal")
|
|
168
|
+
if path in seen:
|
|
169
|
+
continue
|
|
170
|
+
seen.add(path)
|
|
171
|
+
flush_tree(path)
|
|
172
|
+
descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
173
|
+
try:
|
|
174
|
+
os.fsync(descriptor)
|
|
175
|
+
finally:
|
|
176
|
+
os.close(descriptor)
|
|
177
|
+
_full_barrier(path / "VERSION")
|
|
178
|
+
if not activation._release_matches(path, release):
|
|
179
|
+
raise RuntimeError("durability runtime identity changed")
|
|
180
|
+
descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
181
|
+
try:
|
|
182
|
+
os.fsync(descriptor)
|
|
183
|
+
finally:
|
|
184
|
+
os.close(descriptor)
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""Credential HTTP transport pinned to the connected native process.
|
|
2
|
+
|
|
3
|
+
The Linux /proc and Darwin lsof tuple checks are shared in design with
|
|
4
|
+
install.sh:pinned_managed_http_get. No HTTP bytes, including credentials, leave
|
|
5
|
+
the connected socket before process ownership and caller authority are proven.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ipaddress
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import socket
|
|
14
|
+
import subprocess
|
|
15
|
+
import time
|
|
16
|
+
from typing import Callable
|
|
17
|
+
from urllib.parse import urlsplit
|
|
18
|
+
|
|
19
|
+
MAXIMUM_HEADERS = 64 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def address_from_proc(raw, ipv6):
|
|
23
|
+
packed = bytes.fromhex(raw)
|
|
24
|
+
if not ipv6:
|
|
25
|
+
packed = packed[::-1]
|
|
26
|
+
else:
|
|
27
|
+
packed = b"".join(
|
|
28
|
+
packed[offset : offset + 4][::-1]
|
|
29
|
+
for offset in range(0, 16, 4)
|
|
30
|
+
)
|
|
31
|
+
return ipaddress.ip_address(packed)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def linux_connection_owned(pid, server, client):
|
|
35
|
+
inodes = set()
|
|
36
|
+
try:
|
|
37
|
+
entries = (Path("/proc") / str(pid) / "fd").iterdir()
|
|
38
|
+
for entry in entries:
|
|
39
|
+
try:
|
|
40
|
+
target = os.readlink(entry)
|
|
41
|
+
except OSError:
|
|
42
|
+
continue
|
|
43
|
+
if target.startswith("socket:[") and target.endswith("]"):
|
|
44
|
+
inodes.add(target[8:-1])
|
|
45
|
+
except OSError:
|
|
46
|
+
return False
|
|
47
|
+
if not inodes:
|
|
48
|
+
return False
|
|
49
|
+
expected_server = (ipaddress.ip_address(server[0]), int(server[1]))
|
|
50
|
+
expected_client = (ipaddress.ip_address(client[0]), int(client[1]))
|
|
51
|
+
for table, ipv6 in ((Path("/proc/net/tcp"), False), (Path("/proc/net/tcp6"), True)):
|
|
52
|
+
try:
|
|
53
|
+
lines = table.read_text(encoding="ascii").splitlines()[1:]
|
|
54
|
+
except OSError:
|
|
55
|
+
continue
|
|
56
|
+
for line in lines:
|
|
57
|
+
fields = line.split()
|
|
58
|
+
if len(fields) < 10 or fields[3] != "01" or fields[9] not in inodes:
|
|
59
|
+
continue
|
|
60
|
+
try:
|
|
61
|
+
local_address, local_port = fields[1].rsplit(":", 1)
|
|
62
|
+
remote_address, remote_port = fields[2].rsplit(":", 1)
|
|
63
|
+
observed_server = (
|
|
64
|
+
address_from_proc(local_address, ipv6),
|
|
65
|
+
int(local_port, 16),
|
|
66
|
+
)
|
|
67
|
+
observed_client = (
|
|
68
|
+
address_from_proc(remote_address, ipv6),
|
|
69
|
+
int(remote_port, 16),
|
|
70
|
+
)
|
|
71
|
+
except (IndexError, ValueError):
|
|
72
|
+
continue
|
|
73
|
+
if observed_server == expected_server and observed_client == expected_client:
|
|
74
|
+
return True
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def parse_lsof_endpoint(value):
|
|
79
|
+
value = value.strip().split(None, 1)[0]
|
|
80
|
+
if value.startswith("["):
|
|
81
|
+
closing = value.find("]:")
|
|
82
|
+
if closing < 0:
|
|
83
|
+
raise ValueError
|
|
84
|
+
host = value[1:closing]
|
|
85
|
+
port = value[closing + 2 :]
|
|
86
|
+
else:
|
|
87
|
+
host, port = value.rsplit(":", 1)
|
|
88
|
+
return ipaddress.ip_address(host), int(port)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def darwin_connection_owned(pid, server, client, timeout):
|
|
92
|
+
try:
|
|
93
|
+
result = subprocess.run(
|
|
94
|
+
[
|
|
95
|
+
"/usr/sbin/lsof",
|
|
96
|
+
"-nP",
|
|
97
|
+
"-a",
|
|
98
|
+
"-p",
|
|
99
|
+
str(pid),
|
|
100
|
+
"-iTCP",
|
|
101
|
+
"-sTCP:ESTABLISHED",
|
|
102
|
+
"-Fn",
|
|
103
|
+
],
|
|
104
|
+
stdin=subprocess.DEVNULL,
|
|
105
|
+
stdout=subprocess.PIPE,
|
|
106
|
+
stderr=subprocess.DEVNULL,
|
|
107
|
+
timeout=max(0.05, timeout),
|
|
108
|
+
check=False,
|
|
109
|
+
env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "LC_ALL": "C"},
|
|
110
|
+
)
|
|
111
|
+
except (OSError, subprocess.SubprocessError):
|
|
112
|
+
return False
|
|
113
|
+
if result.returncode != 0:
|
|
114
|
+
return False
|
|
115
|
+
expected_server = (ipaddress.ip_address(server[0]), int(server[1]))
|
|
116
|
+
expected_client = (ipaddress.ip_address(client[0]), int(client[1]))
|
|
117
|
+
for raw_line in result.stdout.decode("ascii", "strict").splitlines():
|
|
118
|
+
if not raw_line.startswith("n") or "->" not in raw_line:
|
|
119
|
+
continue
|
|
120
|
+
try:
|
|
121
|
+
local, remote = raw_line[1:].split("->", 1)
|
|
122
|
+
if (
|
|
123
|
+
parse_lsof_endpoint(local) == expected_server
|
|
124
|
+
and parse_lsof_endpoint(remote) == expected_client
|
|
125
|
+
):
|
|
126
|
+
return True
|
|
127
|
+
except (UnicodeError, ValueError):
|
|
128
|
+
continue
|
|
129
|
+
return False
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def receive_response(connection, deadline, maximum_body):
|
|
133
|
+
payload = bytearray()
|
|
134
|
+
header_end = -1
|
|
135
|
+
while header_end < 0:
|
|
136
|
+
remaining = deadline - time.monotonic()
|
|
137
|
+
if remaining <= 0:
|
|
138
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
139
|
+
connection.settimeout(remaining)
|
|
140
|
+
chunk = connection.recv(min(64 * 1024, MAXIMUM_HEADERS + 4 - len(payload)))
|
|
141
|
+
if not chunk:
|
|
142
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
143
|
+
payload.extend(chunk)
|
|
144
|
+
header_end = payload.find(b"\r\n\r\n")
|
|
145
|
+
if header_end > MAXIMUM_HEADERS or (header_end < 0 and len(payload) > MAXIMUM_HEADERS):
|
|
146
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
147
|
+
raw_headers = bytes(payload[:header_end])
|
|
148
|
+
body = bytearray(payload[header_end + 4 :])
|
|
149
|
+
try:
|
|
150
|
+
lines = raw_headers.decode("latin-1").split("\r\n")
|
|
151
|
+
except UnicodeError:
|
|
152
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
153
|
+
status_parts = lines[0].split(" ", 2) if lines else []
|
|
154
|
+
if (
|
|
155
|
+
len(status_parts) < 2
|
|
156
|
+
or status_parts[0] not in ("HTTP/1.0", "HTTP/1.1")
|
|
157
|
+
or status_parts[1] != "200"
|
|
158
|
+
):
|
|
159
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
160
|
+
headers = {}
|
|
161
|
+
for line in lines[1:]:
|
|
162
|
+
if not line or line[:1] in " \t" or ":" not in line:
|
|
163
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
164
|
+
name, value = line.split(":", 1)
|
|
165
|
+
name = name.strip().lower()
|
|
166
|
+
value = value.strip()
|
|
167
|
+
if not name or name in headers:
|
|
168
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
169
|
+
headers[name] = value
|
|
170
|
+
if "transfer-encoding" in headers:
|
|
171
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
172
|
+
content_length = None
|
|
173
|
+
if "content-length" in headers:
|
|
174
|
+
try:
|
|
175
|
+
content_length = int(headers["content-length"], 10)
|
|
176
|
+
except ValueError:
|
|
177
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
178
|
+
if content_length < 0 or content_length > maximum_body:
|
|
179
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
180
|
+
if len(body) > maximum_body or (
|
|
181
|
+
content_length is not None and len(body) > content_length
|
|
182
|
+
):
|
|
183
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
184
|
+
while content_length is None or len(body) < content_length:
|
|
185
|
+
remaining = deadline - time.monotonic()
|
|
186
|
+
if remaining <= 0:
|
|
187
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
188
|
+
connection.settimeout(remaining)
|
|
189
|
+
chunk = connection.recv(min(64 * 1024, maximum_body + 1 - len(body)))
|
|
190
|
+
if not chunk:
|
|
191
|
+
break
|
|
192
|
+
body.extend(chunk)
|
|
193
|
+
if len(body) > maximum_body:
|
|
194
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
195
|
+
if content_length is not None and len(body) != content_length:
|
|
196
|
+
raise RuntimeError("pinned HTTP response is invalid")
|
|
197
|
+
return bytes(body)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def request_json(url: str, token: str, *, expected_pid: int, platform: str,
|
|
201
|
+
verify_owner: Callable[[], None], body: dict | None = None,
|
|
202
|
+
timeout: float = 5.0, maximum_body: int = 1024 * 1024) -> dict:
|
|
203
|
+
"""Connect once, prove the exact peer PID, then send one GET or POST."""
|
|
204
|
+
if type(expected_pid) is not int or expected_pid <= 1 or platform not in {"Linux", "Darwin"}:
|
|
205
|
+
raise ValueError("pinned HTTP process identity is invalid")
|
|
206
|
+
parsed = urlsplit(url)
|
|
207
|
+
if (parsed.scheme != "http" or not parsed.hostname or not parsed.port
|
|
208
|
+
or parsed.username or parsed.password or parsed.fragment):
|
|
209
|
+
raise ValueError("pinned HTTP requires an exact local IP endpoint")
|
|
210
|
+
# Numeric addresses avoid DNS or proxy rewriting the admitted endpoint.
|
|
211
|
+
ipaddress.ip_address(parsed.hostname)
|
|
212
|
+
path = parsed.path or "/"
|
|
213
|
+
if parsed.query:
|
|
214
|
+
path += "?" + parsed.query
|
|
215
|
+
if any(ord(char) < 33 or ord(char) > 126 for char in path):
|
|
216
|
+
raise ValueError("pinned HTTP path is invalid")
|
|
217
|
+
if (not isinstance(token, str) or not 1 <= len(token) <= 4096
|
|
218
|
+
or any(ord(char) < 33 or ord(char) > 126 for char in token)):
|
|
219
|
+
raise ValueError("pinned HTTP credential format is invalid")
|
|
220
|
+
if not 0 < timeout <= 60 or not 0 < maximum_body <= 16 * 1024 * 1024:
|
|
221
|
+
raise ValueError("pinned HTTP bounds are invalid")
|
|
222
|
+
payload = b"" if body is None else json.dumps(body, separators=(",", ":"), allow_nan=False).encode()
|
|
223
|
+
if len(payload) > maximum_body:
|
|
224
|
+
raise ValueError("pinned HTTP request exceeds limit")
|
|
225
|
+
deadline = time.monotonic() + timeout
|
|
226
|
+
with socket.create_connection((parsed.hostname, parsed.port), timeout=timeout) as connection:
|
|
227
|
+
server, client = connection.getpeername()[:2], connection.getsockname()[:2]
|
|
228
|
+
while True:
|
|
229
|
+
remaining = deadline - time.monotonic()
|
|
230
|
+
if remaining <= 0:
|
|
231
|
+
raise RuntimeError("pinned HTTP connection ownership could not be proven")
|
|
232
|
+
owned = (linux_connection_owned(expected_pid, server, client) if platform == "Linux" else
|
|
233
|
+
darwin_connection_owned(expected_pid, server, client, remaining))
|
|
234
|
+
if owned:
|
|
235
|
+
break
|
|
236
|
+
time.sleep(min(0.01, max(0.0, deadline - time.monotonic())))
|
|
237
|
+
# A valid socket belonging to a stale epoch is still not the admitted
|
|
238
|
+
# native role. Recheck its service manager and owned receipt now.
|
|
239
|
+
verify_owner()
|
|
240
|
+
remaining = deadline - time.monotonic()
|
|
241
|
+
if remaining <= 0:
|
|
242
|
+
raise RuntimeError("pinned HTTP ownership verification timed out")
|
|
243
|
+
host = parsed.hostname
|
|
244
|
+
host_header = f"[{host}]:{parsed.port}" if ":" in host else f"{host}:{parsed.port}"
|
|
245
|
+
headers = (("POST" if body is not None else "GET") + " " + path + " HTTP/1.1\r\n"
|
|
246
|
+
+ "Host: " + host_header + "\r\nAuthorization: Bearer " + token
|
|
247
|
+
+ "\r\nAccept: application/json\r\nConnection: close\r\n")
|
|
248
|
+
if body is not None:
|
|
249
|
+
headers += "Content-Type: application/json\r\nContent-Length: " + str(len(payload)) + "\r\n"
|
|
250
|
+
connection.settimeout(remaining)
|
|
251
|
+
connection.sendall(headers.encode("ascii") + b"\r\n" + payload)
|
|
252
|
+
response = receive_response(connection, deadline, maximum_body)
|
|
253
|
+
result = json.loads(response)
|
|
254
|
+
if not isinstance(result, dict):
|
|
255
|
+
raise RuntimeError("pinned HTTP returned an invalid object")
|
|
256
|
+
return result
|