ccx-native 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +240 -4
- package/bin/ccx +55 -2
- package/ccx/__init__.py +167 -0
- package/ccx/__main__.py +28 -0
- package/ccx/claude/__init__.py +0 -0
- package/ccx/claude/analytics.py +466 -0
- package/ccx/claude/attach.py +807 -0
- package/ccx/claude/context.py +465 -0
- package/ccx/claude/delivery.py +806 -0
- package/ccx/claude/external.py +253 -0
- package/ccx/claude/host/__init__.py +0 -0
- package/ccx/claude/host/events.py +297 -0
- package/ccx/claude/host/hooks.py +438 -0
- package/ccx/claude/host/pty.py +269 -0
- package/ccx/claude/host/session.py +5244 -0
- package/ccx/claude/host/subagents.py +169 -0
- package/ccx/claude/host/successor.py +267 -0
- package/ccx/claude/managed_turns.py +240 -0
- package/ccx/claude/oauth.py +176 -0
- package/ccx/claude/settings.py +339 -0
- package/ccx/claude/spawn.py +557 -0
- package/ccx/claude/tasks.py +72 -0
- package/ccx/claude/telemetry.py +282 -0
- package/ccx/claude/transcript.py +2535 -0
- package/ccx/claude/usage.py +796 -0
- package/ccx/claude/usage_control.py +204 -0
- package/ccx/cli/__init__.py +0 -0
- package/ccx/cli/codex.py +1009 -0
- package/ccx/cli/commands/__init__.py +0 -0
- package/ccx/cli/commands/_common.py +407 -0
- package/ccx/cli/commands/control.py +161 -0
- package/ccx/cli/commands/create.py +669 -0
- package/ccx/cli/commands/models.py +53 -0
- package/ccx/cli/commands/read.py +1104 -0
- package/ccx/cli/commands/turns.py +1081 -0
- package/ccx/cli/controller.py +757 -0
- package/ccx/cli/main.py +302 -0
- package/ccx/cli/parser.py +1439 -0
- package/ccx/cli/skills.py +215 -0
- package/ccx/cli/statusline.py +160 -0
- package/ccx/client.py +1904 -0
- package/ccx/codex/__init__.py +0 -0
- package/ccx/codex/attach_tty.py +267 -0
- package/ccx/codex/client.py +738 -0
- package/ccx/codex/completion.py +121 -0
- package/ccx/codex/control.py +288 -0
- package/ccx/codex/daemon/__init__.py +0 -0
- package/ccx/codex/daemon/__main__.py +4 -0
- package/ccx/codex/daemon/daemon.py +1558 -0
- package/ccx/codex/daemon/models.py +140 -0
- package/ccx/codex/daemon/native.py +1590 -0
- package/ccx/codex/daemon/release.py +367 -0
- package/ccx/codex/daemon/worker.py +2424 -0
- package/ccx/codex/delivery.py +758 -0
- package/ccx/codex/delivery_recovery.py +232 -0
- package/ccx/codex/evidence.py +827 -0
- package/ccx/codex/history.py +158 -0
- package/ccx/codex/history_store.py +131 -0
- package/ccx/codex/notify_history.py +140 -0
- package/ccx/codex/ops.py +666 -0
- package/ccx/codex/paths.py +70 -0
- package/ccx/codex/protocol.py +805 -0
- package/ccx/codex/provenance.py +105 -0
- package/ccx/codex/read.py +29 -0
- package/ccx/codex/records.py +51 -0
- package/ccx/codex/reports.py +177 -0
- package/ccx/codex/runtime.py +1454 -0
- package/ccx/codex/settle.py +363 -0
- package/ccx/codex/telemetry.py +792 -0
- package/ccx/codex/types.py +659 -0
- package/ccx/codex/worker_environment.py +25 -0
- package/ccx/constants.py +288 -0
- package/ccx/controllers/__init__.py +0 -0
- package/ccx/controllers/controller.py +1449 -0
- package/ccx/controllers/live_vars.py +1706 -0
- package/ccx/controllers/live_vars_runner.py +216 -0
- package/ccx/controllers/notify.py +213 -0
- package/ccx/controllers/orchestrator.py +193 -0
- package/ccx/controllers/policy.py +57 -0
- package/ccx/controllers/store.py +681 -0
- package/ccx/errors.py +634 -0
- package/ccx/models.py +225 -0
- package/ccx/notify/__init__.py +0 -0
- package/ccx/notify/__main__.py +5 -0
- package/ccx/notify/client.py +116 -0
- package/ccx/notify/daemon.py +480 -0
- package/ccx/notify/deliver.py +304 -0
- package/ccx/notify/paths.py +51 -0
- package/ccx/notify/registry.py +649 -0
- package/ccx/notify/select.py +750 -0
- package/ccx/notify/settle.py +684 -0
- package/ccx/ops/__init__.py +0 -0
- package/ccx/ops/background.py +115 -0
- package/ccx/ops/cancel.py +296 -0
- package/ccx/ops/codex_import.py +96 -0
- package/ccx/ops/common.py +397 -0
- package/ccx/ops/compact.py +337 -0
- package/ccx/ops/compact_ops.py +550 -0
- package/ccx/ops/context_events.py +48 -0
- package/ccx/ops/conversation.py +1593 -0
- package/ccx/ops/conversation_paging.py +1339 -0
- package/ccx/ops/create.py +56 -0
- package/ccx/ops/creation.py +952 -0
- package/ccx/ops/creation_steps.py +575 -0
- package/ccx/ops/event_feed.py +710 -0
- package/ccx/ops/forking.py +711 -0
- package/ccx/ops/history.py +1041 -0
- package/ccx/ops/input.py +556 -0
- package/ccx/ops/maintenance.py +295 -0
- package/ccx/ops/report.py +773 -0
- package/ccx/ops/semantics.py +490 -0
- package/ccx/ops/send.py +961 -0
- package/ccx/ops/steer_ops.py +867 -0
- package/ccx/ops/terminate.py +383 -0
- package/ccx/ops/turn_reports.py +348 -0
- package/ccx/ops/wait.py +762 -0
- package/ccx/ops/worker_events.py +640 -0
- package/ccx/rpc/__init__.py +0 -0
- package/ccx/rpc/local_rpc.py +1602 -0
- package/ccx/rpc/rpc.py +559 -0
- package/ccx/rpc/serve.py +374 -0
- package/ccx/rpc/transport.py +409 -0
- package/ccx/rpc/view.py +125 -0
- package/ccx/store/__init__.py +0 -0
- package/ccx/store/discovery.py +170 -0
- package/ccx/store/identity.py +105 -0
- package/ccx/store/journal_io.py +79 -0
- package/ccx/store/liveness.py +194 -0
- package/ccx/store/paths.py +250 -0
- package/ccx/store/record_cache.py +40 -0
- package/ccx/store/records.py +309 -0
- package/ccx/store/root.py +789 -0
- package/ccx/store/runtime_presence.py +137 -0
- package/ccx/store/store.py +2332 -0
- package/ccx/store/worker_state.py +86 -0
- package/ccx/system/__init__.py +0 -0
- package/ccx/system/bounded.py +296 -0
- package/ccx/system/cwd_paths.py +25 -0
- package/ccx/system/engine_binary.py +59 -0
- package/ccx/system/executable.py +50 -0
- package/ccx/system/oslayer.py +667 -0
- package/ccx/system/pipes.py +46 -0
- package/ccx/system/private_json.py +54 -0
- package/ccx/system/sanitize.py +43 -0
- package/ccx/types.py +809 -0
- package/ccx/version.py +29 -0
- package/ccx/workers.py +2093 -0
- package/ccx.py +9 -0
- package/package.json +39 -7
- package/pyproject.toml +87 -0
- package/skills/ccx/SKILL.md +166 -0
- package/skills/ccx/references/controllers.md +244 -0
- package/skills/ccx/references/history.md +50 -0
- package/skills/ccx/references/troubleshooting.md +94 -0
- package/skills/ccx/references/variables.md +62 -0
|
@@ -0,0 +1,1454 @@
|
|
|
1
|
+
"""Shared, non-presentational operations for CCX-owned Codex workers.
|
|
2
|
+
|
|
3
|
+
Both ``ccx codex`` and :class:`ccx.CodexClient` use these primitives. Nothing
|
|
4
|
+
here prints, parses command output, or exits the importing process.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from ccx.version import ccx_version
|
|
10
|
+
|
|
11
|
+
import fcntl
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import socket
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Iterator
|
|
22
|
+
|
|
23
|
+
from ccx.codex.paths import (
|
|
24
|
+
codexd_lifetime_lock_path,
|
|
25
|
+
codexd_log_path,
|
|
26
|
+
codexd_sock_path,
|
|
27
|
+
codexd_spawn_lock_path,
|
|
28
|
+
ensure_dirs,
|
|
29
|
+
root as codex_state_root,
|
|
30
|
+
)
|
|
31
|
+
from ccx.constants import (
|
|
32
|
+
CODEXD_RECOVERY_OPS,
|
|
33
|
+
CODEXD_RPC_GENERATION,
|
|
34
|
+
)
|
|
35
|
+
from ccx.codex.protocol import AppServerClient, ProtocolError
|
|
36
|
+
from ccx.codex.reports import report_from_thread
|
|
37
|
+
from ccx.codex.records import read_record
|
|
38
|
+
from ccx.errors import (
|
|
39
|
+
CcxError,
|
|
40
|
+
InvalidRequest,
|
|
41
|
+
SessionNotFound,
|
|
42
|
+
SessionUnavailable,
|
|
43
|
+
SpawnFailed,
|
|
44
|
+
)
|
|
45
|
+
from ccx.store.worker_state import (
|
|
46
|
+
RUNTIME_ABSENT,
|
|
47
|
+
RUNTIME_INCONCLUSIVE,
|
|
48
|
+
RUNTIME_SERVED,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
STABLE_STATES = {"idle", "waiting", "idle-killed", "crashed", "dead"}
|
|
52
|
+
# App-server boot; a load deadline still overrides this.
|
|
53
|
+
CODEXD_BOOT_TIMEOUT_S = 60
|
|
54
|
+
# How long a caller without a deadline waits for a stopping codexd to exit. Its
|
|
55
|
+
# shutdown closes the app-server (up to three 3s waits) before it lets go.
|
|
56
|
+
CODEXD_STOP_WAIT_S = 15
|
|
57
|
+
# One presence probe's reply budget; a slower one is retried, not refused.
|
|
58
|
+
PRESENCE_PROBE_S = 5
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@contextmanager
|
|
62
|
+
def _short_lived_app_server(
|
|
63
|
+
*,
|
|
64
|
+
cwd: str,
|
|
65
|
+
title: str,
|
|
66
|
+
timeout: float,
|
|
67
|
+
rejection_prefix: str,
|
|
68
|
+
) -> Iterator[AppServerClient]:
|
|
69
|
+
"""Open one stable app-server connection for a read-only operation."""
|
|
70
|
+
from ccx.system.engine_binary import resolve_engine_binary
|
|
71
|
+
codex_bin = resolve_engine_binary("codex")
|
|
72
|
+
client: AppServerClient | None = None
|
|
73
|
+
|
|
74
|
+
def reject_request(
|
|
75
|
+
request_id: Any,
|
|
76
|
+
method: str,
|
|
77
|
+
params: dict[str, Any],
|
|
78
|
+
) -> None:
|
|
79
|
+
if client is not None:
|
|
80
|
+
client.reject_request(request_id, f"{rejection_prefix}: {method}")
|
|
81
|
+
|
|
82
|
+
try:
|
|
83
|
+
client = AppServerClient(
|
|
84
|
+
[codex_bin, "app-server", "--stdio"],
|
|
85
|
+
cwd=cwd,
|
|
86
|
+
on_notification=lambda method, params: None,
|
|
87
|
+
on_server_request=reject_request,
|
|
88
|
+
stderr=subprocess.DEVNULL,
|
|
89
|
+
)
|
|
90
|
+
client.request(
|
|
91
|
+
"initialize",
|
|
92
|
+
{
|
|
93
|
+
"clientInfo": {
|
|
94
|
+
"name": "ccx",
|
|
95
|
+
"title": title,
|
|
96
|
+
"version": ccx_version(),
|
|
97
|
+
},
|
|
98
|
+
"capabilities": {"experimentalApi": False},
|
|
99
|
+
},
|
|
100
|
+
timeout=timeout,
|
|
101
|
+
)
|
|
102
|
+
client.notify("initialized")
|
|
103
|
+
yield client
|
|
104
|
+
finally:
|
|
105
|
+
if client is not None:
|
|
106
|
+
client.close()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def list_model_entries(*, cwd: str, timeout: float = 30) -> list[dict]:
|
|
110
|
+
"""Return model/list's item dictionaries (default visibility, every page)."""
|
|
111
|
+
try:
|
|
112
|
+
with _short_lived_app_server(
|
|
113
|
+
cwd=cwd,
|
|
114
|
+
title="CCX Codex model validation",
|
|
115
|
+
timeout=timeout,
|
|
116
|
+
rejection_prefix="unsupported request during model list read",
|
|
117
|
+
) as client:
|
|
118
|
+
entries: list[dict] = []
|
|
119
|
+
cursor: str | None = None
|
|
120
|
+
seen_cursors: set[str] = set()
|
|
121
|
+
while True:
|
|
122
|
+
params = {"cursor": cursor} if cursor is not None else {}
|
|
123
|
+
page = client.request("model/list", params, timeout=timeout)
|
|
124
|
+
for item in page.get("data") or []:
|
|
125
|
+
if isinstance(item, dict):
|
|
126
|
+
entries.append(item)
|
|
127
|
+
next_cursor = page.get("nextCursor")
|
|
128
|
+
if not isinstance(next_cursor, str) or not next_cursor:
|
|
129
|
+
break
|
|
130
|
+
if next_cursor in seen_cursors:
|
|
131
|
+
raise RuntimeError("model/list returned a repeated cursor")
|
|
132
|
+
seen_cursors.add(next_cursor)
|
|
133
|
+
cursor = next_cursor
|
|
134
|
+
if not entries:
|
|
135
|
+
raise RuntimeError("model/list returned no model entries")
|
|
136
|
+
return entries
|
|
137
|
+
except (OSError, RuntimeError) as error:
|
|
138
|
+
raise SessionUnavailable(
|
|
139
|
+
f"Codex model list unavailable: {error}"
|
|
140
|
+
) from error
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def list_models(*, cwd: str, timeout: float = 30) -> list[str]:
|
|
144
|
+
"""Return every model ID advertised by Codex's stable model/list API."""
|
|
145
|
+
models: list[str] = []
|
|
146
|
+
for item in list_model_entries(cwd=cwd, timeout=timeout):
|
|
147
|
+
model_id = item.get("model") or item.get("id")
|
|
148
|
+
if isinstance(model_id, str) and model_id and model_id not in models:
|
|
149
|
+
models.append(model_id)
|
|
150
|
+
if not models:
|
|
151
|
+
raise SessionUnavailable("Codex model list unavailable: model/list returned no model IDs")
|
|
152
|
+
return models
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def validate_model(model: str, *, cwd: str, timeout: float = 30) -> list[str]:
|
|
156
|
+
"""Validate one explicit model and return the complete advertised ID list."""
|
|
157
|
+
if not isinstance(model, str) or not model:
|
|
158
|
+
raise InvalidRequest("Codex model must be a non-empty string")
|
|
159
|
+
valid_models = list_models(cwd=cwd, timeout=timeout)
|
|
160
|
+
if model in valid_models:
|
|
161
|
+
return valid_models
|
|
162
|
+
suffix_matches = [
|
|
163
|
+
value for value in valid_models if value.endswith(f"-{model}")
|
|
164
|
+
]
|
|
165
|
+
suggestion = (
|
|
166
|
+
f" Did you mean {suffix_matches[0]!r}?"
|
|
167
|
+
if len(suffix_matches) == 1
|
|
168
|
+
else ""
|
|
169
|
+
)
|
|
170
|
+
raise InvalidRequest(
|
|
171
|
+
f"unsupported Codex model {model!r}.{suggestion} "
|
|
172
|
+
f"Valid models: {', '.join(valid_models)}"
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def read_account_usage(
|
|
177
|
+
*,
|
|
178
|
+
tokens: bool = False,
|
|
179
|
+
timeout: float = 30,
|
|
180
|
+
) -> dict[str, Any]:
|
|
181
|
+
"""Read Codex account quota or token activity without creating a worker.
|
|
182
|
+
|
|
183
|
+
A running codexd answers from its already-open app-server; only when no
|
|
184
|
+
daemon owns the runtime slot is a short-lived stable connection used. The
|
|
185
|
+
read never starts the daemon.
|
|
186
|
+
"""
|
|
187
|
+
from ccx.system.engine_binary import resolve_engine_binary
|
|
188
|
+
resolve_engine_binary("codex")
|
|
189
|
+
deadline = time.monotonic() + max(0.0, timeout)
|
|
190
|
+
if observation_daemon(deadline):
|
|
191
|
+
reply = daemon_call(
|
|
192
|
+
{"op": "account-read", "tokens": tokens},
|
|
193
|
+
timeout=remaining(deadline, timeout),
|
|
194
|
+
)
|
|
195
|
+
if reply.get("ok"):
|
|
196
|
+
data = reply.get("data")
|
|
197
|
+
if isinstance(data, dict):
|
|
198
|
+
return data
|
|
199
|
+
raise SessionUnavailable(
|
|
200
|
+
"codexd returned no Codex account usage data", short="codexd"
|
|
201
|
+
)
|
|
202
|
+
# The daemon that owns the app-server answered, and its failure is the
|
|
203
|
+
# answer. Starting a second app-server beside it to get a different one
|
|
204
|
+
# would launder a daemon-side failure into a fresh read.
|
|
205
|
+
raise SessionUnavailable(
|
|
206
|
+
_served_failure(reply, "Codex usage unavailable"), short="codexd"
|
|
207
|
+
)
|
|
208
|
+
try:
|
|
209
|
+
with _short_lived_app_server(
|
|
210
|
+
cwd=os.getcwd(),
|
|
211
|
+
title="CCX Codex usage",
|
|
212
|
+
timeout=timeout,
|
|
213
|
+
rejection_prefix="unsupported account request during usage read",
|
|
214
|
+
) as client:
|
|
215
|
+
method = "account/usage/read" if tokens else "account/rateLimits/read"
|
|
216
|
+
return client.request(method, timeout=timeout)
|
|
217
|
+
except (OSError, RuntimeError) as error:
|
|
218
|
+
raise SessionUnavailable(
|
|
219
|
+
f"Codex usage unavailable: {error}"
|
|
220
|
+
) from error
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def observation_daemon(
|
|
224
|
+
deadline: float | None,
|
|
225
|
+
*,
|
|
226
|
+
short: str | None = None,
|
|
227
|
+
) -> bool:
|
|
228
|
+
"""Does the shared daemon own the runtime slot for this observation?
|
|
229
|
+
|
|
230
|
+
There is one app-server per CCX root, and codexd owns it. So if codexd is
|
|
231
|
+
answering at all, an observation goes through it: starting a second
|
|
232
|
+
app-server beside a daemon that is merely still booting would double the
|
|
233
|
+
runtime and route the read around the compatibility boundary.
|
|
234
|
+
|
|
235
|
+
Returns ``True`` when the daemon should serve the read and ``False`` only
|
|
236
|
+
when absence is positively established and a short-lived app-server is safe. A daemon that is
|
|
237
|
+
booting is waited for strictly inside the caller's own deadline — never
|
|
238
|
+
past it — and one of another generation is refused rather than bypassed.
|
|
239
|
+
"""
|
|
240
|
+
while True:
|
|
241
|
+
probe = settled_daemon_status(deadline)
|
|
242
|
+
transport = probe.get("transport")
|
|
243
|
+
if transport == RUNTIME_ABSENT:
|
|
244
|
+
return False
|
|
245
|
+
if transport != RUNTIME_SERVED:
|
|
246
|
+
raise SessionUnavailable(
|
|
247
|
+
inconclusive_presence(probe, "no second app-server was started"),
|
|
248
|
+
short=short or "codexd",
|
|
249
|
+
)
|
|
250
|
+
require_compatible_daemon(probe, short=short)
|
|
251
|
+
if probe.get("state") == "ready":
|
|
252
|
+
return True
|
|
253
|
+
if remaining(deadline, 1) <= 0:
|
|
254
|
+
raise SessionUnavailable(
|
|
255
|
+
"codexd is still booting and the read budget expired; it owns "
|
|
256
|
+
"the shared app-server, so no second one was started",
|
|
257
|
+
short=short or "codexd",
|
|
258
|
+
)
|
|
259
|
+
time.sleep(min(0.05, remaining(deadline, 0.05)))
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def remaining(deadline: float | None, maximum: float) -> float:
|
|
263
|
+
if deadline is None:
|
|
264
|
+
return maximum
|
|
265
|
+
return max(0.0, min(maximum, deadline - time.monotonic()))
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _lifetime_lock_held() -> bool | None:
|
|
269
|
+
"""Read-only positive singleton lease check, no implicit lock creation."""
|
|
270
|
+
from ccx import constants
|
|
271
|
+
from ccx.store.root import require_ccx_active
|
|
272
|
+
import stat
|
|
273
|
+
try:
|
|
274
|
+
store = require_ccx_active(constants.CCX_ROOT)
|
|
275
|
+
with store._gen("runtime-metadata", "pid") as directory:
|
|
276
|
+
try:
|
|
277
|
+
fd = os.open(
|
|
278
|
+
"codexd.lease.lock",
|
|
279
|
+
os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC,
|
|
280
|
+
dir_fd=directory,
|
|
281
|
+
)
|
|
282
|
+
except FileNotFoundError:
|
|
283
|
+
return False
|
|
284
|
+
try:
|
|
285
|
+
info = os.fstat(fd)
|
|
286
|
+
if (not stat.S_ISREG(info.st_mode)
|
|
287
|
+
or info.st_uid != os.getuid()
|
|
288
|
+
or info.st_mode & 0o077):
|
|
289
|
+
return None
|
|
290
|
+
try:
|
|
291
|
+
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
292
|
+
except BlockingIOError:
|
|
293
|
+
return True
|
|
294
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
295
|
+
return False
|
|
296
|
+
finally:
|
|
297
|
+
os.close(fd)
|
|
298
|
+
except FileNotFoundError:
|
|
299
|
+
return False
|
|
300
|
+
except (OSError, ValueError):
|
|
301
|
+
return None
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _transport_failure(error: BaseException) -> dict[str, Any]:
|
|
305
|
+
"""Classify a failed socket observation without confusing timeout with absence."""
|
|
306
|
+
if isinstance(error, (TimeoutError, socket.timeout, EOFError)):
|
|
307
|
+
# A busy machine or a daemon mid-exit: unknown, but worth asking again
|
|
308
|
+
# inside the caller's budget. Never evidence of absence.
|
|
309
|
+
return {
|
|
310
|
+
"ok": False,
|
|
311
|
+
"error": f"codexd unavailable: {error.__class__.__name__}",
|
|
312
|
+
"transport": RUNTIME_INCONCLUSIVE,
|
|
313
|
+
"retry": True,
|
|
314
|
+
}
|
|
315
|
+
elif isinstance(error, OSError) and getattr(error, "errno", None) in {
|
|
316
|
+
2, # ENOENT: no socket file
|
|
317
|
+
32, # EPIPE: the listener closed under the request
|
|
318
|
+
54, # ECONNRESET (BSD/macOS)
|
|
319
|
+
61, # ECONNREFUSED (BSD/macOS)
|
|
320
|
+
104, # ECONNRESET: a backlog connection dropped as the listener closed
|
|
321
|
+
111, # ECONNREFUSED
|
|
322
|
+
}:
|
|
323
|
+
held = _lifetime_lock_held()
|
|
324
|
+
presence = RUNTIME_ABSENT if held is False else RUNTIME_INCONCLUSIVE
|
|
325
|
+
if held is True:
|
|
326
|
+
# Nothing listens, yet a codexd still owns the runtime: one that is
|
|
327
|
+
# binding its socket or shutting down. Both resolve in seconds.
|
|
328
|
+
return {
|
|
329
|
+
"ok": False,
|
|
330
|
+
"error": f"codexd unavailable: {error.__class__.__name__}",
|
|
331
|
+
"transport": presence,
|
|
332
|
+
"transitional": True,
|
|
333
|
+
}
|
|
334
|
+
else:
|
|
335
|
+
presence = RUNTIME_INCONCLUSIVE
|
|
336
|
+
return {
|
|
337
|
+
"ok": False,
|
|
338
|
+
"error": f"codexd unavailable: {error.__class__.__name__}",
|
|
339
|
+
"transport": presence,
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
HISTORY_REPLY_OPS = frozenset({"thread-read", "thread-history", "thread-items-list", "thread-turns-list"})
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def daemon_rpc(message: dict[str, Any], timeout: float = 10, *,
|
|
347
|
+
keep_open: bool = False) -> dict[str, Any]:
|
|
348
|
+
"""Real daemon RPC: verify singleton lease and SO_PEERCRED on same socket.
|
|
349
|
+
|
|
350
|
+
Codex provider native IDs never identify the managed CCX worker, and a
|
|
351
|
+
socket pathname or prior daemon status is not evidence of this daemon's
|
|
352
|
+
PID, generation or still-held runtime role lock.
|
|
353
|
+
"""
|
|
354
|
+
from ccx import constants
|
|
355
|
+
from ccx.store.root import require_ccx_active
|
|
356
|
+
from ccx.rpc.transport import (
|
|
357
|
+
handshake_payload, plan_transport, verify_handshake,
|
|
358
|
+
)
|
|
359
|
+
if timeout <= 0:
|
|
360
|
+
return {
|
|
361
|
+
"ok": False, "error": "codexd unavailable: TimeoutError",
|
|
362
|
+
"transport": RUNTIME_INCONCLUSIVE,
|
|
363
|
+
}
|
|
364
|
+
try:
|
|
365
|
+
store = require_ccx_active(constants.CCX_ROOT)
|
|
366
|
+
plan = plan_transport(store, role="codexd")
|
|
367
|
+
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
|
|
368
|
+
client.settimeout(timeout)
|
|
369
|
+
client.connect(str(plan.final))
|
|
370
|
+
client.sendall((json.dumps({
|
|
371
|
+
"op": "handshake",
|
|
372
|
+
"identity": {"protocolVersion": 2, **plan.evidence()},
|
|
373
|
+
}) + "\n").encode())
|
|
374
|
+
# The native daemon must prove its own ACTIVE lease nonce in its
|
|
375
|
+
# reply. The request supplies only the expected root/role fields:
|
|
376
|
+
# a client cannot know a daemon nonce until the daemon presents
|
|
377
|
+
# it, and knowledge of the nonce alone never authorizes a peer.
|
|
378
|
+
data = bytearray()
|
|
379
|
+
# thread/read and thread/items/list responses are caller-requested
|
|
380
|
+
# native history, not bounded control envelopes. A large rollout or
|
|
381
|
+
# one page of big tool outputs can exceed the normal 1 MiB RPC reply
|
|
382
|
+
# guard; rejecting it here masqueraded as "codexd unavailable:
|
|
383
|
+
# ValueError".
|
|
384
|
+
def read_one(*, bounded: bool = True) -> dict:
|
|
385
|
+
while b"\n" not in data:
|
|
386
|
+
chunk = client.recv(65536)
|
|
387
|
+
if not chunk:
|
|
388
|
+
raise EOFError("Codex daemon closed socket mid-RPC")
|
|
389
|
+
data.extend(chunk)
|
|
390
|
+
if bounded and len(data) > 1024 * 1024:
|
|
391
|
+
raise ValueError("Codex daemon reply exceeds budget")
|
|
392
|
+
line, rest = data.split(b"\n", 1)
|
|
393
|
+
data[:] = rest
|
|
394
|
+
value = json.loads(line)
|
|
395
|
+
if not isinstance(value, dict):
|
|
396
|
+
raise ValueError("Codex daemon replied with non-object")
|
|
397
|
+
return value
|
|
398
|
+
hello = read_one()
|
|
399
|
+
if not hello.get("ok"):
|
|
400
|
+
raise ValueError(
|
|
401
|
+
"Codex daemon handshake refused: "
|
|
402
|
+
+ str(hello.get("error"))
|
|
403
|
+
)
|
|
404
|
+
verify_handshake(
|
|
405
|
+
store, plan, received=hello.get("identity"),
|
|
406
|
+
connected_socket=client,
|
|
407
|
+
)
|
|
408
|
+
store.revalidate()
|
|
409
|
+
client.sendall((json.dumps(message) + "\n").encode())
|
|
410
|
+
result = read_one(bounded=message.get("op") not in HISTORY_REPLY_OPS)
|
|
411
|
+
if keep_open and result.get("ok") is True:
|
|
412
|
+
return {**result, "transport": RUNTIME_SERVED,
|
|
413
|
+
"_held_fd": client.detach()}
|
|
414
|
+
return {**result, "transport": RUNTIME_SERVED}
|
|
415
|
+
except (OSError, ValueError, EOFError) as error:
|
|
416
|
+
return _transport_failure(error)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@contextmanager
|
|
420
|
+
def attached_worker(target: str, *, timeout: float = 90) -> Iterator[dict[str, Any]]:
|
|
421
|
+
"""Hold one authenticated codexd connection while the native TUI runs."""
|
|
422
|
+
from ccx import constants
|
|
423
|
+
from ccx.store.root import require_ccx_active
|
|
424
|
+
from ccx.store.identity import IdentityError
|
|
425
|
+
|
|
426
|
+
store = require_ccx_active(constants.CCX_ROOT)
|
|
427
|
+
try:
|
|
428
|
+
owner = store.resolve(target)
|
|
429
|
+
if owner.engine != "codex":
|
|
430
|
+
raise InvalidRequest("attach requires a Codex worker")
|
|
431
|
+
store.authorize_native_action(owner.worker, "resume")
|
|
432
|
+
record = store.record(owner.worker)
|
|
433
|
+
require_workspace(record)
|
|
434
|
+
except IdentityError as error:
|
|
435
|
+
raise InvalidRequest(f"Codex attach worker unavailable: {error}") from error
|
|
436
|
+
|
|
437
|
+
deadline = time.monotonic() + timeout
|
|
438
|
+
ensure_daemon(deadline=deadline)
|
|
439
|
+
reply = daemon_rpc({
|
|
440
|
+
"op": "attach", "short": owner.worker_id,
|
|
441
|
+
"rpcGeneration": CODEXD_RPC_GENERATION,
|
|
442
|
+
}, timeout=max(.1, deadline - time.monotonic()), keep_open=True)
|
|
443
|
+
fd = reply.pop("_held_fd", None)
|
|
444
|
+
if fd is None:
|
|
445
|
+
if reply.get("transport") == RUNTIME_SERVED:
|
|
446
|
+
require_served_generation(reply, short=owner.short)
|
|
447
|
+
raise SessionUnavailable(
|
|
448
|
+
f"Codex attach unavailable: {reply.get('error') or reply.get('state') or reply.get('transport')}",
|
|
449
|
+
short=owner.short)
|
|
450
|
+
with socket.socket(fileno=fd):
|
|
451
|
+
require_served_generation(reply, short=owner.short)
|
|
452
|
+
store.revalidate()
|
|
453
|
+
args = record.get("createArgs") or {}
|
|
454
|
+
yield {**reply, "workerId": owner.worker_id,
|
|
455
|
+
"model": args.get("model"), "effort": args.get("effort"),
|
|
456
|
+
"hasTurn": bool(record.get("lastTurn"))}
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
# Inspection and teardown stay available on an incompatible daemon: they are
|
|
460
|
+
# how an operator sees it and gets rid of it. Everything else is refused.
|
|
461
|
+
RECOVERY_OPS = CODEXD_RECOVERY_OPS
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def daemon_call(
|
|
465
|
+
message: dict[str, Any],
|
|
466
|
+
timeout: float = 10,
|
|
467
|
+
*,
|
|
468
|
+
short: str | None = None,
|
|
469
|
+
) -> dict[str, Any]:
|
|
470
|
+
"""Send a non-recovery request that declares this client's RPC generation.
|
|
471
|
+
|
|
472
|
+
The declaration is what lets a current daemon refuse an old client before
|
|
473
|
+
running the handler, so it belongs on every request that can change
|
|
474
|
+
anything — never on the recovery operations that must keep answering an
|
|
475
|
+
incompatible peer.
|
|
476
|
+
|
|
477
|
+
Whatever comes back is held to the same standard in reverse. Once a request
|
|
478
|
+
has actually been written, a served envelope must declare this generation
|
|
479
|
+
whether it reports success or failure: a daemon replaced between the
|
|
480
|
+
preflight and the write could otherwise return an old ``ok: false`` and
|
|
481
|
+
have its application error read as a current one.
|
|
482
|
+
"""
|
|
483
|
+
reply = daemon_rpc(
|
|
484
|
+
{**message, "rpcGeneration": CODEXD_RPC_GENERATION}, timeout
|
|
485
|
+
)
|
|
486
|
+
if reply.get("transport") != RUNTIME_SERVED:
|
|
487
|
+
return reply
|
|
488
|
+
return require_served_generation(reply, short=short)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def daemon_generation(reply: dict[str, Any]) -> int | None:
|
|
492
|
+
"""The RPC generation an answering daemon declared, if it declared one."""
|
|
493
|
+
generation = reply.get("rpcGeneration")
|
|
494
|
+
return generation if isinstance(generation, int) else None
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def require_served_generation(
|
|
498
|
+
reply: dict[str, Any],
|
|
499
|
+
*,
|
|
500
|
+
short: str | None = None,
|
|
501
|
+
) -> dict[str, Any]:
|
|
502
|
+
"""Refuse an envelope a codexd of another RPC generation served us.
|
|
503
|
+
|
|
504
|
+
A daemon that predates the generation field declares nothing and is treated
|
|
505
|
+
as incompatible: its replies are missing members current callers rely on,
|
|
506
|
+
and using them anyway is how a successful turn gets reported as a failure —
|
|
507
|
+
or an old rejection as a current one. Success is irrelevant here; only who
|
|
508
|
+
answered is.
|
|
509
|
+
"""
|
|
510
|
+
generation = daemon_generation(reply)
|
|
511
|
+
if generation != CODEXD_RPC_GENERATION:
|
|
512
|
+
raise SessionUnavailable(
|
|
513
|
+
"the running codexd speaks RPC generation "
|
|
514
|
+
f"{generation if generation is not None else 'unknown (legacy)'}, "
|
|
515
|
+
f"but this CCX speaks {CODEXD_RPC_GENERATION}. Finish or kill its "
|
|
516
|
+
"workers and run `ccx shutdown --engine codex` to replace it.",
|
|
517
|
+
short=short or "codexd",
|
|
518
|
+
)
|
|
519
|
+
return reply
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def require_compatible_daemon(
|
|
523
|
+
reply: dict[str, Any],
|
|
524
|
+
*,
|
|
525
|
+
short: str | None = None,
|
|
526
|
+
) -> dict[str, Any]:
|
|
527
|
+
"""Hold a raw availability probe to this RPC generation.
|
|
528
|
+
|
|
529
|
+
Only a reply that reports success proves a daemon is there to be
|
|
530
|
+
incompatible with: a not-ok probe mutated nothing and may simply mean no
|
|
531
|
+
daemon, so those paths stay free to fall through to :func:`ensure_daemon`,
|
|
532
|
+
which knows how to replace an idle one. Requests that were actually written
|
|
533
|
+
are stricter — see :func:`require_served_generation`.
|
|
534
|
+
"""
|
|
535
|
+
if reply.get("transport") != RUNTIME_SERVED:
|
|
536
|
+
return reply
|
|
537
|
+
return require_served_generation(reply, short=short)
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def preflight_daemon(
|
|
541
|
+
deadline: float | None,
|
|
542
|
+
*,
|
|
543
|
+
short: str | None = None,
|
|
544
|
+
) -> dict[str, Any] | None:
|
|
545
|
+
"""Establish daemon compatibility before a mutating operation is written.
|
|
546
|
+
|
|
547
|
+
Returns ``None`` once the answering daemon has positively declared this
|
|
548
|
+
RPC generation and the positive-absence reply when no daemon owns the
|
|
549
|
+
runtime. An inconclusive probe raises: it must never authorize mutation or
|
|
550
|
+
daemon replacement.
|
|
551
|
+
"""
|
|
552
|
+
probe = settled_daemon_status(deadline)
|
|
553
|
+
if probe.get("transport") == RUNTIME_ABSENT:
|
|
554
|
+
return probe
|
|
555
|
+
if probe.get("transport") != RUNTIME_SERVED:
|
|
556
|
+
raise SessionUnavailable(
|
|
557
|
+
inconclusive_presence(probe, "nothing was sent"),
|
|
558
|
+
short=short or "codexd",
|
|
559
|
+
)
|
|
560
|
+
require_compatible_daemon(probe, short=short)
|
|
561
|
+
return None
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def rpc(
|
|
565
|
+
short: str,
|
|
566
|
+
message: dict[str, Any],
|
|
567
|
+
timeout: float = 10,
|
|
568
|
+
) -> dict[str, Any]:
|
|
569
|
+
"""Send a worker-addressed operation to the shared codexd daemon.
|
|
570
|
+
|
|
571
|
+
Every operation that is not a recovery operation is fenced on the daemon's
|
|
572
|
+
RPC generation before it is written, so an incompatible daemon never
|
|
573
|
+
processes a steer, an approval, or a turn and leaves the caller to notice
|
|
574
|
+
afterwards. ``timeout`` is the end-to-end budget for the preflight and the
|
|
575
|
+
operation together; the reply-side check stays as a guard against a daemon
|
|
576
|
+
that is replaced between the two.
|
|
577
|
+
"""
|
|
578
|
+
if message.get("op") in RECOVERY_OPS:
|
|
579
|
+
return daemon_rpc({**message, "short": short}, timeout)
|
|
580
|
+
deadline = time.monotonic() + max(0.0, timeout)
|
|
581
|
+
unreachable = preflight_daemon(deadline, short=short)
|
|
582
|
+
if unreachable is not None:
|
|
583
|
+
return unreachable
|
|
584
|
+
return daemon_call(
|
|
585
|
+
{**message, "short": short},
|
|
586
|
+
timeout=remaining(deadline, timeout),
|
|
587
|
+
short=short,
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def daemon_status(timeout: float = 2, *, presence_only: bool = False) -> dict[str, Any]:
|
|
592
|
+
"""Probe the daemon. Deliberately unchecked: status is a recovery tool.
|
|
593
|
+
|
|
594
|
+
``presence_only`` skips the per-worker states, which take every worker's
|
|
595
|
+
lock; a daemon that predates the flag ignores it and lists them anyway.
|
|
596
|
+
"""
|
|
597
|
+
message: dict[str, Any] = {"op": "daemon-status"}
|
|
598
|
+
if presence_only:
|
|
599
|
+
message["workers"] = False
|
|
600
|
+
return daemon_rpc(message, timeout=timeout)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _stopping(probe: dict[str, Any]) -> bool:
|
|
604
|
+
"""A codexd is between owning the runtime and having let go of it."""
|
|
605
|
+
if probe.get("transport") == RUNTIME_SERVED:
|
|
606
|
+
return probe.get("state") == "stopping"
|
|
607
|
+
return probe.get("transport") == RUNTIME_INCONCLUSIVE and bool(
|
|
608
|
+
probe.get("transitional")
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def settled_daemon_status(deadline: float | None) -> dict[str, Any]:
|
|
613
|
+
"""Probe the daemon, waiting out one that is slow, binding or shutting down.
|
|
614
|
+
|
|
615
|
+
A stopping codexd refuses new loads and is about to release the runtime,
|
|
616
|
+
so neither "use it" nor "start another" is right yet; a probe that timed
|
|
617
|
+
out says nothing either way. Both are asked again until the caller's
|
|
618
|
+
deadline (or ``CODEXD_STOP_WAIT_S``); what is still unknown then is
|
|
619
|
+
returned as the inconclusive probe it is, with how long it was waited for.
|
|
620
|
+
"""
|
|
621
|
+
started = time.monotonic()
|
|
622
|
+
limit = deadline if deadline is not None else started + CODEXD_STOP_WAIT_S
|
|
623
|
+
attempts = 0
|
|
624
|
+
while True:
|
|
625
|
+
attempts += 1
|
|
626
|
+
probe = daemon_status(timeout=remaining(limit, PRESENCE_PROBE_S),
|
|
627
|
+
presence_only=True)
|
|
628
|
+
now = time.monotonic()
|
|
629
|
+
if not (_stopping(probe) or probe.get("retry")) or now >= limit:
|
|
630
|
+
if probe.get("transport") == RUNTIME_INCONCLUSIVE:
|
|
631
|
+
probe = {**probe, "attempts": attempts,
|
|
632
|
+
"waitedSeconds": round(now - started, 1)}
|
|
633
|
+
return probe
|
|
634
|
+
time.sleep(min(0.1, max(0.0, limit - now)))
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def inconclusive_presence(probe: dict[str, Any], outcome: str) -> str:
|
|
638
|
+
"""The refusal for a presence probe that never settled, with its evidence."""
|
|
639
|
+
error = str(probe.get("error") or "no reply").removeprefix("codexd unavailable: ")
|
|
640
|
+
attempts = probe.get("attempts") or 1
|
|
641
|
+
tried = (f" after {attempts} probes over {probe.get('waitedSeconds')}s"
|
|
642
|
+
if attempts > 1 else "")
|
|
643
|
+
return f"codexd presence is inconclusive ({error}{tried}); {outcome}"
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def observe_worker(
|
|
647
|
+
short: str,
|
|
648
|
+
*,
|
|
649
|
+
timeout: float = 1.0,
|
|
650
|
+
) -> dict[str, Any]:
|
|
651
|
+
"""Canonical live observation for one Codex worker."""
|
|
652
|
+
reply = daemon_rpc({"op": "status", "short": short}, timeout=max(0.0, timeout))
|
|
653
|
+
transport = reply.get("transport")
|
|
654
|
+
if transport == RUNTIME_ABSENT:
|
|
655
|
+
return {
|
|
656
|
+
"presence": RUNTIME_ABSENT,
|
|
657
|
+
"daemon_served": False,
|
|
658
|
+
"reply": reply,
|
|
659
|
+
}
|
|
660
|
+
if transport != RUNTIME_SERVED:
|
|
661
|
+
return {
|
|
662
|
+
"presence": RUNTIME_INCONCLUSIVE,
|
|
663
|
+
"daemon_served": False,
|
|
664
|
+
"reply": reply,
|
|
665
|
+
}
|
|
666
|
+
if daemon_generation(reply) != CODEXD_RPC_GENERATION:
|
|
667
|
+
return {
|
|
668
|
+
"presence": RUNTIME_INCONCLUSIVE,
|
|
669
|
+
"daemon_served": True,
|
|
670
|
+
"reply": reply,
|
|
671
|
+
}
|
|
672
|
+
if reply.get("ok"):
|
|
673
|
+
return {
|
|
674
|
+
"presence": RUNTIME_SERVED,
|
|
675
|
+
"daemon_served": True,
|
|
676
|
+
"reply": reply,
|
|
677
|
+
}
|
|
678
|
+
if reply.get("loaded") is False:
|
|
679
|
+
return {
|
|
680
|
+
"presence": RUNTIME_ABSENT,
|
|
681
|
+
"daemon_served": True,
|
|
682
|
+
"reply": reply,
|
|
683
|
+
}
|
|
684
|
+
return {
|
|
685
|
+
"presence": RUNTIME_INCONCLUSIVE,
|
|
686
|
+
"daemon_served": True,
|
|
687
|
+
"reply": reply,
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def observe_registry(*, timeout: float = 1.0) -> dict[str, Any]:
|
|
692
|
+
"""Observe the shared daemon registry once for list-style projection."""
|
|
693
|
+
reply = daemon_status(timeout=timeout)
|
|
694
|
+
transport = reply.get("transport")
|
|
695
|
+
if transport == RUNTIME_ABSENT:
|
|
696
|
+
return {"presence": RUNTIME_ABSENT, "workers": {}, "reply": reply}
|
|
697
|
+
if (
|
|
698
|
+
transport == RUNTIME_SERVED
|
|
699
|
+
and daemon_generation(reply) == CODEXD_RPC_GENERATION
|
|
700
|
+
and reply.get("ok")
|
|
701
|
+
):
|
|
702
|
+
workers = reply.get("workers")
|
|
703
|
+
return {
|
|
704
|
+
"presence": RUNTIME_SERVED,
|
|
705
|
+
"workers": workers if isinstance(workers, dict) else {},
|
|
706
|
+
"reply": reply,
|
|
707
|
+
}
|
|
708
|
+
return {"presence": RUNTIME_INCONCLUSIVE, "workers": {}, "reply": reply}
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def explain_native_failure(message: str) -> str:
|
|
712
|
+
"""Put a plain-language cause in front of known native Codex failures."""
|
|
713
|
+
if "missing source rollout" in message:
|
|
714
|
+
return ("the Codex thread this worker was forked or imported from was "
|
|
715
|
+
f"deleted, so its history can't be read ({message})")
|
|
716
|
+
return message
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def _served_failure(reply: dict[str, Any], fallback: str) -> str:
|
|
720
|
+
"""The most specific message a failed daemon reply carries.
|
|
721
|
+
|
|
722
|
+
A current daemon attaches the native ``ProtocolError`` as ``protocolError``;
|
|
723
|
+
an envelope without one only carries the flattened string.
|
|
724
|
+
"""
|
|
725
|
+
structured = reply.get("protocolError")
|
|
726
|
+
if isinstance(structured, dict) and isinstance(structured.get("message"), str):
|
|
727
|
+
return explain_native_failure(structured["message"])
|
|
728
|
+
return explain_native_failure(str(reply.get("error") or fallback))
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _history_failure(error: BaseException, thread_id: str) -> CcxError:
|
|
732
|
+
message = explain_native_failure(getattr(error, "message", str(error)))
|
|
733
|
+
if _is_missing_history(message):
|
|
734
|
+
return SessionNotFound(message, short=thread_id)
|
|
735
|
+
return SessionUnavailable(message, short=thread_id)
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _is_missing_history(error: Any) -> bool:
|
|
739
|
+
"""Recognize native missing-thread errors by message, never by code.
|
|
740
|
+
|
|
741
|
+
Codex 0.150.1 answers both a missing thread ("thread not loaded: <id>")
|
|
742
|
+
and a malformed one with the same broad ``-32600``, so the code proves
|
|
743
|
+
nothing. These phrases are specific enough that a model, config, or
|
|
744
|
+
resource failure cannot match them; anything else stays unavailable.
|
|
745
|
+
"""
|
|
746
|
+
message = getattr(error, "message", str(error)).lower()
|
|
747
|
+
return (
|
|
748
|
+
"thread not loaded:" in message
|
|
749
|
+
or "unknown thread" in message
|
|
750
|
+
or "no rollout found" in message
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def observation_cwd(cwd: Any) -> str:
|
|
755
|
+
"""The directory a read-only app-server may be launched in.
|
|
756
|
+
|
|
757
|
+
``thread/read`` is addressed by native thread id and never uses the launch
|
|
758
|
+
directory, so a deleted worktree must not also destroy access to persisted
|
|
759
|
+
history. Only observation falls back, and only to CCX's own state root; the
|
|
760
|
+
fallback is never written back to the worker record, the native thread's
|
|
761
|
+
own cwd, or conversation metadata. Starting project work is the opposite
|
|
762
|
+
policy and lives in :func:`require_workspace`.
|
|
763
|
+
"""
|
|
764
|
+
if isinstance(cwd, str) and cwd and os.path.isdir(cwd):
|
|
765
|
+
return cwd
|
|
766
|
+
neutral = codex_state_root()
|
|
767
|
+
if not neutral.is_dir():
|
|
768
|
+
raise SessionUnavailable(
|
|
769
|
+
"cannot read Codex history: the worker workspace is gone and the "
|
|
770
|
+
f"CCX state directory {neutral} does not exist"
|
|
771
|
+
)
|
|
772
|
+
return str(neutral)
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def require_workspace(value: dict[str, Any]) -> str:
|
|
776
|
+
"""The worker's recorded project directory, which must still exist.
|
|
777
|
+
|
|
778
|
+
Native ``thread/resume`` will happily reattach a persisted thread whose
|
|
779
|
+
``cwd`` is gone, but CCX will not start or restart project work in a
|
|
780
|
+
directory that no longer exists: the turn would run somewhere the record
|
|
781
|
+
does not describe, and its shell commands would land in whatever the
|
|
782
|
+
app-server falls back to. This is a deliberate CCX policy, stricter than
|
|
783
|
+
the native one, and it gates every operation that starts native work on
|
|
784
|
+
the thread — ``resume``, ``send``, and ``compact``. Creation is
|
|
785
|
+
independently strict about its own ``--cwd``.
|
|
786
|
+
|
|
787
|
+
Read-only observation keeps working without the workspace (see
|
|
788
|
+
:func:`observation_cwd`): report, conversation, export, ``thread/read``,
|
|
789
|
+
and status. So does controlling a turn that is already running — steer,
|
|
790
|
+
approve, reject, interrupt, stop, kill — because a workspace that
|
|
791
|
+
disappears mid-turn must not trap the turn.
|
|
792
|
+
"""
|
|
793
|
+
cwd = value.get("cwd")
|
|
794
|
+
if isinstance(cwd, str) and cwd and os.path.isdir(cwd):
|
|
795
|
+
return cwd
|
|
796
|
+
raise SessionUnavailable(
|
|
797
|
+
f"the Codex worker workspace is gone: {cwd!r}. History stays readable "
|
|
798
|
+
f"(`ccx report`, `conversation`, `export`), but resuming or "
|
|
799
|
+
f"sending needs the recorded directory to exist.",
|
|
800
|
+
short=value.get("short"),
|
|
801
|
+
)
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
def read_thread(
|
|
805
|
+
thread_id: str,
|
|
806
|
+
*,
|
|
807
|
+
cwd: str,
|
|
808
|
+
include_turns: bool = True,
|
|
809
|
+
timeout: float = 30,
|
|
810
|
+
worker_id: str | None = None,
|
|
811
|
+
expected_source: str | None = None,
|
|
812
|
+
limit: int | None = None,
|
|
813
|
+
newest_first: bool = False,
|
|
814
|
+
exact_turn_id: str | None = None,
|
|
815
|
+
) -> dict[str, Any]:
|
|
816
|
+
"""Read source-verified Codex metadata, optionally assembling paged turns."""
|
|
817
|
+
if not isinstance(thread_id, str) or not thread_id:
|
|
818
|
+
raise SessionNotFound("Codex conversation is unavailable; check worker status")
|
|
819
|
+
if timeout <= 0:
|
|
820
|
+
raise InvalidRequest("history read timeout must be greater than zero")
|
|
821
|
+
if worker_id is not None and (not isinstance(expected_source, str) or not expected_source):
|
|
822
|
+
raise InvalidRequest("Codex conversation source is not verified; inspect the worker")
|
|
823
|
+
deadline = time.monotonic() + timeout
|
|
824
|
+
message = {"threadId": thread_id, "includeTurns": False}
|
|
825
|
+
from ccx.codex.history import fill_thread
|
|
826
|
+
# Never start a second app-server beside a daemon that owns the runtime
|
|
827
|
+
# slot — including one still booting — nor to skip a compatibility problem
|
|
828
|
+
# the operator needs to see.
|
|
829
|
+
if observation_daemon(deadline, short=thread_id):
|
|
830
|
+
budget = remaining(deadline, timeout)
|
|
831
|
+
reply = daemon_call({
|
|
832
|
+
"op": "thread-history" if include_turns else "thread-read",
|
|
833
|
+
**message,
|
|
834
|
+
**({"workerId": worker_id} if worker_id is not None else {}),
|
|
835
|
+
**({"limit": limit} if limit is not None else {}),
|
|
836
|
+
**({"newestFirst": True} if newest_first else {}),
|
|
837
|
+
**({"turnId": exact_turn_id} if exact_turn_id else {}),
|
|
838
|
+
"observeSeconds": budget,
|
|
839
|
+
}, timeout=budget, short=thread_id)
|
|
840
|
+
if not reply.get("ok"):
|
|
841
|
+
native = _served_failure(reply, "Codex history read failed")
|
|
842
|
+
if _is_missing_history(native):
|
|
843
|
+
raise SessionNotFound(native, short=thread_id)
|
|
844
|
+
raise SessionUnavailable(native, short=thread_id)
|
|
845
|
+
thread = reply.get("thread")
|
|
846
|
+
if not isinstance(thread, dict):
|
|
847
|
+
raise SessionNotFound("no Codex thread available", short=thread_id)
|
|
848
|
+
if worker_id is not None and (
|
|
849
|
+
reply.get("workerId") != worker_id
|
|
850
|
+
or reply.get("threadId") != thread_id
|
|
851
|
+
or thread.get("id") != thread_id
|
|
852
|
+
or thread.get("threadSource") != expected_source
|
|
853
|
+
):
|
|
854
|
+
raise SessionUnavailable(
|
|
855
|
+
"Codex history belongs to another worker; inspect the handle", short=thread_id,
|
|
856
|
+
)
|
|
857
|
+
return thread
|
|
858
|
+
|
|
859
|
+
try:
|
|
860
|
+
with _short_lived_app_server(
|
|
861
|
+
cwd=observation_cwd(cwd),
|
|
862
|
+
title="CCX Codex history read",
|
|
863
|
+
timeout=remaining(deadline, timeout),
|
|
864
|
+
rejection_prefix="unsupported request during history read",
|
|
865
|
+
) as client:
|
|
866
|
+
result = client.request(
|
|
867
|
+
"thread/read", message, timeout=remaining(deadline, timeout)
|
|
868
|
+
)
|
|
869
|
+
thread = result.get("thread")
|
|
870
|
+
if not isinstance(thread, dict):
|
|
871
|
+
raise SessionNotFound("no Codex thread available", short=thread_id)
|
|
872
|
+
if worker_id is not None and (
|
|
873
|
+
thread.get("id") != thread_id
|
|
874
|
+
or thread.get("threadSource") != expected_source
|
|
875
|
+
):
|
|
876
|
+
raise SessionUnavailable(
|
|
877
|
+
"Codex history belongs to another worker; inspect the handle", short=thread_id,
|
|
878
|
+
)
|
|
879
|
+
if not include_turns:
|
|
880
|
+
return thread
|
|
881
|
+
return fill_thread(thread, client.request, deadline=deadline,
|
|
882
|
+
limit=limit, newest_first=newest_first,
|
|
883
|
+
exact_turn_id=exact_turn_id)
|
|
884
|
+
except (ProtocolError, OSError) as error:
|
|
885
|
+
raise _history_failure(error, thread_id) from error
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def read_import_source(thread_id: str, *, timeout: float = 20.) -> dict:
|
|
889
|
+
"""Read one external native thread's last completed-point candidate.
|
|
890
|
+
|
|
891
|
+
No stored owner is required, and this operation never resumes the source.
|
|
892
|
+
A running CCX app-server must serve it; an absent one permits one short
|
|
893
|
+
lived observer without access to the machine control socket.
|
|
894
|
+
"""
|
|
895
|
+
import os
|
|
896
|
+
from ccx.codex.history import read_fork_source_point
|
|
897
|
+
if not isinstance(thread_id, str):
|
|
898
|
+
raise InvalidRequest("Codex import requires a full native thread UUID")
|
|
899
|
+
deadline = time.monotonic() + timeout
|
|
900
|
+
if observation_daemon(deadline, short=thread_id):
|
|
901
|
+
reply = daemon_call({
|
|
902
|
+
"op": "import-source-point", "threadId": thread_id,
|
|
903
|
+
"observeSeconds": remaining(deadline, timeout),
|
|
904
|
+
}, timeout=remaining(deadline, timeout), short=thread_id)
|
|
905
|
+
if not reply.get("ok"):
|
|
906
|
+
error = _served_failure(reply, "Codex import source unavailable")
|
|
907
|
+
if _is_missing_history(error):
|
|
908
|
+
raise SessionNotFound(error, short=thread_id)
|
|
909
|
+
raise SessionUnavailable(error, short=thread_id)
|
|
910
|
+
return {"thread": reply["thread"], "lastTurn": reply["lastTurn"]}
|
|
911
|
+
try:
|
|
912
|
+
with _short_lived_app_server(
|
|
913
|
+
cwd=os.getcwd(), title="CCX Codex native import source read",
|
|
914
|
+
timeout=remaining(deadline, timeout),
|
|
915
|
+
rejection_prefix="unsupported request during Codex native import",
|
|
916
|
+
) as client:
|
|
917
|
+
return read_fork_source_point(
|
|
918
|
+
client.request, thread_id, deadline=deadline,
|
|
919
|
+
)
|
|
920
|
+
except ProtocolError as error:
|
|
921
|
+
raise _history_failure(error, thread_id) from error
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def list_thread_turns(
|
|
925
|
+
thread_id: str,
|
|
926
|
+
*,
|
|
927
|
+
cwd: str,
|
|
928
|
+
cursor: str | None = None,
|
|
929
|
+
limit: int = 100,
|
|
930
|
+
sort_direction: str = "asc",
|
|
931
|
+
items_view: str = "full",
|
|
932
|
+
timeout: float = 30,
|
|
933
|
+
worker_id: str | None = None,
|
|
934
|
+
expected_source: str | None = None,
|
|
935
|
+
) -> dict[str, Any]:
|
|
936
|
+
"""One owner-fenced native turn page, including original statuses."""
|
|
937
|
+
from ccx.store.identity import InvalidTarget, canonical_worker_id
|
|
938
|
+
|
|
939
|
+
try:
|
|
940
|
+
canonical_worker_id(worker_id)
|
|
941
|
+
except InvalidTarget as exc:
|
|
942
|
+
raise InvalidRequest("Codex history needs a complete worker handle") from exc
|
|
943
|
+
if not isinstance(expected_source, str) or not expected_source:
|
|
944
|
+
raise InvalidRequest("Codex history source is not verified; restart paging")
|
|
945
|
+
if not isinstance(thread_id, str) or not thread_id:
|
|
946
|
+
raise SessionNotFound("Codex conversation is unavailable; check worker status")
|
|
947
|
+
if type(limit) is not int or not 1 <= limit <= 100:
|
|
948
|
+
raise InvalidRequest("thread turn limit must be between 1 and 100")
|
|
949
|
+
if (not isinstance(sort_direction, str) or sort_direction not in {"asc", "desc"}
|
|
950
|
+
or not isinstance(items_view, str) or items_view not in {"notLoaded", "summary", "full"}):
|
|
951
|
+
raise InvalidRequest("invalid Codex turn page direction or items view")
|
|
952
|
+
if cursor is not None and (not isinstance(cursor, str) or not cursor):
|
|
953
|
+
raise InvalidRequest("native turn cursor must be non-empty")
|
|
954
|
+
params: dict[str, Any] = {
|
|
955
|
+
"threadId": thread_id, "limit": limit,
|
|
956
|
+
"sortDirection": sort_direction, "itemsView": items_view,
|
|
957
|
+
}
|
|
958
|
+
if cursor is not None:
|
|
959
|
+
params["cursor"] = cursor
|
|
960
|
+
return _paged_history_request(
|
|
961
|
+
method="thread/turns/list", daemon_op="thread-turns-list", params=params,
|
|
962
|
+
cwd=cwd, timeout=timeout, short=thread_id, worker_id=worker_id,
|
|
963
|
+
expected_source=expected_source,
|
|
964
|
+
)
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
def list_thread_items(
|
|
968
|
+
thread_id: str,
|
|
969
|
+
*,
|
|
970
|
+
cwd: str,
|
|
971
|
+
cursor: str | None = None,
|
|
972
|
+
limit: int = 25,
|
|
973
|
+
sort_direction: str = "desc",
|
|
974
|
+
timeout: float = 30,
|
|
975
|
+
turn_id: str | None = None,
|
|
976
|
+
worker_id: str | None = None,
|
|
977
|
+
expected_source: str | None = None,
|
|
978
|
+
) -> dict[str, Any]:
|
|
979
|
+
"""Read original full-worker Codex items without loading or resuming.
|
|
980
|
+
|
|
981
|
+
A native thread ID alone cannot authorize a page. The running daemon
|
|
982
|
+
independently proves the CCX owner; a cold read proves threadSource first
|
|
983
|
+
on the same observer connection, without materializing native turns.
|
|
984
|
+
"""
|
|
985
|
+
from ccx.store.identity import InvalidTarget, canonical_worker_id
|
|
986
|
+
|
|
987
|
+
try:
|
|
988
|
+
canonical_worker_id(worker_id)
|
|
989
|
+
except InvalidTarget as exc:
|
|
990
|
+
raise InvalidRequest("Codex history needs a complete worker handle") from exc
|
|
991
|
+
if not isinstance(expected_source, str) or not expected_source:
|
|
992
|
+
raise InvalidRequest("Codex history source is not verified; restart paging")
|
|
993
|
+
if not isinstance(thread_id, str) or not thread_id:
|
|
994
|
+
raise SessionNotFound("Codex conversation is unavailable; check worker status")
|
|
995
|
+
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or timeout <= 0:
|
|
996
|
+
raise InvalidRequest("history read timeout must be greater than zero")
|
|
997
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 100:
|
|
998
|
+
raise InvalidRequest("thread item limit must be between 1 and 100")
|
|
999
|
+
if not isinstance(sort_direction, str) or sort_direction not in {"asc", "desc"}:
|
|
1000
|
+
raise InvalidRequest("thread item sort direction must be asc or desc")
|
|
1001
|
+
if cursor is not None and (not isinstance(cursor, str) or not cursor):
|
|
1002
|
+
raise InvalidRequest("native item cursor must be a non-empty string")
|
|
1003
|
+
|
|
1004
|
+
params: dict[str, Any] = {
|
|
1005
|
+
"threadId": thread_id,
|
|
1006
|
+
"limit": limit,
|
|
1007
|
+
"sortDirection": sort_direction,
|
|
1008
|
+
}
|
|
1009
|
+
if cursor is not None:
|
|
1010
|
+
params["cursor"] = cursor
|
|
1011
|
+
if turn_id is not None:
|
|
1012
|
+
if not isinstance(turn_id, str) or not turn_id:
|
|
1013
|
+
raise InvalidRequest("native item turn id must be non-empty")
|
|
1014
|
+
params["turnId"] = turn_id
|
|
1015
|
+
return _paged_history_request(
|
|
1016
|
+
method="thread/items/list", daemon_op="thread-items-list", params=params,
|
|
1017
|
+
cwd=cwd, timeout=timeout, short=thread_id, worker_id=worker_id,
|
|
1018
|
+
expected_source=expected_source,
|
|
1019
|
+
)
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def _paged_history_request(
|
|
1023
|
+
*,
|
|
1024
|
+
method: str,
|
|
1025
|
+
daemon_op: str,
|
|
1026
|
+
params: dict[str, Any],
|
|
1027
|
+
cwd: str,
|
|
1028
|
+
timeout: float,
|
|
1029
|
+
short: str | None,
|
|
1030
|
+
worker_id: str,
|
|
1031
|
+
expected_source: str,
|
|
1032
|
+
) -> dict[str, Any]:
|
|
1033
|
+
"""Read one original source-attested page within the caller's deadline."""
|
|
1034
|
+
from ccx.errors import OperationTimeout
|
|
1035
|
+
|
|
1036
|
+
if (method, daemon_op) not in {
|
|
1037
|
+
("thread/items/list", "thread-items-list"),
|
|
1038
|
+
("thread/turns/list", "thread-turns-list"),
|
|
1039
|
+
}:
|
|
1040
|
+
raise InvalidRequest("Only CCX-managed Codex workers can be read; use a worker handle")
|
|
1041
|
+
deadline = time.monotonic() + timeout
|
|
1042
|
+
thread_id = params["threadId"]
|
|
1043
|
+
source = expected_source
|
|
1044
|
+
if observation_daemon(deadline, short=short):
|
|
1045
|
+
remaining_time = remaining(deadline, timeout)
|
|
1046
|
+
if remaining_time <= 0:
|
|
1047
|
+
raise OperationTimeout("Codex item-page deadline expired", short=short)
|
|
1048
|
+
reply = daemon_call(
|
|
1049
|
+
{"op": daemon_op, "workerId": worker_id, "params": params},
|
|
1050
|
+
timeout=remaining_time, short=short,
|
|
1051
|
+
)
|
|
1052
|
+
if not reply.get("ok"):
|
|
1053
|
+
native = _served_failure(reply, f"{method} failed")
|
|
1054
|
+
if reply.get("reason") == "worker-not-found" or _is_missing_history(native):
|
|
1055
|
+
raise SessionNotFound(native, short=short)
|
|
1056
|
+
raise SessionUnavailable(native, short=short)
|
|
1057
|
+
if (reply.get("workerId") != worker_id
|
|
1058
|
+
or reply.get("threadId") != thread_id
|
|
1059
|
+
or reply.get("threadSource") != source):
|
|
1060
|
+
raise SessionUnavailable(
|
|
1061
|
+
"Codex history page belongs to another worker; restart paging", short=short,
|
|
1062
|
+
)
|
|
1063
|
+
page = reply.get("data")
|
|
1064
|
+
else:
|
|
1065
|
+
try:
|
|
1066
|
+
remaining_time = remaining(deadline, timeout)
|
|
1067
|
+
if remaining_time <= 0:
|
|
1068
|
+
raise OperationTimeout("Codex item-page deadline expired", short=short)
|
|
1069
|
+
with _short_lived_app_server(
|
|
1070
|
+
cwd=observation_cwd(cwd),
|
|
1071
|
+
title="CCX Codex paginated history read",
|
|
1072
|
+
timeout=remaining_time,
|
|
1073
|
+
rejection_prefix="unsupported request during paginated history read",
|
|
1074
|
+
) as client:
|
|
1075
|
+
remaining_time = remaining(deadline, timeout)
|
|
1076
|
+
if remaining_time <= 0:
|
|
1077
|
+
raise OperationTimeout("Codex item-page deadline expired", short=short)
|
|
1078
|
+
observed = client.request(
|
|
1079
|
+
"thread/read", {"threadId": thread_id, "includeTurns": False},
|
|
1080
|
+
timeout=remaining_time,
|
|
1081
|
+
)
|
|
1082
|
+
thread = observed.get("thread") if isinstance(observed, dict) else None
|
|
1083
|
+
if not isinstance(thread, dict):
|
|
1084
|
+
raise SessionNotFound("no Codex thread available", short=short)
|
|
1085
|
+
if thread.get("id") != thread_id or thread.get("threadSource") != source:
|
|
1086
|
+
raise SessionUnavailable(
|
|
1087
|
+
"Codex history page belongs to another worker; restart paging", short=short,
|
|
1088
|
+
)
|
|
1089
|
+
remaining_time = remaining(deadline, timeout)
|
|
1090
|
+
if remaining_time <= 0:
|
|
1091
|
+
raise OperationTimeout("Codex item-page deadline expired", short=short)
|
|
1092
|
+
page = client.request(method, params, timeout=remaining_time)
|
|
1093
|
+
except (ProtocolError, OSError) as error:
|
|
1094
|
+
raise _history_failure(error, thread_id) from error
|
|
1095
|
+
if (not isinstance(page, dict)
|
|
1096
|
+
or not isinstance(page.get("data"), list)
|
|
1097
|
+
or (page.get("nextCursor") is not None
|
|
1098
|
+
and not isinstance(page.get("nextCursor"), str))):
|
|
1099
|
+
raise SessionUnavailable(f"{method} returned malformed native page", short=short)
|
|
1100
|
+
return page
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
def daemon_stop(*, force: bool = False, timeout: float = 10) -> dict[str, Any]:
|
|
1104
|
+
return daemon_rpc({"op": "daemon-stop", "force": force}, timeout=timeout)
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
CALLER_PYTHONPATH_ENV = "CCX_CALLER_PYTHONPATH"
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def package_env() -> dict[str, str]:
|
|
1111
|
+
"""Make the currently-running CCX package importable to a child process."""
|
|
1112
|
+
env = os.environ.copy()
|
|
1113
|
+
package_parent = str(Path(__file__).resolve().parents[2])
|
|
1114
|
+
inherited = env.get("PYTHONPATH")
|
|
1115
|
+
# Remember the caller's own value so grandchildren that run user commands
|
|
1116
|
+
# (codex app-server) get it back instead of CCX's install path.
|
|
1117
|
+
env.setdefault(CALLER_PYTHONPATH_ENV, inherited or "")
|
|
1118
|
+
env["PYTHONPATH"] = (
|
|
1119
|
+
package_parent
|
|
1120
|
+
if not inherited
|
|
1121
|
+
else package_parent + os.pathsep + inherited
|
|
1122
|
+
)
|
|
1123
|
+
return env
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def daemon_env() -> dict[str, str]:
|
|
1127
|
+
"""Environment for the shared daemon: importable CCX, no caller identity.
|
|
1128
|
+
|
|
1129
|
+
The singleton cannot inherit the initiating worker's identity, but its
|
|
1130
|
+
user environment (auth, proxies, PATH and tool config) must survive.
|
|
1131
|
+
Each native thread gets its own attested full-UUID stamp separately.
|
|
1132
|
+
"""
|
|
1133
|
+
from ccx.codex.worker_environment import scrub_shared_codex_identity
|
|
1134
|
+
return scrub_shared_codex_identity(package_env())
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def _acquire_spawn_lock(handle, deadline: float | None) -> None:
|
|
1138
|
+
while True:
|
|
1139
|
+
try:
|
|
1140
|
+
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1141
|
+
return
|
|
1142
|
+
except BlockingIOError:
|
|
1143
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1144
|
+
raise SessionUnavailable("timed out waiting for Codex spawn lock")
|
|
1145
|
+
time.sleep(0.02)
|
|
1146
|
+
|
|
1147
|
+
|
|
1148
|
+
@contextmanager
|
|
1149
|
+
def daemon_spawn_lock(deadline: float | None) -> Iterator[None]:
|
|
1150
|
+
"""Existing singleton spawn fence on stable owner-private generation inode.
|
|
1151
|
+
|
|
1152
|
+
Create its lock only after ACTIVE, then hold that inode across daemon boot,
|
|
1153
|
+
without holding the activation lock during subprocess/provider startup.
|
|
1154
|
+
"""
|
|
1155
|
+
from ccx import constants
|
|
1156
|
+
from ccx.store.root import GenerationBusy, require_ccx_active, _flock_at
|
|
1157
|
+
store = require_ccx_active(constants.CCX_ROOT)
|
|
1158
|
+
with store.lease():
|
|
1159
|
+
with store._gen("runtime-metadata", "pid", create=True):
|
|
1160
|
+
pass
|
|
1161
|
+
# Storage lock acquisitions are individually bounded to three seconds, but
|
|
1162
|
+
# a cold app-server boot may take longer. Wait for that same daemon through
|
|
1163
|
+
# the caller's deadline, then probe again while holding the existing fence.
|
|
1164
|
+
until = deadline if deadline is not None else time.monotonic() + CODEXD_BOOT_TIMEOUT_S
|
|
1165
|
+
while True:
|
|
1166
|
+
left = until - time.monotonic()
|
|
1167
|
+
if left <= 0:
|
|
1168
|
+
raise SessionUnavailable("timed out waiting for Codex spawn lock")
|
|
1169
|
+
acquired = False
|
|
1170
|
+
try:
|
|
1171
|
+
with store._gen("runtime-metadata", "pid") as fd:
|
|
1172
|
+
with _flock_at(fd, "codexd.spawn.lock", exclusive=True,
|
|
1173
|
+
create=True, timeout=min(3.0, left)):
|
|
1174
|
+
acquired = True
|
|
1175
|
+
store.revalidate()
|
|
1176
|
+
yield
|
|
1177
|
+
return
|
|
1178
|
+
except GenerationBusy:
|
|
1179
|
+
# A failure in the protected body must propagate; never re-enter
|
|
1180
|
+
# it and run a native operation a second time.
|
|
1181
|
+
if acquired:
|
|
1182
|
+
raise
|
|
1183
|
+
if time.monotonic() >= until:
|
|
1184
|
+
raise SessionUnavailable("timed out waiting for Codex spawn lock")
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def _open_daemon_log():
|
|
1188
|
+
"""Descriptor-pinned, nofollow, uid-private daemon stdout/stderr sidecar."""
|
|
1189
|
+
from ccx import constants
|
|
1190
|
+
from ccx.store.root import require_ccx_active
|
|
1191
|
+
import stat
|
|
1192
|
+
store = require_ccx_active(constants.CCX_ROOT)
|
|
1193
|
+
with store.lease():
|
|
1194
|
+
with store._gen("runtime-metadata", create=True) as fd:
|
|
1195
|
+
for attempt in range(2):
|
|
1196
|
+
log_fd = os.open(
|
|
1197
|
+
"codexd.log", os.O_WRONLY | os.O_CREAT | os.O_APPEND |
|
|
1198
|
+
os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=fd,
|
|
1199
|
+
)
|
|
1200
|
+
info = os.fstat(log_fd)
|
|
1201
|
+
if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid()
|
|
1202
|
+
or info.st_mode & 0o077):
|
|
1203
|
+
os.close(log_fd)
|
|
1204
|
+
raise PermissionError("untrusted Codex daemon log inode")
|
|
1205
|
+
if attempt == 0 and info.st_size > 10 * 1024 * 1024:
|
|
1206
|
+
os.close(log_fd)
|
|
1207
|
+
os.replace("codexd.log", "codexd.log.1", src_dir_fd=fd,
|
|
1208
|
+
dst_dir_fd=fd)
|
|
1209
|
+
continue
|
|
1210
|
+
break
|
|
1211
|
+
return os.fdopen(log_fd, "a", encoding="utf-8", buffering=1)
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def _codexd_log_tail() -> str:
|
|
1217
|
+
try:
|
|
1218
|
+
lines = codexd_log_path().read_text().splitlines()
|
|
1219
|
+
except OSError:
|
|
1220
|
+
return ""
|
|
1221
|
+
return " | ".join(line.strip() for line in lines[-3:] if line.strip())
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
def ensure_daemon(deadline: float | None = None) -> dict[str, Any]:
|
|
1225
|
+
"""Start the shared codexd daemon if it is not already serving.
|
|
1226
|
+
|
|
1227
|
+
The daemon gets its own process session and a reaper thread waits for it,
|
|
1228
|
+
so it survives the importing process while never becoming a zombie in
|
|
1229
|
+
long-lived callers. The spawn lock is held for the whole boot; a daemon
|
|
1230
|
+
that is already booting is waited for rather than duplicated.
|
|
1231
|
+
"""
|
|
1232
|
+
ensure_dirs()
|
|
1233
|
+
probe = settled_daemon_status(deadline)
|
|
1234
|
+
if probe.get("transport") == RUNTIME_SERVED:
|
|
1235
|
+
require_compatible_daemon(probe)
|
|
1236
|
+
if probe.get("state") == "ready":
|
|
1237
|
+
return probe
|
|
1238
|
+
elif probe.get("transport") != RUNTIME_ABSENT and not (
|
|
1239
|
+
probe.get("transport") == RUNTIME_INCONCLUSIVE and probe.get("transitional")
|
|
1240
|
+
):
|
|
1241
|
+
raise SessionUnavailable(
|
|
1242
|
+
inconclusive_presence(probe, "refusing to start a second daemon"),
|
|
1243
|
+
short="codexd",
|
|
1244
|
+
)
|
|
1245
|
+
with daemon_spawn_lock(deadline):
|
|
1246
|
+
probe = settled_daemon_status(deadline)
|
|
1247
|
+
if probe.get("transport") == RUNTIME_SERVED:
|
|
1248
|
+
require_compatible_daemon(probe)
|
|
1249
|
+
if probe.get("state") == "ready":
|
|
1250
|
+
return probe
|
|
1251
|
+
elif probe.get("transport") == RUNTIME_INCONCLUSIVE and not probe.get(
|
|
1252
|
+
"transitional"
|
|
1253
|
+
):
|
|
1254
|
+
raise SessionUnavailable(
|
|
1255
|
+
inconclusive_presence(probe, "refusing to start a second daemon"),
|
|
1256
|
+
short="codexd",
|
|
1257
|
+
)
|
|
1258
|
+
process = None
|
|
1259
|
+
if probe.get("transport") == RUNTIME_ABSENT:
|
|
1260
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1261
|
+
raise SessionUnavailable(
|
|
1262
|
+
"timed out before codexd could be started", short="codexd"
|
|
1263
|
+
)
|
|
1264
|
+
from ccx.system.engine_binary import resolve_engine_binary
|
|
1265
|
+
resolve_engine_binary("codex") # No daemon start for absent CLI.
|
|
1266
|
+
try:
|
|
1267
|
+
from ccx import constants
|
|
1268
|
+
from ccx.store.root import require_ccx_active
|
|
1269
|
+
from ccx.rpc.transport import plan_transport, ensure_transport_parent
|
|
1270
|
+
ctx = require_ccx_active(constants.CCX_ROOT)
|
|
1271
|
+
plan = plan_transport(ctx, role="codexd")
|
|
1272
|
+
ensure_transport_parent(ctx, plan)
|
|
1273
|
+
log = _open_daemon_log()
|
|
1274
|
+
except OSError as error:
|
|
1275
|
+
raise SpawnFailed(
|
|
1276
|
+
f"cannot prepare codexd: {error}",
|
|
1277
|
+
short="codexd",
|
|
1278
|
+
phase="settings-io",
|
|
1279
|
+
detail=str(error),
|
|
1280
|
+
) from error
|
|
1281
|
+
try:
|
|
1282
|
+
process = subprocess.Popen(
|
|
1283
|
+
[sys.executable, "-m", 'ccx.codex.daemon'],
|
|
1284
|
+
cwd=str(codexd_sock_path().parent),
|
|
1285
|
+
stdin=subprocess.DEVNULL,
|
|
1286
|
+
stdout=log,
|
|
1287
|
+
stderr=log,
|
|
1288
|
+
start_new_session=True,
|
|
1289
|
+
close_fds=True,
|
|
1290
|
+
env=daemon_env(),
|
|
1291
|
+
)
|
|
1292
|
+
except OSError as error:
|
|
1293
|
+
raise SpawnFailed(
|
|
1294
|
+
f"could not start codexd: {error}",
|
|
1295
|
+
short="codexd",
|
|
1296
|
+
phase="launcher-failed",
|
|
1297
|
+
detail=str(error),
|
|
1298
|
+
) from error
|
|
1299
|
+
finally:
|
|
1300
|
+
if not log.closed:
|
|
1301
|
+
log.close()
|
|
1302
|
+
threading.Thread(
|
|
1303
|
+
target=process.wait,
|
|
1304
|
+
name="ccx-codexd-reaper",
|
|
1305
|
+
daemon=True,
|
|
1306
|
+
).start()
|
|
1307
|
+
|
|
1308
|
+
# The caller's deadline governs the boot wait.
|
|
1309
|
+
boot_deadline = (
|
|
1310
|
+
deadline if deadline is not None
|
|
1311
|
+
else time.monotonic() + CODEXD_BOOT_TIMEOUT_S
|
|
1312
|
+
)
|
|
1313
|
+
while time.monotonic() < boot_deadline:
|
|
1314
|
+
probe = daemon_status(timeout=1)
|
|
1315
|
+
if probe.get("transport") == RUNTIME_SERVED:
|
|
1316
|
+
require_compatible_daemon(probe)
|
|
1317
|
+
if probe.get("state") == "ready":
|
|
1318
|
+
return probe
|
|
1319
|
+
if process is not None and process.poll() is not None:
|
|
1320
|
+
# The daemon we started died before serving (e.g. no codex
|
|
1321
|
+
# binary); waiting out the boot budget would only hide that.
|
|
1322
|
+
detail = _codexd_log_tail()
|
|
1323
|
+
raise SpawnFailed(
|
|
1324
|
+
"codexd exited during startup"
|
|
1325
|
+
+ (f": {detail}" if detail else "")
|
|
1326
|
+
+ f" (set CCX_CODEX_BIN or put codex on PATH; log: {codexd_log_path()})",
|
|
1327
|
+
short="codexd",
|
|
1328
|
+
phase="launcher-failed",
|
|
1329
|
+
detail=detail or None,
|
|
1330
|
+
)
|
|
1331
|
+
time.sleep(0.05)
|
|
1332
|
+
detail = _codexd_log_tail()
|
|
1333
|
+
raise SessionUnavailable(
|
|
1334
|
+
"codexd did not become ready" + (f": {detail}" if detail else "")
|
|
1335
|
+
+ f" (log: {codexd_log_path()})",
|
|
1336
|
+
short="codexd",
|
|
1337
|
+
)
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
def record(target: str) -> dict[str, Any]:
|
|
1341
|
+
"""Read only an exact owned Codex full UUID, bare or Codex-qualified."""
|
|
1342
|
+
if isinstance(target, str):
|
|
1343
|
+
engine, sep, suffix = target.partition(":")
|
|
1344
|
+
if sep and engine == "claude" and suffix:
|
|
1345
|
+
raise InvalidRequest(
|
|
1346
|
+
f"this command is scoped to codex; got {target!r} (a Claude handle)"
|
|
1347
|
+
)
|
|
1348
|
+
# Only error labels use a bare UUID; owner lookup receives the original target.
|
|
1349
|
+
label = (target[6:] if isinstance(target, str) and target.startswith("codex:")
|
|
1350
|
+
and target[6:] else target)
|
|
1351
|
+
try:
|
|
1352
|
+
value = read_record(target)
|
|
1353
|
+
except OSError as error:
|
|
1354
|
+
raise SessionUnavailable(
|
|
1355
|
+
f"cannot read Codex worker records: {error}",
|
|
1356
|
+
short=label,
|
|
1357
|
+
) from error
|
|
1358
|
+
if not value:
|
|
1359
|
+
raise SessionNotFound(f"no such Codex worker: {label}", short=label)
|
|
1360
|
+
return value
|
|
1361
|
+
|
|
1362
|
+
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
def identity(value: dict[str, Any]) -> dict[str, Any]:
|
|
1366
|
+
lifecycle = value.get("lifecycle") or {}
|
|
1367
|
+
return {
|
|
1368
|
+
"engine": "codex",
|
|
1369
|
+
"short": value["short"],
|
|
1370
|
+
"worker_id": value.get("workerId"),
|
|
1371
|
+
"thread_id": value.get("threadId"),
|
|
1372
|
+
"name": value.get("name"),
|
|
1373
|
+
"cwd": value.get("cwd"),
|
|
1374
|
+
"state": lifecycle.get("state"),
|
|
1375
|
+
"active_turn_id": lifecycle.get("activeTurnId"),
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
def wait(short: str, timeout: float | None) -> dict[str, Any]:
|
|
1380
|
+
deadline = time.monotonic() + timeout if timeout is not None else None
|
|
1381
|
+
polled = False
|
|
1382
|
+
while not polled or deadline is None or time.monotonic() < deadline:
|
|
1383
|
+
polled = True
|
|
1384
|
+
observation = observe_worker(short, timeout=remaining(deadline, 2))
|
|
1385
|
+
reply = observation["reply"]
|
|
1386
|
+
if (
|
|
1387
|
+
observation["presence"] == RUNTIME_SERVED
|
|
1388
|
+
and reply.get("state") in STABLE_STATES
|
|
1389
|
+
):
|
|
1390
|
+
return reply
|
|
1391
|
+
current = read_record(short) or {}
|
|
1392
|
+
if observation["presence"] == RUNTIME_ABSENT:
|
|
1393
|
+
recorded_state = (current.get("lifecycle") or {}).get("state")
|
|
1394
|
+
if recorded_state in {"busy", "waiting", "resuming"}:
|
|
1395
|
+
return {
|
|
1396
|
+
**identity(current),
|
|
1397
|
+
"lastTurn": current.get("lastTurn"),
|
|
1398
|
+
"ok": False,
|
|
1399
|
+
"loaded": False,
|
|
1400
|
+
"normalized_state": "failed",
|
|
1401
|
+
"error": "Codex worker is not loaded; recorded active state is orphaned",
|
|
1402
|
+
}
|
|
1403
|
+
return {
|
|
1404
|
+
**identity(current),
|
|
1405
|
+
"lastTurn": current.get("lastTurn"),
|
|
1406
|
+
"ok": True,
|
|
1407
|
+
"loaded": False,
|
|
1408
|
+
"normalized_state": (
|
|
1409
|
+
"stopped" if recorded_state == "idle" else None
|
|
1410
|
+
),
|
|
1411
|
+
}
|
|
1412
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1413
|
+
break
|
|
1414
|
+
time.sleep(min(0.1, remaining(deadline, 0.1)))
|
|
1415
|
+
current = read_record(short) or {"short": short, "lifecycle": {}}
|
|
1416
|
+
return {
|
|
1417
|
+
**identity(current),
|
|
1418
|
+
"lastTurn": current.get("lastTurn"),
|
|
1419
|
+
"ok": False,
|
|
1420
|
+
"error": "wait timed out",
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
__all__ = [
|
|
1425
|
+
"STABLE_STATES",
|
|
1426
|
+
"daemon_call",
|
|
1427
|
+
"daemon_env",
|
|
1428
|
+
"daemon_rpc",
|
|
1429
|
+
"daemon_status",
|
|
1430
|
+
"daemon_stop",
|
|
1431
|
+
"ensure_daemon",
|
|
1432
|
+
"identity",
|
|
1433
|
+
"list_model_entries",
|
|
1434
|
+
"list_models",
|
|
1435
|
+
"daemon_generation",
|
|
1436
|
+
"daemon_spawn_lock",
|
|
1437
|
+
"observe_registry",
|
|
1438
|
+
"observe_worker",
|
|
1439
|
+
"package_env",
|
|
1440
|
+
"preflight_daemon",
|
|
1441
|
+
"read_account_usage",
|
|
1442
|
+
"record",
|
|
1443
|
+
"require_compatible_daemon",
|
|
1444
|
+
"require_served_generation",
|
|
1445
|
+
"require_workspace",
|
|
1446
|
+
"remaining",
|
|
1447
|
+
"observation_cwd",
|
|
1448
|
+
"observation_daemon",
|
|
1449
|
+
"read_thread",
|
|
1450
|
+
"report_from_thread",
|
|
1451
|
+
"rpc",
|
|
1452
|
+
"validate_model",
|
|
1453
|
+
"wait",
|
|
1454
|
+
]
|