ccx-native 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +241 -2
- package/bin/ccx +56 -0
- package/ccx/__init__.py +163 -0
- package/ccx/__main__.py +28 -0
- package/ccx/claude/__init__.py +0 -0
- package/ccx/claude/analytics.py +466 -0
- package/ccx/claude/attach.py +762 -0
- package/ccx/claude/context.py +465 -0
- package/ccx/claude/delivery.py +806 -0
- package/ccx/claude/external.py +253 -0
- package/ccx/claude/host/__init__.py +0 -0
- package/ccx/claude/host/events.py +297 -0
- package/ccx/claude/host/hooks.py +438 -0
- package/ccx/claude/host/pty.py +250 -0
- package/ccx/claude/host/session.py +5184 -0
- package/ccx/claude/host/subagents.py +169 -0
- package/ccx/claude/host/successor.py +267 -0
- package/ccx/claude/managed_turns.py +240 -0
- package/ccx/claude/oauth.py +176 -0
- package/ccx/claude/settings.py +339 -0
- package/ccx/claude/spawn.py +557 -0
- package/ccx/claude/tasks.py +72 -0
- package/ccx/claude/telemetry.py +282 -0
- package/ccx/claude/transcript.py +2281 -0
- package/ccx/claude/usage.py +837 -0
- package/ccx/claude/usage_control.py +152 -0
- package/ccx/cli/__init__.py +0 -0
- package/ccx/cli/codex.py +1009 -0
- package/ccx/cli/commands/__init__.py +0 -0
- package/ccx/cli/commands/_common.py +407 -0
- package/ccx/cli/commands/control.py +161 -0
- package/ccx/cli/commands/create.py +669 -0
- package/ccx/cli/commands/read.py +1020 -0
- package/ccx/cli/commands/turns.py +1081 -0
- package/ccx/cli/controller.py +757 -0
- package/ccx/cli/main.py +301 -0
- package/ccx/cli/parser.py +1324 -0
- package/ccx/cli/skills.py +215 -0
- package/ccx/cli/statusline.py +160 -0
- package/ccx/client.py +1900 -0
- package/ccx/codex/__init__.py +0 -0
- package/ccx/codex/attach_tty.py +267 -0
- package/ccx/codex/client.py +732 -0
- package/ccx/codex/completion.py +121 -0
- package/ccx/codex/control.py +288 -0
- package/ccx/codex/daemon/__init__.py +0 -0
- package/ccx/codex/daemon/__main__.py +4 -0
- package/ccx/codex/daemon/daemon.py +1556 -0
- package/ccx/codex/daemon/models.py +140 -0
- package/ccx/codex/daemon/native.py +1590 -0
- package/ccx/codex/daemon/release.py +367 -0
- package/ccx/codex/daemon/worker.py +2396 -0
- package/ccx/codex/delivery.py +758 -0
- package/ccx/codex/delivery_recovery.py +232 -0
- package/ccx/codex/evidence.py +827 -0
- package/ccx/codex/history.py +158 -0
- package/ccx/codex/notify_history.py +140 -0
- package/ccx/codex/ops.py +666 -0
- package/ccx/codex/paths.py +70 -0
- package/ccx/codex/protocol.py +805 -0
- package/ccx/codex/provenance.py +105 -0
- package/ccx/codex/read.py +29 -0
- package/ccx/codex/records.py +51 -0
- package/ccx/codex/reports.py +177 -0
- package/ccx/codex/runtime.py +1412 -0
- package/ccx/codex/settle.py +363 -0
- package/ccx/codex/telemetry.py +792 -0
- package/ccx/codex/types.py +659 -0
- package/ccx/codex/worker_environment.py +25 -0
- package/ccx/constants.py +288 -0
- package/ccx/controllers/__init__.py +0 -0
- package/ccx/controllers/controller.py +1449 -0
- package/ccx/controllers/live_vars.py +1706 -0
- package/ccx/controllers/live_vars_runner.py +216 -0
- package/ccx/controllers/notify.py +213 -0
- package/ccx/controllers/orchestrator.py +193 -0
- package/ccx/controllers/policy.py +57 -0
- package/ccx/controllers/store.py +681 -0
- package/ccx/errors.py +634 -0
- package/ccx/notify/__init__.py +0 -0
- package/ccx/notify/__main__.py +5 -0
- package/ccx/notify/client.py +116 -0
- package/ccx/notify/daemon.py +480 -0
- package/ccx/notify/deliver.py +304 -0
- package/ccx/notify/paths.py +51 -0
- package/ccx/notify/registry.py +649 -0
- package/ccx/notify/select.py +750 -0
- package/ccx/notify/settle.py +684 -0
- package/ccx/ops/__init__.py +0 -0
- package/ccx/ops/background.py +115 -0
- package/ccx/ops/cancel.py +289 -0
- package/ccx/ops/codex_import.py +96 -0
- package/ccx/ops/common.py +397 -0
- package/ccx/ops/compact.py +337 -0
- package/ccx/ops/compact_ops.py +550 -0
- package/ccx/ops/context_events.py +48 -0
- package/ccx/ops/conversation.py +1540 -0
- package/ccx/ops/conversation_paging.py +1338 -0
- package/ccx/ops/create.py +56 -0
- package/ccx/ops/creation.py +952 -0
- package/ccx/ops/creation_steps.py +579 -0
- package/ccx/ops/event_feed.py +710 -0
- package/ccx/ops/forking.py +711 -0
- package/ccx/ops/input.py +556 -0
- package/ccx/ops/maintenance.py +295 -0
- package/ccx/ops/report.py +770 -0
- package/ccx/ops/semantics.py +490 -0
- package/ccx/ops/send.py +961 -0
- package/ccx/ops/steer_ops.py +867 -0
- package/ccx/ops/terminate.py +383 -0
- package/ccx/ops/turn_reports.py +306 -0
- package/ccx/ops/wait.py +762 -0
- package/ccx/ops/worker_events.py +636 -0
- package/ccx/rpc/__init__.py +0 -0
- package/ccx/rpc/local_rpc.py +1602 -0
- package/ccx/rpc/rpc.py +559 -0
- package/ccx/rpc/serve.py +374 -0
- package/ccx/rpc/transport.py +409 -0
- package/ccx/rpc/view.py +125 -0
- package/ccx/store/__init__.py +0 -0
- package/ccx/store/discovery.py +170 -0
- package/ccx/store/identity.py +105 -0
- package/ccx/store/journal_io.py +79 -0
- package/ccx/store/liveness.py +194 -0
- package/ccx/store/paths.py +254 -0
- package/ccx/store/record_cache.py +40 -0
- package/ccx/store/records.py +309 -0
- package/ccx/store/root.py +804 -0
- package/ccx/store/runtime_presence.py +137 -0
- package/ccx/store/store.py +2332 -0
- package/ccx/store/worker_state.py +86 -0
- package/ccx/system/__init__.py +0 -0
- package/ccx/system/bounded.py +296 -0
- package/ccx/system/cwd_paths.py +25 -0
- package/ccx/system/engine_binary.py +59 -0
- package/ccx/system/executable.py +50 -0
- package/ccx/system/oslayer.py +667 -0
- package/ccx/system/pipes.py +46 -0
- package/ccx/system/sanitize.py +43 -0
- package/ccx/types.py +705 -0
- package/ccx/version.py +29 -0
- package/ccx/workers.py +2081 -0
- package/ccx.py +9 -0
- package/package.json +40 -4
- package/pyproject.toml +87 -0
- package/skills/ccx/SKILL.md +155 -0
- package/skills/ccx/references/controllers.md +244 -0
- package/skills/ccx/references/troubleshooting.md +94 -0
- package/skills/ccx/references/variables.md +62 -0
package/ccx/ops/send.py
ADDED
|
@@ -0,0 +1,961 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import json
|
|
3
|
+
import time
|
|
4
|
+
from typing import Any, Callable
|
|
5
|
+
from ccx.codex import delivery as codex_delivery
|
|
6
|
+
from ccx.controllers.orchestrator import require_notify_parent
|
|
7
|
+
from ccx.notify.registry import create_registration as create_parent_notification, public_registration as public_parent_notification, read_registration as read_parent_notification
|
|
8
|
+
from ccx.errors import (
|
|
9
|
+
DeliveryRejected,
|
|
10
|
+
InvalidRequest,
|
|
11
|
+
OperationConflict,
|
|
12
|
+
SessionUnavailable,
|
|
13
|
+
)
|
|
14
|
+
from ccx.ops.common import _normalize_caller_metadata, split_target
|
|
15
|
+
|
|
16
|
+
# How long a send keeps checking the receipt after the host's reply timed out.
|
|
17
|
+
_OWNERSHIP_GRACE_S = 10.0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def send_turn(
|
|
21
|
+
target: str, text: str, *, delivery_id: str,
|
|
22
|
+
mode: str = "new_turn_only", timeout: float | None = None,
|
|
23
|
+
cap: dict | None = None,
|
|
24
|
+
notify_parent: bool = False,
|
|
25
|
+
settle: bool = False,
|
|
26
|
+
caller_metadata: dict[str, Any] | None = None,
|
|
27
|
+
expected_generation: str | None = None,
|
|
28
|
+
_native_submit: Callable | None = None,
|
|
29
|
+
_observe_settlement: Callable | None = None,
|
|
30
|
+
) -> dict[str, Any]:
|
|
31
|
+
"""Actual Claude semantic send with full-ID native fence.
|
|
32
|
+
|
|
33
|
+
Retains original claude_delivery prepare/owned/accepted/rejected state
|
|
34
|
+
transitions, original delivery ID/prompt hash/turn ref. The injected
|
|
35
|
+
bounded sender is fake/test-only until real host socket adoption is
|
|
36
|
+
reviewed. With no sender, refuse before any receipt side effects.
|
|
37
|
+
Native model completion is observed outside the activation lease.
|
|
38
|
+
"""
|
|
39
|
+
from ccx.claude import delivery as claude_delivery
|
|
40
|
+
from ccx import constants
|
|
41
|
+
from ccx.store.root import require_ccx_active, NativeEffectIndeterminate, NativeActionDenied, _HASH
|
|
42
|
+
real_native_transport = _native_submit is None
|
|
43
|
+
if real_native_transport:
|
|
44
|
+
owner_handle, owner_engine, owner_id = split_target(target)
|
|
45
|
+
if owner_engine == "codex":
|
|
46
|
+
from ccx.codex.runtime import daemon_rpc
|
|
47
|
+
from ccx.codex.delivery import read_receipt as _read_native_semantic
|
|
48
|
+
from ccx.constants import CODEXD_RPC_GENERATION
|
|
49
|
+
def _native_submit(handle, native_thread, prompt, token, budget):
|
|
50
|
+
original = _read_native_semantic(handle, token)
|
|
51
|
+
if not isinstance(original, dict):
|
|
52
|
+
raise NativeEffectIndeterminate(
|
|
53
|
+
"Codex original semantic receipt missing before native request"
|
|
54
|
+
)
|
|
55
|
+
return daemon_rpc({
|
|
56
|
+
"op": "native-send",
|
|
57
|
+
"workerId": owner_id,
|
|
58
|
+
"threadId": native_thread,
|
|
59
|
+
"text": prompt, "deliveryId": token,
|
|
60
|
+
"clientUserMessageId": original["clientUserMessageId"],
|
|
61
|
+
"cap": cap,
|
|
62
|
+
"rpcGeneration": CODEXD_RPC_GENERATION,
|
|
63
|
+
}, timeout=budget)
|
|
64
|
+
else:
|
|
65
|
+
# Existing original Claude host IPC verifies the exact server
|
|
66
|
+
# root/generation/full-worker peer and Linux SO_PEERCRED PID/start.
|
|
67
|
+
from ccx.rpc.rpc import _rpc
|
|
68
|
+
def _native_submit(handle, native_id, prompt, token, budget):
|
|
69
|
+
msg = {"op": "deliver", "text": prompt,
|
|
70
|
+
"delivery_id": token, "mode": mode}
|
|
71
|
+
if cap is not None:
|
|
72
|
+
msg["cap"] = cap
|
|
73
|
+
return _rpc(handle, msg, timeout=budget)
|
|
74
|
+
if not callable(_native_submit):
|
|
75
|
+
raise SessionUnavailable("Worker connection unavailable; check its status and retry")
|
|
76
|
+
if caller_metadata is not None and not notify_parent:
|
|
77
|
+
raise InvalidRequest("send: caller_metadata requires notify_parent=true")
|
|
78
|
+
handle, engine, worker_id = split_target(target)
|
|
79
|
+
if settle and (engine != "codex" or not notify_parent):
|
|
80
|
+
raise InvalidRequest("send: --settle requires Codex --notify-parent")
|
|
81
|
+
if engine == "codex":
|
|
82
|
+
if mode not in codex_delivery.MODES:
|
|
83
|
+
raise InvalidRequest(f"send: unsupported Codex mode {mode!r}")
|
|
84
|
+
if notify_parent and mode != "new_turn_only":
|
|
85
|
+
raise InvalidRequest("Codex send --notify-parent requires new_turn_only")
|
|
86
|
+
if cap is not None:
|
|
87
|
+
from ccx.workers import _check_caps
|
|
88
|
+
_check_caps(cap, engine="codex")
|
|
89
|
+
if engine == "claude":
|
|
90
|
+
if mode not in claude_delivery.MODES:
|
|
91
|
+
raise InvalidRequest(f"send: unsupported Claude mode {mode!r}")
|
|
92
|
+
if notify_parent and mode != "new_turn_only":
|
|
93
|
+
raise InvalidRequest("Claude send --notify-parent requires new_turn_only")
|
|
94
|
+
try:
|
|
95
|
+
cap = claude_delivery.validate_cap(cap)
|
|
96
|
+
except claude_delivery.ClaudeDeliveryError as exc:
|
|
97
|
+
raise InvalidRequest(str(exc)) from exc
|
|
98
|
+
if not isinstance(text, str) or not text.strip():
|
|
99
|
+
raise InvalidRequest("send: prompt text is required")
|
|
100
|
+
if not claude_delivery.valid_delivery_id(delivery_id):
|
|
101
|
+
raise InvalidRequest("send: delivery_id is missing or unsafe")
|
|
102
|
+
if engine == "codex":
|
|
103
|
+
from ccx.errors import SendIndeterminate
|
|
104
|
+
# codexd refuses a request frame over 1 MiB before Codex sees it;
|
|
105
|
+
# refuse here, before any receipt, so the send is not left uncertain.
|
|
106
|
+
if len(json.dumps(text)) > 1024 * 1024 - 64 * 1024:
|
|
107
|
+
raise InvalidRequest("send: prompt is too large for Codex (limit is about 1 MB)")
|
|
108
|
+
metadata = _normalize_caller_metadata(caller_metadata, notify_parent=notify_parent)
|
|
109
|
+
parent = require_notify_parent() if notify_parent else None
|
|
110
|
+
try:
|
|
111
|
+
return _send_codex_original(
|
|
112
|
+
handle, worker_id, text, delivery_id,
|
|
113
|
+
_native_submit, _observe_settlement, mode=mode,
|
|
114
|
+
real_native_transport=real_native_transport,
|
|
115
|
+
expected_generation=expected_generation,
|
|
116
|
+
parent=parent, caller_metadata=metadata, settle=settle,
|
|
117
|
+
cap=cap,
|
|
118
|
+
)
|
|
119
|
+
except codex_delivery.DeliveryReceiptConflict as exc:
|
|
120
|
+
raise OperationConflict(
|
|
121
|
+
f"original Codex delivery {delivery_id!r} conflicts: {exc}",
|
|
122
|
+
short=worker_id[:8], worker_id=worker_id,
|
|
123
|
+
) from exc
|
|
124
|
+
except InvalidRequest as exc:
|
|
125
|
+
if "notification operation is owned by a different parent or input" in str(exc):
|
|
126
|
+
raise OperationConflict(str(exc), short=worker_id[:8],
|
|
127
|
+
worker_id=worker_id) from exc
|
|
128
|
+
raise
|
|
129
|
+
except NativeEffectIndeterminate as exc:
|
|
130
|
+
# Deliberate correction for BOTH ordinary and notification Codex
|
|
131
|
+
# sends: a previously submitted native effect is never exit 3.
|
|
132
|
+
current = codex_delivery.read_receipt(handle, delivery_id)
|
|
133
|
+
raise SendIndeterminate(
|
|
134
|
+
f"original Codex native delivery uncertain; reconcile {delivery_id!r}, never resend: {exc}",
|
|
135
|
+
short=worker_id[:8], delivery_id=delivery_id,
|
|
136
|
+
delivery_status=(current.get("state") if isinstance(current, dict)
|
|
137
|
+
else "submitted-unknown"),
|
|
138
|
+
) from exc
|
|
139
|
+
metadata = _normalize_caller_metadata(caller_metadata, notify_parent=notify_parent)
|
|
140
|
+
parent = require_notify_parent() if notify_parent else None
|
|
141
|
+
store = require_ccx_active(
|
|
142
|
+
constants.CCX_ROOT, expected_generation=expected_generation,
|
|
143
|
+
)
|
|
144
|
+
rec = store.record(handle)
|
|
145
|
+
native = rec.get("native")
|
|
146
|
+
if not isinstance(native, dict) or not isinstance(native.get("sessionId"), str):
|
|
147
|
+
raise NativeActionDenied("Claude session is missing; inspect the worker before sending")
|
|
148
|
+
native_source = "claude-session:" + native["sessionId"]
|
|
149
|
+
# No incidental receipt/lock directory creation for unactivated,
|
|
150
|
+
# quarantined, wrong-engine or unowned workers. A terminal/owned original
|
|
151
|
+
# delivery may be observed even if native capability was revoked later,
|
|
152
|
+
# but a fresh or merely prepared send MUST pass the native guard first.
|
|
153
|
+
try:
|
|
154
|
+
prior = claude_delivery.require_matching(
|
|
155
|
+
handle, delivery_id, text, mode=mode, cap=cap,
|
|
156
|
+
)
|
|
157
|
+
except claude_delivery.ClaudeDeliveryConflict as exc:
|
|
158
|
+
if not notify_parent:
|
|
159
|
+
raise
|
|
160
|
+
raise OperationConflict(f"original Claude delivery {delivery_id!r} conflicts: {exc}",
|
|
161
|
+
short=worker_id[:8], worker_id=worker_id) from exc
|
|
162
|
+
if (parent is not None and prior is not None
|
|
163
|
+
and prior.get("callerMetadata") != metadata):
|
|
164
|
+
raise OperationConflict("Delivery details changed; retry with the same message and delivery ID",
|
|
165
|
+
short=worker_id[:8], worker_id=worker_id)
|
|
166
|
+
if prior is None or prior.get("state") == "prepared":
|
|
167
|
+
store.authorize_native_action(handle, "send")
|
|
168
|
+
# A prepared semantic receipt can coexist with an already submitted
|
|
169
|
+
# original native effect. Never revive or dispatch this old ID again.
|
|
170
|
+
effect = store.read_receipt(handle, "claude-delivery", delivery_id)
|
|
171
|
+
if effect is not None and effect.get("state") != "prepared":
|
|
172
|
+
raise NativeEffectIndeterminate(
|
|
173
|
+
"original Claude delivery already submitted; reconcile exact id, never revive/replay"
|
|
174
|
+
)
|
|
175
|
+
if cap:
|
|
176
|
+
from ccx.claude.usage import fetch_usage, cap_tripped
|
|
177
|
+
from ccx.constants import USAGE_STALE_LIMIT_S
|
|
178
|
+
from ccx.errors import BudgetStopped
|
|
179
|
+
trip = cap_tripped(fetch_usage(stale_limit=USAGE_STALE_LIMIT_S), cap)
|
|
180
|
+
if trip:
|
|
181
|
+
key, used, limit = trip
|
|
182
|
+
reason = (f"usage cap {key} already exceeded "
|
|
183
|
+
f"({used:g}% >= {limit:g}%); no turn delivered")
|
|
184
|
+
raise BudgetStopped(f"budget stop: {reason}", short=worker_id[:8],
|
|
185
|
+
reason=reason, phase="preflight")
|
|
186
|
+
notification = None
|
|
187
|
+
if parent is not None and prior is not None:
|
|
188
|
+
# The semantic key rejects a different parent before even
|
|
189
|
+
# an original Claude host resume. Never retroactively arm accepted
|
|
190
|
+
# ordinary sends or re-arm a cancelled notification.
|
|
191
|
+
try:
|
|
192
|
+
notification = create_parent_notification(
|
|
193
|
+
parent=parent, worker_engine="claude", worker_id=worker_id,
|
|
194
|
+
worker_name=rec.get("name"), source="send",
|
|
195
|
+
operation={"kind": "claude-turn", "deliveryId": delivery_id},
|
|
196
|
+
)
|
|
197
|
+
except InvalidRequest as error:
|
|
198
|
+
if "notification operation is owned by a different parent or input" in str(error):
|
|
199
|
+
raise OperationConflict(str(error), short=worker_id[:8],
|
|
200
|
+
worker_id=worker_id) from error
|
|
201
|
+
raise
|
|
202
|
+
if prior is None or prior.get("state") == "prepared":
|
|
203
|
+
if real_native_transport:
|
|
204
|
+
# Strict sends may revive; auto may revive only a
|
|
205
|
+
# recorded stopped original, never a served BUSY/BOOTING turn.
|
|
206
|
+
# Same-token native effects were fenced before this branch.
|
|
207
|
+
if (mode == "new_turn_only"
|
|
208
|
+
or (mode == "auto" and
|
|
209
|
+
(rec.get("lifecycle") or {}).get("state")
|
|
210
|
+
in ("dead", "crashed", "idle-killed", "budget-stopped"))):
|
|
211
|
+
ensure_ready(handle, expected_generation=store.generation)
|
|
212
|
+
rec = store.record(handle)
|
|
213
|
+
# No real host -> no first semantic/native receipt creation.
|
|
214
|
+
# Same-connection SO_PEERCRED server PID/start/root/owner proof
|
|
215
|
+
# and independent original provider SessionStart must exist.
|
|
216
|
+
from ccx.rpc.rpc import _rpc
|
|
217
|
+
try:
|
|
218
|
+
response = _rpc(handle, {"op": "startup-status"}, timeout=.5)
|
|
219
|
+
except (OSError, ValueError) as exc:
|
|
220
|
+
if mode == "steer_only":
|
|
221
|
+
raise SessionUnavailable(
|
|
222
|
+
f"steer: original Claude startup status unavailable: {exc}",
|
|
223
|
+
short=worker_id[:8],
|
|
224
|
+
) from exc
|
|
225
|
+
raise
|
|
226
|
+
lifecycle = rec.get("lifecycle") or {}
|
|
227
|
+
source = lifecycle.get("nativeStartProof")
|
|
228
|
+
if (not isinstance(response,dict)
|
|
229
|
+
or response.get("ok") is not True
|
|
230
|
+
or response.get("workerId") != worker_id
|
|
231
|
+
or response.get("engine") != "claude"
|
|
232
|
+
or response.get("generation") != store.generation
|
|
233
|
+
or not isinstance(source, dict)
|
|
234
|
+
or source.get("sessionId") != native["sessionId"]
|
|
235
|
+
or response.get("nativeStartProof") != source):
|
|
236
|
+
raise SessionUnavailable(
|
|
237
|
+
"Claude worker connection could not be verified for this session; "
|
|
238
|
+
"check its status; no delivery receipt written"
|
|
239
|
+
)
|
|
240
|
+
if mode == "steer_only":
|
|
241
|
+
# A revived worker can have an owned new-turn receipt before
|
|
242
|
+
# Claude reports the native BUSY state. Wait only for a recent
|
|
243
|
+
# turn on a previously used owner, never for an idle worker.
|
|
244
|
+
try:
|
|
245
|
+
recent = [
|
|
246
|
+
row for row in claude_delivery.iter_receipts(handle)
|
|
247
|
+
if row.get("requestedMode") == "new_turn_only"
|
|
248
|
+
and row.get("state") in {"owned", "accepted"}
|
|
249
|
+
and row.get("settlement") is None
|
|
250
|
+
and (row.get("ownedAtMs") or row.get("acceptedAtMs") or 0)
|
|
251
|
+
> (rec.get("lastSettledAt") or 0)
|
|
252
|
+
and 0 <= int(time.time() * 1000) - (
|
|
253
|
+
row.get("ownedAtMs") or row.get("acceptedAtMs") or 0
|
|
254
|
+
) <= 10_000
|
|
255
|
+
]
|
|
256
|
+
except (claude_delivery.ClaudeDeliveryError, OSError, ValueError) as exc:
|
|
257
|
+
raise SessionUnavailable(
|
|
258
|
+
f"steer: original Claude receipt inventory unavailable: {exc}",
|
|
259
|
+
short=worker_id[:8],
|
|
260
|
+
) from exc
|
|
261
|
+
revived_pending = (
|
|
262
|
+
bool((rec.get("stats") or {}).get("turns"))
|
|
263
|
+
and bool(recent)
|
|
264
|
+
and lifecycle.get("state") in {"idle", "busy", "resuming", "booting"}
|
|
265
|
+
)
|
|
266
|
+
deadline_busy = time.monotonic() + (4.0 if revived_pending else 0.0)
|
|
267
|
+
while True:
|
|
268
|
+
try:
|
|
269
|
+
status = _rpc(
|
|
270
|
+
handle, {"op": "status"},
|
|
271
|
+
timeout=min(.5, max(.05, deadline_busy-time.monotonic()))
|
|
272
|
+
if revived_pending else .5,
|
|
273
|
+
)
|
|
274
|
+
except (OSError, ValueError) as exc:
|
|
275
|
+
raise SessionUnavailable(
|
|
276
|
+
f"steer: original Claude host status unavailable: {exc}",
|
|
277
|
+
short=worker_id[:8],
|
|
278
|
+
) from exc
|
|
279
|
+
if not isinstance(status, dict) or status.get("ok") is not True:
|
|
280
|
+
raise SessionUnavailable(
|
|
281
|
+
"steer: original Claude host status not authenticated",
|
|
282
|
+
short=worker_id[:8],
|
|
283
|
+
)
|
|
284
|
+
if status.get("state") == "busy":
|
|
285
|
+
break
|
|
286
|
+
if (not revived_pending or status.get("state") not in
|
|
287
|
+
{"idle", "resuming", "booting"}
|
|
288
|
+
or time.monotonic() >= deadline_busy):
|
|
289
|
+
raise DeliveryRejected(
|
|
290
|
+
"steer requires a currently busy Claude turn",
|
|
291
|
+
short=worker_id[:8], state=status.get("state"),
|
|
292
|
+
refusal="waiting" if status.get("state") == "waiting" else "idle",
|
|
293
|
+
)
|
|
294
|
+
store.revalidate()
|
|
295
|
+
fresh = store.record(handle)
|
|
296
|
+
if ((fresh.get("lifecycle") or {}).get("nativeStartProof") != source):
|
|
297
|
+
raise SessionUnavailable(
|
|
298
|
+
"steer: revived Claude host identity changed during readiness",
|
|
299
|
+
short=worker_id[:8],
|
|
300
|
+
)
|
|
301
|
+
time.sleep(min(.15, max(0.0, deadline_busy-time.monotonic())))
|
|
302
|
+
# The original semantic receipt is the public acceptance/pending state.
|
|
303
|
+
# A previously owned token must remain observable even if the worker
|
|
304
|
+
# later becomes terminal-only; observation never invokes a provider.
|
|
305
|
+
try:
|
|
306
|
+
existing, _fresh = claude_delivery.prepare(
|
|
307
|
+
handle, delivery_id, text, mode=mode, cap=cap, timeout=timeout,
|
|
308
|
+
**({"caller_metadata": metadata} if parent is not None else {}),
|
|
309
|
+
)
|
|
310
|
+
except claude_delivery.ClaudeDeliveryConflict as exc:
|
|
311
|
+
if not notify_parent:
|
|
312
|
+
raise
|
|
313
|
+
raise OperationConflict(f"original Claude delivery {delivery_id!r} conflicts: {exc}",
|
|
314
|
+
short=worker_id[:8], worker_id=worker_id) from exc
|
|
315
|
+
except claude_delivery.ClaudeDeliveryBusy as exc:
|
|
316
|
+
raise SessionUnavailable(
|
|
317
|
+
f"steer: Claude delivery receipt lock contested: {exc}"
|
|
318
|
+
if mode == "steer_only" else f"Claude delivery receipt lock contested: {exc}",
|
|
319
|
+
short=worker_id[:8],
|
|
320
|
+
) from exc
|
|
321
|
+
if parent is not None:
|
|
322
|
+
if notification is None:
|
|
323
|
+
notification = create_parent_notification(
|
|
324
|
+
parent=parent, worker_engine="claude", worker_id=worker_id,
|
|
325
|
+
worker_name=rec.get("name"), source="send",
|
|
326
|
+
operation={"kind": "claude-turn", "deliveryId": delivery_id},
|
|
327
|
+
)
|
|
328
|
+
# The original notification's file AND directory are already fsynced.
|
|
329
|
+
# Never submit original Claude host input if supervisor cannot start.
|
|
330
|
+
from ccx.notify.daemon import ensure_notify_daemon
|
|
331
|
+
ensure_notify_daemon()
|
|
332
|
+
def public(value):
|
|
333
|
+
return {
|
|
334
|
+
"worker": {"worker": handle, "worker_id": worker_id,
|
|
335
|
+
"engine": "claude", "short": worker_id[:8]},
|
|
336
|
+
"delivery": claude_delivery.public_receipt(value),
|
|
337
|
+
"status": value["state"],
|
|
338
|
+
"settled": isinstance(value.get("settlement"), dict),
|
|
339
|
+
"notification": (public_parent_notification(
|
|
340
|
+
read_parent_notification(notification["eventId"]) or notification,
|
|
341
|
+
) if notification is not None else None),
|
|
342
|
+
}
|
|
343
|
+
if existing["state"] in ("accepted", "rejected"):
|
|
344
|
+
if (existing["state"] == "accepted"
|
|
345
|
+
and existing.get("settlement") is None
|
|
346
|
+
and _observe_settlement is not None
|
|
347
|
+
and existing.get("placement") == "new_turn"):
|
|
348
|
+
existing = _settle_claude_original(
|
|
349
|
+
handle, worker_id, native_source, delivery_id,
|
|
350
|
+
existing["turnRef"], _observe_settlement,
|
|
351
|
+
)
|
|
352
|
+
return public(existing)
|
|
353
|
+
identity = claude_delivery.spec_digest(existing)
|
|
354
|
+
if existing["state"] == "owned":
|
|
355
|
+
durable_effect = store.read_receipt(
|
|
356
|
+
handle, "claude-delivery", delivery_id,
|
|
357
|
+
)
|
|
358
|
+
if (isinstance(durable_effect, dict)
|
|
359
|
+
and durable_effect.get("intentSha256") == identity
|
|
360
|
+
and durable_effect.get("state") == "submitted-unknown"):
|
|
361
|
+
# A retry may report the original HOST-OWNED pending receipt.
|
|
362
|
+
# It MUST NOT dispatch again or infer a completed native turn.
|
|
363
|
+
return public(existing)
|
|
364
|
+
raise NativeEffectIndeterminate(
|
|
365
|
+
"owned semantic token lacks original full-UUID native effect proof"
|
|
366
|
+
)
|
|
367
|
+
store.authorize_native_action(handle, "send")
|
|
368
|
+
attempt = store.begin_native_intent(
|
|
369
|
+
handle, action="send", family="claude-delivery",
|
|
370
|
+
operation_id=delivery_id, intent_sha256=identity,
|
|
371
|
+
)
|
|
372
|
+
if attempt.replayed:
|
|
373
|
+
raise NativeEffectIndeterminate(
|
|
374
|
+
"native fence terminal but original placement receipt absent; "
|
|
375
|
+
"reconcile original provider turn, never replay"
|
|
376
|
+
)
|
|
377
|
+
# The native-effect claim is the only pre-IPC submitted-unknown fence.
|
|
378
|
+
# Leave the semantic receipt PREPARED until the host takes ownership.
|
|
379
|
+
# If we pre-mark it owned, the real host correctly interprets deliver as
|
|
380
|
+
# a replay and returns without ever sending bytes to the PTY.
|
|
381
|
+
response = store.dispatch_native_intent_once(
|
|
382
|
+
attempt,
|
|
383
|
+
bounded_submit=lambda budget: _native_submit(
|
|
384
|
+
handle, native["sessionId"], text, delivery_id, budget,
|
|
385
|
+
),
|
|
386
|
+
)
|
|
387
|
+
# The real host ACKs *durable ownership*, not native UserPromptSubmit/Stop.
|
|
388
|
+
# Its reply can be lost (timeout) or carry a stale view, so the host's
|
|
389
|
+
# durable receipt decides: if it owns or accepted the message, report
|
|
390
|
+
# that instead of an indeterminate send.
|
|
391
|
+
if "nativeEvidenceSha256" not in response:
|
|
392
|
+
observed = claude_delivery.read_receipt(handle, delivery_id)
|
|
393
|
+
# A busy host (e.g. just restarted) can take the message after our
|
|
394
|
+
# short reply wait. Keep reading its receipt for a while; never resend.
|
|
395
|
+
deadline = time.monotonic() + _OWNERSHIP_GRACE_S
|
|
396
|
+
while (response.get("error") == "timeout"
|
|
397
|
+
and (observed is None or observed.get("state") == "prepared")
|
|
398
|
+
and time.monotonic() < deadline):
|
|
399
|
+
time.sleep(0.1)
|
|
400
|
+
observed = claude_delivery.read_receipt(handle, delivery_id)
|
|
401
|
+
if not (response.get("ok") is True
|
|
402
|
+
and isinstance(response.get("delivery"), dict)
|
|
403
|
+
and response["delivery"].get("delivery_id") == delivery_id):
|
|
404
|
+
try:
|
|
405
|
+
import os
|
|
406
|
+
from ccx.claude.host.events import _open_event_journal, _stamp_event
|
|
407
|
+
preview = json.dumps({
|
|
408
|
+
"keys": sorted(str(key) for key in response),
|
|
409
|
+
"error": str(response.get("error") or ""),
|
|
410
|
+
}, ensure_ascii=False)[:300]
|
|
411
|
+
event = _stamp_event({
|
|
412
|
+
"type": "delivery_reply_unrecognized", "delivery_id": delivery_id,
|
|
413
|
+
"reply": preview,
|
|
414
|
+
"receipt_state": observed.get("state") if observed else None,
|
|
415
|
+
}, short=worker_id, seq=1,
|
|
416
|
+
epoch=f"client-{os.getpid()}-{time.time_ns()}", ts=time.time())
|
|
417
|
+
with _open_event_journal(store, worker_id) as events:
|
|
418
|
+
events.write((json.dumps(event) + "\n").encode())
|
|
419
|
+
except Exception:
|
|
420
|
+
pass
|
|
421
|
+
if (observed is not None
|
|
422
|
+
and observed.get("state") in ("owned", "accepted")
|
|
423
|
+
and ((observed.get("route") == "new_turn"
|
|
424
|
+
and mode in ("new_turn_only", "auto"))
|
|
425
|
+
or (observed.get("route") == "steer"
|
|
426
|
+
and mode in ("auto", "steer_only")
|
|
427
|
+
and isinstance(observed.get("targetTurnRef"), str)
|
|
428
|
+
and observed.get("targetTurnRef")))):
|
|
429
|
+
store.revalidate()
|
|
430
|
+
return public(observed)
|
|
431
|
+
raise NativeEffectIndeterminate(
|
|
432
|
+
"host ownership of the message is unconfirmed"
|
|
433
|
+
+ (f" ({response['error']})" if response.get("error") else "")
|
|
434
|
+
)
|
|
435
|
+
if (response.get("workerId") != worker_id
|
|
436
|
+
or response.get("engine") != "claude"
|
|
437
|
+
or response.get("deliveryId") != delivery_id
|
|
438
|
+
or response.get("nativeSource") != native_source
|
|
439
|
+
or response.get("sessionId") != native["sessionId"]
|
|
440
|
+
or not isinstance(response.get("nativeEvidenceSha256"), str)
|
|
441
|
+
or not _HASH.fullmatch(response["nativeEvidenceSha256"])):
|
|
442
|
+
raise NativeEffectIndeterminate(
|
|
443
|
+
"native acknowledgement not tied to original source/delivery"
|
|
444
|
+
)
|
|
445
|
+
if (response.get("accepted") is False
|
|
446
|
+
and response.get("nativeStatus") == "rejected"):
|
|
447
|
+
code = response.get("rejectionCode")
|
|
448
|
+
if not isinstance(code, str) or not code:
|
|
449
|
+
raise NativeEffectIndeterminate("native rejection has no exact original code")
|
|
450
|
+
rejected = claude_delivery.mark_rejected(
|
|
451
|
+
handle, delivery_id, code=code,
|
|
452
|
+
message=str(response.get("rejectionMessage") or code),
|
|
453
|
+
timeout=timeout,
|
|
454
|
+
)
|
|
455
|
+
store.settle_native_intent(
|
|
456
|
+
attempt, state="rejected",
|
|
457
|
+
native_evidence_sha256=response["nativeEvidenceSha256"],
|
|
458
|
+
)
|
|
459
|
+
return public(rejected)
|
|
460
|
+
if (response.get("accepted") is not True
|
|
461
|
+
or not isinstance(response.get("turnRef"), str)
|
|
462
|
+
or not response["turnRef"].startswith("ccx-turn:claude:")):
|
|
463
|
+
raise NativeEffectIndeterminate(
|
|
464
|
+
"original Claude native accepted turn reference unproven; no replay"
|
|
465
|
+
)
|
|
466
|
+
# The direct positive-ACK route is injected/test-only. Unlike the real
|
|
467
|
+
# authenticated host's ownership receipt it must explicitly prove its
|
|
468
|
+
# original route; auto must never silently become a new turn.
|
|
469
|
+
ack_route = "new_turn" if mode == "new_turn_only" else response.get("route")
|
|
470
|
+
target_ref = response.get("targetTurnRef")
|
|
471
|
+
if (ack_route not in ({"steer"} if mode == "steer_only"
|
|
472
|
+
else {"new_turn", "steer"})
|
|
473
|
+
or (ack_route == "steer"
|
|
474
|
+
and (not isinstance(target_ref, str)
|
|
475
|
+
or target_ref != response["turnRef"]))):
|
|
476
|
+
raise NativeEffectIndeterminate(
|
|
477
|
+
"original direct native ACK lacks exact auto/steer route or target"
|
|
478
|
+
)
|
|
479
|
+
# After original positive acceptance only, advance existing semantics.
|
|
480
|
+
# Ownership-only host replies are not valid positive native evidence.
|
|
481
|
+
current_semantic = claude_delivery.read_receipt(handle, delivery_id)
|
|
482
|
+
if current_semantic["state"] == "prepared":
|
|
483
|
+
current_semantic = claude_delivery.mark_owned(
|
|
484
|
+
handle, delivery_id,
|
|
485
|
+
route=ack_route,
|
|
486
|
+
target_turn_ref=target_ref if ack_route == "steer" else None,
|
|
487
|
+
timeout=timeout,
|
|
488
|
+
)
|
|
489
|
+
if (current_semantic["state"] == "owned"
|
|
490
|
+
and not current_semantic.get("maybeWritten")):
|
|
491
|
+
claude_delivery.mark_attempt(
|
|
492
|
+
handle, delivery_id, maybe_written=True, timeout=timeout,
|
|
493
|
+
)
|
|
494
|
+
accepted = claude_delivery.mark_accepted(
|
|
495
|
+
handle, delivery_id,
|
|
496
|
+
placement="current_turn" if ack_route == "steer" else "new_turn",
|
|
497
|
+
turn_ref=response["turnRef"],
|
|
498
|
+
evidence=response["nativeEvidenceSha256"], timeout=timeout,
|
|
499
|
+
)
|
|
500
|
+
store.settle_native_intent(
|
|
501
|
+
attempt, state="accepted",
|
|
502
|
+
native_evidence_sha256=response["nativeEvidenceSha256"],
|
|
503
|
+
)
|
|
504
|
+
if _observe_settlement is not None and accepted.get("placement") == "new_turn":
|
|
505
|
+
accepted = _settle_claude_original(
|
|
506
|
+
handle, worker_id, native_source, delivery_id,
|
|
507
|
+
response["turnRef"], _observe_settlement,
|
|
508
|
+
)
|
|
509
|
+
return public(accepted)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _settle_claude_original(
|
|
513
|
+
handle: str, worker_id: str, source: str, delivery_id: str,
|
|
514
|
+
turn_ref: str, observe_completion: Callable,
|
|
515
|
+
) -> dict[str, Any]:
|
|
516
|
+
"""Confirm original turn after release of the activation dispatch lease."""
|
|
517
|
+
from ccx.claude import delivery as claude_delivery
|
|
518
|
+
from ccx.store.root import NativeEffectIndeterminate
|
|
519
|
+
proof = observe_completion(handle, turn_ref)
|
|
520
|
+
if (not isinstance(proof, dict)
|
|
521
|
+
or proof.get("workerId") != worker_id
|
|
522
|
+
or proof.get("nativeSource") != source
|
|
523
|
+
or proof.get("deliveryId") != delivery_id
|
|
524
|
+
or proof.get("turnRef") != turn_ref
|
|
525
|
+
or proof.get("status") != "completed"):
|
|
526
|
+
raise NativeEffectIndeterminate(
|
|
527
|
+
"provider-native completion source or status unproven"
|
|
528
|
+
)
|
|
529
|
+
return claude_delivery.mark_settled(
|
|
530
|
+
handle, delivery_id, turn_ref=turn_ref,
|
|
531
|
+
original_positive_completion=proof,
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _ensure_original_claude_ready(
|
|
536
|
+
handle: str, worker_id: str, store, timeout: float | None,
|
|
537
|
+
) -> dict[str, Any]:
|
|
538
|
+
"""One source-bound original host lifetime, with no semantic model turn."""
|
|
539
|
+
from ccx.errors import DeliveryRejected, SessionUnavailable
|
|
540
|
+
from ccx.rpc.rpc import _rpc, _respawn
|
|
541
|
+
from ccx.store.liveness import owned_pid_alive
|
|
542
|
+
from ccx.store.root import NativeActionDenied
|
|
543
|
+
import os
|
|
544
|
+
|
|
545
|
+
try:
|
|
546
|
+
store.authorize_native_action(handle, "resume")
|
|
547
|
+
except NativeActionDenied as exc:
|
|
548
|
+
raise SessionUnavailable(
|
|
549
|
+
f"Claude cannot resume; inspect the worker: {exc}",
|
|
550
|
+
short=worker_id[:8],
|
|
551
|
+
) from exc
|
|
552
|
+
initial = store.record(handle)
|
|
553
|
+
original_session = (initial.get("native") or {}).get("sessionId")
|
|
554
|
+
if not isinstance(original_session, str) or not original_session:
|
|
555
|
+
raise SessionUnavailable("Claude session is unavailable; inspect the worker",
|
|
556
|
+
short=worker_id[:8])
|
|
557
|
+
deadline = time.monotonic() + (180 if timeout is None else timeout)
|
|
558
|
+
|
|
559
|
+
def probe():
|
|
560
|
+
# This same-connection source proof is read-only. In particular an
|
|
561
|
+
# unproven host never receives even a general status/native send probe.
|
|
562
|
+
attested = _rpc(handle, {"op": "startup-status"}, timeout=.35)
|
|
563
|
+
current = store.record(handle)
|
|
564
|
+
if not isinstance(attested, dict) or attested.get("ok") is not True:
|
|
565
|
+
return False, current
|
|
566
|
+
reply = _rpc(handle, {"op": "status"}, timeout=.35)
|
|
567
|
+
lc = current.get("lifecycle") or {}
|
|
568
|
+
state = reply.get("state") if isinstance(reply, dict) and reply.get("ok") is True else None
|
|
569
|
+
if state in ("busy", "waiting"):
|
|
570
|
+
refusal = "busy" if state == "busy" else "state_unavailable"
|
|
571
|
+
raise DeliveryRejected(
|
|
572
|
+
f"Worker is {state}; wait until idle to send a new message.",
|
|
573
|
+
short=worker_id[:8], refusal=refusal, state=state,
|
|
574
|
+
delivery_status="rejected",
|
|
575
|
+
)
|
|
576
|
+
proof = lc.get("nativeStartProof")
|
|
577
|
+
identity = (isinstance(proof, dict)
|
|
578
|
+
and proof.get("sessionId") == original_session
|
|
579
|
+
and proof.get("hostPid") == lc.get("hostPid")
|
|
580
|
+
and str(proof.get("hostStart")) == str(lc.get("hostStart"))
|
|
581
|
+
and proof.get("workerPid") == lc.get("workerPid")
|
|
582
|
+
and str(proof.get("workerStart")) == str(lc.get("workerStart"))
|
|
583
|
+
and owned_pid_alive(lc.get("hostPid"), lc.get("hostStart"))
|
|
584
|
+
and owned_pid_alive(lc.get("workerPid"), lc.get("workerStart")))
|
|
585
|
+
if (state == "idle" and identity
|
|
586
|
+
and attested.get("workerId") == worker_id
|
|
587
|
+
and attested.get("generation") == store.generation
|
|
588
|
+
and attested.get("nativeStartProof") == proof):
|
|
589
|
+
return True, current
|
|
590
|
+
return False, current
|
|
591
|
+
|
|
592
|
+
ready, current = probe()
|
|
593
|
+
if ready:
|
|
594
|
+
return {"worker": handle, "worker_id": worker_id,
|
|
595
|
+
"engine": "claude", "short": worker_id[:8],
|
|
596
|
+
"session_id": original_session, "state": "idle", "resumed": False}
|
|
597
|
+
# Original cwd is required to *relaunch*, but not to talk to an existing
|
|
598
|
+
# authenticated live host whose cwd inode may outlive its pathname.
|
|
599
|
+
if not isinstance(initial.get("cwd"), str) or not os.path.isdir(initial["cwd"]):
|
|
600
|
+
raise SessionUnavailable("Claude working directory is unavailable; check its path",
|
|
601
|
+
short=worker_id[:8])
|
|
602
|
+
prior = current.get("lifecycle") or {}
|
|
603
|
+
new_host = False
|
|
604
|
+
if prior.get("state") not in ("booting", "resuming"):
|
|
605
|
+
# The same recorded native JSONL drives both this eligibility check
|
|
606
|
+
# and host._run_host's --resume argv. No history => no replacement
|
|
607
|
+
# conversation under the old CCX/native IDs.
|
|
608
|
+
import stat
|
|
609
|
+
transcript = current.get("transcript")
|
|
610
|
+
try:
|
|
611
|
+
history = os.stat(transcript) if isinstance(transcript, str) else None
|
|
612
|
+
except OSError:
|
|
613
|
+
history = None
|
|
614
|
+
if (history is None or not stat.S_ISREG(history.st_mode)
|
|
615
|
+
or history.st_size == 0):
|
|
616
|
+
raise SessionUnavailable(
|
|
617
|
+
"Claude cannot resume without saved conversation history; "
|
|
618
|
+
"empty transcript cannot be safely restarted",
|
|
619
|
+
short=worker_id[:8],
|
|
620
|
+
)
|
|
621
|
+
# Existing respawn.lock decides whether THIS caller launched; it
|
|
622
|
+
# serializes independent Python/CLI/protocol processes for one full UUID.
|
|
623
|
+
remaining = deadline - time.monotonic()
|
|
624
|
+
if remaining <= 0:
|
|
625
|
+
raise SessionUnavailable("Claude resume observation budget expired",
|
|
626
|
+
short=worker_id[:8])
|
|
627
|
+
rc, new_host = _respawn(handle, deadline=time.time() + remaining,
|
|
628
|
+
report_launch=True)
|
|
629
|
+
if rc != 0:
|
|
630
|
+
raise SessionUnavailable(
|
|
631
|
+
"Claude did not resume; check worker status before retrying",
|
|
632
|
+
short=worker_id[:8],
|
|
633
|
+
)
|
|
634
|
+
# A concurrently BOOTING/RESUMING original is observed, never replaced.
|
|
635
|
+
while time.monotonic() < deadline:
|
|
636
|
+
ready, _current = probe()
|
|
637
|
+
if ready:
|
|
638
|
+
store.revalidate()
|
|
639
|
+
return {"worker": handle, "worker_id": worker_id,
|
|
640
|
+
"engine": "claude", "short": worker_id[:8],
|
|
641
|
+
"session_id": original_session, "state": "idle",
|
|
642
|
+
"resumed": bool(new_host)}
|
|
643
|
+
time.sleep(min(.08, max(.0, deadline - time.monotonic())))
|
|
644
|
+
raise SessionUnavailable("Claude was not ready before the timeout; check status and retry",
|
|
645
|
+
short=worker_id[:8])
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def ensure_ready(
|
|
649
|
+
target: str, *, timeout: float | None = 90,
|
|
650
|
+
expected_generation: str | None = None,
|
|
651
|
+
) -> dict[str, Any]:
|
|
652
|
+
"""Ensure one *original* full-UUID native runtime is positively idle.
|
|
653
|
+
|
|
654
|
+
Runtime-only: the Claude original host or Codex original singleton is
|
|
655
|
+
restored without starting any model turn or creating native identity.
|
|
656
|
+
"""
|
|
657
|
+
from ccx import constants
|
|
658
|
+
from ccx.errors import InvalidRequest, SessionUnavailable
|
|
659
|
+
from ccx.store.root import require_ccx_active, NativeActionDenied
|
|
660
|
+
handle, engine, worker_id = split_target(target)
|
|
661
|
+
if engine == "claude":
|
|
662
|
+
if timeout is not None and (type(timeout) not in (int, float)
|
|
663
|
+
or not 0 < timeout <= 600):
|
|
664
|
+
raise InvalidRequest("Claude resume timeout must be positive or None")
|
|
665
|
+
store = require_ccx_active(constants.CCX_ROOT,
|
|
666
|
+
expected_generation=expected_generation)
|
|
667
|
+
return _ensure_original_claude_ready(handle, worker_id, store, timeout)
|
|
668
|
+
from ccx.codex.runtime import ensure_daemon, daemon_rpc, require_served_generation
|
|
669
|
+
from ccx.constants import CODEXD_RPC_GENERATION
|
|
670
|
+
if timeout is not None and (type(timeout) not in (int, float)
|
|
671
|
+
or not 0 < timeout <= 600):
|
|
672
|
+
raise InvalidRequest("Codex resume timeout must be positive and bounded or None")
|
|
673
|
+
store = require_ccx_active(constants.CCX_ROOT,
|
|
674
|
+
expected_generation=expected_generation)
|
|
675
|
+
try:
|
|
676
|
+
store.authorize_native_action(handle, "resume")
|
|
677
|
+
except NativeActionDenied as error:
|
|
678
|
+
raise SessionUnavailable("Codex cannot resume; inspect the worker: " + str(error),
|
|
679
|
+
short=worker_id[:8]) from error
|
|
680
|
+
record = store.record(handle)
|
|
681
|
+
from ccx.codex.runtime import require_workspace
|
|
682
|
+
require_workspace(record)
|
|
683
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
684
|
+
ensure_daemon(deadline=deadline)
|
|
685
|
+
store.revalidate()
|
|
686
|
+
remaining = (90.0 if deadline is None else max(0.0, deadline-time.monotonic()))
|
|
687
|
+
if remaining <= 0:
|
|
688
|
+
raise SessionUnavailable("Codex native resume budget expired", short=worker_id[:8])
|
|
689
|
+
result = daemon_rpc({
|
|
690
|
+
"op": "load", "short": worker_id,
|
|
691
|
+
"timeout": min(90., remaining), "rpcGeneration": CODEXD_RPC_GENERATION,
|
|
692
|
+
}, timeout=min(90.,remaining))
|
|
693
|
+
if result.get("transport") == "served":
|
|
694
|
+
require_served_generation(result, short=worker_id)
|
|
695
|
+
state = result.get("state")
|
|
696
|
+
if (result.get("transport") == "served" and result.get("ok") is True
|
|
697
|
+
and result.get("loaded") is True and state in ("busy", "waiting")):
|
|
698
|
+
# The live subscribed original reports a running turn or open gate:
|
|
699
|
+
# a definite state refusal (legacy Codex send shape), not unavailable.
|
|
700
|
+
raise DeliveryRejected(f"worker is {state}", short=worker_id[:8],
|
|
701
|
+
reason=f"worker is {state}", state=state,
|
|
702
|
+
refusal=("compaction-busy" if result.get("compacting")
|
|
703
|
+
else "worker-state"))
|
|
704
|
+
if (result.get("transport") != "served"
|
|
705
|
+
or result.get("ok") is not True
|
|
706
|
+
or result.get("loaded") is not True
|
|
707
|
+
or result.get("state") != "idle"):
|
|
708
|
+
from ccx.codex.runtime import explain_native_failure
|
|
709
|
+
raise SessionUnavailable("Codex could not resume to an idle state; check worker status: "
|
|
710
|
+
+ explain_native_failure(
|
|
711
|
+
str(result.get("error") or result.get("state"))),
|
|
712
|
+
short=worker_id[:8])
|
|
713
|
+
store.revalidate()
|
|
714
|
+
return {"worker": handle, "worker_id": worker_id,
|
|
715
|
+
"engine": "codex", "short": worker_id[:8],
|
|
716
|
+
"thread_id": record["native"]["threadId"],
|
|
717
|
+
"state": "idle", "resumed": bool(result.get("changed")),
|
|
718
|
+
"loaded": True}
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _send_codex_original(
|
|
722
|
+
handle: str, worker_id: str, text: str, delivery_id: str,
|
|
723
|
+
submit_native: Callable, observe_settlement: Callable | None,
|
|
724
|
+
*, mode: str = "new_turn_only",
|
|
725
|
+
real_native_transport: bool = False,
|
|
726
|
+
expected_generation: str | None = None,
|
|
727
|
+
parent: dict[str, Any] | None = None,
|
|
728
|
+
caller_metadata: dict[str, Any] | None = None,
|
|
729
|
+
settle: bool = False,
|
|
730
|
+
cap: dict | None = None,
|
|
731
|
+
) -> dict[str, Any]:
|
|
732
|
+
"""Codex half of EXISTING protocol_ops.send_turn, original semantic IDs."""
|
|
733
|
+
from ccx.codex import delivery as codex_delivery
|
|
734
|
+
from ccx import constants
|
|
735
|
+
from ccx.store.root import NativeActionDenied, NativeEffectIndeterminate, _HASH, require_ccx_active
|
|
736
|
+
from ccx.store.store import _digest
|
|
737
|
+
store = require_ccx_active(
|
|
738
|
+
constants.CCX_ROOT, expected_generation=expected_generation,
|
|
739
|
+
)
|
|
740
|
+
rec = store.record(handle)
|
|
741
|
+
native = rec.get("native")
|
|
742
|
+
if (not isinstance(native, dict)
|
|
743
|
+
or not isinstance(native.get("threadId"), str)
|
|
744
|
+
or native.get("threadSource") != store.codex_source(handle)):
|
|
745
|
+
raise NativeActionDenied("Codex conversation is not ready; check worker status and retry")
|
|
746
|
+
original = codex_delivery.read_receipt(handle, delivery_id)
|
|
747
|
+
notification = None
|
|
748
|
+
if original is not None:
|
|
749
|
+
# Validate the immutable original token before any daemon start, native
|
|
750
|
+
# resume, new semantic receipt or turn/start side effect.
|
|
751
|
+
codex_delivery.require_matching(
|
|
752
|
+
handle, delivery_id, text, mode=mode, cap=cap,
|
|
753
|
+
**({"caller_metadata": caller_metadata} if parent is not None else {}),
|
|
754
|
+
)
|
|
755
|
+
if original is None or original.get("state") == "prepared":
|
|
756
|
+
store.authorize_native_action(handle, "send")
|
|
757
|
+
original_effect = store.read_receipt(handle, "codex-send", delivery_id)
|
|
758
|
+
if (original_effect is not None
|
|
759
|
+
and original_effect.get("state") not in ("prepared",)):
|
|
760
|
+
raise NativeEffectIndeterminate(
|
|
761
|
+
"Message was already submitted; reconcile its delivery instead of resending"
|
|
762
|
+
)
|
|
763
|
+
# A preexisting token must reject a different parent before a possible
|
|
764
|
+
# original native thread reload. Notification registration owns the same-ID reuse rules.
|
|
765
|
+
if parent is not None and original is not None:
|
|
766
|
+
notification = create_parent_notification(
|
|
767
|
+
parent=parent, worker_engine="codex", worker_id=worker_id,
|
|
768
|
+
worker_name=rec.get("name"), source="send",
|
|
769
|
+
operation={"kind": "codex-turn", "deliveryId": delivery_id},
|
|
770
|
+
settle=settle,
|
|
771
|
+
)
|
|
772
|
+
if original is None or original.get("state") == "prepared":
|
|
773
|
+
# Native auto picks the route in codexd, not from this stale record.
|
|
774
|
+
# The idle-only load path is needed for a cold worker, never for an
|
|
775
|
+
# already busy turn; explicit steer_only never loads/revives.
|
|
776
|
+
if real_native_transport and (mode == "new_turn_only" or (
|
|
777
|
+
mode == "auto" and
|
|
778
|
+
(rec.get("lifecycle") or {}).get("state") not in ("busy", "waiting")
|
|
779
|
+
)):
|
|
780
|
+
try:
|
|
781
|
+
ensure_ready(handle, expected_generation=store.generation)
|
|
782
|
+
except DeliveryRejected as error:
|
|
783
|
+
error.delivery_id = delivery_id
|
|
784
|
+
raise
|
|
785
|
+
# Existing owned/submitted/accepted/rejected receipts are read-only and
|
|
786
|
+
# must never start a provider merely to reconcile the same delivery id.
|
|
787
|
+
receipt, _fresh = codex_delivery.prepare(
|
|
788
|
+
handle, delivery_id, text, mode=mode, cap=cap,
|
|
789
|
+
**({"caller_metadata": caller_metadata} if parent is not None else {}),
|
|
790
|
+
)
|
|
791
|
+
if parent is not None:
|
|
792
|
+
if notification is None:
|
|
793
|
+
# Registration fsyncs the original receipt-bound event before any native
|
|
794
|
+
# intent/submitted marker or first turn/start can be issued.
|
|
795
|
+
notification = create_parent_notification(
|
|
796
|
+
parent=parent, worker_engine="codex", worker_id=worker_id,
|
|
797
|
+
worker_name=rec.get("name"), source="send",
|
|
798
|
+
operation={"kind": "codex-turn", "deliveryId": delivery_id},
|
|
799
|
+
settle=settle,
|
|
800
|
+
)
|
|
801
|
+
from ccx.notify.daemon import ensure_notify_daemon
|
|
802
|
+
ensure_notify_daemon() # failure leaves a prepared receipt + event, no native effect
|
|
803
|
+
def public(value):
|
|
804
|
+
return {
|
|
805
|
+
"worker": {"worker": handle, "worker_id": worker_id,
|
|
806
|
+
"engine": "codex", "short": worker_id[:8]},
|
|
807
|
+
"delivery": codex_delivery.public_receipt(value),
|
|
808
|
+
"status": value["state"],
|
|
809
|
+
"settled": isinstance(value.get("settlement"), dict),
|
|
810
|
+
"notification": (public_parent_notification(
|
|
811
|
+
read_parent_notification(notification["eventId"]) or notification,
|
|
812
|
+
) if notification is not None else None),
|
|
813
|
+
}
|
|
814
|
+
if receipt["state"] in ("accepted", "rejected"):
|
|
815
|
+
if receipt["state"] == "rejected" and receipt.get("rejectionCode") == "budget-stopped":
|
|
816
|
+
from ccx.errors import BudgetStopped
|
|
817
|
+
reason = receipt.get("rejectionMessage") or "usage cap already reached"
|
|
818
|
+
raise BudgetStopped(f"budget stop: {reason}",
|
|
819
|
+
short=worker_id[:8], reason=reason, phase="preflight")
|
|
820
|
+
if (receipt["state"] == "accepted"
|
|
821
|
+
and receipt.get("settlement") is None
|
|
822
|
+
and observe_settlement is not None
|
|
823
|
+
and codex_delivery.route(receipt) != "steer"):
|
|
824
|
+
receipt = _settle_codex_original(
|
|
825
|
+
handle, worker_id, native, delivery_id,
|
|
826
|
+
receipt["turnId"], observe_settlement,
|
|
827
|
+
expected_source=store.codex_source(handle),
|
|
828
|
+
)
|
|
829
|
+
return public(receipt)
|
|
830
|
+
# For native auto/steer the immutable intent precedes route selection:
|
|
831
|
+
# its client ID remains None even after codexd owns a start/steer kind.
|
|
832
|
+
identity = _digest(codex_delivery.intent_spec(receipt))
|
|
833
|
+
intent = store.begin_native_intent(
|
|
834
|
+
handle, family="codex-send", action="send",
|
|
835
|
+
operation_id=delivery_id, intent_sha256=identity,
|
|
836
|
+
)
|
|
837
|
+
if intent.replayed:
|
|
838
|
+
raise NativeEffectIndeterminate(
|
|
839
|
+
"Codex message outcome is recorded but the turn is missing; reconcile the delivery"
|
|
840
|
+
)
|
|
841
|
+
# Only strict sends freeze the preminted start ID in the caller.
|
|
842
|
+
# Auto/steer stay prepared; the daemon owns their route under its lock.
|
|
843
|
+
if mode == "new_turn_only":
|
|
844
|
+
submitted = codex_delivery.mark_submitted(handle, delivery_id)
|
|
845
|
+
if submitted.get("state") != "submitted":
|
|
846
|
+
raise NativeEffectIndeterminate("Codex message was not submitted; inspect its delivery status")
|
|
847
|
+
answer = store.dispatch_native_intent_once(
|
|
848
|
+
intent, bounded_submit=lambda budget: submit_native(
|
|
849
|
+
handle, native["threadId"], text, delivery_id, budget,
|
|
850
|
+
),
|
|
851
|
+
)
|
|
852
|
+
# Auto/steer freeze the native route and kind-correct client id in codexd.
|
|
853
|
+
# Read its original receipt afresh; never compare ACK to pre-route None.
|
|
854
|
+
frozen = codex_delivery.read_receipt(handle, delivery_id)
|
|
855
|
+
frozen_route = codex_delivery.route(frozen) if isinstance(frozen, dict) else None
|
|
856
|
+
frozen_target = frozen.get("targetTurnId") if isinstance(frozen, dict) else None
|
|
857
|
+
frozen_client = frozen.get("clientUserMessageId") if isinstance(frozen, dict) else None
|
|
858
|
+
rejected = answer.get("accepted") is False and answer.get("nativeStatus") == "rejected"
|
|
859
|
+
if (not isinstance(frozen, dict)
|
|
860
|
+
or _digest(codex_delivery.intent_spec(frozen)) != identity
|
|
861
|
+
or frozen.get("requestedMode") != mode
|
|
862
|
+
or (not rejected and frozen.get("state") != "submitted"
|
|
863
|
+
and not (frozen.get("state") == "accepted"
|
|
864
|
+
# Already recovered from this answer's own journaled
|
|
865
|
+
# user message: the same turn, never a different one.
|
|
866
|
+
and frozen.get("turnId") == answer.get("turnId")))
|
|
867
|
+
or answer.get("workerId") != worker_id
|
|
868
|
+
or answer.get("engine") != "codex"
|
|
869
|
+
or answer.get("deliveryId") != delivery_id
|
|
870
|
+
or answer.get("nativeSource") != store.codex_source(handle)
|
|
871
|
+
or answer.get("threadId") != native["threadId"]
|
|
872
|
+
or answer.get("clientUserMessageId") != frozen_client
|
|
873
|
+
or (mode != "new_turn_only"
|
|
874
|
+
and answer.get("route") != frozen_route)
|
|
875
|
+
or (frozen_route == "steer"
|
|
876
|
+
and (not isinstance(frozen_target, str) or not frozen_target
|
|
877
|
+
or answer.get("targetTurnId") != frozen_target
|
|
878
|
+
or (not rejected and answer.get("turnId") != frozen_target)))
|
|
879
|
+
or not isinstance(answer.get("nativeEvidenceSha256"), str)
|
|
880
|
+
or not _HASH.fullmatch(answer["nativeEvidenceSha256"])):
|
|
881
|
+
raise NativeEffectIndeterminate(
|
|
882
|
+
"Codex response did not match this worker; inspect the delivery"
|
|
883
|
+
"thread/source/route/turn/client identity; never replay"
|
|
884
|
+
)
|
|
885
|
+
if answer.get("accepted") is False and answer.get("nativeStatus") == "rejected":
|
|
886
|
+
code = answer.get("rejectionCode")
|
|
887
|
+
if not isinstance(code, str) or not code:
|
|
888
|
+
raise NativeEffectIndeterminate("native rejection has no code")
|
|
889
|
+
finished = codex_delivery.mark_rejected(
|
|
890
|
+
handle, delivery_id, code=code,
|
|
891
|
+
message=str(answer.get("rejectionMessage") or code),
|
|
892
|
+
)
|
|
893
|
+
store.settle_native_intent(
|
|
894
|
+
intent, state="rejected",
|
|
895
|
+
native_evidence_sha256=answer["nativeEvidenceSha256"],
|
|
896
|
+
)
|
|
897
|
+
if code == "budget-stopped":
|
|
898
|
+
from ccx.errors import BudgetStopped
|
|
899
|
+
reason = str(answer.get("rejectionMessage") or "usage cap reached")
|
|
900
|
+
raise BudgetStopped(f"budget stop: {reason}", short=worker_id[:8],
|
|
901
|
+
reason=reason, phase="preflight")
|
|
902
|
+
return public(finished)
|
|
903
|
+
if answer.get("accepted") is not True or not isinstance(answer.get("turnId"), str) or not answer["turnId"]:
|
|
904
|
+
raise NativeEffectIndeterminate("Codex turn was not confirmed accepted; reconcile its delivery")
|
|
905
|
+
accepted = codex_delivery.mark_accepted(
|
|
906
|
+
handle, delivery_id, turn_id=answer["turnId"],
|
|
907
|
+
)
|
|
908
|
+
store.settle_native_intent(
|
|
909
|
+
intent, state="accepted",
|
|
910
|
+
native_evidence_sha256=answer["nativeEvidenceSha256"],
|
|
911
|
+
)
|
|
912
|
+
if frozen_route != "steer":
|
|
913
|
+
# A fast app-server can publish turn/completed before the caller
|
|
914
|
+
# writes acceptance. Its original fsynced notification is enough to
|
|
915
|
+
# settle without hydrating history or issuing another native turn.
|
|
916
|
+
from ccx.codex.completion import settle_from_journal
|
|
917
|
+
try:
|
|
918
|
+
completed = settle_from_journal(store, handle, answer["turnId"])
|
|
919
|
+
if completed is not None:
|
|
920
|
+
accepted = completed
|
|
921
|
+
except Exception:
|
|
922
|
+
# Accepted placement remains authoritative. A bad/missing journal
|
|
923
|
+
# is recoverable later by source-fenced native observation.
|
|
924
|
+
pass
|
|
925
|
+
if (observe_settlement is not None and frozen_route != "steer"
|
|
926
|
+
and not accepted.get("settlement")):
|
|
927
|
+
accepted = _settle_codex_original(
|
|
928
|
+
handle, worker_id, native, delivery_id, answer["turnId"],
|
|
929
|
+
observe_settlement, expected_source=store.codex_source(handle),
|
|
930
|
+
)
|
|
931
|
+
return public(accepted)
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def _settle_codex_original(
|
|
935
|
+
handle: str, worker_id: str, native: dict, delivery_id: str,
|
|
936
|
+
turn_id: str, observe_completion: Callable,
|
|
937
|
+
*, expected_source: str,
|
|
938
|
+
) -> dict[str, Any]:
|
|
939
|
+
"""Observe native turn/completed independently, without activation lock."""
|
|
940
|
+
from ccx.codex import delivery as codex_delivery
|
|
941
|
+
from ccx.store.root import NativeEffectIndeterminate
|
|
942
|
+
owned = codex_delivery.read_receipt(handle, delivery_id)
|
|
943
|
+
if isinstance(owned, dict) and codex_delivery.route(owned) == "steer":
|
|
944
|
+
raise NativeEffectIndeterminate(
|
|
945
|
+
"Codex steer acceptance is placement, not a completed-turn receipt"
|
|
946
|
+
)
|
|
947
|
+
original = observe_completion(handle, codex_delivery.turn_ref_for(turn_id))
|
|
948
|
+
if (not isinstance(original, dict)
|
|
949
|
+
or original.get("workerId") != worker_id
|
|
950
|
+
or original.get("deliveryId") != delivery_id
|
|
951
|
+
or original.get("nativeSource") != expected_source
|
|
952
|
+
or original.get("threadId") != native["threadId"]
|
|
953
|
+
or original.get("turnId") != turn_id
|
|
954
|
+
or original.get("status") != "completed"):
|
|
955
|
+
raise NativeEffectIndeterminate(
|
|
956
|
+
"Codex turn completion could not be verified; inspect its status"
|
|
957
|
+
)
|
|
958
|
+
return codex_delivery.mark_settled(
|
|
959
|
+
handle, delivery_id, turn_id=turn_id,
|
|
960
|
+
original_positive_completion=original,
|
|
961
|
+
)
|