ccx-native 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +240 -4
- package/bin/ccx +55 -2
- package/ccx/__init__.py +167 -0
- package/ccx/__main__.py +28 -0
- package/ccx/claude/__init__.py +0 -0
- package/ccx/claude/analytics.py +466 -0
- package/ccx/claude/attach.py +807 -0
- package/ccx/claude/context.py +465 -0
- package/ccx/claude/delivery.py +806 -0
- package/ccx/claude/external.py +253 -0
- package/ccx/claude/host/__init__.py +0 -0
- package/ccx/claude/host/events.py +297 -0
- package/ccx/claude/host/hooks.py +438 -0
- package/ccx/claude/host/pty.py +269 -0
- package/ccx/claude/host/session.py +5244 -0
- package/ccx/claude/host/subagents.py +169 -0
- package/ccx/claude/host/successor.py +267 -0
- package/ccx/claude/managed_turns.py +240 -0
- package/ccx/claude/oauth.py +176 -0
- package/ccx/claude/settings.py +339 -0
- package/ccx/claude/spawn.py +557 -0
- package/ccx/claude/tasks.py +72 -0
- package/ccx/claude/telemetry.py +282 -0
- package/ccx/claude/transcript.py +2535 -0
- package/ccx/claude/usage.py +796 -0
- package/ccx/claude/usage_control.py +204 -0
- package/ccx/cli/__init__.py +0 -0
- package/ccx/cli/codex.py +1009 -0
- package/ccx/cli/commands/__init__.py +0 -0
- package/ccx/cli/commands/_common.py +407 -0
- package/ccx/cli/commands/control.py +161 -0
- package/ccx/cli/commands/create.py +669 -0
- package/ccx/cli/commands/models.py +53 -0
- package/ccx/cli/commands/read.py +1104 -0
- package/ccx/cli/commands/turns.py +1081 -0
- package/ccx/cli/controller.py +757 -0
- package/ccx/cli/main.py +302 -0
- package/ccx/cli/parser.py +1439 -0
- package/ccx/cli/skills.py +215 -0
- package/ccx/cli/statusline.py +160 -0
- package/ccx/client.py +1904 -0
- package/ccx/codex/__init__.py +0 -0
- package/ccx/codex/attach_tty.py +267 -0
- package/ccx/codex/client.py +738 -0
- package/ccx/codex/completion.py +121 -0
- package/ccx/codex/control.py +288 -0
- package/ccx/codex/daemon/__init__.py +0 -0
- package/ccx/codex/daemon/__main__.py +4 -0
- package/ccx/codex/daemon/daemon.py +1558 -0
- package/ccx/codex/daemon/models.py +140 -0
- package/ccx/codex/daemon/native.py +1590 -0
- package/ccx/codex/daemon/release.py +367 -0
- package/ccx/codex/daemon/worker.py +2424 -0
- package/ccx/codex/delivery.py +758 -0
- package/ccx/codex/delivery_recovery.py +232 -0
- package/ccx/codex/evidence.py +827 -0
- package/ccx/codex/history.py +158 -0
- package/ccx/codex/history_store.py +131 -0
- package/ccx/codex/notify_history.py +140 -0
- package/ccx/codex/ops.py +666 -0
- package/ccx/codex/paths.py +70 -0
- package/ccx/codex/protocol.py +805 -0
- package/ccx/codex/provenance.py +105 -0
- package/ccx/codex/read.py +29 -0
- package/ccx/codex/records.py +51 -0
- package/ccx/codex/reports.py +177 -0
- package/ccx/codex/runtime.py +1454 -0
- package/ccx/codex/settle.py +363 -0
- package/ccx/codex/telemetry.py +792 -0
- package/ccx/codex/types.py +659 -0
- package/ccx/codex/worker_environment.py +25 -0
- package/ccx/constants.py +288 -0
- package/ccx/controllers/__init__.py +0 -0
- package/ccx/controllers/controller.py +1449 -0
- package/ccx/controllers/live_vars.py +1706 -0
- package/ccx/controllers/live_vars_runner.py +216 -0
- package/ccx/controllers/notify.py +213 -0
- package/ccx/controllers/orchestrator.py +193 -0
- package/ccx/controllers/policy.py +57 -0
- package/ccx/controllers/store.py +681 -0
- package/ccx/errors.py +634 -0
- package/ccx/models.py +225 -0
- package/ccx/notify/__init__.py +0 -0
- package/ccx/notify/__main__.py +5 -0
- package/ccx/notify/client.py +116 -0
- package/ccx/notify/daemon.py +480 -0
- package/ccx/notify/deliver.py +304 -0
- package/ccx/notify/paths.py +51 -0
- package/ccx/notify/registry.py +649 -0
- package/ccx/notify/select.py +750 -0
- package/ccx/notify/settle.py +684 -0
- package/ccx/ops/__init__.py +0 -0
- package/ccx/ops/background.py +115 -0
- package/ccx/ops/cancel.py +296 -0
- package/ccx/ops/codex_import.py +96 -0
- package/ccx/ops/common.py +397 -0
- package/ccx/ops/compact.py +337 -0
- package/ccx/ops/compact_ops.py +550 -0
- package/ccx/ops/context_events.py +48 -0
- package/ccx/ops/conversation.py +1593 -0
- package/ccx/ops/conversation_paging.py +1339 -0
- package/ccx/ops/create.py +56 -0
- package/ccx/ops/creation.py +952 -0
- package/ccx/ops/creation_steps.py +575 -0
- package/ccx/ops/event_feed.py +710 -0
- package/ccx/ops/forking.py +711 -0
- package/ccx/ops/history.py +1041 -0
- package/ccx/ops/input.py +556 -0
- package/ccx/ops/maintenance.py +295 -0
- package/ccx/ops/report.py +773 -0
- package/ccx/ops/semantics.py +490 -0
- package/ccx/ops/send.py +961 -0
- package/ccx/ops/steer_ops.py +867 -0
- package/ccx/ops/terminate.py +383 -0
- package/ccx/ops/turn_reports.py +348 -0
- package/ccx/ops/wait.py +762 -0
- package/ccx/ops/worker_events.py +640 -0
- package/ccx/rpc/__init__.py +0 -0
- package/ccx/rpc/local_rpc.py +1602 -0
- package/ccx/rpc/rpc.py +559 -0
- package/ccx/rpc/serve.py +374 -0
- package/ccx/rpc/transport.py +409 -0
- package/ccx/rpc/view.py +125 -0
- package/ccx/store/__init__.py +0 -0
- package/ccx/store/discovery.py +170 -0
- package/ccx/store/identity.py +105 -0
- package/ccx/store/journal_io.py +79 -0
- package/ccx/store/liveness.py +194 -0
- package/ccx/store/paths.py +250 -0
- package/ccx/store/record_cache.py +40 -0
- package/ccx/store/records.py +309 -0
- package/ccx/store/root.py +789 -0
- package/ccx/store/runtime_presence.py +137 -0
- package/ccx/store/store.py +2332 -0
- package/ccx/store/worker_state.py +86 -0
- package/ccx/system/__init__.py +0 -0
- package/ccx/system/bounded.py +296 -0
- package/ccx/system/cwd_paths.py +25 -0
- package/ccx/system/engine_binary.py +59 -0
- package/ccx/system/executable.py +50 -0
- package/ccx/system/oslayer.py +667 -0
- package/ccx/system/pipes.py +46 -0
- package/ccx/system/private_json.py +54 -0
- package/ccx/system/sanitize.py +43 -0
- package/ccx/types.py +809 -0
- package/ccx/version.py +29 -0
- package/ccx/workers.py +2093 -0
- package/ccx.py +9 -0
- package/package.json +39 -7
- package/pyproject.toml +87 -0
- package/skills/ccx/SKILL.md +166 -0
- package/skills/ccx/references/controllers.md +244 -0
- package/skills/ccx/references/history.md +50 -0
- package/skills/ccx/references/troubleshooting.md +94 -0
- package/skills/ccx/references/variables.md +62 -0
|
@@ -0,0 +1,2424 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
import uuid
|
|
8
|
+
from copy import deepcopy
|
|
9
|
+
from typing import Any
|
|
10
|
+
from ccx.codex.evidence import CONTEXT_CHANGED_METHOD, BACKGROUND_CHANGED_METHOD, COMPACTION_CONTEXT_METHOD, COMPACT_STARTED_METHOD, COMPACT_ENDED_METHOD, INPUT_REQUESTED_METHOD, INPUT_RESOLVED_METHOD, JOURNALED_METHODS, LIFECYCLE_METHOD, EvidenceJournal, read_index as read_evidence_index, read_journal as read_compact_journal
|
|
11
|
+
from ccx.codex.notify_history import notification_delivery_match
|
|
12
|
+
from ccx.ops.background import label as background_label, summary as background_summary, materially_changed
|
|
13
|
+
from ccx.ops.context_events import ContextChanges
|
|
14
|
+
from ccx.codex.protocol import AppServerClient, ProtocolError, ProtocolNotSent, ProtocolTransportError
|
|
15
|
+
from ccx.codex.reports import native_timestamp_ms, report_from_item, should_replace_live_report
|
|
16
|
+
from ccx.codex.records import now_ms
|
|
17
|
+
from ccx.codex.daemon.models import _abandon_active_compact, WorkerKilled, _LAUNCH_SETTLE_CAP_S, _TurnLaunch, _bounded, _protocol_failure
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from ccx.codex.daemon.daemon import CodexDaemon
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
_TERMINAL_STATES = {"dead", "idle-killed", "crashed", "budget-stopped"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _system_error_reason(error) -> str:
|
|
27
|
+
"""The crash reason, with Codex's own error message when there is one."""
|
|
28
|
+
message = error.get("message") if isinstance(error, dict) else None
|
|
29
|
+
if isinstance(message, str) and message.strip():
|
|
30
|
+
return "Codex thread system error: " + message.strip()[:500]
|
|
31
|
+
return "Codex thread system error"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
_PARENT_NOTIFY_FENCES = 32
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
TURN_OUTCOME_STATUSES = {"completed", "interrupted", "failed"}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
APPROVAL_METHODS: dict[str, str] = {
|
|
41
|
+
"item/commandExecution/requestApproval": "command",
|
|
42
|
+
"item/fileChange/requestApproval": "fileChange",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _turn_id(value: dict[str, Any]) -> str | None:
|
|
47
|
+
turn = value.get("turn")
|
|
48
|
+
return turn.get("id") if isinstance(turn, dict) else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _last_turn_from_native(turn: dict[str, Any]) -> dict[str, Any]:
|
|
52
|
+
"""Normalize one native ``Turn`` into the durable outcome projection.
|
|
53
|
+
|
|
54
|
+
``lastTurn`` deliberately contains only the terminal turn facts. In
|
|
55
|
+
particular, timestamps come from Codex's Turn object; a missing native
|
|
56
|
+
timestamp remains ``None`` instead of being replaced with CCX wall-clock
|
|
57
|
+
time. Unknown native statuses are not promoted into the public outcome
|
|
58
|
+
vocabulary.
|
|
59
|
+
"""
|
|
60
|
+
status = turn.get("status")
|
|
61
|
+
return {
|
|
62
|
+
"turnId": turn.get("id"),
|
|
63
|
+
"status": status if status in TURN_OUTCOME_STATUSES else None,
|
|
64
|
+
"error": turn.get("error"),
|
|
65
|
+
"startedAtMs": native_timestamp_ms(turn.get("startedAt")),
|
|
66
|
+
"completedAtMs": native_timestamp_ms(turn.get("completedAt")),
|
|
67
|
+
"durationMs": (
|
|
68
|
+
turn.get("durationMs")
|
|
69
|
+
if isinstance(turn.get("durationMs"), (int, float))
|
|
70
|
+
and not isinstance(turn.get("durationMs"), bool)
|
|
71
|
+
else None
|
|
72
|
+
),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _approval_key(approval: dict[str, Any]) -> str:
|
|
79
|
+
"""One pending approval's identity: its JSON-RPC request id, typed."""
|
|
80
|
+
return json.dumps(approval.get("requestId"), sort_keys=True, default=str)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class Worker:
|
|
84
|
+
"""One CCX-owned Codex worker, driven over the daemon's shared client."""
|
|
85
|
+
|
|
86
|
+
def __init__(self, daemon: "CodexDaemon", record: dict[str, Any]):
|
|
87
|
+
self.daemon = daemon
|
|
88
|
+
# Original local short variable is internally a canonical full owner.
|
|
89
|
+
# Eight-character display prefixes never identify a worker here.
|
|
90
|
+
from ccx.store.identity import canonical_worker_id
|
|
91
|
+
self.worker_id = canonical_worker_id(record.get("workerId"))
|
|
92
|
+
if record.get("engine") != "codex":
|
|
93
|
+
raise ValueError("daemon Worker requires globally owned Codex ID")
|
|
94
|
+
self.record = deepcopy(record)
|
|
95
|
+
self._disk_baseline = deepcopy(record)
|
|
96
|
+
self.short = self.worker_id
|
|
97
|
+
self.lock = threading.RLock()
|
|
98
|
+
self.operation_lock = threading.Lock()
|
|
99
|
+
self.load_lock = threading.Lock()
|
|
100
|
+
self.loaded = False
|
|
101
|
+
# Set by `kill` before it unloads, and never cleared: a Worker object
|
|
102
|
+
# that has been killed is dead forever, and a fresh `load` builds a new
|
|
103
|
+
# one. This is the whole kill/load race protocol — see `_abort_if_killed`.
|
|
104
|
+
self.killed = threading.Event()
|
|
105
|
+
# Published before kill waits for `operation_lock`, and never cleared.
|
|
106
|
+
# A kill that has to wait behind an in-flight steer/approval must not
|
|
107
|
+
# meanwhile let new operations in — but it also must not bypass the
|
|
108
|
+
# lock and race the operation that is already running. The sticky
|
|
109
|
+
# intent separates the two: new work is refused immediately, the
|
|
110
|
+
# running op is allowed to settle, and only then does the kill act.
|
|
111
|
+
self.kill_intent = threading.Event()
|
|
112
|
+
# Non-None exactly while a `turn/start` request is in flight. Written
|
|
113
|
+
# only under `self.lock`, which is what makes kill-vs-launch atomic.
|
|
114
|
+
self.launch: _TurnLaunch | None = None
|
|
115
|
+
# Caps belong to one original native turn, never to the worker's next turn.
|
|
116
|
+
self.cap_turn_id: str | None = None
|
|
117
|
+
self.active_caps: dict | None = None
|
|
118
|
+
# A source-proven resume may return the preceding failed turn's
|
|
119
|
+
# persisted systemError. Until the next native turn/status transition,
|
|
120
|
+
# another systemError edge is that old state, not a new crash.
|
|
121
|
+
self._resumed_system_error_pending = False
|
|
122
|
+
# Native threads this object's load may have subscribed the shared
|
|
123
|
+
# connection to. Codex keeps a thread loaded (with its per-thread MCP
|
|
124
|
+
# servers) until its last subscriber leaves, so `_unload` releases
|
|
125
|
+
# these. Kept rather than cleared: unsubscribing twice is harmless, and
|
|
126
|
+
# a kill that lands mid-resume unloads before the subscription exists.
|
|
127
|
+
self.native_threads: set[str] = set()
|
|
128
|
+
# Parent-notification deliveries whose `turn/start` this object saw
|
|
129
|
+
# answered, so a retry walking history outside the operation lock
|
|
130
|
+
# cannot launch one twice. delivery id -> native turn id.
|
|
131
|
+
self.launched_notifications: dict[str, str | None] = {}
|
|
132
|
+
# Claimed under `daemon.lock` by the one `_unload` that tears this
|
|
133
|
+
# object down; see there.
|
|
134
|
+
self.teardown_claimed = False
|
|
135
|
+
self.last_activity = time.monotonic()
|
|
136
|
+
self.attach_count = 0 # live attach RPC connections, never persisted
|
|
137
|
+
# Native commands become background only if still open at turn/completed.
|
|
138
|
+
# The journal is the source; these are this app-server's owned live IDs.
|
|
139
|
+
# Public context emission is one sparse stream, independent of raw native
|
|
140
|
+
# usage notifications. Reuse the existing daemon janitor for pending flush.
|
|
141
|
+
self._context_changes = ContextChanges()
|
|
142
|
+
self._bg_candidates: dict[str, dict] = {}
|
|
143
|
+
self._bg_open: dict[str, dict] = {}
|
|
144
|
+
# No replay ledger: a persisted snapshot is enough to compare current
|
|
145
|
+
# registry truth when this worker is adopted or notified again.
|
|
146
|
+
prior_items = (record.get("backgroundTaskSummary") or {}).get("items") or []
|
|
147
|
+
self._bg_notifications = {
|
|
148
|
+
item["id"]: dict(item) for item in prior_items
|
|
149
|
+
if isinstance(item, dict) and item.get("kind") == "notification"
|
|
150
|
+
and item.get("status") == "pending"
|
|
151
|
+
and isinstance(item.get("id"), str)
|
|
152
|
+
}
|
|
153
|
+
# The last record write a notification handler could not make (disk
|
|
154
|
+
# full, permissions). The in-memory record stays authoritative.
|
|
155
|
+
self.persistence_error: dict[str, Any] | None = None
|
|
156
|
+
# An external terminal CAS wins over this cached Worker forever.
|
|
157
|
+
self._externally_terminalized = False
|
|
158
|
+
# Sparse durable evidence for this worker's turns. Opened lazily on
|
|
159
|
+
# the first journaled notification, so a worker that never runs one
|
|
160
|
+
# leaves no file — which is itself the truthful answer to "was CCX
|
|
161
|
+
# watching?".
|
|
162
|
+
# `health` rehydrates a degradation an earlier process already
|
|
163
|
+
# recorded: a fresh writer must not present a worker whose evidence is
|
|
164
|
+
# known to be incomplete as complete again.
|
|
165
|
+
self.evidence = EvidenceJournal(
|
|
166
|
+
self.short,
|
|
167
|
+
codex_version=getattr(daemon, "codex_version", None),
|
|
168
|
+
health=record.get("evidence"),
|
|
169
|
+
)
|
|
170
|
+
try:
|
|
171
|
+
evidence_index = read_evidence_index(self.short)
|
|
172
|
+
except Exception:
|
|
173
|
+
evidence_index = {}
|
|
174
|
+
self.completed_turn_ids = {
|
|
175
|
+
str(turn_id)
|
|
176
|
+
for turn_id, facts in (evidence_index.get("turns") or {}).items()
|
|
177
|
+
if isinstance(facts, dict) and facts.get("sawTurnCompleted")
|
|
178
|
+
}
|
|
179
|
+
# Journal-derived identities keep compaction end publication idempotent
|
|
180
|
+
# when native notifications repeat or codexd reloads an owned worker.
|
|
181
|
+
self._compact_started: dict[str, set[str]] = {}
|
|
182
|
+
self._compact_finished: dict[str, set[str]] = {}
|
|
183
|
+
self._compact_ended: set[str] = set()
|
|
184
|
+
self._compact_legacy: set[str] = set()
|
|
185
|
+
try:
|
|
186
|
+
compact_history = read_compact_journal(self.short).events
|
|
187
|
+
except Exception:
|
|
188
|
+
compact_history = []
|
|
189
|
+
for row in compact_history:
|
|
190
|
+
tid, item_id, method = row.get("turnId"), row.get("itemId"), row.get("method")
|
|
191
|
+
if method == "thread/compacted" and isinstance(tid, str):
|
|
192
|
+
self._compact_legacy.add(tid)
|
|
193
|
+
if not isinstance(tid, str) or not tid:
|
|
194
|
+
continue
|
|
195
|
+
# Historical native items are still forensic proof, not feed
|
|
196
|
+
# events. Only CCX-owned sparse lines rehydrate publisher dedupe.
|
|
197
|
+
# Older journals lacking them intentionally gain no backfill.
|
|
198
|
+
if (row.get("itemType") == "contextCompaction"
|
|
199
|
+
and isinstance(item_id, str)
|
|
200
|
+
and method == "item/completed"):
|
|
201
|
+
self._compact_finished.setdefault(tid, set()).add(item_id)
|
|
202
|
+
if method == COMPACT_STARTED_METHOD:
|
|
203
|
+
key = (row.get("params") or {}).get("compactId")
|
|
204
|
+
if isinstance(key, str):
|
|
205
|
+
self._compact_started.setdefault(tid, set()).add(key)
|
|
206
|
+
if method == COMPACT_ENDED_METHOD:
|
|
207
|
+
key = (row.get("params") or {}).get("compactId")
|
|
208
|
+
if isinstance(key, str):
|
|
209
|
+
self._compact_ended.add(key)
|
|
210
|
+
# What the journal last said about this worker's state and approvals.
|
|
211
|
+
# `_journal_lifecycle_locked` diffs the record against it on every
|
|
212
|
+
# save, so each transition is journaled exactly once whichever of the
|
|
213
|
+
# many record-mutating paths made it.
|
|
214
|
+
self.journaled_state = (record.get("lifecycle") or {}).get("state")
|
|
215
|
+
self.journaled_approvals = {
|
|
216
|
+
_approval_key(approval): approval
|
|
217
|
+
for approval in record.get("pendingApprovals") or []
|
|
218
|
+
if isinstance(approval, dict)
|
|
219
|
+
}
|
|
220
|
+
# Decisions CCX itself sent, keyed like `journaled_approvals`, so the
|
|
221
|
+
# resolution line can say how a request was answered.
|
|
222
|
+
self.approval_decisions: dict[str, str] = {}
|
|
223
|
+
# The caller's response id for a decision, journaled with it so a
|
|
224
|
+
# caller that lost the reply can prove its decision landed.
|
|
225
|
+
self.approval_response_ids: dict[str, str] = {}
|
|
226
|
+
last_turn = record.get("lastTurn")
|
|
227
|
+
if (
|
|
228
|
+
isinstance(last_turn, dict)
|
|
229
|
+
and isinstance(last_turn.get("turnId"), str)
|
|
230
|
+
and last_turn.get("turnId")
|
|
231
|
+
):
|
|
232
|
+
self.completed_turn_ids.add(str(last_turn["turnId"]))
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def client(self) -> AppServerClient:
|
|
236
|
+
client = self.daemon.client
|
|
237
|
+
if client is None:
|
|
238
|
+
raise RuntimeError("app-server client is not running")
|
|
239
|
+
return client
|
|
240
|
+
|
|
241
|
+
def _save(self) -> None:
|
|
242
|
+
"""Save once with field CAS; retry only a proved safe fresh-record rebase.
|
|
243
|
+
|
|
244
|
+
A failed CAS is not authority to drop expectations. In particular a
|
|
245
|
+
concurrent CLI kill owns the terminal lifecycle, even if an app-server
|
|
246
|
+
completion races it. The original native event remains in the evidence
|
|
247
|
+
journal when a conflict cannot be reconciled.
|
|
248
|
+
"""
|
|
249
|
+
from ccx.store.root import RecordConflict
|
|
250
|
+
with self.lock:
|
|
251
|
+
if self._externally_terminalized and (
|
|
252
|
+
self.record.get("lifecycle") != self._disk_baseline.get("lifecycle")
|
|
253
|
+
or self.record.get("pendingApprovals")
|
|
254
|
+
!= self._disk_baseline.get("pendingApprovals")
|
|
255
|
+
):
|
|
256
|
+
# This object saw an external terminal CAS. Even another
|
|
257
|
+
# daemon-local _state/approval writer cannot bring it back;
|
|
258
|
+
# restore the current persisted terminal view before refusing.
|
|
259
|
+
latest = self.daemon._uuid_store.record(f"codex:{self.worker_id}")
|
|
260
|
+
self.record = deepcopy(latest)
|
|
261
|
+
self._disk_baseline = deepcopy(latest)
|
|
262
|
+
raise RecordConflict("externally terminalized Codex Worker cannot revive")
|
|
263
|
+
self._journal_lifecycle_locked()
|
|
264
|
+
try:
|
|
265
|
+
self._save_current_locked()
|
|
266
|
+
except RecordConflict:
|
|
267
|
+
self._rebase_conflicting_save_locked()
|
|
268
|
+
self._journal_lifecycle_locked()
|
|
269
|
+
# One bounded retry against the newly observed exact fields.
|
|
270
|
+
# A second conflict is surfaced per worker, never overwritten.
|
|
271
|
+
self._save_current_locked()
|
|
272
|
+
|
|
273
|
+
def _save_current_locked(self) -> None:
|
|
274
|
+
"""One semantic CAS publication; caller holds self.lock."""
|
|
275
|
+
from ccx.store.root import StorageIntegrityError, RecordConflict
|
|
276
|
+
if (self._externally_terminalized
|
|
277
|
+
and self.record["lifecycle"].get("state") not in _TERMINAL_STATES):
|
|
278
|
+
raise RecordConflict("external terminal state cannot be resurrected")
|
|
279
|
+
before = self._disk_baseline
|
|
280
|
+
desired = self.record
|
|
281
|
+
patches: dict[str, object] = {}
|
|
282
|
+
expect: dict[str, object] = {}
|
|
283
|
+
increments: dict[str, int] = {}
|
|
284
|
+
containers = ("lifecycle", "stats", "createArgs")
|
|
285
|
+
top_allowed = {
|
|
286
|
+
"name", "waitingFor", "lastTurnAt", "lastSettledAt",
|
|
287
|
+
"status", "lastTurn", "lastReport", "activeTurnError",
|
|
288
|
+
"pendingApprovals", "context", "activeCompact", "lastCompact",
|
|
289
|
+
"compactCount", "evidence", "tokenUsage", "historyKnownEmpty",
|
|
290
|
+
"nativeArchive", "resolvedModel", "parentNotifyFences",
|
|
291
|
+
"lastTurnError", "pendingSend", "lastConversation", "nativeTurnStart",
|
|
292
|
+
"quotaStop", "backgroundTaskSummary",
|
|
293
|
+
}
|
|
294
|
+
for key in set(before) | set(desired):
|
|
295
|
+
if key in ("recordRevision", "updatedAt"):
|
|
296
|
+
continue
|
|
297
|
+
old = before.get(key)
|
|
298
|
+
new = desired.get(key)
|
|
299
|
+
if new == old:
|
|
300
|
+
continue
|
|
301
|
+
if key not in desired:
|
|
302
|
+
raise StorageIntegrityError(
|
|
303
|
+
"Codex Worker cannot delete a durable record field"
|
|
304
|
+
)
|
|
305
|
+
if key in containers:
|
|
306
|
+
if not isinstance(old, dict) or not isinstance(new, dict):
|
|
307
|
+
raise StorageIntegrityError(
|
|
308
|
+
"Codex Worker cannot replace structured record"
|
|
309
|
+
)
|
|
310
|
+
if not set(old).issubset(new):
|
|
311
|
+
raise StorageIntegrityError(
|
|
312
|
+
"Codex Worker cannot remove structured fields"
|
|
313
|
+
)
|
|
314
|
+
for member in new:
|
|
315
|
+
if member in old and old[member] == new[member]:
|
|
316
|
+
continue
|
|
317
|
+
field = f"{key}.{member}"
|
|
318
|
+
if (key == "stats"
|
|
319
|
+
and type(new[member]) is int
|
|
320
|
+
and type(old.get(member)) is int
|
|
321
|
+
and new[member] >= old[member]):
|
|
322
|
+
increments[field] = new[member] - old[member]
|
|
323
|
+
else:
|
|
324
|
+
patches[field] = new[member]
|
|
325
|
+
expect[field] = old.get(member)
|
|
326
|
+
elif key in top_allowed:
|
|
327
|
+
patches[key] = new
|
|
328
|
+
expect[key] = old
|
|
329
|
+
else:
|
|
330
|
+
raise StorageIntegrityError(
|
|
331
|
+
f"Codex Worker cannot mutate immutable native/owner field {key}"
|
|
332
|
+
)
|
|
333
|
+
ctx = self.daemon._uuid_store
|
|
334
|
+
if patches or increments:
|
|
335
|
+
durable = ctx.mutate_record(
|
|
336
|
+
f"codex:{self.worker_id}",
|
|
337
|
+
patches=patches, increments=increments,
|
|
338
|
+
expect=expect,
|
|
339
|
+
)
|
|
340
|
+
else:
|
|
341
|
+
durable = ctx.record(f"codex:{self.worker_id}")
|
|
342
|
+
# A nonconflicting metadata/evidence patch (or even a pure read) can
|
|
343
|
+
# absorb another process's terminal kill. Recognize its authority on
|
|
344
|
+
# every fresh durable observation, not only in the conflict rebase.
|
|
345
|
+
old_state = (before.get("lifecycle") or {}).get("state")
|
|
346
|
+
committed_state = (durable.get("lifecycle") or {}).get("state")
|
|
347
|
+
proposed_state = (desired.get("lifecycle") or {}).get("state")
|
|
348
|
+
if (committed_state in _TERMINAL_STATES
|
|
349
|
+
and old_state not in _TERMINAL_STATES
|
|
350
|
+
and proposed_state != committed_state):
|
|
351
|
+
self._externally_terminalized = True
|
|
352
|
+
self.killed.set()
|
|
353
|
+
self.record = durable
|
|
354
|
+
self._disk_baseline = deepcopy(durable)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _rebase_conflicting_save_locked(self) -> None:
|
|
358
|
+
"""Reconcile one same-worker CAS conflict, never ignore its expectations.
|
|
359
|
+
|
|
360
|
+
Fresh latest data is the base. Replay a local edit only if its exact
|
|
361
|
+
previous value is still present (or the edit is already durable). A
|
|
362
|
+
positive integer stat increment is commutative, except a completion
|
|
363
|
+
already recorded for the same native turn must not be counted twice.
|
|
364
|
+
A concurrent terminal state wins over *all* cached lifecycle and
|
|
365
|
+
approval projections. Unprovable overlaps remain visibly unresolved.
|
|
366
|
+
"""
|
|
367
|
+
from ccx.store.root import RecordConflict, StorageIntegrityError
|
|
368
|
+
before, desired = self._disk_baseline, self.record
|
|
369
|
+
latest = self.daemon._uuid_store.record(f"codex:{self.worker_id}")
|
|
370
|
+
if (latest.get("workerId") != self.worker_id
|
|
371
|
+
or latest.get("engine") != "codex"
|
|
372
|
+
or latest.get("native") != before.get("native")):
|
|
373
|
+
raise StorageIntegrityError("Codex rebase changed immutable native owner")
|
|
374
|
+
state = (latest.get("lifecycle") or {}).get("state")
|
|
375
|
+
terminal_won = (state in _TERMINAL_STATES
|
|
376
|
+
and (before.get("lifecycle") or {}).get("state")
|
|
377
|
+
not in _TERMINAL_STATES)
|
|
378
|
+
if terminal_won:
|
|
379
|
+
self._externally_terminalized = True
|
|
380
|
+
self.killed.set() # also fences this cached worker's future launches
|
|
381
|
+
|
|
382
|
+
def refuse(message: str) -> None:
|
|
383
|
+
# Do not retain stale state that will conflict on every later event,
|
|
384
|
+
# or an approval that was never durably admitted. Native evidence
|
|
385
|
+
# was journaled independently; the handler exposes the lost patch.
|
|
386
|
+
self.record = deepcopy(latest)
|
|
387
|
+
self._disk_baseline = deepcopy(latest)
|
|
388
|
+
raise RecordConflict(message)
|
|
389
|
+
|
|
390
|
+
old_turn = (before.get("lastTurn") or {}).get("turnId")
|
|
391
|
+
wanted_turn = (desired.get("lastTurn") or {}).get("turnId")
|
|
392
|
+
disk_turn = (latest.get("lastTurn") or {}).get("turnId")
|
|
393
|
+
if terminal_won:
|
|
394
|
+
old_approvals = before.get("pendingApprovals") or []
|
|
395
|
+
new_approvals = desired.get("pendingApprovals") or []
|
|
396
|
+
if any(approval not in old_approvals for approval in new_approvals):
|
|
397
|
+
refuse("external kill won before Codex approval was durable")
|
|
398
|
+
if wanted_turn != old_turn and (not isinstance(wanted_turn, str)
|
|
399
|
+
or (before.get("lifecycle") or {}).get("activeTurnId")
|
|
400
|
+
!= wanted_turn or disk_turn not in (old_turn, wanted_turn)):
|
|
401
|
+
refuse("terminal worker has no positive original turn to reconcile")
|
|
402
|
+
|
|
403
|
+
rebased = deepcopy(latest)
|
|
404
|
+
containers = ("lifecycle", "stats", "createArgs")
|
|
405
|
+
for key in set(before) | set(desired):
|
|
406
|
+
if key in ("recordRevision", "updatedAt") or before.get(key) == desired.get(key):
|
|
407
|
+
continue
|
|
408
|
+
old, new = before.get(key), desired.get(key)
|
|
409
|
+
if key not in desired:
|
|
410
|
+
raise StorageIntegrityError("Codex cannot rebase a deleted field")
|
|
411
|
+
if key in containers:
|
|
412
|
+
if (not isinstance(old, dict) or not isinstance(new, dict)
|
|
413
|
+
or not isinstance(rebased.get(key), dict)
|
|
414
|
+
or not set(old).issubset(new)):
|
|
415
|
+
raise StorageIntegrityError("Codex cannot rebase malformed structured fields")
|
|
416
|
+
changes = ((f"{key}.{member}", old.get(member), value)
|
|
417
|
+
for member, value in new.items()
|
|
418
|
+
if member not in old or old[member] != value)
|
|
419
|
+
else:
|
|
420
|
+
changes = ((key, old, new),)
|
|
421
|
+
for field, previous, value in changes:
|
|
422
|
+
# Terminal status/cleared active turn and approvals are not
|
|
423
|
+
# native completion effects. Never revive them via a stale save.
|
|
424
|
+
if terminal_won and (field.startswith("lifecycle.") or field in {
|
|
425
|
+
"pendingApprovals", "waitingFor", "pendingSend",
|
|
426
|
+
"activeTurnError", "name"} or field.startswith("createArgs.")):
|
|
427
|
+
continue
|
|
428
|
+
parts = field.split(".")
|
|
429
|
+
container = rebased if len(parts) == 1 else rebased[parts[0]]
|
|
430
|
+
current = container.get(parts[-1])
|
|
431
|
+
if (field.startswith("stats.")
|
|
432
|
+
and type(previous) is int and type(value) is int
|
|
433
|
+
and value >= previous):
|
|
434
|
+
if type(current) is not int:
|
|
435
|
+
refuse(f"{field} cannot be merged as a counter")
|
|
436
|
+
delta = value - previous
|
|
437
|
+
if field == "stats.turns" and wanted_turn != old_turn:
|
|
438
|
+
if disk_turn == wanted_turn:
|
|
439
|
+
delta = 0 # this same turn was already committed
|
|
440
|
+
elif disk_turn != old_turn:
|
|
441
|
+
refuse("newer native turn prevents counter reconciliation")
|
|
442
|
+
container[parts[-1]] = current + delta
|
|
443
|
+
elif current == previous:
|
|
444
|
+
container[parts[-1]] = deepcopy(value)
|
|
445
|
+
elif current != value:
|
|
446
|
+
refuse(f"record semantic field changed before CAS: {field}")
|
|
447
|
+
self.record = rebased
|
|
448
|
+
self._disk_baseline = deepcopy(latest)
|
|
449
|
+
|
|
450
|
+
def _journal_lifecycle_locked(
|
|
451
|
+
self,
|
|
452
|
+
*,
|
|
453
|
+
state: str | None = None,
|
|
454
|
+
approvals: list[dict[str, Any]] | None = None,
|
|
455
|
+
) -> None:
|
|
456
|
+
"""Journal state and approval transitions not yet in the journal.
|
|
457
|
+
|
|
458
|
+
Caller holds ``self.lock``. ``state``/``approvals`` override the
|
|
459
|
+
record's, for a teardown that journals its terminal facts before it
|
|
460
|
+
closes the journal. Never raises (``EvidenceJournal.record`` does
|
|
461
|
+
not), and writes nothing once the journal is closed: reopening would
|
|
462
|
+
start a new header for a worker that is already gone.
|
|
463
|
+
"""
|
|
464
|
+
if self.evidence.closed:
|
|
465
|
+
return
|
|
466
|
+
lifecycle = self.record.get("lifecycle") or {}
|
|
467
|
+
current_state = state if state is not None else lifecycle.get("state")
|
|
468
|
+
current = {
|
|
469
|
+
_approval_key(approval): approval
|
|
470
|
+
for approval in (
|
|
471
|
+
approvals if approvals is not None
|
|
472
|
+
else self.record.get("pendingApprovals") or []
|
|
473
|
+
)
|
|
474
|
+
if isinstance(approval, dict)
|
|
475
|
+
}
|
|
476
|
+
wrote = False
|
|
477
|
+
thread_id = (self.record.get("native") or {}).get("threadId")
|
|
478
|
+
for key, approval in self.journaled_approvals.items():
|
|
479
|
+
if key not in current:
|
|
480
|
+
params = {
|
|
481
|
+
"requestId": approval.get("requestId"),
|
|
482
|
+
"gateId": approval.get("gateId"),
|
|
483
|
+
"kind": approval.get("kind"),
|
|
484
|
+
"decision": self.approval_decisions.pop(key, None),
|
|
485
|
+
}
|
|
486
|
+
response_id = self.approval_response_ids.pop(key, None)
|
|
487
|
+
if response_id is not None:
|
|
488
|
+
params["responseId"] = response_id
|
|
489
|
+
self.evidence.record(
|
|
490
|
+
INPUT_RESOLVED_METHOD, params, thread_id=thread_id,
|
|
491
|
+
)
|
|
492
|
+
wrote = True
|
|
493
|
+
for key, approval in current.items():
|
|
494
|
+
if key not in self.journaled_approvals:
|
|
495
|
+
self.evidence.record(
|
|
496
|
+
INPUT_REQUESTED_METHOD, {"request": approval},
|
|
497
|
+
thread_id=thread_id,
|
|
498
|
+
)
|
|
499
|
+
wrote = True
|
|
500
|
+
self.journaled_approvals = current
|
|
501
|
+
if current_state and current_state != self.journaled_state:
|
|
502
|
+
params: dict[str, Any] = {
|
|
503
|
+
"from": self.journaled_state,
|
|
504
|
+
"to": current_state,
|
|
505
|
+
}
|
|
506
|
+
if current_state in {"crashed", "dead", "idle-killed"}:
|
|
507
|
+
params["reason"] = lifecycle.get("lastError")
|
|
508
|
+
self.evidence.record(LIFECYCLE_METHOD, params, thread_id=thread_id)
|
|
509
|
+
self.journaled_state = current_state
|
|
510
|
+
wrote = True
|
|
511
|
+
if wrote:
|
|
512
|
+
self.record["evidence"] = self.evidence.state()
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _abort_if_killed(self) -> None:
|
|
516
|
+
"""Unwind a load that a concurrent ``kill`` has already invalidated.
|
|
517
|
+
|
|
518
|
+
Original resume spends most of its time inside app-server RPCs holding
|
|
519
|
+
no daemon or worker lock — deliberately, so the daemon stays responsive.
|
|
520
|
+
That means a `kill` can complete (record ``dead``, worker removed from
|
|
521
|
+
`workers`/`by_thread`) while this load is still in flight, and every
|
|
522
|
+
step after the RPC would then resurrect it: re-binding the thread,
|
|
523
|
+
rewriting the record and finally reporting ``ok``.
|
|
524
|
+
|
|
525
|
+
Rather than have `kill` block on `load_lock` — which would stall it
|
|
526
|
+
behind a 90s app-server call — the loader checks this event at each of
|
|
527
|
+
those points and unwinds itself. Reading an Event takes no lock, so it
|
|
528
|
+
cannot deadlock against the killer.
|
|
529
|
+
"""
|
|
530
|
+
if self.killed.is_set():
|
|
531
|
+
raise WorkerKilled(f"worker {self.short} was killed during load")
|
|
532
|
+
|
|
533
|
+
def _commit_state(self, state: str, **lifecycle_patch) -> None:
|
|
534
|
+
"""Write a loader state transition atomically against ``kill``.
|
|
535
|
+
|
|
536
|
+
A bare `_abort_if_killed()` before `_state()` is not enough: a kill
|
|
537
|
+
landing in the gap writes ``dead`` and then this write puts the record
|
|
538
|
+
back to ``idle``/``busy``, leaving a killed worker looking alive. The
|
|
539
|
+
check and the write therefore happen inside one hold of ``self.lock``,
|
|
540
|
+
which is the same lock ``kill`` sets its marker under — so the two are
|
|
541
|
+
totally ordered and the loser is the one that does nothing.
|
|
542
|
+
"""
|
|
543
|
+
with self.lock:
|
|
544
|
+
if self.killed.is_set():
|
|
545
|
+
raise WorkerKilled(
|
|
546
|
+
f"worker {self.short} was killed during load")
|
|
547
|
+
lifecycle = self.record["lifecycle"]
|
|
548
|
+
lifecycle.update(lifecycle_patch)
|
|
549
|
+
lifecycle["state"] = state
|
|
550
|
+
lifecycle["since"] = now_ms()
|
|
551
|
+
self.last_activity = time.monotonic()
|
|
552
|
+
self._save()
|
|
553
|
+
|
|
554
|
+
def _interrupt_turn(self, turn_id: str | None) -> bool:
|
|
555
|
+
"""Best-effort native cancellation of one turn. Safe to call twice."""
|
|
556
|
+
thread_id = (self.record.get("native") or {}).get("threadId") or self.record.get("threadId")
|
|
557
|
+
if not turn_id or not thread_id:
|
|
558
|
+
return False
|
|
559
|
+
try:
|
|
560
|
+
self.client.request(
|
|
561
|
+
"turn/interrupt", {"threadId": thread_id, "turnId": turn_id})
|
|
562
|
+
return True
|
|
563
|
+
except Exception:
|
|
564
|
+
return False
|
|
565
|
+
|
|
566
|
+
def check_active_caps(self, rate_limits: dict | None) -> None:
|
|
567
|
+
"""Check a pushed quota update without RPC on the notification reader."""
|
|
568
|
+
from ccx.codex.types import codex_cap_tripped
|
|
569
|
+
with self.lock:
|
|
570
|
+
turn_id = self.cap_turn_id
|
|
571
|
+
if (not turn_id or not self.active_caps
|
|
572
|
+
or self.record["lifecycle"].get("activeTurnId") != turn_id):
|
|
573
|
+
return
|
|
574
|
+
trip = codex_cap_tripped(rate_limits, self.active_caps)
|
|
575
|
+
if not trip:
|
|
576
|
+
return
|
|
577
|
+
name, used, cap = trip
|
|
578
|
+
reason = f"usage cap {name} reached: {used:g}% >= {cap:g}%"
|
|
579
|
+
self.record["quotaStop"] = {"turnId": turn_id, "reason": reason}
|
|
580
|
+
self.active_caps = None # at most one interrupt for this native turn
|
|
581
|
+
self._save()
|
|
582
|
+
threading.Thread(target=self._interrupt_turn, args=(turn_id,),
|
|
583
|
+
daemon=True, name="ccx-codex-cap-interrupt").start()
|
|
584
|
+
|
|
585
|
+
def _abandon_active_compact_locked(self, reason: str) -> bool:
|
|
586
|
+
"""Close a persisted compact that can no longer yield native proof.
|
|
587
|
+
|
|
588
|
+
Caller holds ``self.lock``. See `_abandon_active_compact`.
|
|
589
|
+
"""
|
|
590
|
+
return _abandon_active_compact(self.record, reason)
|
|
591
|
+
|
|
592
|
+
def _state(self, state: str, *, error: str | None = None) -> None:
|
|
593
|
+
with self.lock:
|
|
594
|
+
lifecycle = self.record["lifecycle"]
|
|
595
|
+
lifecycle["state"] = state
|
|
596
|
+
lifecycle["since"] = now_ms()
|
|
597
|
+
if error is not None:
|
|
598
|
+
lifecycle["lastError"] = error
|
|
599
|
+
self.last_activity = time.monotonic()
|
|
600
|
+
self._save()
|
|
601
|
+
|
|
602
|
+
def state(self) -> str:
|
|
603
|
+
with self.lock:
|
|
604
|
+
return self.record["lifecycle"]["state"]
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _native_create_daemon_id(self) -> str:
|
|
608
|
+
return str(getattr(self.daemon, "instance_id", "legacy-daemon"))
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _resume_exact_thread(
|
|
616
|
+
self, thread_id: str, params: dict[str, Any], *,
|
|
617
|
+
archived: bool = False, timeout: float = 90,
|
|
618
|
+
before_submit=None,
|
|
619
|
+
) -> dict[str, Any]:
|
|
620
|
+
"""One original native thread/resume, never a second thread/start.
|
|
621
|
+
|
|
622
|
+
The pre-write release fence and native_threads registration also cover
|
|
623
|
+
a lost reply: a concurrent kill must still unsubscribe this thread.
|
|
624
|
+
"""
|
|
625
|
+
self.daemon._await_thread_release(thread_id)
|
|
626
|
+
self.daemon.claim_thread(thread_id, self.short)
|
|
627
|
+
self.native_threads.add(thread_id)
|
|
628
|
+
if archived:
|
|
629
|
+
self.client.request(
|
|
630
|
+
"thread/unarchive", {"threadId": thread_id}, timeout=timeout,
|
|
631
|
+
)
|
|
632
|
+
resume_params = dict(params)
|
|
633
|
+
resume_params["threadId"] = thread_id
|
|
634
|
+
# Only the thread's id, source and status are read from the reply;
|
|
635
|
+
# rebuilding a long thread's full history can take Codex minutes.
|
|
636
|
+
resume_params["excludeTurns"] = True
|
|
637
|
+
# Kill can win while this load waits for a prior unsubscribe. The
|
|
638
|
+
# already-published killed marker is the existing no-late-load fence.
|
|
639
|
+
self._abort_if_killed()
|
|
640
|
+
if before_submit is not None:
|
|
641
|
+
before_submit() # persist the current native attempt only after release preflight
|
|
642
|
+
try:
|
|
643
|
+
return self.client.request(
|
|
644
|
+
"thread/resume", resume_params, timeout=timeout,
|
|
645
|
+
)
|
|
646
|
+
except ProtocolError as error:
|
|
647
|
+
if "no rollout found" in error.message:
|
|
648
|
+
raise ProtocolError(
|
|
649
|
+
"original Codex thread has no persisted rollout; "
|
|
650
|
+
"refusing native thread/start replacement",
|
|
651
|
+
method="thread/resume",
|
|
652
|
+
) from error
|
|
653
|
+
raise
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _native_turn_start_marker(
|
|
658
|
+
self,
|
|
659
|
+
*,
|
|
660
|
+
kind: str,
|
|
661
|
+
client_message_id: str | None = None,
|
|
662
|
+
delivery_id: str | None = None,
|
|
663
|
+
) -> dict[str, Any]:
|
|
664
|
+
return {
|
|
665
|
+
"schemaVersion": 1,
|
|
666
|
+
"kind": kind,
|
|
667
|
+
"state": "submitted",
|
|
668
|
+
"daemonInstanceId": self._native_create_daemon_id(),
|
|
669
|
+
"clientUserMessageId": client_message_id,
|
|
670
|
+
"deliveryId": delivery_id,
|
|
671
|
+
"submittedAtMs": now_ms(),
|
|
672
|
+
"observedAtMs": None,
|
|
673
|
+
"lastError": None,
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
def _mark_native_turn_start_unknown_locked(self, error: BaseException) -> None:
|
|
677
|
+
marker = self.record.get("nativeTurnStart")
|
|
678
|
+
if not isinstance(marker, dict):
|
|
679
|
+
return
|
|
680
|
+
marker["state"] = "unknown"
|
|
681
|
+
marker["observedAtMs"] = now_ms()
|
|
682
|
+
marker["lastError"] = str(error)
|
|
683
|
+
self._save()
|
|
684
|
+
|
|
685
|
+
def _forget_native_turn_start_locked(self) -> None:
|
|
686
|
+
# Full UUID GenStore CAS does not erase persistent top-level fields;
|
|
687
|
+
# clear the existing launch marker to null under the same CAS instead.
|
|
688
|
+
if self.record.get("fingerprintVersion") == 2:
|
|
689
|
+
if "nativeTurnStart" in self.record:
|
|
690
|
+
self.record["nativeTurnStart"] = None
|
|
691
|
+
else:
|
|
692
|
+
self.record.pop("nativeTurnStart", None)
|
|
693
|
+
|
|
694
|
+
def _clear_native_turn_start_locked(self) -> None:
|
|
695
|
+
if self.record.get("nativeTurnStart") is not None:
|
|
696
|
+
self._forget_native_turn_start_locked()
|
|
697
|
+
self._save()
|
|
698
|
+
|
|
699
|
+
@staticmethod
|
|
700
|
+
def _native_mutation_error_is_ambiguous(error: ProtocolError) -> bool:
|
|
701
|
+
if isinstance(error, ProtocolNotSent):
|
|
702
|
+
return False
|
|
703
|
+
return isinstance(error, ProtocolTransportError) or error.request_id is None
|
|
704
|
+
|
|
705
|
+
def _journal(self, method: str, params: dict[str, Any]) -> bool:
|
|
706
|
+
"""Record one authoritative native event before it is reduced.
|
|
707
|
+
|
|
708
|
+
Returns whether this notification is journaled at all, so the caller
|
|
709
|
+
can persist the journal's health in the record write it is already
|
|
710
|
+
doing. `EvidenceJournal.record` swallows its own failures on purpose:
|
|
711
|
+
a handler that raised here would fail the shared app-server client
|
|
712
|
+
(`AppServerClient._read_loop` treats a handler exception as fatal),
|
|
713
|
+
killing every worker's live turn over a file that could not be
|
|
714
|
+
appended to.
|
|
715
|
+
"""
|
|
716
|
+
if method not in JOURNALED_METHODS:
|
|
717
|
+
return False
|
|
718
|
+
durable = None
|
|
719
|
+
if method == "thread/tokenUsage/updated":
|
|
720
|
+
usage_turn_id = params.get("turnId") or _turn_id(params)
|
|
721
|
+
durable = usage_turn_id in self.completed_turn_ids
|
|
722
|
+
self.evidence.record(
|
|
723
|
+
method,
|
|
724
|
+
params,
|
|
725
|
+
thread_id=self.record.get("threadId"),
|
|
726
|
+
durable=durable,
|
|
727
|
+
)
|
|
728
|
+
return True
|
|
729
|
+
|
|
730
|
+
def _capture_compaction_context_locked(
|
|
731
|
+
self, compact: dict[str, Any]
|
|
732
|
+
) -> None:
|
|
733
|
+
"""Persist one bounded compaction before/after telemetry enrichment."""
|
|
734
|
+
context_after = self.record.get("context")
|
|
735
|
+
earlier = compact.get("contextBefore")
|
|
736
|
+
completed = compact.get("completedAtMs")
|
|
737
|
+
valid = (
|
|
738
|
+
isinstance(context_after, dict)
|
|
739
|
+
and type(context_after.get("as_of_ms")) is int
|
|
740
|
+
and type(completed) is int
|
|
741
|
+
and context_after["as_of_ms"] > completed
|
|
742
|
+
and (not isinstance(earlier, dict)
|
|
743
|
+
or type(earlier.get("as_of_ms")) is not int
|
|
744
|
+
or context_after["as_of_ms"] > earlier["as_of_ms"])
|
|
745
|
+
and (self.record.get("tokenUsage") or {}).get("asOfTurnId") != compact.get("turnId")
|
|
746
|
+
)
|
|
747
|
+
compact["contextAfter"] = dict(context_after) if valid else None
|
|
748
|
+
params = {
|
|
749
|
+
"turnId": compact.get("turnId"),
|
|
750
|
+
"item": {
|
|
751
|
+
"id": compact.get("contextItemId"),
|
|
752
|
+
"type": "contextCompaction",
|
|
753
|
+
},
|
|
754
|
+
"operationId": compact.get("operationId"),
|
|
755
|
+
"compactCount": self.record.get("compactCount"),
|
|
756
|
+
"contextBefore": compact.get("contextBefore"),
|
|
757
|
+
"contextAfter": compact.get("contextAfter"),
|
|
758
|
+
"completedAtMs": compact.get("completedAtMs"),
|
|
759
|
+
}
|
|
760
|
+
self.evidence.record(
|
|
761
|
+
COMPACTION_CONTEXT_METHOD,
|
|
762
|
+
params,
|
|
763
|
+
thread_id=self.record.get("threadId"),
|
|
764
|
+
)
|
|
765
|
+
self.record["evidence"] = self.evidence.state()
|
|
766
|
+
|
|
767
|
+
def _complete_active_compact_locked(
|
|
768
|
+
self, *, turn_id: str | None = None, completed_at_ms: int | None = None
|
|
769
|
+
) -> bool:
|
|
770
|
+
"""Publish one successful requested compact. Caller holds ``self.lock``."""
|
|
771
|
+
active = self.record.get("activeCompact")
|
|
772
|
+
if (not isinstance(active, dict) or not isinstance(turn_id, str)
|
|
773
|
+
or not turn_id or active.get("turnId") != turn_id
|
|
774
|
+
or active.get("nativeTurnStarted") is not True
|
|
775
|
+
or not isinstance(active.get("contextItemId"), str)
|
|
776
|
+
or not active["contextItemId"]
|
|
777
|
+
or active.get("nativeItemCompletedAtMs") is None
|
|
778
|
+
or turn_id not in self.completed_turn_ids):
|
|
779
|
+
return False
|
|
780
|
+
compact_count = int(self.record.get("compactCount") or 0) + 1
|
|
781
|
+
self.record["compactCount"] = compact_count
|
|
782
|
+
self.record["lastCompact"] = {
|
|
783
|
+
"operationId": active.get("operationId"),
|
|
784
|
+
"baselineCompactCount": active.get("baselineCompactCount"),
|
|
785
|
+
"targetCompactCount": active.get("targetCompactCount") or compact_count,
|
|
786
|
+
"startedAtMs": active.get("startedAtMs"),
|
|
787
|
+
"completedAtMs": completed_at_ms or now_ms(),
|
|
788
|
+
"contextBefore": active.get("contextBefore"),
|
|
789
|
+
"contextAfter": None,
|
|
790
|
+
"turnId": turn_id or active.get("turnId"),
|
|
791
|
+
"contextItemId": active.get("contextItemId"),
|
|
792
|
+
"nativeTurnStarted": True,
|
|
793
|
+
"nativeTurnSettled": True,
|
|
794
|
+
"nativeTurnStatus": "completed",
|
|
795
|
+
"status": "succeeded",
|
|
796
|
+
}
|
|
797
|
+
# Older carried tokens are invalid immediately after a native
|
|
798
|
+
# compaction; retain no synthetic newer context in the public record.
|
|
799
|
+
self.record["context"] = None
|
|
800
|
+
self._capture_compaction_context_locked(self.record["lastCompact"])
|
|
801
|
+
self.record["activeCompact"] = None
|
|
802
|
+
return True
|
|
803
|
+
|
|
804
|
+
def _observe_context_compaction_locked(
|
|
805
|
+
self, *, turn_id: str | None, item_id: str | None, completed_at_ms: int | None
|
|
806
|
+
) -> None:
|
|
807
|
+
"""Bind modern ``contextCompaction`` item evidence to compact state."""
|
|
808
|
+
active = self.record.get("activeCompact")
|
|
809
|
+
if isinstance(active, dict):
|
|
810
|
+
# Requested compact item is evidence only for the internal turn
|
|
811
|
+
# positively pinned by turn/started, never an unrelated item.
|
|
812
|
+
if (not isinstance(turn_id, str) or not turn_id
|
|
813
|
+
or active.get("turnId") != turn_id
|
|
814
|
+
or active.get("nativeTurnStarted") is not True
|
|
815
|
+
or not isinstance(item_id, str) or not item_id):
|
|
816
|
+
return
|
|
817
|
+
active["contextItemId"] = item_id
|
|
818
|
+
active["nativeItemCompletedAtMs"] = completed_at_ms or now_ms()
|
|
819
|
+
return
|
|
820
|
+
# Automatic/native compaction outside an explicit CCX compact request is
|
|
821
|
+
# still worth counting for context/history consumers. Do not double-count
|
|
822
|
+
# the deprecated thread/compacted notification for the same turn.
|
|
823
|
+
last = self.record.get("lastCompact")
|
|
824
|
+
if isinstance(last, dict) and turn_id and last.get("turnId") == turn_id:
|
|
825
|
+
return
|
|
826
|
+
compact_count = int(self.record.get("compactCount") or 0) + 1
|
|
827
|
+
self.record["compactCount"] = compact_count
|
|
828
|
+
self.record["lastCompact"] = {
|
|
829
|
+
"operationId": None,
|
|
830
|
+
"baselineCompactCount": compact_count - 1,
|
|
831
|
+
"targetCompactCount": compact_count,
|
|
832
|
+
"startedAtMs": None,
|
|
833
|
+
"completedAtMs": completed_at_ms or now_ms(),
|
|
834
|
+
"contextBefore": None,
|
|
835
|
+
"contextAfter": None,
|
|
836
|
+
"turnId": turn_id,
|
|
837
|
+
"contextItemId": item_id,
|
|
838
|
+
"status": "succeeded",
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
def _emit_compact_feed_locked(
|
|
842
|
+
self, method: str, turn_id: str, compact_id: str, *,
|
|
843
|
+
outcome: str | None = None, operation_id: str | None = None,
|
|
844
|
+
item_id: str | None = None, reason: str | None = None,
|
|
845
|
+
) -> None:
|
|
846
|
+
if method == COMPACT_STARTED_METHOD:
|
|
847
|
+
if compact_id in self._compact_started.get(turn_id, set()):
|
|
848
|
+
return
|
|
849
|
+
elif method == COMPACT_ENDED_METHOD and compact_id in self._compact_ended:
|
|
850
|
+
return
|
|
851
|
+
params = {
|
|
852
|
+
"turnId": turn_id, "compactId": compact_id,
|
|
853
|
+
"operationId": operation_id, "itemId": item_id,
|
|
854
|
+
}
|
|
855
|
+
if outcome is not None:
|
|
856
|
+
params["outcome"] = outcome
|
|
857
|
+
if reason is not None:
|
|
858
|
+
params["reason"] = reason[:500]
|
|
859
|
+
wrote = self.evidence.record(
|
|
860
|
+
method, params, thread_id=self.record.get("threadId"),
|
|
861
|
+
)
|
|
862
|
+
if wrote is None:
|
|
863
|
+
return
|
|
864
|
+
if method == COMPACT_STARTED_METHOD:
|
|
865
|
+
self._compact_started.setdefault(turn_id, set()).add(compact_id)
|
|
866
|
+
else:
|
|
867
|
+
self._compact_ended.add(compact_id)
|
|
868
|
+
self.record["evidence"] = self.evidence.state()
|
|
869
|
+
|
|
870
|
+
def _close_unfinished_compacts_locked(
|
|
871
|
+
self, turn_id: str, *, outcome: str, operation_id: str | None,
|
|
872
|
+
reason: str | None,
|
|
873
|
+
) -> None:
|
|
874
|
+
starts = self._compact_started.get(turn_id, set())
|
|
875
|
+
# A requested compact can have an admitted turn but no item started.
|
|
876
|
+
# Emit its own sparse start before writing the terminal.
|
|
877
|
+
if not starts and isinstance(operation_id, str) and operation_id:
|
|
878
|
+
self._emit_compact_feed_locked(
|
|
879
|
+
COMPACT_STARTED_METHOD, turn_id, f"operation:{operation_id}",
|
|
880
|
+
operation_id=operation_id,
|
|
881
|
+
)
|
|
882
|
+
starts = self._compact_started.get(turn_id, set())
|
|
883
|
+
for compact_id in sorted(starts):
|
|
884
|
+
item_id = None
|
|
885
|
+
prefix = f"{turn_id}:"
|
|
886
|
+
if compact_id.startswith(prefix):
|
|
887
|
+
item_id = compact_id[len(prefix):]
|
|
888
|
+
self._emit_compact_feed_locked(
|
|
889
|
+
COMPACT_ENDED_METHOD, turn_id, compact_id,
|
|
890
|
+
outcome=outcome, operation_id=operation_id,
|
|
891
|
+
item_id=item_id, reason=reason,
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
def _context_snapshot_locked(self) -> dict:
|
|
895
|
+
from ccx.claude.context import current_context, public_context
|
|
896
|
+
return public_context(
|
|
897
|
+
current_context(self.record, engine="codex", live=self.record.get("context")),
|
|
898
|
+
engine="codex", compacting=isinstance(self.record.get("activeCompact"), dict),
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
def _append_context_locked(self, value: dict, *, save: bool = True) -> None:
|
|
902
|
+
turn_id = ((self.record.get("lifecycle") or {}).get("activeTurnId")
|
|
903
|
+
or (self.record.get("lastTurn") or {}).get("turnId"))
|
|
904
|
+
self.evidence.record(
|
|
905
|
+
CONTEXT_CHANGED_METHOD,
|
|
906
|
+
{"context": value, "turnId": turn_id},
|
|
907
|
+
thread_id=(self.record.get("native") or {}).get("threadId"),
|
|
908
|
+
)
|
|
909
|
+
self.record["evidence"] = self.evidence.state()
|
|
910
|
+
if save:
|
|
911
|
+
self._save()
|
|
912
|
+
|
|
913
|
+
def _publish_context_locked(self, *, force: bool = False,
|
|
914
|
+
save: bool = True) -> None:
|
|
915
|
+
view = self._context_snapshot_locked()
|
|
916
|
+
previous = self._context_changes.last
|
|
917
|
+
fenced = (view["reason"] in {"compacting", "just_compacted"}
|
|
918
|
+
and (previous or {}).get("reason") != view["reason"])
|
|
919
|
+
active = (self.record.get("lifecycle") or {}).get("activeTurnId")
|
|
920
|
+
result = self._context_changes.observe(
|
|
921
|
+
view, force=bool(force or fenced or not active),
|
|
922
|
+
)
|
|
923
|
+
if result is not None:
|
|
924
|
+
self._append_context_locked(result, save=save)
|
|
925
|
+
|
|
926
|
+
def flush_context(self) -> None:
|
|
927
|
+
"""Publish the last native usage sample on the existing janitor tick."""
|
|
928
|
+
with self.lock:
|
|
929
|
+
if self.teardown_claimed or self._externally_terminalized:
|
|
930
|
+
return
|
|
931
|
+
# The terminal notification itself flushes context synchronously.
|
|
932
|
+
# Never let the janitor publish an old queued mid-turn sample after
|
|
933
|
+
# turn/compact settlement; a read-only compact-status must stay pure.
|
|
934
|
+
if (not (self.record.get("lifecycle") or {}).get("activeTurnId")
|
|
935
|
+
or isinstance(self.record.get("activeCompact"), dict)):
|
|
936
|
+
self._context_changes.pending = None
|
|
937
|
+
return
|
|
938
|
+
value = self._context_changes.flush()
|
|
939
|
+
if value is not None:
|
|
940
|
+
self._append_context_locked(value)
|
|
941
|
+
|
|
942
|
+
def _publish_background_locked(self, transition: str, item: dict | None) -> None:
|
|
943
|
+
"""Journal one actual native change; the caller holds worker.lock."""
|
|
944
|
+
snapshot = background_summary([
|
|
945
|
+
*self._bg_open.values(), *self._bg_notifications.values(),
|
|
946
|
+
])
|
|
947
|
+
if item is None and not materially_changed(
|
|
948
|
+
self.record.get("backgroundTaskSummary"), snapshot
|
|
949
|
+
):
|
|
950
|
+
return
|
|
951
|
+
self.record["backgroundTaskSummary"] = snapshot
|
|
952
|
+
self.evidence.record(
|
|
953
|
+
BACKGROUND_CHANGED_METHOD,
|
|
954
|
+
{"transition": transition, "item": item, "summary": snapshot},
|
|
955
|
+
thread_id=(self.record.get("native") or {}).get("threadId"),
|
|
956
|
+
)
|
|
957
|
+
|
|
958
|
+
def _lose_native_background_locked(self) -> bool:
|
|
959
|
+
"""Drop old-runtime command guards; never assert a native exit.
|
|
960
|
+
|
|
961
|
+
Called under worker.lock when this app-server goes away or a new
|
|
962
|
+
owner adopts a recorded worker. A lost command is no longer tracked
|
|
963
|
+
(its outcome is unknown, so no ``finished`` item is published); the
|
|
964
|
+
correction keeps the registry-backed pending notifications.
|
|
965
|
+
"""
|
|
966
|
+
previous = self.record.get("backgroundTaskSummary")
|
|
967
|
+
had_native = bool(self._bg_open or self._bg_candidates)
|
|
968
|
+
self._bg_open.clear()
|
|
969
|
+
self._bg_candidates.clear()
|
|
970
|
+
stale_running = (isinstance(previous, dict)
|
|
971
|
+
and type(previous.get("running_count")) is int
|
|
972
|
+
and previous["running_count"] > 0)
|
|
973
|
+
if not (had_native or stale_running):
|
|
974
|
+
return False
|
|
975
|
+
# Only native command tracking is lost. Pending parent notifications
|
|
976
|
+
# come from the durable registry, so they stay in the summary.
|
|
977
|
+
corrected = background_summary(list(self._bg_notifications.values()))
|
|
978
|
+
if not materially_changed(previous, corrected):
|
|
979
|
+
return False
|
|
980
|
+
self.record["backgroundTaskSummary"] = corrected
|
|
981
|
+
self.evidence.record(
|
|
982
|
+
BACKGROUND_CHANGED_METHOD,
|
|
983
|
+
{"transition": "event", "item": None, "summary": corrected},
|
|
984
|
+
thread_id=(self.record.get("native") or {}).get("threadId"),
|
|
985
|
+
)
|
|
986
|
+
self.record["evidence"] = self.evidence.state()
|
|
987
|
+
return True
|
|
988
|
+
|
|
989
|
+
def refresh_parent_notifications(self) -> bool:
|
|
990
|
+
"""Read the current registry once; publish only a changed snapshot.
|
|
991
|
+
|
|
992
|
+
Only codexd owns this worker's evidence journal. Best-effort IPC hints
|
|
993
|
+
do not contain facts, and an absent owner never starts a daemon.
|
|
994
|
+
"""
|
|
995
|
+
from ccx.notify.registry import iter_registrations, _delivery_id
|
|
996
|
+
rows = iter_registrations(include_terminal=True)
|
|
997
|
+
with self.lock:
|
|
998
|
+
if self.teardown_claimed or self._externally_terminalized:
|
|
999
|
+
return False
|
|
1000
|
+
updated = {}
|
|
1001
|
+
outcomes = {}
|
|
1002
|
+
for reg in rows:
|
|
1003
|
+
parent = reg.get("parent") or {}
|
|
1004
|
+
if (parent.get("kind") != "ccx-codex"
|
|
1005
|
+
or parent.get("workerId") != self.worker_id
|
|
1006
|
+
or parent.get("generation") != self.daemon._uuid_store.generation):
|
|
1007
|
+
continue
|
|
1008
|
+
event_id = reg.get("eventId")
|
|
1009
|
+
if not isinstance(event_id, str):
|
|
1010
|
+
continue
|
|
1011
|
+
delivery_id = _delivery_id(event_id)
|
|
1012
|
+
state = reg.get("state")
|
|
1013
|
+
if state in {"armed", "ready"}:
|
|
1014
|
+
updated[delivery_id] = {
|
|
1015
|
+
"id": delivery_id, "kind": "notification",
|
|
1016
|
+
"status": "pending", "label": "child result pending",
|
|
1017
|
+
"notification_id": event_id, "delivery_id": delivery_id,
|
|
1018
|
+
"started_at_ms": reg.get("createdAtMs"),
|
|
1019
|
+
}
|
|
1020
|
+
else:
|
|
1021
|
+
outcomes[delivery_id] = state
|
|
1022
|
+
previous = self._bg_notifications
|
|
1023
|
+
if previous == updated:
|
|
1024
|
+
return False
|
|
1025
|
+
self._bg_notifications = updated
|
|
1026
|
+
additions = [id for id in updated if id not in previous]
|
|
1027
|
+
removals = [id for id in previous if id not in updated]
|
|
1028
|
+
if len(additions) == 1 and not removals:
|
|
1029
|
+
transition, item = "started", updated[additions[0]]
|
|
1030
|
+
elif len(removals) == 1 and not additions:
|
|
1031
|
+
item = dict(previous[removals[0]])
|
|
1032
|
+
result = outcomes.get(removals[0])
|
|
1033
|
+
outcome = {
|
|
1034
|
+
"delivered": ("finished", "completed"),
|
|
1035
|
+
"cancelled": ("stopped", "stopped"),
|
|
1036
|
+
"undeliverable": ("failed", "failed"),
|
|
1037
|
+
}.get(result)
|
|
1038
|
+
if outcome is None:
|
|
1039
|
+
# Missing, unreadable or unknown terminal registry row
|
|
1040
|
+
# proves only a snapshot correction, not delivery.
|
|
1041
|
+
transition, item = "event", None
|
|
1042
|
+
else:
|
|
1043
|
+
transition, item["status"] = outcome
|
|
1044
|
+
else:
|
|
1045
|
+
transition, item = "event", None
|
|
1046
|
+
snapshot = background_summary([
|
|
1047
|
+
*self._bg_open.values(), *updated.values(),
|
|
1048
|
+
])
|
|
1049
|
+
if not materially_changed(self.record.get("backgroundTaskSummary"), snapshot):
|
|
1050
|
+
return False
|
|
1051
|
+
self.record["backgroundTaskSummary"] = snapshot
|
|
1052
|
+
self.evidence.record(
|
|
1053
|
+
BACKGROUND_CHANGED_METHOD,
|
|
1054
|
+
{"transition": transition, "item": item, "summary": snapshot},
|
|
1055
|
+
thread_id=(self.record.get("native") or {}).get("threadId"),
|
|
1056
|
+
)
|
|
1057
|
+
self.record["evidence"] = self.evidence.state()
|
|
1058
|
+
self._save()
|
|
1059
|
+
return True
|
|
1060
|
+
|
|
1061
|
+
def _background_native_locked(
|
|
1062
|
+
self, method: str, params: dict, completed_turn: bool,
|
|
1063
|
+
) -> None:
|
|
1064
|
+
native = self.record.get("native") or {}
|
|
1065
|
+
if params.get("threadId") != native.get("threadId"):
|
|
1066
|
+
return
|
|
1067
|
+
if method == "item/started":
|
|
1068
|
+
item = params.get("item") or {}
|
|
1069
|
+
tid = params.get("turnId")
|
|
1070
|
+
item_id = item.get("id") if isinstance(item, dict) else None
|
|
1071
|
+
if (item.get("type") == "commandExecution"
|
|
1072
|
+
and isinstance(item_id, str) and item_id
|
|
1073
|
+
and isinstance(tid, str) and tid
|
|
1074
|
+
and tid == self.record["lifecycle"].get("activeTurnId")):
|
|
1075
|
+
self._bg_candidates[item_id] = {
|
|
1076
|
+
"turn": tid, "id": "command:" + item_id,
|
|
1077
|
+
"kind": "command", "status": "running",
|
|
1078
|
+
# Deliberately no raw native command, paths or output.
|
|
1079
|
+
"label": "background shell", "tool_use_id": item_id,
|
|
1080
|
+
"started_at_ms": params.get("startedAtMs"),
|
|
1081
|
+
}
|
|
1082
|
+
elif method == "turn/completed" and completed_turn:
|
|
1083
|
+
turn_id = _turn_id(params)
|
|
1084
|
+
for item_id, item in list(self._bg_candidates.items()):
|
|
1085
|
+
if item["turn"] != turn_id:
|
|
1086
|
+
continue
|
|
1087
|
+
del self._bg_candidates[item_id]
|
|
1088
|
+
if item_id not in self._bg_open:
|
|
1089
|
+
self._bg_open[item_id] = item
|
|
1090
|
+
self._publish_background_locked("started", item)
|
|
1091
|
+
elif method == "item/completed":
|
|
1092
|
+
item = params.get("item") or {}
|
|
1093
|
+
item_id = item.get("id") if isinstance(item, dict) else None
|
|
1094
|
+
if not isinstance(item_id, str):
|
|
1095
|
+
return
|
|
1096
|
+
candidate = self._bg_candidates.get(item_id)
|
|
1097
|
+
if candidate is not None and candidate["turn"] == params.get("turnId"):
|
|
1098
|
+
del self._bg_candidates[item_id] # normal foreground command
|
|
1099
|
+
active = self._bg_open.get(item_id)
|
|
1100
|
+
if active is None or active["turn"] != params.get("turnId"):
|
|
1101
|
+
return
|
|
1102
|
+
exit_code = item.get("exitCode")
|
|
1103
|
+
if type(exit_code) is int:
|
|
1104
|
+
status = "completed" if exit_code == 0 else "failed"
|
|
1105
|
+
elif item.get("status") == "failed":
|
|
1106
|
+
status = "failed"
|
|
1107
|
+
else:
|
|
1108
|
+
status = "unknown"
|
|
1109
|
+
terminal = {key: value for key, value in active.items() if key != "turn"}
|
|
1110
|
+
terminal["status"] = status
|
|
1111
|
+
if type(exit_code) is int:
|
|
1112
|
+
terminal["exit_code"] = exit_code
|
|
1113
|
+
del self._bg_open[item_id]
|
|
1114
|
+
self._publish_background_locked(
|
|
1115
|
+
{"completed": "finished", "failed": "failed"}.get(status, "event"),
|
|
1116
|
+
terminal,
|
|
1117
|
+
)
|
|
1118
|
+
|
|
1119
|
+
def on_notification(self, method: str, params: dict[str, Any]) -> None:
|
|
1120
|
+
# Journal the authoritative native event first, while it is still the
|
|
1121
|
+
# thing the app-server sent, rather than after it has been folded into
|
|
1122
|
+
# lifecycle state, `lastTurn`, and `lastReport`. Nothing below reads
|
|
1123
|
+
# the journal, so a journal fault cannot change worker health or turn
|
|
1124
|
+
# outcome — it can only make the evidence explicitly degraded.
|
|
1125
|
+
if self.teardown_claimed:
|
|
1126
|
+
# Teardown owns/closed this worker's evidence journal.
|
|
1127
|
+
return
|
|
1128
|
+
# A separate terminal writer only revoked native *mutations*. Its
|
|
1129
|
+
# still-open journal can record late provider facts without resurrecting
|
|
1130
|
+
# this worker. This is forensic evidence, not outcome authorization.
|
|
1131
|
+
# A valid item/completed can arrive without item/started. Admit only
|
|
1132
|
+
# the live exact turn, then publish its missing sparse start before
|
|
1133
|
+
# journaling the native item. This is source ownership, not a reader
|
|
1134
|
+
# inference or a restatement of native item lines.
|
|
1135
|
+
if method == "item/completed":
|
|
1136
|
+
native_item = params.get("item")
|
|
1137
|
+
turn_id = params.get("turnId")
|
|
1138
|
+
item_id = native_item.get("id") if isinstance(native_item, dict) else None
|
|
1139
|
+
if (isinstance(native_item, dict)
|
|
1140
|
+
and native_item.get("type") == "contextCompaction"
|
|
1141
|
+
and isinstance(turn_id, str) and turn_id
|
|
1142
|
+
and isinstance(item_id, str) and item_id):
|
|
1143
|
+
with self.lock:
|
|
1144
|
+
if self.record["lifecycle"].get("activeTurnId") == turn_id:
|
|
1145
|
+
active = self.record.get("activeCompact")
|
|
1146
|
+
operation_id = (
|
|
1147
|
+
active.get("operationId")
|
|
1148
|
+
if isinstance(active, dict)
|
|
1149
|
+
and active.get("turnId") == turn_id else None
|
|
1150
|
+
)
|
|
1151
|
+
self._emit_compact_feed_locked(
|
|
1152
|
+
COMPACT_STARTED_METHOD, turn_id,
|
|
1153
|
+
f"{turn_id}:{item_id}", item_id=item_id,
|
|
1154
|
+
operation_id=operation_id,
|
|
1155
|
+
)
|
|
1156
|
+
journaled = self._journal(method, params)
|
|
1157
|
+
if method == "item/completed":
|
|
1158
|
+
# The completed user-message item is native's final statement of
|
|
1159
|
+
# that input; by then every copy of it has been journaled.
|
|
1160
|
+
from ccx.codex.delivery_recovery import ccx_client_id
|
|
1161
|
+
client_id = ccx_client_id(params.get("item"), self.worker_id)
|
|
1162
|
+
if client_id is not None:
|
|
1163
|
+
# Journaled above; resolved off this shared reader thread.
|
|
1164
|
+
self.daemon.queue_delivery_recovery(self, client_id)
|
|
1165
|
+
if self._externally_terminalized:
|
|
1166
|
+
return
|
|
1167
|
+
changed = False
|
|
1168
|
+
completed_turn = False
|
|
1169
|
+
with self.lock:
|
|
1170
|
+
if self.teardown_claimed or self._externally_terminalized:
|
|
1171
|
+
return
|
|
1172
|
+
started_id = _turn_id(params) if method == "turn/started" else None
|
|
1173
|
+
if started_id and (
|
|
1174
|
+
started_id in self.completed_turn_ids
|
|
1175
|
+
or started_id == (self.record.get("lastTurn") or {}).get("turnId")):
|
|
1176
|
+
# A re-delivered start of a turn already known completed (also
|
|
1177
|
+
# after a restart, from the durable evidence index) must not
|
|
1178
|
+
# reopen it: its duplicate completion is ignored, so the
|
|
1179
|
+
# worker would stay busy forever. Journaled above only.
|
|
1180
|
+
pass
|
|
1181
|
+
elif method == "turn/started":
|
|
1182
|
+
changed = True
|
|
1183
|
+
self._resumed_system_error_pending = False
|
|
1184
|
+
self.record["historyKnownEmpty"] = False
|
|
1185
|
+
active = self.record.get("activeCompact")
|
|
1186
|
+
if (isinstance(active, dict) and active.get("awaitingTurn")
|
|
1187
|
+
and not active.get("turnId") and started_id
|
|
1188
|
+
and started_id not in self.completed_turn_ids
|
|
1189
|
+
and not self.record.get("nativeTurnStart")
|
|
1190
|
+
and self.launch is None
|
|
1191
|
+
and not self.record["lifecycle"].get("activeTurnId")):
|
|
1192
|
+
active["turnId"] = started_id
|
|
1193
|
+
active["awaitingTurn"] = False
|
|
1194
|
+
active["nativeTurnStarted"] = True
|
|
1195
|
+
self._forget_native_turn_start_locked()
|
|
1196
|
+
self.record["lifecycle"]["state"] = "busy"
|
|
1197
|
+
self.record["lifecycle"]["activeTurnId"] = _turn_id(params)
|
|
1198
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
1199
|
+
elif method == "item/started":
|
|
1200
|
+
item = params.get("item")
|
|
1201
|
+
turn_id = params.get("turnId")
|
|
1202
|
+
item_id = item.get("id") if isinstance(item, dict) else None
|
|
1203
|
+
if (isinstance(item, dict)
|
|
1204
|
+
and item.get("type") == "contextCompaction"
|
|
1205
|
+
and isinstance(turn_id, str) and turn_id
|
|
1206
|
+
and isinstance(item_id, str) and item_id
|
|
1207
|
+
and self.record["lifecycle"].get("activeTurnId") == turn_id):
|
|
1208
|
+
active = self.record.get("activeCompact")
|
|
1209
|
+
operation_id = (
|
|
1210
|
+
active.get("operationId")
|
|
1211
|
+
if isinstance(active, dict)
|
|
1212
|
+
and active.get("turnId") == turn_id else None
|
|
1213
|
+
)
|
|
1214
|
+
# Only this producer writes neutral compaction starts.
|
|
1215
|
+
# Native item/started remains raw forensic evidence.
|
|
1216
|
+
self._emit_compact_feed_locked(
|
|
1217
|
+
COMPACT_STARTED_METHOD, turn_id,
|
|
1218
|
+
f"{turn_id}:{item_id}", operation_id=operation_id,
|
|
1219
|
+
item_id=item_id,
|
|
1220
|
+
)
|
|
1221
|
+
elif method == "item/completed":
|
|
1222
|
+
changed = True
|
|
1223
|
+
item = params.get("item") or {}
|
|
1224
|
+
if item.get("type") == "contextCompaction":
|
|
1225
|
+
item_turn, item_id = params.get("turnId"), item.get("id")
|
|
1226
|
+
if (isinstance(item_turn, str) and item_turn
|
|
1227
|
+
and isinstance(item_id, str) and item_id
|
|
1228
|
+
and self.record["lifecycle"].get("activeTurnId") == item_turn):
|
|
1229
|
+
self._compact_finished.setdefault(item_turn, set()).add(item_id)
|
|
1230
|
+
active = self.record.get("activeCompact")
|
|
1231
|
+
operation_id = (
|
|
1232
|
+
active.get("operationId")
|
|
1233
|
+
if isinstance(active, dict)
|
|
1234
|
+
and active.get("turnId") == item_turn else None
|
|
1235
|
+
)
|
|
1236
|
+
# Journaled after the original native item: exactly
|
|
1237
|
+
# one admitted sparse terminal for this compactId.
|
|
1238
|
+
self._emit_compact_feed_locked(
|
|
1239
|
+
COMPACT_ENDED_METHOD, item_turn,
|
|
1240
|
+
f"{item_turn}:{item_id}", outcome="completed",
|
|
1241
|
+
operation_id=operation_id, item_id=item_id,
|
|
1242
|
+
)
|
|
1243
|
+
self._observe_context_compaction_locked(
|
|
1244
|
+
turn_id=item_turn,
|
|
1245
|
+
item_id=item_id,
|
|
1246
|
+
completed_at_ms=params.get("completedAtMs"),
|
|
1247
|
+
)
|
|
1248
|
+
report = report_from_item(
|
|
1249
|
+
item,
|
|
1250
|
+
turn_id=params.get("turnId"),
|
|
1251
|
+
completed_at=params.get("completedAtMs"),
|
|
1252
|
+
)
|
|
1253
|
+
if report and should_replace_live_report(
|
|
1254
|
+
self.record.get("lastReport"), report
|
|
1255
|
+
):
|
|
1256
|
+
self.record["lastReport"] = report
|
|
1257
|
+
elif method == "thread/tokenUsage/updated":
|
|
1258
|
+
changed = True
|
|
1259
|
+
usage = params.get("tokenUsage") or {}
|
|
1260
|
+
last = usage.get("last") or {}
|
|
1261
|
+
total = usage.get("total") or {}
|
|
1262
|
+
previous_usage = (
|
|
1263
|
+
self.record.get("tokenUsage")
|
|
1264
|
+
if isinstance(self.record.get("tokenUsage"), dict)
|
|
1265
|
+
else {}
|
|
1266
|
+
)
|
|
1267
|
+
previous_total = (
|
|
1268
|
+
previous_usage.get("total")
|
|
1269
|
+
if isinstance(previous_usage.get("total"), dict)
|
|
1270
|
+
else None
|
|
1271
|
+
)
|
|
1272
|
+
reset = (
|
|
1273
|
+
all(
|
|
1274
|
+
int(last.get(key) or 0) == 0
|
|
1275
|
+
for key in (
|
|
1276
|
+
"inputTokens",
|
|
1277
|
+
"cachedInputTokens",
|
|
1278
|
+
"outputTokens",
|
|
1279
|
+
"reasoningOutputTokens",
|
|
1280
|
+
)
|
|
1281
|
+
)
|
|
1282
|
+
and isinstance(previous_total, dict)
|
|
1283
|
+
and dict(total) == dict(previous_total)
|
|
1284
|
+
)
|
|
1285
|
+
tokens = last.get("inputTokens")
|
|
1286
|
+
window = usage.get("modelContextWindow")
|
|
1287
|
+
percent = (
|
|
1288
|
+
tokens / window * 100
|
|
1289
|
+
if (
|
|
1290
|
+
not reset
|
|
1291
|
+
and isinstance(tokens, int)
|
|
1292
|
+
and isinstance(window, int)
|
|
1293
|
+
and window > 0
|
|
1294
|
+
)
|
|
1295
|
+
else None
|
|
1296
|
+
)
|
|
1297
|
+
if reset:
|
|
1298
|
+
self.record["context"] = None
|
|
1299
|
+
else:
|
|
1300
|
+
self.record["context"] = {
|
|
1301
|
+
"state": self.record["lifecycle"]["state"],
|
|
1302
|
+
"tokens_used": tokens,
|
|
1303
|
+
"window_size": window,
|
|
1304
|
+
"percent_used": percent,
|
|
1305
|
+
"as_of_turn_ref": None,
|
|
1306
|
+
"as_of_ms": now_ms(),
|
|
1307
|
+
}
|
|
1308
|
+
self.record["tokenUsage"] = {
|
|
1309
|
+
"last": dict(last),
|
|
1310
|
+
"total": dict(total),
|
|
1311
|
+
"modelContextWindow": window,
|
|
1312
|
+
"asOfTurnId": params.get("turnId"),
|
|
1313
|
+
"asOfMs": now_ms(),
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
last_compact = self.record.get("lastCompact")
|
|
1317
|
+
if (
|
|
1318
|
+
not reset
|
|
1319
|
+
and isinstance(last_compact, dict)
|
|
1320
|
+
and last_compact.get("status") == "succeeded"
|
|
1321
|
+
and last_compact.get("contextAfter") is None
|
|
1322
|
+
and last_compact.get("turnId") != params.get("turnId")
|
|
1323
|
+
and isinstance(self.record.get("context"), dict)
|
|
1324
|
+
):
|
|
1325
|
+
self._capture_compaction_context_locked(last_compact)
|
|
1326
|
+
elif method == "thread/status/changed":
|
|
1327
|
+
changed = True
|
|
1328
|
+
status = params.get("status") or {}
|
|
1329
|
+
kind = status.get("type")
|
|
1330
|
+
flags = status.get("activeFlags") or []
|
|
1331
|
+
if kind in {"active", "idle"}:
|
|
1332
|
+
self._resumed_system_error_pending = False
|
|
1333
|
+
# The status flag can precede the approval request itself;
|
|
1334
|
+
# the request handler sets waiting once there is a gate.
|
|
1335
|
+
if kind == "active" and self.record.get("pendingApprovals") and any(
|
|
1336
|
+
flag in {"waitingOnApproval", "waitingOnUserInput"}
|
|
1337
|
+
for flag in flags
|
|
1338
|
+
):
|
|
1339
|
+
self.record["lifecycle"]["state"] = "waiting"
|
|
1340
|
+
elif kind == "active":
|
|
1341
|
+
self.record["lifecycle"]["state"] = "busy"
|
|
1342
|
+
elif kind == "idle" and not self.record.get("pendingApprovals"):
|
|
1343
|
+
lifecycle = self.record["lifecycle"]
|
|
1344
|
+
if (
|
|
1345
|
+
lifecycle.get("activeTurnId")
|
|
1346
|
+
or self.launch is not None
|
|
1347
|
+
or isinstance(self.record.get("nativeTurnStart"), dict)
|
|
1348
|
+
or isinstance(self.record.get("activeCompact"), dict)
|
|
1349
|
+
):
|
|
1350
|
+
# Codex publishes idle before the authoritative terminal
|
|
1351
|
+
# notification for both model turns and native compact.
|
|
1352
|
+
# Keep the operation unsettled so waiters cannot mistake
|
|
1353
|
+
# this status edge for turn/completion evidence.
|
|
1354
|
+
if lifecycle.get("state") != "crashed":
|
|
1355
|
+
lifecycle["state"] = "busy"
|
|
1356
|
+
else:
|
|
1357
|
+
lifecycle["state"] = "idle"
|
|
1358
|
+
lifecycle["activeTurnId"] = None
|
|
1359
|
+
elif kind == "systemError":
|
|
1360
|
+
if self._resumed_system_error_pending:
|
|
1361
|
+
# The resumed thread still carries the prior failed
|
|
1362
|
+
# turn's systemError. A replayed edge before any new
|
|
1363
|
+
# turn/started or active status cannot fail that turn.
|
|
1364
|
+
changed = False
|
|
1365
|
+
else:
|
|
1366
|
+
active_compact = self.record.get("activeCompact")
|
|
1367
|
+
compact_turn = (
|
|
1368
|
+
active_compact.get("turnId")
|
|
1369
|
+
if isinstance(active_compact, dict) else None
|
|
1370
|
+
)
|
|
1371
|
+
compact_operation = (
|
|
1372
|
+
active_compact.get("operationId")
|
|
1373
|
+
if isinstance(active_compact, dict) else None
|
|
1374
|
+
)
|
|
1375
|
+
compact_turn = compact_turn or self.record["lifecycle"].get("activeTurnId")
|
|
1376
|
+
if isinstance(compact_turn, str) and compact_turn:
|
|
1377
|
+
if compact_operation or self._compact_started.get(compact_turn):
|
|
1378
|
+
self._close_unfinished_compacts_locked(
|
|
1379
|
+
compact_turn, outcome="unknown",
|
|
1380
|
+
operation_id=compact_operation,
|
|
1381
|
+
reason="Codex thread system error",
|
|
1382
|
+
)
|
|
1383
|
+
self._abandon_active_compact_locked("Codex thread system error")
|
|
1384
|
+
# Best effort: name the native cause when it is
|
|
1385
|
+
# already known (Codex may also report it later, on
|
|
1386
|
+
# turn/completed).
|
|
1387
|
+
lifecycle = self.record["lifecycle"]
|
|
1388
|
+
native_error = None
|
|
1389
|
+
active_error = self.record.get("activeTurnError")
|
|
1390
|
+
last_turn = self.record.get("lastTurn")
|
|
1391
|
+
if lifecycle.get("activeTurnId"):
|
|
1392
|
+
if (isinstance(active_error, dict) and active_error.get("turnId")
|
|
1393
|
+
== lifecycle.get("activeTurnId")):
|
|
1394
|
+
native_error = active_error.get("error")
|
|
1395
|
+
elif isinstance(last_turn, dict) and last_turn.get("status") == "failed":
|
|
1396
|
+
native_error = last_turn.get("error")
|
|
1397
|
+
lifecycle["state"] = "crashed"
|
|
1398
|
+
lifecycle["lastError"] = _system_error_reason(native_error)
|
|
1399
|
+
elif method == "turn/completed":
|
|
1400
|
+
changed = True
|
|
1401
|
+
turn = params.get("turn") or {}
|
|
1402
|
+
if not isinstance(turn, dict):
|
|
1403
|
+
turn = {}
|
|
1404
|
+
completed_turn_id = turn.get("id")
|
|
1405
|
+
active_turn_id = self.record["lifecycle"].get("activeTurnId")
|
|
1406
|
+
last_turn_id = (self.record.get("lastTurn") or {}).get("turnId")
|
|
1407
|
+
if (not isinstance(completed_turn_id, str) or not completed_turn_id
|
|
1408
|
+
or completed_turn_id == last_turn_id
|
|
1409
|
+
or completed_turn_id in self.completed_turn_ids
|
|
1410
|
+
or active_turn_id != completed_turn_id):
|
|
1411
|
+
# A completion is a new outcome only for the exact native
|
|
1412
|
+
# turn already admitted as active by turn/started or its
|
|
1413
|
+
# original launch fence. In particular, idle does not prove
|
|
1414
|
+
# an unseen turn: a delayed A after B would roll lastTurn
|
|
1415
|
+
# backwards and count A again. The observed notification
|
|
1416
|
+
# is still journaled, including after a daemon restart.
|
|
1417
|
+
# A duplicate of the current pending save can retry its
|
|
1418
|
+
# original CAS without incrementing again.
|
|
1419
|
+
if journaled:
|
|
1420
|
+
self.record["evidence"] = self.evidence.state()
|
|
1421
|
+
self._save()
|
|
1422
|
+
return
|
|
1423
|
+
self.completed_turn_ids.add(completed_turn_id)
|
|
1424
|
+
if self.cap_turn_id == completed_turn_id:
|
|
1425
|
+
self.cap_turn_id = None
|
|
1426
|
+
self.active_caps = None
|
|
1427
|
+
completed_turn = True
|
|
1428
|
+
previous_state = self.record["lifecycle"].get("state")
|
|
1429
|
+
active_compact = self.record.get("activeCompact")
|
|
1430
|
+
requested_op = (
|
|
1431
|
+
active_compact.get("operationId")
|
|
1432
|
+
if isinstance(active_compact, dict)
|
|
1433
|
+
and active_compact.get("turnId") == completed_turn_id
|
|
1434
|
+
else None
|
|
1435
|
+
)
|
|
1436
|
+
native_compact_ended = self._compact_finished.get(completed_turn_id, set())
|
|
1437
|
+
if turn.get("status") in ("failed", "interrupted", "cancelled"):
|
|
1438
|
+
self._close_unfinished_compacts_locked(
|
|
1439
|
+
completed_turn_id, outcome="failed",
|
|
1440
|
+
operation_id=requested_op,
|
|
1441
|
+
reason=str(turn.get("error") or turn.get("status")),
|
|
1442
|
+
)
|
|
1443
|
+
elif (not native_compact_ended and
|
|
1444
|
+
(requested_op or self._compact_started.get(completed_turn_id))):
|
|
1445
|
+
# Completion of the native turn without the native
|
|
1446
|
+
# compaction item does not prove the compact itself.
|
|
1447
|
+
# Cover CCX-requested and automatic/native compactions.
|
|
1448
|
+
self._close_unfinished_compacts_locked(
|
|
1449
|
+
completed_turn_id, outcome="unknown",
|
|
1450
|
+
operation_id=requested_op,
|
|
1451
|
+
reason="native compact item completion not observed",
|
|
1452
|
+
)
|
|
1453
|
+
elif (completed_turn_id in self._compact_legacy
|
|
1454
|
+
and not self._compact_started.get(completed_turn_id)
|
|
1455
|
+
and not native_compact_ended
|
|
1456
|
+
and requested_op is None):
|
|
1457
|
+
# Legacy-only automatic compaction: defer until the turn's
|
|
1458
|
+
# accepted completion so modern item evidence wins.
|
|
1459
|
+
legacy_id = f"legacy:{completed_turn_id}"
|
|
1460
|
+
self._emit_compact_feed_locked(
|
|
1461
|
+
COMPACT_STARTED_METHOD, completed_turn_id, legacy_id,
|
|
1462
|
+
)
|
|
1463
|
+
self._emit_compact_feed_locked(
|
|
1464
|
+
COMPACT_ENDED_METHOD, completed_turn_id, legacy_id,
|
|
1465
|
+
outcome="completed",
|
|
1466
|
+
)
|
|
1467
|
+
if (
|
|
1468
|
+
isinstance(active_compact, dict)
|
|
1469
|
+
and active_compact.get("turnId") == completed_turn_id
|
|
1470
|
+
and active_compact.get("nativeTurnStarted") is True
|
|
1471
|
+
and isinstance(active_compact.get("contextItemId"), str)
|
|
1472
|
+
and bool(active_compact["contextItemId"])
|
|
1473
|
+
and active_compact.get("nativeItemCompletedAtMs") is not None
|
|
1474
|
+
and turn.get("status") == "completed"
|
|
1475
|
+
):
|
|
1476
|
+
self._complete_active_compact_locked(
|
|
1477
|
+
turn_id=completed_turn_id,
|
|
1478
|
+
completed_at_ms=(
|
|
1479
|
+
int(turn.get("completedAt") * 1000)
|
|
1480
|
+
if isinstance(turn.get("completedAt"), (int, float))
|
|
1481
|
+
else None
|
|
1482
|
+
),
|
|
1483
|
+
)
|
|
1484
|
+
elif (isinstance(active_compact, dict)
|
|
1485
|
+
and active_compact.get("turnId") == completed_turn_id):
|
|
1486
|
+
self._abandon_active_compact_locked(
|
|
1487
|
+
f"compact turn settled without success evidence ({turn.get('status')})"
|
|
1488
|
+
)
|
|
1489
|
+
self.record["lastCompact"].update(
|
|
1490
|
+
turnId=completed_turn_id, nativeTurnSettled=True,
|
|
1491
|
+
nativeTurnStatus=turn.get("status"), nativeTurnError=turn.get("error"),
|
|
1492
|
+
)
|
|
1493
|
+
last_compact = self.record.get("lastCompact")
|
|
1494
|
+
if (
|
|
1495
|
+
isinstance(last_compact, dict)
|
|
1496
|
+
and last_compact.get("turnId") == completed_turn_id
|
|
1497
|
+
and "contextAfter" in last_compact
|
|
1498
|
+
and last_compact.get("contextAfter") is None
|
|
1499
|
+
):
|
|
1500
|
+
self._capture_compaction_context_locked(last_compact)
|
|
1501
|
+
current_turn_id = self.record["lifecycle"].get("activeTurnId")
|
|
1502
|
+
if current_turn_id and current_turn_id != completed_turn_id:
|
|
1503
|
+
# An old completion cannot clear or settle a newer turn.
|
|
1504
|
+
if changed:
|
|
1505
|
+
self._save()
|
|
1506
|
+
return
|
|
1507
|
+
self._forget_native_turn_start_locked()
|
|
1508
|
+
self.record["lifecycle"]["activeTurnId"] = None
|
|
1509
|
+
stats = self.record.setdefault("stats", {})
|
|
1510
|
+
stats["turns"] = stats.get("turns", 0) + 1
|
|
1511
|
+
# Native total is cumulative for the thread. Apply at most
|
|
1512
|
+
# its positive advance (not per-turn last), so the existing
|
|
1513
|
+
# stats counter-CAS path remains safe on a replay/rebase.
|
|
1514
|
+
usage = self.record.get("tokenUsage")
|
|
1515
|
+
total = usage.get("total") if isinstance(usage, dict) else None
|
|
1516
|
+
if isinstance(total, dict):
|
|
1517
|
+
for native_key, field in (("inputTokens", "inputTokens"),
|
|
1518
|
+
("outputTokens", "outputTokens")):
|
|
1519
|
+
value = total.get(native_key)
|
|
1520
|
+
previous = stats.get(field)
|
|
1521
|
+
if (type(value) is int and value >= 0 and
|
|
1522
|
+
(previous is None or value >= previous)):
|
|
1523
|
+
stats[field] = value
|
|
1524
|
+
self.record["lastTurnAt"] = now_ms()
|
|
1525
|
+
self.record["lastTurn"] = _last_turn_from_native(turn)
|
|
1526
|
+
usage = self.record.get("tokenUsage")
|
|
1527
|
+
observed = self.record.get("context")
|
|
1528
|
+
if (isinstance(usage, dict)
|
|
1529
|
+
and usage.get("asOfTurnId") == completed_turn_id
|
|
1530
|
+
and isinstance(observed, dict)):
|
|
1531
|
+
self.record["lastTurn"]["context"] = {
|
|
1532
|
+
**observed,
|
|
1533
|
+
"as_of_turn_ref": "ccx-turn:codex:v1:" + completed_turn_id,
|
|
1534
|
+
}
|
|
1535
|
+
else:
|
|
1536
|
+
self.record["lastTurn"]["context"] = None
|
|
1537
|
+
active_error = self.record.get("activeTurnError")
|
|
1538
|
+
if (
|
|
1539
|
+
isinstance(active_error, dict)
|
|
1540
|
+
and active_error.get("turnId") == completed_turn_id
|
|
1541
|
+
):
|
|
1542
|
+
self.record["activeTurnError"] = None
|
|
1543
|
+
if self.record.get("pendingApprovals"):
|
|
1544
|
+
self.record["lifecycle"]["state"] = "waiting"
|
|
1545
|
+
elif previous_state == "crashed":
|
|
1546
|
+
# A completed turn can arrive after the independent
|
|
1547
|
+
# thread/status systemError. Do not let its settlement
|
|
1548
|
+
# demote a worker whose runtime health is already known to
|
|
1549
|
+
# be bad.
|
|
1550
|
+
self.record["lifecycle"]["state"] = "crashed"
|
|
1551
|
+
# Best effort: name the native cause (a 429, a refused
|
|
1552
|
+
# compact) instead of only the generic status.
|
|
1553
|
+
if self.record["lifecycle"].get("lastError") == _system_error_reason(None):
|
|
1554
|
+
self.record["lifecycle"]["lastError"] = (
|
|
1555
|
+
_system_error_reason(turn.get("error"))
|
|
1556
|
+
)
|
|
1557
|
+
elif isinstance(self.record.get("activeCompact"), dict):
|
|
1558
|
+
# A foreign/native turn can settle while compaction still
|
|
1559
|
+
# owns its own operation fence. Do not publish idle until
|
|
1560
|
+
# thread/compacted (or an explicit compact failure) closes it.
|
|
1561
|
+
self.record["lifecycle"]["state"] = "busy"
|
|
1562
|
+
else:
|
|
1563
|
+
self.record["lifecycle"]["state"] = "idle"
|
|
1564
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
1565
|
+
elif method == "serverRequest/resolved":
|
|
1566
|
+
changed = True
|
|
1567
|
+
request_id = params.get("requestId")
|
|
1568
|
+
self.record["pendingApprovals"] = [
|
|
1569
|
+
approval
|
|
1570
|
+
for approval in self.record.get("pendingApprovals") or []
|
|
1571
|
+
if str(approval.get("requestId")) != str(request_id)
|
|
1572
|
+
]
|
|
1573
|
+
if self.record["lifecycle"]["state"] == "waiting":
|
|
1574
|
+
self.record["lifecycle"]["state"] = (
|
|
1575
|
+
"waiting"
|
|
1576
|
+
if self.record["pendingApprovals"]
|
|
1577
|
+
else (
|
|
1578
|
+
"busy"
|
|
1579
|
+
if (
|
|
1580
|
+
self.record["lifecycle"].get("activeTurnId")
|
|
1581
|
+
or isinstance(self.record.get("nativeTurnStart"), dict)
|
|
1582
|
+
or isinstance(self.record.get("activeCompact"), dict)
|
|
1583
|
+
)
|
|
1584
|
+
else "idle"
|
|
1585
|
+
)
|
|
1586
|
+
)
|
|
1587
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
1588
|
+
elif method == "thread/compacted":
|
|
1589
|
+
changed = True
|
|
1590
|
+
# Legacy Codex signal. Modern releases expose compaction as a
|
|
1591
|
+
# contextCompaction item inside an internal turn. If both arrive,
|
|
1592
|
+
# the turn id dedupes the deprecated notification.
|
|
1593
|
+
legacy_turn_id = params.get("turnId")
|
|
1594
|
+
if isinstance(legacy_turn_id, str) and legacy_turn_id:
|
|
1595
|
+
self._compact_legacy.add(legacy_turn_id)
|
|
1596
|
+
active_compact = self.record.get("activeCompact")
|
|
1597
|
+
last = self.record.get("lastCompact") or {}
|
|
1598
|
+
if legacy_turn_id and last.get("turnId") == legacy_turn_id:
|
|
1599
|
+
return
|
|
1600
|
+
if isinstance(active_compact, dict):
|
|
1601
|
+
# Deprecated thread/compacted is not a modern item and
|
|
1602
|
+
# never settles the requested operation on its own.
|
|
1603
|
+
pass
|
|
1604
|
+
else:
|
|
1605
|
+
last = self.record.get("lastCompact")
|
|
1606
|
+
already_seen = (
|
|
1607
|
+
isinstance(last, dict)
|
|
1608
|
+
and legacy_turn_id
|
|
1609
|
+
and last.get("turnId") == legacy_turn_id
|
|
1610
|
+
)
|
|
1611
|
+
if not already_seen:
|
|
1612
|
+
compact_count = int(self.record.get("compactCount") or 0) + 1
|
|
1613
|
+
self.record["compactCount"] = compact_count
|
|
1614
|
+
self.record["lastCompact"] = {
|
|
1615
|
+
"operationId": None,
|
|
1616
|
+
"baselineCompactCount": compact_count - 1,
|
|
1617
|
+
"targetCompactCount": compact_count,
|
|
1618
|
+
"startedAtMs": None,
|
|
1619
|
+
"completedAtMs": now_ms(),
|
|
1620
|
+
"contextBefore": None,
|
|
1621
|
+
"contextAfter": None,
|
|
1622
|
+
"turnId": legacy_turn_id,
|
|
1623
|
+
"contextItemId": None,
|
|
1624
|
+
"status": "succeeded",
|
|
1625
|
+
}
|
|
1626
|
+
self._capture_compaction_context_locked(
|
|
1627
|
+
self.record["lastCompact"]
|
|
1628
|
+
)
|
|
1629
|
+
# Legacy item evidence is not settlement or an input lease.
|
|
1630
|
+
elif method == "error":
|
|
1631
|
+
changed = True
|
|
1632
|
+
# ``error`` is a turn-scoped diagnostic. It is not evidence of
|
|
1633
|
+
# worker/runtime failure: Codex may recover and retry the same
|
|
1634
|
+
# turn, and the authoritative settled error arrives in
|
|
1635
|
+
# turn/completed.turn.error. Keep the native structured value
|
|
1636
|
+
# intact, but only for the currently active turn.
|
|
1637
|
+
turn_id = params.get("turnId")
|
|
1638
|
+
active_turn_id = self.record["lifecycle"].get("activeTurnId")
|
|
1639
|
+
if (
|
|
1640
|
+
isinstance(turn_id, str)
|
|
1641
|
+
and turn_id
|
|
1642
|
+
and turn_id == active_turn_id
|
|
1643
|
+
):
|
|
1644
|
+
self.record["activeTurnError"] = {
|
|
1645
|
+
"turnId": turn_id,
|
|
1646
|
+
"error": params.get("error"),
|
|
1647
|
+
"willRetry": bool(params.get("willRetry", False)),
|
|
1648
|
+
"observedAtMs": now_ms(),
|
|
1649
|
+
}
|
|
1650
|
+
if method in {"item/started", "item/completed", "turn/completed"}:
|
|
1651
|
+
self._background_native_locked(method, params, completed_turn)
|
|
1652
|
+
if not journaled and not changed:
|
|
1653
|
+
return
|
|
1654
|
+
if journaled:
|
|
1655
|
+
self.record["evidence"] = self.evidence.state()
|
|
1656
|
+
self.last_activity = time.monotonic()
|
|
1657
|
+
# Journal the context view before the same record save that
|
|
1658
|
+
# publishes a new native terminal/compact state. A later ordinary
|
|
1659
|
+
# read-only compact-status call must not race a second record write.
|
|
1660
|
+
self._publish_context_locked(force=completed_turn, save=False)
|
|
1661
|
+
self._save()
|
|
1662
|
+
|
|
1663
|
+
if completed_turn:
|
|
1664
|
+
# The native notification was admitted as this worker's exact
|
|
1665
|
+
# active turn, and its record has been persisted. It is already
|
|
1666
|
+
# the original completion evidence: do not hydrate thread/read.
|
|
1667
|
+
# A turn that finished before the caller persisted acceptance is
|
|
1668
|
+
# caught by the receipt writer's journal check or reconciliation.
|
|
1669
|
+
if turn.get("status") == "completed":
|
|
1670
|
+
from ccx.codex.completion import settle_completed
|
|
1671
|
+
try:
|
|
1672
|
+
settle_completed(
|
|
1673
|
+
self.daemon._uuid_store, f"codex:{self.worker_id}",
|
|
1674
|
+
thread_id=(self.record.get("native") or {}).get("threadId"),
|
|
1675
|
+
turn=turn,
|
|
1676
|
+
live_report=self.record.get("lastReport"),
|
|
1677
|
+
)
|
|
1678
|
+
except Exception as error:
|
|
1679
|
+
# A receipt-side fault cannot take down the shared native
|
|
1680
|
+
# notification reader or change the completed turn. The
|
|
1681
|
+
# original journal and recovery path remain available.
|
|
1682
|
+
print("ccx codexd: original completion settlement pending: "
|
|
1683
|
+
+ type(error).__name__, file=sys.stderr, flush=True)
|
|
1684
|
+
from ccx.notify.daemon import revive_notify_daemon_after_turn
|
|
1685
|
+
revive_notify_daemon_after_turn("codex", self.worker_id)
|
|
1686
|
+
|
|
1687
|
+
def on_server_request(
|
|
1688
|
+
self, request_id: Any, method: str, params: dict[str, Any]
|
|
1689
|
+
) -> None:
|
|
1690
|
+
kind = APPROVAL_METHODS.get(method)
|
|
1691
|
+
if not kind or (
|
|
1692
|
+
method == "item/fileChange/requestApproval"
|
|
1693
|
+
and params.get("grantRoot")
|
|
1694
|
+
):
|
|
1695
|
+
self.client.reject_request(
|
|
1696
|
+
request_id,
|
|
1697
|
+
f"CCX does not broker app-server request {method}",
|
|
1698
|
+
)
|
|
1699
|
+
return
|
|
1700
|
+
preview = {
|
|
1701
|
+
"requestId": request_id,
|
|
1702
|
+
# App-server request ids restart with the app-server, so a stale
|
|
1703
|
+
# id could name a newer request; this one never repeats.
|
|
1704
|
+
"gateId": uuid.uuid4().hex,
|
|
1705
|
+
# Native request IDs are connection-scoped. An abruptly killed
|
|
1706
|
+
# daemon can leave old approvals on disk; they cannot be replayed.
|
|
1707
|
+
"runtimeId": self.daemon.instance_id,
|
|
1708
|
+
"kind": kind,
|
|
1709
|
+
"turnId": params.get("turnId"),
|
|
1710
|
+
"itemId": params.get("itemId"),
|
|
1711
|
+
"command": _bounded(params.get("command")),
|
|
1712
|
+
"cwd": _bounded(params.get("cwd"), 500),
|
|
1713
|
+
"reason": _bounded(params.get("reason")),
|
|
1714
|
+
"createdAt": now_ms(),
|
|
1715
|
+
}
|
|
1716
|
+
with self.lock:
|
|
1717
|
+
torn_down = (self.teardown_claimed or self._externally_terminalized
|
|
1718
|
+
or self.record["lifecycle"].get("state") in _TERMINAL_STATES)
|
|
1719
|
+
if not torn_down:
|
|
1720
|
+
self.record["pendingApprovals"].append(preview)
|
|
1721
|
+
self.record["lifecycle"]["state"] = "waiting"
|
|
1722
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
1723
|
+
self._save()
|
|
1724
|
+
if torn_down:
|
|
1725
|
+
self.client.reject_request(
|
|
1726
|
+
request_id, f"CCX worker {self.short} is being unloaded"
|
|
1727
|
+
)
|
|
1728
|
+
|
|
1729
|
+
def _select_approval(self, gate_id: str) -> dict[str, Any] | None:
|
|
1730
|
+
"""Call under self.lock; only this runtime's exact CCX gateId may act."""
|
|
1731
|
+
if not isinstance(gate_id, str) or not gate_id or len(gate_id) > 128:
|
|
1732
|
+
return None
|
|
1733
|
+
for approval in self.record.get("pendingApprovals") or []:
|
|
1734
|
+
if (approval.get("gateId") == gate_id
|
|
1735
|
+
and approval.get("runtimeId") == self.daemon.instance_id):
|
|
1736
|
+
return approval
|
|
1737
|
+
return None
|
|
1738
|
+
|
|
1739
|
+
def kill(self, observe_seconds: float | None = None) -> dict[str, Any]: # noqa: D401
|
|
1740
|
+
"""Terminate this worker and unload it from the shared daemon.
|
|
1741
|
+
|
|
1742
|
+
Codex has no per-worker process to signal: every loaded worker is a
|
|
1743
|
+
thread multiplexed on the one shared ``codex app-server`` child, and the
|
|
1744
|
+
daemon PID and app-server PID stamped into each record are device-wide,
|
|
1745
|
+
not worker-owned. Killing a worker therefore means unloading exactly this
|
|
1746
|
+
worker — never signalling either shared process, and never disturbing a
|
|
1747
|
+
sibling.
|
|
1748
|
+
|
|
1749
|
+
The active turn is cancelled best-effort first; a refusal or a transport
|
|
1750
|
+
failure there does not stop the unload, because the unload is what the
|
|
1751
|
+
caller asked for. The native ``threadId`` is preserved and the thread is
|
|
1752
|
+
deliberately not archived, so a later ``send``/``resume`` reloads the
|
|
1753
|
+
same conversation immediately.
|
|
1754
|
+
"""
|
|
1755
|
+
# one critical section decides the race. `self.lock` totally orders this
|
|
1756
|
+
# against the loader's two commit points (the native turn launch
|
|
1757
|
+
# publication and `_commit_state`), so exactly one of us wins:
|
|
1758
|
+
# * we go first -> the loader sees `killed` and does nothing at all;
|
|
1759
|
+
# * loader first -> we see its `launch` here and own the cancellation.
|
|
1760
|
+
# It deliberately does not take `load_lock`: waiting there would stall
|
|
1761
|
+
# kill behind a 90s `thread/resume`.
|
|
1762
|
+
with self.lock:
|
|
1763
|
+
launch = self.launch
|
|
1764
|
+
unresolved_start = self.record.get("nativeTurnStart")
|
|
1765
|
+
if launch is None and isinstance(unresolved_start, dict):
|
|
1766
|
+
# The local JSON-RPC request has already settled without exact
|
|
1767
|
+
# native evidence, so the in-memory launch fence can no longer
|
|
1768
|
+
# hand us a turn id. The native request may still land later;
|
|
1769
|
+
# unloading now would falsely confirm a kill.
|
|
1770
|
+
self.kill_intent.clear()
|
|
1771
|
+
return {
|
|
1772
|
+
"ok": False,
|
|
1773
|
+
"pending": True,
|
|
1774
|
+
"error": (
|
|
1775
|
+
f"worker {self.short} has an indeterminate native "
|
|
1776
|
+
"turn/start; kill was NOT completed and is not confirmed "
|
|
1777
|
+
"— retry after turn evidence arrives or the Codex "
|
|
1778
|
+
"app-server is replaced"
|
|
1779
|
+
),
|
|
1780
|
+
"state": self.record["lifecycle"]["state"],
|
|
1781
|
+
}
|
|
1782
|
+
self.kill_intent.set()
|
|
1783
|
+
self.killed.set()
|
|
1784
|
+
state = self.record["lifecycle"]["state"]
|
|
1785
|
+
turn_id = self.record["lifecycle"].get("activeTurnId")
|
|
1786
|
+
thread_id = (self.record.get("native") or {}).get("threadId") or self.record.get("threadId")
|
|
1787
|
+
active_compact = self.record.get("activeCompact")
|
|
1788
|
+
compact_turn = (
|
|
1789
|
+
active_compact.get("turnId")
|
|
1790
|
+
if isinstance(active_compact, dict) else None
|
|
1791
|
+
)
|
|
1792
|
+
compact_operation = (
|
|
1793
|
+
active_compact.get("operationId")
|
|
1794
|
+
if isinstance(active_compact, dict) else None
|
|
1795
|
+
)
|
|
1796
|
+
compact_turn = compact_turn or turn_id
|
|
1797
|
+
if isinstance(compact_turn, str) and compact_turn:
|
|
1798
|
+
if compact_operation or self._compact_started.get(compact_turn):
|
|
1799
|
+
self._close_unfinished_compacts_locked(
|
|
1800
|
+
compact_turn, outcome="unknown",
|
|
1801
|
+
operation_id=compact_operation,
|
|
1802
|
+
reason="worker killed during compaction",
|
|
1803
|
+
)
|
|
1804
|
+
self._abandon_active_compact_locked("worker killed during compaction")
|
|
1805
|
+
|
|
1806
|
+
interrupted = False
|
|
1807
|
+
launch_cancelled = None
|
|
1808
|
+
if launch is not None:
|
|
1809
|
+
# A `turn/start` was already irrevocably in flight when we won. It
|
|
1810
|
+
# is not cancellable until it returns a turn id, so wait for that —
|
|
1811
|
+
# bounded by the caller's own remaining budget, capped.
|
|
1812
|
+
window = (_LAUNCH_SETTLE_CAP_S if observe_seconds is None
|
|
1813
|
+
else max(0.0, min(float(observe_seconds),
|
|
1814
|
+
_LAUNCH_SETTLE_CAP_S)))
|
|
1815
|
+
if not launch.done.wait(window):
|
|
1816
|
+
# unresolved. The turn may be accepted the moment after we
|
|
1817
|
+
# answer, so reporting a confirmed kill here would be exactly
|
|
1818
|
+
# the lie this operation must not tell. Stay loaded and
|
|
1819
|
+
# nonterminal so the caller's reconciliation sees a target that
|
|
1820
|
+
# is still pending; the `killed` marker guarantees the loader
|
|
1821
|
+
# cancels as soon as the turn id becomes knowable and then
|
|
1822
|
+
# terminalizes and unloads. A later kill is idempotent.
|
|
1823
|
+
return {
|
|
1824
|
+
"ok": False,
|
|
1825
|
+
"pending": True,
|
|
1826
|
+
"error": (f"worker {self.short} has a turn launch still in "
|
|
1827
|
+
f"flight after {window:g}s; it will be cancelled "
|
|
1828
|
+
f"and unloaded when the request returns — retry "
|
|
1829
|
+
f"kill to confirm"),
|
|
1830
|
+
"state": self.state(),
|
|
1831
|
+
}
|
|
1832
|
+
if not isinstance(launch.turn_id, str) or not launch.turn_id:
|
|
1833
|
+
# A native start returned without an exact turn ID. The
|
|
1834
|
+
# original mutation may still land; no unload or success.
|
|
1835
|
+
return {"ok": False, "pending": True,
|
|
1836
|
+
"error": "original native turn/start is unresolved; kill not confirmed",
|
|
1837
|
+
"state": self.state()}
|
|
1838
|
+
launch_cancelled = self._interrupt_turn(launch.turn_id)
|
|
1839
|
+
interrupted = launch_cancelled
|
|
1840
|
+
elif state in {"busy", "waiting"} and turn_id and thread_id:
|
|
1841
|
+
interrupted = self._interrupt_turn(turn_id)
|
|
1842
|
+
self._state("dead")
|
|
1843
|
+
# Clears activeTurnId/pendingApprovals, removes this worker alone from
|
|
1844
|
+
# `workers` and `by_thread`, and releases its native thread without
|
|
1845
|
+
# waiting for Codex to answer.
|
|
1846
|
+
self.daemon._unload(self, terminal_state="dead")
|
|
1847
|
+
return {
|
|
1848
|
+
"ok": True,
|
|
1849
|
+
"killed": True,
|
|
1850
|
+
"unloaded": True,
|
|
1851
|
+
"interrupted": interrupted,
|
|
1852
|
+
# Only present when this kill raced a turn launch, so a caller can
|
|
1853
|
+
# tell "nothing was running" from "we cancelled what was starting".
|
|
1854
|
+
"launchCancelled": launch_cancelled,
|
|
1855
|
+
"state": "dead",
|
|
1856
|
+
"threadId": thread_id,
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
def _original_notification_thread(self) -> str:
|
|
1860
|
+
"""The one globally claimed parent thread for both read and wake."""
|
|
1861
|
+
native = self.record.get("native") or {}
|
|
1862
|
+
thread_id = native.get("threadId")
|
|
1863
|
+
if (not isinstance(thread_id, str) or not thread_id
|
|
1864
|
+
or native.get("threadSource") != self.daemon._uuid_store.codex_source(
|
|
1865
|
+
f"codex:{self.worker_id}"
|
|
1866
|
+
)):
|
|
1867
|
+
raise RuntimeError("original parent Codex native thread/source mismatch")
|
|
1868
|
+
self.daemon._uuid_store.authorize_native_action(
|
|
1869
|
+
f"codex:{self.worker_id}", "send",
|
|
1870
|
+
)
|
|
1871
|
+
return thread_id
|
|
1872
|
+
|
|
1873
|
+
def _parent_notification_match(
|
|
1874
|
+
self, delivery_id: str, *, legacy_event_id: str | None = None,
|
|
1875
|
+
name: str = "worker_notification",
|
|
1876
|
+
) -> dict[str, Any] | None:
|
|
1877
|
+
with self.lock:
|
|
1878
|
+
fences = self.record.get("parentNotifyFences")
|
|
1879
|
+
fence = fences.get(delivery_id) if isinstance(fences, dict) else None
|
|
1880
|
+
return notification_delivery_match(
|
|
1881
|
+
self.client.request,
|
|
1882
|
+
self._original_notification_thread(),
|
|
1883
|
+
delivery_id,
|
|
1884
|
+
legacy_event_id=legacy_event_id,
|
|
1885
|
+
stop_at_turn_id=fence if isinstance(fence, str) else None,
|
|
1886
|
+
name=name,
|
|
1887
|
+
)
|
|
1888
|
+
|
|
1889
|
+
def find_parent_notification(
|
|
1890
|
+
self, delivery_id: str, payload: dict[str, Any], *,
|
|
1891
|
+
name: str = "worker_notification",
|
|
1892
|
+
) -> dict[str, Any] | None:
|
|
1893
|
+
"""Read-only: has this notification already landed in the thread?
|
|
1894
|
+
|
|
1895
|
+
Runs without `operation_lock` (a long parent history must not stall
|
|
1896
|
+
the parent's own sends and kills); `_start_parent_notification` then
|
|
1897
|
+
covers an attempt this daemon launched meanwhile.
|
|
1898
|
+
"""
|
|
1899
|
+
legacy_event_id = (
|
|
1900
|
+
payload.get("eventId") if payload.get("deliveryId") is None else None
|
|
1901
|
+
)
|
|
1902
|
+
return self._parent_notification_match(
|
|
1903
|
+
delivery_id,
|
|
1904
|
+
legacy_event_id=str(legacy_event_id) if legacy_event_id else None,
|
|
1905
|
+
name=name,
|
|
1906
|
+
)
|
|
1907
|
+
|
|
1908
|
+
def _fence_parent_notification_locked(self, delivery_id: str) -> None:
|
|
1909
|
+
"""Remember the last turn settled before a delivery's first launch.
|
|
1910
|
+
|
|
1911
|
+
The notification can only land in a later turn, so a retry's history
|
|
1912
|
+
walk stops there. Only the first launch sets it: a retry's baseline may
|
|
1913
|
+
already be newer than the turn an earlier ambiguous attempt joined.
|
|
1914
|
+
"""
|
|
1915
|
+
fences = self.record.get("parentNotifyFences")
|
|
1916
|
+
if not isinstance(fences, dict):
|
|
1917
|
+
fences = self.record["parentNotifyFences"] = {}
|
|
1918
|
+
if delivery_id in fences:
|
|
1919
|
+
return
|
|
1920
|
+
last_turn = self.record.get("lastTurn")
|
|
1921
|
+
fences[delivery_id] = (
|
|
1922
|
+
last_turn.get("turnId") if isinstance(last_turn, dict) else None
|
|
1923
|
+
)
|
|
1924
|
+
for stale in list(fences)[:-_PARENT_NOTIFY_FENCES]:
|
|
1925
|
+
del fences[stale]
|
|
1926
|
+
|
|
1927
|
+
def _compact_refusal(self) -> dict[str, Any] | None:
|
|
1928
|
+
with self.lock:
|
|
1929
|
+
active = self.record.get("activeCompact")
|
|
1930
|
+
if not isinstance(active, dict):
|
|
1931
|
+
return None
|
|
1932
|
+
return {"ok": False, "state": self.record["lifecycle"]["state"],
|
|
1933
|
+
"error": "Compaction owns the input surface",
|
|
1934
|
+
"reason": "compaction-busy", "retryable": True,
|
|
1935
|
+
"definitive": True, "operationId": active.get("operationId")}
|
|
1936
|
+
|
|
1937
|
+
def _start_parent_notification(
|
|
1938
|
+
self,
|
|
1939
|
+
delivery_id: str,
|
|
1940
|
+
payload: dict[str, Any],
|
|
1941
|
+
*,
|
|
1942
|
+
reconcile: bool = True,
|
|
1943
|
+
history: tuple[dict[str, Any] | None] | None = None,
|
|
1944
|
+
namespace: str = "ccx",
|
|
1945
|
+
name: str = "worker_notification",
|
|
1946
|
+
) -> dict[str, Any]:
|
|
1947
|
+
"""Start-or-steer one machine continuation with kill-safe admission.
|
|
1948
|
+
|
|
1949
|
+
Unlike a normal send this intentionally accepts an idle or active parent
|
|
1950
|
+
turn. Codex's turn/start semantics start a new turn when idle and steer
|
|
1951
|
+
the tool output into the active turn when busy. The launch marker keeps
|
|
1952
|
+
kill-vs-turn/start ordering as strong as ordinary user turns.
|
|
1953
|
+
"""
|
|
1954
|
+
if history is not None:
|
|
1955
|
+
# Walked by the caller before it took the operation lock.
|
|
1956
|
+
match = history[0]
|
|
1957
|
+
elif reconcile:
|
|
1958
|
+
match = self.find_parent_notification(
|
|
1959
|
+
delivery_id, payload, name=name,
|
|
1960
|
+
)
|
|
1961
|
+
else:
|
|
1962
|
+
match = None
|
|
1963
|
+
if match is None and delivery_id in self.launched_notifications:
|
|
1964
|
+
# An attempt that held the lock while the walk ran outside it.
|
|
1965
|
+
match = {"turnId": self.launched_notifications[delivery_id]}
|
|
1966
|
+
if match is not None:
|
|
1967
|
+
with self.lock:
|
|
1968
|
+
marker = self.record.get("nativeTurnStart")
|
|
1969
|
+
if (
|
|
1970
|
+
isinstance(marker, dict)
|
|
1971
|
+
and marker.get("kind") == (
|
|
1972
|
+
"parent-notification" if namespace == "ccx" else "machine-wake"
|
|
1973
|
+
)
|
|
1974
|
+
and marker.get("deliveryId") == delivery_id
|
|
1975
|
+
):
|
|
1976
|
+
self._forget_native_turn_start_locked()
|
|
1977
|
+
self._save()
|
|
1978
|
+
return {
|
|
1979
|
+
"ok": True,
|
|
1980
|
+
"state": self.state(),
|
|
1981
|
+
"reconciled": True,
|
|
1982
|
+
"workerId": self.worker_id,
|
|
1983
|
+
"threadId": self._original_notification_thread(),
|
|
1984
|
+
"eventId": payload.get("eventId"),
|
|
1985
|
+
"deliveryId": delivery_id,
|
|
1986
|
+
"turnId": match.get("turnId"),
|
|
1987
|
+
}
|
|
1988
|
+
# A previous attempt can still land after the network reply is lost.
|
|
1989
|
+
# Reconcile-only retries may ACK positive history, never retransmit
|
|
1990
|
+
# merely because the history endpoint finds no matching item.
|
|
1991
|
+
if reconcile:
|
|
1992
|
+
return {"ok": False, "pending": True, "state": self.state(),
|
|
1993
|
+
"error": "original machine wake was attempted; native placement remains unknown"}
|
|
1994
|
+
refusal = self._compact_refusal()
|
|
1995
|
+
if refusal:
|
|
1996
|
+
return refusal
|
|
1997
|
+
params = {
|
|
1998
|
+
"threadId": self._original_notification_thread(),
|
|
1999
|
+
"input": [],
|
|
2000
|
+
"turnTrigger": "ccx-worker-notification",
|
|
2001
|
+
"toolOutput": {
|
|
2002
|
+
"namespace": namespace,
|
|
2003
|
+
"name": name,
|
|
2004
|
+
"output": json.dumps(payload, separators=(",", ":")),
|
|
2005
|
+
},
|
|
2006
|
+
}
|
|
2007
|
+
with self.lock:
|
|
2008
|
+
if self.killed.is_set() or self.kill_intent.is_set():
|
|
2009
|
+
return {
|
|
2010
|
+
"ok": False,
|
|
2011
|
+
**({"definitive": True} if namespace != "ccx" else {}),
|
|
2012
|
+
"error": f"worker {self.short} is being killed",
|
|
2013
|
+
"state": self.record["lifecycle"]["state"],
|
|
2014
|
+
}
|
|
2015
|
+
if (not self.loaded or self.record["lifecycle"]["state"]
|
|
2016
|
+
not in {"idle", "busy", "waiting"}):
|
|
2017
|
+
return {"ok": False, "definitive": True,
|
|
2018
|
+
"error": "original native parent not subscribed/active",
|
|
2019
|
+
"state": self.record["lifecycle"]["state"]}
|
|
2020
|
+
if self.launch is not None:
|
|
2021
|
+
return {"ok": False, "definitive": True,
|
|
2022
|
+
"error": "another original native turn/start is in flight"}
|
|
2023
|
+
if isinstance(self.record.get("nativeTurnStart"), dict):
|
|
2024
|
+
return {"ok": False, "pending": True,
|
|
2025
|
+
"error": "another original native turn/start remains unresolved"}
|
|
2026
|
+
previous_state = self.record["lifecycle"]["state"]
|
|
2027
|
+
previous_turn_id = self.record["lifecycle"].get("activeTurnId")
|
|
2028
|
+
launch = _TurnLaunch()
|
|
2029
|
+
self.launch = launch
|
|
2030
|
+
# A turn blocked on an approval stays `waiting`: the notification
|
|
2031
|
+
# is steered into that same turn, and publishing `busy` would hide
|
|
2032
|
+
# the pending approval from every waiter.
|
|
2033
|
+
if previous_state != "waiting":
|
|
2034
|
+
self.record["lifecycle"]["state"] = "busy"
|
|
2035
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
2036
|
+
self.record["nativeTurnStart"] = self._native_turn_start_marker(
|
|
2037
|
+
kind="parent-notification" if namespace == "ccx" else "machine-wake",
|
|
2038
|
+
delivery_id=delivery_id,
|
|
2039
|
+
)
|
|
2040
|
+
self._fence_parent_notification_locked(delivery_id)
|
|
2041
|
+
self.last_activity = time.monotonic()
|
|
2042
|
+
self._save()
|
|
2043
|
+
turn_id = None
|
|
2044
|
+
request_returned = False
|
|
2045
|
+
try:
|
|
2046
|
+
result = self.client.request("turn/start", params, timeout=60)
|
|
2047
|
+
turn_id = _turn_id(result)
|
|
2048
|
+
request_returned = True
|
|
2049
|
+
except ProtocolError as error:
|
|
2050
|
+
with self.lock:
|
|
2051
|
+
if self._native_mutation_error_is_ambiguous(error):
|
|
2052
|
+
self._mark_native_turn_start_unknown_locked(error)
|
|
2053
|
+
else:
|
|
2054
|
+
self._clear_native_turn_start_locked()
|
|
2055
|
+
# Only an explicit native refusal proves the notification
|
|
2056
|
+
# did not mutate the thread, so only then restore what we
|
|
2057
|
+
# owned before the request.
|
|
2058
|
+
if (
|
|
2059
|
+
self.record["lifecycle"].get("activeTurnId")
|
|
2060
|
+
== previous_turn_id
|
|
2061
|
+
):
|
|
2062
|
+
self.record["lifecycle"]["state"] = previous_state
|
|
2063
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
2064
|
+
self._save()
|
|
2065
|
+
raise
|
|
2066
|
+
except Exception as error:
|
|
2067
|
+
with self.lock:
|
|
2068
|
+
self._mark_native_turn_start_unknown_locked(error)
|
|
2069
|
+
raise
|
|
2070
|
+
finally:
|
|
2071
|
+
with self.lock:
|
|
2072
|
+
if self.launch is launch:
|
|
2073
|
+
self.launch = None
|
|
2074
|
+
launch.turn_id = turn_id
|
|
2075
|
+
if turn_id is not None:
|
|
2076
|
+
self._forget_native_turn_start_locked()
|
|
2077
|
+
if request_returned and isinstance(turn_id, str) and turn_id:
|
|
2078
|
+
self.launched_notifications[delivery_id] = turn_id
|
|
2079
|
+
lifecycle = self.record["lifecycle"]
|
|
2080
|
+
if request_returned and lifecycle["state"] in {"busy", "waiting"}:
|
|
2081
|
+
lifecycle["activeTurnId"] = (
|
|
2082
|
+
lifecycle.get("activeTurnId") or turn_id
|
|
2083
|
+
)
|
|
2084
|
+
if self.record.get("pendingApprovals"):
|
|
2085
|
+
# An approval that arrived while the request was in
|
|
2086
|
+
# flight still gates the turn.
|
|
2087
|
+
lifecycle["state"] = "waiting"
|
|
2088
|
+
self._save()
|
|
2089
|
+
launch.done.set()
|
|
2090
|
+
if self.killed.is_set():
|
|
2091
|
+
self._interrupt_turn(turn_id)
|
|
2092
|
+
return {
|
|
2093
|
+
"ok": False,
|
|
2094
|
+
"error": f"worker {self.short} was killed while starting notification",
|
|
2095
|
+
"state": self.state(),
|
|
2096
|
+
}
|
|
2097
|
+
if not isinstance(turn_id, str) or not turn_id:
|
|
2098
|
+
return {"ok": False, "pending": True, "state": self.state(),
|
|
2099
|
+
"error": "original machine wake has no positive native turn ID"}
|
|
2100
|
+
return {
|
|
2101
|
+
"ok": True,
|
|
2102
|
+
"state": self.state(),
|
|
2103
|
+
"turnId": turn_id,
|
|
2104
|
+
"reconciled": False,
|
|
2105
|
+
"workerId": self.worker_id,
|
|
2106
|
+
"threadId": self._original_notification_thread(),
|
|
2107
|
+
"eventId": payload.get("eventId"),
|
|
2108
|
+
"deliveryId": delivery_id,
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
def handle_operation(
|
|
2112
|
+
self,
|
|
2113
|
+
message: dict[str, Any],
|
|
2114
|
+
*,
|
|
2115
|
+
history: tuple[dict[str, Any] | None] | None = None,
|
|
2116
|
+
) -> dict[str, Any]:
|
|
2117
|
+
op = message.get("op")
|
|
2118
|
+
with self.lock:
|
|
2119
|
+
state = self.record["lifecycle"]["state"]
|
|
2120
|
+
if op == "kill":
|
|
2121
|
+
# Ahead of the booting/resuming guard on purpose: a worker stuck
|
|
2122
|
+
# mid-load is exactly one a caller needs to be able to kill.
|
|
2123
|
+
# `CodexDaemon.handle` routes kills through `_kill_worker` (which
|
|
2124
|
+
# owns the intent/budget protocol); this branch keeps a direct
|
|
2125
|
+
# caller correct.
|
|
2126
|
+
return self.kill(message.get("observeSeconds"))
|
|
2127
|
+
if self.killed.is_set() or self.kill_intent.is_set():
|
|
2128
|
+
# A kill was accepted but could not be confirmed yet (an unresolved
|
|
2129
|
+
# turn launch, or a prior operation that has not settled). The
|
|
2130
|
+
# worker stays registered so reconciliation can see it, but it must
|
|
2131
|
+
# accept no new work. `handle` refuses before taking the operation
|
|
2132
|
+
# lock; this repeats the check for an op that passed that gate and
|
|
2133
|
+
# only then found the intent published.
|
|
2134
|
+
return {"ok": False, "definitive": True,
|
|
2135
|
+
"error": f"worker {self.short} is being killed", "state": state}
|
|
2136
|
+
if state in {"booting", "resuming"}:
|
|
2137
|
+
return {"ok": False, "error": f"worker is {state}", "state": state,
|
|
2138
|
+
**({"definitive": True} if op == "machine-wake" else {})}
|
|
2139
|
+
|
|
2140
|
+
if op in {"parent-notify", "machine-wake"}:
|
|
2141
|
+
event_id = message.get("eventId")
|
|
2142
|
+
delivery_id = message.get("deliveryId") or event_id
|
|
2143
|
+
payload = message.get("payload")
|
|
2144
|
+
if not isinstance(event_id, str) or not event_id:
|
|
2145
|
+
return {"ok": False, "error": "parent-notify requires eventId", "state": state}
|
|
2146
|
+
if not isinstance(delivery_id, str) or not delivery_id:
|
|
2147
|
+
return {"ok": False, "error": "parent-notify requires deliveryId", "state": state}
|
|
2148
|
+
if not isinstance(payload, dict):
|
|
2149
|
+
return {"ok": False, "error": "parent-notify requires payload", "state": state}
|
|
2150
|
+
try:
|
|
2151
|
+
return self._start_parent_notification(
|
|
2152
|
+
delivery_id, payload,
|
|
2153
|
+
reconcile=bool(message.get("reconcile", True)),
|
|
2154
|
+
history=history,
|
|
2155
|
+
namespace=(message["namespace"] if op == "machine-wake" else "ccx"),
|
|
2156
|
+
name=(message["name"] if op == "machine-wake" else "worker_notification"),
|
|
2157
|
+
)
|
|
2158
|
+
except ProtocolError as error:
|
|
2159
|
+
failure = _protocol_failure(error)
|
|
2160
|
+
failure["definitive"] = not self._native_mutation_error_is_ambiguous(error)
|
|
2161
|
+
return failure
|
|
2162
|
+
|
|
2163
|
+
if op == "interrupt":
|
|
2164
|
+
# The notification reader publishes a successor turn under the
|
|
2165
|
+
# worker lock, so read state, turn and thread together once and
|
|
2166
|
+
# interrupt exactly the turn that was checked.
|
|
2167
|
+
with self.lock:
|
|
2168
|
+
state = self.record["lifecycle"]["state"]
|
|
2169
|
+
turn_id = self.record["lifecycle"].get("activeTurnId")
|
|
2170
|
+
# Full-UUID records keep the original thread under native.threadId.
|
|
2171
|
+
native = self.record.get("native") or {}
|
|
2172
|
+
thread_id = native.get("threadId") or self.record.get("threadId")
|
|
2173
|
+
expected_turn_id = message.get("expectedTurnId")
|
|
2174
|
+
if expected_turn_id is not None and expected_turn_id != turn_id:
|
|
2175
|
+
return {
|
|
2176
|
+
"ok": False,
|
|
2177
|
+
"error": "active turn changed",
|
|
2178
|
+
"state": state,
|
|
2179
|
+
"turnId": turn_id,
|
|
2180
|
+
"definitive": True,
|
|
2181
|
+
}
|
|
2182
|
+
if state not in {"busy", "waiting"}:
|
|
2183
|
+
return {"ok": False, "definitive": True,
|
|
2184
|
+
"error": "no active turn", "state": state}
|
|
2185
|
+
if not turn_id:
|
|
2186
|
+
return {"ok": False, "definitive": True,
|
|
2187
|
+
"error": "active turn id is unavailable", "state": state}
|
|
2188
|
+
self.client.request("turn/interrupt", {
|
|
2189
|
+
"threadId": thread_id, "turnId": turn_id,
|
|
2190
|
+
})
|
|
2191
|
+
return {"ok": True, "state": state, "turnId": turn_id}
|
|
2192
|
+
if op in {"approve", "reject"}:
|
|
2193
|
+
# Existing operation_lock serializes decisions and unload. The
|
|
2194
|
+
# reader can still publish/resolve gates: select under worker lock,
|
|
2195
|
+
# then release it before the native server-request response.
|
|
2196
|
+
with self.lock:
|
|
2197
|
+
approval = self._select_approval(message.get("gateId"))
|
|
2198
|
+
gate_state = self.record["lifecycle"]["state"]
|
|
2199
|
+
native = self.record.get("native") or {}
|
|
2200
|
+
turn = self.record["lifecycle"].get("activeTurnId")
|
|
2201
|
+
thread = native.get("threadId")
|
|
2202
|
+
if (approval is None or gate_state != "waiting"
|
|
2203
|
+
or not isinstance(turn, str) or not turn
|
|
2204
|
+
or approval.get("turnId") != turn
|
|
2205
|
+
or not isinstance(thread, str) or not thread
|
|
2206
|
+
or native.get("threadSource") != self.daemon._uuid_store.codex_source(
|
|
2207
|
+
f"codex:{self.worker_id}"
|
|
2208
|
+
)
|
|
2209
|
+
or self.daemon.by_thread.get(thread) != self.worker_id):
|
|
2210
|
+
return {"ok": False, "definitive": True,
|
|
2211
|
+
"error": "no current original Codex approval for gateId",
|
|
2212
|
+
"state": gate_state}
|
|
2213
|
+
self.approval_decisions[_approval_key(approval)] = (
|
|
2214
|
+
"accept" if op == "approve" else "decline"
|
|
2215
|
+
)
|
|
2216
|
+
if isinstance(message.get("responseId"), str):
|
|
2217
|
+
self.approval_response_ids[_approval_key(approval)] = (
|
|
2218
|
+
message["responseId"]
|
|
2219
|
+
)
|
|
2220
|
+
self.client.respond(
|
|
2221
|
+
approval["requestId"],
|
|
2222
|
+
{"decision": "accept" if op == "approve" else "decline"},
|
|
2223
|
+
)
|
|
2224
|
+
with self.lock:
|
|
2225
|
+
self.record["pendingApprovals"] = [
|
|
2226
|
+
value for value in self.record["pendingApprovals"]
|
|
2227
|
+
if value is not approval
|
|
2228
|
+
]
|
|
2229
|
+
# Same resolution as the serverRequest/resolved notification:
|
|
2230
|
+
# without an active turn the worker is idle, not busy — forcing
|
|
2231
|
+
# busy here would wedge a worker whose turn already completed.
|
|
2232
|
+
self.record["lifecycle"]["state"] = (
|
|
2233
|
+
"waiting"
|
|
2234
|
+
if self.record["pendingApprovals"]
|
|
2235
|
+
else (
|
|
2236
|
+
"busy"
|
|
2237
|
+
if self.record["lifecycle"].get("activeTurnId")
|
|
2238
|
+
else "idle"
|
|
2239
|
+
)
|
|
2240
|
+
)
|
|
2241
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
2242
|
+
self._save()
|
|
2243
|
+
return {
|
|
2244
|
+
"ok": True,
|
|
2245
|
+
"state": self.record["lifecycle"]["state"],
|
|
2246
|
+
"requestId": approval["requestId"],
|
|
2247
|
+
"gateId": approval.get("gateId"),
|
|
2248
|
+
"workerId": self.worker_id,
|
|
2249
|
+
"threadId": thread,
|
|
2250
|
+
"turnId": approval.get("turnId"),
|
|
2251
|
+
"decision": "accept" if op == "approve" else "decline",
|
|
2252
|
+
}
|
|
2253
|
+
if op == "compact":
|
|
2254
|
+
from ccx.codex.ops import compact_intent_digest
|
|
2255
|
+
from ccx.store.store import _digest
|
|
2256
|
+
|
|
2257
|
+
operation_id = message.get("operationId")
|
|
2258
|
+
if (not isinstance(operation_id, str) or not operation_id
|
|
2259
|
+
or len(operation_id) > 128 or "\x00" in operation_id):
|
|
2260
|
+
return {"ok": False, "definitive": True,
|
|
2261
|
+
"error": "compact requires caller operationId; no native request"}
|
|
2262
|
+
ctx = self.daemon._uuid_store
|
|
2263
|
+
try:
|
|
2264
|
+
owner = ctx.resolve(f"codex:{self.worker_id}")
|
|
2265
|
+
original = ctx.record(owner.worker)
|
|
2266
|
+
native = original.get("native") or {}
|
|
2267
|
+
source = ctx.codex_source(owner.worker)
|
|
2268
|
+
thread_id = native.get("threadId")
|
|
2269
|
+
activation = ctx.native_activation(owner.worker)
|
|
2270
|
+
if (owner.engine != "codex"
|
|
2271
|
+
or activation.get("state") != "confirmed"
|
|
2272
|
+
or activation.get("source") != source
|
|
2273
|
+
or not isinstance(thread_id, str) or not thread_id
|
|
2274
|
+
or native.get("threadSource") != source
|
|
2275
|
+
or (self.record.get("native") or {}).get("threadId") != thread_id
|
|
2276
|
+
or self.daemon.by_thread.get(thread_id) != self.worker_id
|
|
2277
|
+
or message.get("threadId") != thread_id):
|
|
2278
|
+
raise ValueError("original Codex native thread/source unverified")
|
|
2279
|
+
effect = ctx.read_receipt(owner.worker, "codex-compact", operation_id)
|
|
2280
|
+
digest = compact_intent_digest(
|
|
2281
|
+
owner.worker_id, ctx.generation, thread_id, source, operation_id,
|
|
2282
|
+
)
|
|
2283
|
+
if (not isinstance(effect, dict)
|
|
2284
|
+
or effect.get("intentSha256") != digest
|
|
2285
|
+
or effect.get("workerId") != owner.worker_id
|
|
2286
|
+
or effect.get("engine") != "codex"
|
|
2287
|
+
or effect.get("generation") != ctx.generation
|
|
2288
|
+
or effect.get("family") != "codex-compact"
|
|
2289
|
+
or effect.get("action") != "compact"
|
|
2290
|
+
or effect.get("state") not in (
|
|
2291
|
+
"submitted-unknown", "accepted", "rejected"
|
|
2292
|
+
)):
|
|
2293
|
+
raise ValueError("original full-UUID submitted compact receipt absent")
|
|
2294
|
+
ctx.revalidate()
|
|
2295
|
+
except Exception as error:
|
|
2296
|
+
return {"ok": False, "definitive": True,
|
|
2297
|
+
"error": f"Codex compact original admission denied: {error}"}
|
|
2298
|
+
|
|
2299
|
+
identity = {
|
|
2300
|
+
"workerId": owner.worker_id, "engine": "codex",
|
|
2301
|
+
"generation": ctx.generation, "nativeSource": source,
|
|
2302
|
+
"threadId": thread_id, "operationId": operation_id,
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
def acknowledged(started_at: int, *, accepted: bool,
|
|
2306
|
+
reason: str | None = None, replayed: bool = False) -> dict:
|
|
2307
|
+
data = {
|
|
2308
|
+
**identity, "startedAtMs": started_at,
|
|
2309
|
+
"nativeEvidenceSha256": _digest({
|
|
2310
|
+
**identity, "startedAtMs": started_at,
|
|
2311
|
+
"admission": "thread/compact/start",
|
|
2312
|
+
}),
|
|
2313
|
+
}
|
|
2314
|
+
if accepted:
|
|
2315
|
+
return {**data, "ok": True, "nativeStatus": "accepted",
|
|
2316
|
+
"replayed": replayed, "state": self.record["lifecycle"]["state"]}
|
|
2317
|
+
return {**data, "ok": False, "definitive": True, "noEffect": True,
|
|
2318
|
+
"error": reason or "compact refused before native request",
|
|
2319
|
+
"state": self.record["lifecycle"]["state"]}
|
|
2320
|
+
|
|
2321
|
+
# Same-ID replay must precede the idle gate: a live compact owns
|
|
2322
|
+
# busy and may have outlived a lost caller ACK. Never paste twice.
|
|
2323
|
+
with self.lock:
|
|
2324
|
+
active = self.record.get("activeCompact")
|
|
2325
|
+
last = self.record.get("lastCompact")
|
|
2326
|
+
match = next((
|
|
2327
|
+
marker for marker in (active, last)
|
|
2328
|
+
if isinstance(marker, dict)
|
|
2329
|
+
and marker.get("operationId") == operation_id
|
|
2330
|
+
), None)
|
|
2331
|
+
if match is not None:
|
|
2332
|
+
if effect["state"] == "rejected":
|
|
2333
|
+
return {"ok": False, "error": "rejected compact has a marker",
|
|
2334
|
+
"reconcileRequired": True}
|
|
2335
|
+
started_at = match.get("startedAtMs")
|
|
2336
|
+
if type(started_at) is not int:
|
|
2337
|
+
return {"ok": False, "reconcileRequired": True,
|
|
2338
|
+
"error": "existing compact marker has no original start"}
|
|
2339
|
+
return acknowledged(started_at, accepted=True, replayed=True)
|
|
2340
|
+
if effect["state"] != "submitted-unknown":
|
|
2341
|
+
return acknowledged(now_ms(), accepted=False,
|
|
2342
|
+
reason="compact effect already settled; no resubmit")
|
|
2343
|
+
state = self.record["lifecycle"]["state"]
|
|
2344
|
+
if (state != "idle" or isinstance(active, dict)
|
|
2345
|
+
or self.launch is not None
|
|
2346
|
+
or self.record.get("pendingApprovals")):
|
|
2347
|
+
return acknowledged(now_ms(), accepted=False,
|
|
2348
|
+
reason=f"compact requires idle worker, state={state}")
|
|
2349
|
+
before = self.record.get("context")
|
|
2350
|
+
if isinstance(before, dict):
|
|
2351
|
+
before = {**before, "state": state}
|
|
2352
|
+
baseline = int(self.record.get("compactCount") or 0)
|
|
2353
|
+
started_at = now_ms()
|
|
2354
|
+
self.record["activeCompact"] = {
|
|
2355
|
+
"operationId": operation_id,
|
|
2356
|
+
"awaitingTurn": True,
|
|
2357
|
+
"daemonInstanceId": self._native_create_daemon_id(),
|
|
2358
|
+
"baselineCompactCount": baseline,
|
|
2359
|
+
"targetCompactCount": baseline + 1,
|
|
2360
|
+
"startedAtMs": started_at,
|
|
2361
|
+
"contextBefore": before,
|
|
2362
|
+
}
|
|
2363
|
+
self.record["lifecycle"]["state"] = "busy"
|
|
2364
|
+
self.record["lifecycle"]["since"] = started_at
|
|
2365
|
+
# Native submission is impossible until its exact durable
|
|
2366
|
+
# marker and earlier global submitted intent both exist.
|
|
2367
|
+
self._publish_context_locked(force=True, save=False)
|
|
2368
|
+
self._save()
|
|
2369
|
+
try:
|
|
2370
|
+
self.client.request(
|
|
2371
|
+
"thread/compact/start", {"threadId": thread_id}, timeout=1.5,
|
|
2372
|
+
)
|
|
2373
|
+
except ProtocolError as error:
|
|
2374
|
+
definite = isinstance(error, ProtocolNotSent) or (
|
|
2375
|
+
error.request_id is not None
|
|
2376
|
+
and not isinstance(error, ProtocolTransportError)
|
|
2377
|
+
)
|
|
2378
|
+
if definite:
|
|
2379
|
+
with self.lock:
|
|
2380
|
+
active = self.record.get("activeCompact")
|
|
2381
|
+
if (isinstance(active, dict)
|
|
2382
|
+
and active.get("operationId") == operation_id):
|
|
2383
|
+
self.record["activeCompact"] = None
|
|
2384
|
+
self.record["lifecycle"]["state"] = "idle"
|
|
2385
|
+
self.record["lifecycle"]["since"] = now_ms()
|
|
2386
|
+
self._publish_context_locked(force=True, save=False)
|
|
2387
|
+
self._save()
|
|
2388
|
+
return acknowledged(
|
|
2389
|
+
started_at, accepted=False, reason=str(error),
|
|
2390
|
+
)
|
|
2391
|
+
# Transport ambiguity: persisted activeCompact stays busy,
|
|
2392
|
+
# fenced by this exact global receipt for same-ID inspection.
|
|
2393
|
+
return {**identity, **_protocol_failure(error),
|
|
2394
|
+
"state": "busy", "reconcileRequired": True}
|
|
2395
|
+
# Native admission ACK is not contextCompaction completion.
|
|
2396
|
+
return acknowledged(started_at, accepted=True)
|
|
2397
|
+
return {"ok": False, "error": f"unknown worker operation {op!r}"}
|
|
2398
|
+
|
|
2399
|
+
def public_status(self) -> dict[str, Any]:
|
|
2400
|
+
with self.lock:
|
|
2401
|
+
lifecycle = dict(self.record["lifecycle"])
|
|
2402
|
+
context = self.record.get("context")
|
|
2403
|
+
if context:
|
|
2404
|
+
context = {**context, "state": lifecycle.get("state")}
|
|
2405
|
+
return {
|
|
2406
|
+
"short": self.short,
|
|
2407
|
+
"threadId": (self.record.get("native") or {}).get("threadId"),
|
|
2408
|
+
"name": self.record.get("name"),
|
|
2409
|
+
"cwd": self.record.get("cwd"),
|
|
2410
|
+
"state": lifecycle.get("state"),
|
|
2411
|
+
"activeTurnId": lifecycle.get("activeTurnId"),
|
|
2412
|
+
"compacting": bool(self.record.get("activeCompact")),
|
|
2413
|
+
"pendingApprovals": list(self.record.get("pendingApprovals") or []),
|
|
2414
|
+
"lastReport": self.record.get("lastReport"),
|
|
2415
|
+
"lastTurn": self.record.get("lastTurn"),
|
|
2416
|
+
"context": context,
|
|
2417
|
+
"lastError": lifecycle.get("lastError"),
|
|
2418
|
+
"activeTurnError": self.record.get("activeTurnError"),
|
|
2419
|
+
# Evidence health travels beside worker health, never as part
|
|
2420
|
+
# of it: a worker whose journal degraded is still idle. Null
|
|
2421
|
+
# for a worker that has produced no evidence at all.
|
|
2422
|
+
"evidence": self.record.get("evidence"),
|
|
2423
|
+
"persistenceError": self.persistence_error,
|
|
2424
|
+
}
|