ccx-native 0.0.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +240 -4
  3. package/bin/ccx +55 -2
  4. package/ccx/__init__.py +167 -0
  5. package/ccx/__main__.py +28 -0
  6. package/ccx/claude/__init__.py +0 -0
  7. package/ccx/claude/analytics.py +466 -0
  8. package/ccx/claude/attach.py +807 -0
  9. package/ccx/claude/context.py +465 -0
  10. package/ccx/claude/delivery.py +806 -0
  11. package/ccx/claude/external.py +253 -0
  12. package/ccx/claude/host/__init__.py +0 -0
  13. package/ccx/claude/host/events.py +297 -0
  14. package/ccx/claude/host/hooks.py +438 -0
  15. package/ccx/claude/host/pty.py +269 -0
  16. package/ccx/claude/host/session.py +5244 -0
  17. package/ccx/claude/host/subagents.py +169 -0
  18. package/ccx/claude/host/successor.py +267 -0
  19. package/ccx/claude/managed_turns.py +240 -0
  20. package/ccx/claude/oauth.py +176 -0
  21. package/ccx/claude/settings.py +339 -0
  22. package/ccx/claude/spawn.py +557 -0
  23. package/ccx/claude/tasks.py +72 -0
  24. package/ccx/claude/telemetry.py +282 -0
  25. package/ccx/claude/transcript.py +2535 -0
  26. package/ccx/claude/usage.py +796 -0
  27. package/ccx/claude/usage_control.py +204 -0
  28. package/ccx/cli/__init__.py +0 -0
  29. package/ccx/cli/codex.py +1009 -0
  30. package/ccx/cli/commands/__init__.py +0 -0
  31. package/ccx/cli/commands/_common.py +407 -0
  32. package/ccx/cli/commands/control.py +161 -0
  33. package/ccx/cli/commands/create.py +669 -0
  34. package/ccx/cli/commands/models.py +53 -0
  35. package/ccx/cli/commands/read.py +1104 -0
  36. package/ccx/cli/commands/turns.py +1081 -0
  37. package/ccx/cli/controller.py +757 -0
  38. package/ccx/cli/main.py +302 -0
  39. package/ccx/cli/parser.py +1439 -0
  40. package/ccx/cli/skills.py +215 -0
  41. package/ccx/cli/statusline.py +160 -0
  42. package/ccx/client.py +1904 -0
  43. package/ccx/codex/__init__.py +0 -0
  44. package/ccx/codex/attach_tty.py +267 -0
  45. package/ccx/codex/client.py +738 -0
  46. package/ccx/codex/completion.py +121 -0
  47. package/ccx/codex/control.py +288 -0
  48. package/ccx/codex/daemon/__init__.py +0 -0
  49. package/ccx/codex/daemon/__main__.py +4 -0
  50. package/ccx/codex/daemon/daemon.py +1558 -0
  51. package/ccx/codex/daemon/models.py +140 -0
  52. package/ccx/codex/daemon/native.py +1590 -0
  53. package/ccx/codex/daemon/release.py +367 -0
  54. package/ccx/codex/daemon/worker.py +2424 -0
  55. package/ccx/codex/delivery.py +758 -0
  56. package/ccx/codex/delivery_recovery.py +232 -0
  57. package/ccx/codex/evidence.py +827 -0
  58. package/ccx/codex/history.py +158 -0
  59. package/ccx/codex/history_store.py +131 -0
  60. package/ccx/codex/notify_history.py +140 -0
  61. package/ccx/codex/ops.py +666 -0
  62. package/ccx/codex/paths.py +70 -0
  63. package/ccx/codex/protocol.py +805 -0
  64. package/ccx/codex/provenance.py +105 -0
  65. package/ccx/codex/read.py +29 -0
  66. package/ccx/codex/records.py +51 -0
  67. package/ccx/codex/reports.py +177 -0
  68. package/ccx/codex/runtime.py +1454 -0
  69. package/ccx/codex/settle.py +363 -0
  70. package/ccx/codex/telemetry.py +792 -0
  71. package/ccx/codex/types.py +659 -0
  72. package/ccx/codex/worker_environment.py +25 -0
  73. package/ccx/constants.py +288 -0
  74. package/ccx/controllers/__init__.py +0 -0
  75. package/ccx/controllers/controller.py +1449 -0
  76. package/ccx/controllers/live_vars.py +1706 -0
  77. package/ccx/controllers/live_vars_runner.py +216 -0
  78. package/ccx/controllers/notify.py +213 -0
  79. package/ccx/controllers/orchestrator.py +193 -0
  80. package/ccx/controllers/policy.py +57 -0
  81. package/ccx/controllers/store.py +681 -0
  82. package/ccx/errors.py +634 -0
  83. package/ccx/models.py +225 -0
  84. package/ccx/notify/__init__.py +0 -0
  85. package/ccx/notify/__main__.py +5 -0
  86. package/ccx/notify/client.py +116 -0
  87. package/ccx/notify/daemon.py +480 -0
  88. package/ccx/notify/deliver.py +304 -0
  89. package/ccx/notify/paths.py +51 -0
  90. package/ccx/notify/registry.py +649 -0
  91. package/ccx/notify/select.py +750 -0
  92. package/ccx/notify/settle.py +684 -0
  93. package/ccx/ops/__init__.py +0 -0
  94. package/ccx/ops/background.py +115 -0
  95. package/ccx/ops/cancel.py +296 -0
  96. package/ccx/ops/codex_import.py +96 -0
  97. package/ccx/ops/common.py +397 -0
  98. package/ccx/ops/compact.py +337 -0
  99. package/ccx/ops/compact_ops.py +550 -0
  100. package/ccx/ops/context_events.py +48 -0
  101. package/ccx/ops/conversation.py +1593 -0
  102. package/ccx/ops/conversation_paging.py +1339 -0
  103. package/ccx/ops/create.py +56 -0
  104. package/ccx/ops/creation.py +952 -0
  105. package/ccx/ops/creation_steps.py +575 -0
  106. package/ccx/ops/event_feed.py +710 -0
  107. package/ccx/ops/forking.py +711 -0
  108. package/ccx/ops/history.py +1041 -0
  109. package/ccx/ops/input.py +556 -0
  110. package/ccx/ops/maintenance.py +295 -0
  111. package/ccx/ops/report.py +773 -0
  112. package/ccx/ops/semantics.py +490 -0
  113. package/ccx/ops/send.py +961 -0
  114. package/ccx/ops/steer_ops.py +867 -0
  115. package/ccx/ops/terminate.py +383 -0
  116. package/ccx/ops/turn_reports.py +348 -0
  117. package/ccx/ops/wait.py +762 -0
  118. package/ccx/ops/worker_events.py +640 -0
  119. package/ccx/rpc/__init__.py +0 -0
  120. package/ccx/rpc/local_rpc.py +1602 -0
  121. package/ccx/rpc/rpc.py +559 -0
  122. package/ccx/rpc/serve.py +374 -0
  123. package/ccx/rpc/transport.py +409 -0
  124. package/ccx/rpc/view.py +125 -0
  125. package/ccx/store/__init__.py +0 -0
  126. package/ccx/store/discovery.py +170 -0
  127. package/ccx/store/identity.py +105 -0
  128. package/ccx/store/journal_io.py +79 -0
  129. package/ccx/store/liveness.py +194 -0
  130. package/ccx/store/paths.py +250 -0
  131. package/ccx/store/record_cache.py +40 -0
  132. package/ccx/store/records.py +309 -0
  133. package/ccx/store/root.py +789 -0
  134. package/ccx/store/runtime_presence.py +137 -0
  135. package/ccx/store/store.py +2332 -0
  136. package/ccx/store/worker_state.py +86 -0
  137. package/ccx/system/__init__.py +0 -0
  138. package/ccx/system/bounded.py +296 -0
  139. package/ccx/system/cwd_paths.py +25 -0
  140. package/ccx/system/engine_binary.py +59 -0
  141. package/ccx/system/executable.py +50 -0
  142. package/ccx/system/oslayer.py +667 -0
  143. package/ccx/system/pipes.py +46 -0
  144. package/ccx/system/private_json.py +54 -0
  145. package/ccx/system/sanitize.py +43 -0
  146. package/ccx/types.py +809 -0
  147. package/ccx/version.py +29 -0
  148. package/ccx/workers.py +2093 -0
  149. package/ccx.py +9 -0
  150. package/package.json +39 -7
  151. package/pyproject.toml +87 -0
  152. package/skills/ccx/SKILL.md +166 -0
  153. package/skills/ccx/references/controllers.md +244 -0
  154. package/skills/ccx/references/history.md +50 -0
  155. package/skills/ccx/references/troubleshooting.md +94 -0
  156. package/skills/ccx/references/variables.md +62 -0
@@ -0,0 +1,2424 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import sys
5
+ import threading
6
+ import time
7
+ import uuid
8
+ from copy import deepcopy
9
+ from typing import Any
10
+ from ccx.codex.evidence import CONTEXT_CHANGED_METHOD, BACKGROUND_CHANGED_METHOD, COMPACTION_CONTEXT_METHOD, COMPACT_STARTED_METHOD, COMPACT_ENDED_METHOD, INPUT_REQUESTED_METHOD, INPUT_RESOLVED_METHOD, JOURNALED_METHODS, LIFECYCLE_METHOD, EvidenceJournal, read_index as read_evidence_index, read_journal as read_compact_journal
11
+ from ccx.codex.notify_history import notification_delivery_match
12
+ from ccx.ops.background import label as background_label, summary as background_summary, materially_changed
13
+ from ccx.ops.context_events import ContextChanges
14
+ from ccx.codex.protocol import AppServerClient, ProtocolError, ProtocolNotSent, ProtocolTransportError
15
+ from ccx.codex.reports import native_timestamp_ms, report_from_item, should_replace_live_report
16
+ from ccx.codex.records import now_ms
17
+ from ccx.codex.daemon.models import _abandon_active_compact, WorkerKilled, _LAUNCH_SETTLE_CAP_S, _TurnLaunch, _bounded, _protocol_failure
18
+ from typing import TYPE_CHECKING
19
+ if TYPE_CHECKING:
20
+ from ccx.codex.daemon.daemon import CodexDaemon
21
+
22
+
23
+ _TERMINAL_STATES = {"dead", "idle-killed", "crashed", "budget-stopped"}
24
+
25
+
26
+ def _system_error_reason(error) -> str:
27
+ """The crash reason, with Codex's own error message when there is one."""
28
+ message = error.get("message") if isinstance(error, dict) else None
29
+ if isinstance(message, str) and message.strip():
30
+ return "Codex thread system error: " + message.strip()[:500]
31
+ return "Codex thread system error"
32
+
33
+
34
+ _PARENT_NOTIFY_FENCES = 32
35
+
36
+
37
+ TURN_OUTCOME_STATUSES = {"completed", "interrupted", "failed"}
38
+
39
+
40
+ APPROVAL_METHODS: dict[str, str] = {
41
+ "item/commandExecution/requestApproval": "command",
42
+ "item/fileChange/requestApproval": "fileChange",
43
+ }
44
+
45
+
46
+ def _turn_id(value: dict[str, Any]) -> str | None:
47
+ turn = value.get("turn")
48
+ return turn.get("id") if isinstance(turn, dict) else None
49
+
50
+
51
+ def _last_turn_from_native(turn: dict[str, Any]) -> dict[str, Any]:
52
+ """Normalize one native ``Turn`` into the durable outcome projection.
53
+
54
+ ``lastTurn`` deliberately contains only the terminal turn facts. In
55
+ particular, timestamps come from Codex's Turn object; a missing native
56
+ timestamp remains ``None`` instead of being replaced with CCX wall-clock
57
+ time. Unknown native statuses are not promoted into the public outcome
58
+ vocabulary.
59
+ """
60
+ status = turn.get("status")
61
+ return {
62
+ "turnId": turn.get("id"),
63
+ "status": status if status in TURN_OUTCOME_STATUSES else None,
64
+ "error": turn.get("error"),
65
+ "startedAtMs": native_timestamp_ms(turn.get("startedAt")),
66
+ "completedAtMs": native_timestamp_ms(turn.get("completedAt")),
67
+ "durationMs": (
68
+ turn.get("durationMs")
69
+ if isinstance(turn.get("durationMs"), (int, float))
70
+ and not isinstance(turn.get("durationMs"), bool)
71
+ else None
72
+ ),
73
+ }
74
+
75
+
76
+
77
+
78
+ def _approval_key(approval: dict[str, Any]) -> str:
79
+ """One pending approval's identity: its JSON-RPC request id, typed."""
80
+ return json.dumps(approval.get("requestId"), sort_keys=True, default=str)
81
+
82
+
83
+ class Worker:
84
+ """One CCX-owned Codex worker, driven over the daemon's shared client."""
85
+
86
+ def __init__(self, daemon: "CodexDaemon", record: dict[str, Any]):
87
+ self.daemon = daemon
88
+ # Original local short variable is internally a canonical full owner.
89
+ # Eight-character display prefixes never identify a worker here.
90
+ from ccx.store.identity import canonical_worker_id
91
+ self.worker_id = canonical_worker_id(record.get("workerId"))
92
+ if record.get("engine") != "codex":
93
+ raise ValueError("daemon Worker requires globally owned Codex ID")
94
+ self.record = deepcopy(record)
95
+ self._disk_baseline = deepcopy(record)
96
+ self.short = self.worker_id
97
+ self.lock = threading.RLock()
98
+ self.operation_lock = threading.Lock()
99
+ self.load_lock = threading.Lock()
100
+ self.loaded = False
101
+ # Set by `kill` before it unloads, and never cleared: a Worker object
102
+ # that has been killed is dead forever, and a fresh `load` builds a new
103
+ # one. This is the whole kill/load race protocol — see `_abort_if_killed`.
104
+ self.killed = threading.Event()
105
+ # Published before kill waits for `operation_lock`, and never cleared.
106
+ # A kill that has to wait behind an in-flight steer/approval must not
107
+ # meanwhile let new operations in — but it also must not bypass the
108
+ # lock and race the operation that is already running. The sticky
109
+ # intent separates the two: new work is refused immediately, the
110
+ # running op is allowed to settle, and only then does the kill act.
111
+ self.kill_intent = threading.Event()
112
+ # Non-None exactly while a `turn/start` request is in flight. Written
113
+ # only under `self.lock`, which is what makes kill-vs-launch atomic.
114
+ self.launch: _TurnLaunch | None = None
115
+ # Caps belong to one original native turn, never to the worker's next turn.
116
+ self.cap_turn_id: str | None = None
117
+ self.active_caps: dict | None = None
118
+ # A source-proven resume may return the preceding failed turn's
119
+ # persisted systemError. Until the next native turn/status transition,
120
+ # another systemError edge is that old state, not a new crash.
121
+ self._resumed_system_error_pending = False
122
+ # Native threads this object's load may have subscribed the shared
123
+ # connection to. Codex keeps a thread loaded (with its per-thread MCP
124
+ # servers) until its last subscriber leaves, so `_unload` releases
125
+ # these. Kept rather than cleared: unsubscribing twice is harmless, and
126
+ # a kill that lands mid-resume unloads before the subscription exists.
127
+ self.native_threads: set[str] = set()
128
+ # Parent-notification deliveries whose `turn/start` this object saw
129
+ # answered, so a retry walking history outside the operation lock
130
+ # cannot launch one twice. delivery id -> native turn id.
131
+ self.launched_notifications: dict[str, str | None] = {}
132
+ # Claimed under `daemon.lock` by the one `_unload` that tears this
133
+ # object down; see there.
134
+ self.teardown_claimed = False
135
+ self.last_activity = time.monotonic()
136
+ self.attach_count = 0 # live attach RPC connections, never persisted
137
+ # Native commands become background only if still open at turn/completed.
138
+ # The journal is the source; these are this app-server's owned live IDs.
139
+ # Public context emission is one sparse stream, independent of raw native
140
+ # usage notifications. Reuse the existing daemon janitor for pending flush.
141
+ self._context_changes = ContextChanges()
142
+ self._bg_candidates: dict[str, dict] = {}
143
+ self._bg_open: dict[str, dict] = {}
144
+ # No replay ledger: a persisted snapshot is enough to compare current
145
+ # registry truth when this worker is adopted or notified again.
146
+ prior_items = (record.get("backgroundTaskSummary") or {}).get("items") or []
147
+ self._bg_notifications = {
148
+ item["id"]: dict(item) for item in prior_items
149
+ if isinstance(item, dict) and item.get("kind") == "notification"
150
+ and item.get("status") == "pending"
151
+ and isinstance(item.get("id"), str)
152
+ }
153
+ # The last record write a notification handler could not make (disk
154
+ # full, permissions). The in-memory record stays authoritative.
155
+ self.persistence_error: dict[str, Any] | None = None
156
+ # An external terminal CAS wins over this cached Worker forever.
157
+ self._externally_terminalized = False
158
+ # Sparse durable evidence for this worker's turns. Opened lazily on
159
+ # the first journaled notification, so a worker that never runs one
160
+ # leaves no file — which is itself the truthful answer to "was CCX
161
+ # watching?".
162
+ # `health` rehydrates a degradation an earlier process already
163
+ # recorded: a fresh writer must not present a worker whose evidence is
164
+ # known to be incomplete as complete again.
165
+ self.evidence = EvidenceJournal(
166
+ self.short,
167
+ codex_version=getattr(daemon, "codex_version", None),
168
+ health=record.get("evidence"),
169
+ )
170
+ try:
171
+ evidence_index = read_evidence_index(self.short)
172
+ except Exception:
173
+ evidence_index = {}
174
+ self.completed_turn_ids = {
175
+ str(turn_id)
176
+ for turn_id, facts in (evidence_index.get("turns") or {}).items()
177
+ if isinstance(facts, dict) and facts.get("sawTurnCompleted")
178
+ }
179
+ # Journal-derived identities keep compaction end publication idempotent
180
+ # when native notifications repeat or codexd reloads an owned worker.
181
+ self._compact_started: dict[str, set[str]] = {}
182
+ self._compact_finished: dict[str, set[str]] = {}
183
+ self._compact_ended: set[str] = set()
184
+ self._compact_legacy: set[str] = set()
185
+ try:
186
+ compact_history = read_compact_journal(self.short).events
187
+ except Exception:
188
+ compact_history = []
189
+ for row in compact_history:
190
+ tid, item_id, method = row.get("turnId"), row.get("itemId"), row.get("method")
191
+ if method == "thread/compacted" and isinstance(tid, str):
192
+ self._compact_legacy.add(tid)
193
+ if not isinstance(tid, str) or not tid:
194
+ continue
195
+ # Historical native items are still forensic proof, not feed
196
+ # events. Only CCX-owned sparse lines rehydrate publisher dedupe.
197
+ # Older journals lacking them intentionally gain no backfill.
198
+ if (row.get("itemType") == "contextCompaction"
199
+ and isinstance(item_id, str)
200
+ and method == "item/completed"):
201
+ self._compact_finished.setdefault(tid, set()).add(item_id)
202
+ if method == COMPACT_STARTED_METHOD:
203
+ key = (row.get("params") or {}).get("compactId")
204
+ if isinstance(key, str):
205
+ self._compact_started.setdefault(tid, set()).add(key)
206
+ if method == COMPACT_ENDED_METHOD:
207
+ key = (row.get("params") or {}).get("compactId")
208
+ if isinstance(key, str):
209
+ self._compact_ended.add(key)
210
+ # What the journal last said about this worker's state and approvals.
211
+ # `_journal_lifecycle_locked` diffs the record against it on every
212
+ # save, so each transition is journaled exactly once whichever of the
213
+ # many record-mutating paths made it.
214
+ self.journaled_state = (record.get("lifecycle") or {}).get("state")
215
+ self.journaled_approvals = {
216
+ _approval_key(approval): approval
217
+ for approval in record.get("pendingApprovals") or []
218
+ if isinstance(approval, dict)
219
+ }
220
+ # Decisions CCX itself sent, keyed like `journaled_approvals`, so the
221
+ # resolution line can say how a request was answered.
222
+ self.approval_decisions: dict[str, str] = {}
223
+ # The caller's response id for a decision, journaled with it so a
224
+ # caller that lost the reply can prove its decision landed.
225
+ self.approval_response_ids: dict[str, str] = {}
226
+ last_turn = record.get("lastTurn")
227
+ if (
228
+ isinstance(last_turn, dict)
229
+ and isinstance(last_turn.get("turnId"), str)
230
+ and last_turn.get("turnId")
231
+ ):
232
+ self.completed_turn_ids.add(str(last_turn["turnId"]))
233
+
234
+ @property
235
+ def client(self) -> AppServerClient:
236
+ client = self.daemon.client
237
+ if client is None:
238
+ raise RuntimeError("app-server client is not running")
239
+ return client
240
+
241
+ def _save(self) -> None:
242
+ """Save once with field CAS; retry only a proved safe fresh-record rebase.
243
+
244
+ A failed CAS is not authority to drop expectations. In particular a
245
+ concurrent CLI kill owns the terminal lifecycle, even if an app-server
246
+ completion races it. The original native event remains in the evidence
247
+ journal when a conflict cannot be reconciled.
248
+ """
249
+ from ccx.store.root import RecordConflict
250
+ with self.lock:
251
+ if self._externally_terminalized and (
252
+ self.record.get("lifecycle") != self._disk_baseline.get("lifecycle")
253
+ or self.record.get("pendingApprovals")
254
+ != self._disk_baseline.get("pendingApprovals")
255
+ ):
256
+ # This object saw an external terminal CAS. Even another
257
+ # daemon-local _state/approval writer cannot bring it back;
258
+ # restore the current persisted terminal view before refusing.
259
+ latest = self.daemon._uuid_store.record(f"codex:{self.worker_id}")
260
+ self.record = deepcopy(latest)
261
+ self._disk_baseline = deepcopy(latest)
262
+ raise RecordConflict("externally terminalized Codex Worker cannot revive")
263
+ self._journal_lifecycle_locked()
264
+ try:
265
+ self._save_current_locked()
266
+ except RecordConflict:
267
+ self._rebase_conflicting_save_locked()
268
+ self._journal_lifecycle_locked()
269
+ # One bounded retry against the newly observed exact fields.
270
+ # A second conflict is surfaced per worker, never overwritten.
271
+ self._save_current_locked()
272
+
273
+ def _save_current_locked(self) -> None:
274
+ """One semantic CAS publication; caller holds self.lock."""
275
+ from ccx.store.root import StorageIntegrityError, RecordConflict
276
+ if (self._externally_terminalized
277
+ and self.record["lifecycle"].get("state") not in _TERMINAL_STATES):
278
+ raise RecordConflict("external terminal state cannot be resurrected")
279
+ before = self._disk_baseline
280
+ desired = self.record
281
+ patches: dict[str, object] = {}
282
+ expect: dict[str, object] = {}
283
+ increments: dict[str, int] = {}
284
+ containers = ("lifecycle", "stats", "createArgs")
285
+ top_allowed = {
286
+ "name", "waitingFor", "lastTurnAt", "lastSettledAt",
287
+ "status", "lastTurn", "lastReport", "activeTurnError",
288
+ "pendingApprovals", "context", "activeCompact", "lastCompact",
289
+ "compactCount", "evidence", "tokenUsage", "historyKnownEmpty",
290
+ "nativeArchive", "resolvedModel", "parentNotifyFences",
291
+ "lastTurnError", "pendingSend", "lastConversation", "nativeTurnStart",
292
+ "quotaStop", "backgroundTaskSummary",
293
+ }
294
+ for key in set(before) | set(desired):
295
+ if key in ("recordRevision", "updatedAt"):
296
+ continue
297
+ old = before.get(key)
298
+ new = desired.get(key)
299
+ if new == old:
300
+ continue
301
+ if key not in desired:
302
+ raise StorageIntegrityError(
303
+ "Codex Worker cannot delete a durable record field"
304
+ )
305
+ if key in containers:
306
+ if not isinstance(old, dict) or not isinstance(new, dict):
307
+ raise StorageIntegrityError(
308
+ "Codex Worker cannot replace structured record"
309
+ )
310
+ if not set(old).issubset(new):
311
+ raise StorageIntegrityError(
312
+ "Codex Worker cannot remove structured fields"
313
+ )
314
+ for member in new:
315
+ if member in old and old[member] == new[member]:
316
+ continue
317
+ field = f"{key}.{member}"
318
+ if (key == "stats"
319
+ and type(new[member]) is int
320
+ and type(old.get(member)) is int
321
+ and new[member] >= old[member]):
322
+ increments[field] = new[member] - old[member]
323
+ else:
324
+ patches[field] = new[member]
325
+ expect[field] = old.get(member)
326
+ elif key in top_allowed:
327
+ patches[key] = new
328
+ expect[key] = old
329
+ else:
330
+ raise StorageIntegrityError(
331
+ f"Codex Worker cannot mutate immutable native/owner field {key}"
332
+ )
333
+ ctx = self.daemon._uuid_store
334
+ if patches or increments:
335
+ durable = ctx.mutate_record(
336
+ f"codex:{self.worker_id}",
337
+ patches=patches, increments=increments,
338
+ expect=expect,
339
+ )
340
+ else:
341
+ durable = ctx.record(f"codex:{self.worker_id}")
342
+ # A nonconflicting metadata/evidence patch (or even a pure read) can
343
+ # absorb another process's terminal kill. Recognize its authority on
344
+ # every fresh durable observation, not only in the conflict rebase.
345
+ old_state = (before.get("lifecycle") or {}).get("state")
346
+ committed_state = (durable.get("lifecycle") or {}).get("state")
347
+ proposed_state = (desired.get("lifecycle") or {}).get("state")
348
+ if (committed_state in _TERMINAL_STATES
349
+ and old_state not in _TERMINAL_STATES
350
+ and proposed_state != committed_state):
351
+ self._externally_terminalized = True
352
+ self.killed.set()
353
+ self.record = durable
354
+ self._disk_baseline = deepcopy(durable)
355
+
356
+
357
+ def _rebase_conflicting_save_locked(self) -> None:
358
+ """Reconcile one same-worker CAS conflict, never ignore its expectations.
359
+
360
+ Fresh latest data is the base. Replay a local edit only if its exact
361
+ previous value is still present (or the edit is already durable). A
362
+ positive integer stat increment is commutative, except a completion
363
+ already recorded for the same native turn must not be counted twice.
364
+ A concurrent terminal state wins over *all* cached lifecycle and
365
+ approval projections. Unprovable overlaps remain visibly unresolved.
366
+ """
367
+ from ccx.store.root import RecordConflict, StorageIntegrityError
368
+ before, desired = self._disk_baseline, self.record
369
+ latest = self.daemon._uuid_store.record(f"codex:{self.worker_id}")
370
+ if (latest.get("workerId") != self.worker_id
371
+ or latest.get("engine") != "codex"
372
+ or latest.get("native") != before.get("native")):
373
+ raise StorageIntegrityError("Codex rebase changed immutable native owner")
374
+ state = (latest.get("lifecycle") or {}).get("state")
375
+ terminal_won = (state in _TERMINAL_STATES
376
+ and (before.get("lifecycle") or {}).get("state")
377
+ not in _TERMINAL_STATES)
378
+ if terminal_won:
379
+ self._externally_terminalized = True
380
+ self.killed.set() # also fences this cached worker's future launches
381
+
382
+ def refuse(message: str) -> None:
383
+ # Do not retain stale state that will conflict on every later event,
384
+ # or an approval that was never durably admitted. Native evidence
385
+ # was journaled independently; the handler exposes the lost patch.
386
+ self.record = deepcopy(latest)
387
+ self._disk_baseline = deepcopy(latest)
388
+ raise RecordConflict(message)
389
+
390
+ old_turn = (before.get("lastTurn") or {}).get("turnId")
391
+ wanted_turn = (desired.get("lastTurn") or {}).get("turnId")
392
+ disk_turn = (latest.get("lastTurn") or {}).get("turnId")
393
+ if terminal_won:
394
+ old_approvals = before.get("pendingApprovals") or []
395
+ new_approvals = desired.get("pendingApprovals") or []
396
+ if any(approval not in old_approvals for approval in new_approvals):
397
+ refuse("external kill won before Codex approval was durable")
398
+ if wanted_turn != old_turn and (not isinstance(wanted_turn, str)
399
+ or (before.get("lifecycle") or {}).get("activeTurnId")
400
+ != wanted_turn or disk_turn not in (old_turn, wanted_turn)):
401
+ refuse("terminal worker has no positive original turn to reconcile")
402
+
403
+ rebased = deepcopy(latest)
404
+ containers = ("lifecycle", "stats", "createArgs")
405
+ for key in set(before) | set(desired):
406
+ if key in ("recordRevision", "updatedAt") or before.get(key) == desired.get(key):
407
+ continue
408
+ old, new = before.get(key), desired.get(key)
409
+ if key not in desired:
410
+ raise StorageIntegrityError("Codex cannot rebase a deleted field")
411
+ if key in containers:
412
+ if (not isinstance(old, dict) or not isinstance(new, dict)
413
+ or not isinstance(rebased.get(key), dict)
414
+ or not set(old).issubset(new)):
415
+ raise StorageIntegrityError("Codex cannot rebase malformed structured fields")
416
+ changes = ((f"{key}.{member}", old.get(member), value)
417
+ for member, value in new.items()
418
+ if member not in old or old[member] != value)
419
+ else:
420
+ changes = ((key, old, new),)
421
+ for field, previous, value in changes:
422
+ # Terminal status/cleared active turn and approvals are not
423
+ # native completion effects. Never revive them via a stale save.
424
+ if terminal_won and (field.startswith("lifecycle.") or field in {
425
+ "pendingApprovals", "waitingFor", "pendingSend",
426
+ "activeTurnError", "name"} or field.startswith("createArgs.")):
427
+ continue
428
+ parts = field.split(".")
429
+ container = rebased if len(parts) == 1 else rebased[parts[0]]
430
+ current = container.get(parts[-1])
431
+ if (field.startswith("stats.")
432
+ and type(previous) is int and type(value) is int
433
+ and value >= previous):
434
+ if type(current) is not int:
435
+ refuse(f"{field} cannot be merged as a counter")
436
+ delta = value - previous
437
+ if field == "stats.turns" and wanted_turn != old_turn:
438
+ if disk_turn == wanted_turn:
439
+ delta = 0 # this same turn was already committed
440
+ elif disk_turn != old_turn:
441
+ refuse("newer native turn prevents counter reconciliation")
442
+ container[parts[-1]] = current + delta
443
+ elif current == previous:
444
+ container[parts[-1]] = deepcopy(value)
445
+ elif current != value:
446
+ refuse(f"record semantic field changed before CAS: {field}")
447
+ self.record = rebased
448
+ self._disk_baseline = deepcopy(latest)
449
+
450
+ def _journal_lifecycle_locked(
451
+ self,
452
+ *,
453
+ state: str | None = None,
454
+ approvals: list[dict[str, Any]] | None = None,
455
+ ) -> None:
456
+ """Journal state and approval transitions not yet in the journal.
457
+
458
+ Caller holds ``self.lock``. ``state``/``approvals`` override the
459
+ record's, for a teardown that journals its terminal facts before it
460
+ closes the journal. Never raises (``EvidenceJournal.record`` does
461
+ not), and writes nothing once the journal is closed: reopening would
462
+ start a new header for a worker that is already gone.
463
+ """
464
+ if self.evidence.closed:
465
+ return
466
+ lifecycle = self.record.get("lifecycle") or {}
467
+ current_state = state if state is not None else lifecycle.get("state")
468
+ current = {
469
+ _approval_key(approval): approval
470
+ for approval in (
471
+ approvals if approvals is not None
472
+ else self.record.get("pendingApprovals") or []
473
+ )
474
+ if isinstance(approval, dict)
475
+ }
476
+ wrote = False
477
+ thread_id = (self.record.get("native") or {}).get("threadId")
478
+ for key, approval in self.journaled_approvals.items():
479
+ if key not in current:
480
+ params = {
481
+ "requestId": approval.get("requestId"),
482
+ "gateId": approval.get("gateId"),
483
+ "kind": approval.get("kind"),
484
+ "decision": self.approval_decisions.pop(key, None),
485
+ }
486
+ response_id = self.approval_response_ids.pop(key, None)
487
+ if response_id is not None:
488
+ params["responseId"] = response_id
489
+ self.evidence.record(
490
+ INPUT_RESOLVED_METHOD, params, thread_id=thread_id,
491
+ )
492
+ wrote = True
493
+ for key, approval in current.items():
494
+ if key not in self.journaled_approvals:
495
+ self.evidence.record(
496
+ INPUT_REQUESTED_METHOD, {"request": approval},
497
+ thread_id=thread_id,
498
+ )
499
+ wrote = True
500
+ self.journaled_approvals = current
501
+ if current_state and current_state != self.journaled_state:
502
+ params: dict[str, Any] = {
503
+ "from": self.journaled_state,
504
+ "to": current_state,
505
+ }
506
+ if current_state in {"crashed", "dead", "idle-killed"}:
507
+ params["reason"] = lifecycle.get("lastError")
508
+ self.evidence.record(LIFECYCLE_METHOD, params, thread_id=thread_id)
509
+ self.journaled_state = current_state
510
+ wrote = True
511
+ if wrote:
512
+ self.record["evidence"] = self.evidence.state()
513
+
514
+
515
+ def _abort_if_killed(self) -> None:
516
+ """Unwind a load that a concurrent ``kill`` has already invalidated.
517
+
518
+ Original resume spends most of its time inside app-server RPCs holding
519
+ no daemon or worker lock — deliberately, so the daemon stays responsive.
520
+ That means a `kill` can complete (record ``dead``, worker removed from
521
+ `workers`/`by_thread`) while this load is still in flight, and every
522
+ step after the RPC would then resurrect it: re-binding the thread,
523
+ rewriting the record and finally reporting ``ok``.
524
+
525
+ Rather than have `kill` block on `load_lock` — which would stall it
526
+ behind a 90s app-server call — the loader checks this event at each of
527
+ those points and unwinds itself. Reading an Event takes no lock, so it
528
+ cannot deadlock against the killer.
529
+ """
530
+ if self.killed.is_set():
531
+ raise WorkerKilled(f"worker {self.short} was killed during load")
532
+
533
+ def _commit_state(self, state: str, **lifecycle_patch) -> None:
534
+ """Write a loader state transition atomically against ``kill``.
535
+
536
+ A bare `_abort_if_killed()` before `_state()` is not enough: a kill
537
+ landing in the gap writes ``dead`` and then this write puts the record
538
+ back to ``idle``/``busy``, leaving a killed worker looking alive. The
539
+ check and the write therefore happen inside one hold of ``self.lock``,
540
+ which is the same lock ``kill`` sets its marker under — so the two are
541
+ totally ordered and the loser is the one that does nothing.
542
+ """
543
+ with self.lock:
544
+ if self.killed.is_set():
545
+ raise WorkerKilled(
546
+ f"worker {self.short} was killed during load")
547
+ lifecycle = self.record["lifecycle"]
548
+ lifecycle.update(lifecycle_patch)
549
+ lifecycle["state"] = state
550
+ lifecycle["since"] = now_ms()
551
+ self.last_activity = time.monotonic()
552
+ self._save()
553
+
554
+ def _interrupt_turn(self, turn_id: str | None) -> bool:
555
+ """Best-effort native cancellation of one turn. Safe to call twice."""
556
+ thread_id = (self.record.get("native") or {}).get("threadId") or self.record.get("threadId")
557
+ if not turn_id or not thread_id:
558
+ return False
559
+ try:
560
+ self.client.request(
561
+ "turn/interrupt", {"threadId": thread_id, "turnId": turn_id})
562
+ return True
563
+ except Exception:
564
+ return False
565
+
566
+ def check_active_caps(self, rate_limits: dict | None) -> None:
567
+ """Check a pushed quota update without RPC on the notification reader."""
568
+ from ccx.codex.types import codex_cap_tripped
569
+ with self.lock:
570
+ turn_id = self.cap_turn_id
571
+ if (not turn_id or not self.active_caps
572
+ or self.record["lifecycle"].get("activeTurnId") != turn_id):
573
+ return
574
+ trip = codex_cap_tripped(rate_limits, self.active_caps)
575
+ if not trip:
576
+ return
577
+ name, used, cap = trip
578
+ reason = f"usage cap {name} reached: {used:g}% >= {cap:g}%"
579
+ self.record["quotaStop"] = {"turnId": turn_id, "reason": reason}
580
+ self.active_caps = None # at most one interrupt for this native turn
581
+ self._save()
582
+ threading.Thread(target=self._interrupt_turn, args=(turn_id,),
583
+ daemon=True, name="ccx-codex-cap-interrupt").start()
584
+
585
+ def _abandon_active_compact_locked(self, reason: str) -> bool:
586
+ """Close a persisted compact that can no longer yield native proof.
587
+
588
+ Caller holds ``self.lock``. See `_abandon_active_compact`.
589
+ """
590
+ return _abandon_active_compact(self.record, reason)
591
+
592
+ def _state(self, state: str, *, error: str | None = None) -> None:
593
+ with self.lock:
594
+ lifecycle = self.record["lifecycle"]
595
+ lifecycle["state"] = state
596
+ lifecycle["since"] = now_ms()
597
+ if error is not None:
598
+ lifecycle["lastError"] = error
599
+ self.last_activity = time.monotonic()
600
+ self._save()
601
+
602
+ def state(self) -> str:
603
+ with self.lock:
604
+ return self.record["lifecycle"]["state"]
605
+
606
+
607
+ def _native_create_daemon_id(self) -> str:
608
+ return str(getattr(self.daemon, "instance_id", "legacy-daemon"))
609
+
610
+
611
+
612
+
613
+
614
+
615
+ def _resume_exact_thread(
616
+ self, thread_id: str, params: dict[str, Any], *,
617
+ archived: bool = False, timeout: float = 90,
618
+ before_submit=None,
619
+ ) -> dict[str, Any]:
620
+ """One original native thread/resume, never a second thread/start.
621
+
622
+ The pre-write release fence and native_threads registration also cover
623
+ a lost reply: a concurrent kill must still unsubscribe this thread.
624
+ """
625
+ self.daemon._await_thread_release(thread_id)
626
+ self.daemon.claim_thread(thread_id, self.short)
627
+ self.native_threads.add(thread_id)
628
+ if archived:
629
+ self.client.request(
630
+ "thread/unarchive", {"threadId": thread_id}, timeout=timeout,
631
+ )
632
+ resume_params = dict(params)
633
+ resume_params["threadId"] = thread_id
634
+ # Only the thread's id, source and status are read from the reply;
635
+ # rebuilding a long thread's full history can take Codex minutes.
636
+ resume_params["excludeTurns"] = True
637
+ # Kill can win while this load waits for a prior unsubscribe. The
638
+ # already-published killed marker is the existing no-late-load fence.
639
+ self._abort_if_killed()
640
+ if before_submit is not None:
641
+ before_submit() # persist the current native attempt only after release preflight
642
+ try:
643
+ return self.client.request(
644
+ "thread/resume", resume_params, timeout=timeout,
645
+ )
646
+ except ProtocolError as error:
647
+ if "no rollout found" in error.message:
648
+ raise ProtocolError(
649
+ "original Codex thread has no persisted rollout; "
650
+ "refusing native thread/start replacement",
651
+ method="thread/resume",
652
+ ) from error
653
+ raise
654
+
655
+
656
+
657
+ def _native_turn_start_marker(
658
+ self,
659
+ *,
660
+ kind: str,
661
+ client_message_id: str | None = None,
662
+ delivery_id: str | None = None,
663
+ ) -> dict[str, Any]:
664
+ return {
665
+ "schemaVersion": 1,
666
+ "kind": kind,
667
+ "state": "submitted",
668
+ "daemonInstanceId": self._native_create_daemon_id(),
669
+ "clientUserMessageId": client_message_id,
670
+ "deliveryId": delivery_id,
671
+ "submittedAtMs": now_ms(),
672
+ "observedAtMs": None,
673
+ "lastError": None,
674
+ }
675
+
676
+ def _mark_native_turn_start_unknown_locked(self, error: BaseException) -> None:
677
+ marker = self.record.get("nativeTurnStart")
678
+ if not isinstance(marker, dict):
679
+ return
680
+ marker["state"] = "unknown"
681
+ marker["observedAtMs"] = now_ms()
682
+ marker["lastError"] = str(error)
683
+ self._save()
684
+
685
+ def _forget_native_turn_start_locked(self) -> None:
686
+ # Full UUID GenStore CAS does not erase persistent top-level fields;
687
+ # clear the existing launch marker to null under the same CAS instead.
688
+ if self.record.get("fingerprintVersion") == 2:
689
+ if "nativeTurnStart" in self.record:
690
+ self.record["nativeTurnStart"] = None
691
+ else:
692
+ self.record.pop("nativeTurnStart", None)
693
+
694
+ def _clear_native_turn_start_locked(self) -> None:
695
+ if self.record.get("nativeTurnStart") is not None:
696
+ self._forget_native_turn_start_locked()
697
+ self._save()
698
+
699
+ @staticmethod
700
+ def _native_mutation_error_is_ambiguous(error: ProtocolError) -> bool:
701
+ if isinstance(error, ProtocolNotSent):
702
+ return False
703
+ return isinstance(error, ProtocolTransportError) or error.request_id is None
704
+
705
+ def _journal(self, method: str, params: dict[str, Any]) -> bool:
706
+ """Record one authoritative native event before it is reduced.
707
+
708
+ Returns whether this notification is journaled at all, so the caller
709
+ can persist the journal's health in the record write it is already
710
+ doing. `EvidenceJournal.record` swallows its own failures on purpose:
711
+ a handler that raised here would fail the shared app-server client
712
+ (`AppServerClient._read_loop` treats a handler exception as fatal),
713
+ killing every worker's live turn over a file that could not be
714
+ appended to.
715
+ """
716
+ if method not in JOURNALED_METHODS:
717
+ return False
718
+ durable = None
719
+ if method == "thread/tokenUsage/updated":
720
+ usage_turn_id = params.get("turnId") or _turn_id(params)
721
+ durable = usage_turn_id in self.completed_turn_ids
722
+ self.evidence.record(
723
+ method,
724
+ params,
725
+ thread_id=self.record.get("threadId"),
726
+ durable=durable,
727
+ )
728
+ return True
729
+
730
+ def _capture_compaction_context_locked(
731
+ self, compact: dict[str, Any]
732
+ ) -> None:
733
+ """Persist one bounded compaction before/after telemetry enrichment."""
734
+ context_after = self.record.get("context")
735
+ earlier = compact.get("contextBefore")
736
+ completed = compact.get("completedAtMs")
737
+ valid = (
738
+ isinstance(context_after, dict)
739
+ and type(context_after.get("as_of_ms")) is int
740
+ and type(completed) is int
741
+ and context_after["as_of_ms"] > completed
742
+ and (not isinstance(earlier, dict)
743
+ or type(earlier.get("as_of_ms")) is not int
744
+ or context_after["as_of_ms"] > earlier["as_of_ms"])
745
+ and (self.record.get("tokenUsage") or {}).get("asOfTurnId") != compact.get("turnId")
746
+ )
747
+ compact["contextAfter"] = dict(context_after) if valid else None
748
+ params = {
749
+ "turnId": compact.get("turnId"),
750
+ "item": {
751
+ "id": compact.get("contextItemId"),
752
+ "type": "contextCompaction",
753
+ },
754
+ "operationId": compact.get("operationId"),
755
+ "compactCount": self.record.get("compactCount"),
756
+ "contextBefore": compact.get("contextBefore"),
757
+ "contextAfter": compact.get("contextAfter"),
758
+ "completedAtMs": compact.get("completedAtMs"),
759
+ }
760
+ self.evidence.record(
761
+ COMPACTION_CONTEXT_METHOD,
762
+ params,
763
+ thread_id=self.record.get("threadId"),
764
+ )
765
+ self.record["evidence"] = self.evidence.state()
766
+
767
+ def _complete_active_compact_locked(
768
+ self, *, turn_id: str | None = None, completed_at_ms: int | None = None
769
+ ) -> bool:
770
+ """Publish one successful requested compact. Caller holds ``self.lock``."""
771
+ active = self.record.get("activeCompact")
772
+ if (not isinstance(active, dict) or not isinstance(turn_id, str)
773
+ or not turn_id or active.get("turnId") != turn_id
774
+ or active.get("nativeTurnStarted") is not True
775
+ or not isinstance(active.get("contextItemId"), str)
776
+ or not active["contextItemId"]
777
+ or active.get("nativeItemCompletedAtMs") is None
778
+ or turn_id not in self.completed_turn_ids):
779
+ return False
780
+ compact_count = int(self.record.get("compactCount") or 0) + 1
781
+ self.record["compactCount"] = compact_count
782
+ self.record["lastCompact"] = {
783
+ "operationId": active.get("operationId"),
784
+ "baselineCompactCount": active.get("baselineCompactCount"),
785
+ "targetCompactCount": active.get("targetCompactCount") or compact_count,
786
+ "startedAtMs": active.get("startedAtMs"),
787
+ "completedAtMs": completed_at_ms or now_ms(),
788
+ "contextBefore": active.get("contextBefore"),
789
+ "contextAfter": None,
790
+ "turnId": turn_id or active.get("turnId"),
791
+ "contextItemId": active.get("contextItemId"),
792
+ "nativeTurnStarted": True,
793
+ "nativeTurnSettled": True,
794
+ "nativeTurnStatus": "completed",
795
+ "status": "succeeded",
796
+ }
797
+ # Older carried tokens are invalid immediately after a native
798
+ # compaction; retain no synthetic newer context in the public record.
799
+ self.record["context"] = None
800
+ self._capture_compaction_context_locked(self.record["lastCompact"])
801
+ self.record["activeCompact"] = None
802
+ return True
803
+
804
+ def _observe_context_compaction_locked(
805
+ self, *, turn_id: str | None, item_id: str | None, completed_at_ms: int | None
806
+ ) -> None:
807
+ """Bind modern ``contextCompaction`` item evidence to compact state."""
808
+ active = self.record.get("activeCompact")
809
+ if isinstance(active, dict):
810
+ # Requested compact item is evidence only for the internal turn
811
+ # positively pinned by turn/started, never an unrelated item.
812
+ if (not isinstance(turn_id, str) or not turn_id
813
+ or active.get("turnId") != turn_id
814
+ or active.get("nativeTurnStarted") is not True
815
+ or not isinstance(item_id, str) or not item_id):
816
+ return
817
+ active["contextItemId"] = item_id
818
+ active["nativeItemCompletedAtMs"] = completed_at_ms or now_ms()
819
+ return
820
+ # Automatic/native compaction outside an explicit CCX compact request is
821
+ # still worth counting for context/history consumers. Do not double-count
822
+ # the deprecated thread/compacted notification for the same turn.
823
+ last = self.record.get("lastCompact")
824
+ if isinstance(last, dict) and turn_id and last.get("turnId") == turn_id:
825
+ return
826
+ compact_count = int(self.record.get("compactCount") or 0) + 1
827
+ self.record["compactCount"] = compact_count
828
+ self.record["lastCompact"] = {
829
+ "operationId": None,
830
+ "baselineCompactCount": compact_count - 1,
831
+ "targetCompactCount": compact_count,
832
+ "startedAtMs": None,
833
+ "completedAtMs": completed_at_ms or now_ms(),
834
+ "contextBefore": None,
835
+ "contextAfter": None,
836
+ "turnId": turn_id,
837
+ "contextItemId": item_id,
838
+ "status": "succeeded",
839
+ }
840
+
841
+ def _emit_compact_feed_locked(
842
+ self, method: str, turn_id: str, compact_id: str, *,
843
+ outcome: str | None = None, operation_id: str | None = None,
844
+ item_id: str | None = None, reason: str | None = None,
845
+ ) -> None:
846
+ if method == COMPACT_STARTED_METHOD:
847
+ if compact_id in self._compact_started.get(turn_id, set()):
848
+ return
849
+ elif method == COMPACT_ENDED_METHOD and compact_id in self._compact_ended:
850
+ return
851
+ params = {
852
+ "turnId": turn_id, "compactId": compact_id,
853
+ "operationId": operation_id, "itemId": item_id,
854
+ }
855
+ if outcome is not None:
856
+ params["outcome"] = outcome
857
+ if reason is not None:
858
+ params["reason"] = reason[:500]
859
+ wrote = self.evidence.record(
860
+ method, params, thread_id=self.record.get("threadId"),
861
+ )
862
+ if wrote is None:
863
+ return
864
+ if method == COMPACT_STARTED_METHOD:
865
+ self._compact_started.setdefault(turn_id, set()).add(compact_id)
866
+ else:
867
+ self._compact_ended.add(compact_id)
868
+ self.record["evidence"] = self.evidence.state()
869
+
870
+ def _close_unfinished_compacts_locked(
871
+ self, turn_id: str, *, outcome: str, operation_id: str | None,
872
+ reason: str | None,
873
+ ) -> None:
874
+ starts = self._compact_started.get(turn_id, set())
875
+ # A requested compact can have an admitted turn but no item started.
876
+ # Emit its own sparse start before writing the terminal.
877
+ if not starts and isinstance(operation_id, str) and operation_id:
878
+ self._emit_compact_feed_locked(
879
+ COMPACT_STARTED_METHOD, turn_id, f"operation:{operation_id}",
880
+ operation_id=operation_id,
881
+ )
882
+ starts = self._compact_started.get(turn_id, set())
883
+ for compact_id in sorted(starts):
884
+ item_id = None
885
+ prefix = f"{turn_id}:"
886
+ if compact_id.startswith(prefix):
887
+ item_id = compact_id[len(prefix):]
888
+ self._emit_compact_feed_locked(
889
+ COMPACT_ENDED_METHOD, turn_id, compact_id,
890
+ outcome=outcome, operation_id=operation_id,
891
+ item_id=item_id, reason=reason,
892
+ )
893
+
894
+ def _context_snapshot_locked(self) -> dict:
895
+ from ccx.claude.context import current_context, public_context
896
+ return public_context(
897
+ current_context(self.record, engine="codex", live=self.record.get("context")),
898
+ engine="codex", compacting=isinstance(self.record.get("activeCompact"), dict),
899
+ )
900
+
901
+ def _append_context_locked(self, value: dict, *, save: bool = True) -> None:
902
+ turn_id = ((self.record.get("lifecycle") or {}).get("activeTurnId")
903
+ or (self.record.get("lastTurn") or {}).get("turnId"))
904
+ self.evidence.record(
905
+ CONTEXT_CHANGED_METHOD,
906
+ {"context": value, "turnId": turn_id},
907
+ thread_id=(self.record.get("native") or {}).get("threadId"),
908
+ )
909
+ self.record["evidence"] = self.evidence.state()
910
+ if save:
911
+ self._save()
912
+
913
+ def _publish_context_locked(self, *, force: bool = False,
914
+ save: bool = True) -> None:
915
+ view = self._context_snapshot_locked()
916
+ previous = self._context_changes.last
917
+ fenced = (view["reason"] in {"compacting", "just_compacted"}
918
+ and (previous or {}).get("reason") != view["reason"])
919
+ active = (self.record.get("lifecycle") or {}).get("activeTurnId")
920
+ result = self._context_changes.observe(
921
+ view, force=bool(force or fenced or not active),
922
+ )
923
+ if result is not None:
924
+ self._append_context_locked(result, save=save)
925
+
926
+ def flush_context(self) -> None:
927
+ """Publish the last native usage sample on the existing janitor tick."""
928
+ with self.lock:
929
+ if self.teardown_claimed or self._externally_terminalized:
930
+ return
931
+ # The terminal notification itself flushes context synchronously.
932
+ # Never let the janitor publish an old queued mid-turn sample after
933
+ # turn/compact settlement; a read-only compact-status must stay pure.
934
+ if (not (self.record.get("lifecycle") or {}).get("activeTurnId")
935
+ or isinstance(self.record.get("activeCompact"), dict)):
936
+ self._context_changes.pending = None
937
+ return
938
+ value = self._context_changes.flush()
939
+ if value is not None:
940
+ self._append_context_locked(value)
941
+
942
+ def _publish_background_locked(self, transition: str, item: dict | None) -> None:
943
+ """Journal one actual native change; the caller holds worker.lock."""
944
+ snapshot = background_summary([
945
+ *self._bg_open.values(), *self._bg_notifications.values(),
946
+ ])
947
+ if item is None and not materially_changed(
948
+ self.record.get("backgroundTaskSummary"), snapshot
949
+ ):
950
+ return
951
+ self.record["backgroundTaskSummary"] = snapshot
952
+ self.evidence.record(
953
+ BACKGROUND_CHANGED_METHOD,
954
+ {"transition": transition, "item": item, "summary": snapshot},
955
+ thread_id=(self.record.get("native") or {}).get("threadId"),
956
+ )
957
+
958
+ def _lose_native_background_locked(self) -> bool:
959
+ """Drop old-runtime command guards; never assert a native exit.
960
+
961
+ Called under worker.lock when this app-server goes away or a new
962
+ owner adopts a recorded worker. A lost command is no longer tracked
963
+ (its outcome is unknown, so no ``finished`` item is published); the
964
+ correction keeps the registry-backed pending notifications.
965
+ """
966
+ previous = self.record.get("backgroundTaskSummary")
967
+ had_native = bool(self._bg_open or self._bg_candidates)
968
+ self._bg_open.clear()
969
+ self._bg_candidates.clear()
970
+ stale_running = (isinstance(previous, dict)
971
+ and type(previous.get("running_count")) is int
972
+ and previous["running_count"] > 0)
973
+ if not (had_native or stale_running):
974
+ return False
975
+ # Only native command tracking is lost. Pending parent notifications
976
+ # come from the durable registry, so they stay in the summary.
977
+ corrected = background_summary(list(self._bg_notifications.values()))
978
+ if not materially_changed(previous, corrected):
979
+ return False
980
+ self.record["backgroundTaskSummary"] = corrected
981
+ self.evidence.record(
982
+ BACKGROUND_CHANGED_METHOD,
983
+ {"transition": "event", "item": None, "summary": corrected},
984
+ thread_id=(self.record.get("native") or {}).get("threadId"),
985
+ )
986
+ self.record["evidence"] = self.evidence.state()
987
+ return True
988
+
989
+ def refresh_parent_notifications(self) -> bool:
990
+ """Read the current registry once; publish only a changed snapshot.
991
+
992
+ Only codexd owns this worker's evidence journal. Best-effort IPC hints
993
+ do not contain facts, and an absent owner never starts a daemon.
994
+ """
995
+ from ccx.notify.registry import iter_registrations, _delivery_id
996
+ rows = iter_registrations(include_terminal=True)
997
+ with self.lock:
998
+ if self.teardown_claimed or self._externally_terminalized:
999
+ return False
1000
+ updated = {}
1001
+ outcomes = {}
1002
+ for reg in rows:
1003
+ parent = reg.get("parent") or {}
1004
+ if (parent.get("kind") != "ccx-codex"
1005
+ or parent.get("workerId") != self.worker_id
1006
+ or parent.get("generation") != self.daemon._uuid_store.generation):
1007
+ continue
1008
+ event_id = reg.get("eventId")
1009
+ if not isinstance(event_id, str):
1010
+ continue
1011
+ delivery_id = _delivery_id(event_id)
1012
+ state = reg.get("state")
1013
+ if state in {"armed", "ready"}:
1014
+ updated[delivery_id] = {
1015
+ "id": delivery_id, "kind": "notification",
1016
+ "status": "pending", "label": "child result pending",
1017
+ "notification_id": event_id, "delivery_id": delivery_id,
1018
+ "started_at_ms": reg.get("createdAtMs"),
1019
+ }
1020
+ else:
1021
+ outcomes[delivery_id] = state
1022
+ previous = self._bg_notifications
1023
+ if previous == updated:
1024
+ return False
1025
+ self._bg_notifications = updated
1026
+ additions = [id for id in updated if id not in previous]
1027
+ removals = [id for id in previous if id not in updated]
1028
+ if len(additions) == 1 and not removals:
1029
+ transition, item = "started", updated[additions[0]]
1030
+ elif len(removals) == 1 and not additions:
1031
+ item = dict(previous[removals[0]])
1032
+ result = outcomes.get(removals[0])
1033
+ outcome = {
1034
+ "delivered": ("finished", "completed"),
1035
+ "cancelled": ("stopped", "stopped"),
1036
+ "undeliverable": ("failed", "failed"),
1037
+ }.get(result)
1038
+ if outcome is None:
1039
+ # Missing, unreadable or unknown terminal registry row
1040
+ # proves only a snapshot correction, not delivery.
1041
+ transition, item = "event", None
1042
+ else:
1043
+ transition, item["status"] = outcome
1044
+ else:
1045
+ transition, item = "event", None
1046
+ snapshot = background_summary([
1047
+ *self._bg_open.values(), *updated.values(),
1048
+ ])
1049
+ if not materially_changed(self.record.get("backgroundTaskSummary"), snapshot):
1050
+ return False
1051
+ self.record["backgroundTaskSummary"] = snapshot
1052
+ self.evidence.record(
1053
+ BACKGROUND_CHANGED_METHOD,
1054
+ {"transition": transition, "item": item, "summary": snapshot},
1055
+ thread_id=(self.record.get("native") or {}).get("threadId"),
1056
+ )
1057
+ self.record["evidence"] = self.evidence.state()
1058
+ self._save()
1059
+ return True
1060
+
1061
+ def _background_native_locked(
1062
+ self, method: str, params: dict, completed_turn: bool,
1063
+ ) -> None:
1064
+ native = self.record.get("native") or {}
1065
+ if params.get("threadId") != native.get("threadId"):
1066
+ return
1067
+ if method == "item/started":
1068
+ item = params.get("item") or {}
1069
+ tid = params.get("turnId")
1070
+ item_id = item.get("id") if isinstance(item, dict) else None
1071
+ if (item.get("type") == "commandExecution"
1072
+ and isinstance(item_id, str) and item_id
1073
+ and isinstance(tid, str) and tid
1074
+ and tid == self.record["lifecycle"].get("activeTurnId")):
1075
+ self._bg_candidates[item_id] = {
1076
+ "turn": tid, "id": "command:" + item_id,
1077
+ "kind": "command", "status": "running",
1078
+ # Deliberately no raw native command, paths or output.
1079
+ "label": "background shell", "tool_use_id": item_id,
1080
+ "started_at_ms": params.get("startedAtMs"),
1081
+ }
1082
+ elif method == "turn/completed" and completed_turn:
1083
+ turn_id = _turn_id(params)
1084
+ for item_id, item in list(self._bg_candidates.items()):
1085
+ if item["turn"] != turn_id:
1086
+ continue
1087
+ del self._bg_candidates[item_id]
1088
+ if item_id not in self._bg_open:
1089
+ self._bg_open[item_id] = item
1090
+ self._publish_background_locked("started", item)
1091
+ elif method == "item/completed":
1092
+ item = params.get("item") or {}
1093
+ item_id = item.get("id") if isinstance(item, dict) else None
1094
+ if not isinstance(item_id, str):
1095
+ return
1096
+ candidate = self._bg_candidates.get(item_id)
1097
+ if candidate is not None and candidate["turn"] == params.get("turnId"):
1098
+ del self._bg_candidates[item_id] # normal foreground command
1099
+ active = self._bg_open.get(item_id)
1100
+ if active is None or active["turn"] != params.get("turnId"):
1101
+ return
1102
+ exit_code = item.get("exitCode")
1103
+ if type(exit_code) is int:
1104
+ status = "completed" if exit_code == 0 else "failed"
1105
+ elif item.get("status") == "failed":
1106
+ status = "failed"
1107
+ else:
1108
+ status = "unknown"
1109
+ terminal = {key: value for key, value in active.items() if key != "turn"}
1110
+ terminal["status"] = status
1111
+ if type(exit_code) is int:
1112
+ terminal["exit_code"] = exit_code
1113
+ del self._bg_open[item_id]
1114
+ self._publish_background_locked(
1115
+ {"completed": "finished", "failed": "failed"}.get(status, "event"),
1116
+ terminal,
1117
+ )
1118
+
1119
+ def on_notification(self, method: str, params: dict[str, Any]) -> None:
1120
+ # Journal the authoritative native event first, while it is still the
1121
+ # thing the app-server sent, rather than after it has been folded into
1122
+ # lifecycle state, `lastTurn`, and `lastReport`. Nothing below reads
1123
+ # the journal, so a journal fault cannot change worker health or turn
1124
+ # outcome — it can only make the evidence explicitly degraded.
1125
+ if self.teardown_claimed:
1126
+ # Teardown owns/closed this worker's evidence journal.
1127
+ return
1128
+ # A separate terminal writer only revoked native *mutations*. Its
1129
+ # still-open journal can record late provider facts without resurrecting
1130
+ # this worker. This is forensic evidence, not outcome authorization.
1131
+ # A valid item/completed can arrive without item/started. Admit only
1132
+ # the live exact turn, then publish its missing sparse start before
1133
+ # journaling the native item. This is source ownership, not a reader
1134
+ # inference or a restatement of native item lines.
1135
+ if method == "item/completed":
1136
+ native_item = params.get("item")
1137
+ turn_id = params.get("turnId")
1138
+ item_id = native_item.get("id") if isinstance(native_item, dict) else None
1139
+ if (isinstance(native_item, dict)
1140
+ and native_item.get("type") == "contextCompaction"
1141
+ and isinstance(turn_id, str) and turn_id
1142
+ and isinstance(item_id, str) and item_id):
1143
+ with self.lock:
1144
+ if self.record["lifecycle"].get("activeTurnId") == turn_id:
1145
+ active = self.record.get("activeCompact")
1146
+ operation_id = (
1147
+ active.get("operationId")
1148
+ if isinstance(active, dict)
1149
+ and active.get("turnId") == turn_id else None
1150
+ )
1151
+ self._emit_compact_feed_locked(
1152
+ COMPACT_STARTED_METHOD, turn_id,
1153
+ f"{turn_id}:{item_id}", item_id=item_id,
1154
+ operation_id=operation_id,
1155
+ )
1156
+ journaled = self._journal(method, params)
1157
+ if method == "item/completed":
1158
+ # The completed user-message item is native's final statement of
1159
+ # that input; by then every copy of it has been journaled.
1160
+ from ccx.codex.delivery_recovery import ccx_client_id
1161
+ client_id = ccx_client_id(params.get("item"), self.worker_id)
1162
+ if client_id is not None:
1163
+ # Journaled above; resolved off this shared reader thread.
1164
+ self.daemon.queue_delivery_recovery(self, client_id)
1165
+ if self._externally_terminalized:
1166
+ return
1167
+ changed = False
1168
+ completed_turn = False
1169
+ with self.lock:
1170
+ if self.teardown_claimed or self._externally_terminalized:
1171
+ return
1172
+ started_id = _turn_id(params) if method == "turn/started" else None
1173
+ if started_id and (
1174
+ started_id in self.completed_turn_ids
1175
+ or started_id == (self.record.get("lastTurn") or {}).get("turnId")):
1176
+ # A re-delivered start of a turn already known completed (also
1177
+ # after a restart, from the durable evidence index) must not
1178
+ # reopen it: its duplicate completion is ignored, so the
1179
+ # worker would stay busy forever. Journaled above only.
1180
+ pass
1181
+ elif method == "turn/started":
1182
+ changed = True
1183
+ self._resumed_system_error_pending = False
1184
+ self.record["historyKnownEmpty"] = False
1185
+ active = self.record.get("activeCompact")
1186
+ if (isinstance(active, dict) and active.get("awaitingTurn")
1187
+ and not active.get("turnId") and started_id
1188
+ and started_id not in self.completed_turn_ids
1189
+ and not self.record.get("nativeTurnStart")
1190
+ and self.launch is None
1191
+ and not self.record["lifecycle"].get("activeTurnId")):
1192
+ active["turnId"] = started_id
1193
+ active["awaitingTurn"] = False
1194
+ active["nativeTurnStarted"] = True
1195
+ self._forget_native_turn_start_locked()
1196
+ self.record["lifecycle"]["state"] = "busy"
1197
+ self.record["lifecycle"]["activeTurnId"] = _turn_id(params)
1198
+ self.record["lifecycle"]["since"] = now_ms()
1199
+ elif method == "item/started":
1200
+ item = params.get("item")
1201
+ turn_id = params.get("turnId")
1202
+ item_id = item.get("id") if isinstance(item, dict) else None
1203
+ if (isinstance(item, dict)
1204
+ and item.get("type") == "contextCompaction"
1205
+ and isinstance(turn_id, str) and turn_id
1206
+ and isinstance(item_id, str) and item_id
1207
+ and self.record["lifecycle"].get("activeTurnId") == turn_id):
1208
+ active = self.record.get("activeCompact")
1209
+ operation_id = (
1210
+ active.get("operationId")
1211
+ if isinstance(active, dict)
1212
+ and active.get("turnId") == turn_id else None
1213
+ )
1214
+ # Only this producer writes neutral compaction starts.
1215
+ # Native item/started remains raw forensic evidence.
1216
+ self._emit_compact_feed_locked(
1217
+ COMPACT_STARTED_METHOD, turn_id,
1218
+ f"{turn_id}:{item_id}", operation_id=operation_id,
1219
+ item_id=item_id,
1220
+ )
1221
+ elif method == "item/completed":
1222
+ changed = True
1223
+ item = params.get("item") or {}
1224
+ if item.get("type") == "contextCompaction":
1225
+ item_turn, item_id = params.get("turnId"), item.get("id")
1226
+ if (isinstance(item_turn, str) and item_turn
1227
+ and isinstance(item_id, str) and item_id
1228
+ and self.record["lifecycle"].get("activeTurnId") == item_turn):
1229
+ self._compact_finished.setdefault(item_turn, set()).add(item_id)
1230
+ active = self.record.get("activeCompact")
1231
+ operation_id = (
1232
+ active.get("operationId")
1233
+ if isinstance(active, dict)
1234
+ and active.get("turnId") == item_turn else None
1235
+ )
1236
+ # Journaled after the original native item: exactly
1237
+ # one admitted sparse terminal for this compactId.
1238
+ self._emit_compact_feed_locked(
1239
+ COMPACT_ENDED_METHOD, item_turn,
1240
+ f"{item_turn}:{item_id}", outcome="completed",
1241
+ operation_id=operation_id, item_id=item_id,
1242
+ )
1243
+ self._observe_context_compaction_locked(
1244
+ turn_id=item_turn,
1245
+ item_id=item_id,
1246
+ completed_at_ms=params.get("completedAtMs"),
1247
+ )
1248
+ report = report_from_item(
1249
+ item,
1250
+ turn_id=params.get("turnId"),
1251
+ completed_at=params.get("completedAtMs"),
1252
+ )
1253
+ if report and should_replace_live_report(
1254
+ self.record.get("lastReport"), report
1255
+ ):
1256
+ self.record["lastReport"] = report
1257
+ elif method == "thread/tokenUsage/updated":
1258
+ changed = True
1259
+ usage = params.get("tokenUsage") or {}
1260
+ last = usage.get("last") or {}
1261
+ total = usage.get("total") or {}
1262
+ previous_usage = (
1263
+ self.record.get("tokenUsage")
1264
+ if isinstance(self.record.get("tokenUsage"), dict)
1265
+ else {}
1266
+ )
1267
+ previous_total = (
1268
+ previous_usage.get("total")
1269
+ if isinstance(previous_usage.get("total"), dict)
1270
+ else None
1271
+ )
1272
+ reset = (
1273
+ all(
1274
+ int(last.get(key) or 0) == 0
1275
+ for key in (
1276
+ "inputTokens",
1277
+ "cachedInputTokens",
1278
+ "outputTokens",
1279
+ "reasoningOutputTokens",
1280
+ )
1281
+ )
1282
+ and isinstance(previous_total, dict)
1283
+ and dict(total) == dict(previous_total)
1284
+ )
1285
+ tokens = last.get("inputTokens")
1286
+ window = usage.get("modelContextWindow")
1287
+ percent = (
1288
+ tokens / window * 100
1289
+ if (
1290
+ not reset
1291
+ and isinstance(tokens, int)
1292
+ and isinstance(window, int)
1293
+ and window > 0
1294
+ )
1295
+ else None
1296
+ )
1297
+ if reset:
1298
+ self.record["context"] = None
1299
+ else:
1300
+ self.record["context"] = {
1301
+ "state": self.record["lifecycle"]["state"],
1302
+ "tokens_used": tokens,
1303
+ "window_size": window,
1304
+ "percent_used": percent,
1305
+ "as_of_turn_ref": None,
1306
+ "as_of_ms": now_ms(),
1307
+ }
1308
+ self.record["tokenUsage"] = {
1309
+ "last": dict(last),
1310
+ "total": dict(total),
1311
+ "modelContextWindow": window,
1312
+ "asOfTurnId": params.get("turnId"),
1313
+ "asOfMs": now_ms(),
1314
+ }
1315
+
1316
+ last_compact = self.record.get("lastCompact")
1317
+ if (
1318
+ not reset
1319
+ and isinstance(last_compact, dict)
1320
+ and last_compact.get("status") == "succeeded"
1321
+ and last_compact.get("contextAfter") is None
1322
+ and last_compact.get("turnId") != params.get("turnId")
1323
+ and isinstance(self.record.get("context"), dict)
1324
+ ):
1325
+ self._capture_compaction_context_locked(last_compact)
1326
+ elif method == "thread/status/changed":
1327
+ changed = True
1328
+ status = params.get("status") or {}
1329
+ kind = status.get("type")
1330
+ flags = status.get("activeFlags") or []
1331
+ if kind in {"active", "idle"}:
1332
+ self._resumed_system_error_pending = False
1333
+ # The status flag can precede the approval request itself;
1334
+ # the request handler sets waiting once there is a gate.
1335
+ if kind == "active" and self.record.get("pendingApprovals") and any(
1336
+ flag in {"waitingOnApproval", "waitingOnUserInput"}
1337
+ for flag in flags
1338
+ ):
1339
+ self.record["lifecycle"]["state"] = "waiting"
1340
+ elif kind == "active":
1341
+ self.record["lifecycle"]["state"] = "busy"
1342
+ elif kind == "idle" and not self.record.get("pendingApprovals"):
1343
+ lifecycle = self.record["lifecycle"]
1344
+ if (
1345
+ lifecycle.get("activeTurnId")
1346
+ or self.launch is not None
1347
+ or isinstance(self.record.get("nativeTurnStart"), dict)
1348
+ or isinstance(self.record.get("activeCompact"), dict)
1349
+ ):
1350
+ # Codex publishes idle before the authoritative terminal
1351
+ # notification for both model turns and native compact.
1352
+ # Keep the operation unsettled so waiters cannot mistake
1353
+ # this status edge for turn/completion evidence.
1354
+ if lifecycle.get("state") != "crashed":
1355
+ lifecycle["state"] = "busy"
1356
+ else:
1357
+ lifecycle["state"] = "idle"
1358
+ lifecycle["activeTurnId"] = None
1359
+ elif kind == "systemError":
1360
+ if self._resumed_system_error_pending:
1361
+ # The resumed thread still carries the prior failed
1362
+ # turn's systemError. A replayed edge before any new
1363
+ # turn/started or active status cannot fail that turn.
1364
+ changed = False
1365
+ else:
1366
+ active_compact = self.record.get("activeCompact")
1367
+ compact_turn = (
1368
+ active_compact.get("turnId")
1369
+ if isinstance(active_compact, dict) else None
1370
+ )
1371
+ compact_operation = (
1372
+ active_compact.get("operationId")
1373
+ if isinstance(active_compact, dict) else None
1374
+ )
1375
+ compact_turn = compact_turn or self.record["lifecycle"].get("activeTurnId")
1376
+ if isinstance(compact_turn, str) and compact_turn:
1377
+ if compact_operation or self._compact_started.get(compact_turn):
1378
+ self._close_unfinished_compacts_locked(
1379
+ compact_turn, outcome="unknown",
1380
+ operation_id=compact_operation,
1381
+ reason="Codex thread system error",
1382
+ )
1383
+ self._abandon_active_compact_locked("Codex thread system error")
1384
+ # Best effort: name the native cause when it is
1385
+ # already known (Codex may also report it later, on
1386
+ # turn/completed).
1387
+ lifecycle = self.record["lifecycle"]
1388
+ native_error = None
1389
+ active_error = self.record.get("activeTurnError")
1390
+ last_turn = self.record.get("lastTurn")
1391
+ if lifecycle.get("activeTurnId"):
1392
+ if (isinstance(active_error, dict) and active_error.get("turnId")
1393
+ == lifecycle.get("activeTurnId")):
1394
+ native_error = active_error.get("error")
1395
+ elif isinstance(last_turn, dict) and last_turn.get("status") == "failed":
1396
+ native_error = last_turn.get("error")
1397
+ lifecycle["state"] = "crashed"
1398
+ lifecycle["lastError"] = _system_error_reason(native_error)
1399
+ elif method == "turn/completed":
1400
+ changed = True
1401
+ turn = params.get("turn") or {}
1402
+ if not isinstance(turn, dict):
1403
+ turn = {}
1404
+ completed_turn_id = turn.get("id")
1405
+ active_turn_id = self.record["lifecycle"].get("activeTurnId")
1406
+ last_turn_id = (self.record.get("lastTurn") or {}).get("turnId")
1407
+ if (not isinstance(completed_turn_id, str) or not completed_turn_id
1408
+ or completed_turn_id == last_turn_id
1409
+ or completed_turn_id in self.completed_turn_ids
1410
+ or active_turn_id != completed_turn_id):
1411
+ # A completion is a new outcome only for the exact native
1412
+ # turn already admitted as active by turn/started or its
1413
+ # original launch fence. In particular, idle does not prove
1414
+ # an unseen turn: a delayed A after B would roll lastTurn
1415
+ # backwards and count A again. The observed notification
1416
+ # is still journaled, including after a daemon restart.
1417
+ # A duplicate of the current pending save can retry its
1418
+ # original CAS without incrementing again.
1419
+ if journaled:
1420
+ self.record["evidence"] = self.evidence.state()
1421
+ self._save()
1422
+ return
1423
+ self.completed_turn_ids.add(completed_turn_id)
1424
+ if self.cap_turn_id == completed_turn_id:
1425
+ self.cap_turn_id = None
1426
+ self.active_caps = None
1427
+ completed_turn = True
1428
+ previous_state = self.record["lifecycle"].get("state")
1429
+ active_compact = self.record.get("activeCompact")
1430
+ requested_op = (
1431
+ active_compact.get("operationId")
1432
+ if isinstance(active_compact, dict)
1433
+ and active_compact.get("turnId") == completed_turn_id
1434
+ else None
1435
+ )
1436
+ native_compact_ended = self._compact_finished.get(completed_turn_id, set())
1437
+ if turn.get("status") in ("failed", "interrupted", "cancelled"):
1438
+ self._close_unfinished_compacts_locked(
1439
+ completed_turn_id, outcome="failed",
1440
+ operation_id=requested_op,
1441
+ reason=str(turn.get("error") or turn.get("status")),
1442
+ )
1443
+ elif (not native_compact_ended and
1444
+ (requested_op or self._compact_started.get(completed_turn_id))):
1445
+ # Completion of the native turn without the native
1446
+ # compaction item does not prove the compact itself.
1447
+ # Cover CCX-requested and automatic/native compactions.
1448
+ self._close_unfinished_compacts_locked(
1449
+ completed_turn_id, outcome="unknown",
1450
+ operation_id=requested_op,
1451
+ reason="native compact item completion not observed",
1452
+ )
1453
+ elif (completed_turn_id in self._compact_legacy
1454
+ and not self._compact_started.get(completed_turn_id)
1455
+ and not native_compact_ended
1456
+ and requested_op is None):
1457
+ # Legacy-only automatic compaction: defer until the turn's
1458
+ # accepted completion so modern item evidence wins.
1459
+ legacy_id = f"legacy:{completed_turn_id}"
1460
+ self._emit_compact_feed_locked(
1461
+ COMPACT_STARTED_METHOD, completed_turn_id, legacy_id,
1462
+ )
1463
+ self._emit_compact_feed_locked(
1464
+ COMPACT_ENDED_METHOD, completed_turn_id, legacy_id,
1465
+ outcome="completed",
1466
+ )
1467
+ if (
1468
+ isinstance(active_compact, dict)
1469
+ and active_compact.get("turnId") == completed_turn_id
1470
+ and active_compact.get("nativeTurnStarted") is True
1471
+ and isinstance(active_compact.get("contextItemId"), str)
1472
+ and bool(active_compact["contextItemId"])
1473
+ and active_compact.get("nativeItemCompletedAtMs") is not None
1474
+ and turn.get("status") == "completed"
1475
+ ):
1476
+ self._complete_active_compact_locked(
1477
+ turn_id=completed_turn_id,
1478
+ completed_at_ms=(
1479
+ int(turn.get("completedAt") * 1000)
1480
+ if isinstance(turn.get("completedAt"), (int, float))
1481
+ else None
1482
+ ),
1483
+ )
1484
+ elif (isinstance(active_compact, dict)
1485
+ and active_compact.get("turnId") == completed_turn_id):
1486
+ self._abandon_active_compact_locked(
1487
+ f"compact turn settled without success evidence ({turn.get('status')})"
1488
+ )
1489
+ self.record["lastCompact"].update(
1490
+ turnId=completed_turn_id, nativeTurnSettled=True,
1491
+ nativeTurnStatus=turn.get("status"), nativeTurnError=turn.get("error"),
1492
+ )
1493
+ last_compact = self.record.get("lastCompact")
1494
+ if (
1495
+ isinstance(last_compact, dict)
1496
+ and last_compact.get("turnId") == completed_turn_id
1497
+ and "contextAfter" in last_compact
1498
+ and last_compact.get("contextAfter") is None
1499
+ ):
1500
+ self._capture_compaction_context_locked(last_compact)
1501
+ current_turn_id = self.record["lifecycle"].get("activeTurnId")
1502
+ if current_turn_id and current_turn_id != completed_turn_id:
1503
+ # An old completion cannot clear or settle a newer turn.
1504
+ if changed:
1505
+ self._save()
1506
+ return
1507
+ self._forget_native_turn_start_locked()
1508
+ self.record["lifecycle"]["activeTurnId"] = None
1509
+ stats = self.record.setdefault("stats", {})
1510
+ stats["turns"] = stats.get("turns", 0) + 1
1511
+ # Native total is cumulative for the thread. Apply at most
1512
+ # its positive advance (not per-turn last), so the existing
1513
+ # stats counter-CAS path remains safe on a replay/rebase.
1514
+ usage = self.record.get("tokenUsage")
1515
+ total = usage.get("total") if isinstance(usage, dict) else None
1516
+ if isinstance(total, dict):
1517
+ for native_key, field in (("inputTokens", "inputTokens"),
1518
+ ("outputTokens", "outputTokens")):
1519
+ value = total.get(native_key)
1520
+ previous = stats.get(field)
1521
+ if (type(value) is int and value >= 0 and
1522
+ (previous is None or value >= previous)):
1523
+ stats[field] = value
1524
+ self.record["lastTurnAt"] = now_ms()
1525
+ self.record["lastTurn"] = _last_turn_from_native(turn)
1526
+ usage = self.record.get("tokenUsage")
1527
+ observed = self.record.get("context")
1528
+ if (isinstance(usage, dict)
1529
+ and usage.get("asOfTurnId") == completed_turn_id
1530
+ and isinstance(observed, dict)):
1531
+ self.record["lastTurn"]["context"] = {
1532
+ **observed,
1533
+ "as_of_turn_ref": "ccx-turn:codex:v1:" + completed_turn_id,
1534
+ }
1535
+ else:
1536
+ self.record["lastTurn"]["context"] = None
1537
+ active_error = self.record.get("activeTurnError")
1538
+ if (
1539
+ isinstance(active_error, dict)
1540
+ and active_error.get("turnId") == completed_turn_id
1541
+ ):
1542
+ self.record["activeTurnError"] = None
1543
+ if self.record.get("pendingApprovals"):
1544
+ self.record["lifecycle"]["state"] = "waiting"
1545
+ elif previous_state == "crashed":
1546
+ # A completed turn can arrive after the independent
1547
+ # thread/status systemError. Do not let its settlement
1548
+ # demote a worker whose runtime health is already known to
1549
+ # be bad.
1550
+ self.record["lifecycle"]["state"] = "crashed"
1551
+ # Best effort: name the native cause (a 429, a refused
1552
+ # compact) instead of only the generic status.
1553
+ if self.record["lifecycle"].get("lastError") == _system_error_reason(None):
1554
+ self.record["lifecycle"]["lastError"] = (
1555
+ _system_error_reason(turn.get("error"))
1556
+ )
1557
+ elif isinstance(self.record.get("activeCompact"), dict):
1558
+ # A foreign/native turn can settle while compaction still
1559
+ # owns its own operation fence. Do not publish idle until
1560
+ # thread/compacted (or an explicit compact failure) closes it.
1561
+ self.record["lifecycle"]["state"] = "busy"
1562
+ else:
1563
+ self.record["lifecycle"]["state"] = "idle"
1564
+ self.record["lifecycle"]["since"] = now_ms()
1565
+ elif method == "serverRequest/resolved":
1566
+ changed = True
1567
+ request_id = params.get("requestId")
1568
+ self.record["pendingApprovals"] = [
1569
+ approval
1570
+ for approval in self.record.get("pendingApprovals") or []
1571
+ if str(approval.get("requestId")) != str(request_id)
1572
+ ]
1573
+ if self.record["lifecycle"]["state"] == "waiting":
1574
+ self.record["lifecycle"]["state"] = (
1575
+ "waiting"
1576
+ if self.record["pendingApprovals"]
1577
+ else (
1578
+ "busy"
1579
+ if (
1580
+ self.record["lifecycle"].get("activeTurnId")
1581
+ or isinstance(self.record.get("nativeTurnStart"), dict)
1582
+ or isinstance(self.record.get("activeCompact"), dict)
1583
+ )
1584
+ else "idle"
1585
+ )
1586
+ )
1587
+ self.record["lifecycle"]["since"] = now_ms()
1588
+ elif method == "thread/compacted":
1589
+ changed = True
1590
+ # Legacy Codex signal. Modern releases expose compaction as a
1591
+ # contextCompaction item inside an internal turn. If both arrive,
1592
+ # the turn id dedupes the deprecated notification.
1593
+ legacy_turn_id = params.get("turnId")
1594
+ if isinstance(legacy_turn_id, str) and legacy_turn_id:
1595
+ self._compact_legacy.add(legacy_turn_id)
1596
+ active_compact = self.record.get("activeCompact")
1597
+ last = self.record.get("lastCompact") or {}
1598
+ if legacy_turn_id and last.get("turnId") == legacy_turn_id:
1599
+ return
1600
+ if isinstance(active_compact, dict):
1601
+ # Deprecated thread/compacted is not a modern item and
1602
+ # never settles the requested operation on its own.
1603
+ pass
1604
+ else:
1605
+ last = self.record.get("lastCompact")
1606
+ already_seen = (
1607
+ isinstance(last, dict)
1608
+ and legacy_turn_id
1609
+ and last.get("turnId") == legacy_turn_id
1610
+ )
1611
+ if not already_seen:
1612
+ compact_count = int(self.record.get("compactCount") or 0) + 1
1613
+ self.record["compactCount"] = compact_count
1614
+ self.record["lastCompact"] = {
1615
+ "operationId": None,
1616
+ "baselineCompactCount": compact_count - 1,
1617
+ "targetCompactCount": compact_count,
1618
+ "startedAtMs": None,
1619
+ "completedAtMs": now_ms(),
1620
+ "contextBefore": None,
1621
+ "contextAfter": None,
1622
+ "turnId": legacy_turn_id,
1623
+ "contextItemId": None,
1624
+ "status": "succeeded",
1625
+ }
1626
+ self._capture_compaction_context_locked(
1627
+ self.record["lastCompact"]
1628
+ )
1629
+ # Legacy item evidence is not settlement or an input lease.
1630
+ elif method == "error":
1631
+ changed = True
1632
+ # ``error`` is a turn-scoped diagnostic. It is not evidence of
1633
+ # worker/runtime failure: Codex may recover and retry the same
1634
+ # turn, and the authoritative settled error arrives in
1635
+ # turn/completed.turn.error. Keep the native structured value
1636
+ # intact, but only for the currently active turn.
1637
+ turn_id = params.get("turnId")
1638
+ active_turn_id = self.record["lifecycle"].get("activeTurnId")
1639
+ if (
1640
+ isinstance(turn_id, str)
1641
+ and turn_id
1642
+ and turn_id == active_turn_id
1643
+ ):
1644
+ self.record["activeTurnError"] = {
1645
+ "turnId": turn_id,
1646
+ "error": params.get("error"),
1647
+ "willRetry": bool(params.get("willRetry", False)),
1648
+ "observedAtMs": now_ms(),
1649
+ }
1650
+ if method in {"item/started", "item/completed", "turn/completed"}:
1651
+ self._background_native_locked(method, params, completed_turn)
1652
+ if not journaled and not changed:
1653
+ return
1654
+ if journaled:
1655
+ self.record["evidence"] = self.evidence.state()
1656
+ self.last_activity = time.monotonic()
1657
+ # Journal the context view before the same record save that
1658
+ # publishes a new native terminal/compact state. A later ordinary
1659
+ # read-only compact-status call must not race a second record write.
1660
+ self._publish_context_locked(force=completed_turn, save=False)
1661
+ self._save()
1662
+
1663
+ if completed_turn:
1664
+ # The native notification was admitted as this worker's exact
1665
+ # active turn, and its record has been persisted. It is already
1666
+ # the original completion evidence: do not hydrate thread/read.
1667
+ # A turn that finished before the caller persisted acceptance is
1668
+ # caught by the receipt writer's journal check or reconciliation.
1669
+ if turn.get("status") == "completed":
1670
+ from ccx.codex.completion import settle_completed
1671
+ try:
1672
+ settle_completed(
1673
+ self.daemon._uuid_store, f"codex:{self.worker_id}",
1674
+ thread_id=(self.record.get("native") or {}).get("threadId"),
1675
+ turn=turn,
1676
+ live_report=self.record.get("lastReport"),
1677
+ )
1678
+ except Exception as error:
1679
+ # A receipt-side fault cannot take down the shared native
1680
+ # notification reader or change the completed turn. The
1681
+ # original journal and recovery path remain available.
1682
+ print("ccx codexd: original completion settlement pending: "
1683
+ + type(error).__name__, file=sys.stderr, flush=True)
1684
+ from ccx.notify.daemon import revive_notify_daemon_after_turn
1685
+ revive_notify_daemon_after_turn("codex", self.worker_id)
1686
+
1687
+ def on_server_request(
1688
+ self, request_id: Any, method: str, params: dict[str, Any]
1689
+ ) -> None:
1690
+ kind = APPROVAL_METHODS.get(method)
1691
+ if not kind or (
1692
+ method == "item/fileChange/requestApproval"
1693
+ and params.get("grantRoot")
1694
+ ):
1695
+ self.client.reject_request(
1696
+ request_id,
1697
+ f"CCX does not broker app-server request {method}",
1698
+ )
1699
+ return
1700
+ preview = {
1701
+ "requestId": request_id,
1702
+ # App-server request ids restart with the app-server, so a stale
1703
+ # id could name a newer request; this one never repeats.
1704
+ "gateId": uuid.uuid4().hex,
1705
+ # Native request IDs are connection-scoped. An abruptly killed
1706
+ # daemon can leave old approvals on disk; they cannot be replayed.
1707
+ "runtimeId": self.daemon.instance_id,
1708
+ "kind": kind,
1709
+ "turnId": params.get("turnId"),
1710
+ "itemId": params.get("itemId"),
1711
+ "command": _bounded(params.get("command")),
1712
+ "cwd": _bounded(params.get("cwd"), 500),
1713
+ "reason": _bounded(params.get("reason")),
1714
+ "createdAt": now_ms(),
1715
+ }
1716
+ with self.lock:
1717
+ torn_down = (self.teardown_claimed or self._externally_terminalized
1718
+ or self.record["lifecycle"].get("state") in _TERMINAL_STATES)
1719
+ if not torn_down:
1720
+ self.record["pendingApprovals"].append(preview)
1721
+ self.record["lifecycle"]["state"] = "waiting"
1722
+ self.record["lifecycle"]["since"] = now_ms()
1723
+ self._save()
1724
+ if torn_down:
1725
+ self.client.reject_request(
1726
+ request_id, f"CCX worker {self.short} is being unloaded"
1727
+ )
1728
+
1729
+ def _select_approval(self, gate_id: str) -> dict[str, Any] | None:
1730
+ """Call under self.lock; only this runtime's exact CCX gateId may act."""
1731
+ if not isinstance(gate_id, str) or not gate_id or len(gate_id) > 128:
1732
+ return None
1733
+ for approval in self.record.get("pendingApprovals") or []:
1734
+ if (approval.get("gateId") == gate_id
1735
+ and approval.get("runtimeId") == self.daemon.instance_id):
1736
+ return approval
1737
+ return None
1738
+
1739
+ def kill(self, observe_seconds: float | None = None) -> dict[str, Any]: # noqa: D401
1740
+ """Terminate this worker and unload it from the shared daemon.
1741
+
1742
+ Codex has no per-worker process to signal: every loaded worker is a
1743
+ thread multiplexed on the one shared ``codex app-server`` child, and the
1744
+ daemon PID and app-server PID stamped into each record are device-wide,
1745
+ not worker-owned. Killing a worker therefore means unloading exactly this
1746
+ worker — never signalling either shared process, and never disturbing a
1747
+ sibling.
1748
+
1749
+ The active turn is cancelled best-effort first; a refusal or a transport
1750
+ failure there does not stop the unload, because the unload is what the
1751
+ caller asked for. The native ``threadId`` is preserved and the thread is
1752
+ deliberately not archived, so a later ``send``/``resume`` reloads the
1753
+ same conversation immediately.
1754
+ """
1755
+ # one critical section decides the race. `self.lock` totally orders this
1756
+ # against the loader's two commit points (the native turn launch
1757
+ # publication and `_commit_state`), so exactly one of us wins:
1758
+ # * we go first -> the loader sees `killed` and does nothing at all;
1759
+ # * loader first -> we see its `launch` here and own the cancellation.
1760
+ # It deliberately does not take `load_lock`: waiting there would stall
1761
+ # kill behind a 90s `thread/resume`.
1762
+ with self.lock:
1763
+ launch = self.launch
1764
+ unresolved_start = self.record.get("nativeTurnStart")
1765
+ if launch is None and isinstance(unresolved_start, dict):
1766
+ # The local JSON-RPC request has already settled without exact
1767
+ # native evidence, so the in-memory launch fence can no longer
1768
+ # hand us a turn id. The native request may still land later;
1769
+ # unloading now would falsely confirm a kill.
1770
+ self.kill_intent.clear()
1771
+ return {
1772
+ "ok": False,
1773
+ "pending": True,
1774
+ "error": (
1775
+ f"worker {self.short} has an indeterminate native "
1776
+ "turn/start; kill was NOT completed and is not confirmed "
1777
+ "— retry after turn evidence arrives or the Codex "
1778
+ "app-server is replaced"
1779
+ ),
1780
+ "state": self.record["lifecycle"]["state"],
1781
+ }
1782
+ self.kill_intent.set()
1783
+ self.killed.set()
1784
+ state = self.record["lifecycle"]["state"]
1785
+ turn_id = self.record["lifecycle"].get("activeTurnId")
1786
+ thread_id = (self.record.get("native") or {}).get("threadId") or self.record.get("threadId")
1787
+ active_compact = self.record.get("activeCompact")
1788
+ compact_turn = (
1789
+ active_compact.get("turnId")
1790
+ if isinstance(active_compact, dict) else None
1791
+ )
1792
+ compact_operation = (
1793
+ active_compact.get("operationId")
1794
+ if isinstance(active_compact, dict) else None
1795
+ )
1796
+ compact_turn = compact_turn or turn_id
1797
+ if isinstance(compact_turn, str) and compact_turn:
1798
+ if compact_operation or self._compact_started.get(compact_turn):
1799
+ self._close_unfinished_compacts_locked(
1800
+ compact_turn, outcome="unknown",
1801
+ operation_id=compact_operation,
1802
+ reason="worker killed during compaction",
1803
+ )
1804
+ self._abandon_active_compact_locked("worker killed during compaction")
1805
+
1806
+ interrupted = False
1807
+ launch_cancelled = None
1808
+ if launch is not None:
1809
+ # A `turn/start` was already irrevocably in flight when we won. It
1810
+ # is not cancellable until it returns a turn id, so wait for that —
1811
+ # bounded by the caller's own remaining budget, capped.
1812
+ window = (_LAUNCH_SETTLE_CAP_S if observe_seconds is None
1813
+ else max(0.0, min(float(observe_seconds),
1814
+ _LAUNCH_SETTLE_CAP_S)))
1815
+ if not launch.done.wait(window):
1816
+ # unresolved. The turn may be accepted the moment after we
1817
+ # answer, so reporting a confirmed kill here would be exactly
1818
+ # the lie this operation must not tell. Stay loaded and
1819
+ # nonterminal so the caller's reconciliation sees a target that
1820
+ # is still pending; the `killed` marker guarantees the loader
1821
+ # cancels as soon as the turn id becomes knowable and then
1822
+ # terminalizes and unloads. A later kill is idempotent.
1823
+ return {
1824
+ "ok": False,
1825
+ "pending": True,
1826
+ "error": (f"worker {self.short} has a turn launch still in "
1827
+ f"flight after {window:g}s; it will be cancelled "
1828
+ f"and unloaded when the request returns — retry "
1829
+ f"kill to confirm"),
1830
+ "state": self.state(),
1831
+ }
1832
+ if not isinstance(launch.turn_id, str) or not launch.turn_id:
1833
+ # A native start returned without an exact turn ID. The
1834
+ # original mutation may still land; no unload or success.
1835
+ return {"ok": False, "pending": True,
1836
+ "error": "original native turn/start is unresolved; kill not confirmed",
1837
+ "state": self.state()}
1838
+ launch_cancelled = self._interrupt_turn(launch.turn_id)
1839
+ interrupted = launch_cancelled
1840
+ elif state in {"busy", "waiting"} and turn_id and thread_id:
1841
+ interrupted = self._interrupt_turn(turn_id)
1842
+ self._state("dead")
1843
+ # Clears activeTurnId/pendingApprovals, removes this worker alone from
1844
+ # `workers` and `by_thread`, and releases its native thread without
1845
+ # waiting for Codex to answer.
1846
+ self.daemon._unload(self, terminal_state="dead")
1847
+ return {
1848
+ "ok": True,
1849
+ "killed": True,
1850
+ "unloaded": True,
1851
+ "interrupted": interrupted,
1852
+ # Only present when this kill raced a turn launch, so a caller can
1853
+ # tell "nothing was running" from "we cancelled what was starting".
1854
+ "launchCancelled": launch_cancelled,
1855
+ "state": "dead",
1856
+ "threadId": thread_id,
1857
+ }
1858
+
1859
+ def _original_notification_thread(self) -> str:
1860
+ """The one globally claimed parent thread for both read and wake."""
1861
+ native = self.record.get("native") or {}
1862
+ thread_id = native.get("threadId")
1863
+ if (not isinstance(thread_id, str) or not thread_id
1864
+ or native.get("threadSource") != self.daemon._uuid_store.codex_source(
1865
+ f"codex:{self.worker_id}"
1866
+ )):
1867
+ raise RuntimeError("original parent Codex native thread/source mismatch")
1868
+ self.daemon._uuid_store.authorize_native_action(
1869
+ f"codex:{self.worker_id}", "send",
1870
+ )
1871
+ return thread_id
1872
+
1873
+ def _parent_notification_match(
1874
+ self, delivery_id: str, *, legacy_event_id: str | None = None,
1875
+ name: str = "worker_notification",
1876
+ ) -> dict[str, Any] | None:
1877
+ with self.lock:
1878
+ fences = self.record.get("parentNotifyFences")
1879
+ fence = fences.get(delivery_id) if isinstance(fences, dict) else None
1880
+ return notification_delivery_match(
1881
+ self.client.request,
1882
+ self._original_notification_thread(),
1883
+ delivery_id,
1884
+ legacy_event_id=legacy_event_id,
1885
+ stop_at_turn_id=fence if isinstance(fence, str) else None,
1886
+ name=name,
1887
+ )
1888
+
1889
+ def find_parent_notification(
1890
+ self, delivery_id: str, payload: dict[str, Any], *,
1891
+ name: str = "worker_notification",
1892
+ ) -> dict[str, Any] | None:
1893
+ """Read-only: has this notification already landed in the thread?
1894
+
1895
+ Runs without `operation_lock` (a long parent history must not stall
1896
+ the parent's own sends and kills); `_start_parent_notification` then
1897
+ covers an attempt this daemon launched meanwhile.
1898
+ """
1899
+ legacy_event_id = (
1900
+ payload.get("eventId") if payload.get("deliveryId") is None else None
1901
+ )
1902
+ return self._parent_notification_match(
1903
+ delivery_id,
1904
+ legacy_event_id=str(legacy_event_id) if legacy_event_id else None,
1905
+ name=name,
1906
+ )
1907
+
1908
+ def _fence_parent_notification_locked(self, delivery_id: str) -> None:
1909
+ """Remember the last turn settled before a delivery's first launch.
1910
+
1911
+ The notification can only land in a later turn, so a retry's history
1912
+ walk stops there. Only the first launch sets it: a retry's baseline may
1913
+ already be newer than the turn an earlier ambiguous attempt joined.
1914
+ """
1915
+ fences = self.record.get("parentNotifyFences")
1916
+ if not isinstance(fences, dict):
1917
+ fences = self.record["parentNotifyFences"] = {}
1918
+ if delivery_id in fences:
1919
+ return
1920
+ last_turn = self.record.get("lastTurn")
1921
+ fences[delivery_id] = (
1922
+ last_turn.get("turnId") if isinstance(last_turn, dict) else None
1923
+ )
1924
+ for stale in list(fences)[:-_PARENT_NOTIFY_FENCES]:
1925
+ del fences[stale]
1926
+
1927
+ def _compact_refusal(self) -> dict[str, Any] | None:
1928
+ with self.lock:
1929
+ active = self.record.get("activeCompact")
1930
+ if not isinstance(active, dict):
1931
+ return None
1932
+ return {"ok": False, "state": self.record["lifecycle"]["state"],
1933
+ "error": "Compaction owns the input surface",
1934
+ "reason": "compaction-busy", "retryable": True,
1935
+ "definitive": True, "operationId": active.get("operationId")}
1936
+
1937
+ def _start_parent_notification(
1938
+ self,
1939
+ delivery_id: str,
1940
+ payload: dict[str, Any],
1941
+ *,
1942
+ reconcile: bool = True,
1943
+ history: tuple[dict[str, Any] | None] | None = None,
1944
+ namespace: str = "ccx",
1945
+ name: str = "worker_notification",
1946
+ ) -> dict[str, Any]:
1947
+ """Start-or-steer one machine continuation with kill-safe admission.
1948
+
1949
+ Unlike a normal send this intentionally accepts an idle or active parent
1950
+ turn. Codex's turn/start semantics start a new turn when idle and steer
1951
+ the tool output into the active turn when busy. The launch marker keeps
1952
+ kill-vs-turn/start ordering as strong as ordinary user turns.
1953
+ """
1954
+ if history is not None:
1955
+ # Walked by the caller before it took the operation lock.
1956
+ match = history[0]
1957
+ elif reconcile:
1958
+ match = self.find_parent_notification(
1959
+ delivery_id, payload, name=name,
1960
+ )
1961
+ else:
1962
+ match = None
1963
+ if match is None and delivery_id in self.launched_notifications:
1964
+ # An attempt that held the lock while the walk ran outside it.
1965
+ match = {"turnId": self.launched_notifications[delivery_id]}
1966
+ if match is not None:
1967
+ with self.lock:
1968
+ marker = self.record.get("nativeTurnStart")
1969
+ if (
1970
+ isinstance(marker, dict)
1971
+ and marker.get("kind") == (
1972
+ "parent-notification" if namespace == "ccx" else "machine-wake"
1973
+ )
1974
+ and marker.get("deliveryId") == delivery_id
1975
+ ):
1976
+ self._forget_native_turn_start_locked()
1977
+ self._save()
1978
+ return {
1979
+ "ok": True,
1980
+ "state": self.state(),
1981
+ "reconciled": True,
1982
+ "workerId": self.worker_id,
1983
+ "threadId": self._original_notification_thread(),
1984
+ "eventId": payload.get("eventId"),
1985
+ "deliveryId": delivery_id,
1986
+ "turnId": match.get("turnId"),
1987
+ }
1988
+ # A previous attempt can still land after the network reply is lost.
1989
+ # Reconcile-only retries may ACK positive history, never retransmit
1990
+ # merely because the history endpoint finds no matching item.
1991
+ if reconcile:
1992
+ return {"ok": False, "pending": True, "state": self.state(),
1993
+ "error": "original machine wake was attempted; native placement remains unknown"}
1994
+ refusal = self._compact_refusal()
1995
+ if refusal:
1996
+ return refusal
1997
+ params = {
1998
+ "threadId": self._original_notification_thread(),
1999
+ "input": [],
2000
+ "turnTrigger": "ccx-worker-notification",
2001
+ "toolOutput": {
2002
+ "namespace": namespace,
2003
+ "name": name,
2004
+ "output": json.dumps(payload, separators=(",", ":")),
2005
+ },
2006
+ }
2007
+ with self.lock:
2008
+ if self.killed.is_set() or self.kill_intent.is_set():
2009
+ return {
2010
+ "ok": False,
2011
+ **({"definitive": True} if namespace != "ccx" else {}),
2012
+ "error": f"worker {self.short} is being killed",
2013
+ "state": self.record["lifecycle"]["state"],
2014
+ }
2015
+ if (not self.loaded or self.record["lifecycle"]["state"]
2016
+ not in {"idle", "busy", "waiting"}):
2017
+ return {"ok": False, "definitive": True,
2018
+ "error": "original native parent not subscribed/active",
2019
+ "state": self.record["lifecycle"]["state"]}
2020
+ if self.launch is not None:
2021
+ return {"ok": False, "definitive": True,
2022
+ "error": "another original native turn/start is in flight"}
2023
+ if isinstance(self.record.get("nativeTurnStart"), dict):
2024
+ return {"ok": False, "pending": True,
2025
+ "error": "another original native turn/start remains unresolved"}
2026
+ previous_state = self.record["lifecycle"]["state"]
2027
+ previous_turn_id = self.record["lifecycle"].get("activeTurnId")
2028
+ launch = _TurnLaunch()
2029
+ self.launch = launch
2030
+ # A turn blocked on an approval stays `waiting`: the notification
2031
+ # is steered into that same turn, and publishing `busy` would hide
2032
+ # the pending approval from every waiter.
2033
+ if previous_state != "waiting":
2034
+ self.record["lifecycle"]["state"] = "busy"
2035
+ self.record["lifecycle"]["since"] = now_ms()
2036
+ self.record["nativeTurnStart"] = self._native_turn_start_marker(
2037
+ kind="parent-notification" if namespace == "ccx" else "machine-wake",
2038
+ delivery_id=delivery_id,
2039
+ )
2040
+ self._fence_parent_notification_locked(delivery_id)
2041
+ self.last_activity = time.monotonic()
2042
+ self._save()
2043
+ turn_id = None
2044
+ request_returned = False
2045
+ try:
2046
+ result = self.client.request("turn/start", params, timeout=60)
2047
+ turn_id = _turn_id(result)
2048
+ request_returned = True
2049
+ except ProtocolError as error:
2050
+ with self.lock:
2051
+ if self._native_mutation_error_is_ambiguous(error):
2052
+ self._mark_native_turn_start_unknown_locked(error)
2053
+ else:
2054
+ self._clear_native_turn_start_locked()
2055
+ # Only an explicit native refusal proves the notification
2056
+ # did not mutate the thread, so only then restore what we
2057
+ # owned before the request.
2058
+ if (
2059
+ self.record["lifecycle"].get("activeTurnId")
2060
+ == previous_turn_id
2061
+ ):
2062
+ self.record["lifecycle"]["state"] = previous_state
2063
+ self.record["lifecycle"]["since"] = now_ms()
2064
+ self._save()
2065
+ raise
2066
+ except Exception as error:
2067
+ with self.lock:
2068
+ self._mark_native_turn_start_unknown_locked(error)
2069
+ raise
2070
+ finally:
2071
+ with self.lock:
2072
+ if self.launch is launch:
2073
+ self.launch = None
2074
+ launch.turn_id = turn_id
2075
+ if turn_id is not None:
2076
+ self._forget_native_turn_start_locked()
2077
+ if request_returned and isinstance(turn_id, str) and turn_id:
2078
+ self.launched_notifications[delivery_id] = turn_id
2079
+ lifecycle = self.record["lifecycle"]
2080
+ if request_returned and lifecycle["state"] in {"busy", "waiting"}:
2081
+ lifecycle["activeTurnId"] = (
2082
+ lifecycle.get("activeTurnId") or turn_id
2083
+ )
2084
+ if self.record.get("pendingApprovals"):
2085
+ # An approval that arrived while the request was in
2086
+ # flight still gates the turn.
2087
+ lifecycle["state"] = "waiting"
2088
+ self._save()
2089
+ launch.done.set()
2090
+ if self.killed.is_set():
2091
+ self._interrupt_turn(turn_id)
2092
+ return {
2093
+ "ok": False,
2094
+ "error": f"worker {self.short} was killed while starting notification",
2095
+ "state": self.state(),
2096
+ }
2097
+ if not isinstance(turn_id, str) or not turn_id:
2098
+ return {"ok": False, "pending": True, "state": self.state(),
2099
+ "error": "original machine wake has no positive native turn ID"}
2100
+ return {
2101
+ "ok": True,
2102
+ "state": self.state(),
2103
+ "turnId": turn_id,
2104
+ "reconciled": False,
2105
+ "workerId": self.worker_id,
2106
+ "threadId": self._original_notification_thread(),
2107
+ "eventId": payload.get("eventId"),
2108
+ "deliveryId": delivery_id,
2109
+ }
2110
+
2111
+ def handle_operation(
2112
+ self,
2113
+ message: dict[str, Any],
2114
+ *,
2115
+ history: tuple[dict[str, Any] | None] | None = None,
2116
+ ) -> dict[str, Any]:
2117
+ op = message.get("op")
2118
+ with self.lock:
2119
+ state = self.record["lifecycle"]["state"]
2120
+ if op == "kill":
2121
+ # Ahead of the booting/resuming guard on purpose: a worker stuck
2122
+ # mid-load is exactly one a caller needs to be able to kill.
2123
+ # `CodexDaemon.handle` routes kills through `_kill_worker` (which
2124
+ # owns the intent/budget protocol); this branch keeps a direct
2125
+ # caller correct.
2126
+ return self.kill(message.get("observeSeconds"))
2127
+ if self.killed.is_set() or self.kill_intent.is_set():
2128
+ # A kill was accepted but could not be confirmed yet (an unresolved
2129
+ # turn launch, or a prior operation that has not settled). The
2130
+ # worker stays registered so reconciliation can see it, but it must
2131
+ # accept no new work. `handle` refuses before taking the operation
2132
+ # lock; this repeats the check for an op that passed that gate and
2133
+ # only then found the intent published.
2134
+ return {"ok": False, "definitive": True,
2135
+ "error": f"worker {self.short} is being killed", "state": state}
2136
+ if state in {"booting", "resuming"}:
2137
+ return {"ok": False, "error": f"worker is {state}", "state": state,
2138
+ **({"definitive": True} if op == "machine-wake" else {})}
2139
+
2140
+ if op in {"parent-notify", "machine-wake"}:
2141
+ event_id = message.get("eventId")
2142
+ delivery_id = message.get("deliveryId") or event_id
2143
+ payload = message.get("payload")
2144
+ if not isinstance(event_id, str) or not event_id:
2145
+ return {"ok": False, "error": "parent-notify requires eventId", "state": state}
2146
+ if not isinstance(delivery_id, str) or not delivery_id:
2147
+ return {"ok": False, "error": "parent-notify requires deliveryId", "state": state}
2148
+ if not isinstance(payload, dict):
2149
+ return {"ok": False, "error": "parent-notify requires payload", "state": state}
2150
+ try:
2151
+ return self._start_parent_notification(
2152
+ delivery_id, payload,
2153
+ reconcile=bool(message.get("reconcile", True)),
2154
+ history=history,
2155
+ namespace=(message["namespace"] if op == "machine-wake" else "ccx"),
2156
+ name=(message["name"] if op == "machine-wake" else "worker_notification"),
2157
+ )
2158
+ except ProtocolError as error:
2159
+ failure = _protocol_failure(error)
2160
+ failure["definitive"] = not self._native_mutation_error_is_ambiguous(error)
2161
+ return failure
2162
+
2163
+ if op == "interrupt":
2164
+ # The notification reader publishes a successor turn under the
2165
+ # worker lock, so read state, turn and thread together once and
2166
+ # interrupt exactly the turn that was checked.
2167
+ with self.lock:
2168
+ state = self.record["lifecycle"]["state"]
2169
+ turn_id = self.record["lifecycle"].get("activeTurnId")
2170
+ # Full-UUID records keep the original thread under native.threadId.
2171
+ native = self.record.get("native") or {}
2172
+ thread_id = native.get("threadId") or self.record.get("threadId")
2173
+ expected_turn_id = message.get("expectedTurnId")
2174
+ if expected_turn_id is not None and expected_turn_id != turn_id:
2175
+ return {
2176
+ "ok": False,
2177
+ "error": "active turn changed",
2178
+ "state": state,
2179
+ "turnId": turn_id,
2180
+ "definitive": True,
2181
+ }
2182
+ if state not in {"busy", "waiting"}:
2183
+ return {"ok": False, "definitive": True,
2184
+ "error": "no active turn", "state": state}
2185
+ if not turn_id:
2186
+ return {"ok": False, "definitive": True,
2187
+ "error": "active turn id is unavailable", "state": state}
2188
+ self.client.request("turn/interrupt", {
2189
+ "threadId": thread_id, "turnId": turn_id,
2190
+ })
2191
+ return {"ok": True, "state": state, "turnId": turn_id}
2192
+ if op in {"approve", "reject"}:
2193
+ # Existing operation_lock serializes decisions and unload. The
2194
+ # reader can still publish/resolve gates: select under worker lock,
2195
+ # then release it before the native server-request response.
2196
+ with self.lock:
2197
+ approval = self._select_approval(message.get("gateId"))
2198
+ gate_state = self.record["lifecycle"]["state"]
2199
+ native = self.record.get("native") or {}
2200
+ turn = self.record["lifecycle"].get("activeTurnId")
2201
+ thread = native.get("threadId")
2202
+ if (approval is None or gate_state != "waiting"
2203
+ or not isinstance(turn, str) or not turn
2204
+ or approval.get("turnId") != turn
2205
+ or not isinstance(thread, str) or not thread
2206
+ or native.get("threadSource") != self.daemon._uuid_store.codex_source(
2207
+ f"codex:{self.worker_id}"
2208
+ )
2209
+ or self.daemon.by_thread.get(thread) != self.worker_id):
2210
+ return {"ok": False, "definitive": True,
2211
+ "error": "no current original Codex approval for gateId",
2212
+ "state": gate_state}
2213
+ self.approval_decisions[_approval_key(approval)] = (
2214
+ "accept" if op == "approve" else "decline"
2215
+ )
2216
+ if isinstance(message.get("responseId"), str):
2217
+ self.approval_response_ids[_approval_key(approval)] = (
2218
+ message["responseId"]
2219
+ )
2220
+ self.client.respond(
2221
+ approval["requestId"],
2222
+ {"decision": "accept" if op == "approve" else "decline"},
2223
+ )
2224
+ with self.lock:
2225
+ self.record["pendingApprovals"] = [
2226
+ value for value in self.record["pendingApprovals"]
2227
+ if value is not approval
2228
+ ]
2229
+ # Same resolution as the serverRequest/resolved notification:
2230
+ # without an active turn the worker is idle, not busy — forcing
2231
+ # busy here would wedge a worker whose turn already completed.
2232
+ self.record["lifecycle"]["state"] = (
2233
+ "waiting"
2234
+ if self.record["pendingApprovals"]
2235
+ else (
2236
+ "busy"
2237
+ if self.record["lifecycle"].get("activeTurnId")
2238
+ else "idle"
2239
+ )
2240
+ )
2241
+ self.record["lifecycle"]["since"] = now_ms()
2242
+ self._save()
2243
+ return {
2244
+ "ok": True,
2245
+ "state": self.record["lifecycle"]["state"],
2246
+ "requestId": approval["requestId"],
2247
+ "gateId": approval.get("gateId"),
2248
+ "workerId": self.worker_id,
2249
+ "threadId": thread,
2250
+ "turnId": approval.get("turnId"),
2251
+ "decision": "accept" if op == "approve" else "decline",
2252
+ }
2253
+ if op == "compact":
2254
+ from ccx.codex.ops import compact_intent_digest
2255
+ from ccx.store.store import _digest
2256
+
2257
+ operation_id = message.get("operationId")
2258
+ if (not isinstance(operation_id, str) or not operation_id
2259
+ or len(operation_id) > 128 or "\x00" in operation_id):
2260
+ return {"ok": False, "definitive": True,
2261
+ "error": "compact requires caller operationId; no native request"}
2262
+ ctx = self.daemon._uuid_store
2263
+ try:
2264
+ owner = ctx.resolve(f"codex:{self.worker_id}")
2265
+ original = ctx.record(owner.worker)
2266
+ native = original.get("native") or {}
2267
+ source = ctx.codex_source(owner.worker)
2268
+ thread_id = native.get("threadId")
2269
+ activation = ctx.native_activation(owner.worker)
2270
+ if (owner.engine != "codex"
2271
+ or activation.get("state") != "confirmed"
2272
+ or activation.get("source") != source
2273
+ or not isinstance(thread_id, str) or not thread_id
2274
+ or native.get("threadSource") != source
2275
+ or (self.record.get("native") or {}).get("threadId") != thread_id
2276
+ or self.daemon.by_thread.get(thread_id) != self.worker_id
2277
+ or message.get("threadId") != thread_id):
2278
+ raise ValueError("original Codex native thread/source unverified")
2279
+ effect = ctx.read_receipt(owner.worker, "codex-compact", operation_id)
2280
+ digest = compact_intent_digest(
2281
+ owner.worker_id, ctx.generation, thread_id, source, operation_id,
2282
+ )
2283
+ if (not isinstance(effect, dict)
2284
+ or effect.get("intentSha256") != digest
2285
+ or effect.get("workerId") != owner.worker_id
2286
+ or effect.get("engine") != "codex"
2287
+ or effect.get("generation") != ctx.generation
2288
+ or effect.get("family") != "codex-compact"
2289
+ or effect.get("action") != "compact"
2290
+ or effect.get("state") not in (
2291
+ "submitted-unknown", "accepted", "rejected"
2292
+ )):
2293
+ raise ValueError("original full-UUID submitted compact receipt absent")
2294
+ ctx.revalidate()
2295
+ except Exception as error:
2296
+ return {"ok": False, "definitive": True,
2297
+ "error": f"Codex compact original admission denied: {error}"}
2298
+
2299
+ identity = {
2300
+ "workerId": owner.worker_id, "engine": "codex",
2301
+ "generation": ctx.generation, "nativeSource": source,
2302
+ "threadId": thread_id, "operationId": operation_id,
2303
+ }
2304
+
2305
+ def acknowledged(started_at: int, *, accepted: bool,
2306
+ reason: str | None = None, replayed: bool = False) -> dict:
2307
+ data = {
2308
+ **identity, "startedAtMs": started_at,
2309
+ "nativeEvidenceSha256": _digest({
2310
+ **identity, "startedAtMs": started_at,
2311
+ "admission": "thread/compact/start",
2312
+ }),
2313
+ }
2314
+ if accepted:
2315
+ return {**data, "ok": True, "nativeStatus": "accepted",
2316
+ "replayed": replayed, "state": self.record["lifecycle"]["state"]}
2317
+ return {**data, "ok": False, "definitive": True, "noEffect": True,
2318
+ "error": reason or "compact refused before native request",
2319
+ "state": self.record["lifecycle"]["state"]}
2320
+
2321
+ # Same-ID replay must precede the idle gate: a live compact owns
2322
+ # busy and may have outlived a lost caller ACK. Never paste twice.
2323
+ with self.lock:
2324
+ active = self.record.get("activeCompact")
2325
+ last = self.record.get("lastCompact")
2326
+ match = next((
2327
+ marker for marker in (active, last)
2328
+ if isinstance(marker, dict)
2329
+ and marker.get("operationId") == operation_id
2330
+ ), None)
2331
+ if match is not None:
2332
+ if effect["state"] == "rejected":
2333
+ return {"ok": False, "error": "rejected compact has a marker",
2334
+ "reconcileRequired": True}
2335
+ started_at = match.get("startedAtMs")
2336
+ if type(started_at) is not int:
2337
+ return {"ok": False, "reconcileRequired": True,
2338
+ "error": "existing compact marker has no original start"}
2339
+ return acknowledged(started_at, accepted=True, replayed=True)
2340
+ if effect["state"] != "submitted-unknown":
2341
+ return acknowledged(now_ms(), accepted=False,
2342
+ reason="compact effect already settled; no resubmit")
2343
+ state = self.record["lifecycle"]["state"]
2344
+ if (state != "idle" or isinstance(active, dict)
2345
+ or self.launch is not None
2346
+ or self.record.get("pendingApprovals")):
2347
+ return acknowledged(now_ms(), accepted=False,
2348
+ reason=f"compact requires idle worker, state={state}")
2349
+ before = self.record.get("context")
2350
+ if isinstance(before, dict):
2351
+ before = {**before, "state": state}
2352
+ baseline = int(self.record.get("compactCount") or 0)
2353
+ started_at = now_ms()
2354
+ self.record["activeCompact"] = {
2355
+ "operationId": operation_id,
2356
+ "awaitingTurn": True,
2357
+ "daemonInstanceId": self._native_create_daemon_id(),
2358
+ "baselineCompactCount": baseline,
2359
+ "targetCompactCount": baseline + 1,
2360
+ "startedAtMs": started_at,
2361
+ "contextBefore": before,
2362
+ }
2363
+ self.record["lifecycle"]["state"] = "busy"
2364
+ self.record["lifecycle"]["since"] = started_at
2365
+ # Native submission is impossible until its exact durable
2366
+ # marker and earlier global submitted intent both exist.
2367
+ self._publish_context_locked(force=True, save=False)
2368
+ self._save()
2369
+ try:
2370
+ self.client.request(
2371
+ "thread/compact/start", {"threadId": thread_id}, timeout=1.5,
2372
+ )
2373
+ except ProtocolError as error:
2374
+ definite = isinstance(error, ProtocolNotSent) or (
2375
+ error.request_id is not None
2376
+ and not isinstance(error, ProtocolTransportError)
2377
+ )
2378
+ if definite:
2379
+ with self.lock:
2380
+ active = self.record.get("activeCompact")
2381
+ if (isinstance(active, dict)
2382
+ and active.get("operationId") == operation_id):
2383
+ self.record["activeCompact"] = None
2384
+ self.record["lifecycle"]["state"] = "idle"
2385
+ self.record["lifecycle"]["since"] = now_ms()
2386
+ self._publish_context_locked(force=True, save=False)
2387
+ self._save()
2388
+ return acknowledged(
2389
+ started_at, accepted=False, reason=str(error),
2390
+ )
2391
+ # Transport ambiguity: persisted activeCompact stays busy,
2392
+ # fenced by this exact global receipt for same-ID inspection.
2393
+ return {**identity, **_protocol_failure(error),
2394
+ "state": "busy", "reconcileRequired": True}
2395
+ # Native admission ACK is not contextCompaction completion.
2396
+ return acknowledged(started_at, accepted=True)
2397
+ return {"ok": False, "error": f"unknown worker operation {op!r}"}
2398
+
2399
+ def public_status(self) -> dict[str, Any]:
2400
+ with self.lock:
2401
+ lifecycle = dict(self.record["lifecycle"])
2402
+ context = self.record.get("context")
2403
+ if context:
2404
+ context = {**context, "state": lifecycle.get("state")}
2405
+ return {
2406
+ "short": self.short,
2407
+ "threadId": (self.record.get("native") or {}).get("threadId"),
2408
+ "name": self.record.get("name"),
2409
+ "cwd": self.record.get("cwd"),
2410
+ "state": lifecycle.get("state"),
2411
+ "activeTurnId": lifecycle.get("activeTurnId"),
2412
+ "compacting": bool(self.record.get("activeCompact")),
2413
+ "pendingApprovals": list(self.record.get("pendingApprovals") or []),
2414
+ "lastReport": self.record.get("lastReport"),
2415
+ "lastTurn": self.record.get("lastTurn"),
2416
+ "context": context,
2417
+ "lastError": lifecycle.get("lastError"),
2418
+ "activeTurnError": self.record.get("activeTurnError"),
2419
+ # Evidence health travels beside worker health, never as part
2420
+ # of it: a worker whose journal degraded is still idle. Null
2421
+ # for a worker that has produced no evidence at all.
2422
+ "evidence": self.record.get("evidence"),
2423
+ "persistenceError": self.persistence_error,
2424
+ }