ccx-native 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +241 -4
  3. package/bin/ccx +55 -2
  4. package/ccx/__init__.py +163 -0
  5. package/ccx/__main__.py +28 -0
  6. package/ccx/claude/__init__.py +0 -0
  7. package/ccx/claude/analytics.py +466 -0
  8. package/ccx/claude/attach.py +762 -0
  9. package/ccx/claude/context.py +465 -0
  10. package/ccx/claude/delivery.py +806 -0
  11. package/ccx/claude/external.py +253 -0
  12. package/ccx/claude/host/__init__.py +0 -0
  13. package/ccx/claude/host/events.py +297 -0
  14. package/ccx/claude/host/hooks.py +438 -0
  15. package/ccx/claude/host/pty.py +250 -0
  16. package/ccx/claude/host/session.py +5184 -0
  17. package/ccx/claude/host/subagents.py +169 -0
  18. package/ccx/claude/host/successor.py +267 -0
  19. package/ccx/claude/managed_turns.py +240 -0
  20. package/ccx/claude/oauth.py +176 -0
  21. package/ccx/claude/settings.py +339 -0
  22. package/ccx/claude/spawn.py +557 -0
  23. package/ccx/claude/tasks.py +72 -0
  24. package/ccx/claude/telemetry.py +282 -0
  25. package/ccx/claude/transcript.py +2281 -0
  26. package/ccx/claude/usage.py +837 -0
  27. package/ccx/claude/usage_control.py +152 -0
  28. package/ccx/cli/__init__.py +0 -0
  29. package/ccx/cli/codex.py +1009 -0
  30. package/ccx/cli/commands/__init__.py +0 -0
  31. package/ccx/cli/commands/_common.py +407 -0
  32. package/ccx/cli/commands/control.py +161 -0
  33. package/ccx/cli/commands/create.py +669 -0
  34. package/ccx/cli/commands/read.py +1020 -0
  35. package/ccx/cli/commands/turns.py +1081 -0
  36. package/ccx/cli/controller.py +757 -0
  37. package/ccx/cli/main.py +301 -0
  38. package/ccx/cli/parser.py +1324 -0
  39. package/ccx/cli/skills.py +215 -0
  40. package/ccx/cli/statusline.py +160 -0
  41. package/ccx/client.py +1900 -0
  42. package/ccx/codex/__init__.py +0 -0
  43. package/ccx/codex/attach_tty.py +267 -0
  44. package/ccx/codex/client.py +732 -0
  45. package/ccx/codex/completion.py +121 -0
  46. package/ccx/codex/control.py +288 -0
  47. package/ccx/codex/daemon/__init__.py +0 -0
  48. package/ccx/codex/daemon/__main__.py +4 -0
  49. package/ccx/codex/daemon/daemon.py +1556 -0
  50. package/ccx/codex/daemon/models.py +140 -0
  51. package/ccx/codex/daemon/native.py +1590 -0
  52. package/ccx/codex/daemon/release.py +367 -0
  53. package/ccx/codex/daemon/worker.py +2396 -0
  54. package/ccx/codex/delivery.py +758 -0
  55. package/ccx/codex/delivery_recovery.py +232 -0
  56. package/ccx/codex/evidence.py +827 -0
  57. package/ccx/codex/history.py +158 -0
  58. package/ccx/codex/notify_history.py +140 -0
  59. package/ccx/codex/ops.py +666 -0
  60. package/ccx/codex/paths.py +70 -0
  61. package/ccx/codex/protocol.py +805 -0
  62. package/ccx/codex/provenance.py +105 -0
  63. package/ccx/codex/read.py +29 -0
  64. package/ccx/codex/records.py +51 -0
  65. package/ccx/codex/reports.py +177 -0
  66. package/ccx/codex/runtime.py +1412 -0
  67. package/ccx/codex/settle.py +363 -0
  68. package/ccx/codex/telemetry.py +792 -0
  69. package/ccx/codex/types.py +659 -0
  70. package/ccx/codex/worker_environment.py +25 -0
  71. package/ccx/constants.py +288 -0
  72. package/ccx/controllers/__init__.py +0 -0
  73. package/ccx/controllers/controller.py +1449 -0
  74. package/ccx/controllers/live_vars.py +1706 -0
  75. package/ccx/controllers/live_vars_runner.py +216 -0
  76. package/ccx/controllers/notify.py +213 -0
  77. package/ccx/controllers/orchestrator.py +193 -0
  78. package/ccx/controllers/policy.py +57 -0
  79. package/ccx/controllers/store.py +681 -0
  80. package/ccx/errors.py +634 -0
  81. package/ccx/notify/__init__.py +0 -0
  82. package/ccx/notify/__main__.py +5 -0
  83. package/ccx/notify/client.py +116 -0
  84. package/ccx/notify/daemon.py +480 -0
  85. package/ccx/notify/deliver.py +304 -0
  86. package/ccx/notify/paths.py +51 -0
  87. package/ccx/notify/registry.py +649 -0
  88. package/ccx/notify/select.py +750 -0
  89. package/ccx/notify/settle.py +684 -0
  90. package/ccx/ops/__init__.py +0 -0
  91. package/ccx/ops/background.py +115 -0
  92. package/ccx/ops/cancel.py +289 -0
  93. package/ccx/ops/codex_import.py +96 -0
  94. package/ccx/ops/common.py +397 -0
  95. package/ccx/ops/compact.py +337 -0
  96. package/ccx/ops/compact_ops.py +550 -0
  97. package/ccx/ops/context_events.py +48 -0
  98. package/ccx/ops/conversation.py +1540 -0
  99. package/ccx/ops/conversation_paging.py +1338 -0
  100. package/ccx/ops/create.py +56 -0
  101. package/ccx/ops/creation.py +952 -0
  102. package/ccx/ops/creation_steps.py +579 -0
  103. package/ccx/ops/event_feed.py +710 -0
  104. package/ccx/ops/forking.py +711 -0
  105. package/ccx/ops/input.py +556 -0
  106. package/ccx/ops/maintenance.py +295 -0
  107. package/ccx/ops/report.py +770 -0
  108. package/ccx/ops/semantics.py +490 -0
  109. package/ccx/ops/send.py +961 -0
  110. package/ccx/ops/steer_ops.py +867 -0
  111. package/ccx/ops/terminate.py +383 -0
  112. package/ccx/ops/turn_reports.py +306 -0
  113. package/ccx/ops/wait.py +762 -0
  114. package/ccx/ops/worker_events.py +636 -0
  115. package/ccx/rpc/__init__.py +0 -0
  116. package/ccx/rpc/local_rpc.py +1602 -0
  117. package/ccx/rpc/rpc.py +559 -0
  118. package/ccx/rpc/serve.py +374 -0
  119. package/ccx/rpc/transport.py +409 -0
  120. package/ccx/rpc/view.py +125 -0
  121. package/ccx/store/__init__.py +0 -0
  122. package/ccx/store/discovery.py +170 -0
  123. package/ccx/store/identity.py +105 -0
  124. package/ccx/store/journal_io.py +79 -0
  125. package/ccx/store/liveness.py +194 -0
  126. package/ccx/store/paths.py +254 -0
  127. package/ccx/store/record_cache.py +40 -0
  128. package/ccx/store/records.py +309 -0
  129. package/ccx/store/root.py +804 -0
  130. package/ccx/store/runtime_presence.py +137 -0
  131. package/ccx/store/store.py +2332 -0
  132. package/ccx/store/worker_state.py +86 -0
  133. package/ccx/system/__init__.py +0 -0
  134. package/ccx/system/bounded.py +296 -0
  135. package/ccx/system/cwd_paths.py +25 -0
  136. package/ccx/system/engine_binary.py +59 -0
  137. package/ccx/system/executable.py +50 -0
  138. package/ccx/system/oslayer.py +667 -0
  139. package/ccx/system/pipes.py +46 -0
  140. package/ccx/system/sanitize.py +43 -0
  141. package/ccx/types.py +705 -0
  142. package/ccx/version.py +29 -0
  143. package/ccx/workers.py +2081 -0
  144. package/ccx.py +9 -0
  145. package/package.json +39 -7
  146. package/pyproject.toml +87 -0
  147. package/skills/ccx/SKILL.md +155 -0
  148. package/skills/ccx/references/controllers.md +244 -0
  149. package/skills/ccx/references/troubleshooting.md +94 -0
  150. package/skills/ccx/references/variables.md +62 -0
@@ -0,0 +1,2396 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import sys
5
+ import threading
6
+ import time
7
+ import uuid
8
+ from copy import deepcopy
9
+ from typing import Any
10
+ from ccx.codex.evidence import CONTEXT_CHANGED_METHOD, BACKGROUND_CHANGED_METHOD, COMPACTION_CONTEXT_METHOD, COMPACT_STARTED_METHOD, COMPACT_ENDED_METHOD, INPUT_REQUESTED_METHOD, INPUT_RESOLVED_METHOD, JOURNALED_METHODS, LIFECYCLE_METHOD, EvidenceJournal, read_index as read_evidence_index, read_journal as read_compact_journal
11
+ from ccx.codex.notify_history import notification_delivery_match
12
+ from ccx.ops.background import label as background_label, summary as background_summary, materially_changed
13
+ from ccx.ops.context_events import ContextChanges
14
+ from ccx.codex.protocol import AppServerClient, ProtocolError, ProtocolNotSent, ProtocolTransportError
15
+ from ccx.codex.reports import native_timestamp_ms, report_from_item, should_replace_live_report
16
+ from ccx.codex.records import now_ms
17
+ from ccx.codex.daemon.models import _abandon_active_compact, WorkerKilled, _LAUNCH_SETTLE_CAP_S, _TurnLaunch, _bounded, _protocol_failure
18
+ from typing import TYPE_CHECKING
19
+ if TYPE_CHECKING:
20
+ from ccx.codex.daemon.daemon import CodexDaemon
21
+
22
+
23
+ _TERMINAL_STATES = {"dead", "idle-killed", "crashed", "budget-stopped"}
24
+
25
+
26
+ _PARENT_NOTIFY_FENCES = 32
27
+
28
+
29
+ TURN_OUTCOME_STATUSES = {"completed", "interrupted", "failed"}
30
+
31
+
32
+ APPROVAL_METHODS: dict[str, str] = {
33
+ "item/commandExecution/requestApproval": "command",
34
+ "item/fileChange/requestApproval": "fileChange",
35
+ }
36
+
37
+
38
+ def _turn_id(value: dict[str, Any]) -> str | None:
39
+ turn = value.get("turn")
40
+ return turn.get("id") if isinstance(turn, dict) else None
41
+
42
+
43
+ def _last_turn_from_native(turn: dict[str, Any]) -> dict[str, Any]:
44
+ """Normalize one native ``Turn`` into the durable outcome projection.
45
+
46
+ ``lastTurn`` deliberately contains only the terminal turn facts. In
47
+ particular, timestamps come from Codex's Turn object; a missing native
48
+ timestamp remains ``None`` instead of being replaced with CCX wall-clock
49
+ time. Unknown native statuses are not promoted into the public outcome
50
+ vocabulary.
51
+ """
52
+ status = turn.get("status")
53
+ return {
54
+ "turnId": turn.get("id"),
55
+ "status": status if status in TURN_OUTCOME_STATUSES else None,
56
+ "error": turn.get("error"),
57
+ "startedAtMs": native_timestamp_ms(turn.get("startedAt")),
58
+ "completedAtMs": native_timestamp_ms(turn.get("completedAt")),
59
+ "durationMs": (
60
+ turn.get("durationMs")
61
+ if isinstance(turn.get("durationMs"), (int, float))
62
+ and not isinstance(turn.get("durationMs"), bool)
63
+ else None
64
+ ),
65
+ }
66
+
67
+
68
+
69
+
70
+ def _approval_key(approval: dict[str, Any]) -> str:
71
+ """One pending approval's identity: its JSON-RPC request id, typed."""
72
+ return json.dumps(approval.get("requestId"), sort_keys=True, default=str)
73
+
74
+
75
+ class Worker:
76
+ """One CCX-owned Codex worker, driven over the daemon's shared client."""
77
+
78
+ def __init__(self, daemon: "CodexDaemon", record: dict[str, Any]):
79
+ self.daemon = daemon
80
+ # Original local short variable is internally a canonical full owner.
81
+ # Eight-character display prefixes never identify a worker here.
82
+ from ccx.store.identity import canonical_worker_id
83
+ self.worker_id = canonical_worker_id(record.get("workerId"))
84
+ if record.get("engine") != "codex":
85
+ raise ValueError("daemon Worker requires globally owned Codex ID")
86
+ self.record = deepcopy(record)
87
+ self._disk_baseline = deepcopy(record)
88
+ self.short = self.worker_id
89
+ self.lock = threading.RLock()
90
+ self.operation_lock = threading.Lock()
91
+ self.load_lock = threading.Lock()
92
+ self.loaded = False
93
+ # Set by `kill` before it unloads, and never cleared: a Worker object
94
+ # that has been killed is dead forever, and a fresh `load` builds a new
95
+ # one. This is the whole kill/load race protocol — see `_abort_if_killed`.
96
+ self.killed = threading.Event()
97
+ # Published before kill waits for `operation_lock`, and never cleared.
98
+ # A kill that has to wait behind an in-flight steer/approval must not
99
+ # meanwhile let new operations in — but it also must not bypass the
100
+ # lock and race the operation that is already running. The sticky
101
+ # intent separates the two: new work is refused immediately, the
102
+ # running op is allowed to settle, and only then does the kill act.
103
+ self.kill_intent = threading.Event()
104
+ # Non-None exactly while a `turn/start` request is in flight. Written
105
+ # only under `self.lock`, which is what makes kill-vs-launch atomic.
106
+ self.launch: _TurnLaunch | None = None
107
+ # Caps belong to one original native turn, never to the worker's next turn.
108
+ self.cap_turn_id: str | None = None
109
+ self.active_caps: dict | None = None
110
+ # A source-proven resume may return the preceding failed turn's
111
+ # persisted systemError. Until the next native turn/status transition,
112
+ # another systemError edge is that old state, not a new crash.
113
+ self._resumed_system_error_pending = False
114
+ # Native threads this object's load may have subscribed the shared
115
+ # connection to. Codex keeps a thread loaded (with its per-thread MCP
116
+ # servers) until its last subscriber leaves, so `_unload` releases
117
+ # these. Kept rather than cleared: unsubscribing twice is harmless, and
118
+ # a kill that lands mid-resume unloads before the subscription exists.
119
+ self.native_threads: set[str] = set()
120
+ # Parent-notification deliveries whose `turn/start` this object saw
121
+ # answered, so a retry walking history outside the operation lock
122
+ # cannot launch one twice. delivery id -> native turn id.
123
+ self.launched_notifications: dict[str, str | None] = {}
124
+ # Claimed under `daemon.lock` by the one `_unload` that tears this
125
+ # object down; see there.
126
+ self.teardown_claimed = False
127
+ self.last_activity = time.monotonic()
128
+ self.attach_count = 0 # live attach RPC connections, never persisted
129
+ # Native commands become background only if still open at turn/completed.
130
+ # The journal is the source; these are this app-server's owned live IDs.
131
+ # Public context emission is one sparse stream, independent of raw native
132
+ # usage notifications. Reuse the existing daemon janitor for pending flush.
133
+ self._context_changes = ContextChanges()
134
+ self._bg_candidates: dict[str, dict] = {}
135
+ self._bg_open: dict[str, dict] = {}
136
+ # No replay ledger: a persisted snapshot is enough to compare current
137
+ # registry truth when this worker is adopted or notified again.
138
+ prior_items = (record.get("backgroundTaskSummary") or {}).get("items") or []
139
+ self._bg_notifications = {
140
+ item["id"]: dict(item) for item in prior_items
141
+ if isinstance(item, dict) and item.get("kind") == "notification"
142
+ and item.get("status") == "pending"
143
+ and isinstance(item.get("id"), str)
144
+ }
145
+ # The last record write a notification handler could not make (disk
146
+ # full, permissions). The in-memory record stays authoritative.
147
+ self.persistence_error: dict[str, Any] | None = None
148
+ # An external terminal CAS wins over this cached Worker forever.
149
+ self._externally_terminalized = False
150
+ # Sparse durable evidence for this worker's turns. Opened lazily on
151
+ # the first journaled notification, so a worker that never runs one
152
+ # leaves no file — which is itself the truthful answer to "was CCX
153
+ # watching?".
154
+ # `health` rehydrates a degradation an earlier process already
155
+ # recorded: a fresh writer must not present a worker whose evidence is
156
+ # known to be incomplete as complete again.
157
+ self.evidence = EvidenceJournal(
158
+ self.short,
159
+ codex_version=getattr(daemon, "codex_version", None),
160
+ health=record.get("evidence"),
161
+ )
162
+ try:
163
+ evidence_index = read_evidence_index(self.short)
164
+ except Exception:
165
+ evidence_index = {}
166
+ self.completed_turn_ids = {
167
+ str(turn_id)
168
+ for turn_id, facts in (evidence_index.get("turns") or {}).items()
169
+ if isinstance(facts, dict) and facts.get("sawTurnCompleted")
170
+ }
171
+ # Journal-derived identities keep compaction end publication idempotent
172
+ # when native notifications repeat or codexd reloads an owned worker.
173
+ self._compact_started: dict[str, set[str]] = {}
174
+ self._compact_finished: dict[str, set[str]] = {}
175
+ self._compact_ended: set[str] = set()
176
+ self._compact_legacy: set[str] = set()
177
+ try:
178
+ compact_history = read_compact_journal(self.short).events
179
+ except Exception:
180
+ compact_history = []
181
+ for row in compact_history:
182
+ tid, item_id, method = row.get("turnId"), row.get("itemId"), row.get("method")
183
+ if method == "thread/compacted" and isinstance(tid, str):
184
+ self._compact_legacy.add(tid)
185
+ if not isinstance(tid, str) or not tid:
186
+ continue
187
+ # Historical native items are still forensic proof, not feed
188
+ # events. Only CCX-owned sparse lines rehydrate publisher dedupe.
189
+ # Older journals lacking them intentionally gain no backfill.
190
+ if (row.get("itemType") == "contextCompaction"
191
+ and isinstance(item_id, str)
192
+ and method == "item/completed"):
193
+ self._compact_finished.setdefault(tid, set()).add(item_id)
194
+ if method == COMPACT_STARTED_METHOD:
195
+ key = (row.get("params") or {}).get("compactId")
196
+ if isinstance(key, str):
197
+ self._compact_started.setdefault(tid, set()).add(key)
198
+ if method == COMPACT_ENDED_METHOD:
199
+ key = (row.get("params") or {}).get("compactId")
200
+ if isinstance(key, str):
201
+ self._compact_ended.add(key)
202
+ # What the journal last said about this worker's state and approvals.
203
+ # `_journal_lifecycle_locked` diffs the record against it on every
204
+ # save, so each transition is journaled exactly once whichever of the
205
+ # many record-mutating paths made it.
206
+ self.journaled_state = (record.get("lifecycle") or {}).get("state")
207
+ self.journaled_approvals = {
208
+ _approval_key(approval): approval
209
+ for approval in record.get("pendingApprovals") or []
210
+ if isinstance(approval, dict)
211
+ }
212
+ # Decisions CCX itself sent, keyed like `journaled_approvals`, so the
213
+ # resolution line can say how a request was answered.
214
+ self.approval_decisions: dict[str, str] = {}
215
+ # The caller's response id for a decision, journaled with it so a
216
+ # caller that lost the reply can prove its decision landed.
217
+ self.approval_response_ids: dict[str, str] = {}
218
+ last_turn = record.get("lastTurn")
219
+ if (
220
+ isinstance(last_turn, dict)
221
+ and isinstance(last_turn.get("turnId"), str)
222
+ and last_turn.get("turnId")
223
+ ):
224
+ self.completed_turn_ids.add(str(last_turn["turnId"]))
225
+
226
+ @property
227
+ def client(self) -> AppServerClient:
228
+ client = self.daemon.client
229
+ if client is None:
230
+ raise RuntimeError("app-server client is not running")
231
+ return client
232
+
233
+ def _save(self) -> None:
234
+ """Save once with field CAS; retry only a proved safe fresh-record rebase.
235
+
236
+ A failed CAS is not authority to drop expectations. In particular a
237
+ concurrent CLI kill owns the terminal lifecycle, even if an app-server
238
+ completion races it. The original native event remains in the evidence
239
+ journal when a conflict cannot be reconciled.
240
+ """
241
+ from ccx.store.root import RecordConflict
242
+ with self.lock:
243
+ if self._externally_terminalized and (
244
+ self.record.get("lifecycle") != self._disk_baseline.get("lifecycle")
245
+ or self.record.get("pendingApprovals")
246
+ != self._disk_baseline.get("pendingApprovals")
247
+ ):
248
+ # This object saw an external terminal CAS. Even another
249
+ # daemon-local _state/approval writer cannot bring it back;
250
+ # restore the current persisted terminal view before refusing.
251
+ latest = self.daemon._uuid_store.record(f"codex:{self.worker_id}")
252
+ self.record = deepcopy(latest)
253
+ self._disk_baseline = deepcopy(latest)
254
+ raise RecordConflict("externally terminalized Codex Worker cannot revive")
255
+ self._journal_lifecycle_locked()
256
+ try:
257
+ self._save_current_locked()
258
+ except RecordConflict:
259
+ self._rebase_conflicting_save_locked()
260
+ self._journal_lifecycle_locked()
261
+ # One bounded retry against the newly observed exact fields.
262
+ # A second conflict is surfaced per worker, never overwritten.
263
+ self._save_current_locked()
264
+
265
+ def _save_current_locked(self) -> None:
266
+ """One semantic CAS publication; caller holds self.lock."""
267
+ from ccx.store.root import StorageIntegrityError, RecordConflict
268
+ if (self._externally_terminalized
269
+ and self.record["lifecycle"].get("state") not in _TERMINAL_STATES):
270
+ raise RecordConflict("external terminal state cannot be resurrected")
271
+ before = self._disk_baseline
272
+ desired = self.record
273
+ patches: dict[str, object] = {}
274
+ expect: dict[str, object] = {}
275
+ increments: dict[str, int] = {}
276
+ containers = ("lifecycle", "stats", "createArgs")
277
+ top_allowed = {
278
+ "name", "waitingFor", "lastTurnAt", "lastSettledAt",
279
+ "status", "lastTurn", "lastReport", "activeTurnError",
280
+ "pendingApprovals", "context", "activeCompact", "lastCompact",
281
+ "compactCount", "evidence", "tokenUsage", "historyKnownEmpty",
282
+ "nativeArchive", "resolvedModel", "parentNotifyFences",
283
+ "lastTurnError", "pendingSend", "lastConversation", "nativeTurnStart",
284
+ "quotaStop", "backgroundTaskSummary",
285
+ }
286
+ for key in set(before) | set(desired):
287
+ if key in ("recordRevision", "updatedAt"):
288
+ continue
289
+ old = before.get(key)
290
+ new = desired.get(key)
291
+ if new == old:
292
+ continue
293
+ if key not in desired:
294
+ raise StorageIntegrityError(
295
+ "Codex Worker cannot delete a durable record field"
296
+ )
297
+ if key in containers:
298
+ if not isinstance(old, dict) or not isinstance(new, dict):
299
+ raise StorageIntegrityError(
300
+ "Codex Worker cannot replace structured record"
301
+ )
302
+ if not set(old).issubset(new):
303
+ raise StorageIntegrityError(
304
+ "Codex Worker cannot remove structured fields"
305
+ )
306
+ for member in new:
307
+ if member in old and old[member] == new[member]:
308
+ continue
309
+ field = f"{key}.{member}"
310
+ if (key == "stats"
311
+ and type(new[member]) is int
312
+ and type(old.get(member)) is int
313
+ and new[member] >= old[member]):
314
+ increments[field] = new[member] - old[member]
315
+ else:
316
+ patches[field] = new[member]
317
+ expect[field] = old.get(member)
318
+ elif key in top_allowed:
319
+ patches[key] = new
320
+ expect[key] = old
321
+ else:
322
+ raise StorageIntegrityError(
323
+ f"Codex Worker cannot mutate immutable native/owner field {key}"
324
+ )
325
+ ctx = self.daemon._uuid_store
326
+ if patches or increments:
327
+ durable = ctx.mutate_record(
328
+ f"codex:{self.worker_id}",
329
+ patches=patches, increments=increments,
330
+ expect=expect,
331
+ )
332
+ else:
333
+ durable = ctx.record(f"codex:{self.worker_id}")
334
+ # A nonconflicting metadata/evidence patch (or even a pure read) can
335
+ # absorb another process's terminal kill. Recognize its authority on
336
+ # every fresh durable observation, not only in the conflict rebase.
337
+ old_state = (before.get("lifecycle") or {}).get("state")
338
+ committed_state = (durable.get("lifecycle") or {}).get("state")
339
+ proposed_state = (desired.get("lifecycle") or {}).get("state")
340
+ if (committed_state in _TERMINAL_STATES
341
+ and old_state not in _TERMINAL_STATES
342
+ and proposed_state != committed_state):
343
+ self._externally_terminalized = True
344
+ self.killed.set()
345
+ self.record = durable
346
+ self._disk_baseline = deepcopy(durable)
347
+
348
+
349
+ def _rebase_conflicting_save_locked(self) -> None:
350
+ """Reconcile one same-worker CAS conflict, never ignore its expectations.
351
+
352
+ Fresh latest data is the base. Replay a local edit only if its exact
353
+ previous value is still present (or the edit is already durable). A
354
+ positive integer stat increment is commutative, except a completion
355
+ already recorded for the same native turn must not be counted twice.
356
+ A concurrent terminal state wins over *all* cached lifecycle and
357
+ approval projections. Unprovable overlaps remain visibly unresolved.
358
+ """
359
+ from ccx.store.root import RecordConflict, StorageIntegrityError
360
+ before, desired = self._disk_baseline, self.record
361
+ latest = self.daemon._uuid_store.record(f"codex:{self.worker_id}")
362
+ if (latest.get("workerId") != self.worker_id
363
+ or latest.get("engine") != "codex"
364
+ or latest.get("native") != before.get("native")):
365
+ raise StorageIntegrityError("Codex rebase changed immutable native owner")
366
+ state = (latest.get("lifecycle") or {}).get("state")
367
+ terminal_won = (state in _TERMINAL_STATES
368
+ and (before.get("lifecycle") or {}).get("state")
369
+ not in _TERMINAL_STATES)
370
+ if terminal_won:
371
+ self._externally_terminalized = True
372
+ self.killed.set() # also fences this cached worker's future launches
373
+
374
+ def refuse(message: str) -> None:
375
+ # Do not retain stale state that will conflict on every later event,
376
+ # or an approval that was never durably admitted. Native evidence
377
+ # was journaled independently; the handler exposes the lost patch.
378
+ self.record = deepcopy(latest)
379
+ self._disk_baseline = deepcopy(latest)
380
+ raise RecordConflict(message)
381
+
382
+ old_turn = (before.get("lastTurn") or {}).get("turnId")
383
+ wanted_turn = (desired.get("lastTurn") or {}).get("turnId")
384
+ disk_turn = (latest.get("lastTurn") or {}).get("turnId")
385
+ if terminal_won:
386
+ old_approvals = before.get("pendingApprovals") or []
387
+ new_approvals = desired.get("pendingApprovals") or []
388
+ if any(approval not in old_approvals for approval in new_approvals):
389
+ refuse("external kill won before Codex approval was durable")
390
+ if wanted_turn != old_turn and (not isinstance(wanted_turn, str)
391
+ or (before.get("lifecycle") or {}).get("activeTurnId")
392
+ != wanted_turn or disk_turn not in (old_turn, wanted_turn)):
393
+ refuse("terminal worker has no positive original turn to reconcile")
394
+
395
+ rebased = deepcopy(latest)
396
+ containers = ("lifecycle", "stats", "createArgs")
397
+ for key in set(before) | set(desired):
398
+ if key in ("recordRevision", "updatedAt") or before.get(key) == desired.get(key):
399
+ continue
400
+ old, new = before.get(key), desired.get(key)
401
+ if key not in desired:
402
+ raise StorageIntegrityError("Codex cannot rebase a deleted field")
403
+ if key in containers:
404
+ if (not isinstance(old, dict) or not isinstance(new, dict)
405
+ or not isinstance(rebased.get(key), dict)
406
+ or not set(old).issubset(new)):
407
+ raise StorageIntegrityError("Codex cannot rebase malformed structured fields")
408
+ changes = ((f"{key}.{member}", old.get(member), value)
409
+ for member, value in new.items()
410
+ if member not in old or old[member] != value)
411
+ else:
412
+ changes = ((key, old, new),)
413
+ for field, previous, value in changes:
414
+ # Terminal status/cleared active turn and approvals are not
415
+ # native completion effects. Never revive them via a stale save.
416
+ if terminal_won and (field.startswith("lifecycle.") or field in {
417
+ "pendingApprovals", "waitingFor", "pendingSend",
418
+ "activeTurnError", "name"} or field.startswith("createArgs.")):
419
+ continue
420
+ parts = field.split(".")
421
+ container = rebased if len(parts) == 1 else rebased[parts[0]]
422
+ current = container.get(parts[-1])
423
+ if (field.startswith("stats.")
424
+ and type(previous) is int and type(value) is int
425
+ and value >= previous):
426
+ if type(current) is not int:
427
+ refuse(f"{field} cannot be merged as a counter")
428
+ delta = value - previous
429
+ if field == "stats.turns" and wanted_turn != old_turn:
430
+ if disk_turn == wanted_turn:
431
+ delta = 0 # this same turn was already committed
432
+ elif disk_turn != old_turn:
433
+ refuse("newer native turn prevents counter reconciliation")
434
+ container[parts[-1]] = current + delta
435
+ elif current == previous:
436
+ container[parts[-1]] = deepcopy(value)
437
+ elif current != value:
438
+ refuse(f"record semantic field changed before CAS: {field}")
439
+ self.record = rebased
440
+ self._disk_baseline = deepcopy(latest)
441
+
442
+ def _journal_lifecycle_locked(
443
+ self,
444
+ *,
445
+ state: str | None = None,
446
+ approvals: list[dict[str, Any]] | None = None,
447
+ ) -> None:
448
+ """Journal state and approval transitions not yet in the journal.
449
+
450
+ Caller holds ``self.lock``. ``state``/``approvals`` override the
451
+ record's, for a teardown that journals its terminal facts before it
452
+ closes the journal. Never raises (``EvidenceJournal.record`` does
453
+ not), and writes nothing once the journal is closed: reopening would
454
+ start a new header for a worker that is already gone.
455
+ """
456
+ if self.evidence.closed:
457
+ return
458
+ lifecycle = self.record.get("lifecycle") or {}
459
+ current_state = state if state is not None else lifecycle.get("state")
460
+ current = {
461
+ _approval_key(approval): approval
462
+ for approval in (
463
+ approvals if approvals is not None
464
+ else self.record.get("pendingApprovals") or []
465
+ )
466
+ if isinstance(approval, dict)
467
+ }
468
+ wrote = False
469
+ thread_id = (self.record.get("native") or {}).get("threadId")
470
+ for key, approval in self.journaled_approvals.items():
471
+ if key not in current:
472
+ params = {
473
+ "requestId": approval.get("requestId"),
474
+ "gateId": approval.get("gateId"),
475
+ "kind": approval.get("kind"),
476
+ "decision": self.approval_decisions.pop(key, None),
477
+ }
478
+ response_id = self.approval_response_ids.pop(key, None)
479
+ if response_id is not None:
480
+ params["responseId"] = response_id
481
+ self.evidence.record(
482
+ INPUT_RESOLVED_METHOD, params, thread_id=thread_id,
483
+ )
484
+ wrote = True
485
+ for key, approval in current.items():
486
+ if key not in self.journaled_approvals:
487
+ self.evidence.record(
488
+ INPUT_REQUESTED_METHOD, {"request": approval},
489
+ thread_id=thread_id,
490
+ )
491
+ wrote = True
492
+ self.journaled_approvals = current
493
+ if current_state and current_state != self.journaled_state:
494
+ params: dict[str, Any] = {
495
+ "from": self.journaled_state,
496
+ "to": current_state,
497
+ }
498
+ if current_state in {"crashed", "dead", "idle-killed"}:
499
+ params["reason"] = lifecycle.get("lastError")
500
+ self.evidence.record(LIFECYCLE_METHOD, params, thread_id=thread_id)
501
+ self.journaled_state = current_state
502
+ wrote = True
503
+ if wrote:
504
+ self.record["evidence"] = self.evidence.state()
505
+
506
+
507
+ def _abort_if_killed(self) -> None:
508
+ """Unwind a load that a concurrent ``kill`` has already invalidated.
509
+
510
+ Original resume spends most of its time inside app-server RPCs holding
511
+ no daemon or worker lock — deliberately, so the daemon stays responsive.
512
+ That means a `kill` can complete (record ``dead``, worker removed from
513
+ `workers`/`by_thread`) while this load is still in flight, and every
514
+ step after the RPC would then resurrect it: re-binding the thread,
515
+ rewriting the record and finally reporting ``ok``.
516
+
517
+ Rather than have `kill` block on `load_lock` — which would stall it
518
+ behind a 90s app-server call — the loader checks this event at each of
519
+ those points and unwinds itself. Reading an Event takes no lock, so it
520
+ cannot deadlock against the killer.
521
+ """
522
+ if self.killed.is_set():
523
+ raise WorkerKilled(f"worker {self.short} was killed during load")
524
+
525
+ def _commit_state(self, state: str, **lifecycle_patch) -> None:
526
+ """Write a loader state transition atomically against ``kill``.
527
+
528
+ A bare `_abort_if_killed()` before `_state()` is not enough: a kill
529
+ landing in the gap writes ``dead`` and then this write puts the record
530
+ back to ``idle``/``busy``, leaving a killed worker looking alive. The
531
+ check and the write therefore happen inside one hold of ``self.lock``,
532
+ which is the same lock ``kill`` sets its marker under — so the two are
533
+ totally ordered and the loser is the one that does nothing.
534
+ """
535
+ with self.lock:
536
+ if self.killed.is_set():
537
+ raise WorkerKilled(
538
+ f"worker {self.short} was killed during load")
539
+ lifecycle = self.record["lifecycle"]
540
+ lifecycle.update(lifecycle_patch)
541
+ lifecycle["state"] = state
542
+ lifecycle["since"] = now_ms()
543
+ self.last_activity = time.monotonic()
544
+ self._save()
545
+
546
+ def _interrupt_turn(self, turn_id: str | None) -> bool:
547
+ """Best-effort native cancellation of one turn. Safe to call twice."""
548
+ thread_id = (self.record.get("native") or {}).get("threadId") or self.record.get("threadId")
549
+ if not turn_id or not thread_id:
550
+ return False
551
+ try:
552
+ self.client.request(
553
+ "turn/interrupt", {"threadId": thread_id, "turnId": turn_id})
554
+ return True
555
+ except Exception:
556
+ return False
557
+
558
+ def check_active_caps(self, rate_limits: dict | None) -> None:
559
+ """Check a pushed quota update without RPC on the notification reader."""
560
+ from ccx.codex.types import codex_cap_tripped
561
+ with self.lock:
562
+ turn_id = self.cap_turn_id
563
+ if (not turn_id or not self.active_caps
564
+ or self.record["lifecycle"].get("activeTurnId") != turn_id):
565
+ return
566
+ trip = codex_cap_tripped(rate_limits, self.active_caps)
567
+ if not trip:
568
+ return
569
+ name, used, cap = trip
570
+ reason = f"usage cap {name} reached: {used:g}% >= {cap:g}%"
571
+ self.record["quotaStop"] = {"turnId": turn_id, "reason": reason}
572
+ self.active_caps = None # at most one interrupt for this native turn
573
+ self._save()
574
+ threading.Thread(target=self._interrupt_turn, args=(turn_id,),
575
+ daemon=True, name="ccx-codex-cap-interrupt").start()
576
+
577
+ def _abandon_active_compact_locked(self, reason: str) -> bool:
578
+ """Close a persisted compact that can no longer yield native proof.
579
+
580
+ Caller holds ``self.lock``. See `_abandon_active_compact`.
581
+ """
582
+ return _abandon_active_compact(self.record, reason)
583
+
584
+ def _state(self, state: str, *, error: str | None = None) -> None:
585
+ with self.lock:
586
+ lifecycle = self.record["lifecycle"]
587
+ lifecycle["state"] = state
588
+ lifecycle["since"] = now_ms()
589
+ if error is not None:
590
+ lifecycle["lastError"] = error
591
+ self.last_activity = time.monotonic()
592
+ self._save()
593
+
594
+ def state(self) -> str:
595
+ with self.lock:
596
+ return self.record["lifecycle"]["state"]
597
+
598
+
599
+ def _native_create_daemon_id(self) -> str:
600
+ return str(getattr(self.daemon, "instance_id", "legacy-daemon"))
601
+
602
+
603
+
604
+
605
+
606
+
607
+ def _resume_exact_thread(
608
+ self, thread_id: str, params: dict[str, Any], *,
609
+ archived: bool = False, timeout: float = 90,
610
+ before_submit=None,
611
+ ) -> dict[str, Any]:
612
+ """One original native thread/resume, never a second thread/start.
613
+
614
+ The pre-write release fence and native_threads registration also cover
615
+ a lost reply: a concurrent kill must still unsubscribe this thread.
616
+ """
617
+ self.daemon._await_thread_release(thread_id)
618
+ self.daemon.claim_thread(thread_id, self.short)
619
+ self.native_threads.add(thread_id)
620
+ if archived:
621
+ self.client.request(
622
+ "thread/unarchive", {"threadId": thread_id}, timeout=timeout,
623
+ )
624
+ resume_params = dict(params)
625
+ resume_params["threadId"] = thread_id
626
+ # Only the thread's id, source and status are read from the reply;
627
+ # rebuilding a long thread's full history can take Codex minutes.
628
+ resume_params["excludeTurns"] = True
629
+ # Kill can win while this load waits for a prior unsubscribe. The
630
+ # already-published killed marker is the existing no-late-load fence.
631
+ self._abort_if_killed()
632
+ if before_submit is not None:
633
+ before_submit() # persist the current native attempt only after release preflight
634
+ try:
635
+ return self.client.request(
636
+ "thread/resume", resume_params, timeout=timeout,
637
+ )
638
+ except ProtocolError as error:
639
+ if "no rollout found" in error.message:
640
+ raise ProtocolError(
641
+ "original Codex thread has no persisted rollout; "
642
+ "refusing native thread/start replacement",
643
+ method="thread/resume",
644
+ ) from error
645
+ raise
646
+
647
+
648
+
649
+ def _native_turn_start_marker(
650
+ self,
651
+ *,
652
+ kind: str,
653
+ client_message_id: str | None = None,
654
+ delivery_id: str | None = None,
655
+ ) -> dict[str, Any]:
656
+ return {
657
+ "schemaVersion": 1,
658
+ "kind": kind,
659
+ "state": "submitted",
660
+ "daemonInstanceId": self._native_create_daemon_id(),
661
+ "clientUserMessageId": client_message_id,
662
+ "deliveryId": delivery_id,
663
+ "submittedAtMs": now_ms(),
664
+ "observedAtMs": None,
665
+ "lastError": None,
666
+ }
667
+
668
+ def _mark_native_turn_start_unknown_locked(self, error: BaseException) -> None:
669
+ marker = self.record.get("nativeTurnStart")
670
+ if not isinstance(marker, dict):
671
+ return
672
+ marker["state"] = "unknown"
673
+ marker["observedAtMs"] = now_ms()
674
+ marker["lastError"] = str(error)
675
+ self._save()
676
+
677
+ def _forget_native_turn_start_locked(self) -> None:
678
+ # Full UUID GenStore CAS does not erase persistent top-level fields;
679
+ # clear the existing launch marker to null under the same CAS instead.
680
+ if self.record.get("fingerprintVersion") == 2:
681
+ if "nativeTurnStart" in self.record:
682
+ self.record["nativeTurnStart"] = None
683
+ else:
684
+ self.record.pop("nativeTurnStart", None)
685
+
686
+ def _clear_native_turn_start_locked(self) -> None:
687
+ if self.record.get("nativeTurnStart") is not None:
688
+ self._forget_native_turn_start_locked()
689
+ self._save()
690
+
691
+ @staticmethod
692
+ def _native_mutation_error_is_ambiguous(error: ProtocolError) -> bool:
693
+ if isinstance(error, ProtocolNotSent):
694
+ return False
695
+ return isinstance(error, ProtocolTransportError) or error.request_id is None
696
+
697
+ def _journal(self, method: str, params: dict[str, Any]) -> bool:
698
+ """Record one authoritative native event before it is reduced.
699
+
700
+ Returns whether this notification is journaled at all, so the caller
701
+ can persist the journal's health in the record write it is already
702
+ doing. `EvidenceJournal.record` swallows its own failures on purpose:
703
+ a handler that raised here would fail the shared app-server client
704
+ (`AppServerClient._read_loop` treats a handler exception as fatal),
705
+ killing every worker's live turn over a file that could not be
706
+ appended to.
707
+ """
708
+ if method not in JOURNALED_METHODS:
709
+ return False
710
+ durable = None
711
+ if method == "thread/tokenUsage/updated":
712
+ usage_turn_id = params.get("turnId") or _turn_id(params)
713
+ durable = usage_turn_id in self.completed_turn_ids
714
+ self.evidence.record(
715
+ method,
716
+ params,
717
+ thread_id=self.record.get("threadId"),
718
+ durable=durable,
719
+ )
720
+ return True
721
+
722
+ def _capture_compaction_context_locked(
723
+ self, compact: dict[str, Any]
724
+ ) -> None:
725
+ """Persist one bounded compaction before/after telemetry enrichment."""
726
+ context_after = self.record.get("context")
727
+ earlier = compact.get("contextBefore")
728
+ completed = compact.get("completedAtMs")
729
+ valid = (
730
+ isinstance(context_after, dict)
731
+ and type(context_after.get("as_of_ms")) is int
732
+ and type(completed) is int
733
+ and context_after["as_of_ms"] > completed
734
+ and (not isinstance(earlier, dict)
735
+ or type(earlier.get("as_of_ms")) is not int
736
+ or context_after["as_of_ms"] > earlier["as_of_ms"])
737
+ and (self.record.get("tokenUsage") or {}).get("asOfTurnId") != compact.get("turnId")
738
+ )
739
+ compact["contextAfter"] = dict(context_after) if valid else None
740
+ params = {
741
+ "turnId": compact.get("turnId"),
742
+ "item": {
743
+ "id": compact.get("contextItemId"),
744
+ "type": "contextCompaction",
745
+ },
746
+ "operationId": compact.get("operationId"),
747
+ "compactCount": self.record.get("compactCount"),
748
+ "contextBefore": compact.get("contextBefore"),
749
+ "contextAfter": compact.get("contextAfter"),
750
+ "completedAtMs": compact.get("completedAtMs"),
751
+ }
752
+ self.evidence.record(
753
+ COMPACTION_CONTEXT_METHOD,
754
+ params,
755
+ thread_id=self.record.get("threadId"),
756
+ )
757
+ self.record["evidence"] = self.evidence.state()
758
+
759
+ def _complete_active_compact_locked(
760
+ self, *, turn_id: str | None = None, completed_at_ms: int | None = None
761
+ ) -> bool:
762
+ """Publish one successful requested compact. Caller holds ``self.lock``."""
763
+ active = self.record.get("activeCompact")
764
+ if (not isinstance(active, dict) or not isinstance(turn_id, str)
765
+ or not turn_id or active.get("turnId") != turn_id
766
+ or active.get("nativeTurnStarted") is not True
767
+ or not isinstance(active.get("contextItemId"), str)
768
+ or not active["contextItemId"]
769
+ or active.get("nativeItemCompletedAtMs") is None
770
+ or turn_id not in self.completed_turn_ids):
771
+ return False
772
+ compact_count = int(self.record.get("compactCount") or 0) + 1
773
+ self.record["compactCount"] = compact_count
774
+ self.record["lastCompact"] = {
775
+ "operationId": active.get("operationId"),
776
+ "baselineCompactCount": active.get("baselineCompactCount"),
777
+ "targetCompactCount": active.get("targetCompactCount") or compact_count,
778
+ "startedAtMs": active.get("startedAtMs"),
779
+ "completedAtMs": completed_at_ms or now_ms(),
780
+ "contextBefore": active.get("contextBefore"),
781
+ "contextAfter": None,
782
+ "turnId": turn_id or active.get("turnId"),
783
+ "contextItemId": active.get("contextItemId"),
784
+ "nativeTurnStarted": True,
785
+ "nativeTurnSettled": True,
786
+ "nativeTurnStatus": "completed",
787
+ "status": "succeeded",
788
+ }
789
+ # Older carried tokens are invalid immediately after a native
790
+ # compaction; retain no synthetic newer context in the public record.
791
+ self.record["context"] = None
792
+ self._capture_compaction_context_locked(self.record["lastCompact"])
793
+ self.record["activeCompact"] = None
794
+ return True
795
+
796
+ def _observe_context_compaction_locked(
797
+ self, *, turn_id: str | None, item_id: str | None, completed_at_ms: int | None
798
+ ) -> None:
799
+ """Bind modern ``contextCompaction`` item evidence to compact state."""
800
+ active = self.record.get("activeCompact")
801
+ if isinstance(active, dict):
802
+ # Requested compact item is evidence only for the internal turn
803
+ # positively pinned by turn/started, never an unrelated item.
804
+ if (not isinstance(turn_id, str) or not turn_id
805
+ or active.get("turnId") != turn_id
806
+ or active.get("nativeTurnStarted") is not True
807
+ or not isinstance(item_id, str) or not item_id):
808
+ return
809
+ active["contextItemId"] = item_id
810
+ active["nativeItemCompletedAtMs"] = completed_at_ms or now_ms()
811
+ return
812
+ # Automatic/native compaction outside an explicit CCX compact request is
813
+ # still worth counting for context/history consumers. Do not double-count
814
+ # the deprecated thread/compacted notification for the same turn.
815
+ last = self.record.get("lastCompact")
816
+ if isinstance(last, dict) and turn_id and last.get("turnId") == turn_id:
817
+ return
818
+ compact_count = int(self.record.get("compactCount") or 0) + 1
819
+ self.record["compactCount"] = compact_count
820
+ self.record["lastCompact"] = {
821
+ "operationId": None,
822
+ "baselineCompactCount": compact_count - 1,
823
+ "targetCompactCount": compact_count,
824
+ "startedAtMs": None,
825
+ "completedAtMs": completed_at_ms or now_ms(),
826
+ "contextBefore": None,
827
+ "contextAfter": None,
828
+ "turnId": turn_id,
829
+ "contextItemId": item_id,
830
+ "status": "succeeded",
831
+ }
832
+
833
+ def _emit_compact_feed_locked(
834
+ self, method: str, turn_id: str, compact_id: str, *,
835
+ outcome: str | None = None, operation_id: str | None = None,
836
+ item_id: str | None = None, reason: str | None = None,
837
+ ) -> None:
838
+ if method == COMPACT_STARTED_METHOD:
839
+ if compact_id in self._compact_started.get(turn_id, set()):
840
+ return
841
+ elif method == COMPACT_ENDED_METHOD and compact_id in self._compact_ended:
842
+ return
843
+ params = {
844
+ "turnId": turn_id, "compactId": compact_id,
845
+ "operationId": operation_id, "itemId": item_id,
846
+ }
847
+ if outcome is not None:
848
+ params["outcome"] = outcome
849
+ if reason is not None:
850
+ params["reason"] = reason[:500]
851
+ wrote = self.evidence.record(
852
+ method, params, thread_id=self.record.get("threadId"),
853
+ )
854
+ if wrote is None:
855
+ return
856
+ if method == COMPACT_STARTED_METHOD:
857
+ self._compact_started.setdefault(turn_id, set()).add(compact_id)
858
+ else:
859
+ self._compact_ended.add(compact_id)
860
+ self.record["evidence"] = self.evidence.state()
861
+
862
+ def _close_unfinished_compacts_locked(
863
+ self, turn_id: str, *, outcome: str, operation_id: str | None,
864
+ reason: str | None,
865
+ ) -> None:
866
+ starts = self._compact_started.get(turn_id, set())
867
+ # A requested compact can have an admitted turn but no item started.
868
+ # Emit its own sparse start before writing the terminal.
869
+ if not starts and isinstance(operation_id, str) and operation_id:
870
+ self._emit_compact_feed_locked(
871
+ COMPACT_STARTED_METHOD, turn_id, f"operation:{operation_id}",
872
+ operation_id=operation_id,
873
+ )
874
+ starts = self._compact_started.get(turn_id, set())
875
+ for compact_id in sorted(starts):
876
+ item_id = None
877
+ prefix = f"{turn_id}:"
878
+ if compact_id.startswith(prefix):
879
+ item_id = compact_id[len(prefix):]
880
+ self._emit_compact_feed_locked(
881
+ COMPACT_ENDED_METHOD, turn_id, compact_id,
882
+ outcome=outcome, operation_id=operation_id,
883
+ item_id=item_id, reason=reason,
884
+ )
885
+
886
+ def _context_snapshot_locked(self) -> dict:
887
+ from ccx.claude.context import current_context, public_context
888
+ return public_context(
889
+ current_context(self.record, engine="codex", live=self.record.get("context")),
890
+ engine="codex", compacting=isinstance(self.record.get("activeCompact"), dict),
891
+ )
892
+
893
+ def _append_context_locked(self, value: dict, *, save: bool = True) -> None:
894
+ turn_id = ((self.record.get("lifecycle") or {}).get("activeTurnId")
895
+ or (self.record.get("lastTurn") or {}).get("turnId"))
896
+ self.evidence.record(
897
+ CONTEXT_CHANGED_METHOD,
898
+ {"context": value, "turnId": turn_id},
899
+ thread_id=(self.record.get("native") or {}).get("threadId"),
900
+ )
901
+ self.record["evidence"] = self.evidence.state()
902
+ if save:
903
+ self._save()
904
+
905
+ def _publish_context_locked(self, *, force: bool = False,
906
+ save: bool = True) -> None:
907
+ view = self._context_snapshot_locked()
908
+ previous = self._context_changes.last
909
+ fenced = (view["reason"] in {"compacting", "just_compacted"}
910
+ and (previous or {}).get("reason") != view["reason"])
911
+ active = (self.record.get("lifecycle") or {}).get("activeTurnId")
912
+ result = self._context_changes.observe(
913
+ view, force=bool(force or fenced or not active),
914
+ )
915
+ if result is not None:
916
+ self._append_context_locked(result, save=save)
917
+
918
+ def flush_context(self) -> None:
919
+ """Publish the last native usage sample on the existing janitor tick."""
920
+ with self.lock:
921
+ if self.teardown_claimed or self._externally_terminalized:
922
+ return
923
+ # The terminal notification itself flushes context synchronously.
924
+ # Never let the janitor publish an old queued mid-turn sample after
925
+ # turn/compact settlement; a read-only compact-status must stay pure.
926
+ if (not (self.record.get("lifecycle") or {}).get("activeTurnId")
927
+ or isinstance(self.record.get("activeCompact"), dict)):
928
+ self._context_changes.pending = None
929
+ return
930
+ value = self._context_changes.flush()
931
+ if value is not None:
932
+ self._append_context_locked(value)
933
+
934
+ def _publish_background_locked(self, transition: str, item: dict | None) -> None:
935
+ """Journal one actual native change; the caller holds worker.lock."""
936
+ snapshot = background_summary([
937
+ *self._bg_open.values(), *self._bg_notifications.values(),
938
+ ])
939
+ if item is None and not materially_changed(
940
+ self.record.get("backgroundTaskSummary"), snapshot
941
+ ):
942
+ return
943
+ self.record["backgroundTaskSummary"] = snapshot
944
+ self.evidence.record(
945
+ BACKGROUND_CHANGED_METHOD,
946
+ {"transition": transition, "item": item, "summary": snapshot},
947
+ thread_id=(self.record.get("native") or {}).get("threadId"),
948
+ )
949
+
950
+ def _lose_native_background_locked(self) -> bool:
951
+ """Drop old-runtime command guards; never assert a native exit.
952
+
953
+ Called under worker.lock when this app-server goes away or a new
954
+ owner adopts a recorded worker. A lost command is no longer tracked
955
+ (its outcome is unknown, so no ``finished`` item is published); the
956
+ correction keeps the registry-backed pending notifications.
957
+ """
958
+ previous = self.record.get("backgroundTaskSummary")
959
+ had_native = bool(self._bg_open or self._bg_candidates)
960
+ self._bg_open.clear()
961
+ self._bg_candidates.clear()
962
+ stale_running = (isinstance(previous, dict)
963
+ and type(previous.get("running_count")) is int
964
+ and previous["running_count"] > 0)
965
+ if not (had_native or stale_running):
966
+ return False
967
+ # Only native command tracking is lost. Pending parent notifications
968
+ # come from the durable registry, so they stay in the summary.
969
+ corrected = background_summary(list(self._bg_notifications.values()))
970
+ if not materially_changed(previous, corrected):
971
+ return False
972
+ self.record["backgroundTaskSummary"] = corrected
973
+ self.evidence.record(
974
+ BACKGROUND_CHANGED_METHOD,
975
+ {"transition": "event", "item": None, "summary": corrected},
976
+ thread_id=(self.record.get("native") or {}).get("threadId"),
977
+ )
978
+ self.record["evidence"] = self.evidence.state()
979
+ return True
980
+
981
+ def refresh_parent_notifications(self) -> bool:
982
+ """Read the current registry once; publish only a changed snapshot.
983
+
984
+ Only codexd owns this worker's evidence journal. Best-effort IPC hints
985
+ do not contain facts, and an absent owner never starts a daemon.
986
+ """
987
+ from ccx.notify.registry import iter_registrations, _delivery_id
988
+ rows = iter_registrations(include_terminal=True)
989
+ with self.lock:
990
+ if self.teardown_claimed or self._externally_terminalized:
991
+ return False
992
+ updated = {}
993
+ outcomes = {}
994
+ for reg in rows:
995
+ parent = reg.get("parent") or {}
996
+ if (parent.get("kind") != "ccx-codex"
997
+ or parent.get("workerId") != self.worker_id
998
+ or parent.get("generation") != self.daemon._uuid_store.generation):
999
+ continue
1000
+ event_id = reg.get("eventId")
1001
+ if not isinstance(event_id, str):
1002
+ continue
1003
+ delivery_id = _delivery_id(event_id)
1004
+ state = reg.get("state")
1005
+ if state in {"armed", "ready"}:
1006
+ updated[delivery_id] = {
1007
+ "id": delivery_id, "kind": "notification",
1008
+ "status": "pending", "label": "child result pending",
1009
+ "notification_id": event_id, "delivery_id": delivery_id,
1010
+ "started_at_ms": reg.get("createdAtMs"),
1011
+ }
1012
+ else:
1013
+ outcomes[delivery_id] = state
1014
+ previous = self._bg_notifications
1015
+ if previous == updated:
1016
+ return False
1017
+ self._bg_notifications = updated
1018
+ additions = [id for id in updated if id not in previous]
1019
+ removals = [id for id in previous if id not in updated]
1020
+ if len(additions) == 1 and not removals:
1021
+ transition, item = "started", updated[additions[0]]
1022
+ elif len(removals) == 1 and not additions:
1023
+ item = dict(previous[removals[0]])
1024
+ result = outcomes.get(removals[0])
1025
+ outcome = {
1026
+ "delivered": ("finished", "completed"),
1027
+ "cancelled": ("stopped", "stopped"),
1028
+ "undeliverable": ("failed", "failed"),
1029
+ }.get(result)
1030
+ if outcome is None:
1031
+ # Missing, unreadable or unknown terminal registry row
1032
+ # proves only a snapshot correction, not delivery.
1033
+ transition, item = "event", None
1034
+ else:
1035
+ transition, item["status"] = outcome
1036
+ else:
1037
+ transition, item = "event", None
1038
+ snapshot = background_summary([
1039
+ *self._bg_open.values(), *updated.values(),
1040
+ ])
1041
+ if not materially_changed(self.record.get("backgroundTaskSummary"), snapshot):
1042
+ return False
1043
+ self.record["backgroundTaskSummary"] = snapshot
1044
+ self.evidence.record(
1045
+ BACKGROUND_CHANGED_METHOD,
1046
+ {"transition": transition, "item": item, "summary": snapshot},
1047
+ thread_id=(self.record.get("native") or {}).get("threadId"),
1048
+ )
1049
+ self.record["evidence"] = self.evidence.state()
1050
+ self._save()
1051
+ return True
1052
+
1053
+ def _background_native_locked(
1054
+ self, method: str, params: dict, completed_turn: bool,
1055
+ ) -> None:
1056
+ native = self.record.get("native") or {}
1057
+ if params.get("threadId") != native.get("threadId"):
1058
+ return
1059
+ if method == "item/started":
1060
+ item = params.get("item") or {}
1061
+ tid = params.get("turnId")
1062
+ item_id = item.get("id") if isinstance(item, dict) else None
1063
+ if (item.get("type") == "commandExecution"
1064
+ and isinstance(item_id, str) and item_id
1065
+ and isinstance(tid, str) and tid
1066
+ and tid == self.record["lifecycle"].get("activeTurnId")):
1067
+ self._bg_candidates[item_id] = {
1068
+ "turn": tid, "id": "command:" + item_id,
1069
+ "kind": "command", "status": "running",
1070
+ # Deliberately no raw native command, paths or output.
1071
+ "label": "background shell", "tool_use_id": item_id,
1072
+ "started_at_ms": params.get("startedAtMs"),
1073
+ }
1074
+ elif method == "turn/completed" and completed_turn:
1075
+ turn_id = _turn_id(params)
1076
+ for item_id, item in list(self._bg_candidates.items()):
1077
+ if item["turn"] != turn_id:
1078
+ continue
1079
+ del self._bg_candidates[item_id]
1080
+ if item_id not in self._bg_open:
1081
+ self._bg_open[item_id] = item
1082
+ self._publish_background_locked("started", item)
1083
+ elif method == "item/completed":
1084
+ item = params.get("item") or {}
1085
+ item_id = item.get("id") if isinstance(item, dict) else None
1086
+ if not isinstance(item_id, str):
1087
+ return
1088
+ candidate = self._bg_candidates.get(item_id)
1089
+ if candidate is not None and candidate["turn"] == params.get("turnId"):
1090
+ del self._bg_candidates[item_id] # normal foreground command
1091
+ active = self._bg_open.get(item_id)
1092
+ if active is None or active["turn"] != params.get("turnId"):
1093
+ return
1094
+ exit_code = item.get("exitCode")
1095
+ if type(exit_code) is int:
1096
+ status = "completed" if exit_code == 0 else "failed"
1097
+ elif item.get("status") == "failed":
1098
+ status = "failed"
1099
+ else:
1100
+ status = "unknown"
1101
+ terminal = {key: value for key, value in active.items() if key != "turn"}
1102
+ terminal["status"] = status
1103
+ if type(exit_code) is int:
1104
+ terminal["exit_code"] = exit_code
1105
+ del self._bg_open[item_id]
1106
+ self._publish_background_locked(
1107
+ {"completed": "finished", "failed": "failed"}.get(status, "event"),
1108
+ terminal,
1109
+ )
1110
+
1111
+ def on_notification(self, method: str, params: dict[str, Any]) -> None:
1112
+ # Journal the authoritative native event first, while it is still the
1113
+ # thing the app-server sent, rather than after it has been folded into
1114
+ # lifecycle state, `lastTurn`, and `lastReport`. Nothing below reads
1115
+ # the journal, so a journal fault cannot change worker health or turn
1116
+ # outcome — it can only make the evidence explicitly degraded.
1117
+ if self.teardown_claimed:
1118
+ # Teardown owns/closed this worker's evidence journal.
1119
+ return
1120
+ # A separate terminal writer only revoked native *mutations*. Its
1121
+ # still-open journal can record late provider facts without resurrecting
1122
+ # this worker. This is forensic evidence, not outcome authorization.
1123
+ # A valid item/completed can arrive without item/started. Admit only
1124
+ # the live exact turn, then publish its missing sparse start before
1125
+ # journaling the native item. This is source ownership, not a reader
1126
+ # inference or a restatement of native item lines.
1127
+ if method == "item/completed":
1128
+ native_item = params.get("item")
1129
+ turn_id = params.get("turnId")
1130
+ item_id = native_item.get("id") if isinstance(native_item, dict) else None
1131
+ if (isinstance(native_item, dict)
1132
+ and native_item.get("type") == "contextCompaction"
1133
+ and isinstance(turn_id, str) and turn_id
1134
+ and isinstance(item_id, str) and item_id):
1135
+ with self.lock:
1136
+ if self.record["lifecycle"].get("activeTurnId") == turn_id:
1137
+ active = self.record.get("activeCompact")
1138
+ operation_id = (
1139
+ active.get("operationId")
1140
+ if isinstance(active, dict)
1141
+ and active.get("turnId") == turn_id else None
1142
+ )
1143
+ self._emit_compact_feed_locked(
1144
+ COMPACT_STARTED_METHOD, turn_id,
1145
+ f"{turn_id}:{item_id}", item_id=item_id,
1146
+ operation_id=operation_id,
1147
+ )
1148
+ journaled = self._journal(method, params)
1149
+ if method == "item/completed":
1150
+ # The completed user-message item is native's final statement of
1151
+ # that input; by then every copy of it has been journaled.
1152
+ from ccx.codex.delivery_recovery import ccx_client_id
1153
+ client_id = ccx_client_id(params.get("item"), self.worker_id)
1154
+ if client_id is not None:
1155
+ # Journaled above; resolved off this shared reader thread.
1156
+ self.daemon.queue_delivery_recovery(self, client_id)
1157
+ if self._externally_terminalized:
1158
+ return
1159
+ changed = False
1160
+ completed_turn = False
1161
+ with self.lock:
1162
+ if self.teardown_claimed or self._externally_terminalized:
1163
+ return
1164
+ started_id = _turn_id(params) if method == "turn/started" else None
1165
+ if started_id and (
1166
+ started_id in self.completed_turn_ids
1167
+ or started_id == (self.record.get("lastTurn") or {}).get("turnId")):
1168
+ # A re-delivered start of a turn already known completed (also
1169
+ # after a restart, from the durable evidence index) must not
1170
+ # reopen it: its duplicate completion is ignored, so the
1171
+ # worker would stay busy forever. Journaled above only.
1172
+ pass
1173
+ elif method == "turn/started":
1174
+ changed = True
1175
+ self._resumed_system_error_pending = False
1176
+ self.record["historyKnownEmpty"] = False
1177
+ active = self.record.get("activeCompact")
1178
+ if (isinstance(active, dict) and active.get("awaitingTurn")
1179
+ and not active.get("turnId") and started_id
1180
+ and started_id not in self.completed_turn_ids
1181
+ and not self.record.get("nativeTurnStart")
1182
+ and self.launch is None
1183
+ and not self.record["lifecycle"].get("activeTurnId")):
1184
+ active["turnId"] = started_id
1185
+ active["awaitingTurn"] = False
1186
+ active["nativeTurnStarted"] = True
1187
+ self._forget_native_turn_start_locked()
1188
+ self.record["lifecycle"]["state"] = "busy"
1189
+ self.record["lifecycle"]["activeTurnId"] = _turn_id(params)
1190
+ self.record["lifecycle"]["since"] = now_ms()
1191
+ elif method == "item/started":
1192
+ item = params.get("item")
1193
+ turn_id = params.get("turnId")
1194
+ item_id = item.get("id") if isinstance(item, dict) else None
1195
+ if (isinstance(item, dict)
1196
+ and item.get("type") == "contextCompaction"
1197
+ and isinstance(turn_id, str) and turn_id
1198
+ and isinstance(item_id, str) and item_id
1199
+ and self.record["lifecycle"].get("activeTurnId") == turn_id):
1200
+ active = self.record.get("activeCompact")
1201
+ operation_id = (
1202
+ active.get("operationId")
1203
+ if isinstance(active, dict)
1204
+ and active.get("turnId") == turn_id else None
1205
+ )
1206
+ # Only this producer writes neutral compaction starts.
1207
+ # Native item/started remains raw forensic evidence.
1208
+ self._emit_compact_feed_locked(
1209
+ COMPACT_STARTED_METHOD, turn_id,
1210
+ f"{turn_id}:{item_id}", operation_id=operation_id,
1211
+ item_id=item_id,
1212
+ )
1213
+ elif method == "item/completed":
1214
+ changed = True
1215
+ item = params.get("item") or {}
1216
+ if item.get("type") == "contextCompaction":
1217
+ item_turn, item_id = params.get("turnId"), item.get("id")
1218
+ if (isinstance(item_turn, str) and item_turn
1219
+ and isinstance(item_id, str) and item_id
1220
+ and self.record["lifecycle"].get("activeTurnId") == item_turn):
1221
+ self._compact_finished.setdefault(item_turn, set()).add(item_id)
1222
+ active = self.record.get("activeCompact")
1223
+ operation_id = (
1224
+ active.get("operationId")
1225
+ if isinstance(active, dict)
1226
+ and active.get("turnId") == item_turn else None
1227
+ )
1228
+ # Journaled after the original native item: exactly
1229
+ # one admitted sparse terminal for this compactId.
1230
+ self._emit_compact_feed_locked(
1231
+ COMPACT_ENDED_METHOD, item_turn,
1232
+ f"{item_turn}:{item_id}", outcome="completed",
1233
+ operation_id=operation_id, item_id=item_id,
1234
+ )
1235
+ self._observe_context_compaction_locked(
1236
+ turn_id=item_turn,
1237
+ item_id=item_id,
1238
+ completed_at_ms=params.get("completedAtMs"),
1239
+ )
1240
+ report = report_from_item(
1241
+ item,
1242
+ turn_id=params.get("turnId"),
1243
+ completed_at=params.get("completedAtMs"),
1244
+ )
1245
+ if report and should_replace_live_report(
1246
+ self.record.get("lastReport"), report
1247
+ ):
1248
+ self.record["lastReport"] = report
1249
+ elif method == "thread/tokenUsage/updated":
1250
+ changed = True
1251
+ usage = params.get("tokenUsage") or {}
1252
+ last = usage.get("last") or {}
1253
+ total = usage.get("total") or {}
1254
+ previous_usage = (
1255
+ self.record.get("tokenUsage")
1256
+ if isinstance(self.record.get("tokenUsage"), dict)
1257
+ else {}
1258
+ )
1259
+ previous_total = (
1260
+ previous_usage.get("total")
1261
+ if isinstance(previous_usage.get("total"), dict)
1262
+ else None
1263
+ )
1264
+ reset = (
1265
+ all(
1266
+ int(last.get(key) or 0) == 0
1267
+ for key in (
1268
+ "inputTokens",
1269
+ "cachedInputTokens",
1270
+ "outputTokens",
1271
+ "reasoningOutputTokens",
1272
+ )
1273
+ )
1274
+ and isinstance(previous_total, dict)
1275
+ and dict(total) == dict(previous_total)
1276
+ )
1277
+ tokens = last.get("inputTokens")
1278
+ window = usage.get("modelContextWindow")
1279
+ percent = (
1280
+ tokens / window * 100
1281
+ if (
1282
+ not reset
1283
+ and isinstance(tokens, int)
1284
+ and isinstance(window, int)
1285
+ and window > 0
1286
+ )
1287
+ else None
1288
+ )
1289
+ if reset:
1290
+ self.record["context"] = None
1291
+ else:
1292
+ self.record["context"] = {
1293
+ "state": self.record["lifecycle"]["state"],
1294
+ "tokens_used": tokens,
1295
+ "window_size": window,
1296
+ "percent_used": percent,
1297
+ "as_of_turn_ref": None,
1298
+ "as_of_ms": now_ms(),
1299
+ }
1300
+ self.record["tokenUsage"] = {
1301
+ "last": dict(last),
1302
+ "total": dict(total),
1303
+ "modelContextWindow": window,
1304
+ "asOfTurnId": params.get("turnId"),
1305
+ "asOfMs": now_ms(),
1306
+ }
1307
+
1308
+ last_compact = self.record.get("lastCompact")
1309
+ if (
1310
+ not reset
1311
+ and isinstance(last_compact, dict)
1312
+ and last_compact.get("status") == "succeeded"
1313
+ and last_compact.get("contextAfter") is None
1314
+ and last_compact.get("turnId") != params.get("turnId")
1315
+ and isinstance(self.record.get("context"), dict)
1316
+ ):
1317
+ self._capture_compaction_context_locked(last_compact)
1318
+ elif method == "thread/status/changed":
1319
+ changed = True
1320
+ status = params.get("status") or {}
1321
+ kind = status.get("type")
1322
+ flags = status.get("activeFlags") or []
1323
+ if kind in {"active", "idle"}:
1324
+ self._resumed_system_error_pending = False
1325
+ # The status flag can precede the approval request itself;
1326
+ # the request handler sets waiting once there is a gate.
1327
+ if kind == "active" and self.record.get("pendingApprovals") and any(
1328
+ flag in {"waitingOnApproval", "waitingOnUserInput"}
1329
+ for flag in flags
1330
+ ):
1331
+ self.record["lifecycle"]["state"] = "waiting"
1332
+ elif kind == "active":
1333
+ self.record["lifecycle"]["state"] = "busy"
1334
+ elif kind == "idle" and not self.record.get("pendingApprovals"):
1335
+ lifecycle = self.record["lifecycle"]
1336
+ if (
1337
+ lifecycle.get("activeTurnId")
1338
+ or self.launch is not None
1339
+ or isinstance(self.record.get("nativeTurnStart"), dict)
1340
+ or isinstance(self.record.get("activeCompact"), dict)
1341
+ ):
1342
+ # Codex publishes idle before the authoritative terminal
1343
+ # notification for both model turns and native compact.
1344
+ # Keep the operation unsettled so waiters cannot mistake
1345
+ # this status edge for turn/completion evidence.
1346
+ if lifecycle.get("state") != "crashed":
1347
+ lifecycle["state"] = "busy"
1348
+ else:
1349
+ lifecycle["state"] = "idle"
1350
+ lifecycle["activeTurnId"] = None
1351
+ elif kind == "systemError":
1352
+ if self._resumed_system_error_pending:
1353
+ # The resumed thread still carries the prior failed
1354
+ # turn's systemError. A replayed edge before any new
1355
+ # turn/started or active status cannot fail that turn.
1356
+ changed = False
1357
+ else:
1358
+ active_compact = self.record.get("activeCompact")
1359
+ compact_turn = (
1360
+ active_compact.get("turnId")
1361
+ if isinstance(active_compact, dict) else None
1362
+ )
1363
+ compact_operation = (
1364
+ active_compact.get("operationId")
1365
+ if isinstance(active_compact, dict) else None
1366
+ )
1367
+ compact_turn = compact_turn or self.record["lifecycle"].get("activeTurnId")
1368
+ if isinstance(compact_turn, str) and compact_turn:
1369
+ if compact_operation or self._compact_started.get(compact_turn):
1370
+ self._close_unfinished_compacts_locked(
1371
+ compact_turn, outcome="unknown",
1372
+ operation_id=compact_operation,
1373
+ reason="Codex thread system error",
1374
+ )
1375
+ self._abandon_active_compact_locked("Codex thread system error")
1376
+ self.record["lifecycle"]["state"] = "crashed"
1377
+ self.record["lifecycle"]["lastError"] = "Codex thread system error"
1378
+ elif method == "turn/completed":
1379
+ changed = True
1380
+ turn = params.get("turn") or {}
1381
+ if not isinstance(turn, dict):
1382
+ turn = {}
1383
+ completed_turn_id = turn.get("id")
1384
+ active_turn_id = self.record["lifecycle"].get("activeTurnId")
1385
+ last_turn_id = (self.record.get("lastTurn") or {}).get("turnId")
1386
+ if (not isinstance(completed_turn_id, str) or not completed_turn_id
1387
+ or completed_turn_id == last_turn_id
1388
+ or completed_turn_id in self.completed_turn_ids
1389
+ or active_turn_id != completed_turn_id):
1390
+ # A completion is a new outcome only for the exact native
1391
+ # turn already admitted as active by turn/started or its
1392
+ # original launch fence. In particular, idle does not prove
1393
+ # an unseen turn: a delayed A after B would roll lastTurn
1394
+ # backwards and count A again. The observed notification
1395
+ # is still journaled, including after a daemon restart.
1396
+ # A duplicate of the current pending save can retry its
1397
+ # original CAS without incrementing again.
1398
+ if journaled:
1399
+ self.record["evidence"] = self.evidence.state()
1400
+ self._save()
1401
+ return
1402
+ self.completed_turn_ids.add(completed_turn_id)
1403
+ if self.cap_turn_id == completed_turn_id:
1404
+ self.cap_turn_id = None
1405
+ self.active_caps = None
1406
+ completed_turn = True
1407
+ previous_state = self.record["lifecycle"].get("state")
1408
+ active_compact = self.record.get("activeCompact")
1409
+ requested_op = (
1410
+ active_compact.get("operationId")
1411
+ if isinstance(active_compact, dict)
1412
+ and active_compact.get("turnId") == completed_turn_id
1413
+ else None
1414
+ )
1415
+ native_compact_ended = self._compact_finished.get(completed_turn_id, set())
1416
+ if turn.get("status") in ("failed", "interrupted", "cancelled"):
1417
+ self._close_unfinished_compacts_locked(
1418
+ completed_turn_id, outcome="failed",
1419
+ operation_id=requested_op,
1420
+ reason=str(turn.get("error") or turn.get("status")),
1421
+ )
1422
+ elif (not native_compact_ended and
1423
+ (requested_op or self._compact_started.get(completed_turn_id))):
1424
+ # Completion of the native turn without the native
1425
+ # compaction item does not prove the compact itself.
1426
+ # Cover CCX-requested and automatic/native compactions.
1427
+ self._close_unfinished_compacts_locked(
1428
+ completed_turn_id, outcome="unknown",
1429
+ operation_id=requested_op,
1430
+ reason="native compact item completion not observed",
1431
+ )
1432
+ elif (completed_turn_id in self._compact_legacy
1433
+ and not self._compact_started.get(completed_turn_id)
1434
+ and not native_compact_ended
1435
+ and requested_op is None):
1436
+ # Legacy-only automatic compaction: defer until the turn's
1437
+ # accepted completion so modern item evidence wins.
1438
+ legacy_id = f"legacy:{completed_turn_id}"
1439
+ self._emit_compact_feed_locked(
1440
+ COMPACT_STARTED_METHOD, completed_turn_id, legacy_id,
1441
+ )
1442
+ self._emit_compact_feed_locked(
1443
+ COMPACT_ENDED_METHOD, completed_turn_id, legacy_id,
1444
+ outcome="completed",
1445
+ )
1446
+ if (
1447
+ isinstance(active_compact, dict)
1448
+ and active_compact.get("turnId") == completed_turn_id
1449
+ and active_compact.get("nativeTurnStarted") is True
1450
+ and isinstance(active_compact.get("contextItemId"), str)
1451
+ and bool(active_compact["contextItemId"])
1452
+ and active_compact.get("nativeItemCompletedAtMs") is not None
1453
+ and turn.get("status") == "completed"
1454
+ ):
1455
+ self._complete_active_compact_locked(
1456
+ turn_id=completed_turn_id,
1457
+ completed_at_ms=(
1458
+ int(turn.get("completedAt") * 1000)
1459
+ if isinstance(turn.get("completedAt"), (int, float))
1460
+ else None
1461
+ ),
1462
+ )
1463
+ elif (isinstance(active_compact, dict)
1464
+ and active_compact.get("turnId") == completed_turn_id):
1465
+ self._abandon_active_compact_locked(
1466
+ f"compact turn settled without success evidence ({turn.get('status')})"
1467
+ )
1468
+ self.record["lastCompact"].update(
1469
+ turnId=completed_turn_id, nativeTurnSettled=True,
1470
+ nativeTurnStatus=turn.get("status"), nativeTurnError=turn.get("error"),
1471
+ )
1472
+ last_compact = self.record.get("lastCompact")
1473
+ if (
1474
+ isinstance(last_compact, dict)
1475
+ and last_compact.get("turnId") == completed_turn_id
1476
+ and "contextAfter" in last_compact
1477
+ and last_compact.get("contextAfter") is None
1478
+ ):
1479
+ self._capture_compaction_context_locked(last_compact)
1480
+ current_turn_id = self.record["lifecycle"].get("activeTurnId")
1481
+ if current_turn_id and current_turn_id != completed_turn_id:
1482
+ # An old completion cannot clear or settle a newer turn.
1483
+ if changed:
1484
+ self._save()
1485
+ return
1486
+ self._forget_native_turn_start_locked()
1487
+ self.record["lifecycle"]["activeTurnId"] = None
1488
+ stats = self.record.setdefault("stats", {})
1489
+ stats["turns"] = stats.get("turns", 0) + 1
1490
+ # Native total is cumulative for the thread. Apply at most
1491
+ # its positive advance (not per-turn last), so the existing
1492
+ # stats counter-CAS path remains safe on a replay/rebase.
1493
+ usage = self.record.get("tokenUsage")
1494
+ total = usage.get("total") if isinstance(usage, dict) else None
1495
+ if isinstance(total, dict):
1496
+ for native_key, field in (("inputTokens", "inputTokens"),
1497
+ ("outputTokens", "outputTokens")):
1498
+ value = total.get(native_key)
1499
+ previous = stats.get(field)
1500
+ if (type(value) is int and value >= 0 and
1501
+ (previous is None or value >= previous)):
1502
+ stats[field] = value
1503
+ self.record["lastTurnAt"] = now_ms()
1504
+ self.record["lastTurn"] = _last_turn_from_native(turn)
1505
+ usage = self.record.get("tokenUsage")
1506
+ observed = self.record.get("context")
1507
+ if (isinstance(usage, dict)
1508
+ and usage.get("asOfTurnId") == completed_turn_id
1509
+ and isinstance(observed, dict)):
1510
+ self.record["lastTurn"]["context"] = {
1511
+ **observed,
1512
+ "as_of_turn_ref": "ccx-turn:codex:v1:" + completed_turn_id,
1513
+ }
1514
+ else:
1515
+ self.record["lastTurn"]["context"] = None
1516
+ active_error = self.record.get("activeTurnError")
1517
+ if (
1518
+ isinstance(active_error, dict)
1519
+ and active_error.get("turnId") == completed_turn_id
1520
+ ):
1521
+ self.record["activeTurnError"] = None
1522
+ if self.record.get("pendingApprovals"):
1523
+ self.record["lifecycle"]["state"] = "waiting"
1524
+ elif previous_state == "crashed":
1525
+ # A completed turn can arrive after the independent
1526
+ # thread/status systemError. Do not let its settlement
1527
+ # demote a worker whose runtime health is already known to
1528
+ # be bad.
1529
+ self.record["lifecycle"]["state"] = "crashed"
1530
+ elif isinstance(self.record.get("activeCompact"), dict):
1531
+ # A foreign/native turn can settle while compaction still
1532
+ # owns its own operation fence. Do not publish idle until
1533
+ # thread/compacted (or an explicit compact failure) closes it.
1534
+ self.record["lifecycle"]["state"] = "busy"
1535
+ else:
1536
+ self.record["lifecycle"]["state"] = "idle"
1537
+ self.record["lifecycle"]["since"] = now_ms()
1538
+ elif method == "serverRequest/resolved":
1539
+ changed = True
1540
+ request_id = params.get("requestId")
1541
+ self.record["pendingApprovals"] = [
1542
+ approval
1543
+ for approval in self.record.get("pendingApprovals") or []
1544
+ if str(approval.get("requestId")) != str(request_id)
1545
+ ]
1546
+ if self.record["lifecycle"]["state"] == "waiting":
1547
+ self.record["lifecycle"]["state"] = (
1548
+ "waiting"
1549
+ if self.record["pendingApprovals"]
1550
+ else (
1551
+ "busy"
1552
+ if (
1553
+ self.record["lifecycle"].get("activeTurnId")
1554
+ or isinstance(self.record.get("nativeTurnStart"), dict)
1555
+ or isinstance(self.record.get("activeCompact"), dict)
1556
+ )
1557
+ else "idle"
1558
+ )
1559
+ )
1560
+ self.record["lifecycle"]["since"] = now_ms()
1561
+ elif method == "thread/compacted":
1562
+ changed = True
1563
+ # Legacy Codex signal. Modern releases expose compaction as a
1564
+ # contextCompaction item inside an internal turn. If both arrive,
1565
+ # the turn id dedupes the deprecated notification.
1566
+ legacy_turn_id = params.get("turnId")
1567
+ if isinstance(legacy_turn_id, str) and legacy_turn_id:
1568
+ self._compact_legacy.add(legacy_turn_id)
1569
+ active_compact = self.record.get("activeCompact")
1570
+ last = self.record.get("lastCompact") or {}
1571
+ if legacy_turn_id and last.get("turnId") == legacy_turn_id:
1572
+ return
1573
+ if isinstance(active_compact, dict):
1574
+ # Deprecated thread/compacted is not a modern item and
1575
+ # never settles the requested operation on its own.
1576
+ pass
1577
+ else:
1578
+ last = self.record.get("lastCompact")
1579
+ already_seen = (
1580
+ isinstance(last, dict)
1581
+ and legacy_turn_id
1582
+ and last.get("turnId") == legacy_turn_id
1583
+ )
1584
+ if not already_seen:
1585
+ compact_count = int(self.record.get("compactCount") or 0) + 1
1586
+ self.record["compactCount"] = compact_count
1587
+ self.record["lastCompact"] = {
1588
+ "operationId": None,
1589
+ "baselineCompactCount": compact_count - 1,
1590
+ "targetCompactCount": compact_count,
1591
+ "startedAtMs": None,
1592
+ "completedAtMs": now_ms(),
1593
+ "contextBefore": None,
1594
+ "contextAfter": None,
1595
+ "turnId": legacy_turn_id,
1596
+ "contextItemId": None,
1597
+ "status": "succeeded",
1598
+ }
1599
+ self._capture_compaction_context_locked(
1600
+ self.record["lastCompact"]
1601
+ )
1602
+ # Legacy item evidence is not settlement or an input lease.
1603
+ elif method == "error":
1604
+ changed = True
1605
+ # ``error`` is a turn-scoped diagnostic. It is not evidence of
1606
+ # worker/runtime failure: Codex may recover and retry the same
1607
+ # turn, and the authoritative settled error arrives in
1608
+ # turn/completed.turn.error. Keep the native structured value
1609
+ # intact, but only for the currently active turn.
1610
+ turn_id = params.get("turnId")
1611
+ active_turn_id = self.record["lifecycle"].get("activeTurnId")
1612
+ if (
1613
+ isinstance(turn_id, str)
1614
+ and turn_id
1615
+ and turn_id == active_turn_id
1616
+ ):
1617
+ self.record["activeTurnError"] = {
1618
+ "turnId": turn_id,
1619
+ "error": params.get("error"),
1620
+ "willRetry": bool(params.get("willRetry", False)),
1621
+ "observedAtMs": now_ms(),
1622
+ }
1623
+ if method in {"item/started", "item/completed", "turn/completed"}:
1624
+ self._background_native_locked(method, params, completed_turn)
1625
+ if not journaled and not changed:
1626
+ return
1627
+ if journaled:
1628
+ self.record["evidence"] = self.evidence.state()
1629
+ self.last_activity = time.monotonic()
1630
+ # Journal the context view before the same record save that
1631
+ # publishes a new native terminal/compact state. A later ordinary
1632
+ # read-only compact-status call must not race a second record write.
1633
+ self._publish_context_locked(force=completed_turn, save=False)
1634
+ self._save()
1635
+
1636
+ if completed_turn:
1637
+ # The native notification was admitted as this worker's exact
1638
+ # active turn, and its record has been persisted. It is already
1639
+ # the original completion evidence: do not hydrate thread/read.
1640
+ # A turn that finished before the caller persisted acceptance is
1641
+ # caught by the receipt writer's journal check or reconciliation.
1642
+ if turn.get("status") == "completed":
1643
+ from ccx.codex.completion import settle_completed
1644
+ try:
1645
+ settle_completed(
1646
+ self.daemon._uuid_store, f"codex:{self.worker_id}",
1647
+ thread_id=(self.record.get("native") or {}).get("threadId"),
1648
+ turn=turn,
1649
+ live_report=self.record.get("lastReport"),
1650
+ )
1651
+ except Exception as error:
1652
+ # A receipt-side fault cannot take down the shared native
1653
+ # notification reader or change the completed turn. The
1654
+ # original journal and recovery path remain available.
1655
+ print("ccx codexd: original completion settlement pending: "
1656
+ + type(error).__name__, file=sys.stderr, flush=True)
1657
+ from ccx.notify.daemon import revive_notify_daemon_after_turn
1658
+ revive_notify_daemon_after_turn("codex", self.worker_id)
1659
+
1660
+ def on_server_request(
1661
+ self, request_id: Any, method: str, params: dict[str, Any]
1662
+ ) -> None:
1663
+ kind = APPROVAL_METHODS.get(method)
1664
+ if not kind or (
1665
+ method == "item/fileChange/requestApproval"
1666
+ and params.get("grantRoot")
1667
+ ):
1668
+ self.client.reject_request(
1669
+ request_id,
1670
+ f"CCX does not broker app-server request {method}",
1671
+ )
1672
+ return
1673
+ preview = {
1674
+ "requestId": request_id,
1675
+ # App-server request ids restart with the app-server, so a stale
1676
+ # id could name a newer request; this one never repeats.
1677
+ "gateId": uuid.uuid4().hex,
1678
+ # Native request IDs are connection-scoped. An abruptly killed
1679
+ # daemon can leave old approvals on disk; they cannot be replayed.
1680
+ "runtimeId": self.daemon.instance_id,
1681
+ "kind": kind,
1682
+ "turnId": params.get("turnId"),
1683
+ "itemId": params.get("itemId"),
1684
+ "command": _bounded(params.get("command")),
1685
+ "cwd": _bounded(params.get("cwd"), 500),
1686
+ "reason": _bounded(params.get("reason")),
1687
+ "createdAt": now_ms(),
1688
+ }
1689
+ with self.lock:
1690
+ torn_down = (self.teardown_claimed or self._externally_terminalized
1691
+ or self.record["lifecycle"].get("state") in _TERMINAL_STATES)
1692
+ if not torn_down:
1693
+ self.record["pendingApprovals"].append(preview)
1694
+ self.record["lifecycle"]["state"] = "waiting"
1695
+ self.record["lifecycle"]["since"] = now_ms()
1696
+ self._save()
1697
+ if torn_down:
1698
+ self.client.reject_request(
1699
+ request_id, f"CCX worker {self.short} is being unloaded"
1700
+ )
1701
+
1702
+ def _select_approval(self, gate_id: str) -> dict[str, Any] | None:
1703
+ """Call under self.lock; only this runtime's exact CCX gateId may act."""
1704
+ if not isinstance(gate_id, str) or not gate_id or len(gate_id) > 128:
1705
+ return None
1706
+ for approval in self.record.get("pendingApprovals") or []:
1707
+ if (approval.get("gateId") == gate_id
1708
+ and approval.get("runtimeId") == self.daemon.instance_id):
1709
+ return approval
1710
+ return None
1711
+
1712
+ def kill(self, observe_seconds: float | None = None) -> dict[str, Any]: # noqa: D401
1713
+ """Terminate this worker and unload it from the shared daemon.
1714
+
1715
+ Codex has no per-worker process to signal: every loaded worker is a
1716
+ thread multiplexed on the one shared ``codex app-server`` child, and the
1717
+ daemon PID and app-server PID stamped into each record are device-wide,
1718
+ not worker-owned. Killing a worker therefore means unloading exactly this
1719
+ worker — never signalling either shared process, and never disturbing a
1720
+ sibling.
1721
+
1722
+ The active turn is cancelled best-effort first; a refusal or a transport
1723
+ failure there does not stop the unload, because the unload is what the
1724
+ caller asked for. The native ``threadId`` is preserved and the thread is
1725
+ deliberately not archived, so a later ``send``/``resume`` reloads the
1726
+ same conversation immediately.
1727
+ """
1728
+ # one critical section decides the race. `self.lock` totally orders this
1729
+ # against the loader's two commit points (the native turn launch
1730
+ # publication and `_commit_state`), so exactly one of us wins:
1731
+ # * we go first -> the loader sees `killed` and does nothing at all;
1732
+ # * loader first -> we see its `launch` here and own the cancellation.
1733
+ # It deliberately does not take `load_lock`: waiting there would stall
1734
+ # kill behind a 90s `thread/resume`.
1735
+ with self.lock:
1736
+ launch = self.launch
1737
+ unresolved_start = self.record.get("nativeTurnStart")
1738
+ if launch is None and isinstance(unresolved_start, dict):
1739
+ # The local JSON-RPC request has already settled without exact
1740
+ # native evidence, so the in-memory launch fence can no longer
1741
+ # hand us a turn id. The native request may still land later;
1742
+ # unloading now would falsely confirm a kill.
1743
+ self.kill_intent.clear()
1744
+ return {
1745
+ "ok": False,
1746
+ "pending": True,
1747
+ "error": (
1748
+ f"worker {self.short} has an indeterminate native "
1749
+ "turn/start; kill was NOT completed and is not confirmed "
1750
+ "— retry after turn evidence arrives or the Codex "
1751
+ "app-server is replaced"
1752
+ ),
1753
+ "state": self.record["lifecycle"]["state"],
1754
+ }
1755
+ self.kill_intent.set()
1756
+ self.killed.set()
1757
+ state = self.record["lifecycle"]["state"]
1758
+ turn_id = self.record["lifecycle"].get("activeTurnId")
1759
+ thread_id = (self.record.get("native") or {}).get("threadId") or self.record.get("threadId")
1760
+ active_compact = self.record.get("activeCompact")
1761
+ compact_turn = (
1762
+ active_compact.get("turnId")
1763
+ if isinstance(active_compact, dict) else None
1764
+ )
1765
+ compact_operation = (
1766
+ active_compact.get("operationId")
1767
+ if isinstance(active_compact, dict) else None
1768
+ )
1769
+ compact_turn = compact_turn or turn_id
1770
+ if isinstance(compact_turn, str) and compact_turn:
1771
+ if compact_operation or self._compact_started.get(compact_turn):
1772
+ self._close_unfinished_compacts_locked(
1773
+ compact_turn, outcome="unknown",
1774
+ operation_id=compact_operation,
1775
+ reason="worker killed during compaction",
1776
+ )
1777
+ self._abandon_active_compact_locked("worker killed during compaction")
1778
+
1779
+ interrupted = False
1780
+ launch_cancelled = None
1781
+ if launch is not None:
1782
+ # A `turn/start` was already irrevocably in flight when we won. It
1783
+ # is not cancellable until it returns a turn id, so wait for that —
1784
+ # bounded by the caller's own remaining budget, capped.
1785
+ window = (_LAUNCH_SETTLE_CAP_S if observe_seconds is None
1786
+ else max(0.0, min(float(observe_seconds),
1787
+ _LAUNCH_SETTLE_CAP_S)))
1788
+ if not launch.done.wait(window):
1789
+ # unresolved. The turn may be accepted the moment after we
1790
+ # answer, so reporting a confirmed kill here would be exactly
1791
+ # the lie this operation must not tell. Stay loaded and
1792
+ # nonterminal so the caller's reconciliation sees a target that
1793
+ # is still pending; the `killed` marker guarantees the loader
1794
+ # cancels as soon as the turn id becomes knowable and then
1795
+ # terminalizes and unloads. A later kill is idempotent.
1796
+ return {
1797
+ "ok": False,
1798
+ "pending": True,
1799
+ "error": (f"worker {self.short} has a turn launch still in "
1800
+ f"flight after {window:g}s; it will be cancelled "
1801
+ f"and unloaded when the request returns — retry "
1802
+ f"kill to confirm"),
1803
+ "state": self.state(),
1804
+ }
1805
+ if not isinstance(launch.turn_id, str) or not launch.turn_id:
1806
+ # A native start returned without an exact turn ID. The
1807
+ # original mutation may still land; no unload or success.
1808
+ return {"ok": False, "pending": True,
1809
+ "error": "original native turn/start is unresolved; kill not confirmed",
1810
+ "state": self.state()}
1811
+ launch_cancelled = self._interrupt_turn(launch.turn_id)
1812
+ interrupted = launch_cancelled
1813
+ elif state in {"busy", "waiting"} and turn_id and thread_id:
1814
+ interrupted = self._interrupt_turn(turn_id)
1815
+ self._state("dead")
1816
+ # Clears activeTurnId/pendingApprovals, removes this worker alone from
1817
+ # `workers` and `by_thread`, and releases its native thread without
1818
+ # waiting for Codex to answer.
1819
+ self.daemon._unload(self, terminal_state="dead")
1820
+ return {
1821
+ "ok": True,
1822
+ "killed": True,
1823
+ "unloaded": True,
1824
+ "interrupted": interrupted,
1825
+ # Only present when this kill raced a turn launch, so a caller can
1826
+ # tell "nothing was running" from "we cancelled what was starting".
1827
+ "launchCancelled": launch_cancelled,
1828
+ "state": "dead",
1829
+ "threadId": thread_id,
1830
+ }
1831
+
1832
+ def _original_notification_thread(self) -> str:
1833
+ """The one globally claimed parent thread for both read and wake."""
1834
+ native = self.record.get("native") or {}
1835
+ thread_id = native.get("threadId")
1836
+ if (not isinstance(thread_id, str) or not thread_id
1837
+ or native.get("threadSource") != self.daemon._uuid_store.codex_source(
1838
+ f"codex:{self.worker_id}"
1839
+ )):
1840
+ raise RuntimeError("original parent Codex native thread/source mismatch")
1841
+ self.daemon._uuid_store.authorize_native_action(
1842
+ f"codex:{self.worker_id}", "send",
1843
+ )
1844
+ return thread_id
1845
+
1846
+ def _parent_notification_match(
1847
+ self, delivery_id: str, *, legacy_event_id: str | None = None,
1848
+ name: str = "worker_notification",
1849
+ ) -> dict[str, Any] | None:
1850
+ with self.lock:
1851
+ fences = self.record.get("parentNotifyFences")
1852
+ fence = fences.get(delivery_id) if isinstance(fences, dict) else None
1853
+ return notification_delivery_match(
1854
+ self.client.request,
1855
+ self._original_notification_thread(),
1856
+ delivery_id,
1857
+ legacy_event_id=legacy_event_id,
1858
+ stop_at_turn_id=fence if isinstance(fence, str) else None,
1859
+ name=name,
1860
+ )
1861
+
1862
+ def find_parent_notification(
1863
+ self, delivery_id: str, payload: dict[str, Any], *,
1864
+ name: str = "worker_notification",
1865
+ ) -> dict[str, Any] | None:
1866
+ """Read-only: has this notification already landed in the thread?
1867
+
1868
+ Runs without `operation_lock` (a long parent history must not stall
1869
+ the parent's own sends and kills); `_start_parent_notification` then
1870
+ covers an attempt this daemon launched meanwhile.
1871
+ """
1872
+ legacy_event_id = (
1873
+ payload.get("eventId") if payload.get("deliveryId") is None else None
1874
+ )
1875
+ return self._parent_notification_match(
1876
+ delivery_id,
1877
+ legacy_event_id=str(legacy_event_id) if legacy_event_id else None,
1878
+ name=name,
1879
+ )
1880
+
1881
+ def _fence_parent_notification_locked(self, delivery_id: str) -> None:
1882
+ """Remember the last turn settled before a delivery's first launch.
1883
+
1884
+ The notification can only land in a later turn, so a retry's history
1885
+ walk stops there. Only the first launch sets it: a retry's baseline may
1886
+ already be newer than the turn an earlier ambiguous attempt joined.
1887
+ """
1888
+ fences = self.record.get("parentNotifyFences")
1889
+ if not isinstance(fences, dict):
1890
+ fences = self.record["parentNotifyFences"] = {}
1891
+ if delivery_id in fences:
1892
+ return
1893
+ last_turn = self.record.get("lastTurn")
1894
+ fences[delivery_id] = (
1895
+ last_turn.get("turnId") if isinstance(last_turn, dict) else None
1896
+ )
1897
+ for stale in list(fences)[:-_PARENT_NOTIFY_FENCES]:
1898
+ del fences[stale]
1899
+
1900
+ def _compact_refusal(self) -> dict[str, Any] | None:
1901
+ with self.lock:
1902
+ active = self.record.get("activeCompact")
1903
+ if not isinstance(active, dict):
1904
+ return None
1905
+ return {"ok": False, "state": self.record["lifecycle"]["state"],
1906
+ "error": "Compaction owns the input surface",
1907
+ "reason": "compaction-busy", "retryable": True,
1908
+ "definitive": True, "operationId": active.get("operationId")}
1909
+
1910
+ def _start_parent_notification(
1911
+ self,
1912
+ delivery_id: str,
1913
+ payload: dict[str, Any],
1914
+ *,
1915
+ reconcile: bool = True,
1916
+ history: tuple[dict[str, Any] | None] | None = None,
1917
+ namespace: str = "ccx",
1918
+ name: str = "worker_notification",
1919
+ ) -> dict[str, Any]:
1920
+ """Start-or-steer one machine continuation with kill-safe admission.
1921
+
1922
+ Unlike a normal send this intentionally accepts an idle or active parent
1923
+ turn. Codex's turn/start semantics start a new turn when idle and steer
1924
+ the tool output into the active turn when busy. The launch marker keeps
1925
+ kill-vs-turn/start ordering as strong as ordinary user turns.
1926
+ """
1927
+ if history is not None:
1928
+ # Walked by the caller before it took the operation lock.
1929
+ match = history[0]
1930
+ elif reconcile:
1931
+ match = self.find_parent_notification(
1932
+ delivery_id, payload, name=name,
1933
+ )
1934
+ else:
1935
+ match = None
1936
+ if match is None and delivery_id in self.launched_notifications:
1937
+ # An attempt that held the lock while the walk ran outside it.
1938
+ match = {"turnId": self.launched_notifications[delivery_id]}
1939
+ if match is not None:
1940
+ with self.lock:
1941
+ marker = self.record.get("nativeTurnStart")
1942
+ if (
1943
+ isinstance(marker, dict)
1944
+ and marker.get("kind") == (
1945
+ "parent-notification" if namespace == "ccx" else "machine-wake"
1946
+ )
1947
+ and marker.get("deliveryId") == delivery_id
1948
+ ):
1949
+ self._forget_native_turn_start_locked()
1950
+ self._save()
1951
+ return {
1952
+ "ok": True,
1953
+ "state": self.state(),
1954
+ "reconciled": True,
1955
+ "workerId": self.worker_id,
1956
+ "threadId": self._original_notification_thread(),
1957
+ "eventId": payload.get("eventId"),
1958
+ "deliveryId": delivery_id,
1959
+ "turnId": match.get("turnId"),
1960
+ }
1961
+ # A previous attempt can still land after the network reply is lost.
1962
+ # Reconcile-only retries may ACK positive history, never retransmit
1963
+ # merely because the history endpoint finds no matching item.
1964
+ if reconcile:
1965
+ return {"ok": False, "pending": True, "state": self.state(),
1966
+ "error": "original machine wake was attempted; native placement remains unknown"}
1967
+ refusal = self._compact_refusal()
1968
+ if refusal:
1969
+ return refusal
1970
+ params = {
1971
+ "threadId": self._original_notification_thread(),
1972
+ "input": [],
1973
+ "turnTrigger": "ccx-worker-notification",
1974
+ "toolOutput": {
1975
+ "namespace": namespace,
1976
+ "name": name,
1977
+ "output": json.dumps(payload, separators=(",", ":")),
1978
+ },
1979
+ }
1980
+ with self.lock:
1981
+ if self.killed.is_set() or self.kill_intent.is_set():
1982
+ return {
1983
+ "ok": False,
1984
+ **({"definitive": True} if namespace != "ccx" else {}),
1985
+ "error": f"worker {self.short} is being killed",
1986
+ "state": self.record["lifecycle"]["state"],
1987
+ }
1988
+ if (not self.loaded or self.record["lifecycle"]["state"]
1989
+ not in {"idle", "busy", "waiting"}):
1990
+ return {"ok": False, "definitive": True,
1991
+ "error": "original native parent not subscribed/active",
1992
+ "state": self.record["lifecycle"]["state"]}
1993
+ if self.launch is not None:
1994
+ return {"ok": False, "definitive": True,
1995
+ "error": "another original native turn/start is in flight"}
1996
+ if isinstance(self.record.get("nativeTurnStart"), dict):
1997
+ return {"ok": False, "pending": True,
1998
+ "error": "another original native turn/start remains unresolved"}
1999
+ previous_state = self.record["lifecycle"]["state"]
2000
+ previous_turn_id = self.record["lifecycle"].get("activeTurnId")
2001
+ launch = _TurnLaunch()
2002
+ self.launch = launch
2003
+ # A turn blocked on an approval stays `waiting`: the notification
2004
+ # is steered into that same turn, and publishing `busy` would hide
2005
+ # the pending approval from every waiter.
2006
+ if previous_state != "waiting":
2007
+ self.record["lifecycle"]["state"] = "busy"
2008
+ self.record["lifecycle"]["since"] = now_ms()
2009
+ self.record["nativeTurnStart"] = self._native_turn_start_marker(
2010
+ kind="parent-notification" if namespace == "ccx" else "machine-wake",
2011
+ delivery_id=delivery_id,
2012
+ )
2013
+ self._fence_parent_notification_locked(delivery_id)
2014
+ self.last_activity = time.monotonic()
2015
+ self._save()
2016
+ turn_id = None
2017
+ request_returned = False
2018
+ try:
2019
+ result = self.client.request("turn/start", params, timeout=60)
2020
+ turn_id = _turn_id(result)
2021
+ request_returned = True
2022
+ except ProtocolError as error:
2023
+ with self.lock:
2024
+ if self._native_mutation_error_is_ambiguous(error):
2025
+ self._mark_native_turn_start_unknown_locked(error)
2026
+ else:
2027
+ self._clear_native_turn_start_locked()
2028
+ # Only an explicit native refusal proves the notification
2029
+ # did not mutate the thread, so only then restore what we
2030
+ # owned before the request.
2031
+ if (
2032
+ self.record["lifecycle"].get("activeTurnId")
2033
+ == previous_turn_id
2034
+ ):
2035
+ self.record["lifecycle"]["state"] = previous_state
2036
+ self.record["lifecycle"]["since"] = now_ms()
2037
+ self._save()
2038
+ raise
2039
+ except Exception as error:
2040
+ with self.lock:
2041
+ self._mark_native_turn_start_unknown_locked(error)
2042
+ raise
2043
+ finally:
2044
+ with self.lock:
2045
+ if self.launch is launch:
2046
+ self.launch = None
2047
+ launch.turn_id = turn_id
2048
+ if turn_id is not None:
2049
+ self._forget_native_turn_start_locked()
2050
+ if request_returned and isinstance(turn_id, str) and turn_id:
2051
+ self.launched_notifications[delivery_id] = turn_id
2052
+ lifecycle = self.record["lifecycle"]
2053
+ if request_returned and lifecycle["state"] in {"busy", "waiting"}:
2054
+ lifecycle["activeTurnId"] = (
2055
+ lifecycle.get("activeTurnId") or turn_id
2056
+ )
2057
+ if self.record.get("pendingApprovals"):
2058
+ # An approval that arrived while the request was in
2059
+ # flight still gates the turn.
2060
+ lifecycle["state"] = "waiting"
2061
+ self._save()
2062
+ launch.done.set()
2063
+ if self.killed.is_set():
2064
+ self._interrupt_turn(turn_id)
2065
+ return {
2066
+ "ok": False,
2067
+ "error": f"worker {self.short} was killed while starting notification",
2068
+ "state": self.state(),
2069
+ }
2070
+ if not isinstance(turn_id, str) or not turn_id:
2071
+ return {"ok": False, "pending": True, "state": self.state(),
2072
+ "error": "original machine wake has no positive native turn ID"}
2073
+ return {
2074
+ "ok": True,
2075
+ "state": self.state(),
2076
+ "turnId": turn_id,
2077
+ "reconciled": False,
2078
+ "workerId": self.worker_id,
2079
+ "threadId": self._original_notification_thread(),
2080
+ "eventId": payload.get("eventId"),
2081
+ "deliveryId": delivery_id,
2082
+ }
2083
+
2084
+ def handle_operation(
2085
+ self,
2086
+ message: dict[str, Any],
2087
+ *,
2088
+ history: tuple[dict[str, Any] | None] | None = None,
2089
+ ) -> dict[str, Any]:
2090
+ op = message.get("op")
2091
+ with self.lock:
2092
+ state = self.record["lifecycle"]["state"]
2093
+ if op == "kill":
2094
+ # Ahead of the booting/resuming guard on purpose: a worker stuck
2095
+ # mid-load is exactly one a caller needs to be able to kill.
2096
+ # `CodexDaemon.handle` routes kills through `_kill_worker` (which
2097
+ # owns the intent/budget protocol); this branch keeps a direct
2098
+ # caller correct.
2099
+ return self.kill(message.get("observeSeconds"))
2100
+ if self.killed.is_set() or self.kill_intent.is_set():
2101
+ # A kill was accepted but could not be confirmed yet (an unresolved
2102
+ # turn launch, or a prior operation that has not settled). The
2103
+ # worker stays registered so reconciliation can see it, but it must
2104
+ # accept no new work. `handle` refuses before taking the operation
2105
+ # lock; this repeats the check for an op that passed that gate and
2106
+ # only then found the intent published.
2107
+ return {"ok": False, "definitive": True,
2108
+ "error": f"worker {self.short} is being killed", "state": state}
2109
+ if state in {"booting", "resuming"}:
2110
+ return {"ok": False, "error": f"worker is {state}", "state": state,
2111
+ **({"definitive": True} if op == "machine-wake" else {})}
2112
+
2113
+ if op in {"parent-notify", "machine-wake"}:
2114
+ event_id = message.get("eventId")
2115
+ delivery_id = message.get("deliveryId") or event_id
2116
+ payload = message.get("payload")
2117
+ if not isinstance(event_id, str) or not event_id:
2118
+ return {"ok": False, "error": "parent-notify requires eventId", "state": state}
2119
+ if not isinstance(delivery_id, str) or not delivery_id:
2120
+ return {"ok": False, "error": "parent-notify requires deliveryId", "state": state}
2121
+ if not isinstance(payload, dict):
2122
+ return {"ok": False, "error": "parent-notify requires payload", "state": state}
2123
+ try:
2124
+ return self._start_parent_notification(
2125
+ delivery_id, payload,
2126
+ reconcile=bool(message.get("reconcile", True)),
2127
+ history=history,
2128
+ namespace=(message["namespace"] if op == "machine-wake" else "ccx"),
2129
+ name=(message["name"] if op == "machine-wake" else "worker_notification"),
2130
+ )
2131
+ except ProtocolError as error:
2132
+ failure = _protocol_failure(error)
2133
+ failure["definitive"] = not self._native_mutation_error_is_ambiguous(error)
2134
+ return failure
2135
+
2136
+ if op == "interrupt":
2137
+ expected_turn_id = message.get("expectedTurnId")
2138
+ current_turn_id = self.record["lifecycle"].get("activeTurnId")
2139
+ if (
2140
+ expected_turn_id is not None
2141
+ and expected_turn_id != current_turn_id
2142
+ ):
2143
+ return {
2144
+ "ok": False,
2145
+ "error": "active turn changed",
2146
+ "state": state,
2147
+ "turnId": current_turn_id,
2148
+ "definitive": True,
2149
+ }
2150
+ if state not in {"busy", "waiting"}:
2151
+ return {"ok": False, "definitive": True,
2152
+ "error": "no active turn", "state": state}
2153
+ turn_id = self.record["lifecycle"].get("activeTurnId")
2154
+ if not turn_id:
2155
+ return {"ok": False, "definitive": True,
2156
+ "error": "active turn id is unavailable", "state": state}
2157
+ # Full-UUID records keep the original thread under native.threadId.
2158
+ native = self.record.get("native") or {}
2159
+ self.client.request("turn/interrupt", {
2160
+ "threadId": native.get("threadId") or self.record["threadId"],
2161
+ "turnId": turn_id,
2162
+ })
2163
+ return {"ok": True, "state": state, "turnId": turn_id}
2164
+ if op in {"approve", "reject"}:
2165
+ # Existing operation_lock serializes decisions and unload. The
2166
+ # reader can still publish/resolve gates: select under worker lock,
2167
+ # then release it before the native server-request response.
2168
+ with self.lock:
2169
+ approval = self._select_approval(message.get("gateId"))
2170
+ gate_state = self.record["lifecycle"]["state"]
2171
+ native = self.record.get("native") or {}
2172
+ turn = self.record["lifecycle"].get("activeTurnId")
2173
+ thread = native.get("threadId")
2174
+ if (approval is None or gate_state != "waiting"
2175
+ or not isinstance(turn, str) or not turn
2176
+ or approval.get("turnId") != turn
2177
+ or not isinstance(thread, str) or not thread
2178
+ or native.get("threadSource") != self.daemon._uuid_store.codex_source(
2179
+ f"codex:{self.worker_id}"
2180
+ )
2181
+ or self.daemon.by_thread.get(thread) != self.worker_id):
2182
+ return {"ok": False, "definitive": True,
2183
+ "error": "no current original Codex approval for gateId",
2184
+ "state": gate_state}
2185
+ self.approval_decisions[_approval_key(approval)] = (
2186
+ "accept" if op == "approve" else "decline"
2187
+ )
2188
+ if isinstance(message.get("responseId"), str):
2189
+ self.approval_response_ids[_approval_key(approval)] = (
2190
+ message["responseId"]
2191
+ )
2192
+ self.client.respond(
2193
+ approval["requestId"],
2194
+ {"decision": "accept" if op == "approve" else "decline"},
2195
+ )
2196
+ with self.lock:
2197
+ self.record["pendingApprovals"] = [
2198
+ value for value in self.record["pendingApprovals"]
2199
+ if value is not approval
2200
+ ]
2201
+ # Same resolution as the serverRequest/resolved notification:
2202
+ # without an active turn the worker is idle, not busy — forcing
2203
+ # busy here would wedge a worker whose turn already completed.
2204
+ self.record["lifecycle"]["state"] = (
2205
+ "waiting"
2206
+ if self.record["pendingApprovals"]
2207
+ else (
2208
+ "busy"
2209
+ if self.record["lifecycle"].get("activeTurnId")
2210
+ else "idle"
2211
+ )
2212
+ )
2213
+ self.record["lifecycle"]["since"] = now_ms()
2214
+ self._save()
2215
+ return {
2216
+ "ok": True,
2217
+ "state": self.record["lifecycle"]["state"],
2218
+ "requestId": approval["requestId"],
2219
+ "gateId": approval.get("gateId"),
2220
+ "workerId": self.worker_id,
2221
+ "threadId": thread,
2222
+ "turnId": approval.get("turnId"),
2223
+ "decision": "accept" if op == "approve" else "decline",
2224
+ }
2225
+ if op == "compact":
2226
+ from ccx.codex.ops import compact_intent_digest
2227
+ from ccx.store.store import _digest
2228
+
2229
+ operation_id = message.get("operationId")
2230
+ if (not isinstance(operation_id, str) or not operation_id
2231
+ or len(operation_id) > 128 or "\x00" in operation_id):
2232
+ return {"ok": False, "definitive": True,
2233
+ "error": "compact requires caller operationId; no native request"}
2234
+ ctx = self.daemon._uuid_store
2235
+ try:
2236
+ owner = ctx.resolve(f"codex:{self.worker_id}")
2237
+ original = ctx.record(owner.worker)
2238
+ native = original.get("native") or {}
2239
+ source = ctx.codex_source(owner.worker)
2240
+ thread_id = native.get("threadId")
2241
+ activation = ctx.native_activation(owner.worker)
2242
+ if (owner.engine != "codex"
2243
+ or activation.get("state") != "confirmed"
2244
+ or activation.get("source") != source
2245
+ or not isinstance(thread_id, str) or not thread_id
2246
+ or native.get("threadSource") != source
2247
+ or (self.record.get("native") or {}).get("threadId") != thread_id
2248
+ or self.daemon.by_thread.get(thread_id) != self.worker_id
2249
+ or message.get("threadId") != thread_id):
2250
+ raise ValueError("original Codex native thread/source unverified")
2251
+ effect = ctx.read_receipt(owner.worker, "codex-compact", operation_id)
2252
+ digest = compact_intent_digest(
2253
+ owner.worker_id, ctx.generation, thread_id, source, operation_id,
2254
+ )
2255
+ if (not isinstance(effect, dict)
2256
+ or effect.get("intentSha256") != digest
2257
+ or effect.get("workerId") != owner.worker_id
2258
+ or effect.get("engine") != "codex"
2259
+ or effect.get("generation") != ctx.generation
2260
+ or effect.get("family") != "codex-compact"
2261
+ or effect.get("action") != "compact"
2262
+ or effect.get("state") not in (
2263
+ "submitted-unknown", "accepted", "rejected"
2264
+ )):
2265
+ raise ValueError("original full-UUID submitted compact receipt absent")
2266
+ ctx.revalidate()
2267
+ except Exception as error:
2268
+ return {"ok": False, "definitive": True,
2269
+ "error": f"Codex compact original admission denied: {error}"}
2270
+
2271
+ identity = {
2272
+ "workerId": owner.worker_id, "engine": "codex",
2273
+ "generation": ctx.generation, "nativeSource": source,
2274
+ "threadId": thread_id, "operationId": operation_id,
2275
+ }
2276
+
2277
+ def acknowledged(started_at: int, *, accepted: bool,
2278
+ reason: str | None = None, replayed: bool = False) -> dict:
2279
+ data = {
2280
+ **identity, "startedAtMs": started_at,
2281
+ "nativeEvidenceSha256": _digest({
2282
+ **identity, "startedAtMs": started_at,
2283
+ "admission": "thread/compact/start",
2284
+ }),
2285
+ }
2286
+ if accepted:
2287
+ return {**data, "ok": True, "nativeStatus": "accepted",
2288
+ "replayed": replayed, "state": self.record["lifecycle"]["state"]}
2289
+ return {**data, "ok": False, "definitive": True, "noEffect": True,
2290
+ "error": reason or "compact refused before native request",
2291
+ "state": self.record["lifecycle"]["state"]}
2292
+
2293
+ # Same-ID replay must precede the idle gate: a live compact owns
2294
+ # busy and may have outlived a lost caller ACK. Never paste twice.
2295
+ with self.lock:
2296
+ active = self.record.get("activeCompact")
2297
+ last = self.record.get("lastCompact")
2298
+ match = next((
2299
+ marker for marker in (active, last)
2300
+ if isinstance(marker, dict)
2301
+ and marker.get("operationId") == operation_id
2302
+ ), None)
2303
+ if match is not None:
2304
+ if effect["state"] == "rejected":
2305
+ return {"ok": False, "error": "rejected compact has a marker",
2306
+ "reconcileRequired": True}
2307
+ started_at = match.get("startedAtMs")
2308
+ if type(started_at) is not int:
2309
+ return {"ok": False, "reconcileRequired": True,
2310
+ "error": "existing compact marker has no original start"}
2311
+ return acknowledged(started_at, accepted=True, replayed=True)
2312
+ if effect["state"] != "submitted-unknown":
2313
+ return acknowledged(now_ms(), accepted=False,
2314
+ reason="compact effect already settled; no resubmit")
2315
+ state = self.record["lifecycle"]["state"]
2316
+ if (state != "idle" or isinstance(active, dict)
2317
+ or self.launch is not None
2318
+ or self.record.get("pendingApprovals")):
2319
+ return acknowledged(now_ms(), accepted=False,
2320
+ reason=f"compact requires idle worker, state={state}")
2321
+ before = self.record.get("context")
2322
+ if isinstance(before, dict):
2323
+ before = {**before, "state": state}
2324
+ baseline = int(self.record.get("compactCount") or 0)
2325
+ started_at = now_ms()
2326
+ self.record["activeCompact"] = {
2327
+ "operationId": operation_id,
2328
+ "awaitingTurn": True,
2329
+ "daemonInstanceId": self._native_create_daemon_id(),
2330
+ "baselineCompactCount": baseline,
2331
+ "targetCompactCount": baseline + 1,
2332
+ "startedAtMs": started_at,
2333
+ "contextBefore": before,
2334
+ }
2335
+ self.record["lifecycle"]["state"] = "busy"
2336
+ self.record["lifecycle"]["since"] = started_at
2337
+ # Native submission is impossible until its exact durable
2338
+ # marker and earlier global submitted intent both exist.
2339
+ self._publish_context_locked(force=True, save=False)
2340
+ self._save()
2341
+ try:
2342
+ self.client.request(
2343
+ "thread/compact/start", {"threadId": thread_id}, timeout=1.5,
2344
+ )
2345
+ except ProtocolError as error:
2346
+ definite = isinstance(error, ProtocolNotSent) or (
2347
+ error.request_id is not None
2348
+ and not isinstance(error, ProtocolTransportError)
2349
+ )
2350
+ if definite:
2351
+ with self.lock:
2352
+ active = self.record.get("activeCompact")
2353
+ if (isinstance(active, dict)
2354
+ and active.get("operationId") == operation_id):
2355
+ self.record["activeCompact"] = None
2356
+ self.record["lifecycle"]["state"] = "idle"
2357
+ self.record["lifecycle"]["since"] = now_ms()
2358
+ self._publish_context_locked(force=True, save=False)
2359
+ self._save()
2360
+ return acknowledged(
2361
+ started_at, accepted=False, reason=str(error),
2362
+ )
2363
+ # Transport ambiguity: persisted activeCompact stays busy,
2364
+ # fenced by this exact global receipt for same-ID inspection.
2365
+ return {**identity, **_protocol_failure(error),
2366
+ "state": "busy", "reconcileRequired": True}
2367
+ # Native admission ACK is not contextCompaction completion.
2368
+ return acknowledged(started_at, accepted=True)
2369
+ return {"ok": False, "error": f"unknown worker operation {op!r}"}
2370
+
2371
+ def public_status(self) -> dict[str, Any]:
2372
+ with self.lock:
2373
+ lifecycle = dict(self.record["lifecycle"])
2374
+ context = self.record.get("context")
2375
+ if context:
2376
+ context = {**context, "state": lifecycle.get("state")}
2377
+ return {
2378
+ "short": self.short,
2379
+ "threadId": (self.record.get("native") or {}).get("threadId"),
2380
+ "name": self.record.get("name"),
2381
+ "cwd": self.record.get("cwd"),
2382
+ "state": lifecycle.get("state"),
2383
+ "activeTurnId": lifecycle.get("activeTurnId"),
2384
+ "compacting": bool(self.record.get("activeCompact")),
2385
+ "pendingApprovals": list(self.record.get("pendingApprovals") or []),
2386
+ "lastReport": self.record.get("lastReport"),
2387
+ "lastTurn": self.record.get("lastTurn"),
2388
+ "context": context,
2389
+ "lastError": lifecycle.get("lastError"),
2390
+ "activeTurnError": self.record.get("activeTurnError"),
2391
+ # Evidence health travels beside worker health, never as part
2392
+ # of it: a worker whose journal degraded is still idle. Null
2393
+ # for a worker that has produced no evidence at all.
2394
+ "evidence": self.record.get("evidence"),
2395
+ "persistenceError": self.persistence_error,
2396
+ }