@agentsdock/server 1.0.7-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +19 -0
- package/README.md +60 -0
- package/npm/cli.cjs +289 -0
- package/package.json +32 -0
- package/server/LICENSE +202 -0
- package/server/NOTICE +19 -0
- package/server/VERSION +1 -0
- package/server/activation_transaction.py +2458 -0
- package/server/agent_server.py +93226 -0
- package/server/agentsdock_chats.py +1237 -0
- package/server/agentsdock_emergency.py +338 -0
- package/server/agentsdock_jobs.py +640 -0
- package/server/agentsdock_mail.py +325 -0
- package/server/agentsdock_publish.py +404 -0
- package/server/agentsdock_team.py +793 -0
- package/server/agentsdock_team_hub/__init__.py +33 -0
- package/server/agentsdock_team_hub/auth.py +710 -0
- package/server/agentsdock_team_hub/cli.py +647 -0
- package/server/agentsdock_team_hub/database.py +310 -0
- package/server/agentsdock_team_hub/mail_hint_streams.py +176 -0
- package/server/agentsdock_team_hub/mail_hints.py +201 -0
- package/server/agentsdock_team_hub/migrations/0001_identity_auth.sql +690 -0
- package/server/agentsdock_team_hub/migrations/0002_teamspace_ledger.sql +1336 -0
- package/server/agentsdock_team_hub/migrations/0003_service_runtime.sql +361 -0
- package/server/agentsdock_team_hub/migrations/0004_managed_host_binding.sql +22 -0
- package/server/agentsdock_team_hub/migrations/0005_tailnet_bootstrap_delegations.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0006_team_network_mailbox.sql +423 -0
- package/server/agentsdock_team_hub/migrations/0007_local_agent_mail.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0008_managed_server_session.sql +73 -0
- package/server/agentsdock_team_hub/migrations/0009_team_messages.sql +324 -0
- package/server/agentsdock_team_hub/migrations/0010_team_attachment_orphan_reclamation.sql +52 -0
- package/server/agentsdock_team_hub/migrations/0011_human_admin_paging.sql +65 -0
- package/server/agentsdock_team_hub/migrations/0012_network_content_deletions.sql +69 -0
- package/server/agentsdock_team_hub/migrations/0013_team_message_revisions.sql +48 -0
- package/server/agentsdock_team_hub/migrations/0014_managed_network_owner.sql +50 -0
- package/server/agentsdock_team_hub/migrations/0015_team_message_inbox_dismissals.sql +3 -0
- package/server/agentsdock_team_hub/migrations/0016_team_message_all_servers.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0017_skill_announcement_deletions.sql +16 -0
- package/server/agentsdock_team_hub/migrations/0018_team_mail_subjects.sql +8 -0
- package/server/agentsdock_team_hub/migrations/0019_team_mailbox_state.sql +5 -0
- package/server/agentsdock_team_hub/migrations/0020_team_mail_arrivals.sql +44 -0
- package/server/agentsdock_team_hub/migrations/0021_team_mail_threads.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0022_team_bulletin_changes.sql +49 -0
- package/server/agentsdock_team_hub/migrations/0023_team_message_search.sql +93 -0
- package/server/agentsdock_team_hub/migrations/__init__.py +1 -0
- package/server/agentsdock_team_hub/notification_hints.py +306 -0
- package/server/agentsdock_team_hub/secure_peer.py +11215 -0
- package/server/agentsdock_team_hub/secure_peer_hub.py +1372 -0
- package/server/agentsdock_team_hub/security.py +318 -0
- package/server/agentsdock_team_hub/service.py +1737 -0
- package/server/agentsdock_team_hub/store.py +16276 -0
- package/server/chat_mailbox.py +448 -0
- package/server/claude_background_reconciliation.py +92 -0
- package/server/claude_goals.py +245 -0
- package/server/claude_history_provenance.py +137 -0
- package/server/claude_history_repair.py +1601 -0
- package/server/claude_sdk_client.py +3731 -0
- package/server/claude_side_question.py +212 -0
- package/server/codex_app_server.py +3030 -0
- package/server/codex_auth.py +93 -0
- package/server/codex_history_repair.py +1247 -0
- package/server/codex_provider.py +1120 -0
- package/server/codex_side_question.py +301 -0
- package/server/cursor_agent_client.py +654 -0
- package/server/cursor_process_guard.py +347 -0
- package/server/execution_activation.py +610 -0
- package/server/execution_control.py +233 -0
- package/server/execution_durability.py +184 -0
- package/server/execution_http.py +256 -0
- package/server/execution_install.py +792 -0
- package/server/execution_legacy_runner.py +406 -0
- package/server/execution_maintenance.py +137 -0
- package/server/execution_manage.py +615 -0
- package/server/execution_ownership.py +83 -0
- package/server/execution_preparation.py +362 -0
- package/server/execution_recovery.py +845 -0
- package/server/execution_recovery_status.py +195 -0
- package/server/execution_service.py +387 -0
- package/server/execution_transport.py +622 -0
- package/server/execution_uninstall.py +406 -0
- package/server/execution_update_status.py +49 -0
- package/server/install.sh +7074 -0
- package/server/interactive_chat_controls.py +153 -0
- package/server/interactive_chat_native.py +66 -0
- package/server/interactive_chat_projection.py +152 -0
- package/server/interactive_chat_runtime.py +132 -0
- package/server/interactive_chat_share_routes.py +534 -0
- package/server/interactive_chat_share_web.py +12 -0
- package/server/interactive_chat_shares.py +238 -0
- package/server/provider_commands.py +425 -0
- package/server/public_chat_share_routes.py +426 -0
- package/server/public_chat_shares.py +759 -0
- package/server/public_chat_transcript.py +357 -0
- package/server/pyproject.toml +29 -0
- package/server/release-public-key.pem +3 -0
- package/server/secure_peer_delivery.py +1236 -0
- package/server/secure_peer_runtime.py +7694 -0
- package/server/shared_chat_video_stream.py +262 -0
- package/server/shared_chat_videos.py +155 -0
- package/server/side_questions.py +594 -0
- package/server/team_hub_host.py +1354 -0
- package/server/team_mail_grants.py +136 -0
- package/server/team_mail_runtime.py +302 -0
- package/server/team_mail_websocket.py +211 -0
- package/server/title_generation.py +333 -0
- package/server/uninstall.sh +359 -0
- package/server/update_handoff.py +237 -0
- package/server/update_preparation.py +264 -0
- package/server/update_recovery.py +140 -0
- package/server/update_runner.py +1819 -0
- package/server/uv.lock +1101 -0
- package/server/workspace_git.py +545 -0
|
@@ -0,0 +1,1247 @@
|
|
|
1
|
+
"""Read-only, checkpoint-proven repair of Codex runtime and native replay imports.
|
|
2
|
+
|
|
3
|
+
Preparation is explicit, bounded and once per requested chat/provider. Event
|
|
4
|
+
projection is memory-only. Missing, changing or ambiguous evidence stays visible.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from bisect import bisect_right
|
|
9
|
+
from collections import Counter, OrderedDict
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
import hashlib
|
|
12
|
+
import hmac
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
import re
|
|
17
|
+
import stat
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from typing import Callable
|
|
21
|
+
from datetime import datetime
|
|
22
|
+
|
|
23
|
+
from claude_history_repair import (
|
|
24
|
+
MAX_EVENTS_BYTES, MAX_KEYS, MAX_SESSIONS, MAX_TARGETS,
|
|
25
|
+
_DIGEST, _Unproven, _records, _stamp, _text_key,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
MAX_PRIOR_SOURCE_PATHS = 2
|
|
30
|
+
MAX_AGGREGATE_SOURCE_BYTES = 96 * 1024 * 1024
|
|
31
|
+
MAX_AGGREGATE_SOURCE_RECORDS = 100_000
|
|
32
|
+
MAX_FORK_META_HEADERS = 32
|
|
33
|
+
_PROVIDER_ID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I)
|
|
34
|
+
_NATIVE_LEADING_DECORATION_RE = re.compile(
|
|
35
|
+
r"(?m)^[ \t]*(?:(?::[A-Za-z0-9_+\-]+:|[\U0001F300-\U0001FAFF\u2600-\u27BF]\ufe0f?)[ \t]*)+"
|
|
36
|
+
)
|
|
37
|
+
NATIVE_PROOF_LINE_BYTES = 4 * 1024 * 1024
|
|
38
|
+
NATIVE_PROOF_SECONDS = 30.0
|
|
39
|
+
_COMPACTION_PREFIX = (
|
|
40
|
+
"Another language model started to solve this problem and produced a summary of its thinking process. "
|
|
41
|
+
"You also have access to the state of the tools that were used by that language model. "
|
|
42
|
+
"Use this to build on the work that has already been done and avoid duplicating work. "
|
|
43
|
+
"Here is the summary produced by the other language model, use the information in this summary to assist with your own analysis:\n"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CodexCompactionSummaryTracker:
|
|
48
|
+
"""Match one native compaction response, never classify assistant wording.
|
|
49
|
+
|
|
50
|
+
Some endpoints persist the compactor's response as an ordinary assistant
|
|
51
|
+
item. Only its immediately following usage receipt and typed replacement
|
|
52
|
+
history can prove that item belongs to the compaction, not the conversation.
|
|
53
|
+
"""
|
|
54
|
+
def __init__(self, provider_id: str | None = None):
|
|
55
|
+
self.provider_id = provider_id
|
|
56
|
+
self.pending = None
|
|
57
|
+
|
|
58
|
+
def observe(self, record: dict) -> dict | None:
|
|
59
|
+
payload = record.get("payload")
|
|
60
|
+
if not isinstance(payload, dict):
|
|
61
|
+
self.pending = None
|
|
62
|
+
return None
|
|
63
|
+
kind = record.get("type")
|
|
64
|
+
if kind == "response_item":
|
|
65
|
+
self.pending = None
|
|
66
|
+
origin = codex_public_item_origin(record, self.provider_id or "")
|
|
67
|
+
metadata = payload.get("internal_chat_message_metadata_passthrough")
|
|
68
|
+
content = payload.get("content")
|
|
69
|
+
if (origin and origin["kind"] == "assistant" and payload.get("phase") == "final_answer"
|
|
70
|
+
and isinstance(metadata, dict) and metadata.get("content_item_kinds") == ["unknown"]
|
|
71
|
+
and isinstance(content, list) and len(content) == 1 and isinstance(content[0], dict)
|
|
72
|
+
and content[0].get("type") == "output_text" and isinstance(content[0].get("text"), str)
|
|
73
|
+
and 0 < len(content[0]["text"]) <= NATIVE_PROOF_LINE_BYTES):
|
|
74
|
+
self.pending = {"origin": origin, "text": content[0]["text"], "response_id": None}
|
|
75
|
+
return None
|
|
76
|
+
pending = self.pending
|
|
77
|
+
if pending is None:
|
|
78
|
+
return None
|
|
79
|
+
if kind == "event_msg" and payload.get("type") == "token_count":
|
|
80
|
+
return None
|
|
81
|
+
if kind == "token_usage_record":
|
|
82
|
+
response = payload.get("response_id")
|
|
83
|
+
thread = payload.get("thread_id")
|
|
84
|
+
if (pending["response_id"] is None and isinstance(thread, str) and _PROVIDER_ID.fullmatch(thread)
|
|
85
|
+
and (self.provider_id is None or thread == self.provider_id)
|
|
86
|
+
and payload.get("turn_id") == pending["origin"]["turn_id"]
|
|
87
|
+
and isinstance(response, str) and 0 < len(response) <= 4096):
|
|
88
|
+
pending["response_id"] = response
|
|
89
|
+
pending["thread_id"] = thread
|
|
90
|
+
return None
|
|
91
|
+
self.pending = None
|
|
92
|
+
if kind != "compacted" or pending["response_id"] is None:
|
|
93
|
+
return None
|
|
94
|
+
latest = payload.get("latest_token_usage_record")
|
|
95
|
+
replacement = payload.get("replacement_history")
|
|
96
|
+
if not isinstance(latest, dict) or not isinstance(replacement, list) or not replacement:
|
|
97
|
+
return None
|
|
98
|
+
summary = replacement[-1]
|
|
99
|
+
if not isinstance(summary, dict):
|
|
100
|
+
return None
|
|
101
|
+
metadata = summary.get("internal_chat_message_metadata_passthrough")
|
|
102
|
+
text = _COMPACTION_PREFIX + pending["text"]
|
|
103
|
+
start, end = _delivery_timestamp(pending["origin"]["timestamp"]), _delivery_timestamp(record.get("timestamp"))
|
|
104
|
+
if (payload.get("compaction_response_id") != pending["response_id"]
|
|
105
|
+
or latest.get("response_id") != pending["response_id"] or latest.get("thread_id") != pending["thread_id"]
|
|
106
|
+
or latest.get("turn_id") != pending["origin"]["turn_id"]
|
|
107
|
+
or start is None or end is None or end < start or payload.get("message") != text
|
|
108
|
+
or summary.get("type") != "message" or summary.get("role") != "user"
|
|
109
|
+
or not isinstance(metadata, dict) or metadata.get("turn_id") != pending["origin"]["turn_id"]
|
|
110
|
+
or metadata.get("content_item_kinds") != ["compaction.summary"]
|
|
111
|
+
or summary.get("content") != [{"type": "input_text", "text": text}]):
|
|
112
|
+
return None
|
|
113
|
+
return {**pending["origin"], "kind": "compaction_summary", "source_text_sha256": _text_key(pending["text"])}
|
|
114
|
+
|
|
115
|
+
def discard_summary(self, items, record: dict | None) -> bool:
|
|
116
|
+
"""Use the existing parse pass; incomplete ranges keep their last item."""
|
|
117
|
+
if record is None:
|
|
118
|
+
self.pending = None
|
|
119
|
+
return False
|
|
120
|
+
proof = self.observe(record)
|
|
121
|
+
if proof is None or not items:
|
|
122
|
+
return False
|
|
123
|
+
item = items[-1]
|
|
124
|
+
origin = item.get("provider_origin")
|
|
125
|
+
if (item.get("kind") == "assistant" and isinstance(origin, dict)
|
|
126
|
+
and all(origin.get(key) == proof.get(key) for key in ("event_id", "turn_id", "timestamp"))):
|
|
127
|
+
items.pop()
|
|
128
|
+
return True
|
|
129
|
+
return False
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class CodexNativeHistoryProofUnavailable(ValueError):
|
|
133
|
+
"""Incomplete evidence must defer import, not become a new human message."""
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass
|
|
137
|
+
class _NativeReadBudget:
|
|
138
|
+
cancelled: Callable[[], bool] | None = None
|
|
139
|
+
deadline: float | None = None
|
|
140
|
+
|
|
141
|
+
def __post_init__(self) -> None:
|
|
142
|
+
if self.deadline is None:
|
|
143
|
+
self.deadline = time.monotonic() + NATIVE_PROOF_SECONDS
|
|
144
|
+
|
|
145
|
+
def check(self) -> None:
|
|
146
|
+
if ((self.cancelled is not None and self.cancelled())
|
|
147
|
+
or time.monotonic() >= self.deadline):
|
|
148
|
+
raise CodexNativeHistoryProofUnavailable("Codex history proof was cancelled or exceeded its work budget")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _native_stamp(path: Path) -> tuple[int, int, int, int]:
|
|
152
|
+
"""Pin a regular file without a total-size cutoff or following symlinks."""
|
|
153
|
+
value = path.lstat()
|
|
154
|
+
if not stat.S_ISREG(value.st_mode):
|
|
155
|
+
raise _Unproven()
|
|
156
|
+
return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _native_records(path: Path, expected: tuple[int, int, int, int], budget: _NativeReadBudget,
|
|
160
|
+
*, end: int | None = None):
|
|
161
|
+
"""Stream one fixed prefix; tool volume does not consume retained-key budget.
|
|
162
|
+
|
|
163
|
+
A source proof needs only the prefix ending at its newest checkpoint. Never
|
|
164
|
+
parse or hash a later unrelated source tail. Cancellation is checked between
|
|
165
|
+
individually bounded records, including records that carry no public text.
|
|
166
|
+
"""
|
|
167
|
+
budget.check()
|
|
168
|
+
boundary = expected[2] if end is None else end
|
|
169
|
+
if type(boundary) is not int or not 0 <= boundary <= expected[2]:
|
|
170
|
+
raise _Unproven()
|
|
171
|
+
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
|
172
|
+
| getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NONBLOCK", 0))
|
|
173
|
+
with os.fdopen(descriptor, "rb") as stream:
|
|
174
|
+
info = os.fstat(stream.fileno())
|
|
175
|
+
if not stat.S_ISREG(info.st_mode) or (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != expected:
|
|
176
|
+
raise _Unproven()
|
|
177
|
+
offset = 0
|
|
178
|
+
while offset < boundary:
|
|
179
|
+
budget.check()
|
|
180
|
+
line = stream.readline(min(NATIVE_PROOF_LINE_BYTES + 1, boundary - offset))
|
|
181
|
+
if not line or len(line) > NATIVE_PROOF_LINE_BYTES or not line.endswith(b"\n"):
|
|
182
|
+
raise _Unproven()
|
|
183
|
+
offset += len(line)
|
|
184
|
+
record = json.loads(line)
|
|
185
|
+
if not isinstance(record, dict):
|
|
186
|
+
raise _Unproven()
|
|
187
|
+
yield record, offset, line
|
|
188
|
+
info = os.fstat(stream.fileno())
|
|
189
|
+
if (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != expected:
|
|
190
|
+
raise _Unproven()
|
|
191
|
+
budget.check()
|
|
192
|
+
if _native_stamp(path) != expected:
|
|
193
|
+
raise _Unproven()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _native_event_identity(event: dict) -> dict:
|
|
197
|
+
"""Keep bounded public identity, never retain tool bodies or message text."""
|
|
198
|
+
return {key: event[key] for key in ("id", "seq", "type", "phase", "item_id")
|
|
199
|
+
if key in event and (type(event[key]) is int or isinstance(event[key], str) and len(event[key]) <= 256)}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _native_import_batch(event: dict) -> dict:
|
|
203
|
+
result = {key: event.get(key) for key in ("seq", "provider_session_id", "source_path")}
|
|
204
|
+
checkpoint = event.get("_history_sync_checkpoint")
|
|
205
|
+
if isinstance(checkpoint, dict) and isinstance(checkpoint.get("cursor"), dict):
|
|
206
|
+
result["_history_sync_checkpoint"] = {
|
|
207
|
+
**{key: checkpoint.get(key) for key in (
|
|
208
|
+
"version", "previous_present", "previous_source_offset", "previous_source_digest")},
|
|
209
|
+
"cursor": {key: checkpoint["cursor"].get(key) for key in (
|
|
210
|
+
"version", "backend", "provider_session_id", "source_path", "source_offset",
|
|
211
|
+
"source_dev", "source_ino", "source_digest")},
|
|
212
|
+
}
|
|
213
|
+
# Unexpected huge metadata cannot make the retained compact proof unbounded.
|
|
214
|
+
if len(json.dumps(result)) > 16 * 1024:
|
|
215
|
+
raise _Unproven()
|
|
216
|
+
return result
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _native_assistant_text(text: str) -> str:
|
|
220
|
+
"""Mirror native clean_assistant_text; never normalize source proof keys."""
|
|
221
|
+
return _NATIVE_LEADING_DECORATION_RE.sub("", str(text or "")).strip()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _public_assistant_item_id(event: dict) -> str | None:
|
|
225
|
+
item_id = event.get("item_id")
|
|
226
|
+
public = event.get("type") == "assistant_text" or (
|
|
227
|
+
event.get("type") == "reasoning_summary" and event.get("phase") == "commentary"
|
|
228
|
+
)
|
|
229
|
+
return item_id if public and isinstance(item_id, str) and 0 < len(item_id) <= 256 else None
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@dataclass
|
|
233
|
+
class _SourceBudget:
|
|
234
|
+
bytes_remaining: int
|
|
235
|
+
records_remaining: int
|
|
236
|
+
|
|
237
|
+
def reserve(self, size: int) -> None:
|
|
238
|
+
if size > self.bytes_remaining or self.records_remaining <= 0:
|
|
239
|
+
raise _Unproven()
|
|
240
|
+
self.bytes_remaining -= size
|
|
241
|
+
|
|
242
|
+
def consume_record(self) -> None:
|
|
243
|
+
if self.records_remaining <= 0:
|
|
244
|
+
raise _Unproven()
|
|
245
|
+
self.records_remaining -= 1
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _target(event: dict) -> tuple[int, str, str] | None:
|
|
249
|
+
seq, run, prompt = event.get("seq"), event.get("run_id"), event.get("prompt")
|
|
250
|
+
if (
|
|
251
|
+
event.get("type") != "turn_started" or event.get("imported") is not True
|
|
252
|
+
or event.get("backend") != "codex" or event.get("provider_user_authored") is True
|
|
253
|
+
or type(seq) is not int or seq <= 0
|
|
254
|
+
or not isinstance(run, str) or not run.startswith("import_")
|
|
255
|
+
or not isinstance(prompt, str) or not prompt or len(prompt) > 4 * 1024 * 1024
|
|
256
|
+
):
|
|
257
|
+
return None
|
|
258
|
+
return seq, run, _text_key(prompt)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@dataclass(frozen=True)
|
|
262
|
+
class _Proof:
|
|
263
|
+
provider_id: str
|
|
264
|
+
targets: frozenset[tuple[int, str, str]] = frozenset()
|
|
265
|
+
positions: frozenset[tuple[int, str]] = field(init=False, repr=False)
|
|
266
|
+
|
|
267
|
+
def __post_init__(self) -> None:
|
|
268
|
+
object.__setattr__(self, "positions", frozenset((seq, run) for seq, run, _key in self.targets))
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _prove(session_id: str, provider_id: str, events: Path, source: Path | None, root: Path,
|
|
272
|
+
normalize_user: Callable[[dict], str | None],
|
|
273
|
+
classify_user: Callable[[dict], str | None]) -> _Proof:
|
|
274
|
+
"""Inspect one ledger and only its current/two newest recorded prior sources."""
|
|
275
|
+
if not _PROVIDER_ID.fullmatch(provider_id):
|
|
276
|
+
raise _Unproven()
|
|
277
|
+
events_stamp = _stamp(events)
|
|
278
|
+
if events_stamp[2] > MAX_EVENTS_BYTES:
|
|
279
|
+
raise _Unproven()
|
|
280
|
+
batches, terminals, candidates = {}, {}, []
|
|
281
|
+
terminal_counts = Counter()
|
|
282
|
+
for event, _offset, _line in _records(events, events_stamp):
|
|
283
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
284
|
+
continue
|
|
285
|
+
run = event.get("run_id")
|
|
286
|
+
if not isinstance(run, str) or not run.startswith("import_"):
|
|
287
|
+
continue
|
|
288
|
+
if event.get("type") == "history_imported":
|
|
289
|
+
checkpoint = event.get("_history_sync_checkpoint")
|
|
290
|
+
cursor = checkpoint.get("cursor") if isinstance(checkpoint, dict) else None
|
|
291
|
+
batch_provider = event.get("provider_session_id")
|
|
292
|
+
batch_source = event.get("source_path")
|
|
293
|
+
if (
|
|
294
|
+
event.get("backend") == "codex"
|
|
295
|
+
and isinstance(batch_provider, str) and _PROVIDER_ID.fullmatch(batch_provider)
|
|
296
|
+
and isinstance(batch_source, str) and 0 < len(batch_source) <= 4096
|
|
297
|
+
and isinstance(cursor, dict) and cursor.get("version") == 1
|
|
298
|
+
and cursor.get("backend") == "codex" and cursor.get("provider_session_id") == batch_provider
|
|
299
|
+
and batch_source == cursor.get("source_path")
|
|
300
|
+
and checkpoint.get("version") == 1 and type(event.get("seq")) is int
|
|
301
|
+
):
|
|
302
|
+
if run in batches:
|
|
303
|
+
raise _Unproven()
|
|
304
|
+
batches[run] = (event["seq"], checkpoint)
|
|
305
|
+
elif (event.get("type") == "turn_finished" and event.get("imported") is True
|
|
306
|
+
and event.get("backend") == "codex" and type(event.get("seq")) is int):
|
|
307
|
+
terminals[run] = event["seq"]
|
|
308
|
+
terminal_counts[run] += 1
|
|
309
|
+
else:
|
|
310
|
+
target = _target(event)
|
|
311
|
+
if target:
|
|
312
|
+
candidates.append(target)
|
|
313
|
+
if max(len(batches), len(candidates), len(terminals)) > MAX_TARGETS:
|
|
314
|
+
raise _Unproven()
|
|
315
|
+
if not batches or not candidates:
|
|
316
|
+
return _Proof(provider_id)
|
|
317
|
+
candidate_runs = {run for _seq, run, _key in candidates}
|
|
318
|
+
groups = {}
|
|
319
|
+
for run, (seq, checkpoint) in batches.items():
|
|
320
|
+
if run not in candidate_runs or terminals.get(run, 0) <= seq or terminal_counts[run] != 1:
|
|
321
|
+
continue
|
|
322
|
+
cursor = checkpoint["cursor"]
|
|
323
|
+
key = (cursor["provider_session_id"], cursor["source_path"])
|
|
324
|
+
group = groups.setdefault(key, {})
|
|
325
|
+
group[run] = (seq, checkpoint)
|
|
326
|
+
current_key = (provider_id, str(source)) if source is not None else None
|
|
327
|
+
prior_keys = sorted(
|
|
328
|
+
(key for key in groups if key[0] != provider_id),
|
|
329
|
+
key=lambda key: max(seq for seq, _checkpoint in groups[key].values()),
|
|
330
|
+
reverse=True,
|
|
331
|
+
)[:MAX_PRIOR_SOURCE_PATHS]
|
|
332
|
+
selected = ([current_key] if current_key in groups else []) + prior_keys
|
|
333
|
+
budget = _SourceBudget(MAX_AGGREGATE_SOURCE_BYTES, MAX_AGGREGATE_SOURCE_RECORDS)
|
|
334
|
+
targets = set()
|
|
335
|
+
for source_provider, source_path in selected:
|
|
336
|
+
selected_batches = groups[(source_provider, source_path)]
|
|
337
|
+
selected_candidates = [target for target in candidates if target[1] in selected_batches]
|
|
338
|
+
try:
|
|
339
|
+
targets.update(_prove_source(
|
|
340
|
+
source_provider, Path(source_path), root, selected_batches, terminals,
|
|
341
|
+
selected_candidates, normalize_user, classify_user, budget,
|
|
342
|
+
))
|
|
343
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError):
|
|
344
|
+
# A missing, changed, oversized or ambiguous prior source cannot
|
|
345
|
+
# invalidate a proof independently established from another source.
|
|
346
|
+
continue
|
|
347
|
+
return _Proof(provider_id, frozenset(targets))
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _prove_source(provider_id: str, source: Path, root: Path,
|
|
351
|
+
batches: dict, terminals: dict, candidates: list,
|
|
352
|
+
normalize_user: Callable[[dict], str | None],
|
|
353
|
+
classify_user: Callable[[dict], str | None],
|
|
354
|
+
budget: _SourceBudget) -> set[tuple[int, str, str]]:
|
|
355
|
+
if (
|
|
356
|
+
not source.is_absolute() or source.suffix != ".jsonl" or source.is_symlink()
|
|
357
|
+
or not (source.stem == provider_id or source.stem.endswith("-" + provider_id))
|
|
358
|
+
):
|
|
359
|
+
raise _Unproven()
|
|
360
|
+
canonical_source = source.resolve(strict=True)
|
|
361
|
+
canonical_source.relative_to(root.resolve(strict=True))
|
|
362
|
+
source_stamp = _stamp(canonical_source)
|
|
363
|
+
wanted, eligible = {}, {}
|
|
364
|
+
for run, (seq, checkpoint) in batches.items():
|
|
365
|
+
cursor = checkpoint["cursor"]
|
|
366
|
+
start, end = checkpoint.get("previous_source_offset"), cursor.get("source_offset")
|
|
367
|
+
expected, previous = cursor.get("source_digest"), checkpoint.get("previous_source_digest")
|
|
368
|
+
if (
|
|
369
|
+
type(start) is not int or type(end) is not int or not 0 <= start < end <= source_stamp[2]
|
|
370
|
+
or (cursor.get("source_dev"), cursor.get("source_ino")) != source_stamp[:2]
|
|
371
|
+
or not isinstance(expected, str) or not _DIGEST.fullmatch(expected)
|
|
372
|
+
or type(checkpoint.get("previous_present")) is not bool
|
|
373
|
+
or (start > 0 and checkpoint.get("previous_present") is not True)
|
|
374
|
+
or (start == 0 and previous != "")
|
|
375
|
+
or (start > 0 and (not isinstance(previous, str) or not _DIGEST.fullmatch(previous)))
|
|
376
|
+
):
|
|
377
|
+
continue
|
|
378
|
+
wanted.setdefault(end, set()).add(expected)
|
|
379
|
+
if start:
|
|
380
|
+
wanted.setdefault(start, set()).add(previous)
|
|
381
|
+
eligible[run] = (seq, terminals[run], start, end, expected, previous)
|
|
382
|
+
if not eligible:
|
|
383
|
+
return set()
|
|
384
|
+
budget.reserve(source_stamp[2])
|
|
385
|
+
digest, verified, goals, other_users = hashlib.sha256(), set(), {}, set()
|
|
386
|
+
source_owner_seen, goal_count = False, 0
|
|
387
|
+
for event, offset, line in _records(canonical_source, source_stamp):
|
|
388
|
+
budget.consume_record()
|
|
389
|
+
if budget.records_remaining == 0 and offset < source_stamp[2]:
|
|
390
|
+
raise _Unproven()
|
|
391
|
+
payload = event.get("payload")
|
|
392
|
+
if not source_owner_seen:
|
|
393
|
+
if event.get("type") != "session_meta" or not isinstance(payload, dict) or payload.get("id") != provider_id:
|
|
394
|
+
raise _Unproven()
|
|
395
|
+
source_owner_seen = True
|
|
396
|
+
elif event.get("type") == "session_meta":
|
|
397
|
+
if not isinstance(payload, dict) or payload.get("id") != provider_id:
|
|
398
|
+
raise _Unproven()
|
|
399
|
+
digest.update(line)
|
|
400
|
+
for expected in wanted.get(offset, ()):
|
|
401
|
+
if hmac.compare_digest(digest.hexdigest(), expected):
|
|
402
|
+
verified.add((offset, expected))
|
|
403
|
+
classification = classify_user(event)
|
|
404
|
+
if classification is None:
|
|
405
|
+
continue
|
|
406
|
+
text = normalize_user(event)
|
|
407
|
+
if not isinstance(text, str) or not text:
|
|
408
|
+
continue
|
|
409
|
+
key = _text_key(text)
|
|
410
|
+
if classification == "goal":
|
|
411
|
+
goals.setdefault(key, []).append(offset)
|
|
412
|
+
goal_count += 1
|
|
413
|
+
if goal_count > MAX_TARGETS:
|
|
414
|
+
raise _Unproven()
|
|
415
|
+
else:
|
|
416
|
+
# Even an unknown user source makes identical quoted text ambiguous.
|
|
417
|
+
other_users.add(key)
|
|
418
|
+
if len(goals) + len(other_users) > MAX_KEYS:
|
|
419
|
+
raise _Unproven()
|
|
420
|
+
counts = Counter((run, key) for _seq, run, key in candidates)
|
|
421
|
+
targets = set()
|
|
422
|
+
for target in candidates:
|
|
423
|
+
seq, run, key = target
|
|
424
|
+
batch = eligible.get(run)
|
|
425
|
+
if batch is None or key in other_users:
|
|
426
|
+
continue
|
|
427
|
+
first_seq, last_seq, start, end, expected, previous = batch
|
|
428
|
+
offsets = goals.get(key, ())
|
|
429
|
+
occurrences = bisect_right(offsets, end) - bisect_right(offsets, start)
|
|
430
|
+
if (
|
|
431
|
+
first_seq < seq < last_seq and (end, expected) in verified
|
|
432
|
+
and (start == 0 or (start, previous) in verified)
|
|
433
|
+
and occurrences > 0 and counts[(run, key)] == occurrences
|
|
434
|
+
):
|
|
435
|
+
targets.add(target)
|
|
436
|
+
return targets
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
class CodexGoalHistoryRepairCache:
|
|
440
|
+
def __init__(self) -> None:
|
|
441
|
+
self._lock = threading.RLock()
|
|
442
|
+
self._prepare_lock = threading.Lock()
|
|
443
|
+
self._proofs: OrderedDict[str, _Proof] = OrderedDict()
|
|
444
|
+
self._preparing: str | None = None
|
|
445
|
+
self._cancelled = False
|
|
446
|
+
|
|
447
|
+
def is_prepared(self, session_id: str, provider_id: str) -> bool:
|
|
448
|
+
with self._lock:
|
|
449
|
+
proof = self._proofs.get(session_id)
|
|
450
|
+
return proof is not None and proof.provider_id == provider_id
|
|
451
|
+
|
|
452
|
+
def prepare(self, session_id: str, provider_id: str, events_path: Path,
|
|
453
|
+
source_path: Path | None, root: Path,
|
|
454
|
+
normalize_user: Callable[[dict], str | None],
|
|
455
|
+
classify_user: Callable[[dict], str | None]) -> bool:
|
|
456
|
+
with self._prepare_lock:
|
|
457
|
+
with self._lock:
|
|
458
|
+
previous = self._proofs.get(session_id)
|
|
459
|
+
if previous and previous.provider_id == provider_id:
|
|
460
|
+
self._proofs.move_to_end(session_id)
|
|
461
|
+
return False
|
|
462
|
+
self._preparing, self._cancelled = session_id, False
|
|
463
|
+
try:
|
|
464
|
+
proof = _prove(session_id, provider_id, events_path, source_path, root,
|
|
465
|
+
normalize_user, classify_user)
|
|
466
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError):
|
|
467
|
+
proof = _Proof(provider_id)
|
|
468
|
+
with self._lock:
|
|
469
|
+
self._preparing = None
|
|
470
|
+
if self._cancelled:
|
|
471
|
+
return bool(previous and previous.targets)
|
|
472
|
+
if previous and previous.provider_id != provider_id:
|
|
473
|
+
# Proven immutable ledger positions stay proven when this
|
|
474
|
+
# chat rotates provider threads. Fresh credentials/source
|
|
475
|
+
# discovery must not resurrect old runtime-only bubbles.
|
|
476
|
+
remaining = max(0, MAX_TARGETS - len(previous.targets))
|
|
477
|
+
added = sorted(proof.targets - previous.targets)[:remaining]
|
|
478
|
+
proof = _Proof(provider_id, previous.targets.union(added))
|
|
479
|
+
self._proofs[session_id] = proof
|
|
480
|
+
self._proofs.move_to_end(session_id)
|
|
481
|
+
while len(self._proofs) > MAX_SESSIONS:
|
|
482
|
+
self._proofs.popitem(last=False)
|
|
483
|
+
return (previous.targets if previous else frozenset()) != proof.targets
|
|
484
|
+
|
|
485
|
+
def signature(self, session_id: str) -> frozenset:
|
|
486
|
+
with self._lock:
|
|
487
|
+
proof = self._proofs.get(session_id)
|
|
488
|
+
return proof.targets if proof else frozenset()
|
|
489
|
+
|
|
490
|
+
def forget(self, session_id: str) -> None:
|
|
491
|
+
with self._lock:
|
|
492
|
+
self._proofs.pop(session_id, None)
|
|
493
|
+
if self._preparing == session_id:
|
|
494
|
+
self._cancelled = True
|
|
495
|
+
|
|
496
|
+
def is_hidden(self, session_id: str, event: dict) -> bool:
|
|
497
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
498
|
+
return False
|
|
499
|
+
with self._lock:
|
|
500
|
+
proof = self._proofs.get(session_id)
|
|
501
|
+
if proof is None or not proof.targets:
|
|
502
|
+
return False
|
|
503
|
+
try:
|
|
504
|
+
if (event.get("seq"), event.get("run_id")) not in proof.positions:
|
|
505
|
+
return False
|
|
506
|
+
return _target(event) in proof.targets
|
|
507
|
+
except (TypeError, ValueError):
|
|
508
|
+
return False
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def codex_public_item_origin(event: dict, provider_id: str | None = None) -> dict | None:
|
|
512
|
+
"""Explicit public response identity only; never infer identity from text."""
|
|
513
|
+
payload = event.get("payload")
|
|
514
|
+
if event.get("type") != "response_item" or not isinstance(payload, dict) or payload.get("type") != "message":
|
|
515
|
+
return None
|
|
516
|
+
metadata = payload.get("internal_chat_message_metadata_passthrough")
|
|
517
|
+
item_id, turn_id = payload.get("id"), metadata.get("turn_id") if isinstance(metadata, dict) else None
|
|
518
|
+
timestamp, role = event.get("timestamp"), payload.get("role")
|
|
519
|
+
if role not in ("user", "assistant") or not all(isinstance(value, str) and 0 < len(value) <= 256 for value in (item_id, turn_id)):
|
|
520
|
+
return None
|
|
521
|
+
try:
|
|
522
|
+
if not isinstance(timestamp, str) or len(timestamp) > 64 or datetime.fromisoformat(timestamp.replace("Z", "+00:00")).utcoffset() is None:
|
|
523
|
+
return None
|
|
524
|
+
except ValueError:
|
|
525
|
+
return None
|
|
526
|
+
origin = {"provider": "codex", "kind": role, "event_id": item_id, "turn_id": turn_id, "timestamp": timestamp}
|
|
527
|
+
if provider_id and _PROVIDER_ID.fullmatch(provider_id):
|
|
528
|
+
origin["session_id"] = provider_id
|
|
529
|
+
return origin
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _runtime_human_provenance(event: dict) -> bool:
|
|
533
|
+
metadata = event.get("internal_chat_message_metadata_passthrough")
|
|
534
|
+
kinds = metadata.get("content_item_kinds") if isinstance(metadata, dict) else None
|
|
535
|
+
return event.get("provider_user_authored") is True or isinstance(kinds, list) and "user.text" in kinds or any(
|
|
536
|
+
isinstance(event.get(field), str) and event[field].strip()
|
|
537
|
+
for field in ("clientUserMessageId", "clientId", "client_user_message_id", "client_id")
|
|
538
|
+
) or any(isinstance(origin, dict) and origin.get("kind") in ("human", "user", "user_input", "user-input")
|
|
539
|
+
for origin in (event.get("origin"), event.get("provider_origin")))
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _persisted_runtime_marker(event: dict, session_id: str) -> bool:
|
|
543
|
+
origin = event.get("provider_origin")
|
|
544
|
+
runtime_kind = event.get("provider_runtime_context")
|
|
545
|
+
if (runtime_kind not in ("subagent_notification", "turn_aborted", "provider_notice")
|
|
546
|
+
or event.get("metadata_only") is not True or event.get("backend") != "codex"
|
|
547
|
+
or event.get("imported") is not True or not isinstance(event.get("run_id"), str)
|
|
548
|
+
or not event["run_id"].startswith("import_") or event.get("session_id") != session_id
|
|
549
|
+
or not isinstance(event.get("id"), str) or not 0 < len(event["id"].strip()) <= 256
|
|
550
|
+
or type(event.get("seq")) is not int or event["seq"] <= 0
|
|
551
|
+
or event.get("type") != "turn_started" or event.get("prompt") != ""
|
|
552
|
+
or _runtime_human_provenance(event) or not isinstance(origin, dict)
|
|
553
|
+
or origin.get("provider") != "codex" or origin.get("kind") != runtime_kind
|
|
554
|
+
or not all(isinstance(origin.get(key), str) and 0 < len(origin[key].strip()) <= 256
|
|
555
|
+
for key in ("event_id", "session_id", "turn_id"))
|
|
556
|
+
or not isinstance(origin.get("source_text_sha256"), str) or not _DIGEST.fullmatch(origin["source_text_sha256"])):
|
|
557
|
+
return False
|
|
558
|
+
timestamp = origin.get("timestamp")
|
|
559
|
+
if (not isinstance(timestamp, str) or timestamp != event.get("ts") or not re.fullmatch(
|
|
560
|
+
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)", timestamp)):
|
|
561
|
+
return False
|
|
562
|
+
try:
|
|
563
|
+
return datetime.fromisoformat(timestamp.replace("Z", "+00:00")).utcoffset() is not None
|
|
564
|
+
except ValueError:
|
|
565
|
+
return False
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _replay_target(event: dict) -> tuple | None:
|
|
569
|
+
kind = "user" if event.get("type") == "turn_started" else "assistant" if event.get("type") in ("assistant_text", "reasoning_summary") else None
|
|
570
|
+
body = event.get("prompt") if kind == "user" else event.get("text")
|
|
571
|
+
if (kind is None or event.get("backend") != "codex" or event.get("imported") is not True
|
|
572
|
+
or not isinstance(event.get("run_id"), str) or not event["run_id"].startswith("import_")
|
|
573
|
+
or type(event.get("seq")) is not int or event["seq"] <= 0
|
|
574
|
+
or not isinstance(event.get("id"), str) or not 0 < len(event["id"]) <= 256
|
|
575
|
+
or not isinstance(body, str) or not body or len(body) > 4 * 1024 * 1024
|
|
576
|
+
or not isinstance(event.get("ts"), str) or not 0 < len(event["ts"]) <= 64
|
|
577
|
+
or event.get("source_text_sha256") is not None and (
|
|
578
|
+
not isinstance(event.get("source_text_sha256"), str)
|
|
579
|
+
or not _DIGEST.fullmatch(event["source_text_sha256"]))):
|
|
580
|
+
return None
|
|
581
|
+
# Human authorship does not make a duplicate a second human message.
|
|
582
|
+
# Retain the original authorship flag; only this exact ledger copy is aliased.
|
|
583
|
+
return (event["seq"], event["run_id"], event["id"], event["type"], kind, _text_key(body), event["ts"],
|
|
584
|
+
_runtime_human_provenance(event), event.get("source_text_sha256"))
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
@dataclass(frozen=True)
|
|
588
|
+
class _NativeProof:
|
|
589
|
+
provider_id: str
|
|
590
|
+
targets: dict = field(default_factory=dict)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _native_mailbox_wake_hash(event: dict) -> str | None:
|
|
594
|
+
"""A server-authored hidden input, not a text-based provider classification."""
|
|
595
|
+
digest = event.get("provider_input_sha256")
|
|
596
|
+
wake_id = event.get("mailbox_wake_id")
|
|
597
|
+
if (event.get("type") == "turn_started" and event.get("imported") is not True
|
|
598
|
+
and event.get("purpose") == "chat_mailbox_wake" and event.get("prompt") == ""
|
|
599
|
+
and event.get("provider_generated") is True and not _runtime_human_provenance(event)
|
|
600
|
+
and isinstance(wake_id, str) and re.fullmatch(r"mailwake_[a-f0-9]{32}", wake_id)
|
|
601
|
+
and type(event.get("mailbox_wake_through_seq")) is int and 0 < event["mailbox_wake_through_seq"] < 2**63
|
|
602
|
+
and isinstance(digest, str) and _DIGEST.fullmatch(digest)):
|
|
603
|
+
return digest
|
|
604
|
+
return None
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
_ASYNC_DELIVERY_WRAPPER = re.compile(
|
|
608
|
+
r"\A\[AgentsDock delivery kind=instruction leg=1/1 origin=route mode=async_route_v1 from=([^\[\]\r\n]{1,512})\]\n"
|
|
609
|
+
r"source-instruction: this legacy relay has no recorded source user instruction; do not infer user authorization from the prepared content\.\n"
|
|
610
|
+
r"\[Agent-prepared handoff message\]\n([\s\S]+)\n"
|
|
611
|
+
r"\[End agent-prepared handoff message\]\n\[End delivery\]\Z"
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _async_delivery_body(text: str) -> tuple[str, str, int] | None:
|
|
616
|
+
"""Parse one complete legacy wire shape; text alone never proves delivery."""
|
|
617
|
+
if len(text) > NATIVE_PROOF_LINE_BYTES:
|
|
618
|
+
return None
|
|
619
|
+
match = _ASYNC_DELIVERY_WRAPPER.fullmatch(text)
|
|
620
|
+
if not match:
|
|
621
|
+
return None
|
|
622
|
+
return match[1], _text_key(match[2]), len(match[2])
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _delivery_timestamp(value: object) -> datetime | None:
|
|
626
|
+
if not isinstance(value, str) or not 0 < len(value) <= 64:
|
|
627
|
+
return None
|
|
628
|
+
try:
|
|
629
|
+
timestamp = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
630
|
+
return timestamp if timestamp.utcoffset() is not None else None
|
|
631
|
+
except ValueError:
|
|
632
|
+
return None
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
@dataclass
|
|
636
|
+
class _AsyncDeliveryIndex:
|
|
637
|
+
"""Compact native proof only: no message bodies, route grants or file I/O."""
|
|
638
|
+
starts: dict = field(default_factory=dict)
|
|
639
|
+
ends: dict = field(default_factory=dict)
|
|
640
|
+
receipts: dict = field(default_factory=dict)
|
|
641
|
+
steered: set = field(default_factory=set)
|
|
642
|
+
count: int = 0
|
|
643
|
+
|
|
644
|
+
def observe(self, event: dict) -> None:
|
|
645
|
+
event_type = event.get("type")
|
|
646
|
+
if (event_type not in {"turn_started", "turn_finished", "turn_steered", "turn_stopped", "turn_queue_run_now",
|
|
647
|
+
"chat_conversation_message_received", "chat_conversation_message_started", "chat_conversation_message_delivered"}
|
|
648
|
+
or event.get("imported") is True or event.get("forked") is True
|
|
649
|
+
or event.get("backend") not in (None, "codex")):
|
|
650
|
+
return
|
|
651
|
+
run = (event.get("interrupted_run_id") or event.get("run_id")) if event_type == "turn_queue_run_now" else event.get("run_id")
|
|
652
|
+
if (event_type in {"turn_steered", "turn_stopped"} or event.get("native_steer") is True):
|
|
653
|
+
if isinstance(run, str) and 0 < len(run) <= 256 and not run.startswith("import_"):
|
|
654
|
+
self.count += run not in self.steered
|
|
655
|
+
self.steered.add(run)
|
|
656
|
+
if self.count > MAX_KEYS:
|
|
657
|
+
raise _Unproven()
|
|
658
|
+
return
|
|
659
|
+
if event_type == "turn_queue_run_now" or (
|
|
660
|
+
event_type in {"turn_started", "turn_finished"} and event.get("purpose") != "cross_chat_handoff_delivery"
|
|
661
|
+
):
|
|
662
|
+
return
|
|
663
|
+
envelope = event.get("cross_chat_envelope_id")
|
|
664
|
+
fields = ("id", "seq", "ts", "backend", "purpose", "run_id", "transport", "exit_code", "stopped", "is_error",
|
|
665
|
+
"provider_thread_id", "provider_turn_id", "cross_chat_envelope_id", "handoff_id", "message_id",
|
|
666
|
+
"source_session_id", "target_session_id", "source_title", "target_run_id", "conversation_mode",
|
|
667
|
+
"kind", "action", "handoff_action", "handoff_body_chars", "handoff_body_sha256",
|
|
668
|
+
"message_revision", "message_edited_by_user", "exchange_id", "cross_chat_exchange_id")
|
|
669
|
+
record = {key: event.get(key) for key in fields
|
|
670
|
+
if event.get(key) is None or type(event.get(key)) in (int, bool)
|
|
671
|
+
or isinstance(event.get(key), str) and len(event[key]) <= 512}
|
|
672
|
+
record["malformed"] = len(record) != len(fields)
|
|
673
|
+
record["type"] = event_type
|
|
674
|
+
if isinstance(run, str) and 0 < len(run) <= 256 and not run.startswith("import_"):
|
|
675
|
+
if event_type == "turn_started":
|
|
676
|
+
body = event.get("prompt")
|
|
677
|
+
if isinstance(body, str) and 0 < len(body) <= NATIVE_PROOF_LINE_BYTES and not _runtime_human_provenance(event):
|
|
678
|
+
record.update(body_key=_text_key(body), body_chars=len(body))
|
|
679
|
+
self.starts.setdefault(run, []).append(record)
|
|
680
|
+
self.count += 1
|
|
681
|
+
elif event_type == "turn_finished":
|
|
682
|
+
self.ends.setdefault(run, []).append(record)
|
|
683
|
+
self.count += 1
|
|
684
|
+
if (event_type in {"chat_conversation_message_received", "chat_conversation_message_started", "chat_conversation_message_delivered"}
|
|
685
|
+
and isinstance(envelope, str) and 0 < len(envelope) <= 256):
|
|
686
|
+
self.receipts.setdefault(envelope, []).append(record)
|
|
687
|
+
self.count += 1
|
|
688
|
+
if self.count > MAX_KEYS:
|
|
689
|
+
raise _Unproven()
|
|
690
|
+
|
|
691
|
+
def match(self, session_id: str, provider_id: str, run: str, origin: dict,
|
|
692
|
+
body: tuple[str, str, int] | None, source_ids: set) -> dict | None:
|
|
693
|
+
starts, ends = self.starts.get(run, []), self.ends.get(run, [])
|
|
694
|
+
if body is None or len(starts) != 1 or len(ends) != 1 or run in self.steered:
|
|
695
|
+
return None
|
|
696
|
+
start, end = starts[0], ends[0]
|
|
697
|
+
envelope = start.get("cross_chat_envelope_id")
|
|
698
|
+
if (start.get("malformed") or end.get("malformed")
|
|
699
|
+
or not isinstance(envelope, str) or not envelope or start.get("backend") != "codex"
|
|
700
|
+
or start.get("conversation_mode") != "async_route_v1" or start.get("target_session_id") != session_id
|
|
701
|
+
or not isinstance(start.get("source_session_id"), str) or not start["source_session_id"]
|
|
702
|
+
or start["source_session_id"] == session_id
|
|
703
|
+
or start.get("exchange_id") or start.get("cross_chat_exchange_id")
|
|
704
|
+
or start.get("body_key") != body[1] or start.get("body_chars") != body[2]
|
|
705
|
+
or start.get("message_edited_by_user") not in (None, False) or start.get("message_revision") not in (None, 0)
|
|
706
|
+
or end.get("backend") != "codex" or end.get("transport") != "app-server"
|
|
707
|
+
or end.get("provider_thread_id") != provider_id or end.get("provider_turn_id") != origin.get("turn_id")
|
|
708
|
+
or end.get("cross_chat_envelope_id") != envelope or type(end.get("exit_code")) is not int or end["exit_code"] != 0
|
|
709
|
+
or end.get("stopped") not in (None, False) or end.get("is_error") not in (None, False)
|
|
710
|
+
or any(end.get(key) != start.get(key) for key in (
|
|
711
|
+
"source_session_id", "target_session_id", "conversation_mode", "handoff_id", "message_id",
|
|
712
|
+
"message_revision", "message_edited_by_user", "exchange_id", "cross_chat_exchange_id"))
|
|
713
|
+
or source_ids != {origin.get("event_id")} or not origin.get("event_id")
|
|
714
|
+
or type(start.get("seq")) is not int or type(end.get("seq")) is not int or start["seq"] >= end["seq"]
|
|
715
|
+
or not isinstance(start.get("id"), str) or not start["id"]):
|
|
716
|
+
return None
|
|
717
|
+
times = [_delivery_timestamp(value) for value in (start.get("ts"), origin.get("timestamp"), end.get("ts"))]
|
|
718
|
+
if any(value is None for value in times) or not times[0] <= times[1] <= times[2]:
|
|
719
|
+
return None
|
|
720
|
+
receipts = self.receipts.get(envelope, [])
|
|
721
|
+
stages = set()
|
|
722
|
+
for receipt in receipts:
|
|
723
|
+
if (receipt.get("malformed") or receipt.get("conversation_mode") != "async_route_v1" or receipt.get("kind") != "instruction"
|
|
724
|
+
or receipt.get("action") != "instruction" or receipt.get("handoff_action") != "instruction"
|
|
725
|
+
or receipt.get("handoff_id") != envelope or receipt.get("message_id") != envelope
|
|
726
|
+
or receipt.get("target_session_id") != session_id or receipt.get("source_session_id") != start.get("source_session_id")
|
|
727
|
+
or receipt.get("source_title") != body[0] or receipt.get("handoff_body_sha256") != body[1]
|
|
728
|
+
or type(receipt.get("handoff_body_chars")) is not int or receipt["handoff_body_chars"] != body[2]
|
|
729
|
+
or receipt.get("message_edited_by_user") not in (None, False) or receipt.get("message_revision") not in (None, 0)
|
|
730
|
+
or receipt.get("exchange_id") or receipt.get("cross_chat_exchange_id")
|
|
731
|
+
or type(receipt.get("seq")) is not int):
|
|
732
|
+
return None
|
|
733
|
+
stage = receipt["type"]
|
|
734
|
+
if stage == "chat_conversation_message_received":
|
|
735
|
+
if receipt["seq"] >= start["seq"] or receipt.get("target_run_id") not in (None, run):
|
|
736
|
+
return None
|
|
737
|
+
elif receipt.get("target_run_id") != run:
|
|
738
|
+
return None
|
|
739
|
+
elif stage == "chat_conversation_message_started" and not start["seq"] < receipt["seq"] < end["seq"]:
|
|
740
|
+
return None
|
|
741
|
+
elif stage == "chat_conversation_message_delivered" and receipt["seq"] <= end["seq"]:
|
|
742
|
+
return None
|
|
743
|
+
stages.add(stage)
|
|
744
|
+
if stages != {"chat_conversation_message_received", "chat_conversation_message_started", "chat_conversation_message_delivered"}:
|
|
745
|
+
return None
|
|
746
|
+
return _native_event_identity(start)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _prove_native_replays(session_id: str, provider_id: str, events: Path, source: Path | None,
|
|
750
|
+
root: Path, parse_item: Callable[[dict], dict | None],
|
|
751
|
+
budget: _NativeReadBudget) -> _NativeProof:
|
|
752
|
+
if not _PROVIDER_ID.fullmatch(provider_id):
|
|
753
|
+
raise _Unproven()
|
|
754
|
+
stamp = _native_stamp(events)
|
|
755
|
+
batches, terminals, candidates, native, owners = {}, {}, [], {}, {}
|
|
756
|
+
deliveries = _AsyncDeliveryIndex()
|
|
757
|
+
previous_seq, retained = 0, 0
|
|
758
|
+
for event, _offset, _line in _native_records(events, stamp, budget):
|
|
759
|
+
seq = event.get("seq")
|
|
760
|
+
if type(seq) is not int or seq <= previous_seq:
|
|
761
|
+
raise _Unproven()
|
|
762
|
+
previous_seq = seq
|
|
763
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
764
|
+
continue
|
|
765
|
+
deliveries.observe(event)
|
|
766
|
+
if len(candidates) + len(batches) + len(terminals) + retained + deliveries.count > MAX_KEYS:
|
|
767
|
+
raise _Unproven()
|
|
768
|
+
run = event.get("run_id")
|
|
769
|
+
if not isinstance(run, str) or not 0 < len(run) <= 256:
|
|
770
|
+
continue
|
|
771
|
+
if run.startswith("import_"):
|
|
772
|
+
if event.get("type") == "history_imported" and event.get("backend") == "codex":
|
|
773
|
+
if run in batches:
|
|
774
|
+
raise _Unproven()
|
|
775
|
+
batches[run] = _native_import_batch(event)
|
|
776
|
+
elif event.get("type") == "turn_finished" and event.get("imported") is True and event.get("backend") == "codex":
|
|
777
|
+
ends = terminals.setdefault(run, [])
|
|
778
|
+
if len(ends) < 2:
|
|
779
|
+
ends.append(seq)
|
|
780
|
+
else:
|
|
781
|
+
target = _replay_target(event)
|
|
782
|
+
if target:
|
|
783
|
+
candidates.append(target)
|
|
784
|
+
elif event.get("imported") is not True and event.get("backend") in (None, "codex"):
|
|
785
|
+
if event.get("type") == "turn_finished" and event.get("backend") == "codex" and event.get("transport") == "app-server":
|
|
786
|
+
thread, turn = event.get("provider_thread_id"), event.get("provider_turn_id")
|
|
787
|
+
if isinstance(thread, str) and _PROVIDER_ID.fullmatch(thread) and isinstance(turn, str) and 0 < len(turn) <= 256:
|
|
788
|
+
owned = owners.setdefault((thread, turn), set())
|
|
789
|
+
retained += run not in owned
|
|
790
|
+
owned.add(run)
|
|
791
|
+
kind = "user" if event.get("type") == "turn_started" else "assistant" if event.get("type") in ("assistant_text", "reasoning_summary", "turn_finished") else None
|
|
792
|
+
body = event.get("prompt") if kind == "user" else event.get("result_text") if event.get("type") == "turn_finished" else event.get("text")
|
|
793
|
+
if kind and isinstance(body, str) and body and len(body) <= 4 * 1024 * 1024:
|
|
794
|
+
native.setdefault((run, kind, _text_key(body)), []).append(_native_event_identity(event))
|
|
795
|
+
retained += 1
|
|
796
|
+
elif kind == "user" and (wake_hash := _native_mailbox_wake_hash(event)):
|
|
797
|
+
native.setdefault((run, kind, wake_hash), []).append(_native_event_identity(event))
|
|
798
|
+
retained += 1
|
|
799
|
+
if len(candidates) + len(batches) + len(terminals) + retained + deliveries.count > MAX_KEYS:
|
|
800
|
+
raise _Unproven()
|
|
801
|
+
if not candidates:
|
|
802
|
+
return _NativeProof(provider_id)
|
|
803
|
+
groups = {}
|
|
804
|
+
for run, batch in batches.items():
|
|
805
|
+
budget.check()
|
|
806
|
+
checkpoint = batch.get("_history_sync_checkpoint")
|
|
807
|
+
cursor = checkpoint.get("cursor") if isinstance(checkpoint, dict) else None
|
|
808
|
+
ends = terminals.get(run, [])
|
|
809
|
+
if (not isinstance(cursor, dict) or checkpoint.get("version") != 1 or cursor.get("version") != 1
|
|
810
|
+
or cursor.get("backend") != "codex" or cursor.get("provider_session_id") != batch.get("provider_session_id")
|
|
811
|
+
or cursor.get("source_path") != batch.get("source_path") or len(ends) != 1
|
|
812
|
+
or type(ends[0]) is not int or type(batch.get("seq")) is not int or ends[0] <= batch["seq"]):
|
|
813
|
+
continue
|
|
814
|
+
key = (batch.get("provider_session_id"), batch.get("source_path"))
|
|
815
|
+
if not all(isinstance(value, str) for value in key) or not _PROVIDER_ID.fullmatch(key[0]):
|
|
816
|
+
continue
|
|
817
|
+
groups.setdefault(key, {})[run] = (batch["seq"], ends[0], checkpoint)
|
|
818
|
+
current = (provider_id, str(source)) if source else None
|
|
819
|
+
selected = ([current] if current in groups else []) + sorted(
|
|
820
|
+
(key for key in groups if key != current), key=lambda key: max(row[0] for row in groups[key].values()), reverse=True,
|
|
821
|
+
)[:MAX_PRIOR_SOURCE_PATHS]
|
|
822
|
+
proofs = {}
|
|
823
|
+
for thread, path in selected:
|
|
824
|
+
proofs.update(_prove_native_source(thread, Path(path), root, groups[(thread, path)], candidates, native, owners, parse_item, budget,
|
|
825
|
+
session_id=session_id, deliveries=deliveries))
|
|
826
|
+
budget.check()
|
|
827
|
+
return _NativeProof(provider_id, proofs)
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _prove_native_source(thread: str, source: Path, root: Path, batches: dict, candidates: list,
|
|
831
|
+
native: dict, owners: dict, parse_item: Callable, budget: _NativeReadBudget,
|
|
832
|
+
*, session_id: str, deliveries: _AsyncDeliveryIndex,
|
|
833
|
+
require_verified_checkpoint: bool = False) -> dict:
|
|
834
|
+
if (not source.is_absolute() or source.is_symlink() or source.suffix != ".jsonl"
|
|
835
|
+
or not (source.stem == thread or source.stem.endswith("-" + thread))):
|
|
836
|
+
raise _Unproven()
|
|
837
|
+
source = source.resolve(strict=True)
|
|
838
|
+
source.relative_to(root.resolve(strict=True))
|
|
839
|
+
stamp = _native_stamp(source)
|
|
840
|
+
wanted, eligible = {}, {}
|
|
841
|
+
for run, (first, last, checkpoint) in batches.items():
|
|
842
|
+
cursor = checkpoint["cursor"]
|
|
843
|
+
start, end = checkpoint.get("previous_source_offset"), cursor.get("source_offset")
|
|
844
|
+
previous, expected = checkpoint.get("previous_source_digest"), cursor.get("source_digest")
|
|
845
|
+
if (type(start) is not int or type(end) is not int or not 0 <= start < end <= stamp[2]
|
|
846
|
+
or (cursor.get("source_dev"), cursor.get("source_ino")) != stamp[:2]
|
|
847
|
+
or not isinstance(expected, str) or not _DIGEST.fullmatch(expected)
|
|
848
|
+
or type(checkpoint.get("previous_present")) is not bool
|
|
849
|
+
or start == 0 and previous != ""
|
|
850
|
+
or start > 0 and (checkpoint.get("previous_present") is not True or not isinstance(previous, str) or not _DIGEST.fullmatch(previous))):
|
|
851
|
+
if require_verified_checkpoint:
|
|
852
|
+
raise _Unproven()
|
|
853
|
+
continue
|
|
854
|
+
wanted.setdefault(end, set()).add(expected)
|
|
855
|
+
if start:
|
|
856
|
+
wanted.setdefault(start, set()).add(previous)
|
|
857
|
+
eligible[run] = (first, last, start, end, expected, previous)
|
|
858
|
+
if not eligible:
|
|
859
|
+
if require_verified_checkpoint:
|
|
860
|
+
raise _Unproven()
|
|
861
|
+
return {}
|
|
862
|
+
# Only source messages capable of proving requested imported rows need to
|
|
863
|
+
# occupy memory. Tools/private reasoning still contribute to exact digests.
|
|
864
|
+
candidate_keys = {(target[4], target[5], target[8]) for target in candidates
|
|
865
|
+
if target[1] in eligible}
|
|
866
|
+
digest, verified, canonical, occurrences = hashlib.sha256(), set(), {}, {}
|
|
867
|
+
assistant_native_keys = {}
|
|
868
|
+
compactions, candidate_assistants = {}, set()
|
|
869
|
+
compaction_tracker = CodexCompactionSummaryTracker(thread)
|
|
870
|
+
delivery_bodies, delivery_source_ids = {}, {}
|
|
871
|
+
allowed_header_owners, header_parents = {thread}, {}
|
|
872
|
+
context_turn = None
|
|
873
|
+
retained = 0
|
|
874
|
+
for record, offset, line in _native_records(source, stamp, budget, end=max(wanted)):
|
|
875
|
+
payload = record.get("payload")
|
|
876
|
+
if offset == len(line) or record.get("type") == "session_meta":
|
|
877
|
+
if record.get("type") != "session_meta" or not isinstance(payload, dict):
|
|
878
|
+
raise _Unproven()
|
|
879
|
+
owner, parent = payload.get("id"), payload.get("forked_from_id")
|
|
880
|
+
if (not isinstance(owner, str) or owner not in allowed_header_owners
|
|
881
|
+
or offset == len(line) and owner != thread
|
|
882
|
+
or parent is not None and (not isinstance(parent, str) or not _PROVIDER_ID.fullmatch(parent))
|
|
883
|
+
or owner in header_parents and header_parents[owner] != parent
|
|
884
|
+
or owner not in header_parents and parent in header_parents):
|
|
885
|
+
raise _Unproven()
|
|
886
|
+
# Forked rollouts retain an ancestral session_meta immediately
|
|
887
|
+
# after their own header. Only the declared parent chain is valid;
|
|
888
|
+
# an unrelated embedded owner cannot relabel this source file.
|
|
889
|
+
header_parents[owner] = parent
|
|
890
|
+
if len(header_parents) > MAX_FORK_META_HEADERS:
|
|
891
|
+
raise _Unproven()
|
|
892
|
+
if parent is not None:
|
|
893
|
+
allowed_header_owners.add(parent)
|
|
894
|
+
digest.update(line)
|
|
895
|
+
for expected in wanted.get(offset, ()):
|
|
896
|
+
if hmac.compare_digest(digest.hexdigest(), expected):
|
|
897
|
+
verified.add((offset, expected))
|
|
898
|
+
compaction = compaction_tracker.observe(record)
|
|
899
|
+
if compaction is not None:
|
|
900
|
+
identity = (compaction["event_id"], compaction["turn_id"], compaction["timestamp"])
|
|
901
|
+
if identity in candidate_assistants:
|
|
902
|
+
compactions[identity] = (offset, compaction)
|
|
903
|
+
retained += 1
|
|
904
|
+
if retained > MAX_KEYS:
|
|
905
|
+
raise _Unproven()
|
|
906
|
+
if not isinstance(payload, dict):
|
|
907
|
+
continue
|
|
908
|
+
if record.get("type") == "turn_context" or record.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
909
|
+
context_turn = payload.get("turn_id") if isinstance(payload.get("turn_id"), str) else None
|
|
910
|
+
item = parse_item(record)
|
|
911
|
+
if not item or item.get("kind") not in ("user", "assistant") or not isinstance(item.get("text"), str):
|
|
912
|
+
continue
|
|
913
|
+
origin = codex_public_item_origin(record, thread)
|
|
914
|
+
runtime_kind = item.get("provider_runtime_context")
|
|
915
|
+
if (origin and runtime_kind in ("subagent_notification", "turn_aborted", "provider_notice")
|
|
916
|
+
and item.get("provider_user_authored") is not True
|
|
917
|
+
and (item.get("provider_origin") or {}).get("kind") == runtime_kind):
|
|
918
|
+
origin = {**origin, "kind": runtime_kind}
|
|
919
|
+
turn = origin["turn_id"] if origin else context_turn
|
|
920
|
+
timestamp = record.get("timestamp")
|
|
921
|
+
if (not isinstance(turn, str) or not 0 < len(turn) <= 256
|
|
922
|
+
or not isinstance(timestamp, str) or not 0 < len(timestamp) <= 64):
|
|
923
|
+
continue
|
|
924
|
+
key = (turn, item["kind"], _text_key(item["text"]), item.get("source_text_sha256"))
|
|
925
|
+
if origin and item["kind"] == "user" and any(run in deliveries.starts for run in owners.get((thread, turn), ())):
|
|
926
|
+
ids = delivery_source_ids.setdefault(turn, set())
|
|
927
|
+
if len(ids) < 2:
|
|
928
|
+
retained += origin["event_id"] not in ids
|
|
929
|
+
ids.add(origin["event_id"])
|
|
930
|
+
if retained > MAX_KEYS:
|
|
931
|
+
raise _Unproven()
|
|
932
|
+
if (item["kind"], key[2], key[3]) not in candidate_keys:
|
|
933
|
+
continue
|
|
934
|
+
if item["kind"] == "user" and (body := _async_delivery_body(item["text"])):
|
|
935
|
+
delivery_bodies[key] = body
|
|
936
|
+
if item["kind"] == "assistant":
|
|
937
|
+
cleaned = _native_assistant_text(item["text"])
|
|
938
|
+
if cleaned:
|
|
939
|
+
assistant_native_keys[key] = _text_key(cleaned)
|
|
940
|
+
if origin:
|
|
941
|
+
candidate_assistants.add((origin["event_id"], origin["turn_id"], origin["timestamp"]))
|
|
942
|
+
if origin:
|
|
943
|
+
ids = canonical.setdefault(key, {})
|
|
944
|
+
retained += origin["event_id"] not in ids
|
|
945
|
+
ids[origin["event_id"]] = origin
|
|
946
|
+
occurrences.setdefault((item["kind"], key[2], timestamp, key[3]), []).append((offset, key))
|
|
947
|
+
retained += 1
|
|
948
|
+
if retained > MAX_KEYS:
|
|
949
|
+
raise _Unproven()
|
|
950
|
+
if require_verified_checkpoint and any((offset, expected) not in verified
|
|
951
|
+
for offset, expectations in wanted.items() for expected in expectations):
|
|
952
|
+
raise _Unproven()
|
|
953
|
+
proofs = {}
|
|
954
|
+
for target in candidates:
|
|
955
|
+
budget.check()
|
|
956
|
+
seq, run, _id, _type, kind, body_key, timestamp, _human, source_hash = target
|
|
957
|
+
batch = eligible.get(run)
|
|
958
|
+
if not batch:
|
|
959
|
+
continue
|
|
960
|
+
first, last, start, end, expected, previous = batch
|
|
961
|
+
if not (first < seq < last and (end, expected) in verified and (start == 0 or (start, previous) in verified)):
|
|
962
|
+
continue
|
|
963
|
+
matches = set()
|
|
964
|
+
for offset, key in occurrences.get((kind, body_key, timestamp, source_hash), []):
|
|
965
|
+
budget.check()
|
|
966
|
+
if start < offset <= end:
|
|
967
|
+
matches.add(key)
|
|
968
|
+
if len(matches) != 1:
|
|
969
|
+
continue
|
|
970
|
+
key = next(iter(matches))
|
|
971
|
+
source_ids, owned_runs = canonical.get(key, {}), owners.get((thread, key[0]), set())
|
|
972
|
+
# Compaction owns a particular source item and timestamp. Another
|
|
973
|
+
# genuine answer with the same body in this turn is a distinct item.
|
|
974
|
+
timestamp_origins = [origin for origin in source_ids.values() if origin["timestamp"] == timestamp]
|
|
975
|
+
compaction_matches = [compactions[identity] for origin in timestamp_origins
|
|
976
|
+
if len(timestamp_origins) == 1
|
|
977
|
+
and (identity := (origin["event_id"], origin["turn_id"], origin["timestamp"])) in compactions
|
|
978
|
+
and compactions[identity][0] <= end]
|
|
979
|
+
if kind == "assistant" and source_hash is None and len(compaction_matches) == 1:
|
|
980
|
+
proofs[target] = compaction_matches[0][1]
|
|
981
|
+
if len(proofs) > MAX_TARGETS:
|
|
982
|
+
raise _Unproven()
|
|
983
|
+
continue
|
|
984
|
+
if len(source_ids) != 1 or source_hash is not None:
|
|
985
|
+
# Truncated source hashes need a separately retained native full-body
|
|
986
|
+
# hash. Until one exists, the bounded preview cannot prove equality.
|
|
987
|
+
continue
|
|
988
|
+
source_origin = next(iter(source_ids.values()))
|
|
989
|
+
if source_origin.get("kind") in ("subagent_notification", "turn_aborted", "provider_notice"):
|
|
990
|
+
if kind == "user" and not _human:
|
|
991
|
+
proofs[target] = {**source_origin, "source_text_sha256": body_key}
|
|
992
|
+
if len(proofs) > MAX_TARGETS:
|
|
993
|
+
raise _Unproven()
|
|
994
|
+
continue
|
|
995
|
+
if len(owned_runs) != 1:
|
|
996
|
+
continue
|
|
997
|
+
native_run = next(iter(owned_runs))
|
|
998
|
+
native_matches = native.get((native_run, kind, body_key), [])
|
|
999
|
+
if not native_matches and kind == "user":
|
|
1000
|
+
delivery = deliveries.match(session_id, thread, native_run, source_origin,
|
|
1001
|
+
delivery_bodies.get(key), delivery_source_ids.get(key[0], set()))
|
|
1002
|
+
if delivery is not None:
|
|
1003
|
+
native_matches = [delivery]
|
|
1004
|
+
if not native_matches and kind == "assistant":
|
|
1005
|
+
# Native delivery removes line-leading decorations. Only the same
|
|
1006
|
+
# public provider item may use that normalization; a similar body
|
|
1007
|
+
# in another item/turn is not replay evidence.
|
|
1008
|
+
native_key = assistant_native_keys.get(key)
|
|
1009
|
+
if native_key and native_key != body_key:
|
|
1010
|
+
native_matches = [event for event in native.get((native_run, kind, native_key), [])
|
|
1011
|
+
if _public_assistant_item_id(event) == source_origin["event_id"]]
|
|
1012
|
+
native_matches = [event for event in native_matches if type(event.get("seq")) is int and event["seq"] < first and isinstance(event.get("id"), str)]
|
|
1013
|
+
if not native_matches:
|
|
1014
|
+
continue
|
|
1015
|
+
representative = min(native_matches, key=lambda event: event["seq"])
|
|
1016
|
+
proofs[target] = {**next(iter(source_ids.values())), "native_event_id": representative["id"], "source_text_sha256": body_key}
|
|
1017
|
+
if len(proofs) > MAX_TARGETS:
|
|
1018
|
+
raise _Unproven()
|
|
1019
|
+
budget.check()
|
|
1020
|
+
return proofs
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
class CodexNativeHistoryRepairCache(CodexGoalHistoryRepairCache):
|
|
1024
|
+
"""An explicit history boundary prepares proof; per-event projection does no IO."""
|
|
1025
|
+
def prepare(self, session_id: str, provider_id: str, events_path: Path, source_path: Path | None,
|
|
1026
|
+
root: Path, parse_item: Callable[[dict], dict | None], *,
|
|
1027
|
+
cancelled: Callable[[], bool] | None = None, deadline: float | None = None) -> bool:
|
|
1028
|
+
with self._prepare_lock:
|
|
1029
|
+
with self._lock:
|
|
1030
|
+
previous = self._proofs.get(session_id)
|
|
1031
|
+
if previous and previous.provider_id == provider_id:
|
|
1032
|
+
return False
|
|
1033
|
+
self._preparing, self._cancelled = session_id, False
|
|
1034
|
+
budget = _NativeReadBudget(lambda: self._cancelled or bool(cancelled and cancelled()), deadline)
|
|
1035
|
+
try:
|
|
1036
|
+
proof = _prove_native_replays(
|
|
1037
|
+
session_id, provider_id, events_path, source_path, root, parse_item,
|
|
1038
|
+
budget,
|
|
1039
|
+
)
|
|
1040
|
+
budget.check()
|
|
1041
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError) as exc:
|
|
1042
|
+
with self._lock:
|
|
1043
|
+
self._preparing = None
|
|
1044
|
+
# Unavailable proof is neither a negative answer nor a prepared
|
|
1045
|
+
# cache entry. The explicit worker decides when to retry it.
|
|
1046
|
+
raise CodexNativeHistoryProofUnavailable("Codex native history proof is incomplete") from exc
|
|
1047
|
+
with self._lock:
|
|
1048
|
+
self._preparing = None
|
|
1049
|
+
budget.check()
|
|
1050
|
+
self._proofs[session_id] = proof
|
|
1051
|
+
self._proofs.move_to_end(session_id)
|
|
1052
|
+
while len(self._proofs) > MAX_SESSIONS:
|
|
1053
|
+
self._proofs.popitem(last=False)
|
|
1054
|
+
return bool(proof.targets)
|
|
1055
|
+
|
|
1056
|
+
def signature(self, session_id: str) -> frozenset:
|
|
1057
|
+
with self._lock:
|
|
1058
|
+
proof = self._proofs.get(session_id)
|
|
1059
|
+
return frozenset(proof.targets) if proof else frozenset()
|
|
1060
|
+
|
|
1061
|
+
def project_event(self, session_id: str, event: dict) -> dict | None:
|
|
1062
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
1063
|
+
return None
|
|
1064
|
+
if _persisted_runtime_marker(event, session_id):
|
|
1065
|
+
return {**event, "_agentsdock_imported_prompt_hidden": True}
|
|
1066
|
+
if (event.get("provider_history_repair") == "source_proven_native_replay"
|
|
1067
|
+
and event.get("metadata_only") is True and event.get("backend") == "codex"
|
|
1068
|
+
and event.get("imported") is True and str(event.get("run_id") or "").startswith("import_")
|
|
1069
|
+
and event.get("type") == "turn_started" and event.get("prompt") == ""
|
|
1070
|
+
and isinstance(event.get("provider_origin"), dict)
|
|
1071
|
+
and event["provider_origin"].get("provider") == "codex"
|
|
1072
|
+
and event["provider_origin"].get("native_event_id")):
|
|
1073
|
+
return {**event, "_agentsdock_imported_prompt_hidden": True}
|
|
1074
|
+
with self._lock:
|
|
1075
|
+
proof = self._proofs.get(session_id)
|
|
1076
|
+
if not proof:
|
|
1077
|
+
return None
|
|
1078
|
+
target = _replay_target(event)
|
|
1079
|
+
origin = proof.targets.get(target) if target else None
|
|
1080
|
+
if origin is None:
|
|
1081
|
+
return None
|
|
1082
|
+
if origin.get("kind") == "compaction_summary":
|
|
1083
|
+
return {**event, "text": "", "metadata_only": True,
|
|
1084
|
+
"provider_history_repair": "source_proven_compaction", "provider_origin": origin}
|
|
1085
|
+
if origin.get("kind") in ("subagent_notification", "turn_aborted", "provider_notice"):
|
|
1086
|
+
return {**event, "prompt": "", "metadata_only": True,
|
|
1087
|
+
"provider_runtime_context": origin["kind"], "provider_origin": origin,
|
|
1088
|
+
"_agentsdock_imported_prompt_hidden": True}
|
|
1089
|
+
return {**event, "prompt" if event["type"] == "turn_started" else "text": "",
|
|
1090
|
+
"metadata_only": True, "provider_history_repair": "source_proven_native_replay", "provider_origin": origin,
|
|
1091
|
+
**({"_agentsdock_imported_prompt_hidden": True} if event["type"] == "turn_started" else {})}
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _prove_pending_async_deliveries(session_id: str, provider_id: str, items: list[dict],
|
|
1095
|
+
source: Path, root: Path, checkpoint: dict, parse_item: Callable,
|
|
1096
|
+
previous_seq: int, owners: dict, deliveries: _AsyncDeliveryIndex,
|
|
1097
|
+
budget: _NativeReadBudget) -> dict:
|
|
1098
|
+
"""Reuse pinned-prefix proof before broadcast; the synthetic batch is never stored."""
|
|
1099
|
+
cursor = checkpoint.get("cursor")
|
|
1100
|
+
if (checkpoint.get("version") != 1 or not isinstance(cursor, dict) or cursor.get("version") != 1
|
|
1101
|
+
or cursor.get("backend") != "codex" or cursor.get("provider_session_id") != provider_id
|
|
1102
|
+
or cursor.get("source_path") != str(source)):
|
|
1103
|
+
raise _Unproven()
|
|
1104
|
+
run = "import_pending_native_delivery_proof"
|
|
1105
|
+
candidates, indexes = [], {}
|
|
1106
|
+
for index, item in enumerate(items):
|
|
1107
|
+
budget.check()
|
|
1108
|
+
origin = item.get("provider_origin")
|
|
1109
|
+
if (item.get("kind") != "user" or not isinstance(item.get("text"), str)
|
|
1110
|
+
or not isinstance(origin, dict) or origin.get("provider") != "codex" or origin.get("kind") != "user"
|
|
1111
|
+
or origin.get("session_id", provider_id) != provider_id or item.get("source_text_sha256") is not None
|
|
1112
|
+
or not _async_delivery_body(item["text"])):
|
|
1113
|
+
continue
|
|
1114
|
+
candidate = _replay_target({"seq": previous_seq + index + 2, "id": f"pending:{index}", "run_id": run,
|
|
1115
|
+
"type": "turn_started", "backend": "codex", "imported": True, "prompt": item["text"],
|
|
1116
|
+
"ts": origin.get("timestamp"), "provider_user_authored": _runtime_human_provenance(item)})
|
|
1117
|
+
if candidate is not None:
|
|
1118
|
+
candidates.append(candidate)
|
|
1119
|
+
indexes[candidate] = index
|
|
1120
|
+
if len(candidates) > MAX_TARGETS:
|
|
1121
|
+
raise _Unproven()
|
|
1122
|
+
if not candidates:
|
|
1123
|
+
return {}
|
|
1124
|
+
proof = _prove_native_source(provider_id, source, root,
|
|
1125
|
+
{run: (previous_seq + 1, previous_seq + len(items) + 2, checkpoint)}, candidates, {},
|
|
1126
|
+
{(provider_id, turn): runs for turn, runs in owners.items()}, parse_item, budget,
|
|
1127
|
+
session_id=session_id, deliveries=deliveries, require_verified_checkpoint=True)
|
|
1128
|
+
result = {}
|
|
1129
|
+
for candidate, origin in proof.items():
|
|
1130
|
+
index = indexes[candidate]
|
|
1131
|
+
item_origin = items[index]["provider_origin"]
|
|
1132
|
+
if all(origin.get(key) == item_origin.get(key) for key in ("provider", "kind", "event_id", "turn_id", "timestamp")):
|
|
1133
|
+
result[index] = origin
|
|
1134
|
+
return result
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def filter_native_codex_history_items(session_id: str, provider_id: str, events: Path, items: list[dict], *,
|
|
1138
|
+
cancelled: Callable[[], bool] | None = None,
|
|
1139
|
+
deadline: float | None = None,
|
|
1140
|
+
source_path: Path | None = None, root: Path | None = None,
|
|
1141
|
+
sync_checkpoint: dict | None = None,
|
|
1142
|
+
parse_item: Callable[[dict], dict | None] | None = None) -> list[dict]:
|
|
1143
|
+
"""An existing verified import boundary can omit exact completed native copies.
|
|
1144
|
+
|
|
1145
|
+
Exact native bodies need only the ledger. Legacy async wrappers additionally
|
|
1146
|
+
require the caller's source/checkpoint/parser and one verified source-prefix
|
|
1147
|
+
scan: a delta range alone cannot disambiguate a later same-turn human quote.
|
|
1148
|
+
Missing source context leaves those wrappers visible for later read repair.
|
|
1149
|
+
Incomplete proof raises; the caller must defer import and its cursor commit.
|
|
1150
|
+
"""
|
|
1151
|
+
if not items:
|
|
1152
|
+
return []
|
|
1153
|
+
try:
|
|
1154
|
+
if not _PROVIDER_ID.fullmatch(provider_id):
|
|
1155
|
+
raise _Unproven()
|
|
1156
|
+
budget = _NativeReadBudget(cancelled, deadline)
|
|
1157
|
+
stamp = _native_stamp(events)
|
|
1158
|
+
owners, native, assistant_items, wake_keys, native_count = {}, {}, {}, set(), 0
|
|
1159
|
+
deliveries = _AsyncDeliveryIndex()
|
|
1160
|
+
prove_deliveries = (source_path is not None and root is not None
|
|
1161
|
+
and isinstance(sync_checkpoint, dict) and callable(parse_item)
|
|
1162
|
+
and any(item.get("kind") == "user" and isinstance(item.get("text"), str)
|
|
1163
|
+
and _async_delivery_body(item["text"]) for item in items))
|
|
1164
|
+
previous_seq, owner_count = 0, 0
|
|
1165
|
+
for event, _offset, _line in _native_records(events, stamp, budget):
|
|
1166
|
+
seq = event.get("seq")
|
|
1167
|
+
if type(seq) is not int or seq <= previous_seq:
|
|
1168
|
+
raise _Unproven()
|
|
1169
|
+
previous_seq = seq
|
|
1170
|
+
run = event.get("run_id")
|
|
1171
|
+
if prove_deliveries and event.get("session_id") in (None, "", session_id):
|
|
1172
|
+
deliveries.observe(event)
|
|
1173
|
+
if native_count + owner_count + len(assistant_items) + deliveries.count > MAX_KEYS:
|
|
1174
|
+
raise _Unproven()
|
|
1175
|
+
if (event.get("session_id") not in (None, "", session_id) or event.get("imported") is True
|
|
1176
|
+
or event.get("backend") not in (None, "codex") or not isinstance(run, str)
|
|
1177
|
+
or not 0 < len(run) <= 256 or run.startswith("import_")):
|
|
1178
|
+
continue
|
|
1179
|
+
if (event.get("type") == "turn_finished" and event.get("backend") == "codex"
|
|
1180
|
+
and event.get("transport") == "app-server" and event.get("provider_thread_id") == provider_id
|
|
1181
|
+
and isinstance(event.get("provider_turn_id"), str) and 0 < len(event["provider_turn_id"]) <= 256):
|
|
1182
|
+
owned = owners.setdefault(event["provider_turn_id"], set())
|
|
1183
|
+
owner_count += run not in owned
|
|
1184
|
+
owned.add(run)
|
|
1185
|
+
kind = "user" if event.get("type") == "turn_started" else "assistant" if event.get("type") in ("assistant_text", "reasoning_summary", "turn_finished") else None
|
|
1186
|
+
body = event.get("prompt") if kind == "user" else event.get("result_text") if event.get("type") == "turn_finished" else event.get("text")
|
|
1187
|
+
wake_hash = _native_mailbox_wake_hash(event) if kind == "user" else None
|
|
1188
|
+
if kind and (isinstance(body, str) and body or wake_hash):
|
|
1189
|
+
keys = native.setdefault(run, {})
|
|
1190
|
+
key = (kind, wake_hash or _text_key(body))
|
|
1191
|
+
native_count += key not in keys
|
|
1192
|
+
if isinstance(event.get("id"), str) and 0 < len(event["id"]) <= 256:
|
|
1193
|
+
keys.setdefault(key, event["id"])
|
|
1194
|
+
if wake_hash:
|
|
1195
|
+
wake_keys.add((run, wake_hash))
|
|
1196
|
+
item_id = _public_assistant_item_id(event)
|
|
1197
|
+
if item_id is not None:
|
|
1198
|
+
assistant_items[(run, item_id, key[1])] = event["id"]
|
|
1199
|
+
if native_count + owner_count + len(assistant_items) + deliveries.count > MAX_KEYS:
|
|
1200
|
+
raise _Unproven()
|
|
1201
|
+
delivery_proofs = _prove_pending_async_deliveries(
|
|
1202
|
+
session_id, provider_id, items, source_path, root, sync_checkpoint, parse_item,
|
|
1203
|
+
previous_seq, owners, deliveries, budget,
|
|
1204
|
+
) if prove_deliveries and deliveries.starts else {}
|
|
1205
|
+
# A hidden wake has no public prompt body. Require a single exact source
|
|
1206
|
+
# item in this verified import range, in addition to native turn ownership.
|
|
1207
|
+
wake_source_ids = {}
|
|
1208
|
+
for item in items if wake_keys else ():
|
|
1209
|
+
budget.check()
|
|
1210
|
+
origin = item.get("provider_origin")
|
|
1211
|
+
if (item.get("kind") == "user" and isinstance(item.get("text"), str)
|
|
1212
|
+
and isinstance(origin, dict)):
|
|
1213
|
+
key = (origin.get("turn_id"), _text_key(item["text"]))
|
|
1214
|
+
wake_source_ids.setdefault(key, set()).add(origin.get("event_id"))
|
|
1215
|
+
result = []
|
|
1216
|
+
for index, item in enumerate(items):
|
|
1217
|
+
budget.check()
|
|
1218
|
+
origin = item.get("provider_origin")
|
|
1219
|
+
runs = owners.get(origin.get("turn_id"), set()) if isinstance(origin, dict) else set()
|
|
1220
|
+
owned = (isinstance(origin, dict) and origin.get("provider") == "codex"
|
|
1221
|
+
and origin.get("kind") == item.get("kind") and isinstance(origin.get("event_id"), str)
|
|
1222
|
+
and origin.get("session_id", provider_id) == provider_id and len(runs) == 1
|
|
1223
|
+
and item.get("source_text_sha256") is None and isinstance(item.get("text"), str))
|
|
1224
|
+
known = owned and (item["kind"], _text_key(item["text"])) in native.get(next(iter(runs)), {})
|
|
1225
|
+
delivery = delivery_proofs.get(index) if owned else None
|
|
1226
|
+
known = known or delivery is not None
|
|
1227
|
+
if known and (next(iter(runs)), _text_key(item["text"])) in wake_keys:
|
|
1228
|
+
known = (bool(origin.get("event_id")) and bool(origin.get("timestamp"))
|
|
1229
|
+
and len(wake_source_ids.get((origin.get("turn_id"), _text_key(item["text"])), set())) == 1)
|
|
1230
|
+
if owned and not known and item["kind"] == "assistant":
|
|
1231
|
+
cleaned = _native_assistant_text(item["text"])
|
|
1232
|
+
known = bool(cleaned and (next(iter(runs)), origin["event_id"], _text_key(cleaned)) in assistant_items)
|
|
1233
|
+
if not known:
|
|
1234
|
+
result.append(item)
|
|
1235
|
+
elif item["kind"] == "user":
|
|
1236
|
+
# Retain a silent exact input boundary: a later unmatched
|
|
1237
|
+
# answer must not become the answer to an earlier real input.
|
|
1238
|
+
key = (item["kind"], _text_key(item["text"]))
|
|
1239
|
+
result.append({**item, "text": "", "metadata_only": True,
|
|
1240
|
+
"provider_history_repair": "source_proven_native_replay",
|
|
1241
|
+
"provider_origin": {**origin, "session_id": provider_id,
|
|
1242
|
+
"native_event_id": delivery["native_event_id"] if delivery else native[next(iter(runs))][key],
|
|
1243
|
+
"source_text_sha256": key[1]}})
|
|
1244
|
+
budget.check()
|
|
1245
|
+
return result
|
|
1246
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError) as exc:
|
|
1247
|
+
raise CodexNativeHistoryProofUnavailable("Codex native import ownership proof is incomplete") from exc
|