@agentsdock/server 1.0.7-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +19 -0
- package/README.md +60 -0
- package/npm/cli.cjs +289 -0
- package/package.json +32 -0
- package/server/LICENSE +202 -0
- package/server/NOTICE +19 -0
- package/server/VERSION +1 -0
- package/server/activation_transaction.py +2458 -0
- package/server/agent_server.py +93226 -0
- package/server/agentsdock_chats.py +1237 -0
- package/server/agentsdock_emergency.py +338 -0
- package/server/agentsdock_jobs.py +640 -0
- package/server/agentsdock_mail.py +325 -0
- package/server/agentsdock_publish.py +404 -0
- package/server/agentsdock_team.py +793 -0
- package/server/agentsdock_team_hub/__init__.py +33 -0
- package/server/agentsdock_team_hub/auth.py +710 -0
- package/server/agentsdock_team_hub/cli.py +647 -0
- package/server/agentsdock_team_hub/database.py +310 -0
- package/server/agentsdock_team_hub/mail_hint_streams.py +176 -0
- package/server/agentsdock_team_hub/mail_hints.py +201 -0
- package/server/agentsdock_team_hub/migrations/0001_identity_auth.sql +690 -0
- package/server/agentsdock_team_hub/migrations/0002_teamspace_ledger.sql +1336 -0
- package/server/agentsdock_team_hub/migrations/0003_service_runtime.sql +361 -0
- package/server/agentsdock_team_hub/migrations/0004_managed_host_binding.sql +22 -0
- package/server/agentsdock_team_hub/migrations/0005_tailnet_bootstrap_delegations.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0006_team_network_mailbox.sql +423 -0
- package/server/agentsdock_team_hub/migrations/0007_local_agent_mail.sql +63 -0
- package/server/agentsdock_team_hub/migrations/0008_managed_server_session.sql +73 -0
- package/server/agentsdock_team_hub/migrations/0009_team_messages.sql +324 -0
- package/server/agentsdock_team_hub/migrations/0010_team_attachment_orphan_reclamation.sql +52 -0
- package/server/agentsdock_team_hub/migrations/0011_human_admin_paging.sql +65 -0
- package/server/agentsdock_team_hub/migrations/0012_network_content_deletions.sql +69 -0
- package/server/agentsdock_team_hub/migrations/0013_team_message_revisions.sql +48 -0
- package/server/agentsdock_team_hub/migrations/0014_managed_network_owner.sql +50 -0
- package/server/agentsdock_team_hub/migrations/0015_team_message_inbox_dismissals.sql +3 -0
- package/server/agentsdock_team_hub/migrations/0016_team_message_all_servers.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0017_skill_announcement_deletions.sql +16 -0
- package/server/agentsdock_team_hub/migrations/0018_team_mail_subjects.sql +8 -0
- package/server/agentsdock_team_hub/migrations/0019_team_mailbox_state.sql +5 -0
- package/server/agentsdock_team_hub/migrations/0020_team_mail_arrivals.sql +44 -0
- package/server/agentsdock_team_hub/migrations/0021_team_mail_threads.sql +4 -0
- package/server/agentsdock_team_hub/migrations/0022_team_bulletin_changes.sql +49 -0
- package/server/agentsdock_team_hub/migrations/0023_team_message_search.sql +93 -0
- package/server/agentsdock_team_hub/migrations/__init__.py +1 -0
- package/server/agentsdock_team_hub/notification_hints.py +306 -0
- package/server/agentsdock_team_hub/secure_peer.py +11215 -0
- package/server/agentsdock_team_hub/secure_peer_hub.py +1372 -0
- package/server/agentsdock_team_hub/security.py +318 -0
- package/server/agentsdock_team_hub/service.py +1737 -0
- package/server/agentsdock_team_hub/store.py +16276 -0
- package/server/chat_mailbox.py +448 -0
- package/server/claude_background_reconciliation.py +92 -0
- package/server/claude_goals.py +245 -0
- package/server/claude_history_provenance.py +137 -0
- package/server/claude_history_repair.py +1601 -0
- package/server/claude_sdk_client.py +3731 -0
- package/server/claude_side_question.py +212 -0
- package/server/codex_app_server.py +3030 -0
- package/server/codex_auth.py +93 -0
- package/server/codex_history_repair.py +1247 -0
- package/server/codex_provider.py +1120 -0
- package/server/codex_side_question.py +301 -0
- package/server/cursor_agent_client.py +654 -0
- package/server/cursor_process_guard.py +347 -0
- package/server/execution_activation.py +610 -0
- package/server/execution_control.py +233 -0
- package/server/execution_durability.py +184 -0
- package/server/execution_http.py +256 -0
- package/server/execution_install.py +792 -0
- package/server/execution_legacy_runner.py +406 -0
- package/server/execution_maintenance.py +137 -0
- package/server/execution_manage.py +615 -0
- package/server/execution_ownership.py +83 -0
- package/server/execution_preparation.py +362 -0
- package/server/execution_recovery.py +845 -0
- package/server/execution_recovery_status.py +195 -0
- package/server/execution_service.py +387 -0
- package/server/execution_transport.py +622 -0
- package/server/execution_uninstall.py +406 -0
- package/server/execution_update_status.py +49 -0
- package/server/install.sh +7074 -0
- package/server/interactive_chat_controls.py +153 -0
- package/server/interactive_chat_native.py +66 -0
- package/server/interactive_chat_projection.py +152 -0
- package/server/interactive_chat_runtime.py +132 -0
- package/server/interactive_chat_share_routes.py +534 -0
- package/server/interactive_chat_share_web.py +12 -0
- package/server/interactive_chat_shares.py +238 -0
- package/server/provider_commands.py +425 -0
- package/server/public_chat_share_routes.py +426 -0
- package/server/public_chat_shares.py +759 -0
- package/server/public_chat_transcript.py +357 -0
- package/server/pyproject.toml +29 -0
- package/server/release-public-key.pem +3 -0
- package/server/secure_peer_delivery.py +1236 -0
- package/server/secure_peer_runtime.py +7694 -0
- package/server/shared_chat_video_stream.py +262 -0
- package/server/shared_chat_videos.py +155 -0
- package/server/side_questions.py +594 -0
- package/server/team_hub_host.py +1354 -0
- package/server/team_mail_grants.py +136 -0
- package/server/team_mail_runtime.py +302 -0
- package/server/team_mail_websocket.py +211 -0
- package/server/title_generation.py +333 -0
- package/server/uninstall.sh +359 -0
- package/server/update_handoff.py +237 -0
- package/server/update_preparation.py +264 -0
- package/server/update_recovery.py +140 -0
- package/server/update_runner.py +1819 -0
- package/server/uv.lock +1101 -0
- package/server/workspace_git.py +545 -0
|
@@ -0,0 +1,1601 @@
|
|
|
1
|
+
"""Bounded, source-proven projection repair for old Claude metadata imports.
|
|
2
|
+
|
|
3
|
+
No transcript discovery, polling, durable mutation, or server imports. A caller
|
|
4
|
+
explicitly prepares one session; subsequent per-event checks use memory only.
|
|
5
|
+
Uncheckpointed imports and ambiguous user quotations are intentionally retained.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections import OrderedDict
|
|
10
|
+
from bisect import bisect_right
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
import hashlib
|
|
14
|
+
import hmac
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
import re
|
|
19
|
+
import stat
|
|
20
|
+
import threading
|
|
21
|
+
from types import MappingProxyType
|
|
22
|
+
from typing import Callable
|
|
23
|
+
|
|
24
|
+
from claude_history_provenance import ClaudeInterruptionTracker
|
|
25
|
+
from claude_goals import ClaudeGoalHistoryNormalizer, is_claude_synthetic_no_response
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
MAX_BYTES = 96 * 1024 * 1024
|
|
29
|
+
MAX_EVENTS_BYTES = 32 * 1024 * 1024
|
|
30
|
+
MAX_LINE_BYTES = 4 * 1024 * 1024
|
|
31
|
+
MAX_RECORDS = 100_000
|
|
32
|
+
MAX_KEYS = 20_000
|
|
33
|
+
MAX_TARGETS = 4_000
|
|
34
|
+
MAX_SESSIONS = 24
|
|
35
|
+
MAX_WINDOWS = 48
|
|
36
|
+
_DIGEST = re.compile(r"[a-f0-9]{64}\Z")
|
|
37
|
+
_PAIR = re.compile(r"pair_[a-f0-9]{32}\Z")
|
|
38
|
+
_WAKE = re.compile(r"mailwake_[a-f0-9]{32}\Z")
|
|
39
|
+
_ASYNC_WRAPPER = re.compile(
|
|
40
|
+
r"\[AgentsDock delivery kind=instruction leg=1/1 origin=route mode=async_route_v1 from=([^\]\r\n]+)\]\n"
|
|
41
|
+
r"source-instruction: this legacy relay has no recorded source user instruction; do not infer user authorization from the prepared content\.\n"
|
|
42
|
+
r"\[Agent-prepared handoff message\]\n(.*)\n\[End agent-prepared handoff message\]\n\[End delivery\]",
|
|
43
|
+
re.DOTALL,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _Unproven(ValueError):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _Oversized(_Unproven):
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _regular_stamp(path: Path) -> tuple[int, int, int, int]:
|
|
56
|
+
value = path.lstat()
|
|
57
|
+
if not stat.S_ISREG(value.st_mode):
|
|
58
|
+
raise _Unproven()
|
|
59
|
+
return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _stamp(path: Path) -> tuple[int, int, int, int]:
|
|
63
|
+
value = _regular_stamp(path)
|
|
64
|
+
if value[2] > MAX_BYTES:
|
|
65
|
+
raise _Oversized()
|
|
66
|
+
return value
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _records(path: Path, expected: tuple[int, int, int, int]):
|
|
70
|
+
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
71
|
+
with os.fdopen(descriptor, "rb") as stream:
|
|
72
|
+
info = os.fstat(stream.fileno())
|
|
73
|
+
if (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != expected:
|
|
74
|
+
raise _Unproven()
|
|
75
|
+
count = 0
|
|
76
|
+
offset = 0
|
|
77
|
+
while offset < expected[2]:
|
|
78
|
+
line = stream.readline(min(MAX_LINE_BYTES + 1, expected[2] - offset))
|
|
79
|
+
if len(line) > MAX_LINE_BYTES or not line.endswith(b"\n"):
|
|
80
|
+
raise _Unproven()
|
|
81
|
+
count += 1
|
|
82
|
+
if count > MAX_RECORDS:
|
|
83
|
+
raise _Unproven()
|
|
84
|
+
offset += len(line)
|
|
85
|
+
event = json.loads(line)
|
|
86
|
+
if not isinstance(event, dict):
|
|
87
|
+
raise _Unproven()
|
|
88
|
+
yield event, offset, line
|
|
89
|
+
final = os.fstat(stream.fileno())
|
|
90
|
+
if (final.st_dev, final.st_ino, final.st_size, final.st_mtime_ns) != expected:
|
|
91
|
+
raise _Unproven()
|
|
92
|
+
if _stamp(path) != expected:
|
|
93
|
+
raise _Unproven()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _text_key(text: str) -> str:
|
|
97
|
+
return hashlib.sha256(text.encode("utf-8", errors="surrogatepass")).hexdigest()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _target(event: dict) -> tuple[int, str, str] | None:
|
|
101
|
+
seq = event.get("seq")
|
|
102
|
+
run = event.get("run_id")
|
|
103
|
+
prompt = event.get("prompt")
|
|
104
|
+
if (
|
|
105
|
+
event.get("type") != "turn_started" or event.get("imported") is not True
|
|
106
|
+
or event.get("backend") != "claude" or type(seq) is not int or seq <= 0
|
|
107
|
+
or not isinstance(run, str) or not run.startswith("import_")
|
|
108
|
+
or not isinstance(prompt, str) or len(prompt) > MAX_LINE_BYTES
|
|
109
|
+
):
|
|
110
|
+
return None
|
|
111
|
+
return seq, run, _text_key(prompt)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _assistant_target(event: dict) -> tuple | None:
|
|
115
|
+
origin = event.get("provider_origin")
|
|
116
|
+
text, seq, run = event.get("text"), event.get("seq"), event.get("run_id")
|
|
117
|
+
if (event.get("type") not in ("assistant_text", "reasoning_summary")
|
|
118
|
+
or event.get("imported") is not True or event.get("backend") != "claude"
|
|
119
|
+
or type(seq) is not int or seq <= 0 or not isinstance(run, str) or not run.startswith("import_")
|
|
120
|
+
or not isinstance(text, str) or not text.strip() or len(text) > MAX_LINE_BYTES
|
|
121
|
+
or not isinstance(origin, dict) or origin.get("provider") != "claude"
|
|
122
|
+
or origin.get("kind") not in (None, "assistant")
|
|
123
|
+
or event.get("provider_user_authored") is True
|
|
124
|
+
or any(event.get(key) for key in ("clientUserMessageId", "clientId", "client_user_message_id", "client_id"))):
|
|
125
|
+
return None
|
|
126
|
+
identity = tuple(origin.get(key) for key in ("event_id", "session_id", "timestamp"))
|
|
127
|
+
if (not all(isinstance(value, str) and 0 < len(value) <= 256 for value in identity)
|
|
128
|
+
or _timestamp(identity[2]) is None):
|
|
129
|
+
return None
|
|
130
|
+
return event["type"], seq, run, _text_key(text.strip()), identity
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _async_delivery_identity(event: dict) -> tuple | None:
|
|
134
|
+
pair, message = event.get("conversation_id"), event.get("message_id")
|
|
135
|
+
source, target = event.get("source_session_id"), event.get("target_session_id")
|
|
136
|
+
if (event.get("conversation_mode") != "async_route_v1"
|
|
137
|
+
or not isinstance(pair, str) or not _PAIR.fullmatch(pair)
|
|
138
|
+
or not isinstance(message, str) or not message or message != event.get("cross_chat_envelope_id")
|
|
139
|
+
or not isinstance(source, str) or not source or not isinstance(target, str) or not target
|
|
140
|
+
or source == target or event.get("session_id") != target):
|
|
141
|
+
return None
|
|
142
|
+
return pair, message, source, target
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _mailbox_wake_identity(event: dict) -> tuple | None:
|
|
146
|
+
claim, cutoff, digest = (event.get(key) for key in (
|
|
147
|
+
"mailbox_wake_id", "mailbox_wake_through_seq", "provider_input_sha256"))
|
|
148
|
+
if (event.get("purpose") != "chat_mailbox_wake" or event.get("provider_generated") is not True
|
|
149
|
+
or event.get("prompt") != ""
|
|
150
|
+
or event.get("provider_user_authored") is True
|
|
151
|
+
or any(event.get(key) for key in ("clientUserMessageId", "clientId", "client_user_message_id", "client_id"))
|
|
152
|
+
or not isinstance(claim, str) or not _WAKE.fullmatch(claim)
|
|
153
|
+
or type(cutoff) is not int or not 0 < cutoff < 2**63
|
|
154
|
+
or not isinstance(digest, str) or not _DIGEST.fullmatch(digest)):
|
|
155
|
+
return None
|
|
156
|
+
return claim, cutoff, digest
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class _AssistantReplays:
|
|
160
|
+
"""Bounded exact source/output correlation, fed by the existing two reads."""
|
|
161
|
+
def __init__(self, provider_id: str, normalize_assistant: Callable[[str], str] | None = None) -> None:
|
|
162
|
+
self.provider_id = provider_id
|
|
163
|
+
self.normalize_assistant = normalize_assistant
|
|
164
|
+
self.starts, self.ends, self.owners, self.native = {}, {}, {}, []
|
|
165
|
+
self.candidates, self.sources, self.identities, self.source_credits = [], {}, {}, {}
|
|
166
|
+
self.source_display, self.source_message_counts = {}, {}
|
|
167
|
+
self.source_parents, self.source_users, self.source_input = OrderedDict(), {}, {}
|
|
168
|
+
|
|
169
|
+
def display_key(self, text: str) -> str | None:
|
|
170
|
+
if self.normalize_assistant is None:
|
|
171
|
+
return None
|
|
172
|
+
display = self.normalize_assistant(text)
|
|
173
|
+
return (_text_key(display) if isinstance(display, str) and display.strip()
|
|
174
|
+
and len(display) <= MAX_LINE_BYTES else None)
|
|
175
|
+
|
|
176
|
+
def event(self, event: dict) -> None:
|
|
177
|
+
target = _assistant_target(event)
|
|
178
|
+
if target is not None and target[4][1] == self.provider_id:
|
|
179
|
+
self.candidates.append((target, event.get("phase")))
|
|
180
|
+
run = event.get("run_id")
|
|
181
|
+
if (isinstance(run, str) and run and not run.startswith("import_")
|
|
182
|
+
and event.get("backend") == "claude" and event.get("imported") is not True):
|
|
183
|
+
kind = event.get("type")
|
|
184
|
+
if kind == "turn_started":
|
|
185
|
+
self.starts.setdefault(run, []).append(event)
|
|
186
|
+
elif kind == "turn_finished":
|
|
187
|
+
self.ends.setdefault(run, []).append(event)
|
|
188
|
+
elif kind == "provider_session":
|
|
189
|
+
self.owners.setdefault(run, []).append(event)
|
|
190
|
+
elif (kind in ("reasoning_summary", "assistant_text")
|
|
191
|
+
and (event.get("phase") in ("commentary", "final_answer")
|
|
192
|
+
or (kind == "assistant_text" and event.get("phase") is None))
|
|
193
|
+
and isinstance(event.get("text"), str) and event["text"].strip()
|
|
194
|
+
and len(event["text"]) <= MAX_LINE_BYTES):
|
|
195
|
+
self.native.append((event, _text_key(event["text"].strip())))
|
|
196
|
+
if sum(map(len, (self.starts, self.ends, self.owners, self.native, self.candidates))) > MAX_TARGETS:
|
|
197
|
+
raise _Unproven()
|
|
198
|
+
|
|
199
|
+
def source(self, event: dict, offset: int) -> None:
|
|
200
|
+
if event.get("sessionId") != self.provider_id or event.get("isSidechain") is True:
|
|
201
|
+
return
|
|
202
|
+
identity = (event.get("uuid"), self.provider_id, event.get("timestamp"))
|
|
203
|
+
if not all(isinstance(value, str) and 0 < len(value) <= 256 for value in identity):
|
|
204
|
+
return
|
|
205
|
+
message = event.get("message")
|
|
206
|
+
content = message.get("content") if isinstance(message, dict) else None
|
|
207
|
+
text = content if isinstance(content, str) else "\n".join(
|
|
208
|
+
part["text"] for part in content if isinstance(part, dict) and part.get("type") == "text"
|
|
209
|
+
and isinstance(part.get("text"), str)) if isinstance(content, list) else ""
|
|
210
|
+
parent_input = self.source_parents.get(event.get("parentUuid"))
|
|
211
|
+
tool_result = isinstance(content, list) and any(isinstance(part, dict) and part.get("type") == "tool_result" for part in content)
|
|
212
|
+
if event.get("type") == "user" and event.get("isMeta") is not True and text.strip() and not tool_result:
|
|
213
|
+
self.source_users.setdefault(identity, []).append((offset, _text_key(text.strip())))
|
|
214
|
+
parent_input = identity
|
|
215
|
+
self.source_parents[identity[0]] = parent_input
|
|
216
|
+
while len(self.source_parents) > 256:
|
|
217
|
+
self.source_parents.popitem(last=False)
|
|
218
|
+
if len(self.source_users) > MAX_KEYS:
|
|
219
|
+
raise _Unproven()
|
|
220
|
+
if event.get("type") != "assistant":
|
|
221
|
+
return
|
|
222
|
+
if parent_input is not None:
|
|
223
|
+
self.source_input[identity] = parent_input
|
|
224
|
+
self.identities[identity] = self.identities.get(identity, 0) + 1
|
|
225
|
+
self.source_message_counts[identity[0]] = self.source_message_counts.get(identity[0], 0) + 1
|
|
226
|
+
if event.get("isSidechain") is True or event.get("isMeta") is True:
|
|
227
|
+
return
|
|
228
|
+
message = event.get("message")
|
|
229
|
+
content = message.get("content") if isinstance(message, dict) else None
|
|
230
|
+
# Only public text blocks. Thinking/tool payloads cannot establish a
|
|
231
|
+
# public replay, even if their wording happens to match another row.
|
|
232
|
+
if isinstance(content, str):
|
|
233
|
+
text = content
|
|
234
|
+
elif isinstance(content, list):
|
|
235
|
+
text = "\n".join(block["text"] for block in content if isinstance(block, dict)
|
|
236
|
+
and block.get("type") == "text" and isinstance(block.get("text"), str))
|
|
237
|
+
else:
|
|
238
|
+
return
|
|
239
|
+
if text.strip() and len(text) <= MAX_LINE_BYTES:
|
|
240
|
+
digest = _text_key(text.strip())
|
|
241
|
+
self.sources.setdefault((identity, digest), []).append(offset)
|
|
242
|
+
self.source_display[(identity, digest)] = self.display_key(text)
|
|
243
|
+
timestamp = _timestamp(identity[2])
|
|
244
|
+
if timestamp is not None:
|
|
245
|
+
self.source_credits.setdefault((int(timestamp), digest), set()).add(identity)
|
|
246
|
+
if len(self.identities) + len(self.sources) > MAX_KEYS:
|
|
247
|
+
raise _Unproven()
|
|
248
|
+
|
|
249
|
+
def exact_owned_sources(self) -> tuple[set, set]:
|
|
250
|
+
"""Exact native assistant UUID also proves its source input ancestry."""
|
|
251
|
+
owners = {}
|
|
252
|
+
for event, digest in self.native:
|
|
253
|
+
message_id, run = event.get("provider_message_id"), event["run_id"]
|
|
254
|
+
starts, ends = self.starts.get(run, ()), self.ends.get(run, ())
|
|
255
|
+
if (not isinstance(message_id, str) or not 0 < len(message_id) <= 256
|
|
256
|
+
or len(starts) != 1 or len(ends) != 1
|
|
257
|
+
or starts[0].get("purpose") in ("scheduled_job", "cross_chat_handoff_delivery", "chat_mailbox_wake")):
|
|
258
|
+
continue
|
|
259
|
+
records = [starts[0], event, ends[0], *self.owners.get(run, ())]
|
|
260
|
+
ids = {row.get("provider_session_id") for row in records if row.get("provider_session_id")}
|
|
261
|
+
if (ids != {self.provider_id} or not all(type(row.get("seq")) is int for row in (starts[0], event, ends[0]))
|
|
262
|
+
or not starts[0]["seq"] < event["seq"] < ends[0]["seq"]):
|
|
263
|
+
continue
|
|
264
|
+
owners.setdefault((message_id, digest), []).append(starts[0].get("prompt"))
|
|
265
|
+
assistants, users = set(), set()
|
|
266
|
+
for (identity, digest), offsets in self.sources.items():
|
|
267
|
+
matches = owners.get((identity[0], digest), ())
|
|
268
|
+
if len(matches) != 1 or len(offsets) != 1 or self.source_message_counts.get(identity[0]) != 1:
|
|
269
|
+
continue
|
|
270
|
+
assistants.add((identity, digest))
|
|
271
|
+
source_input = self.source_input.get(identity)
|
|
272
|
+
inputs = self.source_users.get(source_input, ())
|
|
273
|
+
if len(inputs) == 1 and isinstance(matches[0], str) and inputs[0][1] == _text_key(matches[0].strip()):
|
|
274
|
+
users.add((source_input, inputs[0][1], inputs[0][0]))
|
|
275
|
+
return assistants, users
|
|
276
|
+
|
|
277
|
+
def prove(self, eligible: dict) -> frozenset:
|
|
278
|
+
native, identified = {}, {}
|
|
279
|
+
for event, digest in self.native:
|
|
280
|
+
run = event["run_id"]
|
|
281
|
+
starts, ends = self.starts.get(run, ()), self.ends.get(run, ())
|
|
282
|
+
if len(starts) != 1 or len(ends) != 1:
|
|
283
|
+
continue
|
|
284
|
+
start, end = starts[0], ends[0]
|
|
285
|
+
owners = (start, event, end, *self.owners.get(run, ()))
|
|
286
|
+
provider_ids = {row.get("provider_session_id") for row in owners
|
|
287
|
+
if row.get("provider_session_id") not in (None, "")}
|
|
288
|
+
delivery = _async_delivery_identity(start)
|
|
289
|
+
scheduled = start.get("purpose") == "scheduled_job" and bool(start.get("job_id"))
|
|
290
|
+
async_delivery = (start.get("purpose") == "cross_chat_handoff_delivery" and delivery
|
|
291
|
+
and _async_delivery_identity(end) == delivery
|
|
292
|
+
and _async_delivery_identity(event) == delivery
|
|
293
|
+
and isinstance(event.get("provider_message_id"), str) and event["provider_message_id"])
|
|
294
|
+
mailbox_wake = (_mailbox_wake_identity(start)
|
|
295
|
+
and isinstance(event.get("provider_message_id"), str) and event["provider_message_id"])
|
|
296
|
+
if (not (scheduled or async_delivery or mailbox_wake)
|
|
297
|
+
or provider_ids != {self.provider_id}
|
|
298
|
+
or end.get("exit_code") != 0 or end.get("stopped") is True
|
|
299
|
+
or any(type(row.get("seq")) is not int or not start["seq"] <= row["seq"] <= end["seq"]
|
|
300
|
+
for row in self.owners.get(run, ()))
|
|
301
|
+
or (scheduled and event.get("job_id") not in (None, "", start["job_id"]))):
|
|
302
|
+
continue
|
|
303
|
+
if (event.get("phase") is None
|
|
304
|
+
and (not isinstance(end.get("result_text"), str)
|
|
305
|
+
or _text_key(end["result_text"].strip()) != digest)):
|
|
306
|
+
continue
|
|
307
|
+
times = [_timestamp(row.get("ts")) for row in (start, event, end)]
|
|
308
|
+
seqs = [row.get("seq") for row in (start, event, end)]
|
|
309
|
+
if (any(value is None for value in times) or not times[0] <= times[1] <= times[2]
|
|
310
|
+
or not all(type(value) is int for value in seqs) or not seqs[0] < seqs[1] < seqs[2]):
|
|
311
|
+
continue
|
|
312
|
+
native.setdefault((int(times[1]), digest), []).append(event)
|
|
313
|
+
message_id = event.get("provider_message_id")
|
|
314
|
+
if isinstance(message_id, str) and 0 < len(message_id) <= 256:
|
|
315
|
+
identified.setdefault(message_id, []).append((event, digest))
|
|
316
|
+
counts = {}
|
|
317
|
+
for target, _phase in self.candidates:
|
|
318
|
+
key = (target[2], target[4])
|
|
319
|
+
counts[key] = counts.get(key, 0) + 1
|
|
320
|
+
proven = set()
|
|
321
|
+
for target, phase in self.candidates:
|
|
322
|
+
_kind, seq, run, digest, identity = target
|
|
323
|
+
batch = eligible.get(run)
|
|
324
|
+
matches = native.get((int(_timestamp(identity[2])), digest), ())
|
|
325
|
+
offsets = self.sources.get((identity, digest), ())
|
|
326
|
+
owned = identified.get(identity[0], ())
|
|
327
|
+
if owned:
|
|
328
|
+
# Prefer the immutable message owner over text/time buckets.
|
|
329
|
+
# A normalized match is permitted ONLY for one native UUID
|
|
330
|
+
# and one exact raw source record. The imported raw digest,
|
|
331
|
+
# checkpoint, phase and timestamp checks remain authoritative.
|
|
332
|
+
# Never use decoration equivalence for identityless messages.
|
|
333
|
+
matches = ()
|
|
334
|
+
if len(owned) == 1 and self.source_message_counts.get(identity[0]) == 1:
|
|
335
|
+
event, native_digest = owned[0]
|
|
336
|
+
canonical = self.source_display.get((identity, digest))
|
|
337
|
+
if (int(_timestamp(event.get("ts"))) == int(_timestamp(identity[2]))
|
|
338
|
+
and (native_digest == digest or
|
|
339
|
+
(canonical and canonical == self.display_key(event["text"])))):
|
|
340
|
+
matches = (event,)
|
|
341
|
+
if (batch is None or len(matches) != 1 or len(offsets) != 1
|
|
342
|
+
or self.identities.get(identity) != 1 or counts[(run, identity)] != 1):
|
|
343
|
+
continue
|
|
344
|
+
match = matches[0]
|
|
345
|
+
expected_phase = match.get("phase") or "final_answer"
|
|
346
|
+
if (phase not in (None, expected_phase)
|
|
347
|
+
or match.get("provider_message_id") not in (None, "", identity[0])):
|
|
348
|
+
continue
|
|
349
|
+
source_time, native_time = _timestamp(identity[2]), _timestamp(match.get("ts"))
|
|
350
|
+
if ("." in match["ts"] and source_time != native_time):
|
|
351
|
+
continue
|
|
352
|
+
# One coarse native timestamp is not credit for two distinct
|
|
353
|
+
# source messages. An exact native message UUID may disambiguate.
|
|
354
|
+
if (not match.get("provider_message_id")
|
|
355
|
+
and len(self.source_credits.get((int(source_time), digest), ())) != 1):
|
|
356
|
+
continue
|
|
357
|
+
first, last, start, end = batch[:4]
|
|
358
|
+
if first < seq < last and start < offsets[0] <= end:
|
|
359
|
+
proven.add(target)
|
|
360
|
+
exact, _inputs = self.exact_owned_sources()
|
|
361
|
+
for target, _phase in self.candidates:
|
|
362
|
+
_kind, seq, run, digest, identity = target
|
|
363
|
+
batch = eligible.get(run)
|
|
364
|
+
offsets = self.sources.get((identity, digest), ())
|
|
365
|
+
if (batch and (identity, digest) in exact and len(offsets) == 1
|
|
366
|
+
and batch[0] < seq < batch[1] and batch[2] < offsets[0] <= batch[3]):
|
|
367
|
+
proven.add(target)
|
|
368
|
+
return frozenset(proven)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
class _AsyncDeliveryInputs:
|
|
372
|
+
"""Exact completed delivery receipt + complete source wrapper, not text guessing."""
|
|
373
|
+
def __init__(self, provider_id, native, normalize_user, normalize_full_user):
|
|
374
|
+
self.provider_id, self.native = provider_id, native
|
|
375
|
+
self.normalize_user, self.normalize_full_user = normalize_user, normalize_full_user
|
|
376
|
+
self.receipts, self.candidates, self.sources, self.source_counts = {}, [], {}, {}
|
|
377
|
+
|
|
378
|
+
def event(self, event):
|
|
379
|
+
if event.get("type") == "chat_conversation_message_started" and _async_delivery_identity(event):
|
|
380
|
+
run = event.get("target_run_id")
|
|
381
|
+
if isinstance(run, str) and run and not run.startswith("import_"):
|
|
382
|
+
self.receipts.setdefault(run, []).append(event)
|
|
383
|
+
target, origin = _target(event), event.get("provider_origin")
|
|
384
|
+
if (target and isinstance(origin, dict) and origin.get("provider") == "claude"
|
|
385
|
+
and origin.get("kind") in (None, "user") and origin.get("session_id") == self.provider_id
|
|
386
|
+
and event.get("provider_user_authored") is not True
|
|
387
|
+
and not any(event.get(key) for key in ("clientUserMessageId", "clientId", "client_user_message_id", "client_id"))):
|
|
388
|
+
identity = tuple(origin.get(key) for key in ("event_id", "session_id", "timestamp"))
|
|
389
|
+
if all(isinstance(value, str) and 0 < len(value) <= 256 for value in identity):
|
|
390
|
+
self.candidates.append((target, identity))
|
|
391
|
+
if len(self.receipts) + len(self.candidates) > MAX_TARGETS:
|
|
392
|
+
raise _Unproven()
|
|
393
|
+
|
|
394
|
+
def source(self, event, offset):
|
|
395
|
+
if event.get("type") != "user" or event.get("sessionId") != self.provider_id or self.normalize_full_user is None:
|
|
396
|
+
return
|
|
397
|
+
identity = (event.get("uuid"), self.provider_id, event.get("timestamp"))
|
|
398
|
+
if not all(isinstance(value, str) and 0 < len(value) <= 256 for value in identity):
|
|
399
|
+
return
|
|
400
|
+
self.source_counts[identity] = self.source_counts.get(identity, 0) + 1
|
|
401
|
+
if (any(event.get(key) is True for key in ("isMeta", "isCompactSummary", "isSidechain"))
|
|
402
|
+
or any(event.get(key) for key in ("clientUserMessageId", "clientId", "client_user_message_id", "client_id"))):
|
|
403
|
+
return
|
|
404
|
+
full, display = self.normalize_full_user(event), self.normalize_user(event)
|
|
405
|
+
match = _ASYNC_WRAPPER.fullmatch(full.strip()) if isinstance(full, str) and len(full) <= MAX_LINE_BYTES else None
|
|
406
|
+
timestamp = _timestamp(identity[2])
|
|
407
|
+
if match and isinstance(display, str) and display and timestamp is not None:
|
|
408
|
+
self.sources.setdefault((match[1], _text_key(match[2]), len(match[2]), int(timestamp)), []).append(
|
|
409
|
+
(identity, _text_key(display), offset))
|
|
410
|
+
if len(self.source_counts) + len(self.sources) > MAX_KEYS:
|
|
411
|
+
raise _Unproven()
|
|
412
|
+
|
|
413
|
+
def prove(self, eligible):
|
|
414
|
+
matches, credits, candidate_counts = {}, {}, {}
|
|
415
|
+
for target, identity in self.candidates:
|
|
416
|
+
candidate_counts[(target[1], identity)] = candidate_counts.get((target[1], identity), 0) + 1
|
|
417
|
+
for run, receipts in self.receipts.items():
|
|
418
|
+
starts, ends = self.native.starts.get(run, ()), self.native.ends.get(run, ())
|
|
419
|
+
if len(receipts) != 1 or len(starts) != 1 or len(ends) != 1:
|
|
420
|
+
continue
|
|
421
|
+
start, end, receipt = starts[0], ends[0], receipts[0]
|
|
422
|
+
delivery = _async_delivery_identity(start)
|
|
423
|
+
owners = (start, end, *self.native.owners.get(run, ()))
|
|
424
|
+
provider_ids = {row.get("provider_session_id") for row in owners if row.get("provider_session_id") not in (None, "")}
|
|
425
|
+
start_time, end_time = _timestamp(start.get("ts")), _timestamp(end.get("ts"))
|
|
426
|
+
if (start.get("purpose") != "cross_chat_handoff_delivery" or not delivery
|
|
427
|
+
or _async_delivery_identity(end) != delivery or _async_delivery_identity(receipt) != delivery
|
|
428
|
+
or receipt.get("handoff_authorization_kind") != "configured_route"
|
|
429
|
+
or receipt.get("handoff_status") != "running" or receipt.get("queued_id") != start.get("queued_id")
|
|
430
|
+
or not isinstance(receipt.get("handoff_body_sha256"), str) or not _DIGEST.fullmatch(receipt["handoff_body_sha256"])
|
|
431
|
+
or type(receipt.get("handoff_body_chars")) is not int
|
|
432
|
+
or provider_ids != {self.provider_id} or end.get("exit_code") != 0 or end.get("stopped") is True
|
|
433
|
+
or start_time is None or end_time is None or start_time > end_time
|
|
434
|
+
or any(type(row.get("seq")) is not int for row in (start, receipt, end))
|
|
435
|
+
or not start["seq"] < receipt["seq"] < end["seq"]
|
|
436
|
+
or any(type(row.get("seq")) is not int or not start["seq"] <= row["seq"] <= end["seq"]
|
|
437
|
+
for row in self.native.owners.get(run, ()))):
|
|
438
|
+
continue
|
|
439
|
+
source_key = (receipt.get("source_title"), receipt["handoff_body_sha256"], receipt["handoff_body_chars"], int(start_time))
|
|
440
|
+
for identity, key, offset in self.sources.get(source_key, ()):
|
|
441
|
+
timestamp = _timestamp(identity[2])
|
|
442
|
+
if (self.source_counts.get(identity) != 1 or timestamp is None
|
|
443
|
+
or not start_time <= timestamp <= end_time or int(timestamp) != int(start_time)
|
|
444
|
+
or ("." in start["ts"] and timestamp != start_time)):
|
|
445
|
+
continue
|
|
446
|
+
matches.setdefault((identity, key), []).append((run, offset))
|
|
447
|
+
credits[run] = credits.get(run, 0) + 1
|
|
448
|
+
targets = set()
|
|
449
|
+
for target, identity in self.candidates:
|
|
450
|
+
seq, run, key = target
|
|
451
|
+
batch, matched = eligible.get(run), matches.get((identity, key), ())
|
|
452
|
+
if batch and len(matched) == 1 and candidate_counts[(run, identity)] == 1 and credits.get(matched[0][0]) == 1:
|
|
453
|
+
first, last, start, end = batch[:4]
|
|
454
|
+
if first < seq < last and start < matched[0][1] <= end:
|
|
455
|
+
targets.add(target)
|
|
456
|
+
return targets
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
class _MailboxWakeInputs(_AsyncDeliveryInputs):
|
|
460
|
+
"""Exact generated native input hash, current provider and checkpoint identity."""
|
|
461
|
+
def __init__(self, *args):
|
|
462
|
+
super().__init__(*args)
|
|
463
|
+
self.wake_hashes = None
|
|
464
|
+
|
|
465
|
+
def source(self, event, offset):
|
|
466
|
+
if self.wake_hashes is None:
|
|
467
|
+
self.wake_hashes = {wake[2] for starts in self.native.starts.values() for start in starts
|
|
468
|
+
if (wake := _mailbox_wake_identity(start)) is not None}
|
|
469
|
+
if not self.wake_hashes:
|
|
470
|
+
return
|
|
471
|
+
if event.get("type") != "user" or event.get("sessionId") != self.provider_id or self.normalize_full_user is None:
|
|
472
|
+
return
|
|
473
|
+
identity = (event.get("uuid"), self.provider_id, event.get("timestamp"))
|
|
474
|
+
if not all(isinstance(value, str) and 0 < len(value) <= 256 for value in identity):
|
|
475
|
+
return
|
|
476
|
+
self.source_counts[identity] = self.source_counts.get(identity, 0) + 1
|
|
477
|
+
if len(self.source_counts) > MAX_KEYS:
|
|
478
|
+
raise _Unproven()
|
|
479
|
+
if (any(event.get(key) is True for key in ("isMeta", "isCompactSummary", "isSidechain", "provider_user_authored"))
|
|
480
|
+
or any(event.get(key) for key in ("clientUserMessageId", "clientId", "client_user_message_id", "client_id"))):
|
|
481
|
+
return
|
|
482
|
+
full, display = self.normalize_full_user(event), self.normalize_user(event)
|
|
483
|
+
timestamp = _timestamp(identity[2])
|
|
484
|
+
if (isinstance(full, str) and full and len(full) <= MAX_LINE_BYTES
|
|
485
|
+
and _text_key(full) in self.wake_hashes
|
|
486
|
+
and isinstance(display, str) and display and timestamp is not None):
|
|
487
|
+
self.sources.setdefault(_text_key(full), []).append(
|
|
488
|
+
(identity, _text_key(display), offset))
|
|
489
|
+
if len(self.source_counts) + len(self.sources) > MAX_KEYS:
|
|
490
|
+
raise _Unproven()
|
|
491
|
+
|
|
492
|
+
def prove(self, eligible):
|
|
493
|
+
matches, credits, candidate_counts = {}, {}, {}
|
|
494
|
+
for target, identity in self.candidates:
|
|
495
|
+
key = (target[1], identity)
|
|
496
|
+
candidate_counts[key] = candidate_counts.get(key, 0) + 1
|
|
497
|
+
for run, starts in self.native.starts.items():
|
|
498
|
+
ends = self.native.ends.get(run, ())
|
|
499
|
+
if len(starts) != 1 or len(ends) != 1:
|
|
500
|
+
continue
|
|
501
|
+
start, end = starts[0], ends[0]
|
|
502
|
+
wake = _mailbox_wake_identity(start)
|
|
503
|
+
owners = (start, end, *self.native.owners.get(run, ()))
|
|
504
|
+
provider_ids = {row.get("provider_session_id") for row in owners
|
|
505
|
+
if row.get("provider_session_id") not in (None, "")}
|
|
506
|
+
start_time, end_time = _timestamp(start.get("ts")), _timestamp(end.get("ts"))
|
|
507
|
+
# The exact generated input remains internal when its run is
|
|
508
|
+
# stopped or fails. Completion is needed to bound ownership, not
|
|
509
|
+
# to prove success; assistant replay keeps its separate checks.
|
|
510
|
+
if (wake is None or provider_ids != {self.provider_id}
|
|
511
|
+
or start_time is None or end_time is None or start_time > end_time
|
|
512
|
+
or type(start.get("seq")) is not int or type(end.get("seq")) is not int
|
|
513
|
+
or start["seq"] >= end["seq"]
|
|
514
|
+
or any(type(row.get("seq")) is not int or not start["seq"] <= row["seq"] <= end["seq"]
|
|
515
|
+
for row in self.native.owners.get(run, ()))):
|
|
516
|
+
continue
|
|
517
|
+
for identity, key, offset in self.sources.get(wake[2], ()):
|
|
518
|
+
timestamp = _timestamp(identity[2])
|
|
519
|
+
if (self.source_counts.get(identity) != 1 or timestamp is None
|
|
520
|
+
or not start_time <= timestamp <= end_time):
|
|
521
|
+
continue
|
|
522
|
+
matches.setdefault((identity, key), []).append((run, offset))
|
|
523
|
+
credits[run] = credits.get(run, 0) + 1
|
|
524
|
+
targets = set()
|
|
525
|
+
for target, identity in self.candidates:
|
|
526
|
+
seq, run, key = target
|
|
527
|
+
batch, matched = eligible.get(run), matches.get((identity, key), ())
|
|
528
|
+
if batch and len(matched) == 1 and candidate_counts[(run, identity)] == 1 and credits.get(matched[0][0]) == 1:
|
|
529
|
+
first, last, start, end = batch[:4]
|
|
530
|
+
if first < seq < last and start < matched[0][1] <= end:
|
|
531
|
+
targets.add(target)
|
|
532
|
+
return targets
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def filter_native_claude_mailbox_wake_items(
|
|
536
|
+
session_id: str, provider_id: str, events: Path, items: list[dict], *,
|
|
537
|
+
sync_checkpoint: dict, root: Path, normalize_user: Callable,
|
|
538
|
+
normalize_full_user: Callable, source_path: Path | None = None,
|
|
539
|
+
normalize_assistant: Callable[[str], str] | None = None,
|
|
540
|
+
) -> list[dict]:
|
|
541
|
+
"""First-import native input/output proof; never a text-only filter.
|
|
542
|
+
|
|
543
|
+
Parsed items omit human/client flags and may collapse repeated text, so the
|
|
544
|
+
exact checkpoint source is required to prove unique native ownership.
|
|
545
|
+
All failures retain the original items. Live output is never removed.
|
|
546
|
+
"""
|
|
547
|
+
try:
|
|
548
|
+
if not items or len(items) > MAX_TARGETS:
|
|
549
|
+
return items
|
|
550
|
+
stamp = _regular_stamp(events)
|
|
551
|
+
native = _AssistantReplays(provider_id, normalize_assistant)
|
|
552
|
+
for event, _offset in _bounded_records(events, stamp, max(0, stamp[2] - MAX_EVENTS_BYTES), stamp[2]):
|
|
553
|
+
if event.get("session_id") in (None, "", session_id):
|
|
554
|
+
native.event(event)
|
|
555
|
+
if not native.starts:
|
|
556
|
+
return items
|
|
557
|
+
cursor = sync_checkpoint.get("cursor")
|
|
558
|
+
if (sync_checkpoint.get("version") != 1 or not isinstance(cursor, dict)
|
|
559
|
+
or cursor.get("version") != 1 or cursor.get("backend") != "claude"
|
|
560
|
+
or cursor.get("provider_session_id") != provider_id):
|
|
561
|
+
return items
|
|
562
|
+
path = cursor.get("source_path")
|
|
563
|
+
if not isinstance(path, str):
|
|
564
|
+
return items
|
|
565
|
+
source = Path(path)
|
|
566
|
+
if source_path is not None and source != source_path:
|
|
567
|
+
return items
|
|
568
|
+
if not source.is_absolute() or source.suffix != ".jsonl" or source.stem != provider_id or source.is_symlink():
|
|
569
|
+
return items
|
|
570
|
+
source = source.resolve(strict=True)
|
|
571
|
+
source.relative_to(root.resolve(strict=True))
|
|
572
|
+
source_stamp = _regular_stamp(source)
|
|
573
|
+
start, end = sync_checkpoint.get("previous_source_offset"), cursor.get("source_offset")
|
|
574
|
+
previous, expected = sync_checkpoint.get("previous_source_digest"), cursor.get("source_digest")
|
|
575
|
+
if (type(start) is not int or type(end) is not int or not 0 <= start < end <= source_stamp[2]
|
|
576
|
+
or (cursor.get("source_dev"), cursor.get("source_ino")) != source_stamp[:2]
|
|
577
|
+
or type(sync_checkpoint.get("previous_present")) is not bool
|
|
578
|
+
or not isinstance(expected, str) or not _DIGEST.fullmatch(expected)
|
|
579
|
+
or start == 0 and previous != ""
|
|
580
|
+
or start > 0 and (sync_checkpoint["previous_present"] is not True
|
|
581
|
+
or not isinstance(previous, str) or not _DIGEST.fullmatch(previous))):
|
|
582
|
+
return items
|
|
583
|
+
proof = _MailboxWakeInputs(provider_id, native, normalize_user, normalize_full_user)
|
|
584
|
+
# Temporary proof coordinates only; no synthetic event is persisted.
|
|
585
|
+
batch = "import_mailbox_wake_filter"
|
|
586
|
+
assistant_targets = {}
|
|
587
|
+
for index, item in enumerate(items, 1):
|
|
588
|
+
if item.get("kind") == "user" and item.get("source_text_sha256") is None:
|
|
589
|
+
proof.event({**item, "seq": index, "run_id": batch, "type": "turn_started",
|
|
590
|
+
"backend": "claude", "imported": True, "prompt": item.get("text")})
|
|
591
|
+
elif item.get("kind") == "assistant":
|
|
592
|
+
candidate = {**item, "seq": index, "run_id": batch, "type": "assistant_text",
|
|
593
|
+
"backend": "claude", "imported": True}
|
|
594
|
+
native.event(candidate)
|
|
595
|
+
assistant_targets[index] = _assistant_target(candidate)
|
|
596
|
+
if source_stamp[2] <= MAX_EVENTS_BYTES:
|
|
597
|
+
digest, verified = hashlib.sha256(), set()
|
|
598
|
+
for event, offset, line in _records(source, source_stamp):
|
|
599
|
+
digest.update(line)
|
|
600
|
+
if offset == start and hmac.compare_digest(digest.hexdigest(), previous):
|
|
601
|
+
verified.add(start)
|
|
602
|
+
if offset == end and hmac.compare_digest(digest.hexdigest(), expected):
|
|
603
|
+
verified.add(end)
|
|
604
|
+
if offset <= end:
|
|
605
|
+
proof.source(event, offset)
|
|
606
|
+
native.source(event, offset)
|
|
607
|
+
if end not in verified or start and start not in verified:
|
|
608
|
+
return items
|
|
609
|
+
else:
|
|
610
|
+
# Same bounded proof as historical semantic windows: the caller has
|
|
611
|
+
# just verified this checkpoint. Recheck file identity and every
|
|
612
|
+
# candidate's exact source UUID/full hash, without a prefix rescan.
|
|
613
|
+
window_start = max(0, end - MAX_EVENTS_BYTES)
|
|
614
|
+
if start < window_start:
|
|
615
|
+
return items
|
|
616
|
+
for event, offset in _bounded_records(source, source_stamp, window_start, end):
|
|
617
|
+
proof.source(event, offset)
|
|
618
|
+
native.source(event, offset)
|
|
619
|
+
eligible = {batch: (0, len(items) + 1, start, end)}
|
|
620
|
+
targets, assistant_replays = proof.prove(eligible), native.prove(eligible)
|
|
621
|
+
_assistants, owned_inputs = native.exact_owned_sources()
|
|
622
|
+
projected = []
|
|
623
|
+
for index, item in enumerate(items, 1):
|
|
624
|
+
origin = item.get("provider_origin") or {}
|
|
625
|
+
identity = (origin.get("event_id"), origin.get("session_id"), origin.get("timestamp"))
|
|
626
|
+
input_owned = item.get("kind") == "user" and any(
|
|
627
|
+
source == identity and digest == _text_key(item.get("text", "").strip()) and start < offset <= end
|
|
628
|
+
for source, digest, offset in owned_inputs)
|
|
629
|
+
reason = ("source_proven_import" if (index, batch, _text_key(item.get("text", ""))) in targets
|
|
630
|
+
or input_owned
|
|
631
|
+
else "source_proven_assistant_replay" if assistant_targets.get(index) in assistant_replays
|
|
632
|
+
else None)
|
|
633
|
+
projected.append({**item, "text": "", "metadata_only": True,
|
|
634
|
+
"provider_history_repair": reason} if reason else item)
|
|
635
|
+
return projected
|
|
636
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError):
|
|
637
|
+
return items
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
@dataclass(frozen=True)
|
|
641
|
+
class _Proof:
|
|
642
|
+
provider_id: str
|
|
643
|
+
events_stamp: tuple[int, int, int, int]
|
|
644
|
+
source: Path | None
|
|
645
|
+
source_stamp: tuple[int, int, int, int] | None
|
|
646
|
+
targets: frozenset[tuple[int, str, str]]
|
|
647
|
+
interruptions: tuple[tuple[tuple[int, str, str], str], ...] = ()
|
|
648
|
+
companions: frozenset[tuple[str, int, str]] = frozenset()
|
|
649
|
+
assistant_replays: frozenset[tuple] = frozenset()
|
|
650
|
+
goal_commands: tuple = ()
|
|
651
|
+
interruption_index: MappingProxyType = field(init=False, repr=False)
|
|
652
|
+
goal_command_index: MappingProxyType = field(init=False, repr=False)
|
|
653
|
+
cache_signature: frozenset = field(init=False, repr=False)
|
|
654
|
+
|
|
655
|
+
def __post_init__(self) -> None:
|
|
656
|
+
object.__setattr__(self, "interruption_index", MappingProxyType(dict(self.interruptions)))
|
|
657
|
+
object.__setattr__(self, "goal_command_index", MappingProxyType(dict(self.goal_commands)))
|
|
658
|
+
object.__setattr__(self, "cache_signature", self.targets | frozenset(
|
|
659
|
+
("interruption", key, origin) for key, origin in self.interruptions)
|
|
660
|
+
| frozenset(("companion", *key) for key in self.companions)
|
|
661
|
+
| frozenset(("assistant_replay", *key) for key in self.assistant_replays)
|
|
662
|
+
| frozenset(("goal_command", key, text) for key, text in self.goal_commands))
|
|
663
|
+
|
|
664
|
+
def signature(self) -> frozenset:
|
|
665
|
+
return self.cache_signature
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def _timestamp(value) -> float | None:
|
|
669
|
+
if not isinstance(value, str) or len(value) > 64:
|
|
670
|
+
return None
|
|
671
|
+
try:
|
|
672
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
673
|
+
return parsed.timestamp() if parsed.tzinfo is not None else None
|
|
674
|
+
except (ValueError, OverflowError, OSError):
|
|
675
|
+
return None
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _steer_intervals(events: list[dict], provider_id: str) -> list[tuple[float, float]]:
|
|
679
|
+
"""Only a complete native handoff can establish why a provider was interrupted."""
|
|
680
|
+
queues, finished, starts = {}, {}, {}
|
|
681
|
+
for event in events:
|
|
682
|
+
kind = event.get("type")
|
|
683
|
+
if kind == "turn_queue_run_now":
|
|
684
|
+
key = event.get("interrupted_run_id")
|
|
685
|
+
destination = queues
|
|
686
|
+
elif kind in ("turn_finished", "turn_stopped"):
|
|
687
|
+
key = event.get("run_id")
|
|
688
|
+
destination = finished
|
|
689
|
+
else:
|
|
690
|
+
key = event.get("steer_interrupted_run_id")
|
|
691
|
+
destination = starts
|
|
692
|
+
if isinstance(key, str) and key and not key.startswith("import_"):
|
|
693
|
+
destination.setdefault(key, []).append(event)
|
|
694
|
+
intervals = []
|
|
695
|
+
for run, queued in queues.items():
|
|
696
|
+
ended, following = finished.get(run, ()), starts.get(run, ())
|
|
697
|
+
if len(queued) != 1 or len(ended) != 1 or len(following) != 1:
|
|
698
|
+
continue
|
|
699
|
+
queue, terminal, start = queued[0], ended[0], following[0]
|
|
700
|
+
queued_id = queue.get("queued_id")
|
|
701
|
+
next_run = start.get("run_id")
|
|
702
|
+
terminal_proven = (
|
|
703
|
+
terminal.get("type") == "turn_finished"
|
|
704
|
+
and terminal.get("stopped") is True
|
|
705
|
+
and terminal.get("exit_code") is None
|
|
706
|
+
) or (
|
|
707
|
+
terminal.get("type") == "turn_stopped"
|
|
708
|
+
and terminal.get("native_steer") is True
|
|
709
|
+
and terminal.get("superseded_by_run_id") == next_run
|
|
710
|
+
)
|
|
711
|
+
if (
|
|
712
|
+
not terminal_proven
|
|
713
|
+
or terminal.get("provider_session_id") != provider_id
|
|
714
|
+
or not isinstance(queued_id, str) or not queued_id
|
|
715
|
+
or start.get("queued_id") != queued_id
|
|
716
|
+
or not isinstance(next_run, str) or not next_run or next_run == run
|
|
717
|
+
or next_run.startswith("import_")
|
|
718
|
+
or any(event.get("provider_session_id") not in (None, "", provider_id)
|
|
719
|
+
for event in (queue, start))
|
|
720
|
+
):
|
|
721
|
+
continue
|
|
722
|
+
times = [_timestamp(event.get("ts")) for event in (queue, terminal, start)]
|
|
723
|
+
if any(value is None for value in times):
|
|
724
|
+
continue
|
|
725
|
+
# An interruption must be within five seconds of every corroborating
|
|
726
|
+
# record, not merely near an unrelated stop elsewhere in this chat.
|
|
727
|
+
lower, upper = max(times) - 5.0, min(times) + 5.0
|
|
728
|
+
if lower <= upper:
|
|
729
|
+
intervals.append((lower, upper))
|
|
730
|
+
return intervals
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _interruption_origin(origin: dict, provider_id: str, intervals) -> dict | None:
|
|
734
|
+
if (
|
|
735
|
+
not isinstance(origin, dict) or origin.get("provider") != "claude"
|
|
736
|
+
or origin.get("kind") != "interruption"
|
|
737
|
+
or origin.get("session_id") != provider_id
|
|
738
|
+
or not isinstance(origin.get("event_id"), str)
|
|
739
|
+
):
|
|
740
|
+
return None
|
|
741
|
+
timestamp = _timestamp(origin.get("timestamp"))
|
|
742
|
+
if timestamp is None:
|
|
743
|
+
return None
|
|
744
|
+
result = {key: origin[key] for key in (
|
|
745
|
+
"provider", "kind", "event_id", "session_id", "timestamp",
|
|
746
|
+
"parent_event_id", "prompt_id",
|
|
747
|
+
) if key in origin}
|
|
748
|
+
result["cause"] = "steer" if any(low <= timestamp <= high for low, high in intervals) else "unknown"
|
|
749
|
+
return result
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _native_control(event: dict, session_id: str) -> dict | None:
|
|
753
|
+
if not (
|
|
754
|
+
event.get("session_id") == session_id and event.get("backend") == "claude"
|
|
755
|
+
and event.get("imported") is not True
|
|
756
|
+
and event.get("type") in ("turn_queue_run_now", "turn_finished", "turn_stopped", "turn_started")
|
|
757
|
+
):
|
|
758
|
+
return None
|
|
759
|
+
return {key: event[key] for key in (
|
|
760
|
+
"type", "ts", "run_id", "queued_id", "interrupted_run_id",
|
|
761
|
+
"steer_interrupted_run_id", "provider_session_id", "stopped", "exit_code",
|
|
762
|
+
"native_steer", "superseded_by_run_id",
|
|
763
|
+
) if key in event}
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def enrich_interruption_origins(events_path: Path, provider_id: str, origins: list[dict],
|
|
767
|
+
*, session_id: str) -> list[dict]:
|
|
768
|
+
"""Explicit import-worker enrichment; bounded native-log read, no source discovery.
|
|
769
|
+
|
|
770
|
+
Valid origins retain their order and are copied with unknown cause on any
|
|
771
|
+
read/proof failure. Invalid entries become empty dictionaries; an oversized
|
|
772
|
+
input returns an empty list. Callers must not replace an origin with either.
|
|
773
|
+
"""
|
|
774
|
+
if not isinstance(origins, list) or len(origins) > MAX_TARGETS:
|
|
775
|
+
return []
|
|
776
|
+
unknown = [_interruption_origin(origin, provider_id, ()) or {} for origin in origins]
|
|
777
|
+
if not any(unknown) or not isinstance(session_id, str) or not session_id:
|
|
778
|
+
return unknown
|
|
779
|
+
try:
|
|
780
|
+
stamp = _stamp(events_path)
|
|
781
|
+
if stamp[2] > MAX_EVENTS_BYTES:
|
|
782
|
+
raise _Unproven()
|
|
783
|
+
controls = []
|
|
784
|
+
for event, _offset, _line in _records(events_path, stamp):
|
|
785
|
+
control = _native_control(event, session_id)
|
|
786
|
+
if control is not None:
|
|
787
|
+
controls.append(control)
|
|
788
|
+
if len(controls) > MAX_TARGETS:
|
|
789
|
+
raise _Unproven()
|
|
790
|
+
intervals = _steer_intervals(controls, provider_id)
|
|
791
|
+
return [_interruption_origin(origin, provider_id, intervals) or {} for origin in unknown]
|
|
792
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError):
|
|
793
|
+
return unknown
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _prove(session_id: str, provider_id: str, events: Path, root: Path,
|
|
797
|
+
normalize_user: Callable[[dict], str | None], events_stamp,
|
|
798
|
+
normalize_full_user: Callable[[dict], str | None] | None = None,
|
|
799
|
+
normalize_assistant: Callable[[str], str] | None = None) -> _Proof:
|
|
800
|
+
empty = _Proof(provider_id, events_stamp, None, None, frozenset())
|
|
801
|
+
batches = {}
|
|
802
|
+
candidates = []
|
|
803
|
+
terminals = {}
|
|
804
|
+
run_rows = {}
|
|
805
|
+
terminal_counts = {}
|
|
806
|
+
native_events = []
|
|
807
|
+
scheduled_starts, scheduled_ends, candidate_origins = {}, {}, {}
|
|
808
|
+
assistant_replays = _AssistantReplays(provider_id, normalize_assistant)
|
|
809
|
+
async_inputs = _AsyncDeliveryInputs(provider_id, assistant_replays, normalize_user, normalize_full_user)
|
|
810
|
+
wake_inputs = _MailboxWakeInputs(provider_id, assistant_replays, normalize_user, normalize_full_user)
|
|
811
|
+
for event, _offset, _line in _records(events, events_stamp):
|
|
812
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
813
|
+
continue
|
|
814
|
+
assistant_replays.event(event)
|
|
815
|
+
async_inputs.event(event)
|
|
816
|
+
wake_inputs.event(event)
|
|
817
|
+
control = _native_control(event, session_id)
|
|
818
|
+
if control is not None:
|
|
819
|
+
native_events.append(control)
|
|
820
|
+
if len(native_events) > MAX_TARGETS:
|
|
821
|
+
raise _Unproven()
|
|
822
|
+
run = event.get("run_id")
|
|
823
|
+
if (
|
|
824
|
+
isinstance(run, str) and run and not run.startswith("import_")
|
|
825
|
+
and event.get("backend") == "claude" and event.get("imported") is not True
|
|
826
|
+
):
|
|
827
|
+
if (event.get("type") == "turn_started"
|
|
828
|
+
and event.get("purpose") == "scheduled_job" and event.get("job_id")
|
|
829
|
+
and isinstance(event.get("prompt"), str)):
|
|
830
|
+
scheduled_starts.setdefault(run, []).append((
|
|
831
|
+
_text_key(" ".join(event["prompt"].split())), _timestamp(event.get("ts")),
|
|
832
|
+
))
|
|
833
|
+
elif event.get("type") == "turn_finished":
|
|
834
|
+
scheduled_ends.setdefault(run, []).append(_timestamp(event.get("ts")))
|
|
835
|
+
if len(scheduled_starts) + len(scheduled_ends) > MAX_TARGETS:
|
|
836
|
+
raise _Unproven()
|
|
837
|
+
if not isinstance(run, str) or not run.startswith("import_"):
|
|
838
|
+
continue
|
|
839
|
+
run_rows[run] = run_rows.get(run, 0) + 1
|
|
840
|
+
if len(run_rows) > MAX_TARGETS:
|
|
841
|
+
raise _Unproven()
|
|
842
|
+
if event.get("type") == "history_imported":
|
|
843
|
+
checkpoint = event.get("_history_sync_checkpoint")
|
|
844
|
+
cursor = checkpoint.get("cursor") if isinstance(checkpoint, dict) else None
|
|
845
|
+
if (
|
|
846
|
+
event.get("backend") == "claude"
|
|
847
|
+
and event.get("provider_session_id") == provider_id
|
|
848
|
+
and isinstance(cursor, dict) and cursor.get("version") == 1
|
|
849
|
+
and cursor.get("backend") == "claude"
|
|
850
|
+
and cursor.get("provider_session_id") == provider_id
|
|
851
|
+
and event.get("source_path") == cursor.get("source_path")
|
|
852
|
+
and checkpoint.get("version") == 1
|
|
853
|
+
and type(event.get("seq")) is int
|
|
854
|
+
):
|
|
855
|
+
if run in batches:
|
|
856
|
+
raise _Unproven()
|
|
857
|
+
batches[run] = (event["seq"], checkpoint)
|
|
858
|
+
elif event.get("type") == "turn_finished" and event.get("imported") is True:
|
|
859
|
+
if event.get("backend") == "claude" and type(event.get("seq")) is int:
|
|
860
|
+
terminals[run] = event["seq"]
|
|
861
|
+
terminal_counts[run] = terminal_counts.get(run, 0) + 1
|
|
862
|
+
else:
|
|
863
|
+
target = _target(event)
|
|
864
|
+
if target:
|
|
865
|
+
candidates.append(target)
|
|
866
|
+
origin = event.get("provider_origin")
|
|
867
|
+
if isinstance(origin, dict) and origin.get("provider") == "claude":
|
|
868
|
+
candidate_origins[target] = (
|
|
869
|
+
origin.get("event_id"), origin.get("session_id"), origin.get("timestamp"),
|
|
870
|
+
)
|
|
871
|
+
if max(len(batches), len(candidates), len(terminals)) > MAX_TARGETS:
|
|
872
|
+
raise _Unproven()
|
|
873
|
+
if not batches or not (candidates or assistant_replays.candidates):
|
|
874
|
+
return empty
|
|
875
|
+
paths = {batch[1]["cursor"].get("source_path") for batch in batches.values()}
|
|
876
|
+
if len(paths) != 1:
|
|
877
|
+
raise _Unproven()
|
|
878
|
+
raw_path = paths.pop()
|
|
879
|
+
if not isinstance(raw_path, str):
|
|
880
|
+
raise _Unproven()
|
|
881
|
+
source = Path(raw_path)
|
|
882
|
+
if not source.is_absolute() or source.suffix != ".jsonl" or source.stem != provider_id:
|
|
883
|
+
raise _Unproven()
|
|
884
|
+
if source.is_symlink():
|
|
885
|
+
raise _Unproven()
|
|
886
|
+
source = source.resolve(strict=True)
|
|
887
|
+
source.relative_to(root.resolve(strict=True))
|
|
888
|
+
source_stamp = _stamp(source)
|
|
889
|
+
prefix_digests = {}
|
|
890
|
+
eligible_batches = {}
|
|
891
|
+
for run, (seq, checkpoint) in batches.items():
|
|
892
|
+
cursor = checkpoint["cursor"]
|
|
893
|
+
end = cursor.get("source_offset")
|
|
894
|
+
start = checkpoint.get("previous_source_offset")
|
|
895
|
+
expected_digest = cursor.get("source_digest")
|
|
896
|
+
previous_digest = checkpoint.get("previous_source_digest")
|
|
897
|
+
if (
|
|
898
|
+
type(start) is not int or type(end) is not int
|
|
899
|
+
or not 0 <= start < end <= source_stamp[2]
|
|
900
|
+
or (cursor.get("source_dev"), cursor.get("source_ino")) != source_stamp[:2]
|
|
901
|
+
or not isinstance(expected_digest, str) or not _DIGEST.fullmatch(expected_digest)
|
|
902
|
+
or type(checkpoint.get("previous_present")) is not bool
|
|
903
|
+
or (start > 0 and checkpoint.get("previous_present") is not True)
|
|
904
|
+
or (start == 0 and previous_digest != "")
|
|
905
|
+
or (start > 0 and (not isinstance(previous_digest, str) or not _DIGEST.fullmatch(previous_digest)))
|
|
906
|
+
or terminals.get(run, 0) <= seq
|
|
907
|
+
):
|
|
908
|
+
continue
|
|
909
|
+
prefix_digests.setdefault(end, set()).add(expected_digest)
|
|
910
|
+
if start:
|
|
911
|
+
prefix_digests.setdefault(start, set()).add(previous_digest)
|
|
912
|
+
eligible_batches[run] = (seq, terminals[run], start, end, expected_digest, previous_digest)
|
|
913
|
+
if not eligible_batches:
|
|
914
|
+
return _Proof(provider_id, events_stamp, source, source_stamp, frozenset())
|
|
915
|
+
digest = hashlib.sha256()
|
|
916
|
+
verified = set()
|
|
917
|
+
metadata = {}
|
|
918
|
+
humans = set()
|
|
919
|
+
human_offsets = {}
|
|
920
|
+
interruptions = {}
|
|
921
|
+
interruption_count = 0
|
|
922
|
+
tracker = ClaudeInterruptionTracker()
|
|
923
|
+
goal_history = ClaudeGoalHistoryNormalizer()
|
|
924
|
+
goal_sources = {}
|
|
925
|
+
synthetic_sources = {}
|
|
926
|
+
steer_intervals = _steer_intervals(native_events, provider_id)
|
|
927
|
+
scheduled_ranges = {}
|
|
928
|
+
for run, starts in scheduled_starts.items():
|
|
929
|
+
ends = scheduled_ends.get(run, ())
|
|
930
|
+
if len(starts) == len(ends) == 1:
|
|
931
|
+
key, start_time = starts[0]
|
|
932
|
+
end_time = ends[0]
|
|
933
|
+
if start_time is not None and end_time is not None and start_time <= end_time:
|
|
934
|
+
scheduled_ranges.setdefault(key, []).append((run, start_time, end_time))
|
|
935
|
+
scheduled_sources, scheduled_counts = {}, {}
|
|
936
|
+
for event, offset, line in _records(source, source_stamp):
|
|
937
|
+
digest.update(line)
|
|
938
|
+
for wanted in prefix_digests.get(offset, ()):
|
|
939
|
+
if hmac.compare_digest(digest.hexdigest(), wanted):
|
|
940
|
+
verified.add((offset, wanted))
|
|
941
|
+
assistant_replays.source(event, offset)
|
|
942
|
+
async_inputs.source(event, offset)
|
|
943
|
+
wake_inputs.source(event, offset)
|
|
944
|
+
origin = tracker.consume(event)
|
|
945
|
+
normalized_goal = goal_history.consume(event)
|
|
946
|
+
identity = (event.get("uuid"), event.get("sessionId"), event.get("timestamp"))
|
|
947
|
+
if is_claude_synthetic_no_response(event) and identity[1] == provider_id:
|
|
948
|
+
synthetic_sources.setdefault(identity, []).append(offset)
|
|
949
|
+
if normalized_goal is not event and identity[1] == provider_id:
|
|
950
|
+
raw_goal = normalize_user(event)
|
|
951
|
+
if isinstance(raw_goal, str):
|
|
952
|
+
goal_sources.setdefault((identity, _text_key(raw_goal)), []).append(
|
|
953
|
+
(offset, normalized_goal["message"]["content"]))
|
|
954
|
+
if len(goal_sources) + len(synthetic_sources) > MAX_TARGETS:
|
|
955
|
+
raise _Unproven()
|
|
956
|
+
if event.get("type") != "user":
|
|
957
|
+
continue
|
|
958
|
+
text = normalize_user(event)
|
|
959
|
+
if not isinstance(text, str) or not text:
|
|
960
|
+
continue
|
|
961
|
+
key = _text_key(text)
|
|
962
|
+
if (normalize_full_user is not None and scheduled_ranges
|
|
963
|
+
and event.get("sessionId") == provider_id and event.get("isMeta") is not True
|
|
964
|
+
and event.get("isSidechain") is not True):
|
|
965
|
+
full_text = normalize_full_user(event)
|
|
966
|
+
source_time = _timestamp(event.get("timestamp"))
|
|
967
|
+
source_identity = (event.get("uuid"), provider_id, event.get("timestamp"))
|
|
968
|
+
if isinstance(full_text, str) and isinstance(source_identity[0], str) and source_time is not None:
|
|
969
|
+
matches = [run for run, start_time, end_time in scheduled_ranges.get(
|
|
970
|
+
_text_key(" ".join(full_text.split())), (),
|
|
971
|
+
) if start_time <= source_time <= end_time]
|
|
972
|
+
if len(matches) == 1:
|
|
973
|
+
run = matches[0]
|
|
974
|
+
scheduled_counts[run] = scheduled_counts.get(run, 0) + 1
|
|
975
|
+
scheduled_sources.setdefault((source_identity, key), []).append((offset, run))
|
|
976
|
+
origin = _interruption_origin(origin, provider_id, steer_intervals)
|
|
977
|
+
if origin is not None:
|
|
978
|
+
interruptions.setdefault(key, []).append((offset, origin))
|
|
979
|
+
interruption_count += 1
|
|
980
|
+
if interruption_count > MAX_TARGETS:
|
|
981
|
+
raise _Unproven()
|
|
982
|
+
if event.get("isMeta") is True or event.get("isCompactSummary") is True or event.get("isSidechain") is True:
|
|
983
|
+
metadata.setdefault(key, []).append(offset)
|
|
984
|
+
else:
|
|
985
|
+
# Preserve the existing global ambiguity rule for isMeta repairs.
|
|
986
|
+
humans.add(key)
|
|
987
|
+
if origin is None:
|
|
988
|
+
human_offsets.setdefault(key, []).append(offset)
|
|
989
|
+
if len(metadata) + len(humans) + len(interruptions) > MAX_KEYS:
|
|
990
|
+
raise _Unproven()
|
|
991
|
+
targets = set()
|
|
992
|
+
corrected = []
|
|
993
|
+
goal_commands = []
|
|
994
|
+
_owned_assistants, owned_inputs = assistant_replays.exact_owned_sources()
|
|
995
|
+
candidate_counts = {}
|
|
996
|
+
candidate_origin_counts = {}
|
|
997
|
+
run_candidate_counts = {}
|
|
998
|
+
for candidate in candidates:
|
|
999
|
+
_seq, run, key = candidate
|
|
1000
|
+
candidate_counts[(run, key)] = candidate_counts.get((run, key), 0) + 1
|
|
1001
|
+
origin_key = (run, candidate_origins.get(candidate))
|
|
1002
|
+
candidate_origin_counts[origin_key] = candidate_origin_counts.get(origin_key, 0) + 1
|
|
1003
|
+
run_candidate_counts[run] = run_candidate_counts.get(run, 0) + 1
|
|
1004
|
+
for target in candidates:
|
|
1005
|
+
seq, run, key = target
|
|
1006
|
+
batch = eligible_batches.get(run)
|
|
1007
|
+
if batch is None:
|
|
1008
|
+
continue
|
|
1009
|
+
first_seq, last_seq, start, end, expected, previous = batch
|
|
1010
|
+
if not (
|
|
1011
|
+
first_seq < seq < last_seq
|
|
1012
|
+
and (end, expected) in verified
|
|
1013
|
+
and (start == 0 or (start, previous) in verified)
|
|
1014
|
+
):
|
|
1015
|
+
continue
|
|
1016
|
+
if any(identity == candidate_origins.get(target) and digest == key and start < offset <= end
|
|
1017
|
+
for identity, digest, offset in owned_inputs):
|
|
1018
|
+
targets.add(target)
|
|
1019
|
+
continue
|
|
1020
|
+
commands = goal_sources.get((candidate_origins.get(target), key), ())
|
|
1021
|
+
if len(commands) == 1 and start < commands[0][0] <= end:
|
|
1022
|
+
canonical = commands[0][1]
|
|
1023
|
+
native_time = _timestamp(candidate_origins[target][2])
|
|
1024
|
+
owners = []
|
|
1025
|
+
for native_run, starts in assistant_replays.starts.items():
|
|
1026
|
+
ends = assistant_replays.ends.get(native_run, ())
|
|
1027
|
+
if len(starts) != 1 or len(ends) != 1 or starts[0].get("prompt") != canonical:
|
|
1028
|
+
continue
|
|
1029
|
+
records = [starts[0], ends[0], *assistant_replays.owners.get(native_run, ())]
|
|
1030
|
+
provider_ids = {row.get("provider_session_id") for row in records if row.get("provider_session_id")}
|
|
1031
|
+
first, last = _timestamp(starts[0].get("ts")), _timestamp(ends[0].get("ts"))
|
|
1032
|
+
if provider_ids == {provider_id} and first is not None and last is not None and native_time is not None and first <= native_time <= last:
|
|
1033
|
+
owners.append(native_run)
|
|
1034
|
+
if len(owners) == 1:
|
|
1035
|
+
targets.add(target)
|
|
1036
|
+
else:
|
|
1037
|
+
goal_commands.append((target, canonical))
|
|
1038
|
+
continue
|
|
1039
|
+
# A scheduled wake is ordinary provider user input, not isMeta. Prove
|
|
1040
|
+
# its complete text, native occurrence interval, exact source identity
|
|
1041
|
+
# and original import checkpoint. A shared trimmed prefix is not proof.
|
|
1042
|
+
scheduled = scheduled_sources.get((candidate_origins.get(target), key), ())
|
|
1043
|
+
if (len(scheduled) == 1 and candidate_origin_counts[(run, candidate_origins.get(target))] == 1
|
|
1044
|
+
and start < scheduled[0][0] <= end
|
|
1045
|
+
and scheduled_counts.get(scheduled[0][1]) == 1):
|
|
1046
|
+
targets.add(target)
|
|
1047
|
+
continue
|
|
1048
|
+
proven = [(offset, origin) for offset, origin in interruptions.get(key, ())
|
|
1049
|
+
if start < offset <= end]
|
|
1050
|
+
ordinary = human_offsets.get(key, ())
|
|
1051
|
+
if (
|
|
1052
|
+
len(proven) == 1 and candidate_counts[(run, key)] == 1
|
|
1053
|
+
and bisect_right(ordinary, end) - bisect_right(ordinary, start) == 0
|
|
1054
|
+
):
|
|
1055
|
+
corrected.append((target, json.dumps(proven[0][1], sort_keys=True, separators=(",", ":"))))
|
|
1056
|
+
continue
|
|
1057
|
+
# An ambiguous interruption is never demoted to a hidden metadata row.
|
|
1058
|
+
if proven:
|
|
1059
|
+
continue
|
|
1060
|
+
offsets = metadata.get(key, ())
|
|
1061
|
+
if key not in humans and bisect_right(offsets, end) - bisect_right(offsets, start) == 1:
|
|
1062
|
+
targets.add(target)
|
|
1063
|
+
verified_batches = {run: batch for run, batch in eligible_batches.items()
|
|
1064
|
+
if (batch[3], batch[4]) in verified
|
|
1065
|
+
and (batch[2] == 0 or (batch[2], batch[5]) in verified)}
|
|
1066
|
+
targets.update(async_inputs.prove(verified_batches))
|
|
1067
|
+
targets.update(wake_inputs.prove(verified_batches))
|
|
1068
|
+
proven_assistants = assistant_replays.prove(verified_batches)
|
|
1069
|
+
synthetic_targets = set()
|
|
1070
|
+
for target, _phase in assistant_replays.candidates:
|
|
1071
|
+
_kind, seq, run, digest_key, identity = target
|
|
1072
|
+
batch = verified_batches.get(run)
|
|
1073
|
+
offsets = synthetic_sources.get(identity, ())
|
|
1074
|
+
if (batch and batch[0] < seq < batch[1] and len(offsets) == 1
|
|
1075
|
+
and batch[2] < offsets[0] <= batch[3]
|
|
1076
|
+
and digest_key == _text_key("No response requested.")):
|
|
1077
|
+
synthetic_targets.add(target)
|
|
1078
|
+
proven_assistants = proven_assistants | frozenset(synthetic_targets)
|
|
1079
|
+
corrected_counts = {}
|
|
1080
|
+
for (_seq, run, _key), _origin in corrected:
|
|
1081
|
+
corrected_counts[run] = corrected_counts.get(run, 0) + 1
|
|
1082
|
+
for _seq, run, _key in targets:
|
|
1083
|
+
corrected_counts[run] = corrected_counts.get(run, 0) + 1
|
|
1084
|
+
for _kind, _seq, run, _digest, _identity in proven_assistants:
|
|
1085
|
+
corrected_counts[run] = corrected_counts.get(run, 0) + 1
|
|
1086
|
+
companions = set()
|
|
1087
|
+
for run, count in corrected_counts.items():
|
|
1088
|
+
# Only a complete marker-only batch is lifecycle-neutral. Any genuine
|
|
1089
|
+
# user, assistant/tool/unknown row, duplicate terminal or unproven target
|
|
1090
|
+
# leaves both import companions unchanged.
|
|
1091
|
+
if (run_rows[run] == count + 2
|
|
1092
|
+
and terminal_counts.get(run) == 1):
|
|
1093
|
+
companions.add(("history_imported", eligible_batches[run][0], run))
|
|
1094
|
+
companions.add(("turn_finished", eligible_batches[run][1], run))
|
|
1095
|
+
return _Proof(provider_id, events_stamp, source, source_stamp, frozenset(targets),
|
|
1096
|
+
tuple(corrected), frozenset(companions), proven_assistants, tuple(goal_commands))
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
def _bounded_records(path: Path, expected, start: int, end: int):
|
|
1100
|
+
"""Read complete records in one fixed window, never the oversized prefix."""
|
|
1101
|
+
if not 0 <= start < end <= expected[2] or end - start > MAX_EVENTS_BYTES:
|
|
1102
|
+
raise _Unproven()
|
|
1103
|
+
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
1104
|
+
with os.fdopen(descriptor, "rb") as stream:
|
|
1105
|
+
info = os.fstat(stream.fileno())
|
|
1106
|
+
if (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != expected:
|
|
1107
|
+
raise _Unproven()
|
|
1108
|
+
stream.seek(start)
|
|
1109
|
+
if start:
|
|
1110
|
+
stream.seek(start - 1)
|
|
1111
|
+
if stream.read(1) != b"\n":
|
|
1112
|
+
skipped = stream.readline(min(MAX_LINE_BYTES + 1, end - start))
|
|
1113
|
+
if len(skipped) > MAX_LINE_BYTES or not skipped.endswith(b"\n"):
|
|
1114
|
+
raise _Unproven()
|
|
1115
|
+
count = 0
|
|
1116
|
+
while stream.tell() < end:
|
|
1117
|
+
offset = stream.tell()
|
|
1118
|
+
line = stream.readline(min(MAX_LINE_BYTES + 1, end - offset))
|
|
1119
|
+
if len(line) > MAX_LINE_BYTES or not line.endswith(b"\n"):
|
|
1120
|
+
raise _Unproven()
|
|
1121
|
+
count += 1
|
|
1122
|
+
if count > MAX_RECORDS:
|
|
1123
|
+
raise _Unproven()
|
|
1124
|
+
event = json.loads(line)
|
|
1125
|
+
if not isinstance(event, dict):
|
|
1126
|
+
raise _Unproven()
|
|
1127
|
+
yield event, stream.tell()
|
|
1128
|
+
final = os.fstat(stream.fileno())
|
|
1129
|
+
if (final.st_dev, final.st_ino, final.st_size, final.st_mtime_ns) != expected:
|
|
1130
|
+
raise _Unproven()
|
|
1131
|
+
if _regular_stamp(path) != expected:
|
|
1132
|
+
raise _Unproven()
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def _prove_recent_scheduled(session_id: str, provider_id: str, events: Path, root: Path,
|
|
1136
|
+
normalize_user, events_stamp, normalize_full_user, *,
|
|
1137
|
+
normalize_assistant: Callable[[str], str] | None = None,
|
|
1138
|
+
event_window_end: int | None = None) -> _Proof:
|
|
1139
|
+
"""Exact-origin scheduled duplicates and provider metadata in recent history.
|
|
1140
|
+
|
|
1141
|
+
An oversized transcript cannot establish global metadata/interruption proof.
|
|
1142
|
+
Scheduled input requires a complete native occurrence and full source-text
|
|
1143
|
+
match. Metadata requires its explicit structured source flag. Both require
|
|
1144
|
+
an exact, unique origin within the original checkpoint; wording is not proof.
|
|
1145
|
+
Both reads are bounded to 32 MiB; source growth before preparation is allowed.
|
|
1146
|
+
Any change during either read fails visible.
|
|
1147
|
+
"""
|
|
1148
|
+
empty = _Proof(provider_id, events_stamp, None, None, frozenset())
|
|
1149
|
+
if normalize_full_user is None or not events_stamp[2]:
|
|
1150
|
+
return empty
|
|
1151
|
+
window_end = events_stamp[2] if event_window_end is None else event_window_end
|
|
1152
|
+
if type(window_end) is not int or not 0 < window_end <= events_stamp[2]:
|
|
1153
|
+
raise _Unproven()
|
|
1154
|
+
starts, ends, batches, terminals, rows, candidates = {}, {}, {}, {}, {}, []
|
|
1155
|
+
origin_counts = {}
|
|
1156
|
+
assistant_replays = _AssistantReplays(provider_id, normalize_assistant)
|
|
1157
|
+
async_inputs = _AsyncDeliveryInputs(provider_id, assistant_replays, normalize_user, normalize_full_user)
|
|
1158
|
+
wake_inputs = _MailboxWakeInputs(provider_id, assistant_replays, normalize_user, normalize_full_user)
|
|
1159
|
+
previous_seq = 0
|
|
1160
|
+
for event, _offset in _bounded_records(
|
|
1161
|
+
events, events_stamp, max(0, window_end - MAX_EVENTS_BYTES), window_end,
|
|
1162
|
+
):
|
|
1163
|
+
seq = event.get("seq")
|
|
1164
|
+
if type(seq) is not int or seq <= previous_seq:
|
|
1165
|
+
raise _Unproven()
|
|
1166
|
+
previous_seq = seq
|
|
1167
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
1168
|
+
continue
|
|
1169
|
+
assistant_replays.event(event)
|
|
1170
|
+
async_inputs.event(event)
|
|
1171
|
+
wake_inputs.event(event)
|
|
1172
|
+
run = event.get("run_id")
|
|
1173
|
+
if not isinstance(run, str) or not run:
|
|
1174
|
+
continue
|
|
1175
|
+
if not run.startswith("import_"):
|
|
1176
|
+
if (event.get("backend") != "claude" or event.get("imported") is True
|
|
1177
|
+
or event.get("provider_session_id") not in (None, "", provider_id)):
|
|
1178
|
+
continue
|
|
1179
|
+
if (event.get("type") == "turn_started" and event.get("purpose") == "scheduled_job"
|
|
1180
|
+
and event.get("job_id") and isinstance(event.get("prompt"), str)):
|
|
1181
|
+
starts.setdefault(run, []).append((
|
|
1182
|
+
_text_key(" ".join(event["prompt"].split())), _timestamp(event.get("ts")),
|
|
1183
|
+
))
|
|
1184
|
+
elif event.get("type") == "turn_finished":
|
|
1185
|
+
ends.setdefault(run, []).append(_timestamp(event.get("ts")))
|
|
1186
|
+
else:
|
|
1187
|
+
rows[run] = rows.get(run, 0) + 1
|
|
1188
|
+
if event.get("type") == "history_imported":
|
|
1189
|
+
checkpoint = event.get("_history_sync_checkpoint")
|
|
1190
|
+
cursor = checkpoint.get("cursor") if isinstance(checkpoint, dict) else None
|
|
1191
|
+
if (event.get("backend") == "claude" and event.get("provider_session_id") == provider_id
|
|
1192
|
+
and isinstance(cursor, dict) and cursor.get("version") == 1
|
|
1193
|
+
and cursor.get("backend") == "claude" and cursor.get("provider_session_id") == provider_id
|
|
1194
|
+
and checkpoint.get("version") == 1 and event.get("source_path") == cursor.get("source_path")):
|
|
1195
|
+
batches.setdefault(run, []).append((seq, checkpoint))
|
|
1196
|
+
elif (event.get("type") == "turn_finished" and event.get("imported") is True
|
|
1197
|
+
and event.get("backend") == "claude"):
|
|
1198
|
+
terminals.setdefault(run, []).append(seq)
|
|
1199
|
+
target = _target(event)
|
|
1200
|
+
if target:
|
|
1201
|
+
origin = event.get("provider_origin")
|
|
1202
|
+
identity = (origin.get("event_id"), origin.get("session_id"), origin.get("timestamp")) if isinstance(origin, dict) else ()
|
|
1203
|
+
if (isinstance(origin, dict) and origin.get("provider") == "claude"
|
|
1204
|
+
and len(identity) == 3 and all(isinstance(value, str) and value for value in identity)
|
|
1205
|
+
and identity[1] == provider_id and _timestamp(identity[2]) is not None):
|
|
1206
|
+
candidates.append((target, identity))
|
|
1207
|
+
origin_key = (run, identity)
|
|
1208
|
+
origin_counts[origin_key] = origin_counts.get(origin_key, 0) + 1
|
|
1209
|
+
if (sum(map(len, (starts, ends, batches, terminals, rows))) > MAX_TARGETS
|
|
1210
|
+
or len(candidates) > MAX_TARGETS):
|
|
1211
|
+
raise _Unproven()
|
|
1212
|
+
if not batches or not (candidates or assistant_replays.candidates):
|
|
1213
|
+
return empty
|
|
1214
|
+
eligible = {}
|
|
1215
|
+
for run, checkpoints in batches.items():
|
|
1216
|
+
finished = terminals.get(run, ())
|
|
1217
|
+
if len(checkpoints) != 1 or len(finished) != 1:
|
|
1218
|
+
continue
|
|
1219
|
+
seq, checkpoint = checkpoints[0]
|
|
1220
|
+
cursor = checkpoint["cursor"]
|
|
1221
|
+
start, end = checkpoint.get("previous_source_offset"), cursor.get("source_offset")
|
|
1222
|
+
previous, digest = checkpoint.get("previous_source_digest"), cursor.get("source_digest")
|
|
1223
|
+
if (type(start) is int and type(end) is int and 0 <= start < end
|
|
1224
|
+
and type(checkpoint.get("previous_present")) is bool
|
|
1225
|
+
and ((start == 0 and previous == "") or
|
|
1226
|
+
(start > 0 and checkpoint["previous_present"] is True
|
|
1227
|
+
and isinstance(previous, str) and _DIGEST.fullmatch(previous)))
|
|
1228
|
+
and isinstance(digest, str) and _DIGEST.fullmatch(digest) and seq < finished[0]):
|
|
1229
|
+
eligible[run] = (seq, finished[0], start, end, cursor)
|
|
1230
|
+
if not eligible:
|
|
1231
|
+
return empty
|
|
1232
|
+
paths = {batch[4].get("source_path") for batch in eligible.values()}
|
|
1233
|
+
if len(paths) != 1:
|
|
1234
|
+
raise _Unproven()
|
|
1235
|
+
raw_path = paths.pop()
|
|
1236
|
+
if not isinstance(raw_path, str):
|
|
1237
|
+
raise _Unproven()
|
|
1238
|
+
source = Path(raw_path)
|
|
1239
|
+
if not source.is_absolute() or source.suffix != ".jsonl" or source.stem != provider_id or source.is_symlink():
|
|
1240
|
+
raise _Unproven()
|
|
1241
|
+
source = source.resolve(strict=True)
|
|
1242
|
+
source.relative_to(root.resolve(strict=True))
|
|
1243
|
+
source_stamp = _regular_stamp(source)
|
|
1244
|
+
eligible = {run: batch for run, batch in eligible.items()
|
|
1245
|
+
if (batch[4].get("source_dev"), batch[4].get("source_ino")) == source_stamp[:2]
|
|
1246
|
+
and batch[3] <= source_stamp[2]}
|
|
1247
|
+
if not eligible:
|
|
1248
|
+
return empty
|
|
1249
|
+
checkpoint_end = max(batch[3] for batch in eligible.values())
|
|
1250
|
+
window_start = max(0, checkpoint_end - MAX_EVENTS_BYTES)
|
|
1251
|
+
ranges = {}
|
|
1252
|
+
for run, occurrences in starts.items():
|
|
1253
|
+
finished = ends.get(run, ())
|
|
1254
|
+
if len(occurrences) == len(finished) == 1:
|
|
1255
|
+
key, start_time = occurrences[0]
|
|
1256
|
+
end_time = finished[0]
|
|
1257
|
+
if start_time is not None and end_time is not None and start_time <= end_time:
|
|
1258
|
+
ranges.setdefault(key, []).append((run, start_time, end_time))
|
|
1259
|
+
source_matches, occurrence_counts, identity_counts, source_metadata = {}, {}, {}, {}
|
|
1260
|
+
for event, offset in _bounded_records(source, source_stamp, window_start, checkpoint_end):
|
|
1261
|
+
assistant_replays.source(event, offset)
|
|
1262
|
+
async_inputs.source(event, offset)
|
|
1263
|
+
wake_inputs.source(event, offset)
|
|
1264
|
+
if event.get("type") != "user" or event.get("sessionId") != provider_id:
|
|
1265
|
+
continue
|
|
1266
|
+
identity = (event.get("uuid"), provider_id, event.get("timestamp"))
|
|
1267
|
+
if not all(isinstance(value, str) and value for value in identity):
|
|
1268
|
+
continue
|
|
1269
|
+
identity_counts[identity] = identity_counts.get(identity, 0) + 1
|
|
1270
|
+
display_text = normalize_user(event)
|
|
1271
|
+
timestamp = _timestamp(identity[2])
|
|
1272
|
+
if not isinstance(display_text, str) or not display_text or timestamp is None:
|
|
1273
|
+
continue
|
|
1274
|
+
if event.get("isMeta") is True or event.get("isCompactSummary") is True or event.get("isSidechain") is True:
|
|
1275
|
+
source_metadata.setdefault((identity, _text_key(display_text)), []).append(offset)
|
|
1276
|
+
if len(identity_counts) + len(source_metadata) > MAX_KEYS:
|
|
1277
|
+
raise _Unproven()
|
|
1278
|
+
continue
|
|
1279
|
+
full_text = normalize_full_user(event)
|
|
1280
|
+
if not isinstance(full_text, str) or not full_text:
|
|
1281
|
+
continue
|
|
1282
|
+
matches = [run for run, start, end in ranges.get(_text_key(" ".join(full_text.split())), ())
|
|
1283
|
+
if start <= timestamp <= end]
|
|
1284
|
+
if len(matches) == 1:
|
|
1285
|
+
occurrence = matches[0]
|
|
1286
|
+
occurrence_counts[occurrence] = occurrence_counts.get(occurrence, 0) + 1
|
|
1287
|
+
source_matches.setdefault((identity, _text_key(display_text)), []).append((offset, occurrence))
|
|
1288
|
+
if len(identity_counts) + len(source_matches) > MAX_KEYS:
|
|
1289
|
+
raise _Unproven()
|
|
1290
|
+
targets, counts = set(), {}
|
|
1291
|
+
for target, identity in candidates:
|
|
1292
|
+
seq, run, key = target
|
|
1293
|
+
batch, matched = eligible.get(run), source_matches.get((identity, key), ())
|
|
1294
|
+
if (batch is None or origin_counts[(run, identity)] != 1
|
|
1295
|
+
or identity_counts.get(identity) != 1):
|
|
1296
|
+
continue
|
|
1297
|
+
first, last, start, end, _cursor = batch
|
|
1298
|
+
metadata = source_metadata.get((identity, key), ())
|
|
1299
|
+
proven_metadata = len(metadata) == 1 and start < metadata[0] <= end
|
|
1300
|
+
proven_scheduled = (len(matched) == 1 and occurrence_counts.get(matched[0][1]) == 1
|
|
1301
|
+
and start < matched[0][0] <= end)
|
|
1302
|
+
if first < seq < last and (proven_metadata or proven_scheduled):
|
|
1303
|
+
targets.add(target)
|
|
1304
|
+
counts[run] = counts.get(run, 0) + 1
|
|
1305
|
+
targets.update(async_inputs.prove(eligible))
|
|
1306
|
+
targets.update(wake_inputs.prove(eligible))
|
|
1307
|
+
proven_assistants = assistant_replays.prove(eligible)
|
|
1308
|
+
counts = {}
|
|
1309
|
+
for _seq, run, _key in targets:
|
|
1310
|
+
counts[run] = counts.get(run, 0) + 1
|
|
1311
|
+
for _kind, _seq, run, _digest, _identity in proven_assistants:
|
|
1312
|
+
counts[run] = counts.get(run, 0) + 1
|
|
1313
|
+
companions = set()
|
|
1314
|
+
for run, count in counts.items():
|
|
1315
|
+
if rows[run] == count + 2:
|
|
1316
|
+
companions.add(("history_imported", eligible[run][0], run))
|
|
1317
|
+
companions.add(("turn_finished", eligible[run][1], run))
|
|
1318
|
+
return _Proof(provider_id, events_stamp, source, source_stamp, frozenset(targets),
|
|
1319
|
+
companions=frozenset(companions), assistant_replays=proven_assistants)
|
|
1320
|
+
|
|
1321
|
+
|
|
1322
|
+
@dataclass(frozen=True)
|
|
1323
|
+
class ClaudeMetadataRepairWindow:
|
|
1324
|
+
"""Immutable page-owned proof; lookups never read files or change the main index."""
|
|
1325
|
+
|
|
1326
|
+
session_id: str
|
|
1327
|
+
_proof: _Proof = field(repr=False)
|
|
1328
|
+
|
|
1329
|
+
def is_hidden(self, event: dict) -> bool:
|
|
1330
|
+
if event.get("session_id") not in (None, "", self.session_id):
|
|
1331
|
+
return False
|
|
1332
|
+
try:
|
|
1333
|
+
return _target(event) in self._proof.targets
|
|
1334
|
+
except (ValueError, TypeError):
|
|
1335
|
+
return False
|
|
1336
|
+
|
|
1337
|
+
def project_event(self, event: dict) -> dict | None:
|
|
1338
|
+
return _project_proof_event(self.session_id, event, self._proof)
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def _project_proof_event(session_id: str, event: dict, proof: _Proof | None) -> dict | None:
|
|
1342
|
+
if proof is None or event.get("session_id") not in (None, "", session_id):
|
|
1343
|
+
return None
|
|
1344
|
+
canonical = proof.goal_command_index.get(_target(event))
|
|
1345
|
+
if canonical is not None:
|
|
1346
|
+
return {**event, "prompt": canonical, "provider_history_repair": "source_proven_goal_command"}
|
|
1347
|
+
if proof.assistant_replays and _assistant_target(event) in proof.assistant_replays:
|
|
1348
|
+
return {**event, "text": "", "metadata_only": True,
|
|
1349
|
+
"provider_history_repair": "source_proven_assistant_replay"}
|
|
1350
|
+
kind, seq, run = event.get("type"), event.get("seq"), event.get("run_id")
|
|
1351
|
+
if (kind in ("history_imported", "turn_finished") and type(seq) is int
|
|
1352
|
+
and isinstance(run, str) and event.get("backend") == "claude"
|
|
1353
|
+
and (kind == "history_imported" or event.get("imported") is True)
|
|
1354
|
+
and (kind, seq, run) in proof.companions
|
|
1355
|
+
and event.get("provider_session_id") in (None, "", proof.provider_id)):
|
|
1356
|
+
return {**event, "imported": True, "metadata_only": True}
|
|
1357
|
+
try:
|
|
1358
|
+
encoded_origin = proof.interruption_index.get(_target(event))
|
|
1359
|
+
except (ValueError, TypeError):
|
|
1360
|
+
return None
|
|
1361
|
+
if encoded_origin is None:
|
|
1362
|
+
return None
|
|
1363
|
+
origin = json.loads(encoded_origin)
|
|
1364
|
+
projected = dict(event)
|
|
1365
|
+
for key in ("prompt", "text", "_agentsdock_imported_prompt_hidden"):
|
|
1366
|
+
projected.pop(key, None)
|
|
1367
|
+
return {**projected, "type": "provider_interruption", "imported": True,
|
|
1368
|
+
"ts": origin["timestamp"], "provider_origin": origin}
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
class ClaudeMetadataRepairCache:
|
|
1372
|
+
def __init__(self) -> None:
|
|
1373
|
+
self._lock = threading.RLock()
|
|
1374
|
+
# Serialize explicit preparation without blocking per-event lookups
|
|
1375
|
+
# (which can execute on the event loop) behind any filesystem work.
|
|
1376
|
+
self._prepare_lock = threading.Lock()
|
|
1377
|
+
self._proofs: OrderedDict[str, _Proof] = OrderedDict()
|
|
1378
|
+
self._windows: OrderedDict[tuple, ClaudeMetadataRepairWindow] = OrderedDict()
|
|
1379
|
+
self._preparing_session: str | None = None
|
|
1380
|
+
self._preparation_cancelled = False
|
|
1381
|
+
|
|
1382
|
+
def prepare_window(self, session_id: str, provider_id: str, events: Path, root: Path,
|
|
1383
|
+
normalize_user: Callable[[dict], str | None], *, event_window_end: int,
|
|
1384
|
+
normalize_full_user: Callable[[dict], str | None] | None = None,
|
|
1385
|
+
normalize_assistant: Callable[[str], str] | None = None) -> ClaudeMetadataRepairWindow:
|
|
1386
|
+
"""Prove one explicitly requested historical page, never the entire ledger.
|
|
1387
|
+
|
|
1388
|
+
Unchanged positive and negative windows are cached. Any observed log
|
|
1389
|
+
change, including append growth, requires fresh bounded proof; inode
|
|
1390
|
+
equality alone cannot certify that an old prefix was not rewritten.
|
|
1391
|
+
This cache never contributes to the global timeline-index signature.
|
|
1392
|
+
"""
|
|
1393
|
+
try:
|
|
1394
|
+
stamp = _regular_stamp(events)
|
|
1395
|
+
except (OSError, ValueError):
|
|
1396
|
+
stamp = (0, 0, 0, 0)
|
|
1397
|
+
empty = ClaudeMetadataRepairWindow(session_id, _Proof(provider_id, stamp, None, None, frozenset()))
|
|
1398
|
+
if type(event_window_end) is not int or not 0 < event_window_end <= stamp[2]:
|
|
1399
|
+
return empty
|
|
1400
|
+
key = (session_id, provider_id, str(events.absolute()), str(root.absolute()), stamp,
|
|
1401
|
+
max(0, event_window_end - MAX_EVENTS_BYTES), event_window_end)
|
|
1402
|
+
|
|
1403
|
+
def cached_window() -> ClaudeMetadataRepairWindow | None:
|
|
1404
|
+
with self._lock:
|
|
1405
|
+
cached = self._windows.get(key)
|
|
1406
|
+
if cached is not None and cached._proof.source is not None:
|
|
1407
|
+
try:
|
|
1408
|
+
if _regular_stamp(cached._proof.source) != cached._proof.source_stamp:
|
|
1409
|
+
return None
|
|
1410
|
+
except (OSError, ValueError):
|
|
1411
|
+
return None
|
|
1412
|
+
with self._lock:
|
|
1413
|
+
if cached is not None and self._windows.get(key) is cached:
|
|
1414
|
+
self._windows.move_to_end(key)
|
|
1415
|
+
return cached
|
|
1416
|
+
return None
|
|
1417
|
+
|
|
1418
|
+
cached = cached_window()
|
|
1419
|
+
if cached is not None:
|
|
1420
|
+
return cached
|
|
1421
|
+
with self._prepare_lock:
|
|
1422
|
+
cached = cached_window()
|
|
1423
|
+
if cached is not None:
|
|
1424
|
+
return cached
|
|
1425
|
+
with self._lock:
|
|
1426
|
+
self._preparing_session = session_id
|
|
1427
|
+
self._preparation_cancelled = False
|
|
1428
|
+
try:
|
|
1429
|
+
proof = _prove_recent_scheduled(session_id, provider_id, events, root,
|
|
1430
|
+
normalize_user, stamp, normalize_full_user,
|
|
1431
|
+
normalize_assistant=normalize_assistant,
|
|
1432
|
+
event_window_end=event_window_end)
|
|
1433
|
+
window = ClaudeMetadataRepairWindow(session_id, proof)
|
|
1434
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError):
|
|
1435
|
+
window = empty
|
|
1436
|
+
with self._lock:
|
|
1437
|
+
cancelled = self._preparation_cancelled
|
|
1438
|
+
self._preparing_session = None
|
|
1439
|
+
if cancelled:
|
|
1440
|
+
return empty
|
|
1441
|
+
self._windows[key] = window
|
|
1442
|
+
self._windows.move_to_end(key)
|
|
1443
|
+
while len(self._windows) > MAX_WINDOWS:
|
|
1444
|
+
self._windows.popitem(last=False)
|
|
1445
|
+
return window
|
|
1446
|
+
|
|
1447
|
+
def prepare(self, session_id: str, provider_id: str, events: Path, root: Path,
|
|
1448
|
+
normalize_user: Callable[[dict], str | None], *,
|
|
1449
|
+
normalize_full_user: Callable[[dict], str | None] | None = None,
|
|
1450
|
+
normalize_assistant: Callable[[str], str] | None = None,
|
|
1451
|
+
refresh: bool = False) -> bool:
|
|
1452
|
+
"""Prepare only this requested session; report a changed suppression map."""
|
|
1453
|
+
with self._lock:
|
|
1454
|
+
previous = self._proofs.get(session_id)
|
|
1455
|
+
if previous and previous.provider_id == provider_id and not refresh:
|
|
1456
|
+
self._proofs.move_to_end(session_id)
|
|
1457
|
+
return False
|
|
1458
|
+
with self._prepare_lock:
|
|
1459
|
+
with self._lock:
|
|
1460
|
+
previous = self._proofs.get(session_id)
|
|
1461
|
+
if previous and previous.provider_id == provider_id and not refresh:
|
|
1462
|
+
# A verified historical target remains a fact when either
|
|
1463
|
+
# append-only log grows. Do not turn ordinary chat refreshes
|
|
1464
|
+
# into repeated source scans. New imports already filter
|
|
1465
|
+
# metadata; negative admissions are cached as well.
|
|
1466
|
+
self._proofs.move_to_end(session_id)
|
|
1467
|
+
return False
|
|
1468
|
+
self._preparing_session = session_id
|
|
1469
|
+
self._preparation_cancelled = False
|
|
1470
|
+
try:
|
|
1471
|
+
try:
|
|
1472
|
+
stamp = _stamp(events)
|
|
1473
|
+
except _Oversized:
|
|
1474
|
+
if normalize_full_user is None:
|
|
1475
|
+
raise
|
|
1476
|
+
stamp = _regular_stamp(events)
|
|
1477
|
+
# During an explicit import refresh keep the old immutable
|
|
1478
|
+
# proof available to memory-only event readers until replace.
|
|
1479
|
+
if not refresh:
|
|
1480
|
+
with self._lock:
|
|
1481
|
+
self._proofs.pop(session_id, None)
|
|
1482
|
+
try:
|
|
1483
|
+
if stamp[2] > MAX_EVENTS_BYTES:
|
|
1484
|
+
raise _Oversized()
|
|
1485
|
+
proof = _prove(session_id, provider_id, events, root, normalize_user, stamp,
|
|
1486
|
+
normalize_full_user, normalize_assistant)
|
|
1487
|
+
except _Oversized:
|
|
1488
|
+
if normalize_full_user is None:
|
|
1489
|
+
raise
|
|
1490
|
+
proof = _prove_recent_scheduled(session_id, provider_id, events, root,
|
|
1491
|
+
normalize_user, stamp, normalize_full_user,
|
|
1492
|
+
normalize_assistant=normalize_assistant)
|
|
1493
|
+
except (OSError, ValueError, TypeError, KeyError, RuntimeError):
|
|
1494
|
+
# Fail visible, including incomplete or oversized files. A
|
|
1495
|
+
# failed admission must not retry on every page/socket read.
|
|
1496
|
+
proof = _Proof(provider_id, (0, 0, 0, 0), None, None, frozenset())
|
|
1497
|
+
if (refresh and previous and previous.provider_id == provider_id
|
|
1498
|
+
and previous.source is not None and previous.source == proof.source
|
|
1499
|
+
and previous.source_stamp is not None and proof.source_stamp is not None
|
|
1500
|
+
and previous.events_stamp[:2] == proof.events_stamp[:2]
|
|
1501
|
+
and previous.source_stamp[:2] == proof.source_stamp[:2]
|
|
1502
|
+
and previous.events_stamp[2] <= proof.events_stamp[2]
|
|
1503
|
+
and previous.source_stamp[2] <= proof.source_stamp[2]
|
|
1504
|
+
and (previous.events_stamp[2] < proof.events_stamp[2] or previous.events_stamp == proof.events_stamp)
|
|
1505
|
+
and (previous.source_stamp[2] < proof.source_stamp[2] or previous.source_stamp == proof.source_stamp)):
|
|
1506
|
+
# Exact file ownership and nondecreasing immutable-log prefixes
|
|
1507
|
+
# retain older proofs outside the bounded recent read. A replaced,
|
|
1508
|
+
# truncated, unavailable or different source never inherits them.
|
|
1509
|
+
inputs = previous.targets | proof.targets
|
|
1510
|
+
interruptions = {**dict(previous.interruptions), **dict(proof.interruptions)}
|
|
1511
|
+
assistants = previous.assistant_replays | proof.assistant_replays
|
|
1512
|
+
commands = {**dict(previous.goal_commands), **dict(proof.goal_commands)}
|
|
1513
|
+
entries = ([(key[0], key[1], "input", key) for key in inputs]
|
|
1514
|
+
+ [(key[0], key[1], "interruption", key) for key in interruptions]
|
|
1515
|
+
+ [(key[1], key[2], "assistant", key) for key in assistants]
|
|
1516
|
+
+ [(key[0], key[1], "goal_command", key) for key in commands])
|
|
1517
|
+
entries.sort(key=lambda entry: entry[0], reverse=True)
|
|
1518
|
+
evicted_runs = {entry[1] for entry in entries[MAX_TARGETS:]}
|
|
1519
|
+
retained = entries[:MAX_TARGETS]
|
|
1520
|
+
companions = sorted((key for key in previous.companions | proof.companions
|
|
1521
|
+
if key[2] not in evicted_runs), key=lambda key: key[1], reverse=True)[:MAX_TARGETS]
|
|
1522
|
+
# One combined retained-repair budget, not a new allowance on
|
|
1523
|
+
# every import. Evicted proof fails visible, including its
|
|
1524
|
+
# batch companions; no partial batch becomes metadata-only.
|
|
1525
|
+
proof = _Proof(provider_id, proof.events_stamp, proof.source, proof.source_stamp,
|
|
1526
|
+
frozenset(key for _, _, kind, key in retained if kind == "input"),
|
|
1527
|
+
tuple((key, interruptions[key]) for _, _, kind, key in retained if kind == "interruption"),
|
|
1528
|
+
frozenset(companions),
|
|
1529
|
+
frozenset(key for _, _, kind, key in retained if kind == "assistant"),
|
|
1530
|
+
tuple((key, commands[key]) for _, _, kind, key in retained if kind == "goal_command"))
|
|
1531
|
+
with self._lock:
|
|
1532
|
+
cancelled = self._preparation_cancelled
|
|
1533
|
+
self._preparing_session = None
|
|
1534
|
+
if not cancelled:
|
|
1535
|
+
self._proofs[session_id] = proof
|
|
1536
|
+
self._proofs.move_to_end(session_id)
|
|
1537
|
+
while len(self._proofs) > MAX_SESSIONS:
|
|
1538
|
+
self._proofs.popitem(last=False)
|
|
1539
|
+
else:
|
|
1540
|
+
return bool(previous and previous.signature())
|
|
1541
|
+
return bool((previous.signature() if previous else frozenset()) != proof.signature())
|
|
1542
|
+
|
|
1543
|
+
def signature(self, session_id: str) -> frozenset:
|
|
1544
|
+
"""Memory-only cache identity: repaired timeline indexes cannot go stale."""
|
|
1545
|
+
with self._lock:
|
|
1546
|
+
proof = self._proofs.get(session_id)
|
|
1547
|
+
return proof.signature() if proof else frozenset()
|
|
1548
|
+
|
|
1549
|
+
def forget(self, session_id: str) -> None:
|
|
1550
|
+
with self._lock:
|
|
1551
|
+
self._proofs.pop(session_id, None)
|
|
1552
|
+
for key in tuple(self._windows):
|
|
1553
|
+
if key[0] == session_id:
|
|
1554
|
+
self._windows.pop(key, None)
|
|
1555
|
+
if self._preparing_session == session_id:
|
|
1556
|
+
self._preparation_cancelled = True
|
|
1557
|
+
|
|
1558
|
+
def is_hidden(self, session_id: str, event: dict) -> bool:
|
|
1559
|
+
"""Memory-only per-event lookup; missing or evicted proof stays visible."""
|
|
1560
|
+
with self._lock:
|
|
1561
|
+
proof = self._proofs.get(session_id)
|
|
1562
|
+
if not proof or not proof.targets:
|
|
1563
|
+
return False
|
|
1564
|
+
try:
|
|
1565
|
+
target = _target(event)
|
|
1566
|
+
except (ValueError, TypeError):
|
|
1567
|
+
return False
|
|
1568
|
+
return target in proof.targets
|
|
1569
|
+
|
|
1570
|
+
def interruption_origin(self, session_id: str, event: dict) -> dict | None:
|
|
1571
|
+
"""Return copied, verified provenance without filesystem access."""
|
|
1572
|
+
if event.get("session_id") not in (None, "", session_id):
|
|
1573
|
+
return None
|
|
1574
|
+
with self._lock:
|
|
1575
|
+
proof = self._proofs.get(session_id)
|
|
1576
|
+
if not proof or not proof.interruptions:
|
|
1577
|
+
return None
|
|
1578
|
+
try:
|
|
1579
|
+
target = _target(event)
|
|
1580
|
+
except (ValueError, TypeError):
|
|
1581
|
+
return None
|
|
1582
|
+
origin = proof.interruption_index.get(target)
|
|
1583
|
+
return json.loads(origin) if origin is not None else None
|
|
1584
|
+
|
|
1585
|
+
def project_interruption(self, session_id: str, event: dict) -> dict | None:
|
|
1586
|
+
origin = self.interruption_origin(session_id, event)
|
|
1587
|
+
if origin is None:
|
|
1588
|
+
return None
|
|
1589
|
+
projected = dict(event)
|
|
1590
|
+
projected.pop("prompt", None)
|
|
1591
|
+
projected.pop("text", None)
|
|
1592
|
+
projected.pop("_agentsdock_imported_prompt_hidden", None)
|
|
1593
|
+
projected.update(type="provider_interruption", imported=True,
|
|
1594
|
+
ts=origin["timestamp"], provider_origin=origin)
|
|
1595
|
+
return projected
|
|
1596
|
+
|
|
1597
|
+
def project_event(self, session_id: str, event: dict) -> dict | None:
|
|
1598
|
+
"""Memory-only correction of exact source-proven historical records."""
|
|
1599
|
+
with self._lock:
|
|
1600
|
+
proof = self._proofs.get(session_id)
|
|
1601
|
+
return _project_proof_event(session_id, event, proof)
|