cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
"""Pure write-time account attribution kernel for Claude cost records (#944).
|
|
2
|
+
|
|
3
|
+
Claude cost ingest used to stamp every newly read JSONL line with the account
|
|
4
|
+
active when the ingest RAN. This kernel decides the account active when the
|
|
5
|
+
line was WRITTEN instead, from the Claude account timeline the #825/#832 usage
|
|
6
|
+
fence already captures (spec
|
|
7
|
+
``docs/superpowers/specs/2026-10-29-944-claude-write-time-cost-attribution-design.md``,
|
|
8
|
+
sections 5.2 and 5.3), and falls back to the ingest-time identity only when
|
|
9
|
+
that evidence is absent or inconclusive.
|
|
10
|
+
|
|
11
|
+
Two halves:
|
|
12
|
+
|
|
13
|
+
* the rule (``decide``): a record's timestamp interval against its timeline
|
|
14
|
+
neighbours -- the latest observation instant entirely before it, every
|
|
15
|
+
instant overlapping it, and the earliest instant entirely after it (or the
|
|
16
|
+
sync's closing capture when no observation follows). Agreement of every one
|
|
17
|
+
of them names the account; anything else falls back with a typed reason;
|
|
18
|
+
* the identity encoders: the durable decision key a record is filed under
|
|
19
|
+
(keyed by its logical ``(msg_id, req_id)``, physical by its exact raw bytes,
|
|
20
|
+
or the legacy projection of an already-cached row), and legacy matching.
|
|
21
|
+
|
|
22
|
+
Pure stdlib leaf: no I/O, no clock, no SQLite, no project imports.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import dataclasses
|
|
27
|
+
import datetime as dt
|
|
28
|
+
import hashlib
|
|
29
|
+
import json
|
|
30
|
+
import re
|
|
31
|
+
from typing import Iterable, Mapping
|
|
32
|
+
|
|
33
|
+
#: The attribution rule's version. A change invalidates the derived timeline
|
|
34
|
+
#: and rebuilds it from the journal; existing decisions are never
|
|
35
|
+
#: reinterpreted (spec 5.1).
|
|
36
|
+
RULE_VERSION = 1
|
|
37
|
+
|
|
38
|
+
BASIS_AGREEMENT = "timeline_agreement"
|
|
39
|
+
BASIS_FALLBACK = "fallback"
|
|
40
|
+
BASIS_PRESERVED = "preserved_history"
|
|
41
|
+
BASES = (BASIS_AGREEMENT, BASIS_FALLBACK, BASIS_PRESERVED)
|
|
42
|
+
|
|
43
|
+
#: The fallback reasons, in the order the rule evaluates them (spec 5.2).
|
|
44
|
+
FALLBACK_REASONS = (
|
|
45
|
+
"missing_timestamp", "invalid_timestamp", "clock_order", "pre_timeline",
|
|
46
|
+
"ambiguous_instant", "open_tail", "switch_window",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
#: The binding sentinel a ``stably_absent`` capture carries (spec 5.1).
|
|
50
|
+
UNATTRIBUTED = "unattributed"
|
|
51
|
+
|
|
52
|
+
FORM_KEYED = "k1"
|
|
53
|
+
FORM_PHYSICAL = "p1"
|
|
54
|
+
FORM_LEGACY = "l1"
|
|
55
|
+
FORMS = (FORM_KEYED, FORM_PHYSICAL, FORM_LEGACY)
|
|
56
|
+
|
|
57
|
+
#: The retained accounting fields of a cost row, in projection order (5.3).
|
|
58
|
+
PROJECTION_FIELDS = (
|
|
59
|
+
"timestamp_utc", "model", "input_tokens", "output_tokens",
|
|
60
|
+
"cache_create_tokens", "cache_read_tokens", "cache_create_1h_tokens",
|
|
61
|
+
"cache_create_5m_tokens", "speed", "cost_usd_raw",
|
|
62
|
+
)
|
|
63
|
+
_INT_FIELDS = frozenset((
|
|
64
|
+
"input_tokens", "output_tokens", "cache_create_tokens",
|
|
65
|
+
"cache_read_tokens", "cache_create_1h_tokens", "cache_create_5m_tokens",
|
|
66
|
+
))
|
|
67
|
+
_SPLIT_INDICES = (6, 7)
|
|
68
|
+
|
|
69
|
+
MATCH_EXACT = "exact"
|
|
70
|
+
MATCH_UNKNOWN_SPLIT = "unknown_split"
|
|
71
|
+
|
|
72
|
+
_LOC_DOMAIN = "cctally-claude-loc-v1\0"
|
|
73
|
+
_US = 1_000_000
|
|
74
|
+
_EPOCH = dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc)
|
|
75
|
+
_ISO_RE = re.compile(
|
|
76
|
+
r"(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})"
|
|
77
|
+
r"(?:\.(\d+))?(Z|z|[+-]\d{2}:?\d{2})?\Z")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# --------------------------------------------------------------------------
|
|
81
|
+
# instants and intervals
|
|
82
|
+
# --------------------------------------------------------------------------
|
|
83
|
+
|
|
84
|
+
@dataclasses.dataclass(frozen=True)
|
|
85
|
+
class Instant:
|
|
86
|
+
"""One coalesced observation instant: every admissible observation whose
|
|
87
|
+
precision interval is exactly ``[lo_us, lo_us + precision_us)``, as the
|
|
88
|
+
set of accounts they name (``None`` is an explicit NULL-account
|
|
89
|
+
observation). More than one account makes the instant ambiguous."""
|
|
90
|
+
|
|
91
|
+
lo_us: int
|
|
92
|
+
precision_us: int
|
|
93
|
+
accounts: frozenset
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def interval(self) -> tuple[int, int]:
|
|
97
|
+
return (self.lo_us, self.lo_us + self.precision_us)
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def ambiguous(self) -> bool:
|
|
101
|
+
return len(self.accounts) != 1
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclasses.dataclass(frozen=True)
|
|
105
|
+
class Neighbours:
|
|
106
|
+
"""A record's timeline neighbours (spec 5.2): ``before`` is ``Pb``,
|
|
107
|
+
``overlapping`` is ``Ov`` and ``after`` is ``Sa``."""
|
|
108
|
+
|
|
109
|
+
before: "Instant | None"
|
|
110
|
+
overlapping: tuple
|
|
111
|
+
after: "Instant | None"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclasses.dataclass(frozen=True)
|
|
115
|
+
class ClosingCapture:
|
|
116
|
+
"""The stable ingest identity resolved after the sync froze its source
|
|
117
|
+
prefixes, normalized like an observation, at the ``clock_cutoff`` instant."""
|
|
118
|
+
|
|
119
|
+
account_key: "str | None"
|
|
120
|
+
at_us: int
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@dataclasses.dataclass(frozen=True)
|
|
124
|
+
class Decision:
|
|
125
|
+
account_key: "str | None"
|
|
126
|
+
basis: str
|
|
127
|
+
reason: "str | None"
|
|
128
|
+
used_closing_capture: bool
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _parse_instant(text: str) -> tuple[int, int]:
|
|
132
|
+
"""``(lo_us, precision_us)`` of an ISO-8601 instant, the precision
|
|
133
|
+
following from the fractional digits it was written with (none: one
|
|
134
|
+
second). A value without an offset is UTC. ``ValueError`` otherwise."""
|
|
135
|
+
if not isinstance(text, str):
|
|
136
|
+
raise ValueError("instant is not a string")
|
|
137
|
+
match = _ISO_RE.match(text.strip())
|
|
138
|
+
if match is None:
|
|
139
|
+
raise ValueError(f"unparseable instant {text!r}")
|
|
140
|
+
year, month, day, hour, minute, second, frac, offset = match.groups()
|
|
141
|
+
moment = dt.datetime(int(year), int(month), int(day), int(hour),
|
|
142
|
+
int(minute), int(second), tzinfo=dt.timezone.utc)
|
|
143
|
+
if offset and offset not in ("Z", "z"):
|
|
144
|
+
sign = 1 if offset[0] == "+" else -1
|
|
145
|
+
digits = offset[1:].replace(":", "")
|
|
146
|
+
delta = dt.timedelta(hours=int(digits[:2]), minutes=int(digits[2:]))
|
|
147
|
+
moment = moment - sign * delta
|
|
148
|
+
whole = moment - _EPOCH
|
|
149
|
+
lo_us = (whole.days * 86_400 + whole.seconds) * _US
|
|
150
|
+
if frac:
|
|
151
|
+
digits = len(frac)
|
|
152
|
+
if digits <= 6:
|
|
153
|
+
precision_us = 10 ** (6 - digits)
|
|
154
|
+
lo_us += int(frac) * precision_us
|
|
155
|
+
else:
|
|
156
|
+
precision_us = 1
|
|
157
|
+
lo_us += int(frac[:6])
|
|
158
|
+
else:
|
|
159
|
+
precision_us = _US
|
|
160
|
+
return lo_us, precision_us
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def capture_interval(captured_at: str) -> tuple[int, int]:
|
|
164
|
+
"""The precision interval ``(lo_us, hi_us)`` of an observation's
|
|
165
|
+
``captured_at``; ``ValueError`` when it does not parse."""
|
|
166
|
+
lo_us, precision_us = _parse_instant(captured_at)
|
|
167
|
+
return lo_us, lo_us + precision_us
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def record_interval(value: object) -> "tuple[int, int] | str":
|
|
171
|
+
"""The interval of a record timestamp, or the fallback reason
|
|
172
|
+
``missing_timestamp`` / ``invalid_timestamp``."""
|
|
173
|
+
if value is None or (isinstance(value, str) and not value.strip()):
|
|
174
|
+
return "missing_timestamp"
|
|
175
|
+
try:
|
|
176
|
+
lo_us, precision_us = _parse_instant(value)
|
|
177
|
+
except (ValueError, OverflowError):
|
|
178
|
+
return "invalid_timestamp"
|
|
179
|
+
return lo_us, lo_us + precision_us
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def relation(obs: tuple[int, int], rec: tuple[int, int]) -> str:
|
|
183
|
+
"""``before`` when ``obs`` ends at or before ``rec`` starts, ``after``
|
|
184
|
+
when it starts at or after ``rec`` ends, ``overlapping`` otherwise (its
|
|
185
|
+
order relative to the record is unknown)."""
|
|
186
|
+
if obs[1] <= rec[0]:
|
|
187
|
+
return "before"
|
|
188
|
+
if obs[0] >= rec[1]:
|
|
189
|
+
return "after"
|
|
190
|
+
return "overlapping"
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def normalize_binding_account(account_key: str) -> "str | None":
|
|
194
|
+
"""A binding ``accountKey``: the ``unattributed`` sentinel becomes NULL."""
|
|
195
|
+
return None if account_key == UNATTRIBUTED else account_key
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def valid_record_id(value) -> bool:
|
|
199
|
+
"""Whether a transcript ``message.id`` / ``requestId`` can name a record:
|
|
200
|
+
only a non-empty string can. Anything else (empty, numeric, an object)
|
|
201
|
+
counts as missing, so the record is identified by its bytes (physical) or
|
|
202
|
+
its projection (legacy), never by a keyed item the journal would reject."""
|
|
203
|
+
return isinstance(value, str) and bool(value)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def keyed_ids(msg_id, req_id) -> "tuple[str, str] | None":
|
|
207
|
+
"""The ``(msg_id, req_id)`` a keyed identity uses, or None when either
|
|
208
|
+
is not a valid record id (the one identity rule every path shares)."""
|
|
209
|
+
if valid_record_id(msg_id) and valid_record_id(req_id):
|
|
210
|
+
return msg_id, req_id
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def preserved_stamp(account_key) -> "str | None":
|
|
215
|
+
"""A cached row's stamp as ``preserved_history`` carries it: a real
|
|
216
|
+
account key, otherwise NULL (the ``unattributed`` sentinel, an empty
|
|
217
|
+
string or any non-string value; a decision never carries those)."""
|
|
218
|
+
if not isinstance(account_key, str) or not account_key:
|
|
219
|
+
return None
|
|
220
|
+
return normalize_binding_account(account_key)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def iso_from_us(value_us: int) -> str:
|
|
224
|
+
"""A fixed-width UTC ISO string (microseconds, ``Z``), so lexicographic
|
|
225
|
+
order is chronological order."""
|
|
226
|
+
moment = _EPOCH + dt.timedelta(microseconds=int(value_us))
|
|
227
|
+
return moment.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def us_from_iso(text: str) -> int:
|
|
231
|
+
return _parse_instant(text)[0]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
# --------------------------------------------------------------------------
|
|
235
|
+
# the rule
|
|
236
|
+
# --------------------------------------------------------------------------
|
|
237
|
+
|
|
238
|
+
def _fallback(ingest_account, reason) -> Decision:
|
|
239
|
+
return Decision(ingest_account, BASIS_FALLBACK, reason, False)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def decide(interval, neighbours, *, clock_cutoff_us: int,
|
|
243
|
+
closing_capture: "ClosingCapture | None",
|
|
244
|
+
ingest_account: "str | None") -> Decision:
|
|
245
|
+
"""The spec 5.2 table, evaluated in its order.
|
|
246
|
+
|
|
247
|
+
``interval`` is ``record_interval``'s result. ``neighbours`` holds the
|
|
248
|
+
observation instants around it (``None``: no timeline). When no
|
|
249
|
+
observation follows the record and a closing capture is given, the
|
|
250
|
+
capture serves as ``Sa``; it is after the record whenever the clock-order
|
|
251
|
+
check passed. An ambiguous instant is never skipped to reach an
|
|
252
|
+
unambiguous one, and an overlapping instant never stands in for ``Pb``
|
|
253
|
+
or ``Sa``."""
|
|
254
|
+
if isinstance(interval, str):
|
|
255
|
+
return _fallback(ingest_account, interval)
|
|
256
|
+
if interval[1] > clock_cutoff_us:
|
|
257
|
+
return _fallback(ingest_account, "clock_order")
|
|
258
|
+
before = None if neighbours is None else neighbours.before
|
|
259
|
+
if before is None:
|
|
260
|
+
return _fallback(ingest_account, "pre_timeline")
|
|
261
|
+
overlapping = tuple(neighbours.overlapping)
|
|
262
|
+
after = neighbours.after
|
|
263
|
+
used_capture = False
|
|
264
|
+
if after is None and closing_capture is not None:
|
|
265
|
+
after = Instant(int(closing_capture.at_us), 1,
|
|
266
|
+
frozenset((closing_capture.account_key,)))
|
|
267
|
+
used_capture = True
|
|
268
|
+
if before.ambiguous or any(i.ambiguous for i in overlapping) or (
|
|
269
|
+
after is not None and after.ambiguous):
|
|
270
|
+
return _fallback(ingest_account, "ambiguous_instant")
|
|
271
|
+
if after is None:
|
|
272
|
+
return _fallback(ingest_account, "open_tail")
|
|
273
|
+
accounts = set(before.accounts) | set(after.accounts)
|
|
274
|
+
for instant in overlapping:
|
|
275
|
+
accounts |= instant.accounts
|
|
276
|
+
if len(accounts) == 1:
|
|
277
|
+
(account,) = accounts
|
|
278
|
+
return Decision(account, BASIS_AGREEMENT, None, used_capture)
|
|
279
|
+
return _fallback(ingest_account, "switch_window")
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# --------------------------------------------------------------------------
|
|
283
|
+
# identity encoders (spec 5.3)
|
|
284
|
+
# --------------------------------------------------------------------------
|
|
285
|
+
|
|
286
|
+
def _compact(value) -> str:
|
|
287
|
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False,
|
|
288
|
+
allow_nan=False)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _sha256_text(text: str) -> str:
|
|
292
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def loc_for(source_path: str) -> str:
|
|
296
|
+
"""The 32-hex location of the exact stored ``session_entries.source_path``.
|
|
297
|
+
Deliberately not canonicalized: two spellings of one root are two
|
|
298
|
+
physical rows, so a physical or legacy identity names at most one."""
|
|
299
|
+
data = (_LOC_DOMAIN + source_path).encode("utf-8", "surrogateescape")
|
|
300
|
+
return hashlib.sha256(data).hexdigest()[:32]
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def keyed_key(msg_id: str, req_id: str) -> str:
|
|
304
|
+
return FORM_KEYED + ":" + _compact([msg_id, req_id])
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def record_sha256(record_bytes: bytes) -> str:
|
|
308
|
+
"""The SHA-256 of a record's exact raw bytes, terminator removed."""
|
|
309
|
+
return hashlib.sha256(record_bytes).hexdigest()
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def physical_key_from_digest(loc: str, byte_offset: int, digest: str) -> str:
|
|
313
|
+
return FORM_PHYSICAL + ":" + _compact([loc, int(byte_offset), digest])
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def physical_key(loc: str, byte_offset: int, record_bytes: bytes) -> str:
|
|
317
|
+
return physical_key_from_digest(loc, byte_offset,
|
|
318
|
+
record_sha256(record_bytes))
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _projection_value(name: str, value):
|
|
322
|
+
if value is None:
|
|
323
|
+
return None
|
|
324
|
+
if name in _INT_FIELDS:
|
|
325
|
+
if isinstance(value, bool):
|
|
326
|
+
raise ValueError(f"{name} is not an integer")
|
|
327
|
+
return int(value)
|
|
328
|
+
if name == "cost_usd_raw":
|
|
329
|
+
if isinstance(value, bool):
|
|
330
|
+
raise ValueError("cost_usd_raw is not a number")
|
|
331
|
+
return float(value)
|
|
332
|
+
if isinstance(value, str):
|
|
333
|
+
return value
|
|
334
|
+
# A TEXT column (``speed``) stores a non-string scalar as its text form;
|
|
335
|
+
# project the value a parsed record carries the same way the stored row
|
|
336
|
+
# reads back, so a walk and a cached row agree.
|
|
337
|
+
if isinstance(value, bool):
|
|
338
|
+
return str(int(value))
|
|
339
|
+
return str(value)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def legacy_projection(row: Mapping) -> list:
|
|
343
|
+
"""Every retained accounting field of a cached row, in projection order:
|
|
344
|
+
SQL NULL as ``None``, integers as integers, ``cost_usd_raw`` as a float
|
|
345
|
+
(JSON renders it as its shortest round-trip form)."""
|
|
346
|
+
return [_projection_value(name, row[name]) for name in PROJECTION_FIELDS]
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def projection_sha256(projection: list) -> str:
|
|
350
|
+
return _sha256_text(_compact(list(projection)))
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def projection_json(projection: list) -> str:
|
|
354
|
+
return _compact(list(projection))
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def legacy_key(loc: str, line_offset: int, projection: list) -> str:
|
|
358
|
+
return FORM_LEGACY + ":" + _compact(
|
|
359
|
+
[loc, int(line_offset), projection_sha256(projection)])
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def legacy_key_from_digest(loc: str, line_offset: int, digest: str) -> str:
|
|
363
|
+
return FORM_LEGACY + ":" + _compact([loc, int(line_offset), digest])
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _split_unknown(projection) -> bool:
|
|
367
|
+
return all(projection[i] is None for i in _SPLIT_INDICES)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def legacy_match(decision_projection: list, record_projection: list
|
|
371
|
+
) -> "str | None":
|
|
372
|
+
"""Whether a record matches a legacy decision: ``exact`` when every field
|
|
373
|
+
is equal, ``unknown_split`` when the decision recorded the TTL split as
|
|
374
|
+
unknown (both split fields NULL) and every other field is equal, else
|
|
375
|
+
``None``. The exception is exactly the #195 re-walk's enrichment of a
|
|
376
|
+
retained row."""
|
|
377
|
+
if len(decision_projection) != len(PROJECTION_FIELDS) or len(
|
|
378
|
+
record_projection) != len(PROJECTION_FIELDS):
|
|
379
|
+
return None
|
|
380
|
+
for index, (left, right) in enumerate(
|
|
381
|
+
zip(decision_projection, record_projection)):
|
|
382
|
+
if index in _SPLIT_INDICES:
|
|
383
|
+
continue
|
|
384
|
+
if left != right or type(left) is not type(right):
|
|
385
|
+
return None
|
|
386
|
+
split_equal = all(
|
|
387
|
+
decision_projection[i] == record_projection[i]
|
|
388
|
+
and type(decision_projection[i]) is type(record_projection[i])
|
|
389
|
+
for i in _SPLIT_INDICES)
|
|
390
|
+
if split_equal:
|
|
391
|
+
return MATCH_EXACT
|
|
392
|
+
if _split_unknown(decision_projection):
|
|
393
|
+
return MATCH_UNKNOWN_SPLIT
|
|
394
|
+
return None
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def journal_position_key(position) -> tuple:
|
|
398
|
+
"""Canonical order of a ``(segment, offset)`` journal position: bootstrap
|
|
399
|
+
segments first, then observation segments, each by name (the journal's
|
|
400
|
+
own segment order), then offset."""
|
|
401
|
+
segment, offset = position
|
|
402
|
+
if segment.startswith("bootstrap-"):
|
|
403
|
+
klass = 0
|
|
404
|
+
elif segment.startswith("observations-"):
|
|
405
|
+
klass = 1
|
|
406
|
+
else:
|
|
407
|
+
klass = 2
|
|
408
|
+
return (klass, segment, int(offset))
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
_MATCH_RANK = {MATCH_EXACT: 0, MATCH_UNKNOWN_SPLIT: 1}
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def order_legacy_candidates(candidates: Iterable) -> list:
|
|
415
|
+
"""Order matching legacy candidates per spec 5.3: a match that needed no
|
|
416
|
+
recorded-unknown split first, then earliest journal position. Each
|
|
417
|
+
candidate is ``(match_kind, (segment, offset), ...)``; extra elements ride
|
|
418
|
+
along unchanged. Non-matches (``None`` kind) are dropped."""
|
|
419
|
+
kept = [c for c in candidates if c[0] in _MATCH_RANK]
|
|
420
|
+
return sorted(kept, key=lambda c: (_MATCH_RANK[c[0]],
|
|
421
|
+
journal_position_key(c[1])))
|
package/bin/_lib_clone_copy.py
CHANGED
|
@@ -18,6 +18,13 @@ for:
|
|
|
18
18
|
- :func:`copytree` is ``shutil.copytree`` with :func:`copy2` as its file
|
|
19
19
|
copier, so symlink policy, ignore lists, dangling-link errors and
|
|
20
20
|
destination collisions are the stdlib's own code.
|
|
21
|
+
- :func:`clone_regular_file` is the strict clone (#1000): it clones the
|
|
22
|
+
regular file `src` to the new path `dst` or raises ``OSError`` with the
|
|
23
|
+
clone's own errno. It never copies, hardlinks or overwrites, never follows
|
|
24
|
+
a symlinked source (ELOOP), refuses a non-regular source (EINVAL) without
|
|
25
|
+
blocking on it, raises ENOTSUP where no ``fclonefileat`` exists, and keeps
|
|
26
|
+
the clone syscall's own metadata (no ``copystat``). It is for a caller that
|
|
27
|
+
must never write a byte copy, such as the transcript integrity probe.
|
|
21
28
|
|
|
22
29
|
A clone is attempted only on Darwin. An unsupported filesystem, a
|
|
23
30
|
cross-volume destination, a missing ``fclonefileat`` and any non-Darwin
|
|
@@ -124,6 +131,29 @@ def try_clone(src, dst) -> bool:
|
|
|
124
131
|
os.close(fd)
|
|
125
132
|
|
|
126
133
|
|
|
134
|
+
def clone_regular_file(src, dst, *, counts=None):
|
|
135
|
+
"""Clone the regular file `src` to the NEW path `dst`, or raise (#1000).
|
|
136
|
+
|
|
137
|
+
Returns `dst`. Raises ``OSError(ENOTSUP)`` when the platform has no
|
|
138
|
+
``fclonefileat``, ``OSError(EINVAL)`` when `src` is not a regular file,
|
|
139
|
+
ELOOP when `src` is a symlink, and the clone syscall's own ``OSError``
|
|
140
|
+
(ENOTSUP, EXDEV, EEXIST, EIO, ENOSPC, ...) unchanged on any clone failure.
|
|
141
|
+
Nothing is created on failure, and nothing is ever byte-copied.
|
|
142
|
+
"""
|
|
143
|
+
if _fclonefileat() is None:
|
|
144
|
+
raise OSError(errno.ENOTSUP, "fclonefileat is unavailable", str(dst))
|
|
145
|
+
fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0)
|
|
146
|
+
| getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
|
|
147
|
+
try:
|
|
148
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
149
|
+
raise OSError(errno.EINVAL, "not a regular file", str(src))
|
|
150
|
+
_clone_fd(fd, os.fspath(dst))
|
|
151
|
+
finally:
|
|
152
|
+
os.close(fd)
|
|
153
|
+
_count(counts, "clone")
|
|
154
|
+
return dst
|
|
155
|
+
|
|
156
|
+
|
|
127
157
|
def _count(counts, method: str) -> None:
|
|
128
158
|
if counts is not None:
|
|
129
159
|
counts[method] = counts.get(method, 0) + 1
|
|
@@ -5116,11 +5116,40 @@ class _CodexPayloadContainmentError(RuntimeError):
|
|
|
5116
5116
|
"""Internal 404 signal: the selected physical container escaped its root."""
|
|
5117
5117
|
|
|
5118
5118
|
|
|
5119
|
-
def
|
|
5119
|
+
def _codex_payload_archive_dir(logical, root_path):
|
|
5120
|
+
"""The archive directory holding ``logical``'s archive containers (#970).
|
|
5121
|
+
|
|
5122
|
+
Only a row whose stored provider root is a native home, and whose logical
|
|
5123
|
+
path is dated under that home's ``sessions/``, has one. The root is the
|
|
5124
|
+
stored canonical path, so the logical path is compared canonically too
|
|
5125
|
+
when its own spelling differs (only its directory is resolved: the
|
|
5126
|
+
rollout itself may be gone).
|
|
5127
|
+
"""
|
|
5128
|
+
if not root_path:
|
|
5129
|
+
return None
|
|
5130
|
+
locations = _rc.home_locations(root_path)
|
|
5131
|
+
if locations is None or locations.archive_root is None:
|
|
5132
|
+
return None
|
|
5133
|
+
found = _rc.archive_dir_for(logical, (locations,))
|
|
5134
|
+
if found is not None:
|
|
5135
|
+
return found
|
|
5136
|
+
try:
|
|
5137
|
+
canonical = os.path.join(
|
|
5138
|
+
os.path.realpath(os.path.dirname(os.fspath(logical))),
|
|
5139
|
+
os.path.basename(os.fspath(logical)))
|
|
5140
|
+
except (OSError, ValueError):
|
|
5141
|
+
return None
|
|
5142
|
+
return _rc.archive_dir_for(canonical, (locations,))
|
|
5143
|
+
|
|
5144
|
+
|
|
5145
|
+
def _select_codex_payload_container(logical, *, archive_dir=None):
|
|
5120
5146
|
# The discovery leaf deliberately treats stat failures as absent. A payload
|
|
5121
5147
|
# request must distinguish a missing container (410) from inaccessible data
|
|
5122
5148
|
# (500), including an inaccessible plain file beside a readable zstd twin.
|
|
5123
|
-
|
|
5149
|
+
# #970: the archive candidates follow in the leaf's selection order, with
|
|
5150
|
+
# the same strictness (an unreadable archive is 500, never gone).
|
|
5151
|
+
for _kind, _location, candidate in _rc.container_candidates(
|
|
5152
|
+
logical, archive_dir=archive_dir):
|
|
5124
5153
|
try:
|
|
5125
5154
|
info = os.stat(candidate)
|
|
5126
5155
|
except (FileNotFoundError, NotADirectoryError):
|
|
@@ -5129,10 +5158,10 @@ def _select_codex_payload_container(logical):
|
|
|
5129
5158
|
raise CodexContainerReadError("read-error", str(logical)) from exc
|
|
5130
5159
|
if not stat.S_ISREG(info.st_mode):
|
|
5131
5160
|
raise CodexContainerReadError("read-error", str(logical))
|
|
5132
|
-
selected = _rc.select_container(logical)
|
|
5161
|
+
selected = _rc.select_container(logical, archive_dir=archive_dir)
|
|
5133
5162
|
if selected is not None:
|
|
5134
5163
|
return selected
|
|
5135
|
-
# It disappeared between the strict stat and selection; try the
|
|
5164
|
+
# It disappeared between the strict stat and selection; try the next.
|
|
5136
5165
|
return None
|
|
5137
5166
|
|
|
5138
5167
|
|
|
@@ -5152,20 +5181,26 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict
|
|
|
5152
5181
|
|
|
5153
5182
|
#968: select plain first, else zstd, and seek in decoded offsets. Streams are
|
|
5154
5183
|
request-local and reusable; no decoded copies persist. Present unreadable
|
|
5155
|
-
containers raise CodexContainerReadError, never a false gone result.
|
|
5184
|
+
containers raise CodexContainerReadError, never a false gone result.
|
|
5185
|
+
#970: then the archive containers of a native home's dated rollout, the
|
|
5186
|
+
archive directory derived from the row's stored provider root; the
|
|
5187
|
+
provider-root containment and the reselect-on-``OSError`` retry cover an
|
|
5188
|
+
archive move exactly as they cover a compression."""
|
|
5156
5189
|
stored = conn.execute(
|
|
5157
5190
|
"SELECT payload_json FROM codex_conversation_events "
|
|
5158
5191
|
"WHERE source_path = ? AND line_offset = ?",
|
|
5159
5192
|
(row.source_path, row.line_offset)).fetchone()
|
|
5160
5193
|
if stored is None:
|
|
5161
5194
|
return None
|
|
5195
|
+
root_path = _codex_source_root_path(conn, row.source_root_key)
|
|
5196
|
+
archive_dir = _codex_payload_archive_dir(row.source_path, root_path)
|
|
5162
5197
|
for attempt in range(2):
|
|
5163
|
-
selected = _select_codex_payload_container(
|
|
5198
|
+
selected = _select_codex_payload_container(
|
|
5199
|
+
row.source_path, archive_dir=archive_dir)
|
|
5164
5200
|
if selected is None:
|
|
5165
5201
|
return None
|
|
5166
5202
|
# Validate exactly the descriptor we open, including after a compression
|
|
5167
5203
|
# race. The caller has already checked the logical path and known root.
|
|
5168
|
-
root_path = _codex_source_root_path(conn, row.source_root_key)
|
|
5169
5204
|
if not _within_root(str(selected.path), root_path):
|
|
5170
5205
|
raise _CodexPayloadContainmentError()
|
|
5171
5206
|
try:
|
|
@@ -5184,11 +5219,12 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict
|
|
|
5184
5219
|
except OSError as exc:
|
|
5185
5220
|
# ContainerError subclasses OSError; preserve its approved leaf
|
|
5186
5221
|
# category instead of flattening decoder/integrity errors.
|
|
5187
|
-
replacement = _select_codex_payload_container(
|
|
5222
|
+
replacement = _select_codex_payload_container(
|
|
5223
|
+
row.source_path, archive_dir=archive_dir)
|
|
5188
5224
|
if replacement is None:
|
|
5189
5225
|
return None
|
|
5190
5226
|
if replacement.path != selected.path and attempt == 0:
|
|
5191
|
-
continue #
|
|
5227
|
+
continue # moved during Codex's compression or archive window
|
|
5192
5228
|
category = exc.category if isinstance(exc, _rc.ContainerError) else "read-error"
|
|
5193
5229
|
raise CodexContainerReadError(category, str(row.source_path)) from exc
|
|
5194
5230
|
try:
|