cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -0,0 +1,613 @@
|
|
|
1
|
+
"""Pure kernel for correction-to-fit lineage (#1015).
|
|
2
|
+
|
|
3
|
+
No I/O, no clock, no database, no printing, no environment read; stdlib only.
|
|
4
|
+
The glue that reads the stats projections lives in
|
|
5
|
+
`bin/_cctally_quota_calibration.py` (`read_correction_facts`) and in the
|
|
6
|
+
journal paths that write them (`bin/_cctally_journal.py`).
|
|
7
|
+
|
|
8
|
+
Spec: docs/superpowers/specs/2026-10-31-1015-calibration-correction-lineage-design.md
|
|
9
|
+
(sections 5.1 to 5.3).
|
|
10
|
+
|
|
11
|
+
This module owns two things the data plane needs:
|
|
12
|
+
|
|
13
|
+
* :func:`batch_footprint` — the ``(provider, account_key)`` pairs one completed
|
|
14
|
+
correction batch touched. Every path that records a batch's lineage (the
|
|
15
|
+
selector's completion step on live ingest, the live emitters and the
|
|
16
|
+
apply-only rebuild) computes it here, so the paths cannot disagree.
|
|
17
|
+
* The value types of the correction facts a fit is assessed against
|
|
18
|
+
(:class:`CorrectionFacts`, :class:`CorrectionRow`, :class:`BatchFact`).
|
|
19
|
+
|
|
20
|
+
The lineage predicate, the window construction and the transition lineage are
|
|
21
|
+
added beside them by the rest of #1015.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import bisect
|
|
26
|
+
from typing import Callable, Iterable, Literal, NamedTuple
|
|
27
|
+
|
|
28
|
+
#: An owner as the fold stores it: ``(provider, account_key)``. ``'*'`` in the
|
|
29
|
+
#: account position is a stored account-blind row (a `*`-family latch, an
|
|
30
|
+
#: effects-only event); ``('*', '*')`` is written only when ownership is
|
|
31
|
+
#: genuinely unknowable.
|
|
32
|
+
Owner = tuple[str, str]
|
|
33
|
+
|
|
34
|
+
#: The footprint entry for a revision whose owner cannot be resolved.
|
|
35
|
+
UNKNOWN_OWNER: Owner = ("*", "*")
|
|
36
|
+
|
|
37
|
+
ActionKind = Literal["replace", "tombstone", "new"]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Action(NamedTuple):
|
|
41
|
+
"""One action of a completed correction batch, in manifest order.
|
|
42
|
+
|
|
43
|
+
``kind`` is ``"replace"`` (a replacement revision), ``"tombstone"`` or
|
|
44
|
+
``"new"`` (a replacement of an event id with no prior revision; a
|
|
45
|
+
``"replace"`` whose predecessor resolves to None is treated the same).
|
|
46
|
+
``payload_owner`` is the owner the fold would store for the new payload,
|
|
47
|
+
None for a tombstone or when it cannot be resolved (a replacement or new
|
|
48
|
+
event with an unresolvable payload owner contributes :data:`UNKNOWN_OWNER`).
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
event_id: str
|
|
52
|
+
rev: int
|
|
53
|
+
kind: ActionKind
|
|
54
|
+
payload_owner: "Owner | None"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Revision(NamedTuple):
|
|
58
|
+
"""The revision of an event id effective immediately before an action.
|
|
59
|
+
|
|
60
|
+
``status`` is ``"active"`` for a revision carrying data, whose ``owner`` is
|
|
61
|
+
the owner the fold stores for it (None when that cannot be resolved), or
|
|
62
|
+
``"tombstone"`` for a revision that carries no data — a tombstoned id,
|
|
63
|
+
whose replacement adds only its own payload and whose re-tombstoning
|
|
64
|
+
removes nothing. ``predecessor_of`` returns None when the id has no
|
|
65
|
+
effective revision at all.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
status: Literal["active", "tombstone"]
|
|
69
|
+
owner: "Owner | None" = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _active_owner(revision: Revision) -> "Owner | None":
|
|
73
|
+
"""The data owner a revision contributes when it is replaced or removed:
|
|
74
|
+
nothing for a tombstone, its owner (or :data:`UNKNOWN_OWNER`) otherwise."""
|
|
75
|
+
if revision.status != "active":
|
|
76
|
+
return None
|
|
77
|
+
return revision.owner if revision.owner is not None else UNKNOWN_OWNER
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def batch_footprint(
|
|
81
|
+
actions: Iterable[Action],
|
|
82
|
+
predecessor_of: Callable[[str, int], "Revision | None"],
|
|
83
|
+
) -> frozenset:
|
|
84
|
+
"""The ``(provider, account_key)`` pairs one completed batch touched.
|
|
85
|
+
|
|
86
|
+
For each action, in manifest order:
|
|
87
|
+
|
|
88
|
+
* a replacement contributes the owner of the revision it replaces and the
|
|
89
|
+
owner of its new payload, so a move from account A to account B touches
|
|
90
|
+
both;
|
|
91
|
+
* a tombstone contributes the owner of the revision it removes;
|
|
92
|
+
* a new event contributes the owner of its payload.
|
|
93
|
+
|
|
94
|
+
The predecessor is the revision effective immediately before that action:
|
|
95
|
+
the revision an earlier action of the same batch produced for the same id,
|
|
96
|
+
else ``predecessor_of(event_id, rev)`` (the caller's view of the effective
|
|
97
|
+
revision before the batch). A replacement whose predecessor resolves to
|
|
98
|
+
None is a new event. :data:`UNKNOWN_OWNER` appears only when an owner
|
|
99
|
+
cannot be resolved. A batch with no actions has an empty footprint.
|
|
100
|
+
"""
|
|
101
|
+
touched: set = set()
|
|
102
|
+
produced: dict = {}
|
|
103
|
+
for action in actions:
|
|
104
|
+
if action.event_id in produced:
|
|
105
|
+
predecessor = produced[action.event_id]
|
|
106
|
+
elif action.kind == "new":
|
|
107
|
+
predecessor = None
|
|
108
|
+
else:
|
|
109
|
+
predecessor = predecessor_of(action.event_id, action.rev)
|
|
110
|
+
if action.kind == "tombstone":
|
|
111
|
+
if predecessor is not None:
|
|
112
|
+
removed = _active_owner(predecessor)
|
|
113
|
+
if removed is not None:
|
|
114
|
+
touched.add(removed)
|
|
115
|
+
produced[action.event_id] = Revision("tombstone")
|
|
116
|
+
continue
|
|
117
|
+
payload = (action.payload_owner if action.payload_owner is not None
|
|
118
|
+
else UNKNOWN_OWNER)
|
|
119
|
+
if predecessor is not None:
|
|
120
|
+
replaced = _active_owner(predecessor)
|
|
121
|
+
if replaced is not None:
|
|
122
|
+
touched.add(replaced)
|
|
123
|
+
touched.add(payload)
|
|
124
|
+
produced[action.event_id] = Revision("active", action.payload_owner)
|
|
125
|
+
return frozenset(touched)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# --------------------------------------------------------------------------
|
|
129
|
+
# correction facts (read from the published stats projections)
|
|
130
|
+
# --------------------------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
#: A journal coordinate ``(segment, end_offset)``: the end of the record's
|
|
133
|
+
#: line. Coordinates compare in journal order through
|
|
134
|
+
#: `_cctally_journal.journal_coordinate_key`; this kernel never orders them
|
|
135
|
+
#: itself, the caller supplies comparable keys where it needs to.
|
|
136
|
+
Coordinate = tuple[str, int]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class CorrectionRow(NamedTuple):
|
|
140
|
+
"""One reviewed decision of one observation, joined to the observation
|
|
141
|
+
index and to its op's coordinate. ``account_key`` and ``captured_at`` are
|
|
142
|
+
None when no index row resolves the observation (an unresolved
|
|
143
|
+
reference)."""
|
|
144
|
+
|
|
145
|
+
op_id: str
|
|
146
|
+
observation_id: str
|
|
147
|
+
disposition: Literal["hold", "retract", "accept"]
|
|
148
|
+
account_key: "str | None"
|
|
149
|
+
captured_at: "str | None"
|
|
150
|
+
op_coord: Coordinate
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class BatchFact(NamedTuple):
|
|
154
|
+
"""One completed correction batch: its commit coordinate and its
|
|
155
|
+
footprint (a frozenset of ``(provider, account_key)`` pairs; empty for an
|
|
156
|
+
empty batch)."""
|
|
157
|
+
|
|
158
|
+
batch_id: str
|
|
159
|
+
commit_coord: Coordinate
|
|
160
|
+
footprint: frozenset
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class CorrectionFacts(NamedTuple):
|
|
164
|
+
"""Everything a fit's lineage is assessed against, read from one coherent
|
|
165
|
+
stats snapshot: the correction rows in the account scope, every completed
|
|
166
|
+
batch, and the current basis (the ids of every completed batch)."""
|
|
167
|
+
|
|
168
|
+
corrections: tuple
|
|
169
|
+
batches: tuple
|
|
170
|
+
basis: frozenset
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
# --------------------------------------------------------------------------
|
|
174
|
+
# fit lineage: windows, digest, the lineage records (#1015 spec 5.2)
|
|
175
|
+
# --------------------------------------------------------------------------
|
|
176
|
+
|
|
177
|
+
#: The `fitLineage` / `transitionLineage` record version.
|
|
178
|
+
LINEAGE_VERSION = 1
|
|
179
|
+
|
|
180
|
+
#: Regime keys a non-fitting transition legitimately changes (closing,
|
|
181
|
+
#: stale-marking, the #1015 lifecycle annotations, the frozen transition
|
|
182
|
+
#: lineage with its own digest) plus the writer-provenance stamps. Every
|
|
183
|
+
#: other key a fit writes is bound by the lineage digest (spec 5.2).
|
|
184
|
+
DIGEST_EXCLUDED_KEYS = frozenset((
|
|
185
|
+
"effectiveUntil", "status", "supersededByCorrection",
|
|
186
|
+
"recalibrationAttempt", "transitionLineage",
|
|
187
|
+
"wpg", "wpgDigest", "wpr", "wprDigest", "wpx", "wpxDigest",
|
|
188
|
+
))
|
|
189
|
+
|
|
190
|
+
FIT_LINEAGE_KEY = "fitLineage"
|
|
191
|
+
TRANSITION_LINEAGE_KEY = "transitionLineage"
|
|
192
|
+
|
|
193
|
+
WindowKind = Literal["nosplit", "successor", "baseline"]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class Assessment(NamedTuple):
|
|
197
|
+
"""A fit's (or a transition's) lineage against the correction facts.
|
|
198
|
+
|
|
199
|
+
``corrections`` is ``((op_id, observation_count), ...)`` in journal order
|
|
200
|
+
of the op, populated only for ``"tainted"``.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
state: Literal["clean", "tainted", "unavailable"]
|
|
204
|
+
corrections: tuple = ()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
CLEAN = Assessment("clean")
|
|
208
|
+
UNAVAILABLE = Assessment("unavailable")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _canonical(value) -> str:
|
|
212
|
+
import json
|
|
213
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"),
|
|
214
|
+
ensure_ascii=False, default=str)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _sha256(text: str) -> str:
|
|
218
|
+
import hashlib
|
|
219
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _instant(value):
|
|
223
|
+
"""An aware UTC datetime from a datetime or an ISO string, else None."""
|
|
224
|
+
import datetime as dt
|
|
225
|
+
if isinstance(value, dt.datetime):
|
|
226
|
+
parsed = value
|
|
227
|
+
elif isinstance(value, str) and value:
|
|
228
|
+
try:
|
|
229
|
+
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
230
|
+
except ValueError:
|
|
231
|
+
return None
|
|
232
|
+
else:
|
|
233
|
+
return None
|
|
234
|
+
if parsed.tzinfo is None:
|
|
235
|
+
return None
|
|
236
|
+
return parsed.astimezone(dt.timezone.utc)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _iso(value) -> "str | None":
|
|
240
|
+
instant = _instant(value)
|
|
241
|
+
return None if instant is None else instant.isoformat()
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def window(start, until, *, until_inclusive: bool) -> dict:
|
|
245
|
+
"""One dependency window as stored: canonical UTC instants, None open."""
|
|
246
|
+
return {"from": _iso(start), "until": _iso(until),
|
|
247
|
+
"untilInclusive": bool(until_inclusive)}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def dependency_windows(kind: WindowKind, *, a, s=None, t=None) -> tuple:
|
|
251
|
+
"""The correction dependency windows of one fit (spec 5.2).
|
|
252
|
+
|
|
253
|
+
``a`` is the analysis start (`PersistMode.regime_start`), ``s`` the
|
|
254
|
+
split's first instant, ``t`` the fit's `asOf`. A no-split fit depends on
|
|
255
|
+
``[A, T]``; a successor on ``[S, T]`` plus the pre-split ``[A, S)`` that
|
|
256
|
+
supplies its fence and baseline composition; a baseline created in the
|
|
257
|
+
same run on ``[A, S)``. A correction exactly at ``S`` belongs to the
|
|
258
|
+
successor, one exactly at ``T`` is included.
|
|
259
|
+
"""
|
|
260
|
+
if kind == "nosplit":
|
|
261
|
+
return (window(a, t, until_inclusive=True),)
|
|
262
|
+
if kind == "successor":
|
|
263
|
+
return (window(s, t, until_inclusive=True),
|
|
264
|
+
window(a, s, until_inclusive=False))
|
|
265
|
+
if kind == "baseline":
|
|
266
|
+
return (window(a, s, until_inclusive=False),)
|
|
267
|
+
raise ValueError(f"unknown dependency-window kind {kind!r}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _regime_core(regime) -> dict:
|
|
271
|
+
return {k: v for k, v in dict(regime).items()
|
|
272
|
+
if k not in DIGEST_EXCLUDED_KEYS and k != FIT_LINEAGE_KEY}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def lineage_digest(regime) -> str:
|
|
276
|
+
"""The digest a regime's `fitLineage.digest` must equal (spec 5.2).
|
|
277
|
+
|
|
278
|
+
It binds every field a fit writes — the fitted values and their context,
|
|
279
|
+
`asOf`, `effectiveFrom` — and the lineage itself (windows, estimator,
|
|
280
|
+
scope, basis, `fitId`), and excludes only `DIGEST_EXCLUDED_KEYS`. A
|
|
281
|
+
previous release's in-place `update()` always writes a new `asOf`, so it
|
|
282
|
+
breaks this digest and the regime is assessed as legacy.
|
|
283
|
+
"""
|
|
284
|
+
lineage = regime.get(FIT_LINEAGE_KEY) if isinstance(regime, dict) else None
|
|
285
|
+
lineage_body = ({k: v for k, v in lineage.items() if k != "digest"}
|
|
286
|
+
if isinstance(lineage, dict) else None)
|
|
287
|
+
return "sha256:" + _sha256(_canonical(
|
|
288
|
+
{"regime": _regime_core(regime), "lineage": lineage_body}))
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def fit_lineage(*, scope, basis, windows, estimator, regime) -> dict:
|
|
292
|
+
"""The immutable `fitLineage` of a fit this binary creates (spec 5.2).
|
|
293
|
+
|
|
294
|
+
``regime`` is the record the lineage describes (any `fitLineage` it
|
|
295
|
+
already carries is ignored); the returned object's `digest` verifies
|
|
296
|
+
against ``{**regime, "fitLineage": <returned>}``.
|
|
297
|
+
"""
|
|
298
|
+
body = {
|
|
299
|
+
"version": LINEAGE_VERSION,
|
|
300
|
+
"accountScope": str(scope),
|
|
301
|
+
"basis": sorted(str(b) for b in basis),
|
|
302
|
+
"estimator": None if estimator is None else dict(estimator),
|
|
303
|
+
"dependencyWindows": [dict(w) for w in windows],
|
|
304
|
+
"legacy": False,
|
|
305
|
+
}
|
|
306
|
+
fit_id = "fit:" + _sha256(_canonical(
|
|
307
|
+
{"regime": _regime_core(regime), "lineage": body}))
|
|
308
|
+
lineage = {"version": LINEAGE_VERSION, "fitId": fit_id, **body}
|
|
309
|
+
lineage = {k: lineage[k] for k in (
|
|
310
|
+
"version", "fitId", "accountScope", "basis", "estimator",
|
|
311
|
+
"dependencyWindows", "legacy")}
|
|
312
|
+
lineage["digest"] = lineage_digest(
|
|
313
|
+
{**dict(regime), FIT_LINEAGE_KEY: lineage})
|
|
314
|
+
return lineage
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def fit_lineage_verifies(regime) -> bool:
|
|
318
|
+
"""Whether ``regime`` carries a current `fitLineage` whose digest holds."""
|
|
319
|
+
if not isinstance(regime, dict):
|
|
320
|
+
return False
|
|
321
|
+
lineage = regime.get(FIT_LINEAGE_KEY)
|
|
322
|
+
return (isinstance(lineage, dict)
|
|
323
|
+
and lineage.get("version") == LINEAGE_VERSION
|
|
324
|
+
and lineage.get("legacy") is False
|
|
325
|
+
and isinstance(lineage.get("fitId"), str)
|
|
326
|
+
and lineage.get("digest") == lineage_digest(regime))
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def legacy_lineage(regime, era_start, *, scope="*") -> dict:
|
|
330
|
+
"""The conservative lineage of a regime with no verifying `fitLineage`.
|
|
331
|
+
|
|
332
|
+
``basis = []`` and one window ``[era start of its coefficientEra, asOf]``;
|
|
333
|
+
an unreadable era start or `asOf` leaves that end open. ``fitId`` is the
|
|
334
|
+
sha256 of the regime's canonical content. ``scope`` is the bucket the
|
|
335
|
+
regime is stored under (``"*"``, every account, when unknown).
|
|
336
|
+
"""
|
|
337
|
+
record = dict(regime) if isinstance(regime, dict) else {}
|
|
338
|
+
return {
|
|
339
|
+
"version": LINEAGE_VERSION,
|
|
340
|
+
"fitId": "fit:" + _sha256(_canonical(record)),
|
|
341
|
+
"accountScope": str(scope),
|
|
342
|
+
"basis": [],
|
|
343
|
+
"estimator": None,
|
|
344
|
+
"dependencyWindows": [window(era_start, record.get("asOf"),
|
|
345
|
+
until_inclusive=True)],
|
|
346
|
+
"legacy": True,
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def regime_lineage(regime, *, scope, era_start) -> dict:
|
|
351
|
+
"""The lineage a regime is assessed under: its own verifying
|
|
352
|
+
`fitLineage`, else :func:`legacy_lineage`."""
|
|
353
|
+
if fit_lineage_verifies(regime):
|
|
354
|
+
return dict(regime[FIT_LINEAGE_KEY])
|
|
355
|
+
return legacy_lineage(regime, era_start, scope=scope)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def transition_lineage(*, windows, scope, basis, descriptor) -> dict:
|
|
359
|
+
"""The frozen lineage of one detected transition (spec 5.7).
|
|
360
|
+
|
|
361
|
+
``windows`` is the union of both sides' dependency windows and the
|
|
362
|
+
detector coverage; ``descriptor`` is every value the `meter_rate_change`
|
|
363
|
+
event journals other than the lineage itself. The digest covers all of
|
|
364
|
+
it and is independent of either regime's `fitLineage`.
|
|
365
|
+
"""
|
|
366
|
+
unique: list = []
|
|
367
|
+
for w in windows:
|
|
368
|
+
w = dict(w)
|
|
369
|
+
if w not in unique:
|
|
370
|
+
unique.append(w)
|
|
371
|
+
body = {
|
|
372
|
+
"version": LINEAGE_VERSION,
|
|
373
|
+
"accountScope": str(scope),
|
|
374
|
+
"basis": sorted(str(b) for b in basis),
|
|
375
|
+
"dependencyWindows": unique,
|
|
376
|
+
"descriptor": dict(descriptor),
|
|
377
|
+
}
|
|
378
|
+
body["digest"] = "sha256:" + _sha256(_canonical(body))
|
|
379
|
+
return body
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def transition_lineage_verifies(lineage) -> bool:
|
|
383
|
+
if not isinstance(lineage, dict) or lineage.get("version") \
|
|
384
|
+
!= LINEAGE_VERSION or not isinstance(lineage.get("descriptor"),
|
|
385
|
+
dict):
|
|
386
|
+
return False
|
|
387
|
+
body = {k: v for k, v in lineage.items() if k != "digest"}
|
|
388
|
+
return lineage.get("digest") == "sha256:" + _sha256(_canonical(body))
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def legacy_transition_lineage(*, account_key, detected_at,
|
|
392
|
+
unattributed="unattributed") -> dict:
|
|
393
|
+
"""The conservative lineage of a transition row with a null
|
|
394
|
+
`fit_lineage` (spec 5.7): ``basis = []``, window
|
|
395
|
+
``(-inf, detected_at]``, and the row's account as scope — except the
|
|
396
|
+
`unattributed` sentinel, which the historical writer used for a merged
|
|
397
|
+
fit and is therefore assessed with the merged scope."""
|
|
398
|
+
scope = "*" if account_key in (None, unattributed) else str(account_key)
|
|
399
|
+
return {
|
|
400
|
+
"version": LINEAGE_VERSION,
|
|
401
|
+
"accountScope": scope,
|
|
402
|
+
"basis": [],
|
|
403
|
+
"dependencyWindows": [window(None, detected_at,
|
|
404
|
+
until_inclusive=True)],
|
|
405
|
+
"legacy": True,
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
# --------------------------------------------------------------------------
|
|
410
|
+
# the predicate (spec 5.3)
|
|
411
|
+
# --------------------------------------------------------------------------
|
|
412
|
+
|
|
413
|
+
def _coordinate_order(coord) -> tuple:
|
|
414
|
+
"""Journal order of a ``(segment, offset)`` coordinate: the key
|
|
415
|
+
`_cctally_journal.journal_coordinate_key` uses."""
|
|
416
|
+
import _lib_journal
|
|
417
|
+
return (_lib_journal.segment_sort_key(str(coord[0])), int(coord[1]))
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _touches(footprint, account_key) -> bool:
|
|
421
|
+
return (("claude", account_key) in footprint
|
|
422
|
+
or ("claude", "*") in footprint or ("*", "*") in footprint)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _touches_claude(footprint) -> bool:
|
|
426
|
+
return any(provider in ("claude", "*") for provider, _a in footprint)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _parsed_windows(windows) -> tuple:
|
|
430
|
+
"""`(start, until, until_inclusive)` per well-formed window, parsed once
|
|
431
|
+
per assessment rather than once per correction."""
|
|
432
|
+
return tuple((_instant(w.get("from")), _instant(w.get("until")),
|
|
433
|
+
bool(w.get("untilInclusive")))
|
|
434
|
+
for w in windows if isinstance(w, dict))
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _in_windows(instant, windows) -> bool:
|
|
438
|
+
for start, until, inclusive in windows:
|
|
439
|
+
if start is not None and instant < start:
|
|
440
|
+
continue
|
|
441
|
+
if until is not None and (
|
|
442
|
+
instant > until or (instant == until and not inclusive)):
|
|
443
|
+
continue
|
|
444
|
+
return True
|
|
445
|
+
return False
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
class _Prepared(NamedTuple):
|
|
449
|
+
"""One facts object with every coordinate ordered and every capture
|
|
450
|
+
parsed once: `(op_order, row, captured_instant)` per correction and
|
|
451
|
+
`(commit_order, batch)` per batch (R6)."""
|
|
452
|
+
|
|
453
|
+
rows: tuple
|
|
454
|
+
batches: tuple
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
#: The last facts object prepared, with the key it was prepared under. The
|
|
458
|
+
#: facts are immutable tuples and the reference is held, so an identity
|
|
459
|
+
#: match is the same content; a consumer assessing many lineages against one
|
|
460
|
+
#: facts object (the sweep, the transition rows) parses each capture once.
|
|
461
|
+
_PREPARED_MEMO: tuple = (None, None, None)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _prepared(facts, key) -> _Prepared:
|
|
465
|
+
global _PREPARED_MEMO
|
|
466
|
+
memo_facts, memo_key, memo = _PREPARED_MEMO
|
|
467
|
+
if memo_facts is facts and memo_key is key:
|
|
468
|
+
return memo
|
|
469
|
+
prepared = _Prepared(
|
|
470
|
+
tuple((key(row.op_coord), row, _instant(row.captured_at))
|
|
471
|
+
for row in facts.corrections),
|
|
472
|
+
tuple(sorted(((key(b.commit_coord), b) for b in facts.batches),
|
|
473
|
+
key=lambda item: item[0])))
|
|
474
|
+
_PREPARED_MEMO = (facts, key, prepared)
|
|
475
|
+
return prepared
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def assess(lineage, facts, *, coord_key=None) -> Assessment:
|
|
479
|
+
"""Assess one lineage against the correction facts (spec 5.3).
|
|
480
|
+
|
|
481
|
+
Tainted iff some correction ``(op, obs)`` has a completed batch ``b``
|
|
482
|
+
outside the lineage's basis whose commit follows the op's coordinate,
|
|
483
|
+
whose footprint contains ``('claude', obs.account)``, ``('claude', '*')``
|
|
484
|
+
or ``('*', '*')``, with the observation inside the lineage's scope and
|
|
485
|
+
one of its dependency windows. An `accept` counts like any correction.
|
|
486
|
+
An unresolved reference (no index row) followed by such a batch makes
|
|
487
|
+
the assessment `unavailable`; it is never read as "no correction". A
|
|
488
|
+
None lineage or None facts is `unavailable`. A lineage whose basis
|
|
489
|
+
holds every completed batch is clean without a probe.
|
|
490
|
+
"""
|
|
491
|
+
if lineage is None or facts is None or not isinstance(lineage, dict):
|
|
492
|
+
return UNAVAILABLE
|
|
493
|
+
key = coord_key or _coordinate_order
|
|
494
|
+
basis = frozenset(str(b) for b in (lineage.get("basis") or ()))
|
|
495
|
+
outside = [(commit, frozenset(b.footprint))
|
|
496
|
+
for commit, b in _prepared(facts, key).batches
|
|
497
|
+
if b.batch_id not in basis]
|
|
498
|
+
if not outside:
|
|
499
|
+
return CLEAN
|
|
500
|
+
# The batches committed after an op are a suffix of `outside` in commit
|
|
501
|
+
# order, and both footprint predicates are membership tests, so "some
|
|
502
|
+
# later batch's footprint matches" is "the union of the suffix's
|
|
503
|
+
# footprints matches": one bisection per correction instead of a scan of
|
|
504
|
+
# every batch (R6).
|
|
505
|
+
commits = [commit for commit, _fp in outside]
|
|
506
|
+
suffix: list = [frozenset()] * (len(outside) + 1)
|
|
507
|
+
for i in range(len(outside) - 1, -1, -1):
|
|
508
|
+
suffix[i] = suffix[i + 1] | outside[i][1]
|
|
509
|
+
scope = str(lineage.get("accountScope") or "*")
|
|
510
|
+
windows = _parsed_windows(lineage.get("dependencyWindows") or ())
|
|
511
|
+
tainted: dict = {}
|
|
512
|
+
unresolved = False
|
|
513
|
+
for op_order, row, captured in _prepared(facts, key).rows:
|
|
514
|
+
first = bisect.bisect_right(commits, op_order)
|
|
515
|
+
if first == len(commits):
|
|
516
|
+
continue
|
|
517
|
+
later = suffix[first]
|
|
518
|
+
if row.account_key is None or captured is None:
|
|
519
|
+
if _touches_claude(later):
|
|
520
|
+
unresolved = True
|
|
521
|
+
continue
|
|
522
|
+
if scope != "*" and row.account_key != scope:
|
|
523
|
+
continue
|
|
524
|
+
if not _in_windows(captured, windows):
|
|
525
|
+
continue
|
|
526
|
+
if _touches(later, row.account_key):
|
|
527
|
+
tainted.setdefault(row.op_id, set()).add(row.observation_id)
|
|
528
|
+
if tainted:
|
|
529
|
+
return Assessment("tainted", tuple(
|
|
530
|
+
(op, len(observations)) for op, observations in tainted.items()))
|
|
531
|
+
if unresolved:
|
|
532
|
+
return UNAVAILABLE
|
|
533
|
+
return CLEAN
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# --------------------------------------------------------------------------
|
|
537
|
+
# what a set of lineages needs from the correction rows (R6)
|
|
538
|
+
# --------------------------------------------------------------------------
|
|
539
|
+
|
|
540
|
+
class RowDemand(NamedTuple):
|
|
541
|
+
"""The correction rows a set of lineages can be affected by.
|
|
542
|
+
|
|
543
|
+
`assess` ignores a correction whose op no batch outside the lineage's
|
|
544
|
+
basis touching Claude follows, and a resolved, parseable observation
|
|
545
|
+
outside the lineage's scope or dependency windows. A reader may
|
|
546
|
+
therefore omit every row of an op ordered at or after ``op_before`` (the
|
|
547
|
+
latest such batch over the set), and every resolved row whose capture is
|
|
548
|
+
a parseable instant outside ``[start, until]`` (None is open; ``empty``
|
|
549
|
+
means the set has no window) or, when ``account`` is set, of another
|
|
550
|
+
account. Unresolved and unparseable rows are always kept: the kernel
|
|
551
|
+
reads them as unresolved references. ``op_before`` None keeps every op.
|
|
552
|
+
"""
|
|
553
|
+
|
|
554
|
+
op_before: "tuple | None"
|
|
555
|
+
start: "object | None"
|
|
556
|
+
until: "object | None"
|
|
557
|
+
empty: bool
|
|
558
|
+
account: "str | None"
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
#: Every row: what a consumer that cannot name its lineages in advance (the
|
|
562
|
+
#: sweep, which assesses the state it re-reads under the lock) reads.
|
|
563
|
+
ALL_ROWS = RowDemand(None, None, None, False, None)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def row_demand(lineages, batches, *, coord_key=None) -> "RowDemand | None":
|
|
567
|
+
"""What ``lineages`` need from the correction rows, given every
|
|
568
|
+
completed batch; None when no row can affect any of them (each basis
|
|
569
|
+
holds every batch touching Claude), so the rows need not be read.
|
|
570
|
+
|
|
571
|
+
The facts a reader returns under this demand assess every lineage in
|
|
572
|
+
the set exactly as the full facts do (spec 5.3): the op filter keeps an
|
|
573
|
+
op whenever some lineage has a later outside batch touching Claude (the
|
|
574
|
+
precondition of both a taint and an unresolved verdict), and the window
|
|
575
|
+
and account bounds are the union over the lineages that need rows.
|
|
576
|
+
"""
|
|
577
|
+
key = coord_key or _coordinate_order
|
|
578
|
+
latest = None
|
|
579
|
+
starts: list = []
|
|
580
|
+
untils: list = []
|
|
581
|
+
open_start = open_until = False
|
|
582
|
+
scopes: set = set()
|
|
583
|
+
for lineage in lineages:
|
|
584
|
+
if not isinstance(lineage, dict):
|
|
585
|
+
continue # assessed `unavailable` without any fact
|
|
586
|
+
basis = frozenset(str(b) for b in (lineage.get("basis") or ()))
|
|
587
|
+
commits = [key(b.commit_coord) for b in batches
|
|
588
|
+
if b.batch_id not in basis and _touches_claude(b.footprint)]
|
|
589
|
+
if not commits:
|
|
590
|
+
continue
|
|
591
|
+
newest = max(commits)
|
|
592
|
+
latest = newest if latest is None or newest > latest else latest
|
|
593
|
+
scopes.add(str(lineage.get("accountScope") or "*"))
|
|
594
|
+
for start, until, _inclusive in _parsed_windows(
|
|
595
|
+
lineage.get("dependencyWindows") or ()):
|
|
596
|
+
if start is None:
|
|
597
|
+
open_start = True
|
|
598
|
+
else:
|
|
599
|
+
starts.append(start)
|
|
600
|
+
if until is None:
|
|
601
|
+
open_until = True
|
|
602
|
+
else:
|
|
603
|
+
untils.append(until)
|
|
604
|
+
if latest is None:
|
|
605
|
+
return None
|
|
606
|
+
empty = not (starts or open_start)
|
|
607
|
+
return RowDemand(
|
|
608
|
+
op_before=latest,
|
|
609
|
+
start=None if open_start or not starts else min(starts),
|
|
610
|
+
until=None if open_until or not untils else max(untils),
|
|
611
|
+
empty=empty,
|
|
612
|
+
account=(next(iter(scopes))
|
|
613
|
+
if len(scopes) == 1 and "*" not in scopes else None))
|