cctally 1.115.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/bin/_cctally_cache.py +85 -4
- package/bin/_cctally_completion.py +80 -12
- package/bin/_cctally_core.py +106 -9
- package/bin/_cctally_dashboard.py +6 -9
- package/bin/_cctally_dashboard_envelope.py +26 -32
- package/bin/_cctally_db.py +85 -4
- package/bin/_cctally_doctor.py +38 -5
- package/bin/_cctally_five_hour.py +83 -1017
- package/bin/_cctally_journal.py +1307 -180
- package/bin/_cctally_journal_repair.py +9 -7
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_quarantine.py +14 -4
- package/bin/_cctally_quota.py +70 -8
- package/bin/_cctally_quota_model.py +35 -4
- package/bin/_cctally_record.py +414 -123
- package/bin/_cctally_rederive.py +632 -235
- package/bin/_cctally_refresh.py +29 -0
- package/bin/_cctally_statusline.py +984 -47
- package/bin/_cctally_store.py +972 -58
- package/bin/_cctally_tui.py +62 -7
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_protocol.py +201 -18
- package/bin/_lib_clone_copy.py +46 -11
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +273 -52
- package/bin/_lib_doctor.py +302 -1
- package/bin/_lib_journal.py +145 -25
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +3 -2
- package/bin/_lib_quota_model.py +96 -6
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +299 -3
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +65 -5
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +6 -11
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +7 -2
- package/bin/_lib_merge_gate.py +0 -452
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
|
@@ -75,7 +75,7 @@ def _read_prefix(high_water):
|
|
|
75
75
|
malformed = 0
|
|
76
76
|
prior_high_water = None
|
|
77
77
|
cutover_captured = _journal._CUTOVER_UNSEEN
|
|
78
|
-
hasher =
|
|
78
|
+
hasher = _journal._new_prefix_hasher()
|
|
79
79
|
# #825/#832 (spec 5.3): admission is resolved within this one pass and
|
|
80
80
|
# applied after it, before selection or normalization. An excluded record
|
|
81
81
|
# — an old process's repair acknowledgement or correction batch included —
|
|
@@ -87,7 +87,8 @@ def _read_prefix(high_water):
|
|
|
87
87
|
None,
|
|
88
88
|
high_water,
|
|
89
89
|
segments,
|
|
90
|
-
on_segment=lambda name: hasher.begin_segment(
|
|
90
|
+
on_segment=lambda name, size: hasher.begin_segment(
|
|
91
|
+
name, prior_high_water, size=size),
|
|
91
92
|
on_bytes=hasher.extend,
|
|
92
93
|
):
|
|
93
94
|
record = _lib_journal.decode_line(raw)
|
|
@@ -122,11 +123,12 @@ def _read_prefix(high_water):
|
|
|
122
123
|
coordinates.append((segment, offset))
|
|
123
124
|
prior_high_water = record_end
|
|
124
125
|
prefix_hash = hasher.digest_at(high_water)
|
|
125
|
-
# The accumulator
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
#
|
|
126
|
+
# The accumulator streams its hash and retains no segment bytes (#997
|
|
127
|
+
# D12), but it still holds hash states, its registered boundaries and the
|
|
128
|
+
# segment opener, and nothing after the pass needs them. It is dropped the
|
|
129
|
+
# moment its pass ends, before the normalization loop below, exactly as
|
|
130
|
+
# `rebuild_stats_index` drops it, and before the raise too, so a malformed
|
|
131
|
+
# prefix does not pin it on the traceback while the exception unwinds.
|
|
130
132
|
hasher = None
|
|
131
133
|
if malformed:
|
|
132
134
|
raise _lib_journal.JournalProtocolError(
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""Provider-sample projection glue (#797/#823/#840 spec 4.3, D8 and D9).
|
|
2
|
+
|
|
3
|
+
`claude_oauth_samples` (stats epoch 1019) holds one row per ADMITTED Claude
|
|
4
|
+
api observation carrying a valid `oauth_request_started_at`, materialized
|
|
5
|
+
from the raw journal record by :func:`project_sample` — the one projection
|
|
6
|
+
implementation every derivation path shares:
|
|
7
|
+
|
|
8
|
+
* live ingest and the scratch replays (plain `db rederive`, the derivation-
|
|
9
|
+
completion step) through `_cctally_record._pipeline_claude_usage`, which
|
|
10
|
+
projects every admitted reading before reset/credit detection and the
|
|
11
|
+
snapshot fold decision, so equal readings that the fold deduplicates still
|
|
12
|
+
each produce a sample, and neither the five-hour MAX adjustment nor credit
|
|
13
|
+
suppression or retirement touches a sample;
|
|
14
|
+
* ordinary and correction rebuilds (`_rebuild_stats_index_locked`), which are
|
|
15
|
+
apply-only and never run the pipeline, through :class:`RebuildProjector`
|
|
16
|
+
offered every admitted record of the rebuild's single read pass.
|
|
17
|
+
|
|
18
|
+
Reviewed weekly decisions (`claude_weekly_observation_decision`, its `_v2`
|
|
19
|
+
successor) are projected beside it into `claude_weekly_observation_decisions`
|
|
20
|
+
by the op fold applier :func:`apply_weekly_observation_decision_op`, latest
|
|
21
|
+
decision per observation winning in journal order, so the weekly eligibility
|
|
22
|
+
of a sample is resolved at READ time and a later decision rewrites no sample
|
|
23
|
+
row. A `hold` makes only the WEEKLY sample ineligible; a later effective
|
|
24
|
+
`accept` restores it; five-hour samples are never excluded by a weekly
|
|
25
|
+
decision. (The schema-3 `retract` projects each retraction as an ordinary
|
|
26
|
+
`hold` in `weekly_axis_decisions`, so it needs nothing further here.)
|
|
27
|
+
|
|
28
|
+
Rows are retained: nothing in this work prunes the table. The read is exact-
|
|
29
|
+
window (spec D9) and an unavailable projection raises
|
|
30
|
+
:class:`ProjectionUnavailable` rather than answering "no sample".
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import sqlite3
|
|
35
|
+
from typing import Literal
|
|
36
|
+
|
|
37
|
+
import _lib_oauth_samples
|
|
38
|
+
|
|
39
|
+
SAMPLES_TABLE = "claude_oauth_samples"
|
|
40
|
+
DECISIONS_TABLE = "claude_weekly_observation_decisions"
|
|
41
|
+
#: The reviewed weekly decision kinds and the payload key holding each schema
|
|
42
|
+
#: version's decision list.
|
|
43
|
+
_DECISION_LISTS = {1: "decisions", 2: "weekly_axis_decisions",
|
|
44
|
+
3: "weekly_axis_decisions"}
|
|
45
|
+
_DISPOSITIONS = frozenset(("hold", "accept"))
|
|
46
|
+
|
|
47
|
+
_INSERT = (
|
|
48
|
+
"INSERT OR IGNORE INTO claude_oauth_samples (account_key, observation_id,"
|
|
49
|
+
" requested_at_utc, captured_at_utc, weekly_percent, weekly_resets_at_utc,"
|
|
50
|
+
" five_hour_percent, five_hour_resets_at_utc, five_hour_window_key)"
|
|
51
|
+
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
52
|
+
)
|
|
53
|
+
_COLUMNS = (
|
|
54
|
+
"s.account_key, s.observation_id, s.requested_at_utc, s.captured_at_utc,"
|
|
55
|
+
" s.weekly_percent, s.weekly_resets_at_utc, s.five_hour_percent,"
|
|
56
|
+
" s.five_hour_resets_at_utc, s.five_hour_window_key"
|
|
57
|
+
)
|
|
58
|
+
_READ_WEEKLY = (
|
|
59
|
+
f"SELECT {_COLUMNS} FROM claude_oauth_samples AS s"
|
|
60
|
+
" WHERE s.account_key = ? AND s.weekly_resets_at_utc = ?"
|
|
61
|
+
" AND s.requested_at_utc > ? AND s.weekly_percent IS NOT NULL"
|
|
62
|
+
" AND unixepoch(s.captured_at_utc) < unixepoch(s.weekly_resets_at_utc)"
|
|
63
|
+
" AND NOT EXISTS (SELECT 1 FROM claude_weekly_observation_decisions AS d"
|
|
64
|
+
" WHERE d.observation_id = s.observation_id"
|
|
65
|
+
" AND d.disposition = 'hold')"
|
|
66
|
+
" ORDER BY s.requested_at_utc, s.observation_id"
|
|
67
|
+
)
|
|
68
|
+
_READ_FIVE_HOUR = (
|
|
69
|
+
f"SELECT {_COLUMNS} FROM claude_oauth_samples AS s"
|
|
70
|
+
" WHERE s.account_key = ? AND s.five_hour_window_key = ?"
|
|
71
|
+
" AND s.requested_at_utc > ? AND s.five_hour_percent IS NOT NULL"
|
|
72
|
+
" AND unixepoch(s.captured_at_utc)"
|
|
73
|
+
" < MIN(unixepoch(s.five_hour_resets_at_utc), s.five_hour_window_key)"
|
|
74
|
+
" ORDER BY s.requested_at_utc, s.observation_id"
|
|
75
|
+
)
|
|
76
|
+
_UPSERT_DECISION = (
|
|
77
|
+
"INSERT INTO claude_weekly_observation_decisions (observation_id,"
|
|
78
|
+
" disposition, decision_op_id) VALUES (?, ?, ?)"
|
|
79
|
+
" ON CONFLICT(observation_id) DO UPDATE SET"
|
|
80
|
+
" disposition = excluded.disposition,"
|
|
81
|
+
" decision_op_id = excluded.decision_op_id"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class ProjectionUnavailable(RuntimeError):
|
|
86
|
+
"""The samples projection cannot be read on this connection (absent,
|
|
87
|
+
older index shape): a classification that needs it is unavailable,
|
|
88
|
+
never "no proof"."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def sample_from_record(rec) -> "_lib_oauth_samples.OAuthSample | None":
|
|
92
|
+
"""The sample of one journal record, or None. Only a current-revision
|
|
93
|
+
(`wpr: 2`) Claude usage observation from a recording source can carry a
|
|
94
|
+
sample — the same records `_pipeline_claude_usage` derives — so no key is
|
|
95
|
+
ever honoured on a historical or previous-release record."""
|
|
96
|
+
if not isinstance(rec, dict) or rec.get("t") != "obs":
|
|
97
|
+
return None
|
|
98
|
+
payload = rec.get("payload")
|
|
99
|
+
if (not isinstance(payload, dict)
|
|
100
|
+
or _lib_oauth_samples.REQUEST_STARTED_KEY not in payload
|
|
101
|
+
or rec.get("provider") != "claude"):
|
|
102
|
+
return None
|
|
103
|
+
if "resets_at" not in payload or "weekly_percent" not in payload:
|
|
104
|
+
return None
|
|
105
|
+
import _cctally_record
|
|
106
|
+
import _lib_account_protocol
|
|
107
|
+
import _lib_accounts
|
|
108
|
+
if rec.get("src") not in _cctally_record._CLAUDE_OBS_SRCS:
|
|
109
|
+
return None
|
|
110
|
+
if not _lib_account_protocol.has_current_revision(rec):
|
|
111
|
+
return None
|
|
112
|
+
return _lib_oauth_samples.sample_from_observation(
|
|
113
|
+
payload,
|
|
114
|
+
account_key=rec.get("account") or _lib_accounts.UNATTRIBUTED,
|
|
115
|
+
observation_id=rec.get("id"),
|
|
116
|
+
captured_at_utc=payload.get("captured_at"),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def project_sample(conn, sample: "_lib_oauth_samples.OAuthSample") -> None:
|
|
121
|
+
"""Materialize ``sample`` on ``conn`` (the caller's transaction).
|
|
122
|
+
Idempotent: an observation id is a content id, so a replay of the same
|
|
123
|
+
record inserts nothing."""
|
|
124
|
+
conn.execute(_INSERT, sample.row())
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def project_record(conn, rec) -> None:
|
|
128
|
+
"""Project ``rec``'s sample when it carries one. The caller has already
|
|
129
|
+
admitted ``rec``."""
|
|
130
|
+
sample = sample_from_record(rec)
|
|
131
|
+
if sample is not None:
|
|
132
|
+
project_sample(conn, sample)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class RebuildProjector:
|
|
136
|
+
"""The apply-only rebuild's leg of the projection: its single read pass
|
|
137
|
+
offers every admitted record, and samples are projected into the scratch
|
|
138
|
+
index in bounded batches (one transaction each), so the pass never holds
|
|
139
|
+
the whole sample population in memory."""
|
|
140
|
+
|
|
141
|
+
BATCH = 2048
|
|
142
|
+
|
|
143
|
+
def __init__(self, conn) -> None:
|
|
144
|
+
self._conn = conn
|
|
145
|
+
self._pending: list = []
|
|
146
|
+
self.projected = 0
|
|
147
|
+
|
|
148
|
+
def offer(self, rec) -> None:
|
|
149
|
+
sample = sample_from_record(rec)
|
|
150
|
+
if sample is None:
|
|
151
|
+
return
|
|
152
|
+
self._pending.append(sample)
|
|
153
|
+
if len(self._pending) >= self.BATCH:
|
|
154
|
+
self.flush()
|
|
155
|
+
|
|
156
|
+
def flush(self) -> None:
|
|
157
|
+
if not self._pending:
|
|
158
|
+
return
|
|
159
|
+
conn = self._conn
|
|
160
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
161
|
+
try:
|
|
162
|
+
for sample in self._pending:
|
|
163
|
+
project_sample(conn, sample)
|
|
164
|
+
conn.commit()
|
|
165
|
+
except BaseException:
|
|
166
|
+
conn.rollback()
|
|
167
|
+
raise
|
|
168
|
+
self.projected += len(self._pending)
|
|
169
|
+
self._pending = []
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def apply_weekly_observation_decision_op(conn, record) -> None:
|
|
173
|
+
"""Fold applier for a reviewed weekly decision op: each well-formed
|
|
174
|
+
``{observationId, disposition}`` entry becomes the observation's
|
|
175
|
+
effective decision (latest op wins, in journal order). Malformed entries
|
|
176
|
+
are skipped here; `db rederive` validates every retained decision op and
|
|
177
|
+
refuses a malformed one."""
|
|
178
|
+
payload = record.get("payload") if isinstance(record, dict) else None
|
|
179
|
+
if not isinstance(payload, dict):
|
|
180
|
+
return
|
|
181
|
+
version = payload.get("schema_version")
|
|
182
|
+
key = _DECISION_LISTS.get(version) if type(version) is int else None
|
|
183
|
+
decisions = payload.get(key) if key is not None else None
|
|
184
|
+
if not isinstance(decisions, list):
|
|
185
|
+
return
|
|
186
|
+
op_id = record.get("id")
|
|
187
|
+
if not isinstance(op_id, str):
|
|
188
|
+
return
|
|
189
|
+
for item in decisions:
|
|
190
|
+
if not isinstance(item, dict):
|
|
191
|
+
continue
|
|
192
|
+
observation_id = item.get("observationId")
|
|
193
|
+
disposition = item.get("disposition")
|
|
194
|
+
if (isinstance(observation_id, str) and observation_id
|
|
195
|
+
and disposition in _DISPOSITIONS):
|
|
196
|
+
conn.execute(_UPSERT_DECISION,
|
|
197
|
+
(observation_id, disposition, op_id))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def read_samples(conn, *, account_key: str,
|
|
201
|
+
axis: Literal["weekly", "five_hour"],
|
|
202
|
+
window: "str | int",
|
|
203
|
+
requested_after_utc: str,
|
|
204
|
+
) -> "tuple[_lib_oauth_samples.OAuthSample, ...]":
|
|
205
|
+
"""The account's samples of exactly one authoritative window whose `q` is
|
|
206
|
+
strictly after ``requested_after_utc`` and whose response was captured
|
|
207
|
+
before that window's reset, ordered by `q`.
|
|
208
|
+
|
|
209
|
+
``window`` is the weekly reset (an epoch or a tz-aware ISO instant, raw or
|
|
210
|
+
normalized: the nearest-hour normalization is the identity) for
|
|
211
|
+
``"weekly"``, and the canonical five-hour window key for
|
|
212
|
+
``"five_hour"``. The weekly read excludes an observation whose effective
|
|
213
|
+
reviewed weekly decision is `hold`; the five-hour read never does. The
|
|
214
|
+
capture bound uses the sample's own window (for five hours, the earlier
|
|
215
|
+
of its raw reset and its window key); the caller still compares against
|
|
216
|
+
its member's advertised reset. Raises :class:`ProjectionUnavailable`
|
|
217
|
+
when the projection cannot be read."""
|
|
218
|
+
if not isinstance(account_key, str) or not account_key:
|
|
219
|
+
raise ValueError("read_samples requires an exact account")
|
|
220
|
+
after = _lib_oauth_samples.canonical_instant(requested_after_utc)
|
|
221
|
+
if axis == "weekly":
|
|
222
|
+
sql = _READ_WEEKLY
|
|
223
|
+
key = _lib_oauth_samples.weekly_window(window)
|
|
224
|
+
elif axis == "five_hour":
|
|
225
|
+
if type(window) is not int:
|
|
226
|
+
raise ValueError("a five-hour window is its canonical int key")
|
|
227
|
+
sql = _READ_FIVE_HOUR
|
|
228
|
+
key = window
|
|
229
|
+
else:
|
|
230
|
+
raise ValueError(f"unknown sample axis: {axis!r}")
|
|
231
|
+
try:
|
|
232
|
+
rows = conn.execute(sql, (account_key, key, after)).fetchall()
|
|
233
|
+
except sqlite3.OperationalError as exc:
|
|
234
|
+
raise ProjectionUnavailable(str(exc)) from exc
|
|
235
|
+
return tuple(_lib_oauth_samples.OAuthSample(*row) for row in rows)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
_READ_ONE = (
|
|
239
|
+
f"SELECT {_COLUMNS} FROM claude_oauth_samples AS s"
|
|
240
|
+
" WHERE s.account_key = ? AND s.observation_id = ?"
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def sample_for_observation(conn, *, account_key: str, observation_id
|
|
245
|
+
) -> "_lib_oauth_samples.OAuthSample | None":
|
|
246
|
+
"""The account's admitted sample of exactly one observation, or None when
|
|
247
|
+
that observation is not an admitted sample (or the projection cannot be
|
|
248
|
+
read). #797 spec 4.4: a five-hour credit's causal facts name the
|
|
249
|
+
observations it was armed and baselined from; their request instants
|
|
250
|
+
and RAW values are read here, never from the clamped snapshot column."""
|
|
251
|
+
if not isinstance(account_key, str) or not account_key or observation_id is None:
|
|
252
|
+
return None
|
|
253
|
+
try:
|
|
254
|
+
row = conn.execute(_READ_ONE, (account_key, str(observation_id))).fetchone()
|
|
255
|
+
except sqlite3.Error:
|
|
256
|
+
return None
|
|
257
|
+
return None if row is None else _lib_oauth_samples.OAuthSample(*row)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def projection_covered(conn, *, account_key: str) -> bool:
|
|
261
|
+
"""Whether ``conn`` holds a complete samples projection for
|
|
262
|
+
``account_key``: the index is at this binary's epoch with both
|
|
263
|
+
projection tables. The projection commits in the same transaction as
|
|
264
|
+
the journal cursor on every path, so a current index covers every
|
|
265
|
+
admitted observation up to its cursor for every account; anything else
|
|
266
|
+
(an older or newer index shape, a missing table, an unreadable
|
|
267
|
+
connection) is uncovered."""
|
|
268
|
+
if not isinstance(account_key, str) or not account_key:
|
|
269
|
+
return False
|
|
270
|
+
import _cctally_core
|
|
271
|
+
try:
|
|
272
|
+
epoch = conn.execute("PRAGMA user_version").fetchone()[0]
|
|
273
|
+
if epoch != _cctally_core.STATS_INDEX_EPOCH:
|
|
274
|
+
return False
|
|
275
|
+
names = {str(row[0]) for row in conn.execute(
|
|
276
|
+
"SELECT name FROM sqlite_schema WHERE type = 'table'"
|
|
277
|
+
" AND name IN (?, ?)", (SAMPLES_TABLE, DECISIONS_TABLE))}
|
|
278
|
+
except sqlite3.Error:
|
|
279
|
+
return False
|
|
280
|
+
return names == {SAMPLES_TABLE, DECISIONS_TABLE}
|
package/bin/_cctally_parser.py
CHANGED
|
@@ -1849,6 +1849,11 @@ def _build_record_usage_parser(subparsers, name, *, help_text, xref=None):
|
|
|
1849
1849
|
fi
|
|
1850
1850
|
cctally record-usage $record_args &
|
|
1851
1851
|
fi
|
|
1852
|
+
|
|
1853
|
+
Note: this direct recipe cannot tell an idle session's cached
|
|
1854
|
+
reading from a fresh one after a credit, so an idle session can
|
|
1855
|
+
record a value the credit already retired. `cctally statusline`
|
|
1856
|
+
can tell them apart; prefer it as the status line command.
|
|
1852
1857
|
"""
|
|
1853
1858
|
),
|
|
1854
1859
|
)
|
|
@@ -380,16 +380,26 @@ class QuarantineFold:
|
|
|
380
380
|
kind=TRIGGER_LEGACY_SPOOL,
|
|
381
381
|
reason=str(payload.get("reason") or "changed"),
|
|
382
382
|
segment=segment, offset=offset, at=rec.get("at"))
|
|
383
|
-
elif kind
|
|
383
|
+
elif kind in (_lib_account_protocol.CUTOVER_OP_KIND,
|
|
384
|
+
_lib_account_protocol.REVISION_OP_KIND):
|
|
384
385
|
# Spec revision 5 (section 7, *Before the initial cutover*):
|
|
385
386
|
# a cutover that could not complete its legacy spool
|
|
386
387
|
# inventory records the uncertainty in itself, and is the
|
|
387
388
|
# same unresolved-marker trigger an `inventory-incomplete`
|
|
388
|
-
# op is, at the cutover's own position.
|
|
389
|
+
# op is, at the cutover's own position. #797/#823/#840
|
|
390
|
+
# (spec 4.6 B2): so is a revision-2 boundary whose expanded
|
|
391
|
+
# inventory could not complete.
|
|
392
|
+
expected_id = (
|
|
393
|
+
_lib_account_protocol.CUTOVER_OP_ID
|
|
394
|
+
if kind == _lib_account_protocol.CUTOVER_OP_KIND
|
|
395
|
+
else _lib_account_protocol.REVISION_OP_ID)
|
|
389
396
|
if (_lib_account_protocol.is_stamped(rec)
|
|
390
|
-
and rec.get("id") ==
|
|
397
|
+
and rec.get("id") == expected_id
|
|
391
398
|
and payload.get("legacySpoolInventoryFailure")
|
|
392
|
-
is not None
|
|
399
|
+
is not None
|
|
400
|
+
# A revision op continuing an open failure episode
|
|
401
|
+
# adds no trigger: the episode already has one.
|
|
402
|
+
and payload.get("inventoryEpisode") is None):
|
|
393
403
|
self.state.on_trigger(
|
|
394
404
|
kind=TRIGGER_LEGACY_SPOOL,
|
|
395
405
|
reason=_lib_account_protocol.INVENTORY_INCOMPLETE_REASON,
|
package/bin/_cctally_quota.py
CHANGED
|
@@ -2063,6 +2063,63 @@ def _apply_codex_window_attribution_overlay(
|
|
|
2063
2063
|
))
|
|
2064
2064
|
|
|
2065
2065
|
|
|
2066
|
+
class _QuotaLoadValues:
|
|
2067
|
+
"""The values one `load_codex_quota_observations` call shares (#997 D17).
|
|
2068
|
+
|
|
2069
|
+
A full load materializes every retained observation (266 k on the
|
|
2070
|
+
maintainer's copy, about 2.3 KB each), and every row brought its own copy
|
|
2071
|
+
of each repeated text value, its own parsed reset instants and its own
|
|
2072
|
+
`QuotaWindowIdentity`. Within one call this keeps one object per distinct
|
|
2073
|
+
text value, one parsed instant per distinct stored reset text, and one
|
|
2074
|
+
identity per distinct tuple of ALL its fields. `limit_id` and
|
|
2075
|
+
`limit_name` are outside the identity's equality, so sharing by equality
|
|
2076
|
+
alone would hand one observation another's labels; the tuple includes them
|
|
2077
|
+
(with their types, so a hand-written ``1`` and ``1.0`` stay apart). Every
|
|
2078
|
+
shared value is immutable, so values, equality and order are unchanged.
|
|
2079
|
+
"""
|
|
2080
|
+
|
|
2081
|
+
__slots__ = ("_texts", "_resets", "_identities")
|
|
2082
|
+
|
|
2083
|
+
def __init__(self) -> None:
|
|
2084
|
+
self._texts: dict = {}
|
|
2085
|
+
self._resets: dict = {}
|
|
2086
|
+
self._identities: dict = {}
|
|
2087
|
+
|
|
2088
|
+
def text(self, value):
|
|
2089
|
+
"""``value``'s shared object; a non-string is returned unchanged."""
|
|
2090
|
+
if type(value) is not str:
|
|
2091
|
+
return value
|
|
2092
|
+
return self._texts.setdefault(value, value)
|
|
2093
|
+
|
|
2094
|
+
def reset(self, text: str, label: str) -> dt.datetime:
|
|
2095
|
+
"""`_parse_utc(text, label)`, parsed once per distinct text."""
|
|
2096
|
+
parsed = self._resets.get(text)
|
|
2097
|
+
if parsed is None:
|
|
2098
|
+
parsed = self._resets[text] = _parse_utc(text, label)
|
|
2099
|
+
return parsed
|
|
2100
|
+
|
|
2101
|
+
def identity(self, *, source, source_root_key, account_key,
|
|
2102
|
+
logical_limit_key, observed_slot, window_minutes, limit_id,
|
|
2103
|
+
limit_name) -> QuotaWindowIdentity:
|
|
2104
|
+
key = (source, source_root_key, account_key, logical_limit_key,
|
|
2105
|
+
observed_slot, type(window_minutes), window_minutes,
|
|
2106
|
+
type(limit_id), limit_id, type(limit_name), limit_name)
|
|
2107
|
+
identity = self._identities.get(key)
|
|
2108
|
+
if identity is None:
|
|
2109
|
+
text = self.text
|
|
2110
|
+
identity = self._identities[key] = QuotaWindowIdentity(
|
|
2111
|
+
source=text(source),
|
|
2112
|
+
source_root_key=text(source_root_key),
|
|
2113
|
+
account_key=text(account_key),
|
|
2114
|
+
logical_limit_key=text(logical_limit_key),
|
|
2115
|
+
observed_slot=text(observed_slot),
|
|
2116
|
+
window_minutes=window_minutes,
|
|
2117
|
+
limit_id=text(limit_id),
|
|
2118
|
+
limit_name=text(limit_name),
|
|
2119
|
+
)
|
|
2120
|
+
return identity
|
|
2121
|
+
|
|
2122
|
+
|
|
2066
2123
|
def load_codex_quota_observations(
|
|
2067
2124
|
*,
|
|
2068
2125
|
source_root_keys: Iterable[str] | None = None,
|
|
@@ -2463,6 +2520,9 @@ def load_codex_quota_observations(
|
|
|
2463
2520
|
row_source = _iter_shard_rows(conn, shards)
|
|
2464
2521
|
result: list[QuotaObservation] = []
|
|
2465
2522
|
signature_tuples: dict[str, dict[str, list[tuple[object, ...]]]] = {}
|
|
2523
|
+
# #997 D17: repeated values are shared within this call.
|
|
2524
|
+
values = _QuotaLoadValues()
|
|
2525
|
+
text = values.text
|
|
2466
2526
|
for row in row_source:
|
|
2467
2527
|
if not _codex_quota_required_text_present(
|
|
2468
2528
|
row[name] for name in _CODEX_QUOTA_REQUIRED_TEXT
|
|
@@ -2498,7 +2558,7 @@ def load_codex_quota_observations(
|
|
|
2498
2558
|
str(raw_account) if raw_account not in (None, "")
|
|
2499
2559
|
else _lib_accounts.UNATTRIBUTED
|
|
2500
2560
|
)
|
|
2501
|
-
identity =
|
|
2561
|
+
identity = values.identity(
|
|
2502
2562
|
source=str(row["source"]),
|
|
2503
2563
|
source_root_key=str(row["source_root_key"]),
|
|
2504
2564
|
account_key=account_key,
|
|
@@ -2512,16 +2572,18 @@ def load_codex_quota_observations(
|
|
|
2512
2572
|
identity=identity,
|
|
2513
2573
|
captured_at=_parse_utc(str(row["captured_at_utc"]), "captured_at_utc"),
|
|
2514
2574
|
used_percent=float(row["used_percent"]),
|
|
2515
|
-
resets_at=
|
|
2516
|
-
|
|
2575
|
+
resets_at=values.reset(
|
|
2576
|
+
str(row["resets_at_utc"]), "resets_at_utc"),
|
|
2577
|
+
source_path=text(str(row["source_path"])),
|
|
2517
2578
|
line_offset=int(row["line_offset"]),
|
|
2518
|
-
plan_type=row["plan_type"],
|
|
2519
|
-
individual_limit_json=row["individual_limit_json"],
|
|
2520
|
-
reached_type=row["reached_type"],
|
|
2579
|
+
plan_type=text(row["plan_type"]),
|
|
2580
|
+
individual_limit_json=text(row["individual_limit_json"]),
|
|
2581
|
+
reached_type=text(row["reached_type"]),
|
|
2521
2582
|
canonical_resets_at=(
|
|
2522
2583
|
None if row["canonical_resets_at_utc"] in (None, "")
|
|
2523
|
-
else
|
|
2524
|
-
|
|
2584
|
+
else values.reset(
|
|
2585
|
+
str(row["canonical_resets_at_utc"]),
|
|
2586
|
+
"canonical_resets_at_utc")
|
|
2525
2587
|
),
|
|
2526
2588
|
)
|
|
2527
2589
|
except (TypeError, ValueError, OverflowError):
|
|
@@ -1211,9 +1211,22 @@ def stored_regimes(state: dict, account_key) -> list:
|
|
|
1211
1211
|
# breaks the digest, so it reads as legacy too.
|
|
1212
1212
|
_REGIME_WRITER_KEY = "wpg"
|
|
1213
1213
|
_REGIME_WRITER_DIGEST_KEY = "wpgDigest"
|
|
1214
|
+
# #797/#823/#840 spec 4.6 B5: recovery also requires revision-specific
|
|
1215
|
+
# provenance. A revision-1 process (the previous release) writes a regime
|
|
1216
|
+
# with a verifying `wpg` stamp, so that stamp alone cannot tell its
|
|
1217
|
+
# transitions from this binary's. Every regime this binary writes also
|
|
1218
|
+
# carries `wpr` and a revision-specific digest over its content without any
|
|
1219
|
+
# of the four writer keys; a previous-release in-place `update()` keeps
|
|
1220
|
+
# `wpr` but cannot recompute that digest.
|
|
1221
|
+
_REGIME_REVISION_KEY = "wpr"
|
|
1222
|
+
_REGIME_REVISION_DIGEST_KEY = "wprDigest"
|
|
1223
|
+
_REGIME_WRITER_KEYS = (_REGIME_WRITER_KEY, _REGIME_WRITER_DIGEST_KEY,
|
|
1224
|
+
_REGIME_REVISION_KEY, _REGIME_REVISION_DIGEST_KEY)
|
|
1214
1225
|
|
|
1215
1226
|
|
|
1216
1227
|
def _regime_content_digest(regime: dict) -> str:
|
|
1228
|
+
# The revision-1 digest, unchanged: over everything but the two `wpg`
|
|
1229
|
+
# keys, so the previous release still verifies this binary's stamp.
|
|
1217
1230
|
core = {k: v for k, v in regime.items()
|
|
1218
1231
|
if k not in (_REGIME_WRITER_KEY, _REGIME_WRITER_DIGEST_KEY)}
|
|
1219
1232
|
data = json.dumps(core, sort_keys=True, separators=(",", ":"),
|
|
@@ -1221,11 +1234,24 @@ def _regime_content_digest(regime: dict) -> str:
|
|
|
1221
1234
|
return "sha256:" + hashlib.sha256(data.encode("utf-8")).hexdigest()
|
|
1222
1235
|
|
|
1223
1236
|
|
|
1237
|
+
def _regime_revision_digest(regime: dict) -> str:
|
|
1238
|
+
import _lib_account_protocol
|
|
1239
|
+
core = {k: v for k, v in regime.items() if k not in _REGIME_WRITER_KEYS}
|
|
1240
|
+
data = json.dumps(
|
|
1241
|
+
{"revision": _lib_account_protocol.PROTOCOL_REVISION,
|
|
1242
|
+
"regime": core},
|
|
1243
|
+
sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
1244
|
+
default=str)
|
|
1245
|
+
return "sha256:" + hashlib.sha256(data.encode("utf-8")).hexdigest()
|
|
1246
|
+
|
|
1247
|
+
|
|
1224
1248
|
def stamp_regime(regime: dict) -> dict:
|
|
1225
|
-
"""``regime`` stamped as written by the current protocol generation
|
|
1249
|
+
"""``regime`` stamped as written by the current protocol generation and
|
|
1250
|
+
revision."""
|
|
1226
1251
|
import _lib_account_protocol
|
|
1227
|
-
out = {k: v for k, v in regime.items()
|
|
1228
|
-
|
|
1252
|
+
out = {k: v for k, v in regime.items() if k not in _REGIME_WRITER_KEYS}
|
|
1253
|
+
out[_REGIME_REVISION_KEY] = _lib_account_protocol.PROTOCOL_REVISION
|
|
1254
|
+
out[_REGIME_REVISION_DIGEST_KEY] = _regime_revision_digest(out)
|
|
1229
1255
|
out[_REGIME_WRITER_KEY] = _lib_account_protocol.PROTOCOL_GENERATION
|
|
1230
1256
|
out[_REGIME_WRITER_DIGEST_KEY] = _regime_content_digest(out)
|
|
1231
1257
|
return out
|
|
@@ -1237,7 +1263,12 @@ def regime_writer_is_current(regime) -> bool:
|
|
|
1237
1263
|
and regime.get(_REGIME_WRITER_KEY)
|
|
1238
1264
|
== _lib_account_protocol.PROTOCOL_GENERATION
|
|
1239
1265
|
and regime.get(_REGIME_WRITER_DIGEST_KEY)
|
|
1240
|
-
== _regime_content_digest(regime)
|
|
1266
|
+
== _regime_content_digest(regime)
|
|
1267
|
+
and _lib_account_protocol.has_current_revision(
|
|
1268
|
+
{_lib_account_protocol.REVISION_KEY:
|
|
1269
|
+
regime.get(_REGIME_REVISION_KEY)})
|
|
1270
|
+
and regime.get(_REGIME_REVISION_DIGEST_KEY)
|
|
1271
|
+
== _regime_revision_digest(regime))
|
|
1241
1272
|
|
|
1242
1273
|
|
|
1243
1274
|
def stamp_current_writes(stored: dict, reduced: dict, account_key) -> dict:
|