cctally 1.65.0 → 1.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/bin/_cctally_alerts.py +1 -1
- package/bin/_cctally_cache.py +203 -22
- package/bin/_cctally_cache_report.py +7 -5
- package/bin/_cctally_core.py +49 -14
- package/bin/_cctally_dashboard.py +481 -4891
- package/bin/_cctally_dashboard_cache_report.py +714 -0
- package/bin/_cctally_dashboard_conversation.py +771 -0
- package/bin/_cctally_dashboard_envelope.py +1520 -0
- package/bin/_cctally_dashboard_share.py +2012 -0
- package/bin/_cctally_db.py +127 -5
- package/bin/_cctally_doctor.py +104 -3
- package/bin/_cctally_five_hour.py +2 -1
- package/bin/_cctally_forecast.py +78 -135
- package/bin/_cctally_parser.py +919 -784
- package/bin/_cctally_percent_breakdown.py +1 -1
- package/bin/_cctally_pricing_check.py +39 -0
- package/bin/_cctally_project.py +8 -5
- package/bin/_cctally_record.py +279 -274
- package/bin/_cctally_reporting.py +1 -1
- package/bin/_cctally_sync_week.py +1 -1
- package/bin/_cctally_telemetry.py +3 -5
- package/bin/_cctally_tui.py +42 -18
- package/bin/_lib_alert_dispatch.py +1 -1
- package/bin/_lib_blocks.py +6 -1
- package/bin/_lib_credit.py +133 -0
- package/bin/_lib_doctor.py +150 -1
- package/bin/_lib_five_hour.py +34 -0
- package/bin/_lib_forecast.py +144 -0
- package/bin/_lib_json_envelope.py +38 -0
- package/bin/_lib_jsonl.py +115 -73
- package/bin/_lib_log.py +96 -0
- package/bin/_lib_pricing.py +47 -1
- package/bin/_lib_pricing_check.py +25 -3
- package/bin/_lib_record.py +179 -0
- package/bin/_lib_render.py +18 -10
- package/bin/_lib_snapshot_cache.py +61 -0
- package/bin/cctally +51 -7
- package/bin/cctally-dashboard +2 -2
- package/bin/cctally-statusline +1 -0
- package/bin/cctally-tui +2 -2
- package/package.json +10 -1
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Pure JSON wire-format helpers: schemaVersion envelope + canonical UTC-Z serializer.
|
|
2
|
+
|
|
3
|
+
stamp_schema_version() is the single chokepoint for the additive camelCase
|
|
4
|
+
``schemaVersion: 1`` envelope adopted across reporting ``--json`` surfaces
|
|
5
|
+
(#279 S6 W1; convention: docs/cli-contract.md). _iso_z() is the canonical
|
|
6
|
+
None-safe seconds-precision UTC-Z serializer (the forecast/dashboard-envelope
|
|
7
|
+
behavior; _lib_doctor._iso_z deliberately diverges — see its docstring).
|
|
8
|
+
|
|
9
|
+
Pure kernel: stdlib-only, no cctally back-import (split design §5.3).
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import datetime as dt
|
|
14
|
+
|
|
15
|
+
SCHEMA_VERSION_KEY = "schemaVersion"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def stamp_schema_version(payload: dict, *, version: int = 1,
|
|
19
|
+
key: str = SCHEMA_VERSION_KEY) -> dict:
|
|
20
|
+
"""Return a new dict with ``key: version`` first.
|
|
21
|
+
|
|
22
|
+
Always returns a shallow copy; never mutates ``payload``. If ``key`` is
|
|
23
|
+
already present the copy preserves the existing value AND key order
|
|
24
|
+
(value- and order-idempotent no-op). Formatting is the caller's concern:
|
|
25
|
+
this returns a dict, each emitter keeps its own json.dumps arguments.
|
|
26
|
+
"""
|
|
27
|
+
if key in payload:
|
|
28
|
+
return dict(payload)
|
|
29
|
+
out = {key: version}
|
|
30
|
+
out.update(payload)
|
|
31
|
+
return out
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _iso_z(d: "dt.datetime | None") -> "str | None":
|
|
35
|
+
"""None-safe UTC ISO-8601 with Z suffix, seconds precision."""
|
|
36
|
+
if d is None:
|
|
37
|
+
return None
|
|
38
|
+
return d.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
package/bin/_lib_jsonl.py
CHANGED
|
@@ -25,7 +25,7 @@ import json
|
|
|
25
25
|
import pathlib
|
|
26
26
|
import re
|
|
27
27
|
import sys
|
|
28
|
-
from dataclasses import dataclass
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
29
|
from typing import Any
|
|
30
30
|
|
|
31
31
|
|
|
@@ -109,6 +109,13 @@ def _parse_usage_entries(
|
|
|
109
109
|
) -> list[UsageEntry]:
|
|
110
110
|
"""Parse one JSONL file's assistant entries within [range_start, range_end].
|
|
111
111
|
|
|
112
|
+
Per-line gating is `parse_cost_entry`'s — the SINGLE implementation shared
|
|
113
|
+
with the cache-ingest path (#279 S3 F2). This fallback (used when cache.db
|
|
114
|
+
can't be opened) no longer re-implements the type -> timestamp -> usage ->
|
|
115
|
+
model -> synthetic -> costUSD gating inline, so the two paths can never
|
|
116
|
+
drift. The only fallback-specific step is the range filter, applied to the
|
|
117
|
+
constructed `entry.timestamp` (inclusive bounds preserved).
|
|
118
|
+
|
|
112
119
|
Dedup contract (matches ccusage's `push_deduped_entry`):
|
|
113
120
|
- Entries with non-null (msg_id, req_id) go into `dedupe_map`; if a key
|
|
114
121
|
already maps to an entry, replace iff `_should_replace(candidate, existing)`.
|
|
@@ -117,13 +124,14 @@ def _parse_usage_entries(
|
|
|
117
124
|
land in a separate list — partial UNIQUE index on the cache mirrors
|
|
118
125
|
this behavior.
|
|
119
126
|
- `<synthetic>` model rows are dropped entirely (matches ccusage's
|
|
120
|
-
claude_loader.rs:454).
|
|
127
|
+
claude_loader.rs:454) — `parse_cost_entry` returns None for them.
|
|
121
128
|
|
|
122
129
|
Caller is responsible for sorting the returned list by timestamp if
|
|
123
130
|
needed; `_collect_entries_direct` does this once across all files
|
|
124
131
|
after flattening `dedupe_map.values()`.
|
|
125
132
|
"""
|
|
126
133
|
no_key_entries: list[UsageEntry] = []
|
|
134
|
+
path_str = str(jsonl_path)
|
|
127
135
|
try:
|
|
128
136
|
with open(jsonl_path, "r", encoding="utf-8", errors="replace") as fh:
|
|
129
137
|
for line in fh:
|
|
@@ -134,60 +142,12 @@ def _parse_usage_entries(
|
|
|
134
142
|
obj = json.loads(line)
|
|
135
143
|
except json.JSONDecodeError:
|
|
136
144
|
continue
|
|
137
|
-
|
|
138
|
-
if
|
|
139
|
-
continue
|
|
140
|
-
|
|
141
|
-
ts_raw = obj.get("timestamp")
|
|
142
|
-
if not isinstance(ts_raw, str) or not ts_raw.strip():
|
|
143
|
-
continue
|
|
144
|
-
|
|
145
|
-
msg = obj.get("message")
|
|
146
|
-
if not isinstance(msg, dict):
|
|
147
|
-
msg = obj
|
|
148
|
-
|
|
149
|
-
usage = msg.get("usage")
|
|
150
|
-
if not isinstance(usage, dict):
|
|
151
|
-
continue
|
|
152
|
-
|
|
153
|
-
model = msg.get("model") or obj.get("model")
|
|
154
|
-
if not isinstance(model, str) or not model.strip():
|
|
155
|
-
continue
|
|
156
|
-
model = model.strip()
|
|
157
|
-
if model == "<synthetic>":
|
|
158
|
-
# Matches ccusage's claude_loader.rs:454 — synthetic
|
|
159
|
-
# placeholder rows carry no billable usage.
|
|
145
|
+
parsed = parse_cost_entry(obj, path_str)
|
|
146
|
+
if parsed is None:
|
|
160
147
|
continue
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
ts = dt.datetime.fromisoformat(
|
|
164
|
-
ts_raw.strip().replace("Z", "+00:00")
|
|
165
|
-
)
|
|
166
|
-
if ts.tzinfo is None:
|
|
167
|
-
ts = ts.replace(tzinfo=dt.timezone.utc)
|
|
168
|
-
except ValueError:
|
|
169
|
-
continue
|
|
170
|
-
|
|
171
|
-
if ts < range_start or ts > range_end:
|
|
148
|
+
entry, msg_id, req_id = parsed
|
|
149
|
+
if entry.timestamp < range_start or entry.timestamp > range_end:
|
|
172
150
|
continue
|
|
173
|
-
|
|
174
|
-
msg_id = msg.get("id")
|
|
175
|
-
req_id = obj.get("requestId")
|
|
176
|
-
cost_usd_raw = obj.get("costUSD")
|
|
177
|
-
cost_usd = (
|
|
178
|
-
float(cost_usd_raw)
|
|
179
|
-
if cost_usd_raw is not None
|
|
180
|
-
else None
|
|
181
|
-
)
|
|
182
|
-
|
|
183
|
-
entry = UsageEntry(
|
|
184
|
-
timestamp=ts,
|
|
185
|
-
model=model,
|
|
186
|
-
usage=usage,
|
|
187
|
-
cost_usd=cost_usd,
|
|
188
|
-
source_path=str(jsonl_path),
|
|
189
|
-
)
|
|
190
|
-
|
|
191
151
|
if msg_id is None or req_id is None:
|
|
192
152
|
no_key_entries.append(entry)
|
|
193
153
|
continue
|
|
@@ -203,24 +163,25 @@ def _parse_usage_entries(
|
|
|
203
163
|
return no_key_entries
|
|
204
164
|
|
|
205
165
|
|
|
206
|
-
|
|
207
|
-
"""Pure per-line cost parser: given a parsed JSONL object, return
|
|
208
|
-
``(UsageEntry, msg_id, req_id)`` when it is a billable assistant entry, or
|
|
209
|
-
``None`` otherwise (non-assistant, missing/invalid usage, model, or
|
|
210
|
-
timestamp, or a ``<synthetic>`` placeholder). No I/O, no byte offset — the
|
|
211
|
-
caller owns the readline()+tell() loop.
|
|
166
|
+
_DELIBERATE_SKIP_REASONS = ("not-assistant", "synthetic")
|
|
212
167
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
168
|
+
|
|
169
|
+
def _classify_cost_entry(obj, path_str: str):
|
|
170
|
+
"""Reason-returning core of ``parse_cost_entry`` (#279 S2 F1). Returns
|
|
171
|
+
``(parsed, reason)`` where exactly one is non-None; ``parsed`` is the
|
|
172
|
+
``(UsageEntry, msg_id, req_id)`` tuple. Reasons: ``not-assistant`` /
|
|
173
|
+
``synthetic`` (deliberate skips) and the drift trio ``bad-timestamp`` /
|
|
174
|
+
``no-usage`` / ``no-model``. The gating ORDER is the contract — it must
|
|
175
|
+
stay identical to the pre-#279 ``parse_cost_entry`` body (type ->
|
|
176
|
+
raw-timestamp -> usage -> model -> synthetic -> timestamp-parse) so the
|
|
177
|
+
public wrapper's behavior is unchanged.
|
|
217
178
|
"""
|
|
218
179
|
if obj.get("type") != "assistant":
|
|
219
|
-
return None
|
|
180
|
+
return None, "not-assistant"
|
|
220
181
|
|
|
221
182
|
ts_raw = obj.get("timestamp")
|
|
222
183
|
if not isinstance(ts_raw, str) or not ts_raw.strip():
|
|
223
|
-
return None
|
|
184
|
+
return None, "bad-timestamp"
|
|
224
185
|
|
|
225
186
|
msg = obj.get("message")
|
|
226
187
|
if not isinstance(msg, dict):
|
|
@@ -228,29 +189,35 @@ def parse_cost_entry(obj, path_str: str):
|
|
|
228
189
|
|
|
229
190
|
usage = msg.get("usage")
|
|
230
191
|
if not isinstance(usage, dict):
|
|
231
|
-
return None
|
|
192
|
+
return None, "no-usage"
|
|
232
193
|
|
|
233
194
|
model = msg.get("model") or obj.get("model")
|
|
234
195
|
if not isinstance(model, str) or not model.strip():
|
|
235
|
-
return None
|
|
196
|
+
return None, "no-model"
|
|
236
197
|
model = model.strip()
|
|
237
198
|
if model == "<synthetic>":
|
|
238
199
|
# Matches ccusage's claude_loader.rs:454. Filtered here so the cache
|
|
239
200
|
# ingest path can't accidentally store these rows even if a downstream
|
|
240
201
|
# loop forgets to double-check (see `sync_cache` in _cctally_cache.py).
|
|
241
|
-
return None
|
|
202
|
+
return None, "synthetic"
|
|
242
203
|
|
|
243
204
|
try:
|
|
244
205
|
ts = dt.datetime.fromisoformat(ts_raw.strip().replace("Z", "+00:00"))
|
|
245
206
|
if ts.tzinfo is None:
|
|
246
207
|
ts = ts.replace(tzinfo=dt.timezone.utc)
|
|
247
208
|
except ValueError:
|
|
248
|
-
return None
|
|
209
|
+
return None, "bad-timestamp"
|
|
249
210
|
|
|
250
211
|
msg_id = msg.get("id")
|
|
251
212
|
req_id = obj.get("requestId")
|
|
252
213
|
cost_usd_raw = obj.get("costUSD")
|
|
253
|
-
|
|
214
|
+
try:
|
|
215
|
+
cost_usd = float(cost_usd_raw) if cost_usd_raw is not None else None
|
|
216
|
+
except (TypeError, ValueError):
|
|
217
|
+
# Drift-hardened (#279 S3, Codex gate F1): a malformed costUSD must
|
|
218
|
+
# not abort a whole sync/read — degrade to "no raw cost"; the
|
|
219
|
+
# token-derived cost still computes at query time.
|
|
220
|
+
cost_usd = None
|
|
254
221
|
|
|
255
222
|
return (
|
|
256
223
|
UsageEntry(
|
|
@@ -262,7 +229,41 @@ def parse_cost_entry(obj, path_str: str):
|
|
|
262
229
|
),
|
|
263
230
|
msg_id,
|
|
264
231
|
req_id,
|
|
265
|
-
)
|
|
232
|
+
), None
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def parse_cost_entry(obj, path_str: str):
|
|
236
|
+
"""Pure per-line cost parser: given a parsed JSONL object, return
|
|
237
|
+
``(UsageEntry, msg_id, req_id)`` when it is a billable assistant entry, or
|
|
238
|
+
``None`` otherwise (non-assistant, missing/invalid usage, model, or
|
|
239
|
+
timestamp, or a ``<synthetic>`` placeholder). No I/O, no byte offset — the
|
|
240
|
+
caller owns the readline()+tell() loop.
|
|
241
|
+
|
|
242
|
+
Extracted (#138) so the streaming ``_iter_jsonl_entries_with_offsets`` reader
|
|
243
|
+
and the fused single-pass sync walker (``_cctally_cache._iter_sync_entries``)
|
|
244
|
+
share ONE gating implementation — each JSONL line is ``json.loads``-parsed
|
|
245
|
+
once and classified once, never re-parsed for a separate second walk.
|
|
246
|
+
|
|
247
|
+
The body now delegates to ``_classify_cost_entry`` (#279 S2 F1) so the
|
|
248
|
+
parse-health drift classifier and this cost parser can never disagree on
|
|
249
|
+
the gating order; the public contract (returns the tuple or ``None``) is
|
|
250
|
+
unchanged.
|
|
251
|
+
"""
|
|
252
|
+
parsed, _reason = _classify_cost_entry(obj, path_str)
|
|
253
|
+
return parsed
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def assistant_skip_reason(obj) -> "str | None":
|
|
257
|
+
"""Drift classifier for parse-health counters (#279 S2 F1): the reason
|
|
258
|
+
a line that LOOKS like an assistant cost entry was skipped, or None
|
|
259
|
+
when the skip is deliberate (non-assistant, ``<synthetic>``) or the
|
|
260
|
+
line parses fine. Callers invoke this only for lines where
|
|
261
|
+
``parse_cost_entry`` returned None, so the double-classify cost is
|
|
262
|
+
bounded to skipped lines."""
|
|
263
|
+
parsed, reason = _classify_cost_entry(obj, "")
|
|
264
|
+
if parsed is not None or reason in _DELIBERATE_SKIP_REASONS:
|
|
265
|
+
return None
|
|
266
|
+
return reason
|
|
266
267
|
|
|
267
268
|
|
|
268
269
|
def _iter_jsonl_entries_with_offsets(fh, path_str: str):
|
|
@@ -319,7 +320,27 @@ class _CodexIterState:
|
|
|
319
320
|
"""
|
|
320
321
|
session_id: str | None = None
|
|
321
322
|
model: str | None = None
|
|
323
|
+
# #279 S3 F1: the REAL dedup watermark. Seeded by the caller (non-zero
|
|
324
|
+
# wins over the initial_total_tokens kwarg) and STAMPED by the iterator on
|
|
325
|
+
# every yield to the cumulative `total_token_usage.total_tokens` the guard
|
|
326
|
+
# admitted — so after the iterator drains this equals the guard's terminal
|
|
327
|
+
# watermark by construction, and the caller persists exactly it (replacing
|
|
328
|
+
# the old reconstructed initial+Σ(per-turn) sum, which could diverge).
|
|
322
329
|
total_tokens: int = 0
|
|
330
|
+
# #279 S2 F1 parse-health counters — per-iterator-call; sync_codex_cache
|
|
331
|
+
# folds them into CodexIngestStats after each file drains. Reason
|
|
332
|
+
# vocabulary: info-non-dict / no-last-token-usage / bad-timestamp /
|
|
333
|
+
# no-session-id. Rate-limit-only events (info None) and cumulative-dedup
|
|
334
|
+
# re-emissions are NORMAL and never counted.
|
|
335
|
+
lines_seen: int = 0
|
|
336
|
+
lines_malformed: int = 0
|
|
337
|
+
token_events_skipped: int = 0
|
|
338
|
+
skip_reasons: dict = field(default_factory=dict)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _codex_skip(state: "_CodexIterState", reason: str) -> None:
|
|
342
|
+
state.token_events_skipped += 1
|
|
343
|
+
state.skip_reasons[reason] = state.skip_reasons.get(reason, 0) + 1
|
|
323
344
|
|
|
324
345
|
|
|
325
346
|
def _iter_codex_jsonl_entries_with_offsets(
|
|
@@ -375,7 +396,14 @@ def _iter_codex_jsonl_entries_with_offsets(
|
|
|
375
396
|
state.session_id = initial_session_id
|
|
376
397
|
if state.model is None and initial_model is not None:
|
|
377
398
|
state.model = initial_model
|
|
378
|
-
|
|
399
|
+
# #279 S3 F1: the state carries the REAL dedup watermark. Seeding
|
|
400
|
+
# precedence mirrors session_id/model: a caller-supplied NON-ZERO state
|
|
401
|
+
# wins; the kwarg seeds otherwise. (total_tokens has no None sentinel —
|
|
402
|
+
# 0 IS the unset value; a genuine 0 watermark and "unset" are the same
|
|
403
|
+
# thing: both mean "dedupe against nothing".)
|
|
404
|
+
if state.total_tokens == 0 and initial_total_tokens:
|
|
405
|
+
state.total_tokens = int(initial_total_tokens)
|
|
406
|
+
last_total_tokens: int = state.total_tokens
|
|
379
407
|
# Suppress the filename-UUID fallback warning when we already have a
|
|
380
408
|
# seeded session_id (delta resume path). Without this, every resume
|
|
381
409
|
# into a slice of the file that doesn't re-observe session_meta would
|
|
@@ -395,9 +423,11 @@ def _iter_codex_jsonl_entries_with_offsets(
|
|
|
395
423
|
stripped = line.strip()
|
|
396
424
|
if not stripped:
|
|
397
425
|
continue
|
|
426
|
+
state.lines_seen += 1 # #279 S2 F1: non-blank lines (malformed included)
|
|
398
427
|
try:
|
|
399
428
|
obj = json.loads(stripped)
|
|
400
429
|
except json.JSONDecodeError:
|
|
430
|
+
state.lines_malformed += 1
|
|
401
431
|
continue
|
|
402
432
|
|
|
403
433
|
rtype = obj.get("type")
|
|
@@ -421,10 +451,14 @@ def _iter_codex_jsonl_entries_with_offsets(
|
|
|
421
451
|
if payload.get("type") != "token_count":
|
|
422
452
|
continue
|
|
423
453
|
info = payload.get("info")
|
|
454
|
+
if info is None:
|
|
455
|
+
continue # rate-limit-only event — normal, not drift
|
|
424
456
|
if not isinstance(info, dict):
|
|
457
|
+
_codex_skip(state, "info-non-dict")
|
|
425
458
|
continue
|
|
426
459
|
ltu = info.get("last_token_usage")
|
|
427
460
|
if not isinstance(ltu, dict):
|
|
461
|
+
_codex_skip(state, "no-last-token-usage")
|
|
428
462
|
continue
|
|
429
463
|
|
|
430
464
|
# Dedupe re-emitted token_count events. Codex re-emits `last_token_usage`
|
|
@@ -445,12 +479,14 @@ def _iter_codex_jsonl_entries_with_offsets(
|
|
|
445
479
|
|
|
446
480
|
ts_raw = obj.get("timestamp")
|
|
447
481
|
if not isinstance(ts_raw, str) or not ts_raw.strip():
|
|
482
|
+
_codex_skip(state, "bad-timestamp")
|
|
448
483
|
continue
|
|
449
484
|
try:
|
|
450
485
|
ts = dt.datetime.fromisoformat(ts_raw.strip().replace("Z", "+00:00"))
|
|
451
486
|
if ts.tzinfo is None:
|
|
452
487
|
ts = ts.replace(tzinfo=dt.timezone.utc)
|
|
453
488
|
except ValueError:
|
|
489
|
+
_codex_skip(state, "bad-timestamp")
|
|
454
490
|
continue
|
|
455
491
|
|
|
456
492
|
session_id = state.session_id
|
|
@@ -464,6 +500,7 @@ def _iter_codex_jsonl_entries_with_offsets(
|
|
|
464
500
|
filename_session_id_warned = True
|
|
465
501
|
if session_id is None:
|
|
466
502
|
# No session_meta and no parseable filename UUID — skip row.
|
|
503
|
+
_codex_skip(state, "no-session-id")
|
|
467
504
|
continue
|
|
468
505
|
|
|
469
506
|
model = state.model or "unknown"
|
|
@@ -491,5 +528,10 @@ def _iter_codex_jsonl_entries_with_offsets(
|
|
|
491
528
|
)
|
|
492
529
|
# Advance the cumulative watermark only after a successful yield so
|
|
493
530
|
# resume-from-offset continues to dedupe against the last counted turn.
|
|
531
|
+
# #279 S3 F1: stamp the state too — after the iterator drains,
|
|
532
|
+
# state.total_tokens IS the guard's terminal watermark, which the
|
|
533
|
+
# caller persists (the old reconstructed initial+Σ(per-turn) sum could
|
|
534
|
+
# diverge from it and double-count/skip on resume).
|
|
494
535
|
if isinstance(ttu, dict) and cumulative is not None:
|
|
495
536
|
last_total_tokens = cumulative
|
|
537
|
+
state.total_tokens = cumulative
|
package/bin/_lib_log.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Stdlib-logging chokepoint (issue #279 S2, F2).
|
|
2
|
+
|
|
3
|
+
One place decides backend verbosity: ``CCTALLY_DEBUG`` gates DEBUG-level
|
|
4
|
+
stderr output (tracebacks via ``exc_info=True``); the default posture is
|
|
5
|
+
WARNING+ only, so adopters like the dashboard ``log_error`` surface
|
|
6
|
+
errors without any env flag. Pure leaf — no sibling imports (mirrors
|
|
7
|
+
``_lib_perf``'s env handling). The falsey set matches
|
|
8
|
+
``_cctally_core._truthy_env`` EXACTLY (pinned by tests/test_lib_log.py,
|
|
9
|
+
not by import coupling): unset/empty/``0``/``false``/``no`` are off,
|
|
10
|
+
anything else is on. Deliberately NOT ``_lib_perf._FALSEY`` (which also
|
|
11
|
+
treats ``off`` as false) — #279 S2 Codex review P2-5.
|
|
12
|
+
|
|
13
|
+
Optional file sink: ``CCTALLY_DEBUG_LOG=<path>`` attaches an append-mode
|
|
14
|
+
FileHandler at that explicit path (no LOG_DIR default — the module stays
|
|
15
|
+
leaf-pure). Handler-attach failures degrade silently to stderr-only.
|
|
16
|
+
|
|
17
|
+
The stderr handler resolves ``sys.stderr`` at EMIT time (late binding) so
|
|
18
|
+
pytest capsys replacement and hook-tick's dup2 stdio redirect both see
|
|
19
|
+
records; a handler bound at configure time would write to a dead stream.
|
|
20
|
+
|
|
21
|
+
The configure-once latch lives on ``logging.getLogger("cctally").handlers``
|
|
22
|
+
(process-global), NOT on a module global — a SourceFileLoader re-import of
|
|
23
|
+
this module must not attach a second handler and double-print.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import logging
|
|
28
|
+
import os
|
|
29
|
+
import sys
|
|
30
|
+
|
|
31
|
+
_FALSEY = ("", "0", "false", "no")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _env_truthy(name: str) -> bool:
|
|
35
|
+
v = os.environ.get(name)
|
|
36
|
+
return v is not None and v.strip().lower() not in _FALSEY
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
_DEBUG = _env_truthy("CCTALLY_DEBUG")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def debug_enabled() -> bool:
|
|
43
|
+
return _DEBUG
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def set_debug(value: bool) -> None:
|
|
47
|
+
"""Flip debug at runtime (tests). Re-levels an already-configured
|
|
48
|
+
logger so a latch created before the flip follows it."""
|
|
49
|
+
global _DEBUG
|
|
50
|
+
_DEBUG = bool(value)
|
|
51
|
+
root = logging.getLogger("cctally")
|
|
52
|
+
if root.handlers:
|
|
53
|
+
root.setLevel(logging.DEBUG if _DEBUG else logging.WARNING)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _LateStderrHandler(logging.StreamHandler):
|
|
57
|
+
"""StreamHandler that resolves sys.stderr at emit time."""
|
|
58
|
+
|
|
59
|
+
def __init__(self):
|
|
60
|
+
super().__init__(sys.stderr)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def stream(self):
|
|
64
|
+
return sys.stderr
|
|
65
|
+
|
|
66
|
+
@stream.setter
|
|
67
|
+
def stream(self, value): # StreamHandler.__init__ assigns; ignore.
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
|
|
72
|
+
_DATEFMT = "%Y-%m-%dT%H:%M:%S"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def get_logger(name: str = "cctally") -> logging.Logger:
|
|
76
|
+
"""Return the chokepoint logger. Pass the bare suffix:
|
|
77
|
+
``get_logger("dashboard")`` -> logger ``cctally.dashboard``."""
|
|
78
|
+
root = logging.getLogger("cctally")
|
|
79
|
+
if not root.handlers: # configure-once latch (process-global)
|
|
80
|
+
formatter = logging.Formatter(_FORMAT, datefmt=_DATEFMT)
|
|
81
|
+
handler = _LateStderrHandler()
|
|
82
|
+
handler.setFormatter(formatter)
|
|
83
|
+
root.addHandler(handler)
|
|
84
|
+
sink = (os.environ.get("CCTALLY_DEBUG_LOG") or "").strip()
|
|
85
|
+
if sink:
|
|
86
|
+
try:
|
|
87
|
+
fh = logging.FileHandler(sink, encoding="utf-8")
|
|
88
|
+
fh.setFormatter(formatter)
|
|
89
|
+
root.addHandler(fh)
|
|
90
|
+
except OSError:
|
|
91
|
+
pass # degrade to stderr-only
|
|
92
|
+
root.setLevel(logging.DEBUG if _DEBUG else logging.WARNING)
|
|
93
|
+
root.propagate = False
|
|
94
|
+
if name in ("", "cctally"):
|
|
95
|
+
return root
|
|
96
|
+
return logging.getLogger(f"cctally.{name}")
|
package/bin/_lib_pricing.py
CHANGED
|
@@ -53,7 +53,7 @@ def _chip_for_model(name: str) -> str:
|
|
|
53
53
|
# Date the embedded pricing snapshots below were last verified against
|
|
54
54
|
# vendor sources. Bump whenever CLAUDE_MODEL_PRICING / CODEX_MODEL_PRICING
|
|
55
55
|
# is synced. Read by `pricing-check` + the release pre-flight staleness nudge.
|
|
56
|
-
PRICING_SNAPSHOT_DATE = "2026-07-
|
|
56
|
+
PRICING_SNAPSHOT_DATE = "2026-07-10"
|
|
57
57
|
PRICING_STALENESS_DAYS = 60 # release pre-flight WARNs past this age
|
|
58
58
|
|
|
59
59
|
# Canonical machine-readable pricing source (Claude values + Codex values).
|
|
@@ -77,6 +77,11 @@ PRICING_DRIFT_ALLOWLIST: list[dict] = [
|
|
|
77
77
|
{
|
|
78
78
|
"model": "claude-sonnet-5",
|
|
79
79
|
"field": field,
|
|
80
|
+
# Structured cutover date (#279 S7 W7): the intro rate is in effect
|
|
81
|
+
# THROUGH 2026-08-31, so this suppression is valid through then and
|
|
82
|
+
# `expired_allowlist_entries` flags it the day after. Keep the prose in
|
|
83
|
+
# `reason`; the date here is what the offline expiry check + the cron read.
|
|
84
|
+
"expires": "2026-08-31",
|
|
80
85
|
"reason": (
|
|
81
86
|
"LiteLLM tracks the claude-sonnet-5 introductory rate "
|
|
82
87
|
"($2/$10 per MTok, through 2026-08-31); we deliberately embed the "
|
|
@@ -322,6 +327,11 @@ _unknown_model_warnings: set[str] = set()
|
|
|
322
327
|
# nothing missing. Models absent from this table still fall back to `gpt-5`
|
|
323
328
|
# pricing with isFallback=true (matches upstream's LEGACY_FALLBACK_MODEL
|
|
324
329
|
# behavior); a one-shot stderr warning is emitted per unknown model name.
|
|
330
|
+
# 2026-07-10: added the gpt-5.6 family (gpt-5.6, gpt-5.6-sol, gpt-5.6-terra,
|
|
331
|
+
# gpt-5.6-luna) that LiteLLM published since the last sync — flagged by
|
|
332
|
+
# `pricing-check` as missing_from_us (gpt-5.6-sol had real usage falling back
|
|
333
|
+
# to gpt-5). Exact model_prices_and_context_window.json values, each with the
|
|
334
|
+
# above-272k tier (max_input_tokens 1050000).
|
|
325
335
|
#
|
|
326
336
|
# Billing rules:
|
|
327
337
|
# - reasoning_output_tokens is billed at the *output* rate (matches
|
|
@@ -414,6 +424,42 @@ CODEX_MODEL_PRICING: dict[str, dict[str, Any]] = {
|
|
|
414
424
|
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
|
|
415
425
|
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
|
416
426
|
},
|
|
427
|
+
# ── 2026-07-10 sync: gpt-5.6 family (LiteLLM openai-provider entries) ──
|
|
428
|
+
# Exact model_prices_and_context_window.json values; each carries the
|
|
429
|
+
# above-272k tier (max_input_tokens 1050000). gpt-5.6 and gpt-5.6-sol share
|
|
430
|
+
# gpt-5.5's rate card; -terra matches gpt-5.4; -luna is the cheapest tier.
|
|
431
|
+
"gpt-5.6": {
|
|
432
|
+
"input_cost_per_token": 5e-06,
|
|
433
|
+
"cache_read_input_token_cost": 5e-07,
|
|
434
|
+
"output_cost_per_token": 3e-05,
|
|
435
|
+
"input_cost_per_token_above_272k_tokens": 1e-05,
|
|
436
|
+
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
|
|
437
|
+
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
|
438
|
+
},
|
|
439
|
+
"gpt-5.6-sol": {
|
|
440
|
+
"input_cost_per_token": 5e-06,
|
|
441
|
+
"cache_read_input_token_cost": 5e-07,
|
|
442
|
+
"output_cost_per_token": 3e-05,
|
|
443
|
+
"input_cost_per_token_above_272k_tokens": 1e-05,
|
|
444
|
+
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
|
|
445
|
+
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
|
446
|
+
},
|
|
447
|
+
"gpt-5.6-terra": {
|
|
448
|
+
"input_cost_per_token": 2.5e-06,
|
|
449
|
+
"cache_read_input_token_cost": 2.5e-07,
|
|
450
|
+
"output_cost_per_token": 1.5e-05,
|
|
451
|
+
"input_cost_per_token_above_272k_tokens": 5e-06,
|
|
452
|
+
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
|
|
453
|
+
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
|
454
|
+
},
|
|
455
|
+
"gpt-5.6-luna": {
|
|
456
|
+
"input_cost_per_token": 1e-06,
|
|
457
|
+
"cache_read_input_token_cost": 1e-07,
|
|
458
|
+
"output_cost_per_token": 6e-06,
|
|
459
|
+
"input_cost_per_token_above_272k_tokens": 2e-06,
|
|
460
|
+
"cache_read_input_token_cost_above_272k_tokens": 2e-07,
|
|
461
|
+
"output_cost_per_token_above_272k_tokens": 9e-06,
|
|
462
|
+
},
|
|
417
463
|
# ── Issue #123: full gpt-5.x LiteLLM sync (2026-05-30 snapshot) ──
|
|
418
464
|
# Exact model_prices_and_context_window.json values for every
|
|
419
465
|
# openai-provider gpt-5* model `pricing-check`'s scope flags but the
|
|
@@ -162,6 +162,23 @@ def stale_allowlist_entries(allowlist, claude_tbl, codex_tbl, litellm_scoped) ->
|
|
|
162
162
|
return stale
|
|
163
163
|
|
|
164
164
|
|
|
165
|
+
def expired_allowlist_entries(allowlist, as_of_date) -> list:
|
|
166
|
+
"""Return allowlist entries whose optional ``expires`` (``YYYY-MM-DD``) date
|
|
167
|
+
has PASSED as of ``as_of_date`` (a date/datetime or ISO string).
|
|
168
|
+
|
|
169
|
+
Date-derived, so it needs no network (runs offline too — #279 S7 W7): an
|
|
170
|
+
expired suppression is a signal to remove the entry / re-sync the embedded
|
|
171
|
+
snapshot even before LiteLLM has reverted, so the deliberate divergence
|
|
172
|
+
doesn't silently ossify past its stated cutover. An entry with no
|
|
173
|
+
``expires`` field never expires. Comparison is strict — an entry expiring ON
|
|
174
|
+
a date is still valid THROUGH that date and only expired the day AFTER."""
|
|
175
|
+
as_of = str(as_of_date)[:10]
|
|
176
|
+
return [
|
|
177
|
+
e for e in (allowlist or [])
|
|
178
|
+
if e.get("expires") and str(e["expires"])[:10] < as_of
|
|
179
|
+
]
|
|
180
|
+
|
|
181
|
+
|
|
165
182
|
_CLAUDE_REQUIRED = ("input_cost_per_token", "output_cost_per_token",
|
|
166
183
|
"cache_creation_input_token_cost", "cache_read_input_token_cost")
|
|
167
184
|
_CODEX_REQUIRED = ("input_cost_per_token", "cache_read_input_token_cost",
|
|
@@ -194,8 +211,13 @@ def check_table_shapes(claude_tbl, codex_tbl, zero_sentinels) -> list:
|
|
|
194
211
|
return problems
|
|
195
212
|
|
|
196
213
|
|
|
197
|
-
def pricing_issue_action(
|
|
198
|
-
"""Decide the cron's GitHub-issue action. Pure; the YAML executes it.
|
|
199
|
-
|
|
214
|
+
def pricing_issue_action(findings_present: bool, existing_open: bool) -> str:
|
|
215
|
+
"""Decide the cron's GitHub-issue action. Pure; the YAML executes it.
|
|
216
|
+
|
|
217
|
+
``findings_present`` is ANY actionable finding the cron tracks — value
|
|
218
|
+
drift, missing-from-us, OR (as of #279 S7 W7) a stale/expired allowlist
|
|
219
|
+
suppression. The name generalizes the former ``drift_present``; the
|
|
220
|
+
two-state machine is unchanged."""
|
|
221
|
+
if findings_present:
|
|
200
222
|
return "update" if existing_open else "create"
|
|
201
223
|
return "close" if existing_open else "noop"
|