cctally 1.109.0 → 1.111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +6 -6
- package/bin/_cctally_cache.py +62 -13
- package/bin/_cctally_core.py +8 -1
- package/bin/_cctally_dashboard.py +422 -195
- package/bin/_cctally_dashboard_conversation.py +231 -40
- package/bin/_cctally_dashboard_envelope.py +48 -32
- package/bin/_cctally_dashboard_sources.py +503 -109
- package/bin/_cctally_db.py +44 -0
- package/bin/_cctally_diagnosis_sources.py +181 -15
- package/bin/_cctally_doctor.py +15 -3
- package/bin/_cctally_forecast.py +25 -12
- package/bin/_cctally_journal.py +59 -32
- package/bin/_cctally_milestone_history.py +65 -9
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_record.py +50 -12
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_source_analytics.py +567 -133
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +39 -6
- package/bin/_cctally_update.py +420 -14
- package/bin/_lib_cache_report.py +7 -10
- package/bin/_lib_codex_conversation_query.py +286 -63
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation_query.py +254 -37
- package/bin/_lib_doctor.py +16 -2
- package/bin/_lib_pricing.py +80 -276
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_view_models.py +5 -0
- package/bin/cctally +18 -5
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-BZGXOlc2.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-KbO_KWa7.js → ProjectsDrillPanel-DAv39uSo.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BrOPLVRL.js → SourceDetailModal-DSAWo2Fp.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-CYgEFAi4.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/{useKeymap-DQx_A1MB.js → useKeymap-CW7Up0Ay.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +2 -1
- package/dashboard/static/assets/ConversationsView-TO5bpIyc.js +0 -72
- package/dashboard/static/assets/ModalRoot-C_fvugTl.js +0 -1
- package/dashboard/static/assets/index-Cj_l7T7u.css +0 -1
- package/dashboard/static/assets/index-DLVwACUM.js +0 -13
- package/dashboard/static/assets/outlineNavigation-DXX-RjKl.js +0 -9
package/bin/_cctally_db.py
CHANGED
|
@@ -2083,6 +2083,8 @@ def _run_pending_migrations(
|
|
|
2083
2083
|
db_label, db_version=cur_version, max_known=len(registry),
|
|
2084
2084
|
)
|
|
2085
2085
|
if cur_version == len(registry):
|
|
2086
|
+
if db_label == "conversations.db":
|
|
2087
|
+
_ensure_codex_session_meta_provenance_index(conn)
|
|
2086
2088
|
# When the registry is currently empty (today's cache.db case),
|
|
2087
2089
|
# still leave the schema_migrations table behind so a later
|
|
2088
2090
|
# transition to len(registry) >= 1 can distinguish populated
|
|
@@ -2124,6 +2126,9 @@ def _run_pending_migrations(
|
|
|
2124
2126
|
db_label, _first_pending_migration_name(conn, registry, cur_version)
|
|
2125
2127
|
)
|
|
2126
2128
|
|
|
2129
|
+
if db_label == "conversations.db":
|
|
2130
|
+
_ensure_codex_session_meta_provenance_index(conn)
|
|
2131
|
+
|
|
2127
2132
|
# Track whether schema_migrations existed before this open so we can
|
|
2128
2133
|
# detect the fresh-install path. After bootstrap, even a "first time
|
|
2129
2134
|
# opened with framework code" DB might have rows from the legacy
|
|
@@ -5475,6 +5480,45 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5475
5480
|
_apply_codex_conversation_source_identity(conn)
|
|
5476
5481
|
|
|
5477
5482
|
|
|
5483
|
+
def _ensure_codex_session_meta_provenance_index(
|
|
5484
|
+
conn: sqlite3.Connection,
|
|
5485
|
+
) -> bool:
|
|
5486
|
+
"""Ensure the account-scoped anonymization provenance index exists.
|
|
5487
|
+
|
|
5488
|
+
``codex_conversation_events`` is an append-only physical log and can be
|
|
5489
|
+
much larger than the normalized transcript. The scoped anonymizer reads
|
|
5490
|
+
only ``session_meta`` payloads, so an index restricted to that record type
|
|
5491
|
+
makes the read proportional to the retained provenance rows rather than
|
|
5492
|
+
every event. This is an idempotent, data-free schema ensure run by the
|
|
5493
|
+
conversations dispatcher on both its fast and pending paths; keeping the
|
|
5494
|
+
helper here also gives the schema migration layer one stable delivery
|
|
5495
|
+
point for fresh and already-current stores.
|
|
5496
|
+
|
|
5497
|
+
A pre-conversations store may not have the event table yet. That is a
|
|
5498
|
+
valid no-op; the schema apply creates it before the dispatcher runs.
|
|
5499
|
+
Other SQLite errors propagate so a damaged store cannot silently fall back
|
|
5500
|
+
to a full physical-event scan.
|
|
5501
|
+
"""
|
|
5502
|
+
if conn.execute(
|
|
5503
|
+
"SELECT 1 FROM sqlite_master "
|
|
5504
|
+
"WHERE type='table' AND name='codex_conversation_events'"
|
|
5505
|
+
).fetchone() is None:
|
|
5506
|
+
return False
|
|
5507
|
+
if conn.execute(
|
|
5508
|
+
"SELECT 1 FROM sqlite_master "
|
|
5509
|
+
"WHERE type='index' AND name='idx_codex_events_session_meta_provenance'"
|
|
5510
|
+
).fetchone() is not None:
|
|
5511
|
+
return True
|
|
5512
|
+
conn.execute(
|
|
5513
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_events_session_meta_provenance "
|
|
5514
|
+
"ON codex_conversation_events("
|
|
5515
|
+
"source_root_key, account_key, conversation_key, payload_json) "
|
|
5516
|
+
"WHERE record_type='session_meta'"
|
|
5517
|
+
)
|
|
5518
|
+
conn.commit()
|
|
5519
|
+
return True
|
|
5520
|
+
|
|
5521
|
+
|
|
5478
5522
|
#: `cache_meta` keys the stamp backfill uses. The cursor is the durable resume
|
|
5479
5523
|
#: point (the last source path whose rows are fully stamped); the completion
|
|
5480
5524
|
#: marker is what a rebuild's publication gate reads. Both are values, not
|
|
@@ -40,10 +40,12 @@ import json
|
|
|
40
40
|
import os
|
|
41
41
|
import pickle
|
|
42
42
|
import re
|
|
43
|
+
import signal
|
|
43
44
|
import shlex
|
|
44
45
|
import sqlite3
|
|
45
46
|
import sys
|
|
46
47
|
import threading
|
|
48
|
+
import time
|
|
47
49
|
from collections import defaultdict
|
|
48
50
|
from dataclasses import dataclass, field
|
|
49
51
|
from functools import cached_property
|
|
@@ -71,6 +73,14 @@ UTC = dt.timezone.utc
|
|
|
71
73
|
# bounded wait rather than an immediate `database is locked`.
|
|
72
74
|
READ_BUSY_TIMEOUT_MS = 4000
|
|
73
75
|
|
|
76
|
+
# A provider read can lose its component-local coherence check to the writer
|
|
77
|
+
# that is publishing the next generation. Retry the whole provider build a
|
|
78
|
+
# small number of times while a monotonic admission budget remains. The
|
|
79
|
+
# `_establish` retry remains deliberately narrower: it still rereads one
|
|
80
|
+
# component once and refuses a second divergence within that build.
|
|
81
|
+
DIAGNOSIS_RETRY_MAX_ATTEMPTS = 3
|
|
82
|
+
DIAGNOSIS_RETRY_DEADLINE_SECONDS = 1.0
|
|
83
|
+
|
|
74
84
|
_STORE_PATHS = {
|
|
75
85
|
"cache": "CACHE_DB_PATH",
|
|
76
86
|
"stats": "DB_PATH",
|
|
@@ -1045,28 +1055,22 @@ def _compiled_codex_pricer(model: str, speed: str):
|
|
|
1045
1055
|
reasoning_output_tokens: int) -> float:
|
|
1046
1056
|
del reasoning_output_tokens
|
|
1047
1057
|
non_cached_input = max(0, input_tokens - cached_input_tokens)
|
|
1058
|
+
long_context = input_tokens > threshold
|
|
1048
1059
|
if non_cached_input <= 0 or not input_rate:
|
|
1049
1060
|
input_cost = 0.0
|
|
1050
|
-
elif non_cached_input > threshold and input_tier is not None:
|
|
1051
|
-
input_cost = (threshold * input_rate
|
|
1052
|
-
+ (non_cached_input - threshold) * input_tier)
|
|
1053
1061
|
else:
|
|
1054
|
-
input_cost = non_cached_input *
|
|
1062
|
+
input_cost = non_cached_input * (
|
|
1063
|
+
input_tier if long_context and input_tier is not None else input_rate)
|
|
1055
1064
|
if cached_input_tokens <= 0 or not cache_rate:
|
|
1056
1065
|
cached_input_cost = 0.0
|
|
1057
|
-
elif cached_input_tokens > threshold and cache_tier is not None:
|
|
1058
|
-
cached_input_cost = (
|
|
1059
|
-
threshold * cache_rate
|
|
1060
|
-
+ (cached_input_tokens - threshold) * cache_tier)
|
|
1061
1066
|
else:
|
|
1062
|
-
cached_input_cost = cached_input_tokens *
|
|
1067
|
+
cached_input_cost = cached_input_tokens * (
|
|
1068
|
+
cache_tier if long_context and cache_tier is not None else cache_rate)
|
|
1063
1069
|
if output_tokens <= 0 or not output_rate:
|
|
1064
1070
|
output_cost = 0.0
|
|
1065
|
-
elif output_tokens > threshold and output_tier is not None:
|
|
1066
|
-
output_cost = (threshold * output_rate
|
|
1067
|
-
+ (output_tokens - threshold) * output_tier)
|
|
1068
1071
|
else:
|
|
1069
|
-
output_cost = output_tokens *
|
|
1072
|
+
output_cost = output_tokens * (
|
|
1073
|
+
output_tier if long_context and output_tier is not None else output_rate)
|
|
1070
1074
|
base = input_cost + cached_input_cost + output_cost
|
|
1071
1075
|
if speed == "fast":
|
|
1072
1076
|
base *= multiplier
|
|
@@ -4345,11 +4349,43 @@ def build_provider_diagnosis(scope: DiagnosisScope, *,
|
|
|
4345
4349
|
)
|
|
4346
4350
|
|
|
4347
4351
|
|
|
4352
|
+
def _build_provider_with_retry(scope: DiagnosisScope, *,
|
|
4353
|
+
transcripts_visible: bool
|
|
4354
|
+
) -> kernel.ProviderResult:
|
|
4355
|
+
"""Build one provider, admitting only bounded fresh retries.
|
|
4356
|
+
|
|
4357
|
+
A `generation_incoherent` result means the provider's read crossed two
|
|
4358
|
+
writes during one attempt. Restarting the provider build is safe because
|
|
4359
|
+
every attempt owns fresh read-only connections and facts. Other typed
|
|
4360
|
+
establishment failures and unexpected exceptions retain their existing
|
|
4361
|
+
handling at the caller; they are never converted into retries.
|
|
4362
|
+
"""
|
|
4363
|
+
deadline = time.monotonic() + DIAGNOSIS_RETRY_DEADLINE_SECONDS
|
|
4364
|
+
for attempt in range(DIAGNOSIS_RETRY_MAX_ATTEMPTS):
|
|
4365
|
+
try:
|
|
4366
|
+
return build_provider_diagnosis(
|
|
4367
|
+
scope, transcripts_visible=transcripts_visible)
|
|
4368
|
+
except EstablishmentFailure as exc:
|
|
4369
|
+
if exc.code != EstablishmentError.GENERATION_INCOHERENT.value:
|
|
4370
|
+
raise
|
|
4371
|
+
if attempt + 1 >= DIAGNOSIS_RETRY_MAX_ATTEMPTS:
|
|
4372
|
+
raise
|
|
4373
|
+
remaining = deadline - time.monotonic()
|
|
4374
|
+
if remaining <= 0:
|
|
4375
|
+
raise
|
|
4376
|
+
# Give an in-flight writer a chance to finish before reopening
|
|
4377
|
+
# every store. Normal coherent reads pay no delay.
|
|
4378
|
+
time.sleep(min(0.1 * (attempt + 1), remaining))
|
|
4379
|
+
if time.monotonic() >= deadline:
|
|
4380
|
+
raise
|
|
4381
|
+
raise AssertionError("provider diagnosis retry loop did not return")
|
|
4382
|
+
|
|
4383
|
+
|
|
4348
4384
|
def _build_provider_task(args) -> kernel.ProviderResult:
|
|
4349
4385
|
"""Pickle-safe provider build for the independent `all` branches."""
|
|
4350
4386
|
scope, transcripts_visible = args
|
|
4351
4387
|
try:
|
|
4352
|
-
result =
|
|
4388
|
+
result = _build_provider_with_retry(
|
|
4353
4389
|
scope, transcripts_visible=transcripts_visible)
|
|
4354
4390
|
except EstablishmentFailure as exc:
|
|
4355
4391
|
if exc.code != EstablishmentError.STORE_UNAVAILABLE.value:
|
|
@@ -4419,6 +4455,136 @@ def _spawn_worker_init(cctally_path: str,
|
|
|
4419
4455
|
setattr(_cctally_core, name, pathlib.Path(raw))
|
|
4420
4456
|
|
|
4421
4457
|
|
|
4458
|
+
_DIAGNOSIS_PROCESS_TERMINATE_GRACE_SECONDS = 0.5
|
|
4459
|
+
|
|
4460
|
+
|
|
4461
|
+
def _bounded_process_entry(result_path: str, task, args,
|
|
4462
|
+
initializer, initargs) -> None:
|
|
4463
|
+
"""Run one pickle-safe task in its own killable process group."""
|
|
4464
|
+
if hasattr(os, "setsid"):
|
|
4465
|
+
try:
|
|
4466
|
+
os.setsid()
|
|
4467
|
+
except OSError:
|
|
4468
|
+
# The parent still terminates this process directly if the group
|
|
4469
|
+
# was not established. No descendants exist before this point.
|
|
4470
|
+
pass
|
|
4471
|
+
try:
|
|
4472
|
+
if initializer is not None:
|
|
4473
|
+
initializer(*initargs)
|
|
4474
|
+
payload = ("ok", task(*args))
|
|
4475
|
+
except EstablishmentFailure as exc:
|
|
4476
|
+
payload = ("establishment", exc.code, exc.message)
|
|
4477
|
+
except BaseException as exc:
|
|
4478
|
+
payload = ("unexpected", type(exc).__name__, str(exc))
|
|
4479
|
+
with open(result_path, "wb") as stream:
|
|
4480
|
+
pickle.dump(payload, stream, protocol=5)
|
|
4481
|
+
|
|
4482
|
+
|
|
4483
|
+
def _terminate_process_group(process) -> None:
|
|
4484
|
+
"""Terminate and reap one bounded worker plus any provider descendants."""
|
|
4485
|
+
grouped = False
|
|
4486
|
+
if process.pid is not None and hasattr(os, "killpg"):
|
|
4487
|
+
try:
|
|
4488
|
+
os.killpg(process.pid, signal.SIGTERM)
|
|
4489
|
+
grouped = True
|
|
4490
|
+
except ProcessLookupError:
|
|
4491
|
+
pass
|
|
4492
|
+
except OSError:
|
|
4493
|
+
pass
|
|
4494
|
+
if process.is_alive() and not grouped:
|
|
4495
|
+
process.terminate()
|
|
4496
|
+
process.join(_DIAGNOSIS_PROCESS_TERMINATE_GRACE_SECONDS)
|
|
4497
|
+
if grouped and process.pid is not None:
|
|
4498
|
+
try:
|
|
4499
|
+
# The leader may have exited on SIGTERM while a provider child
|
|
4500
|
+
# ignored it. The process group remains addressable until every
|
|
4501
|
+
# descendant is gone, so always finish the group with SIGKILL.
|
|
4502
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
4503
|
+
except ProcessLookupError:
|
|
4504
|
+
pass
|
|
4505
|
+
except OSError:
|
|
4506
|
+
pass
|
|
4507
|
+
elif process.is_alive():
|
|
4508
|
+
process.kill()
|
|
4509
|
+
process.join(_DIAGNOSIS_PROCESS_TERMINATE_GRACE_SECONDS)
|
|
4510
|
+
|
|
4511
|
+
|
|
4512
|
+
def _run_process_with_deadline(task, args, *, timeout_seconds: float,
|
|
4513
|
+
initializer=None, initargs=()):
|
|
4514
|
+
"""Run TASK in a one-shot process and forcibly retire it at the deadline."""
|
|
4515
|
+
import multiprocessing
|
|
4516
|
+
import tempfile
|
|
4517
|
+
|
|
4518
|
+
if timeout_seconds <= 0:
|
|
4519
|
+
raise TimeoutError("diagnosis process deadline expired")
|
|
4520
|
+
methods = multiprocessing.get_all_start_methods()
|
|
4521
|
+
method = "forkserver" if "forkserver" in methods else "spawn"
|
|
4522
|
+
context = multiprocessing.get_context(method)
|
|
4523
|
+
fd, result_path = tempfile.mkstemp(prefix="cctally-diagnosis-")
|
|
4524
|
+
os.close(fd)
|
|
4525
|
+
process = context.Process(
|
|
4526
|
+
target=_bounded_process_entry,
|
|
4527
|
+
args=(result_path, task, args, initializer, initargs),
|
|
4528
|
+
name="cctally-diagnosis-process",
|
|
4529
|
+
)
|
|
4530
|
+
try:
|
|
4531
|
+
process.start()
|
|
4532
|
+
process.join(timeout_seconds)
|
|
4533
|
+
if process.is_alive():
|
|
4534
|
+
_terminate_process_group(process)
|
|
4535
|
+
raise TimeoutError("diagnosis process deadline expired")
|
|
4536
|
+
if process.exitcode != 0:
|
|
4537
|
+
raise RuntimeError(
|
|
4538
|
+
f"diagnosis process exited {process.exitcode}")
|
|
4539
|
+
try:
|
|
4540
|
+
with open(result_path, "rb") as stream:
|
|
4541
|
+
payload = pickle.load(stream)
|
|
4542
|
+
except (EOFError, OSError, pickle.PickleError) as exc:
|
|
4543
|
+
raise RuntimeError("diagnosis process returned no result") from exc
|
|
4544
|
+
finally:
|
|
4545
|
+
if process.is_alive():
|
|
4546
|
+
_terminate_process_group(process)
|
|
4547
|
+
try:
|
|
4548
|
+
os.unlink(result_path)
|
|
4549
|
+
except FileNotFoundError:
|
|
4550
|
+
pass
|
|
4551
|
+
process.close()
|
|
4552
|
+
if payload[0] == "ok":
|
|
4553
|
+
return payload[1]
|
|
4554
|
+
if payload[0] == "establishment":
|
|
4555
|
+
raise EstablishmentFailure(payload[1], payload[2])
|
|
4556
|
+
raise RuntimeError(f"{payload[1]}: {payload[2]}")
|
|
4557
|
+
|
|
4558
|
+
|
|
4559
|
+
def _build_diagnosis_process_task(scope: DiagnosisScope,
|
|
4560
|
+
measured_at: dt.datetime | None,
|
|
4561
|
+
transcripts_visible: bool):
|
|
4562
|
+
return build_diagnosis(
|
|
4563
|
+
scope, measured_at=measured_at,
|
|
4564
|
+
transcripts_visible=transcripts_visible,
|
|
4565
|
+
)
|
|
4566
|
+
|
|
4567
|
+
|
|
4568
|
+
def build_diagnosis_bounded(scope: DiagnosisScope, *,
|
|
4569
|
+
measured_at: dt.datetime | None = None,
|
|
4570
|
+
transcripts_visible: bool,
|
|
4571
|
+
timeout_seconds: float) -> kernel.DiagnosisReport:
|
|
4572
|
+
"""Build one dashboard diagnosis in a process retired at TIMEOUT_SECONDS."""
|
|
4573
|
+
c = _cctally()
|
|
4574
|
+
paths = {
|
|
4575
|
+
name: str(getattr(_cctally_core, name))
|
|
4576
|
+
for name in ("CACHE_DB_PATH", "DB_PATH",
|
|
4577
|
+
"CONVERSATIONS_DB_PATH", "CONFIG_PATH")
|
|
4578
|
+
}
|
|
4579
|
+
return _run_process_with_deadline(
|
|
4580
|
+
_build_diagnosis_process_task,
|
|
4581
|
+
(scope, measured_at, transcripts_visible),
|
|
4582
|
+
timeout_seconds=timeout_seconds,
|
|
4583
|
+
initializer=_spawn_worker_init,
|
|
4584
|
+
initargs=(str(c.__file__), paths),
|
|
4585
|
+
)
|
|
4586
|
+
|
|
4587
|
+
|
|
4422
4588
|
def _build_isolated_provider(args):
|
|
4423
4589
|
"""Run one provider in a portable, one-shot safe process worker."""
|
|
4424
4590
|
import concurrent.futures
|
|
@@ -4467,7 +4633,7 @@ def build_diagnosis(scope: DiagnosisScope,
|
|
|
4467
4633
|
label=scope.label,
|
|
4468
4634
|
)
|
|
4469
4635
|
try:
|
|
4470
|
-
return
|
|
4636
|
+
return _build_provider_with_retry(
|
|
4471
4637
|
provider_scope, transcripts_visible=transcripts_visible)
|
|
4472
4638
|
except EstablishmentFailure as exc:
|
|
4473
4639
|
# A store that cannot be OPENED withholds its provider rather than
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -1810,9 +1810,15 @@ def _doctor_gather_state_impl(
|
|
|
1810
1810
|
if row:
|
|
1811
1811
|
codex_entries_count = int(row[0]) if row[0] is not None else 0
|
|
1812
1812
|
if row[1]:
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1813
|
+
try:
|
|
1814
|
+
codex_last_entry_at = parse_iso_datetime(
|
|
1815
|
+
row[1], "codex_session_entries.timestamp_utc",
|
|
1816
|
+
).astimezone(dt.timezone.utc)
|
|
1817
|
+
except ValueError:
|
|
1818
|
+
# A malformed newest row cannot establish cache
|
|
1819
|
+
# freshness. Continue to the all-history health
|
|
1820
|
+
# partition so doctor can count the bad row.
|
|
1821
|
+
codex_last_entry_at = None
|
|
1816
1822
|
try:
|
|
1817
1823
|
row = conn.execute(
|
|
1818
1824
|
"SELECT COUNT(*) FROM quota_window_snapshots "
|
|
@@ -1839,7 +1845,13 @@ def _doctor_gather_state_impl(
|
|
|
1839
1845
|
"qualified_rows": health.qualified_rows,
|
|
1840
1846
|
"missing_conversation_key_rows": health.missing_conversation_key_rows,
|
|
1841
1847
|
"missing_thread_join_rows": health.missing_thread_join_rows,
|
|
1848
|
+
"undecodable_metadata_rows": (
|
|
1849
|
+
health.undecodable_metadata_rows),
|
|
1842
1850
|
}
|
|
1851
|
+
if health.malformed_accounting_rows:
|
|
1852
|
+
codex_project_metadata_health[
|
|
1853
|
+
"malformed_accounting_rows"] = (
|
|
1854
|
+
health.malformed_accounting_rows)
|
|
1843
1855
|
except Exception as exc:
|
|
1844
1856
|
codex_project_metadata_error = type(exc).__name__
|
|
1845
1857
|
except sqlite3.OperationalError as exc:
|
package/bin/_cctally_forecast.py
CHANGED
|
@@ -127,16 +127,16 @@ def _resolve_forecast_now(as_of: str | None) -> dt.datetime:
|
|
|
127
127
|
return _command_as_of()
|
|
128
128
|
|
|
129
129
|
|
|
130
|
-
def _shape_week_samples(rows, *, include_held: bool
|
|
130
|
+
def _shape_week_samples(rows, *, include_held: bool,
|
|
131
|
+
include_account: bool = False) -> list:
|
|
131
132
|
"""Turn snapshot rows into the sample tuples `_fetch_current_week_snapshots`
|
|
132
133
|
returns. Both of that function's row-fetch legs go through here so the two
|
|
133
134
|
cannot drift on the tuple width.
|
|
134
135
|
|
|
135
136
|
`rows` are `(captured_at_utc, weekly_percent, five_hour_percent,
|
|
136
|
-
weekly_observation_held)`. The held flag is appended
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
existing consumer unpacks.
|
|
137
|
+
weekly_observation_held[, account_key])`. The held flag is appended only
|
|
138
|
+
for a held-inclusive read, and the account only for an explicit identity
|
|
139
|
+
read. Existing consumers keep the historical three- or four-field shape.
|
|
140
140
|
"""
|
|
141
141
|
out = []
|
|
142
142
|
for row in rows:
|
|
@@ -147,13 +147,16 @@ def _shape_week_samples(rows, *, include_held: bool) -> list:
|
|
|
147
147
|
)
|
|
148
148
|
if include_held:
|
|
149
149
|
sample = sample + (int(row[3] or 0),)
|
|
150
|
+
if include_account:
|
|
151
|
+
sample = sample + (row[4],)
|
|
150
152
|
out.append(sample)
|
|
151
153
|
return out
|
|
152
154
|
|
|
153
155
|
|
|
154
156
|
def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime,
|
|
155
157
|
*, account_key: "str | None" = None,
|
|
156
|
-
include_held: bool = False
|
|
158
|
+
include_held: bool = False,
|
|
159
|
+
include_account: bool = False):
|
|
157
160
|
"""Return (week_start_at, week_end_at, list[(captured_at, percent, five_hr)])
|
|
158
161
|
for the subscription week containing `now_utc`, or None if no snapshot
|
|
159
162
|
exists for the current week.
|
|
@@ -199,6 +202,10 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
|
|
|
199
202
|
five-hour axis takes the newest sample, a caller reading the weekly
|
|
200
203
|
axis takes the newest sample whose flag is 0, and neither can mistake
|
|
201
204
|
a carried-forward weekly value for an observed one.
|
|
205
|
+
|
|
206
|
+
``include_account`` appends the physical account key after the held flag
|
|
207
|
+
for the multi-account Current Week card only. The default wire-independent
|
|
208
|
+
sample shape and merged forecast/cost reads remain unchanged.
|
|
202
209
|
"""
|
|
203
210
|
# #341: optional account scoping — same predicate on every leg so the whole
|
|
204
211
|
# window resolution stays consistent (a real key or the `unattributed`
|
|
@@ -209,6 +216,10 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
|
|
|
209
216
|
# `include_held=True` is a pure superset of the default read rather than a
|
|
210
217
|
# read that can also resolve a different week.
|
|
211
218
|
_held_pred = "" if include_held else " AND weekly_observation_held = 0"
|
|
219
|
+
# The TUI's multi-account card needs the physical owner of the sample it
|
|
220
|
+
# actually displays. Keep the historical sample shape for every other read.
|
|
221
|
+
_owner_column = ", account_key" if include_account else ""
|
|
222
|
+
_sample_order = " ASC, id ASC" if include_account else " ASC"
|
|
212
223
|
candidates = conn.execute(
|
|
213
224
|
"SELECT week_start_at, week_end_at, week_start_date, MAX(captured_at_utc) AS latest_cap "
|
|
214
225
|
"FROM weekly_usage_snapshots "
|
|
@@ -255,13 +266,14 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
|
|
|
255
266
|
week_end_at = dt.datetime.combine(we_date + dt.timedelta(days=1), dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
|
|
256
267
|
rows = conn.execute(
|
|
257
268
|
"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
|
|
258
|
-
" weekly_observation_held "
|
|
269
|
+
" weekly_observation_held" + _owner_column + " "
|
|
259
270
|
"FROM weekly_usage_snapshots "
|
|
260
271
|
"WHERE week_start_date = ?" + _acct_pred + _held_pred +
|
|
261
|
-
" ORDER BY captured_at_utc
|
|
272
|
+
" ORDER BY captured_at_utc" + _sample_order,
|
|
262
273
|
(drow[0],) + _acct_p,
|
|
263
274
|
).fetchall()
|
|
264
|
-
samples = _shape_week_samples(rows, include_held=include_held
|
|
275
|
+
samples = _shape_week_samples(rows, include_held=include_held,
|
|
276
|
+
include_account=include_account)
|
|
265
277
|
samples = [s for s in samples if s[0] <= now_utc]
|
|
266
278
|
return week_start_at, week_end_at, samples
|
|
267
279
|
row = chosen
|
|
@@ -283,15 +295,16 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
|
|
|
283
295
|
placeholders = ",".join("?" * len(matching_texts))
|
|
284
296
|
rows = conn.execute(
|
|
285
297
|
f"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
|
|
286
|
-
f" weekly_observation_held "
|
|
298
|
+
f" weekly_observation_held{_owner_column} "
|
|
287
299
|
f"FROM weekly_usage_snapshots "
|
|
288
300
|
f"WHERE (week_start_at IN ({placeholders}) "
|
|
289
301
|
f" OR (week_start_at IS NULL AND week_start_date = ?))"
|
|
290
302
|
f"{_acct_pred}{_held_pred} "
|
|
291
|
-
f"ORDER BY captured_at_utc
|
|
303
|
+
f"ORDER BY captured_at_utc{_sample_order}",
|
|
292
304
|
tuple(matching_texts) + (chosen_date,) + _acct_p,
|
|
293
305
|
).fetchall()
|
|
294
|
-
samples = _shape_week_samples(rows, include_held=include_held
|
|
306
|
+
samples = _shape_week_samples(rows, include_held=include_held,
|
|
307
|
+
include_account=include_account)
|
|
295
308
|
samples = [s for s in samples if s[0] <= now_utc]
|
|
296
309
|
return week_start_at, week_end_at, samples
|
|
297
310
|
|
package/bin/_cctally_journal.py
CHANGED
|
@@ -1415,6 +1415,9 @@ class IngestContext:
|
|
|
1415
1415
|
# writes while still exercising the same SQLite derivation/fold code.
|
|
1416
1416
|
event_sink: "list | None" = None
|
|
1417
1417
|
projection_writes: bool = True
|
|
1418
|
+
# Operator-reviewed scratch replay only. Normal ingest leaves this empty.
|
|
1419
|
+
held_weekly_observation_ids: frozenset[str] = field(default_factory=frozenset)
|
|
1420
|
+
reviewed_weekly_basis_by_account: dict = field(default_factory=dict)
|
|
1418
1421
|
# #374 write boundary: emissions WITHHELD this cycle because they would have
|
|
1419
1422
|
# violated the same-revision rule. Each entry is a `DroppedConflict`; the row
|
|
1420
1423
|
# was converged from the already-journaled effective event instead.
|
|
@@ -3424,7 +3427,8 @@ def _now_iso() -> str:
|
|
|
3424
3427
|
#: the reset-aware in-window maximum, so it IS the week's evidence.
|
|
3425
3428
|
#: ``WEEKLY_HELD_CLAMP`` — it is strictly below that maximum, so the weekly axis
|
|
3426
3429
|
#: is held at the maximum and the incoming value is recorded only as the raw
|
|
3427
|
-
#: reading.
|
|
3430
|
+
#: reading. On reviewed scratch replay, an exact observation id can also hold
|
|
3431
|
+
#: a reading above the basis; either way it is not fresh weekly evidence.
|
|
3428
3432
|
WEEKLY_OBSERVED = "observed"
|
|
3429
3433
|
WEEKLY_HELD_CLAMP = "held_clamp"
|
|
3430
3434
|
|
|
@@ -3469,6 +3473,30 @@ class WeeklyBasis:
|
|
|
3469
3473
|
week_end_at: "str | None"
|
|
3470
3474
|
|
|
3471
3475
|
|
|
3476
|
+
def _latest_weekly_basis(conn, account_key, *, week_start_date=None):
|
|
3477
|
+
"""Return accepted weekly evidence before a later axis may delete rows."""
|
|
3478
|
+
sql = (
|
|
3479
|
+
"SELECT id, captured_at_utc, weekly_percent, week_start_date, "
|
|
3480
|
+
" week_end_date, week_start_at, week_end_at "
|
|
3481
|
+
"FROM weekly_usage_snapshots "
|
|
3482
|
+
"WHERE account_key = ? AND weekly_observation_held = 0 "
|
|
3483
|
+
)
|
|
3484
|
+
args = (account_key,)
|
|
3485
|
+
if week_start_date is not None:
|
|
3486
|
+
sql += "AND week_start_date = ? "
|
|
3487
|
+
args += (week_start_date,)
|
|
3488
|
+
row = conn.execute(
|
|
3489
|
+
sql + "ORDER BY captured_at_utc DESC, id DESC LIMIT 1", args,
|
|
3490
|
+
).fetchone()
|
|
3491
|
+
if row is None:
|
|
3492
|
+
return None
|
|
3493
|
+
return WeeklyBasis(
|
|
3494
|
+
snapshot_id=int(row[0]), captured_at_utc=str(row[1]),
|
|
3495
|
+
weekly_percent=float(row[2]), week_start_date=row[3],
|
|
3496
|
+
week_end_date=row[4], week_start_at=row[5], week_end_at=row[6],
|
|
3497
|
+
)
|
|
3498
|
+
|
|
3499
|
+
|
|
3472
3500
|
@dataclass(frozen=True)
|
|
3473
3501
|
class WeeklyFold:
|
|
3474
3502
|
"""The weekly axis of one fold decision."""
|
|
@@ -3514,7 +3542,9 @@ class UsageSnapshotFoldResult:
|
|
|
3514
3542
|
snapshot_action: str
|
|
3515
3543
|
|
|
3516
3544
|
|
|
3517
|
-
def _usage_snapshot_fold_decision(
|
|
3545
|
+
def _usage_snapshot_fold_decision(
|
|
3546
|
+
conn, payload, *, force_weekly_hold=False, reviewed_weekly_basis=None,
|
|
3547
|
+
) -> UsageSnapshotFoldResult:
|
|
3518
3548
|
"""The apply-time accept/hold/skip decision for a Claude rate-limit obs —
|
|
3519
3549
|
the predicate ported from `cmd_record_usage`'s insert guard
|
|
3520
3550
|
(bin/_cctally_record.py), made at fold time (spec §4.5 / §5.3).
|
|
@@ -3535,9 +3565,9 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
|
|
|
3535
3565
|
The clamp/hold distinction is what `_pipeline_claude_usage` gates the weekly
|
|
3536
3566
|
milestone on, and the reason it must survive: a dedup skip AGREES with the
|
|
3537
3567
|
stored row, so re-running the derivation chokepoints against it heals a
|
|
3538
|
-
killed tick; a
|
|
3539
|
-
|
|
3540
|
-
|
|
3568
|
+
killed tick; a hold CONTRADICTS it, so deriving a weekly milestone from the
|
|
3569
|
+
carried value fabricates a crossing (the 2026-09-01 incident, a fabricated
|
|
3570
|
+
13% milestone, and milestones are forward-only within an epoch).
|
|
3541
3571
|
|
|
3542
3572
|
Every value returned is derived from the same `conn` state and payload, so
|
|
3543
3573
|
the whole result is exactly as replay-deterministic as the boolean it
|
|
@@ -3573,30 +3603,24 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
|
|
|
3573
3603
|
(week_start_date, account_key, clamp_floor_iso),
|
|
3574
3604
|
).fetchone()
|
|
3575
3605
|
max_v = max_row[0] if max_row else None
|
|
3576
|
-
weekly_held = _lib_record.hwm_clamp_applies(
|
|
3577
|
-
|
|
3606
|
+
weekly_held = force_weekly_hold or _lib_record.hwm_clamp_applies(
|
|
3607
|
+
weekly_percent, max_v)
|
|
3578
3608
|
|
|
3579
3609
|
basis = None
|
|
3580
3610
|
if weekly_held:
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
if
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
weekly_percent=float(basis_row[2]),
|
|
3595
|
-
week_start_date=basis_row[3],
|
|
3596
|
-
week_end_date=basis_row[4],
|
|
3597
|
-
week_start_at=basis_row[5],
|
|
3598
|
-
week_end_at=basis_row[6],
|
|
3599
|
-
)
|
|
3611
|
+
# A reviewed hold may carry an old raw boundary. Its basis is selected
|
|
3612
|
+
# before reset/credit detection because five-hour credit can delete
|
|
3613
|
+
# that very row while preserving the held observation's 5h effect.
|
|
3614
|
+
basis = (
|
|
3615
|
+
reviewed_weekly_basis if force_weekly_hold
|
|
3616
|
+
else _latest_weekly_basis(
|
|
3617
|
+
conn, account_key, week_start_date=week_start_date)
|
|
3618
|
+
)
|
|
3619
|
+
weekly_effective = (
|
|
3620
|
+
basis.weekly_percent if force_weekly_hold and basis is not None
|
|
3621
|
+
else float(max_v) if weekly_held and max_v is not None
|
|
3622
|
+
else weekly_percent
|
|
3623
|
+
)
|
|
3600
3624
|
|
|
3601
3625
|
weekly = WeeklyFold(
|
|
3602
3626
|
raw_pct=weekly_percent,
|
|
@@ -3636,12 +3660,16 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
|
|
|
3636
3660
|
# it would make the next held tick look like new five-hour evidence and
|
|
3637
3661
|
# write a duplicate row every tick. Its weekly value is by construction a
|
|
3638
3662
|
# copy of the basis's, so including it changes no weekly comparison.
|
|
3663
|
+
last_week_start_date = (
|
|
3664
|
+
basis.week_start_date if force_weekly_hold and basis is not None
|
|
3665
|
+
else week_start_date
|
|
3666
|
+
)
|
|
3639
3667
|
last = conn.execute(
|
|
3640
3668
|
"SELECT weekly_percent, five_hour_percent, five_hour_window_key "
|
|
3641
3669
|
"FROM weekly_usage_snapshots "
|
|
3642
3670
|
"WHERE week_start_date = ? AND account_key = ? "
|
|
3643
3671
|
"ORDER BY captured_at_utc DESC, id DESC LIMIT 1",
|
|
3644
|
-
(
|
|
3672
|
+
(last_week_start_date, account_key),
|
|
3645
3673
|
).fetchone()
|
|
3646
3674
|
last_5h = last[1] if last is not None else None
|
|
3647
3675
|
last_window_key = last[2] if last is not None else None
|
|
@@ -3673,11 +3701,9 @@ def _usage_snapshot_fold_decision(conn, payload) -> UsageSnapshotFoldResult:
|
|
|
3673
3701
|
|
|
3674
3702
|
# ── The materialized outcome ───────────────────────────────────────────
|
|
3675
3703
|
if weekly_held:
|
|
3676
|
-
# The weekly axis carries nothing new
|
|
3677
|
-
#
|
|
3678
|
-
#
|
|
3679
|
-
# at all (no boundary to copy), so it degrades to a skip rather than
|
|
3680
|
-
# writing a row with a fabricated week.
|
|
3704
|
+
# The weekly axis carries nothing new. An ordinary clamp writes only
|
|
3705
|
+
# for five-hour evidence, including on an explicit reviewed hold.
|
|
3706
|
+
# A held row also needs an accepted basis boundary to carry forward.
|
|
3681
3707
|
if five_hour_unchanged or basis is None:
|
|
3682
3708
|
action = SNAPSHOT_SKIP_NO_CHANGE
|
|
3683
3709
|
else:
|
|
@@ -3913,6 +3939,7 @@ _EVT_KIND_PROVIDER = {
|
|
|
3913
3939
|
# planner raises `RederiveConflict` on every run.
|
|
3914
3940
|
_ACCOUNTS_MACHINERY_KINDS = frozenset(
|
|
3915
3941
|
("account_observe", "account_label", "accounts_cutover",
|
|
3942
|
+
"claude_weekly_observation_decision",
|
|
3916
3943
|
"codex_file_account",
|
|
3917
3944
|
"codex_window_attribution", "codex_window_attribution_retract"))
|
|
3918
3945
|
|