cctally 1.117.2 → 1.118.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +7 -3
- package/bin/_cctally_cache.py +2873 -193
- package/bin/_cctally_completion.py +106 -33
- package/bin/_cctally_core.py +69 -4
- package/bin/_cctally_dashboard.py +188 -13
- package/bin/_cctally_dashboard_conversation.py +5 -1
- package/bin/_cctally_dashboard_envelope.py +50 -19
- package/bin/_cctally_dashboard_sources.py +23 -1
- package/bin/_cctally_db.py +193 -153
- package/bin/_cctally_diagnosis_sources.py +63 -7
- package/bin/_cctally_doctor.py +865 -93
- package/bin/_cctally_five_hour.py +67 -50
- package/bin/_cctally_forecast.py +35 -13
- package/bin/_cctally_journal.py +186 -18
- package/bin/_cctally_milestone_history.py +859 -62
- package/bin/_cctally_oauth_samples.py +286 -15
- package/bin/_cctally_parser.py +2 -2
- package/bin/_cctally_quota.py +104 -12
- package/bin/_cctally_record.py +153 -29
- package/bin/_cctally_rederive.py +608 -68
- package/bin/_cctally_statusline.py +108 -81
- package/bin/_cctally_store.py +179 -38
- package/bin/_cctally_tui.py +892 -181
- package/bin/_cctally_update.py +32 -13
- package/bin/_lib_codex_conversation_query.py +52 -14
- package/bin/_lib_codex_drain_admission.py +224 -6
- package/bin/_lib_codex_metadata.py +39 -0
- package/bin/_lib_conversation_anon.py +15 -8
- package/bin/_lib_conversation_dispatch.py +7 -0
- package/bin/_lib_conversation_query.py +289 -70
- package/bin/_lib_dashboard_sources.py +37 -1
- package/bin/_lib_detector_access_registry.py +497 -26
- package/bin/_lib_doctor.py +85 -20
- package/bin/_lib_doctor_scan_index.py +321 -0
- package/bin/_lib_ingest_frontier.py +20 -1
- package/bin/_lib_oauth_samples.py +23 -7
- package/bin/_lib_pricing.py +146 -16
- package/bin/_lib_quota.py +19 -0
- package/bin/_lib_rederive.py +11 -6
- package/bin/_lib_retained_records.py +516 -0
- package/bin/_lib_snapshot_cache.py +10 -0
- package/bin/_lib_stale_proof.py +272 -43
- package/bin/_lib_stats_content_revision.py +831 -0
- package/bin/_lib_write_io.py +11 -4
- package/bin/cctally +4 -1
- package/dashboard/static/assets/ConversationsView-D4Iv4ku0.js +72 -0
- package/dashboard/static/assets/{DoctorModal-B0DnU9d9.js → DoctorModal-D4QSq6uz.js} +1 -1
- package/dashboard/static/assets/ModalRoot-C9F5dhZi.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-DSxXQSa-.js → ProjectsDrillPanel-WxAdRDMt.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BleDpQqG.js → SourceDetailModal-NyyEfXmc.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-CMqaAkSK.js → UpdateModal-CFLq9pT1.js} +1 -1
- package/dashboard/static/assets/{index-BjQV-0-D.js → index-BKTWH9JE.js} +2 -2
- package/dashboard/static/assets/{outlineNavigation-CO-VWBm7.js → outlineNavigation-BnGUwCS3.js} +3 -3
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-oqNsWzWV.js +0 -72
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -504,8 +504,10 @@ def _stats_corruption_guidance() -> str:
|
|
|
504
504
|
# #195: the cache_meta flag cache migration 030 sets to arm the one-time
|
|
505
505
|
# cost-side re-walk that lands the cache-write TTL split on existing rows.
|
|
506
506
|
# `sync_cache` reads it to select SESSION_ENTRY_UPSERT_SQL_REWALK (the chained
|
|
507
|
-
# ON CONFLICT variant) and clears it at the end of a
|
|
508
|
-
#
|
|
507
|
+
# ON CONFLICT variant) and clears it at the end of a non-targeted full walk
|
|
508
|
+
# that is clean on disk (fully clean, or #916 residual-only: its only defect
|
|
509
|
+
# is orphan rows, and each orphan's own -1 cursor keeps selecting the chained
|
|
510
|
+
# variant per file if it reappears). Deliberately NOT `claude_ingest_walk_complete` — see the migration's
|
|
509
511
|
# docstring for why reusing that marker is both unnecessary and over-broad.
|
|
510
512
|
CACHE_CREATION_SPLIT_REWALK_KEY = "cache_creation_split_rewalk_pending"
|
|
511
513
|
|
|
@@ -4158,20 +4160,29 @@ def _codex_accounting_ledger_ddl() -> tuple[str, ...]:
|
|
|
4158
4160
|
"UPDATE cache_meta SET value=CAST(value AS INTEGER) + 1 "
|
|
4159
4161
|
"WHERE key='codex_accounting_mutation_seq';"
|
|
4160
4162
|
)
|
|
4163
|
+
# #971 spec D1: the paths a thread row names, as an index-backed UNION.
|
|
4164
|
+
# Arm 1 is the conversation arm; arm 2 is the alias arm, which only ever
|
|
4165
|
+
# matched through a real entry join, so it is the matching files that have
|
|
4166
|
+
# at least one entry. UNION supplies the DISTINCT, and every comparison
|
|
4167
|
+
# stays ``=``, so a NULL matches nothing in either arm. The unary ``+`` on
|
|
4168
|
+
# arm 1's root term is load-bearing: without it the planner seeks
|
|
4169
|
+
# idx_codex_entries_root_path inside the INSERT's join with cache_meta and
|
|
4170
|
+
# walks the whole root's history. Arm 2 seeks idx_codex_files_root_thread.
|
|
4161
4171
|
def thread_paths(image: str) -> str:
|
|
4162
4172
|
return f"""
|
|
4163
|
-
SELECT
|
|
4173
|
+
SELECT entries.source_root_key, entries.source_path
|
|
4164
4174
|
FROM codex_session_entries AS entries
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
+
WHERE +entries.source_root_key = {image}.source_root_key
|
|
4176
|
+
AND entries.conversation_key = {image}.conversation_key
|
|
4177
|
+
UNION
|
|
4178
|
+
SELECT files.source_root_key, files.path
|
|
4179
|
+
FROM codex_session_files AS files
|
|
4180
|
+
WHERE files.source_root_key = {image}.source_root_key
|
|
4181
|
+
AND files.last_native_thread_id = {image}.native_thread_id
|
|
4182
|
+
AND EXISTS (
|
|
4183
|
+
SELECT 1 FROM codex_session_entries AS present
|
|
4184
|
+
WHERE present.source_root_key = files.source_root_key
|
|
4185
|
+
AND present.source_path = files.path)
|
|
4175
4186
|
"""
|
|
4176
4187
|
old_thread_paths = thread_paths("OLD")
|
|
4177
4188
|
new_thread_paths = thread_paths("NEW")
|
|
@@ -4269,6 +4280,9 @@ def _codex_accounting_ledger_ddl() -> tuple[str, ...]:
|
|
|
4269
4280
|
WHERE key='codex_accounting_mutation_seq';
|
|
4270
4281
|
END
|
|
4271
4282
|
""",
|
|
4283
|
+
# #971 spec D1: the alias arm of thread_paths() seeks this index.
|
|
4284
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_files_root_thread "
|
|
4285
|
+
"ON codex_session_files(source_root_key, last_native_thread_id)",
|
|
4272
4286
|
"DROP TRIGGER IF EXISTS trg_codex_accounting_thread_ins",
|
|
4273
4287
|
f"""
|
|
4274
4288
|
CREATE TRIGGER trg_codex_accounting_thread_ins
|
|
@@ -4722,22 +4736,33 @@ def _codex_quota_latest_base_match(image: str) -> str:
|
|
|
4722
4736
|
|
|
4723
4737
|
|
|
4724
4738
|
def _codex_quota_latest_recompute_sql(image: str, *, only_if_absent: bool) -> str:
|
|
4725
|
-
"""Re-derive one partition's maximum
|
|
4726
|
-
|
|
4727
|
-
|
|
4739
|
+
"""Re-derive one partition's maximum through bounded index probes.
|
|
4740
|
+
|
|
4741
|
+
#971 spec D2: the maximum is a SCALAR subquery, which SQLite answers with
|
|
4742
|
+
the min/max short-circuit on idx_qws_partition_capture; the earlier
|
|
4743
|
+
FROM-subquery form filtered on ``latest IS NOT NULL`` read the whole
|
|
4744
|
+
partition instead. With ``only_if_absent`` the CASE evaluates the
|
|
4745
|
+
aggregate only when no summary row survives, which for a deleted
|
|
4746
|
+
non-latest row is never.
|
|
4747
|
+
"""
|
|
4748
|
+
aggregate = (
|
|
4749
|
+
"(SELECT MAX(unixepoch(captured_at_utc)) FROM quota_window_snapshots"
|
|
4750
|
+
f" WHERE {_codex_quota_latest_validity_sql('')}"
|
|
4751
|
+
f" AND {_codex_quota_latest_base_match(image)})"
|
|
4752
|
+
)
|
|
4753
|
+
latest = (
|
|
4754
|
+
"CASE WHEN EXISTS (SELECT 1 FROM codex_quota_partition_latest"
|
|
4728
4755
|
f" WHERE {_codex_quota_latest_summary_match(image)})"
|
|
4729
|
-
|
|
4756
|
+
f" THEN NULL ELSE {aggregate} END"
|
|
4757
|
+
if only_if_absent else aggregate
|
|
4730
4758
|
)
|
|
4731
4759
|
return (
|
|
4732
4760
|
"INSERT INTO codex_quota_partition_latest ("
|
|
4733
4761
|
+ ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
|
|
4734
4762
|
+ ", latest_capture_epoch)"
|
|
4735
4763
|
f" SELECT {', '.join(_codex_quota_latest_key_exprs(image + '.'))},"
|
|
4736
|
-
" latest FROM (SELECT
|
|
4737
|
-
"
|
|
4738
|
-
f" WHERE {_codex_quota_latest_validity_sql('')}"
|
|
4739
|
-
f" AND {_codex_quota_latest_base_match(image)})"
|
|
4740
|
-
f" WHERE latest IS NOT NULL{absent_clause}"
|
|
4764
|
+
f" latest FROM (SELECT {latest} AS latest)"
|
|
4765
|
+
" WHERE latest IS NOT NULL"
|
|
4741
4766
|
)
|
|
4742
4767
|
|
|
4743
4768
|
|
|
@@ -5497,6 +5522,11 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
5497
5522
|
kind, name, "_apply_codex_retained_identity_inventory",
|
|
5498
5523
|
"051_codex_retained_identities")
|
|
5499
5524
|
for kind, name in _CODEX_RETAINED_IDENTITY_OBJECTS),
|
|
5525
|
+
# #971 spec D4: migration 052's handler re-runs the accounting helper.
|
|
5526
|
+
SchemaDeliveryObject(
|
|
5527
|
+
"index", "idx_codex_files_root_thread",
|
|
5528
|
+
"_apply_codex_accounting_change_ledger",
|
|
5529
|
+
"052_codex_delete_trigger_plans"),
|
|
5500
5530
|
)
|
|
5501
5531
|
|
|
5502
5532
|
|
|
@@ -5654,9 +5684,9 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
5654
5684
|
``add_column_if_missing``). Does NOT run the dispatcher and does NOT include
|
|
5655
5685
|
the Codex ``last_total_tokens`` ALTER, which carries a one-time purge
|
|
5656
5686
|
side-effect that stays in ``open_cache_db``. The eager dispatcher can run
|
|
5657
|
-
024 under the dispatcher's global writer flock
|
|
5658
|
-
|
|
5659
|
-
reads nor requires ``last_total_tokens``.
|
|
5687
|
+
024 under the dispatcher's global writer flock; since #983 that handler
|
|
5688
|
+
only arms the guarded Codex byte-zero replay (one ``cache_meta`` row) and
|
|
5689
|
+
neither reads nor requires ``last_total_tokens``.
|
|
5660
5690
|
Normal ``open_cache_db`` remains responsible for the ALTER and its historic
|
|
5661
5691
|
purge, so leaving it outside this shared schema is safe.
|
|
5662
5692
|
"""
|
|
@@ -6112,8 +6142,9 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
6112
6142
|
add_column_if_missing(conn, "session_files", "session_id", "TEXT")
|
|
6113
6143
|
add_column_if_missing(conn, "session_files", "project_path", "TEXT")
|
|
6114
6144
|
# #294 S1: old Codex accounting rows have no truthful provider-root or
|
|
6115
|
-
# conversation identity, so the linkage remains nullable until
|
|
6116
|
-
#
|
|
6145
|
+
# conversation identity, so the linkage remains nullable until the guarded
|
|
6146
|
+
# byte-zero replay migration 024 arms re-derives them from source (#983).
|
|
6147
|
+
# Existing report selectors
|
|
6117
6148
|
# intentionally keep selecting their shipped columns only.
|
|
6118
6149
|
add_column_if_missing(conn, "codex_session_entries", "source_root_key", "TEXT")
|
|
6119
6150
|
add_column_if_missing(conn, "codex_session_entries", "conversation_key", "TEXT")
|
|
@@ -8410,8 +8441,10 @@ def _acquire_cache_db_codex_provider_flock(
|
|
|
8410
8441
|
"""Take the Codex provider flock inside the dispatcher's global flock.
|
|
8411
8442
|
|
|
8412
8443
|
Production dispatchers already hold the global cache writer flock. The
|
|
8413
|
-
provider-local lock remains inside handlers
|
|
8414
|
-
tests and provider-scoped
|
|
8444
|
+
provider-local lock remains inside the handlers that write Codex-derived
|
|
8445
|
+
tables (025, 034, 043, 047) so direct migration tests and provider-scoped
|
|
8446
|
+
coordination retain their fail-closed behavior. Since #983 the legacy
|
|
8447
|
+
024/026/027 handlers only arm the replay marker and take no lock.
|
|
8415
8448
|
"""
|
|
8416
8449
|
provider_path = _cache_db_codex_lock_path_for_conn(conn)
|
|
8417
8450
|
if provider_path is None:
|
|
@@ -9374,59 +9407,38 @@ def _023_conversation_sessions_enrichment_columns(conn: sqlite3.Connection) -> N
|
|
|
9374
9407
|
|
|
9375
9408
|
@cache_migration("024_codex_fused_ingest_rebuild")
|
|
9376
9409
|
def _024_codex_fused_ingest_rebuild(conn: sqlite3.Connection) -> None:
|
|
9377
|
-
"""
|
|
9378
|
-
|
|
9379
|
-
|
|
9380
|
-
|
|
9381
|
-
|
|
9382
|
-
|
|
9383
|
-
|
|
9384
|
-
|
|
9385
|
-
|
|
9386
|
-
|
|
9387
|
-
|
|
9388
|
-
|
|
9389
|
-
|
|
9390
|
-
|
|
9391
|
-
|
|
9392
|
-
|
|
9393
|
-
|
|
9410
|
+
"""Arm the guarded byte-zero Codex replay for #294 S1's fused reingest.
|
|
9411
|
+
|
|
9412
|
+
Originally this migration cleared the Codex accounting, file, quota,
|
|
9413
|
+
thread, event and root families directly, because pre-S1 rows lack
|
|
9414
|
+
provider-root identity, qualified conversation linkage, quota observations
|
|
9415
|
+
and physical event payloads, none of which the cache can recover on its
|
|
9416
|
+
own. The next Codex sync then rederived them from the rollouts.
|
|
9417
|
+
|
|
9418
|
+
Since #983 that repair is the guarded byte-zero replay ``sync_codex_cache``
|
|
9419
|
+
performs: this handler arms ``CODEX_REPLAY_FROM_ZERO_KEY`` through
|
|
9420
|
+
``arm_codex_replay_from_zero`` and nothing else, and the sync ORs the
|
|
9421
|
+
marker into its own ``rebuild``. (Over a replay heal record left by the
|
|
9422
|
+
#982 executor the helper also marks that record for a fresh clear, so a
|
|
9423
|
+
refill in progress cannot swallow the re-arm.) The clear then
|
|
9424
|
+
runs under the cache writer flocks after the #416 D1 identity snapshot
|
|
9425
|
+
(which also resolves pre-S1 NULL-root cursors to their configured root),
|
|
9426
|
+
the #485 root-evidence partition and the #968 compressed preflight. On a
|
|
9427
|
+
Python without a zstd decoder that preflight refuses and keeps every
|
|
9428
|
+
retained row and the marker, a recoverable state the next sync with a
|
|
9429
|
+
decoder replays. Clearing here instead destroyed retained compressed-only
|
|
9430
|
+
history and emptied the D1 snapshot.
|
|
9431
|
+
|
|
9432
|
+
No provider flock, no DML on a Codex family, no sequence bump and no
|
|
9433
|
+
certificate change: one ``cache_meta`` row is all it writes, so a mid-walk
|
|
9434
|
+
sync has nothing to interleave with. Idempotent: re-running rewrites the
|
|
9435
|
+
same marker. NO self-stamp — the dispatcher central-stamps on a clean
|
|
9436
|
+
return (#140).
|
|
9394
9437
|
"""
|
|
9395
|
-
|
|
9396
|
-
|
|
9397
|
-
)
|
|
9398
|
-
|
|
9399
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
9400
|
-
try:
|
|
9401
|
-
conn.execute("DELETE FROM codex_session_entries")
|
|
9402
|
-
conn.execute("DELETE FROM codex_session_files")
|
|
9403
|
-
conn.execute("DELETE FROM quota_window_snapshots WHERE source = 'codex'")
|
|
9404
|
-
conn.execute("DELETE FROM codex_conversation_threads")
|
|
9405
|
-
conn.execute("DELETE FROM codex_conversation_events")
|
|
9406
|
-
conn.execute("DELETE FROM codex_source_roots")
|
|
9407
|
-
# F3 (#313): clearing Codex quota state without invalidating the
|
|
9408
|
-
# quota-projection certificate would leave a stale-valid cert
|
|
9409
|
-
# (cache sequence unchanged) that lets the reconcile short-circuit
|
|
9410
|
-
# skip over now-deleted data. Delete it in the same transaction.
|
|
9411
|
-
conn.execute(
|
|
9412
|
-
"DELETE FROM cache_meta WHERE key='codex_quota_projection_certificate'"
|
|
9413
|
-
)
|
|
9414
|
-
# #496 S5b §4.3, the identical hazard one layer over: this deletes
|
|
9415
|
-
# every Codex `quota_window_snapshots` row while the journal still
|
|
9416
|
-
# retains the observations behind them, and it does NOT bump
|
|
9417
|
-
# `codex_physical_mutation_seq`, so a stored coverage certificate
|
|
9418
|
-
# would stay stale-VALID and the next rebuild's fast path would skip
|
|
9419
|
-
# a replay over a cache holding zero Codex quota rows. Reachable
|
|
9420
|
-
# through `db skip 024` followed by `db unskip 024`. Same
|
|
9421
|
-
# transaction as the deletes.
|
|
9422
|
-
import _cctally_cache
|
|
9423
|
-
_cctally_cache._invalidate_codex_journal_coverage_certificate(conn)
|
|
9424
|
-
conn.commit()
|
|
9425
|
-
except Exception:
|
|
9426
|
-
conn.rollback()
|
|
9427
|
-
raise
|
|
9428
|
-
finally:
|
|
9429
|
-
_release_cache_db_writer_flocks(held)
|
|
9438
|
+
import _cctally_cache
|
|
9439
|
+
|
|
9440
|
+
_cctally_cache.arm_codex_replay_from_zero(conn)
|
|
9441
|
+
conn.commit()
|
|
9430
9442
|
|
|
9431
9443
|
|
|
9432
9444
|
@cache_migration("025_codex_conversation_normalization")
|
|
@@ -9475,76 +9487,60 @@ def _025_codex_conversation_normalization(conn: sqlite3.Connection) -> None:
|
|
|
9475
9487
|
|
|
9476
9488
|
@cache_migration("026_codex_conversation_key_backfill")
|
|
9477
9489
|
def _026_codex_conversation_key_backfill(conn: sqlite3.Connection) -> None:
|
|
9478
|
-
"""Arm
|
|
9490
|
+
"""Arm the guarded byte-zero Codex replay for missing conversation keys (#312).
|
|
9479
9491
|
|
|
9480
9492
|
Historical accounting rows predate the source-derived conversation-key
|
|
9481
|
-
enrichment
|
|
9482
|
-
|
|
9483
|
-
sync
|
|
9484
|
-
|
|
9485
|
-
|
|
9486
|
-
|
|
9487
|
-
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
|
|
9492
|
-
|
|
9493
|
-
|
|
9493
|
+
enrichment, and the cache cannot truthfully recover those keys on its own.
|
|
9494
|
+
Originally this migration cleared every Codex-derived family itself so the
|
|
9495
|
+
next Codex sync replayed the retained rollouts from byte zero.
|
|
9496
|
+
|
|
9497
|
+
Since #983 that repair is the guarded byte-zero replay ``sync_codex_cache``
|
|
9498
|
+
performs: this handler only arms ``CODEX_REPLAY_FROM_ZERO_KEY`` (through
|
|
9499
|
+
``arm_codex_replay_from_zero``, like every arming site), and the
|
|
9500
|
+
sync's rebuild path clears and replays behind the #416 D1 identity
|
|
9501
|
+
snapshot, the #485 root-evidence partition and the #968 compressed
|
|
9502
|
+
preflight. Without a zstd decoder the preflight refuses and the retained
|
|
9503
|
+
rows stay (recoverable once a decoder exists) instead of being cleared out
|
|
9504
|
+
of band.
|
|
9505
|
+
|
|
9506
|
+
No provider flock, no row change, no sequence bump, no certificate change.
|
|
9507
|
+
Idempotent: re-running rewrites the same marker. NO self-stamp — the
|
|
9508
|
+
dispatcher central-stamps on a clean return (#140).
|
|
9494
9509
|
"""
|
|
9495
|
-
|
|
9496
|
-
conn, migration="026 conversation-key backfill (#312)"
|
|
9497
|
-
)
|
|
9498
|
-
try:
|
|
9499
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
9500
|
-
try:
|
|
9501
|
-
import _cctally_cache
|
|
9510
|
+
import _cctally_cache
|
|
9502
9511
|
|
|
9503
|
-
|
|
9504
|
-
|
|
9505
|
-
conn.commit()
|
|
9506
|
-
except Exception:
|
|
9507
|
-
conn.rollback()
|
|
9508
|
-
raise
|
|
9509
|
-
finally:
|
|
9510
|
-
_release_cache_db_writer_flocks(held)
|
|
9512
|
+
_cctally_cache.arm_codex_replay_from_zero(conn)
|
|
9513
|
+
conn.commit()
|
|
9511
9514
|
|
|
9512
9515
|
|
|
9513
9516
|
@cache_migration("027_codex_fork_preamble_rebuild")
|
|
9514
9517
|
def _027_codex_fork_preamble_rebuild(conn: sqlite3.Connection) -> None:
|
|
9515
|
-
"""
|
|
9518
|
+
"""Arm the guarded byte-zero Codex replay that drops copied fork-preamble
|
|
9519
|
+
accounting.
|
|
9516
9520
|
|
|
9517
9521
|
Forked/subagent JSONL can retain parent ``token_count`` records before the
|
|
9518
|
-
child's first model-bearing ``turn_context``.
|
|
9519
|
-
copied records into ``codex_session_entries`` as ``model='unknown'``,
|
|
9520
|
-
counting usage already present in the parent rollout.
|
|
9521
|
-
|
|
9522
|
-
corrected fused parser
|
|
9523
|
-
|
|
9524
|
-
|
|
9525
|
-
|
|
9526
|
-
|
|
9527
|
-
|
|
9528
|
-
|
|
9529
|
-
|
|
9530
|
-
|
|
9522
|
+
child's first model-bearing ``turn_context``. Older ingest projected those
|
|
9523
|
+
copied records into ``codex_session_entries`` as ``model='unknown'``,
|
|
9524
|
+
double counting usage already present in the parent rollout. Originally
|
|
9525
|
+
this migration cleared every re-derivable Codex family itself so the
|
|
9526
|
+
corrected fused parser replayed the source from byte zero.
|
|
9527
|
+
|
|
9528
|
+
Since #983 that repair is the guarded byte-zero replay ``sync_codex_cache``
|
|
9529
|
+
performs: this handler only arms ``CODEX_REPLAY_FROM_ZERO_KEY`` (through
|
|
9530
|
+
``arm_codex_replay_from_zero``, like every arming site), and the
|
|
9531
|
+
sync's rebuild path clears and replays behind the identity snapshot, the
|
|
9532
|
+
root-evidence partition and the compressed preflight. Until it runs the
|
|
9533
|
+
copied rows remain what the previous binary showed; without a zstd decoder
|
|
9534
|
+
the state is kept and recoverable rather than cleared.
|
|
9535
|
+
|
|
9536
|
+
No provider flock, no row change, no sequence bump, no certificate change.
|
|
9537
|
+
Idempotent: re-running rewrites the same marker. NO self-stamp — the
|
|
9538
|
+
dispatcher central-stamps on a clean return (#140).
|
|
9531
9539
|
"""
|
|
9532
|
-
|
|
9533
|
-
conn, migration="027 fork-preamble rebuild"
|
|
9534
|
-
)
|
|
9535
|
-
try:
|
|
9536
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
9537
|
-
try:
|
|
9538
|
-
import _cctally_cache
|
|
9540
|
+
import _cctally_cache
|
|
9539
9541
|
|
|
9540
|
-
|
|
9541
|
-
|
|
9542
|
-
conn.commit()
|
|
9543
|
-
except Exception:
|
|
9544
|
-
conn.rollback()
|
|
9545
|
-
raise
|
|
9546
|
-
finally:
|
|
9547
|
-
_release_cache_db_writer_flocks(held)
|
|
9542
|
+
_cctally_cache.arm_codex_replay_from_zero(conn)
|
|
9543
|
+
conn.commit()
|
|
9548
9544
|
|
|
9549
9545
|
|
|
9550
9546
|
#: Pages 028 returns per committed transaction when it reclaims the dropped
|
|
@@ -9813,8 +9809,12 @@ def _030_session_entries_cache_creation_split(conn: sqlite3.Connection) -> None:
|
|
|
9813
9809
|
``session_entries``, so its absence from disk leaves no orphan, and
|
|
9814
9810
|
fixtures seed exactly such rows to stay out of the orphan set. The next
|
|
9815
9811
|
successful ingest of a surviving file overwrites the sentinel with its real
|
|
9816
|
-
size, so it is transient for everything still on disk.
|
|
9817
|
-
|
|
9812
|
+
size, so it is transient for everything still on disk. #916 added a third
|
|
9813
|
+
reader: ``sync_cache``'s per-file UPSERT selection treats a negative
|
|
9814
|
+
``size_bytes`` as a file still owed its re-walk and replays it with the
|
|
9815
|
+
chained-conflict variant even after the flag below has retired (a pre-030
|
|
9816
|
+
orphan whose transcript reappears). Only these three readers exist; grep
|
|
9817
|
+
``session_files`` before adding a fourth.
|
|
9818
9818
|
|
|
9819
9819
|
Unlike the truncation path this does NOT wipe session_entries: rows are
|
|
9820
9820
|
preserved and UPSERTed in place, which is why sync_cache must use the
|
|
@@ -9842,8 +9842,13 @@ def _030_session_entries_cache_creation_split(conn: sqlite3.Connection) -> None:
|
|
|
9842
9842
|
020's deliberately LOUD duplicate-physical-key backstop intact on every
|
|
9843
9843
|
other ingest path, which is the resolution the spec states it prefers.
|
|
9844
9844
|
|
|
9845
|
-
``sync_cache`` clears the flag at the end of a
|
|
9846
|
-
|
|
9845
|
+
``sync_cache`` clears the flag at the end of a non-targeted full walk that
|
|
9846
|
+
is clean on disk: the condition under which it (re)writes
|
|
9847
|
+
``claude_ingest_walk_complete``, and also a #916 residual-only walk (clean
|
|
9848
|
+
on disk, its only defect orphan rows), which writes the
|
|
9849
|
+
``claude_ingest_walk_orphan_residual`` token instead. The only files still
|
|
9850
|
+
owed the re-walk after a residual-only walk are absent orphans, and each
|
|
9851
|
+
one's own ``-1`` cursor selects the chained variant if it reappears.
|
|
9847
9852
|
|
|
9848
9853
|
Idempotent: re-running re-arms a walk that is already correct. NO self-stamp
|
|
9849
9854
|
— the dispatcher central-stamps on a clean return (#140); the handler commits
|
|
@@ -10062,13 +10067,13 @@ def _034_codex_window_spend_adoption(conn: sqlite3.Connection) -> None:
|
|
|
10062
10067
|
empty are candidates, so a re-run over its own output writes nothing. NO
|
|
10063
10068
|
self-stamp — the dispatcher central-stamps on a clean return (#140).
|
|
10064
10069
|
|
|
10065
|
-
Takes the Codex provider flock like
|
|
10070
|
+
Takes the Codex provider flock like handler 025: this writes a
|
|
10066
10071
|
Codex-derived table, so a mid-walk ``sync_codex_cache`` must not interleave.
|
|
10067
10072
|
On contention it DEFERS (``MigrationGateNotMet``) before touching any data —
|
|
10068
10073
|
the safe side, and free here because that same sync runs the identical pass
|
|
10069
10074
|
at its own end anyway.
|
|
10070
10075
|
|
|
10071
|
-
``BEGIN IMMEDIATE`` before the read, like handler
|
|
10076
|
+
``BEGIN IMMEDIATE`` before the read, like handler 025: the pass reads the
|
|
10072
10077
|
folded window evidence and then writes back a plan derived from it, so it
|
|
10073
10078
|
must hold the write lock for the whole read-then-write rather than upgrade
|
|
10074
10079
|
part way through it.
|
|
@@ -10101,17 +10106,20 @@ def _035_codex_thread_source_inference_replay(conn: sqlite3.Connection) -> None:
|
|
|
10101
10106
|
The repair must re-read the rollout bytes: the retained events carry NULL
|
|
10102
10107
|
conversation keys and the in-place normalization replay preserves them.
|
|
10103
10108
|
|
|
10104
|
-
Writes a marker ONLY
|
|
10109
|
+
Writes a marker ONLY, through ``arm_codex_replay_from_zero`` (which, over
|
|
10110
|
+
a #982 replay heal record, also marks that record for a fresh clear).
|
|
10111
|
+
``sync_codex_cache`` consumes it and ORs it into its
|
|
10105
10112
|
own ``rebuild``, which is what makes the rebuild path capture
|
|
10106
10113
|
``rebuild_known_identities`` BEFORE the clear. Clearing here instead would
|
|
10107
10114
|
delete ``codex_session_files`` out of band, leaving the next ordinary sync
|
|
10108
10115
|
with an empty snapshot — every re-read rollout would fall through to the
|
|
10109
10116
|
live-``auth.json`` branch and pre-mechanism Codex spend would be
|
|
10110
|
-
re-attributed to whoever is authenticated now (#416 spec D1). Migrations
|
|
10111
|
-
and 027
|
|
10112
|
-
|
|
10117
|
+
re-attributed to whoever is authenticated now (#416 spec D1). Migrations
|
|
10118
|
+
024, 026 and 027 used to clear out of band exactly that way, and a store
|
|
10119
|
+
still below them reaches them on its next upgrade, so since #983 they arm
|
|
10120
|
+
this same marker instead of clearing.
|
|
10113
10121
|
|
|
10114
|
-
No provider flock: unlike handlers
|
|
10122
|
+
No provider flock: unlike handlers 025 and 034 this writes one
|
|
10115
10123
|
``cache_meta`` row and touches no Codex-derived table, so a concurrent
|
|
10116
10124
|
``sync_codex_cache`` has nothing to interleave with — and if one is mid-walk,
|
|
10117
10125
|
arming the marker simply defers the replay to the following sync.
|
|
@@ -10121,7 +10129,7 @@ def _035_codex_thread_source_inference_replay(conn: sqlite3.Connection) -> None:
|
|
|
10121
10129
|
"""
|
|
10122
10130
|
import _cctally_cache
|
|
10123
10131
|
|
|
10124
|
-
|
|
10132
|
+
_cctally_cache.arm_codex_replay_from_zero(conn)
|
|
10125
10133
|
conn.commit()
|
|
10126
10134
|
|
|
10127
10135
|
|
|
@@ -10433,7 +10441,7 @@ def _043_codex_window_attributions(conn: sqlite3.Connection) -> None:
|
|
|
10433
10441
|
retracted, so a re-run over its own output writes nothing but the cursor.
|
|
10434
10442
|
NO self-stamp — the dispatcher central-stamps on a clean return (#140).
|
|
10435
10443
|
|
|
10436
|
-
Takes the Codex provider flock like handlers
|
|
10444
|
+
Takes the Codex provider flock like handlers 025 and 034: this writes a
|
|
10437
10445
|
Codex-derived table, so a mid-walk ``sync_codex_cache`` must not interleave.
|
|
10438
10446
|
On contention it DEFERS (``MigrationGateNotMet``) before touching any data,
|
|
10439
10447
|
which is free here because that same sync rehydrates this table at its own
|
|
@@ -10606,7 +10614,7 @@ def _047_spill_free_read_paths(conn: sqlite3.Connection) -> None:
|
|
|
10606
10614
|
It also delivers ``idx_codex_window_attributions_read_order`` (Amendment 1
|
|
10607
10615
|
item 1, ``c901-design-2``), the window-attribution read's assertion order.
|
|
10608
10616
|
|
|
10609
|
-
Takes the Codex provider flock like handlers
|
|
10617
|
+
Takes the Codex provider flock like handlers 025, 034 and 043, because
|
|
10610
10618
|
the summary is Codex-derived, and DEFERS (``MigrationGateNotMet``) before
|
|
10611
10619
|
any DDL when a Codex sync holds it. Idempotent: every object uses IF NOT
|
|
10612
10620
|
EXISTS or drop-then-create, and a re-derivation over its own output writes
|
|
@@ -10808,6 +10816,38 @@ def _051_codex_retained_identities(conn: sqlite3.Connection) -> None:
|
|
|
10808
10816
|
conn.commit()
|
|
10809
10817
|
|
|
10810
10818
|
|
|
10819
|
+
@cache_migration("052_codex_delete_trigger_plans")
|
|
10820
|
+
def _052_codex_delete_trigger_plans(conn: sqlite3.Connection) -> None:
|
|
10821
|
+
"""#971 spec D4: deliver the bounded thread and partition-latest triggers.
|
|
10822
|
+
|
|
10823
|
+
The schema apply is version-gated, so a store already at head 51 never
|
|
10824
|
+
receives the rewritten trigger bodies or ``idx_codex_files_root_thread``
|
|
10825
|
+
from ``_apply_cache_schema``; only a registered migration reaches it. The
|
|
10826
|
+
two shared helpers drop and re-create their triggers (the accounting one
|
|
10827
|
+
also creates the index). With all five summary objects present the
|
|
10828
|
+
partition-latest summary is NOT re-bootstrapped, so no history is scanned
|
|
10829
|
+
and no row changes.
|
|
10830
|
+
|
|
10831
|
+
Downgrade: an older head-51 binary opening a head-52 store through
|
|
10832
|
+
``open_cache_db`` sees the gate open (52 != 51), reinstalls its own
|
|
10833
|
+
slower but equivalent bodies, and the dispatcher's version-ahead recovery
|
|
10834
|
+
trims this marker; ``db recover --db cache`` on a raw connection keeps the
|
|
10835
|
+
new bodies, which name no index and stay correct. A re-upgrade opens the
|
|
10836
|
+
gate again and restores them.
|
|
10837
|
+
|
|
10838
|
+
Takes no provider flock: the dispatcher runs it under the opener's
|
|
10839
|
+
maintenance-exclusive and global writer exclusion. The accounting
|
|
10840
|
+
helper's DROP/CREATE pairs ride the transaction its ``cache_meta`` INSERT
|
|
10841
|
+
opens implicitly, the summary helper joins it as a savepoint, and the
|
|
10842
|
+
closing commit publishes every replacement at once. Re-running is a
|
|
10843
|
+
no-op. NO self-stamp — the dispatcher central-stamps on a clean return
|
|
10844
|
+
(#140).
|
|
10845
|
+
"""
|
|
10846
|
+
_apply_codex_accounting_change_ledger(conn)
|
|
10847
|
+
_apply_codex_quota_latest_summary(conn)
|
|
10848
|
+
conn.commit()
|
|
10849
|
+
|
|
10850
|
+
|
|
10811
10851
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
10812
10852
|
|
|
10813
10853
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -1841,6 +1841,49 @@ def _claude_context_window(model: object) -> int | None:
|
|
|
1841
1841
|
return statusline._resolve_context_window(model, lambda *_a, **_k: None)
|
|
1842
1842
|
|
|
1843
1843
|
|
|
1844
|
+
# #1022: prior-generation ids whose bare transcript id may have run in a 200K
|
|
1845
|
+
# or a 1M window. No retained field states which, so only a request's own size
|
|
1846
|
+
# can prove it, and only for itself. This set states an AMBIGUITY, not a
|
|
1847
|
+
# default capacity, which is why it lives here and not in the status line's
|
|
1848
|
+
# tables; the two windows are named here for the same reason, so an edit of
|
|
1849
|
+
# the status line's family defaults cannot move this rule.
|
|
1850
|
+
_CLAUDE_TWO_WINDOW_MODEL_IDS = frozenset({
|
|
1851
|
+
"claude-opus-5", "claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6",
|
|
1852
|
+
"claude-sonnet-5", "claude-sonnet-4-6", "claude-sonnet-4-5",
|
|
1853
|
+
"claude-sonnet-4",
|
|
1854
|
+
})
|
|
1855
|
+
_CLAUDE_TWO_WINDOW_SMALL = 200_000
|
|
1856
|
+
_CLAUDE_TWO_WINDOW_LARGE = 1_000_000
|
|
1857
|
+
|
|
1858
|
+
|
|
1859
|
+
def _classify_claude_context_request(model: object, used: int
|
|
1860
|
+
) -> tuple[str, float | None]:
|
|
1861
|
+
"""One associated Claude request's context-window outcome.
|
|
1862
|
+
|
|
1863
|
+
`("decided", fraction)` at a known or proven capacity, or for a
|
|
1864
|
+
two-window request whose fraction under the small window is already below
|
|
1865
|
+
the floor (an upper bound that cannot qualify, stating no capacity);
|
|
1866
|
+
`("undecidable", None)` for a two-window request that the small window
|
|
1867
|
+
would qualify and the large one would not; `("unknown", None)` when the
|
|
1868
|
+
model publishes no window at all.
|
|
1869
|
+
"""
|
|
1870
|
+
lowered = str(model or "").lower()
|
|
1871
|
+
normalized = re.sub(r"-\d{8}$", "", lowered)
|
|
1872
|
+
if (lowered.endswith("[1m]")
|
|
1873
|
+
or normalized not in _CLAUDE_TWO_WINDOW_MODEL_IDS):
|
|
1874
|
+
capacity = _claude_context_window(model)
|
|
1875
|
+
if not capacity:
|
|
1876
|
+
return ("unknown", None)
|
|
1877
|
+
return ("decided", used / capacity)
|
|
1878
|
+
if used > _CLAUDE_TWO_WINDOW_SMALL:
|
|
1879
|
+
# Only the large window can hold it.
|
|
1880
|
+
return ("decided", used / _CLAUDE_TWO_WINDOW_LARGE)
|
|
1881
|
+
small = used / _CLAUDE_TWO_WINDOW_SMALL
|
|
1882
|
+
if _large_enough(small):
|
|
1883
|
+
return ("undecidable", None)
|
|
1884
|
+
return ("decided", small)
|
|
1885
|
+
|
|
1886
|
+
|
|
1844
1887
|
# --- the reads the evaluators run (#620 S3 §4.2) ------------------------
|
|
1845
1888
|
#
|
|
1846
1889
|
# Every statement below is SET-WISE and chunked. A per-session or per-file
|
|
@@ -2705,28 +2748,41 @@ def _evaluate_claude_short_high_context(scope: DiagnosisScope,
|
|
|
2705
2748
|
for reply in association.replies
|
|
2706
2749
|
for uuid in [str(reply["anchor"]["uuid"])]
|
|
2707
2750
|
if key_of_uuid.get(uuid, (None,))[0] is not None]
|
|
2751
|
+
# #1022: each request is classified on its OWN model id and size,
|
|
2752
|
+
# never on a sibling's, an out-of-window request's or a `/model`
|
|
2753
|
+
# marker's — only a request's own size proves a prior-generation
|
|
2754
|
+
# window, and only for itself. `best` folds DECIDED fractions alone,
|
|
2755
|
+
# so an undecidable request neither qualifies a conversation nor
|
|
2756
|
+
# inflates the published maximum.
|
|
2708
2757
|
best: float | None = None
|
|
2709
2758
|
saw_request = False
|
|
2759
|
+
undecidable = False
|
|
2710
2760
|
for key in dict.fromkeys(keys):
|
|
2711
2761
|
for entry in by_turn.get(key, ()):
|
|
2712
2762
|
saw_request = True
|
|
2713
|
-
capacity = _claude_context_window(entry.model)
|
|
2714
|
-
if not capacity:
|
|
2715
|
-
continue
|
|
2716
2763
|
# The Claude numerator, matching the statusline's own context
|
|
2717
2764
|
# segment: input plus BOTH cache legs.
|
|
2718
2765
|
used = (entry.input_tokens + entry.cache_read_tokens
|
|
2719
2766
|
+ entry.cache_create_tokens)
|
|
2720
|
-
fraction =
|
|
2767
|
+
state, fraction = _classify_claude_context_request(
|
|
2768
|
+
entry.model, used)
|
|
2769
|
+
if state == "undecidable":
|
|
2770
|
+
undecidable = True
|
|
2771
|
+
continue
|
|
2772
|
+
if fraction is None:
|
|
2773
|
+
continue # no published window
|
|
2721
2774
|
best = fraction if best is None else max(best, fraction)
|
|
2722
|
-
if saw_request and best is None:
|
|
2723
|
-
unevaluable[session] = kernel.GAP_UNKNOWN_CONTEXT_WINDOW
|
|
2724
|
-
continue
|
|
2725
2775
|
# With several replies the LARGEST single request fraction is used,
|
|
2726
2776
|
# never their sum: two replies at 0.45 do not make a full window.
|
|
2777
|
+
# Qualification is decided FIRST: a decided request at the floor
|
|
2778
|
+
# qualifies the conversation under either window of its undecidable
|
|
2779
|
+
# siblings.
|
|
2727
2780
|
if best is not None and _large_enough(best):
|
|
2728
2781
|
qualifying.append(session)
|
|
2729
2782
|
fractions[session] = best
|
|
2783
|
+
continue
|
|
2784
|
+
if undecidable or (saw_request and best is None):
|
|
2785
|
+
unevaluable[session] = kernel.GAP_UNKNOWN_CONTEXT_WINDOW
|
|
2730
2786
|
return _short_context_evaluation(
|
|
2731
2787
|
facts, known=sessions, unevaluable=unevaluable, qualifying=qualifying,
|
|
2732
2788
|
turn_counts=turn_counts, fractions=fractions,
|