cctally 1.114.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4491 -295
- package/bin/_cctally_completion.py +1355 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +282 -9
- package/bin/_cctally_dashboard.py +8 -11
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +42 -35
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +401 -9
- package/bin/_cctally_doctor.py +469 -27
- package/bin/_cctally_five_hour.py +166 -1049
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +5076 -177
- package/bin/_cctally_journal_repair.py +110 -19
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +47 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +662 -0
- package/bin/_cctally_quota.py +96 -12
- package/bin/_cctally_quota_model.py +391 -83
- package/bin/_cctally_record.py +1088 -269
- package/bin/_cctally_rederive.py +854 -192
- package/bin/_cctally_refresh.py +113 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +7062 -505
- package/bin/_cctally_store.py +1244 -91
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +119 -60
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +569 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +177 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +4038 -0
- package/bin/_lib_doctor.py +753 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +261 -10
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +155 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +384 -52
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +393 -8
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +356 -27
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +29 -12
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +20 -2
- package/bin/_lib_merge_gate.py +0 -725
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -801,7 +801,7 @@ def _would_block_prod_migration(conn: sqlite3.Connection) -> bool:
|
|
|
801
801
|
test-suite's CCTALLY_DISABLE_DEV_AUTODETECT. Escape: CCTALLY_ALLOW_PROD_MIGRATION."""
|
|
802
802
|
if _cctally_core._truthy_env("CCTALLY_ALLOW_PROD_MIGRATION"):
|
|
803
803
|
return False
|
|
804
|
-
if not
|
|
804
|
+
if not _cctally_core._has_git_checkout_marker():
|
|
805
805
|
return False
|
|
806
806
|
db_dir = _conn_db_dir(conn)
|
|
807
807
|
if db_dir is None:
|
|
@@ -857,7 +857,7 @@ def _would_block_prod_stats(path: pathlib.Path) -> bool:
|
|
|
857
857
|
``CCTALLY_ALLOW_PROD_MIGRATION``."""
|
|
858
858
|
if _cctally_core._truthy_env("CCTALLY_ALLOW_PROD_MIGRATION"):
|
|
859
859
|
return False
|
|
860
|
-
if not
|
|
860
|
+
if not _cctally_core._has_git_checkout_marker():
|
|
861
861
|
return False
|
|
862
862
|
try:
|
|
863
863
|
return (
|
|
@@ -1700,6 +1700,10 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
|
|
|
1700
1700
|
)
|
|
1701
1701
|
forensics = None
|
|
1702
1702
|
incident = None
|
|
1703
|
+
import _cctally_completion
|
|
1704
|
+
# Fix round K11 (K9K10-R7): a rebuild that published but could not record
|
|
1705
|
+
# its residual obligation; reported once every lock is released.
|
|
1706
|
+
obligation_failure = None
|
|
1703
1707
|
try:
|
|
1704
1708
|
fcntl.flock(maint_fd, fcntl.LOCK_EX)
|
|
1705
1709
|
# #386: record the hold so a nested live `open_db()` does not request
|
|
@@ -1751,9 +1755,12 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
|
|
|
1751
1755
|
forensics_path=(
|
|
1752
1756
|
str(forensics) if forensics is not None else None
|
|
1753
1757
|
),
|
|
1754
|
-
)
|
|
1758
|
+
),
|
|
1759
|
+
complete_derivation=True,
|
|
1755
1760
|
)
|
|
1756
1761
|
incident = result.quarantine_dir
|
|
1762
|
+
except _cctally_completion.ResidualObligationNotDurable as exc:
|
|
1763
|
+
obligation_failure = exc
|
|
1757
1764
|
except Exception as exc:
|
|
1758
1765
|
eprint(f"cctally: stats.db rebuild failed: {exc}")
|
|
1759
1766
|
return 3
|
|
@@ -1767,6 +1774,20 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
|
|
|
1767
1774
|
finally:
|
|
1768
1775
|
os.close(maint_fd)
|
|
1769
1776
|
|
|
1777
|
+
if obligation_failure is not None:
|
|
1778
|
+
# The rebuilt index IS published (spec 4.7), so its completion wake
|
|
1779
|
+
# is posted as for any publication; only then is the obligation
|
|
1780
|
+
# failure reported. The inputs past the rebuilt prefix stay in the
|
|
1781
|
+
# journal for the next ingest.
|
|
1782
|
+
_cctally_completion.post_publication_wake()
|
|
1783
|
+
eprint(
|
|
1784
|
+
"cctally: stats.db was rebuilt, but the record that the inputs "
|
|
1785
|
+
"written during the rebuild still need deriving could not be "
|
|
1786
|
+
f"saved ({obligation_failure}). Those inputs stay in the journal "
|
|
1787
|
+
"and the next ingest derives them; check that the data directory "
|
|
1788
|
+
"is writable.")
|
|
1789
|
+
return 3
|
|
1790
|
+
|
|
1770
1791
|
total_rows = sum(result.rows_by_table.values())
|
|
1771
1792
|
if as_json:
|
|
1772
1793
|
payload = {
|
|
@@ -1870,6 +1891,11 @@ def cmd_db_rebuild(args: argparse.Namespace) -> int:
|
|
|
1870
1891
|
print(f" previous stats.db quarantined -> {incident}")
|
|
1871
1892
|
if forensics is not None:
|
|
1872
1893
|
print(f" forensics -> {forensics}")
|
|
1894
|
+
# #825/#832 spec 4.3 revision 2: a completed rebuild is a live index
|
|
1895
|
+
# publication; its completion wake (posted with every lock released)
|
|
1896
|
+
# dispatches the detached continuation, so this command never waits on
|
|
1897
|
+
# it. A file-only no-op when nothing is pending.
|
|
1898
|
+
_cctally_completion.post_publication_wake()
|
|
1873
1899
|
return 0
|
|
1874
1900
|
|
|
1875
1901
|
|
|
@@ -2356,6 +2382,24 @@ def _run_pending_migrations(
|
|
|
2356
2382
|
conn.commit()
|
|
2357
2383
|
|
|
2358
2384
|
|
|
2385
|
+
def _accounts_cutover_value_or_defer(what: str):
|
|
2386
|
+
"""`find_accounts_cutover_op` for an account backfill migration.
|
|
2387
|
+
|
|
2388
|
+
#825/#832 fix round K11 (K9K10-R9): a journal the lookup cannot list,
|
|
2389
|
+
size or read — including the account-bound boundary lookup it applies,
|
|
2390
|
+
`AccountBoundCutoverUnavailable` — proves neither the op nor its
|
|
2391
|
+
absence. That is transient, so the migration defers
|
|
2392
|
+
(`MigrationGateNotMet`: no error log, no banner, retried on the next
|
|
2393
|
+
open) instead of failing hard and stopping the registry."""
|
|
2394
|
+
import _cctally_journal
|
|
2395
|
+
try:
|
|
2396
|
+
return _cctally_journal.find_accounts_cutover_op()
|
|
2397
|
+
except OSError as exc:
|
|
2398
|
+
raise MigrationGateNotMet(
|
|
2399
|
+
f"the journal could not be read to find the accounts cutover op "
|
|
2400
|
+
f"({exc}); deferring the {what}") from exc
|
|
2401
|
+
|
|
2402
|
+
|
|
2359
2403
|
# === Region 3: 001 handler (was bin/cctally:11232-11344) ===
|
|
2360
2404
|
|
|
2361
2405
|
@stats_migration("001_five_hour_block_models_backfill_v1")
|
|
@@ -4743,6 +4787,125 @@ def _apply_codex_window_attribution_read_index(conn: sqlite3.Connection) -> None
|
|
|
4743
4787
|
# === Region 7b2: Eager cache-migration trigger (V4 — same-invocation 008 apply) ===
|
|
4744
4788
|
|
|
4745
4789
|
|
|
4790
|
+
# #968 spec 5.3: container evidence beside each Codex cursor row. Every
|
|
4791
|
+
# persisted key stays the LOGICAL ``.jsonl`` path; these columns record which
|
|
4792
|
+
# physical container's content the cursor accepted. Defaults describe what an
|
|
4793
|
+
# existing row already is: a plain file whose content was accepted. No evidence
|
|
4794
|
+
# column is backfilled by statting a file now, for the reason migration 046
|
|
4795
|
+
# gives for device_id/inode.
|
|
4796
|
+
_CODEX_CONTAINER_CURSOR_COLUMNS: "tuple[tuple[str, str], ...]" = (
|
|
4797
|
+
("container_kind", "TEXT NOT NULL DEFAULT 'plain'"),
|
|
4798
|
+
("container_path", "TEXT"),
|
|
4799
|
+
("container_size_bytes", "INTEGER"),
|
|
4800
|
+
("decoded_size_bytes", "INTEGER"),
|
|
4801
|
+
("content_anchor_version", "INTEGER"),
|
|
4802
|
+
("content_anchor", "BLOB"),
|
|
4803
|
+
("content_anchor_offset", "INTEGER"),
|
|
4804
|
+
("container_state", "TEXT NOT NULL DEFAULT 'accepted'"),
|
|
4805
|
+
("container_state_reason", "TEXT"),
|
|
4806
|
+
("pending_container_fingerprint", "TEXT"),
|
|
4807
|
+
)
|
|
4808
|
+
_CODEX_CONTAINER_CACHE_ONLY_COLUMNS: "tuple[tuple[str, str], ...]" = (
|
|
4809
|
+
("container_resume_strikes", "INTEGER NOT NULL DEFAULT 0"),
|
|
4810
|
+
)
|
|
4811
|
+
_CODEX_968_CACHE_TABLES = (
|
|
4812
|
+
"codex_prune_debt", "codex_barrier_membership", "codex_barrier_targets",
|
|
4813
|
+
"codex_barrier_population", "codex_barrier_certificates",
|
|
4814
|
+
"codex_barrier_repairs", "codex_barrier_entry_threads",
|
|
4815
|
+
)
|
|
4816
|
+
_CODEX_968_CONVERSATIONS_TABLES = (
|
|
4817
|
+
"codex_conversation_prune_debt", "codex_conversation_publications",
|
|
4818
|
+
"codex_conversation_published_threads",
|
|
4819
|
+
)
|
|
4820
|
+
#: The column list of `codex_conversation_threads`, shared by the two
|
|
4821
|
+
#: byte-preserving thread snapshots the publication barrier keeps: a thread
|
|
4822
|
+
#: row is copied with INSERT ... SELECT, never through JSON, so an undecodable
|
|
4823
|
+
#: cwd/git_json (#845) survives exactly as stored.
|
|
4824
|
+
_CODEX_THREAD_SNAPSHOT_COLUMNS_DDL = (
|
|
4825
|
+
" conversation_key TEXT NOT NULL PRIMARY KEY,"
|
|
4826
|
+
" source_root_key TEXT NOT NULL,"
|
|
4827
|
+
" native_thread_id TEXT NOT NULL,"
|
|
4828
|
+
" root_thread_id TEXT NOT NULL,"
|
|
4829
|
+
" parent_thread_id TEXT,"
|
|
4830
|
+
" source_path TEXT NOT NULL,"
|
|
4831
|
+
" cwd TEXT,"
|
|
4832
|
+
" git_json TEXT,"
|
|
4833
|
+
" source_kind TEXT,"
|
|
4834
|
+
" thread_source_json TEXT,"
|
|
4835
|
+
" model_provider TEXT,"
|
|
4836
|
+
" context_window INTEGER,"
|
|
4837
|
+
" first_seen_utc TEXT,"
|
|
4838
|
+
" last_seen_utc TEXT)"
|
|
4839
|
+
)
|
|
4840
|
+
_CODEX_968_CACHE_DDL = (
|
|
4841
|
+
"CREATE TABLE IF NOT EXISTS codex_prune_debt ("
|
|
4842
|
+
" source_path TEXT NOT NULL,"
|
|
4843
|
+
" source_root_key TEXT NOT NULL DEFAULT '',"
|
|
4844
|
+
" recorded_at_utc TEXT NOT NULL,"
|
|
4845
|
+
" PRIMARY KEY (source_path, source_root_key)) WITHOUT ROWID",
|
|
4846
|
+
"CREATE TABLE IF NOT EXISTS codex_barrier_membership ("
|
|
4847
|
+
" file_identity TEXT NOT NULL,"
|
|
4848
|
+
" generation_token TEXT NOT NULL,"
|
|
4849
|
+
" conversation_key TEXT NOT NULL,"
|
|
4850
|
+
" source_path TEXT NOT NULL,"
|
|
4851
|
+
" metadata_json TEXT,"
|
|
4852
|
+
" metadata_offset INTEGER,"
|
|
4853
|
+
" PRIMARY KEY (file_identity, generation_token, conversation_key))"
|
|
4854
|
+
" WITHOUT ROWID",
|
|
4855
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_barrier_membership_conversation"
|
|
4856
|
+
" ON codex_barrier_membership(conversation_key)",
|
|
4857
|
+
"CREATE TABLE IF NOT EXISTS codex_barrier_targets ("
|
|
4858
|
+
" file_identity TEXT PRIMARY KEY,"
|
|
4859
|
+
" source_path TEXT NOT NULL,"
|
|
4860
|
+
" source_root_key TEXT,"
|
|
4861
|
+
" generation_token TEXT NOT NULL,"
|
|
4862
|
+
" scan_target INTEGER NOT NULL,"
|
|
4863
|
+
" complete_line_end INTEGER,"
|
|
4864
|
+
" state TEXT NOT NULL CHECK (state IN ('required','ready','suspended')),"
|
|
4865
|
+
" reason TEXT,"
|
|
4866
|
+
" accepted_generation_token TEXT,"
|
|
4867
|
+
" accepted_complete_line_end INTEGER)",
|
|
4868
|
+
"CREATE TABLE IF NOT EXISTS codex_barrier_population ("
|
|
4869
|
+
" conversation_key TEXT PRIMARY KEY,"
|
|
4870
|
+
" required_revision INTEGER NOT NULL DEFAULT 1,"
|
|
4871
|
+
" entered_at_utc TEXT NOT NULL)",
|
|
4872
|
+
"CREATE TABLE IF NOT EXISTS codex_barrier_entry_threads ("
|
|
4873
|
+
+ _CODEX_THREAD_SNAPSHOT_COLUMNS_DDL,
|
|
4874
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_barrier_entry_threads_parent"
|
|
4875
|
+
" ON codex_barrier_entry_threads"
|
|
4876
|
+
"(source_root_key, native_thread_id, conversation_key)",
|
|
4877
|
+
"CREATE TABLE IF NOT EXISTS codex_barrier_certificates ("
|
|
4878
|
+
" conversation_key TEXT NOT NULL,"
|
|
4879
|
+
" revision INTEGER NOT NULL,"
|
|
4880
|
+
" certificate_json TEXT NOT NULL,"
|
|
4881
|
+
" sealed_at_utc TEXT NOT NULL,"
|
|
4882
|
+
" PRIMARY KEY (conversation_key, revision)) WITHOUT ROWID",
|
|
4883
|
+
"CREATE TABLE IF NOT EXISTS codex_barrier_repairs ("
|
|
4884
|
+
" conversation_key TEXT PRIMARY KEY,"
|
|
4885
|
+
" revision INTEGER NOT NULL,"
|
|
4886
|
+
" reason TEXT NOT NULL)",
|
|
4887
|
+
)
|
|
4888
|
+
_CODEX_968_CONVERSATIONS_DDL = (
|
|
4889
|
+
"CREATE TABLE IF NOT EXISTS codex_conversation_prune_debt ("
|
|
4890
|
+
" source_path TEXT NOT NULL,"
|
|
4891
|
+
" source_root_key TEXT NOT NULL DEFAULT '',"
|
|
4892
|
+
" recorded_at_utc TEXT NOT NULL,"
|
|
4893
|
+
" PRIMARY KEY (source_path, source_root_key)) WITHOUT ROWID",
|
|
4894
|
+
"CREATE TABLE IF NOT EXISTS codex_conversation_publications ("
|
|
4895
|
+
" conversation_key TEXT PRIMARY KEY,"
|
|
4896
|
+
" required_revision INTEGER NOT NULL DEFAULT 0,"
|
|
4897
|
+
" ready_certificate_revision INTEGER,"
|
|
4898
|
+
" published_revision INTEGER NOT NULL DEFAULT 0,"
|
|
4899
|
+
" legacy_uncertified INTEGER NOT NULL DEFAULT 0,"
|
|
4900
|
+
" updated_at_utc TEXT NOT NULL)",
|
|
4901
|
+
"CREATE TABLE IF NOT EXISTS codex_conversation_published_threads ("
|
|
4902
|
+
+ _CODEX_THREAD_SNAPSHOT_COLUMNS_DDL,
|
|
4903
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_published_threads_parent"
|
|
4904
|
+
" ON codex_conversation_published_threads"
|
|
4905
|
+
"(source_root_key, native_thread_id, conversation_key)",
|
|
4906
|
+
)
|
|
4907
|
+
|
|
4908
|
+
|
|
4746
4909
|
class SchemaDeliveryObject(NamedTuple):
|
|
4747
4910
|
"""One re-derivable schema object declared by a version-gated apply.
|
|
4748
4911
|
|
|
@@ -4928,6 +5091,21 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
4928
5091
|
"column", "codex_session_files.inode",
|
|
4929
5092
|
"_apply_codex_session_file_identity",
|
|
4930
5093
|
"046_codex_source_file_identity"),
|
|
5094
|
+
*(SchemaDeliveryObject(
|
|
5095
|
+
"column", f"codex_session_files.{column}",
|
|
5096
|
+
"_apply_codex_container_cache_schema", "049_codex_rollout_containers")
|
|
5097
|
+
for column, _decl in (_CODEX_CONTAINER_CURSOR_COLUMNS
|
|
5098
|
+
+ _CODEX_CONTAINER_CACHE_ONLY_COLUMNS)),
|
|
5099
|
+
*(SchemaDeliveryObject(
|
|
5100
|
+
"table", table, "_apply_codex_container_cache_schema",
|
|
5101
|
+
"049_codex_rollout_containers")
|
|
5102
|
+
for table in _CODEX_968_CACHE_TABLES),
|
|
5103
|
+
SchemaDeliveryObject(
|
|
5104
|
+
"index", "idx_codex_barrier_membership_conversation",
|
|
5105
|
+
"_apply_codex_container_cache_schema", "049_codex_rollout_containers"),
|
|
5106
|
+
SchemaDeliveryObject(
|
|
5107
|
+
"index", "idx_codex_barrier_entry_threads_parent",
|
|
5108
|
+
"_apply_codex_container_cache_schema", "049_codex_rollout_containers"),
|
|
4931
5109
|
SchemaDeliveryObject(
|
|
4932
5110
|
"index", "idx_codex_accounting_change_mutation",
|
|
4933
5111
|
"_apply_codex_accounting_change_ledger",
|
|
@@ -5034,6 +5212,79 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
5034
5212
|
)
|
|
5035
5213
|
|
|
5036
5214
|
|
|
5215
|
+
def _apply_codex_container_cache_schema(conn: sqlite3.Connection) -> None:
|
|
5216
|
+
"""#968 spec 5.3: container columns and the cache-side maintenance state.
|
|
5217
|
+
|
|
5218
|
+
Shared by ``_apply_cache_schema`` and migration 049 so both shapes
|
|
5219
|
+
converge. Columns go through ``add_column_if_missing``; the cursor table is
|
|
5220
|
+
checked first because that helper raises on a missing table, and a table
|
|
5221
|
+
created later is created by the schema apply that calls this helper anyway.
|
|
5222
|
+
"""
|
|
5223
|
+
# Literal column names keep the schema-delivery AST guard complete.
|
|
5224
|
+
if _table_exists(conn, "codex_session_files"):
|
|
5225
|
+
add_column_if_missing(
|
|
5226
|
+
conn, "codex_session_files", "container_kind",
|
|
5227
|
+
"TEXT NOT NULL DEFAULT 'plain'")
|
|
5228
|
+
add_column_if_missing(
|
|
5229
|
+
conn, "codex_session_files", "container_path", "TEXT")
|
|
5230
|
+
add_column_if_missing(
|
|
5231
|
+
conn, "codex_session_files", "container_size_bytes", "INTEGER")
|
|
5232
|
+
add_column_if_missing(
|
|
5233
|
+
conn, "codex_session_files", "decoded_size_bytes", "INTEGER")
|
|
5234
|
+
add_column_if_missing(
|
|
5235
|
+
conn, "codex_session_files", "content_anchor_version", "INTEGER")
|
|
5236
|
+
add_column_if_missing(
|
|
5237
|
+
conn, "codex_session_files", "content_anchor", "BLOB")
|
|
5238
|
+
add_column_if_missing(
|
|
5239
|
+
conn, "codex_session_files", "content_anchor_offset", "INTEGER")
|
|
5240
|
+
add_column_if_missing(
|
|
5241
|
+
conn, "codex_session_files", "container_state",
|
|
5242
|
+
"TEXT NOT NULL DEFAULT 'accepted'")
|
|
5243
|
+
add_column_if_missing(
|
|
5244
|
+
conn, "codex_session_files", "container_state_reason", "TEXT")
|
|
5245
|
+
add_column_if_missing(
|
|
5246
|
+
conn, "codex_session_files", "pending_container_fingerprint", "TEXT")
|
|
5247
|
+
add_column_if_missing(
|
|
5248
|
+
conn, "codex_session_files", "container_resume_strikes",
|
|
5249
|
+
"INTEGER NOT NULL DEFAULT 0")
|
|
5250
|
+
for statement in _CODEX_968_CACHE_DDL:
|
|
5251
|
+
conn.execute(statement)
|
|
5252
|
+
|
|
5253
|
+
|
|
5254
|
+
def _apply_codex_conversation_container_schema(conn: sqlite3.Connection) -> None:
|
|
5255
|
+
"""#968 spec 5.3: the conversations twin of the container evidence.
|
|
5256
|
+
|
|
5257
|
+
Shared by ``_apply_conversations_schema`` (fast path and full path) and
|
|
5258
|
+
conversations migration 012.
|
|
5259
|
+
"""
|
|
5260
|
+
# Keep these in the same order as _CODEX_CONTAINER_CURSOR_COLUMNS.
|
|
5261
|
+
if _table_exists(conn, "codex_conversation_source_files"):
|
|
5262
|
+
add_column_if_missing(
|
|
5263
|
+
conn, "codex_conversation_source_files", "container_kind",
|
|
5264
|
+
"TEXT NOT NULL DEFAULT 'plain'")
|
|
5265
|
+
add_column_if_missing(
|
|
5266
|
+
conn, "codex_conversation_source_files", "container_path", "TEXT")
|
|
5267
|
+
add_column_if_missing(
|
|
5268
|
+
conn, "codex_conversation_source_files", "container_size_bytes", "INTEGER")
|
|
5269
|
+
add_column_if_missing(
|
|
5270
|
+
conn, "codex_conversation_source_files", "decoded_size_bytes", "INTEGER")
|
|
5271
|
+
add_column_if_missing(
|
|
5272
|
+
conn, "codex_conversation_source_files", "content_anchor_version", "INTEGER")
|
|
5273
|
+
add_column_if_missing(
|
|
5274
|
+
conn, "codex_conversation_source_files", "content_anchor", "BLOB")
|
|
5275
|
+
add_column_if_missing(
|
|
5276
|
+
conn, "codex_conversation_source_files", "content_anchor_offset", "INTEGER")
|
|
5277
|
+
add_column_if_missing(
|
|
5278
|
+
conn, "codex_conversation_source_files", "container_state",
|
|
5279
|
+
"TEXT NOT NULL DEFAULT 'accepted'")
|
|
5280
|
+
add_column_if_missing(
|
|
5281
|
+
conn, "codex_conversation_source_files", "container_state_reason", "TEXT")
|
|
5282
|
+
add_column_if_missing(
|
|
5283
|
+
conn, "codex_conversation_source_files", "pending_container_fingerprint", "TEXT")
|
|
5284
|
+
for statement in _CODEX_968_CONVERSATIONS_DDL:
|
|
5285
|
+
conn.execute(statement)
|
|
5286
|
+
|
|
5287
|
+
|
|
5037
5288
|
def _apply_codex_session_file_identity(conn: sqlite3.Connection) -> None:
|
|
5038
5289
|
"""Deliver the Codex accounting cursor's file identity (#769 S6, #716 A).
|
|
5039
5290
|
|
|
@@ -5605,6 +5856,7 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
5605
5856
|
conn, "codex_session_files", "ingest_complete",
|
|
5606
5857
|
"INTEGER NOT NULL DEFAULT 1")
|
|
5607
5858
|
_apply_codex_session_file_identity(conn)
|
|
5859
|
+
_apply_codex_container_cache_schema(conn)
|
|
5608
5860
|
add_column_if_missing(conn, "quota_window_snapshots", "account_key", "TEXT")
|
|
5609
5861
|
# #195: the cache-write TTL split. NULLable with NO DEFAULT — NULL is the
|
|
5610
5862
|
# "split unknown" sentinel a pre-#195 row produces for free, and a real
|
|
@@ -5898,6 +6150,7 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5898
6150
|
_apply_conversation_generation_schema(conn)
|
|
5899
6151
|
_apply_codex_conversation_source_identity(conn)
|
|
5900
6152
|
_apply_conversation_latest_meta_indexes(conn)
|
|
6153
|
+
_apply_codex_conversation_container_schema(conn)
|
|
5901
6154
|
return
|
|
5902
6155
|
|
|
5903
6156
|
_apply_cache_schema(conn)
|
|
@@ -5914,6 +6167,14 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5914
6167
|
-- it), so this only clears the now-orphan ledger and keeps
|
|
5915
6168
|
-- conversations.db transcripts-only.
|
|
5916
6169
|
DROP TABLE IF EXISTS quota_window_change_log;
|
|
6170
|
+
-- #968: only the conversation twins belong in this store.
|
|
6171
|
+
DROP TABLE IF EXISTS codex_prune_debt;
|
|
6172
|
+
DROP TABLE IF EXISTS codex_barrier_membership;
|
|
6173
|
+
DROP TABLE IF EXISTS codex_barrier_targets;
|
|
6174
|
+
DROP TABLE IF EXISTS codex_barrier_population;
|
|
6175
|
+
DROP TABLE IF EXISTS codex_barrier_certificates;
|
|
6176
|
+
DROP TABLE IF EXISTS codex_barrier_repairs;
|
|
6177
|
+
DROP TABLE IF EXISTS codex_barrier_entry_threads;
|
|
5917
6178
|
DROP TABLE IF EXISTS codex_accounting_change_log;
|
|
5918
6179
|
DROP TABLE IF EXISTS codex_conversation_threads;
|
|
5919
6180
|
DROP TABLE IF EXISTS codex_source_roots;
|
|
@@ -5987,6 +6248,7 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5987
6248
|
_apply_conversation_generation_schema(conn)
|
|
5988
6249
|
_apply_codex_conversation_source_identity(conn)
|
|
5989
6250
|
_apply_conversation_latest_meta_indexes(conn)
|
|
6251
|
+
_apply_codex_conversation_container_schema(conn)
|
|
5990
6252
|
|
|
5991
6253
|
|
|
5992
6254
|
def _ensure_codex_session_meta_provenance_index(
|
|
@@ -6364,6 +6626,19 @@ CONVERSATIONS_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
6364
6626
|
"index", "idx_conv_session_latest_git_branch",
|
|
6365
6627
|
"_apply_conversation_latest_meta_indexes",
|
|
6366
6628
|
"011_conversation_latest_meta_indexes"),
|
|
6629
|
+
*(SchemaDeliveryObject(
|
|
6630
|
+
"column", f"codex_conversation_source_files.{column}",
|
|
6631
|
+
"_apply_codex_conversation_container_schema",
|
|
6632
|
+
"012_codex_conversation_containers")
|
|
6633
|
+
for column, _decl in _CODEX_CONTAINER_CURSOR_COLUMNS),
|
|
6634
|
+
*(SchemaDeliveryObject(
|
|
6635
|
+
"table", table, "_apply_codex_conversation_container_schema",
|
|
6636
|
+
"012_codex_conversation_containers")
|
|
6637
|
+
for table in _CODEX_968_CONVERSATIONS_TABLES),
|
|
6638
|
+
SchemaDeliveryObject(
|
|
6639
|
+
"index", "idx_codex_published_threads_parent",
|
|
6640
|
+
"_apply_codex_conversation_container_schema",
|
|
6641
|
+
"012_codex_conversation_containers"),
|
|
6367
6642
|
SchemaDeliveryObject(
|
|
6368
6643
|
"table", "claude_conversation_account_stamp_gaps",
|
|
6369
6644
|
"_apply_conversation_generation_schema",
|
|
@@ -6573,7 +6848,8 @@ def _conv_005_conversation_account_dimension(conn: sqlite3.Connection) -> None:
|
|
|
6573
6848
|
held += _acquire_conversations_db_codex_provider_flock(
|
|
6574
6849
|
conn, migration="conversations 005 account dimension"
|
|
6575
6850
|
)
|
|
6576
|
-
claude_key =
|
|
6851
|
+
claude_key = _accounts_cutover_value_or_defer(
|
|
6852
|
+
"Claude conversation backfill")
|
|
6577
6853
|
if claude_key is None:
|
|
6578
6854
|
pending = conn.execute(
|
|
6579
6855
|
"SELECT 1 FROM conversation_messages "
|
|
@@ -7030,6 +7306,20 @@ def _conv_011_conversation_latest_meta_indexes(
|
|
|
7030
7306
|
_release_cache_db_writer_flocks(held)
|
|
7031
7307
|
|
|
7032
7308
|
|
|
7309
|
+
@conversations_migration("012_codex_conversation_containers")
|
|
7310
|
+
def _conv_012_codex_conversation_containers(conn: sqlite3.Connection) -> None:
|
|
7311
|
+
"""#968 spec 5.3: the conversations twin of cache migration 049.
|
|
7312
|
+
|
|
7313
|
+
Delivers the transcript cursor's container columns, the conversation-side
|
|
7314
|
+
prune debt and the last-good publication records to an already-current
|
|
7315
|
+
store. Same version-gate reason and the same no-backfill rule as 049; no
|
|
7316
|
+
provider flock for the same reason as 010. Re-running is a no-op. NO
|
|
7317
|
+
self-stamp — the dispatcher central-stamps on a clean return (#140).
|
|
7318
|
+
"""
|
|
7319
|
+
_apply_codex_conversation_container_schema(conn)
|
|
7320
|
+
conn.commit()
|
|
7321
|
+
|
|
7322
|
+
|
|
7033
7323
|
# #177 S6: the consolidated multi-column external-content FTS5 table that
|
|
7034
7324
|
# replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
|
|
7035
7325
|
# pair. The three column names MUST match the conversation_messages columns BY
|
|
@@ -8864,6 +9154,13 @@ def _027_codex_fork_preamble_rebuild(conn: sqlite3.Connection) -> None:
|
|
|
8864
9154
|
_release_cache_db_writer_flocks(held)
|
|
8865
9155
|
|
|
8866
9156
|
|
|
9157
|
+
#: Pages 028 returns per committed transaction when it reclaims the dropped
|
|
9158
|
+
#: transcript store. The chunk bounds each transaction's WAL growth at roughly,
|
|
9159
|
+
#: but more than, one SQLite WAL auto-checkpoint (1000 frames): each moved page
|
|
9160
|
+
#: also writes its pointer-map page, its parent and the header page.
|
|
9161
|
+
_028_VACUUM_CHUNK_PAGES = 1000
|
|
9162
|
+
|
|
9163
|
+
|
|
8867
9164
|
@cache_migration("028_split_conversation_store")
|
|
8868
9165
|
def _028_split_conversation_store(conn: sqlite3.Connection) -> None:
|
|
8869
9166
|
"""Move the re-derivable transcript corpus out of cache.db (#320).
|
|
@@ -8973,11 +9270,45 @@ def _028_split_conversation_store(conn: sqlite3.Connection) -> None:
|
|
|
8973
9270
|
raise
|
|
8974
9271
|
try:
|
|
8975
9272
|
if conn.execute("PRAGMA auto_vacuum").fetchone()[0] == 2:
|
|
8976
|
-
#
|
|
8977
|
-
#
|
|
8978
|
-
|
|
9273
|
+
# Return the dropped tables' pages one page per statement. A
|
|
9274
|
+
# bare `PRAGMA incremental_vacuum` returns no columns, and
|
|
9275
|
+
# CPython 3.11's cursor steps such a statement exactly once,
|
|
9276
|
+
# fetchall() or not, so it frees ONE page (#998). A statement
|
|
9277
|
+
# capped at one page is complete after that first step on every
|
|
9278
|
+
# Python (the `_run_incremental_vacuum_chunk` pattern). Each
|
|
9279
|
+
# chunk commits so WAL auto-checkpoints run between chunks, and
|
|
9280
|
+
# the loop stops when a chunk frees nothing, so it is bounded.
|
|
9281
|
+
remaining = conn.execute("PRAGMA freelist_count").fetchone()[0]
|
|
9282
|
+
while remaining:
|
|
9283
|
+
# Any failure inside a chunk, including a non-database
|
|
9284
|
+
# error or an interrupt, rolls that chunk back before it
|
|
9285
|
+
# propagates, so no write transaction outlives the
|
|
9286
|
+
# conversation locks released below.
|
|
9287
|
+
try:
|
|
9288
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
9289
|
+
for _ in range(min(remaining, _028_VACUUM_CHUNK_PAGES)):
|
|
9290
|
+
conn.execute("PRAGMA incremental_vacuum(1)").fetchall()
|
|
9291
|
+
conn.commit()
|
|
9292
|
+
except BaseException:
|
|
9293
|
+
try:
|
|
9294
|
+
conn.rollback()
|
|
9295
|
+
except sqlite3.Error:
|
|
9296
|
+
pass
|
|
9297
|
+
raise
|
|
9298
|
+
left = conn.execute("PRAGMA freelist_count").fetchone()[0]
|
|
9299
|
+
if left >= remaining:
|
|
9300
|
+
break
|
|
9301
|
+
remaining = left
|
|
8979
9302
|
except sqlite3.DatabaseError:
|
|
8980
|
-
|
|
9303
|
+
# Intentional: the reclaim is best effort. Raising here would leave
|
|
9304
|
+
# 028 unapplied, and its next run would re-arm both conversation
|
|
9305
|
+
# store rebuild-from-scratch flags; free pages left behind are
|
|
9306
|
+
# harmless and `db vacuum` returns them. The guarded rollback keeps
|
|
9307
|
+
# a failing rollback from turning that into a handler failure.
|
|
9308
|
+
try:
|
|
9309
|
+
conn.rollback()
|
|
9310
|
+
except sqlite3.Error:
|
|
9311
|
+
pass
|
|
8981
9312
|
finally:
|
|
8982
9313
|
for fh in reversed(held):
|
|
8983
9314
|
try:
|
|
@@ -9012,7 +9343,7 @@ def _029_backfill_claude_account(conn: sqlite3.Connection) -> None:
|
|
|
9012
9343
|
import _cctally_journal as _jr
|
|
9013
9344
|
import _lib_accounts as _acc
|
|
9014
9345
|
|
|
9015
|
-
op_value =
|
|
9346
|
+
op_value = _accounts_cutover_value_or_defer("Claude account backfill")
|
|
9016
9347
|
if op_value is None:
|
|
9017
9348
|
# Cutover hasn't appended its op yet. Defer only if there is legacy Claude
|
|
9018
9349
|
# history to attribute; a fresh install (no such rows) proceeds as a no-op.
|
|
@@ -9918,6 +10249,25 @@ def _048_codex_quota_physical_group_order(conn: sqlite3.Connection) -> None:
|
|
|
9918
10249
|
conn.commit()
|
|
9919
10250
|
|
|
9920
10251
|
|
|
10252
|
+
@cache_migration("049_codex_rollout_containers")
|
|
10253
|
+
def _049_codex_rollout_containers(conn: sqlite3.Connection) -> None:
|
|
10254
|
+
"""#968 spec 5.3: deliver container evidence and maintenance state.
|
|
10255
|
+
|
|
10256
|
+
The schema apply that declares these objects is version-gated, so only a
|
|
10257
|
+
registered migration reaches an already-current store (the reason 046
|
|
10258
|
+
gives). Existing cursor rows read as plain and accepted, which is exactly
|
|
10259
|
+
what they are; no container evidence is backfilled from a stat taken now.
|
|
10260
|
+
The seven tables start empty: prune debt, verification state and the
|
|
10261
|
+
publication barrier are written only by the runtime paths that own them.
|
|
10262
|
+
|
|
10263
|
+
Takes no provider flock: ADD COLUMN with a constant default and CREATE of
|
|
10264
|
+
an empty table rewrite no rows. Re-running is a no-op. NO self-stamp — the
|
|
10265
|
+
dispatcher central-stamps on a clean return (#140).
|
|
10266
|
+
"""
|
|
10267
|
+
_apply_codex_container_cache_schema(conn)
|
|
10268
|
+
conn.commit()
|
|
10269
|
+
|
|
10270
|
+
|
|
9921
10271
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
9922
10272
|
|
|
9923
10273
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -11003,6 +11353,9 @@ def cmd_db_status(args: argparse.Namespace) -> int:
|
|
|
11003
11353
|
),
|
|
11004
11354
|
},
|
|
11005
11355
|
}
|
|
11356
|
+
# #997 D10: additive — each detached rebuild worker's state and its
|
|
11357
|
+
# read-only classification (the doctor `db.rebuild_worker` kernel).
|
|
11358
|
+
payload["rebuildWorkers"] = _db_status_rebuild_workers()
|
|
11006
11359
|
if getattr(args, "json", False):
|
|
11007
11360
|
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
11008
11361
|
return 0
|
|
@@ -11020,9 +11373,46 @@ def cmd_db_status(args: argparse.Namespace) -> int:
|
|
|
11020
11373
|
line = _db_status_format_row(m)
|
|
11021
11374
|
print(line)
|
|
11022
11375
|
print() # blank line between DBs
|
|
11376
|
+
for worker in ("epoch", "heal"):
|
|
11377
|
+
info = payload["rebuildWorkers"][worker]
|
|
11378
|
+
if info["classification"]["row"] != "idle":
|
|
11379
|
+
print(info["classification"]["summary"])
|
|
11023
11380
|
return 0
|
|
11024
11381
|
|
|
11025
11382
|
|
|
11383
|
+
def _db_status_rebuild_workers() -> dict:
|
|
11384
|
+
"""``{"epoch": {...}, "heal": {...}}``: each worker's run state and its
|
|
11385
|
+
classification, gathered without any lock (#997 D8/D10)."""
|
|
11386
|
+
import _cctally_store
|
|
11387
|
+
import _lib_doctor
|
|
11388
|
+
|
|
11389
|
+
now_utc = _cctally_core._command_as_of()
|
|
11390
|
+
out: dict = {}
|
|
11391
|
+
for worker in ("epoch", "heal"):
|
|
11392
|
+
try:
|
|
11393
|
+
snapshot = _cctally_store.rebuild_worker_snapshot(worker)
|
|
11394
|
+
except Exception: # noqa: BLE001 — a diagnostic never fails status
|
|
11395
|
+
snapshot = None
|
|
11396
|
+
assessed = _lib_doctor.assess_rebuild_worker(worker, snapshot,
|
|
11397
|
+
now_utc)
|
|
11398
|
+
details = assessed["details"]
|
|
11399
|
+
out[worker] = {
|
|
11400
|
+
"state": details["state"],
|
|
11401
|
+
"stateUnreadable": details["stateUnreadable"],
|
|
11402
|
+
"ledger": details["ledger"],
|
|
11403
|
+
"liveness": details["liveness"],
|
|
11404
|
+
"pending": details["pending"],
|
|
11405
|
+
"recoveredDeaths": details["recoveredDeaths"],
|
|
11406
|
+
"classification": {
|
|
11407
|
+
"severity": assessed["severity"],
|
|
11408
|
+
"row": assessed["row"],
|
|
11409
|
+
"summary": assessed["summary"],
|
|
11410
|
+
"warnings": assessed["warnings"],
|
|
11411
|
+
},
|
|
11412
|
+
}
|
|
11413
|
+
return out
|
|
11414
|
+
|
|
11415
|
+
|
|
11026
11416
|
def _db_status_for(
|
|
11027
11417
|
db_path: pathlib.Path,
|
|
11028
11418
|
registry: list[Migration],
|
|
@@ -11889,6 +12279,8 @@ def _db_family_open_pids(path: pathlib.Path) -> "set[int] | None":
|
|
|
11889
12279
|
)
|
|
11890
12280
|
except OSError:
|
|
11891
12281
|
return None
|
|
12282
|
+
if result.returncode not in (0, 1):
|
|
12283
|
+
return None
|
|
11892
12284
|
return {
|
|
11893
12285
|
int(line[1:])
|
|
11894
12286
|
for line in result.stdout.splitlines()
|