cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -256,6 +256,17 @@ class DoctorState:
256
256
  # one of them.
257
257
  cache_rollup_pricing_refusal: Optional[dict] = None
258
258
  cache_rollup_pricing_fp: Optional[str] = None
259
+ # #728: the four-state `_lib_pricing.PricingFingerprintObservation` of each
260
+ # store's fingerprint READ, one per store. The two `_fp` fields above carry
261
+ # a value and therefore cannot say whether the read happened at all: a
262
+ # `sqlite3.OperationalError` collapsed to None, which is also what a store
263
+ # that recorded nothing produces, so a locked or schema-less store reported
264
+ # "no refused pricing write recorded" and doctor exited OK over a store it
265
+ # had no evidence about. Typed loosely so this kernel keeps no import of
266
+ # the pricing leaf at module scope; the check reads `.state` and
267
+ # `.error_kind`. None means the gather did not produce one.
268
+ conversation_rollup_pricing_observation: Optional[object] = None
269
+ cache_rollup_pricing_observation: Optional[object] = None
259
270
  # #279 S2 (F5b): PRAGMA quick_check(1) results, gathered ONLY under
260
271
  # doctor_gather_state(deep=True) (CLI cmd_doctor) — the dashboard
261
272
  # rebuild loop calls the gather every rebuild and quick_check on a
@@ -294,6 +305,10 @@ class DoctorState:
294
305
  # visibility, with remediation targeted at its own VACUUM surface.
295
306
  conversations_db_page_count: Optional[int] = None
296
307
  conversations_db_freelist_count: Optional[int] = None
308
+ #: #780 — the durable reclaim backlog record, or None when nothing is
309
+ #: pending. Absent on every store that has never fallen behind, which is
310
+ #: why the check below adds nothing to its output in that case.
311
+ conversations_reclaim_pending: Optional[dict] = None
297
312
  # #294 S2: root-qualified physical Codex quota freshness, per-root native
298
313
  # hook state, and lifecycle activity are gathered by _cctally_doctor.
299
314
  codex_quota_windows: Optional[list[dict]] = None
@@ -996,7 +1011,7 @@ def _check_db_version_ahead(s: DoctorState) -> CheckResult:
996
1011
  # current index as a mismatch; keep this in lockstep with core
997
1012
  # (#496 S5b §6.1). It stays a literal because this kernel is pure and
998
1013
  # must not import `_cctally_core`.
999
- epoch = 1013
1014
+ epoch = 1015
1000
1015
  mismatch = uv > legacy_head and uv != epoch
1001
1016
  return {"user_version": uv, "legacy_head": legacy_head, "epoch": epoch,
1002
1017
  "mismatch": mismatch}
@@ -1470,19 +1485,33 @@ def _check_data_codex_project_metadata(s: DoctorState) -> CheckResult:
1470
1485
  qualified_rows = int(health.get("qualified_rows", 0))
1471
1486
  missing_key_rows = int(health.get("missing_conversation_key_rows", 0))
1472
1487
  missing_join_rows = int(health.get("missing_thread_join_rows", 0))
1473
- incomplete_rows = missing_key_rows + missing_join_rows
1488
+ # #845 §4.3: a third reason, counted all-history like the other two. The
1489
+ # key is additive, so an older consumer reading this JSON keeps working.
1490
+ undecodable_rows = int(health.get("undecodable_metadata_rows", 0))
1491
+ malformed_accounting_rows = int(
1492
+ health.get("malformed_accounting_rows", 0))
1493
+ incomplete_rows = (
1494
+ missing_key_rows + missing_join_rows + undecodable_rows
1495
+ + malformed_accounting_rows)
1474
1496
  details = {
1475
1497
  "total_rows": total_rows,
1476
1498
  "qualified_rows": qualified_rows,
1477
1499
  "missing_conversation_key_rows": missing_key_rows,
1478
1500
  "missing_thread_join_rows": missing_join_rows,
1501
+ "undecodable_metadata_rows": undecodable_rows,
1479
1502
  "incomplete_rows": incomplete_rows,
1480
1503
  }
1504
+ if malformed_accounting_rows:
1505
+ details["malformed_accounting_rows"] = malformed_accounting_rows
1481
1506
  if incomplete_rows:
1482
1507
  return CheckResult(
1483
1508
  id="data.codex_project_metadata", title="Codex project metadata",
1484
1509
  severity="warn",
1485
- summary=f"{incomplete_rows} accounting row(s) need metadata repair",
1510
+ summary=(
1511
+ f"{incomplete_rows} accounting row(s) need repair"
1512
+ if malformed_accounting_rows else
1513
+ f"{incomplete_rows} accounting row(s) need metadata repair"
1514
+ ),
1486
1515
  remediation="Run `cctally cache-sync --source codex --rebuild`.",
1487
1516
  details=details,
1488
1517
  )
@@ -2048,6 +2077,101 @@ ROLLUP_WRITER_UNPARSEABLE_FP_REMEDIATION = (
2048
2077
  "are degraded."
2049
2078
  )
2050
2079
 
2080
+ #: The `--db` value `cctally db checkpoint` takes for each store this check
2081
+ #: reports. The check always knows which store it is talking about, so the
2082
+ #: remediation names that one rather than naming a fixed store and being wrong
2083
+ #: half the time.
2084
+ _ROLLUP_WRITER_STORE_DB_FLAG = {
2085
+ "conversations.db": "conversations",
2086
+ "cache.db": "cache",
2087
+ }
2088
+
2089
+
2090
+ def _rollup_writer_store_db_flag(store: str) -> str:
2091
+ """The `db checkpoint --db` value for a store name, defaulting to
2092
+ `conversations` for an unrecognised one so the text stays a whole
2093
+ sentence."""
2094
+ return _ROLLUP_WRITER_STORE_DB_FLAG.get(store, "conversations")
2095
+
2096
+
2097
+ #: Remediation for a fingerprint READ that did not happen (#728). Neither of
2098
+ #: the two remediations above applies: there is no unparseable row to delete,
2099
+ #: and no refusal has been latched, because the process that would latch one is
2100
+ #: equally unable to read the store. The step is to release whatever is holding
2101
+ #: it and look again.
2102
+ _ROLLUP_WRITER_DEGRADED_READ_REMEDIATION_TEMPLATE = (
2103
+ "cctally could not read the pricing fingerprint recorded in the rollup "
2104
+ "(`conversation_sessions_pricing_fp` in the `cache_meta` table of "
2105
+ "`{store}`), so it cannot say whether that store's materialized cost is "
2106
+ "protected. This is usually a store held by another process: check for a "
2107
+ "`cctally dashboard` or a long ingest running against this installation, "
2108
+ "and run `cctally db checkpoint --db {db}` if the WAL has grown. "
2109
+ "Run `cctally doctor` again once the store is quiet. While the read keeps "
2110
+ "failing every rollup writer refuses rather than assuming the store had "
2111
+ "nothing to protect, so cost sorting and project filtering may be "
2112
+ "degraded, but no session is hidden."
2113
+ )
2114
+
2115
+ #: Remediation for a refusal that was LATCHED by a failed read, reported once
2116
+ #: the store reads cleanly again (#769 S3, Tranche 1 review). Distinct from
2117
+ #: both neighbours: there is no unparseable row to delete, and the version-skew
2118
+ #: remedy above is advice about a pricing table when the refusal was never
2119
+ #: about one. A refusal records the store date its read produced, and a failed
2120
+ #: read produces none, so a null `store_snapshot_date` identifies this case
2121
+ #: exactly — an ABSENT fingerprint is authorized and latches no record at all.
2122
+ _ROLLUP_WRITER_REFUSED_DEGRADED_READ_REMEDIATION_TEMPLATE = (
2123
+ "A rollup write to `{store}` was refused because cctally could not read "
2124
+ "that store's recorded pricing fingerprint "
2125
+ "(`conversation_sessions_pricing_fp` in its `cache_meta` table) at the "
2126
+ "time, so it had no evidence about what the store's materialized cost "
2127
+ "needed protecting from. The store reads cleanly now, so there is no "
2128
+ "unparseable value to delete and no version difference to resolve — "
2129
+ "restarting or upgrading the writing process is not the step here. The "
2130
+ "next authorized sync that recomputes the rollup clears this record: on "
2131
+ "its next tick when the refusal also armed the rollup backfill, and "
2132
+ "otherwise on the next sync that touches a session. If it keeps coming "
2133
+ "back, the store is intermittently unreadable — check for another process "
2134
+ "holding it, and run `cctally db checkpoint --db {db}` if the WAL has "
2135
+ "grown. While the backfill is armed the conversation list falls back to "
2136
+ "live aggregation, so every session stays visible, but cost sorting and "
2137
+ "project filtering are degraded."
2138
+ )
2139
+
2140
+
2141
+ def rollup_writer_degraded_read_remediation(store: str) -> str:
2142
+ """The failed-read remediation, naming the store it is about (#769 S3)."""
2143
+ return _ROLLUP_WRITER_DEGRADED_READ_REMEDIATION_TEMPLATE.format(
2144
+ store=store, db=_rollup_writer_store_db_flag(store))
2145
+
2146
+
2147
+ def rollup_writer_refused_degraded_read_remediation(store: str) -> str:
2148
+ """The remediation for a refusal a failed read latched (#769 S3)."""
2149
+ return _ROLLUP_WRITER_REFUSED_DEGRADED_READ_REMEDIATION_TEMPLATE.format(
2150
+ store=store, db=_rollup_writer_store_db_flag(store))
2151
+
2152
+
2153
+ def _refusal_latched_by_a_failed_read(record) -> bool:
2154
+ """Whether a refusal record describes a read that failed (#769 S3).
2155
+
2156
+ `_record_pricing_write_refusal` stores whatever the observation's raw value
2157
+ was, and a DEGRADED observation carries None. Every other refusal state
2158
+ names a value: MALFORMED records the unparseable text and a store-is-newer
2159
+ refusal records the newer date. An ABSENT fingerprint is authorized, so it
2160
+ never reaches the record at all. A null `store_snapshot_date` is therefore
2161
+ a positive discriminator rather than a missing field.
2162
+ """
2163
+ return isinstance(record, dict) and record.get("store_snapshot_date") is None
2164
+
2165
+
2166
+ def _degraded_pricing_read(observation) -> bool:
2167
+ """Whether a store's fingerprint read failed (#728).
2168
+
2169
+ Reads the observation defensively: a `DoctorState` assembled before this
2170
+ field existed, or by a gather that could not build one, carries None, and
2171
+ that must report as "no evidence of a failed read" rather than raise.
2172
+ """
2173
+ return getattr(observation, "state", None) == "degraded"
2174
+
2051
2175
 
2052
2176
  def _check_pricing_conversation_rollup_writer(s: DoctorState) -> CheckResult:
2053
2177
  """WARN when a process holding older pricing was refused a write to the
@@ -2109,11 +2233,13 @@ def _check_pricing_conversation_rollup_writer(s: DoctorState) -> CheckResult:
2109
2233
 
2110
2234
  stores = (
2111
2235
  ("conversations.db", s.conversation_rollup_pricing_fp,
2112
- s.conversation_rollup_pricing_refusal),
2236
+ s.conversation_rollup_pricing_refusal,
2237
+ s.conversation_rollup_pricing_observation),
2113
2238
  ("cache.db", s.cache_rollup_pricing_fp,
2114
- s.cache_rollup_pricing_refusal),
2239
+ s.cache_rollup_pricing_refusal,
2240
+ s.cache_rollup_pricing_observation),
2115
2241
  )
2116
- for store, stored_fp, record in stores:
2242
+ for store, stored_fp, record, _obs in stores:
2117
2243
  if _lib_pricing.pricing_fingerprint_is_comparable(stored_fp):
2118
2244
  continue
2119
2245
  details = {"store": store, "stored_fingerprint": stored_fp}
@@ -2133,8 +2259,37 @@ def _check_pricing_conversation_rollup_writer(s: DoctorState) -> CheckResult:
2133
2259
  remediation=ROLLUP_WRITER_UNPARSEABLE_FP_REMEDIATION,
2134
2260
  details=details,
2135
2261
  )
2136
- for store, _stored_fp, record in stores:
2137
- if not record:
2262
+ # #728: a fingerprint read that FAILED, reported between the unparseable
2263
+ # value and the recorded refusals. Below the unparseable pass because that
2264
+ # is a diagnosed permanent state with a known step out of it, while this
2265
+ # one is undiagnosed and often transient. Above the record passes because a
2266
+ # store doctor could not read has no verdict at all, and hiding it behind
2267
+ # the other store's recorded refusal is how the collapsed read reported OK.
2268
+ for store, _stored_fp, _record, observation in stores:
2269
+ if not _degraded_pricing_read(observation):
2270
+ continue
2271
+ return CheckResult(
2272
+ id="pricing.conversation_rollup_writer", title="Rollup writer",
2273
+ severity="warn",
2274
+ summary=(
2275
+ "the rollup's stored pricing fingerprint could not be read, "
2276
+ "so this store's materialized cost is unverified"
2277
+ ),
2278
+ remediation=rollup_writer_degraded_read_remediation(store),
2279
+ details={
2280
+ "store": store,
2281
+ "read_error_kind": getattr(observation, "error_kind", None),
2282
+ },
2283
+ )
2284
+ for store, _stored_fp, record, _obs in stores:
2285
+ # #729: ACTIVE episodes only. The record now survives convergence as a
2286
+ # tombstone carrying `active: false`, so a store that diverged and then
2287
+ # recovered is healthy and must not warn forever — that permanent warn
2288
+ # is why the record used to be deleted instead of settled. A record
2289
+ # written before the field existed carries no `active` key and is read
2290
+ # as active, because that older record was latched only on refusal and
2291
+ # deleted on convergence, so its presence alone means an open episode.
2292
+ if not record or record.get("active") is False:
2138
2293
  continue
2139
2294
  if record.get("__malformed__"):
2140
2295
  return CheckResult(
@@ -2144,6 +2299,29 @@ def _check_pricing_conversation_rollup_writer(s: DoctorState) -> CheckResult:
2144
2299
  remediation=ROLLUP_WRITER_REFUSAL_REMEDIATION,
2145
2300
  details={"store": store},
2146
2301
  )
2302
+ if _refusal_latched_by_a_failed_read(record):
2303
+ # #769 S3: the refusal came from a read that failed, and the store
2304
+ # reads cleanly now — the DEGRADED pass above did not claim it.
2305
+ # Rendering it under the version-skew wording printed the absent
2306
+ # store date verbatim ("refused a write over stored None") and
2307
+ # attached a remedy about pricing versions, which is not what
2308
+ # happened.
2309
+ return CheckResult(
2310
+ id="pricing.conversation_rollup_writer", title="Rollup writer",
2311
+ severity="warn",
2312
+ summary=(
2313
+ "a rollup write was refused because this store's pricing "
2314
+ "fingerprint could not be read at the time"
2315
+ ),
2316
+ remediation=rollup_writer_refused_degraded_read_remediation(
2317
+ store),
2318
+ details={
2319
+ "store": store,
2320
+ "process_snapshot_date": record.get("process_snapshot_date"),
2321
+ "store_snapshot_date": record.get("store_snapshot_date"),
2322
+ "first_refused_at_utc": record.get("first_refused_at_utc"),
2323
+ },
2324
+ )
2147
2325
  return CheckResult(
2148
2326
  id="pricing.conversation_rollup_writer", title="Rollup writer",
2149
2327
  severity="warn",
@@ -2757,6 +2935,50 @@ def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
2757
2935
  "conversations_db_free_ratio": ratio,
2758
2936
  "warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
2759
2937
  }
2938
+ # #780: the reclaim backlog rides on this check rather than adding a new
2939
+ # check id, and it contributes NOTHING — no detail key, no severity change
2940
+ # — on a store with no pending record. That is every store that has never
2941
+ # fallen behind, so an install in the ordinary state renders exactly what
2942
+ # it rendered before.
2943
+ pending = s.conversations_reclaim_pending
2944
+ if pending:
2945
+ import _lib_conversation_retention as _conv_retention
2946
+
2947
+ backlog = _conv_retention.reclaim_backlog_bytes(pending)
2948
+ details["reclaim_pending"] = pending
2949
+ details["reclaim_backlog_bytes"] = backlog
2950
+ details["reclaim_ceiling_bytes"] = _conv_retention.RECLAIM_CEILING_BYTES
2951
+ if _conv_retention.reclaim_backlog_over_ceiling(pending):
2952
+ return CheckResult(
2953
+ id="db.conversations_reclaimable",
2954
+ title="Reclaimable transcript space",
2955
+ severity="fail",
2956
+ summary=(
2957
+ f"reclaim backlog {_gib_text(backlog)} is at or over the "
2958
+ f"{_gib_text(_conv_retention.RECLAIM_CEILING_BYTES)} "
2959
+ "ceiling; transcript rebuilds are refused"
2960
+ ),
2961
+ remediation=(
2962
+ "Run `cctally db vacuum --db conversations` to drain the "
2963
+ "backlog, then `cctally db checkpoint --db conversations`."
2964
+ ),
2965
+ details=details,
2966
+ )
2967
+ if _conv_retention.reclaim_backlog_escalated(pending):
2968
+ return CheckResult(
2969
+ id="db.conversations_reclaimable",
2970
+ title="Reclaimable transcript space",
2971
+ severity="warn",
2972
+ summary=(
2973
+ f"reclaim backlog {_gib_text(backlog)} is draining across "
2974
+ "continuation passes"
2975
+ ),
2976
+ remediation=(
2977
+ "No action is required unless it keeps growing; "
2978
+ "`cctally db vacuum --db conversations` drains it now."
2979
+ ),
2980
+ details=details,
2981
+ )
2760
2982
  if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
2761
2983
  return CheckResult(
2762
2984
  id="db.conversations_reclaimable",