cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -0,0 +1,320 @@
1
+ """Cause-keyed retry state machine for degraded source generations (#830).
2
+
3
+ #834 S2. A ``partial`` source generation must not be handed back for the life
4
+ of the process. Refusing reuse outright is the correct statement of coherence
5
+ and is made in ``_lib_dashboard_sources._reusable_provider``; this module
6
+ answers the separate question that refusal raises, which is how often a cause
7
+ that is NOT going to clear on its own should pay for a full rebuild.
8
+
9
+ The kernel is pure and clock-injected: it performs no I/O, reads no wall
10
+ clock, and mutates nothing the caller passed in. It returns a decision and the
11
+ next state, and the caller holds that state for the life of its process.
12
+
13
+ The tree has more partial-producing states than "two producers" suggests:
14
+
15
+ * ``_lib_dashboard_sources.degrade_source_state`` retains a prior generation
16
+ during a transient ingest failure, producing ``partial``/``stale``;
17
+ * a freshly built Codex source becomes ``partial``/``fresh`` on incomplete
18
+ project metadata, a hero projection failure or an unreadable account
19
+ registry (``_cctally_dashboard_sources``);
20
+ * the idle Codex clock can turn an otherwise healthy generation ``partial``
21
+ when a retained cycle expires.
22
+
23
+ Those causes behave differently. An unreadable database is worth retrying at
24
+ once, while metadata that is structurally incomplete will still be incomplete
25
+ one second later and rebuilding for it on every tick buys nothing. The state
26
+ is therefore keyed by the pair of a normalized cause and the source
27
+ ``data_version``, so a changed situation always retries immediately and only a
28
+ genuinely repeated one is throttled.
29
+ """
30
+ from __future__ import annotations
31
+
32
+ import datetime as dt
33
+ from collections.abc import Iterable
34
+ from dataclasses import dataclass
35
+ from types import MappingProxyType
36
+ from typing import Literal, Mapping
37
+
38
+
39
+ RetryAction = Literal["rebuild", "retain"]
40
+
41
+ #: How long one repeated structural cause is retained before it pays for
42
+ #: another rebuild. Chosen to match the production ingest frontier interval
43
+ #: (120s, measured in #769 S6), because a structural Codex cause is cleared by
44
+ #: an ingest and retrying faster than ingest can change the answer spends a
45
+ #: full Codex rebuild to re-derive the same partial generation.
46
+ PARTIAL_RETRY_INTERVAL = dt.timedelta(seconds=120)
47
+
48
+ #: The cause recorded for a ``partial`` generation that states no warning at
49
+ #: all. The hydrating seed in ``_cctally_tui`` is one such generation.
50
+ NO_STATED_CAUSE = "none"
51
+
52
+ #: Normalized causes that are NEVER throttled. Each names a failure of the
53
+ #: read itself rather than a property of the data, so the next tick can
54
+ #: legitimately produce a different answer and must be allowed to try.
55
+ TRANSIENT_PARTIAL_CAUSES = frozenset((
56
+ "ingest_failed",
57
+ "ingest_contended",
58
+ "transient_read_failure",
59
+ ))
60
+
61
+ #: Normalized causes whose degraded generation MAY be retained under the
62
+ #: deadline. This is a positive allowlist and everything else — transient
63
+ #: causes, unrecognized codes, and every cause listed below — is rebuilt on
64
+ #: every tick, so an unclassified cause fails closed.
65
+ #:
66
+ #: Retention is safe only when nothing the tick re-derives can contradict the
67
+ #: retained generation. It does not hold in general. A reuse tick re-resolves
68
+ #: the account scope from the CURRENT registry while republishing the prior
69
+ #: ``data``, so retaining an ``account_scope_unresolved`` generation across a
70
+ #: registry that recovered would publish a scope naming two accounts beside a
71
+ #: ``data`` carrying no accounts and a warning still saying the scope is
72
+ #: unresolved (#819). A repaired quota projection must likewise trigger a
73
+ #: fresh build even when physical accounting did not move, so
74
+ #: ``projection_incoherent`` is not retainable either.
75
+ #:
76
+ #: ``metadata_incomplete`` is retainable because the generation is complete
77
+ #: except for project attribution, and no field the tick re-derives disagrees
78
+ #: with it. It is also the only cause for which refusing reuse adds rebuild
79
+ #: cost that was not already being paid: the two causes above were already
80
+ #: rebuilt on every tick before this kernel existed.
81
+ RETAINABLE_PARTIAL_CAUSES = frozenset((
82
+ "metadata_incomplete",
83
+ ))
84
+
85
+ #: Warning code -> normalized cause. Declared here rather than at the call
86
+ #: sites so one taxonomy governs every producer; an unrecognized code keeps
87
+ #: its own key through ``_OTHER_PREFIX`` instead of collapsing into a shared
88
+ #: bucket that would let two different situations throttle each other.
89
+ _CAUSE_BY_WARNING_CODE: Mapping[str, str] = MappingProxyType({
90
+ "source_ingest_failed": "ingest_failed",
91
+ "source_ingest_contended": "ingest_contended",
92
+ "codex_metadata_incomplete": "metadata_incomplete",
93
+ # No producer emits this code, and that is deliberate rather than an
94
+ # oversight (#834 S2, verified by the browser gate). A transient
95
+ # metadata-health failure never reaches this kernel as a warning, because
96
+ # ``_reusable_provider`` refuses the generation outright on its
97
+ # ``retryable`` carrier and the tick rebuilds. The entry stays so that a
98
+ # producer added later gets the never-throttled behaviour its cause
99
+ # deserves instead of falling into an ``other:`` bucket keyed by a string
100
+ # nobody chose.
101
+ "codex_metadata_health_unknown": "transient_read_failure",
102
+ "codex_projection_incoherent": "projection_incoherent",
103
+ "codex_account_scope_unresolved": "account_scope_unresolved",
104
+ "codex_cycle_unavailable": "cycle_unavailable",
105
+ "source_build_failed": "build_failed",
106
+ })
107
+
108
+ _OTHER_PREFIX = "other:"
109
+ _PROVIDERS = frozenset(("claude", "codex"))
110
+
111
+ EMPTY_RETRY_STATE: "PartialRetryState" = MappingProxyType({})
112
+
113
+
114
+ @dataclass(frozen=True)
115
+ class ProviderRetryEntry:
116
+ """One provider's armed retry key and the instant it was first observed."""
117
+
118
+ cause: str
119
+ data_version: str
120
+ first_seen: dt.datetime
121
+
122
+ @property
123
+ def key(self) -> tuple[str, str]:
124
+ return (self.cause, self.data_version)
125
+
126
+ def deadline(self, interval: dt.timedelta = PARTIAL_RETRY_INTERVAL) -> dt.datetime:
127
+ return self.first_seen + interval
128
+
129
+
130
+ PartialRetryState = Mapping[str, ProviderRetryEntry]
131
+
132
+
133
+ @dataclass(frozen=True)
134
+ class PartialRetryDecision:
135
+ """What the tick must do with this provider's degraded generation."""
136
+
137
+ action: RetryAction
138
+ cause: str
139
+ data_version: str
140
+ #: ``None`` for a transient cause, which arms no deadline at all.
141
+ first_seen: dt.datetime | None
142
+ deadline: dt.datetime | None
143
+ throttled: bool
144
+
145
+ @property
146
+ def rebuild(self) -> bool:
147
+ return self.action == "rebuild"
148
+
149
+
150
+ def _warning_code(item: object) -> str:
151
+ """Return one warning's code, accepting a bare string or a warning object."""
152
+ if isinstance(item, str):
153
+ return item
154
+ code = getattr(item, "code", None)
155
+ if isinstance(code, str) and code:
156
+ return code
157
+ raise ValueError("a partial cause must be a code string or carry .code")
158
+
159
+
160
+ def normalize_partial_cause(cause: object) -> str:
161
+ """Collapse one generation's stated causes into a single stable key.
162
+
163
+ ``cause`` is a warning code, a warning object, an iterable of either, or
164
+ ``None``. The result is order-independent and deduplicated, so the key
165
+ changes when the SET of causes changes and not when a producer happens to
166
+ append its warnings in a different order.
167
+ """
168
+ if cause is None:
169
+ return NO_STATED_CAUSE
170
+ if isinstance(cause, str) or not isinstance(cause, Iterable):
171
+ items: tuple[object, ...] = (cause,)
172
+ else:
173
+ items = tuple(cause)
174
+ if not items:
175
+ return NO_STATED_CAUSE
176
+ normalized = {
177
+ _CAUSE_BY_WARNING_CODE.get(code, _OTHER_PREFIX + code)
178
+ for code in (_warning_code(item) for item in items)
179
+ }
180
+ return "+".join(sorted(normalized))
181
+
182
+
183
+ def is_transient_partial_cause(cause: str) -> bool:
184
+ """Whether a normalized cause names a failure of the read itself.
185
+
186
+ A composite key is transient when ANY of its members is: the tick must be
187
+ allowed to retry a failed read, and the structural cause travelling beside
188
+ it loses nothing by being retried too.
189
+ """
190
+ return any(part in TRANSIENT_PARTIAL_CAUSES for part in cause.split("+"))
191
+
192
+
193
+ def is_retainable_partial_cause(cause: str) -> bool:
194
+ """Whether a degraded generation with this cause may be retained at all.
195
+
196
+ A composite key is retainable only when EVERY member is: one member that
197
+ would publish a self-contradictory envelope is enough to disqualify the
198
+ whole generation, which is the direction an allowlist has to fail.
199
+ """
200
+ return bool(cause) and all(
201
+ part in RETAINABLE_PARTIAL_CAUSES for part in cause.split("+")
202
+ )
203
+
204
+
205
+ def _validated_state(state: object, provider: object) -> tuple[dict, str]:
206
+ if provider not in _PROVIDERS:
207
+ raise ValueError("provider must be one of ['claude', 'codex']")
208
+ if state is None:
209
+ current: dict = {}
210
+ elif isinstance(state, Mapping):
211
+ current = dict(state)
212
+ else:
213
+ raise ValueError("state must be a mapping or None")
214
+ return current, provider # type: ignore[return-value]
215
+
216
+
217
+ def _validated_now(now: object) -> dt.datetime:
218
+ if not isinstance(now, dt.datetime) or now.tzinfo is None or now.utcoffset() is None:
219
+ raise ValueError("now must be a timezone-aware datetime")
220
+ return now
221
+
222
+
223
+ def clear_partial_retry(
224
+ state: PartialRetryState | None, *, provider: str,
225
+ ) -> PartialRetryState:
226
+ """Drop one provider's key entirely, as a healthy publish must.
227
+
228
+ Recovery has to clear the key, not merely stop consulting it. A key left
229
+ armed across a healthy generation would still be inside its deadline when
230
+ the same cause returned, so the first partial after a recovery would be
231
+ RETAINED — the exact permanent-freeze behaviour this kernel exists to end,
232
+ reached by a different route.
233
+ """
234
+ current, provider = _validated_state(state, provider)
235
+ current.pop(provider, None)
236
+ return MappingProxyType(current)
237
+
238
+
239
+ def plan_partial_retry(
240
+ state: PartialRetryState | None,
241
+ *,
242
+ provider: str,
243
+ cause: object,
244
+ data_version: str,
245
+ now: dt.datetime,
246
+ interval: dt.timedelta = PARTIAL_RETRY_INTERVAL,
247
+ ) -> tuple[PartialRetryDecision, PartialRetryState]:
248
+ """Decide whether a degraded generation is rebuilt or retained this tick.
249
+
250
+ Returns ``(decision, next_state)``. The caller's ``state`` is never
251
+ mutated.
252
+
253
+ The rules, in order:
254
+
255
+ * a cause that is not retainable — a transient read failure, an
256
+ unrecognized code, or one of the causes whose retention would publish a
257
+ self-contradictory envelope — always rebuilds and arms nothing, and
258
+ additionally clears any armed key so a retainable cause interrupted by
259
+ one of them is not throttled by a deadline that kept running while it
260
+ was absent;
261
+ * the FIRST observation of a structural key always rebuilds and arms the
262
+ deadline from this instant;
263
+ * a repeat inside the deadline retains and leaves ``first_seen`` exactly
264
+ where it was, because a deadline re-derived from ``now`` never expires
265
+ while its cause persists;
266
+ * a repeat at or after the deadline rebuilds and re-arms from this
267
+ instant, which starts the next throttle window rather than extending the
268
+ last one.
269
+ """
270
+ current, provider = _validated_state(state, provider)
271
+ now = _validated_now(now)
272
+ if not isinstance(data_version, str):
273
+ raise ValueError("data_version must be a string")
274
+ normalized = normalize_partial_cause(cause)
275
+
276
+ if not is_retainable_partial_cause(normalized):
277
+ current.pop(provider, None)
278
+ return (
279
+ PartialRetryDecision(
280
+ action="rebuild",
281
+ cause=normalized,
282
+ data_version=data_version,
283
+ first_seen=None,
284
+ deadline=None,
285
+ throttled=False,
286
+ ),
287
+ MappingProxyType(current),
288
+ )
289
+
290
+ prior = current.get(provider)
291
+ if prior is not None and prior.key == (normalized, data_version):
292
+ deadline = prior.deadline(interval)
293
+ if now < deadline:
294
+ return (
295
+ PartialRetryDecision(
296
+ action="retain",
297
+ cause=normalized,
298
+ data_version=data_version,
299
+ first_seen=prior.first_seen,
300
+ deadline=deadline,
301
+ throttled=True,
302
+ ),
303
+ MappingProxyType(current),
304
+ )
305
+
306
+ entry = ProviderRetryEntry(
307
+ cause=normalized, data_version=data_version, first_seen=now,
308
+ )
309
+ current[provider] = entry
310
+ return (
311
+ PartialRetryDecision(
312
+ action="rebuild",
313
+ cause=normalized,
314
+ data_version=data_version,
315
+ first_seen=now,
316
+ deadline=entry.deadline(interval),
317
+ throttled=False,
318
+ ),
319
+ MappingProxyType(current),
320
+ )