cctally 1.101.0 → 1.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1222 -1364
- package/README.md +8 -2
- package/bin/_cctally_cache.py +31 -0
- package/bin/_cctally_dashboard.py +433 -122
- package/bin/_cctally_dashboard_conversation.py +12 -0
- package/bin/_cctally_dashboard_envelope.py +25 -23
- package/bin/_cctally_diagnosis.py +1172 -0
- package/bin/_cctally_diagnosis_sources.py +4054 -0
- package/bin/_cctally_forecast.py +25 -15
- package/bin/_cctally_parser.py +84 -0
- package/bin/_cctally_quota.py +14 -0
- package/bin/_cctally_record.py +144 -70
- package/bin/_cctally_refresh.py +105 -93
- package/bin/_cctally_statusline.py +8 -1
- package/bin/_lib_blocks.py +12 -0
- package/bin/_lib_codex_conversation.py +14 -0
- package/bin/_lib_codex_conversation_query.py +22 -8
- package/bin/_lib_conversation.py +6 -3
- package/bin/_lib_conversation_query.py +256 -69
- package/bin/_lib_diagnosis.py +1261 -0
- package/bin/_lib_forecast.py +50 -0
- package/bin/_lib_jsonl.py +102 -5
- package/bin/_lib_perf.py +12 -0
- package/bin/_lib_pricing.py +97 -21
- package/bin/_lib_readme_refresh.py +26 -5
- package/bin/cctally +93 -16
- package/bin/cctally-explain +5 -0
- package/dashboard/static/assets/index-Di2hljvB.css +1 -0
- package/dashboard/static/assets/index-XYCIWjVG.js +97 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +5 -1
- package/dashboard/static/assets/index-D6Eb9KDn.js +0 -97
- package/dashboard/static/assets/index-i3g7g8zo.css +0 -1
|
@@ -0,0 +1,1261 @@
|
|
|
1
|
+
"""Pure diagnosis kernel for `cctally explain` (#620 S2).
|
|
2
|
+
|
|
3
|
+
Every type, constant, predicate and the ranking live here. The module opens
|
|
4
|
+
no store, renders nothing and reads no clock, so each classification rule is
|
|
5
|
+
directly unit-testable without a fixture. `bin/_cctally_diagnosis_sources.py`
|
|
6
|
+
is the only component that opens databases; `bin/_cctally_diagnosis.py` is
|
|
7
|
+
the only one that renders.
|
|
8
|
+
|
|
9
|
+
The evidence contract, in one sentence: every measured quantity is an
|
|
10
|
+
`EvidenceField` — either available with a value, its population and zero or
|
|
11
|
+
more qualifications, or withheld with a typed cause and still its
|
|
12
|
+
population. The three cause layers never mix. `EstablishmentError` ends the
|
|
13
|
+
request. `WithheldCause` is field-level. `BaselineOutcome` is its own axis,
|
|
14
|
+
because a thin baseline must not suppress a measured observed-cost answer.
|
|
15
|
+
|
|
16
|
+
Spec: docs/superpowers/specs/2026-08-19-620-s2-on-demand-diagnosis.md
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
from dataclasses import dataclass, field, replace
|
|
22
|
+
from enum import Enum
|
|
23
|
+
from types import MappingProxyType
|
|
24
|
+
from typing import Any, Iterable, Mapping, Sequence
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# --- versioned constants ------------------------------------------------
|
|
28
|
+
#
|
|
29
|
+
# These are published in the JSON envelope and stated on the terminal
|
|
30
|
+
# surface, so a reader can see the rule that produced the verdict rather
|
|
31
|
+
# than inferring it. Changing one is a contract change.
|
|
32
|
+
|
|
33
|
+
DIAGNOSIS_CONTRACT_VERSION: int = 1
|
|
34
|
+
DIAGNOSIS_CONTRIBUTOR_SHARE_FLOOR: float = 0.20
|
|
35
|
+
DIAGNOSIS_TIE_EPSILON_USD: float = 1e-9
|
|
36
|
+
# The slack the share floor is compared with, and the reason it is PUBLISHED
|
|
37
|
+
# rather than private: it decides the verdict. A share is a ratio of two
|
|
38
|
+
# independently accumulated float sums, so a subject genuinely at one fifth of
|
|
39
|
+
# the denominator computes as 0.19999999999999998, and a client applying a bare
|
|
40
|
+
# `share >= floor` to the published share would compute `no_contributor` where
|
|
41
|
+
# the server published `contributor`. The rule both sides apply is
|
|
42
|
+
# `share >= floor - epsilon`, and both halves of it are on the wire.
|
|
43
|
+
DIAGNOSIS_SHARE_FLOOR_EPSILON: float = 1e-9
|
|
44
|
+
CONFIDENCE_HIGH_MIN_SUPPORT: int = 20
|
|
45
|
+
CONFIDENCE_MEDIUM_MIN_SUPPORT: int = 5
|
|
46
|
+
CONFIDENCE_MEDIUM_MIN_COVERAGE: float = 0.80
|
|
47
|
+
WITHHOLD_MIN_COVERAGE: float = 0.50
|
|
48
|
+
WITHHOLD_MIN_SUPPORT: int = 2
|
|
49
|
+
|
|
50
|
+
# Float comparison slack for the COVERAGE thresholds only, so a coverage
|
|
51
|
+
# computed as a ratio of sums does not fall below its own threshold on the last
|
|
52
|
+
# bit. The share floor has its own constant above, because that one is part of
|
|
53
|
+
# the published rule and this one is not.
|
|
54
|
+
#
|
|
55
|
+
# Exported without the underscore because the source adapter applies it to
|
|
56
|
+
# RETENTION coverage, which is the same kind of ratio against the same
|
|
57
|
+
# `WITHHOLD_MIN_COVERAGE` constant. Two copies of one slack value drift, and a
|
|
58
|
+
# retention computing as 0.49999999999999994 for a genuinely half-retained
|
|
59
|
+
# window withheld the whole provider as `stale_evidence`.
|
|
60
|
+
COVERAGE_EPSILON: float = 1e-9
|
|
61
|
+
|
|
62
|
+
# --- S3 versioned constants (#620 S3) -----------------------------------
|
|
63
|
+
#
|
|
64
|
+
# Published with the rest, because each one decides a verdict and a reader
|
|
65
|
+
# who cannot see the threshold cannot reproduce the rule that produced it.
|
|
66
|
+
|
|
67
|
+
# A conversation is "short" at or below this many canonical human turns.
|
|
68
|
+
DIAGNOSIS_SHORT_CONVERSATION_MAX_HUMAN_TURNS: int = 3
|
|
69
|
+
# The fraction of a request's OWN model context window that makes it large.
|
|
70
|
+
DIAGNOSIS_LARGE_CONTEXT_MIN_WINDOW_FRACTION: float = 0.80
|
|
71
|
+
# How many distinct subagent buckets a parent needs before it fans out.
|
|
72
|
+
DIAGNOSIS_MIN_SUBAGENT_BUCKETS: int = 2
|
|
73
|
+
|
|
74
|
+
# The three scan budgets. Each bounds a read path that is capped rather than
|
|
75
|
+
# structurally bounded, and each is allocated per subject rather than
|
|
76
|
+
# globally, so no one session, conversation or source file can starve another
|
|
77
|
+
# and the result never depends on which one happened to be read first.
|
|
78
|
+
DIAGNOSIS_SEED_SCAN_BUDGET_ROWS: int = 50_000
|
|
79
|
+
DIAGNOSIS_CONVERSATION_NORMALIZE_BUDGET_ROWS: int = 2_000
|
|
80
|
+
# #632 measured the inference-relevant production population at max=2,262
|
|
81
|
+
# rows/file (p99=853). The global ceiling keeps arbitrarily wide windows
|
|
82
|
+
# bounded; the per-file ceiling prevents a narrow window's long tail from
|
|
83
|
+
# consuming the whole allowance.
|
|
84
|
+
DIAGNOSIS_CODEX_EVENT_SCAN_BUDGET_ROWS: int = 1_048_576
|
|
85
|
+
DIAGNOSIS_CODEX_EVENT_SCAN_PER_FILE_ROWS: int = 4_096
|
|
86
|
+
|
|
87
|
+
# Gap codes, NOT withheld causes: a gap describes rows the class could not
|
|
88
|
+
# decide, while a cause describes why a whole field was not published.
|
|
89
|
+
#
|
|
90
|
+
# The vocabulary is closed on the server and every member is declared here,
|
|
91
|
+
# because a bare literal at a call site is a code no reader can enumerate and
|
|
92
|
+
# no test can name. Spec revision 4 added the last two: Task 2 emitted
|
|
93
|
+
# `unresolved_subagent_attribution` as an undeclared literal and reported an
|
|
94
|
+
# unreadable Codex origin as `unknown_context_window`, where the context
|
|
95
|
+
# window is not what is unknown. A typed cause that misdescribes itself is
|
|
96
|
+
# worse than none.
|
|
97
|
+
GAP_UNKNOWN_CONTEXT_WINDOW: str = "unknown_context_window"
|
|
98
|
+
GAP_SCAN_BUDGET_EXHAUSTED: str = "scan_budget_exhausted"
|
|
99
|
+
GAP_UNRESOLVED_SUBAGENT_ATTRIBUTION: str = "unresolved_subagent_attribution"
|
|
100
|
+
GAP_AMBIGUOUS_ORIGIN_CATEGORY: str = "ambiguous_origin_category"
|
|
101
|
+
|
|
102
|
+
# The closed set, so a test can enumerate it and a published code that no
|
|
103
|
+
# constant defines fails rather than shipping.
|
|
104
|
+
GAP_CODES: frozenset[str] = frozenset({
|
|
105
|
+
GAP_UNKNOWN_CONTEXT_WINDOW,
|
|
106
|
+
GAP_SCAN_BUDGET_EXHAUSTED,
|
|
107
|
+
GAP_UNRESOLVED_SUBAGENT_ATTRIBUTION,
|
|
108
|
+
GAP_AMBIGUOUS_ORIGIN_CATEGORY,
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
# Qualifications an S3 evidence field can carry.
|
|
112
|
+
QUALIFICATION_IDENTIFIABLE_SUBSET: str = "identifiable_subset_unknown_completeness"
|
|
113
|
+
QUALIFICATION_PARTIAL_ATTRIBUTION: str = "partial_attribution"
|
|
114
|
+
QUALIFICATION_SESSION_LEVEL_CAPACITY: str = "session_level_context_window"
|
|
115
|
+
|
|
116
|
+
# Every S3 class contributes ONE aggregate subject, and its key, kind and
|
|
117
|
+
# label are constants of the class rather than anything derived from a
|
|
118
|
+
# member. `_display_key` aliases only project and session subjects and emits
|
|
119
|
+
# every other key verbatim, so a member-derived key would reach the wire and
|
|
120
|
+
# the screen unaliased.
|
|
121
|
+
SUBJECT_KIND_QUALIFYING_SET: str = "qualifying_set"
|
|
122
|
+
SUBJECT_CACHE_CHURN: str = "qualifying-set/cache-churn"
|
|
123
|
+
SUBJECT_SHORT_HIGH_CONTEXT: str = "qualifying-set/short-high-context"
|
|
124
|
+
SUBJECT_SUBAGENT_FANOUT: str = "qualifying-set/subagent-fanout"
|
|
125
|
+
|
|
126
|
+
# The accounting mode a next-step command is told to apply, and the providers
|
|
127
|
+
# whose target subcommands accept `-m/--mode` at all.
|
|
128
|
+
#
|
|
129
|
+
# `explain` reprices every Claude accounting entry at read time
|
|
130
|
+
# (`_calculate_entry_cost(..., mode="calculate")`), but `cctally claude daily`
|
|
131
|
+
# and `cctally claude session` default to `-m auto`, which returns a non-null
|
|
132
|
+
# stored `session_entries.cost_usd_raw` verbatim. On a store holding stored
|
|
133
|
+
# costs, a reader following the next step would see different dollars from the
|
|
134
|
+
# figure that sent them there, and a next step that contradicts its own
|
|
135
|
+
# evidence is the one thing D4 says a next step must not be.
|
|
136
|
+
#
|
|
137
|
+
# The Codex subcommands register no `-m` at all, because Codex cost is always
|
|
138
|
+
# computed at read time from `CODEX_MODEL_PRICING`, so there is no divergence
|
|
139
|
+
# to close and no flag to pass. `cctally project` and
|
|
140
|
+
# `cctally five-hour-breakdown` register none either, and a flag they reject
|
|
141
|
+
# would turn the next step into an argument error; `docs/commands/explain.md`
|
|
142
|
+
# states that limitation for the reader.
|
|
143
|
+
NEXT_STEP_ACCOUNTING_MODE: str = "calculate"
|
|
144
|
+
NEXT_STEP_MODE_SOURCES: frozenset[str] = frozenset({"claude"})
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def next_step_mode_flag(source: str) -> str:
|
|
148
|
+
"""The `-m/--mode` fragment a next-step command carries, or empty."""
|
|
149
|
+
if source in NEXT_STEP_MODE_SOURCES:
|
|
150
|
+
return f" -m {NEXT_STEP_ACCOUNTING_MODE}"
|
|
151
|
+
return ""
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# Which support minimum a class failed, when it was withheld as
|
|
155
|
+
# `insufficient_population`. The cause alone reads as a contradiction beside a
|
|
156
|
+
# support count that looks ample: `insufficient_population (support 60 units)`
|
|
157
|
+
# is exactly what a class with 60 entries under a single model produces.
|
|
158
|
+
SUPPORT_SHORTFALL_DISTINCT_SUBJECTS: str = "min_distinct_subjects"
|
|
159
|
+
SUPPORT_SHORTFALL_PRICED_ENTRIES: str = "min_priced_entries"
|
|
160
|
+
SUPPORT_SHORTFALL_USD_COVERAGE: str = "min_usd_coverage"
|
|
161
|
+
SUPPORT_SHORTFALL_NO_PRICED_DOLLARS: str = "no_priced_dollars"
|
|
162
|
+
|
|
163
|
+
# The header sentences both surfaces state verbatim. They are constants
|
|
164
|
+
# rather than render-site strings because B11 binds them on both surfaces.
|
|
165
|
+
DIAGNOSIS_SCOPE_SENTENCE: str = (
|
|
166
|
+
"This explains locally retained cost and tokens, not quota percentage."
|
|
167
|
+
)
|
|
168
|
+
DIAGNOSIS_OVERLAP_SENTENCE: str = (
|
|
169
|
+
"The classes overlap — one session is also a project and a model — "
|
|
170
|
+
"so shares do not sum to 100 percent."
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
# The published turn predicate, in one sentence, for the two turn-based
|
|
174
|
+
# classes. It is a constant rather than render-site prose because F8 requires
|
|
175
|
+
# the FULL predicate on the surface: `conversation_sessions.msg_count` groups
|
|
176
|
+
# `conversation_messages` with no `is_sidechain` predicate while subagent files
|
|
177
|
+
# carry the parent's `session_id`, so a reader shown a bare turn count would
|
|
178
|
+
# read a sidechain-inclusive physical count as a human-turn count.
|
|
179
|
+
DIAGNOSIS_TURN_DEFINITION_SENTENCE: str = (
|
|
180
|
+
"Turn definition: main-thread human messages only — non-sidechain, "
|
|
181
|
+
"non-subagent, after command and meta normalisation; assistant replies "
|
|
182
|
+
"associate with the nearest preceding human."
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
# The asymmetry of spec 5.4, stated where it happens: on a denied dashboard
|
|
186
|
+
# the same class name renders two verdicts in one report, because Claude
|
|
187
|
+
# subagent structure lives in `conversations.db` and the Codex signal lives in
|
|
188
|
+
# `cache.db`. A reader who sees one withheld and one measured must be told why
|
|
189
|
+
# rather than left to infer a bug.
|
|
190
|
+
DIAGNOSIS_TRANSCRIPT_ASYMMETRY_SENTENCE: str = (
|
|
191
|
+
"A class is withheld where this dashboard is not authorized to read "
|
|
192
|
+
"transcripts. Codex subagent attribution is derived from accounting "
|
|
193
|
+
"metadata and needs no transcript access."
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# --- the three cause layers ---------------------------------------------
|
|
198
|
+
|
|
199
|
+
class EstablishmentError(str, Enum):
|
|
200
|
+
"""Request-ending failures. Never an evidence field.
|
|
201
|
+
|
|
202
|
+
CLI exit 3. Dashboard 503 for `store_unavailable` and
|
|
203
|
+
`generation_incoherent`, 400 for `range_unresolved` and
|
|
204
|
+
`account_unresolved`.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
RANGE_UNRESOLVED = "range_unresolved"
|
|
208
|
+
ACCOUNT_UNRESOLVED = "account_unresolved"
|
|
209
|
+
STORE_UNAVAILABLE = "store_unavailable"
|
|
210
|
+
GENERATION_INCOHERENT = "generation_incoherent"
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class WithheldCause(str, Enum):
|
|
214
|
+
"""Field-level causes, closed on the server, in precedence order.
|
|
215
|
+
|
|
216
|
+
S3 adds `signal_unavailable` and `transcripts_not_visible` additively.
|
|
217
|
+
The client types every code as a bare string with a required fallback
|
|
218
|
+
branch, so an added cause cannot break an older client.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
PROVIDER_UNAVAILABLE = "provider_unavailable"
|
|
222
|
+
# An authorization denial, decided in plan stage 1 before any store is
|
|
223
|
+
# opened, so it outranks every availability cause: the forbidden store is
|
|
224
|
+
# never probed and `signal_unavailable` — which only means an AUTHORIZED
|
|
225
|
+
# open failed — cannot arise for a denied class.
|
|
226
|
+
TRANSCRIPTS_NOT_VISIBLE = "transcripts_not_visible"
|
|
227
|
+
RETAINED_RANGE_MISMATCH = "retained_range_mismatch"
|
|
228
|
+
PRICING_UNAVAILABLE = "pricing_unavailable"
|
|
229
|
+
UNATTRIBUTED_EVIDENCE = "unattributed_evidence"
|
|
230
|
+
STALE_EVIDENCE = "stale_evidence"
|
|
231
|
+
# The signal could not be established at all, which is more fundamental
|
|
232
|
+
# than establishing it with too little support and less informative than a
|
|
233
|
+
# pruned store, which explains WHY it could not be established.
|
|
234
|
+
SIGNAL_UNAVAILABLE = "signal_unavailable"
|
|
235
|
+
INSUFFICIENT_POPULATION = "insufficient_population"
|
|
236
|
+
CALCULATION_FAILED = "calculation_failed"
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
WITHHELD_CAUSE_PRECEDENCE: tuple[str, ...] = tuple(
|
|
240
|
+
c.value for c in WithheldCause
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class BaselineOutcome(str, Enum):
|
|
245
|
+
"""The baseline axis. A subject missing from an otherwise fully measured
|
|
246
|
+
baseline is the value zero, not a cause."""
|
|
247
|
+
|
|
248
|
+
BASELINE_INSUFFICIENT = "baseline_insufficient"
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
class VerdictState(str, Enum):
|
|
252
|
+
CONTRIBUTOR = "contributor"
|
|
253
|
+
NO_CONTRIBUTOR = "no_contributor"
|
|
254
|
+
WITHHELD = "withheld"
|
|
255
|
+
NOT_APPLICABLE = "not_applicable"
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class OverallVerdict(str, Enum):
|
|
259
|
+
CONTRIBUTOR_DETECTED = "contributor_detected"
|
|
260
|
+
NO_CONTRIBUTOR_DETECTED = "no_contributor_detected"
|
|
261
|
+
WITHHELD = "withheld"
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class EvidenceState(str, Enum):
|
|
265
|
+
AVAILABLE = "available"
|
|
266
|
+
WITHHELD = "withheld"
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class EstablishmentFailure(Exception):
|
|
270
|
+
"""Raised by the adapter when the report cannot be established at all.
|
|
271
|
+
|
|
272
|
+
Defined here rather than in the adapter so the code it carries is
|
|
273
|
+
validated against the same closed enum the wire publishes.
|
|
274
|
+
"""
|
|
275
|
+
|
|
276
|
+
def __init__(self, code: str, message: str = "") -> None:
|
|
277
|
+
super().__init__(message or code)
|
|
278
|
+
self.code = code
|
|
279
|
+
self.message = message or code
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# --- coverage and evidence ----------------------------------------------
|
|
283
|
+
|
|
284
|
+
@dataclass(frozen=True)
|
|
285
|
+
class PopulationCoverage:
|
|
286
|
+
"""Named coverage dimensions rather than one ambiguous fraction.
|
|
287
|
+
|
|
288
|
+
An unmeasurable dimension is absent (`None`) rather than zero, because
|
|
289
|
+
zero is a measurement and absence is not. `gap_codes` names why.
|
|
290
|
+
Confidence is computed on `usd_coverage`, because a diagnosis ranked by
|
|
291
|
+
dollars must gate on dollar coverage rather than row counts.
|
|
292
|
+
"""
|
|
293
|
+
|
|
294
|
+
requested_start: str
|
|
295
|
+
requested_end: str
|
|
296
|
+
observed_start: str | None = None
|
|
297
|
+
observed_end: str | None = None
|
|
298
|
+
count_coverage: float | None = None
|
|
299
|
+
usd_coverage: float | None = None
|
|
300
|
+
identity_coverage: float | None = None
|
|
301
|
+
retention_coverage: float | None = None
|
|
302
|
+
pricing_coverage: float | None = None
|
|
303
|
+
# `None` when no population was ever attributed — a class the provider
|
|
304
|
+
# preempted never shaped its subjects, and zero would assert that it
|
|
305
|
+
# shaped them and found none.
|
|
306
|
+
support_units: int | None = 0
|
|
307
|
+
gap_codes: tuple[str, ...] = ()
|
|
308
|
+
# `evaluated_count / candidate_count`, and the ONLY dimension that falls
|
|
309
|
+
# when a row could not be decided — an unresolvable subagent identity, a
|
|
310
|
+
# parent matching zero or several threads, an ambiguous Codex origin, an
|
|
311
|
+
# unknown context window, an exhausted scan budget. `identity_coverage`
|
|
312
|
+
# cannot also fall for those rows: a population that excludes them has
|
|
313
|
+
# perfect identity coverage by construction. S2 coverage objects leave it
|
|
314
|
+
# `None`, and `_coverage_to_wire` omits a null, so no S2 block moves.
|
|
315
|
+
evaluability_coverage: float | None = None
|
|
316
|
+
# Per dimension, its numerator and denominator in words, so a reader never
|
|
317
|
+
# has to infer which population a figure was computed over. Published by
|
|
318
|
+
# S3 coverage objects only; S2's are byte-frozen.
|
|
319
|
+
dimensions: Mapping[str, str] | None = None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@dataclass(frozen=True)
|
|
323
|
+
class EvidenceField:
|
|
324
|
+
state: str # EvidenceState value
|
|
325
|
+
value: Any | None
|
|
326
|
+
population: PopulationCoverage
|
|
327
|
+
qualifications: tuple[str, ...] = ()
|
|
328
|
+
code: str | None = None
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def available(value: Any, population: PopulationCoverage,
|
|
332
|
+
qualifications: Sequence[str] = ()) -> EvidenceField:
|
|
333
|
+
return EvidenceField(
|
|
334
|
+
state=EvidenceState.AVAILABLE.value, value=value,
|
|
335
|
+
population=population, qualifications=tuple(qualifications),
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def withheld(code: str, population: PopulationCoverage,
|
|
340
|
+
qualifications: Sequence[str] = ()) -> EvidenceField:
|
|
341
|
+
return EvidenceField(
|
|
342
|
+
state=EvidenceState.WITHHELD.value, value=None,
|
|
343
|
+
population=population, qualifications=tuple(qualifications),
|
|
344
|
+
code=code,
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
@dataclass(frozen=True)
|
|
349
|
+
class Denominator:
|
|
350
|
+
"""The named object every share divides by.
|
|
351
|
+
|
|
352
|
+
`identity` is always `totalExplainedRetainedCost`. A row attributed over
|
|
353
|
+
only part of the population divides by its own disclosed explained
|
|
354
|
+
population and says so through its coverage.
|
|
355
|
+
|
|
356
|
+
`usd` is an `EvidenceField`, not a bare float, and it is withheld under
|
|
357
|
+
the same provider-wide cause ladder every class is withheld under. A
|
|
358
|
+
denominator rendered as `$0.00` beside four classes withheld as
|
|
359
|
+
`pricing_unavailable` states a figure the data does not support: the true
|
|
360
|
+
retained cost there is unknown, not zero. `usd_value` is the arithmetic
|
|
361
|
+
view — zero when the field is withheld — and exists so the classification
|
|
362
|
+
rules divide by a number without ever printing one.
|
|
363
|
+
"""
|
|
364
|
+
|
|
365
|
+
identity: str
|
|
366
|
+
source: str
|
|
367
|
+
account_key: str | None
|
|
368
|
+
window_start: str
|
|
369
|
+
window_end: str
|
|
370
|
+
population_digest: str
|
|
371
|
+
usd: EvidenceField
|
|
372
|
+
|
|
373
|
+
@property
|
|
374
|
+
def usd_value(self) -> float:
|
|
375
|
+
value = self.usd.value
|
|
376
|
+
return float(value) if isinstance(value, (int, float)) else 0.0
|
|
377
|
+
|
|
378
|
+
@property
|
|
379
|
+
def usd_is_available(self) -> bool:
|
|
380
|
+
return self.usd.state == EvidenceState.AVAILABLE.value
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
# --- windows, subjects, rows, classes -----------------------------------
|
|
384
|
+
|
|
385
|
+
@dataclass(frozen=True)
|
|
386
|
+
class DiagnosisWindow:
|
|
387
|
+
"""A half-open `[start_at, end_at)` window that states its IANA zone."""
|
|
388
|
+
|
|
389
|
+
start_at: str
|
|
390
|
+
end_at: str
|
|
391
|
+
tz: str
|
|
392
|
+
label: str = ""
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
@dataclass(frozen=True)
|
|
396
|
+
class SubjectFacts:
|
|
397
|
+
"""One subject's priced facts, as the adapter hands them to the kernel.
|
|
398
|
+
|
|
399
|
+
`priced_entry_count` is the support unit for every class: the count of
|
|
400
|
+
priced accounting entries attributed to this subject.
|
|
401
|
+
"""
|
|
402
|
+
|
|
403
|
+
subject_key: str
|
|
404
|
+
subject_label: str
|
|
405
|
+
observed_usd: float
|
|
406
|
+
priced_entry_count: int = 0
|
|
407
|
+
is_fallback_pricing: bool = False
|
|
408
|
+
qualifications: tuple[str, ...] = ()
|
|
409
|
+
baseline_share: float | None = None
|
|
410
|
+
baseline_code: str | None = None
|
|
411
|
+
next_step: str | None = None
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
@dataclass(frozen=True)
|
|
415
|
+
class ContributorRow:
|
|
416
|
+
contributor_class: str
|
|
417
|
+
subject_kind: str
|
|
418
|
+
subject_key: str
|
|
419
|
+
subject_label: str
|
|
420
|
+
rank: int | None
|
|
421
|
+
observed_usd: EvidenceField
|
|
422
|
+
share: EvidenceField
|
|
423
|
+
baseline: EvidenceField
|
|
424
|
+
confidence: str # "high" | "medium" | "low"
|
|
425
|
+
is_fallback_pricing: bool
|
|
426
|
+
next_step: str
|
|
427
|
+
# Class-specific evidence, keyed by camelCase wire name. Declared with a
|
|
428
|
+
# `default_factory` because `dataclasses` rejects a mutable default
|
|
429
|
+
# outright, and because a shared default would leak one row's evidence
|
|
430
|
+
# into the next. `_row_to_wire` emits the key only when the mapping is
|
|
431
|
+
# non-empty, so every S2 row stays byte-identical.
|
|
432
|
+
evidence: "Mapping[str, EvidenceField]" = field(default_factory=dict)
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
@dataclass(frozen=True)
|
|
436
|
+
class ClassResult:
|
|
437
|
+
contributor_class: str
|
|
438
|
+
verdict: str # VerdictState value
|
|
439
|
+
code: str | None
|
|
440
|
+
rows: tuple[ContributorRow, ...]
|
|
441
|
+
population: PopulationCoverage
|
|
442
|
+
# Set only when `code` is `insufficient_population`: WHICH minimum was not
|
|
443
|
+
# met. A class withheld under a provider-wide cause never shaped its
|
|
444
|
+
# subjects, so it has no shortfall to name.
|
|
445
|
+
support_shortfall: str | None = None
|
|
446
|
+
# The confidence of the measurement this class made, and `None` when it
|
|
447
|
+
# made none. `None` is NOT `low`: confidence is a statement about a
|
|
448
|
+
# measurement. Carried on the result rather than recomputed per renderer,
|
|
449
|
+
# so the terminal and the modal state one assessment.
|
|
450
|
+
confidence: str | None = None
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
@dataclass(frozen=True)
|
|
454
|
+
class ClassRule:
|
|
455
|
+
"""The published predicate.
|
|
456
|
+
|
|
457
|
+
`sentence` is what a person reads on the terminal and in the modal;
|
|
458
|
+
`parameters` is the machine-readable form, so a consumer can reproduce the
|
|
459
|
+
rule rather than parse English.
|
|
460
|
+
"""
|
|
461
|
+
|
|
462
|
+
sentence: str
|
|
463
|
+
parameters: "Mapping[str, Any]"
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
@dataclass(frozen=True)
|
|
467
|
+
class ContributorSpec:
|
|
468
|
+
kind: str
|
|
469
|
+
subject_kind: str
|
|
470
|
+
min_distinct_subjects: int
|
|
471
|
+
min_priced_entries: int
|
|
472
|
+
# `None` means this class publishes no generic next step, because its next
|
|
473
|
+
# step names the subject and only the code that built the subject knows
|
|
474
|
+
# how to name it. See `_render_next_step`.
|
|
475
|
+
next_step_template: str | None
|
|
476
|
+
label: str = ""
|
|
477
|
+
# The published predicate, or `None` for the four accounting classes,
|
|
478
|
+
# whose wire shape is byte-frozen. Last field, so existing positional
|
|
479
|
+
# construction is unaffected.
|
|
480
|
+
rule: "ClassRule | None" = None
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
# Fixed order. Registry order is the tie-break after observed USD and the
|
|
484
|
+
# display order for non-ranked rows, so it is part of the contract.
|
|
485
|
+
CONTRIBUTOR_REGISTRY: tuple[ContributorSpec, ...] = (
|
|
486
|
+
ContributorSpec(
|
|
487
|
+
kind="model_mix",
|
|
488
|
+
subject_kind="model",
|
|
489
|
+
min_distinct_subjects=2,
|
|
490
|
+
min_priced_entries=20,
|
|
491
|
+
# The SUBGROUP form, not `--source`: `cctally daily` registers no
|
|
492
|
+
# `--source` flag at all, so `cctally daily --source claude` is an
|
|
493
|
+
# argument error rather than a report. `cctally claude daily` and
|
|
494
|
+
# `cctally codex daily` route to the same command with the provider
|
|
495
|
+
# pinned, which is what the flat form was reaching for.
|
|
496
|
+
next_step_template=(
|
|
497
|
+
"cctally {source} daily{mode_flag} "
|
|
498
|
+
"--since {window_start_date} --until {window_end_date}"
|
|
499
|
+
),
|
|
500
|
+
label="Expensive model mix",
|
|
501
|
+
),
|
|
502
|
+
ContributorSpec(
|
|
503
|
+
kind="project_concentration",
|
|
504
|
+
subject_kind="project",
|
|
505
|
+
min_distinct_subjects=2,
|
|
506
|
+
min_priced_entries=20,
|
|
507
|
+
next_step_template="cctally project --source {source}",
|
|
508
|
+
label="One project dominating",
|
|
509
|
+
),
|
|
510
|
+
ContributorSpec(
|
|
511
|
+
kind="session_concentration",
|
|
512
|
+
subject_kind="session",
|
|
513
|
+
min_distinct_subjects=2,
|
|
514
|
+
min_priced_entries=20,
|
|
515
|
+
# The subgroup form, for the same reason `model_mix` uses it.
|
|
516
|
+
next_step_template=(
|
|
517
|
+
"cctally {source} session{mode_flag} "
|
|
518
|
+
"--since {window_start_date} --until {window_end_date}"
|
|
519
|
+
),
|
|
520
|
+
label="One session dominating",
|
|
521
|
+
),
|
|
522
|
+
ContributorSpec(
|
|
523
|
+
kind="five_hour_bursts",
|
|
524
|
+
subject_kind="block",
|
|
525
|
+
min_distinct_subjects=2,
|
|
526
|
+
min_priced_entries=20,
|
|
527
|
+
# No generic template. This class's next step names ONE block, and a
|
|
528
|
+
# block subject key is the composite `<root>|<pool>|<instant>` that
|
|
529
|
+
# `_native_block_key` builds — so interpolating it rendered
|
|
530
|
+
# `cctally five-hour-breakdown --block-start claude|standard|<iso>`,
|
|
531
|
+
# which the command rejects and which a shell splits at the unquoted
|
|
532
|
+
# `|` into two piped commands. Both block builders set a per-subject
|
|
533
|
+
# override on every block they produce, which is the only form that
|
|
534
|
+
# can be correct, so the hazardous template is deleted rather than
|
|
535
|
+
# rewritten.
|
|
536
|
+
next_step_template=None,
|
|
537
|
+
label="Concentrated 5-hour bursts",
|
|
538
|
+
),
|
|
539
|
+
# --- the three S3 conversation-derived classes, APPENDED ---------------
|
|
540
|
+
#
|
|
541
|
+
# Appending is what keeps every S2 tie-break where it was: `REGISTRY_ORDER`
|
|
542
|
+
# is the tie-break after observed USD, so inserting anywhere else would
|
|
543
|
+
# re-rank rows that did not change. Each takes `min_distinct_subjects=1`
|
|
544
|
+
# because it contributes exactly ONE aggregate subject — the qualifying
|
|
545
|
+
# set — whose members appear only as evidence fields.
|
|
546
|
+
ContributorSpec(
|
|
547
|
+
kind="cache_churn",
|
|
548
|
+
subject_kind=SUBJECT_KIND_QUALIFYING_SET,
|
|
549
|
+
min_distinct_subjects=1,
|
|
550
|
+
min_priced_entries=20,
|
|
551
|
+
# Per-subject override; the loader names the window this class
|
|
552
|
+
# measured rather than any member of it.
|
|
553
|
+
next_step_template=None,
|
|
554
|
+
label="Prompt-cache churn",
|
|
555
|
+
rule=ClassRule(
|
|
556
|
+
sentence=("Turns whose prompt cache was rebuilt: the cached prefix "
|
|
557
|
+
"collapsed and was re-created in the same turn."),
|
|
558
|
+
parameters={"collapseFraction": 0.5, "recreateFraction": 0.75,
|
|
559
|
+
"cacheFloorTokens": 20_000, "createFloorTokens": 20_000},
|
|
560
|
+
),
|
|
561
|
+
),
|
|
562
|
+
ContributorSpec(
|
|
563
|
+
kind="short_high_context",
|
|
564
|
+
subject_kind=SUBJECT_KIND_QUALIFYING_SET,
|
|
565
|
+
min_distinct_subjects=1,
|
|
566
|
+
min_priced_entries=20,
|
|
567
|
+
next_step_template=None,
|
|
568
|
+
label="Short conversations carrying large context",
|
|
569
|
+
rule=ClassRule(
|
|
570
|
+
sentence=("Conversations with at most 3 human turns where at least "
|
|
571
|
+
"one request used at least 80% of that request's model "
|
|
572
|
+
"context window. With several replies the largest single "
|
|
573
|
+
"request fraction is used, never their sum."),
|
|
574
|
+
parameters={
|
|
575
|
+
"maxHumanTurns": DIAGNOSIS_SHORT_CONVERSATION_MAX_HUMAN_TURNS,
|
|
576
|
+
"minWindowFraction": DIAGNOSIS_LARGE_CONTEXT_MIN_WINDOW_FRACTION,
|
|
577
|
+
},
|
|
578
|
+
),
|
|
579
|
+
),
|
|
580
|
+
ContributorSpec(
|
|
581
|
+
kind="subagent_fanout",
|
|
582
|
+
subject_kind=SUBJECT_KIND_QUALIFYING_SET,
|
|
583
|
+
min_distinct_subjects=1,
|
|
584
|
+
min_priced_entries=20,
|
|
585
|
+
next_step_template=None,
|
|
586
|
+
label="Subagent fan-out",
|
|
587
|
+
rule=ClassRule(
|
|
588
|
+
sentence=("Cost attributable to delegated subagent work, grouped by "
|
|
589
|
+
"parent. On Codex this is an identifiable subset of "
|
|
590
|
+
"unknown completeness."),
|
|
591
|
+
parameters={"minBuckets": DIAGNOSIS_MIN_SUBAGENT_BUCKETS},
|
|
592
|
+
),
|
|
593
|
+
),
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
REGISTRY_ORDER: Mapping[str, int] = {
|
|
597
|
+
spec.kind: index for index, spec in enumerate(CONTRIBUTOR_REGISTRY)
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def spec_for(kind: str) -> ContributorSpec:
|
|
602
|
+
for spec in CONTRIBUTOR_REGISTRY:
|
|
603
|
+
if spec.kind == kind:
|
|
604
|
+
return spec
|
|
605
|
+
raise KeyError(kind)
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
# --- the execution plan (#620 S3 §3) ------------------------------------
|
|
609
|
+
#
|
|
610
|
+
# The route behaviour, the withheld causes, the stores opened, the cache
|
|
611
|
+
# projection and the generation hash are all derived from ONE immutable
|
|
612
|
+
# provider-local plan, so they cannot state four different things.
|
|
613
|
+
#
|
|
614
|
+
# Resolution is two-stage, because readability cannot be an input to a
|
|
615
|
+
# decision taken before anything is opened. Stage 1 is decided from
|
|
616
|
+
# authorization and provider capability alone. Only permitted stores are then
|
|
617
|
+
# opened, and stage 2 folds those outcomes into final modes. Stage 2's plan is
|
|
618
|
+
# the one that is hashed, so the identifier describes what actually ran.
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
class ClassMode(str, Enum):
|
|
622
|
+
MEASURE = "measure"
|
|
623
|
+
WITHHOLD = "withhold"
|
|
624
|
+
NOT_APPLICABLE = "not_applicable"
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
@dataclass(frozen=True)
|
|
628
|
+
class ClassPlan:
|
|
629
|
+
"""One class's mode, and what that mode requires of the stores."""
|
|
630
|
+
|
|
631
|
+
kind: str
|
|
632
|
+
mode: str # ClassMode value
|
|
633
|
+
cause: str | None # WithheldCause value on WITHHOLD
|
|
634
|
+
requires_conversations: bool
|
|
635
|
+
requires_s3_projection: bool
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
@dataclass(frozen=True)
|
|
639
|
+
class ExecutionPlan:
|
|
640
|
+
source: str
|
|
641
|
+
# Registry order, always all seven. A plan that omitted a class would make
|
|
642
|
+
# "which classes ran" depend on reading the list rather than on the mode.
|
|
643
|
+
classes: tuple[ClassPlan, ...]
|
|
644
|
+
|
|
645
|
+
def requires_conversations(self) -> bool:
|
|
646
|
+
return any(c.requires_conversations for c in self.classes)
|
|
647
|
+
|
|
648
|
+
def requires_s3_projection(self) -> bool:
|
|
649
|
+
return any(c.requires_s3_projection for c in self.classes)
|
|
650
|
+
|
|
651
|
+
def mode_for(self, kind: str) -> ClassPlan | None:
|
|
652
|
+
for c in self.classes:
|
|
653
|
+
if c.kind == kind:
|
|
654
|
+
return c
|
|
655
|
+
return None
|
|
656
|
+
|
|
657
|
+
def plan_token(self) -> str:
|
|
658
|
+
"""A deterministic token over the source and every class's decision.
|
|
659
|
+
|
|
660
|
+
EVERY plan tokenizes — there is no legacy exemption for the
|
|
661
|
+
four-class shape. A seven-class report with three withheld results is
|
|
662
|
+
a different report from a four-class one and must not share an
|
|
663
|
+
identifier with it.
|
|
664
|
+
"""
|
|
665
|
+
digest = hashlib.sha256()
|
|
666
|
+
digest.update(self.source.encode())
|
|
667
|
+
for c in self.classes:
|
|
668
|
+
digest.update(b"\x1e")
|
|
669
|
+
digest.update(f"{c.kind}={c.mode}:{c.cause or ''}".encode())
|
|
670
|
+
return digest.hexdigest()
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
# The classes whose predicate is defined over transcript rows, so they read
|
|
674
|
+
# `conversations.db` on BOTH providers.
|
|
675
|
+
_CONVERSATION_CLASSES: frozenset[str] = frozenset({
|
|
676
|
+
"cache_churn", "short_high_context",
|
|
677
|
+
})
|
|
678
|
+
# The S3 class whose store depends on the provider: Claude reads normalized
|
|
679
|
+
# subagent structure out of `conversations.db`, while Codex derives the same
|
|
680
|
+
# signal from `codex_conversation_threads` and `codex_session_entries`, which
|
|
681
|
+
# both live in `cache.db` — so it needs no transcript authorization at all.
|
|
682
|
+
_PROVIDER_DEPENDENT_S3_CLASSES: frozenset[str] = frozenset({"subagent_fanout"})
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def resolve_policy_plan(source: str, *,
|
|
686
|
+
transcripts_visible: bool) -> ExecutionPlan:
|
|
687
|
+
"""Stage 1, decided without touching a store.
|
|
688
|
+
|
|
689
|
+
A class denied here is SETTLED: the store it would have needed is never
|
|
690
|
+
opened, probed or digested.
|
|
691
|
+
"""
|
|
692
|
+
classes: list[ClassPlan] = []
|
|
693
|
+
for spec in CONTRIBUTOR_REGISTRY:
|
|
694
|
+
kind = spec.kind
|
|
695
|
+
if (kind not in _CONVERSATION_CLASSES
|
|
696
|
+
and kind not in _PROVIDER_DEPENDENT_S3_CLASSES):
|
|
697
|
+
# The four accounting-native classes. They read `cache.db` and
|
|
698
|
+
# `stats.db` only, which every plan opens.
|
|
699
|
+
classes.append(ClassPlan(kind, ClassMode.MEASURE.value, None,
|
|
700
|
+
False, False))
|
|
701
|
+
continue
|
|
702
|
+
if kind == "cache_churn" and source == "codex":
|
|
703
|
+
# Capability, not availability. Codex retains a cached-input ratio
|
|
704
|
+
# and no loss predicate, so this is never measurable there and is
|
|
705
|
+
# excluded from completeness rather than withheld.
|
|
706
|
+
classes.append(ClassPlan(kind, ClassMode.NOT_APPLICABLE.value, None,
|
|
707
|
+
False, False))
|
|
708
|
+
continue
|
|
709
|
+
needs_conversations = (
|
|
710
|
+
kind in _CONVERSATION_CLASSES
|
|
711
|
+
or (kind in _PROVIDER_DEPENDENT_S3_CLASSES and source == "claude")
|
|
712
|
+
)
|
|
713
|
+
if needs_conversations and not transcripts_visible:
|
|
714
|
+
classes.append(ClassPlan(kind, ClassMode.WITHHOLD.value,
|
|
715
|
+
WithheldCause.TRANSCRIPTS_NOT_VISIBLE.value,
|
|
716
|
+
False, False))
|
|
717
|
+
continue
|
|
718
|
+
classes.append(ClassPlan(kind, ClassMode.MEASURE.value, None,
|
|
719
|
+
needs_conversations, True))
|
|
720
|
+
return ExecutionPlan(source=source, classes=tuple(classes))
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def establish_plan(policy: ExecutionPlan, *, conversations_available: bool,
|
|
724
|
+
provider_cause: str | None) -> ExecutionPlan:
|
|
725
|
+
"""Stage 2, folding in what opening the permitted stores revealed.
|
|
726
|
+
|
|
727
|
+
`provider_cause` is the accounting-store overlay. A provider whose
|
|
728
|
+
accounting store cannot be read has no denominator at all, so it preempts
|
|
729
|
+
every cell — including the ones that measure — except `not_applicable`,
|
|
730
|
+
which is a capability statement rather than an availability one and stays
|
|
731
|
+
true whatever the store did.
|
|
732
|
+
"""
|
|
733
|
+
out: list[ClassPlan] = []
|
|
734
|
+
for c in policy.classes:
|
|
735
|
+
if c.mode == ClassMode.NOT_APPLICABLE.value:
|
|
736
|
+
out.append(c)
|
|
737
|
+
continue
|
|
738
|
+
if provider_cause is not None:
|
|
739
|
+
out.append(ClassPlan(c.kind, ClassMode.WITHHOLD.value,
|
|
740
|
+
provider_cause, False, False))
|
|
741
|
+
continue
|
|
742
|
+
if (c.mode == ClassMode.MEASURE.value and c.requires_conversations
|
|
743
|
+
and not conversations_available):
|
|
744
|
+
out.append(ClassPlan(c.kind, ClassMode.WITHHOLD.value,
|
|
745
|
+
WithheldCause.SIGNAL_UNAVAILABLE.value,
|
|
746
|
+
False, False))
|
|
747
|
+
continue
|
|
748
|
+
out.append(c)
|
|
749
|
+
return ExecutionPlan(source=policy.source, classes=tuple(out))
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
# --- confidence ---------------------------------------------------------
|
|
753
|
+
|
|
754
|
+
def assess_confidence(coverage: PopulationCoverage) -> str:
|
|
755
|
+
"""Three-valued confidence over `usd_coverage`.
|
|
756
|
+
|
|
757
|
+
`high` requires complete coverage and at least
|
|
758
|
+
`CONFIDENCE_HIGH_MIN_SUPPORT` support units. `medium` requires at least
|
|
759
|
+
`CONFIDENCE_MEDIUM_MIN_COVERAGE` and `CONFIDENCE_MEDIUM_MIN_SUPPORT`.
|
|
760
|
+
Anything else is `low`. Absent coverage is `low`, not `high`: we do not
|
|
761
|
+
know that it is complete.
|
|
762
|
+
"""
|
|
763
|
+
usd = coverage.usd_coverage
|
|
764
|
+
if usd is None:
|
|
765
|
+
return "low"
|
|
766
|
+
support = coverage.support_units or 0
|
|
767
|
+
if usd >= 1.0 - COVERAGE_EPSILON and support >= CONFIDENCE_HIGH_MIN_SUPPORT:
|
|
768
|
+
return "high"
|
|
769
|
+
if (usd >= CONFIDENCE_MEDIUM_MIN_COVERAGE - COVERAGE_EPSILON
|
|
770
|
+
and support >= CONFIDENCE_MEDIUM_MIN_SUPPORT):
|
|
771
|
+
return "medium"
|
|
772
|
+
return "low"
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def baseline_is_comparable(confidence: str) -> bool:
|
|
776
|
+
"""A baseline comparison requires at least `medium` confidence.
|
|
777
|
+
|
|
778
|
+
Below that only the baseline field is withheld; the observed-cost result
|
|
779
|
+
still renders. That separation is what keeps a thin baseline from
|
|
780
|
+
suppressing a measured answer.
|
|
781
|
+
"""
|
|
782
|
+
return confidence in ("high", "medium")
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
# --- classification -----------------------------------------------------
|
|
786
|
+
|
|
787
|
+
def _derive_population(subjects: Sequence[SubjectFacts],
|
|
788
|
+
denominator: Denominator) -> PopulationCoverage:
|
|
789
|
+
"""A complete coverage over exactly the handed-in subjects.
|
|
790
|
+
|
|
791
|
+
Used only when the caller supplies none, which is the pure-unit-test
|
|
792
|
+
case: the adapter always measures and passes its own.
|
|
793
|
+
"""
|
|
794
|
+
return PopulationCoverage(
|
|
795
|
+
requested_start=denominator.window_start,
|
|
796
|
+
requested_end=denominator.window_end,
|
|
797
|
+
observed_start=denominator.window_start,
|
|
798
|
+
observed_end=denominator.window_end,
|
|
799
|
+
count_coverage=1.0,
|
|
800
|
+
usd_coverage=1.0,
|
|
801
|
+
support_units=sum(s.priced_entry_count for s in subjects),
|
|
802
|
+
)
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
class _BlankDefaults(dict):
|
|
806
|
+
"""A format mapping that renders an unsupplied key as nothing at all."""
|
|
807
|
+
|
|
808
|
+
def __missing__(self, key: str) -> str:
|
|
809
|
+
return ""
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
# The next step a class with no generic template falls back to. It navigates
|
|
813
|
+
# to the WINDOW rather than to the subject, and interpolates no subject key,
|
|
814
|
+
# so it cannot emit a string a shell would re-interpret. Nothing in production
|
|
815
|
+
# reaches it — every block subject carries its own override — and it exists so
|
|
816
|
+
# that a hand-built subject still yields a runnable command instead of raising
|
|
817
|
+
# into the report or printing an empty `-> Run`.
|
|
818
|
+
_SUBJECTLESS_NEXT_STEP = (
|
|
819
|
+
"cctally {source} daily{mode_flag} "
|
|
820
|
+
"--since {window_start_date} --until {window_end_date}"
|
|
821
|
+
)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _render_next_step(spec: ContributorSpec, subject: SubjectFacts,
|
|
825
|
+
context: Mapping[str, Any]) -> str:
|
|
826
|
+
if subject.next_step:
|
|
827
|
+
return subject.next_step
|
|
828
|
+
template = spec.next_step_template or _SUBJECTLESS_NEXT_STEP
|
|
829
|
+
values = dict(context)
|
|
830
|
+
values.setdefault("source", "claude")
|
|
831
|
+
values.setdefault("mode_flag", "")
|
|
832
|
+
values.setdefault("window_start_date", "")
|
|
833
|
+
values.setdefault("window_end_date", "")
|
|
834
|
+
values["subject_key"] = subject.subject_key
|
|
835
|
+
values["subject_label"] = subject.subject_label
|
|
836
|
+
try:
|
|
837
|
+
return template.format(**values)
|
|
838
|
+
except KeyError:
|
|
839
|
+
# A template naming a key the caller did not supply must still produce
|
|
840
|
+
# a runnable command rather than raising into the report. Substituting
|
|
841
|
+
# blanks keeps the whole command; leaving a raw `{placeholder}` in text
|
|
842
|
+
# a person is told to run would not.
|
|
843
|
+
rendered = template.format_map(_BlankDefaults(values))
|
|
844
|
+
return " ".join(rendered.split())
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _build_row(spec: ContributorSpec, subject: SubjectFacts,
|
|
848
|
+
denominator: Denominator, population: PopulationCoverage,
|
|
849
|
+
confidence: str, context: Mapping[str, Any],
|
|
850
|
+
evidence: Mapping[str, EvidenceField] | None = None
|
|
851
|
+
) -> ContributorRow:
|
|
852
|
+
# The subject's qualifications are attached ONCE, to the observed cost.
|
|
853
|
+
# Attaching the same tuple to the share too shipped `block_precedes_window`
|
|
854
|
+
# twice per row, which reads as two separate qualifications of two separate
|
|
855
|
+
# figures rather than one statement about the subject.
|
|
856
|
+
observed = available(subject.observed_usd, population,
|
|
857
|
+
subject.qualifications)
|
|
858
|
+
# `classify_class` is the single gate on the denominator: it returns
|
|
859
|
+
# before building any row when the denominator is withheld or not
|
|
860
|
+
# positive, so there is no second guard here to fall out of step with it.
|
|
861
|
+
share = available(subject.observed_usd / denominator.usd_value, population)
|
|
862
|
+
|
|
863
|
+
if subject.baseline_code is not None:
|
|
864
|
+
baseline = withheld(subject.baseline_code, population)
|
|
865
|
+
elif not baseline_is_comparable(confidence):
|
|
866
|
+
baseline = withheld(BaselineOutcome.BASELINE_INSUFFICIENT.value,
|
|
867
|
+
population)
|
|
868
|
+
elif subject.baseline_share is None:
|
|
869
|
+
baseline = withheld(BaselineOutcome.BASELINE_INSUFFICIENT.value,
|
|
870
|
+
population)
|
|
871
|
+
else:
|
|
872
|
+
baseline = available(subject.baseline_share, population)
|
|
873
|
+
|
|
874
|
+
return ContributorRow(
|
|
875
|
+
contributor_class=spec.kind,
|
|
876
|
+
subject_kind=spec.subject_kind,
|
|
877
|
+
subject_key=subject.subject_key,
|
|
878
|
+
subject_label=subject.subject_label,
|
|
879
|
+
rank=None,
|
|
880
|
+
observed_usd=observed,
|
|
881
|
+
share=share,
|
|
882
|
+
baseline=baseline,
|
|
883
|
+
confidence=confidence,
|
|
884
|
+
is_fallback_pricing=subject.is_fallback_pricing,
|
|
885
|
+
next_step=_render_next_step(spec, subject, context),
|
|
886
|
+
# An immutable view, so the mapping a caller receives cannot be
|
|
887
|
+
# mutated through the row. Empty for every S2 class, which is what
|
|
888
|
+
# keeps `_row_to_wire` from emitting the key at all.
|
|
889
|
+
evidence=(MappingProxyType(dict(evidence)) if evidence else {}),
|
|
890
|
+
)
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def support_shortfall(spec: ContributorSpec,
|
|
894
|
+
subjects: Sequence[SubjectFacts],
|
|
895
|
+
population: PopulationCoverage,
|
|
896
|
+
*, predicate_evaluated: bool = False) -> str | None:
|
|
897
|
+
"""Which support minimum this class failed, or `None` when support is met.
|
|
898
|
+
|
|
899
|
+
The cause `insufficient_population` is one word for four different
|
|
900
|
+
shortfalls, and the count a surface prints beside it belongs to only one of
|
|
901
|
+
them. A class with sixty priced entries under a single model fails the
|
|
902
|
+
distinct-subject minimum, and printing `support 60 units` beside the cause
|
|
903
|
+
without naming that reads as a contradiction.
|
|
904
|
+
"""
|
|
905
|
+
# An evaluated predicate that matched nothing has no subjects BY RESULT,
|
|
906
|
+
# not by absence of data, so the distinct-subject minimum does not apply to
|
|
907
|
+
# it. Every other minimum still does: a class that looked at four entries
|
|
908
|
+
# and found nothing has not established a healthy window.
|
|
909
|
+
if not (predicate_evaluated and not subjects):
|
|
910
|
+
if len(subjects) < spec.min_distinct_subjects:
|
|
911
|
+
return SUPPORT_SHORTFALL_DISTINCT_SUBJECTS
|
|
912
|
+
units = population.support_units or 0
|
|
913
|
+
if units < max(spec.min_priced_entries, WITHHOLD_MIN_SUPPORT):
|
|
914
|
+
return SUPPORT_SHORTFALL_PRICED_ENTRIES
|
|
915
|
+
usd = population.usd_coverage
|
|
916
|
+
if usd is not None and usd < WITHHOLD_MIN_COVERAGE - COVERAGE_EPSILON:
|
|
917
|
+
return SUPPORT_SHORTFALL_USD_COVERAGE
|
|
918
|
+
return None
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def classify_class(
|
|
922
|
+
spec: ContributorSpec,
|
|
923
|
+
subjects: Sequence[SubjectFacts],
|
|
924
|
+
denominator: Denominator,
|
|
925
|
+
*,
|
|
926
|
+
population: PopulationCoverage | None = None,
|
|
927
|
+
preempting_cause: str | None = None,
|
|
928
|
+
not_applicable: bool = False,
|
|
929
|
+
next_step_context: Mapping[str, Any] | None = None,
|
|
930
|
+
predicate_evaluated: bool = False,
|
|
931
|
+
evidence: Mapping[str, EvidenceField] | None = None,
|
|
932
|
+
) -> ClassResult:
|
|
933
|
+
"""Turn one class's subjects into a verdict and its reported rows.
|
|
934
|
+
|
|
935
|
+
`contributor` when the top subject's share of the class denominator is at
|
|
936
|
+
least `DIAGNOSIS_CONTRIBUTOR_SHARE_FLOOR` and support is met.
|
|
937
|
+
`no_contributor` when support is met and the top share falls below the
|
|
938
|
+
floor — a measured, available answer, not a withheld one. `withheld` with
|
|
939
|
+
`insufficient_population` when support is not met. `not_applicable` when
|
|
940
|
+
the provider structurally cannot support the class.
|
|
941
|
+
|
|
942
|
+
Ties are resolved by reporting every subject within
|
|
943
|
+
`DIAGNOSIS_TIE_EPSILON_USD` of the top, ordered by subject key. The floor
|
|
944
|
+
boundary is inclusive: a share of exactly 0.20 is a contributor.
|
|
945
|
+
"""
|
|
946
|
+
context = dict(next_step_context or {})
|
|
947
|
+
pop = population if population is not None else _derive_population(
|
|
948
|
+
subjects, denominator
|
|
949
|
+
)
|
|
950
|
+
|
|
951
|
+
if not_applicable:
|
|
952
|
+
return ClassResult(spec.kind, VerdictState.NOT_APPLICABLE.value,
|
|
953
|
+
None, (), pop)
|
|
954
|
+
|
|
955
|
+
if preempting_cause is not None:
|
|
956
|
+
return ClassResult(spec.kind, VerdictState.WITHHELD.value,
|
|
957
|
+
preempting_cause, (), pop)
|
|
958
|
+
|
|
959
|
+
shortfall = support_shortfall(spec, subjects, pop,
|
|
960
|
+
predicate_evaluated=predicate_evaluated)
|
|
961
|
+
if shortfall is not None:
|
|
962
|
+
return ClassResult(spec.kind, VerdictState.WITHHELD.value,
|
|
963
|
+
WithheldCause.INSUFFICIENT_POPULATION.value, (), pop,
|
|
964
|
+
support_shortfall=shortfall)
|
|
965
|
+
|
|
966
|
+
if not denominator.usd_is_available:
|
|
967
|
+
# The denominator itself is withheld, so no share can be divided by
|
|
968
|
+
# it. The class carries the denominator's own cause rather than
|
|
969
|
+
# inventing a thinner one.
|
|
970
|
+
return ClassResult(spec.kind, VerdictState.WITHHELD.value,
|
|
971
|
+
denominator.usd.code
|
|
972
|
+
or WithheldCause.CALCULATION_FAILED.value, (), pop)
|
|
973
|
+
|
|
974
|
+
if denominator.usd_value <= 0.0:
|
|
975
|
+
# There is a population but no dollars to divide by, so no share can
|
|
976
|
+
# be stated. That is unmeasurable, not healthy.
|
|
977
|
+
return ClassResult(spec.kind, VerdictState.WITHHELD.value,
|
|
978
|
+
WithheldCause.INSUFFICIENT_POPULATION.value, (), pop,
|
|
979
|
+
support_shortfall=SUPPORT_SHORTFALL_NO_PRICED_DOLLARS)
|
|
980
|
+
|
|
981
|
+
confidence = assess_confidence(pop)
|
|
982
|
+
|
|
983
|
+
# PLACEMENT IS LOAD-BEARING. This sits after BOTH denominator guards and
|
|
984
|
+
# after confidence, and before `ordered[0]`. Placed earlier — the obvious
|
|
985
|
+
# reading of "before indexing" — a class whose denominator was withheld or
|
|
986
|
+
# zero reports healthy, which is the exact defect those two guards exist
|
|
987
|
+
# to prevent. An evaluated predicate that matched nothing has measured
|
|
988
|
+
# something: the healthy inverse, and the class counts as applicable
|
|
989
|
+
# rather than withheld.
|
|
990
|
+
if predicate_evaluated and not subjects:
|
|
991
|
+
return ClassResult(spec.kind, VerdictState.NO_CONTRIBUTOR.value,
|
|
992
|
+
None, (), pop, confidence=confidence)
|
|
993
|
+
|
|
994
|
+
ordered = sorted(subjects, key=lambda s: (-s.observed_usd, s.subject_key))
|
|
995
|
+
top_usd = ordered[0].observed_usd
|
|
996
|
+
top_share = top_usd / denominator.usd_value
|
|
997
|
+
|
|
998
|
+
# `top_share` is a ratio of two independently accumulated float sums, so a
|
|
999
|
+
# subject genuinely at one fifth of the denominator can compute as
|
|
1000
|
+
# 0.19999999999999998. The same slack this repository applies before
|
|
1001
|
+
# `math.floor` on a percent-like float applies here, or a measured
|
|
1002
|
+
# contributor flips to a healthy `no_contributor` on the last bit. The
|
|
1003
|
+
# slack is PUBLISHED (`DIAGNOSIS_SHARE_FLOOR_EPSILON`) because it decides
|
|
1004
|
+
# this verdict, and a client applying a bare `share >= floor` to the
|
|
1005
|
+
# published share would disagree with the server on exactly that row.
|
|
1006
|
+
if top_share < (DIAGNOSIS_CONTRIBUTOR_SHARE_FLOOR
|
|
1007
|
+
- DIAGNOSIS_SHARE_FLOOR_EPSILON):
|
|
1008
|
+
return ClassResult(spec.kind, VerdictState.NO_CONTRIBUTOR.value,
|
|
1009
|
+
None, (), pop, confidence=confidence)
|
|
1010
|
+
|
|
1011
|
+
tied = [s for s in ordered
|
|
1012
|
+
if abs(s.observed_usd - top_usd) <= DIAGNOSIS_TIE_EPSILON_USD]
|
|
1013
|
+
tied.sort(key=lambda s: s.subject_key)
|
|
1014
|
+
rows = tuple(
|
|
1015
|
+
_build_row(spec, s, denominator, pop, confidence, context, evidence)
|
|
1016
|
+
for s in tied
|
|
1017
|
+
)
|
|
1018
|
+
return ClassResult(spec.kind, VerdictState.CONTRIBUTOR.value, None,
|
|
1019
|
+
rows, pop, confidence=confidence)
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
# --- ranking ------------------------------------------------------------
|
|
1023
|
+
|
|
1024
|
+
def _row_usd(row: ContributorRow) -> float:
|
|
1025
|
+
value = row.observed_usd.value
|
|
1026
|
+
return float(value) if isinstance(value, (int, float)) else 0.0
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def rank_rows(rows: Iterable[ContributorRow]) -> tuple[ContributorRow, ...]:
|
|
1030
|
+
"""Rank heterogeneous contributor rows by `(-observedUsd, registryOrder,
|
|
1031
|
+
subject.key)` and stamp a one-based rank onto each."""
|
|
1032
|
+
ordered = sorted(
|
|
1033
|
+
rows,
|
|
1034
|
+
key=lambda r: (
|
|
1035
|
+
-_row_usd(r),
|
|
1036
|
+
REGISTRY_ORDER.get(r.contributor_class, len(CONTRIBUTOR_REGISTRY)),
|
|
1037
|
+
r.subject_key,
|
|
1038
|
+
),
|
|
1039
|
+
)
|
|
1040
|
+
return tuple(
|
|
1041
|
+
replace(row, rank=index) for index, row in enumerate(ordered, start=1)
|
|
1042
|
+
)
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
# --- the overall verdict ------------------------------------------------
|
|
1046
|
+
|
|
1047
|
+
def count_applicable(results: Sequence[ClassResult]) -> int:
|
|
1048
|
+
return sum(1 for r in results
|
|
1049
|
+
if r.verdict != VerdictState.NOT_APPLICABLE.value)
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def count_withheld(results: Sequence[ClassResult]) -> int:
|
|
1053
|
+
return sum(1 for r in results
|
|
1054
|
+
if r.verdict == VerdictState.WITHHELD.value)
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def overall_verdict(results: Sequence[ClassResult]) -> tuple[str, str | None]:
|
|
1058
|
+
"""`no_contributor_detected` only when every applicable class is
|
|
1059
|
+
available and none reports a contributor.
|
|
1060
|
+
|
|
1061
|
+
**The two overall verdicts are deliberately asymmetric.**
|
|
1062
|
+
`no_contributor_detected` is a claim about the whole population — nothing
|
|
1063
|
+
in this window reached the floor — so it requires every applicable class
|
|
1064
|
+
to be available. `contributor_detected` claims only that a contributor was
|
|
1065
|
+
found, which stays true beside a withheld sibling class, so a measured
|
|
1066
|
+
contributor is not erased by an unmeasurable sibling. Because
|
|
1067
|
+
`contributor_detected` is therefore not a complete account, every surface
|
|
1068
|
+
states how many applicable classes were withheld alongside it; see
|
|
1069
|
+
`count_withheld`.
|
|
1070
|
+
|
|
1071
|
+
`not_applicable` classes are excluded from the completeness test. Without
|
|
1072
|
+
that exclusion a provider with a permanently unsupported class could never
|
|
1073
|
+
render as healthy.
|
|
1074
|
+
"""
|
|
1075
|
+
applicable = [r for r in results
|
|
1076
|
+
if r.verdict != VerdictState.NOT_APPLICABLE.value]
|
|
1077
|
+
if any(r.verdict == VerdictState.CONTRIBUTOR.value for r in applicable):
|
|
1078
|
+
return OverallVerdict.CONTRIBUTOR_DETECTED.value, None
|
|
1079
|
+
|
|
1080
|
+
withheld_codes = [r.code for r in applicable
|
|
1081
|
+
if r.verdict == VerdictState.WITHHELD.value]
|
|
1082
|
+
if withheld_codes:
|
|
1083
|
+
for cause in WITHHELD_CAUSE_PRECEDENCE:
|
|
1084
|
+
if cause in withheld_codes:
|
|
1085
|
+
return OverallVerdict.WITHHELD.value, cause
|
|
1086
|
+
return OverallVerdict.WITHHELD.value, withheld_codes[0]
|
|
1087
|
+
|
|
1088
|
+
return OverallVerdict.NO_CONTRIBUTOR_DETECTED.value, None
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
# --- the assembled report -----------------------------------------------
|
|
1092
|
+
|
|
1093
|
+
@dataclass(frozen=True)
|
|
1094
|
+
class ProviderResult:
|
|
1095
|
+
"""One provider's whole answer: its own denominator, classes, ranked
|
|
1096
|
+
contributors and verdict. Nothing is ranked across providers and no
|
|
1097
|
+
denominator spans them."""
|
|
1098
|
+
|
|
1099
|
+
source: str
|
|
1100
|
+
account_key: str | None
|
|
1101
|
+
effective_speed: str | None
|
|
1102
|
+
denominator: Denominator
|
|
1103
|
+
classes: tuple[ClassResult, ...]
|
|
1104
|
+
contributors: tuple[ContributorRow, ...]
|
|
1105
|
+
verdict: str
|
|
1106
|
+
code: str | None
|
|
1107
|
+
applicable_class_count: int = 0
|
|
1108
|
+
withheld_class_count: int = 0
|
|
1109
|
+
generation: Any | None = None
|
|
1110
|
+
coverage: PopulationCoverage | None = None
|
|
1111
|
+
# The ESTABLISHED `ExecutionPlan` this result ran under. Threaded
|
|
1112
|
+
# explicitly rather than hung off the scope, because `_scope_for`
|
|
1113
|
+
# reconstructs scopes and would discard a field merely attached to one.
|
|
1114
|
+
plan: "ExecutionPlan | None" = None
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
def build_provider_result(
|
|
1118
|
+
source: str,
|
|
1119
|
+
account_key: str | None,
|
|
1120
|
+
effective_speed: str | None,
|
|
1121
|
+
denominator: Denominator,
|
|
1122
|
+
classes: Sequence[ClassResult],
|
|
1123
|
+
*,
|
|
1124
|
+
generation: Any | None = None,
|
|
1125
|
+
coverage: PopulationCoverage | None = None,
|
|
1126
|
+
plan: "ExecutionPlan | None" = None,
|
|
1127
|
+
) -> ProviderResult:
|
|
1128
|
+
verdict, code = overall_verdict(classes)
|
|
1129
|
+
ranked = rank_rows([row for result in classes for row in result.rows])
|
|
1130
|
+
# The same row appears in the ranked `contributors` view and in its own
|
|
1131
|
+
# class, so both carry the rank. Publishing `rank: null` on the class row
|
|
1132
|
+
# and a number on the contributor row is two representations of one row
|
|
1133
|
+
# disagreeing about it.
|
|
1134
|
+
ranks = {(row.contributor_class, row.subject_key): row.rank
|
|
1135
|
+
for row in ranked}
|
|
1136
|
+
classes = [
|
|
1137
|
+
replace(result, rows=tuple(
|
|
1138
|
+
replace(row, rank=ranks.get((row.contributor_class,
|
|
1139
|
+
row.subject_key)))
|
|
1140
|
+
for row in result.rows
|
|
1141
|
+
))
|
|
1142
|
+
for result in classes
|
|
1143
|
+
]
|
|
1144
|
+
return ProviderResult(
|
|
1145
|
+
source=source,
|
|
1146
|
+
account_key=account_key,
|
|
1147
|
+
effective_speed=effective_speed,
|
|
1148
|
+
denominator=denominator,
|
|
1149
|
+
classes=tuple(classes),
|
|
1150
|
+
contributors=ranked,
|
|
1151
|
+
verdict=verdict,
|
|
1152
|
+
code=code,
|
|
1153
|
+
applicable_class_count=count_applicable(classes),
|
|
1154
|
+
withheld_class_count=count_withheld(classes),
|
|
1155
|
+
generation=generation,
|
|
1156
|
+
coverage=coverage,
|
|
1157
|
+
plan=plan,
|
|
1158
|
+
)
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
@dataclass(frozen=True)
|
|
1162
|
+
class DiagnosisReport:
|
|
1163
|
+
contract_version: int
|
|
1164
|
+
measured_at: str
|
|
1165
|
+
window: DiagnosisWindow
|
|
1166
|
+
results: tuple[ProviderResult, ...]
|
|
1167
|
+
overall_verdict: str
|
|
1168
|
+
overall_code: str | None
|
|
1169
|
+
applicable_class_count: int = 0
|
|
1170
|
+
withheld_class_count: int = 0
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
def build_report(measured_at: str, window: DiagnosisWindow,
|
|
1174
|
+
results: Sequence[ProviderResult]) -> DiagnosisReport:
|
|
1175
|
+
"""Fold provider verdicts into one report verdict.
|
|
1176
|
+
|
|
1177
|
+
A withheld provider withholds the report, and a contributor anywhere is
|
|
1178
|
+
a positive finding, which is the same precedence `overall_verdict`
|
|
1179
|
+
applies within a provider.
|
|
1180
|
+
"""
|
|
1181
|
+
verdicts = [r.verdict for r in results]
|
|
1182
|
+
if OverallVerdict.CONTRIBUTOR_DETECTED.value in verdicts:
|
|
1183
|
+
overall, code = OverallVerdict.CONTRIBUTOR_DETECTED.value, None
|
|
1184
|
+
elif OverallVerdict.WITHHELD.value in verdicts:
|
|
1185
|
+
codes = [r.code for r in results
|
|
1186
|
+
if r.verdict == OverallVerdict.WITHHELD.value]
|
|
1187
|
+
code = next((c for c in WITHHELD_CAUSE_PRECEDENCE if c in codes), None)
|
|
1188
|
+
overall = OverallVerdict.WITHHELD.value
|
|
1189
|
+
elif verdicts:
|
|
1190
|
+
overall, code = OverallVerdict.NO_CONTRIBUTOR_DETECTED.value, None
|
|
1191
|
+
else:
|
|
1192
|
+
# UNREACHABLE from the product, and kept deliberately.
|
|
1193
|
+
# `build_diagnosis` always passes at least one `ProviderResult` — one
|
|
1194
|
+
# per physical provider the request named, and `--source all` names
|
|
1195
|
+
# two — so `results` is never empty there. It would become reachable if
|
|
1196
|
+
# a caller ever built a report from a filtered result list, and a
|
|
1197
|
+
# report of nothing must then say `withheld`, never
|
|
1198
|
+
# `no_contributor_detected`: claiming no contributor was detected over
|
|
1199
|
+
# zero providers asserts a measurement that was never attempted.
|
|
1200
|
+
overall, code = OverallVerdict.WITHHELD.value, \
|
|
1201
|
+
WithheldCause.PROVIDER_UNAVAILABLE.value
|
|
1202
|
+
return DiagnosisReport(
|
|
1203
|
+
contract_version=DIAGNOSIS_CONTRACT_VERSION,
|
|
1204
|
+
measured_at=measured_at,
|
|
1205
|
+
window=window,
|
|
1206
|
+
results=tuple(results),
|
|
1207
|
+
overall_verdict=overall,
|
|
1208
|
+
overall_code=code,
|
|
1209
|
+
applicable_class_count=sum(r.applicable_class_count for r in results),
|
|
1210
|
+
withheld_class_count=sum(r.withheld_class_count for r in results),
|
|
1211
|
+
)
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def unreadable_store_is_terminal(report: DiagnosisReport) -> bool:
|
|
1215
|
+
"""Whether a requested provider's store could not be read AND nothing else
|
|
1216
|
+
answered.
|
|
1217
|
+
|
|
1218
|
+
A store that cannot be read is an infrastructure failure, not an answer
|
|
1219
|
+
about data availability, so the surfaces report it as a failure: the CLI
|
|
1220
|
+
prints the withheld report and exits 3, and the route publishes the same
|
|
1221
|
+
report with 503. Every other withheld cause —
|
|
1222
|
+
`retained_range_mismatch`, `stale_evidence`, `pricing_unavailable`,
|
|
1223
|
+
`insufficient_population` — is a legitimate answer about what the store
|
|
1224
|
+
holds, and exits 0.
|
|
1225
|
+
|
|
1226
|
+
The second half of the predicate is what keeps `--source all` honest: one
|
|
1227
|
+
provider unreadable while the other reports is a report, so it exits 0.
|
|
1228
|
+
|
|
1229
|
+
**"Answered" is a CLASS-level question, not an overall-verdict one.** A
|
|
1230
|
+
provider whose classes measured three `no_contributor` results and withheld
|
|
1231
|
+
the fourth produced a measurement, but `overall_verdict` returns `withheld`
|
|
1232
|
+
for it, because that verdict is a claim about the whole population and a
|
|
1233
|
+
withheld sibling class makes the population claim incomplete. Reading the
|
|
1234
|
+
overall verdict here therefore counted a complete answer as nothing having
|
|
1235
|
+
answered: a Claude install whose status-line hook was never wired holds no
|
|
1236
|
+
`five_hour_blocks`, so `five_hour_bursts` is withheld while the other three
|
|
1237
|
+
classes measure, and `--source all` beside an unreadable Codex store then
|
|
1238
|
+
exited 3 and served 503 for a report that had measured three classes.
|
|
1239
|
+
`not_applicable` is not a measurement either — it says the class does not
|
|
1240
|
+
apply to this provider, which is not an answer about the window.
|
|
1241
|
+
"""
|
|
1242
|
+
# Both `any()` calls return `False` over an empty `results`, so a
|
|
1243
|
+
# zero-provider report exits 0 rather than 3. That is unreachable for the
|
|
1244
|
+
# same reason the `else` branch of `build_report` is — `build_diagnosis`
|
|
1245
|
+
# always passes at least one provider result — and it is the right answer
|
|
1246
|
+
# anyway: with no provider requested, nothing was unreadable. A caller that
|
|
1247
|
+
# ever builds a report from a filtered result list must decide whether an
|
|
1248
|
+
# empty list is a failure BEFORE reaching here, because this predicate
|
|
1249
|
+
# cannot tell "nothing was asked" from "nothing failed".
|
|
1250
|
+
unreadable = any(
|
|
1251
|
+
result.verdict == OverallVerdict.WITHHELD.value
|
|
1252
|
+
and result.code == WithheldCause.PROVIDER_UNAVAILABLE.value
|
|
1253
|
+
for result in report.results
|
|
1254
|
+
)
|
|
1255
|
+
answered = any(
|
|
1256
|
+
class_result.verdict not in (VerdictState.WITHHELD.value,
|
|
1257
|
+
VerdictState.NOT_APPLICABLE.value)
|
|
1258
|
+
for result in report.results
|
|
1259
|
+
for class_result in result.classes
|
|
1260
|
+
)
|
|
1261
|
+
return unreadable and not answered
|