@trazum/core 1.50.3 → 1.50.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema.d.ts +18 -1
- package/dist/config-schema.d.ts.map +1 -1
- package/dist/config-schema.js +58 -0
- package/dist/config-schema.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/outcome.d.ts +144 -0
- package/dist/outcome.d.ts.map +1 -0
- package/dist/outcome.js +117 -0
- package/dist/outcome.js.map +1 -0
- package/dist/per-outcome.d.ts +129 -0
- package/dist/per-outcome.d.ts.map +1 -0
- package/dist/per-outcome.js +150 -0
- package/dist/per-outcome.js.map +1 -0
- package/dist/usage.d.ts +41 -0
- package/dist/usage.d.ts.map +1 -1
- package/dist/usage.js +85 -1
- package/dist/usage.js.map +1 -1
- package/package.json +1 -1
- package/src/config-schema.ts +97 -0
- package/src/index.ts +23 -0
- package/src/outcome.ts +214 -0
- package/src/per-outcome.ts +235 -0
- package/src/usage.ts +132 -1
package/src/config-schema.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { BASELINE_FILENAME } from './baseline.js';
|
|
2
|
+
import type { OutcomeVocabulary } from './outcome.js';
|
|
2
3
|
import { mostSpecificMatch } from './glob.js';
|
|
3
4
|
import type { PricingCatalogue } from './pricing.js';
|
|
4
5
|
import { isLocale } from './i18n/index.js';
|
|
@@ -213,6 +214,21 @@ export interface TrazumConfig {
|
|
|
213
214
|
* author's terminal and not in CI.
|
|
214
215
|
*/
|
|
215
216
|
baseline?: BaselineConfig;
|
|
217
|
+
/**
|
|
218
|
+
* The product's own outcome vocabulary, and which values are successes.
|
|
219
|
+
*
|
|
220
|
+
* **Declared rather than guessed, and the second half is why.** `resolved`,
|
|
221
|
+
* `escalated`, `deflected`, `abandoned` — every product has its own words,
|
|
222
|
+
* and which of them count as success is a product judgement this tool has no
|
|
223
|
+
* standing to make. A tool that decided `escalated` was a failure would be
|
|
224
|
+
* wrong at every company where escalation is the correct, designed outcome
|
|
225
|
+
* for a whole class of request.
|
|
226
|
+
*
|
|
227
|
+
* A value in a log that this never declared is **named as undeclared**
|
|
228
|
+
* rather than bucketed into either side: a typo in an exporter should show up
|
|
229
|
+
* as a typo, not as a shift in the success rate.
|
|
230
|
+
*/
|
|
231
|
+
outcomes?: OutcomeVocabulary;
|
|
216
232
|
/** File extensions directory mode treats as prompts. */
|
|
217
233
|
extensions?: string[];
|
|
218
234
|
/**
|
|
@@ -244,6 +260,7 @@ export const CONFIG_KEYS = [
|
|
|
244
260
|
'baseline',
|
|
245
261
|
'extensions',
|
|
246
262
|
'pricing',
|
|
263
|
+
'outcomes',
|
|
247
264
|
] as const;
|
|
248
265
|
|
|
249
266
|
export const CONFIG_BASELINE_KEYS = ['path', 'maxGrowthTokens', 'maxGrowthPct'] as const;
|
|
@@ -254,6 +271,8 @@ export const CONFIG_WAIVE_KEYS = ['gate', 'reason', 'until'] as const;
|
|
|
254
271
|
|
|
255
272
|
export const CONFIG_STORE_KEYS = ['keepDays'] as const;
|
|
256
273
|
|
|
274
|
+
export const CONFIG_OUTCOME_KEYS = ['values', 'success'] as const;
|
|
275
|
+
|
|
257
276
|
/**
|
|
258
277
|
* The gates a waiver can silence. The list is closed on purpose: a waiver
|
|
259
278
|
* naming a gate that does not exist is a decision about nothing, and the
|
|
@@ -561,6 +580,81 @@ function parseSources(raw: unknown, source: string): Record<string, string[]> {
|
|
|
561
580
|
* gets deleted, and a policy the tool rounded on the operator's behalf is a
|
|
562
581
|
* policy nobody agreed to.
|
|
563
582
|
*/
|
|
583
|
+
/**
|
|
584
|
+
* `outcomes` — the product's own vocabulary, and which of it is success.
|
|
585
|
+
*
|
|
586
|
+
* The validation here is unusually strict for this file, and deliberately: an
|
|
587
|
+
* outcome vocabulary is the input to every rate this product will ever print,
|
|
588
|
+
* and a typo in it is a silent, permanent distortion of a number people make
|
|
589
|
+
* decisions on.
|
|
590
|
+
*/
|
|
591
|
+
function parseOutcomes(entry: unknown, source: string): OutcomeVocabulary {
|
|
592
|
+
if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {
|
|
593
|
+
throw new ConfigError(
|
|
594
|
+
`"outcomes" in ${source} must be an object, for example {"values": ["resolved", "escalated"], "success": ["resolved"]}.`,
|
|
595
|
+
source,
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
const raw = entry as Record<string, unknown>;
|
|
599
|
+
rejectUnknownKeys(raw, CONFIG_OUTCOME_KEYS, source, 'outcomes.');
|
|
600
|
+
|
|
601
|
+
const stringList = (value: unknown, key: string): string[] => {
|
|
602
|
+
if (
|
|
603
|
+
!Array.isArray(value) ||
|
|
604
|
+
value.some((item) => typeof item !== 'string' || item.trim() === '')
|
|
605
|
+
) {
|
|
606
|
+
throw new ConfigError(
|
|
607
|
+
`"outcomes.${key}" in ${source} must be an array of non-empty strings.`,
|
|
608
|
+
source,
|
|
609
|
+
);
|
|
610
|
+
}
|
|
611
|
+
return value as string[];
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
if (raw.values === undefined) {
|
|
615
|
+
throw new ConfigError(`"outcomes.values" in ${source} is required.`, source);
|
|
616
|
+
}
|
|
617
|
+
const values = stringList(raw.values, 'values');
|
|
618
|
+
const duplicates = values.filter((value, index) => values.indexOf(value) !== index);
|
|
619
|
+
if (duplicates.length > 0) {
|
|
620
|
+
throw new ConfigError(
|
|
621
|
+
`"outcomes.values" in ${source} lists ${JSON.stringify(duplicates[0])} more than once.`,
|
|
622
|
+
source,
|
|
623
|
+
);
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* **Required, and allowed to be empty.**
|
|
628
|
+
*
|
|
629
|
+
* Required because which values mean success is the entire product judgement
|
|
630
|
+
* this tool refuses to make on somebody's behalf — a tool that decided
|
|
631
|
+
* `escalated` was a failure would be wrong at every company where escalation
|
|
632
|
+
* is the correct, designed outcome for a class of request. Leaving it
|
|
633
|
+
* optional would send that question straight back here, to be answered by a
|
|
634
|
+
* default nobody chose.
|
|
635
|
+
*
|
|
636
|
+
* Allowed to be empty because a product that records only failures has
|
|
637
|
+
* declared something real. The report then says it cannot state a rate,
|
|
638
|
+
* rather than inventing one.
|
|
639
|
+
*/
|
|
640
|
+
if (raw.success === undefined) {
|
|
641
|
+
throw new ConfigError(
|
|
642
|
+
`"outcomes.success" in ${source} is required. Which of your outcome values count as success is a judgement about your product rather than about your bill, and this tool has no standing to make it. Use [] if none of them are.`,
|
|
643
|
+
source,
|
|
644
|
+
);
|
|
645
|
+
}
|
|
646
|
+
const success = stringList(raw.success, 'success');
|
|
647
|
+
const undeclared = success.filter((value) => !values.includes(value));
|
|
648
|
+
if (undeclared.length > 0) {
|
|
649
|
+
throw new ConfigError(
|
|
650
|
+
`"outcomes.success" in ${source} names ${undeclared.map((v) => JSON.stringify(v)).join(', ')}, which "outcomes.values" does not declare.`,
|
|
651
|
+
source,
|
|
652
|
+
);
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
return { values, success };
|
|
656
|
+
}
|
|
657
|
+
|
|
564
658
|
function parseStore(raw: unknown, source: string): { keepDays?: number } {
|
|
565
659
|
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
|
|
566
660
|
throw new Error(`"store" in ${source} must be an object, for example {"keepDays": 90}.`);
|
|
@@ -770,6 +864,9 @@ export function parseConfig(raw: string, source = CONFIG_FILENAME): TrazumConfig
|
|
|
770
864
|
if (document.sources !== undefined) config.sources = parseSources(document.sources, source);
|
|
771
865
|
if (document.store !== undefined) config.store = parseStore(document.store, source);
|
|
772
866
|
if (document.waive !== undefined) config.waive = parseWaive(document.waive, source);
|
|
867
|
+
if (document.outcomes !== undefined) {
|
|
868
|
+
config.outcomes = parseOutcomes(document.outcomes, source);
|
|
869
|
+
}
|
|
773
870
|
if (document.baseline !== undefined) {
|
|
774
871
|
config.baseline = parseBaselineConfig(document.baseline, source);
|
|
775
872
|
}
|
package/src/index.ts
CHANGED
|
@@ -49,6 +49,29 @@ export type { PlanParseFailure, PlanParseResult } from './plan.js';
|
|
|
49
49
|
export type { PlanAction, PlanActionKind, PlanAssumption, PlanDocument } from './plan.js';
|
|
50
50
|
export { verifyPlan } from './verify.js';
|
|
51
51
|
export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
|
|
52
|
+
export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
|
|
53
|
+
export {
|
|
54
|
+
perOutcome,
|
|
55
|
+
rankPerOutcome,
|
|
56
|
+
MIN_OUTCOMES_FOR_RATE,
|
|
57
|
+
MIN_COVERAGE_FOR_RATE,
|
|
58
|
+
} from './per-outcome.js';
|
|
59
|
+
export type {
|
|
60
|
+
PerOutcome,
|
|
61
|
+
PerOutcomeRanking,
|
|
62
|
+
RankedSlice,
|
|
63
|
+
SliceInput,
|
|
64
|
+
WithheldReason,
|
|
65
|
+
} from './per-outcome.js';
|
|
66
|
+
export type {
|
|
67
|
+
OutcomeCoverage,
|
|
68
|
+
OutcomeReport,
|
|
69
|
+
OutcomeSlice,
|
|
70
|
+
OutcomeTally,
|
|
71
|
+
OutcomeUnlock,
|
|
72
|
+
OutcomeVerdict,
|
|
73
|
+
OutcomeVocabulary,
|
|
74
|
+
} from './outcome.js';
|
|
52
75
|
export { gatewayDecision, usageFromResponse, FAILURE_POLICIES } from './gateway.js';
|
|
53
76
|
export type {
|
|
54
77
|
CannotTellCause,
|
package/src/outcome.ts
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The counterpart every figure in this product has been missing.
|
|
3
|
+
*
|
|
4
|
+
* Everything Trazum reports is a cost. It can say a workload got forty per cent
|
|
5
|
+
* cheaper and cannot say whether it stopped working — a denominator with no
|
|
6
|
+
* numerator, since the beginning. The missing field is not something this tool
|
|
7
|
+
* can compute. It is something only the caller knows.
|
|
8
|
+
*
|
|
9
|
+
* ## Recorded, never inferred
|
|
10
|
+
*
|
|
11
|
+
* The whole module rests on one refusal. **No absence of complaint counts as
|
|
12
|
+
* success. No short conversation counts as resolution. No retry counts as
|
|
13
|
+
* failure on its own.** Every one of those is a plausible-looking heuristic
|
|
14
|
+
* that would turn a guess into a metric, and a metric somebody would then
|
|
15
|
+
* optimise against — which is how a tool ends up rewarding conversations that
|
|
16
|
+
* end early because the user gave up.
|
|
17
|
+
*
|
|
18
|
+
* Where nothing was recorded, the report says so and names what recording one
|
|
19
|
+
* would unlock. That is the `fieldCoverage` discipline from 1.19, applied to
|
|
20
|
+
* the question that matters most.
|
|
21
|
+
*
|
|
22
|
+
* ## The vocabulary is declared, not guessed
|
|
23
|
+
*
|
|
24
|
+
* `resolved`, `escalated`, `deflected`, `abandoned`, `thumbs-down` — every
|
|
25
|
+
* product has its own words, and **which of them count as success is a product
|
|
26
|
+
* judgement this tool has no standing to make**. A tool that decided
|
|
27
|
+
* `escalated` was a failure would be wrong at every company where escalation is
|
|
28
|
+
* the correct, designed outcome for a whole class of request.
|
|
29
|
+
*
|
|
30
|
+
* So the config names the values and names which are successes. An outcome in
|
|
31
|
+
* the log that the config never declared is **named as undeclared**, not
|
|
32
|
+
* quietly bucketed into one side or the other — a typo in an exporter should
|
|
33
|
+
* show up as a typo, not as a shift in the success rate.
|
|
34
|
+
*
|
|
35
|
+
* ## The privacy line does not move
|
|
36
|
+
*
|
|
37
|
+
* An outcome is a small enumerated value. The store keeps it the way it has
|
|
38
|
+
* kept everything since 1.42: aggregated, never alongside content.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/** What the config declares. Both halves are required; see the module note. */
|
|
42
|
+
export interface OutcomeVocabulary {
|
|
43
|
+
/** Every value this product records. Anything else in a log is undeclared. */
|
|
44
|
+
values: string[];
|
|
45
|
+
/**
|
|
46
|
+
* Which of them count as success.
|
|
47
|
+
*
|
|
48
|
+
* A subset of `values`, and it may be empty — a vocabulary that records only
|
|
49
|
+
* failures is a legitimate thing to declare, and it still gives a rate this
|
|
50
|
+
* tool can report against a cost.
|
|
51
|
+
*/
|
|
52
|
+
success: string[];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export type OutcomeVerdict = 'success' | 'other' | 'undeclared';
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* How a single recorded value is judged. Three outcomes, never two.
|
|
59
|
+
*
|
|
60
|
+
* `other` is a declared value that is not a success. `undeclared` is a value
|
|
61
|
+
* nobody wrote down — which is a data-quality finding and not a result, and is
|
|
62
|
+
* reported as one.
|
|
63
|
+
*/
|
|
64
|
+
export function judgeOutcome(value: string | null, vocabulary: OutcomeVocabulary): OutcomeVerdict | null {
|
|
65
|
+
if (value === null) return null;
|
|
66
|
+
if (!vocabulary.values.includes(value)) return 'undeclared';
|
|
67
|
+
return vocabulary.success.includes(value) ? 'success' : 'other';
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** One value, and what it cost. */
|
|
71
|
+
export interface OutcomeSlice {
|
|
72
|
+
value: string;
|
|
73
|
+
verdict: OutcomeVerdict;
|
|
74
|
+
calls: number;
|
|
75
|
+
usd: number;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface OutcomeCoverage {
|
|
79
|
+
/** Records that carried an outcome at all. */
|
|
80
|
+
recorded: number;
|
|
81
|
+
/** Every record that parsed — the denominator. */
|
|
82
|
+
parsed: number;
|
|
83
|
+
/**
|
|
84
|
+
* Spend on calls that carried no outcome.
|
|
85
|
+
*
|
|
86
|
+
* The figure that decides whether a success rate means anything. A rate
|
|
87
|
+
* computed over eight per cent of the bill is a rate about eight per cent of
|
|
88
|
+
* the bill, and printing it beside the total without this number is how a
|
|
89
|
+
* sample becomes a claim about the whole.
|
|
90
|
+
*/
|
|
91
|
+
unrecordedUsd: number;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface OutcomeReport {
|
|
95
|
+
/** Per recorded value, dearest first. Empty when nothing was recorded. */
|
|
96
|
+
slices: OutcomeSlice[];
|
|
97
|
+
coverage: OutcomeCoverage;
|
|
98
|
+
/**
|
|
99
|
+
* Values found in the log that the config never declared, with what they
|
|
100
|
+
* cost. Named rather than bucketed: a typo in an exporter should surface as
|
|
101
|
+
* a typo and not as a shift in the success rate.
|
|
102
|
+
*/
|
|
103
|
+
undeclared: OutcomeSlice[];
|
|
104
|
+
/**
|
|
105
|
+
* The success rate **by spend**, or null when it cannot honestly be stated.
|
|
106
|
+
*
|
|
107
|
+
* By spend rather than by call, because this product's whole subject is
|
|
108
|
+
* money: a success rate weighted by call count says one thing while the
|
|
109
|
+
* expensive half of the traffic fails, and the two figures diverge exactly
|
|
110
|
+
* when it matters.
|
|
111
|
+
*
|
|
112
|
+
* Null has one meaning and it is not zero: nothing was recorded. A rate of
|
|
113
|
+
* zero is a real, terrible measurement, and a tool that spells "nobody told
|
|
114
|
+
* me" the same way has destroyed the difference between them.
|
|
115
|
+
*/
|
|
116
|
+
successShareOfRecordedUsd: number | null;
|
|
117
|
+
/**
|
|
118
|
+
* Why there is no rate, when there is none. A refusal never arrives bare.
|
|
119
|
+
*/
|
|
120
|
+
noRate: 'nothing-recorded' | 'no-success-values-declared' | null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* What was recorded, before anybody judged it.
|
|
125
|
+
*
|
|
126
|
+
* Deliberately split from the report: `profileUsage` produces this while it
|
|
127
|
+
* reads a log, knowing nothing about which values mean success, and the
|
|
128
|
+
* judgement happens where the config is. Measurement and product judgement are
|
|
129
|
+
* different jobs and this is the seam between them.
|
|
130
|
+
*
|
|
131
|
+
* It is also an **aggregate**, never a list of records — the same shape the
|
|
132
|
+
* store has kept since 1.42. Counting outcomes never means keeping calls.
|
|
133
|
+
*/
|
|
134
|
+
export interface OutcomeTally {
|
|
135
|
+
/** Per recorded value: how many calls carried it and what they cost. */
|
|
136
|
+
byValue: Array<{ value: string; calls: number; usd: number }>;
|
|
137
|
+
recorded: number;
|
|
138
|
+
parsed: number;
|
|
139
|
+
unrecordedUsd: number;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function outcomeReport(
|
|
143
|
+
tally: OutcomeTally,
|
|
144
|
+
vocabulary: OutcomeVocabulary | null,
|
|
145
|
+
): OutcomeReport {
|
|
146
|
+
const coverage: OutcomeCoverage = {
|
|
147
|
+
recorded: tally.recorded,
|
|
148
|
+
parsed: tally.parsed,
|
|
149
|
+
unrecordedUsd: tally.unrecordedUsd,
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* With no vocabulary declared, every recorded value is `undeclared`.
|
|
154
|
+
*
|
|
155
|
+
* That is the honest reading rather than an inconvenience: somebody has been
|
|
156
|
+
* writing outcomes into a log the config never described, so this tool knows
|
|
157
|
+
* what happened and not what any of it means. It reports the values and their
|
|
158
|
+
* cost, and declines the rate.
|
|
159
|
+
*/
|
|
160
|
+
const declared = vocabulary ?? { values: [], success: [] };
|
|
161
|
+
|
|
162
|
+
const all: OutcomeSlice[] = [...tally.byValue]
|
|
163
|
+
.map((entry) => ({
|
|
164
|
+
value: entry.value,
|
|
165
|
+
verdict: judgeOutcome(entry.value, declared) as OutcomeVerdict,
|
|
166
|
+
calls: entry.calls,
|
|
167
|
+
usd: entry.usd,
|
|
168
|
+
}))
|
|
169
|
+
.sort((a, b) => b.usd - a.usd);
|
|
170
|
+
|
|
171
|
+
const slices = all.filter((slice) => slice.verdict !== 'undeclared');
|
|
172
|
+
const undeclared = all.filter((slice) => slice.verdict === 'undeclared');
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The rate's denominator is **declared, recorded spend** — not the whole
|
|
176
|
+
* bill, and not every recorded value.
|
|
177
|
+
*
|
|
178
|
+
* Undeclared values are left out of both halves rather than counted as
|
|
179
|
+
* failures. A misspelled `resolvd` is a broken exporter, and folding it into
|
|
180
|
+
* the failure side would report a product regression that never happened —
|
|
181
|
+
* the direction that gets somebody paged at four in the morning.
|
|
182
|
+
*/
|
|
183
|
+
const declaredUsd = slices.reduce((sum, slice) => sum + slice.usd, 0);
|
|
184
|
+
const successUsd = slices
|
|
185
|
+
.filter((slice) => slice.verdict === 'success')
|
|
186
|
+
.reduce((sum, slice) => sum + slice.usd, 0);
|
|
187
|
+
|
|
188
|
+
let successShareOfRecordedUsd: number | null = null;
|
|
189
|
+
let noRate: OutcomeReport['noRate'] = null;
|
|
190
|
+
if (declared.success.length === 0) {
|
|
191
|
+
noRate = 'no-success-values-declared';
|
|
192
|
+
} else if (declaredUsd <= 0) {
|
|
193
|
+
noRate = 'nothing-recorded';
|
|
194
|
+
} else {
|
|
195
|
+
successShareOfRecordedUsd = successUsd / declaredUsd;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
return { slices, coverage, undeclared, successShareOfRecordedUsd, noRate };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* What recording an outcome would unlock, for a report that has none.
|
|
203
|
+
*
|
|
204
|
+
* Named rather than implied. "No outcome recorded" on its own reads as a
|
|
205
|
+
* missing feature; the point is that one small enumerated field turns every
|
|
206
|
+
* cost figure in this product into a cost *per* something.
|
|
207
|
+
*/
|
|
208
|
+
export const OUTCOME_UNLOCKS = [
|
|
209
|
+
'cost-per-outcome',
|
|
210
|
+
'success-rate-by-spend',
|
|
211
|
+
'cheaper-and-still-working',
|
|
212
|
+
] as const;
|
|
213
|
+
|
|
214
|
+
export type OutcomeUnlock = (typeof OUTCOME_UNLOCKS)[number];
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dollars per outcome — and the three ways this refuses to state one.
|
|
3
|
+
*
|
|
4
|
+
* 1.50.4 recorded the numerator. This divides by it, which sounds like
|
|
5
|
+
* arithmetic and is almost entirely a set of decisions about when *not* to do
|
|
6
|
+
* the arithmetic. A cost per resolution is the most quotable number this
|
|
7
|
+
* product will ever print — it ends up in a slide, in a quarterly review, in an
|
|
8
|
+
* argument about whether to keep a feature — and every way of getting it
|
|
9
|
+
* slightly wrong is a way of getting somebody's decision badly wrong.
|
|
10
|
+
*
|
|
11
|
+
* ## Which bill is the numerator
|
|
12
|
+
*
|
|
13
|
+
* The obvious implementation divides the **whole** slice bill by its successes.
|
|
14
|
+
* It is wrong, and wrong in the direction that makes a product look worse than
|
|
15
|
+
* it is: any call that carried no outcome is spend with no chance of appearing
|
|
16
|
+
* in the denominator, so the ratio is inflated by exactly the uninstrumented
|
|
17
|
+
* share — silently, and by an amount nobody can see from the figure.
|
|
18
|
+
*
|
|
19
|
+
* A team that instruments half its traffic would read a cost per resolution
|
|
20
|
+
* twice the real one, conclude the feature is uneconomic, and kill it.
|
|
21
|
+
*
|
|
22
|
+
* So the numerator is **recorded spend only**: the dollars on calls that
|
|
23
|
+
* carried an outcome. That makes it a ratio over a sample, which is fine and
|
|
24
|
+
* only fine because the coverage is stated beside it every single time, and
|
|
25
|
+
* because below a floor it is not stated at all.
|
|
26
|
+
*
|
|
27
|
+
* ## Three refusals
|
|
28
|
+
*
|
|
29
|
+
* 1. **Too few outcomes is not a rate.** Two resolutions and one figure is
|
|
30
|
+
* noise with a dollar sign. The count is shown instead — the same refusal
|
|
31
|
+
* `route` makes about small case sets and `history` makes about short runs.
|
|
32
|
+
* 2. **Too little coverage is not a rate.** A slice where an eighth of the
|
|
33
|
+
* spend carried an outcome yields a ratio over an unknown denominator. The
|
|
34
|
+
* coverage is shown instead.
|
|
35
|
+
* 3. **No successes recorded is not a rate of infinity.** A slice that spent
|
|
36
|
+
* money and resolved nothing is a real and alarming measurement, and it is
|
|
37
|
+
* reported as what it is rather than as a division by zero dressed up.
|
|
38
|
+
*
|
|
39
|
+
* ## Two rankings, and the product prints both
|
|
40
|
+
*
|
|
41
|
+
* Cheapest per call and cheapest per outcome are **different orders**, and the
|
|
42
|
+
* whole reason this chapter exists is that a workload can move up one while
|
|
43
|
+
* moving down the other. Picking one would be this tool making the choice it
|
|
44
|
+
* spent the last release refusing to make. Both are returned; the disagreement
|
|
45
|
+
* between them is itself a finding, and it is named.
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
import { judgeOutcome } from './outcome.js';
|
|
49
|
+
import type { OutcomeTally, OutcomeVocabulary } from './outcome.js';
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Recorded successes a slice needs before a per-outcome figure is stated.
|
|
53
|
+
*
|
|
54
|
+
* Ten, matching `history`'s `MIN_RUN` reasoning rather than a fresh number: a
|
|
55
|
+
* figure over fewer than ten observations moves more from one more observation
|
|
56
|
+
* than from anything a team could do about it.
|
|
57
|
+
*/
|
|
58
|
+
export const MIN_OUTCOMES_FOR_RATE = 10;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Share of a slice's spend that must carry an outcome before a rate is stated.
|
|
62
|
+
*
|
|
63
|
+
* 0.8, the same floor `watch` uses for a measured day. Below it the figure is a
|
|
64
|
+
* ratio over an unknown denominator, and the gap between what it says and what
|
|
65
|
+
* is true is not bounded by anything the reader can see.
|
|
66
|
+
*/
|
|
67
|
+
export const MIN_COVERAGE_FOR_RATE = 0.8;
|
|
68
|
+
|
|
69
|
+
export type WithheldReason =
|
|
70
|
+
| 'too-few-outcomes'
|
|
71
|
+
| 'too-little-coverage'
|
|
72
|
+
| 'no-successes-recorded'
|
|
73
|
+
| 'no-vocabulary'
|
|
74
|
+
| 'nothing-recorded';
|
|
75
|
+
|
|
76
|
+
export interface PerOutcome {
|
|
77
|
+
/**
|
|
78
|
+
* Dollars per recorded success, over **recorded spend only**, or null.
|
|
79
|
+
*
|
|
80
|
+
* Null is never a zero and never an infinity: `withheld` says which of the
|
|
81
|
+
* five reasons applies, and a caller that prints the figure without reading
|
|
82
|
+
* that field will print nothing rather than something wrong.
|
|
83
|
+
*/
|
|
84
|
+
usdPerSuccess: number | null;
|
|
85
|
+
withheld: WithheldReason | null;
|
|
86
|
+
/** Successes counted. Shown in place of the rate when it is withheld. */
|
|
87
|
+
successes: number;
|
|
88
|
+
/** Spend on calls that carried a declared outcome — the numerator. */
|
|
89
|
+
recordedUsd: number;
|
|
90
|
+
/** Everything the slice spent, recorded or not — for the coverage share. */
|
|
91
|
+
totalUsd: number;
|
|
92
|
+
/**
|
|
93
|
+
* Recorded share of the slice's spend, 0-1. Printed beside the rate every
|
|
94
|
+
* time it is printed, because a ratio over a sample presented without its
|
|
95
|
+
* sample size is a claim about the whole.
|
|
96
|
+
*/
|
|
97
|
+
coverage: number;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function perOutcome(
|
|
101
|
+
tally: OutcomeTally,
|
|
102
|
+
totalUsd: number,
|
|
103
|
+
vocabulary: OutcomeVocabulary | null,
|
|
104
|
+
): PerOutcome {
|
|
105
|
+
const declared = vocabulary ?? { values: [], success: [] };
|
|
106
|
+
|
|
107
|
+
let successes = 0;
|
|
108
|
+
let recordedUsd = 0;
|
|
109
|
+
for (const entry of tally.byValue) {
|
|
110
|
+
const verdict = judgeOutcome(entry.value, declared);
|
|
111
|
+
// Undeclared values are in neither the numerator nor the denominator, the
|
|
112
|
+
// same rule the success rate has: a typo in an exporter is a broken
|
|
113
|
+
// exporter, not a result.
|
|
114
|
+
if (verdict === 'undeclared') continue;
|
|
115
|
+
recordedUsd += entry.usd;
|
|
116
|
+
if (verdict === 'success') successes += entry.calls;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const coverage = totalUsd > 0 ? recordedUsd / totalUsd : 0;
|
|
120
|
+
const base: Omit<PerOutcome, 'usdPerSuccess' | 'withheld'> = {
|
|
121
|
+
successes,
|
|
122
|
+
recordedUsd,
|
|
123
|
+
totalUsd,
|
|
124
|
+
coverage,
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
if (vocabulary === null || declared.success.length === 0) {
|
|
128
|
+
return { ...base, usdPerSuccess: null, withheld: 'no-vocabulary' };
|
|
129
|
+
}
|
|
130
|
+
if (tally.recorded === 0) {
|
|
131
|
+
return { ...base, usdPerSuccess: null, withheld: 'nothing-recorded' };
|
|
132
|
+
}
|
|
133
|
+
if (successes === 0) {
|
|
134
|
+
// Money spent and nothing resolved. A real and alarming measurement, and
|
|
135
|
+
// reported as one rather than as a division by zero dressed up as a figure.
|
|
136
|
+
return { ...base, usdPerSuccess: null, withheld: 'no-successes-recorded' };
|
|
137
|
+
}
|
|
138
|
+
if (successes < MIN_OUTCOMES_FOR_RATE) {
|
|
139
|
+
return { ...base, usdPerSuccess: null, withheld: 'too-few-outcomes' };
|
|
140
|
+
}
|
|
141
|
+
if (coverage < MIN_COVERAGE_FOR_RATE) {
|
|
142
|
+
return { ...base, usdPerSuccess: null, withheld: 'too-little-coverage' };
|
|
143
|
+
}
|
|
144
|
+
return { ...base, usdPerSuccess: recordedUsd / successes, withheld: null };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export interface RankedSlice {
|
|
148
|
+
key: string;
|
|
149
|
+
calls: number;
|
|
150
|
+
totalUsd: number;
|
|
151
|
+
per: PerOutcome;
|
|
152
|
+
/** Dollars per call — the ranking this product has always been able to make. */
|
|
153
|
+
usdPerCall: number;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface PerOutcomeRanking {
|
|
157
|
+
/** Dearest per call first. Every slice appears. */
|
|
158
|
+
byCall: RankedSlice[];
|
|
159
|
+
/**
|
|
160
|
+
* Dearest per **outcome** first. Only slices with a stated rate appear —
|
|
161
|
+
* a withheld figure has no position in an order, and giving it one would put
|
|
162
|
+
* a slice somewhere on the strength of a number this module declined to
|
|
163
|
+
* state.
|
|
164
|
+
*/
|
|
165
|
+
byOutcome: RankedSlice[];
|
|
166
|
+
/**
|
|
167
|
+
* Slices whose two ranks disagree by more than a place, dearest-per-outcome
|
|
168
|
+
* first.
|
|
169
|
+
*
|
|
170
|
+
* **The finding a total cannot make**: the workload that looks cheap per
|
|
171
|
+
* call and is expensive per resolution, or the reverse. Somebody optimising
|
|
172
|
+
* on the first number has been moving the wrong one, and nothing in this
|
|
173
|
+
* product could tell them until now.
|
|
174
|
+
*/
|
|
175
|
+
disagreements: Array<{ slice: RankedSlice; callRank: number; outcomeRank: number }>;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export interface SliceInput {
|
|
179
|
+
key: string;
|
|
180
|
+
calls: number;
|
|
181
|
+
totalUsd: number;
|
|
182
|
+
tally: OutcomeTally;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export function rankPerOutcome(
|
|
186
|
+
slices: readonly SliceInput[],
|
|
187
|
+
vocabulary: OutcomeVocabulary | null,
|
|
188
|
+
): PerOutcomeRanking {
|
|
189
|
+
const ranked: RankedSlice[] = slices.map((slice) => ({
|
|
190
|
+
key: slice.key,
|
|
191
|
+
calls: slice.calls,
|
|
192
|
+
totalUsd: slice.totalUsd,
|
|
193
|
+
per: perOutcome(slice.tally, slice.totalUsd, vocabulary),
|
|
194
|
+
usdPerCall: slice.calls > 0 ? slice.totalUsd / slice.calls : 0,
|
|
195
|
+
}));
|
|
196
|
+
|
|
197
|
+
const byCall = [...ranked].sort((a, b) => b.usdPerCall - a.usdPerCall);
|
|
198
|
+
const byOutcome = ranked
|
|
199
|
+
.filter((slice) => slice.per.usdPerSuccess !== null)
|
|
200
|
+
.sort((a, b) => (b.per.usdPerSuccess as number) - (a.per.usdPerSuccess as number));
|
|
201
|
+
|
|
202
|
+
const callRankOf = new Map(byCall.map((slice, index) => [slice.key, index]));
|
|
203
|
+
const disagreements: PerOutcomeRanking['disagreements'] = [];
|
|
204
|
+
byOutcome.forEach((slice, outcomeRank) => {
|
|
205
|
+
/**
|
|
206
|
+
* Compared against this slice's position among **the rankable slices
|
|
207
|
+
* only**, not among all of them.
|
|
208
|
+
*
|
|
209
|
+
* Ranking it at position 4 of ten by call and 1 of three by outcome would
|
|
210
|
+
* report a disagreement produced entirely by the two lists having different
|
|
211
|
+
* lengths — an artefact, printed as a finding.
|
|
212
|
+
*/
|
|
213
|
+
const callRank = byCall
|
|
214
|
+
.filter((other) => other.per.usdPerSuccess !== null)
|
|
215
|
+
.findIndex((other) => other.key === slice.key);
|
|
216
|
+
/**
|
|
217
|
+
* More than one place — **or** a change at the top.
|
|
218
|
+
*
|
|
219
|
+
* The distance threshold alone misses the sharpest case there is: with two
|
|
220
|
+
* rankable slices a complete reversal is a distance of exactly one, and
|
|
221
|
+
* that is not noise, it is the finding. Whoever is dearest per call and
|
|
222
|
+
* whoever is dearest per resolution are the two names in the conversation,
|
|
223
|
+
* and them being different names is the whole point of computing both.
|
|
224
|
+
*/
|
|
225
|
+
const changedAtTheTop = (callRank === 0) !== (outcomeRank === 0);
|
|
226
|
+
if (Math.abs(callRank - outcomeRank) > 1 || changedAtTheTop) {
|
|
227
|
+
disagreements.push({ slice, callRank, outcomeRank });
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
// `callRankOf` is kept for callers that want the position among everything;
|
|
231
|
+
// the disagreement test deliberately does not use it, for the reason above.
|
|
232
|
+
void callRankOf;
|
|
233
|
+
|
|
234
|
+
return { byCall, byOutcome, disagreements };
|
|
235
|
+
}
|