@fgv/ts-extras 5.1.0-55 → 5.1.0-57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/CAPABILITIES.md +96 -0
  2. package/dist/packlets/ai-assist/cacheRequest.js +118 -0
  3. package/dist/packlets/ai-assist/cacheRequest.js.map +1 -0
  4. package/dist/packlets/ai-assist/chatRequestBuilders.js +72 -7
  5. package/dist/packlets/ai-assist/chatRequestBuilders.js.map +1 -1
  6. package/dist/packlets/ai-assist/completionClient.js +195 -58
  7. package/dist/packlets/ai-assist/completionClient.js.map +1 -1
  8. package/dist/packlets/ai-assist/converters.js +54 -0
  9. package/dist/packlets/ai-assist/converters.js.map +1 -1
  10. package/dist/packlets/ai-assist/embeddingClient.js +16 -1
  11. package/dist/packlets/ai-assist/embeddingClient.js.map +1 -1
  12. package/dist/packlets/ai-assist/imageGenerationClient.js +17 -2
  13. package/dist/packlets/ai-assist/imageGenerationClient.js.map +1 -1
  14. package/dist/packlets/ai-assist/index.js +3 -1
  15. package/dist/packlets/ai-assist/index.js.map +1 -1
  16. package/dist/packlets/ai-assist/listModelsClient.js +19 -1
  17. package/dist/packlets/ai-assist/listModelsClient.js.map +1 -1
  18. package/dist/packlets/ai-assist/model.js.map +1 -1
  19. package/dist/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.js +200 -0
  20. package/dist/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.js.map +1 -0
  21. package/dist/packlets/ai-assist/promptCacheObservabilityHarness.js +137 -0
  22. package/dist/packlets/ai-assist/promptCacheObservabilityHarness.js.map +1 -0
  23. package/dist/packlets/ai-assist/promptCacheRoutingAbHarness.js +156 -0
  24. package/dist/packlets/ai-assist/promptCacheRoutingAbHarness.js.map +1 -0
  25. package/dist/packlets/ai-assist/promptCacheUsageMath.js +124 -0
  26. package/dist/packlets/ai-assist/promptCacheUsageMath.js.map +1 -0
  27. package/dist/packlets/ai-assist/registry.js +9 -11
  28. package/dist/packlets/ai-assist/registry.js.map +1 -1
  29. package/dist/packlets/ai-assist/streamUsageCapability.js +134 -0
  30. package/dist/packlets/ai-assist/streamUsageCapability.js.map +1 -0
  31. package/dist/packlets/ai-assist/streamingAdapters/anthropic.js +32 -9
  32. package/dist/packlets/ai-assist/streamingAdapters/anthropic.js.map +1 -1
  33. package/dist/packlets/ai-assist/streamingAdapters/clientToolContinuationBuilder.js +81 -35
  34. package/dist/packlets/ai-assist/streamingAdapters/clientToolContinuationBuilder.js.map +1 -1
  35. package/dist/packlets/ai-assist/streamingAdapters/common.js +25 -1
  36. package/dist/packlets/ai-assist/streamingAdapters/common.js.map +1 -1
  37. package/dist/packlets/ai-assist/streamingAdapters/gemini.js +12 -5
  38. package/dist/packlets/ai-assist/streamingAdapters/gemini.js.map +1 -1
  39. package/dist/packlets/ai-assist/streamingAdapters/openaiChat.js +45 -11
  40. package/dist/packlets/ai-assist/streamingAdapters/openaiChat.js.map +1 -1
  41. package/dist/packlets/ai-assist/streamingAdapters/openaiResponses.js +33 -22
  42. package/dist/packlets/ai-assist/streamingAdapters/openaiResponses.js.map +1 -1
  43. package/dist/packlets/ai-assist/streamingAdapters/proxy.js +62 -4
  44. package/dist/packlets/ai-assist/streamingAdapters/proxy.js.map +1 -1
  45. package/dist/packlets/ai-assist/streamingClient.js +20 -5
  46. package/dist/packlets/ai-assist/streamingClient.js.map +1 -1
  47. package/dist/packlets/ai-assist/thinkingOptionsResolver.js +23 -5
  48. package/dist/packlets/ai-assist/thinkingOptionsResolver.js.map +1 -1
  49. package/dist/packlets/ai-assist/toolFormats.js +80 -0
  50. package/dist/packlets/ai-assist/toolFormats.js.map +1 -1
  51. package/dist/packlets/ai-assist/toolTypes.js +21 -0
  52. package/dist/packlets/ai-assist/toolTypes.js.map +1 -0
  53. package/dist/packlets/ai-assist/usageNormalization.js +178 -0
  54. package/dist/packlets/ai-assist/usageNormalization.js.map +1 -0
  55. package/dist/packlets/ai-assist/usageTypes.js +21 -0
  56. package/dist/packlets/ai-assist/usageTypes.js.map +1 -0
  57. package/dist/packlets/mustache/index.js.map +1 -1
  58. package/dist/packlets/mustache/interfaces.js.map +1 -1
  59. package/dist/packlets/mustache/mustacheTemplate.js +91 -0
  60. package/dist/packlets/mustache/mustacheTemplate.js.map +1 -1
  61. package/dist/ts-extras.d.ts +764 -48
  62. package/lib/packlets/ai-assist/cacheRequest.d.ts +100 -0
  63. package/lib/packlets/ai-assist/cacheRequest.d.ts.map +1 -0
  64. package/lib/packlets/ai-assist/cacheRequest.js +123 -0
  65. package/lib/packlets/ai-assist/cacheRequest.js.map +1 -0
  66. package/lib/packlets/ai-assist/chatRequestBuilders.d.ts +36 -6
  67. package/lib/packlets/ai-assist/chatRequestBuilders.d.ts.map +1 -1
  68. package/lib/packlets/ai-assist/chatRequestBuilders.js +75 -7
  69. package/lib/packlets/ai-assist/chatRequestBuilders.js.map +1 -1
  70. package/lib/packlets/ai-assist/completionClient.d.ts +19 -0
  71. package/lib/packlets/ai-assist/completionClient.d.ts.map +1 -1
  72. package/lib/packlets/ai-assist/completionClient.js +194 -57
  73. package/lib/packlets/ai-assist/completionClient.js.map +1 -1
  74. package/lib/packlets/ai-assist/converters.d.ts +46 -0
  75. package/lib/packlets/ai-assist/converters.d.ts.map +1 -1
  76. package/lib/packlets/ai-assist/converters.js +55 -1
  77. package/lib/packlets/ai-assist/converters.js.map +1 -1
  78. package/lib/packlets/ai-assist/embeddingClient.d.ts +9 -0
  79. package/lib/packlets/ai-assist/embeddingClient.d.ts.map +1 -1
  80. package/lib/packlets/ai-assist/embeddingClient.js +16 -1
  81. package/lib/packlets/ai-assist/embeddingClient.js.map +1 -1
  82. package/lib/packlets/ai-assist/imageGenerationClient.d.ts +9 -0
  83. package/lib/packlets/ai-assist/imageGenerationClient.d.ts.map +1 -1
  84. package/lib/packlets/ai-assist/imageGenerationClient.js +17 -2
  85. package/lib/packlets/ai-assist/imageGenerationClient.js.map +1 -1
  86. package/lib/packlets/ai-assist/index.d.ts +7 -2
  87. package/lib/packlets/ai-assist/index.d.ts.map +1 -1
  88. package/lib/packlets/ai-assist/index.js +11 -1
  89. package/lib/packlets/ai-assist/index.js.map +1 -1
  90. package/lib/packlets/ai-assist/listModelsClient.d.ts +12 -0
  91. package/lib/packlets/ai-assist/listModelsClient.d.ts.map +1 -1
  92. package/lib/packlets/ai-assist/listModelsClient.js +19 -1
  93. package/lib/packlets/ai-assist/listModelsClient.js.map +1 -1
  94. package/lib/packlets/ai-assist/model.d.ts +81 -295
  95. package/lib/packlets/ai-assist/model.d.ts.map +1 -1
  96. package/lib/packlets/ai-assist/model.js.map +1 -1
  97. package/lib/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.d.ts +124 -0
  98. package/lib/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.d.ts.map +1 -0
  99. package/lib/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.js +208 -0
  100. package/lib/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.js.map +1 -0
  101. package/lib/packlets/ai-assist/promptCacheObservabilityHarness.d.ts +77 -0
  102. package/lib/packlets/ai-assist/promptCacheObservabilityHarness.d.ts.map +1 -0
  103. package/lib/packlets/ai-assist/promptCacheObservabilityHarness.js +143 -0
  104. package/lib/packlets/ai-assist/promptCacheObservabilityHarness.js.map +1 -0
  105. package/lib/packlets/ai-assist/promptCacheRoutingAbHarness.d.ts +84 -0
  106. package/lib/packlets/ai-assist/promptCacheRoutingAbHarness.d.ts.map +1 -0
  107. package/lib/packlets/ai-assist/promptCacheRoutingAbHarness.js +163 -0
  108. package/lib/packlets/ai-assist/promptCacheRoutingAbHarness.js.map +1 -0
  109. package/lib/packlets/ai-assist/promptCacheUsageMath.d.ts +105 -0
  110. package/lib/packlets/ai-assist/promptCacheUsageMath.d.ts.map +1 -0
  111. package/lib/packlets/ai-assist/promptCacheUsageMath.js +130 -0
  112. package/lib/packlets/ai-assist/promptCacheUsageMath.js.map +1 -0
  113. package/lib/packlets/ai-assist/registry.d.ts.map +1 -1
  114. package/lib/packlets/ai-assist/registry.js +9 -11
  115. package/lib/packlets/ai-assist/registry.js.map +1 -1
  116. package/lib/packlets/ai-assist/streamUsageCapability.d.ts +129 -0
  117. package/lib/packlets/ai-assist/streamUsageCapability.d.ts.map +1 -0
  118. package/lib/packlets/ai-assist/streamUsageCapability.js +140 -0
  119. package/lib/packlets/ai-assist/streamUsageCapability.js.map +1 -0
  120. package/lib/packlets/ai-assist/streamingAdapters/anthropic.d.ts +2 -1
  121. package/lib/packlets/ai-assist/streamingAdapters/anthropic.d.ts.map +1 -1
  122. package/lib/packlets/ai-assist/streamingAdapters/anthropic.js +29 -6
  123. package/lib/packlets/ai-assist/streamingAdapters/anthropic.js.map +1 -1
  124. package/lib/packlets/ai-assist/streamingAdapters/clientToolContinuationBuilder.d.ts +58 -14
  125. package/lib/packlets/ai-assist/streamingAdapters/clientToolContinuationBuilder.d.ts.map +1 -1
  126. package/lib/packlets/ai-assist/streamingAdapters/clientToolContinuationBuilder.js +81 -36
  127. package/lib/packlets/ai-assist/streamingAdapters/clientToolContinuationBuilder.js.map +1 -1
  128. package/lib/packlets/ai-assist/streamingAdapters/common.d.ts +38 -3
  129. package/lib/packlets/ai-assist/streamingAdapters/common.d.ts.map +1 -1
  130. package/lib/packlets/ai-assist/streamingAdapters/common.js +25 -1
  131. package/lib/packlets/ai-assist/streamingAdapters/common.js.map +1 -1
  132. package/lib/packlets/ai-assist/streamingAdapters/gemini.d.ts.map +1 -1
  133. package/lib/packlets/ai-assist/streamingAdapters/gemini.js +12 -5
  134. package/lib/packlets/ai-assist/streamingAdapters/gemini.js.map +1 -1
  135. package/lib/packlets/ai-assist/streamingAdapters/openaiChat.d.ts +2 -1
  136. package/lib/packlets/ai-assist/streamingAdapters/openaiChat.d.ts.map +1 -1
  137. package/lib/packlets/ai-assist/streamingAdapters/openaiChat.js +42 -8
  138. package/lib/packlets/ai-assist/streamingAdapters/openaiChat.js.map +1 -1
  139. package/lib/packlets/ai-assist/streamingAdapters/openaiResponses.d.ts +2 -1
  140. package/lib/packlets/ai-assist/streamingAdapters/openaiResponses.d.ts.map +1 -1
  141. package/lib/packlets/ai-assist/streamingAdapters/openaiResponses.js +30 -19
  142. package/lib/packlets/ai-assist/streamingAdapters/openaiResponses.js.map +1 -1
  143. package/lib/packlets/ai-assist/streamingAdapters/proxy.d.ts +22 -0
  144. package/lib/packlets/ai-assist/streamingAdapters/proxy.d.ts.map +1 -1
  145. package/lib/packlets/ai-assist/streamingAdapters/proxy.js +62 -4
  146. package/lib/packlets/ai-assist/streamingAdapters/proxy.js.map +1 -1
  147. package/lib/packlets/ai-assist/streamingClient.d.ts +5 -0
  148. package/lib/packlets/ai-assist/streamingClient.d.ts.map +1 -1
  149. package/lib/packlets/ai-assist/streamingClient.js +20 -5
  150. package/lib/packlets/ai-assist/streamingClient.js.map +1 -1
  151. package/lib/packlets/ai-assist/thinkingOptionsResolver.d.ts +11 -3
  152. package/lib/packlets/ai-assist/thinkingOptionsResolver.d.ts.map +1 -1
  153. package/lib/packlets/ai-assist/thinkingOptionsResolver.js +23 -5
  154. package/lib/packlets/ai-assist/thinkingOptionsResolver.js.map +1 -1
  155. package/lib/packlets/ai-assist/toolFormats.d.ts +55 -1
  156. package/lib/packlets/ai-assist/toolFormats.d.ts.map +1 -1
  157. package/lib/packlets/ai-assist/toolFormats.js +82 -0
  158. package/lib/packlets/ai-assist/toolFormats.js.map +1 -1
  159. package/lib/packlets/ai-assist/toolTypes.d.ts +386 -0
  160. package/lib/packlets/ai-assist/toolTypes.d.ts.map +1 -0
  161. package/lib/packlets/ai-assist/toolTypes.js +22 -0
  162. package/lib/packlets/ai-assist/toolTypes.js.map +1 -0
  163. package/lib/packlets/ai-assist/usageNormalization.d.ts +58 -0
  164. package/lib/packlets/ai-assist/usageNormalization.d.ts.map +1 -0
  165. package/lib/packlets/ai-assist/usageNormalization.js +184 -0
  166. package/lib/packlets/ai-assist/usageNormalization.js.map +1 -0
  167. package/lib/packlets/ai-assist/usageTypes.d.ts +84 -0
  168. package/lib/packlets/ai-assist/usageTypes.d.ts.map +1 -0
  169. package/lib/packlets/ai-assist/usageTypes.js +22 -0
  170. package/lib/packlets/ai-assist/usageTypes.js.map +1 -0
  171. package/lib/packlets/mustache/index.d.ts +1 -1
  172. package/lib/packlets/mustache/index.d.ts.map +1 -1
  173. package/lib/packlets/mustache/index.js.map +1 -1
  174. package/lib/packlets/mustache/interfaces.d.ts +30 -0
  175. package/lib/packlets/mustache/interfaces.d.ts.map +1 -1
  176. package/lib/packlets/mustache/interfaces.js.map +1 -1
  177. package/lib/packlets/mustache/mustacheTemplate.d.ts +56 -1
  178. package/lib/packlets/mustache/mustacheTemplate.d.ts.map +1 -1
  179. package/lib/packlets/mustache/mustacheTemplate.js +91 -0
  180. package/lib/packlets/mustache/mustacheTemplate.js.map +1 -1
  181. package/package.json +16 -16
  182. package/perf/promptCacheAnthropicBreakpoint.js +135 -0
  183. package/perf/promptCacheObservability.js +107 -0
  184. package/perf/promptCacheRoutingAb.js +105 -0
@@ -0,0 +1,208 @@
1
+ "use strict";
2
+ // Copyright (c) 2026 Erik Fortune
3
+ //
4
+ // Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ // of this software and associated documentation files (the "Software"), to deal
6
+ // in the Software without restriction, including without limitation the rights
7
+ // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ // copies of the Software, and to permit persons to whom the Software is
9
+ // furnished to do so, subject to the following conditions:
10
+ //
11
+ // The above copyright notice and this permission notice shall be included in all
12
+ // copies or substantial portions of the Software.
13
+ //
14
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20
+ // SOFTWARE.
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.buildAnthropicBreakpointPrefix = buildAnthropicBreakpointPrefix;
23
+ exports.withAnthropicZeroDefaults = withAnthropicZeroDefaults;
24
+ exports.runPromptCacheAnthropicBreakpointArm = runPromptCacheAnthropicBreakpointArm;
25
+ exports.runPromptCacheAnthropicBreakpointHarness = runPromptCacheAnthropicBreakpointHarness;
26
+ exports.describeAnthropicBreakpointUsage = describeAnthropicBreakpointUsage;
27
+ exports.formatAnthropicBreakpointVerdict = formatAnthropicBreakpointVerdict;
28
+ /**
29
+ * Pure logic and injectable transport for `perf/promptCacheAnthropicBreakpoint.js` — the first
30
+ * measurement of whether C3's Anthropic `cache_control` emission (`IAiCacheRequest.systemBreakpoints`)
31
+ * actually produces a cache hit against a live provider.
32
+ *
33
+ * @remarks
34
+ * Every prompt-cache measurement shipped by this stream so far (`promptCacheObservability.js`,
35
+ * `promptCacheRoutingAb.js`) ran against xAI, whose caching is automatic — no request-side
36
+ * directive is sent, so neither harness exercises C3's emit path at all. Anthropic's cache is the
37
+ * sharp case: it is opt-in **per content block** via `cache_control`, with no automatic path, so a
38
+ * breakpoint is the only thing that can produce a hit. This harness is an A/B rather than a single
39
+ * standing assertion (unlike `promptCacheObservabilityHarness.ts`) precisely because Anthropic has
40
+ * no automatic caching to fall back on: without a negative control that reliably shows no hit, a
41
+ * hit in the positive arm could not be attributed to `systemBreakpoints` at all.
42
+ *
43
+ * Reuses {@link compareCacheHitRatios} / {@link computeCacheUsageRatio} /
44
+ * {@link formatCacheRatioPercent} from `promptCacheUsageMath.ts` rather than re-deriving the
45
+ * ratio/verdict arithmetic — see {@link withAnthropicZeroDefaults} for the one Anthropic-specific
46
+ * wrinkle those shared functions need help with. Also reuses
47
+ * {@link describeObservabilityUsage} from `promptCacheObservabilityHarness.ts` to print reads and
48
+ * writes separately, rather than writing a third formatter for the same shape.
49
+ * @packageDocumentation
50
+ */
51
+ const ts_utils_1 = require("@fgv/ts-utils");
52
+ const promptCacheObservabilityHarness_1 = require("./promptCacheObservabilityHarness");
53
+ const promptCacheUsageMath_1 = require("./promptCacheUsageMath");
54
+ /**
55
+ * Builds one arm's stable system prefix. Same shape as `buildObservabilityStablePrefix` /
56
+ * `buildRoutingAbPrefix` (~8,700 tokens measured for this exact 400-line generator against a
57
+ * comparable provider in this stream's earlier runs) — comfortably above every
58
+ * `minCacheablePrefixTokens` entry design.md §7 lists for Anthropic, including Haiku-class models'
59
+ * 4,096-token floor, not just the 1,024-token floor this harness's target model (`claude-sonnet-5`,
60
+ * the `'base'` tier this stream's other harnesses also default to) actually needs. `salt` and `arm`
61
+ * both feed the text so the two arms — and two separate runs — never warm each other's cache.
62
+ * @internal
63
+ */
64
+ function buildAnthropicBreakpointPrefix(salt, arm) {
65
+ const lines = [];
66
+ for (let i = 0; i < 400; i++) {
67
+ lines.push(`Directive ${salt}-${arm}-${i}: when asked about topic ${i}, respond precisely and cite source ${i}-${i * 7}.`);
68
+ }
69
+ return `You are a careful assistant operating under the following fixed policy document.\n\n${lines.join('\n')}`;
70
+ }
71
+ /**
72
+ * Fills Anthropic's cache-usage fields with their design-mandated zero default when the wire
73
+ * response omits them, before feeding a usage block to {@link compareCacheHitRatios}.
74
+ *
75
+ * @remarks
76
+ * `normalizeAnthropicUsage` always reports `reports: 'reads-and-writes'` for this provider, and
77
+ * design.md §8 states the semantics that follow from it explicitly: *"Under `'reads-and-writes'`,
78
+ * an absent `cacheWriteTokens` genuinely means zero were written."* The same reasoning covers
79
+ * `cachedInputTokens` and `uncachedInputTokens`. So for an Anthropic usage block specifically, an
80
+ * absent field is a **known** zero, not an unknown value — applying the default here does not
81
+ * violate design.md's R-c ("unknown is reported as unknown, never defaulted to a number"); it is
82
+ * what R-c's own `reports` discriminator exists to allow. This is deliberately narrower than
83
+ * {@link computeCacheUsageRatio}, which stays provider-agnostic (a `'reads'`-only provider's
84
+ * absent field genuinely is ambiguous, per design.md §8's own OQ-1 discussion) — the default is
85
+ * applied here, once, rather than widening the shared function's semantics for every caller.
86
+ *
87
+ * Display (`describeObservabilityUsage`) is untouched by this and keeps printing the raw wire
88
+ * value, so a reader can still see whether a field was actually reported or defaulted.
89
+ * @internal
90
+ */
91
+ function withAnthropicZeroDefaults(usage) {
92
+ var _a, _b, _c;
93
+ if (usage === undefined || usage.reports !== 'reads-and-writes') {
94
+ return usage;
95
+ }
96
+ return Object.assign(Object.assign({}, usage), { cachedInputTokens: (_a = usage.cachedInputTokens) !== null && _a !== void 0 ? _a : 0, uncachedInputTokens: (_b = usage.uncachedInputTokens) !== null && _b !== void 0 ? _b : 0, cacheWriteTokens: (_c = usage.cacheWriteTokens) !== null && _c !== void 0 ? _c : 0 });
97
+ }
98
+ /**
99
+ * Runs one arm: a cold call, then a warm call sharing the same
100
+ * {@link buildAnthropicBreakpointPrefix} system prompt and differing only in the final user turn —
101
+ * optionally supplying `cache.systemBreakpoints`. A single breakpoint is placed one character short
102
+ * of the end of `system`, marking essentially the whole fixed prefix as cacheable (design.md §6.1:
103
+ * offsets must be strictly `< system.length`).
104
+ * @internal
105
+ */
106
+ async function runPromptCacheAnthropicBreakpointArm(deps, params) {
107
+ const { descriptor, apiKey, label, salt, arm, withBreakpoint } = params;
108
+ const system = buildAnthropicBreakpointPrefix(salt, arm);
109
+ const cache = withBreakpoint
110
+ ? { systemBreakpoints: [system.length - 1] }
111
+ : undefined;
112
+ const call = (content) => deps.callCompletion({
113
+ descriptor,
114
+ apiKey,
115
+ system,
116
+ messages: [{ role: 'user', content }],
117
+ cache
118
+ });
119
+ const cold = await call('In one sentence, what is directive 12 about?');
120
+ if (cold.isFailure()) {
121
+ return (0, ts_utils_1.fail)(`${label}: cold call failed: ${cold.message}`);
122
+ }
123
+ // Only the tail differs from the cold call — the system prefix is byte-identical.
124
+ const warm = await call('In one sentence, what is directive 99 about?');
125
+ if (warm.isFailure()) {
126
+ return (0, ts_utils_1.fail)(`${label}: warm call failed: ${warm.message}`);
127
+ }
128
+ return (0, ts_utils_1.succeed)({ label, coldUsage: cold.value.usage, warmUsage: warm.value.usage });
129
+ }
130
+ /**
131
+ * Runs both arms of the Anthropic breakpoint A/B and classifies the result.
132
+ *
133
+ * @remarks
134
+ * Both warm usages are passed through {@link withAnthropicZeroDefaults} before comparison, so the
135
+ * negative control (`withoutBreakpoint`) is correctly read as "no cache activity" even if
136
+ * Anthropic's wire response omits the cache fields entirely on a request that sends no
137
+ * `cache_control` — the expected shape for that arm — rather than being scored `'inconclusive'`
138
+ * by {@link compareCacheHitRatios}'s provider-agnostic "absent field = unknown" default.
139
+ * @internal
140
+ */
141
+ async function runPromptCacheAnthropicBreakpointHarness(deps, params) {
142
+ const { descriptor, apiKey, salt } = params;
143
+ const withoutBreakpointResult = await runPromptCacheAnthropicBreakpointArm(deps, {
144
+ descriptor,
145
+ apiKey,
146
+ label: 'arm 1: WITHOUT systemBreakpoints (negative control)',
147
+ salt,
148
+ arm: 'a',
149
+ withBreakpoint: false
150
+ });
151
+ if (withoutBreakpointResult.isFailure()) {
152
+ return (0, ts_utils_1.fail)(withoutBreakpointResult.message);
153
+ }
154
+ const withBreakpointResult = await runPromptCacheAnthropicBreakpointArm(deps, {
155
+ descriptor,
156
+ apiKey,
157
+ label: 'arm 2: WITH systemBreakpoints (C3 emission under test)',
158
+ salt,
159
+ arm: 'b',
160
+ withBreakpoint: true
161
+ });
162
+ if (withBreakpointResult.isFailure()) {
163
+ return (0, ts_utils_1.fail)(withBreakpointResult.message);
164
+ }
165
+ const comparison = (0, promptCacheUsageMath_1.compareCacheHitRatios)(withAnthropicZeroDefaults(withoutBreakpointResult.value.warmUsage), withAnthropicZeroDefaults(withBreakpointResult.value.warmUsage));
166
+ return (0, ts_utils_1.succeed)({
167
+ withoutBreakpoint: withoutBreakpointResult.value,
168
+ withBreakpoint: withBreakpointResult.value,
169
+ verdict: comparison.verdict,
170
+ withoutBreakpointWarmRatio: comparison.baselineRatio,
171
+ withBreakpointWarmRatio: comparison.candidateRatio
172
+ });
173
+ }
174
+ /**
175
+ * One line per call, reusing `describeObservabilityUsage` so reads and writes are printed
176
+ * separately (`uncached=… cached=… written=…`) rather than collapsed into a single ratio, per
177
+ * this measurement's requirement to report Anthropic's `reads-and-writes` fields individually.
178
+ * @internal
179
+ */
180
+ function describeAnthropicBreakpointUsage(label, usage) {
181
+ return (0, promptCacheObservabilityHarness_1.describeObservabilityUsage)(label, usage);
182
+ }
183
+ /**
184
+ * Formats the A/B's closing verdict line(s).
185
+ * @internal
186
+ */
187
+ function formatAnthropicBreakpointVerdict(result) {
188
+ const a = (0, promptCacheUsageMath_1.formatCacheRatioPercent)(result.withoutBreakpointWarmRatio);
189
+ const b = (0, promptCacheUsageMath_1.formatCacheRatioPercent)(result.withBreakpointWarmRatio);
190
+ switch (result.verdict) {
191
+ case 'confirms':
192
+ return (`arm 1 (no breakpoints) warm ${a} -> arm 2 (systemBreakpoints) warm ${b}\n` +
193
+ 'CONFIRMS the prediction: Anthropic cache_control breakpoints produce a cache hit that an ' +
194
+ 'otherwise-identical request with no cache directive does not.');
195
+ case 'misses':
196
+ return (`arm 1 (no breakpoints) warm ${a} -> arm 2 (systemBreakpoints) warm ${b}\n` +
197
+ "MISSES the prediction: arm 2 did not clear the floor. Either C3's Anthropic cache_control " +
198
+ "emission is wrong, or this harness's model/threshold assumptions are — revise the design " +
199
+ 'or the code, do not re-run for a better number. (Re-run once with a fresh SALT first, to ' +
200
+ 'rule out a TTL-tainted reading.)');
201
+ case 'ambiguous':
202
+ return (`arm 1 (no breakpoints) warm ${a} -> arm 2 (systemBreakpoints) warm ${b}\n` +
203
+ 'AMBIGUOUS: neither arm clearly separates. Re-run with a fresh SALT.');
204
+ case 'inconclusive':
205
+ return 'inconclusive: cache-hit ratio not computable for at least one arm — a call may have failed.';
206
+ }
207
+ }
208
+ //# sourceMappingURL=promptCacheAnthropicBreakpointHarness.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptCacheAnthropicBreakpointHarness.js","sourceRoot":"","sources":["../../../src/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.ts"],"names":[],"mappings":";AAAA,kCAAkC;AAClC,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAC/E,4EAA4E;AAC5E,wEAAwE;AACxE,2DAA2D;AAC3D,EAAE;AACF,iFAAiF;AACjF,kDAAkD;AAClD,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,yEAAyE;AACzE,gFAAgF;AAChF,gFAAgF;AAChF,YAAY;;AAsDZ,wEAYC;AAsBD,8DAYC;AAiBD,oFAqCC;AAuBD,4FA6CC;AAQD,4EAKC;AAMD,4EA0BC;AAzQD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,4CAA2D;AAK3D,uFAA+E;AAC/E,iEAIgC;AAQhC;;;;;;;;;GASG;AACH,SAAgB,8BAA8B,CAAC,IAAY,EAAE,GAAW;IACtE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,aAAa,IAAI,IAAI,GAAG,IAAI,CAAC,4BAA4B,CAAC,uCAAuC,CAAC,IAChG,CAAC,GAAG,CACN,GAAG,CACJ,CAAC;IACJ,CAAC;IACD,OAAO,uFAAuF,KAAK,CAAC,IAAI,CACtG,IAAI,CACL,EAAE,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAgB,yBAAyB,CACvC,KAAqC;;IAErC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,kBAAkB,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,uCACK,KAAK,KACR,iBAAiB,EAAE,MAAA,KAAK,CAAC,iBAAiB,mCAAI,CAAC,EAC/C,mBAAmB,EAAE,MAAA,KAAK,CAAC,mBAAmB,mCAAI,CAAC,EACnD,gBAAgB,EAAE,MAAA,KAAK,CAAC,gBAAgB,mCAAI,CAAC,IAC7C;AACJ,CAAC;AASD;;;;;;;GAOG;AACI,KAAK,UAAU,oCAAoC,CACxD,IAAyC,EACzC,MAOC;IAED,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACxE,MAAM,MAAM,GAAG,8BAA8B,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACzD,MAAM,KAAK,GAAgC,cAAc;QACvD,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;QAC5C,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,IAAI,GAAG,CAAC,OAAe,EAA0C,EAAE,CACvE,IAAI,CAAC,cAAc,CAAC;QAClB,UAAU;QACV,MAAM;QACN,MAAM;QACN,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QACrC,KAAK;KACN,CAAC,CAAC;IAEL,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,8CAA8C,CAAC,CAAC;IACxE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;QACrB,OAAO,IAAA,eAAI,EAAC,GAAG,KAAK,uBAAuB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,kFAAkF;IAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,8CAA8C,CAAC,CAAC;IACxE,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;QACrB,OAAO,IAAA,eAAI,EAAC,GAAG,KAAK,uBAAuB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,IAAA,kBAAO,EAAC,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;AACtF,CAAC;AAYD;;;;;;;;;;GAUG;AACI,KAAK,UAAU,wCAAwC,CAC5D,IAAyC,EACzC,MAIC;IAED,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAE5C,MAAM,uBAAuB,GAAG,MAAM,oCAAoC,CAAC,IAAI,EAAE;QAC/E,UAAU;QACV,MAAM;QACN,KAAK,EAAE,qDAAqD;QAC5D,IAAI;QACJ,GAAG,EAAE,GAAG;QACR,cAAc,EAAE,KAAK;KACtB,CAAC,CAAC;IACH,IAAI,uBAAuB,CAAC,SAAS,EAAE,EAAE,CAAC;QACxC,OAAO,IAAA,eAAI,EAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,oBAAoB,GAAG,MAAM,oCAAoC,CAAC,IAAI,EAAE;QAC5E,UAAU;QACV,MAAM;QACN,KAAK,EAAE,wDAAwD;QAC/D,IAAI;QACJ,GAAG,EAAE,GAAG;QACR,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IACH,IAAI,oBAAoB,CAAC,SAAS,EAAE,EAAE,CAAC;QACrC,OAAO,IAAA,eAAI,EAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,UAAU,GAAG,IAAA,4CAAqB,EACtC,yBAAyB,CAAC,uBAAuB,CAAC,KAAK,CAAC,SAAS,CAAC,EAClE,yBAAyB,CAAC,oBAAoB,CAAC,KAAK,CAAC,SAAS,CAAC,CAChE,CAAC;IACF,OAAO,IAAA,kBAAO,EAAC;QACb,iBAAiB,EAAE,uBAAuB,CAAC,KAAK;QAChD,cAAc,EAAE,oBAAoB,CAAC,KAAK;QAC1C,OAAO,EAAE,UAAU,CAAC,OAAO;QAC3B,0BAA0B,EAAE,UAAU,CAAC,aAAa;QACpD,uBAAuB,EAAE,UAAU,CAAC,cAAc;KACnD,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAgB,gCAAgC,CAC9C,KAAa,EACb,KAAqC;IAErC,OAAO,IAAA,4DAA0B,EAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,SAAgB,gCAAgC,CAAC,MAA6C;IAC5F,MAAM,CAAC,GAAG,IAAA,8CAAuB,EAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;IACrE,MAAM,CAAC,GAAG,IAAA,8CAAuB,EAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAClE,QAAQ,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,KAAK,UAAU;YACb,OAAO,CACL,+BAA+B,CAAC,sCAAsC,CAAC,IAAI;gBAC3E,2FAA2F;gBAC3F,+DAA+D,CAChE,CAAC;QACJ,KAAK,QAAQ;YACX,OAAO,CACL,+BAA+B,CAAC,sCAAsC,CAAC,IAAI;gBAC3E,4FAA4F;gBAC5F,2FAA2F;gBAC3F,2FAA2F;gBAC3F,kCAAkC,CACnC,CAAC;QACJ,KAAK,WAAW;YACd,OAAO,CACL,+BAA+B,CAAC,sCAAsC,CAAC,IAAI;gBAC3E,qEAAqE,CACtE,CAAC;QACJ,KAAK,cAAc;YACjB,OAAO,6FAA6F,CAAC;IACzG,CAAC;AACH,CAAC","sourcesContent":["// Copyright (c) 2026 Erik Fortune\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all\n// copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n// SOFTWARE.\n\n/**\n * Pure logic and injectable transport for `perf/promptCacheAnthropicBreakpoint.js` — the first\n * measurement of whether C3's Anthropic `cache_control` emission (`IAiCacheRequest.systemBreakpoints`)\n * actually produces a cache hit against a live provider.\n *\n * @remarks\n * Every prompt-cache measurement shipped by this stream so far (`promptCacheObservability.js`,\n * `promptCacheRoutingAb.js`) ran against xAI, whose caching is automatic — no request-side\n * directive is sent, so neither harness exercises C3's emit path at all. Anthropic's cache is the\n * sharp case: it is opt-in **per content block** via `cache_control`, with no automatic path, so a\n * breakpoint is the only thing that can produce a hit. This harness is an A/B rather than a single\n * standing assertion (unlike `promptCacheObservabilityHarness.ts`) precisely because Anthropic has\n * no automatic caching to fall back on: without a negative control that reliably shows no hit, a\n * hit in the positive arm could not be attributed to `systemBreakpoints` at all.\n *\n * Reuses {@link compareCacheHitRatios} / {@link computeCacheUsageRatio} /\n * {@link formatCacheRatioPercent} from `promptCacheUsageMath.ts` rather than re-deriving the\n * ratio/verdict arithmetic — see {@link withAnthropicZeroDefaults} for the one Anthropic-specific\n * wrinkle those shared functions need help with. Also reuses\n * {@link describeObservabilityUsage} from `promptCacheObservabilityHarness.ts` to print reads and\n * writes separately, rather than writing a third formatter for the same shape.\n * @packageDocumentation\n */\n\nimport { fail, succeed, type Result } from '@fgv/ts-utils';\n\nimport { type IAiCacheRequest } from './cacheRequest';\nimport { type IProviderCompletionParams } from './completionClient';\nimport { type IAiCompletionResponse, type IAiProviderDescriptor } from './model';\nimport { describeObservabilityUsage } from './promptCacheObservabilityHarness';\nimport {\n type CacheHitComparisonVerdict,\n compareCacheHitRatios,\n formatCacheRatioPercent\n} from './promptCacheUsageMath';\nimport { type IAiCompletionUsage } from './usageTypes';\n\n/** Injected transport — only the network call needs a live key. @internal */\nexport interface IPromptCacheAnthropicBreakpointDeps {\n readonly callCompletion: (params: IProviderCompletionParams) => Promise<Result<IAiCompletionResponse>>;\n}\n\n/**\n * Builds one arm's stable system prefix. Same shape as `buildObservabilityStablePrefix` /\n * `buildRoutingAbPrefix` (~8,700 tokens measured for this exact 400-line generator against a\n * comparable provider in this stream's earlier runs) — comfortably above every\n * `minCacheablePrefixTokens` entry design.md §7 lists for Anthropic, including Haiku-class models'\n * 4,096-token floor, not just the 1,024-token floor this harness's target model (`claude-sonnet-5`,\n * the `'base'` tier this stream's other harnesses also default to) actually needs. `salt` and `arm`\n * both feed the text so the two arms — and two separate runs — never warm each other's cache.\n * @internal\n */\nexport function buildAnthropicBreakpointPrefix(salt: string, arm: string): string {\n const lines: string[] = [];\n for (let i = 0; i < 400; i++) {\n lines.push(\n `Directive ${salt}-${arm}-${i}: when asked about topic ${i}, respond precisely and cite source ${i}-${\n i * 7\n }.`\n );\n }\n return `You are a careful assistant operating under the following fixed policy document.\\n\\n${lines.join(\n '\\n'\n )}`;\n}\n\n/**\n * Fills Anthropic's cache-usage fields with their design-mandated zero default when the wire\n * response omits them, before feeding a usage block to {@link compareCacheHitRatios}.\n *\n * @remarks\n * `normalizeAnthropicUsage` always reports `reports: 'reads-and-writes'` for this provider, and\n * design.md §8 states the semantics that follow from it explicitly: *\"Under `'reads-and-writes'`,\n * an absent `cacheWriteTokens` genuinely means zero were written.\"* The same reasoning covers\n * `cachedInputTokens` and `uncachedInputTokens`. So for an Anthropic usage block specifically, an\n * absent field is a **known** zero, not an unknown value — applying the default here does not\n * violate design.md's R-c (\"unknown is reported as unknown, never defaulted to a number\"); it is\n * what R-c's own `reports` discriminator exists to allow. This is deliberately narrower than\n * {@link computeCacheUsageRatio}, which stays provider-agnostic (a `'reads'`-only provider's\n * absent field genuinely is ambiguous, per design.md §8's own OQ-1 discussion) — the default is\n * applied here, once, rather than widening the shared function's semantics for every caller.\n *\n * Display (`describeObservabilityUsage`) is untouched by this and keeps printing the raw wire\n * value, so a reader can still see whether a field was actually reported or defaulted.\n * @internal\n */\nexport function withAnthropicZeroDefaults(\n usage: IAiCompletionUsage | undefined\n): IAiCompletionUsage | undefined {\n if (usage === undefined || usage.reports !== 'reads-and-writes') {\n return usage;\n }\n return {\n ...usage,\n cachedInputTokens: usage.cachedInputTokens ?? 0,\n uncachedInputTokens: usage.uncachedInputTokens ?? 0,\n cacheWriteTokens: usage.cacheWriteTokens ?? 0\n };\n}\n\n/** One arm's cold-then-warm outcome. @internal */\nexport interface IPromptCacheAnthropicBreakpointArmResult {\n readonly label: string;\n readonly coldUsage: IAiCompletionUsage | undefined;\n readonly warmUsage: IAiCompletionUsage | undefined;\n}\n\n/**\n * Runs one arm: a cold call, then a warm call sharing the same\n * {@link buildAnthropicBreakpointPrefix} system prompt and differing only in the final user turn —\n * optionally supplying `cache.systemBreakpoints`. A single breakpoint is placed one character short\n * of the end of `system`, marking essentially the whole fixed prefix as cacheable (design.md §6.1:\n * offsets must be strictly `< system.length`).\n * @internal\n */\nexport async function runPromptCacheAnthropicBreakpointArm(\n deps: IPromptCacheAnthropicBreakpointDeps,\n params: {\n readonly descriptor: IAiProviderDescriptor;\n readonly apiKey: string;\n readonly label: string;\n readonly salt: string;\n readonly arm: string;\n readonly withBreakpoint: boolean;\n }\n): Promise<Result<IPromptCacheAnthropicBreakpointArmResult>> {\n const { descriptor, apiKey, label, salt, arm, withBreakpoint } = params;\n const system = buildAnthropicBreakpointPrefix(salt, arm);\n const cache: IAiCacheRequest | undefined = withBreakpoint\n ? { systemBreakpoints: [system.length - 1] }\n : undefined;\n const call = (content: string): Promise<Result<IAiCompletionResponse>> =>\n deps.callCompletion({\n descriptor,\n apiKey,\n system,\n messages: [{ role: 'user', content }],\n cache\n });\n\n const cold = await call('In one sentence, what is directive 12 about?');\n if (cold.isFailure()) {\n return fail(`${label}: cold call failed: ${cold.message}`);\n }\n\n // Only the tail differs from the cold call — the system prefix is byte-identical.\n const warm = await call('In one sentence, what is directive 99 about?');\n if (warm.isFailure()) {\n return fail(`${label}: warm call failed: ${warm.message}`);\n }\n\n return succeed({ label, coldUsage: cold.value.usage, warmUsage: warm.value.usage });\n}\n\n/** The full A/B: both arms' results and the breakpoint verdict. @internal */\nexport interface IPromptCacheAnthropicBreakpointResult {\n readonly withoutBreakpoint: IPromptCacheAnthropicBreakpointArmResult;\n readonly withBreakpoint: IPromptCacheAnthropicBreakpointArmResult;\n /** Baseline = `withoutBreakpoint`'s warm call; candidate = `withBreakpoint`'s warm call. */\n readonly verdict: CacheHitComparisonVerdict;\n readonly withoutBreakpointWarmRatio: number | undefined;\n readonly withBreakpointWarmRatio: number | undefined;\n}\n\n/**\n * Runs both arms of the Anthropic breakpoint A/B and classifies the result.\n *\n * @remarks\n * Both warm usages are passed through {@link withAnthropicZeroDefaults} before comparison, so the\n * negative control (`withoutBreakpoint`) is correctly read as \"no cache activity\" even if\n * Anthropic's wire response omits the cache fields entirely on a request that sends no\n * `cache_control` — the expected shape for that arm — rather than being scored `'inconclusive'`\n * by {@link compareCacheHitRatios}'s provider-agnostic \"absent field = unknown\" default.\n * @internal\n */\nexport async function runPromptCacheAnthropicBreakpointHarness(\n deps: IPromptCacheAnthropicBreakpointDeps,\n params: {\n readonly descriptor: IAiProviderDescriptor;\n readonly apiKey: string;\n readonly salt: string;\n }\n): Promise<Result<IPromptCacheAnthropicBreakpointResult>> {\n const { descriptor, apiKey, salt } = params;\n\n const withoutBreakpointResult = await runPromptCacheAnthropicBreakpointArm(deps, {\n descriptor,\n apiKey,\n label: 'arm 1: WITHOUT systemBreakpoints (negative control)',\n salt,\n arm: 'a',\n withBreakpoint: false\n });\n if (withoutBreakpointResult.isFailure()) {\n return fail(withoutBreakpointResult.message);\n }\n\n const withBreakpointResult = await runPromptCacheAnthropicBreakpointArm(deps, {\n descriptor,\n apiKey,\n label: 'arm 2: WITH systemBreakpoints (C3 emission under test)',\n salt,\n arm: 'b',\n withBreakpoint: true\n });\n if (withBreakpointResult.isFailure()) {\n return fail(withBreakpointResult.message);\n }\n\n const comparison = compareCacheHitRatios(\n withAnthropicZeroDefaults(withoutBreakpointResult.value.warmUsage),\n withAnthropicZeroDefaults(withBreakpointResult.value.warmUsage)\n );\n return succeed({\n withoutBreakpoint: withoutBreakpointResult.value,\n withBreakpoint: withBreakpointResult.value,\n verdict: comparison.verdict,\n withoutBreakpointWarmRatio: comparison.baselineRatio,\n withBreakpointWarmRatio: comparison.candidateRatio\n });\n}\n\n/**\n * One line per call, reusing `describeObservabilityUsage` so reads and writes are printed\n * separately (`uncached=… cached=… written=…`) rather than collapsed into a single ratio, per\n * this measurement's requirement to report Anthropic's `reads-and-writes` fields individually.\n * @internal\n */\nexport function describeAnthropicBreakpointUsage(\n label: string,\n usage: IAiCompletionUsage | undefined\n): string {\n return describeObservabilityUsage(label, usage);\n}\n\n/**\n * Formats the A/B's closing verdict line(s).\n * @internal\n */\nexport function formatAnthropicBreakpointVerdict(result: IPromptCacheAnthropicBreakpointResult): string {\n const a = formatCacheRatioPercent(result.withoutBreakpointWarmRatio);\n const b = formatCacheRatioPercent(result.withBreakpointWarmRatio);\n switch (result.verdict) {\n case 'confirms':\n return (\n `arm 1 (no breakpoints) warm ${a} -> arm 2 (systemBreakpoints) warm ${b}\\n` +\n 'CONFIRMS the prediction: Anthropic cache_control breakpoints produce a cache hit that an ' +\n 'otherwise-identical request with no cache directive does not.'\n );\n case 'misses':\n return (\n `arm 1 (no breakpoints) warm ${a} -> arm 2 (systemBreakpoints) warm ${b}\\n` +\n \"MISSES the prediction: arm 2 did not clear the floor. Either C3's Anthropic cache_control \" +\n \"emission is wrong, or this harness's model/threshold assumptions are — revise the design \" +\n 'or the code, do not re-run for a better number. (Re-run once with a fresh SALT first, to ' +\n 'rule out a TTL-tainted reading.)'\n );\n case 'ambiguous':\n return (\n `arm 1 (no breakpoints) warm ${a} -> arm 2 (systemBreakpoints) warm ${b}\\n` +\n 'AMBIGUOUS: neither arm clearly separates. Re-run with a fresh SALT.'\n );\n case 'inconclusive':\n return 'inconclusive: cache-hit ratio not computable for at least one arm — a call may have failed.';\n }\n}\n"]}
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Pure logic and injectable transport for `perf/promptCacheObservability.js` — the C1 standing
3
+ * assertion for design.md §8.
4
+ *
5
+ * @remarks
6
+ * Extracted so the ratio arithmetic and the cold/warm verdict can be unit-tested against fixture
7
+ * usage blocks without live traffic or an API key. Mirrors `samples/testbed`'s `xaiCacheProbe`
8
+ * scenario, whose `IXaiCacheProbeDeps` seam this design.md §8's addendum names as the shape to
9
+ * copy: only the network call is injected, everything else is a plain function.
10
+ * @packageDocumentation
11
+ */
12
+ import { type Result } from '@fgv/ts-utils';
13
+ import { type IProviderCompletionParams } from './completionClient';
14
+ import { type IAiCompletionResponse, type IAiProviderDescriptor } from './model';
15
+ import { type IAiCompletionUsage } from './usageTypes';
16
+ /**
17
+ * Injected transport. The harness's value is the arithmetic below; the network call is the only
18
+ * part that needs a live key, so it is the only part injected.
19
+ * @internal
20
+ */
21
+ export interface IPromptCacheObservabilityDeps {
22
+ readonly callCompletion: (params: IProviderCompletionParams) => Promise<Result<IAiCompletionResponse>>;
23
+ }
24
+ /**
25
+ * Builds the stable system prefix both calls share. Ported verbatim from the harness's
26
+ * `stablePrefix()` — repetition, not padding, so the text does not compress into something a
27
+ * provider's tokenizer collapses, and large enough that a genuine cache hit is unmistakable
28
+ * against design.md's recorded 128-token cold floor.
29
+ * @internal
30
+ */
31
+ export declare function buildObservabilityStablePrefix(): string;
32
+ /**
33
+ * One line summarizing a call's usage block, e.g.
34
+ * `cold call : reports=reads uncached=8684 cached=192 (2.2% of input) written=undefined output=20`.
35
+ * Ported from `summarizeUsage`, now returning the line instead of printing it, and computing the
36
+ * ratio via {@link computeCacheUsageRatio} instead of an unguarded division (see that function's
37
+ * remarks for the `NaN%` this replaces).
38
+ * @internal
39
+ */
40
+ export declare function describeObservabilityUsage(label: string, usage: IAiCompletionUsage | undefined): string;
41
+ /**
42
+ * Whether the standing assertion's prediction held: `usage.reports` should be `'reads'` on both
43
+ * calls, and the warm call's cache-hit ratio should clear the cold call's by a wide margin.
44
+ * @internal
45
+ */
46
+ export type PromptCacheObservabilityVerdict = 'confirmed' | 'missed' | 'inconclusive';
47
+ /** The evaluated outcome of one cold/warm pair, per design.md §8's written-down prediction. @internal */
48
+ export interface IPromptCacheObservabilityResult {
49
+ readonly cold: IAiCompletionUsage | undefined;
50
+ readonly warm: IAiCompletionUsage | undefined;
51
+ readonly coldRatio: number | undefined;
52
+ readonly warmRatio: number | undefined;
53
+ /** `true` only when both calls reported `usage.reports === 'reads'`, as design.md §8 predicts for xAI. */
54
+ readonly reportsAsPredicted: boolean;
55
+ readonly verdict: PromptCacheObservabilityVerdict;
56
+ readonly detail: string;
57
+ }
58
+ /**
59
+ * Evaluates the standing assertion's written-down prediction (design.md §8) against a cold/warm
60
+ * pair of usage blocks.
61
+ *
62
+ * @remarks
63
+ * This is new: the original harness printed both summaries and a static reminder for a human to
64
+ * judge by eye ("prediction check: reports should read 'reads' on both calls…"). Formalizing that
65
+ * judgement as a pure function is what makes it testable — and per `TESTING_GUIDELINES.md` §
66
+ * "Measurement Harnesses", an unevaluated prediction is exactly the kind of thing a broken
67
+ * harness never gets caught printing.
68
+ * @internal
69
+ */
70
+ export declare function evaluateObservabilityPrediction(cold: IAiCompletionUsage | undefined, warm: IAiCompletionUsage | undefined): IPromptCacheObservabilityResult;
71
+ /**
72
+ * Runs the standing assertion: a cold call, then a warm call sharing the same
73
+ * {@link buildObservabilityStablePrefix} system prompt and differing only in the final user turn.
74
+ * @internal
75
+ */
76
+ export declare function runPromptCacheObservabilityHarness(deps: IPromptCacheObservabilityDeps, descriptor: IAiProviderDescriptor, apiKey: string): Promise<Result<IPromptCacheObservabilityResult>>;
77
+ //# sourceMappingURL=promptCacheObservabilityHarness.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptCacheObservabilityHarness.d.ts","sourceRoot":"","sources":["../../../src/packlets/ai-assist/promptCacheObservabilityHarness.ts"],"names":[],"mappings":"AAoBA;;;;;;;;;;GAUG;AAEH,OAAO,EAAiB,KAAK,MAAM,EAAE,MAAM,eAAe,CAAC;AAE3D,OAAO,EAAE,KAAK,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,KAAK,qBAAqB,EAAE,KAAK,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAMjF,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEvD;;;;GAIG;AACH,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,yBAAyB,KAAK,OAAO,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC;CACxG;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,IAAI,MAAM,CAUvD;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,SAAS,GAAG,MAAM,CAWvG;AAED;;;;GAIG;AACH,MAAM,MAAM,+BAA+B,GAAG,WAAW,GAAG,QAAQ,GAAG,cAAc,CAAC;AAEtF,yGAAyG;AACzG,MAAM,WAAW,+BAA+B;IAC9C,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,SAAS,CAAC;IAC9C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,0GAA0G;IAC1G,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,+BAA+B,CAAC;IAClD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,+BAA+B,CAC7C,IAAI,EAAE,kBAAkB,GAAG,SAAS,EACpC,IAAI,EAAE,kBAAkB,GAAG,SAAS,GACnC,+BAA+B,CAmDjC;AAED;;;;GAIG;AACH,wBAAsB,kCAAkC,CACtD,IAAI,EAAE,6BAA6B,EACnC,UAAU,EAAE,qBAAqB,EACjC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,+BAA+B,CAAC,CAAC,CAwBlD"}
@@ -0,0 +1,143 @@
1
+ "use strict";
2
+ // Copyright (c) 2026 Erik Fortune
3
+ //
4
+ // Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ // of this software and associated documentation files (the "Software"), to deal
6
+ // in the Software without restriction, including without limitation the rights
7
+ // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ // copies of the Software, and to permit persons to whom the Software is
9
+ // furnished to do so, subject to the following conditions:
10
+ //
11
+ // The above copyright notice and this permission notice shall be included in all
12
+ // copies or substantial portions of the Software.
13
+ //
14
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20
+ // SOFTWARE.
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.buildObservabilityStablePrefix = buildObservabilityStablePrefix;
23
+ exports.describeObservabilityUsage = describeObservabilityUsage;
24
+ exports.evaluateObservabilityPrediction = evaluateObservabilityPrediction;
25
+ exports.runPromptCacheObservabilityHarness = runPromptCacheObservabilityHarness;
26
+ /**
27
+ * Pure logic and injectable transport for `perf/promptCacheObservability.js` — the C1 standing
28
+ * assertion for design.md §8.
29
+ *
30
+ * @remarks
31
+ * Extracted so the ratio arithmetic and the cold/warm verdict can be unit-tested against fixture
32
+ * usage blocks without live traffic or an API key. Mirrors `samples/testbed`'s `xaiCacheProbe`
33
+ * scenario, whose `IXaiCacheProbeDeps` seam this design.md §8's addendum names as the shape to
34
+ * copy: only the network call is injected, everything else is a plain function.
35
+ * @packageDocumentation
36
+ */
37
+ const ts_utils_1 = require("@fgv/ts-utils");
38
+ const promptCacheUsageMath_1 = require("./promptCacheUsageMath");
39
+ /**
40
+ * Builds the stable system prefix both calls share. Ported verbatim from the harness's
41
+ * `stablePrefix()` — repetition, not padding, so the text does not compress into something a
42
+ * provider's tokenizer collapses, and large enough that a genuine cache hit is unmistakable
43
+ * against design.md's recorded 128-token cold floor.
44
+ * @internal
45
+ */
46
+ function buildObservabilityStablePrefix() {
47
+ const lines = [];
48
+ for (let i = 0; i < 400; i++) {
49
+ lines.push(`Directive ${i}: when asked about topic ${i}, respond precisely and cite source ${i}-${i * 7}.`);
50
+ }
51
+ return `You are a careful assistant operating under the following fixed policy document.\n\n${lines.join('\n')}`;
52
+ }
53
+ /**
54
+ * One line summarizing a call's usage block, e.g.
55
+ * `cold call : reports=reads uncached=8684 cached=192 (2.2% of input) written=undefined output=20`.
56
+ * Ported from `summarizeUsage`, now returning the line instead of printing it, and computing the
57
+ * ratio via {@link computeCacheUsageRatio} instead of an unguarded division (see that function's
58
+ * remarks for the `NaN%` this replaces).
59
+ * @internal
60
+ */
61
+ function describeObservabilityUsage(label, usage) {
62
+ if (usage === undefined) {
63
+ return `${label}: usage absent from response`;
64
+ }
65
+ const { cachedRatio } = (0, promptCacheUsageMath_1.computeCacheUsageRatio)(usage);
66
+ return (`${label}: reports=${usage.reports} uncached=${usage.uncachedInputTokens} cached=${usage.cachedInputTokens} ` +
67
+ `(${(0, promptCacheUsageMath_1.formatCacheRatioPercent)(cachedRatio)} of input) written=${usage.cacheWriteTokens} output=${usage.outputTokens}`);
68
+ }
69
+ /**
70
+ * Evaluates the standing assertion's written-down prediction (design.md §8) against a cold/warm
71
+ * pair of usage blocks.
72
+ *
73
+ * @remarks
74
+ * This is new: the original harness printed both summaries and a static reminder for a human to
75
+ * judge by eye ("prediction check: reports should read 'reads' on both calls…"). Formalizing that
76
+ * judgement as a pure function is what makes it testable — and per `TESTING_GUIDELINES.md` §
77
+ * "Measurement Harnesses", an unevaluated prediction is exactly the kind of thing a broken
78
+ * harness never gets caught printing.
79
+ * @internal
80
+ */
81
+ function evaluateObservabilityPrediction(cold, warm) {
82
+ const comparison = (0, promptCacheUsageMath_1.compareCacheHitRatios)(cold, warm);
83
+ // `cold`/`warm` undefined is a strict subset of `comparison.verdict === 'inconclusive'` (an
84
+ // undefined usage block always yields an undefined ratio), but checking it explicitly is what
85
+ // narrows both to defined below — `reports` is a required field, so once narrowed there is no
86
+ // "usage present but reports absent" case left to guard against with `?.`/`?? 'absent'`.
87
+ if (cold === undefined || warm === undefined || comparison.verdict === 'inconclusive') {
88
+ return {
89
+ cold,
90
+ warm,
91
+ coldRatio: comparison.baselineRatio,
92
+ warmRatio: comparison.candidateRatio,
93
+ reportsAsPredicted: false,
94
+ verdict: 'inconclusive',
95
+ detail: 'cache-hit ratio not computable for the cold call, the warm call, or both'
96
+ };
97
+ }
98
+ const reportsAsPredicted = cold.reports === 'reads' && warm.reports === 'reads';
99
+ const common = {
100
+ cold,
101
+ warm,
102
+ coldRatio: comparison.baselineRatio,
103
+ warmRatio: comparison.candidateRatio,
104
+ reportsAsPredicted
105
+ };
106
+ if (!reportsAsPredicted) {
107
+ return Object.assign(Object.assign({}, common), { verdict: 'missed', detail: `usage.reports was not 'reads' on both calls (cold=${cold.reports}, warm=${warm.reports})` });
108
+ }
109
+ if (comparison.verdict === 'confirms') {
110
+ return Object.assign(Object.assign({}, common), { verdict: 'confirmed', detail: `warm cache-hit ratio (${(0, promptCacheUsageMath_1.formatCacheRatioPercent)(comparison.candidateRatio)}) clears the ` +
111
+ `cold floor (${(0, promptCacheUsageMath_1.formatCacheRatioPercent)(comparison.baselineRatio)})` });
112
+ }
113
+ return Object.assign(Object.assign({}, common), { verdict: 'missed', detail: `warm cache-hit ratio (${(0, promptCacheUsageMath_1.formatCacheRatioPercent)(comparison.candidateRatio)}) did not clear ` +
114
+ `the cold floor (${(0, promptCacheUsageMath_1.formatCacheRatioPercent)(comparison.baselineRatio)})` });
115
+ }
116
+ /**
117
+ * Runs the standing assertion: a cold call, then a warm call sharing the same
118
+ * {@link buildObservabilityStablePrefix} system prompt and differing only in the final user turn.
119
+ * @internal
120
+ */
121
+ async function runPromptCacheObservabilityHarness(deps, descriptor, apiKey) {
122
+ const system = buildObservabilityStablePrefix();
123
+ const cold = await deps.callCompletion({
124
+ descriptor,
125
+ apiKey,
126
+ system,
127
+ messages: [{ role: 'user', content: 'In one sentence, what is directive 12 about?' }]
128
+ });
129
+ if (cold.isFailure()) {
130
+ return (0, ts_utils_1.fail)(`cold call failed: ${cold.message}`);
131
+ }
132
+ const warm = await deps.callCompletion({
133
+ descriptor,
134
+ apiKey,
135
+ system,
136
+ messages: [{ role: 'user', content: 'In one sentence, what is directive 99 about?' }]
137
+ });
138
+ if (warm.isFailure()) {
139
+ return (0, ts_utils_1.fail)(`warm call failed: ${warm.message}`);
140
+ }
141
+ return (0, ts_utils_1.succeed)(evaluateObservabilityPrediction(cold.value.usage, warm.value.usage));
142
+ }
143
+ //# sourceMappingURL=promptCacheObservabilityHarness.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptCacheObservabilityHarness.js","sourceRoot":"","sources":["../../../src/packlets/ai-assist/promptCacheObservabilityHarness.ts"],"names":[],"mappings":";AAAA,kCAAkC;AAClC,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAC/E,4EAA4E;AAC5E,wEAAwE;AACxE,2DAA2D;AAC3D,EAAE;AACF,iFAAiF;AACjF,kDAAkD;AAClD,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,yEAAyE;AACzE,gFAAgF;AAChF,gFAAgF;AAChF,YAAY;;AAyCZ,wEAUC;AAUD,gEAWC;AAiCD,0EAsDC;AAOD,gFA4BC;AAhMD;;;;;;;;;;GAUG;AAEH,4CAA2D;AAI3D,iEAIgC;AAYhC;;;;;;GAMG;AACH,SAAgB,8BAA8B;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,aAAa,CAAC,4BAA4B,CAAC,uCAAuC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAChG,CAAC;IACJ,CAAC;IACD,OAAO,uFAAuF,KAAK,CAAC,IAAI,CACtG,IAAI,CACL,EAAE,CAAC;AACN,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,0BAA0B,CAAC,KAAa,EAAE,KAAqC;IAC7F,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,GAAG,KAAK,8BAA8B,CAAC;IAChD,CAAC;IACD,MAAM,EAAE,WAAW,EAAE,GAAG,IAAA,6CAAsB,EAAC,KAAK,CAAC,CAAC;IACtD,OAAO,CACL,GAAG,KAAK,aAAa,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,mBAAmB,WAAW,KAAK,CAAC,iBAAiB,GAAG;QAC7G,IAAI,IAAA,8CAAuB,EAAC,WAAW,CAAC,sBAAsB,KAAK,CAAC,gBAAgB,WAClF,KAAK,CAAC,YACR,EAAE,CACH,CAAC;AACJ,CAAC;AAqBD;;;;;;;;;;;GAWG;AACH,SAAgB,+BAA+B,CAC7C,IAAoC,EACpC,IAAoC;IAEpC,MAAM,UAAU,GAAG,IAAA,4CAAqB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAErD,4FAA4F;IAC5F,8FAA8F;IAC9F,8FAA8F;IAC9F,yFAAyF;IACzF,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;QACtF,OAAO;YACL,IAAI;YACJ,IAAI;YACJ,SAAS,EAAE,UAAU,CAAC,aAAa;YACnC,SAAS,EAAE,UAAU,CAAC,cAAc;YACpC,kBAAkB,EAAE,KAAK;YACzB,OAAO,EAAE,cAAc;YACvB,MAAM,EAAE,0EAA0E;SACnF,CAAC;IACJ,CAAC;IAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC;IAChF,MAAM,MAAM,GAAG;QACb,IAAI;QACJ,IAAI;QACJ,SAAS,EAAE,UAAU,CAAC,aAAa;QACnC,SAAS,EAAE,UAAU,CAAC,cAAc;QACpC,kBAAkB;KACnB,CAAC;IAEF,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,uCACK,MAAM,KACT,OAAO,EAAE,QAAQ,EACjB,MAAM,EAAE,qDAAqD,IAAI,CAAC,OAAO,UAAU,IAAI,CAAC,OAAO,GAAG,IAClG;IACJ,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QACtC,uCACK,MAAM,KACT,OAAO,EAAE,WAAW,EACpB,MAAM,EACJ,yBAAyB,IAAA,8CAAuB,EAAC,UAAU,CAAC,cAAc,CAAC,eAAe;gBAC1F,eAAe,IAAA,8CAAuB,EAAC,UAAU,CAAC,aAAa,CAAC,GAAG,IACrE;IACJ,CAAC;IACD,uCACK,MAAM,KACT,OAAO,EAAE,QAAQ,EACjB,MAAM,EACJ,yBAAyB,IAAA,8CAAuB,EAAC,UAAU,CAAC,cAAc,CAAC,kBAAkB;YAC7F,mBAAmB,IAAA,8CAAuB,EAAC,UAAU,CAAC,aAAa,CAAC,GAAG,IACzE;AACJ,CAAC;AAED;;;;GAIG;AACI,KAAK,UAAU,kCAAkC,CACtD,IAAmC,EACnC,UAAiC,EACjC,MAAc;IAEd,MAAM,MAAM,GAAG,8BAA8B,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;QACrC,UAAU;QACV,MAAM;QACN,MAAM;QACN,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,8CAA8C,EAAE,CAAC;KACtF,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;QACrB,OAAO,IAAA,eAAI,EAAC,qBAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;QACrC,UAAU;QACV,MAAM;QACN,MAAM;QACN,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,8CAA8C,EAAE,CAAC;KACtF,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;QACrB,OAAO,IAAA,eAAI,EAAC,qBAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,IAAA,kBAAO,EAAC,+BAA+B,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AACtF,CAAC","sourcesContent":["// Copyright (c) 2026 Erik Fortune\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all\n// copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n// SOFTWARE.\n\n/**\n * Pure logic and injectable transport for `perf/promptCacheObservability.js` — the C1 standing\n * assertion for design.md §8.\n *\n * @remarks\n * Extracted so the ratio arithmetic and the cold/warm verdict can be unit-tested against fixture\n * usage blocks without live traffic or an API key. Mirrors `samples/testbed`'s `xaiCacheProbe`\n * scenario, whose `IXaiCacheProbeDeps` seam this design.md §8's addendum names as the shape to\n * copy: only the network call is injected, everything else is a plain function.\n * @packageDocumentation\n */\n\nimport { fail, succeed, type Result } from '@fgv/ts-utils';\n\nimport { type IProviderCompletionParams } from './completionClient';\nimport { type IAiCompletionResponse, type IAiProviderDescriptor } from './model';\nimport {\n compareCacheHitRatios,\n computeCacheUsageRatio,\n formatCacheRatioPercent\n} from './promptCacheUsageMath';\nimport { type IAiCompletionUsage } from './usageTypes';\n\n/**\n * Injected transport. The harness's value is the arithmetic below; the network call is the only\n * part that needs a live key, so it is the only part injected.\n * @internal\n */\nexport interface IPromptCacheObservabilityDeps {\n readonly callCompletion: (params: IProviderCompletionParams) => Promise<Result<IAiCompletionResponse>>;\n}\n\n/**\n * Builds the stable system prefix both calls share. Ported verbatim from the harness's\n * `stablePrefix()` — repetition, not padding, so the text does not compress into something a\n * provider's tokenizer collapses, and large enough that a genuine cache hit is unmistakable\n * against design.md's recorded 128-token cold floor.\n * @internal\n */\nexport function buildObservabilityStablePrefix(): string {\n const lines: string[] = [];\n for (let i = 0; i < 400; i++) {\n lines.push(\n `Directive ${i}: when asked about topic ${i}, respond precisely and cite source ${i}-${i * 7}.`\n );\n }\n return `You are a careful assistant operating under the following fixed policy document.\\n\\n${lines.join(\n '\\n'\n )}`;\n}\n\n/**\n * One line summarizing a call's usage block, e.g.\n * `cold call : reports=reads uncached=8684 cached=192 (2.2% of input) written=undefined output=20`.\n * Ported from `summarizeUsage`, now returning the line instead of printing it, and computing the\n * ratio via {@link computeCacheUsageRatio} instead of an unguarded division (see that function's\n * remarks for the `NaN%` this replaces).\n * @internal\n */\nexport function describeObservabilityUsage(label: string, usage: IAiCompletionUsage | undefined): string {\n if (usage === undefined) {\n return `${label}: usage absent from response`;\n }\n const { cachedRatio } = computeCacheUsageRatio(usage);\n return (\n `${label}: reports=${usage.reports} uncached=${usage.uncachedInputTokens} cached=${usage.cachedInputTokens} ` +\n `(${formatCacheRatioPercent(cachedRatio)} of input) written=${usage.cacheWriteTokens} output=${\n usage.outputTokens\n }`\n );\n}\n\n/**\n * Whether the standing assertion's prediction held: `usage.reports` should be `'reads'` on both\n * calls, and the warm call's cache-hit ratio should clear the cold call's by a wide margin.\n * @internal\n */\nexport type PromptCacheObservabilityVerdict = 'confirmed' | 'missed' | 'inconclusive';\n\n/** The evaluated outcome of one cold/warm pair, per design.md §8's written-down prediction. @internal */\nexport interface IPromptCacheObservabilityResult {\n readonly cold: IAiCompletionUsage | undefined;\n readonly warm: IAiCompletionUsage | undefined;\n readonly coldRatio: number | undefined;\n readonly warmRatio: number | undefined;\n /** `true` only when both calls reported `usage.reports === 'reads'`, as design.md §8 predicts for xAI. */\n readonly reportsAsPredicted: boolean;\n readonly verdict: PromptCacheObservabilityVerdict;\n readonly detail: string;\n}\n\n/**\n * Evaluates the standing assertion's written-down prediction (design.md §8) against a cold/warm\n * pair of usage blocks.\n *\n * @remarks\n * This is new: the original harness printed both summaries and a static reminder for a human to\n * judge by eye (\"prediction check: reports should read 'reads' on both calls…\"). Formalizing that\n * judgement as a pure function is what makes it testable — and per `TESTING_GUIDELINES.md` §\n * \"Measurement Harnesses\", an unevaluated prediction is exactly the kind of thing a broken\n * harness never gets caught printing.\n * @internal\n */\nexport function evaluateObservabilityPrediction(\n cold: IAiCompletionUsage | undefined,\n warm: IAiCompletionUsage | undefined\n): IPromptCacheObservabilityResult {\n const comparison = compareCacheHitRatios(cold, warm);\n\n // `cold`/`warm` undefined is a strict subset of `comparison.verdict === 'inconclusive'` (an\n // undefined usage block always yields an undefined ratio), but checking it explicitly is what\n // narrows both to defined below — `reports` is a required field, so once narrowed there is no\n // \"usage present but reports absent\" case left to guard against with `?.`/`?? 'absent'`.\n if (cold === undefined || warm === undefined || comparison.verdict === 'inconclusive') {\n return {\n cold,\n warm,\n coldRatio: comparison.baselineRatio,\n warmRatio: comparison.candidateRatio,\n reportsAsPredicted: false,\n verdict: 'inconclusive',\n detail: 'cache-hit ratio not computable for the cold call, the warm call, or both'\n };\n }\n\n const reportsAsPredicted = cold.reports === 'reads' && warm.reports === 'reads';\n const common = {\n cold,\n warm,\n coldRatio: comparison.baselineRatio,\n warmRatio: comparison.candidateRatio,\n reportsAsPredicted\n };\n\n if (!reportsAsPredicted) {\n return {\n ...common,\n verdict: 'missed',\n detail: `usage.reports was not 'reads' on both calls (cold=${cold.reports}, warm=${warm.reports})`\n };\n }\n if (comparison.verdict === 'confirms') {\n return {\n ...common,\n verdict: 'confirmed',\n detail:\n `warm cache-hit ratio (${formatCacheRatioPercent(comparison.candidateRatio)}) clears the ` +\n `cold floor (${formatCacheRatioPercent(comparison.baselineRatio)})`\n };\n }\n return {\n ...common,\n verdict: 'missed',\n detail:\n `warm cache-hit ratio (${formatCacheRatioPercent(comparison.candidateRatio)}) did not clear ` +\n `the cold floor (${formatCacheRatioPercent(comparison.baselineRatio)})`\n };\n}\n\n/**\n * Runs the standing assertion: a cold call, then a warm call sharing the same\n * {@link buildObservabilityStablePrefix} system prompt and differing only in the final user turn.\n * @internal\n */\nexport async function runPromptCacheObservabilityHarness(\n deps: IPromptCacheObservabilityDeps,\n descriptor: IAiProviderDescriptor,\n apiKey: string\n): Promise<Result<IPromptCacheObservabilityResult>> {\n const system = buildObservabilityStablePrefix();\n\n const cold = await deps.callCompletion({\n descriptor,\n apiKey,\n system,\n messages: [{ role: 'user', content: 'In one sentence, what is directive 12 about?' }]\n });\n if (cold.isFailure()) {\n return fail(`cold call failed: ${cold.message}`);\n }\n\n const warm = await deps.callCompletion({\n descriptor,\n apiKey,\n system,\n messages: [{ role: 'user', content: 'In one sentence, what is directive 99 about?' }]\n });\n if (warm.isFailure()) {\n return fail(`warm call failed: ${warm.message}`);\n }\n\n return succeed(evaluateObservabilityPrediction(cold.value.usage, warm.value.usage));\n}\n"]}
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Pure logic and injectable transport for `perf/promptCacheRoutingAb.js` — the A/B that measures
3
+ * whether `IAiCacheRequest.cacheKey` actually improves the xAI cache-hit rate (design.md §2's
4
+ * 2026-09-19 measurement note, and result.md's "gating defect in C3").
5
+ *
6
+ * @remarks
7
+ * Extracted for the same reason as `promptCacheObservabilityHarness.ts`: the verdict this harness
8
+ * prints (CONFIRMS / MISSES / AMBIGUOUS) "has already been used to justify a merge," per this
9
+ * stream's follow-up task, and had no test checking it classifies correctly.
10
+ * @packageDocumentation
11
+ */
12
+ import { type Result } from '@fgv/ts-utils';
13
+ import { type IProviderCompletionParams } from './completionClient';
14
+ import { type IAiCompletionResponse, type IAiProviderDescriptor } from './model';
15
+ import { type CacheHitComparisonVerdict } from './promptCacheUsageMath';
16
+ import { type IAiCompletionUsage } from './usageTypes';
17
+ /** Injected transport — only the network call needs a live key. @internal */
18
+ export interface IPromptCacheRoutingAbDeps {
19
+ readonly callCompletion: (params: IProviderCompletionParams) => Promise<Result<IAiCompletionResponse>>;
20
+ }
21
+ /**
22
+ * Builds one arm's stable system prefix. Ported verbatim from the harness's `stablePrefix(arm)`.
23
+ * `salt` and `arm` both feed the text so the two arms — and two separate runs — never warm each
24
+ * other's cache.
25
+ * @internal
26
+ */
27
+ export declare function buildRoutingAbPrefix(salt: string, arm: string): string;
28
+ /**
29
+ * One line describing a usage block. Ported from `describeUsage`, now computing the ratio via
30
+ * {@link computeCacheUsageRatio} instead of an inline calculation that coalesced a missing
31
+ * `cachedInputTokens` or `uncachedInputTokens` to `0` (see that function's remarks).
32
+ * @internal
33
+ */
34
+ export declare function describeRoutingAbUsage(usage: IAiCompletionUsage | undefined): string;
35
+ /** One arm's cold-then-warm outcome. @internal */
36
+ export interface IPromptCacheRoutingAbArmResult {
37
+ readonly label: string;
38
+ readonly coldUsage: IAiCompletionUsage | undefined;
39
+ readonly warmUsage: IAiCompletionUsage | undefined;
40
+ }
41
+ /**
42
+ * Runs one arm: a cold call, then a warm call sharing the same {@link buildRoutingAbPrefix} system
43
+ * prompt and differing only in the final user turn — optionally supplying `cacheKey`. Ported from
44
+ * `runArm`, with the network call injected.
45
+ * @internal
46
+ */
47
+ export declare function runPromptCacheRoutingAbArm(deps: IPromptCacheRoutingAbDeps, params: {
48
+ readonly descriptor: IAiProviderDescriptor;
49
+ readonly apiKey: string;
50
+ readonly label: string;
51
+ readonly salt: string;
52
+ readonly arm: string;
53
+ readonly cacheKey?: string;
54
+ }): Promise<Result<IPromptCacheRoutingAbArmResult>>;
55
+ /** The full A/B: both arms' results and the routing-key verdict. @internal */
56
+ export interface IPromptCacheRoutingAbResult {
57
+ readonly withoutKey: IPromptCacheRoutingAbArmResult;
58
+ readonly withKey: IPromptCacheRoutingAbArmResult;
59
+ /** Baseline = `withoutKey`'s warm call; candidate = `withKey`'s warm call. */
60
+ readonly verdict: CacheHitComparisonVerdict;
61
+ readonly withoutKeyWarmRatio: number | undefined;
62
+ readonly withKeyWarmRatio: number | undefined;
63
+ }
64
+ /**
65
+ * Runs both arms of the routing-key A/B and classifies the result.
66
+ *
67
+ * @remarks
68
+ * The verdict compares each arm's **warm** call against the other's — not an arm's own cold vs.
69
+ * warm call — because the question is whether the routing key changes the warm hit rate relative
70
+ * to the no-key baseline, exactly as `promptCacheRoutingAb.js`'s original `main()` computed it
71
+ * (`pct(withoutKey)` / `pct(withKey)`, both taken from each arm's warm response).
72
+ * @internal
73
+ */
74
+ export declare function runPromptCacheRoutingAbHarness(deps: IPromptCacheRoutingAbDeps, params: {
75
+ readonly descriptor: IAiProviderDescriptor;
76
+ readonly apiKey: string;
77
+ readonly salt: string;
78
+ }): Promise<Result<IPromptCacheRoutingAbResult>>;
79
+ /**
80
+ * Formats the A/B's closing verdict line(s), matching the original harness's console output.
81
+ * @internal
82
+ */
83
+ export declare function formatRoutingAbVerdict(result: IPromptCacheRoutingAbResult): string;
84
+ //# sourceMappingURL=promptCacheRoutingAbHarness.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptCacheRoutingAbHarness.d.ts","sourceRoot":"","sources":["../../../src/packlets/ai-assist/promptCacheRoutingAbHarness.ts"],"names":[],"mappings":"AAoBA;;;;;;;;;;GAUG;AAEH,OAAO,EAAiB,KAAK,MAAM,EAAE,MAAM,eAAe,CAAC;AAG3D,OAAO,EAAE,KAAK,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,KAAK,qBAAqB,EAAE,KAAK,qBAAqB,EAAE,MAAM,SAAS,CAAC;AACjF,OAAO,EACL,KAAK,yBAAyB,EAI/B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEvD,6EAA6E;AAC7E,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,yBAAyB,KAAK,OAAO,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC;CACxG;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAYtE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,kBAAkB,GAAG,SAAS,GAAG,MAAM,CAYpF;AAED,kDAAkD;AAClD,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,kBAAkB,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,SAAS,EAAE,kBAAkB,GAAG,SAAS,CAAC;CACpD;AAED;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAC9C,IAAI,EAAE,yBAAyB,EAC/B,MAAM,EAAE;IACN,QAAQ,CAAC,UAAU,EAAE,qBAAqB,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,OAAO,CAAC,MAAM,CAAC,8BAA8B,CAAC,CAAC,CAyBjD;AAED,8EAA8E;AAC9E,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,UAAU,EAAE,8BAA8B,CAAC;IACpD,QAAQ,CAAC,OAAO,EAAE,8BAA8B,CAAC;IACjD,8EAA8E;IAC9E,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/C;AAED;;;;;;;;;GASG;AACH,wBAAsB,8BAA8B,CAClD,IAAI,EAAE,yBAAyB,EAC/B,MAAM,EAAE;IACN,QAAQ,CAAC,UAAU,EAAE,qBAAqB,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,GACA,OAAO,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC,CAkC9C;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,2BAA2B,GAAG,MAAM,CAwBlF"}