@fgv/ts-extras 5.1.0-54 → 5.1.0-56

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fgv/ts-extras",
-  "version": "5.1.0-54",
+  "version": "5.1.0-56",
   "description": "Assorted Typescript Utilities",
   "main": "lib/index.js",
   "types": "dist/ts-extras.d.ts",
@@ -69,10 +69,10 @@
   },
   "homepage": "https://github.com/ErikFortune/fgv/tree/main/libraries/ts-extras#ts-utils",
   "devDependencies": {
-    "@jest/expect-utils": "^29.7.0",
+    "@jest/expect-utils": "^30.5.2",
     "@microsoft/api-documenter": "^7.28.2",
     "@microsoft/api-extractor": "^7.55.2",
-    "@types/jest": "^29.5.14",
+    "@types/jest": "^30.0.0",
     "@types/mustache": "^4.2.5",
     "@types/node": "^20.14.9",
     "@types/papaparse": "^5.3.14",
@@ -82,28 +82,28 @@
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^7.2.1",
-    "jest": "^29.7.0",
+    "jest": "^30.5.2",
     "jest-extended": "^4.0.2",
-    "jest-matcher-utils": "^29.7.0",
+    "jest-matcher-utils": "^30.5.2",
     "rimraf": "^6.1.2",
-    "ts-jest": "^29.4.6",
+    "ts-jest": "^29.4.12",
     "ts-node": "^10.9.2",
     "typescript": "5.9.3",
     "eslint-plugin-n": "^17.23.1",
-    "jest-snapshot": "~29.7.0",
-    "@rushstack/heft": "1.2.7",
-    "@rushstack/heft-node-rig": "2.11.27",
+    "jest-snapshot": "~30.5.2",
+    "@rushstack/heft": "1.3.0",
+    "@rushstack/heft-node-rig": "2.11.50",
     "@rushstack/eslint-config": "4.6.4",
     "@types/heft-jest": "1.0.6",
-    "@rushstack/heft-jest-plugin": "1.2.6",
+    "@rushstack/heft-jest-plugin": "2.0.17",
     "eslint-plugin-tsdoc": "~0.5.2",
     "@types/js-yaml": "~4.0.9",
     "typedoc": "~0.28.16",
     "typedoc-plugin-markdown": "~4.9.0",
-    "@fgv/typedoc-compact-theme": "5.1.0-54",
-    "@fgv/heft-dual-rig": "5.1.0-54",
-    "@fgv/ts-utils-jest": "5.1.0-54",
-    "@fgv/ts-utils": "5.1.0-54"
+    "@fgv/heft-dual-rig": "5.1.0-56",
+    "@fgv/typedoc-compact-theme": "5.1.0-56",
+    "@fgv/ts-utils": "5.1.0-56",
+    "@fgv/ts-utils-jest": "5.1.0-56"
   },
   "dependencies": {
     "@types/luxon": "^3.7.1",
@@ -112,10 +112,10 @@
     "papaparse": "^5.4.1",
     "fflate": "~0.8.2",
     "js-yaml": "~4.1.1",
-    "@fgv/ts-json-base": "5.1.0-54"
+    "@fgv/ts-json-base": "5.1.0-56"
   },
   "peerDependencies": {
-    "@fgv/ts-utils": "5.1.0-54"
+    "@fgv/ts-utils": "5.1.0-56"
   },
   "repository": {
     "type": "git",

package/perf/promptCacheAnthropicBreakpoint.js

@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2026 Erik Fortune
+ * SPDX-License-Identifier: MIT
+ *
+ * A/B for whether C3's Anthropic prompt-cache breakpoint emission (IAiCacheRequest.systemBreakpoints
+ * -> `cache_control`) actually produces a cache hit against a live provider.
+ *
+ * Every prompt-cache measurement shipped by this stream so far (promptCacheObservability.js,
+ * promptCacheRoutingAb.js) ran against xAI, whose caching is automatic — no directive is sent, so
+ * neither harness exercises C3's emit path at all. Anthropic is the sharp case: its cache is
+ * EXPLICIT OPT-IN PER CONTENT BLOCK. There is no automatic path, so a breakpoint is the only thing
+ * that can produce a hit. If C3's Anthropic emission is wrong, nothing else in the repo would show
+ * it — this is the first live exercise of that code path.
+ *
+ *   ANTHROPIC_API_KEY=... node perf/promptCacheAnthropicBreakpoint.js
+ *   SALT=run2 ANTHROPIC_API_KEY=... node perf/promptCacheAnthropicBreakpoint.js   # see "traps" below
+ *
+ * Requires a built lib/ (`rushx build` first). Deliberately NOT a jest test: live traffic and a
+ * bill do not belong behind CI's green (TESTING_GUIDELINES.md § Measurement Harnesses).
+ *
+ * The prefix builder, the ratio arithmetic, and the CONFIRMS/MISSES/AMBIGUOUS/inconclusive verdict
+ * below all live in `src/packlets/ai-assist/promptCacheAnthropicBreakpointHarness.ts`, reusing
+ * `promptCacheUsageMath.ts`'s shared comparison rather than re-deriving it, and unit-tested against
+ * fixture usage blocks in `src/test/unit/ai-assist/promptCacheAnthropicBreakpointHarness.test.ts` —
+ * this file is a thin wrapper that supplies the live network transport and prints the result.
+ *
+ * ---------------------------------------------------------------------------
+ * TARGET MODEL AND FIXTURE SIZING
+ *
+ * Target: `claude-sonnet-5` — this provider's default `'base'` tier (registry.ts), the same tier
+ * this stream's other harnesses default to. design.md §7's verified-minimum table gives Sonnet-class
+ * models (including claude-sonnet-5) a 1,024-token minimum cacheable prefix. Anthropic's minimum is
+ * non-monotonic across the model family (Haiku-class models need 4,096), so this harness's fixture
+ * is sized well above BOTH floors rather than just the one this model needs: the 400-line generator
+ * this file shares with promptCacheObservability.js / promptCacheRoutingAb.js measured ~8,700 tokens
+ * for a comparable prefix earlier in this stream. A fixture below the minimum would produce a real
+ * miss that means nothing about C3 — this one is not close to that line.
+ *
+ * ---------------------------------------------------------------------------
+ * WHAT IT COMPARES
+ *
+ * Both arms send the same large system prefix twice (cold, then warm) and vary only the final user
+ * turn — what a real caller does. The arms differ in exactly one thing: whether `cache.systemBreakpoints`
+ * is supplied, marking almost the entire prefix as a single cacheable block.
+ *
+ *   - arm 1 (WITHOUT systemBreakpoints) — the negative control. Anthropic caching is opt-in per
+ *     block; with no `cache_control` sent, nothing should be written or read, on either call.
+ *   - arm 2 (WITH systemBreakpoints)    — the cold call should show a cache WRITE
+ *     (`cacheWriteTokens` > 0, `cachedInputTokens` ~ 0); the warm call should show a cache READ
+ *     (`cachedInputTokens` clearing roughly the size of the cached prefix).
+ *
+ * `usage.reports` should be `'reads-and-writes'` on every call here — unlike xAI, which this
+ * stream's other harnesses measured as `'reads'`-only on both of its routes. Anthropic's normalizer
+ * (`normalizeAnthropicUsage`) reports both fields unconditionally.
+ *
+ * ---------------------------------------------------------------------------
+ * THE PREDICTION, WRITTEN DOWN BEFORE THE FIRST RUN
+ * (TESTING_GUIDELINES.md § "State the prediction before running it")
+ *
+ *   - arm 1 cold and warm — `cachedInputTokens` and `cacheWriteTokens` both absent or 0 on both
+ *     calls. No directive was sent, so nothing should be written or read.
+ *   - arm 2 cold          — `cacheWriteTokens` > 0 (writing the ~8,700-token prefix into the
+ *     cache), `cachedInputTokens` near 0 (nothing to read yet on a first-ever write).
+ *   - arm 2 warm           — `cachedInputTokens` clears roughly the prefix size — high, not a small
+ *     fixed floor the way xAI's automatic cache showed a ~192-token floor even on a cold call in
+ *     this stream's earlier runs. Anthropic's cache is explicit, so there is no equivalent
+ *     scaffolding-cost floor predicted here.
+ *
+ * A miss means C3's Anthropic emission is wrong, or this design's model of Anthropic caching is
+ * wrong. The response is to revise the design or the code — NOT to lower a threshold or re-run
+ * until a favourable number appears.
+ *
+ * ---------------------------------------------------------------------------
+ * TRAPS THAT MAKE A SINGLE RUN MISLEADING (carried over from promptCacheRoutingAb.js, and still
+ * live even though Anthropic's cache mechanism differs from xAI's per-server routing)
+ *
+ * 1. Re-running within the cache TTL taints the cold reading, because the "cold" call would hit a
+ *    prefix a previous run already wrote. Pass a fresh SALT (or wait out the TTL, ~5 minutes on
+ *    Anthropic's default ephemeral cache) — each run's prefix is salted so re-runs cannot warm a
+ *    prior run's cache, and the two arms are independently salted so they cannot warm each other.
+ *
+ * 2. A failed call must not print a ratio. If either arm's cold or warm call fails outright, the
+ *    verdict reports 'inconclusive' rather than fabricating a 0% or 100% from missing data — see
+ *    `runPromptCacheAnthropicBreakpointHarness`'s propagated failures and
+ *    `compareCacheHitRatios`'s `undefined` handling in `promptCacheUsageMath.ts`.
+ * ---------------------------------------------------------------------------
+ */
+
+/* eslint-disable no-console */
+
+const { AiAssist } = require('../lib/index');
+const {
+  describeAnthropicBreakpointUsage,
+  formatAnthropicBreakpointVerdict,
+  runPromptCacheAnthropicBreakpointHarness
+} = require('../lib/packlets/ai-assist/promptCacheAnthropicBreakpointHarness');
+
+const API_KEY = process.env.ANTHROPIC_API_KEY;
+if (!API_KEY) {
+  console.error('ANTHROPIC_API_KEY is required. See the file header for usage.');
+  process.exit(1);
+}
+const SALT = process.env.SALT ?? 'run1';
+
+async function main() {
+  console.log(`=== Anthropic prompt-cache breakpoint A/B (salt=${SALT}) ===`);
+  console.log('Both arms vary only the final user turn. They differ only in cache.systemBreakpoints.');
+
+  const descriptor = AiAssist.getProviderDescriptor('anthropic').orThrow();
+
+  const result = await runPromptCacheAnthropicBreakpointHarness(
+    { callCompletion: AiAssist.callProviderCompletion },
+    { descriptor, apiKey: API_KEY, salt: SALT }
+  );
+  if (result.isFailure()) {
+    console.error(result.message);
+    process.exit(1);
+  }
+
+  const { value } = result;
+  console.log(`\n--- ${value.withoutBreakpoint.label} ---`);
+  console.log(describeAnthropicBreakpointUsage('  cold', value.withoutBreakpoint.coldUsage));
+  console.log(describeAnthropicBreakpointUsage('  warm', value.withoutBreakpoint.warmUsage));
+  console.log(`\n--- ${value.withBreakpoint.label} ---`);
+  console.log(describeAnthropicBreakpointUsage('  cold', value.withBreakpoint.coldUsage));
+  console.log(describeAnthropicBreakpointUsage('  warm', value.withBreakpoint.warmUsage));
+
+  console.log('\n--- verdict ---');
+  console.log(formatAnthropicBreakpointVerdict(value));
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

package/perf/promptCacheObservability.js

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2026 Erik Fortune
+ * SPDX-License-Identifier: MIT
+ *
+ * Standing assertion for C1 of `ai-assist-prompt-caching` (design.md §8).
+ *
+ * Deliberately NOT a jest test. It is a live-traffic measurement, not a behavior —
+ * putting it in the suite would put CI's runtime and its green/red behind a network
+ * call and a bill (see TESTING_GUIDELINES.md § "Measurement Harnesses"). It lives
+ * here so it can be run on demand and its output pasted into the stream's
+ * `result.md`.
+ *
+ *   XAI_API_KEY=... node perf/promptCacheObservability.js
+ *
+ * Requires a built `lib/` (`rushx build` first).
+ *
+ * The ratio arithmetic, the cold/warm comparison, and the prediction verdict below all live in
+ * `src/packlets/ai-assist/promptCacheObservabilityHarness.ts`, unit-tested against fixture usage
+ * blocks in `src/test/unit/ai-assist/promptCacheObservabilityHarness.test.ts` — this file is a
+ * thin wrapper that supplies the live network transport and prints the result.
+ *
+ * ---------------------------------------------------------------------------
+ * WHY xAI, NOT ANTHROPIC (corrected 2026-09-19 — the previous version of this note argued from a
+ * premise C3 has since retired; see below for what changed):
+ *
+ * xAI (and Gemini, and OpenAI's default mode) cache automatically with no request-side opt-in —
+ * confirmed live against xAI in this stream's design phase (design.md §12, OQ-1, the
+ * `xai-cache-probe` testbed scenario). A repeat request against xAI is therefore a valid test of
+ * whether C1's *reading* of the usage block is correct, on a provider whose cache needs no
+ * cooperation from the request this harness sends.
+ *
+ * Anthropic's prompt cache, by contrast, is opt-in per content block via `cache_control`, and
+ * this harness sends no cache directive of any kind — it calls `AiAssist.callProviderCompletion`
+ * with a plain `system` string and no `cache` field. That is no longer because "C1 sends no cache
+ * directive" (the reason this note originally gave): **C3 shipped and does emit `cache_control`
+ * when a caller supplies `IAiCacheRequest.systemBreakpoints`** — this harness simply doesn't pass
+ * one. So an Anthropic leg is possible now, in a way it structurally was not when this note was
+ * first written; it would need `IAiCacheRequest.systemBreakpoints` set on the request. It has
+ * never been run and is out of scope for this change — see design.md §8's addendum for what is
+ * and is not claimed here.
+ * ---------------------------------------------------------------------------
+ *
+ * THE PREDICTION, WRITTEN DOWN BEFORE THE FIRST RUN (per TESTING_GUIDELINES.md
+ * § "State the prediction before running it"), calibrated to the live numbers
+ * design.md §12 already recorded for this exact provider:
+ *
+ *   Two back-to-back requests share one large, byte-identical system-prompt
+ *   prefix and differ only in their final user turn.
+ *
+ *   - `usage.reports` should be `'reads'` on both calls — xAI never reports
+ *     cache writes on either of its routes (design.md §12, OQ-1).
+ *   - The FIRST (cold) call may still show a small nonzero
+ *     `usage.cachedInputTokens` — design.md §12 recorded a **128-token floor**
+ *     on a genuinely cold call against a prefix xAI had never seen, most
+ *     plausibly fixed chat-template scaffolding. A nonzero cold figure is
+ *     therefore NOT itself evidence of a bug; judge by the ratio, not the raw
+ *     count (design.md §9's explicit warning).
+ *   - The SECOND (warm) call should show `usage.cachedInputTokens` at
+ *     roughly **99%** of `usage.uncachedInputTokens + usage.cachedInputTokens`
+ *     for that call — design.md §12 measured 99.5% (Chat Completions) and
+ *     99.2% (Responses) on the same prefix-caching mechanism this harness
+ *     exercises.
+ *
+ *   A miss (cold and warm ratios roughly equal, or `reports` not `'reads'`)
+ *   means the C1 normalization is not doing what the design claims — the
+ *   response is to revise the design or the code, not to lower a threshold
+ *   until this harness goes green.
+ * ---------------------------------------------------------------------------
+ */
+
+/* eslint-disable no-console */
+
+const { AiAssist } = require('../lib/index');
+const {
+  describeObservabilityUsage,
+  runPromptCacheObservabilityHarness
+} = require('../lib/packlets/ai-assist/promptCacheObservabilityHarness');
+
+const API_KEY = process.env.XAI_API_KEY;
+if (!API_KEY) {
+  console.error('XAI_API_KEY is required. See the file header for usage.');
+  process.exit(1);
+}
+
+async function main() {
+  const descriptor = AiAssist.getProviderDescriptor('xai-grok').orThrow();
+
+  const result = await runPromptCacheObservabilityHarness(
+    { callCompletion: AiAssist.callProviderCompletion },
+    descriptor,
+    API_KEY
+  );
+  if (result.isFailure()) {
+    console.error(result.message);
+    process.exit(1);
+  }
+
+  const { value } = result;
+  console.log(describeObservabilityUsage('cold call ', value.cold));
+  console.log(describeObservabilityUsage('warm call ', value.warm));
+  console.log(`\nverdict: ${value.verdict.toUpperCase()} — ${value.detail}`);
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

package/perf/promptCacheRoutingAb.js

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2026 Erik Fortune
+ * SPDX-License-Identifier: MIT
+ *
+ * A/B for the xAI prompt-cache ROUTING key (IAiCacheRequest.cacheKey).
+ *
+ * Sibling to promptCacheObservability.js, which measures whether C1 *reads* the usage block
+ * correctly. This one measures whether the routing key actually *buys* anything — the claim
+ * this branch ships and the one thing its request-shape tests cannot check.
+ *
+ *   XAI_API_KEY=... node perf/promptCacheRoutingAb.js
+ *   SALT=run2 XAI_API_KEY=... node perf/promptCacheRoutingAb.js    # see "second trap" below
+ *
+ * Requires a built lib/ (`rushx build` first). Deliberately NOT a jest test: live traffic and a
+ * bill do not belong behind CI's green (TESTING_GUIDELINES.md § Measurement Harnesses).
+ *
+ * The prefix builder, the ratio arithmetic, and the CONFIRMS/MISSES/AMBIGUOUS verdict below all
+ * live in `src/packlets/ai-assist/promptCacheRoutingAbHarness.ts`, unit-tested against fixture
+ * usage blocks in `src/test/unit/ai-assist/promptCacheRoutingAbHarness.test.ts` — this file is a
+ * thin wrapper that supplies the live network transport and prints the result.
+ *
+ * ---------------------------------------------------------------------------
+ * WHAT IT COMPARES
+ *
+ * Both arms send the same large system prefix twice and vary only the final user turn — which
+ * is what a real caller does, and what the plain observability harness showed returns ~2%
+ * cached. The arms differ in exactly one thing: whether a stable `cache.cacheKey` is supplied.
+ *
+ * xAI matches byte-for-byte from the start of the messages array, but its cache is PER-SERVER
+ * and evictable, so two requests sharing a prefix can still miss when routed to different
+ * boxes. The key (x-grok-conv-id on Chat Completions) pins them together.
+ *
+ * ---------------------------------------------------------------------------
+ * THE PREDICTION, WRITTEN DOWN BEFORE THE FIRST RUN
+ * (TESTING_GUIDELINES.md § "State the prediction before running it")
+ *
+ *   - arm 1 (no key)  — warm stays near the ~2% floor, matching the observability harness's
+ *     measured 192-of-8876 on this exact shape.
+ *   - arm 2 (key)     — warm jumps toward ~99%, matching what the OQ-1 probe gets from
+ *     byte-identical requests.
+ *
+ * A miss means the routing key does not do what this branch claims. The response is to revise
+ * the change or the claim — NOT to re-run until a favourable number appears.
+ *
+ * ---------------------------------------------------------------------------
+ * TWO TRAPS THAT MAKE A SINGLE RUN MISLEADING
+ *
+ * 1. A hit in arm 1 is NOT a refutation. Without a routing key you can still land on the same
+ *    server by luck — that non-determinism is the whole reason the key exists. The meaningful
+ *    negative result is arm 2 FAILING to jump, not arm 1 happening to succeed.
+ *
+ * 2. Re-running within the cache TTL taints the cold reading, because the "cold" call hits a
+ *    prefix the previous run already warmed. Pass a fresh SALT (or wait out the TTL) — each arm
+ *    already gets its own salted prefix so the arms cannot warm each other, which is the
+ *    mistake the OQ-1 probe had to fix in its own first version.
+ * ---------------------------------------------------------------------------
+ */
+
+/* eslint-disable no-console */
+
+const { AiAssist } = require('../lib/index');
+const {
+  describeRoutingAbUsage,
+  formatRoutingAbVerdict,
+  runPromptCacheRoutingAbHarness
+} = require('../lib/packlets/ai-assist/promptCacheRoutingAbHarness');
+
+const API_KEY = process.env.XAI_API_KEY;
+if (!API_KEY) {
+  console.error('XAI_API_KEY is required. See the file header for usage.');
+  process.exit(1);
+}
+const SALT = process.env.SALT ?? 'run1';
+
+async function main() {
+  console.log(`=== xAI prompt-cache routing A/B (salt=${SALT}) ===`);
+  console.log('Both arms vary only the final user turn. They differ only in the routing key.');
+
+  const descriptor = AiAssist.getProviderDescriptor('xai-grok').orThrow();
+
+  const result = await runPromptCacheRoutingAbHarness(
+    { callCompletion: AiAssist.callProviderCompletion },
+    { descriptor, apiKey: API_KEY, salt: SALT }
+  );
+  if (result.isFailure()) {
+    console.error(result.message);
+    process.exit(1);
+  }
+
+  const { value } = result;
+  console.log(`\n--- ${value.withoutKey.label} ---`);
+  console.log(`  cold: ${describeRoutingAbUsage(value.withoutKey.coldUsage)}`);
+  console.log(`  warm: ${describeRoutingAbUsage(value.withoutKey.warmUsage)}`);
+  console.log(`\n--- ${value.withKey.label} ---`);
+  console.log(`  cold: ${describeRoutingAbUsage(value.withKey.coldUsage)}`);
+  console.log(`  warm: ${describeRoutingAbUsage(value.withKey.warmUsage)}`);
+
+  console.log('\n--- verdict ---');
+  console.log(formatRoutingAbVerdict(value));
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});