@opensearch-project/agent-health 0.5.2 → 0.6.0

package/examples/eval-files/sdk-describe-demo.eval.js

@@ -0,0 +1,50 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * SDK demo with describe() grouping. Each describe() block declares a
+ * benchmark; CLI auto-creates one benchmark per group and runs them.
+ */
+
+const { describe, test, expect } = require('@opensearch-project/agent-health');
+
+describe('RCA Suite', () => {
+  test('payment service is the root cause', {
+    prompt: 'Diagnose why the payment service is failing.',
+    context: [{ description: 'Error log', value: 'ERROR [payment-service] Connection refused to db-primary:5432' }],
+    labels: ['category:RCA', 'difficulty:Medium'],
+  }, async function ({ agent, judge }) {
+    const result = await agent.run();
+    expect(result.trajectory).to.have.length.greaterThan(0);
+    expect(result).to.haveCompletedWithin(60_000);
+    await judge(result, 'Mentions the payment service or its database connection failure');
+  });
+
+  test('budget cap', {
+    prompt: 'Summarize the system in one paragraph.',
+    labels: ['category:Budget', 'difficulty:Easy'],
+  }, async function ({ agent, traces }) {
+    const result = await agent.run();
+    expect(result.agentOutput.length).to.be.greaterThan(0);
+    expect(traces.totalTokens).to.be.lessThan(50_000);
+  });
+});
+
+describe('Smoke', () => {
+  test('says hello', {
+    prompt: 'Say hello in one short sentence.',
+    labels: ['category:Smoke', 'difficulty:Easy'],
+  }, async function ({ agent }) {
+    const result = await agent.run();
+    expect(result.trajectory).to.have.length.greaterThan(0);
+    expect(result.agentOutput.trim()).to.have.length.greaterThan(0);
+    expect(result).to.haveCompletedWithin(30_000);
+  });
+});
+
+// Tests outside any describe() — go to the file-default benchmark
+test('orphan-deterministic', function () {
+  expect(2 + 2).to.equal(4);
+});

package/examples/eval-files/sdk-hooks-demo.eval.js

@@ -14,7 +14,7 @@
  *
  * Run with:
  *   AH_PORT=4002 npx @opensearch-project/agent-health benchmark \
- *     -f evals/sdk-hooks-demo.eval.js -a demo
+ *     -f examples/eval-files/sdk-hooks-demo.eval.js -a demo
  */
 
 const fs = require('fs');

package/lib/dist/lib/bedrockCompat.d.ts

@@ -0,0 +1,27 @@
+/** Whether the given Bedrock model still accepts an explicit `temperature`. */
+export declare function modelSupportsTemperature(modelId: string): boolean;
+/**
+ * Build a Converse `inferenceConfig`, including `temperature` only when the
+ * target model accepts it. Callers pass their preferred temperature; for
+ * models that have deprecated the parameter it is silently omitted (the
+ * model's own default applies).
+ */
+export declare function buildInferenceConfig(modelId: string, opts: {
+    maxTokens: number;
+    temperature?: number;
+}): {
+    maxTokens: number;
+    temperature?: number;
+};
+/** Inference-profile prefix appropriate for the given (or current) AWS region. */
+export declare function regionInferencePrefix(region?: string): string;
+/**
+ * Re-home a cross-region inference-profile id to the active AWS region.
+ *
+ * `us.anthropic.claude-...` → `eu.anthropic.claude-...` when running in an
+ * EU region, etc. Ids that carry no regional prefix (bare model ids,
+ * `global.` profiles, non-Bedrock ids like `mock://` or `gpt-4o`) are
+ * returned unchanged.
+ */
+export declare function resolveRegionAwareModelId(modelId: string, region?: string): string;
+//# sourceMappingURL=bedrockCompat.d.ts.map

package/lib/dist/lib/bedrockCompat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bedrockCompat.d.ts","sourceRoot":"","sources":["../../bedrockCompat.ts"],"names":[],"mappings":"AAmCA,+EAA+E;AAC/E,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAGjE;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAM7C;AAED,kFAAkF;AAClF,wBAAgB,qBAAqB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAU7D;AAID;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CASlF"}

package/lib/dist/lib/bedrockCompat.js

@@ -0,0 +1,83 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Bedrock model compatibility helpers shared by every Bedrock Converse call
+ * site (judge, skill eval generator, skill improver, failure clustering,
+ * assistant fallback).
+ *
+ * Two concerns live here so the policy is defined exactly once:
+ *
+ * 1. **Temperature deprecation** — newer Anthropic models (Claude Opus 4.5+,
+ *    Sonnet 4.6+, and the Claude 5 family incl. Fable/Mythos) reject an
+ *    explicit `temperature` with `ValidationException: 'temperature' is
+ *    deprecated for this model`. `buildInferenceConfig()` omits the field for
+ *    those models and lets the model use its default. (GitHub issue #299)
+ *
+ * 2. **Region-aware inference profiles** — the default model registry pins
+ *    `us.`-prefixed cross-region inference profile ids, which only resolve
+ *    from US regions. `resolveRegionAwareModelId()` swaps the prefix to the
+ *    one matching the active AWS region (`eu.` / `apac.`) at call time, so
+ *    the registry itself stays region-neutral. (GitHub issue #298)
+ */
+/**
+ * Model-id patterns that have deprecated the `temperature` parameter.
+ * Matched case-insensitively against the full model/inference-profile id.
+ */
+const TEMPERATURE_DEPRECATED_PATTERNS = [
+    /claude-opus-4-[5-9]/, // Opus 4.5, 4.6, 4.7, 4.8, ...
+    /claude-sonnet-4-[6-9]/, // Sonnet 4.6+ (4.5 still accepts temperature)
+    /claude-[a-z]+-5(?![0-9])/, // Claude 5 family: fable-5, mythos-5, opus-5, sonnet-5, ...
+];
+/** Whether the given Bedrock model still accepts an explicit `temperature`. */
+export function modelSupportsTemperature(modelId) {
+    const id = (modelId || '').toLowerCase();
+    return !TEMPERATURE_DEPRECATED_PATTERNS.some((p) => p.test(id));
+}
+/**
+ * Build a Converse `inferenceConfig`, including `temperature` only when the
+ * target model accepts it. Callers pass their preferred temperature; for
+ * models that have deprecated the parameter it is silently omitted (the
+ * model's own default applies).
+ */
+export function buildInferenceConfig(modelId, opts) {
+    const cfg = { maxTokens: opts.maxTokens };
+    if (opts.temperature !== undefined && modelSupportsTemperature(modelId)) {
+        cfg.temperature = opts.temperature;
+    }
+    return cfg;
+}
+/** Inference-profile prefix appropriate for the given (or current) AWS region. */
+export function regionInferencePrefix(region) {
+    const r = (region ||
+        process.env.AWS_REGION ||
+        process.env.AWS_DEFAULT_REGION ||
+        'us-east-1').toLowerCase();
+    if (r.startsWith('eu-'))
+        return 'eu.';
+    if (r.startsWith('ap-'))
+        return 'apac.';
+    return 'us.';
+}
+const CROSS_REGION_PREFIXES = ['us.', 'eu.', 'apac.'];
+/**
+ * Re-home a cross-region inference-profile id to the active AWS region.
+ *
+ * `us.anthropic.claude-...` → `eu.anthropic.claude-...` when running in an
+ * EU region, etc. Ids that carry no regional prefix (bare model ids,
+ * `global.` profiles, non-Bedrock ids like `mock://` or `gpt-4o`) are
+ * returned unchanged.
+ */
+export function resolveRegionAwareModelId(modelId, region) {
+    if (!modelId)
+        return modelId;
+    const target = regionInferencePrefix(region);
+    for (const prefix of CROSS_REGION_PREFIXES) {
+        if (modelId.startsWith(prefix)) {
+            return prefix === target ? modelId : target + modelId.slice(prefix.length);
+        }
+    }
+    return modelId;
+}
+//# sourceMappingURL=bedrockCompat.js.map

package/lib/dist/lib/bedrockCompat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bedrockCompat.js","sourceRoot":"","sources":["../../bedrockCompat.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;GAGG;AACH,MAAM,+BAA+B,GAAa;IAChD,qBAAqB,EAAE,+BAA+B;IACtD,uBAAuB,EAAE,8CAA8C;IACvE,0BAA0B,EAAE,4DAA4D;CACzF,CAAC;AAEF,+EAA+E;AAC/E,MAAM,UAAU,wBAAwB,CAAC,OAAe;IACtD,MAAM,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACzC,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAe,EACf,IAAiD;IAEjD,MAAM,GAAG,GAAgD,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;IACvF,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,wBAAwB,CAAC,OAAO,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACrC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,qBAAqB,CAAC,MAAe;IACnD,MAAM,CAAC,GAAG,CACR,MAAM;QACN,OAAO,CAAC,GAAG,CAAC,UAAU;QACtB,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAC9B,WAAW,CACZ,CAAC,WAAW,EAAE,CAAC;IAChB,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACxC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAU,CAAC;AAE/D;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAAe,EAAE,MAAe;IACxE,IAAI,CAAC,OAAO;QAAE,OAAO,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC7C,KAAK,MAAM,MAAM,IAAI,qBAAqB,EAAE,CAAC;QAC3C,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,OAAO,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/lib/dist/lib/benchmarkImage.d.ts

@@ -0,0 +1,52 @@
+import type { BenchmarkImage, TestCase } from '../types/index.js';
+/** Eval conditions frozen into an image digest. */
+export interface ImageEvalConditions {
+    evaluatorId?: string;
+    judgeModelId?: string;
+}
+/** Bump when the digest input shape changes (invalidates old digests). */
+export declare const IMAGE_DIGEST_VERSION = 1;
+/**
+ * Deterministic JSON: objects serialized with sorted keys, recursively.
+ * Arrays keep their order (order-sensitivity is decided by the caller).
+ */
+export declare function canonicalStringify(value: unknown): string;
+/**
+ * Content fingerprint of a single test case. Uses CONTENT only — never the
+ * storage id — so re-imported test cases (fresh ids, same content) fingerprint
+ * identically. Fields chosen to match what actually changes agent behavior
+ * and judging: prompt, context, expected outcomes/trajectory, tools.
+ */
+export declare function fingerprintTestCase(tc: Partial<TestCase> & {
+    name: string;
+}): {
+    name: string;
+    contentHash: string;
+};
+/**
+ * Compute the image digest for a set of test cases + eval conditions.
+ * Order-insensitive over test cases (fingerprints are sorted), content- and
+ * condition-sensitive.
+ */
+export declare function computeImageDigest(input: {
+    testCases: Array<Partial<TestCase> & {
+        name: string;
+    }>;
+    evalConditions?: ImageEvalConditions;
+}): string;
+/** Storage id for an image doc (content-addressed — one doc per digest). */
+export declare function imageIdForDigest(digest: string): string;
+/**
+ * Build a full image doc for storage. `create` on the storage side is
+ * find-or-create keyed on the digest, so calling this repeatedly for the
+ * same inputs is harmless.
+ */
+export declare function buildImageDoc(input: {
+    testCases: Array<Partial<TestCase> & {
+        id?: string;
+        name: string;
+    }>;
+    evalConditions?: ImageEvalConditions;
+    tags?: string[];
+}): BenchmarkImage;
+//# sourceMappingURL=benchmarkImage.d.ts.map

package/lib/dist/lib/benchmarkImage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarkImage.d.ts","sourceRoot":"","sources":["../../benchmarkImage.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExD,mDAAmD;AACnD,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,0EAA0E;AAC1E,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEzD;AAmBD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GACvC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAYvC;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACxC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,mBAAmB,CAAC;CACtC,GAAG,MAAM,CAaT;AAED,4EAA4E;AAC5E,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE;IACnC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpE,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB,GAAG,cAAc,CAkBjB"}

package/lib/dist/lib/benchmarkImage.js

@@ -0,0 +1,113 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Benchmark images — content-addressed identity for evaluation conditions.
+ *
+ * An image freezes the "controls" of an evaluation run: the test-case
+ * contents and the eval conditions (evaluator, judge model). The digest is a
+ * SHA-256 over a canonical serialization, so:
+ *
+ *   - two runs with the same digest are comparable **by construction**
+ *     (same tests, same judge conditions — only the agent varies);
+ *   - re-running the same command yields the same digest → the image is
+ *     found, not re-created (inherent dedup: identity is content, not name).
+ *
+ * Names are docker-style *tags*: mutable labels pointing at a digest, never
+ * identity. See the benchmark-dedup design discussion for rationale.
+ */
+import { createHash } from 'crypto';
+/** Bump when the digest input shape changes (invalidates old digests). */
+export const IMAGE_DIGEST_VERSION = 1;
+/**
+ * Deterministic JSON: objects serialized with sorted keys, recursively.
+ * Arrays keep their order (order-sensitivity is decided by the caller).
+ */
+export function canonicalStringify(value) {
+    return JSON.stringify(sortValue(value));
+}
+function sortValue(value) {
+    if (Array.isArray(value))
+        return value.map(sortValue);
+    if (value && typeof value === 'object') {
+        const out = {};
+        for (const key of Object.keys(value).sort()) {
+            const v = value[key];
+            if (v !== undefined)
+                out[key] = sortValue(v);
+        }
+        return out;
+    }
+    return value;
+}
+function sha256Hex(input) {
+    return createHash('sha256').update(input, 'utf8').digest('hex');
+}
+/**
+ * Content fingerprint of a single test case. Uses CONTENT only — never the
+ * storage id — so re-imported test cases (fresh ids, same content) fingerprint
+ * identically. Fields chosen to match what actually changes agent behavior
+ * and judging: prompt, context, expected outcomes/trajectory, tools.
+ */
+export function fingerprintTestCase(tc) {
+    const content = {
+        name: tc.name,
+        initialPrompt: tc.initialPrompt ?? null,
+        context: tc.context ?? null,
+        tools: tc.tools ?? null,
+        expectedOutcomes: tc.expectedOutcomes ?? null,
+        expectedTrajectory: tc.expectedTrajectory ?? null,
+        // Code-imported cases: sourceHash already fingerprints the executable body.
+        sourceHash: tc.sourceHash ?? null,
+    };
+    return { name: tc.name, contentHash: sha256Hex(canonicalStringify(content)) };
+}
+/**
+ * Compute the image digest for a set of test cases + eval conditions.
+ * Order-insensitive over test cases (fingerprints are sorted), content- and
+ * condition-sensitive.
+ */
+export function computeImageDigest(input) {
+    const fingerprints = input.testCases
+        .map(fingerprintTestCase)
+        .sort((a, b) => a.contentHash.localeCompare(b.contentHash));
+    const payload = {
+        v: IMAGE_DIGEST_VERSION,
+        testCases: fingerprints.map((f) => f.contentHash),
+        evalConditions: {
+            evaluatorId: input.evalConditions?.evaluatorId ?? null,
+            judgeModelId: input.evalConditions?.judgeModelId ?? null,
+        },
+    };
+    return sha256Hex(canonicalStringify(payload));
+}
+/** Storage id for an image doc (content-addressed — one doc per digest). */
+export function imageIdForDigest(digest) {
+    return `img-${digest}`;
+}
+/**
+ * Build a full image doc for storage. `create` on the storage side is
+ * find-or-create keyed on the digest, so calling this repeatedly for the
+ * same inputs is harmless.
+ */
+export function buildImageDoc(input) {
+    const digest = computeImageDigest(input);
+    const fingerprints = input.testCases
+        .map((tc) => ({ ...fingerprintTestCase(tc), id: tc.id }))
+        .sort((a, b) => a.contentHash.localeCompare(b.contentHash));
+    return {
+        id: imageIdForDigest(digest),
+        docType: 'benchmark-image',
+        digest,
+        tags: input.tags ?? [],
+        testCaseFingerprints: fingerprints,
+        testCaseCount: fingerprints.length,
+        evalConditions: {
+            ...(input.evalConditions?.evaluatorId ? { evaluatorId: input.evalConditions.evaluatorId } : {}),
+            ...(input.evalConditions?.judgeModelId ? { judgeModelId: input.evalConditions.judgeModelId } : {}),
+        },
+        createdAt: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=benchmarkImage.js.map

package/lib/dist/lib/benchmarkImage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarkImage.js","sourceRoot":"","sources":["../../benchmarkImage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AASpC,0EAA0E;AAC1E,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAEtC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAgC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACvE,MAAM,CAAC,GAAI,KAAiC,CAAC,GAAG,CAAC,CAAC;YAClD,IAAI,CAAC,KAAK,SAAS;gBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,EAAwC;IAExC,MAAM,OAAO,GAAG;QACd,IAAI,EAAE,EAAE,CAAC,IAAI;QACb,aAAa,EAAE,EAAE,CAAC,aAAa,IAAI,IAAI;QACvC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,IAAI;QAC3B,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,IAAI;QACvB,gBAAgB,EAAE,EAAE,CAAC,gBAAgB,IAAI,IAAI;QAC7C,kBAAkB,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI;QACjD,4EAA4E;QAC5E,UAAU,EAAE,EAAE,CAAC,UAAU,IAAI,IAAI;KAClC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;AAChF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAGlC;IACC,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS;SACjC,GAAG,CAAC,mBAAmB,CAAC;SACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG;QACd,CAAC,EAAE,oBAAoB;QACvB,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;QACjD,cAAc,EAAE;YACd,WAAW,EAAE,KAAK,CAAC,cAAc,EAAE,WAAW,IAAI,IAAI;YACtD,YAAY,EAAE,KAAK,CAAC,cAAc,EAAE,YAAY,IAAI,IAAI;SACzD;KACF,CAAC;IACF,OAAO,SAAS,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,OAAO,OAAO,MAAM,EAAE,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,KAI7B;IACC,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS;SACjC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SACxD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC9D,OAAO;QACL,EAAE,EAAE,gBAAgB,CAAC,MAAM,CAAC;QAC5B,OAAO,EAAE,iBAAiB;QAC1B,MAAM;QACN,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE;QACtB,oBAAoB,EAAE,YAAY;QAClC,aAAa,EAAE,YAAY,CAAC,MAAM;QAClC,cAAc,EAAE;YACd,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnG;QACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;AACJ,CAAC"}

package/lib/dist/lib/benchmarkVersionUtils.d.ts

@@ -34,4 +34,17 @@ export declare function getVersionTestCases(testCases: TestCase[], selectedVersi
  * @param versionFilter - Version number to filter by, or 'all' for all runs
  */
 export declare function filterRunsByVersion(runs: BenchmarkRun[] | undefined, versionFilter: number | 'all'): BenchmarkRun[];
+/**
+ * Resolve the effective run-version filter from a persisted (possibly stale)
+ * value. A persisted version the benchmark doesn't actually have — e.g. a
+ * filter set while viewing a different benchmark under the old global
+ * localStorage key, a deleted version, or corrupted storage — must behave as
+ * 'all' rather than filtering every run out and rendering a bogus
+ * "No runs for vN" empty state that looks like data loss.
+ *
+ * @param raw - persisted filter value
+ * @param availableVersions - the benchmark's actual version numbers
+ *   (undefined while the benchmark is still loading — value passes through)
+ */
+export declare function effectiveRunVersionFilter(raw: number | 'all', availableVersions: number[] | undefined): number | 'all';
 //# sourceMappingURL=benchmarkVersionUtils.d.ts.map

package/lib/dist/lib/benchmarkVersionUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"benchmarkVersionUtils.d.ts","sourceRoot":"","sources":["../../benchmarkVersionUtils.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAoB,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI,GAAG,WAAW,EAAE,CA0B7E;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,WAAW,EAAE,EAC1B,eAAe,EAAE,MAAM,GAAG,IAAI,GAC7B,WAAW,GAAG,IAAI,CAIpB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,QAAQ,EAAE,EACrB,mBAAmB,EAAE,WAAW,GAAG,IAAI,GACtC,QAAQ,EAAE,CAKZ;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,YAAY,EAAE,GAAG,SAAS,EAChC,aAAa,EAAE,MAAM,GAAG,KAAK,GAC5B,YAAY,EAAE,CAShB"}
+{"version":3,"file":"benchmarkVersionUtils.d.ts","sourceRoot":"","sources":["../../benchmarkVersionUtils.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAoB,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI,GAAG,WAAW,EAAE,CA0B7E;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,WAAW,EAAE,EAC1B,eAAe,EAAE,MAAM,GAAG,IAAI,GAC7B,WAAW,GAAG,IAAI,CAIpB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,QAAQ,EAAE,EACrB,mBAAmB,EAAE,WAAW,GAAG,IAAI,GACtC,QAAQ,EAAE,CAKZ;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,YAAY,EAAE,GAAG,SAAS,EAChC,aAAa,EAAE,MAAM,GAAG,KAAK,GAC5B,YAAY,EAAE,CAShB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CACvC,GAAG,EAAE,MAAM,GAAG,KAAK,EACnB,iBAAiB,EAAE,MAAM,EAAE,GAAG,SAAS,GACtC,MAAM,GAAG,KAAK,CAKhB"}

package/lib/dist/lib/benchmarkVersionUtils.js

@@ -65,4 +65,25 @@ export function filterRunsByVersion(runs, versionFilter) {
         return sorted;
     return sorted.filter(run => (run.benchmarkVersion || 1) === versionFilter);
 }
+/**
+ * Resolve the effective run-version filter from a persisted (possibly stale)
+ * value. A persisted version the benchmark doesn't actually have — e.g. a
+ * filter set while viewing a different benchmark under the old global
+ * localStorage key, a deleted version, or corrupted storage — must behave as
+ * 'all' rather than filtering every run out and rendering a bogus
+ * "No runs for vN" empty state that looks like data loss.
+ *
+ * @param raw - persisted filter value
+ * @param availableVersions - the benchmark's actual version numbers
+ *   (undefined while the benchmark is still loading — value passes through)
+ */
+export function effectiveRunVersionFilter(raw, availableVersions) {
+    if (raw === 'all')
+        return 'all';
+    if (!Number.isInteger(raw) || raw < 1)
+        return 'all';
+    if (availableVersions !== undefined && !availableVersions.includes(raw))
+        return 'all';
+    return raw;
+}
 //# sourceMappingURL=benchmarkVersionUtils.js.map

package/lib/dist/lib/benchmarkVersionUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"benchmarkVersionUtils.js","sourceRoot":"","sources":["../../benchmarkVersionUtils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiBH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAA2B;IAC5D,IAAI,CAAC,SAAS,EAAE,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvE,OAAO,SAAS,CAAC,QAAQ;SACtB,KAAK,EAAE,CAAC,2CAA2C;SACnD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,eAAe;SACrD,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,2BAA2B;QAC/D,MAAM,KAAK,GAAG,WAAW;YACvB,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,WAAW;YACzB,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAC1C,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CACxC,CAAC,MAAM,IAAI,CAAC,CAAC;QAEd,OAAO;YACL,GAAG,CAAC;YACJ,QAAQ,EAAE,KAAK,KAAK,CAAC;YACrB,KAAK;YACL,OAAO;YACP,QAAQ;SACT,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,WAA0B,EAC1B,eAA8B;IAE9B,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,eAAe,KAAK,IAAI;QAAE,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;IAC9D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;AAChF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,SAAqB,EACrB,mBAAuC;IAEvC,IAAI,CAAC,mBAAmB;QAAE,OAAO,EAAE,CAAC;IACpC,OAAO,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAC3B,mBAAmB,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAChD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAgC,EAChC,aAA6B;IAE7B,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAC5E,CAAC;IACF,IAAI,aAAa,KAAK,KAAK;QAAE,OAAO,MAAM,CAAC;IAC3C,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACzB,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,CAAC,KAAK,aAAa,CAC9C,CAAC;AACJ,CAAC"}
+{"version":3,"file":"benchmarkVersionUtils.js","sourceRoot":"","sources":["../../benchmarkVersionUtils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiBH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAA2B;IAC5D,IAAI,CAAC,SAAS,EAAE,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvE,OAAO,SAAS,CAAC,QAAQ;SACtB,KAAK,EAAE,CAAC,2CAA2C;SACnD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,eAAe;SACrD,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,2BAA2B;QAC/D,MAAM,KAAK,GAAG,WAAW;YACvB,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,WAAW;YACzB,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAC1C,CAAC,CAAC,CAAC,gBAAgB,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CACxC,CAAC,MAAM,IAAI,CAAC,CAAC;QAEd,OAAO;YACL,GAAG,CAAC;YACJ,QAAQ,EAAE,KAAK,KAAK,CAAC;YACrB,KAAK;YACL,OAAO;YACP,QAAQ;SACT,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,WAA0B,EAC1B,eAA8B;IAE9B,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,eAAe,KAAK,IAAI;QAAE,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;IAC9D,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;AAChF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,SAAqB,EACrB,mBAAuC;IAEvC,IAAI,CAAC,mBAAmB;QAAE,OAAO,EAAE,CAAC;IACpC,OAAO,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAC3B,mBAAmB,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAChD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAgC,EAChC,aAA6B;IAE7B,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAC5E,CAAC;IACF,IAAI,aAAa,KAAK,KAAK;QAAE,OAAO,MAAM,CAAC;IAC3C,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACzB,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,CAAC,KAAK,aAAa,CAC9C,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,yBAAyB,CACvC,GAAmB,EACnB,iBAAuC;IAEvC,IAAI,GAAG,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACpD,IAAI,iBAAiB,KAAK,SAAS,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,OAAO,GAAG,CAAC;AACb,CAAC"}

package/lib/dist/lib/chunkedFetch.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Shared "fetch a big id list without blowing up the request" policy.
+ *
+ * A comparison over a large multi-run pool can need results for hundreds of
+ * ids (report ids, test-case ids, …) in one go. Two failure modes to avoid:
+ *   - one unbounded `?ids=<all>` GET can blow past practical URL/header size
+ *     limits (HTTP 431 — see asyncRunStorage.getReportsByIds's history);
+ *   - firing one request per chunk with no cap can stampede the backend.
+ *
+ * `fetchChunked` splits `ids` into bounded chunks and fans them out with a
+ * capped worker pool — never one unbounded request, never an unbounded burst
+ * of parallel ones either. Used by asyncRunStorage (report ids) and
+ * asyncTestCaseStorage (test-case ids).
+ */
+export declare const DEFAULT_CHUNK_SIZE = 100;
+export declare const DEFAULT_MAX_CONCURRENT_CHUNKS = 8;
+export declare function fetchChunked<T>(ids: string[], chunkSize: number, fetchChunk: (chunk: string[]) => Promise<T[]>, maxConcurrentChunks?: number): Promise<T[]>;
+//# sourceMappingURL=chunkedFetch.d.ts.map

package/lib/dist/lib/chunkedFetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkedFetch.d.ts","sourceRoot":"","sources":["../../chunkedFetch.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,kBAAkB,MAAM,CAAC;AACtC,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,wBAAsB,YAAY,CAAC,CAAC,EAClC,GAAG,EAAE,MAAM,EAAE,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,EAC7C,mBAAmB,GAAE,MAAsC,GAC1D,OAAO,CAAC,CAAC,EAAE,CAAC,CAiBd"}

package/lib/dist/lib/chunkedFetch.js

@@ -0,0 +1,40 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Shared "fetch a big id list without blowing up the request" policy.
+ *
+ * A comparison over a large multi-run pool can need results for hundreds of
+ * ids (report ids, test-case ids, …) in one go. Two failure modes to avoid:
+ *   - one unbounded `?ids=<all>` GET can blow past practical URL/header size
+ *     limits (HTTP 431 — see asyncRunStorage.getReportsByIds's history);
+ *   - firing one request per chunk with no cap can stampede the backend.
+ *
+ * `fetchChunked` splits `ids` into bounded chunks and fans them out with a
+ * capped worker pool — never one unbounded request, never an unbounded burst
+ * of parallel ones either. Used by asyncRunStorage (report ids) and
+ * asyncTestCaseStorage (test-case ids).
+ */
+export const DEFAULT_CHUNK_SIZE = 100;
+export const DEFAULT_MAX_CONCURRENT_CHUNKS = 8;
+export async function fetchChunked(ids, chunkSize, fetchChunk, maxConcurrentChunks = DEFAULT_MAX_CONCURRENT_CHUNKS) {
+    const chunks = [];
+    for (let i = 0; i < ids.length; i += chunkSize) {
+        chunks.push(ids.slice(i, i + chunkSize));
+    }
+    const results = new Array(chunks.length);
+    let nextChunk = 0;
+    async function worker() {
+        while (true) {
+            const i = nextChunk++;
+            if (i >= chunks.length)
+                return;
+            results[i] = await fetchChunk(chunks[i]);
+        }
+    }
+    const workerCount = Math.min(maxConcurrentChunks, chunks.length);
+    await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    return results.flat();
+}
+//# sourceMappingURL=chunkedFetch.js.map

package/lib/dist/lib/chunkedFetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkedFetch.js","sourceRoot":"","sources":["../../chunkedFetch.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACtC,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAa,EACb,SAAiB,EACjB,UAA6C,EAC7C,sBAA8B,6BAA6B;IAE3D,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,OAAO,GAAU,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,UAAU,MAAM;QACnB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,OAAO;YAC/B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvE,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC"}

package/lib/dist/lib/comparisonInsights.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Comparison insights — deterministic agreement + category math for the
+ * comparison page's insights band (and, later, as grounding context for the
+ * "What's actually different" LLM deep-dive).
+ *
+ * Everything here is pure arithmetic over the already-loaded comparison rows:
+ * no API calls, no LLM. The band renders the *what* (which runs agree where,
+ * which categories are weak); the deep-dive explains the *why*.
+ *
+ * Generalized to N runs from the start:
+ *   - 2 runs  → "Both pass / Both fail / Split"
+ *   - 3+ runs → "All pass / All fail / Split"
+ */
+import type { TestCaseComparisonRow } from '../types/index.js';
+/** Agreement bucket for a row across the selected runs. */
+export type AgreementBucket = 'allPass' | 'allFail' | 'split';
+export interface AgreementPartition {
+    allPass: TestCaseComparisonRow[];
+    allFail: TestCaseComparisonRow[];
+    split: TestCaseComparisonRow[];
+    /** Rows where at least one run has no verdict (missing report / not run). */
+    uncovered: TestCaseComparisonRow[];
+}
+/**
+ * Partition rows into agreement buckets across the given runs.
+ * A row only participates when EVERY selected run has a verdict for it —
+ * partially-covered rows go to `uncovered` (they can't agree or disagree).
+ */
+export declare function partitionByAgreement(rows: TestCaseComparisonRow[], runIds: string[]): AgreementPartition;
+/** Bucket a single row (same semantics as {@link partitionByAgreement}); null = uncovered. */
+export declare function bucketRow(row: TestCaseComparisonRow, runIds: string[]): AgreementBucket | null;
+/**
+ * Synthetic bucket names. Parentheses are deliberately outside the
+ * name-tag charset (`[\w-]`), so a real benchmark category can never
+ * collide with these rollup buckets.
+ */
+export declare const UNCATEGORIZED = "(uncategorized)";
+/** Rollup bucket for categories with too few cases to be meaningful. */
+export declare const OTHER_CATEGORY = "(other)";
+/**
+ * Extract a row's category.
+ * Priority:
+ *   1. The `subcategory:<x>` label — the proper, purpose-built tag for this
+ *      (set via the SDK's `test(name, { labels: ['subcategory:basic'] })`,
+ *      the JSON/CLI import's `subcategory` field, or the Test Case editor).
+ *      Prefer this over anything scraped from free text: it's validated,
+ *      exported/round-tripped, and shown elsewhere in the UI (EvalsPage,
+ *      BenchmarkEditor) — a real column, not a regex guess.
+ *   2. A bracketed tag embedded in the test-case name — a convention older
+ *      imported benchmarks used before `subcategory` existed (e.g.
+ *      "qst_0011 [basic] How long is …"). Kept only for benchmarks that
+ *      predate the `subcategory` field; new imports should set it instead.
+ *   3. A `topic:<x>` label (the generic labels system, pre-dates
+ *      `subcategory` as a dedicated concept).
+ *   4. {@link UNCATEGORIZED}.
+ * (`category:<x>` labels are intentionally NOT used here: imported benchmarks
+ * stamp a single `category:RAG` on every case, which would collapse the
+ * breakdown into one column — that's the coarse domain, not this facet.)
+ */
+export declare function extractRowCategory(row: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>): string;
+export interface CategoryCell {
+    passed: number;
+    total: number;
+}
+export interface CategoryBreakdown {
+    /** Display order: by case count desc; `other` (if present) always last. */
+    categories: string[];
+    /** runId → category → {passed,total}. Totals may differ per run (missing results are skipped). */
+    perRun: Record<string, Record<string, CategoryCell>>;
+    /** Overall case count per category (across rows, not per run). */
+    totals: Record<string, number>;
+    /**
+     * Raw categories each displayed column represents. Identity for real
+     * categories; the union of rolled-up raw categories for `(other)`.
+     * Filtering MUST use this mapping so clicking `(other)` matches exactly
+     * the rows its cell counted.
+     */
+    members: Record<string, string[]>;
+}
+/**
+ * Default minimum case count for a category to get its own column; smaller
+ * ones roll up into `other`. Callers with small datasets may lower this —
+ * a cell needs enough cases that a single verdict flip doesn't swing it
+ * by tens of percentage points.
+ */
+export declare const MIN_CATEGORY_CASES = 5;
+/** Per-category pass rates per run, with small categories rolled into `other`. */
+export declare function buildCategoryBreakdown(rows: TestCaseComparisonRow[], runIds: string[], minCases?: number): CategoryBreakdown;
+export interface SharedWeakness {
+    category: string;
+    /** runId → pass-rate percent (0-100) in this category. */
+    rates: Record<string, number>;
+    /** Fraction (0-1) of all-fail cases that belong to this category. */
+    allFailShare: number;
+}
+/**
+ * Detect a category that is the weakest (within {@link WEAKEST_TOLERANCE_PP})
+ * for EVERY selected run — the "shared floor". When present, it usually
+ * indicates a benchmark/corpus-level problem rather than an agent choice.
+ * Returns null when runs disagree about their weakest category or nothing is
+ * genuinely weak. `other`/`uncategorized` rollups are never flagged.
+ */
+export declare function detectSharedWeakness(breakdown: CategoryBreakdown, partition: AgreementPartition, runIds: string[]): SharedWeakness | null;
+//# sourceMappingURL=comparisonInsights.d.ts.map

package/lib/dist/lib/comparisonInsights.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparisonInsights.d.ts","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAGrD,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6EAA6E;IAC7E,SAAS,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAyBD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,kBAAkB,CAgBpB;AAED,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,IAAI,CAO9F;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAC/C,wEAAwE;AACxE,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,GAAG,MAAM,CAQtG;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACrD,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACnC;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,kFAAkF;AAClF,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,GAAE,MAA2B,GACpC,iBAAiB,CAuCnB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,iBAAiB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,cAAc,GAAG,IAAI,CAuCvB"}