@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/lib/matchers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../matchers/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAClG,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,EACxB,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EACL,YAAY,EACZ,UAAU,EACV,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,YAAY,GACb,MAAM,cAAc,CAAC"}

package/lib/dist/lib/matchers/index.js

@@ -0,0 +1,9 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export { expect } from './expect.js';
+export { buildTracesAccessor, emptyTracesAccessor, unavailableTracesAccessor } from './traces.js';
+export { getJudgeMatcherResults, getJudgeReasoningText, recordJudgeMatcherResult, buildJudgeMatcherEntry, formatExpectedOutcomesAsClaim, } from './judgeAccessor.js';
+export { startSession, endSession, isSessionActive, recordVerdict, recordWithTiming, runInSession, } from './session.js';
+//# sourceMappingURL=index.js.map

package/lib/dist/lib/matchers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../matchers/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAElG,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,EACxB,sBAAsB,EACtB,6BAA6B,GAC9B,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,YAAY,EACZ,UAAU,EACV,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,YAAY,GACb,MAAM,cAAc,CAAC"}

package/lib/dist/lib/matchers/judgeAccessor.d.ts

@@ -0,0 +1,113 @@
+/**
+ * Unified judge accessor — single source of truth for "what the LLM judge
+ * said" on an EvaluationReport.
+ *
+ * Background: historically the judge verdict + reasoning lived in two
+ * parallel surfaces:
+ *
+ *   1. report.llmJudgeReasoning (string) — written by the legacy
+ *      auto-judge path (services/evaluation/index.ts and the various
+ *      runners that call callBedrockJudge against expectedOutcomes).
+ *      One per report, last writer wins.
+ *
+ *   2. report.matcherResults[*] with method: 'llm-judge' — written by
+ *      the SDK `judge(result, claim)` matcher. Many per report.
+ *
+ * Both ultimately come from the same Bedrock Judge endpoint. Storing
+ * them in two places caused real bugs (notably issue #230, where the
+ * trace-mode placeholder leaked into the legacy field and bled through
+ * to the UI for deterministic SDK runs).
+ *
+ * The fix: `matcherResults[]` is the canonical surface. The legacy
+ * `llmJudgeReasoning` field is kept populated as a derived
+ * convenience for backward-compat readers, but new code MUST use
+ * {@link getJudgeMatcherResults} on the read side and
+ * {@link recordJudgeMatcherResult} on the write side.
+ */
+import type { MatcherResult } from './types.js';
+/** A minimal duck-typed report shape — accepts any object with the two fields. */
+export interface JudgeAccessReport {
+    matcherResults?: MatcherResult[];
+    llmJudgeReasoning?: string;
+}
+/**
+ * Returns every llm-judge MatcherResult on the report, including a
+ * synthetic entry derived from the legacy `llmJudgeReasoning` field
+ * when the report pre-dates the unified surface.
+ *
+ * Read-only: nothing is written back to the report.
+ *
+ * Behaviour matrix:
+ *
+ * | matcherResults has [llm-judge] | llmJudgeReasoning      | Returns                    |
+ * |--------------------------------|------------------------|----------------------------|
+ * | yes                            | anything               | the existing entries       |
+ * | no                             | empty / placeholder    | []                         |
+ * | no                             | real reasoning         | [<synthetic entry>]        |
+ *
+ * The placeholder string `Waiting for traces to become available...`
+ * is treated as "no judge ran yet" \u2014 see issue #230.
+ */
+export declare function getJudgeMatcherResults(report: JudgeAccessReport): MatcherResult[];
+/**
+ * Convenience: the single most recent llm-judge reasoning string, or
+ * empty when no judge ran. Used by surfaces that historically read
+ * `llmJudgeReasoning` directly (HTML report, skills grader, compare
+ * view) and only need a flat string.
+ */
+export declare function getJudgeReasoningText(report: JudgeAccessReport): string;
+/**
+ * Convert a `JudgeResult` from `callBedrockJudge` into a `MatcherResult`.
+ * Pure: doesn't touch any report. Used by inline-return code paths in
+ * services/evaluation/index.ts where building a fresh report object is
+ * cleaner than the mutate-then-return pattern.
+ */
+export declare function buildJudgeMatcherEntry(judgeResult: JudgeResultLike, options?: {
+    claim?: string;
+    model?: string;
+}): MatcherResult;
+/**
+ * Format an arbitrary `expectedOutcomes` value into a single description
+ * fragment suitable for the matcher entry's `description` field. Accepts
+ * the SDK's flat `string[]` *and* the legacy `{ rootCauses, requiredFacts,
+ * conclusions }` object — both shapes appear on persisted test cases.
+ */
+export declare function formatExpectedOutcomesAsClaim(outcomes?: unknown): string;
+/**
+ * Convert a `JudgeResult` from `callBedrockJudge` into a `MatcherResult`
+ * and append it to `report.matcherResults`. Also keeps the legacy
+ * `report.llmJudgeReasoning` field populated as a derived
+ * backward-compat shim (it equals the reasoning of the just-pushed
+ * entry, so direct readers continue to work).
+ *
+ * Mutates the passed report and returns the matcher entry.
+ *
+ * The `claim` argument is what the auto-judge was evaluating against
+ * (the test case's expected outcomes). The matcher description ends
+ * up as `judge: <claim>` matching the SDK convention.
+ */
+export declare function recordJudgeMatcherResult(report: JudgeAccessReport & {
+    matcherResults?: MatcherResult[];
+    llmJudgeReasoning?: string;
+}, judgeResult: JudgeResultLike, options?: {
+    claim?: string;
+    model?: string;
+}): MatcherResult;
+/** Subset of `JudgeResult` we depend on \u2014 keeps this module type-import-light. */
+export interface JudgeResultLike {
+    passFailStatus: 'passed' | 'failed';
+    metrics: {
+        accuracy?: number;
+        [k: string]: number | undefined;
+    };
+    llmJudgeReasoning: string;
+    judgeDurationMs?: number;
+    improvementStrategies?: Array<{
+        category: string;
+        issue: string;
+        recommendation: string;
+        priority: 'high' | 'medium' | 'low';
+    }>;
+}
+export type { JudgeResult } from '../../types/index.js';
+//# sourceMappingURL=judgeAccessor.d.ts.map

package/lib/dist/lib/matchers/judgeAccessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judgeAccessor.d.ts","sourceRoot":"","sources":["../../../matchers/judgeAccessor.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAGhD,kFAAkF;AAClF,MAAM,WAAW,iBAAiB;IAChC,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,iBAAiB,GAAG,aAAa,EAAE,CAuBjF;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAevE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,eAAe,EAC5B,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,aAAa,CA0Bf;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,CAoBxE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,iBAAiB,GAAG;IAAE,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAE,EAC5F,WAAW,EAAE,eAAe,EAC5B,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,aAAa,CAcf;AAED,uFAAuF;AACvF,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACpC,OAAO,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC;IAChE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,KAAK,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;KACrC,CAAC,CAAC;CACJ;AA+BD,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC"}

package/lib/dist/lib/matchers/judgeAccessor.js

@@ -0,0 +1,183 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Returns every llm-judge MatcherResult on the report, including a
+ * synthetic entry derived from the legacy `llmJudgeReasoning` field
+ * when the report pre-dates the unified surface.
+ *
+ * Read-only: nothing is written back to the report.
+ *
+ * Behaviour matrix:
+ *
+ * | matcherResults has [llm-judge] | llmJudgeReasoning      | Returns                    |
+ * |--------------------------------|------------------------|----------------------------|
+ * | yes                            | anything               | the existing entries       |
+ * | no                             | empty / placeholder    | []                         |
+ * | no                             | real reasoning         | [<synthetic entry>]        |
+ *
+ * The placeholder string `Waiting for traces to become available...`
+ * is treated as "no judge ran yet" \u2014 see issue #230.
+ */
+export function getJudgeMatcherResults(report) {
+    const fromMatcher = (report.matcherResults ?? []).filter(m => m.method === 'llm-judge');
+    if (fromMatcher.length > 0) {
+        return fromMatcher;
+    }
+    const legacy = report.llmJudgeReasoning?.trim();
+    if (!legacy || isPlaceholderReasoning(legacy)) {
+        return [];
+    }
+    // Old report: synthesize a single virtual matcher entry so UIs that
+    // render only matcherResults still have something to show.
+    return [
+        {
+            description: 'judge: expected outcomes',
+            pass: inferPassFromReport(report),
+            method: 'llm-judge',
+            reasoning: legacy,
+        },
+    ];
+}
+/**
+ * Convenience: the single most recent llm-judge reasoning string, or
+ * empty when no judge ran. Used by surfaces that historically read
+ * `llmJudgeReasoning` directly (HTML report, skills grader, compare
+ * view) and only need a flat string.
+ */
+export function getJudgeReasoningText(report) {
+    const entries = getJudgeMatcherResults(report);
+    if (entries.length === 0)
+        return '';
+    // Most-recent-first: prefer the last entry the runner pushed. If
+    // multiple entries exist (SDK with several judge() calls) we join
+    // them with horizontal-rule markers so a flat-string consumer still
+    // sees every claim.
+    if (entries.length === 1)
+        return entries[0].reasoning ?? '';
+    return entries
+        .map((m, i) => {
+        const claim = m.description?.replace(/^judge:\s*/, '') ?? `claim ${i + 1}`;
+        const verdict = m.pass ? 'PASS' : 'FAIL';
+        return `### ${verdict} \u2014 ${claim}\n\n${m.reasoning ?? ''}`;
+    })
+        .join('\n\n---\n\n');
+}
+/**
+ * Convert a `JudgeResult` from `callBedrockJudge` into a `MatcherResult`.
+ * Pure: doesn't touch any report. Used by inline-return code paths in
+ * services/evaluation/index.ts where building a fresh report object is
+ * cleaner than the mutate-then-return pattern.
+ */
+export function buildJudgeMatcherEntry(judgeResult, options) {
+    const claim = options?.claim?.trim() || 'expected outcomes';
+    const accuracy = judgeResult.metrics?.accuracy;
+    const score = typeof accuracy === 'number' ? accuracy / 100 : undefined;
+    const pass = judgeResult.passFailStatus === 'passed';
+    const reasoning = judgeResult.llmJudgeReasoning ?? '';
+    return {
+        description: `judge: ${claim}`,
+        pass,
+        method: 'llm-judge',
+        durationMs: judgeResult.judgeDurationMs,
+        score,
+        reasoning,
+        ...(options?.model ? { model: options.model } : {}),
+        ...(pass ? {} : { errorMessage: reasoning }),
+        // Carry the rest of the judge payload through so SDK
+        // `getJudgeMatcherResults()` consumers see the same data the
+        // legacy report-level fields used to expose. See MatcherResult.
+        ...(Array.isArray(judgeResult.improvementStrategies) && judgeResult.improvementStrategies.length > 0
+            ? { improvementStrategies: judgeResult.improvementStrategies }
+            : {}),
+        ...(judgeResult.metrics && typeof judgeResult.metrics === 'object'
+            ? { judgeMetrics: { ...judgeResult.metrics } }
+            : {}),
+    };
+}
+/**
+ * Format an arbitrary `expectedOutcomes` value into a single description
+ * fragment suitable for the matcher entry's `description` field. Accepts
+ * the SDK's flat `string[]` *and* the legacy `{ rootCauses, requiredFacts,
+ * conclusions }` object — both shapes appear on persisted test cases.
+ */
+export function formatExpectedOutcomesAsClaim(outcomes) {
+    const items = [];
+    if (Array.isArray(outcomes)) {
+        for (const o of outcomes) {
+            if (typeof o === 'string' && o.trim())
+                items.push(o.trim());
+        }
+    }
+    else if (outcomes && typeof outcomes === 'object') {
+        // Legacy shape: { rootCauses?: string[]; requiredFacts?: string[]; conclusions?: string[] }
+        for (const key of Object.keys(outcomes)) {
+            const v = outcomes[key];
+            if (Array.isArray(v)) {
+                for (const s of v) {
+                    if (typeof s === 'string' && s.trim())
+                        items.push(s.trim());
+                }
+            }
+        }
+    }
+    if (items.length === 0)
+        return 'expected outcomes';
+    if (items.length === 1)
+        return items[0];
+    return `${items.length} expected outcomes`;
+}
+/**
+ * Convert a `JudgeResult` from `callBedrockJudge` into a `MatcherResult`
+ * and append it to `report.matcherResults`. Also keeps the legacy
+ * `report.llmJudgeReasoning` field populated as a derived
+ * backward-compat shim (it equals the reasoning of the just-pushed
+ * entry, so direct readers continue to work).
+ *
+ * Mutates the passed report and returns the matcher entry.
+ *
+ * The `claim` argument is what the auto-judge was evaluating against
+ * (the test case's expected outcomes). The matcher description ends
+ * up as `judge: <claim>` matching the SDK convention.
+ */
+export function recordJudgeMatcherResult(report, judgeResult, options) {
+    const entry = buildJudgeMatcherEntry(judgeResult, options);
+    const existing = report.matcherResults ?? [];
+    report.matcherResults = [...existing, entry];
+    // Backward-compat shim: keep the legacy field populated with the most
+    // recent reasoning so external readers that haven't migrated still
+    // work. Empty string is a deliberate signal for SDK-multi-judge runs
+    // (the field can't represent multiple verdicts; consumers must look
+    // at matcherResults).
+    report.llmJudgeReasoning = entry.reasoning ?? '';
+    return entry;
+}
+// ─── internal helpers ───────────────────────────────────────────────────────
+/**
+ * The trace-mode init path sets `llmJudgeReasoning` to a placeholder
+ * before any real verdict exists. We must not synthesize a virtual
+ * matcher entry from that string \u2014 it's a "judge has not run yet"
+ * signal, not real reasoning.
+ */
+function isPlaceholderReasoning(s) {
+    return /^Waiting for traces/i.test(s);
+}
+/**
+ * Best-effort guess at the verdict for a synthesized entry on an old
+ * report. Prefers an explicit `passFailStatus` if present; otherwise
+ * falls back to the report's metrics.accuracy (>=70 ~ pass), and
+ * finally defaults to `false` so a missing verdict never silently
+ * looks green.
+ */
+function inferPassFromReport(report) {
+    if (report.passFailStatus === 'passed')
+        return true;
+    if (report.passFailStatus === 'failed')
+        return false;
+    const acc = report.metrics?.accuracy;
+    if (typeof acc === 'number')
+        return acc >= 70;
+    return false;
+}
+//# sourceMappingURL=judgeAccessor.js.map

package/lib/dist/lib/matchers/judgeAccessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judgeAccessor.js","sourceRoot":"","sources":["../../../matchers/judgeAccessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsCH;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAyB;IAC9D,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,CACtD,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAC9B,CAAC;IACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC;IAChD,IAAI,CAAC,MAAM,IAAI,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,oEAAoE;IACpE,2DAA2D;IAC3D,OAAO;QACL;YACE,WAAW,EAAE,0BAA0B;YACvC,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC;YACjC,MAAM,EAAE,WAAW;YACnB,SAAS,EAAE,MAAM;SAClB;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAyB;IAC7D,MAAM,OAAO,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,iEAAiE;IACjE,kEAAkE;IAClE,oEAAoE;IACpE,oBAAoB;IACpB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;IAC5D,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACzC,OAAO,OAAO,OAAO,WAAW,KAAK,OAAO,CAAC,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;IAClE,CAAC,CAAC;SACD,IAAI,CAAC,aAAa,CAAC,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,WAA4B,EAC5B,OAA4C;IAE5C,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,mBAAmB,CAAC;IAC5D,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACxE,MAAM,IAAI,GAAG,WAAW,CAAC,cAAc,KAAK,QAAQ,CAAC;IACrD,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAEtD,OAAO;QACL,WAAW,EAAE,UAAU,KAAK,EAAE;QAC9B,IAAI;QACJ,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,WAAW,CAAC,eAAe;QACvC,KAAK;QACL,SAAS;QACT,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC;QAC5C,qDAAqD;QACrD,6DAA6D;QAC7D,gEAAgE;QAChE,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,IAAI,WAAW,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC;YAClG,CAAC,CAAC,EAAE,qBAAqB,EAAE,WAAW,CAAC,qBAAqB,EAAE;YAC9D,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,WAAW,CAAC,OAAO,IAAI,OAAO,WAAW,CAAC,OAAO,KAAK,QAAQ;YAChE,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,GAAG,WAAW,CAAC,OAAO,EAAS,EAAE;YACrD,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,6BAA6B,CAAC,QAAkB;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;SAAM,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACpD,4FAA4F;QAC5F,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAmC,CAAC,EAAE,CAAC;YACnE,MAAM,CAAC,GAAI,QAAoC,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrB,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE;wBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,mBAAmB,CAAC;IACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IACxC,OAAO,GAAG,KAAK,CAAC,MAAM,oBAAoB,CAAC;AAC7C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,wBAAwB,CACtC,MAA4F,EAC5F,WAA4B,EAC5B,OAA4C;IAE5C,MAAM,KAAK,GAAG,sBAAsB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAE3D,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC;IAC7C,MAAM,CAAC,cAAc,GAAG,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CAAC;IAE7C,sEAAsE;IACtE,mEAAmE;IACnE,qEAAqE;IACrE,oEAAoE;IACpE,sBAAsB;IACtB,MAAM,CAAC,iBAAiB,GAAG,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC;IAEjD,OAAO,KAAK,CAAC;AACf,CAAC;AAgBD,+EAA+E;AAE/E;;;;;GAKG;AACH,SAAS,sBAAsB,CAAC,CAAS;IACvC,OAAO,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,MAAwF;IACnH,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACpD,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC;IACrC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,IAAI,EAAE,CAAC;IAC9C,OAAO,KAAK,CAAC;AACf,CAAC"}

package/lib/dist/lib/matchers/session.d.ts

@@ -0,0 +1,39 @@
+import type { MatcherResult, MatcherMethod } from './types.js';
+export interface MatcherSession {
+    results: MatcherResult[];
+    startedAt: number;
+}
+/**
+ * Run `fn` with a fresh, isolated matcher session and return the verdicts
+ * recorded during it. Concurrency-safe: each call gets its own session via
+ * AsyncLocalStorage, so interleaved bodies never share verdicts.
+ *
+ * Results are returned even when `fn` throws — the caller can inspect both
+ * the partial results and the error (cleanup hooks that record after a body
+ * failure therefore still surface in the breakdown).
+ */
+export declare function runInSession<T>(fn: () => Promise<T> | T): Promise<{
+    results: MatcherResult[];
+    value?: T;
+    error?: unknown;
+}>;
+/**
+ * Begin a fresh global session and make it active (legacy API).
+ *
+ * Prefer {@link runInSession} for anything that may run concurrently. This
+ * remains for synchronous, single-threaded callers.
+ */
+export declare function startSession(): MatcherSession;
+/** Stop the current global session and return its results (legacy API). */
+export declare function endSession(): MatcherResult[];
+/** True when a session is active (ALS scope or legacy global). */
+export declare function isSessionActive(): boolean;
+/** Record a single matcher verdict on the active session. No-op when none. */
+export declare function recordVerdict(result: MatcherResult): void;
+/**
+ * Record a verdict computed from a try/catch around the matcher body.
+ * Convenience wrapper used by judge() and traces helpers; chai matchers go
+ * through the chai plugin in `./expect.ts`.
+ */
+export declare function recordWithTiming<T>(description: string, method: MatcherMethod, fn: () => Promise<T> | T): Promise<T>;
+//# sourceMappingURL=session.d.ts.map

package/lib/dist/lib/matchers/session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../matchers/session.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE/D,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;CACnB;AAiBD;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAClC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GACvB,OAAO,CAAC;IAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,CAQnE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,IAAI,cAAc,CAG7C;AAED,2EAA2E;AAC3E,wBAAgB,UAAU,IAAI,aAAa,EAAE,CAK5C;AAED,kEAAkE;AAClE,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AAED,8EAA8E;AAC9E,wBAAgB,aAAa,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAIzD;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,aAAa,EACrB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GACvB,OAAO,CAAC,CAAC,CAAC,CAqBZ"}

package/lib/dist/lib/matchers/session.js

@@ -0,0 +1,116 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Matcher session — collects MatcherResult[] for the currently-executing
+ * test body. Matchers (expect/judge/traces) record into the active session
+ * via `recordVerdict`.
+ *
+ * Concurrency model (RFC 004, "kills the global session"):
+ *
+ *   The session is held in an AsyncLocalStorage so that bodies running
+ *   concurrently (the runners dispatch up to `concurrency` tests at once,
+ *   interleaving at every `await`) each record into their OWN session.
+ *   Before this, a single module-level `activeSession` was shared by all
+ *   in-flight bodies, so verdicts cross-contaminated between tests.
+ *
+ *   `runInSession(fn)` is the concurrency-safe entry point and is what the
+ *   runners use: it establishes a fresh session for the duration of `fn`
+ *   (and everything it `await`s) and returns the collected results.
+ *
+ *   The legacy `startSession()` / `endSession()` pair is retained for
+ *   synchronous, non-interleaved callers (unit tests, simple scripts). It
+ *   uses a module-global fallback session. When an AsyncLocalStorage scope
+ *   is active (i.e. we're inside `runInSession`), the ALS session always
+ *   takes precedence over the global, so the two never collide.
+ */
+import { AsyncLocalStorage } from 'async_hooks';
+/** Per-async-context session. Set for the duration of `runInSession(fn)`. */
+const sessionStore = new AsyncLocalStorage();
+/** Legacy module-global session for direct startSession()/endSession() callers. */
+let legacySession = null;
+function newSession() {
+    return { results: [], startedAt: Date.now() };
+}
+/** The session verdicts should be recorded into right now, if any. */
+function currentSession() {
+    return sessionStore.getStore() ?? legacySession;
+}
+/**
+ * Run `fn` with a fresh, isolated matcher session and return the verdicts
+ * recorded during it. Concurrency-safe: each call gets its own session via
+ * AsyncLocalStorage, so interleaved bodies never share verdicts.
+ *
+ * Results are returned even when `fn` throws — the caller can inspect both
+ * the partial results and the error (cleanup hooks that record after a body
+ * failure therefore still surface in the breakdown).
+ */
+export async function runInSession(fn) {
+    const session = newSession();
+    try {
+        const value = await sessionStore.run(session, fn);
+        return { results: session.results, value };
+    }
+    catch (error) {
+        return { results: session.results, error };
+    }
+}
+/**
+ * Begin a fresh global session and make it active (legacy API).
+ *
+ * Prefer {@link runInSession} for anything that may run concurrently. This
+ * remains for synchronous, single-threaded callers.
+ */
+export function startSession() {
+    legacySession = newSession();
+    return legacySession;
+}
+/** Stop the current global session and return its results (legacy API). */
+export function endSession() {
+    if (!legacySession)
+        return [];
+    const out = legacySession.results;
+    legacySession = null;
+    return out;
+}
+/** True when a session is active (ALS scope or legacy global). */
+export function isSessionActive() {
+    return currentSession() !== null;
+}
+/** Record a single matcher verdict on the active session. No-op when none. */
+export function recordVerdict(result) {
+    const session = currentSession();
+    if (!session)
+        return;
+    session.results.push(result);
+}
+/**
+ * Record a verdict computed from a try/catch around the matcher body.
+ * Convenience wrapper used by judge() and traces helpers; chai matchers go
+ * through the chai plugin in `./expect.ts`.
+ */
+export async function recordWithTiming(description, method, fn) {
+    const start = Date.now();
+    try {
+        const value = await fn();
+        recordVerdict({
+            description,
+            pass: true,
+            method,
+            durationMs: Date.now() - start,
+        });
+        return value;
+    }
+    catch (err) {
+        recordVerdict({
+            description,
+            pass: false,
+            method,
+            durationMs: Date.now() - start,
+            errorMessage: err?.message || String(err),
+        });
+        throw err;
+    }
+}
+//# sourceMappingURL=session.js.map

package/lib/dist/lib/matchers/session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.js","sourceRoot":"","sources":["../../../matchers/session.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAQhD,6EAA6E;AAC7E,MAAM,YAAY,GAAG,IAAI,iBAAiB,EAAkB,CAAC;AAE7D,mFAAmF;AACnF,IAAI,aAAa,GAA0B,IAAI,CAAC;AAEhD,SAAS,UAAU;IACjB,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AAChD,CAAC;AAED,sEAAsE;AACtE,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC,QAAQ,EAAE,IAAI,aAAa,CAAC;AAClD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAwB;IAExB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7C,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY;IAC1B,aAAa,GAAG,UAAU,EAAE,CAAC;IAC7B,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC,aAAa;QAAE,OAAO,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC;IAClC,aAAa,GAAG,IAAI,CAAC;IACrB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,eAAe;IAC7B,OAAO,cAAc,EAAE,KAAK,IAAI,CAAC;AACnC,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,aAAa,CAAC,MAAqB;IACjD,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,IAAI,CAAC,OAAO;QAAE,OAAO;IACrB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,WAAmB,EACnB,MAAqB,EACrB,EAAwB;IAExB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,EAAE,CAAC;QACzB,aAAa,CAAC;YACZ,WAAW;YACX,IAAI,EAAE,IAAI;YACV,MAAM;YACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,aAAa,CAAC;YACZ,WAAW;YACX,IAAI,EAAE,KAAK;YACX,MAAM;YACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,YAAY,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;SAC1C,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC"}

package/lib/dist/lib/matchers/traces.d.ts

@@ -0,0 +1,55 @@
+/**
+ * `traces` fixture — read-only access to the OTel traces emitted by the
+ * agent during this run. Pre-loaded by the runner before the test body
+ * starts; sync access from inside the body.
+ *
+ * Three construction modes (the runner picks one):
+ *   - `buildTracesAccessor(spans)` — real spans available, real numbers.
+ *   - `emptyTracesAccessor()` — agent opted out (`useTraces=false`).
+ *     Every accessor returns 0/[] so user matchers don't blow up.
+ *   - `unavailableTracesAccessor(reason)` — agent opted in (`useTraces=true`)
+ *     but spans were not fetchable (no runId, fetch failed, or polling
+ *     timed out). Every accessor *throws* so silent false-passes like
+ *     `expect(traces.totalTokens).to.be.lessThan(10_000)` against an
+ *     empty fixture become loud failures (see issue #230).
+ *
+ * Built on top of `services/traces/index.ts:fetchTracesByRunIds` —
+ * the runner is responsible for invoking that and constructing the
+ * accessor.
+ */
+import type { Span } from '../../types/index.js';
+export interface TracesAccessor {
+    /** Sum of all `gen_ai.usage.prompt_tokens + completion_tokens` across LLM spans. */
+    totalTokens: number;
+    /** Sum of all `gen_ai.usage.cost_usd` (or derived) across LLM spans. */
+    totalCost: number;
+    /** Tool invocation summary derived from spans. */
+    toolCalls: ReadonlyArray<{
+        name: string;
+        durationMs: number;
+    }>;
+    /** Duration of the first span matching the given name, or 0 when not found. */
+    spanDuration(name: string): number;
+    /** All spans (read-only). */
+    spans: ReadonlyArray<Span>;
+}
+/**
+ * Empty accessor — silent zeros. Used when the agent opted out of traces
+ * (`useTraces: false`). Reading any accessor returns `0` / `[]` so user
+ * matchers in opt-out scenarios don't throw.
+ */
+export declare function emptyTracesAccessor(): TracesAccessor;
+/**
+ * Loud-failure accessor — every read throws. Used when the agent opted
+ * in to traces (`useTraces: true`) but spans were not retrievable. This
+ * turns the silent false-pass described in issue #230 into an actionable
+ * error, e.g.:
+ *
+ *   Error: traces fixture unavailable: no spans found for runId=…
+ *
+ * Construction never throws — only attribute / method access does.
+ */
+export declare function unavailableTracesAccessor(reason: string): TracesAccessor;
+/** Build a TracesAccessor from a flat list of spans. */
+export declare function buildTracesAccessor(spans: Span[]): TracesAccessor;
+//# sourceMappingURL=traces.d.ts.map

package/lib/dist/lib/matchers/traces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../../matchers/traces.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,WAAW,cAAc;IAC7B,oFAAoF;IACpF,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,SAAS,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/D,+EAA+E;IAC/E,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC,6BAA6B;IAC7B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,cAAc,CAQpD;AAED;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAWxE;AAED,wDAAwD;AACxD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,cAAc,CA4CjE"}