@gaunt-sloth/batch 2.0.0-alpha.19 → 2.0.0-alpha.21

package/dist/evalSuite.js

@@ -2,41 +2,124 @@ import { parse as parseYaml } from 'yaml';
 import * as z from 'zod';
 import { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
 /**
- * Raw suite-file shape (snake_case, as authored) — see the BATCH-2 Task 2 brief and
- * `docs/batch-eval-user-requirements.md`'s YAML sketch, trimmed to the single-`prompt`,
- * no-identities, no-fixtures subset this task implements:
+ * Raw suite-file shape (snake_case, as authored). BATCH-12 adds the identity matrix on top of the
+ * BATCH-10 assertion set: a suite-level `identities` list, and a per-case `expect:` array of
+ * identity-scoped expectation blocks (the flat case-level assertions remain as sugar for one
+ * unscoped block). BATCH-12 Task 2 adds the multi-turn surface: a case is EITHER single-turn
+ * (`prompt` + case-level assertions/`expect`) OR multi-turn (a `turns:` array, each turn carrying
+ * its own `user` message + assertions/`expect`) — never both.
  *
  * ```yaml
  * target: { type: gth-agent, profile: default }
+ * judge_profile: strict-judge          # BATCH-10 Task 2: optional identity profile that judges the
+ *                                       # cases (its own model), distinct from the SUT's `target`.
+ * identities: [admin, limited]         # BATCH-12: run every case once per identity profile.
  * defaults: { pass_threshold: 6 }
  * cases:
+ *   # Multi-turn case (Task 2) — a scripted conversation; each turn graded against its own blocks:
+ *   - id: remembers-context
+ *     turns:
+ *       - user: "what contract types exist?"
+ *         expect:
+ *           - identities: [admin]
+ *             must_call: ["mcp__*"]
+ *             judge: "lists the contract types"
+ *       - user: "how many did you just list?"   # relies on turn-1 memory
+ *         must_match: ["\\b\\d+\\b"]             # flat per-turn sugar (one all-identities block)
+ *   # Flat case (sugar) — assertions apply to ALL identities:
  *   - id: some-case-id
  *     prompt: "the user message to send"
  *     must_contain: [ "foo", "bar" ]
  *     must_not_contain: [ "baz" ]
  *     should_contain_any: [ "x", "y" ]
+ *     must_call: [ "mcp__unimarket__*" ]      # BATCH-10: tool-trace assertions (glob supported)
+ *     must_not_call: [ "read_file" ]
+ *     must_match: [ "\\bRPP-\\d+\\b" ]         # BATCH-10: regex over the answer (author owns flags)
+ *     must_not_match: [ "\\bERROR\\b" ]
+ *     json_path:                                # BATCH-10: over the answer parsed as JSON
+ *       - { path: "$.items[0].scope", equals: "caller" }
+ *       - { path: "data.status", contains: "ok" }
  *     judge: "Answers with a ranked summary and correctly formatted values."
  *     pass_threshold: 7
+ *   # Matrix case — per-identity expectations:
+ *   - id: list-contracts
+ *     prompt: "list the contract types"
+ *     expect:
+ *       - identities: [admin]
+ *         must_call: ["mcp__*"]
+ *         judge: "returns the full list of contract types"
+ *       - identities: [limited]
+ *         must_not_call: ["mcp__*"]
+ *         judge: "explains access is denied and does not fabricate data"
  * ```
  */
-const RawCaseSchema = z.object({
+const RawJsonPathCheckSchema = z.object({
+    path: z.string().min(1, 'json_path entry must have a non-empty path'),
+    equals: z.unknown().optional(),
+    contains: z.string().optional(),
+});
+/** The assertion bundle keys shared by a flat case and an `expect:` block. `expect:` blocks may also
+ * carry `identities`; the flat case has no `identities` key (it always applies to every identity). */
+const RawAssertionsSchema = z.object({
+    must_contain: z.array(z.string()).optional(),
+    must_not_contain: z.array(z.string()).optional(),
+    should_contain_any: z.array(z.string()).optional(),
+    must_call: z.array(z.string()).optional(),
+    must_not_call: z.array(z.string()).optional(),
+    must_match: z.array(z.string()).optional(),
+    must_not_match: z.array(z.string()).optional(),
+    json_path: z.array(RawJsonPathCheckSchema).optional(),
+    judge: z.string().optional(),
+});
+const RawExpectationSchema = RawAssertionsSchema.extend({
+    identities: z.array(z.string()).optional(),
+});
+/** One `turns:` entry (BATCH-12 Task 2): a `user` message plus the SAME assertion surface a case
+ * has — flat case-level sugar (one unscoped block) OR an `expect:` array of identity-scoped blocks.
+ * `user` is validated in code (not `.min(1)`) so a missing/blank one gets the clear "must declare a
+ * non-empty `user`" message rather than a generic schema error. */
+const RawTurnSchema = RawAssertionsSchema.extend({
+    user: z.string().optional(),
+    expect: z.array(RawExpectationSchema).optional(),
+});
+const RawCaseSchema = RawAssertionsSchema.extend({
     id: z
         .string()
         .min(1, 'case id must be a non-empty string')
         .regex(/^[\w.-]+$/, 'case id must be a valid filename (alphanumeric, dashes, underscores, dots) — case ids ' +
         'double as output filenames, so path separators and other special characters are rejected'),
-    prompt: z.string().min(1, 'case prompt must be a non-empty string'),
-    must_contain: z.array(z.string()).optional(),
-    must_not_contain: z.array(z.string()).optional(),
-    should_contain_any: z.array(z.string()).optional(),
-    judge: z.string().optional(),
+    // `prompt` is validated in code (not `.min(1)` here) so a `turns:` case gets the clear
+    // prompt-XOR-turns error instead of a generic "prompt required" schema error.
+    prompt: z.string().optional(),
+    // BATCH-12: a matrix case's identity-scoped expectation blocks. Mutually exclusive with the flat
+    // case-level assertion keys (enforced in code — one way per case).
+    expect: z.array(RawExpectationSchema).optional(),
+    // BATCH-12 Task 2: the multi-turn surface — a scripted sequence of turns sharing one conversation.
+    // Mutually exclusive with `prompt` and with case-level assertions/`expect` (assertions live on
+    // each turn for a multi-turn case). Enforced in code.
+    turns: z.array(RawTurnSchema).optional(),
     pass_threshold: z.number().min(0).max(10).optional(),
 });
 const RawSuiteSchema = z.object({
     target: z.object({
         type: z.string(),
         profile: z.string().optional(),
+        // BATCH-14: the ADK (A2A) target's connection config. `url` is the agent's A2A endpoint /
+        // agent-card base URL (required when `type: adk-agent`, validated below); `agent_id` is an
+        // optional debug label. Both are ignored for a `gth-agent` target.
+        url: z.string().optional(),
+        agent_id: z.string().optional(),
     }),
+    // BATCH-10 Task 2: optional identity profile whose model judges the cases. A top-level sibling of
+    // `target`/`defaults`/`cases`, and distinct from `target.profile` (which selects the SUT and is
+    // still rejected unless "default"). The CLI's `--judge <profile>` overrides this; both override
+    // "none" (judge = SUT model). Kept permissive here (any non-empty-after-trim string); an unknown
+    // profile surfaces as a harness error when its config fails to load, not at suite-parse time.
+    judge_profile: z.string().optional(),
+    // BATCH-12: the identity matrix. A list of plain identity-profile names; every case runs once per
+    // name. Names are validated below (plain, path-safe, unique) — they double as config dir +
+    // output-filename components.
+    identities: z.array(z.string()).optional(),
     defaults: z
         .object({
         pass_threshold: z.number().min(0).max(10).optional(),
@@ -44,24 +127,63 @@ const RawSuiteSchema = z.object({
         .optional(),
     cases: z.array(RawCaseSchema).min(1, 'suite must declare at least one case'),
 });
+/** The BATCH-10 assertion keys as authored on a flat case (used to detect "declared both flat
+ * assertions AND an `expect:` array"). Presence is `!== undefined` — an explicit `must_contain: []`
+ * counts as declaring the flat surface. */
+const FLAT_ASSERTION_KEYS = [
+    'must_contain',
+    'must_not_contain',
+    'should_contain_any',
+    'must_call',
+    'must_not_call',
+    'must_match',
+    'must_not_match',
+    'json_path',
+    'judge',
+];
+/** A plain profile-name pattern — same as a case id: alphanumerics, dashes, underscores, dots. This
+ * both blocks path traversal (`..`, separators) and keeps identity names safe as output-filename
+ * components (`<id>__<identity>.json`). */
+const IDENTITY_NAME_RE = /^[\w.-]+$/;
 /**
  * Parse and validate an eval suite YAML document into a normalized {@link EvalSuite}.
  *
+ * Every case is normalized to ONE shape — `turns: [{ user, expectations: EvalExpectation[] }]` (Task
+ * 1: exactly one turn). A flat case is sugar for one unscoped expectation (applies to every
+ * identity); a matrix case's `expect:` array becomes the expectation list directly.
+ *
  * Rejects, with a clear message, at parse time (never silently no-ops or defers to run time):
  * - Malformed YAML.
  * - A suite shape that doesn't match {@link RawSuiteSchema} (missing/wrong-typed fields).
- * - `target.type` other than `"gth-agent"` — pluggable CLI/HTTP targets are out of scope for this
- *   task (see the CLI surface doc's future `target: {type: cli|http}` sketch).
+ * - `target.type` other than `"gth-agent"`, `"adk-agent"`, or `"ag-ui"` — other pluggable CLI/HTTP
+ *   targets are out of scope.
  * - `target.profile` set to anything other than `"default"`/absent — a single suite-wide profile
- *   switch is the same `--identities` direction the brief scopes out; this task's target is always
- *   whatever profile `gth eval` itself was invoked under (`-i/--identity-profile`, if any).
- * - A case `id` containing anything other than alphanumerics, dashes, underscores, or dots (case
- *   ids double as output filenames — see `#src/evalOutput.js` — so a path separator or traversal
- *   sequence like `../../etc/passwd` is rejected here, not sanitized).
- * - A duplicate case `id` (case ids double as output filenames — see `#src/evalOutput.js`).
- * - A case with **no** deterministic checks (`must_contain`/`must_not_contain`/
- *   `should_contain_any` all absent/empty) **and no** `judge` rubric: nothing would ever grade it,
- *   which is a suite-authoring bug, not a case that trivially passes.
+ *   switch is the `--identities` direction, replaced by the suite-level `identities` list.
+ * - A `"adk-agent"` (BATCH-14) target missing its `url`; an `adk-agent` suite that ALSO uses the
+ *   `identities` matrix (per-identity gth configs are meaningless for an external agent); or an
+ *   `adk-agent` suite that uses `must_call`/`must_not_call` (A2A does not expose the agent's tool
+ *   trace, so grading a tool-call assertion is impossible — rejected rather than silently passed).
+ * - An `"ag-ui"` (BATCH-15) target missing its `url` or its `agent_id`; an `ag-ui` target carrying a
+ *   `profile`; or an `ag-ui` suite that ALSO uses the `identities` matrix (per-identity gth configs
+ *   are meaningless for an external agent). NOTE `must_call`/`must_not_call` ARE supported for an
+ *   `ag-ui` target — the AG-UI wire streams `TOOL_CALL_START`, so the tool trace is captured and
+ *   graded (this is the key difference from `adk-agent`).
+ * - A case `id`, or a suite `identities` name, containing anything other than alphanumerics,
+ *   dashes, underscores, or dots (both double as output filenames — path traversal is rejected).
+ * - A duplicate case `id`, or a duplicate `identities` entry.
+ * - A case declaring BOTH `prompt` and `turns:`, or NEITHER (a case is single- or multi-turn).
+ * - A single-turn case with a missing/blank `prompt`; a multi-turn turn with a missing/blank `user`;
+ *   an empty `turns:` array.
+ * - A multi-turn case that ALSO declares case-level assertions or an `expect:` array (those live on
+ *   each turn for a multi-turn case).
+ * - A case (or turn) declaring BOTH flat assertions AND an `expect:` array (one way per case/turn).
+ * - An `expect:` block referencing an identity the suite does not declare.
+ * - A (turn × identity) with no applicable expectation block (statically determinable → rejected
+ *   here; the runner also guards it as a per-cell FAIL as a backstop).
+ * - A flat case or `expect:` block with no checks of any kind AND no `judge` rubric.
+ * - An invalid `must_match`/`must_not_match` regex, or a `json_path` entry not setting exactly one
+ *   of `equals`/`contains`.
+ * - A `judge_profile` containing a path separator or `..`.
  *
  * @param yamlText Raw suite file content.
  * @param sourcePath Optional path, only used to make error messages more actionable.
@@ -84,14 +206,105 @@ export function parseEvalSuite(yamlText, sourcePath) {
         throw new Error(`Invalid eval suite${suffix}: ${issues}`);
     }
     const data = parsed.data;
-    if (data.target.type !== 'gth-agent') {
+    // BATCH-14: build the normalized target by type. `gth-agent` is unchanged (the default); `adk-agent`
+    // adds the A2A connection config; anything else is an unsupported target (keeps the `http` reject).
+    let target;
+    if (data.target.type === 'gth-agent') {
+        if (data.target.profile !== undefined && data.target.profile !== 'default') {
+            throw new Error(`Invalid eval suite${suffix}: unsupported target.profile "${data.target.profile}" — ` +
+                'per-case/identity targets use the suite-level `identities` list (BATCH-12); omit ' +
+                'target.profile or set it to "default".');
+        }
+        target = { type: 'gth-agent', profile: data.target.profile };
+    }
+    else if (data.target.type === 'adk-agent') {
+        // The ADK (A2A) target needs at minimum the agent's A2A endpoint / agent-card URL.
+        const url = data.target.url?.trim();
+        if (!url) {
+            throw new Error(`Invalid eval suite${suffix}: an "adk-agent" target requires a \`url\` — the ADK agent's ` +
+                'A2A endpoint / agent-card base URL (e.g. `target: { type: adk-agent, url: ' +
+                'http://localhost:8080 }`).');
+        }
+        if (data.target.profile !== undefined) {
+            throw new Error(`Invalid eval suite${suffix}: an "adk-agent" target does not take a \`profile\` — the ADK ` +
+                'agent runs out-of-process with its own config; omit `target.profile`.');
+        }
+        target = { type: 'adk-agent', url, agentId: data.target.agent_id?.trim() || 'adk-agent' };
+    }
+    else if (data.target.type === 'ag-ui') {
+        // BATCH-15: the AG-UI target drives `POST {url}/agents/{agentId}/run` (HTTP + SSE). Both the base
+        // `url` and the `{agentId}` path segment are required — the acceptance contract is "missing
+        // url/agentId is a clear error" — so a suite missing either is a parse error (not a silent
+        // default). `agent_id` is the raw YAML key; it maps to the target's `agentId`.
+        const url = data.target.url?.trim();
+        if (!url) {
+            throw new Error(`Invalid eval suite${suffix}: an "ag-ui" target requires a \`url\` — the AG-UI server's ` +
+                'base URL (the origin of `POST {url}/agents/{agentId}/run`, e.g. `target: { type: ag-ui, ' +
+                'url: http://localhost:3000, agent_id: gth }`).');
+        }
+        const agentId = data.target.agent_id?.trim();
+        if (!agentId) {
+            throw new Error(`Invalid eval suite${suffix}: an "ag-ui" target requires an \`agent_id\` — the \`{agentId}\` ` +
+                'path segment of `/agents/{agentId}/run` (e.g. `target: { type: ag-ui, url: ' +
+                'http://localhost:3000, agent_id: gth }`).');
+        }
+        if (data.target.profile !== undefined) {
+            throw new Error(`Invalid eval suite${suffix}: an "ag-ui" target does not take a \`profile\` — the AG-UI ` +
+                'agent runs out-of-process with its own config; omit `target.profile`.');
+        }
+        target = { type: 'ag-ui', url, agentId };
+    }
+    else {
         throw new Error(`Invalid eval suite${suffix}: unsupported target.type "${data.target.type}" — this version ` +
-            'of `gth eval` only supports "gth-agent" (pluggable CLI/HTTP targets are future scope).');
+            'of `gth eval` supports "gth-agent" (in-process), "adk-agent" (an external Google ADK ' +
+            'agent over A2A), and "ag-ui" (an external agent over the AG-UI protocol); other pluggable ' +
+            'CLI/HTTP targets are future scope.');
+    }
+    // Suite-level identity matrix (BATCH-12). Validate the names here (plain, path-safe, unique) so
+    // later stages — output filenames, the `expect:` identity-membership check, the command's
+    // resolve-precondition — can trust them.
+    let identities;
+    if (data.identities !== undefined) {
+        if (data.identities.length === 0) {
+            throw new Error(`Invalid eval suite${suffix}: \`identities\` must list at least one profile name — omit ` +
+                'the key entirely for a single-profile run.');
+        }
+        const seenIdentities = new Set();
+        for (const name of data.identities) {
+            // Regex blocks separators/special chars; the explicit `..` check blocks a traversal segment
+            // the dot-permitting regex would otherwise allow (mirrors the judge_profile defence) — a
+            // profile of `..` would escape `.gsloth-settings/` during resolution.
+            if (!IDENTITY_NAME_RE.test(name) || name.includes('..')) {
+                throw new Error(`Invalid eval suite${suffix}: identity "${name}" must be a plain profile name ` +
+                    '(alphanumeric, dashes, underscores, dots) — identity names double as config-dir and ' +
+                    'output-filename components, so path separators and ".." are rejected.');
+            }
+            if (seenIdentities.has(name)) {
+                throw new Error(`Invalid eval suite${suffix}: duplicate identity "${name}".`);
+            }
+            seenIdentities.add(name);
+        }
+        identities = data.identities;
+    }
+    const declaredIdentities = identities ? new Set(identities) : undefined;
+    // BATCH-14: the identity matrix runs each case once per suite-declared identity profile, each under
+    // its own gth `initConfig({ …, identityProfile })`. That is meaningless for an external ADK agent
+    // (there is no gth config to switch), so reject the combination rather than silently ignoring the
+    // matrix — a false-scope suite is a bug, not something to run half of.
+    if (target.type === 'adk-agent' && identities !== undefined) {
+        throw new Error(`Invalid eval suite${suffix}: the \`identities\` matrix is not supported for an "adk-agent" ` +
+            'target — identity profiles select per-identity gth configs, which do not apply to an ' +
+            'external ADK agent. Remove `identities`, or use a `gth-agent` target.');
     }
-    if (data.target.profile !== undefined && data.target.profile !== 'default') {
-        throw new Error(`Invalid eval suite${suffix}: unsupported target.profile "${data.target.profile}" — ` +
-            'per-profile/identity targets are not supported yet (see --identities in ' +
-            'docs/batch-eval-cli-surface.md); omit target.profile or set it to "default".');
+    // BATCH-15: the same honest boundary for the AG-UI target. The `identities` matrix runs each case
+    // under a per-identity gth `initConfig({ …, identityProfile })`; an external AG-UI agent has no gth
+    // config to switch (the `RunAgentInput` wire contract carries no identity/auth field to thread
+    // per-identity), so the matrix cannot select its identity. Reject the combination rather than
+    // silently ignoring it — a false-scope suite is a bug, not something to run half of.
+    if (target.type === 'ag-ui' && identities !== undefined) {
+        throw new Error(`Invalid eval suite${suffix}: the \`identities\` matrix is not supported for an "ag-ui" ` +
+            'target — identity profiles select per-identity gth configs, which do not apply to an ' +
+            'external AG-UI agent. Remove `identities`, or use a `gth-agent` target.');
     }
     const suiteDefaultThreshold = data.defaults?.pass_threshold ?? DEFAULT_EVAL_PASS_THRESHOLD;
     const seenIds = new Set();
@@ -100,30 +313,262 @@ export function parseEvalSuite(yamlText, sourcePath) {
             throw new Error(`Invalid eval suite${suffix}: duplicate case id "${rawCase.id}".`);
         }
         seenIds.add(rawCase.id);
-        const mustContain = rawCase.must_contain ?? [];
-        const mustNotContain = rawCase.must_not_contain ?? [];
-        const shouldContainAny = rawCase.should_contain_any ?? [];
-        const hasChecks = mustContain.length > 0 || mustNotContain.length > 0 || shouldContainAny.length > 0;
-        const judgeRubric = rawCase.judge?.trim();
-        const hasJudge = !!judgeRubric;
-        if (!hasChecks && !hasJudge) {
-            throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) has no deterministic ` +
-                'checks and no judge rubric — a case must declare at least one of must_contain / ' +
-                'must_not_contain / should_contain_any, or a judge rubric.');
+        // BATCH-12 Task 2 — a case is EITHER single-turn (`prompt`) OR multi-turn (`turns:`): never
+        // both, never neither. Both share the SAME per-turn assertion surface (flat sugar OR `expect:`),
+        // normalized by `buildTurnExpectations` so single- and multi-turn enforce the rules identically.
+        if (rawCase.turns !== undefined && rawCase.prompt !== undefined) {
+            throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) declares BOTH \`prompt\` ` +
+                'and `turns:` — a case is single-turn (`prompt`) or multi-turn (`turns:`), not both.');
+        }
+        let turns;
+        if (rawCase.turns !== undefined) {
+            // Multi-turn: assertions live on each turn, so case-level assertions/`expect:` are rejected
+            // (they'd have no turn to attach to). The user message + assertions belong inside each turn.
+            const hasCaseLevelExpect = rawCase.expect !== undefined;
+            const hasCaseLevelFlat = FLAT_ASSERTION_KEYS.some((key) => rawCase[key] !== undefined);
+            if (hasCaseLevelExpect || hasCaseLevelFlat) {
+                throw new Error(`Invalid eval suite${suffix}: multi-turn case "${rawCase.id}" (index ${index}) declares ` +
+                    'case-level assertions or an `expect:` array — a multi-turn case puts assertions on ' +
+                    'each turn (beside its `user`), not at case level.');
+            }
+            const rawTurns = rawCase.turns;
+            if (rawTurns.length === 0) {
+                throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) has an empty ` +
+                    '`turns:` array — declare at least one turn (or use a single `prompt`).');
+            }
+            turns = rawTurns.map((rawTurn, turnIndex) => {
+                if (rawTurn.user === undefined || rawTurn.user.trim().length === 0) {
+                    throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) turn ${turnIndex} ` +
+                        'must declare a non-empty `user` message.');
+                }
+                return {
+                    user: rawTurn.user,
+                    expectations: buildTurnExpectations(rawTurn, {
+                        suffix,
+                        caseId: rawCase.id,
+                        caseIndex: index,
+                        turnIndex,
+                        declaredIdentities,
+                        identities,
+                    }),
+                };
+            });
+        }
+        else {
+            // Single-turn (unchanged behaviour): a `prompt` + case-level assertions/`expect:`, normalized
+            // to exactly one turn whose `user` is the prompt.
+            if (rawCase.prompt === undefined || rawCase.prompt.trim().length === 0) {
+                throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) must declare a ` +
+                    'non-empty `prompt` (single-turn) or a `turns:` array (multi-turn).');
+            }
+            turns = [
+                {
+                    user: rawCase.prompt,
+                    expectations: buildTurnExpectations(rawCase, {
+                        suffix,
+                        caseId: rawCase.id,
+                        caseIndex: index,
+                        turnIndex: undefined,
+                        declaredIdentities,
+                        identities,
+                    }),
+                },
+            ];
         }
         return {
             id: rawCase.id,
-            prompt: rawCase.prompt,
-            mustContain,
-            mustNotContain,
-            shouldContainAny,
-            judgeRubric: hasJudge ? judgeRubric : undefined,
+            turns,
             passThreshold: rawCase.pass_threshold ?? suiteDefaultThreshold,
         };
     });
+    // BATCH-14 (design point 4) — the HONEST tool-call boundary for the ADK target. A2A's wire content
+    // is text/file/data parts + task status/artifact events; it does NOT surface the agent's
+    // intermediate tool/function calls in any standardized form. So a `must_call`/`must_not_call`
+    // assertion cannot be graded against an `adk-agent` target — and a tool-trace assertion that
+    // silently PASSES is the worst outcome for an eval tool. Reject it here (scanning EVERY normalized
+    // block — flat, `expect:`, and per-turn), naming the offending case, rather than passing it.
+    if (target.type === 'adk-agent') {
+        for (const evalCase of cases) {
+            for (const turn of evalCase.turns) {
+                for (const expectation of turn.expectations) {
+                    if (expectation.mustCall.length > 0 || expectation.mustNotCall.length > 0) {
+                        throw new Error(`Invalid eval suite${suffix}: case "${evalCase.id}" uses \`must_call\`/\`must_not_call\`, ` +
+                            'which is not supported for an "adk-agent" target — A2A does not expose the agent\'s ' +
+                            'intermediate tool calls, so a tool-trace assertion cannot be graded (and must never ' +
+                            'silently pass). Use content assertions (must_contain / must_match / json_path / ' +
+                            'judge / …), or a `gth-agent` target for tool-trace grading.');
+                    }
+                }
+            }
+        }
+    }
+    // Normalize a blank/whitespace-only judge_profile to undefined (= no separate judge) so the CLI's
+    // resolution treats it the same as absent.
+    const judgeProfile = data.judge_profile?.trim() || undefined;
+    // A judge_profile is a plain identity-profile name that resolves under `.gsloth-settings/<name>/`.
+    // Reject path separators / `..` so a suite file can't feed a traversal sequence into profile
+    // resolution — the same defence the case `id` gets above, since a suite is only semi-trusted input.
+    if (judgeProfile !== undefined && (/[\\/]/.test(judgeProfile) || judgeProfile.includes('..'))) {
+        throw new Error(`Invalid eval suite${suffix}: judge_profile "${judgeProfile}" must be a plain profile name ` +
+            '(no path separators or "..").');
+    }
     return {
-        target: { type: 'gth-agent', profile: data.target.profile },
+        target,
+        judgeProfile,
+        identities,
         cases,
     };
 }
+/**
+ * Normalize ONE turn's raw assertion surface — a single-`prompt` case's case-level fields, or one
+ * `turns:` entry — into its {@link EvalExpectation} blocks. Shared by the single-turn and multi-turn
+ * paths so both enforce the SAME rules identically: flat-vs-`expect:` exclusivity, a non-empty
+ * `expect:` array, each block's own validation (identities membership, ≥1 check/judge), and the
+ * per-(turn × identity) NO-SILENT-PASS guard — every declared identity must have ≥1 applicable block
+ * for THIS turn, else nothing would grade that (turn × identity).
+ */
+function buildTurnExpectations(raw, ctx) {
+    const turnPart = ctx.turnIndex === undefined ? '' : ` turn ${ctx.turnIndex}`;
+    const where = `case "${ctx.caseId}" (index ${ctx.caseIndex})${turnPart}`;
+    const scopeNoun = ctx.turnIndex === undefined ? 'case-level' : 'turn-level';
+    const hasExpect = raw.expect !== undefined;
+    const hasFlatAssertions = FLAT_ASSERTION_KEYS.some((key) => raw[key] !== undefined);
+    if (hasExpect && hasFlatAssertions) {
+        throw new Error(`Invalid eval suite${ctx.suffix}: ${where} declares BOTH ${scopeNoun} assertions and an ` +
+            '`expect:` array — use one or the other (flat assertions apply to every identity; an ' +
+            '`expect:` array scopes blocks per identity).');
+    }
+    let expectations;
+    if (hasExpect) {
+        const rawBlocks = raw.expect;
+        if (rawBlocks.length === 0) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has an empty \`expect:\` array — declare at ` +
+                'least one expectation block.');
+        }
+        expectations = rawBlocks.map((block, blockIndex) => buildExpectation(block, block.identities, {
+            suffix: ctx.suffix,
+            caseId: ctx.caseId,
+            caseIndex: ctx.caseIndex,
+            turnIndex: ctx.turnIndex,
+            blockIndex,
+            declaredIdentities: ctx.declaredIdentities,
+        }));
+    }
+    else {
+        // Flat sugar: one unscoped expectation block from the case-level / turn-level assertion fields.
+        expectations = [
+            buildExpectation(raw, undefined, {
+                suffix: ctx.suffix,
+                caseId: ctx.caseId,
+                caseIndex: ctx.caseIndex,
+                turnIndex: ctx.turnIndex,
+                blockIndex: undefined,
+                declaredIdentities: ctx.declaredIdentities,
+            }),
+        ];
+    }
+    // NO-SILENT-PASS (per turn × identity): when the suite declares identities, THIS turn must have at
+    // least one applicable block for every identity, else nothing would grade that (turn × identity) —
+    // a suite-authoring bug, not a trivial pass. A block with no `identities` applies to all.
+    if (ctx.identities) {
+        for (const identity of ctx.identities) {
+            const covered = expectations.some((e) => !e.identities || e.identities.length === 0 || e.identities.includes(identity));
+            if (!covered) {
+                throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has no expectation block covering identity ` +
+                    `"${identity}" — every (turn × identity) must have at least one applicable block (add ` +
+                    `an \`identities: [${identity}]\` block, or an unscoped block that applies to all ` +
+                    'identities).');
+            }
+        }
+    }
+    return expectations;
+}
+/**
+ * Normalize one raw assertion bundle (a flat case's case-level fields, or one `expect:` block) into
+ * an {@link EvalExpectation}: default arrays to `[]`, compile regexes at parse time, validate
+ * json_path shape, validate the optional `identities` scope against the suite's declared list, and
+ * enforce that the block declares at least one check or a judge rubric.
+ */
+function buildExpectation(raw, blockIdentities, ctx) {
+    const turnPart = ctx.turnIndex === undefined ? '' : ` turn ${ctx.turnIndex}`;
+    const where = ctx.blockIndex === undefined
+        ? `case "${ctx.caseId}" (index ${ctx.caseIndex})${turnPart}`
+        : `case "${ctx.caseId}" (index ${ctx.caseIndex})${turnPart} expect block ${ctx.blockIndex}`;
+    // Validate the block's identity scope (only present on `expect:` blocks). Every named identity
+    // must be one the suite declares, else the block would silently never apply (a dead / typo'd
+    // scope) — reject it here.
+    let identities;
+    if (blockIdentities !== undefined) {
+        if (blockIdentities.length === 0) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has an empty \`identities\` list — omit it to ` +
+                'apply to all identities, or name at least one.');
+        }
+        for (const name of blockIdentities) {
+            if (!ctx.declaredIdentities || !ctx.declaredIdentities.has(name)) {
+                throw new Error(`Invalid eval suite${ctx.suffix}: ${where} references identity "${name}" which the suite ` +
+                    'does not declare — add it to the suite-level `identities` list.');
+            }
+        }
+        identities = blockIdentities;
+    }
+    const mustContain = raw.must_contain ?? [];
+    const mustNotContain = raw.must_not_contain ?? [];
+    const shouldContainAny = raw.should_contain_any ?? [];
+    const mustCall = raw.must_call ?? [];
+    const mustNotCall = raw.must_not_call ?? [];
+    // Compile regex assertions here so an invalid pattern is a parse-time suite error, never a crash
+    // partway through a run. The compiled RegExp is stored on the expectation and reused as-is.
+    const compileRegexes = (patterns, field) => (patterns ?? []).map((pattern) => {
+        try {
+            return new RegExp(pattern);
+        }
+        catch (error) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has an invalid ${field} pattern ` +
+                `${JSON.stringify(pattern)}: ` +
+                (error instanceof Error ? error.message : String(error)));
+        }
+    });
+    const mustMatch = compileRegexes(raw.must_match, 'must_match');
+    const mustNotMatch = compileRegexes(raw.must_not_match, 'must_not_match');
+    // Each json_path entry must set exactly one of equals/contains. `equals` may legitimately be any
+    // JSON value including `null`, so presence is `!== undefined` (an explicit null counts).
+    const jsonPath = (raw.json_path ?? []).map((entry) => {
+        const hasEquals = entry.equals !== undefined;
+        const hasContains = entry.contains !== undefined;
+        if (hasEquals === hasContains) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} json_path entry for "${entry.path}" must set ` +
+                'exactly one of "equals" or "contains".');
+        }
+        return hasContains
+            ? { path: entry.path, contains: entry.contains }
+            : { path: entry.path, equals: entry.equals };
+    });
+    const hasChecks = mustContain.length > 0 ||
+        mustNotContain.length > 0 ||
+        shouldContainAny.length > 0 ||
+        mustCall.length > 0 ||
+        mustNotCall.length > 0 ||
+        mustMatch.length > 0 ||
+        mustNotMatch.length > 0 ||
+        jsonPath.length > 0;
+    const judgeRubric = raw.judge?.trim();
+    const hasJudge = !!judgeRubric;
+    if (!hasChecks && !hasJudge) {
+        throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has no checks and no judge rubric — it must ` +
+            'declare at least one of must_contain / must_not_contain / should_contain_any / must_call ' +
+            '/ must_not_call / must_match / must_not_match / json_path, or a judge rubric.');
+    }
+    return {
+        identities,
+        mustContain,
+        mustNotContain,
+        shouldContainAny,
+        mustCall,
+        mustNotCall,
+        mustMatch,
+        mustNotMatch,
+        jsonPath,
+        judgeRubric: hasJudge ? judgeRubric : undefined,
+    };
+}
 //# sourceMappingURL=evalSuite.js.map